Hi Akshita,
as promised I wanted to inspect you patches with the separate
generate-bibtex script. I'm afraid I did not properly imported your
patches and I think we need to find some better way to propagate your
changes. I used `git am <your_patches>` (attached again for
verification as `git format-patch` patches). If I try:
$ ./update-and-run.sh generate-bibref
Traceback (most recent call last):
File "/srv/udd.debian.org/udd//udd.py", line 54, in <module>
src_config = config[src]
KeyError: 'generate-bibref'
Before I check the error I would like to ask whether I have really
imported all your changes.
Kind regards
Andreas.
On Fri, Apr 24, 2015 at 05:28:12PM +0530, zenith158@gmail.com wrote:
> From: Akshita Jha <akshita-guest@users.alioth.debian.org>
>
> ---
> config-ullmann.yaml | 5 ++
> scripts/cron_ftpnew_blends.sh | 1 +
> udd/bibref_gatherer.py | 110 +---------------------------
> udd/generate_bibtex.py | 166 ++++++++++++++++++++++++++++++++++++++++++
> 4 files changed, 173 insertions(+), 109 deletions(-)
> create mode 100644 udd/generate_bibtex.py
>
> diff --git a/config-ullmann.yaml b/config-ullmann.yaml
> index 901550e..5764565 100644
> --- a/config-ullmann.yaml
> +++ b/config-ullmann.yaml
> @@ -45,6 +45,7 @@ general:
> i18n-apps: module udd.i18n_apps_gatherer
> hints: module udd.hints_gatherer
> deferred: module udd.deferred_gatherer
> + generate-bibtex: module udd.generate_bibtex
> timestamp-dir: /srv/udd.debian.org/timestamps
> lock-dir: /srv/udd.debian.org/locks
> archs:
> @@ -471,3 +472,7 @@ vcswatch:
>
> reproducible:
> type: reproducible
> +
> +generate-bibtex:
> + type: generate-bibtex
> +
> diff --git a/scripts/cron_ftpnew_blends.sh b/scripts/cron_ftpnew_blends.sh
> index fc0d087..c38c076 100755
> --- a/scripts/cron_ftpnew_blends.sh
> +++ b/scripts/cron_ftpnew_blends.sh
> @@ -11,3 +11,4 @@ $UAR ftpnew
> $UAR blends-prospective
> # $UAR blends-metadata
> $UAR blends-all
> +$UAR generate-bibtex
> diff --git a/udd/bibref_gatherer.py b/udd/bibref_gatherer.py
> index 654d7e7..e86c8f2 100644
> --- a/udd/bibref_gatherer.py
> +++ b/udd/bibref_gatherer.py
> @@ -6,8 +6,7 @@ This script imports bibliographic references from upstream-metadata.debian.net.
>
> from gatherer import gatherer
> from sys import stderr, exit
> -from os import listdir, unlink, rename, access, X_OK
> -from os.path import isfile
> +from os import listdir
> from fnmatch import fnmatch
> import yaml
> from psycopg2 import IntegrityError, InternalError
> @@ -23,43 +22,9 @@ debug=0
> def get_gatherer(connection, config, source):
> return bibref_gatherer(connection, config, source)
>
> -def rm_f(file):
> - try:
> - unlink(file)
> - except OSError:
> - pass
> -
> -def cleanup_tex_logs(basetexfile):
> - rm_f(basetexfile+'.aux')
> - rm_f(basetexfile+'.bbl')
> - rm_f(basetexfile+'.blg')
> - rm_f(basetexfile+'.log')
> -
> # seek for authors separated by ',' rather than by ' and '
> seek_broken_authors_re = re.compile('^[^\s^,]+\s+[^\s^,]+\s*,\s*[^\s^,]+\s+[^\s^,]')
>
> -def open_tex_process(texexe, basetexfile):
> - if texexe == 'pdflatex':
> - ptex = Popen(['pdflatex', '-interaction=batchmode', basetexfile], shell=False, stdout=PIPE)
> - elif texexe == 'bibtex':
> - ptex = Popen(['bibtex', basetexfile], shell=False, stdout=PIPE)
> - else:
> - return(False, 'Wrong exe: '+texexe)
> - errstring=""
> - if ptex.wait():
> - if texexe == 'pdflatex':
> - for logrow in ptex.communicate()[0].splitlines():
> - if logrow.startswith('!'):
> - errstring += logrow
> - return(False, errstring)
> - else:
> - for logrow in ptex.communicate()[0].splitlines():
> - if logrow.startswith('This is BibTeX'):
> - continue
> - errstring += logrow + '\n'
> - return(True, errstring)
> - return(True, errstring)
> -
> other_known_keys = ('Archive',
> 'Bug-Database',
> 'Cite-As',
> @@ -297,10 +262,6 @@ class bibref_gatherer(gatherer):
> handler.setFormatter(formatter)
> self.log.addHandler(handler)
>
> -
> - self.bibtexfile = 'debian.bib'
> - self.bibtex_example_tex = 'debian.tex'
> -
> def run(self):
> my_config = self.my_config
> #start harassing the DB, preparing the final inserts and making place
> @@ -364,75 +325,6 @@ class bibref_gatherer(gatherer):
> # commit before check to make sure the table is not locked in case LaTeX run will fail for whatever reason
> self.connection.commit()
>
> - # if there is a working LaTeX installation try to build a BibTeX database and test it by creating a debian.pdf file
> - if isfile('/usr/bin/pdflatex') and access('/usr/bin/pdflatex', X_OK) and \
> - isfile('/usr/bin/bibtex') and access('/usr/bin/bibtex', X_OK) and \
> - ( isfile('/usr/share/texlive/texmf-dist/fonts/source/jknappen/ec/ecrm.mf') or \
> - isfile('/usr/share/texmf-texlive/fonts/source/jknappen/ec/ecrm.mf') ) :
> - # create BibTeX file
> - bf = open(self.bibtexfile, 'w')
> - cur.execute("SELECT * FROM bibtex()")
> - for row in cur.fetchall():
> - print >>bf, row[0]
> - bf.close()
> -
> - # create LaTeX file to test BibTeX functionality
> - bf = open(self.bibtex_example_tex, 'w')
> - print >>bf, """\documentclass[10]{article}
> -\usepackage[T1]{fontenc}
> -\usepackage[utf8]{inputenc}
> -\usepackage[left=2mm,top=2mm,right=2mm,bottom=2mm,nohead,nofoot]{geometry}
> -\usepackage{longtable}
> -\usepackage[super]{natbib}
> -\setlongtables
> -\\begin{document}
> -\small
> -\\begin{longtable}{llp{70mm}l}
> -\\bf package & \\bf source & \\bf description & BibTeX key \\\\ \hline"""
> -
> - cur.execute("SELECT * FROM bibtex_example_data() AS (package text, source text, bibkey text, description text)")
> - for row in cur.fetchall():
> - print >>bf, row[0], '&', row[1], '&', row[3] , '&', row[2]+'\cite{'+row[2]+'} \\\\'
> -
> - print >>bf, """\end{longtable}
> -
> -% \\bibliographystyle{plain}
> -% Try a bit harder by also including URL+DOI
> -\\bibliographystyle{plainnat}
> -\\bibliography{debian}
> -
> -\end{document}
> -"""
> - bf.close()
> -
> - # try to build debian.pdf file to test aboc LaTeX file
> - basetexfile = self.bibtex_example_tex.replace('.tex','')
> - cleanup_tex_logs(basetexfile)
> - try:
> - rename(basetexfile+'.pdf', basetexfile+'.pdf~')
> - except OSError:
> - pass
> -
> - (retcode,errstring) = open_tex_process('pdflatex', basetexfile)
> - if not retcode:
> - self.log.error("Problem in 1. PdfLaTeX run of %s.tex: `%s` --> please inspect %s.log" % (basetexfile, errstring, basetexfile))
> - exit(1)
> - (retcode,errstring) = open_tex_process('bibtex', basetexfile)
> - if errstring != "":
> - if not retcode:
> - self.log.error("Problem in BibTeX run of %s.bib: `%s`" % (basetexfile, errstring))
> - exit(1)
> - self.log.error("Ignore the following problems in BibTeX run of %s.bib: `%s`" % (basetexfile, errstring))
> - (retcode,errstring) = open_tex_process('pdflatex', basetexfile)
> - if not retcode:
> - self.log.error("Problem in 2. PdfLaTeX run of %s.tex: `%s` --> please inspect %s.log" % (basetexfile, errstring, basetexfile))
> - exit(1)
> - (retcode,errstring) = open_tex_process('pdflatex', basetexfile)
> - if not retcode:
> - self.log.error("Problem in 3. PdfLaTeX run of %s.tex: `%s` --> please inspect %s.log" % (basetexfile, errstring, basetexfile))
> - exit(1)
> -
> - cleanup_tex_logs(basetexfile)
>
> if __name__ == '__main__':
> main()
> diff --git a/udd/generate_bibtex.py b/udd/generate_bibtex.py
> new file mode 100644
> index 0000000..03af254
> --- /dev/null
> +++ b/udd/generate_bibtex.py
> @@ -0,0 +1,166 @@
> +#!/usr/bin/env python
> +
> +"""
> +This script creates debian.bib and debian.tex files for bibliographic references.
> +"""
> +
> +from gatherer import gatherer
> +from os import unlink, rename, access, X_OK
> +from os.path import isfile
> +from subprocess import Popen, PIPE
> +import logging
> +import logging.handlers
> +
> +debug = 0
> +
> +def get_gatherer(connection, config, source):
> + return generate_bibtex(connection, config, source)
> +
> +def rm_f(file):
> + try:
> + unlink(file)
> + except OSError:
> + pass
> +
> +
> +def cleanup_tex_logs(basetexfile):
> + rm_f(basetexfile+'.aux')
> + rm_f(basetexfile+'.bbl')
> + rm_f(basetexfile+'.blg')
> + rm_f(basetexfile+'.log')
> +
> +
> +def open_tex_process(texexe, basetexfile):
> + if texexe == 'pdflatex':
> + ptex = Popen(['pdflatex', '-interaction=batchmode', basetexfile], shell=False, stdout=PIPE)
> + elif texexe == 'bibtex':
> + ptex = Popen(['bibtex', basetexfile], shell=False, stdout=PIPE)
> + else:
> + return(False, 'Wrong exe: '+texexe)
> + errstring=""
> + if ptex.wait():
> + if texexe == 'pdflatex':
> + for logrow in ptex.communicate()[0].splitlines():
> + if logrow.startswith('!'):
> + errstring += logrow
> + return(False, errstring)
> + else:
> + for logrow in ptex.communicate()[0].splitlines():
> + if logrow.startswith('This is BibTeX'):
> + continue
> + errstring += logrow + '\n'
> + return(True, errstring)
> + return(True, errstring)
> +
> +
> +class generate_bibtex(gatherer):
> + """
> + Generate a debian.bib and debian.tex files
> + """
> +
> + def __init__(self, connection, config, source):
> + gatherer.__init__(self, connection, config, source)
> +
> + self.log = logging.getLogger(self.__class__.__name__)
> + if debug==1:
> + self.log.setLevel(logging.DEBUG)
> + else:
> + self.log.setLevel(logging.INFO)
> + handler = logging.handlers.RotatingFileHandler(filename=self.__class__.__name__+'.log',mode='w')
> + formatter = logging.Formatter("%(asctime)s - %(levelname)s - (%(lineno)d): %(message)s")
> + handler.setFormatter(formatter)
> + self.log.addHandler(handler)
> +
> + self.bibtexfile = 'debian.bib'
> + self.bibtex_example_tex = 'debian.tex'
> + self.all_ref = 0 # to include all references from bibref table set it to 1
> +
> + def run(self):
> + cur = self.cursor()
> +
> + # if there is a working LaTeX installation try to build a BibTeX database and test it by creating a debian.pdf file
> + if isfile('/usr/bin/pdflatex') and access('/usr/bin/pdflatex', X_OK) and \
> + isfile('/usr/bin/bibtex') and access('/usr/bin/bibtex', X_OK) and \
> + ( isfile('/usr/share/texlive/texmf-dist/fonts/source/jknappen/ec/ecrm.mf') or \
> + isfile('/usr/share/texmf-texlive/fonts/source/jknappen/ec/ecrm.mf') ) :
> +
> + # create BibTeX file
> + bf = open(self.bibtexfile, 'w')
> +
> + if self.all_ref == 1:
> + bibtable = "SELECT * FROM bibentry(bibref)"
> + query = "SELECT * FROM bibtex(bibtable)"
> + else:
> + bibtable = "SELECT * FROM bibentry(bibref, sources)"
> + query = "SELECT * FROM bibtex(bibtable)"
> +
> + cur.execute(query)
> + for row in cur.fetchall():
> + print >>bf, row[0]
> +
> + bf.close()
> +
> + # create LaTeX file to test BibTeX functionality
> + bf = open(self.bibtex_example_tex, 'w')
> + print >>bf, """\documentclass[10]{article}
> +\usepackage[T1]{fontenc}
> +\usepackage[utf8]{inputenc}
> +\usepackage[left=2mm,top=2mm,right=2mm,bottom=2mm,nohead,nofoot]{geometry}
> +\usepackage{longtable}
> +\usepackage[super]{natbib}
> +\setlongtables
> +\\begin{document}
> +\small
> +\\begin{longtable}{llp{70mm}l}
> +\\bf package & \\bf source & \\bf description & BibTeX key \\\\ \hline"""
> +
> + cur.execute("SELECT * FROM bibtex_example_data() AS (package text, source text, bibkey text, description text)")
> + for row in cur.fetchall():
> + print >>bf, row[0], '&', row[1], '&', row[3] , '&', row[2]+'\cite{'+row[2]+'} \\\\'
> +
> + print >>bf, """\end{longtable}
> +
> +% \\bibliographystyle{plain}
> +% Try a bit harder by also including URL+DOI
> +\\bibliographystyle{plainnat}
> +\\bibliography{debian}
> +
> +\end{document}
> +"""
> + bf.close()
> +
> + # try to build debian.pdf file to test aboc LaTeX file
> + basetexfile = self.bibtex_example_tex.replace('.tex','')
> + cleanup_tex_logs(basetexfile)
> + try:
> + rename(basetexfile+'.pdf', basetexfile+'.pdf~')
> + except OSError:
> + pass
> +
> + (retcode,errstring) = open_tex_process('pdflatex', basetexfile)
> + if not retcode:
> + self.log.error("Problem in 1. PdfLaTeX run of %s.tex: `%s` --> please inspect %s.log" % (basetexfile, errstring, basetexfile))
> + exit(1)
> +
> + (retcode,errstring) = open_tex_process('bibtex', basetexfile)
> + if errstring != "":
> + if not retcode:
> + self.log.error("Problem in BibTeX run of %s.bib: `%s`" % (basetexfile, errstring))
> + exit(1)
> + self.log.error("Ignore the following problems in BibTeX run of %s.bib: `%s`" % (basetexfile, errstring))
> +
> + (retcode,errstring) = open_tex_process('pdflatex', basetexfile)
> + if not retcode:
> + self.log.error("Problem in 2. PdfLaTeX run of %s.tex: `%s` --> please inspect %s.log" % (basetexfile, errstring, basetexfile))
> + exit(1)
> +
> + (retcode,errstring) = open_tex_process('pdflatex', basetexfile)
> + if not retcode:
> + self.log.error("Problem in 3. PdfLaTeX run of %s.tex: `%s` --> please inspect %s.log" % (basetexfile, errstring, basetexfile))
> + exit(1)
> +
> + cleanup_tex_logs(basetexfile)
> +
> +if __name__ == '__main__':
> + main()
> +
> --
> 1.9.1
>
>
> --
> To UNSUBSCRIBE, email to debian-blends-REQUEST@lists.debian.org
> with a subject of "unsubscribe". Trouble? Contact listmaster@lists.debian.org
> Archive: https://lists.debian.org/1429876694-20856-1-git-send-email-akshita-guest@users.alioth.debian.org
>
>
--
http://fam-tille.de
Attachment:
bibtex_patches.tgz
Description: application/gtar-compressed