[PATCH] Add generate_bibtex.py: Make use of a separate script for generation of debian.{bib,tex} files
From: Akshita Jha <akshita-guest@users.alioth.debian.org>
---
config-ullmann.yaml | 5 ++
scripts/cron_ftpnew_blends.sh | 2 +
udd/bibref_gatherer.py | 113 +----------------------------
udd/generate_bibtex.py | 165 ++++++++++++++++++++++++++++++++++++++++++
4 files changed, 175 insertions(+), 110 deletions(-)
create mode 100644 udd/generate_bibtex.py
diff --git a/config-ullmann.yaml b/config-ullmann.yaml
index 901550e..5764565 100644
--- a/config-ullmann.yaml
+++ b/config-ullmann.yaml
@@ -45,6 +45,7 @@ general:
i18n-apps: module udd.i18n_apps_gatherer
hints: module udd.hints_gatherer
deferred: module udd.deferred_gatherer
+ generate-bibtex: module udd.generate_bibtex
timestamp-dir: /srv/udd.debian.org/timestamps
lock-dir: /srv/udd.debian.org/locks
archs:
@@ -471,3 +472,7 @@ vcswatch:
reproducible:
type: reproducible
+
+generate-bibtex:
+ type: generate-bibtex
+
diff --git a/scripts/cron_ftpnew_blends.sh b/scripts/cron_ftpnew_blends.sh
index fc0d087..fbc7daa 100755
--- a/scripts/cron_ftpnew_blends.sh
+++ b/scripts/cron_ftpnew_blends.sh
@@ -11,3 +11,5 @@ $UAR ftpnew
$UAR blends-prospective
# $UAR blends-metadata
$UAR blends-all
+$UAR generate-bibtex
+
diff --git a/udd/bibref_gatherer.py b/udd/bibref_gatherer.py
index 654d7e7..bfff5b4 100644
--- a/udd/bibref_gatherer.py
+++ b/udd/bibref_gatherer.py
@@ -6,8 +6,7 @@ This script imports bibliographic references from upstream-metadata.debian.net.
from gatherer import gatherer
from sys import stderr, exit
-from os import listdir, unlink, rename, access, X_OK
-from os.path import isfile
+from os import listdir
from fnmatch import fnmatch
import yaml
from psycopg2 import IntegrityError, InternalError
@@ -23,43 +22,9 @@ debug=0
def get_gatherer(connection, config, source):
return bibref_gatherer(connection, config, source)
-def rm_f(file):
- try:
- unlink(file)
- except OSError:
- pass
-
-def cleanup_tex_logs(basetexfile):
- rm_f(basetexfile+'.aux')
- rm_f(basetexfile+'.bbl')
- rm_f(basetexfile+'.blg')
- rm_f(basetexfile+'.log')
-
# seek for authors separated by ',' rather than by ' and '
seek_broken_authors_re = re.compile('^[^\s^,]+\s+[^\s^,]+\s*,\s*[^\s^,]+\s+[^\s^,]')
-def open_tex_process(texexe, basetexfile):
- if texexe == 'pdflatex':
- ptex = Popen(['pdflatex', '-interaction=batchmode', basetexfile], shell=False, stdout=PIPE)
- elif texexe == 'bibtex':
- ptex = Popen(['bibtex', basetexfile], shell=False, stdout=PIPE)
- else:
- return(False, 'Wrong exe: '+texexe)
- errstring=""
- if ptex.wait():
- if texexe == 'pdflatex':
- for logrow in ptex.communicate()[0].splitlines():
- if logrow.startswith('!'):
- errstring += logrow
- return(False, errstring)
- else:
- for logrow in ptex.communicate()[0].splitlines():
- if logrow.startswith('This is BibTeX'):
- continue
- errstring += logrow + '\n'
- return(True, errstring)
- return(True, errstring)
-
other_known_keys = ('Archive',
'Bug-Database',
'Cite-As',
@@ -297,10 +262,6 @@ class bibref_gatherer(gatherer):
handler.setFormatter(formatter)
self.log.addHandler(handler)
-
- self.bibtexfile = 'debian.bib'
- self.bibtex_example_tex = 'debian.tex'
-
def run(self):
my_config = self.my_config
#start harassing the DB, preparing the final inserts and making place
@@ -364,77 +325,9 @@ class bibref_gatherer(gatherer):
# commit before check to make sure the table is not locked in case LaTeX run will fail for whatever reason
self.connection.commit()
- # if there is a working LaTeX installation try to build a BibTeX database and test it by creating a debian.pdf file
- if isfile('/usr/bin/pdflatex') and access('/usr/bin/pdflatex', X_OK) and \
- isfile('/usr/bin/bibtex') and access('/usr/bin/bibtex', X_OK) and \
- ( isfile('/usr/share/texlive/texmf-dist/fonts/source/jknappen/ec/ecrm.mf') or \
- isfile('/usr/share/texmf-texlive/fonts/source/jknappen/ec/ecrm.mf') ) :
- # create BibTeX file
- bf = open(self.bibtexfile, 'w')
- cur.execute("SELECT * FROM bibtex()")
- for row in cur.fetchall():
- print >>bf, row[0]
- bf.close()
-
- # create LaTeX file to test BibTeX functionality
- bf = open(self.bibtex_example_tex, 'w')
- print >>bf, """\documentclass[10]{article}
-\usepackage[T1]{fontenc}
-\usepackage[utf8]{inputenc}
-\usepackage[left=2mm,top=2mm,right=2mm,bottom=2mm,nohead,nofoot]{geometry}
-\usepackage{longtable}
-\usepackage[super]{natbib}
-\setlongtables
-\\begin{document}
-\small
-\\begin{longtable}{llp{70mm}l}
-\\bf package & \\bf source & \\bf description & BibTeX key \\\\ \hline"""
-
- cur.execute("SELECT * FROM bibtex_example_data() AS (package text, source text, bibkey text, description text)")
- for row in cur.fetchall():
- print >>bf, row[0], '&', row[1], '&', row[3] , '&', row[2]+'\cite{'+row[2]+'} \\\\'
-
- print >>bf, """\end{longtable}
-
-% \\bibliographystyle{plain}
-% Try a bit harder by also including URL+DOI
-\\bibliographystyle{plainnat}
-\\bibliography{debian}
-
-\end{document}
-"""
- bf.close()
-
- # try to build debian.pdf file to test aboc LaTeX file
- basetexfile = self.bibtex_example_tex.replace('.tex','')
- cleanup_tex_logs(basetexfile)
- try:
- rename(basetexfile+'.pdf', basetexfile+'.pdf~')
- except OSError:
- pass
-
- (retcode,errstring) = open_tex_process('pdflatex', basetexfile)
- if not retcode:
- self.log.error("Problem in 1. PdfLaTeX run of %s.tex: `%s` --> please inspect %s.log" % (basetexfile, errstring, basetexfile))
- exit(1)
- (retcode,errstring) = open_tex_process('bibtex', basetexfile)
- if errstring != "":
- if not retcode:
- self.log.error("Problem in BibTeX run of %s.bib: `%s`" % (basetexfile, errstring))
- exit(1)
- self.log.error("Ignore the following problems in BibTeX run of %s.bib: `%s`" % (basetexfile, errstring))
- (retcode,errstring) = open_tex_process('pdflatex', basetexfile)
- if not retcode:
- self.log.error("Problem in 2. PdfLaTeX run of %s.tex: `%s` --> please inspect %s.log" % (basetexfile, errstring, basetexfile))
- exit(1)
- (retcode,errstring) = open_tex_process('pdflatex', basetexfile)
- if not retcode:
- self.log.error("Problem in 3. PdfLaTeX run of %s.tex: `%s` --> please inspect %s.log" % (basetexfile, errstring, basetexfile))
- exit(1)
-
- cleanup_tex_logs(basetexfile)
-
+
if __name__ == '__main__':
main()
# vim:set et tabstop=2:
+
diff --git a/udd/generate_bibtex.py b/udd/generate_bibtex.py
new file mode 100644
index 0000000..80002e7
--- /dev/null
+++ b/udd/generate_bibtex.py
@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+
+"""
+This script creates debian.bib and debian.tex files for bibliographic references.
+"""
+
+from gatherer import gatherer
+from os import unlink, rename, access, X_OK
+from os.path import isfile
+from subprocess import Popen, PIPE
+import logging
+import logging.handlers
+
+debug = 0
+
+def get_gatherer(connection, config, source):
+ return generate_bibtex(connection, config, source)
+
+def rm_f(file):
+ try:
+ unlink(file)
+ except OSError:
+ pass
+
+
+def cleanup_tex_logs(basetexfile):
+ rm_f(basetexfile+'.aux')
+ rm_f(basetexfile+'.bbl')
+ rm_f(basetexfile+'.blg')
+ rm_f(basetexfile+'.log')
+
+
+def open_tex_process(texexe, basetexfile):
+ if texexe == 'pdflatex':
+ ptex = Popen(['pdflatex', '-interaction=batchmode', basetexfile], shell=False, stdout=PIPE)
+ elif texexe == 'bibtex':
+ ptex = Popen(['bibtex', basetexfile], shell=False, stdout=PIPE)
+ else:
+ return(False, 'Wrong exe: '+texexe)
+ errstring=""
+ if ptex.wait():
+ if texexe == 'pdflatex':
+ for logrow in ptex.communicate()[0].splitlines():
+ if logrow.startswith('!'):
+ errstring += logrow
+ return(False, errstring)
+ else:
+ for logrow in ptex.communicate()[0].splitlines():
+ if logrow.startswith('This is BibTeX'):
+ continue
+ errstring += logrow + '\n'
+ return(True, errstring)
+ return(True, errstring)
+
+
+class generate_bibtex(gatherer):
+ """
+ Generate a debian.bib and debian.tex files
+ """
+
+ def __init__(self, connection, config, source):
+ gatherer.__init__(self, connection, config, source)
+
+ self.log = logging.getLogger(self.__class__.__name__)
+ if debug==1:
+ self.log.setLevel(logging.DEBUG)
+ else:
+ self.log.setLevel(logging.INFO)
+ handler = logging.handlers.RotatingFileHandler(filename=self.__class__.__name__+'.log',mode='w')
+ formatter = logging.Formatter("%(asctime)s - %(levelname)s - (%(lineno)d): %(message)s")
+ handler.setFormatter(formatter)
+ self.log.addHandler(handler)
+
+ self.bibtexfile = 'debian.bib'
+ self.bibtex_example_tex = 'debian.tex'
+ self.all_ref = 0 # to include all references from bibref table set this to 1
+
+ def run(self):
+ cur = self.cursor()
+
+ # if there is a working LaTeX installation try to build a BibTeX database and test it by creating a debian.pdf file
+ if isfile('/usr/bin/pdflatex') and access('/usr/bin/pdflatex', X_OK) and \
+ isfile('/usr/bin/bibtex') and access('/usr/bin/bibtex', X_OK) and \
+ ( isfile('/usr/share/texlive/texmf-dist/fonts/source/jknappen/ec/ecrm.mf') or \
+ isfile('/usr/share/texmf-texlive/fonts/source/jknappen/ec/ecrm.mf') ) :
+
+ # create BibTeX file
+ bf = open(self.bibtexfile, 'w')
+
+ if self.all_ref == 1:
+ query = "SELECT * FROM bibtex()"
+ else:
+ query = "SELECT * FROM bibtex_debian_pool()"
+
+ cur.execute(query)
+ for row in cur.fetchall():
+ print >>bf, row[0]
+
+ bf.close()
+
+ # create LaTeX file to test BibTeX functionality
+ bf = open(self.bibtex_example_tex, 'w')
+ print >>bf, """\documentclass[10]{article}
+\usepackage[T1]{fontenc}
+\usepackage[utf8]{inputenc}
+\usepackage[left=2mm,top=2mm,right=2mm,bottom=2mm,nohead,nofoot]{geometry}
+\usepackage{longtable}
+\usepackage[super]{natbib}
+\setlongtables
+\\begin{document}
+\small
+\\begin{longtable}{llp{70mm}l}
+\\bf package & \\bf source & \\bf description & BibTeX key \\\\ \hline"""
+
+ cur.execute("SELECT * FROM bibtex_example_data() AS (package text, source text, bibkey text, description text)")
+ for row in cur.fetchall():
+ print >>bf, row[0], '&', row[1], '&', row[3] , '&', row[2]+'\cite{'+row[2]+'} \\\\'
+
+ print >>bf, """\end{longtable}
+
+% \\bibliographystyle{plain}
+% Try a bit harder by also including URL+DOI
+\\bibliographystyle{plainnat}
+\\bibliography{debian}
+
+\end{document}
+"""
+ bf.close()
+
+ # try to build debian.pdf file to test aboc LaTeX file
+ basetexfile = self.bibtex_example_tex.replace('.tex','')
+ cleanup_tex_logs(basetexfile)
+ try:
+ rename(basetexfile+'.pdf', basetexfile+'.pdf~')
+ except OSError:
+ pass
+
+ (retcode,errstring) = open_tex_process('pdflatex', basetexfile)
+ if not retcode:
+ self.log.error("Problem in 1. PdfLaTeX run of %s.tex: `%s` --> please inspect %s.log" % (basetexfile, errstring, basetexfile))
+ exit(1)
+
+ (retcode,errstring) = open_tex_process('bibtex', basetexfile)
+ if errstring != "":
+ if not retcode:
+ self.log.error("Problem in BibTeX run of %s.bib: `%s`" % (basetexfile, errstring))
+ exit(1)
+ self.log.error("Ignore the following problems in BibTeX run of %s.bib: `%s`" % (basetexfile, errstring))
+
+ (retcode,errstring) = open_tex_process('pdflatex', basetexfile)
+ if not retcode:
+ self.log.error("Problem in 2. PdfLaTeX run of %s.tex: `%s` --> please inspect %s.log" % (basetexfile, errstring, basetexfile))
+ exit(1)
+
+ (retcode,errstring) = open_tex_process('pdflatex', basetexfile)
+ if not retcode:
+ self.log.error("Problem in 3. PdfLaTeX run of %s.tex: `%s` --> please inspect %s.log" % (basetexfile, errstring, basetexfile))
+ exit(1)
+
+ cleanup_tex_logs(basetexfile)
+
+if __name__ == '__main__':
+ main()
+
+
--
1.9.1
Reply to: