[Date Prev][Date Next] [Thread Prev][Thread Next] [Date Index] [Thread Index]

[dak/master] Remove import-known-changes and import-new-files.



---
 dak/dak.py                  |    2 -
 dak/import_known_changes.py |  355 -------------------------------------------
 dak/import_new_files.py     |  187 -----------------------
 3 files changed, 0 insertions(+), 544 deletions(-)
 delete mode 100755 dak/import_known_changes.py
 delete mode 100755 dak/import_new_files.py

diff --git a/dak/dak.py b/dak/dak.py
index 579548c..1e36d82 100755
--- a/dak/dak.py
+++ b/dak/dak.py
@@ -137,8 +137,6 @@ def init():
          "Generate statistics"),
         ("bts-categorize",
          "Categorize uncategorized bugs filed against ftp.debian.org"),
-        ("import-known-changes",
-         "import old changes files into known_changes table"),
         ("add-user",
          "Add a user to the archive"),
         ("make-changelog",
diff --git a/dak/import_known_changes.py b/dak/import_known_changes.py
deleted file mode 100755
index 4e8068f..0000000
--- a/dak/import_known_changes.py
+++ /dev/null
@@ -1,355 +0,0 @@
-#!/usr/bin/env python
-# coding=utf8
-
-"""
-Import known_changes files
-
-@contact: Debian FTP Master <ftpmaster@debian.org>
-@copyright: 2009  Mike O'Connor <stew@debian.org>
-@license: GNU General Public License version 2 or later
-"""
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-
-################################################################################
-
-
-################################################################################
-
-import sys
-import os
-import logging
-import threading
-from daklib.dbconn import DBConn, get_dbchange, get_policy_queue
-from daklib.config import Config
-import apt_pkg
-from daklib.dak_exceptions import DBUpdateError, InvalidDscError, ChangesUnicodeError
-from daklib.changes import Changes
-from daklib.utils import parse_changes, warn, gpgv_get_status_output, process_gpgv_output
-import traceback
-
-# where in dak.conf all of our configuration will be stowed
-options_prefix = "KnownChanges"
-options_prefix = "%s::Options" % options_prefix
-
-log = logging.getLogger()
-
-################################################################################
-
-
-def usage (exit_code=0):
-    print """Usage: dak import-known-changes [options]
-
-OPTIONS
-     -j n
-        run with n threads concurrently
-
-     -v, --verbose
-        show verbose information messages
-
-     -q, --quiet
-        supress all output but errors
-
-"""
-    sys.exit(exit_code)
-
-def check_signature (sig_filename, data_filename=""):
-    fingerprint = None
-
-    keyrings = [
-        "/home/joerg/keyring/keyrings/debian-keyring.gpg",
-        "/home/joerg/keyring/keyrings/debian-maintainers.gpg",
-        "/home/joerg/keyring/keyrings/debian-role-keys.gpg",
-        "/home/joerg/keyring/keyrings/emeritus-keyring.pgp",
-        "/home/joerg/keyring/keyrings/emeritus-keyring.gpg",
-        "/home/joerg/keyring/keyrings/removed-keys.gpg",
-        "/home/joerg/keyring/keyrings/removed-keys.pgp"
-        ]
-
-    keyringargs = " ".join(["--keyring %s" % x for x in keyrings ])
-
-    # Build the command line
-    status_read, status_write = os.pipe()
-    cmd = "gpgv --status-fd %s %s %s" % (status_write, keyringargs, sig_filename)
-
-    # Invoke gpgv on the file
-    (output, status, exit_status) = gpgv_get_status_output(cmd, status_read, status_write)
-
-    # Process the status-fd output
-    (keywords, internal_error) = process_gpgv_output(status)
-
-    # If we failed to parse the status-fd output, let's just whine and bail now
-    if internal_error:
-        warn("Couldn't parse signature")
-        return None
-
-    # usually one would check for bad things here. We, however, do not care.
-
-    # Next check gpgv exited with a zero return code
-    if exit_status:
-        warn("Couldn't parse signature")
-        return None
-
-    # Sanity check the good stuff we expect
-    if not keywords.has_key("VALIDSIG"):
-        warn("Couldn't parse signature")
-    else:
-        args = keywords["VALIDSIG"]
-        if len(args) < 1:
-            warn("Couldn't parse signature")
-        else:
-            fingerprint = args[0]
-
-    return fingerprint
-
-
-class EndOfChanges(object):
-    """something enqueued to signify the last change"""
-    pass
-
-
-class OneAtATime(object):
-    """
-    a one space queue which sits between multiple possible producers
-    and multiple possible consumers
-    """
-    def __init__(self):
-        self.next_in_line = None
-        self.read_lock = threading.Condition()
-        self.write_lock = threading.Condition()
-        self.die = False
-
-    def plsDie(self):
-        self.die = True
-        self.write_lock.acquire()
-        self.write_lock.notifyAll()
-        self.write_lock.release()
-
-        self.read_lock.acquire()
-        self.read_lock.notifyAll()
-        self.read_lock.release()
-
-    def enqueue(self, next):
-        self.write_lock.acquire()
-        while self.next_in_line:
-            if self.die:
-                return
-            self.write_lock.wait()
-
-        assert( not self.next_in_line )
-        self.next_in_line = next
-        self.write_lock.release()
-        self.read_lock.acquire()
-        self.read_lock.notify()
-        self.read_lock.release()
-
-    def dequeue(self):
-        self.read_lock.acquire()
-        while not self.next_in_line:
-            if self.die:
-                return
-            self.read_lock.wait()
-
-        result = self.next_in_line
-
-        self.next_in_line = None
-        self.read_lock.release()
-        self.write_lock.acquire()
-        self.write_lock.notify()
-        self.write_lock.release()
-
-        if isinstance(result, EndOfChanges):
-            return None
-
-        return result
-
-class ChangesToImport(object):
-    """A changes file to be enqueued to be processed"""
-    def __init__(self, checkdir, changesfile, count):
-        self.dirpath = checkdir
-        self.changesfile = changesfile
-        self.count = count
-
-    def __str__(self):
-        return "#%d: %s in %s" % (self.count, self.changesfile, self.dirpath)
-
-class ChangesGenerator(threading.Thread):
-    """enqueues changes files to be imported"""
-    def __init__(self, parent, queue):
-        threading.Thread.__init__(self)
-        self.queue = queue
-        self.session = DBConn().session()
-        self.parent = parent
-        self.die = False
-
-    def plsDie(self):
-        self.die = True
-
-    def run(self):
-        cnf = Config()
-        count = 1
-
-        dirs = []
-        dirs.append(cnf['Dir::Done'])
-
-        for queue_name in [ "byhand", "new", "proposedupdates", "oldproposedupdates" ]:
-            queue = get_policy_queue(queue_name)
-            if queue:
-                dirs.append(os.path.abspath(queue.path))
-            else:
-                warn("Could not find queue %s in database" % queue_name)
-
-        for checkdir in dirs:
-            if os.path.exists(checkdir):
-                print "Looking into %s" % (checkdir)
-
-                for dirpath, dirnames, filenames in os.walk(checkdir, topdown=True):
-                    if not filenames:
-                        # Empty directory (or only subdirectories), next
-                        continue
-
-                    for changesfile in filenames:
-                        try:
-                            if not changesfile.endswith(".changes"):
-                                # Only interested in changes files.
-                                continue
-                            count += 1
-
-                            if not get_dbchange(changesfile, self.session):
-                                to_import = ChangesToImport(dirpath, changesfile, count)
-                                if self.die:
-                                    return
-                                self.queue.enqueue(to_import)
-                        except KeyboardInterrupt:
-                            print("got Ctrl-c in enqueue thread.  terminating")
-                            self.parent.plsDie()
-                            sys.exit(1)
-
-        self.queue.enqueue(EndOfChanges())
-
-class ImportThread(threading.Thread):
-    def __init__(self, parent, queue):
-        threading.Thread.__init__(self)
-        self.queue = queue
-        self.session = DBConn().session()
-        self.parent = parent
-        self.die = False
-
-    def plsDie(self):
-        self.die = True
-
-    def run(self):
-        while True:
-            try:
-                if self.die:
-                    return
-                to_import = self.queue.dequeue()
-                if not to_import:
-                    return
-
-                print( "Directory %s, file %7d, (%s)" % (to_import.dirpath[-10:], to_import.count, to_import.changesfile) )
-
-                changes = Changes()
-                changes.changes_file = to_import.changesfile
-                changesfile = os.path.join(to_import.dirpath, to_import.changesfile)
-                changes.changes = parse_changes(changesfile, signing_rules=-1)
-                changes.changes["fingerprint"] = check_signature(changesfile)
-                changes.add_known_changes(to_import.dirpath, session=self.session)
-                self.session.commit()
-
-            except InvalidDscError as line:
-                warn("syntax error in .dsc file '%s', line %s." % (f, line))
-
-            except ChangesUnicodeError:
-                warn("found invalid changes file, not properly utf-8 encoded")
-
-            except KeyboardInterrupt:
-                print("Caught C-c; on ImportThread. terminating.")
-                self.parent.plsDie()
-                sys.exit(1)
-
-            except:
-                self.parent.plsDie()
-                sys.exit(1)
-
-class ImportKnownChanges(object):
-    def __init__(self,num_threads):
-        self.queue = OneAtATime()
-        self.threads = [ ChangesGenerator(self,self.queue) ]
-
-        for i in range(num_threads):
-            self.threads.append( ImportThread(self,self.queue) )
-
-        try:
-            for thread in self.threads:
-                thread.start()
-
-        except KeyboardInterrupt:
-            print("Caught C-c; terminating.")
-            warn("Caught C-c; terminating.")
-            self.plsDie()
-
-    def plsDie(self):
-        traceback.print_stack90
-        for thread in self.threads:
-            print( "STU: before ask %s to die" % thread )
-            thread.plsDie()
-            print( "STU: after ask %s to die" % thread )
-
-        self.threads=[]
-        sys.exit(1)
-
-
-def main():
-    cnf = Config()
-
-    arguments = [('h',"help", "%s::%s" % (options_prefix,"Help")),
-                 ('j',"concurrency", "%s::%s" % (options_prefix,"Concurrency"),"HasArg"),
-                 ('q',"quiet", "%s::%s" % (options_prefix,"Quiet")),
-                 ('v',"verbose", "%s::%s" % (options_prefix,"Verbose")),
-                ]
-
-    args = apt_pkg.parse_commandline(cnf.Cnf, arguments,sys.argv)
-
-    num_threads = 1
-
-    if len(args) > 0:
-        usage()
-
-    if cnf.has_key("%s::%s" % (options_prefix,"Help")):
-        usage()
-
-    level=logging.INFO
-    if cnf.has_key("%s::%s" % (options_prefix,"Quiet")):
-        level=logging.ERROR
-
-    elif cnf.has_key("%s::%s" % (options_prefix,"Verbose")):
-        level=logging.DEBUG
-
-
-    logging.basicConfig( level=level,
-                         format='%(asctime)s %(levelname)s %(message)s',
-                         stream = sys.stderr )
-
-    if Config().has_key( "%s::%s" %(options_prefix,"Concurrency")):
-        num_threads = int(Config()[ "%s::%s" %(options_prefix,"Concurrency")])
-
-    ImportKnownChanges(num_threads)
-
-
-
-
-if __name__ == '__main__':
-    main()
diff --git a/dak/import_new_files.py b/dak/import_new_files.py
deleted file mode 100755
index 7a29467..0000000
--- a/dak/import_new_files.py
+++ /dev/null
@@ -1,187 +0,0 @@
-#!/usr/bin/env python
-# coding=utf8
-
-"""
-Import known_changes files
-
-@contact: Debian FTP Master <ftpmaster@debian.org>
-@copyright: 2009  Mike O'Connor <stew@debian.org>
-@license: GNU General Public License version 2 or later
-"""
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-
-################################################################################
-
-
-################################################################################
-
-import sys
-import os
-import logging
-import threading
-import glob
-import apt_pkg
-from daklib.dbconn import DBConn, get_dbchange, get_policy_queue, session_wrapper, ChangePendingFile, get_location, check_poolfile
-from daklib.config import Config
-from daklib.queue import Upload
-from daklib.utils import poolify
-
-# where in dak.conf all of our configuration will be stowed
-options_prefix = "NewFiles"
-options_prefix = "%s::Options" % options_prefix
-
-log = logging.getLogger()
-
-################################################################################
-
-
-def usage (exit_code=0):
-    print """Usage: dak import-new-files [options]
-
-OPTIONS
-     -v, --verbose
-        show verbose information messages
-
-     -q, --quiet
-        supress all output but errors
-
-"""
-    sys.exit(exit_code)
-
-class ImportNewFiles(object):
-    @session_wrapper
-    def __init__(self, session=None):
-        cnf = Config()
-        try:
-            newq = get_policy_queue('new', session)
-            for changes_fn in glob.glob(newq.path + "/*.changes"):
-                changes_bn = os.path.basename(changes_fn)
-                chg = get_dbchange(changes_bn, session)
-
-                u = Upload()
-                success = u.load_changes(changes_fn)
-                u.pkg.changes_file = changes_bn
-                u.check_hashes()
-
-                if not chg:
-                    chg = u.pkg.add_known_changes(newq.path, newq.policy_queue_id, session)
-                    session.add(chg)
-
-                if not success:
-                    log.critical("failed to load %s" % changes_fn)
-                    sys.exit(1)
-                else:
-                    log.critical("ACCLAIM: %s" % changes_fn)
-
-                files=[]
-                for chg_fn in u.pkg.files.keys():
-                    try:
-                        f = open(os.path.join(newq.path, chg_fn))
-                        cpf = ChangePendingFile()
-                        cpf.filename = chg_fn
-                        cpf.size = u.pkg.files[chg_fn]['size']
-                        cpf.md5sum = u.pkg.files[chg_fn]['md5sum']
-
-                        if u.pkg.files[chg_fn].has_key('sha1sum'):
-                            cpf.sha1sum = u.pkg.files[chg_fn]['sha1sum']
-                        else:
-                            log.warning("Having to generate sha1sum for %s" % chg_fn)
-                            f.seek(0)
-                            cpf.sha1sum = apt_pkg.sha1sum(f)
-
-                        if u.pkg.files[chg_fn].has_key('sha256sum'):
-                            cpf.sha256sum = u.pkg.files[chg_fn]['sha256sum']
-                        else:
-                            log.warning("Having to generate sha256sum for %s" % chg_fn)
-                            f.seek(0)
-                            cpf.sha256sum = apt_pkg.sha256sum(f)
-
-                        session.add(cpf)
-                        files.append(cpf)
-                        f.close()
-                    except IOError:
-                        # Can't find the file, try to look it up in the pool
-                        poolname = poolify(u.pkg.changes["source"], u.pkg.files[chg_fn]["component"])
-                        l = get_location(cnf["Dir::Pool"], u.pkg.files[chg_fn]["component"], session=session)
-                        if not l:
-                            log.critical("ERROR: Can't find location for %s (component %s)" % (chg_fn, u.pkg.files[chg_fn]["component"]))
-
-                        found, poolfile = check_poolfile(os.path.join(poolname, chg_fn),
-                                                         u.pkg.files[chg_fn]['size'],
-                                                         u.pkg.files[chg_fn]["md5sum"],
-                                                         l.location_id,
-                                                         session=session)
-
-                        if found is None:
-                            log.critical("ERROR: Found multiple files for %s in pool" % chg_fn)
-                            sys.exit(1)
-                        elif found is False and poolfile is not None:
-                            log.critical("ERROR: md5sum / size mismatch for %s in pool" % chg_fn)
-                            sys.exit(1)
-                        else:
-                            if poolfile is None:
-                                log.critical("ERROR: Could not find %s in pool" % chg_fn)
-                                sys.exit(1)
-                            else:
-                                chg.poolfiles.append(poolfile)
-
-
-                chg.files = files
-
-
-            session.commit()
-
-        except KeyboardInterrupt:
-            print("Caught C-c; terminating.")
-            utils.warn("Caught C-c; terminating.")
-            self.plsDie()
-
-
-def main():
-    cnf = Config()
-
-    arguments = [('h',"help", "%s::%s" % (options_prefix,"Help")),
-                 ('q',"quiet", "%s::%s" % (options_prefix,"Quiet")),
-                 ('v',"verbose", "%s::%s" % (options_prefix,"Verbose")),
-                ]
-
-    args = apt_pkg.parse_commandline(cnf.Cnf, arguments,sys.argv)
-
-    num_threads = 1
-
-    if len(args) > 0:
-        usage(1)
-
-    if cnf.has_key("%s::%s" % (options_prefix,"Help")):
-        usage(0)
-
-    level=logging.INFO
-    if cnf.has_key("%s::%s" % (options_prefix,"Quiet")):
-        level=logging.ERROR
-
-    elif cnf.has_key("%s::%s" % (options_prefix,"Verbose")):
-        level=logging.DEBUG
-
-
-    logging.basicConfig( level=level,
-                         format='%(asctime)s %(levelname)s %(message)s',
-                         stream = sys.stderr )
-
-    ImportNewFiles()
-
-
-if __name__ == '__main__':
-    main()
-- 
1.7.2.5



Reply to: