[dak/master] Improve dak/import.py.
---
dak/import.py | 145 ++++++++++++++++++++++++++++++++++++++++++++++-----------
1 files changed, 117 insertions(+), 28 deletions(-)
diff --git a/dak/import.py b/dak/import.py
index 0e11647..c972293 100644
--- a/dak/import.py
+++ b/dak/import.py
@@ -23,6 +23,7 @@ import sys
from daklib.dbconn import *
import daklib.archive
import daklib.config
+import daklib.daklog
import daklib.upload
import daklib.regexes
@@ -30,33 +31,44 @@ def usage():
print """Usage:
dak import <suite> <component> <files...>
+dak import -D|--dump <file> <suite> <component>
+dak import -E|--export-dump <suite> <component>
+
+WARNING: This command does no sanity checks. Only use it on trusted packages.
Options:
-h, --help: show this help message
-a, --add-overrides: add missing overrides automatically
-c, --changed-by: Changed-By for imported source packages
(default: maintainer)
+ -D, --dump <file>: Import all files listed in <file>. The format
+ is described below.
+ -E, --export-dump: Export list of files in the format required
+ by dak import --dump.
-s, --ignore-signature: ignore signature for imported source packages
-WARNING: This command does no sanity checks. Only use it on trusted packages.
+File format used by --dump:
+
+ <filename>:<md5>:<sha1>:<sha256>:[<fingerprint>]:[<changed-by>]
"""
-def import_source(transaction, suite, component, directory, filename,
- changed_by=None, keyrings=None, require_signature=True,
- add_overrides=False):
+def import_source(log, transaction, suite, component, directory, hashed_file,
+ fingerprint=None, changed_by=None,
+ keyrings=None, require_signature=True, add_overrides=False):
if keyrings is None:
keyrings = []
+ filename = hashed_file.filename
session = transaction.session
- source = daklib.upload.Source.from_file(directory, filename, keyrings, require_signature)
- fingerprint = None
+ source = daklib.upload.Source(directory, [hashed_file], keyrings, require_signature)
if source.valid_signature:
- fingerprint = session.query(Fingerprint).filter_by(fingerprint=source.primary_fingerprint).one()
+ fingerprint = session.query(Fingerprint).filter_by(fingerprint=source.primary_fingerprint).first()
if changed_by is None:
changed_by = source.dsc['Maintainer']
db_changed_by = get_or_set_maintainer(changed_by, session)
transaction.install_source(directory, source, suite, component, db_changed_by, fingerprint=fingerprint)
+ log.log(['import-source', suite.suite_name, component.component_name, filename])
if add_overrides and not session.query(Override).filter_by(suite=suite.get_overridesuite(), component=component, package=source.dsc['Source']).join(OverrideType).filter(OverrideType.overridetype == 'dsc').first():
overridetype = session.query(OverrideType).filter_by(overridetype='dsc').one()
@@ -70,12 +82,15 @@ def import_source(transaction, suite, component, directory, filename,
override = Override(package=control.dsc['Source'], suite=overridesuite, component=component,
section=section, priority=priority, overridetype=overridetype)
session.add(override)
+ log.log(['add-source-override', suite.suite_name, component.component_name, control.dsc['Source'], section.section, priority.priority])
-def import_binary(transaction, suite, component, directory, filename, add_overrides=False):
+def import_binary(log, transaction, suite, component, directory, hashed_file, fingerprint=None, add_overrides=False):
+ filename = hashed_file.filename
session = transaction.session
- binary = daklib.upload.Binary.from_file(directory, filename)
- transaction.install_binary(directory, binary, suite, component)
+ binary = daklib.upload.Binary(directory, hashed_file)
+ transaction.install_binary(directory, binary, suite, component, fingerprint=fingerprint)
+ log.log(['import-binary', suite.suite_name, component.component_name, filename])
if add_overrides and not session.query(Override).filter_by(suite=suite.get_overridesuite(), component=component, package=binary.control['Package']).join(OverrideType).filter(OverrideType.overridetype == binary.type).first():
overridetype = session.query(OverrideType).filter_by(overridetype=binary.type).one()
@@ -86,22 +101,80 @@ def import_binary(transaction, suite, component, directory, filename, add_overri
override = Override(package=binary.control['Package'], suite=overridesuite, component=component,
section=section, priority=priority, overridetype=overridetype)
session.add(override)
+ log.log(['add-binary-override', suite.suite_name, component.component_name, binary.control['Package'], section.section, priority.priority])
-def import_file(transaction, suite, component, directory, filename,
- changed_by=None, keyrings=None, require_signature=False,
+def import_file(log, transaction, suite, component, directory, hashed_file,
+ fingerprint=None, changed_by=None, keyrings=None, require_signature=True,
add_overrides = False):
- if keyrings is None:
- keyrings = []
-
+ filename = hashed_file.filename
if daklib.regexes.re_file_binary.match(filename):
- import_binary(transaction, suite, component, directory, filename, add_overrides)
+ import_binary(log, transaction, suite, component, directory, hashed_file,
+ fingerprint=fingerprint, add_overrides=add_overrides)
elif daklib.regexes.re_file_dsc.match(filename):
- import_source(transaction, suite, component, directory, filename,
- changed_by=changed_by, keyrings=keyrings,
+ import_source(log, transaction, suite, component, directory, hashed_file,
+ fingerprint=fingerprint, changed_by=changed_by, keyrings=keyrings,
require_signature=require_signature, add_overrides=add_overrides)
else:
raise Exception('File is neither source nor binary package: {0}'.format(filename))
+def import_dump(log, transaction, suite, component, fh,
+ keyrings=None, require_signature=True, add_overrides=False):
+ session = transaction.session
+ for line in fh:
+ path, size, md5, sha1, sha256, fpr, changed_by = line.strip().split(':', 6)
+
+ if not changed_by:
+ changed_by = None
+ fingerprint = None
+ if fpr:
+ fingerprint = session.query(Fingerprint).filter_by(fingerprint=fpr).first()
+ if fingerprint is None:
+ print 'W: {0}: unknown fingerprint {1}'.format(filename, fpr)
+
+ directory, filename = os.path.split(os.path.abspath(path))
+ hashed_file = daklib.upload.HashedFile(filename, long(size), md5, sha1, sha256)
+ hashed_file.check(directory)
+
+ import_file(log, transaction, suite, component, directory, hashed_file,
+ fingerprint=fingerprint, changed_by=changed_by,
+ keyrings=keyrings, require_signature=require_signature, add_overrides=add_overrides)
+
+_export_query = r"""
+WITH
+tmp AS
+ (SELECT 1 AS order, s.file AS file_id, s.sig_fpr AS fingerprint_id, s.changedby AS changed_by, sa.suite AS suite_id
+ FROM source s
+ JOIN src_associations sa ON sa.source = s.id
+ UNION
+ SELECT 2 AS order, b.file AS file_id, b.sig_fpr AS fingerprint_id, NULL, ba.suite AS suite_id
+ FROM binaries b
+ JOIN bin_associations ba ON ba.bin = b.id
+ )
+
+SELECT
+ f.filename, f.size::TEXT, f.md5sum, f.sha1sum, f.sha256sum, COALESCE(fpr.fingerprint, ''), COALESCE(m.name, '')
+FROM files f
+JOIN tmp ON f.id = tmp.file_id
+JOIN suite ON suite.id = tmp.suite_id
+JOIN files_archive_map fam ON fam.file_id = f.id AND fam.archive_id = suite.archive_id
+LEFT JOIN fingerprint fpr ON fpr.id = tmp.fingerprint_id
+LEFT JOIN maintainer m ON m.id = tmp.changed_by
+
+WHERE
+ suite.id = :suite_id
+ AND fam.component_id = :component_id
+
+ORDER BY tmp.order, f.filename;
+"""
+
+def export_dump(transaction, suite, component):
+ session = transaction.session
+ query = session.execute(_export_query,
+ {'suite_id': suite.suite_id,
+ 'component_id': component.component_id})
+ for row in query:
+ print ":".join(row)
+
def main(argv=None):
if argv is None:
argv = sys.argv
@@ -110,6 +183,8 @@ def main(argv=None):
('h', 'help', 'Import::Options::Help'),
('a', 'add-overrides', 'Import::Options::AddOverrides'),
('c', 'changed-by', 'Import::Options::ChangedBy', 'HasArg'),
+ ('D', 'dump', 'Import::Options::Dump', 'HasArg'),
+ ('E', 'export-dump', 'Import::Options::Export'),
('s', 'ignore-signature', 'Import::Options::IgnoreSignature'),
]
@@ -118,18 +193,19 @@ def main(argv=None):
argv = apt_pkg.parse_commandline(cnf.Cnf, arguments, argv)
options = cnf.subtree('Import::Options')
- if 'Help' in options or len(argv) < 3:
+ if 'Help' in options or len(argv) < 2:
usage()
sys.exit(0)
suite_name = argv[0]
component_name = argv[1]
- files = argv[2:]
add_overrides = options.find_b('AddOverrides')
require_signature = not options.find_b('IgnoreSignature')
changed_by = options.find('ChangedBy') or None
+ log = daklib.daklog.Logger('import')
+
with daklib.archive.ArchiveTransaction() as transaction:
session = transaction.session
suite = session.query(Suite).filter_by(suite_name=suite_name).one()
@@ -137,14 +213,27 @@ def main(argv=None):
keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority)
keyring_files = [ k.keyring_name for k in keyrings ]
- for f in files:
- directory, filename = os.path.split(os.path.abspath(f))
- import_file(transaction, suite, component, directory, filename,
- changed_by=changed_by,
- keyrings=keyring_files, require_signature=require_signature,
- add_overrides=add_overrides)
-
- transaction.commit()
+ dump = options.find('Dump') or None
+ if options.find_b('Export'):
+ export_dump(transaction, suite, component)
+ transaction.rollback()
+ elif dump is not None:
+ with open(dump, 'r') as fh:
+ import_dump(log, transaction, suite, component, fh, keyring_files,
+ require_signature=require_signature, add_overrides=add_overrides)
+ transaction.commit()
+ else:
+ files = argv[2:]
+ for f in files:
+ directory, filename = os.path.split(os.path.abspath(f))
+ hashed_file = daklib.upload.HashedFile.from_file(directory, filename)
+ import_file(log, transaction, suite, component, directory, hashed_file,
+ changed_by=changed_by,
+ keyrings=keyring_files, require_signature=require_signature,
+ add_overrides=add_overrides)
+ transaction.commit()
+
+ log.close()
if __name__ == '__main__':
main()
--
1.7.2.5
Reply to: