[dak/master] Add process-commands program and activate new DM implementation.
For now packages that use DMUA in the source package are still accepted.
---
dak/dak.py | 2 +
dak/dakdb/update89.py | 53 +++++++++
dak/process-commands.py | 82 +++++++++++++
dak/update_db.py | 2 +-
daklib/checks.py | 7 +-
daklib/command.py | 222 +++++++++++++++++++++++++++++++++++
daklib/dbconn.py | 23 ++++
daklib/gpg.py | 17 +++
templates/process-command.processed | 12 ++
9 files changed, 417 insertions(+), 3 deletions(-)
create mode 100644 dak/dakdb/update89.py
create mode 100644 dak/process-commands.py
create mode 100644 daklib/command.py
create mode 100644 templates/process-command.processed
diff --git a/dak/dak.py b/dak/dak.py
index 306137a..c2a502c 100755
--- a/dak/dak.py
+++ b/dak/dak.py
@@ -71,6 +71,8 @@ def init():
"Process NEW and BYHAND packages"),
("process-upload",
"Process packages in queue/unchecked"),
+ ("process-commands",
+ "Process command files (*.dak-commands)"),
("process-policy",
"Process packages in policy queues from COMMENTS files"),
diff --git a/dak/dakdb/update89.py b/dak/dakdb/update89.py
new file mode 100644
index 0000000..2ecccf5
--- /dev/null
+++ b/dak/dakdb/update89.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# coding=utf8
+
+"""
+add table to keep track of seen signatures
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2012 Ansgar Burchardt <ansgar@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+import psycopg2
+from daklib.dak_exceptions import DBUpdateError
+from daklib.config import Config
+
+################################################################################
+def do_update(self):
+ print __doc__
+ try:
+ cnf = Config()
+
+ c = self.db.cursor()
+
+ c.execute("""CREATE TABLE signature_history (
+ fingerprint TEXT NOT NULL,
+ signature_timestamp TIMESTAMP NOT NULL,
+ contents_sha1 TEXT NOT NULL,
+ seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ PRIMARY KEY (signature_timestamp, fingerprint, contents_sha1)
+ )""")
+
+ c.execute("UPDATE config SET value = '89' WHERE name = 'db_revision'")
+ self.db.commit()
+
+ except psycopg2.ProgrammingError as msg:
+ self.db.rollback()
+ raise DBUpdateError('Unable to apply sick update 89, rollback issued. Error message: {0}'.format(msg))
diff --git a/dak/process-commands.py b/dak/process-commands.py
new file mode 100644
index 0000000..9bc2d3f
--- /dev/null
+++ b/dak/process-commands.py
@@ -0,0 +1,82 @@
+#! /usr/bin/env python
+#
+# Copyright (C) 2012, Ansgar Burchardt <ansgar@debian.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import apt_pkg
+import datetime
+import os
+import sys
+
+from daklib.config import Config
+from daklib.command import CommandError, CommandFile
+from daklib.daklog import Logger
+from daklib.fstransactions import FilesystemTransaction
+from daklib.utils import find_next_free
+
+def usage():
+ print """Usage: dak command <command-file>...
+
+process command files
+"""
+
+def main(argv=None):
+ if argv is None:
+ argv = sys.argv
+
+ arguments = [('h', 'help', 'Process-Commands::Options::Help'),
+ ('d', 'directory', 'Process-Commands::Options::Directory', 'HasArg')]
+
+ cnf = Config()
+ cnf['Command::Options::Dummy'] = ''
+ filenames = apt_pkg.parse_commandline(cnf.Cnf, arguments, argv)
+ options = cnf.subtree('Command::Options')
+
+ if 'Help' in options or (len(filenames) == 0 and 'Directory' not in options):
+ usage()
+ sys.exit(0)
+
+ log = Logger('command')
+
+ now = datetime.datetime.now()
+ donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
+ rejectdir = cnf['Dir::Reject']
+
+ if len(filenames) == 0:
+ filenames = [ fn for fn in os.listdir(options['Directory']) if fn.endswith('.dak-commands') ]
+
+ for fn in filenames:
+ basename = os.path.basename(fn)
+ if not fn.endswith('.dak-commands'):
+ log.log(['unexpected filename', basename])
+ continue
+
+ command = CommandFile(fn, log)
+ if command.evaluate():
+ log.log(['moving to done', basename])
+ dst = find_next_free(os.path.join(donedir, basename))
+ else:
+ log.log(['moving to reject', basename])
+ dst = find_next_free(os.path.join(rejectdir, basename))
+
+ with FilesystemTransaction() as fs:
+ fs.move(fn, dst, mode=0o644)
+ fs.commit()
+
+ log.close()
+
+if __name__ == '__main__':
+ main()
diff --git a/dak/update_db.py b/dak/update_db.py
index a27c32e..97efd3f 100755
--- a/dak/update_db.py
+++ b/dak/update_db.py
@@ -46,7 +46,7 @@ from daklib.daklog import Logger
################################################################################
Cnf = None
-required_database_schema = 88
+required_database_schema = 89
################################################################################
diff --git a/daklib/checks.py b/daklib/checks.py
index 63ea10f..cae801e 100644
--- a/daklib/checks.py
+++ b/daklib/checks.py
@@ -464,8 +464,11 @@ class ACLCheck(Check):
# XXX: Drop DMUA part here and switch to new implementation.
# XXX: Send warning mail once users can set the new DMUA flag
dmua_status, dmua_reason = self._check_dmua(upload)
- if not dmua_status:
- return False, dmua_reason
+ if acl_per_source is None:
+ if not dmua_status:
+ return False, dmua_reason
+ else:
+ upload.warn('DM flag not set, but accepted as DMUA was set.')
#if acl_per_source is None:
# return False, "not allowed to upload source package '{0}'".format(source_name)
if acl.deny_per_source and acl_per_source is not None:
diff --git a/daklib/command.py b/daklib/command.py
new file mode 100644
index 0000000..31551d7
--- /dev/null
+++ b/daklib/command.py
@@ -0,0 +1,222 @@
+"""module to handle command files
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2012, Ansgar Burchardt <ansgar@debian.org>
+@license: GPL-2+
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import apt_pkg
+import os
+import re
+import tempfile
+
+from daklib.config import Config
+from daklib.dbconn import *
+from daklib.gpg import SignedFile
+from daklib.regexes import re_field_package
+from daklib.textutils import fix_maintainer
+from daklib.utils import gpg_get_key_addresses, send_mail, TemplateSubst
+
+class CommandError(Exception):
+ pass
+
+class CommandFile(object):
+ def __init__(self, path, log=None):
+ if log is None:
+ from daklib.daklog import Logger
+ log = Logger()
+ self.cc = []
+ self.result = []
+ self.log = log
+ self.path = path
+ self.filename = os.path.basename(path)
+
+ def _check_replay(self, signed_file, session):
+ """check for replays
+
+ @note: Will commit changes to the database.
+
+ @type signed_file: L{daklib.gpg.SignedFile}
+
+ @param session: database session
+ """
+ # Mark commands file as seen to prevent replays.
+ signature_history = SignatureHistory.from_signed_file(signed_file)
+ session.add(signature_history)
+ session.commit()
+
+ def _evaluate_sections(self, sections, session):
+ session.rollback()
+ try:
+ sections.next()
+ section = sections.section
+
+ action = section.get('Action', None)
+ if action is None:
+ raise CommandError('Encountered section without Action field')
+ self.result.append('Action: {0}'.format(action))
+
+ if action == 'dm':
+ self.action_dm(self.fingerprint, section, session)
+ else:
+ raise CommandError('Unknown action: {0}'.format(action))
+ except StopIteration:
+ pass
+ finally:
+ session.rollback()
+
+ def _notify_uploader(self):
+ cnf = Config()
+
+ bcc = 'X-DAK: dak process-command'
+ if 'Dinstall::Bcc' in cnf:
+ bcc = '{0}\nBcc: {1}'.format(bcc, cnf['Dinstall::Bcc'])
+
+ cc = set(fix_maintainer(address)[1] for address in self.cc)
+
+ subst = {
+ '__DAK_ADDRESS__': cnf['Dinstall::MyEmailAddress'],
+ '__MAINTAINER_TO__': fix_maintainer(self.uploader)[1],
+ '__CC__': ", ".join(cc),
+ '__BCC__': bcc,
+ '__RESULTS__': "\n".join(self.result),
+ '__FILENAME__': self.filename,
+ }
+
+ message = TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-command.processed'))
+
+ send_mail(message)
+
+ def evaluate(self):
+ """evaluate commands file
+
+ @rtype: bool
+ @returns: C{True} if the file was processed sucessfully,
+ C{False} otherwise
+ """
+ result = True
+
+ session = DBConn().session()
+
+ keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority)
+ keyring_files = [ k.keyring_name for k in keyrings ]
+
+ raw_contents = open(self.path, 'r').read()
+ signed_file = SignedFile(raw_contents, keyring_files)
+ if not signed_file.valid:
+ self.log.log(['invalid signature', self.filename])
+ return False
+
+ self.fingerprint = session.query(Fingerprint).filter_by(fingerprint=signed_file.primary_fingerprint).one()
+ if self.fingerprint.keyring is None:
+ self.log.log(['singed by key in unknown keyring', self.filename])
+ return False
+ assert self.fingerprint.keyring.active
+
+ self.log.log(['processing', self.filename, 'signed-by={0}'.format(self.fingerprint.fingerprint)])
+
+ with tempfile.TemporaryFile() as fh:
+ fh.write(signed_file.contents)
+ fh.seek(0)
+ sections = apt_pkg.TagFile(fh)
+
+ self.uploader = None
+ addresses = gpg_get_key_addresses(self.fingerprint.fingerprint)
+ if len(addresses) > 0:
+ self.uploader = addresses[0]
+
+ try:
+ sections.next()
+ section = sections.section
+ if 'Uploader' in section:
+ self.uploader = section['Uploader']
+ # TODO: Verify first section has valid Archive field
+ if 'Archive' not in section:
+ raise CommandError('No Archive field in first section.')
+
+ # TODO: send mail when we detected a replay.
+ self._check_replay(signed_file, session)
+
+ self._evaluate_sections(sections, session)
+ self.result.append('')
+ except Exception as e:
+ self.log.log(['ERROR', e])
+ self.result.append("There was an error processing this section:\n{0}".format(e))
+ result = False
+
+ self._notify_uploader()
+
+ session.close()
+ self.log.log(['done', self.filename])
+
+ return result
+
+ def _split_packages(self, value):
+ names = value.split()
+ for name in names:
+ if not re_field_package.match(name):
+ raise CommandError('Invalid package name "{0}"'.format(name))
+ return names
+
+ def action_dm(self, fingerprint, section, session):
+ cnf = Config()
+
+ if 'Command::DM::AdminKeyrings' not in cnf \
+ or 'Command::DM::ACL' not in cnf \
+ or 'Command::DM::Keyrings' not in cnf:
+ raise CommandError('DM command is not configured for this archive.')
+
+ allowed_keyrings = cnf.value_list('Command::DM::AdminKeyrings')
+ if fingerprint.keyring.keyring_name not in allowed_keyrings:
+ raise CommandError('Key {0} is not allowed to set DM'.format(fingerprint.fingerprint))
+
+ acl_name = cnf.get('Command::DM::ACL', 'dm')
+ acl = session.query(ACL).filter_by(name=acl_name).one()
+
+ fpr = session.query(Fingerprint).filter_by(fingerprint=section['Fingerprint']).one()
+ if fpr.keyring is None or fpr.keyring.keyring_name not in cnf.value_list('Command::DM::Keyrings'):
+ raise CommandError('Key {0} is not in DM keyring.'.format(fpr.fingerprint))
+ addresses = gpg_get_key_addresses(fpr.fingerprint)
+ if len(addresses) > 0:
+ self.cc.append(addresses[0])
+
+ self.log.log(['dm', 'fingerprint', fpr.fingerprint])
+ self.result.append('Fingerprint: {0}'.format(fpr.fingerprint))
+ if len(addresses) > 0:
+ self.log.log(['dm', 'uid', addresses[0]])
+ self.result.append('Uid: {0}'.format(addresses[0]))
+
+ for source in self._split_packages(section.get('Allow', '')):
+ if session.query(ACLPerSource).filter_by(acl=acl, fingerprint=fpr, source=source).first() is None:
+ aps = ACLPerSource()
+ aps.acl = acl
+ aps.fingerprint = fpr
+ aps.source = source
+ session.add(aps)
+ self.log.log(['dm', 'allow', fpr.fingerprint, source])
+ self.result.append('Allowed: {0}'.format(source))
+ else:
+ self.result.append('Already-Allowed: {0}'.format(source))
+
+ session.flush()
+
+ for source in self._split_packages(section.get('Deny', '')):
+ session.query(ACLPerSource).filter_by(acl=acl, fingerprint=fpr, source=source).delete()
+ self.log.log(['dm', 'deny', fpr.fingerprint, source])
+ self.result.append('Denied: {0}'.format(source))
+
+ session.commit()
diff --git a/daklib/dbconn.py b/daklib/dbconn.py
index 41121ff..5217462 100644
--- a/daklib/dbconn.py
+++ b/daklib/dbconn.py
@@ -1850,6 +1850,26 @@ __all__.append('get_sections')
################################################################################
+class SignatureHistory(ORMObject):
+ @classmethod
+ def from_signed_file(cls, signed_file):
+ """signature history entry from signed file
+
+ @type signed_file: L{daklib.gpg.SignedFile}
+ @param signed_file: signed file
+
+ @rtype: L{SignatureHistory}
+ """
+ self = cls()
+ self.fingerprint = signed_file.primary_fingerprint
+ self.signature_timestamp = signed_file.signature_timestamp
+ self.contents_sha1 = signed_file.contents_sha1()
+ return self
+
+__all__.append('SignatureHistory')
+
+################################################################################
+
class SrcContents(ORMObject):
def __init__(self, file = None, source = None):
self.file = file
@@ -2544,6 +2564,7 @@ class DBConn(object):
'policy_queue_byhand_file',
'priority',
'section',
+ 'signature_history',
'source',
'source_metadata',
'src_associations',
@@ -2762,6 +2783,8 @@ class DBConn(object):
properties = dict(section_id = self.tbl_section.c.id,
section=self.tbl_section.c.section))
+ mapper(SignatureHistory, self.tbl_signature_history)
+
mapper(DBSource, self.tbl_source,
properties = dict(source_id = self.tbl_source.c.id,
version = self.tbl_source.c.version,
diff --git a/daklib/gpg.py b/daklib/gpg.py
index 865e9bd..828bf64 100644
--- a/daklib/gpg.py
+++ b/daklib/gpg.py
@@ -19,6 +19,8 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+import apt_pkg
+import datetime
import errno
import fcntl
import os
@@ -141,6 +143,17 @@ class SignedFile(object):
return dict( (fd, "".join(read_lines[fd])) for fd in read_lines.keys() )
+ def _parse_date(self, value):
+ """parse date string in YYYY-MM-DD format
+
+ @rtype: L{datetime.datetime}
+ @returns: datetime objects for 0:00 on the given day
+ """
+ year, month, day = value.split('-')
+ date = datetime.date(int(year), int(month), int(day))
+ time = datetime.time(0, 0)
+ return datetime.datetime.combine(date, time)
+
def _parse_status(self, line):
fields = line.split()
if fields[0] != "[GNUPG:]":
@@ -153,6 +166,7 @@ class SignedFile(object):
self.valid = True
self.fingerprint = fields[2]
self.primary_fingerprint = fields[11]
+ self.signature_timestamp = self._parse_date(fields[3])
if fields[1] == "BADARMOR":
raise GpgException("Bad armor.")
@@ -190,4 +204,7 @@ class SignedFile(object):
finally:
os._exit(1)
+ def contents_sha1(self):
+ return apt_pkg.sha1sum(self.contents)
+
# vim: set sw=4 et:
diff --git a/templates/process-command.processed b/templates/process-command.processed
new file mode 100644
index 0000000..979032f
--- /dev/null
+++ b/templates/process-command.processed
@@ -0,0 +1,12 @@
+From: __DAK_ADDRESS__
+To: __MAINTAINER_TO__
+Cc: __CC__
+__BCC__
+X-Debian: DAK
+Precedence: bulk
+MIME-Version: 1.0
+Content-Type: text/plain; charset="utf-8"
+Content-Transfer-Encoding: 8bit
+Subject: Results of processing __FILENAME__
+
+__RESULTS__
--
1.7.2.5
Reply to: