[dak/master 31/50] initial policy queue implementation in process_upload
Signed-off-by: Mark Hymers <mhy@debian.org>
---
dak/process_upload.py | 92 ++++++++++++++++++++++----
daklib/queue_install.py | 168 ++++++++++++++++++++---------------------------
2 files changed, 150 insertions(+), 110 deletions(-)
diff --git a/dak/process_upload.py b/dak/process_upload.py
index cf1594a..97117b8 100755
--- a/dak/process_upload.py
+++ b/dak/process_upload.py
@@ -125,21 +125,53 @@ Checks Debian packages from Incoming
## pu: create files for BTS
## pu: create entry in queue_build
## pu: check overrides
-import errno
+
+# Integrity checks
+## GPG
+## Parsing changes (check for duplicates)
+## Parse dsc
+## file list checks
+
+# New check layout (TODO: Implement)
+## Permission checks
+### suite mappings
+### ACLs
+### version checks (suite)
+### override checks
+
+## Source checks
+### copy orig
+### unpack
+### BTS changelog
+### src contents
+### lintian
+### urgency log
+
+## Binary checks
+### timestamps
+### control checks
+### src relation check
+### contents
+
+## Database insertion (? copy from stuff)
+### BYHAND / NEW / Policy queues
+### Pool
+
+## Queue builds
+
+from errno import EACCES, EAGAIN
import fcntl
import os
import sys
-#from datetime import datetime
import traceback
import apt_pkg
+from sqlalchemy.orm.exc import NoResultFound
from daklib import daklog
from daklib.queue import *
from daklib.queue_install import *
from daklib import utils
from daklib.dbconn import *
-#from daklib.dak_exceptions import *
-#from daklib.regexes import re_default_answer, re_issource, re_fdnic
from daklib.urgencylog import UrgencyLog
from daklib.summarystats import SummaryStats
from daklib.holding import Holding
@@ -164,13 +196,14 @@ def usage (exit_code=0):
###############################################################################
-def action(u):
+def action(u, dbc):
cnf = Config()
holding = Holding()
+ session = DBConn().session()
# changes["distribution"] may not exist in corner cases
# (e.g. unreadable changes files)
- if not u.pkg.changes.has_key("distribution") or not isinstance(u.pkg.changes["distribution"], DictType):
+ if not u.pkg.changes.has_key("distribution") or not isinstance(u.pkg.changes["distribution"], dict):
u.pkg.changes["distribution"] = {}
(summary, short_summary) = u.build_summaries()
@@ -193,6 +226,8 @@ def action(u):
if Options["Automatic"]:
answer = 'R'
else:
+ # Are we headed for NEW / BYHAND / AUTOBYHAND?
+ # Note that policy queues are no longer handled here
qu = determine_target(u)
if qu:
print "%s for %s\n%s%s" % ( qu.upper(), ", ".join(u.pkg.changes["distribution"].keys()), pi, summary)
@@ -205,10 +240,38 @@ def action(u):
if Options["Automatic"]:
answer = queuekey
else:
- print "ACCEPT\n" + pi + summary,
- prompt = "[A]ccept, Skip, Quit ?"
- if Options["Automatic"]:
- answer = 'A'
+ # TODO: FIX THIS BY HAVING ADDED TO changes TABLE earlier
+ try:
+ dbc = session.query(DBChange).filter_by(changesname=os.path.basename(u.pkg.changes_file)).one()
+ except NoResultFound, e:
+ dbc = None
+
+ # Does suite have a policy_queue configured
+ divert = False
+ for s in u.pkg.changes["distribution"].keys():
+ suite = get_suite(s, session)
+ if suite.policy_queue:
+ if not dbc or dbc.approved_for_id != su.policy_queue.policy_queue_id:
+ # This routine will check whether the upload is a binary
+ # upload when the source is already in the target suite. If
+ # so, we skip the policy queue, otherwise we go there.
+ divert = package_to_suite(u, suite.suite_name, session=session)
+ if divert:
+ print "%s for %s\n%s%s" % ( su.policy_queue.queue_name.upper(),
+ ", ".join(u.pkg.changes["distribution"].keys()),
+ pi, summary)
+ queuekey = "P"
+ prompt = "[P]olicy, Skip, Quit ?"
+ policyqueue = su.policy_queue
+ if Options["Automatic"]:
+ answer = 'P'
+ break
+
+ if not divert:
+ print "ACCEPT\n" + pi + summary,
+ prompt = "[A]ccept, Skip, Quit ?"
+ if Options["Automatic"]:
+ answer = 'A'
while prompt.find(answer) == -1:
answer = utils.our_raw_input(prompt)
@@ -217,8 +280,6 @@ def action(u):
answer = m.group(1)
answer = answer[:1].upper()
- session = DBConn().session()
-
if answer == 'R':
os.chdir(u.pkg.directory)
u.do_reject(0, pi)
@@ -227,6 +288,10 @@ def action(u):
u.accept(summary, short_summary, session)
u.check_override()
u.remove()
+ elif answer == 'P':
+ u.pkg.add_known_changes(holding.holding_dir, session)
+ package_to_queue(u, summary, short_summary, policyqueue, perms=0664, announce=None)
+ u.remove()
elif answer == queuekey:
u.pkg.add_known_changes(holding.holding_dir, session)
QueueInfo[qu]["process"](u, summary, short_summary, session)
@@ -252,6 +317,9 @@ def process_it(changes_file):
holding = Holding()
+ # TODO: Actually implement using pending* tables so that we don't lose track
+ # of what is where
+
u = Upload()
u.pkg.changes_file = changes_file
u.pkg.directory = os.getcwd()
diff --git a/daklib/queue_install.py b/daklib/queue_install.py
index c8fa39e..d6b651b 100644
--- a/daklib/queue_install.py
+++ b/daklib/queue_install.py
@@ -31,60 +31,37 @@ from daklib import utils
from daklib.dbconn import *
from daklib.config import Config
-###############################################################################
-
-def determine_target(u):
- cnf = Config()
-
- queues = [ "New", "Autobyhand", "Byhand" ]
- if cnf.FindB("Dinstall::SecurityQueueHandling"):
- queues += [ "Unembargo", "Embargo" ]
- else:
- queues += [ "OldStableUpdate", "StableUpdate" ]
-
- target = None
- for q in queues:
- if QueueInfo[q]["is"](u):
- target = q
- break
-
- return target
-
################################################################################
-def package_to_suite(u, suite):
+def package_to_suite(u, suite_name, session):
if not u.pkg.changes["distribution"].has_key(suite):
return False
ret = True
if not u.pkg.changes["architecture"].has_key("source"):
- s = DBConn().session()
- q = s.query(SrcAssociation.sa_id)
+ q = session.query(SrcAssociation.sa_id)
q = q.join(Suite).filter_by(suite_name=suite)
q = q.join(DBSource).filter_by(source=u.pkg.changes['source'])
q = q.filter_by(version=u.pkg.changes['version']).limit(1)
# NB: Careful, this logic isn't what you would think it is
- # Source is already in {old-,}proposed-updates so no need to hold
- # Instead, we don't move to the holding area, we just do an ACCEPT
+ # Source is already in the target suite so no need to go to policy
+ # Instead, we don't move to the policy area, we just do an ACCEPT
if q.count() > 0:
ret = False
- s.close()
-
return ret
-def package_to_queue(u, summary, short_summary, queue, perms=0660, build=True, announce=None):
+def package_to_queue(u, summary, short_summary, queue, perms=0660, announce=None):
cnf = Config()
- dir = cnf["Dir::Queue::%s" % queue]
+ dir = queue.path
- print "Moving to %s holding area" % queue.upper()
- u.logger.log(["Moving to %s" % queue, u.pkg.changes_file])
+ print "Moving to %s policy queue" % queue.queue_name.upper()
+ u.logger.log(["Moving to %s" % queue.queue_name, u.pkg.changes_file])
u.move_to_dir(dir, perms=perms)
- if build:
- get_or_set_queue(queue.lower()).autobuild_upload(u.pkg, dir)
+ # TODO: Put building logic in here? We used to take a build=bool argument
# Check for override disparities
u.check_override()
@@ -100,64 +77,49 @@ def package_to_queue(u, summary, short_summary, queue, perms=0660, build=True, a
################################################################################
-def is_unembargo(u):
- session = DBConn().session()
- cnf = Config()
-
- q = session.execute("SELECT package FROM disembargo WHERE package = :source AND version = :version", u.pkg.changes)
- if q.rowcount > 0:
- session.close()
- return True
-
- oldcwd = os.getcwd()
- os.chdir(cnf["Dir::Queue::Disembargo"])
- disdir = os.getcwd()
- os.chdir(oldcwd)
-
- ret = False
-
- if u.pkg.directory == disdir:
- if u.pkg.changes["architecture"].has_key("source"):
- session.execute("INSERT INTO disembargo (package, version) VALUES (:package, :version)", u.pkg.changes)
- session.commit()
-
- ret = True
-
- session.close()
-
- return ret
-
-def queue_unembargo(u, summary, short_summary, session=None):
- return package_to_queue(u, summary, short_summary, "Unembargoed",
- perms=0660, build=True, announce='process-unchecked.accepted')
-
-################################################################################
-
-def is_embargo(u):
- # if embargoed queues are enabled always embargo
- return True
-
-def queue_embargo(u, summary, short_summary, session=None):
- return package_to_queue(u, summary, short_summary, "Unembargoed",
- perms=0660, build=True, announce='process-unchecked.accepted')
-
-################################################################################
-
-def is_stableupdate(u):
- return package_to_suite(u, 'proposed-updates')
-
-def do_stableupdate(u, summary, short_summary, session=None):
- return package_to_queue(u, summary, short_summary, "ProposedUpdates",
- perms=0664, build=False, announce=None)
-
-################################################################################
-
-def is_oldstableupdate(u):
- return package_to_suite(u, 'oldstable-proposed-updates')
-
-def do_oldstableupdate(u, summary, short_summary, session=None):
- return package_to_queue(u, summary, short_summary, "OldProposedUpdates",
- perms=0664, build=False, announce=None)
+# TODO: This logic needs to be replaced with policy queues before we upgrade
+# security master
+
+#def is_unembargo(u):
+# session = DBConn().session()
+# cnf = Config()
+#
+# q = session.execute("SELECT package FROM disembargo WHERE package = :source AND version = :version", u.pkg.changes)
+# if q.rowcount > 0:
+# session.close()
+# return True
+#
+# oldcwd = os.getcwd()
+# os.chdir(cnf["Dir::Queue::Disembargo"])
+# disdir = os.getcwd()
+# os.chdir(oldcwd)
+#
+# ret = False
+#
+# if u.pkg.directory == disdir:
+# if u.pkg.changes["architecture"].has_key("source"):
+# session.execute("INSERT INTO disembargo (package, version) VALUES (:package, :version)", u.pkg.changes)
+# session.commit()
+#
+# ret = True
+#
+# session.close()
+#
+# return ret
+#
+#def queue_unembargo(u, summary, short_summary, session=None):
+# return package_to_queue(u, summary, short_summary, "Unembargoed",
+# perms=0660, build=True, announce='process-unchecked.accepted')
+#
+#################################################################################
+#
+#def is_embargo(u):
+# # if embargoed queues are enabled always embargo
+# return True
+#
+#def queue_embargo(u, summary, short_summary, session=None):
+# return package_to_queue(u, summary, short_summary, "Unembargoed",
+# perms=0660, build=True, announce='process-unchecked.accepted')
################################################################################
@@ -275,12 +237,22 @@ def acknowledge_new(u, summary, short_summary, session=None):
# q-unapproved hax0ring
QueueInfo = {
- "New": { "is": is_new, "process": acknowledge_new },
- "Autobyhand" : { "is" : is_autobyhand, "process": do_autobyhand },
- "Byhand" : { "is": is_byhand, "process": do_byhand },
- "OldStableUpdate" : { "is": is_oldstableupdate,
- "process": do_oldstableupdate },
- "StableUpdate" : { "is": is_stableupdate, "process": do_stableupdate },
- "Unembargo" : { "is": is_unembargo, "process": queue_unembargo },
- "Embargo" : { "is": is_embargo, "process": queue_embargo },
+ "new": { "is": is_new, "process": acknowledge_new },
+ "autobyhand" : { "is" : is_autobyhand, "process": do_autobyhand },
+ "byhand" : { "is": is_byhand, "process": do_byhand },
}
+
+def determine_target(u):
+ cnf = Config()
+
+ # Statically handled queues
+ target = None
+
+ for q in QueueInfo.keys():
+ if QueueInfo[q]["is"](u):
+ target = q
+
+ return target
+
+###############################################################################
+
--
1.6.3.3
Reply to: