[dak/master] NEW
modified process-new and all related stuff so NEW can now also be processed
during (most of) dinstall. The lock where NEW is forbidden is then down to
some 15 til 20 minutes, instead of 2 hours.
For that process-new moves files into a "staging" directory, from which
cron.unchecked will move them into the accepted queue. As c.u only runs when
it is safe to do so this is fine.
The small part where we still are forbidden to do NEW is between the point where
we run process-accepted and right before we start generating Packages files. As
that is exactly the area daily.lock exists for, process-new now locks on that and
simply refuses to do work then.
Signed-off-by: Joerg Jaspert <joerg@debian.org>
---
config/debian/cron.unchecked | 19 +++++++++
config/debian/dak.conf | 3 +-
config/debian/vars | 1 +
dak/process_new.py | 88 ++++++++++++++++++++++--------------------
daklib/dak_exceptions.py | 3 +-
daklib/queue.py | 23 +++++++++--
6 files changed, 89 insertions(+), 48 deletions(-)
diff --git a/config/debian/cron.unchecked b/config/debian/cron.unchecked
index fc146da..0006f80 100755
--- a/config/debian/cron.unchecked
+++ b/config/debian/cron.unchecked
@@ -23,6 +23,25 @@ cleanup() {
fi
}
+# in case there is anything NEW processed, move it over into the accepted queue
+cd $newstage
+for file in $(find . -maxdepth 1 -mindepth 1 -type f -name \*.changes | sed -e "s,./,," | xargs); do
+ sed '1,/Files:/d' "${file}" | sed '/BEGIN PGP SIGNATURE/,$d' \
+ | while read notused1 notused2 notused3 notused4 NAME; do
+ if [ -z "${NAME}" ]; then
+ # Sometimes there is a newline after the Files:, ignore it
+ continue
+ fi
+ if [ -f "${NAME}" ]; then
+ mv --target-directory="${accepted}" "${NAME}"
+ else
+ log_error "Error, couldn't find file ${NAME} to move to ${accepted}"
+ exit 2
+ fi
+ done
+ mv --target-directory="${accepted}" "${file}" "${file%%.changes}.dak"
+done
+
# only run one cron.unchecked
if lockfile -r3 $LOCKFILE; then
trap cleanup 0
diff --git a/config/debian/dak.conf b/config/debian/dak.conf
index 756a37c..baf140e 100644
--- a/config/debian/dak.conf
+++ b/config/debian/dak.conf
@@ -173,7 +173,7 @@ Clean-Suites
Process-New
{
- AcceptedLockFile "/srv/ftp.debian.org/lock/unchecked.lock";
+ DinstallLockFile "/srv/ftp.debian.org/lock/daily.lock";
LockDir "/srv/ftp.debian.org/lock/new/";
};
@@ -563,6 +563,7 @@ Dir
New "/srv/ftp.debian.org/queue/new/";
Reject "/srv/ftp.debian.org/queue/reject/";
Unchecked "/srv/ftp.debian.org/queue/unchecked/";
+ Newstage "/srv/ftp.debian.org/queue/newstage/";
BTSVersionTrack "/srv/ftp.debian.org/queue/bts_version_track/";
};
};
diff --git a/config/debian/vars b/config/debian/vars
index 09a047b..35e9f0e 100644
--- a/config/debian/vars
+++ b/config/debian/vars
@@ -23,6 +23,7 @@ unchecked=$queuedir/unchecked/
accepted=$queuedir/accepted/
mirrordir=$base/mirror/
incoming=$base/incoming
+newstage=$queuedir/newstage/
ftpgroup=debadmin
diff --git a/dak/process_new.py b/dak/process_new.py
index 52e1f4e..d631b53 100755
--- a/dak/process_new.py
+++ b/dak/process_new.py
@@ -59,7 +59,7 @@ from daklib import logging
from daklib import queue
from daklib import utils
from daklib.regexes import re_no_epoch, re_default_answer, re_isanum
-from daklib.dak_exceptions import CantOpenError, AlreadyLockedError
+from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError
# Globals
Cnf = None #: Configuration, apt_pkg.Configuration
@@ -687,7 +687,12 @@ def do_new():
answer = answer[:1].upper()
if answer == 'A' and not Options["Trainee"]:
- done = add_overrides (new)
+ try:
+ check_daily_lock()
+ done = add_overrides (new)
+ except CantGetLockError:
+ print "Hello? Operator! Give me the number for 911!"
+ print "Dinstall in the locked area, cant process packages, come back later"
elif answer == 'C':
check_pkg()
elif answer == 'E' and not Options["Trainee"]:
@@ -821,22 +826,18 @@ def do_byhand():
################################################################################
-def get_accept_lock():
- retry = 0
- while retry < 10:
- try:
- os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
- retry = 10
- except OSError, e:
- if e.errno == errno.EACCES or e.errno == errno.EEXIST:
- retry += 1
- if (retry >= 10):
- utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
- else:
- print("Unable to get accepted lock (try %d of 10)" % retry)
- time.sleep(60)
- else:
- raise
+def check_daily_lock():
+ """
+ Raises CantGetLockError if the dinstall daily.lock exists.
+ """
+
+ try:
+ os.open(Cnf["Process-New::DinstallLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
+ except OSError, e:
+ if e.errno == errno.EEXIST or e.errno == errno.EACCES:
+ raise CantGetLockError
+
+ os.unlink(Cnf["Process-New::DinstallLockFile"])
@contextlib.contextmanager
@@ -898,7 +899,7 @@ def _accept():
if Options["No-Action"]:
return
(summary, short_summary) = Upload.build_summaries()
- Upload.accept(summary, short_summary)
+ Upload.accept(summary, short_summary, targetdir=Cnf["Dir::Queue::Newstage"])
os.unlink(Upload.pkg.changes_file[:-8]+".dak")
def do_accept_stableupdate(suite, q):
@@ -944,30 +945,25 @@ def do_accept_stableupdate(suite, q):
def do_accept():
print "ACCEPT"
if not Options["No-Action"]:
- get_accept_lock()
(summary, short_summary) = Upload.build_summaries()
- try:
- if Cnf.FindB("Dinstall::SecurityQueueHandling"):
- Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
- move_to_dir(Cnf["Dir::Queue::Embargoed"])
- Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
- # Check for override disparities
- Upload.Subst["__SUMMARY__"] = summary
- else:
- # Stable updates need to be copied to proposed-updates holding
- # area instead of accepted. Sourceful uploads need to go
- # to it directly, binaries only if the source has not yet been
- # accepted into p-u.
- for suite, q in [("proposed-updates", "ProposedUpdates"),
- ("oldstable-proposed-updates", "OldProposedUpdates")]:
- if not Upload.pkg.changes["distribution"].has_key(suite):
- continue
- return do_accept_stableupdate(suite, q)
- # Just a normal upload, accept it...
- _accept()
- finally:
- if not Options["No-Action"]:
- os.unlink(Cnf["Process-New::AcceptedLockFile"])
+ if Cnf.FindB("Dinstall::SecurityQueueHandling"):
+ Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
+ move_to_dir(Cnf["Dir::Queue::Embargoed"])
+ Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
+ # Check for override disparities
+ Upload.Subst["__SUMMARY__"] = summary
+ else:
+ # Stable updates need to be copied to proposed-updates holding
+ # area instead of accepted. Sourceful uploads need to go
+ # to it directly, binaries only if the source has not yet been
+ # accepted into p-u.
+ for suite, q in [("proposed-updates", "ProposedUpdates"),
+ ("oldstable-proposed-updates", "OldProposedUpdates")]:
+ if not Upload.pkg.changes["distribution"].has_key(suite):
+ continue
+ return do_accept_stableupdate(suite, q)
+ # Just a normal upload, accept it...
+ _accept()
def check_status(files):
new = byhand = 0
@@ -986,6 +982,13 @@ def do_pkg(changes_file):
files = Upload.pkg.files
try:
+ check_daily_lock()
+ except CantGetLockError:
+ print "Hello? Operator! Give me the number for 911!"
+ print "Dinstall in the locked area, cant process packages, come back later"
+ sys.exit(1)
+
+ try:
with lock_package(Upload.pkg.changes["source"]):
if not recheck():
return
@@ -1100,6 +1103,7 @@ def main():
if not changes_file:
continue
print "\n" + changes_file
+
do_pkg (changes_file)
end()
diff --git a/daklib/dak_exceptions.py b/daklib/dak_exceptions.py
index 21fce9b..ef24fd7 100755
--- a/daklib/dak_exceptions.py
+++ b/daklib/dak_exceptions.py
@@ -61,7 +61,8 @@ dakerrors = {
"MissingContents": """Exception raised - we could not determine contents for this deb""",
"DBUpdateError": """Exception raised - could not update the database""",
"ChangesUnicodeError": """Exception raised - changes file not properly utf-8 encoded""",
- "AlreadyLockedError": """Exception raised - package already locked by someone else"""
+ "AlreadyLockedError": """Exception raised - package already locked by someone else""",
+ "CantGetLockError": """Exception raised - lockfile already in use"""
} #: All dak exceptions
def construct_dak_exception(name, description):
diff --git a/daklib/queue.py b/daklib/queue.py
index 35754c8..ceb4a79 100755
--- a/daklib/queue.py
+++ b/daklib/queue.py
@@ -552,7 +552,7 @@ distribution."""
###########################################################################
- def accept (self, summary, short_summary):
+ def accept (self, summary, short_summary, targetdir=None):
"""
Accept an upload.
@@ -577,16 +577,19 @@ distribution."""
changes_file = self.pkg.changes_file
dsc = self.pkg.dsc
+ if targetdir is None:
+ targetdir = Cnf["Dir::Queue::Accepted"]
+
print "Accepting."
self.Logger.log(["Accepting changes",changes_file])
- self.dump_vars(Cnf["Dir::Queue::Accepted"])
+ self.dump_vars(targetdir)
# Move all the files into the accepted directory
- utils.move(changes_file, Cnf["Dir::Queue::Accepted"])
+ utils.move(changes_file, targetdir)
file_keys = files.keys()
for file_entry in file_keys:
- utils.move(file_entry, Cnf["Dir::Queue::Accepted"])
+ utils.move(file_entry, targetdir)
self.accept_bytes += float(files[file_entry]["size"])
self.accept_count += 1
@@ -635,6 +638,18 @@ distribution."""
os.rename(temp_filename, filename)
os.chmod(filename, 0644)
+ # Its is Cnf["Dir::Queue::Accepted"] here, not targetdir!
+ # <Ganneff> we do call queue_build too
+ # <mhy> well yes, we'd have had to if we were inserting into accepted
+ # <Ganneff> now. thats database only.
+ # <mhy> urgh, that's going to get messy
+ # <Ganneff> so i make the p-n call to it *also* using accepted/
+ # <mhy> but then the packages will be in the queue_build table without the files being there
+ # <Ganneff> as the buildd queue is only regenerated whenever unchecked runs
+ # <mhy> ah, good point
+ # <Ganneff> so it will work out, as unchecked move it over
+ # <mhy> that's all completely sick
+ # <Ganneff> yes
self.queue_build("accepted", Cnf["Dir::Queue::Accepted"])
###########################################################################
--
1.5.6.5
Reply to: