[dak/master] generate-packages-sources
finalize the first step. there needs to be a rework of the apt-trees part
into an aptconfig class. Much to inflexible the way it is now, but we can
start testing this out in real life now.
Signed-off-by: Joerg Jaspert <joerg@debian.org>
---
dak/generate_packages_sources.py | 282 ++++++++++++++++++++++++++++++++++++--
daklib/dbconn.py | 252 ----------------------------------
2 files changed, 272 insertions(+), 262 deletions(-)
diff --git a/dak/generate_packages_sources.py b/dak/generate_packages_sources.py
index 366d722..6e91aed 100755
--- a/dak/generate_packages_sources.py
+++ b/dak/generate_packages_sources.py
@@ -27,20 +27,20 @@
import os
import os.path
-import stat
import sys
-from datetime import datetime
import apt_pkg
+from tempfile import mkstemp, mkdtemp
+import commands
+from multiprocessing import Pool
from daklib import daklog
from daklib.dbconn import *
from daklib.config import Config
-from daklib.threadpool import ThreadPool
################################################################################
-Options = None
-Logger = None
+Options = None #: Commandline arguments parsed into this
+Logger = None #: Our logging object
################################################################################
@@ -62,6 +62,265 @@ SUITE can be a space seperated list, e.g.
################################################################################
+def generate_packages_sources(arch, suite, tmppath):
+ """
+ Generate Packages/Sources files with apt-ftparchive for the given suite/arch
+
+ @type suite: string
+ @param suite: Suite name
+
+ @type arch: string
+ @param arch: Architecture name
+
+ @type tmppath: string
+ @param tmppath: The temporary path to work ing
+ """
+
+ DAILY_APT_CONF="""
+Dir
+{
+ ArchiveDir "/srv/ftp-master.debian.org/ftp/";
+ OverrideDir "/srv/ftp-master.debian.org/scripts/override/";
+ CacheDir "/srv/ftp-master.debian.org/database/";
+};
+
+Default
+{
+ Packages::Compress "bzip2 gzip";
+ Sources::Compress "bzip2 gzip";
+ Contents::Compress "gzip";
+ DeLinkLimit 0;
+ MaxContentsChange 25000;
+ FileMode 0664;
+}
+
+TreeDefault
+{
+ Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top";
+};
+
+"""
+
+ apt_trees={}
+ apt_trees["di"]={}
+ apt_trees["testing"]="""
+tree "dists/testing"
+{
+ FakeDI "dists/unstable";
+ FileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list";
+ SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list";
+ Sections "main contrib non-free";
+ Architectures "%(arch)s";
+ BinOverride "override.squeeze.$(SECTION)";
+ ExtraOverride "override.squeeze.extra.$(SECTION)";
+ SrcOverride "override.squeeze.$(SECTION).src";
+};
+"""
+ apt_trees["di"]["testing"]="""
+tree "dists/testing/main"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list";
+ Sections "debian-installer";
+ Architectures "%(arch)s";
+ BinOverride "override.squeeze.main.$(SECTION)";
+ SrcOverride "override.squeeze.main.src";
+ BinCacheDB "packages-debian-installer-$(ARCH).db";
+ Packages::Extensions ".udeb";
+ %(contentsline)s
+};
+
+tree "dists/testing/non-free"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list";
+ Sections "debian-installer";
+ Architectures "%(arch)s";
+ BinOverride "override.squeeze.main.$(SECTION)";
+ SrcOverride "override.squeeze.main.src";
+ BinCacheDB "packages-debian-installer-$(ARCH).db";
+ Packages::Extensions ".udeb";
+ %(contentsline)s
+};
+"""
+
+ apt_trees["unstable"]="""
+tree "dists/unstable"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_binary-$(ARCH).list";
+ SourceFileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_source.list";
+ Sections "main contrib non-free";
+ Architectures "%(arch)s";
+ BinOverride "override.sid.$(SECTION)";
+ ExtraOverride "override.sid.extra.$(SECTION)";
+ SrcOverride "override.sid.$(SECTION).src";
+};
+"""
+ apt_trees["di"]["unstable"]="""
+tree "dists/unstable/main"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/unstable_main_$(SECTION)_binary-$(ARCH).list";
+ Sections "debian-installer";
+ Architectures "%(arch)s";
+ BinOverride "override.sid.main.$(SECTION)";
+ SrcOverride "override.sid.main.src";
+ BinCacheDB "packages-debian-installer-$(ARCH).db";
+ Packages::Extensions ".udeb";
+ %(contentsline)s
+};
+
+tree "dists/unstable/non-free"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/unstable_non-free_$(SECTION)_binary-$(ARCH).list";
+ Sections "debian-installer";
+ Architectures "%(arch)s";
+ BinOverride "override.sid.main.$(SECTION)";
+ SrcOverride "override.sid.main.src";
+ BinCacheDB "packages-debian-installer-$(ARCH).db";
+ Packages::Extensions ".udeb";
+ %(contentsline)s
+};
+"""
+
+ apt_trees["experimental"]="""
+tree "dists/experimental"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_binary-$(ARCH).list";
+ SourceFileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_source.list";
+ Sections "main contrib non-free";
+ Architectures "%(arch)s";
+ BinOverride "override.sid.$(SECTION)";
+ SrcOverride "override.sid.$(SECTION).src";
+};
+"""
+ apt_trees["di"]["experimental"]="""
+tree "dists/experimental/main"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/experimental_main_$(SECTION)_binary-$(ARCH).list";
+ Sections "debian-installer";
+ Architectures "%(arch)s";
+ BinOverride "override.sid.main.$(SECTION)";
+ SrcOverride "override.sid.main.src";
+ BinCacheDB "packages-debian-installer-$(ARCH).db";
+ Packages::Extensions ".udeb";
+ %(contentsline)s
+};
+
+tree "dists/experimental/non-free"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/experimental_non-free_$(SECTION)_binary-$(ARCH).list";
+ Sections "debian-installer";
+ Architectures "%(arch)s";
+ BinOverride "override.sid.main.$(SECTION)";
+ SrcOverride "override.sid.main.src";
+ BinCacheDB "packages-debian-installer-$(ARCH).db";
+ Packages::Extensions ".udeb";
+ %(contentsline)s
+};
+"""
+
+ apt_trees["testing-proposed-updates"]="""
+tree "dists/testing-proposed-updates"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list";
+ SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list";
+ Sections "main contrib non-free";
+ Architectures "%(arch)s";
+ BinOverride "override.squeeze.$(SECTION)";
+ ExtraOverride "override.squeeze.extra.$(SECTION)";
+ SrcOverride "override.squeeze.$(SECTION).src";
+ Contents " ";
+};
+"""
+ apt_trees["di"]["testing-proposed-updates"]="""
+tree "dists/testing-proposed-updates/main"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
+ Sections "debian-installer";
+ Architectures "%(arch)s";
+ BinOverride "override.squeeze.main.$(SECTION)";
+ SrcOverride "override.squeeze.main.src";
+ BinCacheDB "packages-debian-installer-$(ARCH).db";
+ Packages::Extensions ".udeb";
+ Contents " ";
+};
+"""
+
+ apt_trees["proposed-updates"]="""
+tree "dists/proposed-updates"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list";
+ SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list";
+ Sections "main contrib non-free";
+ Architectures "%(arch)s";
+ BinOverride "override.lenny.$(SECTION)";
+ ExtraOverride "override.lenny.extra.$(SECTION)";
+ SrcOverride "override.lenny.$(SECTION).src";
+ Contents " ";
+};
+"""
+ apt_trees["di"]["proposed-updates"]="""
+tree "dists/proposed-updates/main"
+{
+ FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
+ Sections "debian-installer";
+ Architectures "%(arch)s";
+ BinOverride "override.lenny.main.$(SECTION)";
+ SrcOverride "override.lenny.main.src";
+ BinCacheDB "packages-debian-installer-$(ARCH).db";
+ Packages::Extensions ".udeb";
+ Contents " ";
+};
+"""
+
+ cnf = Config()
+ try:
+ # Write apt.conf
+ (ac_fd, ac_name) = mkstemp(dir=tmppath)
+ os.write(ac_fd, DAILY_APT_CONF)
+ # here we want to generate the tree entries
+ os.write(ac_fd, apt_trees[suite] % {'arch': arch})
+ # this special casing needs to go away, but this whole thing may just want an
+ # aptconfig class anyways
+ if arch != 'source':
+ if arch == 'hurd-i386' and suite == 'experimental':
+ pass
+ else:
+ if arch == "amd64":
+ os.write(ac_fd, apt_trees["di"][suite] %
+ {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'})
+ else:
+ os.write(ac_fd, apt_trees["di"][suite] % {'arch': arch, 'contentsline': ''})
+ os.close(ac_fd)
+
+ print "Going to run apt-ftparchive for %s/%s" % (arch, suite)
+ # Run apt-ftparchive generate
+ # We dont want to add a -q or -qq here, this output should go into our logs, sometimes
+ # it has errormessages we like to see
+ os.environ['GZIP'] = '--rsyncable'
+ os.chdir(tmppath)
+ (result, output) = commands.getstatusoutput('apt-ftparchive generate %s' % os.path.basename(ac_name))
+ sn="a-f %s,%s: " % (suite, arch)
+ print sn + output.replace('\n', '\n%s' % (sn))
+
+ # Clean up any left behind files
+ finally:
+ if ac_fd:
+ try:
+ os.close(ac_fd)
+ except OSError:
+ pass
+
+ if ac_name:
+ try:
+ os.unlink(ac_name)
+ except OSError:
+ pass
+
+def sname(arch):
+ return arch.arch_string
+
+########################################################################
+########################################################################
+
def main ():
global Options, Logger
@@ -98,19 +357,22 @@ def main ():
else:
suites=session.query(Suite).filter(Suite.untouchable == False).all()
- threadpool = ThreadPool()
-
startdir = os.getcwd()
os.chdir(cnf["Dir::TempPath"])
+ # Setup a multiprocessing Pool. As many workers as we have CPU cores.
+ pool = Pool()
+
# For each given suite, each architecture, run one apt-ftparchive
for s in suites:
arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=True, session=session)
- Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, arch_list)])
+ Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))])
for a in arch_list:
- threadpool.queueTask(s.generate_packages_sources, (a.arch_string))
+ pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]))
- threadpool.joinAll()
+ # No more work will be added to our pool, close it and then wait for all to finish
+ pool.close()
+ pool.join()
os.chdir(startdir)
# this script doesn't change the database
session.close()
diff --git a/daklib/dbconn.py b/daklib/dbconn.py
index b987a37..a1bff89 100755
--- a/daklib/dbconn.py
+++ b/daklib/dbconn.py
@@ -2459,202 +2459,6 @@ SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
('OverrideSuite', 'overridesuite'),
('ChangelogBase', 'changelogbase')]
-
-DAILY_APT_CONF="""
-Dir
-{
- ArchiveDir "/srv/ftp-master.debian.org/ftp/";
- OverrideDir "/srv/ftp-master.debian.org/scripts/override/";
- CacheDir "/srv/ftp-master.debian.org/database/";
-};
-
-Default
-{
- Packages::Compress "bzip2 gzip";
- Sources::Compress "bzip2 gzip";
- Contents::Compress "gzip";
- DeLinkLimit 0;
- MaxContentsChange 25000;
- FileMode 0664;
-}
-
-TreeDefault
-{
- Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top";
-};
-
-"""
-
-apt_trees={}
-apt_trees["di"]={}
-apt_trees["testing"]="""
-tree "dists/testing"
-{
- FakeDI "dists/unstable";
- FileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list";
- SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list";
- Sections "main contrib non-free";
- Architectures "%(arch)s";
- BinOverride "override.squeeze.$(SECTION)";
- ExtraOverride "override.squeeze.extra.$(SECTION)";
- SrcOverride "override.squeeze.$(SECTION).src";
-};
-"""
-apt_trees["di"]["testing"]="""
-tree "dists/testing/main"
-{
- FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list";
- Sections "debian-installer";
- Architectures "%(arch)s";
- BinOverride "override.squeeze.main.$(SECTION)";
- SrcOverride "override.squeeze.main.src";
- BinCacheDB "packages-debian-installer-$(ARCH).db";
- Packages::Extensions ".udeb";
- %(contentsline)s
-};
-
-tree "dists/testing/non-free"
-{
- FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list";
- Sections "debian-installer";
- Architectures "%(arch)s";
- BinOverride "override.squeeze.main.$(SECTION)";
- SrcOverride "override.squeeze.main.src";
- BinCacheDB "packages-debian-installer-$(ARCH).db";
- Packages::Extensions ".udeb";
- %(contentsline)s
-};
-"""
-
-apt_trees["unstable"]="""
-tree "dists/unstable"
-{
- FileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_binary-$(ARCH).list";
- SourceFileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_source.list";
- Sections "main contrib non-free";
- Architectures "%(arch)s";
- BinOverride "override.sid.$(SECTION)";
- ExtraOverride "override.sid.extra.$(SECTION)";
- SrcOverride "override.sid.$(SECTION).src";
-};
-"""
-apt_trees["di"]["unstable"]="""
-tree "dists/unstable/main"
-{
- FileList "/srv/ftp-master.debian.org/database/dists/unstable_main_$(SECTION)_binary-$(ARCH).list";
- Sections "debian-installer";
- Architectures "%(arch)s";
- BinOverride "override.sid.main.$(SECTION)";
- SrcOverride "override.sid.main.src";
- BinCacheDB "packages-debian-installer-$(ARCH).db";
- Packages::Extensions ".udeb";
- %(contentsline)s
-};
-
-tree "dists/unstable/non-free"
-{
- FileList "/srv/ftp-master.debian.org/database/dists/unstable_non-free_$(SECTION)_binary-$(ARCH).list";
- Sections "debian-installer";
- Architectures "%(arch)s";
- BinOverride "override.sid.main.$(SECTION)";
- SrcOverride "override.sid.main.src";
- BinCacheDB "packages-debian-installer-$(ARCH).db";
- Packages::Extensions ".udeb";
- %(contentsline)s
-};
-"""
-
-apt_trees["experimental"]="""
-tree "dists/experimental"
-{
- FileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_binary-$(ARCH).list";
- SourceFileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_source.list";
- Sections "main contrib non-free";
- Architectures "%(arch)s";
- BinOverride "override.sid.$(SECTION)";
- SrcOverride "override.sid.$(SECTION).src";
-};
-"""
-apt_trees["di"]["experimental"]="""
-tree "dists/experimental/main"
-{
- FileList "/srv/ftp-master.debian.org/database/dists/experimental_main_$(SECTION)_binary-$(ARCH).list";
- Sections "debian-installer";
- Architectures "%(arch)s";
- BinOverride "override.sid.main.$(SECTION)";
- SrcOverride "override.sid.main.src";
- BinCacheDB "packages-debian-installer-$(ARCH).db";
- Packages::Extensions ".udeb";
- %(contentsline)s
-};
-
-tree "dists/experimental/non-free"
-{
- FileList "/srv/ftp-master.debian.org/database/dists/experimental_non-free_$(SECTION)_binary-$(ARCH).list";
- Sections "debian-installer";
- Architectures "%(arch)s";
- BinOverride "override.sid.main.$(SECTION)";
- SrcOverride "override.sid.main.src";
- BinCacheDB "packages-debian-installer-$(ARCH).db";
- Packages::Extensions ".udeb";
- %(contentsline)s
-};
-"""
-
-apt_trees["testing-proposed-updates"]="""
-tree "dists/testing-proposed-updates"
-{
- FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list";
- SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list";
- Sections "main contrib non-free";
- Architectures "%(arch)s";
- BinOverride "override.squeeze.$(SECTION)";
- ExtraOverride "override.squeeze.extra.$(SECTION)";
- SrcOverride "override.squeeze.$(SECTION).src";
- Contents " ";
-};
-"""
-apt_trees["di"]["testing-proposed-updates"]="""
-tree "dists/testing-proposed-updates/main"
-{
- FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
- Sections "debian-installer";
- Architectures "%(arch)s";
- BinOverride "override.squeeze.main.$(SECTION)";
- SrcOverride "override.squeeze.main.src";
- BinCacheDB "packages-debian-installer-$(ARCH).db";
- Packages::Extensions ".udeb";
- Contents " ";
-};
-"""
-
-apt_trees["proposed-updates"]="""
-tree "dists/proposed-updates"
-{
- FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list";
- SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list";
- Sections "main contrib non-free";
- Architectures "%(arch)s";
- BinOverride "override.lenny.$(SECTION)";
- ExtraOverride "override.lenny.extra.$(SECTION)";
- SrcOverride "override.lenny.$(SECTION).src";
- Contents " ";
-};
-"""
-apt_trees["di"]["proposed-updates"]="""
-tree "dists/proposed-updates/main"
-{
- FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
- Sections "debian-installer";
- Architectures "%(arch)s";
- BinOverride "override.lenny.main.$(SECTION)";
- SrcOverride "override.lenny.main.src";
- BinCacheDB "packages-debian-installer-$(ARCH).db";
- Packages::Extensions ".udeb";
- Contents " ";
-};
-"""
-
class Suite(object):
def __init__(self, *args, **kwargs):
pass
@@ -2683,62 +2487,6 @@ class Suite(object):
return "\n".join(ret)
- def generate_packages_sources(self, arch):
- """
- Generate Packages/Sources files with apt-ftparchive for the given suite/arch
-
- @type suite: string
- @param suite: Suite name
-
- @type arch: string
- @param arch: Architecture name
- """
-
- cnf = Config()
- try:
- # Write apt.conf
- (ac_fd, ac_name) = mkstemp(dir=cnf["Dir::TempPath"])
- os.write(ac_fd, DAILY_APT_CONF)
- # here we want to generate the tree entries
- os.write(ac_fd, apt_trees[self.suite_name] % {'arch': arch})
- # this special casing needs to go away, but this whole thing may just want an
- # aptconfig class anyways
- if arch != 'source':
- if arch == 'hurd-i386' and self.suite_name == 'experimental':
- pass
- else:
- if arch == "amd64":
- os.write(ac_fd, apt_trees["di"][self.suite_name] %
- {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'})
- else:
- os.write(ac_fd, apt_trees["di"][self.suite_name] % {'arch': arch, 'contentsline': ''})
- os.close(ac_fd)
-
- # Run apt-ftparchive generate
- # We might want to add a -q or -qq here
- os.environ['GZIP'] = '--rsyncable'
- os.chdir(cnf["Dir::TempPath"])
- (result, output) = commands.getstatusoutput('apt-ftparchive generate %s' % os.path.basename(ac_name))
- sn="a-f %s,%s: " % (self.suite_name, arch)
- print sn + output.replace('\n', '\n%s' % (sn))
-
- # Clean up any left behind files
- finally:
- if ac_fd:
- try:
- os.close(ac_fd)
- except OSError:
- pass
-
- if ac_name:
- try:
- os.unlink(ac_name)
- except OSError:
- pass
-
-
-
-
__all__.append('Suite')
@session_wrapper
--
1.5.6.5
Reply to: