[dak/master] get bootstrap working again
Signed-off-by: Mike O'Connor <stew@vireo.org>
---
dak/contents.py | 17 ++++++++---------
daklib/binary.py | 17 +++++++++++------
daklib/dbconn.py | 38 +++++++++++++++++++++++++++++++++++++-
daklib/utils.py | 50 --------------------------------------------------
4 files changed, 56 insertions(+), 66 deletions(-)
diff --git a/dak/contents.py b/dak/contents.py
index 963bf49..5ff121b 100644
--- a/dak/contents.py
+++ b/dak/contents.py
@@ -42,6 +42,7 @@ import math
import gzip
import apt_pkg
from daklib import utils
+from daklib.binary import Binary
from daklib.config import Config
from daklib.dbconn import DBConn
################################################################################
@@ -159,7 +160,7 @@ remove_filename_cruft_q = """DELETE FROM content_file_names
WHERE id IN (SELECT cfn.id FROM content_file_names cfn
LEFT JOIN content_associations ca
ON ca.filename=cfn.id
- WHERE ca.id IS NULL)""" );
+ WHERE ca.id IS NULL)"""
# delete any paths we are storing which have no binary associated with them
remove_filepath_cruft_q = """DELETE FROM content_file_paths
@@ -273,9 +274,7 @@ class Contents(object):
else:
debfile = os.path.join( pooldir, deb[1] )
if os.path.exists( debfile ):
- contents = utils.generate_contents_information( debfile )
- DBConn().insert_content_paths(deb[0], contents)
- log.info( "imported (%d/%d): %s" % (count,len(debs),deb[1] ) )
+ Binary(f).scan_package( deb[0] )
else:
log.error( "missing .deb: %s" % deb[1] )
@@ -305,12 +304,12 @@ class Contents(object):
# The MORE fun part. Ok, udebs need their own contents files, udeb, and udeb-nf (not-free)
# This is HORRIBLY debian specific :-/
for section_id, fn_pattern in [("debian-installer","dists/%s/Contents-udeb.gz"),
- ("non-free/debian-installer", "dists/%s/Contents-udeb-nf.gz")]
+ ("non-free/debian-installer", "dists/%s/Contents-udeb-nf.gz")]:
- section_id = DBConn().get_section_id(section_id) # all udebs should be here)
- if section_id != -1:
- cursor.execute("EXECUTE udeb_contents_q(%d,%d,%d)" % (section_id, suite_id, suite_id))
- self._write_content_file(cursor, fn_pattern % suite)
+ section_id = DBConn().get_section_id(section_id) # all udebs should be here)
+ if section_id != -1:
+ cursor.execute("EXECUTE udeb_contents_q(%d,%d,%d)" % (section_id, suite_id, suite_id))
+ self._write_content_file(cursor, fn_pattern % suite)
################################################################################
diff --git a/daklib/binary.py b/daklib/binary.py
index f9cf243..042cd26 100755
--- a/daklib/binary.py
+++ b/daklib/binary.py
@@ -115,7 +115,7 @@ class Binary(object):
return not rejected
- def scan_package(self):
+ def scan_package(self, bootstrap_id=0):
"""
Unpack the .deb, do sanity checking, and gather info from it.
@@ -123,6 +123,11 @@ class Binary(object):
the hopefully near future, it should also include gathering info from the
control file.
+ @ptype bootstrap_id: int
+ @param bootstrap_id: the id of the binary these packages
+ should be associated or zero meaning we are not bootstrapping
+ so insert into a temporary table
+
@return True if the deb is valid and contents were imported
"""
rejected = not self.valid_deb()
@@ -135,14 +140,17 @@ class Binary(object):
if self.chunks[1] == "control.tar.gz":
control = tarfile.open(os.path.join(self.tmpdir, "control.tar.gz" ), "r:gz")
- pkg = deb822.Packages.iter_paragraphs( control.extractfile('./control') ).next()
if self.chunks[2] == "data.tar.gz":
data = tarfile.open(os.path.join(self.tmpdir, "data.tar.gz"), "r:gz")
elif self.chunks[2] == "data.tar.bz2":
data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2")
- return DBConn().insert_content_paths(pkg, [ tarinfo.name for tarinfo in data if not tarinfo.isdir()])
+ if bootstrap_id:
+ return DBConn().insert_content_paths(bootstrap_id, [ tarinfo.name for tarinfo in data if not tarinfo.isdir()])
+ else:
+ pkg = deb822.Packages.iter_paragraphs( control.extractfile('./control') ).next()
+ return DBConn().insert_pending_content_paths(pkg, [ tarinfo.name for tarinfo in data if not tarinfo.isdir()])
except:
traceback.print_exc()
@@ -152,9 +160,6 @@ class Binary(object):
finally:
os.chdir( cwd )
-
-
-
if __name__ == "__main__":
Binary( "/srv/ftp.debian.org/queue/accepted/halevt_0.1.3-2_amd64.deb" ).scan_package()
diff --git a/daklib/dbconn.py b/daklib/dbconn.py
index 21c9cc6..3fad3f5 100755
--- a/daklib/dbconn.py
+++ b/daklib/dbconn.py
@@ -478,7 +478,7 @@ class DBConn(Singleton):
return map(lambda x: x[0], c.fetchall())
- def insert_content_paths(self, package, fullpaths):
+ def insert_content_paths(self, bin_id, fullpaths):
"""
Make sure given path is associated with given binary id
@@ -495,6 +495,42 @@ class DBConn(Singleton):
c.execute("BEGIN WORK")
try:
+ for fullpath in fullpaths:
+ (path, file) = os.path.split(fullpath)
+
+ # Get the necessary IDs ...
+ file_id = self.get_or_set_contents_file_id(file)
+ path_id = self.get_or_set_contents_path_id(path)
+
+ c.execute("""INSERT INTO content_associations
+ (binary_pkg, filepath, filename)
+ VALUES ( '%d', '%d', '%d')""" % (bin_id, path_id, file_id) )
+
+ c.execute("COMMIT")
+ return True
+ except:
+ traceback.print_exc()
+ c.execute("ROLLBACK")
+ return False
+
+ def insert_pending_content_paths(self, package, fullpaths):
+ """
+ Make sure given paths are temporarily associated with given
+ package
+
+ @type package: dict
+ @param package: the package to associate with should have been read in from the binary control file
+ @type fullpaths: list
+ @param fullpaths: the list of paths of the file being associated with the binary
+
+ @return True upon success
+ """
+
+ c = self.db_con.cursor()
+
+ c.execute("BEGIN WORK")
+ try:
+
# Remove any already existing recorded files for this package
c.execute("""DELETE FROM pending_content_associations
WHERE package=%(Package)s
diff --git a/daklib/utils.py b/daklib/utils.py
index 27c3af3..c630053 100755
--- a/daklib/utils.py
+++ b/daklib/utils.py
@@ -1526,54 +1526,4 @@ apt_pkg.ReadConfigFileISC(Cnf,default_config)
if which_conf_file() != default_config:
apt_pkg.ReadConfigFileISC(Cnf,which_conf_file())
-################################################################################
-
-def generate_contents_information(filename):
- """
- Generate a list of flies contained in a .deb
-
- @ptype filename: string
- @param filename: the path to a .deb
-
- @rtype: list
- @return: a list of files in the data.tar.* portion of the .deb
- """
- cmd = "ar t %s" % (filename)
- (result, output) = commands.getstatusoutput(cmd)
- if result != 0:
- reject("%s: 'ar t' invocation failed." % (filename))
- reject(utils.prefix_multi_line_string(output, " [ar output:] "), "")
-
- # Ugh ... this is ugly ... Code ripped from process_unchecked.py
- chunks = output.split('\n')
-
- contents = []
- try:
- cmd = "ar x %s %s" % (filename, chunks[2])
- (result, output) = commands.getstatusoutput(cmd)
- if result != 0:
- reject("%s: '%s' invocation failed." % (filename, cmd))
- reject(utils.prefix_multi_line_string(output, " [ar output:] "), "")
-
- # Got deb tarballs, now lets go through and determine what bits
- # and pieces the deb had ...
- if chunks[2] == "data.tar.gz":
- data = tarfile.open("data.tar.gz", "r:gz")
- elif chunks[2] == "data.tar.bz2":
- data = tarfile.open("data.tar.bz2", "r:bz2")
- else:
- os.remove(chunks[2])
- reject("couldn't find data.tar.*")
-
- for tarinfo in data:
- if not tarinfo.isdir():
- contents.append(tarinfo.name[2:])
-
- finally:
- if os.path.exists( chunks[2] ):
- shutil.rmtree( chunks[2] )
- os.remove( chunks[2] )
-
- return contents
-
###############################################################################
--
1.5.6.5
Reply to: