[dak/master] Start of contents scanner
Signed-off-by: Michael Casadevall <sonicmctails@gmail.com>
---
dak/.generate_contents.py.swp | Bin 0 -> 12288 bytes
dak/dak.py | 2 +
dak/generate_contents.py | 139 +++++++++++++++++++++++++++++++++++++++++
dak/process_accepted.py | 45 +++++++++++++-
4 files changed, 184 insertions(+), 2 deletions(-)
create mode 100644 dak/.generate_contents.py.swp
create mode 100755 dak/generate_contents.py
diff --git a/dak/.generate_contents.py.swp b/dak/.generate_contents.py.swp
new file mode 100644
index 0000000000000000000000000000000000000000..d4e83290431f9beed6b625eb22bb28b4ce8ee265
GIT binary patch
literal 12288
zc-rk+TW=dh6rSEGP(ZXVyuhhL6&tM`Cjr_zO@$Mh#8Phcg({6AGxm<{A?w}M?yOzE
z6oD!MuSkVTT>b$dBm@t<AR$2T&Tjzv0uu0w_-4J%r73MyB@e)U(r0^S&YU^t%sFRT
zwYBU^g#|jD8D+p@jHR5*g)h2?S?vU4CSPaw0?4IJ(Gk{N<gP++L<={i(%&OZVGEBd
zQLQ_+61MU)ZjZ%?5o6a8s7CtCvC8<^XjTu05+~{S<CQorF=E7s5hF&77%^h}kD%Or
z>{Wd4kH#y}_r6=cW5kFNBSwrEF=E7s5hF&782?^yld&TQ7<=O&3;+Fp@b~|lk23ZH
z_&xYK_!^i4e}9CrPr$dpH^CY35cth8#@+$1gBQW$;CBx*_AMBI&x41+pB`fD1~?1O
zfPX&7*dO56;Ah}_U>EFwr@*JdC%~iNUq>1H4g3}S5c~kV23`iwgOlK4a6fqS0mgm-
zZ-DQDSHU@O7TgE^d<3?DUxJ^5pMo!gRj>#?3qA!t2_69VgI^tH>|^jcco94S9tVFs
z1Y5xO!E2xbeefhm;1TdJ_%UMq5%~7ODBdyt^PpSN=F(24m~2qBYV)?JR%vpQhO1S*
zP^}K<7#q41c0<R8qP;@<zBcWKSNDbfHSPG{NXZoSS6t6|Mby<4O_JNpVA;kfIc+yq
z6E9BXa*HrFv1ZW5#GK<RJt6HaQp=8q$y91HCjy^01w7Wj;q{WMq+|Q5i8*1piM3o`
zFq*~D(3B|!wX!EfGBFz&o1Tu0&E+D=L|>sFNG0?(?y9Q0-pnj>&ll59yUlHLUfM!i
zX5_tQ(AL`xzvpMT*X*RW=WJ;HKOdIMo$?cOijp$5hO-ihjm>b#c2sC|i~H4e(M#T&
zo>VT>(QrJHh#;*}qNM{uYq=fOv=+{6H~|Y*lf$iHqrbzr<AGt6Q%2uin2F7Sl9r!o
z3Z+w^k&Hs&1PW_9KE5(B=qHQi)VQVh85`+gXi;X0MwrT!OzlC&hKFx{4AHDmIM$2q
z+aTO7sg@w_)mxaUtH2XJi-b2PVSA1H(vKD{39^O2PKUaVw;pOi970Wla_H-JJ!CaW
z6^6yJiIXe5f&Qg+W7`(oAMUZub;wETVYqB^#c{d_V|AOd+l#pMivpJ@u1J;CA2`qy
zDVB(mHr+BPoz{u7qciy}rwha3K)9^KpcV9W4swhJIYtd>2|sEaI#@WNqDwso$JaOf
zO%MDHqY(B2+O%sjyEjqDyQ-h-i;gW=KefZ3TY6kL-XgUJW;8sft)pSdS_WTLaiuPz
zMkm`YIwd``Rc0_57AT7bQDYknaknTy&~;p4k0B97+tpWLkgvGHShn%oZSI9XyRLDF
z&dqYXGmXZ1avCV80Nooo$Q23{ITa?N?vXZ9v)dA$pzdZ34{}UbTbpvJZR%RQBkd-&
z9qiHdnc245m8ylJwncZKMb|D&@Pm4*pW53usPJ&qhOTUB3a$<CEvIYs$P=!`>s$Ik
zzcltB{xstRwiz00P{FQe40`S?m5~CLpk-}@K_%THX?#3u&=g*1ExC{<%8reWj;2S)
zv!{tFr95<T%Rzv`NQ707+Gu*bJrO->26a8H34UsyXYe3X$!VxA_XKvYIDzgMfo1AY
z3k0DqWCs_+5eK)YW2v{zE>@!6VbO9>vt*s-WnI|5F1S0ahPNJGo+%X94;I81E)=Kd
z@{8sCRAIhQULlCg6v~SiN+p_EDpH=7^Tl#ux-y?H(sHG^yi~f7Au5S5L$JSwMsTT6
zo?EJvDZjWvm-59T)U8Z}QP6EdbVR>-queYBZ{V%RZKeHzg7TI)>0~gauRIC0kc_&1
z3w7DgV7wk(Y2gJD=zF3OScY~viItvy%ABwhq|j%3WFd<^>Il!rl=gH-#JAl11PQ6r
zaUf4)gJ4IoLxb?31LtE4<n$8sxBa_c3UT<SXZ#jis`YN;b3>cuM`lfEU#QM)-2Y}U
z^wD@eM#3vJWuvDZpGH=Ave{CUoKDeLHhad<pJ!=7)>~ZQPZ`|jrs!~l>|DFPF?hig
oHOcLa<2BERdOV?PUW;vegrlN6ITHBZNKM)!!tRjU!@b(<Z?Q(J`2YX_
literal 0
Hc-jL100001
diff --git a/dak/dak.py b/dak/dak.py
index 9dfd026..77999ab 100755
--- a/dak/dak.py
+++ b/dak/dak.py
@@ -105,6 +105,8 @@ def init():
"Generate lists of packages per suite for apt-ftparchive"),
("generate-releases",
"Generate Release files"),
+ ("generate-contents",
+ "Generate contest files"),
("generate-index-diffs",
"Generate .diff/Index files"),
("clean-suites",
diff --git a/dak/generate_contents.py b/dak/generate_contents.py
new file mode 100755
index 0000000..9696bb8
--- /dev/null
+++ b/dak/generate_contents.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+# Create all the contents files
+
+# Copyright (C) 2008, 2009 Michael Casadevall <mcasadevall@debian.org>
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+# <Ganneff> there is the idea to slowly replace contents files
+# <Ganneff> with a new generation of such files.
+# <Ganneff> having more info.
+# <Ganneff> of course that wont help for now where we need to generate them :)
+################################################################################
+
+################################################################################
+
+import sys, os, popen2, tempfile, stat, time, pg
+import apt_pkg
+from daklib import database, utils
+from daklib.dak_exceptions import *
+
+################################################################################
+
+Cnf = None
+projectB = None
+out = None
+AptCnf = None
+
+################################################################################
+
+def usage (exit_code=0):
+ print """Usage: dak generate-contents
+Generate Contents files
+
+ -h, --help show this help and exit
+ -s, --suite=SUITE only write file lists for this suite
+"""
+ sys.exit(exit_code)
+
+################################################################################
+
+def handle_dup_files(file_list):
+ # Sort the list, and then handle finding dups in the filenames key
+
+ # Walk the list, seeing if the current entry and the next one are the same
+ # and if so, join them together
+
+
+ return file_list
+
+################################################################################
+
+def generate_contents(suites):
+ global projectB, Cnf
+ # Ok, the contents information is in the database
+
+ # We need to work and get the contents, and print it out on a per
+ # architectual basis
+
+ # Get our suites, and the architectures
+ for s in suites:
+ suite_id = database.get_suite_id(s)
+
+ q = projectB.query("SELECT architecture FROM suite_architectures WHERE suite = '%d'" % suite_id)
+
+ arch_list = [ ]
+ for r in q.getresult():
+ arch_list.append(r[0])
+
+ arch_all_id = database.get_architecture_id("all")
+
+ # Got the arch all packages, now we need to get the arch dependent packages
+ # attach the arch all, stick them together, and write out the result
+
+ for arch_id in arch_list:
+ print "SELECT b.package, c.file, s.section FROM contents c JOIN binaries b ON (b.id=c.binary_pkg) JOIN bin_associations ba ON (b.id=ba.bin) JOIN override o ON (o.package=b.package) JOIN section s ON (s.id=o.section) WHERE (b.architecture = '%d' OR b.architecture = '%d') AND ba.suite = '%d'" % (arch_id, arch_all_id, suite_id)
+ q = projectB.query("SELECT b.package, c.file, s.section FROM contents c JOIN binaries b ON (b.id=c.binary_pkg) JOIN bin_associations ba ON (b.id=ba.bin) JOIN override o ON (o.package=b.package) JOIN section s ON (s.id=o.section) WHERE (b.architecture = '%d' OR b.architecture = '%d') AND ba.suite = '%d'" % (arch_id, arch_all_id, suite_id))
+ # We need to copy the arch_all packages table into arch packages
+
+ # This is for the corner case of arch dependent packages colliding
+ # with arch all packages only on some architectures.
+ # Ugly, I know ...
+
+ arch_packages = []
+ for r in q.getresult():
+ arch_packages.append((r[1], (r[2] + '/' + r[0])))
+
+ arch_packages = handle_dup_files(arch_packages)
+
+ #for contents in arch_packages:
+ #print contents[0] + '\t\t\t\t' + contents[1]
+
+################################################################################
+
+def main ():
+ global Cnf, projectB, out
+ out = sys.stdout
+
+ Cnf = utils.get_conf()
+
+ Arguments = [('h',"help","Generate-Contents::Options::Help"),
+ ('s',"suite","Generate-Contents::Options::Suite","HasArg"),
+ ]
+ for i in [ "help", "suite" ]:
+ if not Cnf.has_key("Generate-Contents::Options::%s" % (i)):
+ Cnf["Generate-Contents::Options::%s" % (i)] = ""
+
+ suites = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
+ Options = Cnf.SubTree("Generate-Contents::Options")
+
+ if Options["Help"]:
+ usage()
+
+ if Options["Suite"]:
+ suites = utils.split_args(Options["Suite"])
+ else:
+ suites = Cnf.SubTree("Suite").List()
+
+ projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
+ database.init(Cnf, projectB)
+
+ generate_contents(suites)
+
+#######################################################################################
+
+if __name__ == '__main__':
+ main()
diff --git a/dak/process_accepted.py b/dak/process_accepted.py
index 0d5a538..b6f8a31 100755
--- a/dak/process_accepted.py
+++ b/dak/process_accepted.py
@@ -30,7 +30,7 @@
###############################################################################
import errno, fcntl, os, sys, time, re
-import apt_pkg
+import apt_pkg, tarfile, commands
from daklib import database
from daklib import logging
from daklib import queue
@@ -96,6 +96,43 @@ class Urgency_Log:
else:
os.unlink(self.log_filename)
+
+###############################################################################
+
+def generate_contents_information(filename):
+ # Generate all the contents for the database
+ cmd = "ar t %s" % (filename)
+ (result, output) = commands.getstatusoutput(cmd)
+ if result != 0:
+ reject("%s: 'ar t' invocation failed." % (filename))
+ reject(utils.prefix_multi_line_string(output, " [ar output:] "), "")
+
+ # Ugh ... this is ugly ... Code ripped from process_unchecked.py
+ chunks = output.split('\n')
+ cmd = "ar x %s %s" % (filename, chunks[2])
+ (result, output) = commands.getstatusoutput(cmd)
+ if result != 0:
+ reject("%s: 'ar t' invocation failed." % (filename))
+ reject(utils.prefix_multi_line_string(output, " [ar output:] "), "")
+
+ # Got deb tarballs, now lets go through and determine what bits
+ # and pieces the deb had ...
+ if chunks[2] == "data.tar.gz":
+ data = tarfile.open("data.tar.gz", "r:gz")
+ elif data_tar == "data.tar.bz2":
+ data = tarfile.open("data.tar.bz2", "r:bz2")
+ else:
+ os.remove(chunks[2])
+ reject("couldn't find data.tar.*")
+
+ contents = []
+ for tarinfo in data:
+ if not tarinfo.isdir():
+ contents.append(tarinfo.name[2:])
+
+ os.remove(chunks[2])
+ return contents
+
###############################################################################
def reject (str, prefix="Rejected: "):
@@ -354,6 +391,7 @@ def install ():
source = files[file]["source package"]
source_version = files[file]["source version"]
filename = files[file]["pool name"] + file
+ contents = generate_contents_information(file)
if not files[file].has_key("location id") or not files[file]["location id"]:
files[file]["location id"] = database.get_location_id(Cnf["Dir::Pool"],files[file]["component"],utils.where_am_i())
if not files[file].has_key("files id") or not files[file]["files id"]:
@@ -368,6 +406,10 @@ def install ():
suite_id = database.get_suite_id(suite)
projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%d, currval('binaries_id_seq'))" % (suite_id))
+ # insert contents into the database
+ for file in contents:
+ projectB.query("INSERT INTO contents (binary_pkg, file) VALUES (currval('binaries_id_seq'), '%s')" % file)
+
# If the .orig.tar.gz is in a legacy directory we need to poolify
# it, so that apt-get source (and anything else that goes by the
# "Directory:" field in the Sources.gz file) works.
@@ -430,7 +472,6 @@ def install ():
utils.copy(pkg.changes_file, Cnf["Dir::Root"] + dest)
for dest in copy_dot_dak.keys():
utils.copy(Upload.pkg.changes_file[:-8]+".dak", dest)
-
projectB.query("COMMIT WORK")
# Move the .changes into the 'done' directory
--
1.5.6.5
Reply to: