[dak/master 1/2] merge from master with sqla
---
dak/contents.py | 487 ++++++++++++++++++++++++++++---------------------
dak/dakdb/update17.py | 310 +++++++++++++++++++++++++++++++
dak/update_db.py | 9 +-
daklib/dbconn.py | 6 +-
daklib/utils.py | 7 +-
5 files changed, 601 insertions(+), 218 deletions(-)
create mode 100644 dak/dakdb/update17.py
diff --git a/dak/contents.py b/dak/contents.py
index 1b3f3e1..073eb50 100755
--- a/dak/contents.py
+++ b/dak/contents.py
@@ -40,14 +40,18 @@ import logging
import math
import gzip
import threading
+import traceback
import Queue
import apt_pkg
+import datetime #just for debugging, can be removed
from daklib import utils
from daklib.binary import Binary
from daklib.config import Config
from daklib.dbconn import DBConn
################################################################################
+log=None
+
def usage (exit_code=0):
print """Usage: dak contents [options] command [arguments]
@@ -83,7 +87,7 @@ OPTIONS
options_prefix = "Contents"
options_prefix = "%s::Options" % options_prefix
-log = logging.getLogger()
+#log = logging.getLogger()
################################################################################
@@ -103,54 +107,23 @@ debs_q = """PREPARE debs_q(int, int) as
WHERE suite = $1
AND arch = $2"""
-# ask if we already have contents associated with this binary
-olddeb_q = """PREPARE olddeb_q(int) as
- SELECT 1 FROM content_associations
- WHERE binary_pkg = $1
- LIMIT 1"""
-
# find me all of the contents for a given .deb
contents_q = """PREPARE contents_q(int,int) as
- SELECT (p.path||'/'||n.file) AS fn,
- s.section,
- b.package,
- b.architecture
- FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
- JOIN content_file_names n ON (c.filename=n.id)
- JOIN binaries b ON (b.id=c.binary_pkg)
- JOIN override o ON (o.package=b.package)
- JOIN section s ON (s.id=o.section)
- WHERE o.suite = $1 AND o.type = $2
- AND b.type='deb'
- ORDER BY fn"""
-
+ SELECT file, section, package
+ FROM deb_contents
+ WHERE suite = $1
+ AND (arch = $2 or arch=2)"""
+# ORDER BY file"""
+
# find me all of the contents for a given .udeb
-udeb_contents_q = """PREPARE udeb_contents_q(int,int,int) as
- SELECT (p.path||'/'||n.file) AS fn,
- s.section,
- b.package,
- b.architecture
- FROM content_associations c join content_file_paths p ON (c.filepath=p.id)
- JOIN content_file_names n ON (c.filename=n.id)
- JOIN binaries b ON (b.id=c.binary_pkg)
- JOIN override o ON (o.package=b.package)
- JOIN section s ON (s.id=o.section)
- WHERE o.suite = $1 AND o.type = $2
- AND s.id = $3
- AND b.type='udeb'
- ORDER BY fn"""
-
-# FROM content_file_paths p join content_associations c ON (c.filepath=p.id)
-# JOIN content_file_names n ON (c.filename=n.id)
-# JOIN binaries b ON (b.id=c.binary_pkg)
-# JOIN override o ON (o.package=b.package)
-# JOIN section s ON (s.id=o.section)
-# WHERE o.suite = $1 AND o.type = $2
-# AND s.id = $3
-# AND b.id in (SELECT ba.bin from bin_associations ba join binaries b on b.id=ba.bin where (b.architecture=$3 or b.architecture=$4)and ba.suite=$1 and b.type='udeb')
-# GROUP BY fn
-# ORDER BY fn;"""
-
+udeb_contents_q = """PREPARE udeb_contents_q(int,int,text, int) as
+ SELECT file, section, package, arch
+ FROM udeb_contents
+ WHERE suite = $1
+ AND otype = $2
+ AND section = $3
+ and arch = $4
+ ORDER BY file"""
# clear out all of the temporarily stored content associations
@@ -159,121 +132,213 @@ udeb_contents_q = """PREPARE udeb_contents_q(int,int,int) as
# so there should no longer be anything in the queue
remove_pending_contents_cruft_q = """DELETE FROM pending_content_associations"""
-# delete any filenames we are storing which have no binary associated with them
-remove_filename_cruft_q = """DELETE FROM content_file_names
- WHERE id IN (SELECT cfn.id FROM content_file_names cfn
- LEFT JOIN content_associations ca
- ON ca.filename=cfn.id
- WHERE ca.id IS NULL)"""
-
-# delete any paths we are storing which have no binary associated with them
-remove_filepath_cruft_q = """DELETE FROM content_file_paths
- WHERE id IN (SELECT cfn.id FROM content_file_paths cfn
- LEFT JOIN content_associations ca
- ON ca.filepath=cfn.id
- WHERE ca.id IS NULL)"""
-
class EndOfContents(object):
- """
- A sentry object for the end of the filename stream
- """
pass
-class GzippedContentWriter(object):
+class OneAtATime(object):
"""
- An object which will write contents out to a Contents-$arch.gz
- file on a separate thread
"""
+ def __init__(self):
+ self.next_in_line = None
+ self.next_lock = threading.Condition()
+
+ def enqueue(self, next):
+ self.next_lock.acquire()
+ while self.next_in_line:
+ self.next_lock.wait()
+
+ assert( not self.next_in_line )
+ self.next_in_line = next
+ self.next_lock.notify()
+ self.next_lock.release()
+
+ def dequeue(self):
+ self.next_lock.acquire()
+ while not self.next_in_line:
+ self.next_lock.wait()
+ result = self.next_in_line
+ self.next_in_line = None
+ self.next_lock.notify()
+ self.next_lock.release()
+ return result
+
+
+class ContentsWorkThread(threading.Thread):
+ """
+ """
+ def __init__(self, upstream, downstream):
+ threading.Thread.__init__(self)
+ self.upstream = upstream
+ self.downstream = downstream
- header = None # a class object holding the header section of contents file
+ def run(self):
+ while True:
+ try:
+ contents_file = self.upstream.dequeue()
+ if isinstance(contents_file,EndOfContents):
+ if self.downstream:
+ self.downstream.enqueue(contents_file)
+ break
+
+ s = datetime.datetime.now()
+ print("%s start: %s" % (self,contents_file) )
+ self._run(contents_file)
+ print("%s finished: %s in %d seconds" % (self, contents_file, (datetime.datetime.now()-s).seconds ))
+ if self.downstream:
+ self.downstream.enqueue(contents_file)
+ except:
+ traceback.print_exc()
+
+class QueryThread(ContentsWorkThread):
+ def __init__(self, upstream, downstream):
+ ContentsWorkThread.__init__(self, upstream, downstream)
+
+ def __str__(self):
+ return "QueryThread"
+ __repr__ = __str__
+
+ def _run(self, contents_file):
+ contents_file.query()
+
+class IngestThread(ContentsWorkThread):
+ def __init__(self, upstream, downstream):
+ ContentsWorkThread.__init__(self, upstream, downstream)
+
+ def __str__(self):
+ return "IngestThread"
+ __repr__ = __str__
+
+ def _run(self, contents_file):
+ contents_file.ingest()
+
+class SortThread(ContentsWorkThread):
+ def __init__(self, upstream, downstream):
+ ContentsWorkThread.__init__(self, upstream, downstream)
+
+ def __str__(self):
+ return "SortThread"
+ __repr__ = __str__
+
+ def _run(self, contents_file):
+ contents_file.sorted_keys = sorted(contents_file.filenames.keys())
+
+class OutputThread(ContentsWorkThread):
+ def __init__(self, upstream, downstream):
+ ContentsWorkThread.__init__(self, upstream, downstream)
+
+ def __str__(self):
+ return "OutputThread"
+ __repr__ = __str__
+
+ def _run(self, contents_file):
+ contents_file.open_file()
+ for fname in contents_file.sorted_keys:
+ contents_file.filehandle.write("%s\t%s\n" % (fname,contents_file.filenames[fname]))
+ contents_file.sorted_keys = None
+ contents_file.filenames.clear()
+
+class GzipThread(ContentsWorkThread):
+ def __init__(self, upstream, downstream):
+ ContentsWorkThread.__init__(self, upstream, downstream)
+
+ def __str__(self):
+ return "GzipThread"
+ __repr__ = __str__
+
+ def _run(self, contents_file):
+ os.system("gzip -f %s" % contents_file.filename)
+
+class ContentFile(object):
+ def __init__(self,
+ filename,
+ suite_str,
+ suite_id,
+ arch_str,
+ arch_id):
+
+ self.filename = filename
+ self.filenames = {}
+ self.sorted_keys = None
+ self.suite_str = suite_str
+ self.suite_id = suite_id
+ self.arch_str = arch_str
+ self.arch_id = arch_id
+ self.cursor = None
+ self.filehandle = None
+
+ def __str__(self):
+ return self.filename
+ __repr__ = __str__
+
+
+ def cleanup(self):
+ self.filenames = None
+ self.sortedkeys = None
+ self.filehandle.close()
+ self.cursor.close()
+
+ def query(self):
+ self.cursor = DBConn().cursor();
+
+ self.cursor.execute("""SELECT file, section || '/' || package
+ FROM deb_contents
+ WHERE ( arch=2 or arch = %d) AND suite = %d
+ """ % (self.arch_id, self.suite_id))
+
+ def ingest(self):
+ while True:
+ r = self.cursor.fetchone()
+ if not r:
+ break
+ filename, package = r
+ if self.filenames.has_key(filename):
+ self.filenames[filename] += ",%s" % (package)
+ else:
+ self.filenames[filename] = "%s" % (package)
+ self.cursor.close()
- def __init__(self, filename):
- """
- @type filename: string
- @param filename: the name of the file to write to
- """
- self.queue = Queue.Queue()
- self.current_file = None
- self.first_package = True
- self.output = self.open_file(filename)
- self.thread = threading.Thread(target=self.write_thread,
- name='Contents writer')
- self.thread.start()
-
- def open_file(self, filename):
+ def open_file(self):
"""
opens a gzip stream to the contents file
"""
- filepath = Config()["Contents::Root"] + filename
- filedir = os.path.dirname(filepath)
+# filepath = Config()["Contents::Root"] + self.filename
+ self.filename = "/home/stew/contents/" + self.filename
+ filedir = os.path.dirname(self.filename)
if not os.path.isdir(filedir):
os.makedirs(filedir)
- return gzip.open(filepath, "w")
-
- def write(self, filename, section, package):
- """
- enqueue content to be written to the file on a separate thread
- """
- self.queue.put((filename,section,package))
-
- def write_thread(self):
- """
- the target of a Thread which will do the actual writing
- """
- while True:
- next = self.queue.get()
- if isinstance(next, EndOfContents):
- self.output.write('\n')
- self.output.close()
- break
-
- (filename,section,package)=next
- if next != self.current_file:
- # this is the first file, so write the header first
- if not self.current_file:
- self.output.write(self._getHeader())
-
- self.output.write('\n%s\t' % filename)
- self.first_package = True
+# self.filehandle = gzip.open(self.filename, "w")
+ self.filehandle = open(self.filename, "w")
+ self._write_header()
- self.current_file=filename
+ def _write_header(self):
+ self._get_header();
+ self.filehandle.write(ContentFile.header)
- if not self.first_package:
- self.output.write(',')
- else:
- self.first_package=False
- self.output.write('%s/%s' % (section,package))
-
- def finish(self):
- """
- enqueue the sentry object so that writers will know to terminate
- """
- self.queue.put(EndOfContents())
+ header=None
@classmethod
- def _getHeader(self):
+ def _get_header(self):
"""
Internal method to return the header for Contents.gz files
This is boilerplate which explains the contents of the file and how
it can be used.
"""
- if not GzippedContentWriter.header:
+ if not ContentFile.header:
if Config().has_key("Contents::Header"):
try:
h = open(os.path.join( Config()["Dir::Templates"],
Config()["Contents::Header"] ), "r")
- GzippedContentWriter.header = h.read()
+ ContentFile.header = h.read()
h.close()
except:
log.error( "error opening header file: %d\n%s" % (Config()["Contents::Header"],
traceback.format_exc() ))
- GzippedContentWriter.header = None
+ ContentFile.header = None
else:
- GzippedContentWriter.header = None
-
- return GzippedContentWriter.header
+ ContentFile.header = None
+ return ContentFile.header
class Contents(object):
"""
@@ -312,7 +377,6 @@ class Contents(object):
cursor = DBConn().cursor();
DBConn().prepare("debs_q",debs_q)
- DBConn().prepare("olddeb_q",olddeb_q)
DBConn().prepare("arches_q",arches_q)
suites = self._suites()
@@ -331,76 +395,92 @@ class Contents(object):
break
count += 1
cursor1 = DBConn().cursor();
- cursor1.execute( "EXECUTE olddeb_q(%d)" % (deb[0] ) )
+ cursor1.execute( "SELECT 1 FROM deb_contents WHERE binary_id = %d LIMIT 1" % (deb[0] ) )
old = cursor1.fetchone()
if old:
- log.debug( "already imported: %s" % (deb[1]) )
+ log.log( "already imported: %s" % (deb[1]) )
else:
- log.debug( "scanning: %s" % (deb[1]) )
+# log.debug( "scanning: %s" % (deb[1]) )
+ log.log( "scanning: %s" % (deb[1]) )
debfile = os.path.join( pooldir, deb[1] )
if os.path.exists( debfile ):
- Binary(debfile, self.reject).scan_package(deb[0],True)
+ Binary(debfile, self.reject).scan_package(deb[0], True)
else:
log.error("missing .deb: %s" % deb[1])
+
def generate(self):
"""
- Generate Contents-$arch.gz files for every available arch in each given suite.
+ Generate contents files for both deb and udeb
"""
- cursor = DBConn().cursor()
-
DBConn().prepare("arches_q", arches_q)
- DBConn().prepare("contents_q", contents_q)
- DBConn().prepare("udeb_contents_q", udeb_contents_q)
-
- debtype_id=DBConn().get_override_type_id("deb")
- udebtype_id=DBConn().get_override_type_id("udeb")
+ self.deb_generate()
+# self.udeb_generate()
- arch_all_id = DBConn().get_architecture_id("all")
+ def deb_generate(self):
+ """
+ Generate Contents-$arch.gz files for every available arch in each given suite.
+ """
+ cursor = DBConn().cursor()
+ debtype_id = DBConn().get_override_type_id("deb")
suites = self._suites()
-
+ inputtoquery = OneAtATime()
+ querytoingest = OneAtATime()
+ ingesttosort = OneAtATime()
+ sorttooutput = OneAtATime()
+ outputtogzip = OneAtATime()
+
+ qt = QueryThread(inputtoquery,querytoingest)
+ it = IngestThread(querytoingest,ingesttosort)
+# these actually make things worse
+# it2 = IngestThread(querytoingest,ingesttosort)
+# it3 = IngestThread(querytoingest,ingesttosort)
+# it4 = IngestThread(querytoingest,ingesttosort)
+ st = SortThread(ingesttosort,sorttooutput)
+ ot = OutputThread(sorttooutput,outputtogzip)
+ gt = GzipThread(outputtogzip, None)
+
+ qt.start()
+ it.start()
+# it2.start()
+# it3.start()
+# it2.start()
+ st.start()
+ ot.start()
+ gt.start()
+
# Get our suites, and the architectures
for suite in [i.lower() for i in suites]:
suite_id = DBConn().get_suite_id(suite)
arch_list = self._arches(cursor, suite_id)
- file_writers = {}
-
- try:
- for arch_id in arch_list:
- file_writers[arch_id[0]] = GzippedContentWriter("dists/%s/Contents-%s.gz" % (suite, arch_id[1]))
-
- cursor.execute("EXECUTE contents_q(%d,%d);" % (suite_id, debtype_id))
+ for (arch_id,arch_str) in arch_list:
+ print( "suite: %s, arch: %s time: %s" %(suite_id, arch_id, datetime.datetime.now().isoformat()) )
- while True:
- r = cursor.fetchone()
- if not r:
- break
-
- filename, section, package, arch = r
-
- if not file_writers.has_key( arch ):
- continue
+# filename = "dists/%s/Contents-%s.gz" % (suite, arch_str)
+ filename = "dists/%s/Contents-%s" % (suite, arch_str)
+ cf = ContentFile(filename, suite, suite_id, arch_str, arch_id)
+ inputtoquery.enqueue( cf )
- if arch == arch_all_id:
- ## its arch all, so all contents files get it
- for writer in file_writers.values():
- writer.write(filename, section, package)
+ inputtoquery.enqueue( EndOfContents() )
+ gt.join()
- else:
- file_writers[arch].write(filename, section, package)
+ def udeb_generate(self):
+ """
+ Generate Contents-$arch.gz files for every available arch in each given suite.
+ """
+ cursor = DBConn().cursor()
- finally:
- # close all the files
- for writer in file_writers.values():
- writer.finish()
+ DBConn().prepare("udeb_contents_q", udeb_contents_q)
+ udebtype_id=DBConn().get_override_type_id("udeb")
+ suites = self._suites()
+# for section, fn_pattern in [("debian-installer","dists/%s/Contents-udeb-%s.gz"),
+# ("non-free/debian-installer", "dists/%s/Contents-udeb-nf-%s.gz")]:
- # The MORE fun part. Ok, udebs need their own contents files, udeb, and udeb-nf (not-free)
- # This is HORRIBLY debian specific :-/
- for section, fn_pattern in [("debian-installer","dists/%s/Contents-udeb-%s.gz"),
- ("non-free/debian-installer", "dists/%s/Contents-udeb-nf-%s.gz")]:
+ for section, fn_pattern in [("debian-installer","dists/%s/Contents-udeb-%s"),
+ ("non-free/debian-installer", "dists/%s/Contents-udeb-nf-%s")]:
section_id = DBConn().get_section_id(section) # all udebs should be here)
if section_id != -1:
@@ -410,35 +490,22 @@ class Contents(object):
suite_id = DBConn().get_suite_id(suite)
arch_list = self._arches(cursor, suite_id)
- file_writers = {}
+ for arch_id in arch_list:
- try:
- for arch_id in arch_list:
- file_writers[arch_id[0]] = GzippedContentWriter(fn_pattern % (suite, arch_id[1]))
+ writer = GzippedContentWriter(fn_pattern % (suite, arch_id[1]))
+ try:
- cursor.execute("EXECUTE udeb_contents_q(%d,%d,%d)" % (suite_id, udebtype_id, section_id))
+ cursor.execute("EXECUTE udeb_contents_q(%d,%d,%d)" % (suite_id, udebtype_id, section_id, arch_id))
- while True:
- r = cursor.fetchone()
- if not r:
- break
+ while True:
+ r = cursor.fetchone()
+ if not r:
+ break
- filename, section, package, arch = r
-
- if not file_writers.has_key( arch ):
- continue
-
- if arch == arch_all_id:
- ## its arch all, so all contents files get it
- for writer in file_writers.values():
- writer.write(filename, section, package)
-
- else:
- file_writers[arch].write(filename, section, package)
- finally:
- # close all the files
- for writer in file_writers.values():
- writer.finish()
+ filename, section, package, arch = r
+ writer.write(filename, section, package)
+ finally:
+ writer.close()
@@ -451,7 +518,8 @@ class Contents(object):
if Config().has_key( "%s::%s" %(options_prefix,"Suite")):
suites = utils.split_args(Config()[ "%s::%s" %(options_prefix,"Suite")])
else:
- suites = Config().SubTree("Suite").List()
+ suites = [ 'unstable', 'testing' ]
+# suites = Config().SubTree("Suite").List()
return suites
@@ -476,7 +544,8 @@ class Contents(object):
def main():
cnf = Config()
-
+# log = logging.Logger(cnf, "contents")
+
arguments = [('h',"help", "%s::%s" % (options_prefix,"Help")),
('s',"suite", "%s::%s" % (options_prefix,"Suite"),"HasArg"),
('q',"quiet", "%s::%s" % (options_prefix,"Quiet")),
@@ -496,17 +565,17 @@ def main():
if cnf.has_key("%s::%s" % (options_prefix,"Help")):
usage()
- level=logging.INFO
- if cnf.has_key("%s::%s" % (options_prefix,"Quiet")):
- level=logging.ERROR
+# level=logging.INFO
+# if cnf.has_key("%s::%s" % (options_prefix,"Quiet")):
+# level=logging.ERROR
- elif cnf.has_key("%s::%s" % (options_prefix,"Verbose")):
- level=logging.DEBUG
+# elif cnf.has_key("%s::%s" % (options_prefix,"Verbose")):
+# level=logging.DEBUG
- logging.basicConfig( level=level,
- format='%(asctime)s %(levelname)s %(message)s',
- stream = sys.stderr )
+# logging.basicConfig( level=level,
+# format='%(asctime)s %(levelname)s %(message)s',
+# stream = sys.stderr )
commands[args[0]](Contents())
diff --git a/dak/dakdb/update17.py b/dak/dakdb/update17.py
new file mode 100644
index 0000000..a3e134a
--- /dev/null
+++ b/dak/dakdb/update17.py
@@ -0,0 +1,310 @@
+#!/usr/bin/env python
+# coding=utf8
+
+"""
+Adding a trainee field to the process-new notes
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2009 Mike O'Connor <stew@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+
+################################################################################
+
+import psycopg2
+import time
+from daklib.dak_exceptions import DBUpdateError
+
+################################################################################
+
+def suites():
+ """
+ return a list of suites to operate on
+ """
+ if Config().has_key( "%s::%s" %(options_prefix,"Suite")):
+ suites = utils.split_args(Config()[ "%s::%s" %(options_prefix,"Suite")])
+ else:
+ suites = [ 'unstable', 'testing' ]
+# suites = Config().SubTree("Suite").List()
+
+ return suites
+
+def arches(cursor, suite):
+ """
+ return a list of archs to operate on
+ """
+ arch_list = []
+ cursor.execute("EXECUTE arches_q(%d)" % (suite))
+ while True:
+ r = cursor.fetchone()
+ if not r:
+ break
+
+ if r[1] != "source" and r[1] != "all":
+ arch_list.append((r[0], r[1]))
+
+ return arch_list
+
+def do_update(self):
+ """
+ Adding contents table as first step to maybe, finally getting rid
+ of apt-ftparchive
+ """
+
+ print __doc__
+
+ try:
+ c = self.db.cursor()
+ c.execute("""CREATE TABLE deb_contents (
+ file text,
+ section text,
+ package text,
+ binary_id integer,
+ arch integer,
+ suite integer,
+ component integer)""" )
+
+ c.execute("""CREATE TABLE udeb_contents (
+ file text,
+ section text,
+ package text,
+ binary_id integer,
+ suite integer,
+ component integer )""" )
+
+ c.execute("""ALTER TABLE ONLY deb_contents
+ ADD CONSTRAINT deb_contents_arch_fkey
+ FOREIGN KEY (arch) REFERENCES architecture(id)
+ ON DELETE CASCADE;""")
+
+ c.execute("""ALTER TABLE ONLY udeb_contents
+ ADD CONSTRAINT udeb_contents_arch_fkey
+ FOREIGN KEY (arch) REFERENCES architecture(id)
+ ON DELETE CASCADE;""")
+
+ c.execute("""ALTER TABLE ONLY deb_contents
+ ADD CONSTRAINT deb_contents_suite_fkey
+ FOREIGN KEY (suite) REFERENCES suite(id)
+ ON DELETE CASCADE;""")
+
+ c.execute("""ALTER TABLE ONLY udeb_contents
+ ADD CONSTRAINT udeb_contents_suite_fkey
+ FOREIGN KEY (suite) REFERENCES suite(id)
+ ON DELETE CASCADE;""")
+
+ c.execute("""ALTER TABLE ONLY deb_contents
+ ADD CONSTRAINT deb_contents_binary_fkey
+ FOREIGN KEY (binary_id) REFERENCES binaries(id)
+ ON DELETE CASCADE;""")
+
+ c.execute("""ALTER TABLE ONLY udeb_contents
+ ADD CONSTRAINT udeb_contents_binary_fkey
+ FOREIGN KEY (binary_id) REFERENCES binaries(id)
+ ON DELETE CASCADE;""")
+
+ c.execute("""CREATE INDEX ind_deb_contents_binary ON deb_contents(binary_id);""" )
+
+ arches_q = """PREPARE arches_q(int) as
+ SELECT s.architecture, a.arch_string
+ FROM suite_architectures s
+ JOIN architecture a ON (s.architecture=a.id)
+ WHERE suite = $1"""
+
+ suites = self.suites()
+
+ for suite in [i.lower() for i in suites]:
+ suite_id = DBConn().get_suite_id(suite)
+ arch_list = arches(c, suite_id)
+ arch_list = arches(c, suite_id)
+
+ for (arch_id,arch_str) in arch_list:
+ c.execute( "CREATE INDEX ind_deb_contents_%s_%s ON deb_contents (arch,suite) WHERE (arch=2 OR arch=%d) AND suite=$d"%(arch_str,suite,arch_id,suite_id) )
+
+ for section, sname in [("debian-installer","main"),
+ ("non-free/debian-installer", "nonfree")]:
+ c.execute( "CREATE INDEX ind_udeb_contents_%s_%s ON udeb_contents (section,suite) WHERE section=%s AND suite=$d"%(sname,suite,section,suite_id) )
+
+
+ Column | Type | Modifiers
+ ------------+---------+-----------
+ package | text | not null
+ suite | integer | not null
+ component | integer | not null
+ priority | integer |
+ section | integer | not null
+ type | integer | not null
+ maintainer | text |
+
+ c.execute("""CREATE TABLE deb_contents (
+ file text,
+ section text,
+ package text,
+ binary_id integer,
+ arch integer,
+ suite integer,
+ component integer)""" )
+
+
+CREATE FUNCTION update_contents_for_override() RETURNS trigger AS $update_contents_for_override$
+BEGIN
+ UPDATE deb_contents SET section=NEW.section, component=NEW.component
+ WHERE deb_contents.package=OLD.package
+
+
+DELETE FROM
+NEW.last_date := current_timestamp;
+NEW.last_user := current_user;
+RETURN NEW;
+END;
+$update_contents_for_override$ LANGUAGE plpgsql;
+
+
+ self.db.commit()
+
+ except psycopg2.ProgrammingError, msg:
+ self.db.rollback()
+ raise DBUpdateError, "Unable to apply process-new update 14, rollback issued. Error message : %s" % (str(msg))
+"""
+ INSERT INTO deb_contents SELECT (p.path||'/'||n.file) AS file,
+ s.section AS section,
+ b.package AS package,
+ b.id AS binary_id,
+ b.architecture AS arch,
+ o.suite AS suited,
+ o.component AS componentd,
+ o.type AS otype_id
+ FROM content_associations c
+ JOIN content_file_paths p ON (c.filepath=p.id)
+ JOIN content_file_names n ON (c.filename=n.id)
+ JOIN binaries b ON (b.id=c.binary_pkg)
+ JOIN architecture a ON (b.architecture = a.id)
+ JOIN override o ON (o.package=b.package)
+ JOIN bin_associations ba on ba.suite=o.suite and ba.bin=b.id
+ JOIN section s ON (s.id=o.section)
+ where b.type='deb';
+
+ INSERT INTO udeb_contents SELECT (p.path||'/'||n.file) AS file,
+ s.section AS section,
+ b.package AS package,
+ b.id AS binary_id,
+ b.architecture AS arch,
+ o.suite AS suited,
+ o.component AS componentd,
+ o.type AS otype_id
+ FROM content_associations c
+ JOIN content_file_paths p ON (c.filepath=p.id)
+ JOIN content_file_names n ON (c.filename=n.id)
+ JOIN binaries b ON (b.id=c.binary_pkg)
+ JOIN architecture a ON (b.architecture = a.id)
+ JOIN override o ON (o.package=b.package)
+ JOIN section s ON (s.id=o.section)
+ where b.type='udeb'
+"""
+
+"""
+CREATE INDEX ind_archid ON contents(arch);
+CREATE INDEX ind_archid_amd64 ON contents(arch) WHERE arch=16;
+CREATE INDEX ind_suite ON contents(suite);
+CREATE INDEX ind_suite_unstable ON contents(suite) WHERE suite=5;
+CREATE INDEX ind_overridetype ON contents(otype);
+CREATE INDEX ind_overridetype_deb ON contents(otype) WHERE otype=7;
+CREATE INDEX ind_packagetype ON contents(packagetype);
+CREATE INDEX ind_packagetype_deb ON contents(packagetype) WHERE packagetype='deb';
+CREATE INDEX ind_package ON contents(package);
+
+ CREATE INDEX ind_suite_otype ON contents(suite, otype) WHERE suite=5 AND otype=7;
+ CREATE INDEX ind_suite_otype_arch ON contents(suite, otype, arch) WHERE suite=5 AND otype=7 AND arch=16;
+ CREATE INDEX ind_suite_otype_package ON contents(suite, otype, packagetype) WHERE suite=5 AND otype=7 AND packagetype='deb';
+ CREATE INDEX ind_suite_otype_package_notdeb ON contents(suite, otype, packagetype) WHERE suite=5 AND otype=7 AND packagetype!='deb';
+ """
+
+CREATE INDEX ind_deb_contents_binary ON deb_contents(binary_id);
+
+CREATE INDEX ind_deb_contents_arch_alpha_unstable ON deb_contents(arch) where (arch=2 or arch=3) AND suite=5 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_hurd_i386_unstable ON deb_contents(arch) where (arch=2 or arch=4) AND suite=5 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_hppa_unstable ON deb_contents(arch) where (arch=2 or arch=5) AND suite=5 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_arm_unstable ON deb_contents(arch) where (arch=2 or arch=6) AND suite=5 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_i386_unstable ON deb_contents(arch) where (arch=2 or arch=7) AND suite=5 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_m68k_unstable ON deb_contents(arch) where (arch=2 or arch=8) AND suite=5 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_mips_unstable ON deb_contents(arch) where (arch=2 or arch=9) AND suite=5 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_mipsel_unstable ON deb_contents(arch) where (arch=2 or arch=10) AND suite=5 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_powerpc_unstable ON deb_contents(arch) where (arch=2 or arch=11) AND suite=5 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_sh_unstable ON deb_contents(arch) where (arch=2 or arch=12) AND suite=5 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_sparc_unstable ON deb_contents(arch) where (arch=2 or arch=13) AND suite=5 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_s390_unstable ON deb_contents(arch) where (arch=2 or arch=14) AND suite=5 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_ia64_unstable ON deb_contents(arch) where (arch=2 or arch=15) AND suite=5 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_amd64_unstable ON deb_contents(arch) where (arch=2 or arch=16) AND suite=5 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_armel_unstable ON deb_contents(arch) where (arch=2 or arch=17) AND suite=5 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_kfreebsd_i386_unstable ON deb_contents(arch) where (arch=2 or arch=25) AND suite=5 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_kfreebsd_amd64_unstable ON deb_contents(arch) where (arch=2 or arch=26) AND suite=5 AND otype=7;
+
+CREATE INDEX ind_deb_contents_arch_alpha_stable ON deb_contents(arch) where (arch=2 or arch=3) AND suite=2 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_hurd_i386_stable ON deb_contents(arch) where (arch=2 or arch=4) AND suite=2 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_hppa_stable ON deb_contents(arch) where (arch=2 or arch=5) AND suite=2 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_arm_stable ON deb_contents(arch) where (arch=2 or arch=6) AND suite=2 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_i386_stable ON deb_contents(arch) where (arch=2 or arch=7) AND suite=2 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_m68k_stable ON deb_contents(arch) where (arch=2 or arch=8) AND suite=2 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_mips_stable ON deb_contents(arch) where (arch=2 or arch=9) AND suite=2 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_mipsel_stable ON deb_contents(arch) where (arch=2 or arch=10) AND suite=2 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_powerpc_stable ON deb_contents(arch) where (arch=2 or arch=11) AND suite=2 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_sh_stable ON deb_contents(arch) where (arch=2 or arch=12) AND suite=2 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_sparc_stable ON deb_contents(arch) where (arch=2 or arch=13) AND suite=2 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_s390_stable ON deb_contents(arch) where (arch=2 or arch=14) AND suite=2 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_ia64_stable ON deb_contents(arch) where (arch=2 or arch=15) AND suite=2 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_amd64_stable ON deb_contents(arch) where (arch=2 or arch=16) AND suite=2 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_armel_stable ON deb_contents(arch) where (arch=2 or arch=17) AND suite=2 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_kfreebsd_i386_stable ON deb_contents(arch) where (arch=2 or arch=25) AND suite=2 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_kfreebsd_amd64_stable ON deb_contents(arch) where (arch=2 or arch=26) AND suite=2 AND otype=7;
+
+CREATE INDEX ind_deb_contents_arch_alpha_testing ON deb_contents(arch) where (arch=2 or arch=3) AND suite=4 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_hurd_i386_testing ON deb_contents(arch) where (arch=2 or arch=4) AND suite=4 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_hppa_testing ON deb_contents(arch) where (arch=2 or arch=5) AND suite=4 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_arm_testing ON deb_contents(arch) where (arch=2 or arch=6) AND suite=4 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_i386_testing ON deb_contents(arch) where (arch=2 or arch=7) AND suite=4 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_m68k_testing ON deb_contents(arch) where (arch=2 or arch=8) AND suite=4 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_mips_testing ON deb_contents(arch) where (arch=2 or arch=9) AND suite=4 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_mipsel_testing ON deb_contents(arch) where (arch=2 or arch=10) AND suite=4 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_powerpc_testing ON deb_contents(arch) where (arch=2 or arch=11) AND suite=4 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_sh_testing ON deb_contents(arch) where (arch=2 or arch=12) AND suite=4 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_sparc_testing ON deb_contents(arch) where (arch=2 or arch=13) AND suite=4 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_s390_testing ON deb_contents(arch) where (arch=2 or arch=14) AND suite=4 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_ia64_testing ON deb_contents(arch) where (arch=2 or arch=15) AND suite=4 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_amd64_testing ON deb_contents(arch) where (arch=2 or arch=16) AND suite=4 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_armel_testing ON deb_contents(arch) where (arch=2 or arch=17) AND suite=4 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_kfreebsd_i386_testing ON deb_contents(arch) where (arch=2 or arch=25) AND suite=4 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_kfreebsd_amd64_testing ON deb_contents(arch) where (arch=2 or arch=26) AND suite=4 AND otype=7;
+
+CREATE INDEX ind_deb_contents_arch_alpha_oldstable ON deb_contents(arch) where (arch=2 or arch=3) AND suite=14 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_hurd_i386_oldstable ON deb_contents(arch) where (arch=2 or arch=4) AND suite=14 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_hppa_oldstable ON deb_contents(arch) where (arch=2 or arch=5) AND suite=14 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_arm_oldstable ON deb_contents(arch) where (arch=2 or arch=6) AND suite=14 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_i386_oldstable ON deb_contents(arch) where (arch=2 or arch=7) AND suite=14 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_m68k_oldstable ON deb_contents(arch) where (arch=2 or arch=8) AND suite=14 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_mips_oldstable ON deb_contents(arch) where (arch=2 or arch=9) AND suite=14 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_mipsel_oldstable ON deb_contents(arch) where (arch=2 or arch=10) AND suite=14 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_powerpc_oldstable ON deb_contents(arch) where (arch=2 or arch=11) AND suite=14 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_sh_oldstable ON deb_contents(arch) where (arch=2 or arch=12) AND suite=14 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_sparc_oldstable ON deb_contents(arch) where (arch=2 or arch=13) AND suite=14 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_s390_oldstable ON deb_contents(arch) where (arch=2 or arch=14) AND suite=14 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_ia64_oldstable ON deb_contents(arch) where (arch=2 or arch=15) AND suite=14 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_amd64_oldstable ON deb_contents(arch) where (arch=2 or arch=16) AND suite=14 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_armel_oldstable ON deb_contents(arch) where (arch=2 or arch=17) AND suite=14 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_kfreebsd_i386_oldstable ON deb_contents(arch) where (arch=2 or arch=25) AND suite=14 AND otype=7;
+CREATE INDEX ind_deb_contents_arch_kfreebsd_amd64_oldstable ON deb_contents(arch) where (arch=2 or arch=26) AND suite=14 AND otype=7;
diff --git a/dak/update_db.py b/dak/update_db.py
index 5fe6918..e197edd 100755
--- a/dak/update_db.py
+++ b/dak/update_db.py
@@ -45,7 +45,7 @@ from daklib.dak_exceptions import DBUpdateError
Cnf = None
projectB = None
-required_database_schema = 13
+required_database_schema = 14
################################################################################
@@ -110,9 +110,10 @@ Updates dak's database schema to the lastest version. You should disable crontab
try:
# Build a connect string
- connect_str = "dbname=%s"% (Cnf["DB::Name"])
- if Cnf["DB::Host"] != '': connect_str += " host=%s" % (Cnf["DB::Host"])
- if Cnf["DB::Port"] != '-1': connect_str += " port=%d" % (int(Cnf["DB::Port"]))
+# connect_str = "dbname=%s"% (Cnf["DB::Name"])
+ connect_str = "dbname=%s"% "projectbstew"
+# if Cnf["DB::Host"] != '': connect_str += " host=%s" % (Cnf["DB::Host"])
+# if Cnf["DB::Port"] != '-1': connect_str += " port=%d" % (int(Cnf["DB::Port"]))
self.db = psycopg2.connect(connect_str)
diff --git a/daklib/dbconn.py b/daklib/dbconn.py
index 58dd7fc..a1ac703 100755
--- a/daklib/dbconn.py
+++ b/daklib/dbconn.py
@@ -73,7 +73,8 @@ class DBConn(Singleton):
## Connection functions
def __createconn(self):
cnf = Config()
- connstr = "dbname=%s" % cnf["DB::Name"]
+ connstr = "dbname=%s" % "projectbstew" #cnf["DB::Name"]
+ print( "connstr: %s "% connstr)
if cnf["DB::Host"]:
connstr += " host=%s" % cnf["DB::Host"]
if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
@@ -524,7 +525,8 @@ class DBConn(Singleton):
file_id = self.get_or_set_contents_file_id(file)
path_id = self.get_or_set_contents_path_id(path)
- c.execute("""INSERT INTO content_associations
+ c.execute("""INSERT INTO deb_contents
+
(binary_pkg, filepath, filename)
VALUES ( '%d', '%d', '%d')""" % (bin_id, path_id, file_id) )
diff --git a/daklib/utils.py b/daklib/utils.py
index 073b813..04570c9 100755
--- a/daklib/utils.py
+++ b/daklib/utils.py
@@ -48,7 +48,8 @@ from regexes import re_html_escaping, html_escaping, re_single_line_field, \
################################################################################
-default_config = "/etc/dak/dak.conf" #: default dak config, defines host properties
+#default_config = "/etc/dak/dak.conf" #: default dak config, defines host properties
+default_config = "/home/stew/etc/dak/dak.conf" #: default dak config, defines host properties
default_apt_config = "/etc/dak/apt.conf" #: default apt config, not normally used
alias_cache = None #: Cache for email alias checks
@@ -1558,7 +1559,7 @@ apt_pkg.init()
Cnf = apt_pkg.newConfiguration()
apt_pkg.ReadConfigFileISC(Cnf,default_config)
-if which_conf_file() != default_config:
- apt_pkg.ReadConfigFileISC(Cnf,which_conf_file())
+#if which_conf_file() != default_config:
+# apt_pkg.ReadConfigFileISC(Cnf,which_conf_file())
###############################################################################
--
1.6.3.3
Reply to: