[dak/master 1/2] Remove old contents code and tables.
Signed-off-by: Torsten Werner <twerner@debian.org>
---
dak/dakdb/update43.py | 52 +++++++
dak/update_db.py | 2 +-
daklib/binary.py | 367 -------------------------------------------------
daklib/dbconn.py | 135 ------------------
daklib/queue.py | 8 -
5 files changed, 53 insertions(+), 511 deletions(-)
create mode 100755 dak/dakdb/update43.py
delete mode 100644 daklib/binary.py
diff --git a/dak/dakdb/update43.py b/dak/dakdb/update43.py
new file mode 100755
index 0000000..b9c7f58
--- /dev/null
+++ b/dak/dakdb/update43.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# coding=utf8
+
+"""
+Remove old contents tables that are no longer needed by the current
+implementation.
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2011 Torsten Werner <twerner@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+import psycopg2
+from daklib.dak_exceptions import DBUpdateError
+from socket import gethostname;
+
+################################################################################
+def do_update(self):
+ """
+ Remove old contents tables that are no longer needed by the current
+ implementation.
+ """
+ print __doc__
+ try:
+ c = self.db.cursor()
+
+ # remove useless type casts
+ for table in ('pending_bin_contents', 'deb_contents', 'udeb_contents'):
+ c.execute("DROP TABLE %s" % table)
+
+ c.execute("UPDATE config SET value = '43' WHERE name = 'db_revision'")
+ self.db.commit()
+
+ except psycopg2.ProgrammingError, msg:
+ self.db.rollback()
+ raise DBUpdateError, 'Unable to apply sick update 43, rollback issued. Error message : %s' % (str(msg))
diff --git a/dak/update_db.py b/dak/update_db.py
index 928396c..71bac95 100755
--- a/dak/update_db.py
+++ b/dak/update_db.py
@@ -45,7 +45,7 @@ from daklib.dak_exceptions import DBUpdateError
################################################################################
Cnf = None
-required_database_schema = 42
+required_database_schema = 43
################################################################################
diff --git a/daklib/binary.py b/daklib/binary.py
deleted file mode 100644
index d1f78f3..0000000
--- a/daklib/binary.py
+++ /dev/null
@@ -1,367 +0,0 @@
-#!/usr/bin/python
-
-"""
-Functions related debian binary packages
-
-@contact: Debian FTPMaster <ftpmaster@debian.org>
-@copyright: 2009 Mike O'Connor <stew@debian.org>
-@license: GNU General Public License version 2 or later
-"""
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-################################################################################
-
-# <Ganneff> are we going the xorg way?
-# <Ganneff> a dak without a dak.conf?
-# <stew> automatically detect the wrong settings at runtime?
-# <Ganneff> yes!
-# <mhy> well, we'll probably always need dak.conf (how do you get the database setting
-# <mhy> but removing most of the config into the database seems sane
-# <Ganneff> mhy: dont spoil the fun
-# <Ganneff> mhy: and i know how. we nmap localhost and check all open ports
-# <Ganneff> maybe one answers to sql
-# <stew> we will discover projectb via avahi
-# <mhy> you're both sick
-# <mhy> really fucking sick
-
-################################################################################
-
-import os
-import sys
-import shutil
-import tarfile
-import commands
-import traceback
-import atexit
-
-try:
- # starting with squeeze
- from debian import deb822
-except:
- # up to lenny
- from debian_bundle import deb822
-
-from dbconn import *
-from config import Config
-import utils
-
-################################################################################
-
-__all__ = []
-
-################################################################################
-
-class Binary(object):
- def __init__(self, filename, reject=None):
- """
- @type filename: string
- @param filename: path of a .deb
-
- @type reject: function
- @param reject: a function to log reject messages to
- """
- self.filename = filename
- self.tmpdir = None
- self.chunks = None
- self.wrapped_reject = reject
- # Store rejects for later use
- self.rejects = []
-
- def reject(self, message):
- """
- if we were given a reject function, send the reject message,
- otherwise send it to stderr.
- """
- print >> sys.stderr, message
- self.rejects.append(message)
- if self.wrapped_reject:
- self.wrapped_reject(message)
-
- def __del__(self):
- """
- make sure we cleanup when we are garbage collected.
- """
- self._cleanup()
-
- def _cleanup(self):
- """
- we need to remove the temporary directory, if we created one
- """
- if self.tmpdir and os.path.exists(self.tmpdir):
- shutil.rmtree(self.tmpdir)
- self.tmpdir = None
-
- def __scan_ar(self):
- # get a list of the ar contents
- if not self.chunks:
-
- cmd = "ar t %s" % (self.filename)
- (result, output) = commands.getstatusoutput(cmd)
- if result != 0:
- rejected = True
- print("%s: 'ar t' invocation failed." % (self.filename))
- self.reject("%s: 'ar t' invocation failed." % (self.filename))
- self.reject(utils.prefix_multi_line_string(output, " [ar output:] "))
- self.chunks = output.split('\n')
-
-
-
- def __unpack(self):
- # Internal function which extracts the contents of the .ar to
- # a temporary directory
-
- if not self.tmpdir:
- tmpdir = utils.temp_dirname()
- cwd = os.getcwd()
- try:
- os.chdir( tmpdir )
- cmd = "ar x %s %s %s" % (os.path.join(cwd,self.filename), self.chunks[1], self.chunks[2])
- (result, output) = commands.getstatusoutput(cmd)
- if result != 0:
- print("%s: '%s' invocation failed." % (self.filename, cmd))
- self.reject("%s: '%s' invocation failed." % (self.filename, cmd))
- self.reject(utils.prefix_multi_line_string(output, " [ar output:] "))
- else:
- self.tmpdir = tmpdir
- atexit.register( self._cleanup )
-
- finally:
- os.chdir( cwd )
-
- def valid_deb(self, relaxed=False):
- """
- Check deb contents making sure the .deb contains:
- 1. debian-binary
- 2. control.tar.gz
- 3. data.tar.gz or data.tar.bz2
- in that order, and nothing else.
- """
- self.__scan_ar()
- rejected = not self.chunks
- if relaxed:
- if len(self.chunks) < 3:
- rejected = True
- self.reject("%s: found %d chunks, expected at least 3." % (self.filename, len(self.chunks)))
- else:
- if len(self.chunks) != 3:
- rejected = True
- self.reject("%s: found %d chunks, expected 3." % (self.filename, len(self.chunks)))
- if self.chunks[0] != "debian-binary":
- rejected = True
- self.reject("%s: first chunk is '%s', expected 'debian-binary'." % (self.filename, self.chunks[0]))
- if not rejected and self.chunks[1] != "control.tar.gz":
- rejected = True
- self.reject("%s: second chunk is '%s', expected 'control.tar.gz'." % (self.filename, self.chunks[1]))
- if not rejected and self.chunks[2] not in [ "data.tar.bz2", "data.tar.gz" ]:
- rejected = True
- self.reject("%s: third chunk is '%s', expected 'data.tar.gz' or 'data.tar.bz2'." % (self.filename, self.chunks[2]))
-
- return not rejected
-
- def scan_package(self, bootstrap_id=0, relaxed=False, session=None):
- """
- Unpack the .deb, do sanity checking, and gather info from it.
-
- Currently information gathering consists of getting the contents list. In
- the hopefully near future, it should also include gathering info from the
- control file.
-
- @type bootstrap_id: int
- @param bootstrap_id: the id of the binary these packages
- should be associated or zero meaning we are not bootstrapping
- so insert into a temporary table
-
- @return: True if the deb is valid and contents were imported
- """
- result = False
- rejected = not self.valid_deb(relaxed)
- if not rejected:
- self.__unpack()
-
-
- cwd = os.getcwd()
- if not rejected and self.tmpdir:
- try:
- os.chdir(self.tmpdir)
- if self.chunks[1] == "control.tar.gz":
- control = tarfile.open(os.path.join(self.tmpdir, "control.tar.gz" ), "r:gz")
- control.extract('./control', self.tmpdir )
- if self.chunks[2] == "data.tar.gz":
- data = tarfile.open(os.path.join(self.tmpdir, "data.tar.gz"), "r:gz")
- elif self.chunks[2] == "data.tar.bz2":
- data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2")
-
- if bootstrap_id:
- result = insert_content_paths(bootstrap_id, [tarinfo.name for tarinfo in data if not tarinfo.isdir()], session)
- else:
- pkgs = deb822.Packages.iter_paragraphs(file(os.path.join(self.tmpdir,'control')))
- pkg = pkgs.next()
- result = insert_pending_content_paths(pkg,
- self.filename.endswith('.udeb'),
- [tarinfo.name for tarinfo in data if not tarinfo.isdir()],
- session)
-
- except:
- traceback.print_exc()
-
- os.chdir(cwd)
- self._cleanup()
- return result
-
- def check_utf8_package(self, package):
- """
- Unpack the .deb, do sanity checking, and gather info from it.
-
- Currently information gathering consists of getting the contents list. In
- the hopefully near future, it should also include gathering info from the
- control file.
-
- @type package: string
- @param package: the name of the package to be checked
-
- @rtype: boolean
- @return: True if the deb is valid and contents were imported
- """
- rejected = not self.valid_deb(True)
- self.__unpack()
-
- if not rejected and self.tmpdir:
- cwd = os.getcwd()
- try:
- os.chdir(self.tmpdir)
- if self.chunks[1] == "control.tar.gz":
- control = tarfile.open(os.path.join(self.tmpdir, "control.tar.gz" ), "r:gz")
- control.extract('control', self.tmpdir )
- if self.chunks[2] == "data.tar.gz":
- data = tarfile.open(os.path.join(self.tmpdir, "data.tar.gz"), "r:gz")
- elif self.chunks[2] == "data.tar.bz2":
- data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2")
-
- for tarinfo in data:
- try:
- unicode( tarinfo.name )
- except:
- print >> sys.stderr, "E: %s has non-unicode filename: %s" % (package,tarinfo.name)
-
- result = True
-
- except:
- traceback.print_exc()
- result = False
-
- os.chdir(cwd)
-
- return result
-
-__all__.append('Binary')
-
-
-def copy_temporary_contents(binary, bin_association, reject, session=None):
- """
- copy the previously stored contents from the temp table to the permanant one
-
- during process-unchecked, the deb should have been scanned and the
- contents stored in pending_content_associations
- """
-
- cnf = Config()
-
- privatetrans = False
- if session is None:
- session = DBConn().session()
- privatetrans = True
-
- arch = get_architecture(archname, session=session)
-
- pending = session.query(PendingBinContents).filter_by(package=binary.package,
- version=binary.version,
- arch=binary.arch).first()
-
- if pending:
- # This should NOT happen. We should have added contents
- # during process-unchecked. if it did, log an error, and send
- # an email.
- subst = {
- "__PACKAGE__": package,
- "__VERSION__": version,
- "__ARCH__": arch,
- "__TO_ADDRESS__": cnf["Dinstall::MyAdminAddress"],
- "__DAK_ADDRESS__": cnf["Dinstall::MyEmailAddress"] }
-
- message = utils.TemplateSubst(subst, cnf["Dir::Templates"]+"/missing-contents")
- utils.send_mail(message)
-
- # rescan it now
- exists = Binary(deb, reject).scan_package()
-
- if not exists:
- # LOG?
- return False
-
- component = binary.poolfile.location.component
- override = session.query(Override).filter_by(package=binary.package,
- suite=bin_association.suite,
- component=component.id).first()
- if not override:
- # LOG?
- return False
-
-
- if not override.overridetype.type.endswith('deb'):
- return True
-
- if override.overridetype.type == "udeb":
- table = "udeb_contents"
- elif override.overridetype.type == "deb":
- table = "deb_contents"
- else:
- return False
-
-
- if component.name == "main":
- component_str = ""
- else:
- component_str = component.name + "/"
-
- vals = { 'package':binary.package,
- 'version':binary.version,
- 'arch':binary.architecture,
- 'binary_id': binary.id,
- 'component':component_str,
- 'section':override.section.section
- }
-
- session.execute( """INSERT INTO %s
- (binary_id,package,version.component,arch,section,filename)
- SELECT :binary_id, :package, :version, :component, :arch, :section
- FROM pending_bin_contents pbc
- WHERE pbc.package=:package
- AND pbc.version=:version
- AND pbc.arch=:arch""" % table, vals )
-
- session.execute( """DELETE from pending_bin_contents package=:package
- AND version=:version
- AND arch=:arch""", vals )
-
- if privatetrans:
- session.commit()
- session.close()
-
- return exists
-
-__all__.append('copy_temporary_contents')
-
-
diff --git a/daklib/dbconn.py b/daklib/dbconn.py
index 9b757d1..4f6b02d 100755
--- a/daklib/dbconn.py
+++ b/daklib/dbconn.py
@@ -77,9 +77,6 @@ import warnings
warnings.filterwarnings('ignore', \
"The SQLAlchemy PostgreSQL dialect has been renamed from 'postgres' to 'postgresql'.*", \
SADeprecationWarning)
-# TODO: sqlalchemy needs some extra configuration to correctly reflect
-# the ind_deb_contents_* indexes - we ignore the warnings at the moment
-warnings.filterwarnings("ignore", 'Predicate of partial index', SAWarning)
################################################################################
@@ -1948,111 +1945,6 @@ __all__.append('get_override_type')
################################################################################
-class DebContents(object):
- def __init__(self, *args, **kwargs):
- pass
-
- def __repr__(self):
- return '<DebConetnts %s: %s>' % (self.package.package,self.file)
-
-__all__.append('DebContents')
-
-
-class UdebContents(object):
- def __init__(self, *args, **kwargs):
- pass
-
- def __repr__(self):
- return '<UdebConetnts %s: %s>' % (self.package.package,self.file)
-
-__all__.append('UdebContents')
-
-class PendingBinContents(object):
- def __init__(self, *args, **kwargs):
- pass
-
- def __repr__(self):
- return '<PendingBinContents %s>' % self.contents_id
-
-__all__.append('PendingBinContents')
-
-def insert_pending_content_paths(package,
- is_udeb,
- fullpaths,
- session=None):
- """
- Make sure given paths are temporarily associated with given
- package
-
- @type package: dict
- @param package: the package to associate with should have been read in from the binary control file
- @type fullpaths: list
- @param fullpaths: the list of paths of the file being associated with the binary
- @type session: SQLAlchemy session
- @param session: Optional SQLAlchemy session. If this is passed, the caller
- is responsible for ensuring a transaction has begun and committing the
- results or rolling back based on the result code. If not passed, a commit
- will be performed at the end of the function
-
- @return: True upon success, False if there is a problem
- """
-
- privatetrans = False
-
- if session is None:
- session = DBConn().session()
- privatetrans = True
-
- try:
- arch = get_architecture(package['Architecture'], session)
- arch_id = arch.arch_id
-
- # Remove any already existing recorded files for this package
- q = session.query(PendingBinContents)
- q = q.filter_by(package=package['Package'])
- q = q.filter_by(version=package['Version'])
- q = q.filter_by(architecture=arch_id)
- q.delete()
-
- for fullpath in fullpaths:
-
- if fullpath.startswith( "./" ):
- fullpath = fullpath[2:]
-
- pca = PendingBinContents()
- pca.package = package['Package']
- pca.version = package['Version']
- pca.file = fullpath
- pca.architecture = arch_id
-
- if isudeb:
- pca.type = 8 # gross
- else:
- pca.type = 7 # also gross
- session.add(pca)
-
- # Only commit if we set up the session ourself
- if privatetrans:
- session.commit()
- session.close()
- else:
- session.flush()
-
- return True
- except Exception, e:
- traceback.print_exc()
-
- # Only rollback if we set up the session ourself
- if privatetrans:
- session.rollback()
- session.close()
-
- return False
-
-__all__.append('insert_pending_content_paths')
-
-################################################################################
-
class PolicyQueue(object):
def __init__(self, *args, **kwargs):
pass
@@ -2940,7 +2832,6 @@ class DBConn(object):
'maintainer',
'new_comments',
'override_type',
- 'pending_bin_contents',
'policy_queue',
'priority',
'section',
@@ -2958,13 +2849,11 @@ class DBConn(object):
'changes_pending_files_map',
'changes_pending_source_files',
'changes_pool_files',
- 'deb_contents',
# TODO: the maintainer column in table override should be removed.
'override',
'suite_architectures',
'suite_src_formats',
'suite_build_queue_copy',
- 'udeb_contents',
)
views = (
@@ -3028,30 +2917,6 @@ class DBConn(object):
properties = dict(archive_id = self.tbl_archive.c.id,
archive_name = self.tbl_archive.c.name))
- mapper(PendingBinContents, self.tbl_pending_bin_contents,
- properties = dict(contents_id =self.tbl_pending_bin_contents.c.id,
- filename = self.tbl_pending_bin_contents.c.filename,
- package = self.tbl_pending_bin_contents.c.package,
- version = self.tbl_pending_bin_contents.c.version,
- arch = self.tbl_pending_bin_contents.c.arch,
- otype = self.tbl_pending_bin_contents.c.type))
-
- mapper(DebContents, self.tbl_deb_contents,
- properties = dict(binary_id=self.tbl_deb_contents.c.binary_id,
- package=self.tbl_deb_contents.c.package,
- suite=self.tbl_deb_contents.c.suite,
- arch=self.tbl_deb_contents.c.arch,
- section=self.tbl_deb_contents.c.section,
- filename=self.tbl_deb_contents.c.filename))
-
- mapper(UdebContents, self.tbl_udeb_contents,
- properties = dict(binary_id=self.tbl_udeb_contents.c.binary_id,
- package=self.tbl_udeb_contents.c.package,
- suite=self.tbl_udeb_contents.c.suite,
- arch=self.tbl_udeb_contents.c.arch,
- section=self.tbl_udeb_contents.c.section,
- filename=self.tbl_udeb_contents.c.filename))
-
mapper(BuildQueue, self.tbl_build_queue,
properties = dict(queue_id = self.tbl_build_queue.c.id))
diff --git a/daklib/queue.py b/daklib/queue.py
index a7e2d5a..b3e26d7 100755
--- a/daklib/queue.py
+++ b/daklib/queue.py
@@ -53,7 +53,6 @@ from dbconn import *
from summarystats import SummaryStats
from utils import parse_changes, check_dsc_files
from textutils import fix_maintainer
-from binary import Binary
from lintian import parse_lintian_output, generate_reject_messages
# suppress some deprecation warnings in squeeze related to apt_pkg
@@ -866,13 +865,6 @@ class Upload(object):
# Check the version and for file overwrites
self.check_binary_against_db(f, session)
- # Temporarily disable contents generation until we change the table storage layout
- #b = Binary(f)
- #b.scan_package()
- #if len(b.rejects) > 0:
- # for j in b.rejects:
- # self.rejects.append(j)
-
def source_file_checks(self, f, session):
entry = self.pkg.files[f]
--
1.7.2.5
Reply to: