[dak/master 1/2] cruft-report: reimplement 'obsolete source'
Signed-off-by: Torsten Werner <twerner@debian.org>
---
dak/cruft_report.py | 109 +++++++++++++++++++++++-----------------
dak/dakdb/update27.py | 131 +++++++++++++++++++++++++++++++++++++++++++++++++
dak/update_db.py | 2 +-
3 files changed, 194 insertions(+), 48 deletions(-)
create mode 100755 dak/dakdb/update27.py
diff --git a/dak/cruft_report.py b/dak/cruft_report.py
index 73619d5..384ad0e 100755
--- a/dak/cruft_report.py
+++ b/dak/cruft_report.py
@@ -272,47 +272,64 @@ def do_dubious_nbs(dubious_nbs):
################################################################################
-def do_obsolete_source(duplicate_bins, bin2source):
- obsolete = {}
- for key in duplicate_bins.keys():
- (source_a, source_b) = key.split('_')
- for source in [ source_a, source_b ]:
- if not obsolete.has_key(source):
- if not source_binaries.has_key(source):
- # Source has already been removed
- continue
- else:
- obsolete[source] = [ i.strip() for i in source_binaries[source].split(',') ]
- for binary in duplicate_bins[key]:
- if bin2source.has_key(binary) and bin2source[binary]["source"] == source:
- continue
- if binary in obsolete[source]:
- obsolete[source].remove(binary)
-
- to_remove = []
- output = "Obsolete source package\n"
- output += "-----------------------\n\n"
- obsolete_keys = obsolete.keys()
- obsolete_keys.sort()
- for source in obsolete_keys:
- if not obsolete[source]:
- to_remove.append(source)
- output += " * %s (%s)\n" % (source, source_versions[source])
- for binary in [ i.strip() for i in source_binaries[source].split(',') ]:
- if bin2source.has_key(binary):
- output += " o %s (%s) is built by %s.\n" \
- % (binary, bin2source[binary]["version"],
- bin2source[binary]["source"])
- else:
- output += " o %s is not built.\n" % binary
- output += "\n"
-
- if to_remove:
- print output
-
- print "Suggested command:"
- print " dak rm -S -p -m \"[auto-cruft] obsolete source package\" %s" % (" ".join(to_remove))
- print
+def obsolete_source(suite_name, session):
+ """returns obsolete source packages for suite_name sorted by
+ install_date"""
+
+ query = """
+SELECT os.src, os.source, os.version, os.install_date
+ FROM obsolete_source os
+ JOIN suite s on s.id = os.suite
+ WHERE s.suite_name = :suite_name
+ ORDER BY install_date"""
+ args = { 'suite_name': suite_name }
+ return session.execute(query, args)
+
+def source_bin(source, session):
+ """returns binaries built by source for all or no suite"""
+
+ query = """
+SELECT package
+ FROM source_bin
+ WHERE source = :source
+ ORDER BY package"""
+ args = { 'source': source }
+ return session.execute(query, args)
+
+def newest_source_bab(suite_name, package, session):
+ """returns newest source that builds binary package in suite"""
+
+ query = """
+SELECT source, srcver
+ FROM newest_source_bab nsb
+ JOIN suite s on s.id = nsb.suite
+ WHERE s.suite_name = :suite_name AND nsb.package = :package
+ ORDER BY source"""
+ args = { 'suite_name': suite_name, 'package': package }
+ return session.execute(query, args)
+
+def report_obsolete_source(suite_name, session):
+ rows = obsolete_source(suite_name, session)
+ if rows.rowcount == 0:
+ return
+ print \
+"""Obsolete source packages in suite %s
+----------------------------------%s\n""" % \
+ (suite_name, '-' * len(suite_name))
+ for os_row in rows.fetchall():
+ (src, old_source, version, install_date) = os_row
+ print " * obsolete source %s version %s installed at %s" % \
+ (old_source, version, install_date)
+ for sb_row in source_bin(old_source, session):
+ (package, ) = sb_row
+ print " - has built binary %s" % package
+ for nsb_row in newest_source_bab(suite_name, package, session):
+ (new_source, srcver) = nsb_row
+ print " currently built by source %s version %s" % \
+ (new_source, srcver)
+ print " - suggested command:"
+ rm_opts = "-S -p -m \"[auto-cruft] obsolete source package\""
+ print " dak rm -s %s %s %s\n" % (suite_name, rm_opts, old_source)
def get_suite_binaries(suite, session):
# Initalize a large hash table of all binary packages
@@ -358,9 +375,7 @@ def main ():
# Set up checks based on mode
if Options["Mode"] == "daily":
- checks = [ "nbs", "nviu", "nvit" ]
- # 'obsolete source' is broken since the introduction of dak dominate
- #checks = [ "nbs", "nviu", "nvit", "obsolete source" ]
+ checks = [ "nbs", "nviu", "nvit", "obsolete source" ]
elif Options["Mode"] == "full":
checks = [ "nbs", "nviu", "nvit", "obsolete source", "nfu", "dubious nbs", "bnb", "bms", "anais" ]
else:
@@ -385,6 +400,9 @@ def main ():
suite_id = suite.suite_id
suite_name = suite.suite_name.lower()
+ if "obsolete source" in checks:
+ report_obsolete_source(suite_name, session)
+
bin_not_built = {}
if "bnb" in checks:
@@ -503,9 +521,6 @@ def main ():
packages.close()
os.unlink(temp_filename)
- if "obsolete source" in checks:
- do_obsolete_source(duplicate_bins, bin2source)
-
# Distinguish dubious (version numbers match) and 'real' NBS (they don't)
dubious_nbs = {}
real_nbs = {}
diff --git a/dak/dakdb/update27.py b/dak/dakdb/update27.py
new file mode 100755
index 0000000..ee8f584
--- /dev/null
+++ b/dak/dakdb/update27.py
@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+
+"""
+Add views for new obsolete source detection.
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2009 Torsten Werner <twerner@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import psycopg2
+
+def do_update(self):
+ print "Add/modify views for obsolete source detection."
+
+ try:
+ c = self.db.cursor()
+
+ print "Replace old views."
+ # joins src_associations and source
+ c.execute("""
+CREATE OR REPLACE VIEW source_suite AS
+ SELECT src_associations.id, source.id AS src, source.source, source.version,
+ src_associations.suite, suite.suite_name, source.install_date
+ FROM source
+ JOIN src_associations ON source.id = src_associations.source
+ JOIN suite ON suite.id = src_associations.suite;
+ """)
+ # joins bin_associations and binaries
+ c.execute("""
+CREATE OR REPLACE VIEW bin_associations_binaries AS
+ SELECT bin_associations.id, bin_associations.bin, binaries.package,
+ binaries.version, bin_associations.suite, binaries.architecture,
+ binaries.source
+ FROM bin_associations
+ JOIN binaries ON bin_associations.bin = binaries.id;
+ """)
+
+ print "Drop old views."
+ c.execute("DROP VIEW IF EXISTS source_suite_unique CASCADE")
+ c.execute("DROP VIEW IF EXISTS obsolete_source CASCADE")
+ c.execute("DROP VIEW IF EXISTS source_bin CASCADE")
+ c.execute("DROP VIEW IF EXISTS newest_source_bab CASCADE")
+
+ print "Create new views."
+ # returns source package names from suite without duplicates;
+ # rationale: cruft-report and rm cannot handle duplicates (yet)
+ c.execute("""
+CREATE VIEW source_suite_unique
+ AS SELECT source, suite
+ FROM source_suite GROUP BY source, suite HAVING count(*) = 1;
+ """)
+ # returns obsolete sources without binaries in the same suite;
+ # outputs install_date to detect source only (or binary throw away)
+ # uploads; duplicates are skipped
+ c.execute("""
+CREATE VIEW obsolete_source
+ AS SELECT ss.src, ss.source, ss.version, ss.suite,
+ to_char(ss.install_date, 'YYYY-MM-DD') AS install_date
+ FROM source_suite ss
+ JOIN source_suite_unique ssu
+ ON ss.source = ssu.source AND ss.suite = ssu.suite
+ LEFT JOIN bin_associations_binaries bab
+ ON ss.src = bab.source AND ss.suite = bab.suite
+ WHERE bab.id IS NULL;
+ """)
+ # returns source package names and its binaries from any suite
+ c.execute("""
+CREATE VIEW source_bin
+ AS SELECT b.package, MAX(b.version) AS version, sas.source
+ FROM binaries b
+ JOIN src_associations_src sas
+ ON b.source = sas.src
+ GROUP BY b.package, sas.source
+ """)
+ # returns binaries from suite and their source with max(version)
+ # grouped by source name, binary name, and suite
+ c.execute("""
+CREATE VIEW newest_source_bab
+ AS SELECT sas.source, MAX(sas.version) AS srcver, bab.package, bab.suite
+ FROM src_associations_src sas
+ JOIN bin_associations_binaries bab ON sas.src = bab.source
+ GROUP BY sas.source, bab.package, bab.suite;
+ """)
+
+ print "Grant permissions to views."
+ c.execute("GRANT SELECT ON binfiles_suite_component_arch TO PUBLIC;");
+ c.execute("GRANT SELECT ON srcfiles_suite_component TO PUBLIC;");
+ c.execute("GRANT SELECT ON binaries_suite_arch TO PUBLIC;");
+ c.execute("GRANT SELECT ON newest_all_associations TO PUBLIC;");
+ c.execute("GRANT SELECT ON obsolete_any_by_all_associations TO PUBLIC;");
+ c.execute("GRANT SELECT ON newest_any_associations TO PUBLIC;");
+ c.execute("GRANT SELECT ON obsolete_any_associations TO PUBLIC;");
+ c.execute("GRANT SELECT ON source_suite TO PUBLIC;");
+ c.execute("GRANT SELECT ON newest_source TO PUBLIC;");
+ c.execute("GRANT SELECT ON newest_src_association TO PUBLIC;");
+ c.execute("GRANT SELECT ON any_associations_source TO PUBLIC;");
+ c.execute("GRANT SELECT ON src_associations_src TO PUBLIC;");
+ c.execute("GRANT SELECT ON almost_obsolete_src_associations TO PUBLIC;");
+ c.execute("GRANT SELECT ON obsolete_src_associations TO PUBLIC;");
+ c.execute("GRANT SELECT ON bin_associations_binaries TO PUBLIC;");
+ c.execute("GRANT SELECT ON src_associations_bin TO PUBLIC;");
+ c.execute("GRANT SELECT ON almost_obsolete_all_associations TO PUBLIC;");
+ c.execute("GRANT SELECT ON obsolete_all_associations TO PUBLIC;");
+ c.execute("GRANT SELECT ON source_suite_unique TO PUBLIC;");
+ c.execute("GRANT SELECT ON obsolete_source TO PUBLIC;");
+ c.execute("GRANT SELECT ON source_bin TO PUBLIC;");
+ c.execute("GRANT SELECT ON newest_source_bab TO PUBLIC;");
+
+ print "Committing"
+ c.execute("UPDATE config SET value = '27' WHERE name = 'db_revision'")
+ self.db.commit()
+
+ except psycopg2.InternalError, msg:
+ self.db.rollback()
+ raise DBUpdateError, "Database error, rollback issued. Error message : %s" % (str(msg))
+
diff --git a/dak/update_db.py b/dak/update_db.py
index e7b1984..5d1cb86 100755
--- a/dak/update_db.py
+++ b/dak/update_db.py
@@ -45,7 +45,7 @@ from daklib.dak_exceptions import DBUpdateError
################################################################################
Cnf = None
-required_database_schema = 26
+required_database_schema = 27
################################################################################
--
1.6.3.3
Reply to: