tags 632167 + patch thanks Hi again, On Thu, Jun 30, 2011 at 09:13:07AM +0100, Iain Lane wrote:
Package: qa.debian.org Severity: wishlist Hi there, Thanks for adding ubuntu_upload_history. :-) However, it misses some fields. Ubuntu's changes mails contain a few pieces of information that Debian's do not. - Signed-By: same format as Changed-By, but is used to indicate the signer of the upload when it is different from the changer. - Launchpad-Bugs-Fixed: Same as Closes, but for Launchpad rather than the BTS. - Original-Maintainer: Same format as Maintainer, but records the Debian maintainer in the case when Ubuntu has pointed Maintainer to the Ubuntu development team. Also, some fields do not really make sense in ubuntu_upload_history. - NMU: There's no concept of NMUing in Ubuntu - Architecture: All uploads are 'source'. - key_id/fingerprint: These aren't provided, so the columns should be dropped.
Well I got bored just now and implemented this. To keep it all in one file there's now quite a few conditional statements in upload_history_gatherer.py. I don't think it's too bad. You'll want to drop the tables and do a full reimport. I just tried that and it WFM: udd=# SELECT u.source, u.version, u.changed_by, count(lp.bug) as num_bugs_closed FROM ubuntu_upload_history u, ubuntu_upload_history_launchpad_closes lp WHERE u.source = lp.source AND u.version = lp.version GROUP BY u.source, u.version, u.changed_by ORDER BY num_bugs_closed DESC LIMIT 10;source | version | changed_by | num_bugs_closed ------------+------------------------------------+-----------------------------------------+-----------------
linux | 2.6.28-9.30 | Tim Gardner <tim.gardner@canonical.com> | 106 phatch | 0.2.1-4 | stani <spe.stani.be@gmail.com> | 83 openjdk-7 | 7~b136-1.14~pre0-1 | Matthias Klose <doko@ubuntu.com> | 76 pulseaudio | 1:0.9.22~0.9.21+341-g62bf-0ubuntu1 | Daniel T Chen <crimsun@ubuntu.com> | 76 xorg | 1:7.5+1ubuntu1 | Timo Aaltonen <tjaalton@ubuntu.com> | 73 vlc | 1.1.0-1ubuntu1 | Benjamin Drung <bdrung@ubuntu.com> | 70 unity | 3.8.4-0ubuntu1 | Didier Roche <didrocks@ubuntu.com> | 67 unity | 3.8.2-0ubuntu1 | Didier Roche <didrocks@ubuntu.com> | 65 unity-2d | 3.8-0ubuntu1 | Oliver Grawert <ogra@ubuntu.com> | 63 unity | 3.6.6-0ubuntu1 | Didier Roche <didrocks@ubuntu.com> | 54 (10 rows) Patch is attached. Please have a look and commit if you think it's alright. :-) (I didn't test the Debian upload importer, but that should hopefully be untouched) Cheers, Iain
Index: sql/setup.sql
===================================================================
--- sql/setup.sql (revision 1961)
+++ sql/setup.sql (working copy)
@@ -481,33 +481,23 @@
CREATE TABLE ubuntu_upload_history
(source text, version debversion, date timestamp with time zone,
- changed_by text, changed_by_name text, changed_by_email text, maintainer text, maintainer_name text, maintainer_email text, nmu boolean, signed_by text, signed_by_name text, signed_by_email text, key_id text, distribution text, file text,
- fingerprint text,
+ changed_by text, changed_by_name text, changed_by_email text, maintainer text, maintainer_name text, maintainer_email text, nmu boolean, signed_by text, signed_by_name text, signed_by_email text, key_id text, distribution text, file text,
+ fingerprint text, original_maintainer text, original_maintainer_name text, original_maintainer_email text,
PRIMARY KEY (source, version));
-
-CREATE TABLE ubuntu_upload_history_architecture
- (source text, version debversion, architecture text, file text,
- PRIMARY KEY (source, version, architecture),
+CREATE TABLE ubuntu_upload_history_launchpad_closes
+ (source text, version debversion, bug int, file text,
+ PRIMARY KEY (source, version, bug),
FOREIGN KEY (source, version) REFERENCES ubuntu_upload_history DEFERRABLE);
-
+
CREATE TABLE ubuntu_upload_history_closes
(source text, version debversion, bug int, file text,
PRIMARY KEY (source, version, bug),
FOREIGN KEY (source, version) REFERENCES ubuntu_upload_history DEFERRABLE);
GRANT SELECT ON ubuntu_upload_history TO PUBLIC;
-GRANT SELECT ON ubuntu_upload_history_architecture TO PUBLIC;
GRANT SELECT ON ubuntu_upload_history_closes TO PUBLIC;
+GRANT SELECT ON ubuntu_upload_history_launchpad_closes TO PUBLIC;
-CREATE VIEW ubuntu_upload_history_nmus AS
-select uh1.source, count(*) AS nmus
-from ubuntu_upload_history uh1, (select source, max(date) as date from upload_history where nmu = false group by source) uh2
-where uh1.nmu = true
-and uh1.source = uh2.source
-and uh1.date > uh2.date
-group by uh1.source;
-GRANT SELECT ON ubuntu_upload_history_nmus TO PUBLIC;
-
-- Ubuntu bugs
CREATE TABLE ubuntu_bugs (
bug int,
Index: udd/upload_history_gatherer.py
===================================================================
--- udd/upload_history_gatherer.py (revision 1961)
+++ udd/upload_history_gatherer.py (working copy)
@@ -14,18 +14,14 @@
class upload_history_gatherer(gatherer):
def __init__(self, connection, config, source):
+ self.is_ubuntu = source == 'ubuntu-upload-history'
+ self.is_debian = not self.is_ubuntu
gatherer.__init__(self, connection, config, source)
if not 'path' in self.my_config:
raise aux.ConfigException('path not specified for source ' + source)
- def tables(self):
- return [
- self.my_config['table'] + '_architecture',
- self.my_config['table'] + '_closes',
- self.my_config['table']]
+ def run(self):
-
- def run(self):
path = self.my_config['path']
if 'only-recent' in self.my_config:
onlyrecent = self.my_config['only-recent']
@@ -34,12 +30,26 @@
cursor = self.cursor()
- cursor.execute("PREPARE uh_insert AS INSERT INTO %s (source, \
- version, date, changed_by, changed_by_name, changed_by_email, maintainer, maintainer_name, maintainer_email, nmu, signed_by, signed_by_name, signed_by_email, key_id, fingerprint, distribution, file) VALUES \
- ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)" % self.my_config['table'])
- cursor.execute("PREPARE uh_arch_insert AS INSERT INTO %s (source, \
+ tables = ['source', 'version', 'date', 'changed_by', 'changed_by_name', 'changed_by_email', 'maintainer', 'maintainer_name', 'maintainer_email', 'nmu', 'signed_by', 'signed_by_name', 'signed_by_email', 'key_id', 'fingerprint', 'distribution', 'file']
+
+ if self.is_ubuntu:
+ tables = tables + ['original_maintainer', 'original_maintainer_name', 'original_maintainer_email']
+
+ indices = ', '.join(map(lambda x: '$' + str(x), range(1,len(tables)+1)))
+ tables = ', '.join(tables)
+
+ cursor.execute("PREPARE uh_insert AS INSERT INTO %s (%s) VALUES \
+ (%s)" % (self.my_config['table'], tables, indices))
+
+ if self.is_debian:
+ cursor.execute("PREPARE uh_arch_insert AS INSERT INTO %s (source, \
version, architecture, file) VALUES \
($1, $2, $3, $4)" % (self.my_config['table'] + '_architecture'))
+
+ if self.is_ubuntu:
+ cursor.execute("PREPARE uh_launchpad_close_insert AS INSERT INTO %s (source, version, bug, file) \
+ VALUES ($1, $2, $3, $4)" % (self.my_config['table'] + '_launchpad_closes'))
+
cursor.execute("PREPARE uh_close_insert AS INSERT INTO %s (source, version, bug, file) \
VALUES ($1, $2, $3, $4)" % (self.my_config['table'] + '_closes'))
@@ -48,8 +58,18 @@
%(Maintainer)s, %(Maintainer_name)s, %(Maintainer_email)s, %(NMU)s, \
%(Signed-By)s, %(Signed-By_name)s, %(Signed-By_email)s, %(Key)s, \
%(Fingerprint)s, %(Distribution)s, %(File)s)"
- query_archs = "EXECUTE uh_arch_insert(%(Source)s, %(Version)s, %(arch)s, %(File)s)"
+
+ if self.is_ubuntu:
+ query = query[:-1] + ", %(Original-Maintainer)s, %(Original-Maintainer_name)s, %(Original-Maintainer_email)s)"
+
+ if self.is_debian:
+ query_archs = "EXECUTE uh_arch_insert(%(Source)s, %(Version)s, %(arch)s, %(File)s)"
+
query_closes = "EXECUTE uh_close_insert(%(Source)s, %(Version)s, %(closes)s, %(File)s)"
+
+ if self.is_ubuntu:
+ query_launchpad_closes = "EXECUTE uh_launchpad_close_insert(%(Source)s, %(Version)s, %(closes)s, %(File)s)"
+
added = {}
files = glob(path + '/*-changes*mbox*')
files.sort()
@@ -58,12 +78,18 @@
print files
else:
print "Doing full import!"
- cursor.execute("delete from " + self.my_config['table'] + "_architecture")
+ if self.is_debian:
+ cursor.execute("delete from " + self.my_config['table'] + "_architecture")
+ if self.is_ubuntu:
+ cursor.execute("delete from " + self.my_config['table'] + "_launchpad_closes")
cursor.execute("delete from " + self.my_config['table'] + "_closes")
cursor.execute("delete from " + self.my_config['table'])
for name in files:
bname = os.path.basename(name).replace(".gz","").replace(".out","")
- cursor.execute("DELETE FROM " + self.my_config['table'] + "_architecture where file='%s'" % (bname))
+ if self.is_debian:
+ cursor.execute("DELETE FROM " + self.my_config['table'] + "_architecture where file='%s'" % (bname))
+ if self.is_ubuntu:
+ cursor.execute("DELETE FROM " + self.my_config['table'] + "_launchpad_closes where file='%s'" % (bname))
cursor.execute("DELETE FROM " + self.my_config['table'] + "_closes where file='%s'" % (bname))
cursor.execute("DELETE FROM " + self.my_config['table'] + " where file='%s'" % (bname))
@@ -82,6 +108,7 @@
uploads = []
uploads_archs = []
uploads_closes = []
+ uploads_launchpad_closes = []
for line in f:
line_count += 1
@@ -95,6 +122,13 @@
else:
current['Signed-By_name'] = current['Signed-By']
current['Signed-By_email'] = ''
+
+ if current.has_key('Original-Maintainer'):
+ if current['Original-Maintainer'] != 'N/A':
+ current['Original-Maintainer_name'], current['Original-Maintainer_email'] = aux.parse_email(current['Original-Maintainer'])
+ else:
+ current['Original-Maintainer_name'] = current['Original-Maintainer_email'] = 'N/A'
+
current['Message-Date'] = current['Message-Date'].partition('(')[0].replace('+4200','+0000').replace('+4300','+0000').replace('+4100','+0000').replace('+4400','+0000').replace('+4000','+0000')
if (current['Source'], current['Version']) in added or \
(current['Source'], current['Version']) == ('libapache-authznetldap-perl', '0.07-4') or \
@@ -104,27 +138,33 @@
print "Skipping upload: "+current['Source']+" "+current['Version']+" "+current['Date']
current = {}
current['Fingerprint'] = 'N/A' # hack: some entries don't have fp
- current['NMU'] = False
- current['Key'] = ''
- current['File'] = bname
+ current['NMU'] = False
+ current['Key'] = ''
+ current['File'] = bname
last_field = None
continue
added[(current['Source'], current['Version'])] = True
uploads.append(current)
- for arch in set(current['Architecture'].split()):
- current_arch = {'Source': current['Source'], 'Version': current['Version'], 'File': bname}
- current_arch['arch'] = arch
- uploads_archs.append(current_arch)
+ if self.is_debian:
+ for arch in set(current['Architecture'].split()):
+ current_arch = {'Source': current['Source'], 'Version': current['Version'], 'File': bname}
+ current_arch['arch'] = arch
+ uploads_archs.append(current_arch)
if current['Closes'] != 'N/A':
for closes in set(current['Closes'].split()):
current_closes = {'Source': current['Source'], 'Version': current['Version'], 'File': bname}
current_closes['closes'] = closes
uploads_closes.append(current_closes)
+ if current.has_key('Launchpad-Bugs-Fixed') and current['Launchpad-Bugs-Fixed'] != 'N/A':
+ for closes in set(current['Launchpad-Bugs-Fixed'].split()):
+ current_closes = {'Source': current['Source'], 'Version': current['Version'], 'File': bname}
+ current_closes['closes'] = closes
+ uploads_launchpad_closes.append(current_closes)
current = {}
current['Fingerprint'] = 'N/A' # hack: some entries don't have fp
- current['NMU'] = False
- current['Key'] = ''
- current['File'] = bname
+ current['NMU'] = False
+ current['Key'] = ''
+ current['File'] = bname
last_field = None
continue
@@ -145,10 +185,17 @@
# print u
# cursor.execute(query, u)
cursor.executemany(query, uploads)
- cursor.executemany(query_archs, uploads_archs)
+ if self.is_debian:
+ cursor.executemany(query_archs, uploads_archs)
+ if self.is_ubuntu:
+ cursor.executemany(query_launchpad_closes, uploads_launchpad_closes)
cursor.executemany(query_closes, uploads_closes)
cursor.execute("DEALLOCATE uh_insert")
- cursor.execute("ANALYZE " + self.my_config['table'] + '_architecture')
+ if self.is_debian:
+ cursor.execute("ANALYZE " + self.my_config['table'] + '_architecture')
+ if self.is_ubuntu:
+ cursor.execute("ANALYZE " + self.my_config['table'] + '_launchpad_closes')
+
cursor.execute("ANALYZE " + self.my_config['table'] + '_closes')
cursor.execute("ANALYZE " + self.my_config['table'])
Attachment:
signature.asc
Description: Digital signature