Package: qa.debian.org Severity: wishlist Tags: patch X-Debbugs-CC: piuparts-devel@lists.alioth.debian.org thanks Piuparts has a new summary file format, visible at https://piuparts.debian.org/summary.json. The attached patches use that data to integrate piuparts into the DDPO report, to the right of the lintian results. The contents of the column indicate the summary test result for the package; either '-' for pass or unknown, 'F' for fail, 'X' for blocked, or 'W' for waiting. The tooltip describes the test result, lists the applicable distribution(s), and the number of blocked packages resulting from a failure, if any. All entries hyperlink to appropriate pages with further details about the displayed test result. If this sounds like a good idea, I'd appreciate a review of the code, and sponsorship for integration. Thanks in advance. 0001-piuparts-update-Initial-checkin-of-piuparts-cron.patch 0002-extract-piuparts.py-1st-commit-create-piuparts-db.patch 0003-piuparts-update-Add-db-creation-to-piuparts-cron.patch 0004-developer.wml-Add-skeleton-for-piuparts-column.patch 0005-developer.wml-Populate-piuparts-cell-contents.patch -- "Le mieux est l'ennemi du bien" - Voltaire
From bc2c2a4b0b13f8bb09042e43cb5a78a61770a3b9 Mon Sep 17 00:00:00 2001 From: David Steele <dsteele@gmail.com> Date: Fri, 21 Feb 2014 20:15:25 -0500 Subject: [PATCH 1/5] piuparts-update: Initial checkin of piuparts cron Baby steps. Do no more than download the piuparts summary json into the data directory. Do it two hours after the two daily piuparts run. The runs currently take about 80 minutes. --- data/cronjobs/piuparts-update | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100755 data/cronjobs/piuparts-update diff --git a/data/cronjobs/piuparts-update b/data/cronjobs/piuparts-update new file mode 100755 index 0000000..91d561c --- /dev/null +++ b/data/cronjobs/piuparts-update @@ -0,0 +1,7 @@ +#!/bin/sh -e +# CRON=33 2,14 * * * + +umask 002 +wget -q https://piuparts.debian.org/summary.json \ + -O /srv/qa.debian.org/data/piuparts-summary.json + -- 2.0.0.rc0
From 3eca446f2f87ee872e18261a5b37f047b9fe9bb8 Mon Sep 17 00:00:00 2001
From: David Steele <dsteele@gmail.com>
Date: Fri, 21 Feb 2014 23:56:55 -0500
Subject: [PATCH 2/5] extract-piuparts.py: 1st commit-create piuparts db
Parse a JSON piuparts results summary file, and store in a dbm db
file. Results, stored by package name, consist of a single-character
result flag, a package result inspection url, and tooltip text,
separated by spaces.
---
data/piuparts/extract_piuparts.py | 142 ++++++++++++++++++++++++++++++++++++++
1 file changed, 142 insertions(+)
create mode 100755 data/piuparts/extract_piuparts.py
diff --git a/data/piuparts/extract_piuparts.py b/data/piuparts/extract_piuparts.py
new file mode 100755
index 0000000..4987e6f
--- /dev/null
+++ b/data/piuparts/extract_piuparts.py
@@ -0,0 +1,142 @@
+#!/usr/bin/python
+#
+# Copyright (C) 2014 David Steele (dsteele@gmail.com)
+#
+# This file is part of the Debian Developer's Package Overview (DDPO)
+#
+# This is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# This is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+# Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, see <http://www.gnu.org/licenses/>.
+
+# This extracts the information from piuparts for each package
+
+import textwrap
+import argparse
+import os
+import sys
+import json
+import dbm
+from collections import defaultdict
+
+
+DBFILE = 'piuparts-new'
+
+SUMMID = "Piuparts Package Test Results Summary"
+SUMMVER = "1."
+DEFSEC = 'overall'
+
+class SummaryException(Exception):
+ pass
+
+
+dist_sort_order = defaultdict(lambda: 100, [
+ ('oldstable', 10),
+ ('stable', 20),
+ ('testing', 30),
+ ('unstable', 40),
+ ('experimental', 50),
+ ])
+
+
+flag_description = defaultdict(lambda: 'Unknown', [
+ ('P', 'Passed'),
+ ('X', 'Blocked'),
+ ('W', 'Waiting'),
+ ('F', 'Failed'),
+ ('-', 'Unknown'),
+ ])
+
+def tooltip(summary, pkg):
+ """Returns e.g. "Failed in stable and testing, blocking 5 packages"."""
+
+ tip = ''
+ pkgdict = summary['packages']
+
+ if pkg in pkgdict:
+ flag, block_cnt, url = pkgdict[pkg][DEFSEC][0:3]
+
+ if len(flag) != 1 or not isinstance(block_cnt, int) \
+ or not url.startswith('http'):
+ err = "Invalid entry - "
+ err += ', '.join([x.__str__() for x in flag, block_cnt, url])
+ raise SummaryException(err)
+
+ dists = [x for x in pkgdict[pkg] if x != DEFSEC
+ and pkgdict[pkg][x][0] == flag]
+ dists = sorted(dists, key=lambda x: dist_sort_order[x])
+
+ tip = flag_description[flag]
+
+ if len(dists) > 2:
+ tip += ' in ' + ', '.join(dists[:-1]) + ' and ' + dists[-1]
+ elif len(dists) == 2:
+ tip += ' in ' + ' and '.join(dists)
+ elif len(dists) == 1:
+ tip += ' in ' + dists[0]
+
+ if block_cnt:
+ tip += ", blocking %d package" % block_cnt
+ if block_cnt > 1:
+ tip += "s"
+
+ return tip
+
+def summary_read(fname):
+ with open(fname, 'r') as fl:
+ result = json.load(fl)
+
+ if result["_id"] != SUMMID or not result["_version"].startswith(SUMMVER):
+ raise SummaryException('Summary JSON header mismatch')
+
+ return result
+
+class MyParser(argparse.ArgumentParser):
+ def error(self, message):
+ sys.stderr.write('error: %s\n' % message)
+ self.print_help()
+ sys.exit(2)
+
+if __name__ == "__main__":
+
+ parser = MyParser(description="Parse piuparts json to db",
+ epilog=textwrap.fill("""\
+ Parse the "%s" file (version %s), and convert
+ the contents into a Berkley database file,
+ keyed off of the package name, and containing
+ a space-separated list of the extracted
+ parameters.
+ """ % (SUMMID, SUMMVER)))
+
+ parser.add_argument('summfile', metavar='summaryfile',
+ help="Piuparts summary JSON file, as input (e.g. piuparts-summary.json)")
+
+ parser.add_argument('outfile',
+ nargs='?',
+ default=DBFILE,
+ help='The output Berkley DB file (default %s(.db))' % DBFILE,
+ metavar='dbpath',
+ )
+
+ args = parser.parse_args()
+
+ summary = summary_read(args.summfile)
+ db = dbm.open(args.outfile, 'n')
+
+ for pkg in summary['packages']:
+ flag, blocked, url = summary['packages'][pkg][DEFSEC][0:3]
+ tip = tooltip(summary, pkg)
+
+ db[pkg] = " ".join([flag, url, tip])
+
+ db.close()
+
+# vim:et:sw=4:
--
2.0.0.rc0
From 91fe94366facd407186f54fe76b0fade221ff914 Mon Sep 17 00:00:00 2001
From: David Steele <dsteele@gmail.com>
Date: Sat, 22 Feb 2014 00:17:11 -0500
Subject: [PATCH 3/5] piuparts-update: Add db creation to piuparts cron
The database is created in data/piuparts.
---
data/cronjobs/piuparts-update | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/data/cronjobs/piuparts-update b/data/cronjobs/piuparts-update
index 91d561c..fff7901 100755
--- a/data/cronjobs/piuparts-update
+++ b/data/cronjobs/piuparts-update
@@ -5,3 +5,7 @@ umask 002
wget -q https://piuparts.debian.org/summary.json \
-O /srv/qa.debian.org/data/piuparts-summary.json
+cd /srv/qa.debian.org/data/piuparts
+./extract_piuparts.py ../piuparts-summary.json
+mv piuparts-new.db piuparts.db
+
--
2.0.0.rc0
From 9652e0375e1854f12801b72cb46900e8c795b924 Mon Sep 17 00:00:00 2001
From: David Steele <dsteele@gmail.com>
Date: Fri, 21 Feb 2014 19:41:31 -0500
Subject: [PATCH 4/5] developer.wml: Add skeleton for piuparts column
Add a cell to the right of Lintian in the tables, with url/cookie
visibility control support. Cell contents are currently always a dash.
---
wml/developer.wml | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/wml/developer.wml b/wml/developer.wml
index aa4b541..582500c 100644
--- a/wml/developer.wml
+++ b/wml/developer.wml
@@ -87,6 +87,7 @@ $arg_cookie_def = array ( comaint => "yes",
buildd => 1,
problems => 1,
lintian => 1,
+ piuparts => 1,
ci => 1,
popc => 1,
section => 0,
@@ -333,6 +334,7 @@ function print_help() {
settings_entry("Buildd","&buildd=", "buildd") .
settings_entry("Debcheck","&problems=", "problems") .
settings_entry("Lintian","&lintian=", "lintian") .
+ settings_entry("Piuparts","&piuparts=", "piuparts") .
settings_entry("CI","&ci=", "ci") .
settings_entry("Popcon","&popc=", "popc") .
settings_entry("Watch","&watch=", "watch") .
@@ -1061,6 +1063,11 @@ function print_package_entry($package, $uploader, $back_tr_color, $skip_seen)
$line .= html_td(html_a("-", $lintianurl, "dash"));
}
+ if(isdisplayed('piuparts'))
+ {
+ $line .= html_td('-');
+ }
+
if(isdisplayed('ci'))
{
static $ci_db;
@@ -1177,6 +1184,7 @@ function print_package_entries($branch, $packages, $skip_seen)
if(isdisplayed('buildd')) $ths .= html_th("Buildd", "", 2, 1);
if(isdisplayed('problems')) $ths .= html_th("Deb<br />check", "", 2, 1);
if(isdisplayed('lintian')) $ths .= html_th("Lintian", "", 1, 1);
+ if(isdisplayed('piuparts')) $ths .= html_th("Piuparts", "", 2, 1);
if(isdisplayed('ci')) $ths .= html_th(html_a("CI", "http://ci.debian.net";), "", 2, 1);
if(isdisplayed('popc')) $ths .= html_th("Popcon", "", 2, 1);
if(isdisplayed('watch')) $ths .= html_th("Watch", "", 2, 1);
--
2.0.0.rc0
From 8cf818d37e8e8100fcfa04b42d79e889c48a980e Mon Sep 17 00:00:00 2001
From: David Steele <dsteele@gmail.com>
Date: Sat, 22 Feb 2014 18:25:53 -0500
Subject: [PATCH 5/5] developer.wml: Populate piuparts cell contents
Cell contents are '-' for unknown or Pass, 'F' for Fail, 'X' for
Blocked, and 'W' for Waiting. All but unknown link to a piuparts web
page with information on the status. The tooltip text details the
distributions for which the state is valid, and the number of
packages blocked, for the Fail case.
---
wml/developer.wml | 21 ++++++++++++++++++++-
1 file changed, 20 insertions(+), 1 deletion(-)
diff --git a/wml/developer.wml b/wml/developer.wml
index 582500c..a6f3205 100644
--- a/wml/developer.wml
+++ b/wml/developer.wml
@@ -1065,7 +1065,26 @@ function print_package_entry($package, $uploader, $back_tr_color, $skip_seen)
if(isdisplayed('piuparts'))
{
- $line .= html_td('-');
+ static $piuparts_db;
+
+ if(!$piuparts_db)
+ $piuparts_db = dba_open("/srv/qa.debian.org/data/piuparts/piuparts.db", "r", "db4");
+
+ $piuparts_data = dba_fetch($package, $piuparts_db);
+
+ if($piuparts_data)
+ {
+ $parts = explode(" ", $piuparts_data);
+ $pflag = array_shift($parts);
+ if($pflag === "P") $pflag = "-";
+ $purl = array_shift($parts);
+ $ptip = implode(" ", $parts);
+ $ptip = wordwrap($ptip, 30, " \n");
+
+ $line .= html_td(html_a($pflag, $purl, "dash", $ptip));
+ }
+ else
+ $line .= html_td("-");
}
if(isdisplayed('ci'))
--
2.0.0.rc0
Attachment:
signature.asc
Description: OpenPGP digital signature