Group up orphaned packages in order to find adopter faster

["Sandro Tosi" <morph@debian.org>]

Hi all,
I've being playing with python-apt to get used to it[1], and I come
out with the script you can find attached.

What it does is

- take orphaned.txt from qa.d.o
- merge with package already maintained by QA group (using
grep-available.. anything better, that doesn't need "dselect update",
machine indipendent (so no Source file location) and maybe in
python-apt or debian-python?)
- trying to partition this list in different subset, to facilitate the
adoption by people/team intested in a specific category of packages

An example of its output is in attach, run minutes before sending this
email, just to give you an idea of waht the script does.

This is the result of the happy interaction I had once with perl-group
(in particular gregoa) in adopting orphaned perl modules.

So, do you think it's something worth keeping at QA group level? maybe
preparing a webpage or so?

Cheers,
Sandro

[1] but I still HATE it from the bottom of my heart for the lack of a
doc one can use to develop something

-- 
Sandro Tosi (aka morph, Morpheus, matrixhasu)
My website: http://matrixhasu.altervista.org/
Me at Debian: http://wiki.debian.org/SandroTosi

#!/usr/bin/python
# Author: Sandro Tosi <morph@debian.org>
# Date: 2008-10-26
# License: public domain
# 
# This script merges 2 lists of orphaned packages:
#
#  - the ones from http://qa.debian.org/data/orphaned.txt
#    (or from cli, if the file is already on the disk)
#  - the lists of package maintained by Debian QA Group
#
# then it tries to partition this set into many subsets
# in order to facilate the possible adoption by teams or
# people interested in particular packages areas


import sys
import urllib
import apt_pkg
import pprint
import subprocess

def append_to_dict(dict, key, value):
    """Appends a value to a dict entry (if it exists) or create the entry (if not)"""
    if key in dict:
        dict[key] = dict[key] + "; " + value
    else:
        dict[key] = value

orph_not_qa = []

if len(sys.argv) == 1:
    print "I: Reading orphaned.txt packages from qa.debian.org"
    orph = urllib.urlopen('http://qa.debian.org/data/orphaned.txt')
else:
    print "I: Reading orphaned.txt from first cli input parameter"
    orph = open(sys.argv[1])

# Parse orphaned.txt to obtain a list of packages
num = int(orph.readline().strip())
while orph.readline():
    p = {}
    p["package"] = orph.readline().strip()
    if not p["package"]:
        break
    p["maintainer"] = orph.readline().strip()
    p["bugid"] = int(orph.readline().strip())
    p["subject"] = orph.readline().strip()
    p["submitter"] = orph.readline().strip()
    p["age"] = int(orph.readline().strip())
    orph_not_qa.append(p["package"])

orph.close()

# Obtain the packages already maintained by QA Group
# so that have received a QA upload
print "W: please execute dselect update before running this script"
proc = subprocess.Popen("grep-available -F Maintainer 'QA' -s Source:Package | sort -u | awk '{ print $2 }'",
                       shell=True,
                       stdout=subprocess.PIPE,
                       )
orph_qa_out = proc.communicate()[0]
# split the stdout in lines
orph_qa = orph_qa_out.split('\n')

# init apt structures
apt_pkg.init()
srcrecords = apt_pkg.GetPkgSrcRecords()
cache = apt_pkg.GetCache()
depcache = apt_pkg.GetDepCache(cache)

# init subsets (represented as dict)
maybe_python = {}
maybe_perl = {}
maybe_game = {}
maybe_lib = {}
maybe_ruby = {}
maybe_java = {}
maybe_sci = {}

# debugging
#import pdb; pdb.set_trace()

for pkg in orph_not_qa + orph_qa:
    try:
        # setting the current package
        srcrecords.Lookup(pkg)


        # trying to identify category from source package section
        if srcrecords.Section == 'python':
            append_to_dict(maybe_python, pkg, 'section = python')
        if srcrecords.Section == 'perl':
            append_to_dict(maybe_perl, pkg, 'section = perl')
        if srcrecords.Section == 'games':
            append_to_dict(maybe_game, pkg, 'section = games')
        if srcrecords.Section in ('libs', 'oldlibs', 'libdevel'):
            append_to_dict(maybe_lib, pkg, 'section = '+ srcrecords.Section)
        if srcrecords.Section in ('science', 'electronics', 'math'):
            append_to_dict(maybe_sci, pkg, 'section = '+ srcrecords.Section)

        # trying to identify category from source package build-depends
        for builddep in srcrecords.BuildDepends:
            if builddep[0] in ['python','python-all','python-dbg','python-dev']:
                append_to_dict(maybe_python, pkg, 'build-depends on '+builddep[0])
            if builddep[0] in ['perl']:
                append_to_dict(maybe_perl, pkg, 'build-depends on '+builddep[0])
            if builddep[0] in ['ruby']:
                append_to_dict(maybe_ruby, pkg, 'build-depends on '+builddep[0])
            if builddep[0] in ['sun-java5-jdk','sun-java6-jdk','java-gcj-compat','default-jdk']:
                append_to_dict(maybe_java, pkg, 'build-depends on '+builddep[0])

        # trying to identify category from binary packages information
        #  -> TODO <-
        #for bin in srcrecords.Binaries:
        #    binpkg = depcache.GetCandidateVer(cache[bin])

        srcrecords.Restart()
    except:
        print "W: Problem examining", pkg

print "maybe_python: ",
pprint.pprint(maybe_python)
print
print "maybe_perl: ",
pprint.pprint(maybe_perl)
print
print "maybe_game: ",
pprint.pprint(maybe_game)
print
print "maybe_lib: ",
pprint.pprint(maybe_lib)
print
print "maybe_ruby: ",
pprint.pprint(maybe_ruby)
print
print "maybe_java: ",
pprint.pprint(maybe_java)
print
print "maybe_sci: ",
pprint.pprint(maybe_sci)

################################################################
# TODO: add the package already under QA team, from python-apt or debian-python?
#
# maybe python
# -> depends on python
#
# maybe perl
# -> depends on perl
#
# maybe ruby
# -> depends on ruby
#
# maybe java
# -> depends on java|j2sdk|openjdk|whatever?
#
# maybe game
# -> short or long description containing game
#
# maybe scientific
# -> short or long description containing "scien" (ce,tific) or so
#
# maybe library
# -> name lib* (ma non lib-*perl|ruby?) 
# -> section devel
#
# other maybe: sound/mail/net/graphics/web

I: Reading orphaned.txt packages from qa.debian.org
W: please execute dselect update before running this script

Reading package lists... 0%

Reading package lists... 100%

Reading package lists... Done

Building dependency tree... 0%

Building dependency tree... 0%

Building dependency tree... 1%

Building dependency tree... 50%

Building dependency tree... 50%

Building dependency tree... 77%

Building dependency tree       

Reading state information... 0%

Reading state information... 0%

Reading state information... Done
W: Problem examining 
maybe_python: {'adesklets': 'build-depends on python-dev',
 'capisuite': 'build-depends on python-dev',
 'configobj': 'section = python',
 'dict-jargon': 'build-depends on python',
 'ftplib': 'build-depends on python',
 'galternatives': 'build-depends on python-dev',
 'gnochm': 'build-depends on python',
 'grokking-the-gimp': 'build-depends on python',
 'guml': 'build-depends on python-dev',
 'hk-classes': 'build-depends on python-dev',
 'imgseek': 'build-depends on python-dev',
 'knoda': 'build-depends on python-dev',
 'mma': 'build-depends on python-dev',
 'pydance': 'build-depends on python-dev',
 'pypanel': 'build-depends on python-dev',
 'python-happydoc': 'section = python; build-depends on python-dev',
 'python-oss': 'section = python',
 'pyzor': 'build-depends on python-dev',
 'qtorrent': 'build-depends on python',
 'subterfugue': 'build-depends on python-dev',
 'synopsis': 'build-depends on python-dev; build-depends on python',
 'webut': 'section = python; build-depends on python-dev'}

maybe_perl: {'a2ps-perl-ja': 'section = perl',
 'abicheck': 'build-depends on perl',
 'aub': 'build-depends on perl',
 'faqomatic': 'build-depends on perl',
 'flowscan': 'build-depends on perl',
 'g2': 'build-depends on perl',
 'gnuift': 'build-depends on perl',
 'grepmail': 'build-depends on perl',
 'html-munger': 'build-depends on perl',
 'libdata-flow-perl': 'section = perl',
 'libdata-showtable-perl': 'section = perl; build-depends on perl',
 'libdb-file-lock-perl': 'section = perl; build-depends on perl',
 'libdbix-password-perl': 'section = perl; build-depends on perl',
 'libdevel-corestack-perl': 'section = perl; build-depends on perl',
 'libmail-srs-perl': 'section = perl; build-depends on perl',
 'libpalm-perl': 'section = perl; build-depends on perl',
 'libschedule-cron-perl': 'section = perl; build-depends on perl',
 'libx500-dn-perl': 'section = perl; build-depends on perl',
 'libxml-sablot-perl': 'section = perl; build-depends on perl',
 'log-analysis': 'build-depends on perl',
 'note': 'build-depends on perl',
 'perlsgml': 'build-depends on perl',
 'psh': 'build-depends on perl',
 'quiteinsane': 'build-depends on perl',
 'quiteinsanegimpplugin': 'build-depends on perl',
 'sufary': 'build-depends on perl; build-depends on perl',
 'webmagick': 'build-depends on perl'}

maybe_game: {'cookietool': 'section = games',
 'lsnipes': 'section = games',
 'matanza': 'section = games',
 'nighthawk': 'section = games',
 'pouetchess': 'section = games',
 'pydance': 'section = games',
 'pydance-music': 'section = games',
 'sillypoker': 'section = games',
 'sjeng': 'section = games',
 'sopwith': 'section = games',
 'tama': 'section = games',
 'tint': 'section = games',
 'xbl': 'section = games',
 'xemeraldia': 'section = games',
 'xjewel': 'section = games',
 'xjokes': 'section = games',
 'xlaby': 'section = games',
 'xoids': 'section = games',
 'xonix': 'section = games',
 'xstarfish': 'section = games',
 'xtris': 'section = games'}

maybe_lib: {'abicheck': 'section = libs',
 'adplug': 'section = libs',
 'agsync': 'section = libs',
 'bogl': 'section = libs',
 'dlisp': 'section = libs',
 'freetype1': 'section = libs',
 'ftplib': 'section = libs',
 'g2': 'section = libs',
 'gnustep-dl2': 'section = libs',
 'gtkglarea': 'section = libs',
 'hermes1': 'section = libs',
 'hk-classes': 'section = libs',
 'imlib': 'section = oldlibs',
 'lib3ds': 'section = libs',
 'libaal': 'section = libs',
 'libast': 'section = libs',
 'libcgi': 'section = libs',
 'libdc0': 'section = libs',
 'libdockapp': 'section = libs',
 'libdumbnet': 'section = libs',
 'libghttp': 'section = libs',
 'libglpng': 'section = libs',
 'libjconv': 'section = libs',
 'libmimedir': 'section = libs',
 'libnet0': 'section = oldlibs',
 'libpcd': 'section = libs',
 'libtlen': 'section = libs',
 'libvigraimpex': 'section = libs',
 'mffm-fftw': 'section = libs',
 'mffm-gtkclasses': 'section = libs',
 'mffm-libsndfilew': 'section = libs',
 'mffm-timecode': 'section = libs',
 'phat': 'section = libs',
 'synce-multisync-plugin': 'section = libs',
 'tix': 'section = libs',
 'wvstreams': 'section = libs',
 'xbsql': 'section = libs',
 'xclass': 'section = libs',
 'xplc': 'section = libs'}

maybe_ruby: {'cvsdelta': 'build-depends on ruby',
 'kwartz': 'build-depends on ruby',
 'sqlrelay': 'build-depends on ruby'}

maybe_java: {'ikvm': 'build-depends on java-gcj-compat'}

maybe_sci: {'ava': 'section = electronics',
 'avrp': 'section = electronics',
 'cassbeam': 'section = science',
 'dome': 'section = math',
 'eukleides': 'section = math',
 'fv': 'section = science',
 'gnuplot-mode': 'section = math',
 'harminv': 'section = science',
 'ksimus': 'section = electronics',
 'ksimus-boolean': 'section = electronics',
 'ksimus-datarecorder': 'section = electronics',
 'ksimus-floatingpoint': 'section = electronics',
 'mpb': 'section = science',
 'oleo': 'section = math',
 'plotmtv': 'section = math',
 'saods9': 'section = science',
 'skinedit': 'section = math',
 'ssystem': 'section = science',
 'stepulator.app': 'section = math',
 'tiemu': 'section = math',
 'tilp': 'section = math',
 'tkgate': 'section = electronics',
 'tochnog': 'section = science',
 'transcalc': 'section = science',
 'vbpp': 'section = electronics'}