Re: Test suite in github but missing from pypi tarballs

[Edward Betts <edward@4angle.com>]

Here is a list of packages that include a test suite on github but not in
the DPMT git.

csvkit                 https://github.com/onyxfish/csvkit
django-haystack        https://github.com/toastdriven/django-haystack
django-jinja           https://github.com/niwibe/django-jinja
django-recurrence      https://github.com/django-recurrence/django-recurrence
django-webpack-loader  https://github.com/owais/django-webpack-loader
djoser                 https://github.com/gizmag/django-fsm-log
dockerpty              https://github.com/d11wtq/dockerpty
drf-generators         https://github.com/Brobin/drf-generators
mod-wsgi               https://github.com/GrahamDumpleton/mod_wsgi
pdfrw                  https://github.com/pmaupin/pdfrw
pep8-naming            https://github.com/flintwork/pep8-naming
pkgconfig              https://github.com/matze/pkgconfig
pydot                  https://github.com/erocarrera/pydot
pylast                 http://github.com/pylast/pylast
pysrt                  https://github.com/byroot/pysrt
python-args            https://github.com/kennethreitz/args
python-astor           https://github.com/berkerpeksag/astor
python-cachecontrol    https://github.com/ionrock/cachecontrol
python-easywebdav      https://github.com/amnong/easywebdav
python-exif            https://github.com/ianare/exif-py
python-gnutls          https://github.com/AGProjects/python-gnutls
python-hpilo           https://github.com/seveas/python-hpilo
python-humanize        https://github.com/jmoiron/humanize
python-jsonpify        https://github.com/wcdolphin/flask-jsonpify
python-model-mommy     http://github.com/vandersonmota/model_mommy
python-mrjob           http://github.com/Yelp/mrjob
python-pgspecial       https://github.com/dbcli/pgspecial
python-pip             https://github.com/pypa/pip
python-pretend         https://github.com/alex/pretend
python-rply            https://github.com/alex/rply
python-scp             https://github.com/jbardin/scp.py
python-sentinels       https://github.com/vmalloc/sentinels
python-sunlight        https://github.com/sunlightlabs/python-sunlight
python-trezor          https://github.com/trezor/python-trezor
python-zxcvbn          https://github.com/dropbox/python-zxcvbn
responses              https://github.com/getsentry/responses
subliminal             https://github.com/Diaoul/subliminal
txws                   https://github.com/MostAwesomeDude/txWS
vcversioner            https://github.com/habnabit/vcversioner
whichcraft             https://github.com/pydanny/whichcraft

This is the code I ran on git.debian.org to generate the list.

#!/usr/bin/python3

from subprocess import Popen, PIPE
from urllib.request import urlopen
from urllib.error import HTTPError
import os
import re
    
re_github_url = re.compile('(https?://[^/ ]*github.com/[^/]+/[^/ ]+)')

location = '/git/python-modules/packages/'

def repo_includes_tests(repo):
    cmd = ['git', '--git-dir=' + location + repo, 'ls-tree', '-r', 'HEAD', '--name-only']
    with Popen(cmd, stdout=PIPE, stderr=PIPE) as proc:
        for line in proc.stdout:
            filename = line.decode('utf-8')[:-1]
            if not filename.endswith('.py'):
                continue
            if filename.startswith('test') or '/test' in filename:
                return True
    return False

def show_debian_file(repo, filename):
    cmd = ['git', '--git-dir=' + location + repo, 'show', 'HEAD:debian/' + filename]
    return Popen(cmd, stdout=PIPE, stderr=PIPE).stdout.read().decode('utf-8')

def github_urls(content):
    github_urls = set()
    for line in content.splitlines():
        if 'github.com' in line:
            m = re_github_url.search(line)
            if not m:
                print(line)
            github_urls.add(m.group(1))
    return github_urls

for repo in sorted(os.listdir(location)):
    if not repo.endswith('.git') or repo_includes_tests(repo):
        continue

    urls = (github_urls(show_debian_file(repo, 'copyright'))
            or github_urls(show_debian_file(repo, 'watch')))

    if len(urls) != 1:
        continue
    (url,) = urls
    try:
        page = urlopen(url).read()
    except HTTPError:
        continue
    if ' title="test' not in page.decode('utf-8'):
        continue  # no tests in github
    print('{:23s} {}'.format(repo[:-4], url))