[PATCH] DEP-11 support example for dak
Hi there!
In order to complete the DEP-11 / AppStream support, the metadata
which is generated on mekeel.debian.org needs to be added to the
Debian archive, so APT can download it.
The attached patch downloads the data and verifies it, so we don't get
garbage added to the archive.
I haven't tested it completely, since it requires the DSA to set up an
ftp-masters account on mekeel.d.o, but I would like some input from
you on whether there's anything left for me to do which I can help
with, or if everything is ready now and you can take it from here (=
add the code, so we can add the missing bits to APT and have awesome
AppStream support in Debian).
A bit more information about AppStream on Debian can be found at
https://wiki.debian.org/AppStream , I am extending this page from time
to time.
Patch is attached.
Cheers & thanks for your hard work!
Matthias Klumpp
--
I welcome VSRE emails. See http://vsre.info/
commit b1e542912c9b630f01c4419db1b83d519b23804b
Author: Matthias Klumpp <matthias@tenstral.net>
Date: Tue Nov 24 20:46:18 2015 +0100
Pull AppStream metadata from the generator server
This patch pulls AppStream/DEP-11 metadata from the server where it was
generated, then validates it for format issues and puts it into the
archive.
diff --git a/config/debian/cron.dinstall b/config/debian/cron.dinstall
index 701754c..8f1f0a7 100755
--- a/config/debian/cron.dinstall
+++ b/config/debian/cron.dinstall
@@ -206,6 +206,14 @@ GO=(
)
stage $GO
+GO=(
+ FUNC="dep11"
+ TIME="dep11 1"
+ ARGS=""
+ ERR="false"
+)
+stage $GO
+
lockfile "$LOCK_ACCEPTED"
trap remove_all_locks EXIT TERM HUP INT QUIT
diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions
index 57ea18d..3342621 100644
--- a/config/debian/dinstall.functions
+++ b/config/debian/dinstall.functions
@@ -97,6 +97,32 @@ function i18n1() {
fi
}
+# Syncing AppStream/DEP-11 data
+function dep11() {
+ log "Synchronizing AppStream metadata"
+ # First sync their newest data
+ mkdir -p ${scriptdir}/dep11
+ cd ${scriptdir}/dep11
+ rsync -aq --delete --delete-after dep11-sync:/does/not/matter . || true
+
+ # Lets check!
+ if ${scriptsdir}/dep11-basic-validate.py . ${scriptdir}/dep11/; then
+ # Yay, worked, lets copy around
+ for dir in stretch unstable; do
+ if [ -d ${dir}/ ]; then
+ for comp in main contrib non-free; do
+ cd dists/${dir}/${comp}/dep11
+ rsync -aq --delete --delete-after --exclude *.tmp . ${ftpdir}/dists/${dir}/${comp}/dep11/.
+ cd ${scriptdir}/dep11
+ fi
+ fi
+ done
+ else
+ echo "ARRRR, bad guys, wrong files, ARRR"
+ echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" mak@debian.org
+ fi
+}
+
function cruft() {
log "Checking for cruft in overrides"
dak check-overrides
diff --git a/config/homedir/ssh/ftpmaster-config b/config/homedir/ssh/ftpmaster-config
index 5db7c72..bfcc27b 100644
--- a/config/homedir/ssh/ftpmaster-config
+++ b/config/homedir/ssh/ftpmaster-config
@@ -17,6 +17,11 @@ Host ddtp-sync
User ddtp-dak
IdentityFile /srv/ftp-master.debian.org/s3kr1t/ddtp-dak.rsa
+Host dep11-sync
+ Hostname mekeel.debian.org
+ User dep11-dak
+ IdentityFile /srv/ftp-master.debian.org/s3kr1t/dep11-dak.rsa
+
Host morgue-sync
Hostname stabile.debian.org
User dak
diff --git a/scripts/debian/dep11-basic-validate.py b/scripts/debian/dep11-basic-validate.py
new file mode 100755
index 0000000..6f46e14
--- /dev/null
+++ b/scripts/debian/dep11-basic-validate.py
@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2015 Matthias Klumpp <mak@debian.org>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program.
+
+import os
+import sys
+import yaml
+import gzip
+from voluptuous import Schema, Required, All, Any, Length, Range, Match, Url
+from optparse import OptionParser
+
+schema_header = Schema({
+ Required('File'): All(str, 'DEP-11', msg="Must be \"DEP-11\""),
+ Required('Origin'): All(str, Length(min=1)),
+ Required('Version'): All(str, Match(r'(\d+\.?)+$'), msg="Must be a valid version number"),
+ Required('MediaBaseUrl'): All(str, Url()),
+ 'Time': All(str, str),
+ 'Priority': All(str, int),
+})
+
+schema_translated = Schema({
+ Required('C'): All(str, Length(min=1), msg="Must have an unlocalized 'C' key"),
+ dict: All(str, Length(min=1)),
+}, extra = True)
+
+schema_component = Schema({
+ Required('Type'): All(str, Length(min=1)),
+ Required('ID'): All(str, Length(min=1)),
+ Required('Name'): All(dict, Length(min=1), schema_translated),
+ Required('Package'): All(str, Length(min=1)),
+}, extra = True)
+
+def add_issue(msg):
+ print(msg)
+
+def test_custom_objects(lines):
+ ret = True
+ for i in range(0, len(lines)):
+ if "!!python/" in lines[i]:
+ add_issue("Python object encoded in line %i." % (i))
+ ret = False
+ return ret
+
+def is_quoted(s):
+ return (s.startswith("\"") and s.endswith("\"")) or (s.startswith("\'") and s.endswith("\'"))
+
+def test_localized_dict(doc, ldict, id_string):
+ ret = True
+ for lang, value in ldict.items():
+ if lang == 'x-test':
+ add_issue("[%s][%s]: %s" % (doc['ID'], id_string, "Found cruft locale: x-test"))
+ if lang == 'xx':
+ add_issue("[%s][%s]: %s" % (doc['ID'], id_string, "Found cruft locale: xx"))
+ if lang.endswith('.UTF-8'):
+ add_issue("[%s][%s]: %s" % (doc['ID'], id_string, "AppStream locale names should not specify encoding (ends with .UTF-8)"))
+ if is_quoted(value):
+ add_issue("[%s][%s]: %s" % (doc['ID'], id_string, "String is quoted: '%s' @ %s" % (value, lang)))
+ if " " in lang:
+ add_issue("[%s][%s]: %s" % (doc['ID'], id_string, "Locale name contains space: '%s'" % (lang)))
+ # this - as opposed to the other issues - is an error
+ ret = False
+ return ret
+
+def test_localized(doc, key):
+ ldict = doc.get(key, None)
+ if not ldict:
+ return True
+
+ return test_localized_dict(doc, ldict, key)
+
+def validate_data(data):
+ ret = True
+ lines = data.split("\n")
+
+ # see if there are any Python-specific objects encoded
+ ret = test_custom_objects(lines)
+
+ try:
+ docs = yaml.safe_load_all(data)
+ header = next(docs)
+ except Exception as e:
+ add_issue("Could not parse file: %s" % (str(e)))
+ return False
+
+ try:
+ schema_header(header)
+ except Exception as e:
+ add_issue("Invalid DEP-11 header: %s" % (str(e)))
+ ret = False
+
+ for doc in docs:
+ docid = doc.get('ID')
+ pkgname = doc.get('Package')
+ if not pkgname:
+ pkgname = "?unknown?"
+ if not doc:
+ add_issue("FATAL: Empty document found.")
+ ret = False
+ continue
+ if not docid:
+ add_issue("FATAL: Component without ID found.")
+ ret = False
+ continue
+
+ try:
+ schema_component(doc)
+ except Exception as e:
+ add_issue("[%s]: %s" % (docid, str(e)))
+ ret = False
+ continue
+
+ # more tests for the icon key
+ icon = doc.get('Icon')
+ if (doc['Type'] == "desktop-app") or (doc['Type'] == "web-app"):
+ if not doc.get('Icon'):
+ add_issue("[%s]: %s" % (docid, "Components containing an application must have an 'Icon' key."))
+ ret = False
+ if icon:
+ if (not icon.get('stock')) and (not icon.get('cached')) and (not icon.get('local')):
+ add_issue("[%s]: %s" % (docid, "A 'stock', 'cached' or 'local' icon must at least be provided. @ data['Icon']"))
+ ret = False
+
+ if not test_localized(doc, 'Name'):
+ ret = False
+ if not test_localized(doc, 'Summary'):
+ ret = False
+ if not test_localized(doc, 'Description'):
+ ret = False
+ if not test_localized(doc, 'DeveloperName'):
+ ret = False
+
+ for shot in doc.get('Screenshots', list()):
+ caption = shot.get('caption')
+ if caption:
+ if not test_localized_dict(doc, caption, "Screenshots.x.caption"):
+ ret = False
+
+ return ret
+
+def validate_file(fname):
+ f = None
+ if fname.endswith(".gz"):
+ f = gzip.open(fname, 'r')
+ else:
+ f = open(fname, 'r')
+
+ data = str(f.read(), 'utf-8')
+ f.close()
+
+ return validate_data(data)
+
+def validate_dir(dirname):
+ ret = True
+ for root, subfolders, files in os.walk(dirname):
+ for fname in files:
+ if fname.endswith(".yml.gz"):
+ if not validate_file(os.path.join(root, fname)):
+ ret = False
+
+ return ret
+
+def main():
+ parser = OptionParser()
+
+ (options, args) = parser.parse_args()
+
+ if len(args) < 1:
+ print("You need to specify a file to validate!")
+ sys.exit(4)
+ fname = args[0]
+
+ if os.path.isdir(fname):
+ ret = validate_dir(fname)
+ else:
+ ret = validate_file(fname)
+ if ret:
+ msg = "DEP-11 basic validation successful."
+ else:
+ msg = "DEP-11 validation failed!"
+ print(msg)
+
+ if not ret:
+ sys.exit(1)
+
+if __name__ == "__main__":
+ main()
Reply to: