[dak/master] Include AppStream metadata from the generator server
This patch pulls AppStream/DEP-11 metadata from the server where it was
generated, then validates it for format issues and adds it to the
archive.
---
config/debian/cron.dinstall | 8 ++
config/debian/dinstall.functions | 26 +++++
config/homedir/ssh/ftpmaster-config | 5 +
daklib/regexes.py | 2 +-
scripts/debian/dep11-basic-validate.py | 199 +++++++++++++++++++++++++++++++++
5 files changed, 239 insertions(+), 1 deletion(-)
create mode 100755 scripts/debian/dep11-basic-validate.py
diff --git a/config/debian/cron.dinstall b/config/debian/cron.dinstall
index 4e8c1f2..8aaaa64 100755
--- a/config/debian/cron.dinstall
+++ b/config/debian/cron.dinstall
@@ -210,6 +210,14 @@ GO=(
)
stage $GO
+GO=(
+ FUNC="dep11"
+ TIME="dep11 1"
+ ARGS=""
+ ERR="false"
+)
+stage $GO
+
lockfile "$LOCK_ACCEPTED"
trap remove_all_locks EXIT TERM HUP INT QUIT
diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions
index ae62118..f43a0aa 100644
--- a/config/debian/dinstall.functions
+++ b/config/debian/dinstall.functions
@@ -97,6 +97,32 @@ function i18n1() {
fi
}
+# Syncing AppStream/DEP-11 data
+function dep11() {
+ log "Synchronizing AppStream metadata"
+ # First sync their newest data
+ mkdir -p ${scriptdir}/dep11
+ cd ${scriptdir}/dep11
+ rsync -aq --delete --delete-after dep11-sync:/does/not/matter . || true
+
+ # Lets check!
+ if ${scriptsdir}/dep11-basic-validate.py . ${scriptdir}/dep11/; then
+ # Yay, worked, lets copy around
+ for dir in stretch sid; do
+ if [ -d ${dir}/ ]; then
+ for comp in main contrib non-free; do
+ cd dists/${dir}/${comp}/dep11
+ rsync -aq --delete --delete-after --exclude *.tmp . ${ftpdir}/dists/${dir}/${comp}/dep11/.
+ cd ${scriptdir}/dep11
+ fi
+ fi
+ done
+ else
+ echo "ARRRR, bad guys, wrong files, ARRR"
+ echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -a "X-Debian: DAK" -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" mak@debian.org
+ fi
+}
+
function cruft() {
log "Checking for cruft in overrides"
dak check-overrides
diff --git a/config/homedir/ssh/ftpmaster-config b/config/homedir/ssh/ftpmaster-config
index 222338a..083f81f 100644
--- a/config/homedir/ssh/ftpmaster-config
+++ b/config/homedir/ssh/ftpmaster-config
@@ -17,6 +17,11 @@ Host ddtp-sync
User ddtp-dak
IdentityFile /srv/ftp-master.debian.org/s3kr1t/ddtp-dak.rsa
+Host dep11-sync
+ Hostname mekeel.debian.org
+ User appstream
+ IdentityFile /srv/ftp-master.debian.org/s3kr1t/appstream.rsa
+
Host morgue-sync
Hostname stabile.debian.org
User dak
diff --git a/daklib/regexes.py b/daklib/regexes.py
index 21defe9..0a2045d 100644
--- a/daklib/regexes.py
+++ b/daklib/regexes.py
@@ -102,7 +102,7 @@ re_parse_lintian = re.compile(r"^(?P<level>W|E|O): (?P<package>.*?): (?P<tag>[^
# in generate-releases
re_gensubrelease = re.compile (r".*/(binary-[0-9a-z-]+|source)$")
-re_includeinrelease = re.compile (r"(Translation-[a-zA-Z_]+\.(?:bz2|xz)|Contents-[0-9a-z-]+.gz|Index|Packages(.gz|.bz2|.xz)?|Sources(.gz|.bz2|.xz)?|MD5SUMS|SHA256SUMS|Release)$")
+re_includeinrelease = re.compile (r"(Translation-[a-zA-Z_]+\.(?:bz2|xz)|Contents-[0-9a-z-]+.gz|Index|Packages(.gz|.bz2|.xz)?|Sources(.gz|.bz2|.xz)?|Components-[0-9a-z-]+(.gz|.xz)|icons-[0-9x-]+(.gz|.xz)|MD5SUMS|SHA256SUMS|Release)$")
# in generate_index_diffs
re_includeinpdiff = re.compile(r"(Translation-[a-zA-Z_]+\.(?:bz2|xz))")
diff --git a/scripts/debian/dep11-basic-validate.py b/scripts/debian/dep11-basic-validate.py
new file mode 100755
index 0000000..6f46e14
--- /dev/null
+++ b/scripts/debian/dep11-basic-validate.py
@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2015 Matthias Klumpp <mak@debian.org>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3.0 of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program.
+
+import os
+import sys
+import yaml
+import gzip
+from voluptuous import Schema, Required, All, Any, Length, Range, Match, Url
+from optparse import OptionParser
+
+schema_header = Schema({
+ Required('File'): All(str, 'DEP-11', msg="Must be \"DEP-11\""),
+ Required('Origin'): All(str, Length(min=1)),
+ Required('Version'): All(str, Match(r'(\d+\.?)+$'), msg="Must be a valid version number"),
+ Required('MediaBaseUrl'): All(str, Url()),
+ 'Time': All(str, str),
+ 'Priority': All(str, int),
+})
+
+schema_translated = Schema({
+ Required('C'): All(str, Length(min=1), msg="Must have an unlocalized 'C' key"),
+ dict: All(str, Length(min=1)),
+}, extra = True)
+
+schema_component = Schema({
+ Required('Type'): All(str, Length(min=1)),
+ Required('ID'): All(str, Length(min=1)),
+ Required('Name'): All(dict, Length(min=1), schema_translated),
+ Required('Package'): All(str, Length(min=1)),
+}, extra = True)
+
+def add_issue(msg):
+ print(msg)
+
+def test_custom_objects(lines):
+ ret = True
+ for i in range(0, len(lines)):
+ if "!!python/" in lines[i]:
+ add_issue("Python object encoded in line %i." % (i))
+ ret = False
+ return ret
+
+def is_quoted(s):
+ return (s.startswith("\"") and s.endswith("\"")) or (s.startswith("\'") and s.endswith("\'"))
+
+def test_localized_dict(doc, ldict, id_string):
+ ret = True
+ for lang, value in ldict.items():
+ if lang == 'x-test':
+ add_issue("[%s][%s]: %s" % (doc['ID'], id_string, "Found cruft locale: x-test"))
+ if lang == 'xx':
+ add_issue("[%s][%s]: %s" % (doc['ID'], id_string, "Found cruft locale: xx"))
+ if lang.endswith('.UTF-8'):
+ add_issue("[%s][%s]: %s" % (doc['ID'], id_string, "AppStream locale names should not specify encoding (ends with .UTF-8)"))
+ if is_quoted(value):
+ add_issue("[%s][%s]: %s" % (doc['ID'], id_string, "String is quoted: '%s' @ %s" % (value, lang)))
+ if " " in lang:
+ add_issue("[%s][%s]: %s" % (doc['ID'], id_string, "Locale name contains space: '%s'" % (lang)))
+ # this - as opposed to the other issues - is an error
+ ret = False
+ return ret
+
+def test_localized(doc, key):
+ ldict = doc.get(key, None)
+ if not ldict:
+ return True
+
+ return test_localized_dict(doc, ldict, key)
+
+def validate_data(data):
+ ret = True
+ lines = data.split("\n")
+
+ # see if there are any Python-specific objects encoded
+ ret = test_custom_objects(lines)
+
+ try:
+ docs = yaml.safe_load_all(data)
+ header = next(docs)
+ except Exception as e:
+ add_issue("Could not parse file: %s" % (str(e)))
+ return False
+
+ try:
+ schema_header(header)
+ except Exception as e:
+ add_issue("Invalid DEP-11 header: %s" % (str(e)))
+ ret = False
+
+ for doc in docs:
+ docid = doc.get('ID')
+ pkgname = doc.get('Package')
+ if not pkgname:
+ pkgname = "?unknown?"
+ if not doc:
+ add_issue("FATAL: Empty document found.")
+ ret = False
+ continue
+ if not docid:
+ add_issue("FATAL: Component without ID found.")
+ ret = False
+ continue
+
+ try:
+ schema_component(doc)
+ except Exception as e:
+ add_issue("[%s]: %s" % (docid, str(e)))
+ ret = False
+ continue
+
+ # more tests for the icon key
+ icon = doc.get('Icon')
+ if (doc['Type'] == "desktop-app") or (doc['Type'] == "web-app"):
+ if not doc.get('Icon'):
+ add_issue("[%s]: %s" % (docid, "Components containing an application must have an 'Icon' key."))
+ ret = False
+ if icon:
+ if (not icon.get('stock')) and (not icon.get('cached')) and (not icon.get('local')):
+ add_issue("[%s]: %s" % (docid, "A 'stock', 'cached' or 'local' icon must at least be provided. @ data['Icon']"))
+ ret = False
+
+ if not test_localized(doc, 'Name'):
+ ret = False
+ if not test_localized(doc, 'Summary'):
+ ret = False
+ if not test_localized(doc, 'Description'):
+ ret = False
+ if not test_localized(doc, 'DeveloperName'):
+ ret = False
+
+ for shot in doc.get('Screenshots', list()):
+ caption = shot.get('caption')
+ if caption:
+ if not test_localized_dict(doc, caption, "Screenshots.x.caption"):
+ ret = False
+
+ return ret
+
+def validate_file(fname):
+ f = None
+ if fname.endswith(".gz"):
+ f = gzip.open(fname, 'r')
+ else:
+ f = open(fname, 'r')
+
+ data = str(f.read(), 'utf-8')
+ f.close()
+
+ return validate_data(data)
+
+def validate_dir(dirname):
+ ret = True
+ for root, subfolders, files in os.walk(dirname):
+ for fname in files:
+ if fname.endswith(".yml.gz"):
+ if not validate_file(os.path.join(root, fname)):
+ ret = False
+
+ return ret
+
+def main():
+ parser = OptionParser()
+
+ (options, args) = parser.parse_args()
+
+ if len(args) < 1:
+ print("You need to specify a file to validate!")
+ sys.exit(4)
+ fname = args[0]
+
+ if os.path.isdir(fname):
+ ret = validate_dir(fname)
+ else:
+ ret = validate_file(fname)
+ if ret:
+ msg = "DEP-11 basic validation successful."
+ else:
+ msg = "DEP-11 validation failed!"
+ print(msg)
+
+ if not ret:
+ sys.exit(1)
+
+if __name__ == "__main__":
+ main()
--
2.1.4
Reply to: