On Sun, 2015-01-25 at 18:26 +0100, Raphael Hertzog wrote: > Don't you want to delete the action item here if the new status is None or not > known ? In fact you probably want to create it the new action_item only > after that test too! Or given that you delete obsolete action items later, > you might want to do this check first (or outside of the function). Hmm, I got the impression action items that weren't saved didn't need to be explicitly deleted. Anyway, moved the check to the top. > get_build_reproducibility() can never return None with your current > implementation. To follow the pattern used in other tasks, it should > do so when the underlying URL hasn't changed. Instead of using > get_resource_content, you must use the HttpCache() directly to be able > to retrieve the result of cache.update(). Fixed. > This will always raise the exception since you dropped them all at the > start of the transaction. So you can just keep the content of your except > clause AIUI. Fixed. Updated patch attached. -- bye, pabs https://wiki.debian.org/PaulWise
From d6696d2eddf21b2541a66c90fd2010fd9769241b Mon Sep 17 00:00:00 2001
From: Paul Wise <pabs@debian.org>
Date: Wed, 26 Nov 2014 18:55:50 +0800
Subject: [PATCH] Add reproducible builds (Closes: #739497)
Action item long description by Lunar^
---
.../debian/build-reproducibility-action-item.html | 6 ++
.../debian/templates/debian/logcheck-links.html | 4 +
distro_tracker/vendor/debian/tests.py | 107 +++++++++++++++++++++
distro_tracker/vendor/debian/tracker_panels.py | 11 +++
distro_tracker/vendor/debian/tracker_tasks.py | 81 ++++++++++++++++
5 files changed, 209 insertions(+)
create mode 100644 distro_tracker/vendor/debian/templates/debian/build-reproducibility-action-item.html
diff --git a/distro_tracker/vendor/debian/templates/debian/build-reproducibility-action-item.html b/distro_tracker/vendor/debian/templates/debian/build-reproducibility-action-item.html
new file mode 100644
index 0000000..beaa395
--- /dev/null
+++ b/distro_tracker/vendor/debian/templates/debian/build-reproducibility-action-item.html
@@ -0,0 +1,6 @@
+A package <a href="https://wiki.debian.org/ReproducibleBuilds";>building reproducibly</a>
+enables third parties to verify that the source matches the distributed binaries.
+It has been <a href="https://reproducible.debian.net/rb-pkg/{{ item.package.name }}".html>identified</a>
+that this source package produced different results, failed to build or had other issues in a
+<a href="https://wiki.debian.org/ReproducibleBuilds/ExperimentalToolchain";>test environment</a>.
+Please read about <a href="https://wiki.debian.org/ReproducibleBuilds/Howto";>how to improve the situation</a>!
diff --git a/distro_tracker/vendor/debian/templates/debian/logcheck-links.html b/distro_tracker/vendor/debian/templates/debian/logcheck-links.html
index ffaf2d4..006eec5 100644
--- a/distro_tracker/vendor/debian/templates/debian/logcheck-links.html
+++ b/distro_tracker/vendor/debian/templates/debian/logcheck-links.html
@@ -13,4 +13,8 @@
{% endif %}
<span>,</span>
<span><a title="clang compiler build log" href="http://buildd-clang.debian.net/package.php?{{ query_string }}">clang</a></span>
+{% if item.context.has_reproducibility %}
+<span>,</span>
+<span><a title="report about build reproducibility ({{ item.context.reproducibility_status }})" href="{{ item.context.reproducibility_url }}">reproducibility</a></span>
+{% endif %}
{% endwith %}
diff --git a/distro_tracker/vendor/debian/tests.py b/distro_tracker/vendor/debian/tests.py
index c330770..009c029 100644
--- a/distro_tracker/vendor/debian/tests.py
+++ b/distro_tracker/vendor/debian/tests.py
@@ -69,6 +69,7 @@ from distro_tracker.vendor.debian.tracker_tasks \
import UpdateAutoRemovalsStatsTask
from distro_tracker.vendor.debian.tracker_tasks \
import UpdatePackageScreenshotsTask
+from distro_tracker.vendor.debian.tracker_tasks import UpdateBuildReproducibilityTask
from distro_tracker.vendor.debian.models import DebianContributor
from distro_tracker.vendor.debian.models import UbuntuPackage
from distro_tracker.vendor.debian.tracker_tasks import UpdateLintianStatsTask
@@ -5188,3 +5189,109 @@ class UpdatePackageScreenshotsTaskTest(TestCase):
info = self.dummy_package.packageextractedinfo_set.get(key='general')
self.assertEqual(info.value['name'], 'dummy')
+
+
+@mock.patch('distro_tracker.core.utils.http.requests')
+class UpdateBuildReproducibilityTaskTest(TestCase):
+ """
+ Tests for the:class:`distro_tracker.vendor.debian.tracker_tasks.
+ UpdateBuildReproducibilityTask` task.
+ """
+ def setUp(self):
+ self.json_data = """
+ [{
+ "package": "dummy",
+ "version": "1.2-3",
+ "status": "unreproducible",
+ "suite": "sid"
+ }]
+ """
+ self.other_json_data = """
+ [{
+ "package": "other",
+ "version": "1.2-3",
+ "status": "unreproducible",
+ "suite": "sid"
+ }]
+ """
+ self.dummy_package = SourcePackageName.objects.create(name='dummy')
+ PackageExtractedInfo.objects.create(
+ package=self.dummy_package,
+ key='general',
+ value={
+ 'name': 'dummy',
+ 'maintainer': {
+ 'email': 'jane@example.com',
+ }
+ }
+ )
+
+ def run_task(self):
+ """
+ Runs the build reproducibility status update task.
+ """
+ task = UpdateBuildReproducibilityTask()
+ task.execute()
+
+ def test_extractedinfo_item_for_without_reproducibility(self, mock_requests):
+ """
+ Tests that packages without reproducibility info don't claim to have them.
+ """
+ set_mock_response(mock_requests, text=self.json_data)
+ other_package = SourcePackageName.objects.create(name='other-package')
+
+ self.run_task()
+
+ with self.assertRaises(PackageExtractedInfo.DoesNotExist):
+ other_package.packageextractedinfo_set.get(key='reproducibility')
+
+ def test_no_extractedinfo_for_unknown_package(self, mock_requests):
+ """
+ Tests that BuildReproducibilityTask doesn't fail with an unknown package.
+ """
+ set_mock_response(mock_requests, text=self.other_json_data)
+
+ self.run_task()
+
+ count = PackageExtractedInfo.objects.filter(key='reproducibility').count()
+ self.assertEqual(0, count)
+
+ def test_extractedinfo_for_package_with_reproducibility(self, mock_requests):
+ """
+ Tests that PackageExtractedInfo for a package with reproducibility info is correct.
+ """
+ set_mock_response(mock_requests, text=self.json_data)
+
+ self.run_task()
+
+ info = self.dummy_package.packageextractedinfo_set.get(key='reproducibility')
+
+ self.assertEqual(info.value['reproducibility'], 'unreproducible')
+
+ def test_extractedinfo_is_dropped_when_no_more_reproducibility(self, mock_requests):
+ """
+ Tests that PackageExtractedInfo is dropped if reproducibility info goes away.
+ """
+ set_mock_response(mock_requests, text=self.json_data)
+ self.run_task()
+
+ set_mock_response(mock_requests, text=self.other_json_data)
+ self.run_task()
+
+ with self.assertRaises(PackageExtractedInfo.DoesNotExist):
+ self.dummy_package.packageextractedinfo_set.get(key='reproducibility')
+
+ def test_other_extractedinfo_keys_not_dropped(self, mock_requests):
+ """
+ Ensure that other PackageExtractedInfo keys are not dropped when
+ deleting the reproducibility key.
+ """
+ set_mock_response(mock_requests, text=self.json_data)
+ self.run_task()
+
+ set_mock_response(mock_requests, text=self.other_json_data)
+ self.run_task()
+
+ info = self.dummy_package.packageextractedinfo_set.get(key='general')
+
+ self.assertEqual(info.value['name'], 'dummy')
diff --git a/distro_tracker/vendor/debian/tracker_panels.py b/distro_tracker/vendor/debian/tracker_panels.py
index d7cd4bc..e447661 100644
--- a/distro_tracker/vendor/debian/tracker_panels.py
+++ b/distro_tracker/vendor/debian/tracker_panels.py
@@ -77,12 +77,23 @@ class BuildLogCheckLinks(LinksPanel.ItemProvider):
logcheck_url = \
"https://qa.debian.org/bls/packages/{hash}/{pkg}.html".format(
hash=self.package.name[0], pkg=self.package.name)
+ try:
+ infos = self.package.packageextractedinfo_set.get(key='reproducibility')
+ has_reproducibility = True
+ reproducibility_status = infos.value['reproducibility']
+ except PackageExtractedInfo.DoesNotExist:
+ has_reproducibility = False
+ reproducibility_status = None
+ reproducibility_url = "https://reproducible.debian.net/rb-pkg/{pkg}.html".format(pkg=self.package.name)
return [
TemplatePanelItem('debian/logcheck-links.html', {
'package_query_string': query_string,
'has_checks': has_checks,
'logcheck_url': logcheck_url,
+ 'has_reproducibility': has_reproducibility,
+ 'reproducibility_url': reproducibility_url,
+ 'reproducibility_status': reproducibility_status,
'has_experimental': has_experimental,
})
]
diff --git a/distro_tracker/vendor/debian/tracker_tasks.py b/distro_tracker/vendor/debian/tracker_tasks.py
index 8d797b7..5f04c62 100644
--- a/distro_tracker/vendor/debian/tracker_tasks.py
+++ b/distro_tracker/vendor/debian/tracker_tasks.py
@@ -2262,3 +2262,84 @@ class UpdatePackageScreenshotsTask(BaseTask):
extracted_info.append(screenshot_info)
PackageExtractedInfo.objects.bulk_create(extracted_info)
+
+
+class UpdateBuildReproducibilityTask(BaseTask):
+ ACTION_ITEM_TYPE_NAME = 'debian-build-reproducibility'
+ ACTION_ITEM_TEMPLATE = 'debian/build-reproducibility-action-item.html'
+ ITEM_DESCRIPTION = {
+ 'blacklisted': '<a href="{url}">Blacklisted</a> from build reproducibility testing',
+ 'FTBFS': '<a href="{url}">Fails to build</a> during reproducibility testing',
+ 'reproducible': None,
+ 'unreproducible': '<a href="{url}">Does not build reproducibly</a> during testing and no .buildinfo file is created',
+ 'unreproducible-with-buildinfo': '<a href="{url}">Does not build reproducibly</a> during testing',
+ '404': None,
+ 'not for us': None,
+ }
+
+ def __init__(self, force_update=False, *args, **kwargs):
+ super(UpdateBuildReproducibilityTask, self).__init__(*args, **kwargs)
+ self.force_update = force_update
+ self.action_item_type = ActionItemType.objects.create_or_update(
+ type_name=self.ACTION_ITEM_TYPE_NAME,
+ full_description_template=self.ACTION_ITEM_TEMPLATE)
+
+ def set_parameters(self, parameters):
+ if 'force_update' in parameters:
+ self.force_update = parameters['force_update']
+
+ def get_build_reproducibility(self):
+ url = 'https://reproducible.debian.net/reproducible.json'
+ cache = HttpCache(settings.DISTRO_TRACKER_CACHE_DIRECTORY)
+ if not self.force_update and not cache.is_expired(url):
+ return
+ response, updated = cache.update(url, force=self.force_update)
+ response.raise_for_status()
+ if not updated:
+ return
+ reproducibilities = json.loads(response.text)
+ reproducibilities = dict([(item['package'], item['status']) for item in reproducibilities])
+ return reproducibilities
+
+ def update_action_item(self, package, status):
+ if not (status and status in self.ITEM_DESCRIPTION and self.ITEM_DESCRIPTION[status]):
+ return
+
+ action_item = package.get_action_item_for_type(self.action_item_type.type_name)
+ if action_item is None:
+ action_item = ActionItem(
+ package=package,
+ item_type=self.action_item_type,
+ severity=ActionItem.SEVERITY_NORMAL)
+
+ url = "https://reproducible.debian.net/rb-pkg/{pkg}.html".format(pkg=package.name)
+ action_item.short_description = self.ITEM_DESCRIPTION[status].format(url=url)
+ action_item.save()
+
+ def execute(self):
+ reproducibilities = self.get_build_reproducibility()
+ if reproducibilities is None:
+ return
+
+ with transaction.atomic():
+ PackageExtractedInfo.objects.filter(key='reproducibility').delete()
+
+ packages = []
+ extracted_info = []
+
+ for name, status in reproducibilities.items():
+ try:
+ package = SourcePackageName.objects.get(name=name)
+ packages.append(package)
+ self.update_action_item(package, status)
+ except SourcePackageName.DoesNotExist:
+ continue
+
+ reproducibility_info = PackageExtractedInfo(
+ key='reproducibility',
+ package=package,
+ value={'reproducibility': status})
+ extracted_info.append(reproducibility_info)
+
+ ActionItem.objects.delete_obsolete_items([self.action_item_type], packages)
+ PackageExtractedInfo.objects.bulk_create(extracted_info)
--
2.1.4
Attachment:
signature.asc
Description: This is a digitally signed message part