reassign 759886 dpkg-dev
retitle 759886 dpkg-dev: please make mtimes of packaged files deterministic
thanks
Hi!
The attached patch series is an attempt to make the mtimes of packaged
files deterministic. It is taken from the `pu/reproducible_builds`
dpkg branch maintained by the “reproducible builds” folks [1].
The first two patches introduce the idea of a canonical build timestamp
that will be used throughout dpkg-deb.
The first patch will make use of this timestamp to set the mtime in ar
headers (that's #759999). All headers will thus get the same timestamp
instead of recording the current time as they are added.
The second patch will use the --mtime and --clamp-mtime option of tar to
clamp the mtime of files recorded in control.tar and data.tar to the
build timestamp: files created at a later time will see their mtime
set to the build timestamp (that's #759886). As --clamp-mtime is only
available since tar/1.28-1 and has not yet been merged upstream,
dpkg-deb will first look for its availability by looking for the option
in the output of “tar --help”. If it's not available, it will fallback
to the previous behavior.
The third patch adds the ability to set the aforementioned build
timestamp using the SOURCE_DATE_EPOCH environment variable [2].
The forth patch changes dpkg-buildpackage to set SOURCE_DATE_EPOCH to
the time of the latest debian/changelog entry if it hasn't been already
set, effectively making the timestamps recorded by dpkg-deb in the most
common build process deterministic.
[1]: https://anonscm.debian.org/cgit/reproducible/dpkg.git/log/?h=pu/reproducible_builds
[2]: https://reproducible-builds.org/specs/source-date-epoch/
--
Lunar .''`.
lunar@debian.org : :Ⓐ : # apt-get install anarchism
`. `'`
`-
From e56a69e2fa3333c8531c9a45008470ca8aa8dd9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Bobbio?= <lunar@debian.org>
Date: Tue, 27 Aug 2013 22:38:31 +0200
Subject: [PATCH 1/4] dpkg-deb: Use a single timestamp for ar headers when
building a .deb
In order to make build reproducible in the future, we now use a single
timestamp in all ar headers when creating a .deb.
Previously, each ar header would have the current time of its creation.
This level of precision is not really needed and the time of the beginning of
the build is good enough.
Address: #759999
---
dpkg-deb/build.c | 10 +++++++---
dpkg-split/split.c | 4 ++--
lib/dpkg/ar.c | 13 +++++++------
lib/dpkg/ar.h | 4 ++--
4 files changed, 18 insertions(+), 13 deletions(-)
diff --git a/dpkg-deb/build.c b/dpkg-deb/build.c
index 8d9f066..117e424 100644
--- a/dpkg-deb/build.c
+++ b/dpkg-deb/build.c
@@ -37,6 +37,7 @@
#include <stdint.h>
#include <stdlib.h>
#include <stdio.h>
+#include <time.h>
#include <dpkg/i18n.h>
#include <dpkg/c-ctype.h>
@@ -529,6 +530,7 @@ do_build(const char *const *argv)
int arfd;
int p1[2], gzfd;
pid_t c1, c2;
+ time_t build_timestamp;
/* Decode our arguments. */
dir = *argv++;
@@ -559,6 +561,8 @@ do_build(const char *const *argv)
}
m_output(stdout, _("<standard output>"));
+ build_timestamp = time(NULL);
+
/* Now that we have verified everything its time to actually
* build something. Let's start by making the ar-wrapper. */
arfd = creat(debar, 0644);
@@ -636,8 +640,8 @@ do_build(const char *const *argv)
compressor_get_extension(control_compress_params.type));
dpkg_ar_put_magic(debar, arfd);
- dpkg_ar_member_put_mem(debar, arfd, DEBMAGIC, deb_magic, strlen(deb_magic));
- dpkg_ar_member_put_file(debar, arfd, adminmember, gzfd, -1);
+ dpkg_ar_member_put_mem(debar, arfd, DEBMAGIC, deb_magic, build_timestamp, strlen(deb_magic));
+ dpkg_ar_member_put_file(debar, arfd, adminmember, gzfd, build_timestamp, -1);
} else {
internerr("unknown deb format version %d.%d", deb_format.major, deb_format.minor);
}
@@ -679,7 +683,7 @@ do_build(const char *const *argv)
if (lseek(gzfd, 0, SEEK_SET))
ohshite(_("failed to rewind temporary file (%s)"), _("data member"));
- dpkg_ar_member_put_file(debar, arfd, datamember, gzfd, -1);
+ dpkg_ar_member_put_file(debar, arfd, datamember, gzfd, build_timestamp, -1);
close(gzfd);
}
diff --git a/dpkg-split/split.c b/dpkg-split/split.c
index 8137654..d132e3e 100644
--- a/dpkg-split/split.c
+++ b/dpkg-split/split.c
@@ -210,13 +210,13 @@ mksplit(const char *file_src, const char *prefix, off_t maxpartsize,
(intmax_t)st.st_size, (intmax_t)partsize,
curpart, nparts, pkg->available.arch->name);
dpkg_ar_member_put_mem(file_dst.buf, fd_dst, PARTMAGIC,
- partmagic.buf, partmagic.used);
+ partmagic.buf, time(NULL), partmagic.used);
varbuf_reset(&partmagic);
/* Write the data part. */
varbuf_printf(&partname, "data.%d", curpart);
dpkg_ar_member_put_file(file_dst.buf, fd_dst, partname.buf,
- fd_src, cur_partsize);
+ fd_src, time(NULL), cur_partsize);
varbuf_reset(&partname);
close(fd_dst);
diff --git a/lib/dpkg/ar.c b/lib/dpkg/ar.c
index 97eefb6..f31b6a7 100644
--- a/lib/dpkg/ar.c
+++ b/lib/dpkg/ar.c
@@ -36,11 +36,12 @@
#include <dpkg/ar.h>
static void
-dpkg_ar_member_init(struct dpkg_ar_member *member, const char *name, off_t size)
+dpkg_ar_member_init(struct dpkg_ar_member *member, const char *name,
+ time_t timestamp, off_t size)
{
member->name = name;
member->size = size;
- member->time = time(NULL);
+ member->time = timestamp;
member->mode = 0100644;
member->uid = 0;
member->gid = 0;
@@ -124,11 +125,11 @@ dpkg_ar_member_put_header(const char *ar_name, int ar_fd,
void
dpkg_ar_member_put_mem(const char *ar_name, int ar_fd,
- const char *name, const void *data, size_t size)
+ const char *name, const void *data, time_t timestamp, size_t size)
{
struct dpkg_ar_member member;
- dpkg_ar_member_init(&member, name, size);
+ dpkg_ar_member_init(&member, name, timestamp, size);
dpkg_ar_member_put_header(ar_name, ar_fd, &member);
/* Copy data contents. */
@@ -142,7 +143,7 @@ dpkg_ar_member_put_mem(const char *ar_name, int ar_fd,
void
dpkg_ar_member_put_file(const char *ar_name, int ar_fd,
- const char *name, int fd, off_t size)
+ const char *name, int fd, time_t timestamp, off_t size)
{
struct dpkg_error err;
struct dpkg_ar_member member;
@@ -155,7 +156,7 @@ dpkg_ar_member_put_file(const char *ar_name, int ar_fd,
size = st.st_size;
}
- dpkg_ar_member_init(&member, name, size);
+ dpkg_ar_member_init(&member, name, timestamp, size);
dpkg_ar_member_put_header(ar_name, ar_fd, &member);
/* Copy data contents. */
diff --git a/lib/dpkg/ar.h b/lib/dpkg/ar.h
index 81a061e..c1b8bed 100644
--- a/lib/dpkg/ar.h
+++ b/lib/dpkg/ar.h
@@ -59,9 +59,9 @@ void dpkg_ar_put_magic(const char *ar_name, int ar_fd);
void dpkg_ar_member_put_header(const char *ar_name, int ar_fd,
struct dpkg_ar_member *member);
void dpkg_ar_member_put_file(const char *ar_name, int ar_fd, const char *name,
- int fd, off_t size);
+ int fd, time_t timestamp, off_t size);
void dpkg_ar_member_put_mem(const char *ar_name, int ar_fd, const char *name,
- const void *data, size_t size);
+ const void *data, time_t timestamp, size_t size);
off_t dpkg_ar_member_get_size(const char *ar_name, struct ar_hdr *arh);
/** @} */
--
2.1.4
From 365f53d5adec124e327d30e2b8e3646fc2a4fc09 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Bobbio?= <lunar@debian.org>
Date: Fri, 15 Jan 2016 17:00:37 +0000
Subject: [PATCH 2/4] dpkg-deb: Use the common build timestamp for all files
created at a later time
In order to make build reproducible in the future, we now set the mtime of
archived files that has been create during the build to the common build
timestamp when tar supports the --clamp-mtime option.
The latter is available in Debian since tar/1.28-1 but has not been accepted
upstream yet.
Address: #759886
---
dpkg-deb/build.c | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++----
1 file changed, 59 insertions(+), 4 deletions(-)
diff --git a/dpkg-deb/build.c b/dpkg-deb/build.c
index 117e424..2e11e43 100644
--- a/dpkg-deb/build.c
+++ b/dpkg-deb/build.c
@@ -469,6 +469,7 @@ typedef void filenames_feed_func(const char *dir, int fd_out);
*/
static void
tarball_pack(const char *dir, filenames_feed_func *tar_filenames_feeder,
+ time_t build_timestamp, int tar_supports_clamp_mtime,
struct compress_params *tar_compress_params, int fd_out)
{
int pipe_filenames[2], pipe_tarball[2];
@@ -489,8 +490,17 @@ tarball_pack(const char *dir, filenames_feed_func *tar_filenames_feeder,
if (chdir(dir))
ohshite(_("failed to chdir to '%.255s'"), dir);
- execlp(TAR, "tar", "-cf", "-", "--format=gnu", "--null", "--no-unquote",
- "--no-recursion", "-T", "-", NULL);
+ if (tar_supports_clamp_mtime) {
+ char mtime_option[30];
+ snprintf(mtime_option, sizeof(mtime_option), "--mtime=@%ld", build_timestamp);
+ mtime_option[29] = '\0';
+ execlp(TAR, "tar", "-cf", "-", "--format=gnu", "--null", "--no-unquote",
+ mtime_option, "--clamp-mtime", "--no-recursion",
+ "-T", "-", NULL);
+ } else {
+ execlp(TAR, "tar", "-cf", "-", "--format=gnu", "--null", "--no-unquote",
+ "--no-recursion", "-T", "-", NULL);
+ }
ohshite(_("unable to execute %s (%s)"), "tar -cf", TAR);
}
close(pipe_filenames[0]);
@@ -516,6 +526,40 @@ tarball_pack(const char *dir, filenames_feed_func *tar_filenames_feeder,
}
/**
+ * Parse `tar --help` output to see if it supports the given option.
+ */
+static int
+does_tar_support(const char *option)
+{
+ int pipe_tar_output[2];
+ pid_t pid_tar;
+ FILE *tar_output;
+ char line_buf[80];
+ int found = 0;
+
+ m_pipe(pipe_tar_output);
+ pid_tar = subproc_fork();
+ if (pid_tar == 0) {
+ m_dup2(pipe_tar_output[1], 1);
+ close(pipe_tar_output[0]);
+ close(pipe_tar_output[1]);
+ execlp(TAR, "tar", "--help", NULL);
+ ohshite(_("unable to execute %s (%s)"), "tar --help", TAR);
+ }
+ close(pipe_tar_output[1]);
+ tar_output = fdopen(pipe_tar_output[0], "r");
+ while (fgets(line_buf, sizeof(line_buf), tar_output)) {
+ if (strstr(line_buf, option)) {
+ found = 1;
+ /* we don't break to consume all output and avoid getting a SIGPIPE. */
+ }
+ }
+ close(pipe_tar_output[0]);
+ subproc_reap(pid_tar, "tar --help", 0);
+ return found;
+}
+
+/**
* Overly complex function that builds a .deb file.
*/
int
@@ -530,6 +574,7 @@ do_build(const char *const *argv)
int arfd;
int p1[2], gzfd;
pid_t c1, c2;
+ int tar_supports_clamp_mtime;
time_t build_timestamp;
/* Decode our arguments. */
@@ -561,6 +606,8 @@ do_build(const char *const *argv)
}
m_output(stdout, _("<standard output>"));
+ tar_supports_clamp_mtime = does_tar_support("--clamp-mtime");
+
build_timestamp = time(NULL);
/* Now that we have verified everything its time to actually
@@ -576,7 +623,15 @@ do_build(const char *const *argv)
m_dup2(p1[1],1); close(p1[0]); close(p1[1]);
if (chdir(ctrldir))
ohshite(_("failed to chdir to '%.255s'"), ctrldir);
- execlp(TAR, "tar", "-cf", "-", "--format=gnu", "--sort=name", ".", NULL);
+ if (tar_supports_clamp_mtime) {
+ char mtime_option[30];
+ snprintf(mtime_option, sizeof(mtime_option), "--mtime=@%ld", build_timestamp);
+ mtime_option[29] = '\0';
+ execlp(TAR, "tar", "-cf", "-", "--format=gnu", "--sort=name",
+ mtime_option, "--clamp-mtime", ".", NULL);
+ } else {
+ execlp(TAR, "tar", "-cf", "-", "--format=gnu", "--sort=name", ".", NULL);
+ }
ohshite(_("unable to execute %s (%s)"), "tar -cf", TAR);
}
close(p1[1]);
@@ -671,7 +726,7 @@ do_build(const char *const *argv)
}
/* Pack the directory into a tarball, feeding files from the callback. */
- tarball_pack(dir, file_treewalk_feed, &compress_params, gzfd);
+ tarball_pack(dir, file_treewalk_feed, build_timestamp, tar_supports_clamp_mtime, &compress_params, gzfd);
/* Okay, we have data.tar as well now, add it to the ar wrapper. */
if (deb_format.major == 2) {
--
2.1.4
From eb5718a37a12093e43a4ec6df8d3576b7e6ce85d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Bobbio?= <lunar@debian.org>
Date: Thu, 8 Oct 2015 15:02:12 +0000
Subject: [PATCH 3/4] dpkg-deb: Allow to set the build timestamp using
SOURCE_DATE_EPOCH
In order to allow Debian packages to be built reproducibly, we need a way for
users to reproduce the dates captured in the .deb archives.
dpkg-deb now support the SOURCE_DATE_EPOCH environment variable. If set we will
use its value for the build timestamp instead of the current time. The build
timestamp is used to set the modification time in ar headers, and for archived
files created later than the build timestamp.
SOURCE_DATE_EPOCH specification is available at:
https://reproducible-builds.org/specs/source-date-epoch/
---
dpkg-deb/build.c | 20 +++++++++++++++++++-
1 file changed, 19 insertions(+), 1 deletion(-)
diff --git a/dpkg-deb/build.c b/dpkg-deb/build.c
index 2e11e43..c718eac 100644
--- a/dpkg-deb/build.c
+++ b/dpkg-deb/build.c
@@ -462,6 +462,24 @@ gen_dest_pathname_from_pkg(const char *dir, struct pkginfo *pkg)
arch_sep, pkg->available.arch->name, DEBEXT);
}
+static time_t
+get_build_timestamp(void)
+{
+ time_t timestamp;
+ const char *value;
+ char *end;
+
+ errno = 0;
+ value = getenv("SOURCE_DATE_EPOCH");
+ if (!value)
+ return time(NULL);
+
+ timestamp = strtol(value, &end, 10);
+ if (value == end || *end || errno != 0)
+ ohshite(_("unable to parse SOURCE_DATE_EPOCH"));
+ return timestamp;
+}
+
typedef void filenames_feed_func(const char *dir, int fd_out);
/**
@@ -608,7 +626,7 @@ do_build(const char *const *argv)
tar_supports_clamp_mtime = does_tar_support("--clamp-mtime");
- build_timestamp = time(NULL);
+ build_timestamp = get_build_timestamp();
/* Now that we have verified everything its time to actually
* build something. Let's start by making the ar-wrapper. */
--
2.1.4
From aba0bb881de2979a4469d66ad78f86a6e567628e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Bobbio?= <lunar@debian.org>
Date: Tue, 27 Aug 2013 23:53:22 +0200
Subject: [PATCH 4/4] dpkg-buildpackage: Preset build timestamp to latest
changelog entry
dpkg-buildpackage will set the SOURCE_DATE_EPOCH environment variable to the
date of the latest entry in debian/changelog. This enables build to be easily
reproduced as the dates captured in the `.deb` archives will be deterministic.
SOURCE_DATE_EPOCH can also be externaly set before running dpkg-buildpackage.
to reproduce the build with an arbitrary date.
Closes: #759886
Closes: #759999
---
scripts/dpkg-buildpackage.pl | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/scripts/dpkg-buildpackage.pl b/scripts/dpkg-buildpackage.pl
index 17ada97..e14cd26 100755
--- a/scripts/dpkg-buildpackage.pl
+++ b/scripts/dpkg-buildpackage.pl
@@ -439,6 +439,11 @@ if ($changedby) {
$maintainer = mustsetvar($changelog->{maintainer}, g_('source changed by'));
}
+if (!$ENV{SOURCE_DATE_EPOCH}) {
+ my $timestamp = `date -d'$changelog->{date}' +%s`;
+ chomp $timestamp;
+ $ENV{SOURCE_DATE_EPOCH} = $timestamp;
+}
my @arch_opts;
push @arch_opts, ('--host-arch', $host_arch) if $host_arch;
--
2.1.4
Attachment:
signature.asc
Description: Digital signature