Re: support-snapshots.debian.org-in-live-build
On Mon, Jun 18, 2012 at 06:15:19PM +0200, Daniel Baumann wrote:
> On 06/18/2012 03:08 PM, Rui Miguel P. Bernardo wrote:
> > I'm _proposing_ this patch to support snapshots.debian.org in live-build.
>
> good idea, thanks for comming up with it. i'm right now a bit busy, so
> i'll look at it later this week (wed or thu) and provide some feedback.
Following today irc talk here is a fresh patch against debian-next.
This removes the debian-snapshots mode and uses only standard options.
Quick example:
lb config -d wheezy \
--debian-snapshots true \
--debian-snapshot-date 20120101 \
--debian-installer live \
--debian-installer-distribution daily
Should build a debian wheezy live image iso as testing/wheezy was in
January and have sources.list entries set to snapshosts.debian-org. To
override the binary sources.list snapshots entries, setting
LB_MIRROR_BINARY should be enough. In installed systems with daily d-i
the sources.list entries will be set by debian-installer as usual and
track latest wheezy repos (not snapshots.debian.org).
NOTE: in the live image, after booting, the user still needs to change
apt.conf to be able to run "apt-get update". See http://snapshot.debian.org/.
We could add an option resolve this but that is the expected behaviour.
Maybe the wget outputs could be muted to reduce noise?
Maybe --debian-snapshots true could be dropped and if
--debian-shapshots-date is set then use snapshots.debian.org?
>From 51045d2748a49a43212d9d9d6b615065bd1fb15c Mon Sep 17 00:00:00 2001
From: "Rui Miguel P. Bernardo" <rui.bernardo.pt@gmail.com>
Date: Sat, 16 Jun 2012 16:00:11 +0100
Subject: [PATCH] support snapshots.debian.org in live-build
---
functions/debian-snapshots.sh | 65 ++++++++++++++++++++++++++++++
functions/defaults.sh | 38 +++++++++++++++++
scripts/build/lb_binary_debian-installer | 5 +++
scripts/build/lb_chroot_apt | 10 +++++
scripts/build/lb_config | 22 +++++++++-
5 files changed, 139 insertions(+), 1 deletion(-)
create mode 100644 functions/debian-snapshots.sh
diff --git a/functions/debian-snapshots.sh b/functions/debian-snapshots.sh
new file mode 100644
index 0000000..786d8d5
--- /dev/null
+++ b/functions/debian-snapshots.sh
@@ -0,0 +1,65 @@
+#!/bin/sh
+
+## live-build(7) - System Build Scripts
+## Copyright (C) 2006-2012 Daniel Baumann <daniel@debian.org>
+##
+## live-build comes with ABSOLUTELY NO WARRANTY; for details see COPYING.
+## This is free software, and you are welcome to redistribute it
+## under certain conditions; see COPYING for details.
+
+Short_date ()
+{
+ SHORT_DATE="${1}:-${LB_DEBIAN_SNAPSHOT_DATE}}"
+ # returns the date as YYYYMMDD
+ echo "${SHORT_DATE}" | cut -c 1-8
+}
+
+Last_archive_entry ()
+{
+ # returns the last date link
+ LAST_ENTRY="$(cat /tmp/index.html | grep "$(date +%Y%m)" | sed 's|.*<a href="\([^"]*\)".*$|\1|' | tail -n 1 )"
+ echo "$( echo ${LAST_ENTRY} | sed 's|./||' | sed 's|/||' )"
+}
+
+Dated_archive_entry ()
+{
+ # returns the link for a specific date
+ DATED_ENTRY="$(cat /tmp/index.html | grep "$(Short_date ${LB_DEBIAN_SNAPSHOT_DATE})" | sed 's|.*<a href="\([^"]*\)".*$|\1|' | tail -n 1 )"
+ echo "$( echo ${DATED_ENTRY} | sed 's|./||' | sed 's|/||' )"
+}
+
+Latest_debian_archive_snapshot_available ()
+{
+ # returns the complete date/time for the link of the latest (last) available debian archive snapshot date
+ wget 'http://snapshot.debian.org/archive/debian/?year='"$(date +%Y)"';month='"$(date +%m)" -O /tmp/index.html && true
+ LAST_ARCHIVE_SNAPSHOT="$(Last_archive_entry)"
+ echo "${LAST_ARCHIVE_SNAPSHOT}"
+}
+
+Dated_debian_archive_snapshot ()
+{
+ # returns the complete date/time for the link of the latest for a specific date in snapshots.debian.org
+ DATE_YEAR="$(Short_date ${LB_DEBIAN_SNAPSHOT_DATE} | cut -c 1-4 )"
+ DATE_MONTH="$(Short_date ${LB_DEBIAN_SNAPSHOT_DATE} | cut -c 5-6 )"
+ wget 'http://snapshot.debian.org/archive/debian/?year='"${DATE_YEAR}"';month='"${DATE_MONTH}" -O /tmp/index.html && true
+ LAST_ARCHIVE_SNAPSHOT="$(Dated_archive_entry)"
+ echo "${LAST_ARCHIVE_SNAPSHOT}"
+}
+
+Latest_debian_installer_snapshot_available ()
+{
+ # returns the date-hour for the latest date of debian-installer daily build available for an arch
+ # d-i archive uses different date links
+ wget 'http://d-i.debian.org/daily-images/'"${LB_ARCHITECTURES}" -O /tmp/index.html && true
+ LAST_INSTALLER_SNAPSHOT="$(Last_archive_entry)"
+ echo "${LAST_INSTALLER_SNAPSHOT}"
+}
+
+Dated_debian_installer_snapshot ()
+{
+ # returns the date-hour for the specific date of debian-installer daily build available for an arch
+ # d-i archive uses different date links
+ wget 'http://d-i.debian.org/daily-images/'"${LB_ARCHITECTURES}" -O /tmp/index.html && true
+ DATED_INSTALLER_SNAPSHOT="$(Dated_archive_entry)"
+ echo "${DATED_INSTALLER_SNAPSHOT}"
+}
diff --git a/functions/defaults.sh b/functions/defaults.sh
index 5a55503..35a9db6 100755
--- a/functions/defaults.sh
+++ b/functions/defaults.sh
@@ -381,6 +381,28 @@ Set_defaults ()
# Setting bootstrap keyring
# LB_BOOTSTRAP_KEYRING
+ # Setting snapshots.debian.org
+ LB_DEBIAN_SNAPSHOTS="${LB_DEBIAN_SNAPSHOTS:-false}"
+ if [ "${LB_DEBIAN_SNAPSHOTS}" = "true" ]
+ then
+ if [ -z "${LB_DEBIAN_SNAPSHOT_DATE}" ]
+ then
+ # if not set, use latest debian archive snapshot
+ Echo_warning "debian-snapshots option enabled, but no debian-snapshot-date was set, assuming today latest."
+ LB_DEBIAN_SNAPSHOT_DATE="$(Latest_debian_archive_snapshot_available)"
+ Echo_warning " The latest debian archive snapshot available for ${LB_ARCHITECTURES} is ${LB_DEBIAN_SNAPSHOT_DATE}."
+ fi
+ if [ -n "${LB_DEBIAN_SNAPSHOT_DATE}" ]
+ then
+ LB_MIRROR_BOOTSTRAP="${LB_MIRROR_BOOTSTRAP:-http://snapshot.debian.org/archive/debian/${LB_DEBIAN_SNAPSHOT_DATE}}";
+ LB_PARENT_MIRROR_BOOTSTRAP="${LB_MIRROR_BOOTSTRAP}"
+ LB_MIRROR_CHROOT_SECURITY="${LB_MIRROR_CHROOT_SECURITY:-http://snapshot.debian.org/archive/debian-security/${LB_DEBIAN_SNAPSHOT_DATE}}";
+ LB_PARENT_MIRROR_CHROOT_SECURITY="${LB_PARENT_MIRROR_CHROOT_SECURITY:-${LB_MIRROR_CHROOT_SECURITY}}"
+ LB_MIRROR_BINARY="${LB_MIRROR_BINARY:-http://snapshot.debian.org/archive/debian/${LB_DEBIAN_SNAPSHOT_DATE}}";
+ LB_PARENT_MIRROR_BINARY="${LB_PARENT_MIRROR_BINARY:-${LB_MIRROR_BINARY}}"
+ fi
+ fi
+
# Setting mirror to fetch packages from
case "${LB_MODE}" in
debian)
@@ -1257,6 +1279,22 @@ Check_defaults ()
fi
fi
+ if [ "${LB_DEBIAN_INSTALLER}" = "live" ] && [ "${LB_DEBIAN_SNAPSHOTS}" = "true" ] && [ -n "${LB_DEBIAN_SNAPSHOT_DATE}" ] && [ "${LB_DEBIAN_INSTALLER_DISTRIBUTION}" = "daily" ]
+ then
+ # FIXME: this doesn't actually check if the debian
+ # installer had a successfull build for the selected
+ # image type in that date, but only that the build
+ # directory exists. See if selected arch is present in
+ # http://d-i.debian.org/daily-images/. Only some dates
+ # are preseved in the archive.
+ DAILY_LIVE_INSTALLER="$(Dated_debian_installer_snapshot)"
+ if [ -z "${DAILY_LIVE_INSTALLER}" ]
+ then
+ Echo_error "You have selected debian-snapshots option with daily live-installer, but there is no daily live installer available at that date. The selected date was ${LB_DEBIAN_SNAPSHOT_DATE}"
+ exit 1
+ fi
+ fi
+
if [ "${LB_BOOTLOADER}" = "syslinux" ]
then
# syslinux + fat
diff --git a/scripts/build/lb_binary_debian-installer b/scripts/build/lb_binary_debian-installer
index 003051b..1fbc4e7 100755
--- a/scripts/build/lb_binary_debian-installer
+++ b/scripts/build/lb_binary_debian-installer
@@ -261,6 +261,11 @@ case "${LB_DERIVATIVE}" in
# Debian Installer daily builds
URL="http://d-i.debian.org/daily-images/${LB_ARCHITECTURES}/daily/";
+ # use snapshot installer too
+ if [ -n "$LB_DEBIAN_SNAPSHOTS_DATE" ]
+ then
+ URL="http://d-i.debian.org/daily-images/${LB_ARCHITECTURES}/$(Dated_debian_installer_snapshot)/"
+ fi
else
URL="${LB_MIRROR_DEBIAN_INSTALLER}/dists/${LB_DEBIAN_INSTALLER_DISTRIBUTION}/main/installer-${LB_ARCHITECTURES}/current/images/"
fi
diff --git a/scripts/build/lb_chroot_apt b/scripts/build/lb_chroot_apt
index 710b2e0..70b649d 100755
--- a/scripts/build/lb_chroot_apt
+++ b/scripts/build/lb_chroot_apt
@@ -54,6 +54,13 @@ case "${1}" in
echo "Acquire::http::Proxy \"${LB_APT_HTTP_PROXY}\";" > chroot/etc/apt/apt.conf.d/00http-proxy
fi
+ # Configuring snapshots.debian.org
+ if [ "${LB_DEBIAN_SNAPSHOTS}" = "true" ] && [ -n "${LB_DEBIAN_SNAPSHOT_DATE}" ]
+ then
+ echo '// see bug #595801' > chroot/etc/apt/apt.conf.d/00debian-snapshots
+ echo 'Acquire::Check-Valid-Until "false";' >> chroot/etc/apt/apt.conf.d/00debian-snapshots
+ fi
+
# Configuring apt pipeline
if [ -n "${LB_APT_PIPELINE}" ]
then
@@ -240,6 +247,9 @@ EOF
# Deconfiguring aptitude http proxy
rm -f chroot/etc/apt/apt.conf.d/00http-proxy
+ # Deconfiguring snapshots.debian.org
+ rm -f chroot/etc/apt/apt.conf.d/00debian-snapshots
+
# Deconfiguring aptitude pipeline
rm -f chroot/etc/apt/apt.conf.d/00pipeline
diff --git a/scripts/build/lb_config b/scripts/build/lb_config
index f1ae30c..fa5e984 100755
--- a/scripts/build/lb_config
+++ b/scripts/build/lb_config
@@ -66,6 +66,8 @@ USAGE="${PROGRAM} [--apt apt|aptitude]\n\
\t [--debian-installer-distribution daily|CODENAME]\n\
\t [--debian-installer-preseedfile FILE|URL]\n\
\t [--debian-installer-gui true|false]\n\
+\t [--debian-snapshots true|false]\n\
+\t [--debian-snapshot-date yyyymmdd|yyyymmddThhmmssZ\n\
\t [--debug]\n\
\t [-d|--distribution CODENAME]\n\
\t [--parent-distribution CODENAME]\n\
@@ -175,7 +177,7 @@ Local_arguments ()
net-cow-server:,net-tarball:,firmware-binary:,firmware-chroot:,swap-file-path:,swap-file-size:,syslinux-theme:,
win32-loader:,source:,source-images:,breakpoints,conffile:,debug,force,
help,ignore-system-defaults,quiet,usage,verbose,version,bootstrap-qemu-static:,bootstrap-qemu-arch:,
- bootstrap-qemu-exclude:"
+ bootstrap-qemu-exclude:,debian-snapshots:,debian-snapshot-date:"
# Remove spaces added by indentation
LONG_OPTIONS="$(echo ${LONG_OPTIONS} | tr -d ' ')"
ARGUMENTS="$(getopt --longoptions ${LONG_OPTIONS} --name="${PROGRAM}" --options a:f:d:m:l:k:p:b:e:s:c:huv --shell sh -- "${@}")"
@@ -323,6 +325,16 @@ Local_arguments ()
shift 2
;;
+ --debian-snapshots)
+ LB_DEBIAN_SNAPSHOTS="${2}"
+ shift 2
+ ;;
+
+ --debian-snapshot-date)
+ LB_DEBIAN_SNAPSHOT_DATE="${2}"
+ shift 2
+ ;;
+
--initramfs)
LB_INITRAMFS="${2}"
shift 2
@@ -983,6 +995,14 @@ LB_DEBCONF_NOWARNINGS="${LB_DEBCONF_NOWARNINGS}"
# (Default: ${LB_DEBCONF_PRIORITY})
LB_DEBCONF_PRIORITY="${LB_DEBCONF_PRIORITY}"
+# \$LB_DEBIAN_SNAPSHOTS: use debian-snapshots repos
+# (Default: false)
+LB_DEBIAN_SNAPSHOTS="${LB_DEBIAN_SNAPSHOTS}"
+
+# \$LB_DEBIAN_SNAPSHOT_DATE: set debian-snapshots date to use
+# (Default: empty to use latest archive snapshot, or a valid date in snapshots.debian.org URL)
+LB_DEBIAN_SNAPSHOT_DATE="${LB_DEBIAN_SNAPSHOT_DATE}"
+
# \$LB_INITRAMFS: set initramfs hook
# (Default: ${LB_INITRAMFS})
LB_INITRAMFS="${LB_INITRAMFS}"
--
1.7.10
Reply to: