Le vendredi 20 avril à 00h 16mn 13s (+0200), Raphael Hertzog a écrit :
>
> IMO the special casing for ports.debian.org architectures should be
> handled in a dedicated wrapper. And maybe debootstrap needs new features
> to make this wrapper possible.
>
Hi,
It would be beneficial to add a new option to debootstrap
to postprocess downloaded package indices with an external
helper program.
This would allow to enforce the policy : 'debootstrap should install
the newest version of each package'
Please find below a first try to implement it. I would be thankful
for comments, advice or suggestions of improvement.
Regards,
JH Chatenet
diff -Naur a/debootstrap/debootstrap b/debootstrap/debootstrap
--- a/debootstrap/debootstrap 2020-03-13 03:03:08.000000000 +0100
+++ b/debootstrap/debootstrap 2020-07-20 11:08:15.531224245 +0200
@@ -46,6 +46,7 @@
CHECKCERTIF=""
PRIVATEKEY=""
CACHE_DIR=""
+PACKAGES_FILES_PROCESSOR=""
DEF_MIRROR="http://deb.debian.org/debian";
DEF_HTTPS_MIRROR="https://deb.debian.org/debian";
@@ -125,6 +126,10 @@
(requires --second-stage)
--extractor=TYPE override automatic .deb extractor selection
(supported: $EXTRACTORS_SUPPORTED)
+ --packages-files-processor=HELPER
+ specifies the path to an external helper program
+ to postprocess downloaded Packages files
+ (this may be beneficial in conjunction with --extra-suites)
--debian-installer used for internal purposes by debian-installer
--private-key=file read the private key from file
--certificate=file use the client certificate stored in file (PEM)
@@ -323,6 +328,17 @@
fi
EXTRA_SUITES="$(echo "$EXTRA_SUITES" | tr , " ")"
;;
+ --packages-files-processor|--packages-files-processor=?*)
+ if [ "$1" = "--packages-files-processor" ] && [ -n "$2" ]; then
+ PACKAGES_FILES_PROCESSOR="$2"
+ shift 2
+ elif [ "$1" != "${1#--packages-files-processor=}" ]; then
+ PACKAGES_FILES_PROCESSOR="${1#--packages-files-processor=}"
+ shift 1
+ else
+ error 1 NEEDARG "option requires an argument %s" "$1"
+ fi
+ ;;
--components|--components=?*)
if [ "$1" = "--components" ] && [ -n "$2" ]; then
USE_COMPONENTS="$2"
@@ -459,6 +475,9 @@
if [ -e "$DEBOOTSTRAP_DIR/extra-suites" ]; then
EXTRA_SUITES=$(cat "$DEBOOTSTRAP_DIR/extra-suites")
fi
+ if [ -e "$DEBOOTSTRAP_DIR/extra-suites-unprocessed" ]; then
+ extra_suites_unprocessed=$(cat "$DEBOOTSTRAP_DIR/extra-suites-unprocessed")
+ fi
ARCH=$(cat "$DEBOOTSTRAP_DIR/arch")
USER_MIRROR=$(cat "$DEBOOTSTRAP_DIR/mirror")
if [ -e "$DEBOOTSTRAP_DIR/variant" ]; then
@@ -686,6 +705,8 @@
###########################################################################
+packages_files_postprocess_initialize
+
if am_doing_phase finddebs; then
if [ "$FINDDEBS_NEEDS_INDICES" = "true" ] || \
[ "$RESOLVE_DEPS" = "true" ]; then
@@ -751,6 +772,8 @@
echo "$SUITE" >"$TARGET/debootstrap/suite"
[ "" = "$EXTRA_SUITES" ] ||
echo "$EXTRA_SUITES" >"$TARGET/debootstrap/extra-suites"
+ [ "$EXTRA_SUITES" = "$extra_suites_unprocessed" ] ||
+ echo "$extra_suites_unprocessed" >"$TARGET/debootstrap/extra-suites-unprocessed"
[ "" = "$VARIANT" ] ||
echo "$VARIANT" >"$TARGET/debootstrap/variant"
echo "$required" >"$TARGET/debootstrap/required"
@@ -786,8 +809,10 @@
cp "$DEBOOTSTRAP_DIR"/scripts/*-common "$TARGET/debootstrap/"
echo "$ARCH" >"$TARGET/debootstrap/arch"
echo "$SUITE" >"$TARGET/debootstrap/suite"
- [ "" = "$EXTRA_SUITES" ] ||
- echo "$EXTRA_SUITES" >"$TARGET/debootstrap/extra-suites"
+ if [ -n "$EXTRA_SUITES" ]; then
+ echo "$EXTRA_SUITES" >"$TARGET/debootstrap/extra-suites"
+ echo "$extra_suites_unprocessed" >"$TARGET/debootstrap/extra-suites-unprocessed"
+ fi
echo "$USER_MIRROR" >"$TARGET/debootstrap/mirror"
[ "" = "$VARIANT" ] ||
echo "$VARIANT" >"$TARGET/debootstrap/variant"
diff -Naur a/debootstrap/debootstrap.8 b/debootstrap/debootstrap.8
--- a/debootstrap/debootstrap.8 2020-03-09 06:21:13.000000000 +0100
+++ b/debootstrap/debootstrap.8 2020-07-20 11:20:58.187006051 +0200
@@ -63,6 +63,10 @@
.IP "\fB\-\-extra-suites=alpha,beta\fP"
Also use packages from the listed suites of the archive.
.IP
+.IP "\fB\-\-packages-files-processor=HELPER\fP"
+Specifies the path to an external helper program to postprocess downloaded
+Packages files. This may be beneficial in conjunction with --extra-suites.
+.IP
.IP "\fB\-\-components=alpha,beta\fP"
Use packages from the listed components of the archive.
.IP
diff -Naur a/debootstrap/functions b/debootstrap/functions
--- a/debootstrap/functions 2020-03-14 01:53:38.000000000 +0100
+++ b/debootstrap/functions 2020-07-20 11:16:16.485609169 +0200
@@ -728,10 +728,16 @@
if [ ! -f "$pkgdest" ]; then
error 1 COULDNTDL "Couldn't download %s" "$m/$path"
fi
+
+ packages_files_postprocess_accumulate "$pkgdest"
+
donepkgs=$(($donepkgs + ${i#* }))
progress $donepkgs $totalpkgs DOWNPKGS "Downloading Packages files"
done
done
+
+ packages_files_postprocess
+
}
get_package_sizes () {
@@ -807,7 +813,7 @@
leftoverdebs=$(printf "$leftoverdebs"|tr ' ' '\n'|sort -u|tr '\n' ' ')
numdebs=$(printf "$leftoverdebs"|wc -w)
- for s in $SUITE $EXTRA_SUITES; do
+ for s in $SUITE $extra_suites_unprocessed; do
for c in $COMPONENTS; do
if [ "$countdebs" -ge "$numdebs" ]; then break; fi
@@ -839,7 +845,7 @@
:>"$TARGET/debootstrap/debpaths"
pkgs_to_get="$*"
- for s in $SUITE $EXTRA_SUITES; do
+ for s in $SUITE $extra_suites_unprocessed; do
for c in $COMPONENTS; do
path="dists/$s/$c/binary-$ARCH/Packages"
for m in $MIRRORS; do
@@ -880,9 +886,16 @@
elif get "$m/$path" "$pkgdest"; then
true
fi
+
+ if [ -f "$pkgdest" ]; then
+ packages_files_postprocess_accumulate "$pkgdest"
+ fi
done
done
done
+
+ packages_files_postprocess
+
progress 100 100 DOWNMAINPKGS "Downloading Packages file"
}
@@ -892,7 +905,7 @@
:>"$TARGET/debootstrap/debpaths"
for p in "$@"; do
- for s in $SUITE $EXTRA_SUITES; do
+ for s in $SUITE $extra_suites_unprocessed; do
for c in $COMPONENTS; do
local details=""
for m in $MIRRORS; do
@@ -934,7 +947,7 @@
field="$1"
shift
for m1 in $MIRRORS; do
- for s in $SUITE $EXTRA_SUITES; do
+ for s in $SUITE $extra_suites_unprocessed; do
for c in $COMPONENTS; do
path="dists/$s/$c/binary-$ARCH/Packages"
pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")"
@@ -1532,7 +1545,7 @@
local ALLPKGS2="";
while [ "$PKGS" != "" ]; do
local NEWPKGS=""
- for s in $SUITE $EXTRA_SUITES; do
+ for s in $SUITE $extra_suites_unprocessed; do
for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do
local path="dists/$s/$c/binary-$ARCH/Packages"
local pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")"
@@ -1541,7 +1554,7 @@
done
PKGS=$(echo "$PKGS $NEWPKGS" | tr ' ' '\n' | sort | uniq)
local REALPKGS=""
- for s in $SUITE $EXTRA_SUITES; do
+ for s in $SUITE $extra_suites_unprocessed; do
for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do
local path="dists/$s/$c/binary-$ARCH/Packages"
local pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")"
@@ -1560,7 +1573,7 @@
local m1 c path pkgdest pkg
m1="${MIRRORS%% *}"
- for s in $SUITE $EXTRA_SUITES; do
+ for s in $SUITE $extra_suites_unprocessed; do
for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do
path="dists/$s/$c/binary-$ARCH/Packages"
pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")"
@@ -1881,3 +1894,76 @@
fakechroot" >> "$TARGET/var/lib/dpkg/diversions"
}
+
+############################################################### 'Packages' files postprocessing
+
+packages_files_postprocess_initialize () {
+
+ extra_suites_unprocessed="$EXTRA_SUITES"
+ postprocess_packages_files=
+
+ if [ -n "$PACKAGES_FILES_PROCESSOR" ]; then
+
+ if [ -n "$UNPACK_TARBALL" ]; then
+ warning TARBALL_PROC "option --packages-files-processor is ignored when unpacking a tarball"
+ elif [ -x "$PACKAGES_FILES_PROCESSOR" ]; then
+ postprocess_packages_files=true
+ else
+ error 1 NO_PKGPROC "Packages files postprocessor %s not found or not executable" "$PACKAGES_FILES_PROCESSOR"
+ fi
+ fi
+
+ if [ -n "$UNPACK_TARBALL" ] && [ -e "$TARGET/debootstrap/extra-suites-unprocessed" ]; then
+ extra_suites_unprocessed=$(cat "$TARGET/debootstrap/extra-suites-unprocessed")
+ info UNPROCESSED_SET "using extra_suites_unprocessed=%s" "$extra_suites_unprocessed"
+ fi
+
+}
+
+packages_files_postprocess_accumulate () {
+
+ if [ -n "$postprocess_packages_files" ]; then
+ local src dst path
+ src="$1"
+ path="dists/$SUITE/$c/binary-$ARCH/Packages"
+ dst="$TARGET/$($DLDEST pkg "$SUITE" "$c" "$ARCH" "$m" "$path").to_process"
+
+ cat "$src" >> "$dst"
+ fi
+}
+
+
+packages_files_postprocess () {
+
+ if [ -n "$postprocess_packages_files" ]; then
+ local m c src dst path
+
+ for m in $MIRRORS; do
+ for c in $COMPONENTS; do
+ path="dists/$SUITE/$c/binary-$ARCH/Packages"
+ dst="$TARGET/$($DLDEST pkg "$SUITE" "$c" "$ARCH" "$m" "$path")"
+ src="${dst}.to_process"
+
+ # Do nothing on empty Packages files
+ if [ -s "$src" ]; then
+ "$PACKAGES_FILES_PROCESSOR" "$src" > "${dst}.processed"
+ ret=$?
+ if [ $ret -ne 0 ] || [ ! -s "${dst}.processed" ]; then
+ error 1 COULDNTPROC "Couldn't postprocess %s: ret code=%s" "$dst" "$ret"
+ fi
+
+ mv "$dst" "${dst}.unprocessed"
+ mv "${dst}.processed" "$dst"
+ fi
+
+ if [ -f "$src" ]; then
+ # Cleaning up (required after --download-only)
+ rm -f "$src"
+ fi
+ done
+ done
+ # all packages appear in the Packages file of $SUITE after postprocessing
+ # there is no need to consider other Packages files
+ extra_suites_unprocessed=
+ fi
+}
Attachment:
README.debian-ports.gz
Description: application/gzip