Bug#1035567: debootstrap: Does not support APT repositories with modern split arch:all support
Package: debootstrap
X-Debbugs-Cc: mak@debian.org
Version: 1.0.128+nmu2
Severity: normal
Tags: patch
Dear Maintainer,
currently debootstrap fails to work with APT repositories that have
arch:all split out exclusively into a separate Packages file, instead
of having arch:all merged into every arch:any Packages file in archive
metadata.
APT has already been supporting the new scheme for multiple releases,
so it would be nice if debootstrap supported it too (so far, it is the
only tool I found that doesn't do that). This would enable us in
PureOS to use the new APT repository style, as well as potentially
other derivatives and Debian itself in future as well.
The APT repository format (for both styles and the current
"compatibility mode" that the Debian archive runs on) is described
here: https://wiki.debian.org/DebianRepository/Format#Architectures
I attached a patch which implements support for this feature in
debootstrap. So far it has been working find for bootstrapping both
Debian (with the "compatibility" archive metadata mode) as well as a
PureOS scratch repository which uses the newer-style APT repository
mode.
You can also review this patch as MR @ Salsa:
https://salsa.debian.org/installer-team/debootstrap/-/merge_requests/92
Thank you for considering!
I've set this bug as "normal", since I do consider bringing
debootstrap on-par with what features APT supports for a while as
important, but feel free to reduce it to "whishlist" priority if you
think that is more appropriate for a feature request like this.
With kind regards,
Matthias Klumpp
--
I welcome VSRE emails. See http://vsre.info/
From 5dd4cef1a5a03dc180eb2ec1d2bfc97023f74d2b Mon Sep 17 00:00:00 2001
From: Matthias Klumpp <mak@debian.org>
Date: Sun, 23 Apr 2023 22:30:02 +0200
Subject: [PATCH] Implement support for repos with modern-style arch:all
support
This implements support for modern APT repositories which have arch:any
and arch:all packages in separate Packages files, as outlined in the
Debian
repository documentation:
https://wiki.debian.org/DebianRepository/Format#Architectures
---
functions | 274 +++++++++++++++++++++++++++++++++---------------------
1 file changed, 168 insertions(+), 106 deletions(-)
diff --git a/functions b/functions
index 0ff5379..8bffbc6 100644
--- a/functions
+++ b/functions
@@ -557,6 +557,24 @@ extract_release_components () {
fi
}
+repo_supports_arch_all () {
+ local a no_arch_all_support
+ local reldest="$1"; shift
+ TMPARCHS="$(sed -n 's/Architectures: *//p' "$reldest")"
+ ARCH_ALL_SUPPORTED=0
+ for a in $TMPARCHS ; do
+ if [ "$a" = "all" ]; then
+ ARCH_ALL_SUPPORTED=1
+ break
+ fi
+ done
+
+ no_arch_all_support=$(grep "^No-Support-for-Architecture-all: Packages$" "$reldest" || true)
+ if [ "$no_arch_all_support" != "" ]; then
+ ARCH_ALL_SUPPORTED=0
+ fi
+}
+
CODENAME=""
validate_suite () {
local reldest suite s
@@ -667,7 +685,7 @@ download_release_sig () {
download_release_indices () {
local m1 inreldest reldest relsigdest totalpkgs \
subpath xzi bz2i gzi normi i ext \
- donepkgs pkgdest acquirebyhash s c m
+ donepkgs pkgdest acquirebyhash archs s c a m
m1="${MIRRORS%% *}"
for s in $SUITE $EXTRA_SUITES; do
inreldest="$TARGET/$($DLDEST rel "$s" "$m1" "dists/$s/InRelease")"
@@ -680,70 +698,80 @@ download_release_indices () {
extract_release_components "$reldest"
+ repo_supports_arch_all "$reldest"
+
+ archs="$ARCH"
+ if [ $ARCH_ALL_SUPPORTED -eq 1 ]; then
+ archs="all $ARCH"
+ fi
+
acquirebyhash=$(grep "^Acquire-By-Hash: yes$" "$reldest" || true)
- totalpkgs=0
- for c in $COMPONENTS; do
- subpath="$c/binary-$ARCH/Packages"
- xzi="$(get_release_checksum "$reldest" "$subpath.xz")"
- bz2i="$(get_release_checksum "$reldest" "$subpath.bz2")"
- gzi="$(get_release_checksum "$reldest" "$subpath.gz")"
- normi="$(get_release_checksum "$reldest" "$subpath")"
- if [ "$normi" != "" ]; then
- i="$normi"
- elif in_path bunzip2 && [ "$bz2i" != "" ]; then
- i="$bz2i"
- elif in_path unxz && [ "$xzi" != "" ]; then
- i="$xzi"
- elif in_path gunzip && [ "$gzi" != "" ]; then
- i="$gzi"
- fi
- if [ "$i" != "" ]; then
- totalpkgs=$(( $totalpkgs + ${i#* } ))
- else
- mv "$reldest" "$reldest.malformed"
- error 1 MISSINGRELENTRY "Invalid Release file, no entry for %s" "$subpath"
- fi
- done
- donepkgs=0
- progress 0 $totalpkgs DOWNPKGS "Downloading Packages files"
- for c in $COMPONENTS; do
- subpath="$c/binary-$ARCH/Packages"
- path="dists/$s/$subpath"
- xzi="$(get_release_checksum "$reldest" "$subpath.xz")"
- bz2i="$(get_release_checksum "$reldest" "$subpath.bz2")"
- gzi="$(get_release_checksum "$reldest" "$subpath.gz")"
- normi="$(get_release_checksum "$reldest" "$subpath")"
- ext=""
- if [ "$acquirebyhash" != "" ]; then
- ext="$ext byhash"
- fi
- if [ "$normi" != "" ]; then
- ext="$ext $normi ."
- i="$normi"
- fi
- if in_path unxz && [ "$xzi" != "" ]; then
- ext="$ext $xzi xz"
- i="${i:-$xzi}"
- fi
- if in_path bunzip2 && [ "$bz2i" != "" ]; then
- ext="$ext $bz2i bz2"
- i="${i:-$bz2i}"
- fi
- if in_path gunzip && [ "$gzi" != "" ]; then
- ext="$ext $gzi gz"
- i="${i:-$gzi}"
- fi
- progress_next $(($donepkgs + ${i#* }))
- for m in $MIRRORS; do
- pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m" "$path")"
- if get "$m/$path" "$pkgdest" $ext; then break; fi
+ for a in $archs; do
+ totalpkgs=0
+ for c in $COMPONENTS; do
+ subpath="$c/binary-$a/Packages"
+ xzi="$(get_release_checksum "$reldest" "$subpath.xz")"
+ bz2i="$(get_release_checksum "$reldest" "$subpath.bz2")"
+ gzi="$(get_release_checksum "$reldest" "$subpath.gz")"
+ normi="$(get_release_checksum "$reldest" "$subpath")"
+ if [ "$normi" != "" ]; then
+ i="$normi"
+ elif in_path bunzip2 && [ "$bz2i" != "" ]; then
+ i="$bz2i"
+ elif in_path unxz && [ "$xzi" != "" ]; then
+ i="$xzi"
+ elif in_path gunzip && [ "$gzi" != "" ]; then
+ i="$gzi"
+ fi
+ if [ "$i" != "" ]; then
+ totalpkgs=$(( $totalpkgs + ${i#* } ))
+ else
+ mv "$reldest" "$reldest.malformed"
+ error 1 MISSINGRELENTRY "Invalid Release file, no entry for %s" "$subpath"
+ fi
+ done
+
+ donepkgs=0
+ progress 0 $totalpkgs DOWNPKGS "Downloading Packages files for $a"
+ for c in $COMPONENTS; do
+ subpath="$c/binary-$a/Packages"
+ path="dists/$s/$subpath"
+ xzi="$(get_release_checksum "$reldest" "$subpath.xz")"
+ bz2i="$(get_release_checksum "$reldest" "$subpath.bz2")"
+ gzi="$(get_release_checksum "$reldest" "$subpath.gz")"
+ normi="$(get_release_checksum "$reldest" "$subpath")"
+ ext=""
+ if [ "$acquirebyhash" != "" ]; then
+ ext="$ext byhash"
+ fi
+ if [ "$normi" != "" ]; then
+ ext="$ext $normi ."
+ i="$normi"
+ fi
+ if in_path unxz && [ "$xzi" != "" ]; then
+ ext="$ext $xzi xz"
+ i="${i:-$xzi}"
+ fi
+ if in_path bunzip2 && [ "$bz2i" != "" ]; then
+ ext="$ext $bz2i bz2"
+ i="${i:-$bz2i}"
+ fi
+ if in_path gunzip && [ "$gzi" != "" ]; then
+ ext="$ext $gzi gz"
+ i="${i:-$gzi}"
+ fi
+ progress_next $(($donepkgs + ${i#* }))
+ for m in $MIRRORS; do
+ pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m" "$path")"
+ if get "$m/$path" "$pkgdest" $ext; then break; fi
+ done
+ if [ ! -f "$pkgdest" ]; then
+ error 1 COULDNTDL "Couldn't download %s" "$m/$path"
+ fi
+ donepkgs=$(($donepkgs + ${i#* }))
+ progress $donepkgs $totalpkgs DOWNPKGS "Downloading Packages files"
done
- if [ ! -f "$pkgdest" ]; then
- error 1 COULDNTDL "Couldn't download %s" "$m/$path"
- fi
- donepkgs=$(($donepkgs + ${i#* }))
- progress $donepkgs $totalpkgs DOWNPKGS "Downloading Packages files"
done
done
}
@@ -807,7 +835,7 @@ download_debs () {
download_release () {
local m1 numdebs countdebs totaldebs leftoverdebs path pkgdest \
- dloaddebs s c m
+ dloaddebs archs s c a m
m1="${MIRRORS%% *}"
numdebs="$#"
@@ -822,25 +850,32 @@ download_release () {
leftoverdebs=$(printf "$leftoverdebs"|tr ' ' '\n'|sort -u|tr '\n' ' ')
numdebs=$(printf "$leftoverdebs"|wc -w)
+ archs="$ARCH"
+ if [ $ARCH_ALL_SUPPORTED -eq 1 ]; then
+ archs="all $ARCH"
+ fi
+
for s in $SUITE $EXTRA_SUITES; do
- for c in $COMPONENTS; do
- if [ "$countdebs" -ge "$numdebs" ]; then break; fi
+ for a in $archs; do
+ for c in $COMPONENTS; do
+ if [ "$countdebs" -ge "$numdebs" ]; then break; fi
- path="dists/$s/$c/binary-$ARCH/Packages"
- pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")"
- if [ ! -e "$pkgdest" ]; then continue; fi
+ path="dists/$s/$c/binary-$a/Packages"
+ pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m1" "$path")"
+ if [ ! -e "$pkgdest" ]; then continue; fi
- info CHECKINGSIZES "Checking component %s on %s..." "$c" "$m1"
+ info CHECKINGSIZES "Checking component %s on %s..." "$c" "$m1"
- leftoverdebs="$(get_package_sizes "$m1" "$pkgdest" $leftoverdebs)"
+ leftoverdebs="$(get_package_sizes "$m1" "$pkgdest" $leftoverdebs)"
- countdebs=$(($countdebs + ${leftoverdebs%% *}))
- leftoverdebs=${leftoverdebs#* }
+ countdebs=$(($countdebs + ${leftoverdebs%% *}))
+ leftoverdebs=${leftoverdebs#* }
- totaldebs=${leftoverdebs%% *}
- leftoverdebs=${leftoverdebs#* }
+ totaldebs=${leftoverdebs%% *}
+ leftoverdebs=${leftoverdebs#* }
- progress "$countdebs" "$numdebs" SIZEDEBS "Finding package sizes"
+ progress "$countdebs" "$numdebs" SIZEDEBS "Finding package sizes"
+ done
done
done
@@ -855,15 +890,17 @@ download_release () {
pkgs_to_get="$*"
for s in $SUITE $EXTRA_SUITES; do
- for c in $COMPONENTS; do
- path="dists/$s/$c/binary-$ARCH/Packages"
- for m in $MIRRORS; do
- pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m" "$path")"
- if [ ! -e "$pkgdest" ]; then continue; fi
- pkgs_to_get="$(download_debs "$m" "$pkgdest" $pkgs_to_get 5>&1 1>&6)"
+ for a in $archs; do
+ for c in $COMPONENTS; do
+ path="dists/$s/$c/binary-$a/Packages"
+ for m in $MIRRORS; do
+ pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m" "$path")"
+ if [ ! -e "$pkgdest" ]; then continue; fi
+ pkgs_to_get="$(download_debs "$m" "$pkgdest" $pkgs_to_get 5>&1 1>&6)"
+ if [ -z "$pkgs_to_get" ]; then break; fi
+ done 6>&1
if [ -z "$pkgs_to_get" ]; then break; fi
- done 6>&1
- if [ -z "$pkgs_to_get" ]; then break; fi
+ done
done
if [ -z "$pkgs_to_get" ]; then break; fi
done
@@ -947,15 +984,23 @@ download_main () {
###################################################### deb choosing support
get_debs () {
- local field m1 s c path pkgdest
+ local field m1 s c a path pkgdest
field="$1"
shift
+
+ archs="$ARCH"
+ if [ $ARCH_ALL_SUPPORTED -eq 1 ]; then
+ archs="all $ARCH"
+ fi
+
for m1 in $MIRRORS; do
for s in $SUITE $EXTRA_SUITES; do
for c in $COMPONENTS; do
- path="dists/$s/$c/binary-$ARCH/Packages"
- pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")"
- echo "$("$PKGDETAILS" FIELD "$field" "$m1" "$pkgdest" "$@" | sed 's/ .*//')"
+ for a in $archs; do
+ path="dists/$s/$c/binary-$a/Packages"
+ pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m1" "$path")"
+ echo "$("$PKGDETAILS" FIELD "$field" "$m1" "$pkgdest" "$@" | sed 's/ .*//')"
+ done
done
done
done
@@ -1587,14 +1632,22 @@ resolve_deps () {
local PKGS="$*"
local ALLPKGS="$PKGS";
local ALLPKGS2="";
- local s c
+ local s c a
+
+ archs="$ARCH"
+ if [ $ARCH_ALL_SUPPORTED -eq 1 ]; then
+ archs="all $ARCH"
+ fi
+
while [ "$PKGS" != "" ]; do
local NEWPKGS=""
- for s in $SUITE $EXTRA_SUITES; do
- for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do
- local path="dists/$s/$c/binary-$ARCH/Packages"
- local pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")"
- NEWPKGS="$NEWPKGS $("$PKGDETAILS" GETDEPS "$pkgdest" $PKGS)"
+ for a in $archs; do
+ for s in $SUITE $EXTRA_SUITES; do
+ for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do
+ local path="dists/$s/$c/binary-$a/Packages"
+ local pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m1" "$path")"
+ NEWPKGS="$NEWPKGS $("$PKGDETAILS" GETDEPS "$pkgdest" $PKGS)"
+ done
done
done
if [ -n "${EXCLUDE_DEPENDENCY:-}" ]; then
@@ -1602,11 +1655,13 @@ resolve_deps () {
fi
PKGS=$(echo "$PKGS $NEWPKGS" | tr ' ' '\n' | sort | uniq)
local REALPKGS=""
- for s in $SUITE $EXTRA_SUITES; do
- for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do
- local path="dists/$s/$c/binary-$ARCH/Packages"
- local pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")"
- REALPKGS="$REALPKGS $("$PKGDETAILS" PKGS REAL "$pkgdest" $PKGS | sed -n 's/ .*REAL.*$//p')"
+ for a in $archs; do
+ for s in $SUITE $EXTRA_SUITES; do
+ for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do
+ local path="dists/$s/$c/binary-$a/Packages"
+ local pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m1" "$path")"
+ REALPKGS="$REALPKGS $("$PKGDETAILS" PKGS REAL "$pkgdest" $PKGS | sed -n 's/ .*REAL.*$//p')"
+ done
done
done
PKGS="$REALPKGS"
@@ -1621,15 +1676,22 @@ setup_available () {
local m1 c path pkgdest pkg
m1="${MIRRORS%% *}"
+ archs="$ARCH"
+ if [ $ARCH_ALL_SUPPORTED -eq 1 ]; then
+ archs="all $ARCH"
+ fi
+
for s in $SUITE $EXTRA_SUITES; do
- for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do
- path="dists/$s/$c/binary-$ARCH/Packages"
- pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")"
- # XXX: What if a package is in more than one component?
- # -- cjwatson 2009-07-29
- # XXX: ...or suite?
- # -- jrtc27 2019-06-11
- "$PKGDETAILS" STANZAS "$pkgdest" "$@"
+ for a in $archs; do
+ for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do
+ path="dists/$s/$c/binary-$a/Packages"
+ pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m1" "$path")"
+ # XXX: What if a package is in more than one component?
+ # -- cjwatson 2009-07-29
+ # XXX: ...or suite?
+ # -- jrtc27 2019-06-11
+ "$PKGDETAILS" STANZAS "$pkgdest" "$@"
+ done
done
done >"$TARGET/var/lib/dpkg/available"
--
GitLab
Reply to: