Bug#818761: packages.debian.org doesn't list experimental packages anymore
Package: www.debian.org
Followup-For: Bug #818761
I've prepared a patch for this. Please find it attached.
I don't just want to push it though, since it would be good to clean
up the archive/ directory on packages.debian.org. So it would be
preferable that someone with access to the deployment merges it and
then takes care of the deployment immediately.
Regards,
Frank
-- System Information:
Debian Release: 8.3
APT prefers stable
APT policy: (500, 'stable')
Architecture: amd64 (x86_64)
Kernel: Linux 3.16.0-4-amd64 (SMP w/2 CPU cores)
Locale: LANG=de_DE.UTF-8, LC_CTYPE=de_DE.UTF-8 (charmap=UTF-8)
Shell: /bin/sh linked to /bin/dash
Init: systemd (via /run/systemd/system)
>From 63322b969c931107c3d26d3e9c8b1b24a41b7da5 Mon Sep 17 00:00:00 2001
From: Frank Lichtenheld <frank@lichtenheld.de>
Date: Mon, 28 Mar 2016 20:21:12 +0200
Subject: [PATCH] Teach packages.debian.org to handle .xz archive files
Currently only for main archive, not backports or
debports, yet.
Crude mix of autodetection and configuration but
since at least experimental is completely broken
right now my priority was to get it working.
Closes: #818761
---
bin/parse-packages | 38 ++++++++++++++++++++++++++++++++++----
bin/parse-sources | 35 ++++++++++++++++++++++++++++++++---
config.sh.sed.in | 4 ++++
cron.d/100syncarchive | 33 +++++++++++++++++++++------------
4 files changed, 91 insertions(+), 19 deletions(-)
diff --git a/bin/parse-packages b/bin/parse-packages
index a7403a9..280adaf 100755
--- a/bin/parse-packages
+++ b/bin/parse-packages
@@ -63,6 +63,37 @@ mkpath( "$DBDIR/xapian.new" );
my %descriptions_english_db;
tie %descriptions_english_db, "DB_File", "files/db/descriptions_translated_english_only.db", O_RDONLY, 0666, $DB_BTREE;
+my %ext_to_prog = (
+ xz => 'xzcat',
+ gz => 'zcat',
+ );
+sub open_packages_files {
+ my( $suite_dir, $component) = @_;
+
+ my (@files, $prog);
+ for my $ext (qw(xz gz)){
+ my $packages_match = "binary-*/Packages.$ext";
+ @files = ();
+ push @files, <"$suite_dir/$component/$packages_match">;
+ push @files, <"$suite_dir/updates/$component/$packages_match">;
+ push @files, <"$suite_dir/$component/debian-installer/$packages_match">;
+
+ if( @files ){
+ $prog = $ext_to_prog{$ext};
+ last;
+ }
+ }
+
+ if( @files && $prog ){
+ print "\tprog=$prog\n";
+ print "\tfiles=@files\n";
+ open my $fh, '-|', $prog, @files;
+ return $fh;
+ }
+ print "\tno files found, skipping...\n";
+ return;
+}
+
for my $suite (@SUITES) {
my %package_names_suite = ();
my %packages_all_db;
@@ -76,10 +107,9 @@ for my $suite (@SUITES) {
print "\tseems not to exist, skipping...\n";
next;
}
- open PKG, "zcat $TOPDIR/archive/$archive/$suite/$what/binary-*/Packages.gz"
- . " $TOPDIR/archive/$archive/$suite/updates/$what/binary-*/Packages.gz"
- . " $TOPDIR/archive/$archive/$suite/$what/debian-installer/binary-*/Packages.gz|";
- while (<PKG>) {
+
+ my $fh = open_packages_files("$TOPDIR/archive/$archive/$suite/", $what) || next;
+ while (<$fh>) {
next if /^\s*$/;
my $data = "";
my %data = ();
diff --git a/bin/parse-sources b/bin/parse-sources
index 1f73bcc..a4e6d06 100755
--- a/bin/parse-sources
+++ b/bin/parse-sources
@@ -38,6 +38,36 @@ $/ = "";
-d $DBDIR || mkpath( $DBDIR );
+my %ext_to_prog = (
+ xz => 'xzcat',
+ gz => 'zcat',
+ );
+sub open_sources_files {
+ my( $suite_dir, $component) = @_;
+
+ my (@files, $prog);
+ for my $ext (qw(xz gz)){
+ my $sources_match = "source/Sources.$ext";
+ @files = ();
+ push @files, <"$suite_dir/$component/$sources_match">;
+ push @files, <"$suite_dir/updates/$component/$sources_match">;
+
+ if( @files ){
+ $prog = $ext_to_prog{$ext};
+ last;
+ }
+ }
+
+ if( @files && $prog ){
+ print "\tprog=$prog\n";
+ print "\tfiles=@files\n";
+ open my $fh, '-|', $prog, @files;
+ return $fh;
+ }
+ print "\tno files found, skipping...\n";
+ return;
+}
+
for my $archive (@ARCHIVES) {
for my $suite (@SUITES) {
@@ -51,9 +81,8 @@ for my $archive (@ARCHIVES) {
print "\tseems not to exist, skipping...\n";
next;
}
- open PKG, "zcat $TOPDIR/archive/$archive/$suite/$what/source/Sources.gz"
- . " $TOPDIR/archive/$archive/$suite/updates/$what/source/Sources.gz|";
- while (<PKG>) {
+ my $fh = open_sources_files("$TOPDIR/archive/$archive/$suite/", $what) || next;
+ while (<$fh>) {
next if /^\s*$/;
my $data = "";
my %data = ();
diff --git a/config.sh.sed.in b/config.sh.sed.in
index 10970d2..03ffc31 100644
--- a/config.sh.sed.in
+++ b/config.sh.sed.in
@@ -62,6 +62,10 @@ arch_sid="${arch_stretch} alpha avr32 hppa hurd-i386 m68k powerpcspe ppc64el sh4
arch_experimental="${arch_sid}"
arch_wheezy_proposed_updates="${arch_wheezy}"
+ext_wheezy=gz
+#for security Sources
+ext_jessie=gz
+
# from http://ftp.de.debian.org/debian-ports/dists/{sid,experimental}/Release
arch_debports="alpha hppa m68k powerpcspe ppc64 sh4 sparc64 x32"
diff --git a/cron.d/100syncarchive b/cron.d/100syncarchive
index d6459e0..b7f7a47 100755
--- a/cron.d/100syncarchive
+++ b/cron.d/100syncarchive
@@ -17,8 +17,11 @@ then
(cd us/${dist} &&
$wget_cmd -N ${ftpsite}/dists/${dist}/Release &&
$wget_cmd -N ${ftpsite}/dists/${dist}/Release.gpg )
-
- foo=\$arch_${dist//-/_}
+
+ ext=$(eval echo \$ext_${dist%-*})
+ if [ -z "$ext" ]; then ext=xz; fi
+ echo "using $ext as file extension"
+ foo=\$arch_${dist%-*}
for arch in `eval echo $foo`
do
for part in ${parts}
@@ -26,7 +29,7 @@ then
echo retrieve Packages $dist/$part/$arch
test -d us/${dist}/${part}/binary-${arch} || mkdir -p us/${dist}/${part}/binary-${arch}
(cd us/${dist}/${part}/binary-${arch} &&
- $wget_cmd -N ${ftpsite}/dists/${dist}/${part}/binary-${arch}/Packages.gz)
+ $wget_cmd -N ${ftpsite}/dists/${dist}/${part}/binary-${arch}/Packages.$ext)
#FIXME: no Contents files for p-u
echo retrieve Contents $dist/${part}/$arch
(cd us/${dist}/${part} &&
@@ -38,7 +41,7 @@ then
echo retrieve Sources $dist/$part
test -d us/${dist}/${part}/source || mkdir -p us/${dist}/${part}/source
(cd us/${dist}/${part}/source &&
- $wget_cmd -N ${ftpsite}/dists/${dist}/${part}/source/Sources.gz)
+ $wget_cmd -N ${ftpsite}/dists/${dist}/${part}/source/Sources.$ext)
done
$check_release_cmd $archivedir/us/${dist}/Release
@@ -52,15 +55,18 @@ else
rsync -Lt ${localdir}/dists/${dist}/Release us/${dist}/Release
rsync -Lt ${localdir}/dists/${dist}/Release.gpg us/${dist}/Release.gpg
- foo=\$arch_${dist//-/_}
+ ext=$(eval echo \$ext_${dist%-*})
+ if [ -z "$ext" ]; then ext=xz; fi
+ echo "using $ext as file extension"
+ foo=\$arch_${dist%-*}
for arch in `eval echo $foo`
do
for part in ${parts}
do
echo retrieve Packages $dist/$part/$arch
test -d us/${dist}/${part}/binary-${arch} || mkdir -p us/${dist}/${part}/binary-${arch}
- rsync -Lt ${localdir}/dists/${dist}/${part}/binary-${arch}/Packages.gz \
- us/${dist}/${part}/binary-${arch}/Packages.gz
+ rsync -Lt ${localdir}/dists/${dist}/${part}/binary-${arch}/Packages.$ext \
+ us/${dist}/${part}/binary-${arch}/Packages.$ext
#FIXME: no Contents files for p-u
#FIXME: the symlinks for wheezy/sid/experimental will go away
@@ -74,8 +80,8 @@ else
do
echo retrieve Sources $dist/$part
test -d us/${dist}/${part}/source || mkdir -p us/${dist}/${part}/source
- rsync -Lt ${localdir}/dists/${dist}/${part}/source/Sources.gz \
- us/${dist}/${part}/source/Sources.gz
+ rsync -Lt ${localdir}/dists/${dist}/${part}/source/Sources.$ext \
+ us/${dist}/${part}/source/Sources.$ext
done
$check_release_cmd $archivedir/us/${dist}/Release
@@ -92,7 +98,10 @@ do
$wget_cmd -N ${security_ftpsite}/dists/${dist}/updates/Release &&
$wget_cmd -N ${security_ftpsite}/dists/${dist}/updates/Release.gpg )
- foo=\$arch_${dist//-/_}
+ ext=$(eval echo \$ext_${dist%-*})
+ if [ -z "$ext" ]; then ext=xz; fi
+ echo "using $ext as file extension"
+ foo=\$arch_${dist%-*}
for arch in `eval echo $foo`
do
for part in ${parts}
@@ -100,7 +109,7 @@ do
echo retrieve Packages $dist/$part/security/$arch
test -d security/${dist}/${part}/binary-${arch} || mkdir -p security/${dist}/${part}/binary-${arch}
(cd security/${dist}/${part}/binary-${arch} &&
- $wget_cmd -N ${security_ftpsite}/dists/${dist}/updates/${part}/binary-${arch}/Packages.gz)
+ $wget_cmd -N ${security_ftpsite}/dists/${dist}/updates/${part}/binary-${arch}/Packages.$ext)
done
done
for part in ${parts}
@@ -108,7 +117,7 @@ do
echo retrieve Sources $dist/$part/security
test -d security/${dist}/${part}/source || mkdir -p security/${dist}/${part}/source
(cd security/${dist}/${part}/source &&
- $wget_cmd -N ${security_ftpsite}/dists/${dist}/updates/${part}/source/Sources.gz)
+ $wget_cmd -N ${security_ftpsite}/dists/${dist}/updates/${part}/source/Sources.$ext)
done
$check_release_cmd $archivedir/security/${dist}/Release
--
2.1.4
Reply to: