Package: release.debian.org Severity: normal User: release.debian.org@packages.debian.org Usertags: unblock X-Debbugs-Cc: inn2@packages.debian.org Control: affects -1 + src:inn2 Please unblock package inn2 This is tagged as a snapshot but is actually 2.7.1 RC1. It contains many documentation fixes, small improvements and fixes to pullnews, and the new ovsqlite-util program which can be used to debug and repair an ovsqlite database. The new package has been used in production for 3 weeks on one of my servers. I am attaching the git diff between debian/2.7.1_20230306-1 and debian/2.7.1_20230322-1, abridged of documentation changes. The full changelog can be consulted at https://salsa.debian.org/md/inn2/-/commits/master . unblock inn2/2.7.1~20230322-1 -- ciao, Marco
diff --git a/.gitignore b/.gitignore
index 274716315..9960002af 100644
--- a/.gitignore
+++ b/.gitignore
@@ -176,6 +176,7 @@
/storage/ovmethods.h
/storage/buffindexed/buffindexed_d
/storage/ovsqlite/ovsqlite-server
+/storage/ovsqlite/ovsqlite-util
/storage/ovsqlite/sql-init.c
/storage/ovsqlite/sql-init.h
/storage/ovsqlite/sql-main.c
diff --git a/MANIFEST b/MANIFEST
index 35e05aef2..7254d27aa 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -210,6 +210,7 @@ doc/man/ovdb_server.8 Manpage for ovdb_server
doc/man/ovdb_stat.8 Manpage for ovdb_stat
doc/man/overchan.8 Manpage for overchan backend
doc/man/ovsqlite-server.8 Manpage for ovsqlite-server
+doc/man/ovsqlite-util.8 Manpage for ovsqlite-util
doc/man/ovsqlite.5 Manpage for the ovsqlite overview module
doc/man/passwd.nntp.5 Manpage for passwd.nntp config file
doc/man/perl-nocem.8 Manpage for perl-nocem
@@ -331,6 +332,7 @@ doc/pod/ovdb_server.pod Master file for ovdb_server.8
doc/pod/ovdb_stat.pod Master file for ovdb_stat.8
doc/pod/overchan.pod Master file for overchan.8
doc/pod/ovsqlite-server.pod Master file for ovsqlite-server.8
+doc/pod/ovsqlite-util.pod Master file for ovsqlite-util.8
doc/pod/ovsqlite.pod Master file for ovsqlite.5
doc/pod/passwd.nntp.pod Master file for passwd.nntp.5
doc/pod/procbatch.pod Master file for procbatch.8
@@ -774,6 +776,7 @@ storage/ovsqlite/ovmethod.mk Make rules for ovsqlite
storage/ovsqlite/ovsqlite-private.c Private code for ovsqlite
storage/ovsqlite/ovsqlite-private.h Private header for ovsqlite
storage/ovsqlite/ovsqlite-server.c SQLite database exclusive owner
+storage/ovsqlite/ovsqlite-util.in Utility program for ovsqlite
storage/ovsqlite/ovsqlite.c ovsqlite implementation
storage/ovsqlite/ovsqlite.h ovsqlite interface
storage/ovsqlite/sql-init.c Generated database setup implementation
diff --git a/Makefile.global.in b/Makefile.global.in
index 8a185ed39..db42dee2e 100644
--- a/Makefile.global.in
+++ b/Makefile.global.in
@@ -20,7 +20,7 @@
## be complying with the NNTP protocol.
VERSION = 2.7.1
-VERSION_EXTRA = prerelease
+VERSION_EXTRA = rc1 version
## The absolute path to the top of the build directory, used to find the
## libraries built as part of INN. Using relative paths confuses libtool
diff --git a/debian/changelog b/debian/changelog
index ffbb0e6a6..eff319e64 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+inn2 (2.7.1~20230322-1) unstable; urgency=medium
+
+ * New release candidate 1 of the stable branch.
+
+ -- Marco d'Itri <md@linux.it> Mon, 27 Mar 2023 04:30:21 +0200
+
inn2 (2.7.1~20230306-1) unstable; urgency=medium
* New upstream snapshot of the stable branch.
diff --git a/doc/man/Makefile b/doc/man/Makefile
index 906725ebd..30a87587f 100644
--- a/doc/man/Makefile
+++ b/doc/man/Makefile
@@ -30,9 +30,9 @@ SEC8 = actsync.8 archive.8 batcher.8 buffchan.8 ckpasswd.8 \
innupgrade.8 innwatch.8 innxbatch.8 innxmit.8 mailpost.8 makedbz.8 \
makehistory.8 mod-active.8 news.daily.8 news2mail.8 ninpaths.8 \
nnrpd.8 nntpsend.8 ovdb_init.8 ovdb_monitor.8 ovdb_server.8 \
- ovdb_stat.8 overchan.8 ovsqlite-server.8 perl-nocem.8 procbatch.8 \
- prunehistory.8 radius.8 \
- rc.news.8 scanlogs.8 scanspool.8 send-ihave.8 send-uucp.8 sendinpaths.8 \
+ ovdb_stat.8 overchan.8 ovsqlite-server.8 ovsqlite-util.8 perl-nocem.8 \
+ procbatch.8 prunehistory.8 radius.8 rc.news.8 \
+ scanlogs.8 scanspool.8 send-ihave.8 send-uucp.8 sendinpaths.8 \
tally.control.8 tdx-util.8 tinyleaf.8 writelog.8
all:
diff --git a/doc/pod/Makefile b/doc/pod/Makefile
index 792ccf568..2fe219533 100644
--- a/doc/pod/Makefile
+++ b/doc/pod/Makefile
@@ -48,6 +48,7 @@ MAN8 = ../man/actsync.8 ../man/archive.8 ../man/auth_krb5.8 \
../man/nnrpd.8 ../man/nntpsend.8 \
../man/ovdb_init.8 ../man/ovdb_monitor.8 ../man/ovdb_server.8 \
../man/ovdb_stat.8 ../man/overchan.8 ../man/ovsqlite-server.8 \
+ ../man/ovsqlite-util.8 \
../man/procbatch.8 ../man/prunehistory.8 ../man/radius.8 \
../man/rc.news.8 ../man/scanlogs.8 ../man/scanspool.8 \
../man/send-ihave.8 ../man/sendinpaths.8 \
@@ -172,6 +173,7 @@ maintclean: distclean
../man/ovdb_stat.8: ovdb_stat.pod ; $(POD2MAN) -s 8 $? > $@
../man/overchan.8: overchan.pod ; $(POD2MAN) -s 8 $? > $@
../man/ovsqlite-server.8: ovsqlite-server.pod ; $(POD2MAN) -s 8 $? > $@
+../man/ovsqlite-util.8: ovsqlite-util.pod ; $(POD2MAN) -s 8 $? > $@
../man/procbatch.8: procbatch.pod ; $(POD2MAN) -s 8 $? > $@
../man/prunehistory.8: prunehistory.pod ; $(POD2MAN) -s 8 $? > $@
../man/radius.8: radius.pod ; $(POD2MAN) -s 8 $? > $@
diff --git a/frontends/pullnews.in b/frontends/pullnews.in
index 99ebfbf84..b21ce29b4 100644
--- a/frontends/pullnews.in
+++ b/frontends/pullnews.in
@@ -11,9 +11,10 @@
# Full changelog can be found in the Git commit history of the
# INN project. Major changes are:
#
-# February 2023:
+# February-March 2023:
# Julien Élie added TLS support for both downstream and upstream
-# servers. Also made pullnews robust on socket timeout.
+# servers. Also made pullnews robust on socket timeout, and
+# added -L (largest article size wanted).
#
# January 2010:
# Geraint A. Edwards added header-only feeding (-B);
@@ -131,9 +132,10 @@ END {
my $usage = "Usage:
$0 [-BhnOqRx] [-a hashfeed] [-b fraction] [-c config] [-C width]
[-d level] [-f fraction] [-F fakehop] [-g groups] [-G newsgroups]
- [-H headers] [-k checkpt] [-l logfile] [-m header_pats] [-M num] [-N num]
- [-p port] [-P hop_limit] [-Q level] [-r file] [-s host[:port][_tlsmode]]
- [-S num] [-t retries] [-T seconds] [-w num] [-z num] [-Z num]
+ [-H headers] [-k checkpt] [-l logfile] [-L size] [-m header_pats]
+ [-M num] [-N num] [-p port] [-P hop_limit] [-Q level] [-r file]
+ [-s host[:port][_tlsmode]] [-S num] [-t retries] [-T seconds]
+ [-w num] [-z num] [-Z num]
[upstream_host ...]
Options:
@@ -187,6 +189,8 @@ Options:
-l logfile log progress/stats to logfile (default is stdout).
+ -L size largest wanted article size in bytes for articles to download.
+
-m 'Hdr1:regexp1 !Hdr2:regexp2 #Hdr3:regexp3 !#Hdr4:regexp4 ...'
feed article only if:
the Hdr1 header field body matches regexp1;
@@ -258,10 +262,10 @@ sub HELP_MESSAGE {
}
use vars qw($opt_a $opt_b $opt_B $opt_c $opt_C $opt_d $opt_f $opt_F
- $opt_g $opt_G $opt_h $opt_H $opt_k $opt_l $opt_m $opt_M $opt_n
+ $opt_g $opt_G $opt_h $opt_H $opt_k $opt_l $opt_L $opt_m $opt_M $opt_n
$opt_N $opt_O $opt_p $opt_P $opt_q $opt_Q $opt_r $opt_R $opt_s
$opt_S $opt_t $opt_T $opt_w $opt_x $opt_z $opt_Z);
-getopts("a:b:Bc:C:d:f:F:g:G:hH:k:l:m:M:nN:Op:P:qQ:r:Rs:S:t:T:w:xz:Z:")
+getopts("a:b:Bc:C:d:f:F:g:G:hH:k:l:L:m:M:nN:Op:P:qQ:r:Rs:S:t:T:w:xz:Z:")
|| die $usage;
HELP_MESSAGE() if defined $opt_h;
@@ -522,7 +526,7 @@ if (not $rnews) {
if ($localTLS == 1 && !$localcxn->starttls()) {
die " Can't use STARTTLS on $localServer: "
. $localcxn->code() . " "
- . $localcxn->message() . "\n";
+ . join('//', split(/\r?\n/, $localcxn->message())) . "\n";
}
if (exists $passwd{$localServer}
@@ -530,7 +534,8 @@ if (not $rnews) {
{
warn sprintf(
" failed to authorize: %s %s\n",
- $localcxn->code(), $localcxn->message()
+ $localcxn->code(),
+ join('//', split(/\r?\n/, $localcxn->message()))
);
}
}
@@ -627,7 +632,8 @@ foreach my $server (@servers) {
if ($upstreamTLS == 1 && !$upstream->starttls()) {
warn sprintf(
"can't use STARTTLS: %s %s\n",
- $upstream->code(), $upstream->message()
+ $upstream->code(),
+ join('//', split(/\r?\n/, $upstream->message()))
);
next;
}
@@ -635,7 +641,8 @@ foreach my $server (@servers) {
if ($username && !$upstream->authinfo($username, $passwd)) {
warn sprintf(
"failed to authorize: %s %s\n",
- $upstream->code(), $upstream->message()
+ $upstream->code(),
+ join('//', split(/\r?\n/, $upstream->message()))
);
next;
}
@@ -826,7 +833,8 @@ sub crossFeedGroup {
if (!defined($narticles)) { # Group command failed.
warn sprintf(
"Group command failed for $group: %s %s\n",
- $fromServer->code() || 'NO_CODE', $fromServer->message()
+ $fromServer->code() || 'NO_CODE',
+ join('//', split(/\r?\n/, $fromServer->message()))
);
return 0;
}
@@ -874,16 +882,46 @@ sub crossFeedGroup {
my $i;
my @warns;
my $skip_article;
- for ($i = ($first > $high ? $first : $high + 1); $i <= $last; $i++) {
+ my $overview;
+ my $begin = ($first > $high ? $first : $high + 1);
+ for ($i = $begin; $i <= $last; $i++) {
$skip_article = 0;
last if defined $maxArts and $count >= $maxArts;
last if defined $opt_f and $count >= $toget;
$count++;
$art_total_count++;
sleep $opt_z if defined $opt_z and $count > 1;
+
+ # Do not download articles whose size exceeds the largest wanted size.
+ # Field 3 contains the Message-ID, field 5 the article size.
+ if (defined($opt_L)) {
+ # Retrieve overview data by chunks, so that articles keep being
+ # downloaded instead of a possible long wait at the start of the
+ # process of each newsgroup.
+ if (($count % $progressWidth) == 1) {
+ # Do not directly use $i + $progressWidth, as the result may
+ # exceed the maximum article number supported by the server.
+ my @range
+ = ($i,
+ $i + $progressWidth - 1 > $last
+ ? $last
+ : $i + $progressWidth - 1);
+ $overview = $fromServer->xover(\@range);
+ }
+
+ if (defined($$overview{$i}[5]) and $$overview{$i}[5] > $opt_L) {
+ print LOG "." unless $quiet;
+ print LOG "\tDEBUGGING $i\t-- not downloading "
+ . "article $$overview{$i}[3] "
+ . "which has $$overview{$i}[5] bytes\n"
+ if $debug >= 1;
+ $skip_article = 1;
+ }
+ }
+
# "Optimized mode" -- check if the article is wanted
# *before* downloading it.
- if (defined $opt_O) {
+ if (not $skip_article and defined($opt_O)) {
# 223 n <a> article retrieved
# -- request text separately (after STAT)
# 423 no such article number in this group
@@ -896,11 +934,12 @@ sub crossFeedGroup {
my $new_msgid = $toServer->nntpstat($org_msgid);
my $new_code = $toServer->code();
print LOG
- "\tDEBUGGING $i\t$org_msgid ($org_code) => $new_code\n"
+ "\n\tDEBUGGING $i\t$org_msgid ($org_code) => $new_code\n"
if $debug >= 3;
# Skip the article if it already exists
# on the downstream server.
if ($new_code == 223) {
+ print LOG "." unless $quiet;
print LOG "\tDEBUGGING $i\t-- not downloading "
. "already existing message $org_msgid code=$new_code\n"
if $debug >= 1;
@@ -928,6 +967,7 @@ sub crossFeedGroup {
push @{$article}, "\n" if not $is_control_art;
}
}
+
if (not $skip_article
and (not $header_only or $is_control_art or $add_bytes_header))
{
@@ -1071,13 +1111,13 @@ sub crossFeedGroup {
my $cut = join("\n\t", splice(@{$article}, $idx, 1));
$tx_len -= length($cut);
$idx_blank_pre_body--;
- print LOG "\tDEBUGGING $i\tcut1 $cut" if $debug >= 2;
+ print LOG "\tDEBUGGING $i\tcut1 $cut\n" if $debug >= 2;
while ($article->[$idx] =~ /^[[:space:]](.+)/) {
# Folded lines.
my $cut = join("\n\t", splice(@{$article}, $idx, 1));
$tx_len -= length($cut);
$idx_blank_pre_body--;
- print LOG "\tDEBUGGING $i\tcut_ $cut" if $debug >= 2;
+ print LOG "\tDEBUGGING $i\tcut_ $cut\n" if $debug >= 2;
}
}
@@ -1139,6 +1179,7 @@ sub crossFeedGroup {
$pulled->{$server}->{$group}++;
if ($skip_due_to_hdrs) {
+ print LOG "m" unless $quiet;
if ($debug >= 2) {
print LOG "\tDEBUGGING $i\tskip_art: "
. (
@@ -1152,7 +1193,6 @@ sub crossFeedGroup {
)
) . "\n";
}
- print LOG "m" unless $quiet;
} elsif ($rnews) {
printf RNEWS "#! rnews %d\n", $tx_len;
map { print RNEWS $_ } @{$article};
@@ -1171,9 +1211,6 @@ sub crossFeedGroup {
# 441 posting failed
my $code = $toServer->code();
my $msg = $toServer->message();
- print LOG "\tDEBUGGING $i\tPost $code: Msg: <"
- . join('//', split(/\r?\n/, $msg)) . ">\n"
- if $debug >= 1;
$msg =~ s/^340 .*?\n(?=.)//o;
if ($msg =~ /^240 /) {
print LOG "+" unless $quiet;
@@ -1198,6 +1235,10 @@ sub crossFeedGroup {
saveConfig();
exit(1);
}
+ print LOG "\tDEBUGGING $i\tPost $code: Msg: <"
+ . join('//', split(/\r?\n/, $toServer->message()))
+ . ">\n"
+ if $debug >= 1;
} elsif (not $reader
and not $toServer->ihave($msgid, $article))
@@ -1210,9 +1251,6 @@ sub crossFeedGroup {
# 437 article rejected -- do not try again
my $code = $toServer->code();
my $msg = $toServer->message();
- print LOG "\tDEBUGGING $i\tPost $code: Msg: <"
- . join('//', split(/\r?\n/, $msg)) . ">\n"
- if $debug >= 1;
if ($code == 435) {
print LOG "." unless $quiet;
$refused{$group}++;
@@ -1229,14 +1267,17 @@ sub crossFeedGroup {
saveConfig();
exit(1);
}
+ print LOG "\tDEBUGGING $i\tPost $code: Msg: <"
+ . join('//', split(/\r?\n/, $msg)) . ">\n"
+ if $debug >= 1;
} else {
my $code = $toServer->code();
my $msg = $toServer->message();
- print LOG "\tDEBUGGING $i\tPost $code: Msg: <"
- . join('//', split(/\r?\n/, $msg)) . ">\n"
- if $debug >= 1;
print LOG "+" unless $quiet;
+ print LOG "\tDEBUGGING $i\tPost $code: Msg: <"
+ . join('//', split(/\r?\n/, $msg)) . ">\n"
+ if $debug >= 1;
$fed{$group}++;
$info{server}->{$server}->{fed}++;
$info{fed}++;
@@ -1244,8 +1285,9 @@ sub crossFeedGroup {
}
$shash->{$group} = [time, $high = $i];
} elsif ($skip_article) {
- # Optimized mode (-O) decided to skip this article...
- print LOG "." unless $quiet;
+ # Optimized mode (-O) or article size check (-L) decided to skip
+ # this article...
+ # The "." resulting treatment has already been output.
$refused{$group}++;
$info{server}->{$server}->{refused}++;
$info{refused}++;
@@ -1262,20 +1304,23 @@ sub crossFeedGroup {
# Net::NNTP is a subclass) when the connection is no longer
# active.
warn "\nArticle retrieval failed ("
- . $fromServer->message() . ")\n\n";
+ . join('//', split(/\r?\n/, $fromServer->message()))
+ . ")\n\n";
return 2;
} else {
warn "\nUnexpected response from server ("
. $fromServer->code() . " "
- . $fromServer->message() . ")\n";
+ . join('//', split(/\r?\n/, $fromServer->message()))
+ . ")\n";
saveConfig();
exit(1);
}
}
print LOG "x" unless $quiet;
printf LOG (
- "\nDEBUGGING $i %s %s\n", $fromServer->code(),
- $fromServer->message()
+ "\tDEBUGGING $i\t-- article unavailable %s %s\n",
+ $fromServer->code(),
+ join('//', split(/\r?\n/, $fromServer->message()))
) if $debug >= 1;
}
saveConfig() if $checkPoint and ($art_total_count % $checkPoint) == 0;
diff --git a/scripts/innreport_inn.pm b/scripts/innreport_inn.pm
index a63e17fe8..33d79a373 100644
--- a/scripts/innreport_inn.pm
+++ b/scripts/innreport_inn.pm
@@ -1670,6 +1670,8 @@ sub collect($$$$$$) {
=~ /^python: dynamic authorization access type is not known: /o;
# during daily expiration
return 1 if $left =~ /^\S+ rejected Expiring process \d+$/o;
+ # during ovsqlite-util
+ return 1 if $left =~ /^\S+ rejected ovsqlite-util fixes$/o;
# during scanlogs
return 1 if $left =~ /^\S+ rejected Flushing log and syslog files$/o;
return 1 if $left =~ /^\S+ rejected Snapshot log and syslog files$/o;
diff --git a/storage/ovsqlite/ovmethod.config b/storage/ovsqlite/ovmethod.config
index c945ba877..57b932ebb 100644
--- a/storage/ovsqlite/ovmethod.config
+++ b/storage/ovsqlite/ovmethod.config
@@ -2,6 +2,6 @@ name = ovsqlite
number = 5
sources = ovsqlite.c ovsqlite-private.c
extra-sources = ovsqlite-server.c sql-main.c sql-init.c sqlite-helper.c
-programs = ovsqlite-server
+programs = ovsqlite-server ovsqlite-util
clean = sqlite-helper-gen
maintclean = sql-init.c sql-init.h sql-main.c sql-main.h
diff --git a/storage/ovsqlite/ovmethod.mk b/storage/ovsqlite/ovmethod.mk
index 3e56b3f83..009d3d259 100644
--- a/storage/ovsqlite/ovmethod.mk
+++ b/storage/ovsqlite/ovmethod.mk
@@ -8,6 +8,9 @@ ovsqlite/ovsqlite-server: $(OVSQLITEOBJECTS) libinnstorage.$(EXTLIB)
$(LIBSTORAGE) $(LIBHIST) $(LIBINN) $(STORAGE_LIBS) $(SQLITE3_LIBS) \
$(LIBS)
+ovsqlite/ovsqlite-util: ovsqlite/ovsqlite-util.in $(FIXSCRIPT)
+ $(FIX) ovsqlite/ovsqlite-util.in
+
ovsqlite/sqlite-helper-gen: ovsqlite/sqlite-helper-gen.in $(FIXSCRIPT)
$(FIX) -i ovsqlite/sqlite-helper-gen.in
diff --git a/storage/ovsqlite/ovsqlite-util.in b/storage/ovsqlite/ovsqlite-util.in
new file mode 100644
index 000000000..626949b82
--- /dev/null
+++ b/storage/ovsqlite/ovsqlite-util.in
@@ -0,0 +1,501 @@
+#! /usr/bin/perl -w
+# fixscript will replace this line with code to load INN::Config
+
+## Overview manipulation utility for ovsqlite.
+##
+## Initial version written in March 2023 by Julien ÉLIE.
+
+use Compress::Zlib;
+use Getopt::Std;
+use POSIX qw(strftime locale_h);
+use strict;
+
+$0 =~ s!.*/!!;
+
+# Bail out if the needed DBI Perl module is not installed.
+eval {
+ require DBI;
+ require DBD::SQLite;
+ my $err = $DBI::errstr; # Just to silence "used only once" warning.
+ 1;
+}
+ or die "DBI Perl module with SQLite driver needed"
+ . " (usually packaged as libdbd-sqlite3-perl, perl-DBD-SQLite,"
+ . " or p5-DBD-SQLite)";
+
+# Name of the database file (not configurable for ovsqlite).
+my $dbfile = "ovsqlite.db";
+
+my $usage = "Usage:
+ $0 [-AFghioO] [-a article] [-n newsgroup] [-p path]
+
+Options:
+ -a article Specify an article number or a range of article numbers on
+ which to act.
+ -A Audit the overview database for problems, and report them to
+ standard error, without trying to fix them.
+ -F Audit the overview database for problems, fixing them where
+ possible. To see what would be changed, run $0
+ with -A first.
+ -g Dump overall overview information for the newsgroup specified
+ with -n.
+ -h Print this help message.
+ -i Dump newsgroup-related overview information for all newsgroups
+ or the newsgroup specified with -n.
+ -n newsgroup Specify the newsgroup on which to act.
+ -o Dump overview information for articles in the newsgroup
+ specified with -n, in the format returned to clients.
+ -O Dump overview information for articles in the newsgroup
+ specified with -n, in the format used by overchan.
+ -p path Read $dbfile database file in path directory instead of
+ default $INN::Config::pathoverview directory.
+";
+
+sub HELP_MESSAGE {
+ print $usage;
+ exit(0);
+}
+
+my %opt;
+getopts("a:AFghioOn:p:", \%opt) || die $usage;
+
+HELP_MESSAGE() if defined($opt{'h'});
+
+my $modes = 0;
+$modes++ if defined($opt{'A'});
+$modes++ if defined($opt{'F'});
+$modes++ if defined($opt{'g'});
+$modes++ if defined($opt{'i'});
+$modes++ if defined($opt{'o'});
+$modes++ if defined($opt{'O'});
+
+die "Can't use both -A and -F\n\n$usage"
+ if defined($opt{'A'})
+ and defined($opt{'F'});
+die "No action specified\n\n$usage"
+ if $modes == 0;
+die "Only one action allowed at the same time\n\n$usage"
+ if $modes > 1;
+die "A newsgroup must be specified with -n\n\n$usage"
+ if !defined($opt{'n'})
+ and (defined($opt{'g'}) || defined($opt{'o'}) || defined($opt{'O'}));
+
+my ($low, $high, $compress, $basedict);
+my $sql_extraclause_artinfo = "";
+my $sql_extraclause_groupinfo = "";
+my $dbdir = $opt{'p'} || $INN::Config::pathoverview;
+my $datasource = "dbi:SQLite:dbname=$dbdir/$dbfile";
+my $pausemsg = "ovsqlite-util fixes"; # Message when pausing INN.
+ # Known line in innreport.
+my $ispaused = 0;
+
+# To determine the length of compressed overview data.
+my @pack_length_bias = (
+ 0,
+ 0x80,
+ 0x4080,
+ 0x204080,
+ 0x10204080,
+);
+
+# Open the connection. The username and password fields are left empty.
+# Enabling RaiseError permits not checking every return error codes.
+my $dbh = DBI->connect(
+ $datasource, '', '',
+ { PrintError => 0, RaiseError => 1, AutoCommit => 0 }
+) or die "Can't connect to database: $DBI::errstr";
+
+# To process multiple SQL statements in a do() handle.
+$dbh->{sqlite_allow_multiple_statements} = 1;
+
+# Check the specified newsgroup exists, and create appropriate SQL requests.
+if (defined($opt{'n'})) {
+ my $groupid = get_groupid($opt{'n'});
+ if ($groupid == 0) {
+ printf STDERR "Cannot find newsgroup $opt{'n'} in overview\n";
+ exit(1);
+ } else {
+ $sql_extraclause_artinfo = "where groupid = $groupid";
+ $sql_extraclause_groupinfo
+ = "where cast(groupname as text) = '$opt{'n'}'";
+ }
+}
+
+# Parse the specified range of articles.
+if (defined($opt{'a'})) {
+ if ($opt{'a'} =~ /^(\d*)-(\d*)$/) {
+ $low = $1;
+ $high = $2;
+ } elsif ($opt{'a'} =~ /^\d+$/) {
+ $low = $opt{'a'};
+ $high = $low;
+ } else {
+ printf STDERR "Cannot parse $opt{'a'} as article numbers\n";
+ exit(1);
+ }
+ if (defined($low) and length($low) > 0) {
+ $sql_extraclause_artinfo .= " and artnum >= $low";
+ }
+ if (defined($high) and length($high) > 0) {
+ $sql_extraclause_artinfo .= " and artnum <= $high";
+ }
+}
+
+# Grab information from the misc table.
+load_settings();
+
+# Pause the server if changes need being done, so that the overview is not
+# updated by another process at the same time.
+if (defined($opt{'F'})) {
+ if (system "$INN::Config::newsbin/ctlinnd -s pause '$pausemsg'") {
+ die "$0: failed to pause INN, aborting\n";
+ }
+ $ispaused = 1;
+}
+
+if (defined($opt{'A'}) or defined($opt{'F'})) {
+ # Run the checks, and fix them if -F given.
+ check_groupinfo_consistency();
+} elsif (defined($opt{'g'})) {
+ dump_overview();
+} elsif (defined($opt{'i'})) {
+ dump_groupinfo();
+} elsif (defined($opt{'o'})) {
+ dump_artinfo_clients();
+} elsif (defined($opt{'O'})) {
+ dump_artinfo_overchan();
+}
+
+# Close the connection properly.
+$dbh->disconnect;
+
+exit(0);
+
+END {
+ # In case we bail out while being paused, make sure that the show goes on!
+ if ($ispaused) {
+ if (system "$INN::Config::newsbin/ctlinnd -s go '$pausemsg'") {
+ die "$0: failed to resume INN with "
+ . "\"ctlinnd -s go '$pausemsg'\" command "
+ . "=> please check why and *manually* resume it\n";
+ }
+ }
+}
+
+# Grab compression settings from the database.
+sub load_settings {
+ my $getsetting;
+
+ $getsetting = $dbh->prepare("select value from misc where key = ?1");
+ ($compress) = $dbh->selectrow_array($getsetting, undef, "compress");
+ if ($compress > 0) {
+ ($basedict) = $dbh->selectrow_array($getsetting, undef, "basedict");
+ defined($basedict)
+ or die "No basedict value found to decompress overview data\n";
+ }
+}
+
+# Return the ID of the newsgroup given as argument, or 0 if not found.
+sub get_groupid {
+ my $groupname = shift;
+ my ($getgroupid, $groupid);
+
+ $getgroupid = $dbh->prepare(
+ q{
+select groupid from groupinfo
+ where cast(groupname as text) = ?1
+ and deleted = 0;
+}
+ );
+ ($groupid) = $dbh->selectrow_array($getgroupid, undef, $groupname);
+
+ return defined($groupid) ? $groupid : 0;
+}
+
+# Turn enforcement of foreign key constraints on or off, depending on the
+# argument given to the function (either 1 for on, or 0 for off).
+# The AutoCommit attribute needs being true so as not to start a transaction.
+sub pragma_foreign_keys {
+ my $onoff = shift;
+ local $dbh->{AutoCommit} = 1;
+ $dbh->do("pragma foreign_keys = $onoff;");
+}
+
+# Return an array containing the length of the encoded length of decompressed
+# overview data, and the length of actual decompressed overview data, or undef
+# if corrupted.
+# This function can be called even on uncompressed data.
+# The expected argument is the overview data.
+sub overview_length {
+ my $data = shift;
+ my ($lenlen, $len);
+
+ if ($compress > 0) {
+ my $first;
+
+ $first = vec($data, 0, 8);
+ $len = $first;
+ $lenlen = 1;
+ while (($first & 0x80) > 0) {
+ $len = $len << 8 | vec($data, $lenlen, 8);
+ $lenlen++;
+ $first <<= 1;
+ }
+ if ($lenlen > 5) {
+ return (undef, undef);
+ }
+ $len &= (1 << $lenlen * 7) - 1;
+ $len += $pack_length_bias[$lenlen - 1];
+ } else {
+ # Uncompressed overview data.
+ $lenlen = 0;
+ $len = length($data);
+ }
+ return ($lenlen, $len);
+}
+
+# Return decompressed overview data, or undef if a failure occurs.
+# This function can be called even on uncompressed data.
+# The expected arguments are the newsgroup name, the article number, and the
+# associated overview data.
+sub decompress_overview {
+ my ($groupname, $artnum, $data) = @_;
+ my $result;
+
+ if ($compress > 0) {
+ my ($lenlen, $len) = overview_length($data);
+ if (!defined($lenlen)) {
+ warn "$groupname:$artnum: Corrupt overview data\n";
+ return undef;
+ }
+ if ($len == 0) {
+ # No compression.
+ $result = substr($data, $lenlen);
+ } else {
+ my ($inflation, $status);
+
+ ($inflation, $status)
+ = inflateInit(-Dictionary => "$basedict$groupname:$artnum\r\n");
+ if ($status != Z_OK) {
+ warn
+ "$groupname:$artnum: inflateInit failed with code $status\n";
+ return undef;
+ }
+ ($result, $status) = $inflation->inflate(substr($data, $lenlen));
+ if ($status != Z_STREAM_END) {
+ warn "$groupname:$artnum: inflate failed with code $status\n";
+ return undef;
+ }
+ if (length($result) != $len) {
+ warn "$groupname:$artnum: Corrupt overview data\n";
+ return undef;
+ }
+ }
+ } else {
+ # Uncompressed overview data.
+ $result = $data;
+ }
+ return $result;
+}
+
+# Perform consistency checks on low water marks, high water marks, and
+# article counts in groupinfo. SQL commands were provided by Bo Lindbergh.
+sub check_groupinfo_consistency {
+ my ($statement, $result);
+
+ pragma_foreign_keys(0);
+ $dbh->do(
+ q{
+create table temp.repairs (
+ groupid integer
+ primary key,
+ new_low integer
+ not null,
+ low_was_bad integer
+ not null,
+ new_high integer
+ not null,
+ high_was_bad integer
+ not null,
+ new_count integer
+ not null,
+ count_was_bad integer
+ not null,
+ expired integer
+ not null,
+ groupname blob
+ not null,
+ flag_alias blob
+ not null
+);
+
+with new_stats (groupid, new_low, new_high, new_count) as
+ (select groupid,
+ coalesce(min(artnum), low),
+ coalesce(max(artnum), high),
+ count(artnum)
+ from groupinfo
+ natural left join artinfo
+ where not deleted
+ group by groupid)
+insert into repairs
+ (groupid,
+ new_low, low_was_bad,
+ new_high, high_was_bad,
+ new_count, count_was_bad,
+ expired, groupname, flag_alias)
+ select groupid,
+ new_low, new_low != low as low_was_bad,
+ new_high, new_high != high as high_was_bad,
+ new_count, new_count != "count" as count_was_bad,
+ expired, groupname, flag_alias
+ from new_stats
+ natural join groupinfo
+ where low_was_bad
+ or high_was_bad
+ or count_was_bad;
+}
+ );
+ pragma_foreign_keys(1);
+
+ $statement = $dbh->prepare("select count(*) from repairs;");
+ ($result) = $dbh->selectrow_array($statement);
+
+ if ($result > 0) {
+ printf STDERR (
+ "%d groupinfo record%s incoherent\n", $result,
+ ($result > 1) ? "s" : ""
+ );
+
+ # Show incoherent records (L, H and C are for Low, High, Count).
+ $statement = $dbh->prepare(
+ q{
+select case when low_was_bad then 'L' else '_' end
+ || case when high_was_bad then 'H' else '_' end
+ || case when count_was_bad then 'C' else '_' end as bad, groupname
+ from repairs;
+}
+ );
+ $statement->execute();
+
+ while (my @row = $statement->fetchrow_array()) {
+ print STDERR " $row[0] for $row[1]\n";
+ }
+
+ if (defined($opt{'F'})) {
+ # Fix groupinfo table.
+ $result = $dbh->do(
+ q{
+insert or replace into groupinfo
+ (groupid, low, high, "count", expired, groupname, flag_alias)
+ select groupid, new_low, new_high, new_count,
+ expired, groupname, flag_alias
+ from repairs;
+}
+ );
+ $dbh->commit();
+
+ printf STDERR (
+ "%d groupinfo record%s fixed\n", $result,
+ ($result > 1) ? "s" : ""
+ );
+ }
+
+ }
+}
+
+# Dump overview information (-g option).
+sub dump_overview {
+ my $statement;
+
+ $statement = $dbh->prepare(
+ qq{
+select artnum, overview, arrived, expires, quote(token)
+ from artinfo $sql_extraclause_artinfo;
+}
+ );
+ $statement->execute();
+
+ while (my @row = $statement->fetchrow_array()) {
+ # quote(token) returns a string in the form "X'token'" without
+ # surrounding '@' characters.
+ my $len = (overview_length($row[1]))[1];
+ if (!defined($len)) {
+ warn "$opt{'n'}:$row[0]: Corrupt overview data\n";
+ $len = 0;
+ }
+ print "$row[0] $len $row[2] $row[3]";
+ print " @" . substr($row[4], 2, -1) . "@\n";
+ }
+}
+
+# Dump newsgroup-related overview information (-i option).
+sub dump_groupinfo {
+ my $statement;
+
+ $statement = $dbh->prepare(
+ qq{
+select groupname, high, low, count, flag_alias, expired, deleted
+ from groupinfo $sql_extraclause_groupinfo;
+}
+ );
+ $statement->execute();
+
+ while (my @row = $statement->fetchrow_array()) {
+ print join(" ", @row) . "\n";
+ }
+}
+
+# Dump overview information in the format returned to clients (-o option).
+sub dump_artinfo_clients {
+ my $statement;
+
+ $statement = $dbh->prepare(
+ qq{
+select overview, artnum, quote(token), arrived, expires
+ from artinfo $sql_extraclause_artinfo;
+}
+ );
+ $statement->execute();
+
+ # To generate valid Date header fields.
+ setlocale(LC_TIME, 'C');
+
+ while (my @row = $statement->fetchrow_array()) {
+ my $overdata = decompress_overview($opt{'n'}, $row[1], $row[0]);
+ # Remove trailing CRLF from overview data.
+ $overdata =~ s/\r\n//g;
+ print "$overdata";
+ print "\tArticle: $row[1]";
+ print "\tToken: @" . substr($row[2], 2, -1) . "@";
+ print "\tArrived: "
+ . strftime('%a, %d %b %Y %H:%M:%S %z (%Z)', localtime($row[3]));
+ print "\tExpires: "
+ . strftime('%a, %d %b %Y %H:%M:%S %z (%Z)', localtime($row[4]))
+ if $row[4] > 0;
+ print "\n";
+ }
+}
+
+# Dump overview information in the format used by overchan (-O option).
+sub dump_artinfo_overchan {
+ my $statement;
+
+ $statement = $dbh->prepare(
+ qq{
+select quote(token), arrived, expires, overview, artnum
+ from artinfo $sql_extraclause_artinfo;
+}
+ );
+ $statement->execute();
+
+ while (my @row = $statement->fetchrow_array()) {
+ print "@" . substr($row[0], 2, -1) . "@";
+ my $overdata = decompress_overview($opt{'n'}, $row[4], $row[3]);
+ # Remove the first field (article number, not expected by overchan)
+ # and trailing CRLF from overview data.
+ $overdata =~ s/^\d+\t//;
+ $overdata =~ s/\r\n//;
+ print " $row[1] $row[2] $overdata\n";
+ }
+}
diff --git a/support/mkmanifest b/support/mkmanifest
index b4694caf0..484090be5 100755
--- a/support/mkmanifest
+++ b/support/mkmanifest
@@ -266,6 +266,7 @@ site/update
storage/buffindexed/buffindexed_d
storage/buildconfig
storage/ovsqlite/ovsqlite-server
+storage/ovsqlite/ovsqlite-util
storage/ovsqlite/sqlite-helper-gen
storage/tradindexed/tdx-util
support/fixconfig
Attachment:
signature.asc
Description: PGP signature