[SCM] Debian package checker branch, master, updated. 2.5.8-18-g90a8176
The following commit has been merged in the master branch:
commit 90a81765c46aa30f1117a8eb089eddfbb5d0cd23
Author: Niels Thykier <niels@thykier.net>
Date: Thu Jun 14 19:02:00 2012 +0200
coll/hardening-info: Use xargs to reduce processing time
For binary packages with binary ELF binaries, hardening-check takes
unreasonably long when run once on each ELF. Instead, use xargs to
reduce the number of hardening-check invocations.
This requires a few extra hoops to handle the fix for #673112, but
keeps the runtime of linux-image on 45-50 seconds. Lintian/2.5.8
takes at least 3 minutes (likely even more).
Signed-off-by: Niels Thykier <niels@thykier.net>
diff --git a/collection/hardening-info b/collection/hardening-info
index 80b711e..4227359 100755
--- a/collection/hardening-info
+++ b/collection/hardening-info
@@ -29,50 +29,46 @@
use strict;
use warnings;
+use Cwd qw(realpath);
use lib "$ENV{'LINTIAN_ROOT'}/lib";
use Lintian::Collect;
+use Lintian::Command qw(spawn reap);
use Lintian::Util qw(fail);
my ($pkg, $type, $dir) = @ARGV;
my $info = Lintian::Collect->new ($pkg, $type, $dir);
my $file_info = $info->file_info;
+my $helper = realpath("$0-helper");
+
if ( -e "$dir/hardening-info" ) {
unlink "$dir/hardening-info" or fail "unlink hardening-info: $!";
}
-open OUT, '>', "$dir/hardening-info"
- or fail("cannot open hardening-info: $!");
-
# Prepare to examine the file tree.
chdir ("$dir/unpacked")
or fail("unable to chdir to unpacked: $!");
-foreach my $bin (keys %$file_info) {
+my %opts = ( pipe_in => FileHandle->new,
+ out => "$dir/hardening-info",
+ fail => 'never' );
+
+# Use xargs to keep processing times of packages like linux-image
+# reasonble.
+spawn(\%opts, ['xargs', '-0r', 'hardening-check', '--lintian', '--'], '|',
+ [$helper]);
+$opts{pipe_in}->blocking(1);
+
+
+foreach my $bin ($info->sorted_index) {
+ next unless $info->index->{$bin}->is_file;
my $finfo = $file_info->{$bin};
next unless $finfo =~ m/\bELF\b/o;
- if (open (my $pipe, '-|', "hardening-check --lintian -- \Q$bin\E 2>&1")) {
- while ( my $line = <$pipe> ) {
- my $emit = 1;
- if ($line =~ m/^no-fortify-functions:/o ) {
- $emit = 0;
- open my $p2, '-|', "hardening-check --verbose -- \Q$bin\E 2>&1" or fail "hardening-check: $!";
- while ( my $l2 = <$p2> ) {
- if ($l2 =~ m/^\s+unprotected:\s*(\S+)/) {
- next if $1 eq 'memcpy';
- $emit = 1;
- last;
- }
- }
- close $p2;
- }
- print OUT $line if $emit;
- }
- close $pipe;
- }
+ printf {$opts{pipe_in}} "%s\0", $bin;
}
-close OUT or fail("cannot write hardening-info: $!");
+close $opts{pipe_in};
+reap (\%opts);
exit 0;
diff --git a/collection/hardening-info-helper b/collection/hardening-info-helper
new file mode 100755
index 0000000..49864e5
--- /dev/null
+++ b/collection/hardening-info-helper
@@ -0,0 +1,147 @@
+#!/usr/bin/perl
+# hardening-info-helper -- lintian collection script helper
+
+# Copyright (C) 2012 Niels Thykier
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, you can find it on the World Wide
+# Web at http://www.gnu.org/copyleft/gpl.html, or write to the Free
+# Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
+# MA 02110-1301, USA.
+
+use strict;
+use warnings;
+
+use lib "$ENV{'LINTIAN_ROOT'}/lib";
+use Lintian::Command qw(spawn reap);
+use Lintian::Util qw(fail);
+
+# To reduce the number of false-positives in hardening-check for
+# fortify-functions, we have to "double-check" its output in some
+# cases (like we do with file-info).
+#
+# Basic idea - fork and pipe to child in up to two passes.
+# - The parent will filter "hardening-check --lintian" input in first
+# pass.
+# - Filter out (and collect) all no-fortify-function tags
+# - Work around bug #677530
+# - The parent will (in second pass) pipe the verbose hardening-check
+# output to the child.
+# - Only binaries with a no-foritfy-function tag in the first pass
+# will be re-checked with --verbose.
+#
+# - In the first pass, the child will behave like cat.
+# - In the second pass, the child will parse hardening-check --verbose
+# output.
+#
+# Implied by the above - the second pass is only done if needed.
+
+
+my ($in, $out);
+my ($cread, $cwrite);
+my $cpid;
+my @recheck = ();
+# Work around bug in hardening-check when it proceses multiple binaries
+# with --lintian (see #677530).
+my %seen = ();
+
+pipe ($cread, $cwrite) or fail "pipe failed: $!";
+$cpid = fork();
+fail "fork failed: $!" unless defined $cpid;
+if ($cpid) {
+ # parent
+ close $cread; # read end not needed
+ $in = \*STDIN;
+ $out = $cwrite;
+} else {
+ # child
+ close $cwrite; # write end not needed.
+ $in = $cread;
+ $out = \*STDOUT;
+}
+
+while (my $line = <$in>) {
+ chomp $line;
+ if ($cpid) {
+ # The parent filters the duplicate tags (#677530)
+ next if $seen{$line}++;
+ if ($line =~ m/^no-fortify-functions:(.*)$/o) {
+ my $bin = $1;
+ push @recheck, $bin;
+ next;
+ }
+ } else {
+ # End of "first pass" marker (for the child).
+ last if $line eq '__VERBOSE__';
+ }
+ print $out "$line\n";
+}
+
+undef %seen;
+
+if (not $cpid) {
+ # child's second pass
+ my $bin;
+ my $infsf = 0;
+ my $emit = 0;
+ while (my $line = <$in>) {
+ chomp $line;
+ # At this point we are reading "verbose" hardening-check output
+ if ($line =~ m,^(\S.+):$,) {
+ if ($emit) {
+ print $out "no-fortify-functions:$bin\n";
+ }
+ $bin = $1;
+ $infsf = 0;
+ $emit = 0;
+ } elsif ($line =~ m/^\s+Fortify Source functions:/) {
+ $infsf = 1;
+ } elsif ($infsf and $line =~ m/^\s+unprotected:\s*(\S+)/) {
+ next if $1 eq 'memcpy';
+ $emit = 1;
+ } else {
+ $infsf = 0;
+ }
+ }
+ if ($emit) {
+ print $out "no-fortify-functions:$bin\n";
+ }
+ # ensure $out is flushed before exiting.
+ close $out or fail "close output: $!";
+ require POSIX;
+ POSIX::_exit (0);
+} elsif (@recheck) {
+ # (optionally) parent's second pass.
+ my %opts = (
+ pipe_in => FileHandle->new,
+ out => $out,
+ fail => 'never'
+ );
+ # End the first pass for the child
+ print $out "__VERBOSE__\n";
+ spawn(\%opts, ['xargs', '-0r', 'hardening-check', '--verbose', '--']);
+ $opts{pipe_in}->blocking (1);
+ foreach my $file (@recheck) {
+ printf {$opts{pipe_in}} "%s\0", $file;
+ }
+ close $opts{pipe_in};
+ reap (\%opts);
+}
+
+# Close the out handle, else the child process will wait for
+# ever.
+close $out;
+# wait for the child process.
+wait();
+exit $?;
+
diff --git a/collection/hardening-info.desc b/collection/hardening-info.desc
index 5b9a29a..5494aaf 100644
--- a/collection/hardening-info.desc
+++ b/collection/hardening-info.desc
@@ -4,4 +4,4 @@ Info: This script runs hardening-check(1) over all ELF binaries of a binary
package.
Type: binary, udeb
Version: 2
-Needs-Info: bin-pkg-control, file-info, unpacked
+Needs-Info: bin-pkg-control, file-info, index, unpacked
diff --git a/debian/changelog b/debian/changelog
index d7e2253..418f5b9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -25,6 +25,14 @@ lintian (2.5.9) UNRELEASED; urgency=low
+ [NT] Clarify that dev-pkg-without-shlib-symlink is emitted
on library and not "-dev" packages.
+ * collection/hardening-info:
+ + [NT] Process binaries with hardening-check via xargs. This
+ greatly reduces the processing time for packages with many
+ binaries (like some of the linux binaries). Thanks to
+ Bastian Blank for the heads up.
+ * collection/hardening-info-helper:
+ + [NT] New file.
+
* data/*:
+ [NT] Refresh with tools/data from sid.
* data/scripts/interpreters:
--
Debian package checker
Reply to: