Hello, pirmadienis 19 Gegužė 2008, Raphael Hertzog rašė: > I also remember you started optimizing dpkg-shlibdeps's memory usage, but > I haven't seen any followup since my initial answer, are you still > interested in working on that? Yes, I am. I just never got around to sending fixed version. > The output of find_symbols_file depend not only on $pkg but also on > $soname and $lib. You can't assume that you can reuse the same symbols > file simply because a previous call of find_symbols with the same $kg > returned something. The key of %dpkg_symfile_cache should really be > $dpkg_symfile and not $pkg. > Why are you using $pkg and $lib as key for this cache? $lib should be > enough as there's only one objdump output for a given binary file... > If you want however, you can add a cache to the function > symfile_has_soname() to avoid parsing the same file multiple times. All 3 issues resolved. Patch attached (against lenny branch, applies to master without problems). I have been using this patched dpkg quite heavily lately, no problems noticed. -- Modestas Vainius <modestas@vainius.eu>
From 542003e693ba1d3064d5ee9aa5a95de101092c84 Mon Sep 17 00:00:00 2001
From: Modestas Vainius <modestas@vainius.eu>
Date: Fri, 5 Dec 2008 13:43:37 +0200
Subject: [PATCH] Optimize dpkg-shlibdeps by caching symbol file and objdump objects
This patch optimizes dpkg-shlibdeps by caching parsed symbols files and
objdump objects. This way neither of the libraries or symbols files are
parsed more than once. This patch significantly improves performance of
dpkg-shlibdeps bringing it near to performance levels of << 1.14.8
dpkg-shlibdeps without loosing any of new functionally at all. Memory
requirements are reduced too.
This patch SHOULD NOT change the end result of dpkg-shlibdeps. If it
does, it is a bug.
Signed-off-by: Modestas Vainius <modestas@vainius.eu>
---
scripts/Dpkg/Shlibs/Objdump.pm | 12 +++++++---
scripts/Dpkg/Shlibs/SymbolFile.pm | 17 ++++++++++++++++
scripts/dpkg-shlibdeps.pl | 39 +++++++++++++++++++++++++++++++-----
3 files changed, 58 insertions(+), 10 deletions(-)
diff --git a/scripts/Dpkg/Shlibs/Objdump.pm b/scripts/Dpkg/Shlibs/Objdump.pm
index 0cb7ddf..41a0f52 100644
--- a/scripts/Dpkg/Shlibs/Objdump.pm
+++ b/scripts/Dpkg/Shlibs/Objdump.pm
@@ -30,10 +30,8 @@ sub new {
return $self;
}
-sub parse {
- my ($self, $file) = @_;
- my $obj = Dpkg::Shlibs::Objdump::Object->new($file);
-
+sub add_object {
+ my ($self, $obj) = @_;
my $id = $obj->get_id;
if ($id) {
$self->{objects}{$id} = $obj;
@@ -41,6 +39,12 @@ sub parse {
return $id;
}
+sub parse {
+ my ($self, $file) = @_;
+ my $obj = Dpkg::Shlibs::Objdump::Object->new($file);
+
+ return $self->add_object($obj);
+}
sub locate_symbol {
my ($self, $name) = @_;
diff --git a/scripts/Dpkg/Shlibs/SymbolFile.pm b/scripts/Dpkg/Shlibs/SymbolFile.pm
index c79db1e..1f7494c 100644
--- a/scripts/Dpkg/Shlibs/SymbolFile.pm
+++ b/scripts/Dpkg/Shlibs/SymbolFile.pm
@@ -175,6 +175,23 @@ sub load {
delete $seen->{$file};
}
+sub merge_from_symfile {
+ my ($self, $src) = @_;
+ while (($soname, $srcobj) = each(%{$src->{objects}})) {
+ if (exists $self->{objects}{$soname}) {
+ # Update/override infos only
+ $self->{objects}{$soname}{deps} = $srcobj->{deps};
+ } else {
+ # Shallow copy the soname object (because deps can be replaced later)
+ my %obj;
+ while (($key, $val) = each(%$srcobj)) {
+ $obj{$key} = $val;
+ }
+ $self->{objects}{$soname} = \%obj;
+ }
+ }
+}
+
sub save {
my ($self, $file, $with_deprecated) = @_;
$file = $self->{file} unless defined($file);
diff --git a/scripts/dpkg-shlibdeps.pl b/scripts/dpkg-shlibdeps.pl
index e9e7a75..66a5422 100755
--- a/scripts/dpkg-shlibdeps.pl
+++ b/scripts/dpkg-shlibdeps.pl
@@ -128,6 +128,11 @@ my %global_soname_notfound;
my %global_soname_used;
my %global_soname_needed;
+# Symfile and objdump caches
+my %dpkg_symfile_cache;
+my %dpkg_objdump_cache;
+my %symfile_has_soname_cache;
+
my $cur_field;
foreach my $file (keys %exec) {
$cur_field = $exec{$file};
@@ -194,10 +199,16 @@ foreach my $file (keys %exec) {
if ($packagetype eq "deb") {
# Use fine-grained dependencies only on real deb
$dpkg_symfile = find_symbols_file($pkg, $soname, $lib);
- if (defined $dpkg_symfile) {
- # Load symbol information
- print "Using symbols file $dpkg_symfile for $soname\n" if $debug;
- $symfile->load($dpkg_symfile);
+ if (defined($dpkg_symfile)) {
+ if (exists $dpkg_symfile_cache{$dpkg_symfile}) {
+ print "Using symbols file $dpkg_symfile (cached) for $soname\n" if $debug;
+ } else {
+ # Load symbol information
+ print "Using symbols file $dpkg_symfile for $soname\n" if $debug;
+ $dpkg_symfile_cache{$dpkg_symfile} = new Dpkg::Shlibs::SymbolFile();
+ $dpkg_symfile_cache{$dpkg_symfile}->load($dpkg_symfile);
+ }
+ $symfile->merge_from_symfile($dpkg_symfile_cache{$dpkg_symfile});
}
}
if (defined($dpkg_symfile) && $symfile->has_object($soname)) {
@@ -214,13 +225,25 @@ foreach my $file (keys %exec) {
}
} else {
# No symbol file found, fall back to standard shlibs
- my $id = $dumplibs_wo_symfile->parse($lib);
+ my $id;
+ my $libobj;
+ if (exists $dpkg_objdump_cache{$lib}) {
+ $libobj = $dpkg_objdump_cache{$lib};
+ # We don't want to process the same lib more than once (redundant)
+ next if ($dumplibs_wo_symfile->get_object($libobj->get_id()));
+ $id = $dumplibs_wo_symfile->add_object($dpkg_objdump_cache{$lib});
+ print "Using objdump (cached) for $soname (file $lib)\n" if $debug;
+ } else {
+ $id = $dumplibs_wo_symfile->parse($lib);
+ $libobj = $dumplibs_wo_symfile->get_object($id);
+ $dpkg_objdump_cache{$lib} = $libobj;
+ print "Using objdump for $soname (file $lib)\n" if $debug;
+ }
if (($id ne $soname) and ($id ne $lib)) {
warning(_g("%s has an unexpected SONAME (%s)"), $lib, $id);
$alt_soname{$id} = $soname;
}
push @soname_wo_symfile, $soname;
- my $libobj = $dumplibs_wo_symfile->get_object($id);
# Only try to generate a dependency for libraries with a SONAME
if ($libobj->is_public_library() and not
add_shlibs_dep($soname, $pkg, $lib)) {
@@ -659,12 +682,16 @@ sub find_symbols_file {
sub symfile_has_soname {
my ($file, $soname) = @_;
+
+ return 1 if (exists $symfile_has_soname_cache{$file}{$soname});
+
open(SYM_FILE, "<", $file) ||
syserr(_g("cannot open file %s"), $file);
my $result = 0;
while (<SYM_FILE>) {
if (/^\Q$soname\E /) {
$result = 1;
+ $symfile_has_soname_cache{$file}{$soname} = 1;
last;
}
}
--
1.5.6.5
Attachment:
signature.asc
Description: This is a digitally signed message part.