APT now has a ftp method
Hi,
Ladies and Gentelmen, APT now has a (minimally) tested ftp
method.
With an inspired last minute debug session on IRC by Adam
Heath (I was getting the remote size before setting the connection to
binary, and the darned thing would to CRLF conversions, so remote
size would always be larger than the local copy). Setting the
connection to binary early helped.
manoj
__> ~/src/ftp-get ftp://ftp.debian.org/debian/README README
F ftp://ftp.debian.org/debian/README
I Connecting to ftp.debian.org
I setting type to binary
I getting modified time for /debian/README
I getting /debian/README
I resetting modified time for README
I Closing connection
I Trying to get md5sum for README
M 99e7acd0783cfb1b29de5d09c761241a
__> ~/src/ftp-get ftp://ftp.debian.org/debian/README README
F ftp://ftp.debian.org/debian/README
I Connecting to ftp.debian.org
I setting type to binary
I getting modified time for /debian/README
I Local file README exists
I local modified time matches
I getting size for /debian/README
I Already have README with size 1057
--
"Your butt is mine." Michael Jackson, Bad
Manoj Srivastava <srivasta@acm.org> <http://www.datasync.com/%7Esrivasta/>
Key C7261095 fingerprint = CB D9 F4 12 68 07 E4 05 CC 2D 27 12 1D F5 E8 6E
#!/usr/bin/perl -w
# -*- Mode: Perl -*-
# ftp-get ---
# Author : Manoj Srivastava ( srivasta@tiamat.datasync.com )
# Created On : Wed Mar 25 13:53:12 1998
# Created On Node : tiamat.datasync.com
# Last Modified By : Manoj Srivastava
# Last Modified On : Wed May 6 03:33:15 1998
# Last Machine Used: tiamat.datasync.com
# Update Count : 170
# Status : Unknown, Use with caution!
# HISTORY :
# Description :
#
# <Culus> Manoj: on sigint you should abort what you are doing and write the
# proper date to the file
require 5.004;
use strict;
use diagnostics;
use Net::FTP;
use File::Listing qw(parse_dir);
use File::Basename;
use File::stat;
use URI::URL;
use MD5;
use vars qw(%Configuration);
%Configuration =
(
'ConfigFile' => "/etc/apt/ftp.conf",
'Firewall' => "",
'ProxyLogName' => "",
'ProxyPassword' => "",
'TimeOut' => 60,
'Passive' => 0,
'MaxReTry' => 2
);
my $remote_mdtm; # Here since we need it in the sighandler
my $current_filename;
sub catch_zap {
my $signame = shift;
if ($remote_mdtm && $current_filename) {
my $now = time;
my $ret = utime $now, $remote_mdtm, $current_filename;
if ($ret != 1) {
print "E Failed to reset the modified time for $current_filename\n";
}
print "I Dying with signal SIG$signame\n";
exit 0;
}
}
sub read_config {
my %params = @_;
my $lineno = 0;
die "Required parameter 'Config File' missing"
unless $params{'Config File'};
return unless -r $params{'Config File'};
open(CONFIG, "$params{'Config File'}") ||
die "Could not open $params{'Config File'}:$!";
while (<CONFIG>) {
chomp;
$lineno++;
s/\#.*//og;
next if /^\s*$/og;
$_ .= ";" unless /;\s*$/;
if (/^\s*([^=]+)\s*=\s*(\S.*)$/o) {
if (defined $Configuration{$1}) {
$Configuration{$1}=$2 if $2;
}
else {
print STDERR "E Error parsing ftp config file!\n";
print STDERR "E $lineno:$_\n";
}
}
}
}
sub set_defaults {
my %params = @_;
die "Required parameter 'Config File' missing"
unless $params{'Config File'};
return unless -r $params{'Config File'};
# Well, let them over ride stuff and feed us defaults
read_config('Config File' => $params{'Config File'});
if (!$Configuration{'Firewall'} && $ENV{'FTP_FIREWALL'}) {
$Configuration{'Firewall'} = $ENV{'FTP_FIREWALL'};
}
if (!$Configuration{'Passive'} && $ENV{'FTP_PASSIVE'}) {
$Configuration{'Passive'} = $ENV{'FTP_PASSIVE'};
}
}
sub handle_requests {
my %params = @_;
my $status;
die "Required parameter 'Configuration' missing"
unless $params{'Configuration'};
# well, now let us collate all the pairs into lists of files from a host
my @tmparray = ();
push @tmparray, @{$params{'Configuration'}{'Request'}};
while (@tmparray) {
my $try_number = 0;
my $uri = shift @tmparray; # We need to parse this
my $file = shift @tmparray; # Local file name
my $url = new URI::URL "$uri";
print "F ", $url->as_string, "\n"; # Tell apt we are looking at this uri now
if ($url->scheme !~ /ftp/io) {
print "E $uri is not a ftp URI.\n";
next;
}
# well, see if we have the base file name here in this dir, or
# not. $file and $name should be the same, really.
my ($name,$path,$suffix) = fileparse($file,());
my $localfile;
my $md5 = new MD5;
TRY_OPEN:
while (1) {
my @args = ();
$try_number++;
if (-e $name) { $status = stat($name);
$localfile = $name; }
elsif (-e "../$name") { $status = stat("../$name");
$localfile = "../$name"; }
else { $status = undef;
$localfile = $name; }
push @args, $url->host;
push @args, 'Passive' => $Configuration{'Passive'};
push @args, 'Timeout' => $Configuration{'Timeout'};
push @args, 'Firewall' => $Configuration{'Firewall'} if
$Configuration{'Firewall'};
push @args, 'Port' => $url->port if $url->port;
print "I Connecting to ", $url->netloc, "\n";
my $ftp = Net::FTP->new(@args);
if(!$ftp || !$ftp->ok) {
if ($try_number >= $Configuration{'MaxReTry'}) {
print "E Failed to connect to ", $url->netloc, ":" . $ftp->message;
last TRY_OPEN;
}
else {
print "I Re trying to connect to ", $url->netloc, ":" . $ftp->message;
next TRY_OPEN;
}
}
# Authorize to the firewall, if needed.
if ($Configuration{'Firewall'} && $Configuration{'ProxyLogName'}) {
my $ret = $ftp->authorize($Configuration{'ProxyLogName'},
$Configuration{'ProxyPassword'});
if (! $ret) {
if ($try_number >= $Configuration{'MaxReTry'}) {
print "E Failed to authorize to $Configuration{'Firewall'}:"
. $ftp->message;
last TRY_OPEN;
}
else {
print "I Re trying to authorize to $Configuration{'Firewall'}:"
. $ftp->message;
next TRY_OPEN;
}
}
}
# Login
@args = ();
push @args, ($url->user || 'anonymous');
push @args, $url->password if $url->password;
if (! $ftp->login(@args)) {
if ($try_number >= $Configuration{'MaxReTry'}) {
print "E Failed to login to ", $url->host, ":" . $ftp->message;
last TRY_OPEN;
}
else {
print "I Re trying to login to ", $url->host, ":" . $ftp->message;
next TRY_OPEN;
}
}
# set type to binary
print "I setting type to binary\n";
if (! defined $ftp->binary()) {
if ($try_number >= $Configuration{'MaxReTry'}) {
print "E Failed to set type to binary:" . $ftp->message;
$ftp->quit;
last TRY_OPEN;
}
else {
print "I Re trying to set type to binary:" . $ftp->message;
$ftp->quit;
next TRY_OPEN;
}
}
print "I getting modified time for ", $url->epath, "\n";
$remote_mdtm = $ftp->mdtm($url->epath);
my $remote_size = undef;
my $restart_at = 0;
# Only do this if we have a local file ...
if (defined $status) {
print "I Local file $localfile exists\n";
if (! defined $remote_mdtm) {
print "I ", $url->host, " does not report modified time\n";
}
else {
if ($status->mtime == $remote_mdtm ) {
# Hmm. Same time Check sizes now.
$restart_at = $status->size;
print "I local modified time matches\n";
print "I getting size for ", $url->epath, "\n";
$remote_size = $ftp->size($url->epath);
if (! defined $remote_size) {
print "I Could not get size of ", $url->epath, "\n";
print "I Resuming $localfile at ", $status->size, "\n";
}
else {
if ($status->size == $remote_size) {
print "I Already have $localfile with size ",
$status->size, "\n";
$restart_at = 0;
$ftp->quit;
last TRY_OPEN;
}
else {
print "I Resuming $localfile at ", $status->size,
"/$remote_size\n";
}
}
}
}
}
# get the file
print "I getting ", $url->epath, "\n";
@args =();
push @args, $url->epath, $localfile;
push @args, $restart_at if $restart_at;
if ($remote_mdtm) { $current_filename = $localfile; }
my $ftp_retval = $ftp->get(@args);
print "I resetting modified time for $localfile\n";
if (defined $remote_mdtm) {
my $now = time;
my $ret = utime $now, $remote_mdtm, $localfile;
if ($ret != 1) {
print "E Failed to reset the modified time for $localfile\n";
}
}
$current_filename = '';
if (! $ftp_retval) {
if ($try_number >= $Configuration{'MaxReTry'}) {
print "E Failed to get ", $url->epath, ":" . $ftp->message;
$ftp->quit;
last TRY_OPEN;
}
else {
print "I Re trying to get ", $url->epath, ":" . $ftp->message;
$ftp->quit;
next TRY_OPEN;
}
}
print "I Closing connection\n";
$ftp->quit() ||
print "E Failed to close connection cleanly\n";
print "I Trying to get md5sum for $localfile\n";
open (FILE, $localfile) or do {
print "E Failed to open $localfile for MD5 check\n";
last TRY_OPEN;
};
seek(FILE, 0, 0);
$md5->reset;
$md5->addfile(\*FILE);
print "M ", $md5->hexdigest, "\n";
$md5->reset;
last TRY_OPEN;
}
}
}
sub main {
my @params = @_;
# we must get URI local name pairs ...
if (! ($#params & 1)) {
print "E Unbalanced number of arguments.\n";
exit 0;
}
# defaults
set_defaults('Config File' => $Configuration{'ConfigFile'});
push @{$Configuration{'Request'}}, @params;
# set up the signal handler
$SIG{INT} = \&catch_zap;
$SIG{HUP} = \&catch_zap;
$SIG{QUIT} = \&catch_zap;
$SIG{ABRT} = \&catch_zap;
$SIG{TERM} = \&catch_zap;
handle_requests('Configuration' => \%Configuration);
}
&main(@ARGV);
1;
__END__
# Local Variables:
# mode: cperl
# End:
--
To UNSUBSCRIBE, email to deity-request@lists.debian.org
with a subject of "unsubscribe". Trouble? Contact listmaster@lists.debian.org
Reply to: