Bug#955268: udd watch: "429 too many requests" from GitHub

[Xavier <yadd@debian.org>]

Le 29/03/2020 à 17:31, Xavier a écrit :
> Le 29/03/2020 à 16:39, Lucas Nussbaum a écrit :
>> Hi,
>>
>> On 29/03/20 at 00:28 +0100, Adam Borowski wrote:
>>> On Sat, Mar 28, 2020 at 11:00:51PM +0000, Rebecca N. Palmer wrote:
>>>> In watchfile debian/watch, reading webpage
>>>>   https://github.com/statsmodels/statsmodels/tags failed: 429 too many
>>>> requests
>>>>
>>>> Which packages have this error seems to vary over time, but it seems to be
>>>> common (~50-90% of GitHub upstreams - e.g. see https://qa.debian.org/developer.php?email=pkg-go-maintainers%40lists.alioth.debian.org
>>>> but note that a dash can also mean "no debian/watch" or another error).
>>>>
>>>> I don't know if this was triggered by GitHub introducing/lowering their
>>>> limit, by a bug on our end making more requests than we intended to, or by
>>>> our generally increasing size.  (Currently 9490 GitHub-hosted packages each
>>>> checked every 3 days
>>>
>>> IIRC with an access token the limit is 5000 queries per hour.  Without a
>>> token, much, much less.
>>
>> The UDD code calls uscan. Is there a way for uscan to use an access
>> token?
>>
>> Lucas
> 
> Hi,
> 
> not for now, but I can add this feature. Just to insert an additional
> header ?


This little diff adds --http-header option. To use it:

uscan --http-header=Access-Token=qwertyuiop

Sounds good for you ?

diff --git a/lib/Devscripts/Config.pm b/lib/Devscripts/Config.pm
index 636f660b..58730b16 100644
--- a/lib/Devscripts/Config.pm
+++ b/lib/Devscripts/Config.pm
@@ -316,7 +316,8 @@ sub parse_command_line {
         my $name = $kname;
         $kname =~ s/-/_/g;
         if (defined $opts->{$name}) {
-            next if (ref $opts->{$name} and !@{ $opts->{$name} });
+            next if (ref $opts->{$name} eq 'ARRAY' and !@{ $opts->{$name} });
+            next if (ref $opts->{$name} eq 'HASH' and !%{ $opts->{$name} });
             if (defined $check) {
                 if (not(ref $check)) {
                     $check
diff --git a/lib/Devscripts/Uscan/Config.pm b/lib/Devscripts/Uscan/Config.pm
index 8e233a33..450a43f9 100644
--- a/lib/Devscripts/Uscan/Config.pm
+++ b/lib/Devscripts/Uscan/Config.pm
@@ -51,6 +51,7 @@ has log                      => (is => 'rw');
 has orig                     => (is => 'rw');
 has package                  => (is => 'rw');
 has pasv                     => (is => 'rw');
+has http_header              => (is => 'rw', default => sub { {} });
 
 # repack to .tar.$zsuffix if 1
 has repack     => (is => 'rw');
@@ -111,6 +112,8 @@ use constant keys => [
     ['uversion|upstream-version=s'],
     ['watchfile=s'],
     # 2.3 - More complex options
+    # http headers (#955268)
+    ['http-header=s', undef, undef, sub { {} }],
 
     # "download" and its aliases
     [
diff --git a/lib/Devscripts/Uscan/Downloader.pm b/lib/Devscripts/Uscan/Downloader.pm
index abd07e7a..b23b249e 100644
--- a/lib/Devscripts/Uscan/Downloader.pm
+++ b/lib/Devscripts/Uscan/Downloader.pm
@@ -74,6 +74,10 @@ has user_agent => (
 
 has ssl => (is => 'rw', default => sub { $haveSSL });
 
+has headers => (
+    is      => 'ro',
+    default => sub { {} });
+
 sub download ($$$$$$$$) {
     my ($self, $url, $fname, $optref, $base, $pkg_dir, $pkg, $mode) = @_;
     my ($request, $response);
@@ -91,10 +95,15 @@ sub download ($$$$$$$$) {
         my $headers = HTTP::Headers->new;
         $headers->header('Accept'  => '*/*');
         $headers->header('Referer' => $base);
+        foreach my $k (keys %{ $self->headers }) {
+            $headers->header($k => $self->headers->{$k});
+            uscan_verbose "Set custom header $k => $self->{headers}->{$k}";
+        }
         $request  = HTTP::Request->new('GET', $url, $headers);
         $response = $self->user_agent->request($request, $fname);
         if (!$response->is_success) {
-            uscan_warn((defined $pkg_dir ? "In directory $pkg_dir, d" : "D")
+            uscan_warn(
+                  (defined $pkg_dir ? "In directory $pkg_dir, d" : "D")
                 . "ownloading\n  $url failed: "
                   . $response->status_line);
             return 0;
@@ -104,7 +113,8 @@ sub download ($$$$$$$$) {
         $request  = HTTP::Request->new('GET', "$url");
         $response = $self->user_agent->request($request, $fname);
         if (!$response->is_success) {
-            uscan_warn((defined $pkg_dir ? "In directory $pkg_dir, d" : "D")
+            uscan_warn(
+                  (defined $pkg_dir ? "In directory $pkg_dir, d" : "D")
                 . "ownloading\n  $url failed: "
                   . $response->status_line);
             return 0;
diff --git a/lib/Devscripts/Uscan/WatchFile.pm b/lib/Devscripts/Uscan/WatchFile.pm
index 71309cee..2f041554 100644
--- a/lib/Devscripts/Uscan/WatchFile.pm
+++ b/lib/Devscripts/Uscan/WatchFile.pm
@@ -126,11 +126,13 @@ has downloader => (
     is      => 'ro',
     lazy    => 1,
     default => sub {
+        my %h;
         Devscripts::Uscan::Downloader->new({
             timeout => $_[0]->config->timeout,
             agent   => $_[0]->config->user_agent,
             pasv    => $_[0]->config->pasv,
             destdir => $_[0]->config->destdir,
+            headers => $_[0]->config->http_header,
         });
     },
 );