Bug#781509: apt-transport-https: reports uninitialized size → W: Size of file [FILE] is not what the server reported

[Anders Kaseorg <andersk@mit.edu>]

Package: apt-transport-https
Version: 0.9.15.3
Tags: patch

When there are many https repositories in sources.list, ‘apt-get update’ 
often spews warnings like this:

W: Size of file /var/lib/apt/lists/partial/debathena.mit.edu_apt_dists_quantal_debathena_source_Sources.bz2 is not what the server reported 31194 16209
W: Size of file /var/lib/apt/lists/partial/debathena.mit.edu_apt_dists_saucy_debathena_source_Sources.bz2 is not what the server reported 31029 24393
W: Size of file /var/lib/apt/lists/partial/debathena.mit.edu_apt_dists_trusty_debathena_source_Sources.bz2 is not what the server reported 26334 16209
W: Size of file /var/lib/apt/lists/partial/debathena.mit.edu_apt_dists_utopic_debathena_source_Sources.bz2 is not what the server reported 26237 16209
W: Size of file /var/lib/apt/lists/partial/debathena.mit.edu_apt_dists_vivid_debathena_source_Sources.bz2 is not what the server reported 14 251

For more details, see 
https://bugs.launchpad.net/ubuntu/+source/apt/+bug/807303.  The underlying 
problem is that HttpsMethod::write_data was sometimes calling URIStart 
with Res.Size uninitialized.  Here is a patch against 1.0.9.7 with a 
satisfying diffstat.

Anders

-- 8< --
From: Anders Kaseorg <andersk@mit.edu>
Subject: [PATCH] HttpsMethod: Call URIStart in parse_header, not write_data

There was no guarantee that progress_callback was called before the
first write_data, so URIStart was sometimes called with Res.Size
uninitialized.  This is much simpler anyway.

Signed-off-by: Anders Kaseorg <andersk@mit.edu>
---
 methods/https.cc | 23 ++---------------------
 methods/https.h  |  3 ---
 2 files changed, 2 insertions(+), 24 deletions(-)

diff --git a/methods/https.cc b/methods/https.cc
index 3a5981b..a423b3d 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -68,6 +68,8 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
 
       me->File->Truncate(me->Server->StartPos);
       me->File->Seek(me->Server->StartPos);
+      me->Res.Size = me->Server->Size;
+      me->URIStart(me->Res);
    }
    else if (me->Server->HeaderLine(line) == false)
       return 0;
@@ -85,29 +87,12 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
    if (me->Server->JunkSize != 0)
       return buffer_size;
 
-   if (me->ReceivedData == false)
-   {
-      me->URIStart(me->Res);
-      me->ReceivedData = true;
-   }
-
    if(me->File->Write(buffer, buffer_size) != true)
       return false;
 
    return buffer_size;
 }
 
-int
-HttpsMethod::progress_callback(void *clientp, double dltotal, double /*dlnow*/,
-                             double /*ultotal*/, double /*ulnow*/)
-{
-   HttpsMethod *me = (HttpsMethod *)clientp;
-   if(dltotal > 0 && me->Res.Size == 0) {
-      me->Res.Size = (unsigned long long)dltotal;
-   }
-   return 0;
-}
-
 // HttpsServerState::HttpsServerState - Constructor			/*{{{*/
 HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * /*Owner*/) : ServerState(Srv, NULL)
 {
@@ -183,7 +168,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
    char curl_errorstr[CURL_ERROR_SIZE];
    URI Uri = Itm->Uri;
    string remotehost = Uri.Host;
-   ReceivedData = false;
 
    // TODO:
    //       - http::Pipeline-Depth
@@ -201,10 +185,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
    curl_easy_setopt(curl, CURLOPT_WRITEHEADER, this);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, this);
-   curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progress_callback);
-   curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, this);
    // options
-   curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
    curl_easy_setopt(curl, CURLOPT_FILETIME, true);
    // only allow curl to handle https, not the other stuff it supports
    curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTPS);
diff --git a/methods/https.h b/methods/https.h
index 411b714..e3dd709 100644
--- a/methods/https.h
+++ b/methods/https.h
@@ -60,13 +60,10 @@ class HttpsMethod : public pkgAcqMethod
    virtual bool Fetch(FetchItem *);
    static size_t parse_header(void *buffer, size_t size, size_t nmemb, void *userp);
    static size_t write_data(void *buffer, size_t size, size_t nmemb, void *userp);
-   static int progress_callback(void *clientp, double dltotal, double dlnow, 
-				double ultotal, double ulnow);
    void SetupProxy();
    CURL *curl;
    FetchResult Res;
    HttpsServerState *Server;
-   bool ReceivedData;
 
    public:
    FileFd *File;
-- 
2.4.0.rc0