Re: easy task for Debian derivatives census code

[Fabrizio Furnari <fab.furnari@gmail.com>]

Hi, here's attached my (very very simple patch).

Cannot test it live but should work... :)

Fabrizio

On Tue, Apr 7, 2015 at 1:18 PM, Paul Wise <pabs@debian.org> wrote:

Hi all,

For those of you who know Python, an easy task would be to migrate the
Debian derivatives census code from the urllib2 module to the requests
module and add resuming of downloads in the case of broken TCP
connections. Currently the patch generation code is failing to generate
a patch for the nvidia-graphics-drivers package in SteamOS as the Akamai
infrastructure always seems to close randomly connections mid-stream.
Switching to the requests module and adding autoresume would fix that.

--
bye,
pabs

https://wiki.debian.org/PaulWise

From 5d07ff642e6663a9536b769e5392b17d3849edba Mon Sep 17 00:00:00 2001
From: Fabrizio Furnari <fab.furnari@gmail.com>
Date: Wed, 8 Apr 2015 16:57:11 +0200
Subject: [PATCH] moving from urllib2 to requests

---
 bin/compare-source-package-list | 43 ++++++++++++++++++-----------------------
 1 file changed, 19 insertions(+), 24 deletions(-)

diff --git a/bin/compare-source-package-list b/bin/compare-source-package-list
index 08f07fe..13e6afc 100755
--- a/bin/compare-source-package-list
+++ b/bin/compare-source-package-list
@@ -58,8 +58,7 @@
 import re
 import os
 import sys
-import httplib
-import urllib2
+import requests
 import hashlib
 import shutil
 import logging
@@ -304,10 +303,10 @@ def download_and_check_hash(url, dir, hash, hash_type):
 		path = hash_path(dir,hash)
 		logging.debug('downloading %s', url)
 		makedirs(parent)
-		headers = { 'User-Agent' : user_agent }
-		req = urllib2.Request(url, None, headers)
-		u = urllib2.urlopen(req, None, timeout)
-		data = u.read()
+		headers = {'user-agent' : user_agent}
+		req = requests.get(url, headers=headers, timeout=timeout)
+		req.raise_for_status() # to catch HTTP errors
+		data = req.text
 		if hash_type == 'sha256':
 			data_hash = hashlib.sha256(data).hexdigest()
 		elif hash_type == 'md5sum':
@@ -337,14 +336,12 @@ def download_and_check_hash(url, dir, hash, hash_type):
 			symlink(os.path.relpath(sha1_path, os.path.dirname(path)), path)
 			logging.debug('does not exist in snapshot sha1 cache: %s %s %s %s', hash_type, hash, sha1, url)
 			return (False, sha1)
-	except urllib2.URLError, e:
-		if hasattr(e, 'reason'): reason = e.reason
-		elif hasattr(e, 'code'): reason = e.code
-		else: reason = e
-		logging.warning('unable to download hash file, ignoring: %s %s', reason, url)
+	except requests.ConnectionError as e:
+		logging.warning('unable to download hash file, ignoring: %s %s', e, url)
 		return ('unknown', None)
-	except httplib.HTTPException, e:
-		logging.warning('unable to download hash file, ignoring: %s %s', repr(e), url)
+	except requests.HTTPError, e:
+		st = req.status_code
+		logging.warning('unable to download hash file, ignoring: %s: %s %s', st, e, url)
 		return ('unknown', None)
 	except socket.error, e:
 		logging.warning('unable to download hash file, ignoring: %s %s', e, url)
@@ -356,10 +353,10 @@ def download_sha1(url, dir, sha1):
 		path = hash_path(dir,sha1)
 		logging.debug('downloading sha1: %s %s', sha1, url)
 		makedirs(parent)
-		headers = { 'User-Agent' : user_agent }
-		req = urllib2.Request(url, None, headers)
-		u = urllib2.urlopen(req, None, timeout)
-		data = u.read()
+		headers = { 'user-agent' : user_agent }
+		req = requests.get(url, headers=headers, timeout=timeout)
+		req.raise_for_status() # to catch HTTP errors
+		data = u.text
 		data_sha1 = hashlib.sha1(data).hexdigest()
 		if data_sha1 == sha1:
 			logging.debug('correct sha1 for downloaded file, saving: %s %s', sha1, url)
@@ -370,14 +367,12 @@ def download_sha1(url, dir, sha1):
 		else:
 			logging.warning('incorrect sha1 for downloaded file, ignoring: %s != %s %s', sha1, data_sha1, url)
 			return ('unknown', None)
-	except urllib2.URLError, e:
-		if hasattr(e, 'reason'): reason = e.reason
-		elif hasattr(e, 'code'): reason = e.code
-		else: reason = e
-		logging.warning('unable to download sha1 file, ignoring: %s %s', reason, url)
+	except requests.ConnectionError as e:
+		logging.warning('unable to download sha1 file, ignoring: %s %s', e, url)
 		return ('unknown', None)
-	except httplib.HTTPException, e:
-		logging.warning('unable to download hash file, ignoring: %s %s', repr(e), url)
+	except requests.HTTPError as e:
+		st = req.status_code
+		logging.warning('unable to download hash file, ignoring: %s: %s %s', st, e, url)
 		return ('unknown', None)
 	except socket.error, e:
 		logging.warning('unable to download hash file, ignoring: %s %s', e, url)
-- 
2.0.1