Hi, On Wed, Aug 12, 2015 at 02:22:00PM +0200, David Kalnischkies wrote: > 2. Hashes for the compressed pdiff files in .diff/Index […] > 3. different Hash algorithms in .diff/Index […] > (Unfortunately my python knowledge is close to non-existent, so I will > refrain from trying to write a patch for your, daks and my sanity.) I have warned you guys, but nobody listened… so here we are: Attached are 2 patches realising 2. and 3. (in the form of hardcoding SHA256), which have received minimal testing and are monkey-coded in a "heh, the python interpreter doesn't throw error messages at me anymore!" style… (and yes, even I can see that magic numbers are wrong, but you had that in before so I made it only slightly worse…) While 1. sounds easy (adding a single field, how hard can it be, right…) I wasted too much time on these two already without adding new database fields, more python and stuff so I am not going to touch this. P.S.: setup/README is quite useful; I had only a few PEBCAKs, but I have to wonder why it isn't helping me figure out how to get dak.py to work. I finally figured out that "cd dak; rm -rf daklib; ln -s ../daklib ." and python-sqlalchemy/stable work out okay, but that feels oh so wrong… Best regards David Kalnischkies
From 5b4fb4256869052925dc21ff2bade19fd0ca8427 Mon Sep 17 00:00:00 2001 From: David Kalnischkies <david@kalnischkies.de> Date: Wed, 28 Oct 2015 20:36:20 +0100 Subject: [PATCH 1/2] include hashsum for compressed patch file in .diff/Index APT prefers to download files it has hashes for to check it actually got the file it wanted instead of some man-in-the-middled file (by an attacker or simply by a webportal returning a login mask for every request). Giving it the hash of the compressed file saves APT from needing to uncompress the patch before being able to verify its integrity. Signed-off-by: David Kalnischkies <david@kalnischkies.de> --- dak/generate_index_diffs.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/dak/generate_index_diffs.py b/dak/generate_index_diffs.py index 6fe9436..8451371 100755 --- a/dak/generate_index_diffs.py +++ b/dak/generate_index_diffs.py @@ -127,10 +127,13 @@ class Updates: x = f.readline() if not x or x[0] != " ": break l = x.split() - if not self.history.has_key(l[2]): - self.history[l[2]] = [None,None] - self.history_order.append(l[2]) - self.history[l[2]][ind] = (l[0], int(l[1])) + fname = l[2] + if fname.endswith('.gz'): + fname = fname[:-3] + if not self.history.has_key(fname): + self.history[fname] = [None,None,None] + self.history_order.append(fname) + self.history[fname][ind] = (l[0], int(l[1])) return x while x: @@ -148,6 +151,10 @@ class Updates: x = read_hashs(1,f,self) continue + if l[0] == "SHA1-Download:": + x = read_hashs(2,f,self) + continue + if l[0] == "Canonical-Name:" or l[0]=="Canonical-Path:": self.can_path = l[1] @@ -183,6 +190,10 @@ class Updates: out.write("SHA1-Patches:\n") for h in l: out.write(" %s %7d %s\n" % (hs[h][1][0], hs[h][1][1], h)) + out.write("SHA1-Download:\n") + for h in l: + if hs[h][2]: + out.write(" %s %7d %s.gz\n" % (hs[h][2][0], hs[h][2][1], h)) def create_temp_file(r): f = tempfile.TemporaryFile() @@ -270,7 +281,11 @@ def genchanges(Options, outdir, oldfile, origfile, maxdiffs = 56): difsizesha1 = sizesha1(difff) difff.close() - upd.history[patchname] = (oldsizesha1, difsizesha1) + difffgz = open(difffile + ".gz", "r") + difgzsizesha1 = sizesha1(difffgz) + difffgz.close() + + upd.history[patchname] = (oldsizesha1, difsizesha1, difgzsizesha1) upd.history_order.append(patchname) upd.filesizesha1 = newsizesha1 -- 2.6.2
From e6611a4135fd4e70860964b75f9cf91510900a2a Mon Sep 17 00:00:00 2001 From: David Kalnischkies <david@kalnischkies.de> Date: Wed, 28 Oct 2015 21:20:30 +0100 Subject: [PATCH 2/2] generate SHA256 hashes for pdiff patches Preparing for phasing out SHA1 means first and for most eliminating hard dependencies on it, so lets include SHA256 hashes in .diff/Index as new fields as this keeps backward compatibility. Signed-off-by: David Kalnischkies <david@kalnischkies.de> --- dak/generate_index_diffs.py | 87 +++++++++++++++++++++++++++++++++------------ 1 file changed, 65 insertions(+), 22 deletions(-) diff --git a/dak/generate_index_diffs.py b/dak/generate_index_diffs.py index 8451371..ba5e37d 100755 --- a/dak/generate_index_diffs.py +++ b/dak/generate_index_diffs.py @@ -115,14 +115,14 @@ class Updates: self.history_order = [] self.max = max self.readpath = readpath - self.filesizesha1 = None + self.filesizehashes = None if readpath: try: f = open(readpath + "/Index") x = f.readline() - def read_hashs(ind, f, self, x=x): + def read_hashs(ind, hashind, f, self, x=x): while 1: x = f.readline() if not x or x[0] != " ": break @@ -133,7 +133,12 @@ class Updates: if not self.history.has_key(fname): self.history[fname] = [None,None,None] self.history_order.append(fname) - self.history[fname][ind] = (l[0], int(l[1])) + if not self.history[fname][ind]: + self.history[fname][ind] = (int(l[1]), None, None) + if hashind == 1: + self.history[fname][ind] = (int(self.history[fname][ind][0]), l[0], self.history[fname][ind][2]) + else: + self.history[fname][ind] = (int(self.history[fname][ind][0]), self.history[fname][ind][1], l[0]) return x while x: @@ -144,22 +149,41 @@ class Updates: continue if l[0] == "SHA1-History:": - x = read_hashs(0,f,self) + x = read_hashs(0,1,f,self) + continue + + if l[0] == "SHA256-History:": + x = read_hashs(0,2,f,self) continue if l[0] == "SHA1-Patches:": - x = read_hashs(1,f,self) + x = read_hashs(1,1,f,self) + continue + + if l[0] == "SHA256-Patches:": + x = read_hashs(1,2,f,self) continue if l[0] == "SHA1-Download:": - x = read_hashs(2,f,self) + x = read_hashs(2,1,f,self) + continue + + if l[0] == "SHA256-Download:": + x = read_hashs(2,2,f,self) continue if l[0] == "Canonical-Name:" or l[0]=="Canonical-Path:": self.can_path = l[1] if l[0] == "SHA1-Current:" and len(l) == 3: - self.filesizesha1 = (l[1], int(l[2])) + if not self.filesizehashes: + self.filesizehashes = (int(l[2]), None, None) + self.filesizehashes = (int(self.filesizehashes[0]), l[1], self.filesizehashes[2]) + + if l[0] == "SHA256-Current:" and len(l) == 3: + if not self.filesizehashes: + self.filesizehashes = (int(l[2]), None, None) + self.filesizehashes = (int(self.filesizehashes[0]), self.filesizehashes[2], l[1]) x = f.readline() @@ -170,8 +194,11 @@ class Updates: if self.can_path: out.write("Canonical-Path: %s\n" % (self.can_path)) - if self.filesizesha1: - out.write("SHA1-Current: %s %7d\n" % (self.filesizesha1)) + if self.filesizehashes: + if self.filesizehashes[1]: + out.write("SHA1-Current: %s %7d\n" % (self.filesizehashes[1], self.filesizehashes[0])) + if self.filesizehashes[2]: + out.write("SHA256-Current: %s %7d\n" % (self.filesizehashes[2], self.filesizehashes[0])) hs = self.history l = self.history_order[:] @@ -186,14 +213,28 @@ class Updates: out.write("SHA1-History:\n") for h in l: - out.write(" %s %7d %s\n" % (hs[h][0][0], hs[h][0][1], h)) + if hs[h][0] and hs[h][0][1]: + out.write(" %s %7d %s\n" % (hs[h][0][1], hs[h][0][0], h)) + out.write("SHA256-History:\n") + for h in l: + if hs[h][0] and hs[h][0][2]: + out.write(" %s %7d %s\n" % (hs[h][0][2], hs[h][0][0], h)) out.write("SHA1-Patches:\n") for h in l: - out.write(" %s %7d %s\n" % (hs[h][1][0], hs[h][1][1], h)) + if hs[h][1] and hs[h][1][1]: + out.write(" %s %7d %s\n" % (hs[h][1][1], hs[h][1][0], h)) + out.write("SHA256-Patches:\n") + for h in l: + if hs[h][1] and hs[h][1][2]: + out.write(" %s %7d %s\n" % (hs[h][1][2], hs[h][1][0], h)) out.write("SHA1-Download:\n") for h in l: - if hs[h][2]: - out.write(" %s %7d %s.gz\n" % (hs[h][2][0], hs[h][2][1], h)) + if hs[h][2] and hs[h][2][1]: + out.write(" %s %7d %s.gz\n" % (hs[h][2][1], hs[h][2][0], h)) + out.write("SHA256-Download:\n") + for h in l: + if hs[h][2] and hs[h][2][2]: + out.write(" %s %7d %s.gz\n" % (hs[h][2][2], hs[h][2][0], h)) def create_temp_file(r): f = tempfile.TemporaryFile() @@ -207,11 +248,13 @@ def create_temp_file(r): f.seek(0) return f -def sizesha1(f): +def sizehashes(f): size = os.fstat(f.fileno())[6] f.seek(0) sha1sum = apt_pkg.sha1sum(f) - return (sha1sum, size) + f.seek(0) + sha256sum = apt_pkg.sha256sum(f) + return (size, sha1sum, sha256sum) def genchanges(Options, outdir, oldfile, origfile, maxdiffs = 56): if Options.has_key("NoAct"): @@ -247,7 +290,7 @@ def genchanges(Options, outdir, oldfile, origfile, maxdiffs = 56): return oldf = smartopen(oldfile) - oldsizesha1 = sizesha1(oldf) + oldsizehashes = sizehashes(oldf) # should probably early exit if either of these checks fail # alternatively (optionally?) could just trim the patch history @@ -261,10 +304,10 @@ def genchanges(Options, outdir, oldfile, origfile, maxdiffs = 56): if os.path.exists(newfile): os.unlink(newfile) smartlink(origfile, newfile) newf = open(newfile, "r") - newsizesha1 = sizesha1(newf) + newsizehashes = sizehashes(newf) newf.close() - if newsizesha1 == oldsizesha1: + if newsizehashes == oldsizehashes: os.unlink(newfile) oldf.close() #print "%s: unchanged" % (origfile) @@ -278,17 +321,17 @@ def genchanges(Options, outdir, oldfile, origfile, maxdiffs = 56): oldf.close() difff = smartopen(difffile) - difsizesha1 = sizesha1(difff) + difsizehashes = sizehashes(difff) difff.close() difffgz = open(difffile + ".gz", "r") - difgzsizesha1 = sizesha1(difffgz) + difgzsizehashes = sizehashes(difffgz) difffgz.close() - upd.history[patchname] = (oldsizesha1, difsizesha1, difgzsizesha1) + upd.history[patchname] = (oldsizehashes, difsizehashes, difgzsizehashes) upd.history_order.append(patchname) - upd.filesizesha1 = newsizesha1 + upd.filesizehashes = newsizehashes os.unlink(oldfile + oldext) os.link(origfile + origext, oldfile + origext) -- 2.6.2
Attachment:
signature.asc
Description: PGP signature