On Tue, 19 Mar 2013 13:24:28 +0100, intrigeri writes: >> this problem is solved in the version in unstable (0.6.20-3), >Any particular reason why #702563 is not marked as such, then? sorry, i overlooked that; closed now. >Would debian/patches/02unicode.dpatch apply to the duplicity version >currently in Wheezy? If not, how hard would it be to extract the >minimal upstream changes needed by this workaround? both 682837 and 702563 are not hard to retrofit - but isn't that against the freeze policy? the way i understand '3. fixes for severity: important bugs in packages of priority: optional or extra, only when this can be done via unstable' would mean that only what's in unstable can be considered at this time... but please correct me if i'm wrong! >So, I recommend you investigate how targeted fixes for the most >serious bugs could go into Wheezy through t-p-u. i've just completed that and prepared a 0.6.18-7 for tpu; the much much more manageable debdiff is attached. if nobody complains loudly about further problems i'll upload that in a day. regards az
diff -u duplicity-0.6.18/debian/changelog duplicity-0.6.18/debian/changelog
--- duplicity-0.6.18/debian/changelog
+++ duplicity-0.6.18/debian/changelog
@@ -1,3 +1,31 @@
+duplicity (0.6.18-7) testing-proposed-updates; urgency=low
+
+ * backported fixes for #682837 and #702563 for wheezy
+
+ -- Alexander Zangerl <az@debian.org> Wed, 20 Mar 2013 15:37:42 +1000
+
+duplicity (0.6.18-6) unstable; urgency=low
+
+ * fixed WebDAV backend: MKCOL must be iterated for
+ nested directories (closes: #693521)
+
+ -- Alexander Zangerl <az@debian.org> Mon, 19 Nov 2012 13:02:02 +1000
+
+duplicity (0.6.18-5) unstable; urgency=low
+
+ * Ubuntu One backend: fixed a small programming error,
+ added 30s delay for retries.
+
+ -- Alexander Zangerl <az@debian.org> Fri, 26 Oct 2012 15:26:26 +1000
+
+duplicity (0.6.18-4) unstable; urgency=low
+
+ * include new standalone/REST backend for Ubuntu One
+ * updated recommends to include oauth and httplib2, which
+ are required if the Ubuntu One backend is used.
+
+ -- Alexander Zangerl <az@debian.org> Sat, 13 Oct 2012 15:54:50 +1000
+
duplicity (0.6.18-3) unstable; urgency=low
* repaired duplicity's symlink handling for --exclude-if-present
diff -u duplicity-0.6.18/debian/control duplicity-0.6.18/debian/control
--- duplicity-0.6.18/debian/control
+++ duplicity-0.6.18/debian/control
@@ -10,7 +10,7 @@
Architecture: any
Homepage: http://duplicity.nongnu.org/
Depends: ${shlibs:Depends}, ${python:Depends}, ${misc:Depends}, python-gnupginterface (>=0.3.2-9.1)
-Recommends: rsync, python-paramiko
+Recommends: rsync, python-paramiko, python-httplib2, python-oauth
Suggests: python-boto, ncftp, python-pexpect (>=2.3-1), python-cloudfiles, lftp, python-gdata, tahoe-lafs
Description: encrypted bandwidth-efficient backup
Duplicity backs directories by producing encrypted tar-format volumes
diff -u duplicity-0.6.18/debian/NEWS.Debian duplicity-0.6.18/debian/NEWS.Debian
--- duplicity-0.6.18/debian/NEWS.Debian
+++ duplicity-0.6.18/debian/NEWS.Debian
@@ -1,3 +1,15 @@
+duplicity (0.6.18-4) unstable; urgency=low
+
+ Reworked Ubuntu One backend
+ This version includes a reworked standalone backend for Ubuntu One,
+ which no longer requires Gnome, an X11 session or software that's not
+ packaged for Debian. The backend requires the python-oauth and -httplib2
+ packages and duplicity therefore now recommends them.
+
+ Check the man page for details about Ubuntu One authentication.
+
+ -- Alexander Zangerl <az@debian.org> Thu, 18 Oct 2012 13:07:36 +1000
+
duplicity (0.6.17-1) unstable; urgency=low
New sftp/scp backend
diff -u duplicity-0.6.18/debian/patches/01sshbackend.dpatch duplicity-0.6.18/debian/patches/01sshbackend.dpatch
--- duplicity-0.6.18/debian/patches/01sshbackend.dpatch
+++ duplicity-0.6.18/debian/patches/01sshbackend.dpatch
@@ -7,31 +7,99 @@
@DPATCH@
diff -urNad '--exclude=CVS' '--exclude=.svn' '--exclude=.git' '--exclude=.arch' '--exclude=.hg' '--exclude=_darcs' '--exclude=.bzr' duplicity-0.6.18~/duplicity/backends/sshbackend.py duplicity-0.6.18/duplicity/backends/sshbackend.py
--- duplicity-0.6.18~/duplicity/backends/sshbackend.py 2012-03-01 05:24:04.000000000 +1000
-+++ duplicity-0.6.18/duplicity/backends/sshbackend.py 2012-03-11 11:56:10.284325675 +1000
-@@ -2,9 +2,9 @@
- #
++++ duplicity-0.6.18/duplicity/backends/sshbackend.py 2013-03-20 15:53:08.447655360 +1000
+@@ -3,8 +3,9 @@
# Copyright 2002 Ben Escoto <ben@emerose.org>
# Copyright 2007 Kenneth Loafman <kenneth@loafman.com>
--# Copyright 2011 Alexander Zangerl <az@snafu.priv.at>
-+# Copyright 2011 Alexander Zangerl <az@snafu.priv.at>
+ # Copyright 2011 Alexander Zangerl <az@snafu.priv.at>
++# Copyright 2012 edso (ssh_config added)
#
-# $Id: sshbackend.py,v 1.2 2011/12/31 04:44:12 az Exp $
+# $Id: sshbackend.py,v 1.5 2012/03/11 01:55:46 az Exp $
#
# This file is part of duplicity.
#
-@@ -28,6 +28,7 @@
+@@ -27,14 +28,10 @@
+ import os
import errno
import sys
++import time
import getpass
+-
+-# debian squeeze's paramiko is a bit old, so we silence randompool depreciation warning
+-# note also: passphrased private keys work with squeeze's paramiko only if done with DES, not AES
+-import warnings
+-warnings.simplefilter("ignore")
+-import paramiko
+-warnings.resetwarnings()
+import logging
++from binascii import hexlify
+
+ import duplicity.backend
+ from duplicity import globals
+@@ -62,11 +59,7 @@
+ def __init__(self, parsed_url):
+ duplicity.backend.Backend.__init__(self, parsed_url)
+
+- # host string could be [user@]hostname
+- if parsed_url.username:
+- username=parsed_url.username
+- else:
+- username=getpass.getuser()
++ self.retry_delay = 10
+
+ if parsed_url.path:
+ # remove first leading '/'
+@@ -74,16 +67,68 @@
+ else:
+ self.remote_dir = '.'
- # debian squeeze's paramiko is a bit old, so we silence randompool depreciation warning
- # note also: passphrased private keys work with squeeze's paramiko only if done with DES, not AES
-@@ -84,6 +85,30 @@
- else:
- password = None
++ # lazily import paramiko when we need it
++ # debian squeeze's paramiko is a bit old, so we silence randompool depreciation warning
++ # note also: passphrased private keys work with squeeze's paramiko only if done with DES, not AES
++ import warnings
++ warnings.simplefilter("ignore")
++ import paramiko
++ warnings.resetwarnings()
++
++ class AgreedAddPolicy (paramiko.AutoAddPolicy):
++ """
++ Policy for showing a yes/no prompt and adding the hostname and new
++ host key to the known host file accordingly.
++
++ This class simply extends the AutoAddPolicy class with a yes/no prompt.
++ """
++ def missing_host_key(self, client, hostname, key):
++ fp = hexlify(key.get_fingerprint())
++ fingerprint = ':'.join(a+b for a,b in zip(fp[::2], fp[1::2]))
++ question = """The authenticity of host '%s' can't be established.
++%s key fingerprint is %s.
++Are you sure you want to continue connecting (yes/no)? """ % (hostname, key.get_name().upper(), fingerprint)
++ while True:
++ sys.stdout.write(question)
++ choice = raw_input().lower()
++ if choice in ['yes','y']:
++ super(AgreedAddPolicy, self).missing_host_key(client, hostname, key)
++ return
++ elif choice in ['no','n']:
++ raise AuthenticityException( hostname )
++ else:
++ question = "Please type 'yes' or 'no': "
++
++ class AuthenticityException (paramiko.SSHException):
++ def __init__(self, hostname):
++ paramiko.SSHException.__init__(self, 'Host key verification for server %s failed.' % hostname)
+
+- # set up password
+- if globals.ssh_askpass:
+- password = self.get_password()
+- else:
+- if parsed_url.password:
+- password = parsed_url.password
+- else:
+- password = None
self.client = paramiko.SSHClient()
++ self.client.set_missing_host_key_policy(AgreedAddPolicy())
+
+ # paramiko uses logging with the normal python severity levels,
+ # but duplicity uses both custom levels and inverted logic...*sigh*
@@ -58,4 +126,342 @@
-+
++
# load known_hosts files
# paramiko is very picky wrt format and bails out on any problem...
try:
+@@ -92,27 +137,74 @@
+ except Exception, e:
+ raise BackendException("could not load /etc/ssh/ssh_known_hosts, maybe corrupt?")
+ try:
+- self.client.load_system_host_keys()
++ # use load_host_keys() to signal it's writable to paramiko
++ # load if file exists or add filename to create it if needed
++ file = os.path.expanduser('~/.ssh/known_hosts')
++ if os.path.isfile(file):
++ self.client.load_host_keys(file)
++ else:
++ self.client._host_keys_filename = file
+ except Exception, e:
+ raise BackendException("could not load ~/.ssh/known_hosts, maybe corrupt?")
+
+- # alternative ssh private key?
+- keyfilename=None
++ """ the next block reorganizes all host parameters into a
++ dictionary like SSHConfig does. this dictionary 'self.config'
++ becomes the authorative source for these values from here on.
++ rationale is that it is easiest to deal wrt overwriting multiple
++ values from ssh_config file. (ede 03/2012)
++ """
++ self.config={'hostname':parsed_url.hostname}
++ # get system host config entries
++ self.config.update(self.gethostconfig('/etc/ssh/ssh_config',parsed_url.hostname))
++ # update with user's config file
++ self.config.update(self.gethostconfig('~/.ssh/config',parsed_url.hostname))
++ # update with url values
++ ## username from url
++ if parsed_url.username:
++ self.config.update({'user':parsed_url.username})
++ ## username from input
++ if not 'user' in self.config:
++ self.config.update({'user':getpass.getuser()})
++ ## port from url
++ if parsed_url.port:
++ self.config.update({'port':parsed_url.port})
++ ## ensure there is deafult 22 or an int value
++ if 'port' in self.config:
++ self.config.update({'port':int(self.config['port'])})
++ else:
++ self.config.update({'port':22})
++ ## alternative ssh private key, identity file
+ m=re.search("-oidentityfile=(\S+)",globals.ssh_options,re.I)
+ if (m!=None):
+ keyfilename=m.group(1)
+-
+- if parsed_url.port:
+- portnumber=parsed_url.port
++ self.config['identityfile'] = keyfilename
++ ## ensure ~ is expanded and identity exists in dictionary
++ if 'identityfile' in self.config:
++ self.config['identityfile'] = os.path.expanduser(
++ self.config['identityfile'])
+ else:
+- portnumber=22
++ self.config['identityfile'] = None
++
++ # get password, enable prompt if askpass is set
++ self.use_getpass = globals.ssh_askpass
++ ## set url values for beautiful login prompt
++ parsed_url.username = self.config['user']
++ parsed_url.hostname = self.config['hostname']
++ password = self.get_password()
++
+ try:
+- self.client.connect(hostname=parsed_url.hostname, port=portnumber,
+- username=username, password=password,
+- allow_agent=True, look_for_keys=True,
+- key_filename=keyfilename)
++ self.client.connect(hostname=self.config['hostname'],
++ port=self.config['port'],
++ username=self.config['user'],
++ password=password,
++ allow_agent=True,
++ look_for_keys=True,
++ key_filename=self.config['identityfile'])
+ except Exception, e:
+- raise BackendException("ssh connection to %s:%d failed: %s" % (parsed_url.hostname,portnumber,e))
++ raise BackendException("ssh connection to %s@%s:%d failed: %s" % (
++ self.config['user'],
++ self.config['hostname'],
++ self.config['port'],e))
+ self.client.get_transport().set_keepalive((int)(globals.timeout / 2))
+
+ # scp or sftp?
+@@ -160,109 +252,149 @@
+ contain single quotes."""
+ if not remote_filename:
+ remote_filename = source_path.get_filename()
+- if (globals.use_scp):
+- f=file(source_path.name,'rb')
+- try:
+- chan=self.client.get_transport().open_session()
+- chan.settimeout(globals.timeout)
+- chan.exec_command("scp -t '%s'" % self.remote_dir) # scp in sink mode uses the arg as base directory
+- except Exception, e:
+- raise BackendException("scp execution failed: %s" % e)
+- # scp protocol: one 0x0 after startup, one after the Create meta, one after saving
+- # if there's a problem: 0x1 or 0x02 and some error text
+- response=chan.recv(1)
+- if (response!="\0"):
+- raise BackendException("scp remote error: %s" % chan.recv(-1))
+- fstat=os.stat(source_path.name)
+- chan.send('C%s %d %s\n' %(oct(fstat.st_mode)[-4:], fstat.st_size, remote_filename))
+- response=chan.recv(1)
+- if (response!="\0"):
+- raise BackendException("scp remote error: %s" % chan.recv(-1))
+- chan.sendall(f.read()+'\0')
+- f.close()
+- response=chan.recv(1)
+- if (response!="\0"):
+- raise BackendException("scp remote error: %s" % chan.recv(-1))
+- chan.close()
+- else:
++
++ for n in range(1, globals.num_retries+1):
++ if n > 1:
++ # sleep before retry
++ time.sleep(self.retry_delay)
+ try:
+- self.sftp.put(source_path.name,remote_filename)
++ if (globals.use_scp):
++ f=file(source_path.name,'rb')
++ try:
++ chan=self.client.get_transport().open_session()
++ chan.settimeout(globals.timeout)
++ chan.exec_command("scp -t '%s'" % self.remote_dir) # scp in sink mode uses the arg as base directory
++ except Exception, e:
++ raise BackendException("scp execution failed: %s" % e)
++ # scp protocol: one 0x0 after startup, one after the Create meta, one after saving
++ # if there's a problem: 0x1 or 0x02 and some error text
++ response=chan.recv(1)
++ if (response!="\0"):
++ raise BackendException("scp remote error: %s" % chan.recv(-1))
++ fstat=os.stat(source_path.name)
++ chan.send('C%s %d %s\n' %(oct(fstat.st_mode)[-4:], fstat.st_size, remote_filename))
++ response=chan.recv(1)
++ if (response!="\0"):
++ raise BackendException("scp remote error: %s" % chan.recv(-1))
++ chan.sendall(f.read()+'\0')
++ f.close()
++ response=chan.recv(1)
++ if (response!="\0"):
++ raise BackendException("scp remote error: %s" % chan.recv(-1))
++ chan.close()
++ return
++ else:
++ try:
++ self.sftp.put(source_path.name,remote_filename)
++ return
++ except Exception, e:
++ raise BackendException("sftp put of %s (as %s) failed: %s" % (source_path.name,remote_filename,e))
+ except Exception, e:
+- raise BackendException("sftp put of %s (as %s) failed: %s" % (source_path.name,remote_filename,e))
++ log.Warn("%s (Try %d of %d) Will retry in %d seconds." % (e,n,globals.num_retries,self.retry_delay))
++ raise BackendException("Giving up trying to upload '%s' after %d attempts" % (remote_filename,n))
+
+
+ def get(self, remote_filename, local_path):
+ """retrieves a single file from the remote side.
+ In scp mode unavoidable quoting issues will make this fail if the remote directory or file names
+ contain single quotes."""
+- if (globals.use_scp):
++
++ for n in range(1, globals.num_retries+1):
++ if n > 1:
++ # sleep before retry
++ time.sleep(self.retry_delay)
+ try:
+- chan=self.client.get_transport().open_session()
+- chan.settimeout(globals.timeout)
+- chan.exec_command("scp -f '%s/%s'" % (self.remote_dir,remote_filename))
+- except Exception, e:
+- raise BackendException("scp execution failed: %s" % e)
++ if (globals.use_scp):
++ try:
++ chan=self.client.get_transport().open_session()
++ chan.settimeout(globals.timeout)
++ chan.exec_command("scp -f '%s/%s'" % (self.remote_dir,remote_filename))
++ except Exception, e:
++ raise BackendException("scp execution failed: %s" % e)
+
+- chan.send('\0') # overall ready indicator
+- msg=chan.recv(-1)
+- m=re.match(r"C([0-7]{4})\s+(\d+)\s+(\S.*)$",msg)
+- if (m==None or m.group(3)!=remote_filename):
+- raise BackendException("scp get %s failed: incorrect response '%s'" % (remote_filename,msg))
+- chan.recv(1) # dispose of the newline trailing the C message
++ chan.send('\0') # overall ready indicator
++ msg=chan.recv(-1)
++ m=re.match(r"C([0-7]{4})\s+(\d+)\s+(\S.*)$",msg)
++ if (m==None or m.group(3)!=remote_filename):
++ raise BackendException("scp get %s failed: incorrect response '%s'" % (remote_filename,msg))
++ chan.recv(1) # dispose of the newline trailing the C message
+
+- size=int(m.group(2))
+- togo=size
+- f=file(local_path.name,'wb')
+- chan.send('\0') # ready for data
+- try:
+- while togo>0:
+- if togo>read_blocksize:
+- blocksize = read_blocksize
+- else:
+- blocksize = togo
+- buff=chan.recv(blocksize)
+- f.write(buff)
+- togo-=len(buff)
+- except Exception, e:
+- raise BackendException("scp get %s failed: %s" % (remote_filename,e))
++ size=int(m.group(2))
++ togo=size
++ f=file(local_path.name,'wb')
++ chan.send('\0') # ready for data
++ try:
++ while togo>0:
++ if togo>read_blocksize:
++ blocksize = read_blocksize
++ else:
++ blocksize = togo
++ buff=chan.recv(blocksize)
++ f.write(buff)
++ togo-=len(buff)
++ except Exception, e:
++ raise BackendException("scp get %s failed: %s" % (remote_filename,e))
+
+- msg=chan.recv(1) # check the final status
+- if msg!='\0':
+- raise BackendException("scp get %s failed: %s" % (remote_filename,chan.recv(-1)))
+- f.close()
+- chan.send('\0') # send final done indicator
+- chan.close()
+- else:
+- try:
+- self.sftp.get(remote_filename,local_path.name)
++ msg=chan.recv(1) # check the final status
++ if msg!='\0':
++ raise BackendException("scp get %s failed: %s" % (remote_filename,chan.recv(-1)))
++ f.close()
++ chan.send('\0') # send final done indicator
++ chan.close()
++ return
++ else:
++ try:
++ self.sftp.get(remote_filename,local_path.name)
++ return
++ except Exception, e:
++ raise BackendException("sftp get of %s (to %s) failed: %s" % (remote_filename,local_path.name,e))
++ local_path.setdata()
+ except Exception, e:
+- raise BackendException("sftp get of %s (to %s) failed: %s" % (remote_filename,local_path.name,e))
+- local_path.setdata()
++ log.Warn("%s (Try %d of %d) Will retry in %d seconds." % (e,n,globals.num_retries,self.retry_delay))
++ raise BackendException("Giving up trying to download '%s' after %d attempts" % (remote_filename,n))
+
+ def list(self):
+ """lists the contents of the one-and-only duplicity dir on the remote side.
+ In scp mode unavoidable quoting issues will make this fail if the directory name
+ contains single quotes."""
+- if (globals.use_scp):
+- output=self.runremote("ls -1 '%s'" % self.remote_dir,False,"scp dir listing ")
+- return output.splitlines()
+- else:
++ for n in range(1, globals.num_retries+1):
++ if n > 1:
++ # sleep before retry
++ time.sleep(self.retry_delay)
+ try:
+- return self.sftp.listdir()
++ if (globals.use_scp):
++ output=self.runremote("ls -1 '%s'" % self.remote_dir,False,"scp dir listing ")
++ return output.splitlines()
++ else:
++ try:
++ return self.sftp.listdir()
++ except Exception, e:
++ raise BackendException("sftp listing of %s failed: %s" % (self.sftp.getcwd(),e))
+ except Exception, e:
+- raise BackendException("sftp listing of %s failed: %s" % (self.sftp.getcwd(),e))
++ log.Warn("%s (Try %d of %d) Will retry in %d seconds." % (e,n,globals.num_retries,self.retry_delay))
++ raise BackendException("Giving up trying to list '%s' after %d attempts" % (self.remote_dir,n))
+
+ def delete(self, filename_list):
+ """deletes all files in the list on the remote side. In scp mode unavoidable quoting issues
+ will cause failures if filenames containing single quotes are encountered."""
+- for fn in filename_list:
+- if (globals.use_scp):
+- self.runremote("rm '%s/%s'" % (self.remote_dir,fn),False,"scp rm ")
+- else:
+- try:
+- self.sftp.remove(fn)
+- except Exception, e:
+- raise BackendException("sftp rm %s failed: %s" % (fn,e))
++ for n in range(1, globals.num_retries+1):
++ if n > 1:
++ # sleep before retry
++ time.sleep(self.retry_delay)
++ try:
++ for fn in filename_list:
++ if (globals.use_scp):
++ self.runremote("rm '%s/%s'" % (self.remote_dir,fn),False,"scp rm ")
++ else:
++ try:
++ self.sftp.remove(fn)
++ except Exception, e:
++ raise BackendException("sftp rm %s failed: %s" % (fn,e))
++ except Exception, e:
++ if n == globals.num_retries:
++ log.FatalError(str(e), log.ErrorCode.backend_error)
++ else:
++ log.Warn("%s (Try %d of %d) Will retry in %d seconds." % (e,n,globals.num_retries,self.retry_delay))
+
+ def runremote(self,cmd,ignoreexitcode=False,errorprefix=""):
+ """small convenience function that opens a shell channel, runs remote command and returns
+@@ -279,6 +411,21 @@
+ raise BackendException("%sfailed(%d): %s" % (errorprefix,res,chan.recv_stderr(4096)))
+ return output
+
++ def gethostconfig(self, file, host):
++ import paramiko
++
++ file = os.path.expanduser(file)
++ if not os.path.isfile(file):
++ return {}
++
++ sshconfig = paramiko.SSHConfig()
++ try:
++ sshconfig.parse(open(file))
++ except Exception, e:
++ raise BackendException("could not load '%s', maybe corrupt?" % (file))
++
++ return sshconfig.lookup(host)
++
+ duplicity.backend.register_backend("sftp", SftpBackend)
+ duplicity.backend.register_backend("scp", SftpBackend)
+ duplicity.backend.register_backend("ssh", SftpBackend)
diff -u duplicity-0.6.18/debian/patches/00list duplicity-0.6.18/debian/patches/00list
--- duplicity-0.6.18/debian/patches/00list
+++ duplicity-0.6.18/debian/patches/00list
@@ -1,3 +1,4 @@
+01webdavmkcol
01sshbackend
02cachedesync
03forcecleanup
@@ -7,0 +9,2 @@
+08u1
+09unicode
only in patch2:
unchanged:
--- duplicity-0.6.18.orig/debian/patches/09unicode.dpatch
+++ duplicity-0.6.18/debian/patches/09unicode.dpatch
@@ -0,0 +1,61 @@
+#! /bin/sh /usr/share/dpatch/dpatch-run
+## 09unicode.dpatch by <az@debian.org>
+##
+## All lines beginning with `## DP:' are a description of the patch.
+## DP: fix unicode decode-does-encode-and-fail problems with iso8859 locales
+
+@DPATCH@
+diff -urNad '--exclude=CVS' '--exclude=.svn' '--exclude=.git' '--exclude=.arch' '--exclude=.hg' '--exclude=_darcs' '--exclude=.bzr' duplicity-0.6.18~/bin/duplicity duplicity-0.6.18/bin/duplicity
+--- duplicity-0.6.18~/bin/duplicity 2013-03-20 15:56:05.670491444 +1000
++++ duplicity-0.6.18/bin/duplicity 2013-03-20 15:56:05.741487773 +1000
+@@ -30,6 +30,10 @@
+ import getpass, gzip, os, sys, time, types
+ import traceback, platform, statvfs, resource, re
+
++# override locale to avoid bug #682837, until
++# the logger finally deals with locales cleanly
++os.environ['LC_ALL']="POSIX"
++
+ pwd = os.path.abspath(os.path.dirname(sys.argv[0]))
+ if os.path.exists(os.path.join(pwd, "../duplicity")):
+ sys.path.insert(0, os.path.abspath(os.path.join(pwd, "../.")))
+diff -urNad '--exclude=CVS' '--exclude=.svn' '--exclude=.git' '--exclude=.arch' '--exclude=.hg' '--exclude=_darcs' '--exclude=.bzr' duplicity-0.6.18~/duplicity/backends/u1backend.py duplicity-0.6.18/duplicity/backends/u1backend.py
+--- duplicity-0.6.18~/duplicity/backends/u1backend.py 2013-03-20 15:56:05.686490618 +1000
++++ duplicity-0.6.18/duplicity/backends/u1backend.py 2013-03-20 15:56:05.743487669 +1000
+@@ -19,6 +19,7 @@
+ # You should have received a copy of the GNU General Public License
+ # along with duplicity. If not, see <http://www.gnu.org/licenses/>.
+
++import gettext
+ import duplicity.backend
+ from duplicity.errors import BackendException
+ from duplicity import log
+@@ -135,6 +136,7 @@
+ See https://one.ubuntu.com/developer/ for REST documentation.
+ """
+ def __init__(self, url):
++ gettext.install('duplicity', codeset='utf8') # workaround LP: #1050061
+ duplicity.backend.Backend.__init__(self, url)
+
+ # u1://dontcare/volname or u1+http:///volname
+@@ -237,7 +239,7 @@
+ if 'children' in content:
+ for child in content['children']:
+ path = urllib.unquote(child['path'].lstrip('/'))
+- filelist += [path]
++ filelist += [path.encode('utf-8')]
+ return filelist
+
+ def delete(self, filename_list):
+diff -urNad '--exclude=CVS' '--exclude=.svn' '--exclude=.git' '--exclude=.arch' '--exclude=.hg' '--exclude=_darcs' '--exclude=.bzr' duplicity-0.6.18~/duplicity/log.py duplicity-0.6.18/duplicity/log.py
+--- duplicity-0.6.18~/duplicity/log.py 2012-03-01 05:24:04.000000000 +1000
++++ duplicity-0.6.18/duplicity/log.py 2013-03-20 15:56:05.743487669 +1000
+@@ -71,7 +71,7 @@
+ initial_level = _logger.getEffectiveLevel()
+ _logger.setLevel(DupToLoggerLevel(MAX))
+
+- _logger.log(DupToLoggerLevel(verb_level), s.decode("utf8", "ignore"))
++ _logger.log(DupToLoggerLevel(verb_level), s if (isinstance(s,unicode)) else s.decode("utf8", "ignore"))
+ _logger.controlLine = None
+
+ if force_print:
only in patch2:
unchanged:
--- duplicity-0.6.18.orig/debian/patches/01webdavmkcol.dpatch
+++ duplicity-0.6.18/debian/patches/01webdavmkcol.dpatch
@@ -0,0 +1,55 @@
+#! /bin/sh /usr/share/dpatch/dpatch-run
+## 01webdavmkcol.dpatch by <az@debian.org>
+##
+## All lines beginning with `## DP:' are a description of the patch.
+## DP: fix for #693521: nested mkcols
+
+@DPATCH@
+diff -urNad '--exclude=CVS' '--exclude=.svn' '--exclude=.git' '--exclude=.arch' '--exclude=.hg' '--exclude=_darcs' '--exclude=.bzr' duplicity-0.6.18~/duplicity/backends/webdavbackend.py duplicity-0.6.18/duplicity/backends/webdavbackend.py
+--- duplicity-0.6.18~/duplicity/backends/webdavbackend.py 2012-11-19 14:18:20.000000000 +1000
++++ duplicity-0.6.18/duplicity/backends/webdavbackend.py 2012-11-19 14:18:32.914602536 +1000
+@@ -167,9 +167,8 @@
+ del self.headers['Depth']
+ # if the target collection does not exist, create it.
+ if response.status == 404:
+- log.Info("Directory '%s' being created." % self.directory)
+- res = self.request("MKCOL", self.directory)
+- log.Info("WebDAV MKCOL status: %s %s" % (res.status, res.reason))
++ response.close() # otherwise next request fails with ResponseNotReady
++ self.makedir()
+ continue
+ if response.status == 207:
+ document = response.read()
+@@ -188,6 +187,32 @@
+ result.append(filename)
+ return result
+
++ def makedir(self):
++ """Make (nested) directories on the server."""
++ dirs = self.directory.split("/")
++ # url causes directory to start with /, but it might be given
++ # with or without trailing / (which is required)
++ if dirs[-1] == '':
++ dirs=dirs[0:-1]
++ for i in range(1,len(dirs)):
++ d="/".join(dirs[0:i+1])+"/"
++
++ self.close() # or we get previous request's data or exception
++ self.headers['Depth'] = "1"
++ response = self.request("PROPFIND", d)
++ del self.headers['Depth']
++
++ log.Info("Checking existence dir %s: %d" % (d, response.status))
++
++ if response.status == 404:
++ log.Info("Creating missing directory %s" % d)
++ self.close() # or we get previous request's data or exception
++
++ res = self.request("MKCOL", d)
++ if res.status != 201:
++ raise BackendException("WebDAV MKCOL %s failed: %s %s" % (d,res.status,res.reason))
++ self.close()
++
+ def __taste_href(self, href):
+ """
+ Internal helper to taste the given href node and, if
only in patch2:
unchanged:
--- duplicity-0.6.18.orig/debian/patches/08u1.dpatch
+++ duplicity-0.6.18/debian/patches/08u1.dpatch
@@ -0,0 +1,471 @@
+#! /bin/sh /usr/share/dpatch/dpatch-run
+## 08u1.dpatch by <az@debian.org>
+##
+## All lines beginning with `## DP:' are a description of the patch.
+## DP: RESTful ubuntu one backend without external dependencies
+
+@DPATCH@
+diff -urNad '--exclude=CVS' '--exclude=.svn' '--exclude=.git' '--exclude=.arch' '--exclude=.hg' '--exclude=_darcs' '--exclude=.bzr' duplicity-0.6.18~/bin/duplicity.1 duplicity-0.6.18/bin/duplicity.1
+--- duplicity-0.6.18~/bin/duplicity.1 2012-10-26 15:29:42.000000000 +1000
++++ duplicity-0.6.18/bin/duplicity.1 2012-10-26 15:29:42.931275131 +1000
+@@ -833,7 +833,7 @@
+ .PP
+ .BI "Ubuntu One"
+ .br
+-u1://host/volume_path
++u1://host_is_ignored/volume_path
+ .br
+ u1+http://volume_path
+ .br
+@@ -1201,9 +1201,16 @@
+ will distinguish between different backups.
+
+ .SH A NOTE ON UBUNTU ONE
+-Connecting to Ubuntu One requires that you be running duplicity inside of an X
+-session so that you can be prompted for your credentials if necessary by the
+-Ubuntu One session daemon.
++
++To use Ubuntu One you must have an Ubuntu One OAuth access token. Such
++OAuth tokens have a practically unlimited lifetime; you can have multiple
++active tokens and you can revoke tokens using the Ubuntu One web interface.
++.PP
++duplicity expects th token in the environment variable FTP_PASSWORD
++(in the format "consumer_key:consumer_secret:token:token_secret"). If no
++token is present, duplicity asks for your Ubuntu One email address and password
++and requests an access token from the Ubuntu SSO service. The newly
++acquired token is then printed to the console.
+ .PP
+ See https://one.ubuntu.com/ for more information about Ubuntu One.
+
+diff -urNad '--exclude=CVS' '--exclude=.svn' '--exclude=.git' '--exclude=.arch' '--exclude=.hg' '--exclude=_darcs' '--exclude=.bzr' duplicity-0.6.18~/duplicity/backends/u1backend.py duplicity-0.6.18/duplicity/backends/u1backend.py
+--- duplicity-0.6.18~/duplicity/backends/u1backend.py 2012-10-26 15:29:42.000000000 +1000
++++ duplicity-0.6.18/duplicity/backends/u1backend.py 2012-10-26 15:30:21.224304480 +1000
+@@ -2,6 +2,7 @@
+ #
+ # Copyright 2011 Canonical Ltd
+ # Authors: Michael Terry <michael.terry@canonical.com>
++# Alexander Zangerl <az@debian.org>
+ #
+ # This file is part of duplicity.
+ #
+@@ -19,239 +20,241 @@
+ # along with duplicity. If not, see <http://www.gnu.org/licenses/>.
+
+ import duplicity.backend
+-from duplicity.backend import retry
+-from duplicity.errors import BackendException, TemporaryLoadException
++from duplicity.errors import BackendException
++from duplicity import log
++from duplicity import globals
+
+-def ensure_dbus():
+- # GIO requires a dbus session bus which can start the gvfs daemons
+- # when required. So we make sure that such a bus exists and that our
+- # environment points to it.
+- import atexit
+- import os
+- import subprocess
+- import signal
+- if 'DBUS_SESSION_BUS_ADDRESS' not in os.environ:
+- output = subprocess.Popen(['dbus-launch'], stdout=subprocess.PIPE).communicate()[0]
+- lines = output.split('\n')
+- for line in lines:
+- parts = line.split('=', 1)
+- if len(parts) == 2:
+- if parts[0] == 'DBUS_SESSION_BUS_PID': # cleanup at end
+- atexit.register(os.kill, int(parts[1]), signal.SIGTERM)
+- os.environ[parts[0]] = parts[1]
++from httplib2 import Http
++from oauth import oauth
++from urlparse import urlparse, parse_qsl
++from json import loads, dumps
++import urllib
++import getpass
++import os
++import sys
++import time
+
+-class U1Backend(duplicity.backend.Backend):
+- """
+- Backend for Ubuntu One, through the use of the ubuntone module and a REST
+- API. See https://one.ubuntu.com/developer/ for REST documentation.
+- """
+- def __init__(self, url):
+- duplicity.backend.Backend.__init__(self, url)
++class OAuthHttpClient(object):
++ """a simple HTTP client with OAuth added on"""
++ def __init__(self):
++ self.signature_method = oauth.OAuthSignatureMethod_HMAC_SHA1()
++ self.consumer = None
++ self.token = None
++ self.client = Http()
+
+- if self.parsed_url.scheme == 'u1+http':
+- # Use the default Ubuntu One host
+- self.parsed_url.hostname = "one.ubuntu.com"
+- else:
+- assert self.parsed_url.scheme == 'u1'
++ def set_consumer(self, consumer_key, consumer_secret):
++ self.consumer = oauth.OAuthConsumer(consumer_key,
++ consumer_secret)
+
+- path = self.parsed_url.path.lstrip('/')
++ def set_token(self, token, token_secret):
++ self.token = oauth.OAuthToken( token, token_secret)
+
+- self.api_base = "https://%s/api/file_storage/v1"; % self.parsed_url.hostname
+- self.volume_uri = "%s/volumes/~/%s" % (self.api_base, path)
+- self.meta_base = "%s/~/%s/" % (self.api_base, path)
+- # This next line *should* work, but isn't set up correctly server-side yet
+- #self.content_base = self.api_base
+- self.content_base = "https://files.%s"; % self.parsed_url.hostname
++ def _get_oauth_request_header(self, url, method):
++ """Get an oauth request header given the token and the url"""
++ query = urlparse(url).query
+
+- ensure_dbus()
++ oauth_request = oauth.OAuthRequest.from_consumer_and_token(
++ http_url=url,
++ http_method=method,
++ oauth_consumer=self.consumer,
++ token=self.token,
++ parameters=dict(parse_qsl(query))
++ )
++ oauth_request.sign_request(oauth.OAuthSignatureMethod_HMAC_SHA1(),
++ self.consumer, self.token)
++ return oauth_request.to_header()
+
+- if not self.login():
+- from duplicity import log
+- log.FatalError(_("Could not obtain Ubuntu One credentials"),
+- log.ErrorCode.backend_error)
++ def request(self, url, method="GET", body=None, headers={}, ignore=None):
++ oauth_header = self._get_oauth_request_header(url, method)
++ headers.update(oauth_header)
+
+- # Create volume in case it doesn't exist yet
+- self.create_volume()
++ for n in range(1, globals.num_retries+1):
++ log.Info("making %s request to %s (attempt %d)" % (method,url,n))
++ try:
++ resp, content = self.client.request(url, method, headers=headers, body=body)
++ except Exception, e:
++ log.Info("request failed, exception %s" % e);
++ if n == globals.num_retries:
++ log.FatalError("Giving up on request after %d attempts, last exception %s" % (n,e))
++ time.sleep(30)
++ continue
++
++ log.Info("completed request with status %s %s" % (resp.status,resp.reason))
++ oops_id = resp.get('x-oops-id', None)
++ if oops_id:
++ log.Debug("Server Error: method %s url %s Oops-ID %s" % (method, url, oops_id))
+
+- def login(self):
+- from gobject import MainLoop
+- from dbus.mainloop.glib import DBusGMainLoop
+- from ubuntuone.platform.credentials import CredentialsManagementTool
++ if resp['content-type'] == 'application/json':
++ content = loads(content)
+
+- self.login_success = False
++ # were we successful? status either 2xx or code we're told to ignore
++ numcode=int(resp.status)
++ if (numcode>=200 and numcode<300) or (ignore and numcode in ignore):
++ return resp, content
+
+- DBusGMainLoop(set_as_default=True)
+- loop = MainLoop()
++ ecode=log.ErrorCode.backend_error
++ if numcode==404:
++ ecode=log.ErrorCode.backend_not_found
++ elif numcode==507: # webdav no space
++ ecode=log.ErrorCode.backend_no_space
+
+- def quit(result):
+- loop.quit()
+- if result:
+- self.login_success = True
++ if n < globals.num_retries:
++ time.sleep(30)
+
+- cd = CredentialsManagementTool()
+- d = cd.login()
+- d.addCallbacks(quit)
+- loop.run()
+- return self.login_success
++ log.FatalError("Giving up on request after %d attempts, last status %d %s" % (n,numcode,resp.reason),
++ ecode)
+
+- def quote(self, url):
+- import urllib
+- return urllib.quote(url, safe="/~")
+
+- def parse_error(self, headers, ignore=None):
+- from duplicity import log
++ def get_and_set_token(self,email, password, hostname):
++ """Acquire an Ubuntu One access token via OAuth with the Ubuntu SSO service.
++ See https://one.ubuntu.com/developer/account_admin/auth/otherplatforms for details.
++ """
+
+- status = int(headers[0].get('status'))
+- if status >= 200 and status < 300:
+- return None
++ # Request new access token from the Ubuntu SSO service
++ self.client.add_credentials(email,password)
++ resp, content = self.client.request('https://login.ubuntu.com/api/1.0/authentications?'
++ +'ws.op=authenticate&token_name=Ubuntu%%20One%%20@%%20%s' % hostname)
++ if resp.status!=200:
++ log.FatalError("Token request failed: Incorrect Ubuntu One credentials",log.ErrorCode.backend_permission_denied)
++ self.client.clear_credentials()
++
++ tokendata=loads(content)
++ self.set_consumer(tokendata['consumer_key'],tokendata['consumer_secret'])
++ self.set_token(tokendata['token'],tokendata['token_secret'])
+
+- if ignore and status in ignore:
+- return None
++ # and finally tell Ubuntu One about the token
++ resp, content = self.request('https://one.ubuntu.com/oauth/sso-finished-so-get-tokens/')
++ if resp.status!=200:
++ log.FatalError("Ubuntu One token was not accepted: %s %s" % (resp.status,resp.reason))
+
+- if status == 400:
+- code = log.ErrorCode.backend_permission_denied
+- elif status == 404:
+- code = log.ErrorCode.backend_not_found
+- elif status == 507:
+- code = log.ErrorCode.backend_no_space
+- else:
+- code = log.ErrorCode.backend_error
+- return code
++ return tokendata
+
+- def handle_error(self, raise_error, op, headers, file1=None, file2=None, ignore=None):
+- from duplicity import log
+- from duplicity import util
+- import json
++class U1Backend(duplicity.backend.Backend):
++ """
++ Backend for Ubuntu One, through the use of the REST API.
++ See https://one.ubuntu.com/developer/ for REST documentation.
++ """
++ def __init__(self, url):
++ duplicity.backend.Backend.__init__(self, url)
+
+- code = self.parse_error(headers, ignore)
+- if code is None:
+- return
++ # u1://dontcare/volname or u1+http:///volname
++ path = self.parsed_url.path.lstrip('/')
+
+- status = int(headers[0].get('status'))
++ self.api_base = "https://one.ubuntu.com/api/file_storage/v1";
++ self.content_base = "https://files.one.ubuntu.com";
+
+- if file1:
+- file1 = file1.encode("utf8")
+- else:
+- file1 = None
+- if file2:
+- file2 = file2.encode("utf8")
+- else:
+- file2 = None
+- extra = ' '.join([util.escape(x) for x in [file1, file2] if x])
+- extra = ' '.join([op, extra])
+- msg = _("Got status code %s") % status
+- if headers[0].get('x-oops-id') is not None:
+- msg += '\nOops-ID: %s' % headers[0].get('x-oops-id')
+- if headers[0].get('content-type') == 'application/json':
+- node = json.loads(headers[1])
+- if node.get('error'):
+- msg = node.get('error')
++ self.volume_uri = "%s/volumes/~/%s" % (self.api_base, path)
++ self.meta_base = "%s/~/%s/" % (self.api_base, path)
+
+- if raise_error:
+- if status == 503:
+- raise TemporaryLoadException(msg)
+- else:
+- raise BackendException(msg)
++ self.client=OAuthHttpClient();
++
++ if 'FTP_PASSWORD' not in os.environ:
++ sys.stderr.write("No Ubuntu One token found in $FTP_PASSWORD, requesting a new one\n")
++ email=raw_input('Enter Ubuntu One account email: ')
++ password=getpass.getpass("Enter Ubuntu One password: ")
++ hostname=os.uname()[1]
++
++ tokendata=self.client.get_and_set_token(email,password,hostname)
++ sys.stderr.write("\nPlease record your new Ubuntu One access token for future use with duplicity:\n"
++ +"FTP_PASSWORD=%s:%s:%s:%s\n\n"
++ % (tokendata['consumer_key'],tokendata['consumer_secret'],
++ tokendata['token'],tokendata['token_secret']))
+ else:
+- log.FatalError(msg, code, extra)
++ (consumer,consumer_secret,token,token_secret) = os.environ['FTP_PASSWORD'].split(':')
++ self.client.set_consumer(consumer, consumer_secret)
++ self.client.set_token(token, token_secret)
++
++ resp, content = self.client.request(self.api_base,ignore=[400,401,403])
++ if resp['status']!='200':
++ log.FatalError("Access failed: Ubuntu One credentials incorrect",
++ log.ErrorCode.user_error)
+
+- @retry
+- def create_volume(self, raise_errors=False):
+- import ubuntuone.couch.auth as auth
+- answer = auth.request(self.volume_uri, http_method="PUT")
+- self.handle_error(raise_errors, 'put', answer, self.volume_uri)
++ # Create volume, but check existence first
++ resp, content = self.client.request(self.volume_uri,ignore=[404])
++ if resp['status']=='404':
++ resp, content = self.client.request(self.volume_uri,"PUT")
+
+- @retry
+- def put(self, source_path, remote_filename = None, raise_errors=False):
++ def quote(self, url):
++ return urllib.quote(url, safe="/~").replace(" ","%20")
++
++ def put(self, source_path, remote_filename = None):
+ """Copy file to remote"""
+- import json
+- import ubuntuone.couch.auth as auth
+- import mimetypes
+ if not remote_filename:
+ remote_filename = source_path.get_filename()
+ remote_full = self.meta_base + self.quote(remote_filename)
+- answer = auth.request(remote_full,
+- http_method="PUT",
+- request_body='{"kind":"file"}')
+- self.handle_error(raise_errors, 'put', answer, source_path.name, remote_full)
+- node = json.loads(answer[1])
++ # check if it exists already, returns existing content_path
++ resp, content = self.client.request(remote_full,ignore=[404])
++ if resp['status']=='404':
++ # put with path returns new content_path
++ resp, content = self.client.request(remote_full,
++ method="PUT",
++ headers = { 'content-type': 'application/json' },
++ body=dumps({"kind":"file"}))
++ elif resp['status']!='200':
++ raise BackendException("access to %s failed, code %s" % (remote_filename, resp['status']))
+
+- remote_full = self.content_base + self.quote(node.get('content_path'))
+- data = bytearray(open(source_path.name, 'rb').read())
+- size = len(data)
+- content_type = mimetypes.guess_type(source_path.name)[0]
+- content_type = content_type or 'application/octet-stream'
+- headers = {"Content-Length": str(size),
++ assert(content['content_path'] is not None)
++ # content_path allows put of the actual material
++ remote_full = self.content_base + self.quote(content['content_path'])
++ log.Info("uploading file %s to location %s" % (remote_filename, remote_full))
++
++ fh=open(source_path.name,'rb')
++ data = bytearray(fh.read())
++ fh.close()
++
++ content_type = 'application/octet-stream'
++ headers = {"Content-Length": str(len(data)),
+ "Content-Type": content_type}
+- answer = auth.request(remote_full, http_method="PUT",
+- headers=headers, request_body=data)
+- self.handle_error(raise_errors, 'put', answer, source_path.name, remote_full)
++ resp, content = self.client.request(remote_full,
++ method="PUT",
++ body=str(data),
++ headers=headers)
+
+- @retry
+- def get(self, filename, local_path, raise_errors=False):
++ def get(self, filename, local_path):
+ """Get file and put in local_path (Path object)"""
+- import json
+- import ubuntuone.couch.auth as auth
++
++ # get with path returns content_path
+ remote_full = self.meta_base + self.quote(filename)
+- answer = auth.request(remote_full)
+- self.handle_error(raise_errors, 'get', answer, remote_full, filename)
+- node = json.loads(answer[1])
++ resp, content = self.client.request(remote_full)
++
++ assert(content['content_path'] is not None)
++ # now we have content_path to access the actual material
++ remote_full = self.content_base + self.quote(content['content_path'])
++ log.Info("retrieving file %s from location %s" % (filename, remote_full))
++ resp, content = self.client.request(remote_full)
+
+- remote_full = self.content_base + self.quote(node.get('content_path'))
+- answer = auth.request(remote_full)
+- self.handle_error(raise_errors, 'get', answer, remote_full, filename)
+ f = open(local_path.name, 'wb')
+- f.write(answer[1])
++ f.write(content)
++ f.close()
+ local_path.setdata()
+
+- @retry
+- def list(self, raise_errors=False):
++ def list(self):
+ """List files in that directory"""
+- import json
+- import ubuntuone.couch.auth as auth
+- import urllib
+ remote_full = self.meta_base + "?include_children=true"
+- answer = auth.request(remote_full)
+- self.handle_error(raise_errors, 'list', answer, remote_full)
++ resp, content = self.client.request(remote_full)
++
+ filelist = []
+- node = json.loads(answer[1])
+- if node.get('has_children') == True:
+- for child in node.get('children'):
+- path = urllib.unquote(child.get('path')).lstrip('/')
++ if 'children' in content:
++ for child in content['children']:
++ path = urllib.unquote(child['path'].lstrip('/'))
+ filelist += [path]
+ return filelist
+
+- @retry
+- def delete(self, filename_list, raise_errors=False):
++ def delete(self, filename_list):
+ """Delete all files in filename list"""
+ import types
+- import ubuntuone.couch.auth as auth
+ assert type(filename_list) is not types.StringType
++
+ for filename in filename_list:
+ remote_full = self.meta_base + self.quote(filename)
+- answer = auth.request(remote_full, http_method="DELETE")
+- self.handle_error(raise_errors, 'delete', answer, remote_full, ignore=[404])
++ resp, content = self.client.request(remote_full,method="DELETE")
+
+- @retry
+- def _query_file_info(self, filename, raise_errors=False):
++ def _query_file_info(self, filename):
+ """Query attributes on filename"""
+- import json
+- import ubuntuone.couch.auth as auth
+- from duplicity import log
+ remote_full = self.meta_base + self.quote(filename)
+- answer = auth.request(remote_full)
+-
+- code = self.parse_error(answer)
+- if code is not None:
+- if code == log.ErrorCode.backend_not_found:
+- return {'size': -1}
+- elif raise_errors:
+- self.handle_error(raise_errors, 'query', answer, remote_full, filename)
+- else:
+- return {'size': None}
++ resp, content = self.client.request(remote_full)
+
+- node = json.loads(answer[1])
+- size = node.get('size')
++ size = content['size']
+ return {'size': size}
+
+ duplicity.backend.register_backend("u1", U1Backend)
-- Alexander Zangerl + GPG Key 0x42BD645D or 0x5B586291 + http://snafu.priv.at/ This mind intentionally left blank.
Attachment:
signature.asc
Description: Digital Signature