Bug#779317: unblock: gramps/4.1.1~dfsg-3
Package: release.debian.org
Severity: normal
User: release.debian.org@packages.debian.org
Usertags: unblock
Please unblock package gramps
Bug #779308: Gramps 4.1.1 contains several bugs (originally reported upstream
only) that render Gramps unusable for users of languages with special
characters. This was due to the switch to Python 3 causing a database upgrade,
and strings being converted in ASCII format instead of Unicode.
The patch has been cherry-picked from upstream who have only recently fixed the
problem.
diff -Nru gramps-4.1.1~dfsg/debian/changelog gramps-4.1.1~dfsg/debian/changelog
--- gramps-4.1.1~dfsg/debian/changelog 2014-12-08 19:33:21.000000000 +0100
+++ gramps-4.1.1~dfsg/debian/changelog 2015-02-26 21:59:03.000000000 +0100
@@ -1,3 +1,9 @@
+gramps (4.1.1~dfsg-3) unstable; urgency=medium
+
+ * Add patch to fix bugs when database upgraded to Python 3 (Closes: #779308)
+
+ -- Ross Gammon <rossgammon@mail.dk> Thu, 26 Feb 2015 21:57:31 +0100
+
gramps (4.1.1~dfsg-2) unstable; urgency=medium
* Fix clean target (Closes: #772573)
diff -Nru gramps-4.1.1~dfsg/debian/patches/0002-PickleUpgradePython2to3.patch
gramps-4.1.1~dfsg/debian/patches/0002-PickleUpgradePython2to3.patch
--- gramps-4.1.1~dfsg/debian/patches/0002-PickleUpgradePython2to3.patch
1970-01-01 01:00:00.000000000 +0100
+++ gramps-4.1.1~dfsg/debian/patches/0002-PickleUpgradePython2to3.patch
2015-02-26 21:59:03.000000000 +0100
@@ -0,0 +1,251 @@
+From: Nick Hall <nick-h@gramps-project.org> & Tim Lyons <guy.linton@gmail.com>
+Subject: Fix pickle upgrade of python2 strings to python3 str
+
+* Add gramps_upgrade_pickle() function: commit
27648255c0808ee3f097cd1e2debba5850079bbf
+* Fix schema upgrade from 15 to 16 with python3: commit
03579aa4a1e386387e6daa23b4e2517b2e520109
+* Move load of metadata (which can also have pickle upgrade errors) and
+ make zip backup for pickle or schema upgrade: commit
2cd23356b4c91dae6fb8060c6f2e06236e03c45c
+* Add missing import statement: commit
8cc86bd153e95ad1a9323ca894077b22d0232136
+
+Origin: upstream, https://github.com/gramps-
project/gramps/commits/maintenance/gramps41
+Bug: https://gramps-project.org/bugs/view.php?id=8134
+Bug-Debian: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=779308
+
+--- gramps.orig/gramps/gen/db/dbconst.py
++++ gramps/gramps/gen/db/dbconst.py
+@@ -39,7 +39,7 @@
+ ('DBPAGE', 'DBMODE', 'DBCACHE', 'DBLOCKS', 'DBOBJECTS', 'DBUNDO',
+ 'DBEXT', 'DBMODE_R', 'DBMODE_W', 'DBUNDOFN', 'DBLOCKFN',
+ 'DBRECOVFN','BDBVERSFN', 'DBLOGNAME', 'DBFLAGS_O', 'DBFLAGS_R',
+- 'DBFLAGS_D',
++ 'DBFLAGS_D', 'SCHVERSFN', 'PCKVERSFN'
+ ) +
+
+ ('PERSON_KEY', 'FAMILY_KEY', 'SOURCE_KEY', 'CITATION_KEY',
+@@ -55,6 +55,8 @@
+ DBLOCKFN = "lock" # File name of lock file
+ DBRECOVFN = "need_recover" # File name of recovery file
+ BDBVERSFN = "bdbversion.txt"# File name of Berkeley DB version file
++SCHVERSFN = "schemaversion.txt"# File name of schema version file
++PCKVERSFN = "pickleupgrade.txt" # Indicator that pickle has been upgrade t
Python3
+ DBLOGNAME = ".Db" # Name of logger
+ DBMODE_R = "r" # Read-only access
+ DBMODE_W = "w" # Full Read/Write access
+--- gramps.orig/gramps/gen/db/read.py
++++ gramps/gramps/gen/db/read.py
+@@ -1654,10 +1654,6 @@
+ handle = handle.encode('utf-8')
+ try:
+ return table.get(handle, txn=self.txn)
+- except UnicodeDecodeError:
+- #we need to assume we opened data in python3 saved in python2
+- raw = table.db.get(handle, txn=self.txn)
+- return pickle.loads(raw, encoding='utf-8')
+ except DBERRS as msg:
+ self.__log_error()
+ raise DbError(msg)
+--- gramps.orig/gramps/gen/db/upgrade.py
++++ gramps/gramps/gen/db/upgrade.py
+@@ -66,6 +66,27 @@
+
+ LOG = logging.getLogger(".upgrade")
+
++def gramps_upgrade_pickle(self):
++ """
++ Upgrade to python3 pickle protocol.
++ """
++ import pickle
++ tables = (self.person_map, self.event_map, self.family_map,
self.place_map,
++ self.repository_map, self.source_map, self.citation_map,
++ self.media_map, self.note_map, self.tag_map, self.metadata,
++ self.reference_map)
++ self.set_total(sum(map(len, tables)))
++ for data_map in tables:
++ for handle in data_map.keys():
++ raw = data_map.db.get(handle)
++ data = pickle.loads(raw, encoding='utf-8')
++ with BSDDBTxn(self.env, data_map) as txn:
++ txn.put(handle, data)
++ self.update()
++
++ with BSDDBTxn(self.env, self.metadata) as txn:
++ txn.put(b'upgraded', 'Yes')
++
+ def gramps_upgrade_17(self):
+ """
+ Upgrade database from version 16 to 17.
+@@ -859,12 +880,12 @@
+ new_citation = (new_handle, new_gramps_id,
+ date, page, confidence, ref, note_list,
new_media_list,
+ new_data_map, new_change, private)
++ citation_list.append((new_handle))
+ with BSDDBTxn(self.env, self.citation_map) as txn:
+ if isinstance(new_handle, UNITYPE):
+ new_handle = new_handle.encode('utf-8')
+ txn.put(new_handle, new_citation)
+ self.cmap_index += 1
+- citation_list.append((new_handle))
+ return citation_list
+
+ def gramps_upgrade_15(self):
+--- gramps.orig/gramps/gen/db/write.py
++++ gramps/gramps/gen/db/write.py
+@@ -396,6 +396,7 @@
+ self.brief_name = None
+ self.update_env_version = False
+ self.update_python_version = False
++ self.update_pickle_version = False
+
+ def catch_db_error(func):
+ """
+@@ -733,6 +734,25 @@
+ force_bsddb_downgrade)
+
+ self.__check_python_version(name, force_python_upgrade)
++ # Check for pickle upgrade
++ versionpath = os.path.join(self.path, cuni(PCKVERSFN))
++ if sys.version_info[0] >= 3 and not os.path.isfile(versionpath) and \
++ not self.readonly and not self.update_pickle_version:
++ _LOG.debug("Make backup in case there is a pickle upgrade")
++ self.__make_zip_backup(name)
++ self.update_pickle_version = True
++
++ # Check for schema upgrade
++ versionpath = os.path.join(self.path, cuni(SCHVERSFN))
++ if os.path.isfile(versionpath):
++ with open(versionpath, "r") as version_file:
++ schema_version = int(version_file.read().strip())
++ else:
++ schema_version = 0
++ if not self.readonly and schema_version < _DBVERSION and \
++ force_schema_upgrade:
++ _LOG.debug("Make backup in case there is a schema upgrade")
++ self.__make_zip_backup(name)
+
+ # Set up database environment
+ self.env = db.DBEnv()
+@@ -788,7 +808,6 @@
+ self.__close_early()
+ raise DbVersionError(tree_vers, _MINVERSION, _DBVERSION)
+
+- self.__load_metadata()
+ gstats = self.metadata.get(b'gender_stats', default=None)
+
+ # Ensure version info in metadata
+@@ -799,6 +818,7 @@
+ # New database. Set up the current version.
+ #self.metadata.put(b'version', _DBVERSION, txn=the_txn)
+ txn.put(b'version', _DBVERSION)
++ txn.put(b'upgraded', 'Yes')
+ elif b'version' not in self.metadata:
+ # Not new database, but the version is missing.
+ # Use 0, but it is likely to fail anyway.
+@@ -844,7 +864,7 @@
+ if isinstance(version, UNITYPE):
+ version = version.encode('utf-8')
+ version_file.write(version)
+- _LOG.debug("Updated BDBVERSFN file to %s" % str(db.version()))
++ _LOG.debug("Updated bsddb version file to %s" %
str(db.version()))
+
+ if self.update_python_version:
+ versionpath = os.path.join(name, "pythonversion.txt")
+@@ -860,6 +880,21 @@
+ # If secondary indices change, then they should removed
+ # or rebuilt by upgrade as well. In any case, the
+ # self.secondary_connected flag should be set accordingly.
++ if self.update_pickle_version:
++ from . import upgrade
++ UpdateCallback.__init__(self, callback)
++ upgrade.gramps_upgrade_pickle(self)
++ versionpath = os.path.join(name, cuni(PCKVERSFN))
++ with open(versionpath, "w") as version_file:
++ version = "Yes"
++ if sys.version_info[0] <3:
++ if isinstance(version, UNITYPE):
++ version = version.encode('utf-8')
++ version_file.write(version)
++ _LOG.debug("Updated pickle version file to %s" % str(version))
++
++ self.__load_metadata()
++
+ if self.need_schema_upgrade():
+ oldschema = self.metadata.get(b'version', default=0)
+ newschema = _DBVERSION
+@@ -867,6 +902,14 @@
+ (oldschema, newschema))
+ if force_schema_upgrade == True:
+ self.gramps_upgrade(callback)
++ versionpath = os.path.join(name, cuni(SCHVERSFN))
++ with open(versionpath, "w") as version_file:
++ version = str(_DBVERSION)
++ if sys.version_info[0] <3:
++ if isinstance(version, UNITYPE):
++ version = version.encode('utf-8')
++ version_file.write(version)
++ _LOG.debug("Updated schema version file to %s" %
str(version))
+ else:
+ self.__close_early()
+ clear_lock_file(name)
+@@ -936,13 +979,8 @@
+
+ # bookmarks
+ def meta(key):
+- try:
+- return self.metadata.get(key, default=[])
+- except UnicodeDecodeError:
+- #we need to assume we opened data in python3 saved in python2
+- raw = self.metadata.db.get(key, default=[])
+- return pickle.loads(raw, encoding='utf-8') if raw else raw
+-
++ return self.metadata.get(key, default=[])
++
+ self.bookmarks.set(meta(b'bookmarks'))
+ self.family_bookmarks.set(meta(b'family_bookmarks'))
+ self.event_bookmarks.set(meta(b'event_bookmarks'))
+@@ -2135,10 +2173,6 @@
+ handle = handle.encode('utf-8')
+ try:
+ data = data_map.get(handle, txn=self.txn)
+- except UnicodeDecodeError:
+- #we need to assume we opened data in python3 saved in python2
+- raw = data_map.db.get(handle, txn=self.txn)
+- data = pickle.loads(raw, encoding='utf-8')
+ except:
+ data = None
+ # under certain circumstances during a database reload,
+@@ -2330,8 +2364,9 @@
+ version = self.metadata.get(b'version', default=_MINVERSION)
+
+ t = time.time()
+-
++
+ from . import upgrade
++
+ if version < 14:
+ upgrade.gramps_upgrade_14(self)
+ if version < 15:
+@@ -2419,6 +2454,24 @@
+ with open(versionpath, "w") as version_file:
+ version_file.write(version)
+
++ versionpath = os.path.join(name, cuni(PCKVERSFN))
++ _LOG.debug("Write pickle version file to %s" % "Yes")
++ with open(versionpath, "w") as version_file:
++ version = "Yes"
++ if sys.version_info[0] <3:
++ if isinstance(version, UNITYPE):
++ version = version.encode('utf-8')
++ version_file.write(version)
++
++ versionpath = os.path.join(name, cuni(SCHVERSFN))
++ _LOG.debug("Write schema version file to %s" % str(_DBVERSION))
++ with open(versionpath, "w") as version_file:
++ version = str(_DBVERSION)
++ if sys.version_info[0] <3:
++ if isinstance(version, UNITYPE):
++ version = version.encode('utf-8')
++ version_file.write(version)
++
+ versionpath = os.path.join(name, "pythonversion.txt")
+ version = str(version_info[0])
+ if sys.version_info[0] < 3:
diff -Nru gramps-4.1.1~dfsg/debian/patches/series
gramps-4.1.1~dfsg/debian/patches/series
--- gramps-4.1.1~dfsg/debian/patches/series 2014-12-08 19:33:21.000000000
+0100
+++ gramps-4.1.1~dfsg/debian/patches/series 2015-02-26 21:59:03.000000000
+0100
@@ -1 +1,2 @@
0001-Correct-resource-path-in-setup.py.patch
+0002-PickleUpgradePython2to3.patch
unblock gramps/4.1.1~dfsg-3
-- System Information:
Debian Release: jessie/sid
APT prefers trusty-security
APT policy: (990, 'trusty-security'), (900, 'trusty-updates'), (500,
'trusty'), (400, 'trusty-proposed'), (200, 'utopic-proposed'), (100, 'trusty-
backports')
Architecture: amd64 (x86_64)
Foreign Architectures: i386
Kernel: Linux 3.13.0-46-generic (SMP w/4 CPU cores)
Reply to: