Bug#631281: add hook to remove .pyc files
Package: live-build
Version: 3.0~a21-1
Severity: wishlist
Tags: patch
User: ubuntu-devel@lists.ubuntu.com
Usertags: origin-ubuntu ubuntu-patch oneiric
While trying to squeeze more space out of the Ubuntu live CDs, we
discovered that byte-compiled Python modules were using something on the
order of 12MB compressed. This is clearly not to be sneezed at, and
we'd like to remove the byte-compiled versions and arrange to have them
recompiled when installing the system to disk, since the CPU time
required to byte-compile on the fly is unlikely to dominate I/O when
running from a live image. I've attached a patch that adds a hook to do
this.
The only complications I saw in live-build were (a) in some ways it
isn't ideal to do this as a hook, but it seems a bit too hacky to do in
any other way; (b) some care needs to be taken if you accept something
like my patch to run apt-xapian-index in #627716 to avoid it
re-byte-compiling some modules (obviously feel free to adjust the
attached patch if you apply this one first).
The installer side of this is more complicated than might be considered
ideal due to the profusion of Python helpers. For your amusement and/or
in case you want to do something like this in live-installer, here's my
current code from ubiquity:
def configure_python(self):
"""Byte-compile Python modules.
To save space, Ubuntu excludes .pyc files from the live filesystem.
Recreate them now to restore the appearance of a system installed
from .debs."""
cache = Cache()
# Python standard library.
re_minimal = re.compile('^python\d+\.\d+-minimal$')
python_installed = sorted(
[pkg[:-8] for pkg in cache.keys()
if re_minimal.match(pkg) and cache[pkg].is_installed])
for python in python_installed:
re_file = re.compile('^/usr/lib/%s/.*\.py$' % python)
files = [f for f in cache['%s-minimal' % python].installed_files
if re_file.match(f) and
not os.path.exists(os.path.join(self.target,
'%sc' % f[1:]))]
install_misc.chrex(self.target, python,
'/usr/lib/%s/py_compile.py' % python, *files)
files = [f for f in cache[python].installed_files
if re_file.match(f) and
not os.path.exists(os.path.join(self.target,
'%sc' % f[1:]))]
install_misc.chrex(self.target, python,
'/usr/lib/%s/py_compile.py' % python, *files)
# Modules provided by the core Debian Python packages.
default = subprocess.Popen(
['chroot', self.target, 'pyversions', '-d'],
stdout=subprocess.PIPE).communicate()[0].rstrip('\n')
if default:
install_misc.chrex(self.target, default, '-m', 'compileall',
'/usr/share/python/')
if osextras.find_on_path_root(self.target, 'py3compile'):
install_misc.chrex(self.target, 'py3compile', '-p', 'python3',
'/usr/share/python3/')
def run_hooks(path, *args):
for hook in osextras.glob_root(self.target, path):
if not os.access(os.path.join(self.target, hook[1:]), os.X_OK):
continue
install_misc.chrex(self.target, hook, *args)
# Public and private modules provided by other packages.
install_misc.chroot_setup(self.target)
try:
if osextras.find_on_path_root(self.target, 'pyversions'):
supported = subprocess.Popen(
['chroot', self.target, 'pyversions', '-s'],
stdout=subprocess.PIPE).communicate()[0].rstrip('\n')
for python in supported.split():
cachedpython = cache['%s-minimal' % python]
if not cachedpython.is_installed:
continue
version = cachedpython.installed.version
run_hooks('/usr/share/python/runtime.d/*.rtinstall',
'rtinstall', python, '', version)
run_hooks('/usr/share/python/runtime.d/*.rtupdate',
'pre-rtupdate', python, python)
run_hooks('/usr/share/python/runtime.d/*.rtupdate',
'rtupdate', python, python)
run_hooks('/usr/share/python/runtime.d/*.rtupdate',
'post-rtupdate', python, python)
if osextras.find_on_path_root(self.target, 'py3versions'):
supported = subprocess.Popen(
['chroot', self.target, 'py3versions', '-s'],
stdout=subprocess.PIPE).communicate()[0].rstrip('\n')
for python in supported.split():
cachedpython = cache['%s-minimal' % python]
if not cachedpython.is_installed:
continue
version = cachedpython.installed.version
run_hooks('/usr/share/python3/runtime.d/*.rtinstall',
'rtinstall', python, '', version)
run_hooks('/usr/share/python3/runtime.d/*.rtupdate',
'pre-rtupdate', python, python)
run_hooks('/usr/share/python3/runtime.d/*.rtupdate',
'rtupdate', python, python)
run_hooks('/usr/share/python3/runtime.d/*.rtupdate',
'post-rtupdate', python, python)
finally:
install_misc.chroot_cleanup(self.target)
Thanks,
--
Colin Watson [cjwatson@ubuntu.com]
>From b9ed5879f2f293dbde9d4ee8af7a1c495028abcf Mon Sep 17 00:00:00 2001
From: Colin Watson <cjwatson@canonical.com>
Date: Wed, 22 Jun 2011 15:20:17 +0100
Subject: [PATCH] Add hook to remove .pyc files.
---
examples/hooks/all_chroot_pyc-purge.sh | 7 +++++++
scripts/build/lb_chroot_hacks | 2 +-
2 files changed, 8 insertions(+), 1 deletions(-)
create mode 100755 examples/hooks/all_chroot_pyc-purge.sh
diff --git a/examples/hooks/all_chroot_pyc-purge.sh b/examples/hooks/all_chroot_pyc-purge.sh
new file mode 100755
index 0000000..e74820a
--- /dev/null
+++ b/examples/hooks/all_chroot_pyc-purge.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+# This is a hook for live-build(7) to remove byte-compiled Python modules.
+# To enable it, copy or symlink this hook into your config/chroot_local-hooks
+# directory.
+
+find /usr -name \*.pyc -print0 | xargs -0r rm -f
diff --git a/scripts/build/lb_chroot_hacks b/scripts/build/lb_chroot_hacks
index f9e11dd..3f1e582 100755
--- a/scripts/build/lb_chroot_hacks
+++ b/scripts/build/lb_chroot_hacks
@@ -227,7 +227,7 @@ fi
# build the index in the background which will be racy in the context of
# live-build.
if [ -x chroot/usr/sbin/update-apt-xapian-index ]; then
- Chroot chroot /usr/sbin/update-apt-xapian-index --force --quiet
+ Chroot chroot PYTHONDONTWRITEBYTECODE=1 /usr/sbin/update-apt-xapian-index --force --quiet
fi
# Remove build systems clock drift
--
1.7.5.4
Reply to: