[PATCH v2] dpkg-deb: add liblzma support
Use liblzma if available instead of external commands to handle
.xz and .lzma compressed files.
Among other things, this means the lzma package no longer has to
be part of the base system.
Note: on severely memory starved systems, dpkg-deb will refuse
to unpack an xz or lzma compressed deb if it requires more than
10 MiB of memory. Packages with data.tar.lzma or data.tar.xz
built by the command-line xz or lzma encoder with the default
preset are fine. Packages built with the default dpkg-deb
compression level are fine, too.
Signed-off-by: Jonathan Nieder <jrnieder@gmail.com>
---
I wrote:
> Jonathan Nieder (3):
> libdpkg: let backends decide default compression level
> libdpkg: compress_lzma(): decrease default compression level
> dpkg-deb: add liblzma support
Actually, this is just one patch, though it requires the default
compression level patch series and xz support patch I just sent a few
hours ago to apply. Apologies for the broken cover letter. This is
basically the same as patch 15 of the xz support series I sent a week
ago, but rebased on top of the other patches sent today.
Thoughts?
configure.ac | 1 +
debian/control | 6 +-
debian/rules | 3 +-
dpkg-deb/Makefile.am | 1 +
lib/dpkg/compression-backend.c | 276 +++++++++++++++++++++++++++++++++++++++-
m4/libs.m4 | 32 +++++
src/Makefile.am | 1 +
7 files changed, 310 insertions(+), 10 deletions(-)
diff --git a/configure.ac b/configure.ac
index 7a7458c..da99ab8 100644
--- a/configure.ac
+++ b/configure.ac
@@ -65,6 +65,7 @@ AC_SYS_LARGEFILE
# Checks for libraries.
DPKG_LIB_ZLIB
DPKG_LIB_BZ2
+DPKG_LIB_LZMA
DPKG_LIB_SELINUX
if test "x$build_dselect" = "xyes"; then
DPKG_LIB_CURSES
diff --git a/debian/control b/debian/control
index 901f8e2..0d58b9c 100644
--- a/debian/control
+++ b/debian/control
@@ -11,14 +11,14 @@ Vcs-Browser: http://git.debian.org/?p=dpkg/dpkg.git
Vcs-Git: git://git.debian.org/git/dpkg/dpkg.git
Standards-Version: 3.8.3
Build-Depends: debhelper (>= 6.0.7), pkg-config, po4a (>= 0.33.1),
- libncursesw5-dev, zlib1g-dev (>= 1:1.1.3-19.1), libbz2-dev, flex,
- libselinux1-dev (>= 1.28-4) [!hurd-i386 !kfreebsd-i386 !kfreebsd-amd64],
+ libncursesw5-dev, zlib1g-dev (>= 1:1.1.3-19.1), libbz2-dev, liblzma-dev,
+ flex, libselinux1-dev (>= 1.28-4) [!hurd-i386 !kfreebsd-i386 !kfreebsd-amd64],
libtimedate-perl, libio-string-perl
Package: dpkg
Architecture: any
Essential: yes
-Pre-Depends: ${shlibs:Depends}, coreutils (>= 5.93-1), lzma
+Pre-Depends: ${shlibs:Depends}, coreutils (>= 5.93-1)
Conflicts: sysvinit (<< 2.82-1), dpkg-iasearch (<< 0.11),
dpkg-dev (<< 1.14.16), apt (<< 0.7.7), aptitude (<< 0.4.7-1)
Replaces: manpages-de (<= 0.4-3), manpages-pl (<= 20051117-1)
diff --git a/debian/rules b/debian/rules
index 9f84953..1d8f81a 100755
--- a/debian/rules
+++ b/debian/rules
@@ -55,7 +55,8 @@ build-tree/config.status: configure
--sysconfdir=/etc \
--localstatedir=/var/lib \
--with-zlib=static \
- --with-bz2=static
+ --with-bz2=static \
+ --with-liblzma
# Build the package in build-tree
build: build-tree/config.status
diff --git a/dpkg-deb/Makefile.am b/dpkg-deb/Makefile.am
index ee4fd1d..b5d4626 100644
--- a/dpkg-deb/Makefile.am
+++ b/dpkg-deb/Makefile.am
@@ -23,5 +23,6 @@ dpkg_deb_LDADD = \
$(LIBINTL) \
$(ZLIB_LIBS) \
$(BZ2_LIBS) \
+ $(LZMA_LIBS) \
$(SELINUX_LIBS)
diff --git a/lib/dpkg/compression-backend.c b/lib/dpkg/compression-backend.c
index dbc1008..ce79e1a 100644
--- a/lib/dpkg/compression-backend.c
+++ b/lib/dpkg/compression-backend.c
@@ -18,6 +18,9 @@
#ifdef WITH_BZ2
#include <bzlib.h>
#endif
+#ifdef WITH_LIBLZMA
+#include <lzma.h>
+#endif
#include <tuklib/tuklib_physmem.h>
#include <dpkg/dpkg.h>
@@ -28,9 +31,10 @@
#include "compression-backend.h"
+#define DPKG_BUFFER_SIZE 4096
+
static const char default_gz_compression = '9';
static const char default_bz2_compression = '9';
-static const char default_lzma_compression = '\0';
static void fd_fd_filter(int fd_in, int fd_out, const char *desc,
const char *file, const char *cmd, const char *argfmt, ...)
@@ -93,7 +97,7 @@ default_memlimit()
#define DECOMPRESS(format, zFile, zdopen, zread, zerror, ERR_ERRNO, \
fd_in, fd_out, desc) do \
{ \
- char buffer[4096]; \
+ char buffer[DPKG_BUFFER_SIZE]; \
int actualread; \
zFile zfile = zdopen(fd_in, "r"); \
\
@@ -124,7 +128,7 @@ default_memlimit()
/* If compression == '\0', use library default. */ \
char combuf[] = {'w', compression, '\0'}; \
int actualread, actualwrite; \
- char buffer[4096]; \
+ char buffer[DPKG_BUFFER_SIZE]; \
zFile zfile; \
\
zfile = zdopen(fd_out, combuf); \
@@ -226,6 +230,268 @@ compress_bzip2(int fd_in, int fd_out, char compression, const char *desc)
}
#endif
+#ifdef WITH_LIBLZMA
+static const char default_liblzma_compression = '6';
+
+/* liblzma does not do I/O, so we get to. */
+static size_t
+fill_buf(int fd_in, void *buf, size_t bufsz)
+{
+ void *p = buf;
+ while (bufsz > 0) {
+ ssize_t n = read(fd_in, p, bufsz);
+
+ if (n < 0) {
+ if (errno == EINTR)
+ continue;
+ return SIZE_MAX;
+ }
+
+ if (n == 0)
+ /* End of file. */
+ break;
+
+ p += n;
+ bufsz -= n;
+ }
+ return p - buf;
+}
+
+/* Returns nonzero on error. */
+static int
+write_buf(int fd_out, const void *buf, size_t bufsz)
+{
+ while (bufsz > 0) {
+ ssize_t n = write(fd_out, buf, bufsz);
+
+ if (n < 0) {
+ if (errno == EINTR)
+ continue;
+ return -1;
+ }
+
+ if (n == 0) {
+ /* This can’t happen. */
+ errno = EDOM;
+ return -1;
+ }
+
+ buf += n;
+ bufsz -= n;
+ }
+ return 0;
+}
+
+/* liblzma does not expose error messages. */
+static const char *
+message_lzma_ret(lzma_ret code, bool initializing, bool compressing)
+{
+ const char *const impossible = _("Internal error (bug)");
+
+ switch (code) {
+ case LZMA_MEM_ERROR:
+ return strerror(ENOMEM);
+ case LZMA_MEMLIMIT_ERROR:
+ if (!initializing)
+ return _("Memory usage limit reached");
+ return impossible;
+ case LZMA_OPTIONS_ERROR:
+ if (initializing && compressing)
+ return _("Unsupported compression preset");
+ if (!initializing && !compressing)
+ return _("Unsupported options in file header");
+ return impossible;
+ case LZMA_DATA_ERROR:
+ if (!initializing)
+ return _("Compressed data is corrupt");
+ return impossible;
+ case LZMA_BUF_ERROR:
+ if (!initializing)
+ return _("Unexpected end of input");
+ return impossible;
+ case LZMA_FORMAT_ERROR:
+ if (!initializing && !compressing)
+ return _("File format not recognized");
+ return impossible;
+ case LZMA_UNSUPPORTED_CHECK:
+ if (initializing && compressing)
+ return _("Unsupported type of integrity check");
+ return impossible;
+ case LZMA_OK:
+ case LZMA_STREAM_END:
+ case LZMA_NO_CHECK:
+ case LZMA_GET_CHECK:
+ case LZMA_PROG_ERROR:
+ default:
+ return impossible;
+ }
+}
+
+static void
+filter_lzma(int fd_in, int fd_out, lzma_stream *s,
+ bool compressing, uint64_t memlimit, const char *desc)
+{
+ lzma_action action;
+ uint8_t in_buf[DPKG_BUFFER_SIZE];
+ uint8_t out_buf[DPKG_BUFFER_SIZE];
+
+ s->next_out = out_buf;
+ s->avail_out = sizeof(out_buf);
+ action = LZMA_RUN;
+
+ for (;;) {
+ lzma_ret ret;
+
+ if (action != LZMA_FINISH && s->avail_in == 0) {
+ size_t len = fill_buf(fd_in, in_buf, sizeof(in_buf));
+ if (len == SIZE_MAX)
+ ohshite(_("%s: internal lzma error: %s"),
+ desc, "read");
+ s->next_in = in_buf;
+ s->avail_in = len;
+ if (len == 0)
+ action = LZMA_FINISH;
+ }
+
+ ret = lzma_code(s, action);
+
+ if (ret == LZMA_STREAM_END || s->avail_out == 0 ||
+ ret == LZMA_BUF_ERROR) {
+ if (write_buf(fd_out, out_buf, s->next_out - out_buf)) {
+ if (errno == EDOM)
+ ohshit(_("%s: internal lzma error: "
+ "write returned 0"), desc);
+ ohshite(_("%s: internal lzma error: %s"),
+ desc, "write");
+ }
+ s->next_out = out_buf;
+ s->avail_out = sizeof(out_buf);
+ }
+
+ if (ret == LZMA_OK)
+ continue;
+ if (!compressing && ret == LZMA_BUF_ERROR)
+ /*
+ * Compressed file ended early.
+ * Assume the caller meant for that to happen.
+ */
+ break;
+ if (ret == LZMA_STREAM_END)
+ /* End of output. */
+ break;
+ if (!compressing && ret == LZMA_MEMLIMIT_ERROR) {
+ /*
+ * Figure out how much memory it would have
+ * actually needed.
+ */
+ uint64_t memusage = lzma_memusage(s);
+
+ /*
+ * Round the memory limit down and usage up.
+ * This way we don't display a ridiculous
+ * message like "Limit was 9 MiB, but 9 MiB
+ * would have been needed".
+ */
+ memusage = (memusage + (1 << 20) - 1) >> 20;
+ memlimit >>= 20;
+
+ ohshit(_("%s: memory limit was "
+ "%" PRIu64 " MiB, but %" PRIu64 " MiB "
+ "would have been needed"),
+ desc, memlimit, memusage);
+ }
+ ohshit(_("%s: internal lzma error: %s"),
+ desc, message_lzma_ret(ret, false, compressing));
+ }
+}
+
+void
+decompress_lzma(int fd_in, int fd_out, uint64_t memlimit, const char *desc)
+{
+ lzma_stream s = LZMA_STREAM_INIT;
+ lzma_ret ret;
+
+ if (memlimit == 0)
+ memlimit = default_memlimit();
+
+ ret = lzma_alone_decoder(&s, memlimit);
+ if (ret != LZMA_OK)
+ ohshit(_("%s: internal lzma error: %s"),
+ desc, message_lzma_ret(ret, true, false));
+
+ filter_lzma(fd_in, fd_out, &s, false, memlimit, desc);
+ lzma_end(&s);
+ exit(0);
+}
+
+void
+compress_lzma(int fd_in, int fd_out, char compression, const char *desc)
+{
+ lzma_stream s = LZMA_STREAM_INIT;
+ lzma_options_lzma preset;
+ lzma_ret ret;
+
+ if (compression == '\0')
+ compression = default_liblzma_compression;
+ if (compression < '0' || compression > '9' ||
+ lzma_lzma_preset(&preset, compression - '0'))
+ ohshit(_("%s: internal lzma error: %s %c"), desc,
+ message_lzma_ret(LZMA_OPTIONS_ERROR, true, true),
+ compression);
+
+ ret = lzma_alone_encoder(&s, &preset);
+ if (ret != LZMA_OK)
+ ohshit(_("%s: internal lzma error: %s"),
+ desc, message_lzma_ret(ret, true, true));
+
+ filter_lzma(fd_in, fd_out, &s, true, 0, desc);
+ lzma_end(&s);
+ exit(0);
+}
+
+void
+decompress_xz(int fd_in, int fd_out, uint64_t memlimit, const char *desc)
+{
+ lzma_stream s = LZMA_STREAM_INIT;
+ lzma_ret ret;
+
+ if (memlimit == 0)
+ memlimit = default_memlimit();
+
+ ret = lzma_stream_decoder(&s, memlimit, 0);
+ if (ret != LZMA_OK)
+ ohshit(_("%s: internal lzma error: %s"),
+ desc, message_lzma_ret(ret, true, false));
+
+ filter_lzma(fd_in, fd_out, &s, false, memlimit, desc);
+ lzma_end(&s);
+ exit(0);
+}
+
+void
+compress_xz(int fd_in, int fd_out, char compression, const char *desc)
+{
+ lzma_stream s = LZMA_STREAM_INIT;
+ lzma_ret ret;
+
+ if (compression == '\0')
+ compression = default_liblzma_compression;
+ if (compression < '0' || compression > '9')
+ ohshit(_("%s: internal lzma error: %s %c"), desc,
+ message_lzma_ret(LZMA_OPTIONS_ERROR, true, true),
+ compression);
+
+ ret = lzma_easy_encoder(&s, compression - '0', LZMA_CHECK_CRC32);
+
+ if (ret != LZMA_OK)
+ ohshit(_("%s: internal lzma error: %s"),
+ desc, message_lzma_ret(ret, true, true));
+
+ filter_lzma(fd_in, fd_out, &s, true, 0, desc);
+ lzma_end(&s);
+ exit(0);
+}
+#else /* !WITH_LIBLZMA */
static bool
input_matches(FILE *in, const char *str)
{
@@ -303,9 +569,6 @@ decompress_lzma(int fd_in, int fd_out, uint64_t memlimit, const char *desc)
void
compress_lzma(int fd_in, int fd_out, char compression, const char *desc)
{
- if (compression == '\0')
- compression = default_lzma_compression;
-
compress_cmd(fd_in, fd_out, LZMA, "lzma", compression, desc);
}
@@ -323,6 +586,7 @@ compress_xz(int fd_in, int fd_out, char compression, const char *desc)
{
compress_cmd(fd_in, fd_out, XZ, "xz", compression, desc);
}
+#endif
void
decompress_noop(int fd_in, int fd_out, const char *desc)
diff --git a/m4/libs.m4 b/m4/libs.m4
index 255a7ef..1cfe2b0 100644
--- a/m4/libs.m4
+++ b/m4/libs.m4
@@ -58,6 +58,38 @@ if test "x$with_bz2" != "xno"; then
fi
])# DPKG_LIB_BZ2
+# DPKG_LIB_LZMA
+# -------------
+# Check for liblzma.
+AC_DEFUN([DPKG_LIB_LZMA],
+[AC_ARG_VAR([LZMA_LIBS], [linker flags for lzma library])dnl
+AC_ARG_WITH(liblzma,
+ AS_HELP_STRING([--with-liblzma],
+ [use liblzma for compression and decompression]))
+if test "x$with_liblzma" != "xno"; then
+ AC_CHECK_LIB([lzma], [lzma_alone_decoder],
+ [AC_DEFINE(WITH_LIBLZMA, 1,
+ [Define to 1 to use liblzma rather than console tool])
+ if test "x$with_liblzma" = "xstatic"; then
+ dpkg_lzma_libs="-Wl,-Bstatic $(
+ pkg-config --static --libs liblzma
+ ) -Wl,-Bdynamic"
+ else
+ dpkg_lzma_libs=$(pkg-config --libs liblzma)
+ fi
+ LZMA_LIBS="${LZMA_LIBS:+$LZMA_LIBS }$dpkg_lzma_libs"
+ with_liblzma="yes"],
+ [if test -n "$with_liblzma"; then
+ AC_MSG_FAILURE([lzma library not found])
+ fi])
+
+ AC_CHECK_HEADER([lzma.h],,
+ [if test -n "$with_liblzma"; then
+ AC_MSG_FAILURE([lzma header not found])
+ fi])
+fi
+])# DPKG_LIB_LZMA
+
# DPKG_LIB_SELINUX
# ----------------
# Check for selinux library.
diff --git a/src/Makefile.am b/src/Makefile.am
index a29b629..8da53bb 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -43,6 +43,7 @@ dpkg_LDADD = \
$(LIBINTL) \
$(ZLIB_LIBS) \
$(BZ2_LIBS) \
+ $(LZMA_LIBS) \
$(SELINUX_LIBS)
dpkg_query_SOURCES = \
--
1.6.5.rc1.199.g596ec
Reply to: