[PATCH v2 2/4] dpkg-deb: set a memory usage limit for lzma -d
Unlike gzip and bzip2, LZ77-based compressors can continue to
improve their compression ratio quite a bit in some cases by
using more memory. For this reason, the .lzma format allows a
dictionary size (and thus memory usage) up to 4 GiB, which can
present problems for the decompressor, including making a system
unresponsive or summoning the dreaded Linux OOM killer. Make
sure dpkg does not use more than 100 MiB, nor 40% of available
RAM, when decompressing an lzma-compressed package to unpack it
or examine its contents.
If the lzma command is provided by XZ Utils, also make sure _not_
to set a memory usage limit below 10 MiB. Without this change,
dpkg would refuse to install packages compressed with the default
lzma settings on memory-starved systems (with less than 20 MiB of
physical memory).
Add a --memlimit command-line option to allow overriding the
memory usage limit in case it is too low.
Signed-off-by: Jonathan Nieder <jrnieder@gmail.com>
---
configure.ac | 2 +
dpkg-deb/dpkg-deb.h | 3 +
dpkg-deb/extract.c | 2 +-
dpkg-deb/main.c | 24 ++++++++++
lib/dpkg/Makefile.am | 5 ++
lib/dpkg/compression-backend.c | 100 +++++++++++++++++++++++++++++++++++++++-
lib/dpkg/compression-backend.h | 6 ++-
lib/dpkg/compression.c | 8 ++-
lib/dpkg/dpkg.h | 4 +-
man/dpkg-deb.1 | 8 +++
10 files changed, 153 insertions(+), 9 deletions(-)
diff --git a/configure.ac b/configure.ac
index 52f019c..7a7458c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -75,6 +75,7 @@ fi
# Checks for header files.
AC_HEADER_STDC
+AC_HEADER_STDBOOL
AC_CHECK_HEADERS([stddef.h error.h locale.h libintl.h kvm.h \
sys/cdefs.h sys/syscall.h])
DPKG_CHECK_DEFINE(TIOCNOTTY, [sys/ioctl.h])
@@ -102,6 +103,7 @@ DPKG_CHECK_DECL([WCOREDUMP], [sys/wait.h])
DPKG_CHECK_COMPAT_FUNCS([getopt getopt_long obstack_free \
strnlen strerror strsignal \
scandir alphasort unsetenv])
+TUKLIB_PHYSMEM
AC_CHECK_FUNCS([strtoul isascii bcopy memcpy lchown setsid getdtablesize])
DPKG_COMPILER_WARNINGS
diff --git a/dpkg-deb/dpkg-deb.h b/dpkg-deb/dpkg-deb.h
index 2bd6d88..edb8a8f 100644
--- a/dpkg-deb/dpkg-deb.h
+++ b/dpkg-deb/dpkg-deb.h
@@ -22,6 +22,8 @@
#ifndef DPKG_DEB_H
#define DPKG_DEB_H
+#include <stdint.h>
+
typedef void dofunction(const char *const *argv);
dofunction do_build DPKG_ATTR_NORET;
dofunction do_contents, do_control, do_showinfo;
@@ -37,6 +39,7 @@ void extracthalf(const char *debar, const char *directory,
extern const char *compression;
extern const char* showformat;
extern enum compress_type compress_type;
+extern uint64_t compress_memlimit;
#define ARCHIVEVERSION "2.0"
diff --git a/dpkg-deb/extract.c b/dpkg-deb/extract.c
index 4c429d7..f0abce8 100644
--- a/dpkg-deb/extract.c
+++ b/dpkg-deb/extract.c
@@ -292,7 +292,7 @@ void extracthalf(const char *debar, const char *directory,
m_dup2(readfromfd,0);
if (admininfo) close(p1[0]);
if (taroption) { m_dup2(p2[1],1); close(p2[0]); close(p2[1]); }
- decompress_cat(compress_type, 0, 1, _("data"));
+ decompress_cat(compress_type, 0, 1, compress_memlimit, _("data"));
}
if (readfromfd != fileno(ar)) close(readfromfd);
if (taroption) close(p2[1]);
diff --git a/dpkg-deb/main.c b/dpkg-deb/main.c
index b478ba9..0f2ef19 100644
--- a/dpkg-deb/main.c
+++ b/dpkg-deb/main.c
@@ -25,6 +25,7 @@
#include <stdio.h>
#include <string.h>
+#include <stdint.h>
#include <stdlib.h>
#include <signal.h>
#include <sys/stat.h>
@@ -107,6 +108,8 @@ usage(const struct cmdinfo *cip, const char *value)
" -z# Set the compression level when building.\n"
" -Z<type> Set the compression type used when building.\n"
" Allowed values: gzip, bzip2, lzma, none.\n"
+" -M, --memlimit=<bytes> Set the memory usage limit used when\n"
+" examining lzma compressed packages.\n"
"\n"));
printf(_(
@@ -137,11 +140,13 @@ const char printforhelp[]=
int debugflag=0, nocheckflag=0, oldformatflag=BUILDOLDPKGFORMAT;
const char* compression=NULL;
enum compress_type compress_type = compress_type_gzip;
+uint64_t compress_memlimit = 0;
const struct cmdinfo *cipaction = NULL;
dofunction *action = NULL;
static void setaction(const struct cmdinfo *cip, const char *value);
static void setcompresstype(const struct cmdinfo *cip, const char *value);
+static void setmemlimit(const struct cmdinfo *cip, const char *value);
static dofunction *const dofunctions[]= {
do_build,
@@ -174,6 +179,7 @@ static const struct cmdinfo cmdinfos[]= {
{ "nocheck", 0, 0, &nocheckflag, NULL, NULL, 1 },
{ "compression", 'z', 1, NULL, &compression, NULL, 1 },
{ "compress_type", 'Z', 1, NULL, NULL, setcompresstype },
+ { "memlimit", 'M', 1, NULL, NULL, setmemlimit },
{ "showformat", 0, 1, NULL, &showformat, NULL },
{ "help", 'h', 0, NULL, NULL, usage },
{ "version", 0, 0, NULL, NULL, printversion },
@@ -206,6 +212,24 @@ static void setcompresstype(const struct cmdinfo *cip, const char *value) {
ohshit(_("unknown compression type `%s'!"), value);
}
+static void setmemlimit(const struct cmdinfo *cip, const char *value) {
+ const char *endp;
+ unsigned long long limit;
+
+ if (strchr(value, '-') != NULL)
+ ohshit(_("invalid integer for -M: '%s'"), value);
+
+ errno = 0;
+ limit = strtoull(value, (char **)&endp, 10);
+
+ if (value == endp || *endp != '\0')
+ ohshit(_("invalid integer for -M: '%s'"), value);
+ if (errno == ERANGE || limit > UINT64_MAX)
+ ohshit(_("argument to -M out of range: '%s'"), value);
+
+ compress_memlimit = (uint64_t)limit;
+}
+
int main(int argc, const char *const *argv) {
jmp_buf ejbuf;
diff --git a/lib/dpkg/Makefile.am b/lib/dpkg/Makefile.am
index 7428f7c..22cbeff 100644
--- a/lib/dpkg/Makefile.am
+++ b/lib/dpkg/Makefile.am
@@ -23,6 +23,11 @@ libdpkg_a_SOURCES = \
cleanup.c \
compression.c \
compression-backend.c compression-backend.h \
+ $(top_srcdir)/lib/tuklib/tuklib_physmem.c \
+ $(top_srcdir)/lib/tuklib/tuklib_physmem.h \
+ $(top_srcdir)/lib/tuklib/tuklib_common.h \
+ $(top_srcdir)/lib/tuklib/tuklib_config.h \
+ $(top_srcdir)/lib/tuklib/sysdefs.h \
database.c \
dbmodify.c \
dump.c \
diff --git a/lib/dpkg/compression-backend.c b/lib/dpkg/compression-backend.c
index 9bfbcee..3cf561b 100644
--- a/lib/dpkg/compression-backend.c
+++ b/lib/dpkg/compression-backend.c
@@ -4,9 +4,12 @@
#include <dpkg/i18n.h>
#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
+#include <sys/resource.h>
#include <errno.h>
#ifdef WITH_ZLIB
@@ -15,10 +18,12 @@
#ifdef WITH_BZ2
#include <bzlib.h>
#endif
+#include <tuklib/tuklib_physmem.h>
#include <dpkg/dpkg.h>
#include <dpkg/varbuf.h>
#include <dpkg/buffer.h>
+#include <dpkg/subproc.h>
#include <dpkg/macros.h>
#include "compression-backend.h"
@@ -58,6 +63,29 @@ fd_fd_filter(int fd_in, int fd_out, const char *desc,
varbuffree(&argbuf);
}
+/* Default memory usage limit for LZ77-based decompressors. */
+static uint64_t
+default_memlimit()
+{
+ /*
+ * The command-line decoder from XZ Utils limits itself to 40% of
+ * available RAM, so take that as a reasonable default memory
+ * limit. If tuklib_physmem() fails, this is zero.
+ */
+ uint64_t limit = tuklib_physmem() * 2 / 5;
+
+ /*
+ * Assume one can manage to find 10 MiB even on memory-starved
+ * systems, so dpkg won’t refuse to unpack packages shipped by
+ * Debian. Do not use more than 100 MiB, as a safety measure
+ * for servers processing untrusted packages.
+ */
+ limit = max(limit, 10 << 20);
+ limit = min(limit, 100 << 20);
+
+ return limit;
+}
+
#define DECOMPRESS(format, zFile, zdopen, zread, zerror, ERR_ERRNO, \
fd_in, fd_out, desc) do \
{ \
@@ -180,10 +208,78 @@ compress_bzip2(int fd_in, int fd_out, char compression, const char *desc)
}
#endif
+static bool
+input_matches(FILE *in, const char *str)
+{
+ char ch;
+
+ while (ch = *str++)
+ if (fgetc(in) != ch)
+ return false;
+
+ return true;
+}
+
+static bool
+lzma_is_xz(const char *desc)
+{
+ int pipefd[2];
+ pid_t cpid;
+
+ m_pipe(pipefd);
+ cpid = m_fork();
+
+ if (cpid == 0) {
+ m_dup2(pipefd[1], 1);
+ close(pipefd[0]);
+ close(pipefd[1]);
+ execlp(LZMA, "lzma", "--version", NULL);
+ ohshite(_("%s: failed to exec '%s %s'"),
+ desc, "lzma", "--version");
+ } else {
+ FILE *pipef;
+ bool ret;
+
+ close(pipefd[1]);
+ pipef = fdopen(pipefd[0], "r");
+
+ ret = input_matches(pipef, "xz ");
+
+ if (ferror(pipef))
+ ohshite(_("%s: error reading lzma's pipe"), desc);
+ if (fclose(pipef))
+ ohshite(_("%s: error closing lzma's pipe"), desc);
+ waitsubproc(cpid, "lzma --version", PROCPIPE);
+
+ return ret;
+ }
+}
+
void
-decompress_lzma(int fd_in, int fd_out, const char *desc)
+decompress_lzma(int fd_in, int fd_out, uint64_t memlimit, const char *desc)
{
- fd_fd_filter(fd_in, fd_out, desc, LZMA, "lzma", "-dc");
+ if (memlimit == 0)
+ memlimit = default_memlimit();
+
+ if (lzma_is_xz(desc)) {
+ fd_fd_filter(fd_in, fd_out, desc, LZMA, "lzma",
+ "-dcM%" PRIu64, memlimit);
+ } else {
+ struct rlimit lim;
+
+ if (getrlimit(RLIMIT_AS, &lim))
+ ohshite(_("%s: failed to get address space limit"),
+ desc);
+ if (memlimit > lim.rlim_max)
+ lim.rlim_cur = lim.rlim_max;
+ else
+ lim.rlim_cur = (rlim_t)memlimit;
+
+ if (setrlimit(RLIMIT_AS, &lim))
+ ohshite(_("%s: failed to set address space limit"),
+ desc);
+ fd_fd_filter(fd_in, fd_out, desc, LZMA, "lzma", "-dc");
+ }
}
void
diff --git a/lib/dpkg/compression-backend.h b/lib/dpkg/compression-backend.h
index 7f3c5d6..86c55f2 100644
--- a/lib/dpkg/compression-backend.h
+++ b/lib/dpkg/compression-backend.h
@@ -12,14 +12,16 @@
#include <config.h>
#include <compat.h>
+#include <stdint.h>
+
#include <dpkg/macros.h>
void decompress_gzip(int fd_in, int fd_out, const char *desc)
DPKG_ATTR_NORET;
void decompress_bzip2(int fd_in, int fd_out, const char *desc)
DPKG_ATTR_NORET;
-void decompress_lzma(int fd_in, int fd_out, const char *desc)
- DPKG_ATTR_NORET;
+void decompress_lzma(int fd_in, int fd_out, uint64_t memlimit,
+ const char *desc) DPKG_ATTR_NORET;
void decompress_noop(int fd_in, int fd_out, const char *desc)
DPKG_ATTR_NORET;
diff --git a/lib/dpkg/compression.c b/lib/dpkg/compression.c
index 813526f..61068be 100644
--- a/lib/dpkg/compression.c
+++ b/lib/dpkg/compression.c
@@ -1,14 +1,16 @@
#include <config.h>
#include <compat.h>
-#include <stdlib.h>
#include <stdarg.h>
+#include <stdint.h>
+#include <stdlib.h>
#include <dpkg/dpkg.h>
#include <dpkg/varbuf.h>
#include <dpkg/compression-backend.h>
-void decompress_cat(enum compress_type type, int fd_in, int fd_out, char *desc, ...) {
+void decompress_cat(enum compress_type type, int fd_in, int fd_out,
+ uint64_t memlimit, char *desc, ...) {
va_list al;
struct varbuf v = VARBUF_INIT;
@@ -22,7 +24,7 @@ void decompress_cat(enum compress_type type, int fd_in, int fd_out, char *desc,
case compress_type_bzip2:
decompress_bzip2(fd_in, fd_out, v.buf);
case compress_type_lzma:
- decompress_lzma(fd_in, fd_out, v.buf);
+ decompress_lzma(fd_in, fd_out, memlimit, v.buf);
case compress_type_cat:
decompress_noop(fd_in, fd_out, v.buf);
default:
diff --git a/lib/dpkg/dpkg.h b/lib/dpkg/dpkg.h
index afe650f..94a5214 100644
--- a/lib/dpkg/dpkg.h
+++ b/lib/dpkg/dpkg.h
@@ -30,6 +30,7 @@ DPKG_BEGIN_DECLS
#include <setjmp.h>
#include <stdarg.h>
#include <stdio.h>
+#include <stdint.h>
#include <sys/types.h>
#ifdef HAVE_SYS_CDEFS_H
@@ -224,7 +225,8 @@ enum compress_type {
};
void decompress_cat(enum compress_type type, int fd_in, int fd_out,
- char *desc, ...) DPKG_ATTR_NORET DPKG_ATTR_PRINTF(4);
+ uint64_t memlimit, char *desc, ...)
+ DPKG_ATTR_NORET DPKG_ATTR_PRINTF(5);
void compress_cat(enum compress_type type, int fd_in, int fd_out,
const char *compression, char *desc, ...)
DPKG_ATTR_NORET DPKG_ATTR_PRINTF(5);
diff --git a/man/dpkg-deb.1 b/man/dpkg-deb.1
index bb08dc9..37b43b3 100644
--- a/man/dpkg-deb.1
+++ b/man/dpkg-deb.1
@@ -197,6 +197,14 @@ Specify which compression type to use when building a package. Allowed
values are \fIgzip\fP, \fIbzip2\fP, \fIlzma\fP, and \fInone\fP (default
is \fIgzip\fP).
.TP
+.BR \-M ", " \-\-memlimit= \fImemory_limit\fP
+Specify a maximum in bytes for memory usage when decompressing an lzma
+or xz compressed package. The default is 40% of the installed RAM,
+clamped to at most 100 MiB and at least 10 MiB, which allows
+decompression of any package built at the default compression level.
+This option allows one to increase the limit to allow decompression of
+packages built with a higher compression level than the default.
+.TP
.BR \-\-new
Ensures that
.B dpkg\-deb
--
1.6.5.rc1.199.g596ec
Reply to: