[PATCH v2] libdpkg: Use the amount of available memory instead phys_mem/2
On Linux there is the field 'MemAvailable' in '/proc/meminfo' which
holds the amount of memory which is available as of now. It considers
the fact that the page cache can purge (if not all) some memory can be
reclaimed (for instance by writing back dirty inodes) and some memory
should remain free just in case. This amount of memory can be used
without the need to swap-out some memory. The complete definition can
be located at [0].
The advantage over PHYS_MEM/2 is that it considers the current
status/usage of the system with assuming that half of what is physically
available is usable.
[0] https://www.kernel.org/doc/html/latest/filesystems/proc.html
Signed-off-by: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
---
v1…v2:
- replaced `' with ''
- spelling
- replaced !val with val == 0
- using strtoimax() for string to value, setting errno to 0 and
checking it against ERANGE if the max values was returned.
- moved includes to the top
- keeping open+read+close stat() does not work on proc files
lib/dpkg/compress.c | 99 ++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 94 insertions(+), 5 deletions(-)
diff --git a/lib/dpkg/compress.c b/lib/dpkg/compress.c
index 41991317afe53..84adb8b06c1a2 100644
--- a/lib/dpkg/compress.c
+++ b/lib/dpkg/compress.c
@@ -23,8 +23,11 @@
#include <config.h>
#include <compat.h>
+#include <sys/stat.h>
+
#include <errno.h>
#include <string.h>
+#include <fcntl.h>
#include <unistd.h>
#include <stdbool.h>
#include <stdlib.h>
@@ -523,6 +526,89 @@ filter_lzma_error(struct io_lzma *io, lzma_ret ret)
dpkg_lzma_strerror(ret, io->status));
}
+#ifdef HAVE_LZMA_MT
+# ifdef __linux__
+
+/*
+ * An estimate of how much memory is available. Swap will not be used, the page
+ * cache may be purged, not everything will be reclaimed what might be
+ * reclaimed, watermarks are considers.
+ */
+static char str_MemAvailable[] = "MemAvailable";
+
+static int
+get_avail_mem(uint64_t *val)
+{
+ char buf[4096];
+ char *str;
+ ssize_t bytes;
+ int fd;
+
+ *val = 0;
+
+ fd = open("/proc/meminfo", O_RDONLY);
+ if (fd < 0)
+ return -1;
+
+ bytes = read(fd, buf, sizeof(buf));
+ close(fd);
+
+ if (bytes <= 0)
+ return -1;
+
+ buf[bytes] = '\0';
+
+ str = buf;
+ while (1) {
+ char *end;
+
+ end = strchr(str, ':');
+ if (end == 0)
+ break;
+ if ((end - str) == sizeof(str_MemAvailable) - 1) {
+ if (strncmp(str, str_MemAvailable,
+ sizeof(str_MemAvailable) - 1) == 0) {
+ intmax_t num;
+
+ str = end + 1;
+ errno = 0;
+ num = strtoimax(str, &end, 10);
+ if (num <= 0)
+ return -1;
+ if ((num == INTMAX_MAX) && errno == ERANGE)
+ return -1;
+ /* it should end with ' kB\n' */
+ if (*end != ' ' || *(end + 1) != 'k' ||
+ *(end + 2) != 'B')
+ return -1;
+
+ /* This shouldn't overflow, just in case */
+ if (num < (INTMAX_MAX / 1024))
+ num *= 1024;
+ *val = num;
+ return 0;
+ }
+ }
+
+ end = strchr(end + 1, '\n');
+ if (end == 0)
+ break;
+ str = end + 1;
+ }
+ return -1;
+}
+
+# else
+
+static int
+get_avail_mem(uint64_t *val)
+{
+ return -1;
+}
+
+# endif
+#endif
+
static void
filter_unxz_init(struct io_lzma *io, lzma_stream *s)
{
@@ -562,11 +648,14 @@ filter_xz_init(struct io_lzma *io, lzma_stream *s)
#ifdef HAVE_LZMA_MT
mt_options.preset = preset;
- /* Initialize the multi-threaded memory limit to half the physical
- * RAM, or to 128 MiB if we cannot infer the number. */
- mt_memlimit = lzma_physmem() / 2;
- if (mt_memlimit == 0)
- mt_memlimit = 128 * 1024 * 1024;
+ /* Ask the kernel what is currently available for us. If this fails
+ * initialize the memory limit to half the physical RAM, or to 128 MiB
+ * if we cannot infer the number. */
+ if (get_avail_mem(&mt_memlimit) < 0) {
+ mt_memlimit = lzma_physmem() / 2;
+ if (mt_memlimit == 0)
+ mt_memlimit = 128 * 1024 * 1024;
+ }
/* Clamp the multi-threaded memory limit to half the addressable
* memory on this architecture. */
if (mt_memlimit > INTPTR_MAX)
--
2.30.0
Reply to: