[PATCH] libdpkg: Use the amount of available memory instead phys_mem/2
On Linux there is the field `MemAvailable' in `/proc/meminfo' which
holds the amount of memory which is available as of now. It considers
the fact that the page cache can purge (if not all) some memory can be
reclaimed (for instance by writting back dirty inodes) and some memory
should remain free just in case. This amount of memory can be used
without the need to swap-out some memory. The complete definition can
be located at [0].
The advantage over PHYS_MEM/2 is that it considers the current
status/usage of the system with assumung that half of what is physically
avilable is usable.
[0] https://www.kernel.org/doc/html/latest/filesystems/proc.html
Signed-off-by: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
---
This is a bit of my multithreaded XZ decompressor for dpkg. For
compression the PhysMem/2 estimation is probably good enough since
mostly used the buildd and owns the system..
For decompression the amount of memory should be close to reality so it
does not start threads and allocate a lot of memory if the system is
quite busy at the amout if package upgrade/installation.
lib/dpkg/compress.c | 95 ++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 90 insertions(+), 5 deletions(-)
diff --git a/lib/dpkg/compress.c b/lib/dpkg/compress.c
index 41991317afe53..7ec9144a56290 100644
--- a/lib/dpkg/compress.c
+++ b/lib/dpkg/compress.c
@@ -523,6 +523,88 @@ filter_lzma_error(struct io_lzma *io, lzma_ret ret)
dpkg_lzma_strerror(ret, io->status));
}
+#ifdef HAVE_LZMA_MT
+# ifdef __linux__
+
+#include <sys/stat.h>
+#include <fcntl.h>
+
+/*
+ * An estimate of how much memory is available. Swap will not be used, the page
+ * cache may be purged, not everything will be reclaimed what might be
+ * reclaimed, watermarks are considers.
+ */
+static char str_MemAvailable[] = "MemAvailable";
+
+static int
+get_avail_mem(uint64_t *val)
+{
+ char buf[4096];
+ char *str;
+ ssize_t bytes;
+ int fd;
+
+ *val = 0;
+
+ fd = open("/proc/meminfo", O_RDONLY);
+ if (fd < 0)
+ return -1;
+
+ bytes = read(fd, buf, sizeof(buf));
+ close(fd);
+
+ if (bytes <= 0)
+ return -1;
+
+ buf[bytes] = '\0';
+
+ str = buf;
+ while (1) {
+ char *end;
+
+ end = strchr(str, ':');
+ if (!end)
+ break;
+ if ((end - str) == sizeof(str_MemAvailable) - 1) {
+ if (!strncmp(str, str_MemAvailable,
+ sizeof(str_MemAvailable) - 1)) {
+ uint64_t num;
+
+ str = end + 1;
+ num = strtoull(str, &end, 10);
+ if (!num)
+ return -1;
+ if (num == ULLONG_MAX)
+ return -1;
+ /* it should end with ' kb\n' */
+ if (*end != ' ')
+ return -1;
+
+ num *= 1024;
+ *val = num;
+ return 0;
+ }
+ }
+
+ end = strchr(end + 1, '\n');
+ if (!end)
+ break;
+ str = end + 1;
+ }
+ return -1;
+}
+
+# else
+
+static int
+get_avail_mem(uint64_t *val)
+{
+ return -1;
+}
+
+# endif
+#endif
+
static void
filter_unxz_init(struct io_lzma *io, lzma_stream *s)
{
@@ -562,11 +644,14 @@ filter_xz_init(struct io_lzma *io, lzma_stream *s)
#ifdef HAVE_LZMA_MT
mt_options.preset = preset;
- /* Initialize the multi-threaded memory limit to half the physical
- * RAM, or to 128 MiB if we cannot infer the number. */
- mt_memlimit = lzma_physmem() / 2;
- if (mt_memlimit == 0)
- mt_memlimit = 128 * 1024 * 1024;
+ /* Ask the kernel what is currently available for us. If this fails
+ * initialize the memory limit to half the physical RAM, or to 128 MiB
+ * if we cannot infer the number. */
+ if (get_avail_mem(&mt_memlimit) < 0) {
+ mt_memlimit = lzma_physmem() / 2;
+ if (mt_memlimit == 0)
+ mt_memlimit = 128 * 1024 * 1024;
+ }
/* Clamp the multi-threaded memory limit to half the addressable
* memory on this architecture. */
if (mt_memlimit > INTPTR_MAX)
--
2.30.0
Reply to: