Bug#314334: apt: APT doesn't work on filesystems without shared writable mmap(), like JFFS2.
Package: apt
Version: 0.5.28.6
Severity: important
Tags: patch
APT doesn't work on filesystems which don't support shared writable mmap. This is significant for hand-held devices which use NAND flash as a data store. These devices typically use a filesystem like JFFS2, which lacks shared writable mmap support.
The attached patch optionally implements [at compile time] support for mmap() emulation within APT. In order for this support to properly function, a modification to the FileFd class was required. This modification makes the FileFd class keep a cache of open files, such that if two contexts both hold the same file descriptor, the file descriptor isn't closed until both FileFd destructors are called.
The aforementioned patch corrects this issue.
-- Package-specific info:
-- /etc/apt/preferences --
Package: *
Pin: release a=experimental
Pin-Priority: 101
Package: libgcc1-powerpc-cross
Pin: release v=3.4.3-1
Pin-Priority: 1001
-- (/etc/apt/sources.list present, but not submitted) --
-- System Information:
Debian Release: 3.1
APT prefers testing
APT policy: (990, 'testing'), (500, 'unstable'), (101, 'experimental')
Architecture: i386 (i686)
Kernel: Linux 2.6.11.11.msw3
Locale: LANG=C, LC_CTYPE=C (charmap=ANSI_X3.4-1968)
Versions of packages apt depends on:
ii libc6 2.3.2.ds1-21 GNU C Library: Shared libraries an
ii libgcc1 1:3.4.3-12 GCC support library
ii libstdc++5 1:3.3.5-12 The GNU Standard C++ Library v3
-- no debconf information
--- PATCH HERE ---
diff -urp apt-0.6.25.orig/apt-pkg/contrib/fileutl.cc apt-0.6.25/apt-pkg/contrib/fileutl.cc
--- apt-0.6.25.orig/apt-pkg/contrib/fileutl.cc 2002-09-13 23:29:22.000000000 -0600
+++ apt-0.6.25/apt-pkg/contrib/fileutl.cc 2005-06-13 10:49:02.000000000 -0600
@@ -36,6 +36,9 @@
using namespace std;
+
+map<int, int> FileFd::ref_count_map;
+
// CopyFile - Buffered copy of a file /*{{{*/
// ---------------------------------------------------------------------
/* The caller is expected to set things so that failure causes erasure */
@@ -415,6 +418,7 @@ bool FileFd::Open(string FileName,OpenMo
if (iFd < 0)
return _error->Errno("open",_("Could not open file %s"),FileName.c_str());
+ inc_ref(iFd);
this->FileName = FileName;
SetCloseExec(iFd,true);
@@ -573,13 +577,11 @@ unsigned long FileFd::Size()
bool FileFd::Close()
{
bool Res = true;
- if ((Flags & AutoClose) == AutoClose)
- if (iFd >= 0 && close(iFd) != 0)
- Res &= _error->Errno("close",_("Problem closing the file"));
+
+ Res &= dec_ref(iFd);
iFd = -1;
- if ((Flags & Fail) == Fail && (Flags & DelOnFail) == DelOnFail &&
- FileName.empty() == false)
+ if ((Flags & Fail) && (Flags & DelOnFail) && !FileName.empty())
if (unlink(FileName.c_str()) != 0)
Res &= _error->WarningE("unlnk",_("Problem unlinking the file"));
return Res;
diff -urp apt-0.6.25.orig/apt-pkg/contrib/fileutl.h apt-0.6.25/apt-pkg/contrib/fileutl.h
--- apt-0.6.25.orig/apt-pkg/contrib/fileutl.h 2001-05-06 23:06:52.000000000 -0600
+++ apt-0.6.25/apt-pkg/contrib/fileutl.h 2005-06-13 11:43:35.000000000 -0600
@@ -1,4 +1,4 @@
-// -*- mode: cpp; mode: fold -*-
+// -*- mode: cpp; mode: fold; c-basic-offset: 3 -*-
// Description /*{{{*/
// $Id: fileutl.h,v 1.26 2001/05/07 05:06:52 jgg Exp $
/* ######################################################################
@@ -25,12 +25,51 @@
#pragma interface "apt-pkg/fileutl.h"
#endif
+#include <apt-pkg/error.h>
+
+#include <apti18n.h>
+
#include <string>
+#include <map>
+#include <cassert>
using std::string;
+using std::map;
class FileFd
{
+ private:
+ /*
+ * Helpers to impelment reference counting on file descriptors.
+ * This is done to avoid closing a file descriptor which is shared
+ * by two different scopes. If this is to be made thread-safe,
+ * a mutex must be in place. - MSW 06/10/2005
+ */
+ typedef int fd_t;
+ static map<fd_t, int> ref_count_map;
+ void inc_ref(int fd) {
+ if (fd >= 0) {
+ if (ref_count_map.find(fd) == ref_count_map.end())
+ ref_count_map[fd] = 1;
+ else
+ ref_count_map[fd] += 1;
+ }
+ }
+ bool dec_ref(int fd) {
+ int ret = true;
+ if (fd >= 0) {
+ assert(ref_count_map.find(fd) != ref_count_map.end());
+ ref_count_map[fd] -= 1;
+ if (ref_count_map[fd] == 0 && Flags & AutoClose) {
+ if (close(fd) < 0)
+ ret = _error->Errno("close",
+ _("Problem closing the file"));
+ ref_count_map.erase(fd);
+ }
+ }
+ return ret;
+ }
+
protected:
int iFd;
@@ -62,12 +101,13 @@ class FileFd
// Simple manipulators
inline int Fd() {return iFd;};
- inline void Fd(int fd) {iFd = fd;};
+ inline void Fd(int fd) {dec_ref(iFd); iFd = fd; inc_ref(iFd);};
inline bool IsOpen() {return iFd >= 0;};
inline bool Failed() {return (Flags & Fail) == Fail;};
inline void EraseOnFailure() {Flags |= DelOnFail;};
inline void OpFail() {Flags |= Fail;};
inline bool Eof() {return (Flags & HitEof) == HitEof;};
+ inline void SetAutoClose() {Flags |= AutoClose;};
inline string &Name() {return FileName;};
FileFd(string FileName,OpenMode Mode,unsigned long Perms = 0666) : iFd(-1),
@@ -75,8 +115,8 @@ class FileFd
{
Open(FileName,Mode,Perms);
};
- FileFd(int Fd = -1) : iFd(Fd), Flags(AutoClose) {};
- FileFd(int Fd,bool) : iFd(Fd), Flags(0) {};
+ FileFd(int Fd = -1) : iFd(Fd), Flags(AutoClose) { inc_ref(Fd); };
+ FileFd(int Fd,bool) : iFd(Fd), Flags(0) { inc_ref(Fd); };
virtual ~FileFd();
};
diff -urp apt-0.6.25.orig/apt-pkg/contrib/mmap.cc apt-0.6.25/apt-pkg/contrib/mmap.cc
--- apt-0.6.25.orig/apt-pkg/contrib/mmap.cc 2001-05-26 23:19:30.000000000 -0600
+++ apt-0.6.25/apt-pkg/contrib/mmap.cc 2005-06-14 16:53:11.000000000 -0600
@@ -1,4 +1,4 @@
-// -*- mode: cpp; mode: fold -*-
+// -*- mode: cpp; mode: fold; c-basic-offset: 3 -*-
// Description /*{{{*/
// $Id: mmap.cc,v 1.22 2001/05/27 05:19:30 jgg Exp $
/* ######################################################################
@@ -33,9 +33,117 @@
#include <sys/mman.h>
#include <sys/stat.h>
+#include <sys/types.h>
#include <unistd.h>
#include <fcntl.h>
- /*}}}*/
+#include <errno.h>
+
+#ifdef USE_NO_MMAP_FALLBACK
+map<struct key, struct mem_cache, stat_eq > MMap::cached_mem;
+
+
+/*
+ * buf_read and buf_write initially called one common function to do the
+ * real loop, because the code is so similar. This was abandoned, however,
+ * due to C++ type strictness and the slightly different function signatures
+ * of read and write.
+ */
+static inline bool buf_write(int fd, const void *buf, size_t count)
+{
+ size_t bytes_rem = count;
+ ssize_t ret;
+ while (bytes_rem > 0) {
+ int offset = count - bytes_rem;
+ ret = write(fd, ((const char *) buf) + offset, bytes_rem);
+ if (ret < 0)
+ return _error->Errno("write", _("Short write returned on fd %d.\n"), fd);
+ bytes_rem -= ret;
+ }
+ return true;
+}
+
+static inline bool buf_read(int fd, void *buf, size_t count)
+{
+ size_t bytes_rem = count;
+ ssize_t ret;
+ while (bytes_rem > 0) {
+ int offset = count - bytes_rem;
+ ret = read(fd, ((char *) buf) + offset, bytes_rem);
+ if (ret < 0)
+ return _error->Errno("read",
+ _("Short read returned on fd %d.\n"), fd);
+ bytes_rem -= ret;
+ }
+ return true;
+}
+
+bool MMap::fill_buffer(int fd, void **buf, unsigned long size)
+{
+ // First, see if the file has already been 'mapped'
+ struct mem_cache cache;
+ struct stat st_buf;
+ if (fstat(fd, &st_buf) < 0)
+ return _error->Errno("stat", _("Couldn't get file information."));
+
+ my_key.inode = st_buf.st_ino;
+ my_key.device = st_buf.st_dev;
+
+ /* We already have a cache of this file. */
+ if (cached_mem.find(my_key) != cached_mem.end()) {
+ cache = cached_mem[my_key];
+ if (cache.len < size)
+ return _error->Errno("virtual_mmap",
+ _("Virtual mmap can't grow shared memory"));
+ (*buf) = cache.ptr;
+ cache.ref_count++;
+ cached_mem[my_key] = cache;
+ return true;
+ }
+
+ /* There is no cache of this file */
+ (*buf) = malloc(size);
+ if (!(*buf))
+ return _error->Errno("malloc", _("Couldn't allocate %lu bytes memory"),
+ size);
+ /*
+ * If the file is newly created, seek to the size we're requesting,
+ * then write a byte to make it a sparse file.
+ */
+ if (st_buf.st_size < (long) size) {
+ if (lseek(fd, SEEK_SET, size) < 0) {
+ free(*buf);
+ return _error->Errno("lseek", _("couldn't create sparse file."));
+ }
+
+ char eof = 0;
+ if (!buf_write(fd, &eof, 1)) {
+ free(*buf);
+ return false;
+ }
+ } else {
+ lseek(fd, SEEK_SET, 0);
+ if (!buf_read(fd, *buf, size)) {
+ free(*buf);
+ return false;
+ }
+ }
+
+ cache.ptr = (*buf);
+ cache.len = size;
+ cache.ref_count = 1;
+ cached_mem[my_key] = cache;
+ return true;
+}
+
+# define PRINT_MMAP_WARNING()
+#else
+# define PRINT_MMAP_WARNING() \
+ if (errno == EINVAL) \
+ fprintf(stderr, "W: It is possible, though not likely, that your " \
+ "filesystem does not support shared writable mmap. If this is " \
+ "truely the case, recompile apt with --enable-mmap-fallback.\n")
+#endif /* USE_NO_MMAP_FALLBACK */
+
// MMap::MMap - Constructor /*{{{*/
// ---------------------------------------------------------------------
@@ -43,6 +151,9 @@
MMap::MMap(FileFd &F,unsigned long Flags) : Flags(Flags), iSize(0),
Base(0)
{
+#ifdef USE_NO_MMAP_FALLBACK
+ baseMalloced = false;
+#endif
if ((Flags & NoImmMap) != NoImmMap)
Map(F);
}
@@ -53,6 +164,9 @@ MMap::MMap(FileFd &F,unsigned long Flags
MMap::MMap(unsigned long Flags) : Flags(Flags), iSize(0),
Base(0)
{
+#ifdef USE_NO_MMAP_FALLBACK
+ baseMalloced = false;
+#endif
}
/*}}}*/
// MMap::~MMap - Destructor /*{{{*/
@@ -80,12 +194,28 @@ bool MMap::Map(FileFd &Fd)
if (iSize == 0)
return _error->Error(_("Can't mmap an empty file"));
-
- // Map it.
- Base = mmap(0,iSize,Prot,Map,Fd.Fd(),0);
- if (Base == (void *)-1)
- return _error->Errno("mmap",_("Couldn't make mmap of %lu bytes"),iSize);
+#ifdef USE_NO_MMAP_FALLBACK
+ if (!(Map & MAP_SHARED)) {
+#endif
+ // Map it.
+ Base = mmap(0,iSize,Prot,Map,Fd.Fd(),0);
+ if (Base == (void *)-1) {
+ PRINT_MMAP_WARNING();
+ return _error->Errno("mmap", _("Couldn't make mmap of %lu bytes"),
+ iSize);
+ }
+#ifdef USE_NO_MMAP_FALLBACK
+ } else {
+ // Save off the passed file descriptor in case the caller closes it.
+ savedFd.Fd(Fd.Fd());
+ savedFd.SetAutoClose();
+
+ if (fill_buffer(savedFd.Fd(), &Base, iSize) < 0)
+ return false;
+ baseMalloced = true;
+ }
+#endif
return true;
}
/*}}}*/
@@ -96,12 +226,29 @@ bool MMap::Close(bool DoSync)
{
if ((Flags & UnMapped) == UnMapped || Base == 0 || iSize == 0)
return true;
-
+
if (DoSync == true)
Sync();
-
- if (munmap((char *)Base,iSize) != 0)
- _error->Warning("Unable to munmap");
+
+#ifdef USE_NO_MMAP_FALLBACK
+ if (baseMalloced) {
+ /*
+ * We always Sync in the case of malloc()'d memory because a
+ * free() won't sync to disk the way an munmap will.
+ */
+ if (!DoSync)
+ Sync();
+
+ cached_mem[my_key].ref_count--;
+ if (!cached_mem[my_key].ref_count) {
+ cached_mem.erase(my_key);
+ free(Base);
+ baseMalloced = false;
+ }
+ } else
+#endif
+ if (munmap((char *)Base,iSize) != 0)
+ _error->Warning("Unable to munmap");
iSize = 0;
Base = 0;
@@ -116,12 +263,21 @@ bool MMap::Sync()
{
if ((Flags & UnMapped) == UnMapped)
return true;
-
+
+#ifdef USE_NO_MMAP_FALLBACK
+ if (baseMalloced) {
+ if (Flags & ReadOnly)
+ return true;
+
+ lseek(savedFd.Fd(), 0, SEEK_SET);
+ return buf_write(savedFd.Fd(), Base, iSize);
+ }
+#endif
#ifdef _POSIX_SYNCHRONIZED_IO
- if ((Flags & ReadOnly) != ReadOnly)
- if (msync((char *)Base,iSize,MS_SYNC) != 0)
- return _error->Errno("msync","Unable to write mmap");
-#endif
+ if ((Flags & ReadOnly) != ReadOnly)
+ if (msync((char *)Base,iSize,MS_SYNC) != 0)
+ return _error->Errno("msync","Unable to write mmap");
+#endif
return true;
}
/*}}}*/
@@ -133,12 +289,22 @@ bool MMap::Sync(unsigned long Start,unsi
if ((Flags & UnMapped) == UnMapped)
return true;
+#ifdef USE_NO_MMAP_FALLBACK
+ if (baseMalloced) {
+ if (Flags & ReadOnly)
+ return true;
+ unsigned long sz = Stop - Start;
+ lseek(savedFd.Fd(), Start, SEEK_SET);
+ return buf_write(savedFd.Fd(), ((char *) Base) + Start, sz);
+ }
+#endif
#ifdef _POSIX_SYNCHRONIZED_IO
- unsigned long PSize = sysconf(_SC_PAGESIZE);
- if ((Flags & ReadOnly) != ReadOnly)
- if (msync((char *)Base+(int)(Start/PSize)*PSize,Stop - Start,MS_SYNC) != 0)
- return _error->Errno("msync","Unable to write mmap");
-#endif
+ if ((Flags & ReadOnly) != ReadOnly) {
+ unsigned long PSize = sysconf(_SC_PAGESIZE);
+ if (msync((char *)Base+(int)(Start/PSize)*PSize,Stop - Start,MS_SYNC) != 0)
+ return _error->Errno("msync","Unable to write mmap");
+ }
+#endif
return true;
}
/*}}}*/
diff -urp apt-0.6.25.orig/apt-pkg/contrib/mmap.h apt-0.6.25/apt-pkg/contrib/mmap.h
--- apt-0.6.25.orig/apt-pkg/contrib/mmap.h 2001-05-13 23:16:43.000000000 -0600
+++ apt-0.6.25/apt-pkg/contrib/mmap.h 2005-06-14 16:49:21.000000000 -0600
@@ -32,6 +32,39 @@
#include <string>
#include <apt-pkg/fileutl.h>
+#include <config.h>
+
+#ifdef USE_NO_MMAP_FALLBACK
+# include <map>
+# include <unistd.h>
+# include <sys/types.h>
+# include <sys/stat.h>
+
+# include <iostream>
+
+struct key {
+ ino_t inode;
+ dev_t device;
+};
+
+struct mem_cache {
+ void *ptr;
+ size_t len;
+ short ref_count;
+};
+
+
+struct stat_eq {
+ bool operator()(struct key k1, struct key k2) const {
+ if (k1.inode == k2.inode)
+ return (k1.device < k2.device);
+ return (k1.inode < k2.inode);
+ }
+};
+
+using std::map;
+#endif
+
using std::string;
/* This should be a 32 bit type, larger tyes use too much ram and smaller
@@ -41,6 +74,17 @@ typedef unsigned int map_ptrloc;
class MMap
{
+#ifdef USE_NO_MMAP_FALLBACK
+ private:
+
+ static map<struct key, struct mem_cache, stat_eq > cached_mem;
+
+ bool baseMalloced;
+ FileFd savedFd;
+ struct key my_key;
+ bool fill_buffer(int fd, void **buf, unsigned long size);
+#endif
+
protected:
unsigned long Flags;
diff -urp apt-0.6.25.orig/apt-pkg/pkgcachegen.cc apt-0.6.25/apt-pkg/pkgcachegen.cc
--- apt-0.6.25.orig/buildlib/config.h.in 2002-11-22 00:15:23.000000000 -0700
+++ apt-0.6.25/buildlib/config.h.in 2005-06-14 17:05:20.000000000 -0600
@@ -43,3 +43,6 @@
/* The package name string */
#undef PACKAGE
+
+/* Define if mmap() emulation is needed. */
+#undef USE_NO_MMAP_FALLBACK
diff -urp apt-0.6.25.orig/configure.in apt-0.6.25/configure.in
--- apt-0.6.25.orig/configure.in 2004-06-09 06:30:22.000000000 -0600
+++ apt-0.6.25/configure.in 2005-06-14 17:00:49.000000000 -0600
@@ -18,7 +18,7 @@ AC_CONFIG_AUX_DIR(buildlib)
AC_CONFIG_HEADER(include/config.h:buildlib/config.h.in include/apti18n.h:buildlib/apti18n.h.in)
dnl -- SET THIS TO THE RELEASE VERSION --
-AC_DEFINE_UNQUOTED(VERSION,"0.6.25")
+AC_DEFINE_UNQUOTED(VERSION,"0.6.25-realm1")
PACKAGE="apt"
AC_DEFINE_UNQUOTED(PACKAGE,"$PACKAGE")
AC_SUBST(PACKAGE)
@@ -189,6 +189,11 @@ rc_GLIBC_VER
rc_LIBSTDCPP_VER
ah_GCC3DEP
+AC_ARG_ENABLE(mmap-fallback,
+ [ --enable-mmap-fallback Support emulation of mmap() ],
+ AC_DEFINE(USE_NO_MMAP_FALLBACK, 1,
+ ["Define if mmap() emulation is needed"]))
+
dnl It used to be that the user could select translations and that could get
dnl passed to the makefiles, but now that can only work if you use special
dnl gettext approved makefiles, so this feature is unsupported by this.
diff -urp apt-0.6.25.orig/debian/changelog apt-0.6.25/debian/changelog
--- apt-0.6.25.orig/debian/changelog 2004-06-09 06:33:17.000000000 -0600
+++ apt-0.6.25/debian/changelog 2005-06-14 16:54:13.000000000 -0600
@@ -1,4 +1,11 @@
+apt (0.6.25-realm1) experimental; urgency=low
+
+ * Add mmap() emulation to apt for platforms which don't support
+ shared writable mmap.
+
+ -- Matthew S. Wood <matt@realmsys.com> Tue, 14 Jun 2005 16:53:42 -0600
+
apt (0.6.25) experimental; urgency=low
* Fix handling of two-part sources for sources.list deb-src entries in
Reply to: