Bug#864777: initramfs-tools: Please support a tmpfs boot mode
Hi Peter,
Am Donnerstag, den 29.06.2017, 15:55 -0400 schrieb Peter Colberg:
> Hi Benjamin,
>
> On Wed, Jun 14, 2017 at 05:47:54PM +0200, Benjamin Drung wrote:
> > Package: initramfs-tools
> > Version: 0.130
> > Severity: normal
> > Tags: patch
> >
> > Hi,
> >
> > The tmpfs boot mode allows one to operate a disk-less live system
> > by
> > downloading a root tarball (that can be created with debootstrap)
> > and
> > extracting it to a tmpfs root partition.
> >
> > The tmpfs boot mode is similar to the nfs boot mode, but it does
> > not
> > rely on any external service (after booting). We use this boot mode
> > for
> > our compute nodes. I have attached a patch to support this boot
> > mode and
> > tested it with qemu and on real hardware.
>
> Please take a look at the package live-boot (and the man page of the
> same name in live-boot-doc). Specifically, the parameter fetch=URL
> should satisfy your requirements and consume less resources.
>
> live-boot creates an overlay filesystem backed by a squashfs image,
> which means it likely uses much less RAM than copying root to tmpfs.
> (For my system, squashfs reduces the required memory to a third.)
I took a look at the live-boot package and we will evaluate the RAM
savings and performance penalty. Expect a patch for live-boot to
support live-{top,premount,bottom} hooks.
Some statistics: Our xz compressed root tarball is around 180 MB in
size and consumes 690 MB extracted. Our initrd uses 9 MB. The initrd
from live-boot uses 17 MB. Let's see if the live-boot initrd can be
stripped down as well.
For now, I updated my patch to add an option to enable/disable support
for the tmpfs boot mode. Updated patch is attached.
--
Benjamin Drung
System Developer
Debian & Ubuntu Developer
ProfitBricks GmbH
Greifswalder Str. 207
D - 10405 Berlin
Email: benjamin.drung@profitbricks.com
URL: https://www.profitbricks.de
Sitz der Gesellschaft: Berlin
Registergericht: Amtsgericht Charlottenburg, HRB 125506 B
Geschäftsführer: Achim Weiss, Matthias SteinbergFrom 500c0566ef08a460b666c881b6d5a120b01501f4 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <bdrung@debian.org>
Date: Mon, 12 Jun 2017 15:31:58 +0200
Subject: [PATCH 1/2] Fix indentation (spaces to tabs)
---
init | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/init b/init
index a88a64c..035469b 100755
--- a/init
+++ b/init
@@ -85,7 +85,7 @@ for x in $(cat /proc/cmdline); do
if [ -z "${BOOT}" ] && [ "$ROOT" = "/dev/nfs" ]; then
BOOT=nfs
fi
- ;;
+ ;;
rootflags=*)
ROOTFLAGS="-o ${x#rootflags=}"
;;
@@ -115,7 +115,7 @@ for x in $(cat /proc/cmdline); do
resume=*)
RESUME="${x#resume=}"
case $RESUME in
- UUID=*)
+ UUID=*)
RESUME="/dev/disk/by-uuid/${RESUME#UUID=}"
esac
;;
--
2.14.1
From c236415782ea0171a64049d273b1fab557ba6698 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Wed, 7 Jun 2017 18:04:47 +0200
Subject: [PATCH 2/2] Add tmpfs boot mode
The tmpfs boot mode allows one to operate a disk-less live system by downloading
a root tarball (that can be created with debootstrap) and extracting it to a
tmpfs root partition.
Closes: #864777
Signed-off-by: Benjamin Drung <benjamin.drung@profitbricks.com>
---
conf/initramfs.conf | 22 ++++++++
hooks/tmpfs | 39 +++++++++++++
init | 14 ++++-
initramfs-tools.8 | 23 ++++----
initramfs.conf.5 | 14 +++++
mkinitramfs | 1 +
scripts/tmpfs | 158 ++++++++++++++++++++++++++++++++++++++++++++++++++++
7 files changed, 259 insertions(+), 12 deletions(-)
create mode 100755 hooks/tmpfs
create mode 100644 scripts/tmpfs
diff --git a/conf/initramfs.conf b/conf/initramfs.conf
index f0f5071..4af51e4 100644
--- a/conf/initramfs.conf
+++ b/conf/initramfs.conf
@@ -62,3 +62,25 @@ DEVICE=
NFSROOT=auto
+#
+# tmpfs section of the config.
+#
+
+#
+# TMPFS: [ y | n ]
+#
+# Enable tmpfs boot mode support (i.e. copy required files into initramfs).
+#
+
+TMPFS=n
+
+#
+# ROOTSIZE: ...
+#
+# The size of the tmpfs root mount in bytes, and rounded up to entire pages.
+# This option accepts a suffix % to limit the instance to that percentage of
+# your physical RAM or a suffix k, m or g for Ki, Mi, Gi (binary kilo (kibi),
+# binary mega (mebi) and binary giga (gibi)).
+#
+
+ROOTSIZE=50%
diff --git a/hooks/tmpfs b/hooks/tmpfs
new file mode 100755
index 0000000..49bbca0
--- /dev/null
+++ b/hooks/tmpfs
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+PREREQ=""
+
+prereqs()
+{
+ echo "$PREREQ"
+}
+
+case $1 in
+# get pre-requisites
+prereqs)
+ prereqs
+ exit 0
+ ;;
+esac
+
+# Hook for supporting tmpfs boot mode if requested by TMPFS="y"
+if [ "$TMPFS" != "y" ] && [ "$TMPFS" != "Y" ]; then
+ exit 0
+fi
+
+. /usr/share/initramfs-tools/hook-functions
+
+# GNU tar is needed for xattrs support
+if test -e /bin/tar; then
+ test -z "$DESTDIR" || rm -f ${DESTDIR}/bin/tar
+ copy_exec /bin/tar
+fi
+
+# pixz is used to speed up the decompression
+if test -e /usr/bin/pixz; then
+ copy_exec /usr/bin/pixz
+fi
+
+# libnss_dns is needed for resolving hosts by name
+if test -e /lib/x86_64-linux-gnu/libnss_dns.so.2; then
+ copy_exec /lib/x86_64-linux-gnu/libnss_dns.so.2
+fi
diff --git a/init b/init
index 035469b..11b8750 100755
--- a/init
+++ b/init
@@ -82,8 +82,15 @@ for x in $(cat /proc/cmdline); do
;;
root=*)
ROOT=${x#root=}
- if [ -z "${BOOT}" ] && [ "$ROOT" = "/dev/nfs" ]; then
- BOOT=nfs
+ if [ -z "${BOOT}" ]; then
+ case ${ROOT} in
+ /dev/nfs)
+ BOOT=nfs
+ ;;
+ http*)
+ BOOT=tmpfs
+ ;;
+ esac
fi
;;
rootflags=*)
@@ -100,6 +107,9 @@ for x in $(cat /proc/cmdline); do
;;
esac
;;
+ rootsize=*)
+ ROOTSIZE="${x#rootsize=}"
+ ;;
nfsroot=*)
NFSROOT="${x#nfsroot=}"
;;
diff --git a/initramfs-tools.8 b/initramfs-tools.8
index 0eee050..94e76e2 100644
--- a/initramfs-tools.8
+++ b/initramfs-tools.8
@@ -27,6 +27,8 @@ the binary to hand over execution to on the root fs after the initramfs scripts
\fB\fI root= "<path to blockdevice>"
the device node to mount as the root file system.
The recommended usage is to specify the UUID as followed "root=UUID=xxx".
+When using the tmpfs boot mode, specify a URL to a (compressed) root tarball
+here.
.TP
\fB\fI rootfstype
@@ -62,7 +64,7 @@ bootarg.
.TP
\fB\fI boot
-either local or NFS (affects which initramfs scripts are run, see the "Subdirectories" section under boot scripts).
+local, NFS, or tmpfs (affects which initramfs scripts are run, see the "Subdirectories" section under boot scripts).
.TP
\fB\fI resume
@@ -303,7 +305,7 @@ allows arch specific hook additions.
\fB\fI verbose
corresponds to the verbosity of the update-initramfs run.
.TP
-\fB\fI BUSYBOX, KEYMAP, MODULES
+\fB\fI BUSYBOX, KEYMAP, MODULES, TMPFS
are as described in \fIinitramfs.conf\fR(5).
.TP
\fB\fI BUSYBOXDIR
@@ -432,9 +434,9 @@ have been loaded.
.TP
\fB\fI
-local-top OR nfs-top
+local-top, nfs-top, OR tmpfs-top
After these scripts have been executed, the root device node is expected to be
-present (local) or the network interface is expected to be usable (NFS).
+present (local) or the network interface is expected to be usable (NFS, tmpfs).
.TP
\fB\fI
@@ -447,16 +449,17 @@ try again.
.TP
\fB\fI
-local-premount OR nfs-premount
+local-premount, nfs-premount, OR tmpfs-premount
are run after the sanity of the root device has been verified (local) or the
-network interface has been brought up (NFS), but before the actual root fs has
-been mounted.
+network interface has been brought up (NFS, tmpfs) and the empty root partition
+is mounted (tmpfs), but before the actual root fs has been mounted.
.TP
\fB\fI
-local-bottom OR nfs-bottom
-are run after the rootfs has been mounted (local) or the NFS root share has
-been mounted.
+local-bottom, nfs-bottom, OR tmpfs-bottom
+are run after the rootfs has been mounted (local), the NFS root share has
+been mounted, or the root tarball was extract to the mounted tmpfs root
+partition.
.TP
\fB\fI
diff --git a/initramfs.conf.5 b/initramfs.conf.5
index b4eea70..e439cbd 100644
--- a/initramfs.conf.5
+++ b/initramfs.conf.5
@@ -99,6 +99,20 @@ A root bootarg overrides that special setting.
Defaults to \fIauto\fP in order to pick up value from DHCP server.
Otherwise you need to specify \fIHOST:MOUNT\fP.
+.SH VARIABLES FOR TMPFS BOOT
+
+.TP
+\fB TMPFS
+If set to 'y', the required files are copied into the initramfs that are
+neede to support the tmpfs boot mode.
+
+.TP
+\fB ROOTSIZE
+The size of the tmpfs root mount in bytes, and rounded up to entire pages. This
+option accepts a suffix % to limit the instance to that percentage of your
+physical RAM or a suffix k, m or g for Ki, Mi, Gi (binary kilo (kibi), binary
+mega (mebi) and binary giga (gibi)). The default is half of the memory (50%).
+
.SH FILES
.TP
.I /etc/initramfs-tools/initramfs.conf
diff --git a/mkinitramfs b/mkinitramfs
index 24715d5..60841ec 100755
--- a/mkinitramfs
+++ b/mkinitramfs
@@ -206,6 +206,7 @@ export KEYMAP
export MODULES
export BUSYBOX
export RESUME
+export TMPFS
# Private, used by 'catenate_cpiogz'.
export __TMPCPIOGZ
diff --git a/scripts/tmpfs b/scripts/tmpfs
new file mode 100644
index 0000000..e21cbac
--- /dev/null
+++ b/scripts/tmpfs
@@ -0,0 +1,158 @@
+# Diskless system running on tmpfs -*- shell-script -*-
+
+persist_networking()
+{
+ if [ -n "$DNSDOMAIN" ]; then
+ [ "${quiet}" = "y" ] || echo "/etc/resolv.conf: Setting local domain name to $DNSDOMAIN"
+ echo "domain $DNSDOMAIN" >> /etc/resolv.conf
+ fi
+ if [ -n "$IPV4DNS0" -a "$IPV4DNS0" != "0.0.0.0" ]; then
+ [ "${quiet}" = "y" ] || echo "/etc/resolv.conf: Adding DNS ${IPV4DNS0}"
+ echo "nameserver ${IPV4DNS0}" >> /etc/resolv.conf
+ fi
+ if [ -n "$IPV4DNS1" -a "$IPV4DNS1" != "0.0.0.0" ]; then
+ [ "${quiet}" = "y" ] || echo "/etc/resolv.conf: Adding DNS ${IPV4DNS1}"
+ echo "nameserver ${IPV4DNS1}" >> /etc/resolv.conf
+ fi
+ if [ -n "$DOMAINSEARCH" ]; then
+ [ "${quiet}" = "y" ] || echo "/etc/resolv.conf: Setting search list to $DNSDOMAIN $DOMAINSEARCH"
+ echo "search $DNSDOMAIN $DOMAINSEARCH" >> /etc/resolv.conf
+ fi
+ if [ -n "$HOSTNAME" ]; then
+ [ "${quiet}" = "y" ] || echo "Setting hostname to $hostname"
+ echo $HOSTNAME > /etc/hostname
+ if [ -n "$DNSDOMAIN" ]; then
+ names="$HOSTNAME.$DNSDOMAIN $HOSTNAME"
+ else
+ names="$HOSTNAME"
+ fi
+ echo "127.0.0.1 localhost
+127.0.1.1 $names
+
+# The following lines are desirable for IPv6 capable hosts
+::1 localhost ip6-localhost ip6-loopback
+ff02::1 ip6-allnodes
+ff02::2 ip6-allrouters" > /etc/hosts
+ fi
+}
+
+tmpfs_top()
+{
+ if [ "${tmpfs_top_used}" != "yes" ]; then
+ [ "$quiet" != "y" ] && log_begin_msg "Running /scripts/tmpfs-top"
+ run_scripts /scripts/tmpfs-top
+ [ "$quiet" != "y" ] && log_end_msg
+ fi
+ tmpfs_top_used=yes
+
+ # For DHCP
+ modprobe af_packet
+
+ wait_for_udev 10
+
+ [ "$quiet" = "y" ] || log_begin_msg "Starting networking"
+ configure_networking || panic "Network configuration failed."
+ [ "$quiet" = "y" ] || log_end_msg
+
+ persist_networking
+
+ mount -t tmpfs -o size=${ROOTSIZE-50%} tmpfs ${rootmnt} || panic "tmpfs root mount failed."
+}
+
+tmpfs_premount()
+{
+ if [ "${tmpfs_premount_used}" != "yes" ]; then
+ [ "$quiet" != "y" ] && log_begin_msg "Running /scripts/tmpfs-premount"
+ run_scripts /scripts/tmpfs-premount
+ [ "$quiet" != "y" ] && log_end_msg
+ fi
+ tmpfs_premount_used=yes
+}
+
+tmpfs_bottom()
+{
+ if [ "${tmpfs_premount_used}" = "yes" ] || [ "${tmpfs_top_used}" = "yes" ]; then
+ [ "$quiet" != "y" ] && log_begin_msg "Running /scripts/tmpfs-bottom"
+ run_scripts /scripts/tmpfs-bottom
+ [ "$quiet" != "y" ] && log_end_msg
+ fi
+ tmpfs_premount_used=no
+ tmpfs_top_used=no
+}
+
+sleep_break()
+{
+ local timeout=$1
+ if test "$timeout" -ge 1; then
+ echo "Press x (in the next $timeout seconds) to open a shell"
+ read -t $timeout -r -s -n 1 input
+ [ "$input" = "x" ] && panic "Spawning shell within the initramfs as requested"
+ fi
+}
+
+extract()
+{
+ local archive="$1"
+ local TAROPTS=""
+ echo "Extracting ${archive} to ${rootmnt}..."
+ case "$archive" in
+ *.xz)
+ if test -e /usr/bin/pixz; then
+ TAROPTS=-Ipixz
+ fi
+ ;;
+ esac
+ /bin/tar $TAROPTS --xattrs --xattrs-include='*' -xf "$archive" -C "${rootmnt}" || panic "Extracting "${archive}" failed."
+}
+
+tmpfs_mount_root()
+{
+ host=$(hostname)
+ while ! fqdn=$(hostname -f); do
+ local sleep_t="$((RANDOM%30+10))"
+ echo "Failed to determine fully qualified domain name (FQDN), retry in $sleep_t seconds..."
+ sleep_break $sleep_t
+ done
+
+ IFS=','
+ for url in $ROOT; do
+ # Replace %hostname and %fqdn in the URL
+ url=$(echo "$url" | sed "s/%hostname/$host/g;s/%fqdn/$fqdn/g")
+ filename=/tmp/$(basename "$url")
+
+ echo "Downloading ${url}..."
+ while ! wget -O "${filename}" "${url}"; do
+ local sleep_t="$((RANDOM%30+10))"
+ echo "Download failed, retry in $sleep_t seconds..."
+ sleep_break $sleep_t
+ done
+ echo "Download successful"
+
+ extract "${filename}"
+ rm "${filename}"
+ done
+ unset IFS
+}
+
+mountroot()
+{
+ tmpfs_mount_root
+}
+
+mount_top()
+{
+ # Note, also called directly in case it's overridden.
+ tmpfs_top
+}
+
+mount_premount()
+{
+ # Note, also called directly in case it's overridden.
+ tmpfs_premount
+}
+
+mount_bottom()
+{
+ # Note, also called directly in case it's overridden.
+ tmpfs_bottom
+}
--
2.14.1
Reply to: