[RFR] Creating tasks dynamically

To: debian-cd@lists.debian.org
Subject: [RFR] Creating tasks dynamically
From: Frans Pop <elendil@planet.nl>
Date: Thu, 16 Oct 2008 15:21:12 +0200
Message-id: <[🔎] 200810161521.22359.elendil@planet.nl>

Hiya,

I've been working on a patch that dynamically creates some of the 
debian-cd task files instead of having them in SVN:
- debian-installer
- debian-installer+kernel
- task-essential*
- task-full*

The "static" task files and the task file based on popcon data have been 
moved to per-release subdirectories.
The popcon task file is not yet dynamically generated as there are some 
considerations that should be discussed before we do so.

This means that in the new situation you get this:
$ ls tasks/
etch  firmware  lenny  README  sid->lenny
$ ls tasks/lenny/
Debian       empty                 task.list       popularity-contest
Debian-kde   forcd1                task.list.kde
Debian-xfce  interesting-fromcd23  task.list.xfce

The "Debian" task files lose the codename refs in the includes:
$ grep include tasks/lenny/Debian
#include <debian-installer+kernel>
#include <forcd1>
#include <task-essential>
#include <task-full>
#include <interesting-fromcd23>
#include <popularity-contest>

The basic concept of the new setup is that early in a build we copy
tasks/$CODENAME/* to $BDIR/tasks/ and then generate the D-I and tasksel 
task files in the latter dir.

The attached diff shows the changes to scripts etc. The changes in the 
tasks directory have been omitted from the diff.
Documentation updates are TODO.

I've successfully build a BC and a KDE full CD with the new code.

I've dropped EXCLUDE1 and UNEXCLUDE2 and the corresponding task files as 
they no longer seem to be used anywhere.

Cheers,
FJP

diff --git a/CONF.sh b/CONF.sh
index 46d5164..ecf8c51 100644
--- a/CONF.sh
+++ b/CONF.sh
@@ -185,14 +185,6 @@ export DISKTYPE=CD
 #export DISKTYPE=CUSTOM
 #export CUSTOMSIZE=XXXX
 
-# We don't want certain packages to take up space on CD1...
-#export EXCLUDE1="$BASEDIR"/tasks/exclude-$CODENAME
-# ...but they are okay for other CDs (UNEXCLUDEx == may be included
-# on CD x if not already covered)
-#export UNEXCLUDE2="$BASEDIR"/tasks/unexclude-CD2-$CODENAME
-# Any packages listed in EXCLUDE but not in any UNEXCLUDE will be
-# excluded completely.
-
 # Set this if the recommended packages should be skipped when adding 
 # package on the CD.  The default is 'false'.
 export NORECOMMENDS=1
@@ -298,10 +290,10 @@ done
 # and if you're not using build.sh then also make sure you set TASK
 # appropriately here)
 # INSTALLER_CD=0: nothing special (default)
-# INSTALLER_CD=1: just add debian-installer (use TASK=tasks/debian-installer-$CODENAME)
-# INSTALLER_CD=2: add d-i and base (use TASK=tasks/debian-installer+kernel-$CODENAME)
+# INSTALLER_CD=1: just add debian-installer (use TASK=debian-installer)
+# INSTALLER_CD=2: add d-i and base (use TASK=debian-installer+kernel)
 #export INSTALLER_CD=2
-#export TASK=tasks/debian-installer+kernel-$CODENAME
+#export TASK=debian-installer+kernel
 
 # Parameters to pass to kernel (or d-i) when the CD boots. Not currently
 # supported for all architectures.
diff --git a/Makefile b/Makefile
index 8e8964e..0ef5a2e 100755
--- a/Makefile
+++ b/Makefile
@@ -18,7 +18,7 @@ ifndef VERBOSE_MAKE
 Q=@
 endif
 ifndef TASK
-TASK=$(BASEDIR)/tasks/Debian_$(CODENAME)
+TASK=Debian
 endif
 ifndef MKISOFS
 export MKISOFS=$(shell which genisoimage mkisofs | head -1)
@@ -56,6 +56,7 @@ grab_source_list=$(BASEDIR)/tools/grab_source_list
 which_deb=$(BASEDIR)/tools/which_deb
 
 BDIR=$(TDIR)/$(CODENAME)
+TASKDIR=$(BDIR)/tasks
 ADIR=$(APTTMP)
 DB_DIR=$(BDIR)/debootstrap
 
@@ -108,7 +109,7 @@ endif
 ## INITIALIZATION ##
 
 # Creation of the directories needed
-init: ok $(OUT) $(TDIR) $(BDIR) $(ADIR) $(BDIR)/DATE $(DB_DIR) unstable-map
+init: ok $(OUT) $(TDIR) $(BDIR) $(ADIR) $(TASKDIR) $(BDIR)/DATE $(DB_DIR) unstable-map
 $(OUT):
 	$(Q)mkdir -p $(OUT)
 $(TDIR):
@@ -117,6 +118,13 @@ $(BDIR):
 	$(Q)mkdir -p $(BDIR)
 $(ADIR):
 	$(Q)mkdir -p $(ADIR)
+$(TASKDIR):
+	$(Q)mkdir -p $(TASKDIR)
+	cp $(BASEDIR)/tasks/$(CODENAME)/* $(TASKDIR)
+	set -e; cd $(TASKDIR); \
+		$(BASEDIR)/tools/update_tasks; \
+		$(BASEDIR)/tools/generate_di_list; \
+		$(BASEDIR)/tools/generate_di+k_list
 $(BDIR)/DATE:
 	$(Q)date '+%Y%m%d' > $(BDIR)/DATE
 $(DB_DIR): $(LATEST_DB)
@@ -294,11 +302,11 @@ $(BDIR)/rawlist:
 		ARCHUNDEFS="$$ARCHUNDEFS -U $$ARCH"; \
 	done; \
 	if [ "$(SOURCEONLY)"x != "yes"x ] ; then \
-		cat $(TASK) | \
+		cat $(TASKDIR)/$(TASK) | \
 		cpp -nostdinc -nostdinc++ -P -undef $$ARCHDEFS \
 	   		$$ARCHUNDEFS -U i386 -U linux -U unix \
 		    -DFORCENONUSONCD1=0 \
-		    -I $(BASEDIR)/tasks -I $(BDIR) - - >> $(BDIR)/rawlist; \
+		    -I $(TASKDIR) -I $(BDIR) - - >> $(BDIR)/rawlist; \
 	fi
 
     # If we're *only* doing source, then we need to build a list of all the
@@ -386,10 +394,7 @@ mirrorcheck: ok
 	done
 
 update-popcon:
-	$(update_popcon) tasks/popularity-contest-$(CODENAME)
-
-update-tasks:
-	$(update_tasks) $(CODENAME)
+	$(update_popcon) tasks/$(CODENAME)/popularity-contest
 
 # Little trick to simplify things
 official_images: ok init packagelists image-trees images
diff --git a/build.sh b/build.sh
index 95829b2..8423642 100755
--- a/build.sh
+++ b/build.sh
@@ -26,11 +26,11 @@ export PATH
 if [ "$TASK"x = ""x ] ; then
 	case "$INSTALLER_CD"x in
 		"1"x)
-			TASK=tasks/debian-installer-$DI_CODENAME
+			TASK=debian-installer
 			unset COMPLETE
 			;;
 		"2"x)
-			TASK=tasks/debian-installer+kernel-$CODENAME
+			TASK=debian-installer+kernel
 			unset COMPLETE
 			;;
 		*)
diff --git a/build_all.sh b/build_all.sh
index ba378e4..e4be750 100755
--- a/build_all.sh
+++ b/build_all.sh
@@ -19,11 +19,11 @@ export PATH
 if [ "$TASK"x = ""x ] ; then
 	case "$INSTALLER_CD"x in
 		"1"x)
-			TASK=tasks/debian-installer-$DI_CODENAME
+			TASK=debian-installer
 			unset COMPLETE
 			;;
 		"2"x)
-			TASK=tasks/debian-installer+kernel-$CODENAME
+			TASK=debian-installer+kernel
 			unset COMPLETE
 			;;
 		*)
diff --git a/easy-build.sh b/easy-build.sh
index 983a604..6df5230 100755
--- a/easy-build.sh
+++ b/easy-build.sh
@@ -99,8 +99,6 @@ export MAXJIGDOS=0
 # Include proposed-updates
 #export PROPOSED_UPDATES=$CODENAME-proposed-updates
 
-#export EXCLUDE1="$BASEDIR"/tasks/exclude-$CODENAME
-#export UNEXCLUDE2="$BASEDIR"/tasks/unexclude-CD2-$CODENAME
 #export UDEB_INCLUDE="$BASEDIR"/data/$CODENAME/udeb_include
 #export UDEB_EXCLUDE="$BASEDIR"/data/$CODENAME/udeb_exclude
 #export BASE_INCLUDE="$BASEDIR"/data/$CODENAME/base_include
@@ -135,7 +133,7 @@ esac
 
 # By default a GNOME CD/DVD is built, but KDE and Xfce are supported too
 if [ "$desktop" ] && ([ $DISKTYPE = CD ] || [ $DISKTYPE = DVD ]); then
-	TASK=tasks/Debian_${CODENAME}-${desktop}
+	TASK=Debian-${desktop}
 	if [ $CODENAME = etch ]; then
 		KERNEL_PARAMS="tasks=\"${desktop}-desktop, standard\""
 	else
@@ -153,23 +151,6 @@ if [ "$LOCAL" ] && [ "$UPDATE_LOCAL" ]; then
 	done
 fi
 
-if [ -d tasks ]; then
-	if [ $DISKTYPE = CD ] || [ $DISKTYPE = DVD ]; then
-		echo "Updating task files derived from tasksel..."
-		make update-tasks
-	fi
-
-	echo "Updating debian-installer task files..."
-	(
-		cd tasks
-		../tools/generate_di_list
-		../tools/generate_di+k_list
-	)
-else
-	echo "Error: cannot find tasks directory"
-	exit 1
-fi
-
 echo
 echo "Starting the actual debian-cd build..."
 ./build.sh "$ARCHES"
diff --git a/tools/generate_di+k_list b/tools/generate_di+k_list
index dc3f304..43bb513 100755
--- a/tools/generate_di+k_list
+++ b/tools/generate_di+k_list
@@ -4,7 +4,7 @@ set -e
 
 # Generate a list of packages required for debian-installer
 # This script makes use of the following variables that need to be preset:
-# MIRROR, CODENAME, DI_CODENAME
+# MIRROR, CODENAME
 if [ "$MIRROR"x = ""x ] ; then
     echo "\$MIRROR unset; abort!"
     exit 1
@@ -19,7 +19,7 @@ if [ "$DI_CODENAME"x = ""x ] ; then
 fi
 
 DATE=`date`
-cat > debian-installer+kernel-$CODENAME << EOF
+cat > debian-installer+kernel << EOF
 /* These packages + the ones needed by debootstrap are the ones needed for
  * debian-installer to be able to install a base Debian system.
  *
@@ -34,7 +34,7 @@ cat > debian-installer+kernel-$CODENAME << EOF
  * Last update: $DATE
  */
 
-#include <debian-installer-$DI_CODENAME>
+#include <debian-installer>
 eject
 locales
 libdevmapper1.02
diff --git a/tools/generate_di_list b/tools/generate_di_list
index 0a2d5a4..0302bd4 100755
--- a/tools/generate_di_list
+++ b/tools/generate_di_list
@@ -10,7 +10,7 @@ my @ARCHES=qw{alpha arm armel hppa hurd-i386 i386 ia64 mips mipsel
               powerpc s390 sparc amd64};
 my $DATE=`date`;
 chomp $DATE;
-open(OUT, ">debian-installer-$ENV{DI_CODENAME}") || die "write: $!";
+open(OUT, ">debian-installer") || die "write: $!";
 print OUT << "EOF";
 /* List of udebs to be included so that debian-installer works fine 
  *
diff --git a/tools/update_tasks b/tools/update_tasks
index b612743..d027669 100755
--- a/tools/update_tasks
+++ b/tools/update_tasks
@@ -1,35 +1,22 @@
 #!/bin/sh
 set -e
 
-CODENAME="$1"
 if [ -z "$CODENAME" ]; then
-    echo "usage: update_tasks CODENAME" >&2
+    echo "update_tasks: codename not specified" >&2
     exit 1
 fi
 
 if [ "$MIRROR"x = ""x ] ; then
-    echo "update_tasks needs to know where the mirror is" >&2
+    echo "update_tasks: mirror dir not specified" >&2
     exit 1
 fi
 
-# Determine temp dir to use
 if [ "$BDIR"x = ""x ] ; then
-    if [ "$TMPDIR"x != ""x ] ; then
-        BDIR=$TMPDIR
-    else
-        BDIR=/tmp
-    fi
-    echo "update_tasks not given a temp dir, using $BDIR" >&2
-fi
-if [ ! -d "$BDIR" ] ; then
-    echo "update_tasks: temp dir '$BDIR' does not exist" >&2
+    echo "update_tasks: temp dir not specified" >&2
     exit 1
 fi
-
-# Create temp dir and ensure cleanup
-TDIR=$BDIR/update_tasks.$$
+TDIR=$BDIR/update_tasks
 mkdir -p $TDIR
-trap 'rm -rf $TDIR' EXIT HUP INT QUIT TERM
 
 
 update_full_list () {
@@ -37,7 +24,6 @@ update_full_list () {
     tasklist=$2
     pkgfile=$3
 
-    grep '\*' $file > $file.new
     (grep -Ev "^(#.*)?[[:space:]]*$" $tasklist ; echo DONE ; cat $pkgfile) | mawk '
     /DONE/ {
         in_packages = 1
@@ -73,8 +59,7 @@ update_full_list () {
             }
         }
         next
-    }' | sort -n | cut -d: -f2 >> $file.new    
-    mv $file.new $file
+    }' | sort -n | cut -d: -f2 > $file
 }
 
 update_essential_list () {
@@ -83,7 +68,6 @@ update_essential_list () {
     desktoptask=$3
     tasksel=$4
 
-    grep '\*' $file > $file.new
     (grep -Ev "^(#.*)?[[:space:]]*$" $tasklist ;
 		echo DONE ;
 		cat $tasksel/usr/share/tasksel/debian-tasks.desc) | mawk -v DT=$3 '
@@ -129,8 +113,7 @@ update_essential_list () {
             }
         }
         next
-    }' | sort -s -n -k1 | cut -d: -f2 >> $file.new
-    mv $file.new $file
+    }' | sort -s -n -k1 | cut -d: -f2 > $file
 }
 
 # We need to gunzip a copy of the appropriate Packages.gz file
@@ -145,23 +128,19 @@ TASKSEL_DEB=$MIRROR/`mawk '
 
 dpkg -x $TASKSEL_DEB $TDIR/tasksel
 
-update_essential_list tasks/task-essential-$CODENAME \
-    tasks/task.list gnome-desktop \
+update_essential_list task-essential task.list gnome-desktop \
     $TDIR/tasksel
 
-update_essential_list tasks/task-essential-$CODENAME-kde \
-    tasks/task.list.kde kde-desktop \
+update_essential_list task-essential-kde task.list.kde kde-desktop \
     $TDIR/tasksel
 
-update_essential_list tasks/task-essential-$CODENAME-xfce \
-    tasks/task.list.xfce xfce-desktop \
+update_essential_list task-essential-xfce task.list.xfce xfce-desktop \
     $TDIR/tasksel
 
-update_full_list tasks/task-full-$CODENAME \
-    tasks/task.list $TMP_PKG
+update_full_list task-full task.list $TMP_PKG
+
+update_full_list task-full-kde task.list.kde $TMP_PKG
 
-update_full_list tasks/task-full-$CODENAME-kde \
-    tasks/task.list.kde $TMP_PKG
+update_full_list task-full-xfce task.list.xfce $TMP_PKG
 
-update_full_list tasks/task-full-$CODENAME-xfce \
-    tasks/task.list.xfce $TMP_PKG
+rm -rf $TDIR

Attachment: signature.asc
Description: This is a digitally signed message part.

Reply to:

Follow-Ups:
- Re: [RFR] Creating tasks dynamically
  - From: Frans Pop <elendil@planet.nl>
- Re: [RFR] Creating tasks dynamically v3
  - From: Frans Pop <elendil@planet.nl>
- Re: [RFR] Creating tasks dynamically - alternative
  - From: Frans Pop <elendil@planet.nl>

Prev by Date: Re: Bits from the Debian CD team
Next by Date: Re: [RFR] Creating tasks dynamically
Previous by thread: Re: Bits from the Debian CD team
Next by thread: Re: [RFR] Creating tasks dynamically
Index(es):
- Date
- Thread