Re: [RFR] Creating tasks dynamically - alternative

To: debian-cd@lists.debian.org
Subject: Re: [RFR] Creating tasks dynamically - alternative
From: Frans Pop <elendil@planet.nl>
Date: Fri, 17 Oct 2008 08:45:32 +0200
Message-id: <[🔎] 200810170845.44085.elendil@planet.nl>
In-reply-to: <[🔎] 200810161521.22359.elendil@planet.nl>
References: <[🔎] 200810161521.22359.elendil@planet.nl>

On Thursday 16 October 2008, Frans Pop wrote:
> The basic concept of the new setup is that early in a build we copy
> tasks/$CODENAME/* to $BDIR/tasks/ and then generate the D-I and tasksel
> task files in the latter dir.

Here's an alternative implementation where the "static" task files are not 
copied, but the dynamically generated task files get included from the 
static task files.

For this to work two new static task files Debian-businesscard and 
Debian-netinst have been created.

I've not yet tested this and some additional cleanup is possible (for 
example, the name d-i+kernel does not really make sense anymore), but the 
attached patch should show the general idea.

I'm not sure whether I prefer this version or the previous one...

diff --git a/CONF.sh b/CONF.sh
index 46d5164..ed9c160 100644
--- a/CONF.sh
+++ b/CONF.sh
@@ -186,10 +186,10 @@ export DISKTYPE=CD
 #export CUSTOMSIZE=XXXX
 
 # We don't want certain packages to take up space on CD1...
-#export EXCLUDE1="$BASEDIR"/tasks/exclude-$CODENAME
+#export EXCLUDE1=exclude
 # ...but they are okay for other CDs (UNEXCLUDEx == may be included
 # on CD x if not already covered)
-#export UNEXCLUDE2="$BASEDIR"/tasks/unexclude-CD2-$CODENAME
+#export UNEXCLUDE2=unexclude-CD2
 # Any packages listed in EXCLUDE but not in any UNEXCLUDE will be
 # excluded completely.
 
@@ -298,10 +298,10 @@ done
 # and if you're not using build.sh then also make sure you set TASK
 # appropriately here)
 # INSTALLER_CD=0: nothing special (default)
-# INSTALLER_CD=1: just add debian-installer (use TASK=tasks/debian-installer-$CODENAME)
-# INSTALLER_CD=2: add d-i and base (use TASK=tasks/debian-installer+kernel-$CODENAME)
+# INSTALLER_CD=1: just add debian-installer (use TASK=Debian-businesscard)
+# INSTALLER_CD=2: add d-i and base (use TASK=Debian-netinst)
 #export INSTALLER_CD=2
-#export TASK=tasks/debian-installer+kernel-$CODENAME
+#export TASK=Debian-netinst
 
 # Parameters to pass to kernel (or d-i) when the CD boots. Not currently
 # supported for all architectures.
diff --git a/Makefile b/Makefile
index 8e8964e..012ebb8 100755
--- a/Makefile
+++ b/Makefile
@@ -18,7 +18,7 @@ ifndef VERBOSE_MAKE
 Q=@
 endif
 ifndef TASK
-TASK=$(BASEDIR)/tasks/Debian_$(CODENAME)
+TASK=Debian
 endif
 ifndef MKISOFS
 export MKISOFS=$(shell which genisoimage mkisofs | head -1)
@@ -55,13 +55,16 @@ update_tasks=$(BASEDIR)/tools/update_tasks
 grab_source_list=$(BASEDIR)/tools/grab_source_list
 which_deb=$(BASEDIR)/tools/which_deb
 
+TASKDIR=$(BASEDIR)/tasks/$(CODENAME)
 BDIR=$(TDIR)/$(CODENAME)
+DYNTASKDIR=$(BDIR)/tasks
 ADIR=$(APTTMP)
 DB_DIR=$(BDIR)/debootstrap
 
 export DEBOOTSTRAP_DIR := $(DB_DIR)/usr/lib/debootstrap
 export PATH := $(DB_DIR)/usr/sbin:$(PATH)
 export BDIR
+export TASKDIR
 
 LATEST_DB := $(shell $(which_deb) $(MIRROR) $(CODENAME) debootstrap)
 
@@ -108,7 +111,7 @@ endif
 ## INITIALIZATION ##
 
 # Creation of the directories needed
-init: ok $(OUT) $(TDIR) $(BDIR) $(ADIR) $(BDIR)/DATE $(DB_DIR) unstable-map
+init: ok $(OUT) $(TDIR) $(BDIR) $(ADIR) $(TASKDIR) $(BDIR)/DATE $(DB_DIR) unstable-map
 $(OUT):
 	$(Q)mkdir -p $(OUT)
 $(TDIR):
@@ -117,6 +120,12 @@ $(BDIR):
 	$(Q)mkdir -p $(BDIR)
 $(ADIR):
 	$(Q)mkdir -p $(ADIR)
+$(TASKDIR):
+	$(Q)mkdir -p $(TASKDIR)
+	$(Q)set -e; cd $(TASKDIR); \
+		$(BASEDIR)/tools/update_tasks; \
+		$(BASEDIR)/tools/generate_di_list; \
+		$(BASEDIR)/tools/generate_di+k_list
 $(BDIR)/DATE:
 	$(Q)date '+%Y%m%d' > $(BDIR)/DATE
 $(DB_DIR): $(LATEST_DB)
@@ -144,6 +153,7 @@ unstable-map:
 clean: ok dir-clean
 dir-clean:
 	$(Q)rm -rf $(BDIR)/CD[1234567890]*
+	$(Q)rm -rf $(BDIR)/tasks
 	$(Q)rm -f $(BDIR)/*.filelist*
 	$(Q)rm -f  $(BDIR)/packages-stamp $(BDIR)/upgrade-stamp $(BDIR)/md5-check
 
@@ -294,11 +304,12 @@ $(BDIR)/rawlist:
 		ARCHUNDEFS="$$ARCHUNDEFS -U $$ARCH"; \
 	done; \
 	if [ "$(SOURCEONLY)"x != "yes"x ] ; then \
-		cat $(TASK) | \
+		cat $(TASKDIR)/$(TASK) | \
 		cpp -nostdinc -nostdinc++ -P -undef $$ARCHDEFS \
 	   		$$ARCHUNDEFS -U i386 -U linux -U unix \
 		    -DFORCENONUSONCD1=0 \
-		    -I $(BASEDIR)/tasks -I $(BDIR) - - >> $(BDIR)/rawlist; \
+		    -I $(TASKDIR) -I $(DYNTASKDIR) -I $(BDIR) - - \
+		    >> $(BDIR)/rawlist; \
 	fi
 
     # If we're *only* doing source, then we need to build a list of all the
@@ -320,7 +331,7 @@ $(BDIR)/rawlist-exclude:
 			cpp -nostdinc -nostdinc++ -P -undef $$ARCHDEFS \
 			$$ARCHUNDEFS -U i386 -U linux -U unix \
 			-DFORCENONUSONCD1=0 \
-			-I $(BASEDIR)/tasks -I $(BDIR) - - >> $(BDIR)/rawlist-exclude; \
+			-I $(TASKDIR) -I $(BDIR) - - >> $(BDIR)/rawlist-exclude; \
 	else \
 		echo > $(BDIR)/rawlist-exclude; \
 	fi
@@ -386,10 +397,7 @@ mirrorcheck: ok
 	done
 
 update-popcon:
-	$(update_popcon) tasks/popularity-contest-$(CODENAME)
-
-update-tasks:
-	$(update_tasks) $(CODENAME)
+	$(update_popcon) tasks/$(CODENAME)/popularity-contest
 
 # Little trick to simplify things
 official_images: ok init packagelists image-trees images
diff --git a/build.sh b/build.sh
index 95829b2..70c1722 100755
--- a/build.sh
+++ b/build.sh
@@ -26,11 +26,11 @@ export PATH
 if [ "$TASK"x = ""x ] ; then
 	case "$INSTALLER_CD"x in
 		"1"x)
-			TASK=tasks/debian-installer-$DI_CODENAME
+			TASK=Debian-businesscard
 			unset COMPLETE
 			;;
 		"2"x)
-			TASK=tasks/debian-installer+kernel-$CODENAME
+			TASK=Debian-netinst
 			unset COMPLETE
 			;;
 		*)
diff --git a/build_all.sh b/build_all.sh
index ba378e4..0e4739b 100755
--- a/build_all.sh
+++ b/build_all.sh
@@ -19,11 +19,11 @@ export PATH
 if [ "$TASK"x = ""x ] ; then
 	case "$INSTALLER_CD"x in
 		"1"x)
-			TASK=tasks/debian-installer-$DI_CODENAME
+			TASK=Debian-businesscard
 			unset COMPLETE
 			;;
 		"2"x)
-			TASK=tasks/debian-installer+kernel-$CODENAME
+			TASK=Debian-netinst
 			unset COMPLETE
 			;;
 		*)
diff --git a/easy-build.sh b/easy-build.sh
index 983a604..6df5230 100755
--- a/easy-build.sh
+++ b/easy-build.sh
@@ -99,8 +99,6 @@ export MAXJIGDOS=0
 # Include proposed-updates
 #export PROPOSED_UPDATES=$CODENAME-proposed-updates
 
-#export EXCLUDE1="$BASEDIR"/tasks/exclude-$CODENAME
-#export UNEXCLUDE2="$BASEDIR"/tasks/unexclude-CD2-$CODENAME
 #export UDEB_INCLUDE="$BASEDIR"/data/$CODENAME/udeb_include
 #export UDEB_EXCLUDE="$BASEDIR"/data/$CODENAME/udeb_exclude
 #export BASE_INCLUDE="$BASEDIR"/data/$CODENAME/base_include
@@ -135,7 +133,7 @@ esac
 
 # By default a GNOME CD/DVD is built, but KDE and Xfce are supported too
 if [ "$desktop" ] && ([ $DISKTYPE = CD ] || [ $DISKTYPE = DVD ]); then
-	TASK=tasks/Debian_${CODENAME}-${desktop}
+	TASK=Debian-${desktop}
 	if [ $CODENAME = etch ]; then
 		KERNEL_PARAMS="tasks=\"${desktop}-desktop, standard\""
 	else
@@ -153,23 +151,6 @@ if [ "$LOCAL" ] && [ "$UPDATE_LOCAL" ]; then
 	done
 fi
 
-if [ -d tasks ]; then
-	if [ $DISKTYPE = CD ] || [ $DISKTYPE = DVD ]; then
-		echo "Updating task files derived from tasksel..."
-		make update-tasks
-	fi
-
-	echo "Updating debian-installer task files..."
-	(
-		cd tasks
-		../tools/generate_di_list
-		../tools/generate_di+k_list
-	)
-else
-	echo "Error: cannot find tasks directory"
-	exit 1
-fi
-
 echo
 echo "Starting the actual debian-cd build..."
 ./build.sh "$ARCHES"
diff --git a/tools/generate_di+k_list b/tools/generate_di+k_list
index dc3f304..86ba5f8 100755
--- a/tools/generate_di+k_list
+++ b/tools/generate_di+k_list
@@ -4,7 +4,7 @@ set -e
 
 # Generate a list of packages required for debian-installer
 # This script makes use of the following variables that need to be preset:
-# MIRROR, CODENAME, DI_CODENAME
+# MIRROR, CODENAME
 if [ "$MIRROR"x = ""x ] ; then
     echo "\$MIRROR unset; abort!"
     exit 1
@@ -19,7 +19,7 @@ if [ "$DI_CODENAME"x = ""x ] ; then
 fi
 
 DATE=`date`
-cat > debian-installer+kernel-$CODENAME << EOF
+cat > debian-installer+kernel << EOF
 /* These packages + the ones needed by debootstrap are the ones needed for
  * debian-installer to be able to install a base Debian system.
  *
@@ -34,7 +34,6 @@ cat > debian-installer+kernel-$CODENAME << EOF
  * Last update: $DATE
  */
 
-#include <debian-installer-$DI_CODENAME>
 eject
 locales
 libdevmapper1.02
diff --git a/tools/generate_di_list b/tools/generate_di_list
index 0a2d5a4..0302bd4 100755
--- a/tools/generate_di_list
+++ b/tools/generate_di_list
@@ -10,7 +10,7 @@ my @ARCHES=qw{alpha arm armel hppa hurd-i386 i386 ia64 mips mipsel
               powerpc s390 sparc amd64};
 my $DATE=`date`;
 chomp $DATE;
-open(OUT, ">debian-installer-$ENV{DI_CODENAME}") || die "write: $!";
+open(OUT, ">debian-installer") || die "write: $!";
 print OUT << "EOF";
 /* List of udebs to be included so that debian-installer works fine 
  *
diff --git a/tools/make_disc_trees.pl b/tools/make_disc_trees.pl
index 9f7dc10..a3a8909 100755
--- a/tools/make_disc_trees.pl
+++ b/tools/make_disc_trees.pl
@@ -579,7 +579,7 @@ sub start_disc {
     undef @unexclude_packages;
 
     if (defined ($ENV{"EXCLUDE"})) {
-        my $excl_file = $ENV{"EXCLUDE"};
+        my $excl_file = $ENV("$TASKDIR")/$ENV{"EXCLUDE"};
         print LOG "Adding excludes from $excl_file\n";
         open (EXCLUDE_FILE, "< $excl_file") || die "Can't open exclude file $excl_file: $!\n";
         while (defined (my $excl_pkg = <EXCLUDE_FILE>)) {
@@ -589,7 +589,7 @@ sub start_disc {
         close (EXCLUDE_FILE);
     }
     if (defined ($ENV{"EXCLUDE$disknum"})) {
-        my $excl_file = $ENV{"EXCLUDE$disknum"};
+        my $excl_file = $ENV("$TASKDIR")/$ENV{"EXCLUDE$disknum"};
         print LOG "Adding excludes from $excl_file\n";
         open (EXCLUDE_FILE, "< $excl_file") || die "Can't open exclude file $excl_file: $!\n";
         while (defined (my $excl_pkg = <EXCLUDE_FILE>)) {
@@ -599,7 +599,7 @@ sub start_disc {
         close (EXCLUDE_FILE);
     }
     if (defined ($ENV{"UNEXCLUDE$disknum"})) {
-        my $excl_file = $ENV{"UNEXCLUDE$disknum"};
+        my $excl_file = $ENV("$TASKDIR")/$ENV{"UNEXCLUDE$disknum"};
         print LOG "Adding unexcludes from $excl_file\n";
         open (EXCLUDE_FILE, "< $excl_file") || die "Can't open unexclude file $excl_file: $!\n";
         while (defined (my $excl_pkg = <EXCLUDE_FILE>)) {
diff --git a/tools/update_tasks b/tools/update_tasks
index b612743..0471a97 100755
--- a/tools/update_tasks
+++ b/tools/update_tasks
@@ -1,43 +1,33 @@
 #!/bin/sh
 set -e
 
-CODENAME="$1"
 if [ -z "$CODENAME" ]; then
-    echo "usage: update_tasks CODENAME" >&2
+    echo "update_tasks: codename not specified" >&2
     exit 1
 fi
 
 if [ "$MIRROR"x = ""x ] ; then
-    echo "update_tasks needs to know where the mirror is" >&2
+    echo "update_tasks: mirror dir not specified" >&2
     exit 1
 fi
 
-# Determine temp dir to use
-if [ "$BDIR"x = ""x ] ; then
-    if [ "$TMPDIR"x != ""x ] ; then
-        BDIR=$TMPDIR
-    else
-        BDIR=/tmp
-    fi
-    echo "update_tasks not given a temp dir, using $BDIR" >&2
-fi
-if [ ! -d "$BDIR" ] ; then
-    echo "update_tasks: temp dir '$BDIR' does not exist" >&2
+if [ "$TASKDIR"x = ""x ] ; then
+    echo "update_tasks: task dir not specified" >&2
     exit 1
 fi
 
-# Create temp dir and ensure cleanup
-TDIR=$BDIR/update_tasks.$$
+if [ "$BDIR"x = ""x ] ; then
+    echo "update_tasks: temp dir not specified" >&2
+    exit 1
+fi
+TDIR=$BDIR/update_tasks
 mkdir -p $TDIR
-trap 'rm -rf $TDIR' EXIT HUP INT QUIT TERM
-
 
 update_full_list () {
     file=$1
     tasklist=$2
     pkgfile=$3
 
-    grep '\*' $file > $file.new
     (grep -Ev "^(#.*)?[[:space:]]*$" $tasklist ; echo DONE ; cat $pkgfile) | mawk '
     /DONE/ {
         in_packages = 1
@@ -73,8 +63,7 @@ update_full_list () {
             }
         }
         next
-    }' | sort -n | cut -d: -f2 >> $file.new    
-    mv $file.new $file
+    }' | sort -n | cut -d: -f2 > $file
 }
 
 update_essential_list () {
@@ -83,7 +72,6 @@ update_essential_list () {
     desktoptask=$3
     tasksel=$4
 
-    grep '\*' $file > $file.new
     (grep -Ev "^(#.*)?[[:space:]]*$" $tasklist ;
 		echo DONE ;
 		cat $tasksel/usr/share/tasksel/debian-tasks.desc) | mawk -v DT=$3 '
@@ -129,8 +117,7 @@ update_essential_list () {
             }
         }
         next
-    }' | sort -s -n -k1 | cut -d: -f2 >> $file.new
-    mv $file.new $file
+    }' | sort -s -n -k1 | cut -d: -f2 > $file
 }
 
 # We need to gunzip a copy of the appropriate Packages.gz file
@@ -145,23 +132,19 @@ TASKSEL_DEB=$MIRROR/`mawk '
 
 dpkg -x $TASKSEL_DEB $TDIR/tasksel
 
-update_essential_list tasks/task-essential-$CODENAME \
-    tasks/task.list gnome-desktop \
+update_essential_list task-essential $TASKDIR/task.list gnome-desktop \
     $TDIR/tasksel
 
-update_essential_list tasks/task-essential-$CODENAME-kde \
-    tasks/task.list.kde kde-desktop \
+update_essential_list task-essential-kde $TASKDIR/task.list.kde kde-desktop \
     $TDIR/tasksel
 
-update_essential_list tasks/task-essential-$CODENAME-xfce \
-    tasks/task.list.xfce xfce-desktop \
+update_essential_list task-essential-xfce $TASKDIR/task.list.xfce xfce-desktop \
     $TDIR/tasksel
 
-update_full_list tasks/task-full-$CODENAME \
-    tasks/task.list $TMP_PKG
+update_full_list task-full $TASKDIR/task.list $TMP_PKG
+
+update_full_list task-full-kde $TASKDIR/task.list.kde $TMP_PKG
 
-update_full_list tasks/task-full-$CODENAME-kde \
-    tasks/task.list.kde $TMP_PKG
+update_full_list task-full-xfce $TASKDIR/task.list.xfce $TMP_PKG
 
-update_full_list tasks/task-full-$CODENAME-xfce \
-    tasks/task.list.xfce $TMP_PKG
+rm -rf $TDIR

Attachment: signature.asc
Description: This is a digitally signed message part.

Reply to:

References:
- [RFR] Creating tasks dynamically
  - From: Frans Pop <elendil@planet.nl>

Prev by Date: Re: [RFR] Creating tasks dynamically v3
Next by Date: Re: Bits from the Debian CD team
Previous by thread: Re: [RFR] Creating tasks dynamically v3
Next by thread: Scour needs your help
Index(es):
- Date
- Thread