[dak/master] Loadsa changes to make new cronscript way work
- Move functions so they are seen,
- Emit logs not only when called by dinstall
- Use correct variables (TMPFILES, not TEMPFILES)
- Define used variables
- create a scriptname link to the logfiles with datebased names
- in the logdir, not in the configdir
- move linkmorgue from daily to weekly
- remove nonexistant tasks
- make purgeempty and fixsymlinks work on all archives, not just main
- all weekly tasks run in parallel
---
config/debian/common | 26 +++++++++++---------------
config/debian/cronscript | 11 +++++++----
config/debian/daily.functions | 4 ++--
config/debian/daily.tasks | 3 ---
config/debian/dinstall.functions | 10 +++-------
config/debian/dinstall.tasks | 1 -
config/debian/dinstall.variables | 3 +++
config/debian/hourly.functions | 4 ++--
config/debian/hourly.tasks | 2 +-
config/debian/vars | 1 +
config/debian/weekly.functions | 27 ++++++++++++++++++++-------
config/debian/weekly.tasks | 7 ++++---
12 files changed, 54 insertions(+), 45 deletions(-)
diff --git a/config/debian/common b/config/debian/common
index 30237da..5a7ec0b 100644
--- a/config/debian/common
+++ b/config/debian/common
@@ -59,11 +59,15 @@ function cleanup() {
cleantempfiles
- exit $ERRVAL
+ return $ERRVAL
}
TEMPFILES=${TEMPFILES:-""}
TMPFILES=${TMPFILES:-""}
+# Timestamp. Used for dinstall stat graphs
+function ts() {
+ echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
+}
########################################################################
########################################################################
@@ -114,17 +118,13 @@ function punew_do() {
# These versions used in dinstall
function punew() {
- if [ "${PROGRAM}" = "dinstall" ]; then
- log "Doing automated p-u-new processing"
- fi
+ log "Doing automated p-u-new processing"
cd "${queuedir}/p-u-new"
punew_do "$1" "${queuedir}/p-u-new" "debian-release@lists.debian.org"
}
function opunew() {
- if [ "${PROGRAM}" = "dinstall" ]; then
- log "Doing automated o-p-u-new processing"
- fi
+ log "Doing automated o-p-u-new processing"
cd "${queuedir}/o-p-u-new"
punew_do "$1" "${queuedir}/o-p-u-new" "debian-release@lists.debian.org"
}
@@ -134,9 +134,7 @@ function backports_policy() {
local qdir="/srv/backports-master.debian.org/queue/policy"
local to="backports-team@debian.org"
- if [ "${PROGRAM}" = "dinstall" ]; then
- log "Doing automated ${queue} processing"
- fi
+ log "Doing automated ${queue} processing"
cd "${qdir}"
punew_do "${queue}" "${qdir}" "${to}"
@@ -163,9 +161,7 @@ function do_unchecked () {
# process NEW policy queue
function do_new () {
- if [ "${PROGRAM}" = "dinstall" ]; then
- log "Doing NEW processing"
- fi
+ log "Doing NEW processing"
(dak process-policy new; dak process-policy byhand) | mail -a "X-Debian: DAK" -e -s "NEW and BYHAND processing" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" ftpmaster@ftp-master.debian.org
dak process-policy backports-new | mail -a "X-Debian: DAK" -e -s "NEW processing for backports-new" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" backports-team@debian.org
@@ -227,8 +223,8 @@ function get_archiveroot() {
# Prepare the trees for buildds, then push wanna-build
function do_buildd() {
- if lockfile -r3 $NOTICE; then
- TEMPFILES="${TEMPFILES} ${NOTICE}"
+ if lockfile -r3 ${LOCK_DAILY}; then
+ TMPFILES="${TMPFILES} ${LOCK_DAILY}"
make_buildd_dir
wbtrigger
fi
diff --git a/config/debian/cronscript b/config/debian/cronscript
index ffaf977..fec8f93 100755
--- a/config/debian/cronscript
+++ b/config/debian/cronscript
@@ -40,6 +40,10 @@ export LANG=C
export LC_ALL=C
ARG=${1:-"meh"}
+
+# set DEBUG if you want to see a little more logs (needs to be used more)
+DEBUG=${DEBUG:-0}
+
# While this check can be done in the following case, some assumptions
# down there are easier if we sorted out calls without an arg before.
if [[ ${ARG} == meh ]]; then
@@ -101,15 +105,14 @@ case ${ARG,,} in
# log "aborting cron.unchecked because $LOCK_UNCHECKED has already been locked"
exit 0
fi
- TEMPFILES="${TEMPFILES} ${LOCK_UNCHECKED}"
+ TMPFILES="${TMPFILES} ${LOCK_UNCHECKED}"
;;
dinstall)
;;
hourly)
# Only one of me should ever run.
FLOCKER=${FLOCKER:-""}
- [ "${FLOCKER}" != "${configdir}/${PROGRAM}.variables" ] && exec env FLOCKER="${configdir}/${PROGRAM}.variables" flock -E 0 -en "${configdir}/${PROGRAM}.variables" "$0"
- "$@" || :
+ [ "${FLOCKER}" != "${configdir}/${PROGRAM}.variables" ] && exec env FLOCKER="${configdir}/${PROGRAM}.variables" flock -E 0 -en "${configdir}/${PROGRAM}.variables" "$0" "$@" || :
;;
daily)
;;
@@ -124,7 +127,7 @@ case ${ARG,,} in
esac
# An easy access by name for the current log
-ln -sf ${LOGFILE} ${PROGRAM}
+ln -sf ${LOGFILE} ${logdir}/${PROGRAM}
# And from here, all output to the log please
exec >> "$LOGFILE" 2>&1
diff --git a/config/debian/daily.functions b/config/debian/daily.functions
index 2d52836..e2b4b07 100644
--- a/config/debian/daily.functions
+++ b/config/debian/daily.functions
@@ -3,7 +3,7 @@
# get the latest list of wnpp bugs and their source packages
function wnppbugs() {
TMPFILE=$( mktemp -p ${TMPDIR} )
- TEMPFILES="${TEMPFILES} ${TMPFILE}"
+ TMPFILES="${TMPFILES} ${TMPFILE}"
wget -q -O${TMPFILE} --ca-directory=/etc/ssl/ca-debian https://qa.debian.org/data/bts/wnpp_rm
chmod go+r ${TMPFILE}
mv ${TMPFILE} /srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm
@@ -39,7 +39,7 @@ function newstats() {
function contributor() {
log "Submitting data to contributors"
TMPCNTB=$( mktemp -p ${TMPDIR} )
- TEMPFILES="${TEMPFILES} ${TMPCNTB}"
+ TMPFILES="${TMPFILES} ${TMPCNTB}"
REQUESTS_CA_BUNDLE=/etc/ssl/ca-debian/ca-certificates.crt dc-tool --mine="${configdir}/contributor.source" --auth-token @"${base}/s3kr1t/contributor.auth" --source ftp.debian.org --json > ${TMPCNTB}
# Post with curl as a workaround for #801506
diff --git a/config/debian/daily.tasks b/config/debian/daily.tasks
index 7bec97e..51c4e5d 100644
--- a/config/debian/daily.tasks
+++ b/config/debian/daily.tasks
@@ -1,5 +1,4 @@
# FUNC ARGS TIME ERR BG
-linkmorgue none none false true
wnppbugs none none false false
pushfilesdb none none false false
wbdump none none false false
@@ -8,5 +7,3 @@ clean_debbugs none none fal
overridedisp none none false false
newstats none none false false
contributor none none false false
-finalize none none false false
-
diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions
index fcd2de3..f351ad4 100644
--- a/config/debian/dinstall.functions
+++ b/config/debian/dinstall.functions
@@ -1,8 +1,4 @@
# -*- mode:sh -*-
-# Timestamp. Used for dinstall stat graphs
-function ts() {
- echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
-}
# Remove daily lock
# FIXME: Dead code when switched to cronscript
@@ -193,13 +189,13 @@ function mpfm() {
log "Generating package / file mapping"
for archive in "${public_archives[@]}"; do
+ log " archive: ${archive}"
archiveroot="$(get_archiveroot "${archive}")"
dak make-pkg-file-mapping "${archive}" | bzip2 -9 > "${archiveroot}/indices/package-file.map.bz2"
done
}
function packages() {
- log "Generating Packages and Sources files"
for archive in "${public_archives[@]}"; do
log " Generating Packages/Sources for ${archive}"
dak generate-packages-sources2 -a "${archive}"
@@ -216,6 +212,7 @@ function pdiff() {
function release() {
log "Generating Release files"
for archive in "${public_archives[@]}"; do
+ log " archive: ${archive}"
dak generate-releases -a "${archive}"
done
}
@@ -680,9 +677,8 @@ function signotherfiles() {
function startup() {
touch "${DINSTALLSTART}"
ts "startup"
- DINSTALLBEGIN="$(date -u +"%a %b %d %T %Z %Y (%s)")"
lockfile -l 3600 "${LOCK_DAILY}"
trap onerror ERR
- TEMPFILES="${TEMPFILES} ${LOCK_DAILY}"
touch "${LOCK_BRITNEY}"
+ TMPFILES="${TMPFILES} ${LOCK_DAILY} ${LOCK_BRITNEY}"
}
diff --git a/config/debian/dinstall.tasks b/config/debian/dinstall.tasks
index 2185287..79ae575 100644
--- a/config/debian/dinstall.tasks
+++ b/config/debian/dinstall.tasks
@@ -46,5 +46,4 @@ mirrorpush-backports none mirrorpush-backports fal
i18n2 none i18n\ 2 false true
stats none stats false true
testingsourcelist none none false true
-NOSTAGE rm\ -f\ "\${LOCK_BRITNEY}" none false false
cleantransactions none none false false
diff --git a/config/debian/dinstall.variables b/config/debian/dinstall.variables
index 6f5c953..c4efabe 100644
--- a/config/debian/dinstall.variables
+++ b/config/debian/dinstall.variables
@@ -19,6 +19,9 @@ DINSTALLSTART="${lockdir}/dinstallstart"
# Marker for dinstall end
DINSTALLEND="${lockdir}/dinstallend"
+# Timestamp for start
+DINSTALLBEGIN="$(date -u +"%a %b %d %T %Z %Y (%s)")"
+
# Lock cron.unchecked from doing work
LOCK_ACCEPTED="$lockdir/unchecked.lock"
diff --git a/config/debian/hourly.functions b/config/debian/hourly.functions
index 5a43310..3e809fa 100644
--- a/config/debian/hourly.functions
+++ b/config/debian/hourly.functions
@@ -7,7 +7,7 @@ function importusers() {
function newoverview() {
# do not run show-new and other stuff in parallel
if lockfile -r16 $LOCK_UNCHECKED 2> /dev/null; then
- TEMPFILES="${TEMPFILES} ${LOCK_UNCHECKED}"
+ TMPFILES="${TMPFILES} ${LOCK_UNCHECKED}"
do_new
dak show-new > /dev/null || true
fi
@@ -49,7 +49,7 @@ function backportsacl() {
# do the buildd key updates
function builddautosigning() {
BUILDDFUN=$(mktemp -p "${TMPDIR}" BUILDDFUN.XXXXXX)
- TEMPFILES="${TEMPFILEs} ${BUILDDFUN}"
+ TMPFILES="${TMPFILES} ${BUILDDFUN}"
exec >> "${BUILDDFUN}" 2>&1
${scriptsdir}/buildd-remove-keys
${scriptsdir}/buildd-add-keys
diff --git a/config/debian/hourly.tasks b/config/debian/hourly.tasks
index b5417db..93053a4 100644
--- a/config/debian/hourly.tasks
+++ b/config/debian/hourly.tasks
@@ -7,4 +7,4 @@ rss none none fal
ddaccess none none true true
gen_di none none false false
backportsacl none none false false
-builddautosigning none none false false
\ No newline at end of file
+builddautosigning none none false false
diff --git a/config/debian/vars b/config/debian/vars
index c94ece0..83cbcf1 100644
--- a/config/debian/vars
+++ b/config/debian/vars
@@ -43,6 +43,7 @@ GZIP='--rsyncable' ; export GZIP
# lock cron.unchecked (it immediately exits when this exists)
LOCK_DAILY="$lockdir/daily.lock"
LOCK_UNCHECKED="$lockdir/unchecked.lock"
+LOCK_STOP="$lockdir/archive.stop"
# Set the database variables
eval $(dak admin config db-shell)
diff --git a/config/debian/weekly.functions b/config/debian/weekly.functions
index 55a7f23..893813d 100644
--- a/config/debian/weekly.functions
+++ b/config/debian/weekly.functions
@@ -1,11 +1,18 @@
# -*- mode:sh -*-
function purgeempty() {
- log "Purging empty directories in $ftpdir/pool/"
+ log "Purging empty directories"
+ local archiveroot
- if [ ! -z "$(find $ftpdir/pool/ -type d -empty)" ]; then
- find $ftpdir/pool/ -type d -empty | xargs rmdir;
- fi
+ for archive in "${public_archives[@]}"; do
+ log "... archive: ${archive}"
+ archiveroot="$(get_archiveroot "${archive}")"
+
+ # FIXME: double find
+ if [ ! -z "$(find ${archiveroot}/pool/ -type d -empty)" ]; then
+ find ${archiveroot}/pool/ -type d -empty | xargs rmdir;
+ fi
+ done
}
function gitcleanup() {
@@ -18,7 +25,13 @@ function gitcleanup() {
chmod -R g+w logs/
}
-function symlinks() {
- log "Fixing symlinks in $ftpdir"
- symlinks -d -r $ftpdir
+function fixsymlinks() {
+ log "Fixing symlinks"
+ local archiveroot
+
+ for archive in "${public_archives[@]}"; do
+ log "... archive: ${archive}"
+ archiveroot="$(get_archiveroot "${archive}")"
+ symlinks -d -r ${archiveroot}
+ done
}
diff --git a/config/debian/weekly.tasks b/config/debian/weekly.tasks
index 4fb0d92..874a1bc 100644
--- a/config/debian/weekly.tasks
+++ b/config/debian/weekly.tasks
@@ -1,4 +1,5 @@
# FUNC ARGS TIME ERR BG
-purgeempty none none false false
-gitcleanup none none false false
-symlinks none none false false
+purgeempty none none false true
+gitcleanup none none false true
+fixsymlinks none none false true
+linkmorgue none none false true
--
2.1.4
Reply to: