[dak/master] Adjust cronscripts to end up with just one
---
config/debian/common | 178 +++++++++++++++++++++++++++++
config/debian/cronscript | 235 ++++++++++++++++++++++++++++++++++++++
config/debian/daily.functions | 59 ++++++++++
config/debian/daily.tasks | 12 ++
config/debian/dinstall.functions | 21 ++--
config/debian/dinstall.tasks | 50 ++++++++
config/debian/dinstall.variables | 6 -
config/debian/hourly.functions | 67 +++++++++++
config/debian/hourly.tasks | 10 ++
config/debian/monthly.functions | 19 +++
config/debian/monthly.tasks | 3 +
config/debian/monthly.variables | 3 +
config/debian/unchecked.tasks | 9 ++
config/debian/unchecked.variables | 3 +
config/debian/vars | 4 +
config/debian/weekly.functions | 24 ++++
config/debian/weekly.tasks | 4 +
17 files changed, 693 insertions(+), 14 deletions(-)
create mode 100755 config/debian/cronscript
create mode 100644 config/debian/daily.functions
create mode 100644 config/debian/daily.tasks
create mode 100644 config/debian/dinstall.tasks
create mode 100644 config/debian/hourly.functions
create mode 100644 config/debian/hourly.tasks
create mode 100644 config/debian/monthly.functions
create mode 100644 config/debian/monthly.tasks
create mode 100644 config/debian/monthly.variables
create mode 100644 config/debian/unchecked.tasks
create mode 100644 config/debian/unchecked.variables
create mode 100644 config/debian/weekly.functions
create mode 100644 config/debian/weekly.tasks
diff --git a/config/debian/common b/config/debian/common
index bb6f8e4..30237da 100644
--- a/config/debian/common
+++ b/config/debian/common
@@ -23,6 +23,51 @@ function debug () {
fi
}
+# Function that only cleans tempfiles, but does not exit or otherwise
+# care about any exit status
+function cleantempfiles() {
+ resolvetmpfiles
+ for TEMPFILE in $TMPFILES; do
+ if [ -n "${TEMPFILE}" ] && [ -f "${TEMPFILE}" ]; then
+ rm -f "${TEMPFILE}"
+ elif [ -n "${TEMPFILE}" ] && [ -d "${TEMPFILE}" ]; then
+ if [ "${TEMPFILE}" != "/" ] && [ "${TEMPFILE}" != "/*" ]; then
+ rm -rf "${TEMPFILE}"
+ fi
+ fi
+ done
+ TMPFILES=""
+}
+
+function resolvetmpfiles() {
+ # If you don't understand this better not touch the script
+ for TEMPFILE in $TEMPFILES; do
+ TMPFILES="${TMPFILES} ${!TEMPFILE:-""}"
+ done
+ TEMPFILES=""
+}
+
+# Function cleanup
+# No arguments
+# Cleans up any known tempfile.
+# Just ensure your script sets the variable
+# TEMPFILES to the names of variables of tempfiles
+# Or TMPFILES to the pathes of tempfiles
+function cleanup() {
+ ERRVAL=$?
+ trap - ERR EXIT TERM HUP INT QUIT
+
+ cleantempfiles
+
+ exit $ERRVAL
+}
+TEMPFILES=${TEMPFILES:-""}
+TMPFILES=${TMPFILES:-""}
+
+
+########################################################################
+########################################################################
+
function wbtrigger() {
SSHOPT="-n -o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=240"
if lockfile -r 3 -l 3600 "${LOCK_BUILDD}"; then
@@ -109,6 +154,11 @@ function do_unchecked () {
echo "$timestamp": ${changes:-"Nothing to do"} >> $report
dak process-upload -a ${UNCHECKED_WITHOUT_LOCK} -d "$unchecked" >> $report
dak process-commands -d "$unchecked" >> $report
+
+ if [ ! -z "$changes" ]; then
+ sync_debbugs
+ do_buildd
+ fi
}
# process NEW policy queue
@@ -174,3 +224,131 @@ function get_archiveroot() {
fi
echo "${archiveroot}"
}
+
+# Prepare the trees for buildds, then push wanna-build
+function do_buildd() {
+ if lockfile -r3 $NOTICE; then
+ TEMPFILES="${TEMPFILES} ${NOTICE}"
+ make_buildd_dir
+ wbtrigger
+ fi
+}
+
+# Cleanup policy queues
+function cleanpolicy() {
+ dak clean-suites -a backports-policy,policy
+}
+
+# Scan new packages for contents
+function scancontents() {
+ dak contents -l 10000 scan-binary
+ dak contents -l 1000 scan-source
+}
+
+function ddaccess() {
+ # Tell our dd accessible mirror to sync itself up. Including ftp dir.
+ log "Trigger dd accessible parts sync including ftp dir"
+ ${scriptsdir}/sync-dd dd-sync dd-sync1 dd-sync2 sync
+}
+
+
+
+########################################################################
+########################################################################
+########################################################################
+########################################################################
+
+# Function to save which stage we are in, so we can restart an interrupted
+# dinstall. Or even run actions in parallel, if we dare to, by simply
+# backgrounding the call to this function. But that should only really be
+# done for things we don't care much about.
+#
+# This should be called with the first argument being an array, with the
+# members
+# - FUNC - the function name to call
+# - ARGS - Possible arguments to hand to the function. Can be the empty string
+# - TIME - The timestamp name. Can be the empty string
+# - ERR - if this is the string false, then the call will be surrounded by
+# set +e ... set -e calls, so errors in the function do not exit
+# dinstall. Can be the empty string, meaning true.
+#
+# MAKE SURE TO KEEP THIS THE LAST FUNCTION, AFTER ALL THE VARIOUS ONES
+# ADDED FOR DINSTALL FEATURES!
+function stage() {
+ ARGS='GO[@]'
+ local "${!ARGS}"
+
+ local error=${ERR:-"true"}
+
+ ARGS=${ARGS:-""}
+
+ log "########## ${PROGRAM} BEGIN: ${FUNC} ${ARGS} ##########"
+ local STAGEFILE="${stagedir}/${FUNC}_${ARGS}"
+ STAGEFILE=${STAGEFILE// /_}
+ if [ -f "${STAGEFILE}" ]; then
+ local stamptime=$(/usr/bin/stat -c %Z "${STAGEFILE}")
+ local unixtime=$(date +%s)
+ local difference=$(( $unixtime - $stamptime ))
+ if [ ${difference} -ge 14400 ]; then
+ log_error "Did already run ${FUNC}, stagefile exists, but that was ${difference} seconds ago. Please check."
+ else
+ log "Did already run ${FUNC}, not calling again..."
+ fi
+ return
+ fi
+
+ debug "Now calling function ${FUNC}. Arguments: ${ARGS}. Timestamp: ${TIME}"
+
+ # Make sure we are always at the same place. If a function wants
+ # to be elsewhere, it has to cd first!
+ cd ${configdir}
+
+ # Now redirect the output into $STAGEFILE.log. In case it errors
+ # out somewhere our errorhandler trap can then mail the contents
+ # of $STAGEFILE.log only, instead of a whole ${PROGRAM} logfile.
+ # Short error mails ftw!
+ exec >> "${STAGEFILE}.log" 2>&1
+
+ if [ -f "${LOCK_STOP}" ]; then
+ log "${LOCK_STOP} exists, exiting immediately"
+ exit 42
+ fi
+
+ # Do we care about trouble in the function we call?
+ if [ "${error}" = "false" ]; then
+ set +e
+ fi
+ ${FUNC} ${ARGS}
+
+ # No matter what happened in the function, we make sure we have
+ # set -e default state back
+ set -e
+
+ # Make sure we are always at the same place.
+ cd ${configdir}
+
+ # We always use the same umask. If a function wants to do
+ # different, fine, but we reset.
+ umask 022
+
+ touch "${STAGEFILE}"
+
+ if [ -n "${TIME}" ]; then
+ ts "${TIME}"
+ fi
+
+ # And the output goes back to the normal logfile
+ exec >> "${LOGFILE}" 2>&1
+
+ # Now we should make sure that we have a usable ${PROGRAM}.log, so
+ # append the $STAGEFILE.log to it.
+ cat "${STAGEFILE}.log" >> "${LOGFILE}"
+ rm -f "${STAGEFILE}.log"
+
+ log "########## ${PROGRAM} END: ${FUNC} ##########"
+
+ if [ -f "${LOCK_STOP}" ]; then
+ log "${LOCK_STOP} exists, exiting immediately"
+ exit 42
+ fi
+}
diff --git a/config/debian/cronscript b/config/debian/cronscript
new file mode 100755
index 0000000..ffaf977
--- /dev/null
+++ b/config/debian/cronscript
@@ -0,0 +1,235 @@
+#!/bin/bash
+# No way I try to deal with a crippled sh just for POSIX foo.
+
+# Copyright (C) 2009-2015 Joerg Jaspert <joerg@debian.org>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; version 2.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+# Homer: Are you saying you're never going to eat any animal again? What
+# about bacon?
+# Lisa: No.
+# Homer: Ham?
+# Lisa: No.
+# Homer: Pork chops?
+# Lisa: Dad, those all come from the same animal.
+# Homer: Heh heh heh. Ooh, yeah, right, Lisa. A wonderful, magical animal.
+
+# exit on errors
+set -e
+set -o pipefail
+# make sure to only use defined variables
+set -u
+# ERR traps should be inherited from functions too. (And command
+# substitutions and subshells and whatnot, but for us the functions is
+# the important part here)
+set -E
+
+# And use one locale, no matter what the caller has set
+export LANG=C
+export LC_ALL=C
+
+ARG=${1:-"meh"}
+# While this check can be done in the following case, some assumptions
+# down there are easier if we sorted out calls without an arg before.
+if [[ ${ARG} == meh ]]; then
+ cat - <<EOF
+This is the FTPMaster cronscript. It needs an argument or it won't do
+anything for you.
+
+Currently accepted Arguments:
+
+ unchecked - Process the unchecked queue
+ dinstall - Run a dinstall
+ hourly, daily, weekly - Run that part
+
+EOF
+ exit 0
+fi
+
+# Make sure we start out with a sane umask setting
+umask 022
+
+# import the general variable set.
+export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
+. $SCRIPTVARS
+
+# common functions are "outsourced"
+. "${configdir}/common"
+
+# program name is the (lower cased) first argument.
+PROGRAM="${ARG,,}"
+
+# Timestamp when we started
+NOW=$(date "+%Y.%m.%d-%H:%M:%S")
+
+# Which list of tasks should we run?
+TASKLIST="${configdir}/${PROGRAM}.tasks"
+
+# A logfile for every cron script
+LOGFILE="${logdir}/${PROGRAM}_${NOW}.log"
+
+# Each "cronscript" may have a variables and a functions file
+# that we source
+for what in variables functions; do
+ if [[ -f ${configdir}/${PROGRAM}.${what} ]]; then
+ . ${configdir}/${PROGRAM}.${what}
+ fi
+done
+
+# Get rid of tempfiles at the end
+trap cleanup EXIT TERM HUP INT QUIT
+
+case ${ARG,,} in
+ unchecked)
+ # Do not run during dinstall
+ if [[ -e ${LOCK_DAILY} ]]; then
+ exit 0;
+ fi
+ # only run one cron.unchecked and also lock against hourly (newoverview)
+ if ! lockfile -r8 ${LOCK_UNCHECKED} 2> /dev/null; then
+ # log "aborting cron.unchecked because $LOCK_UNCHECKED has already been locked"
+ exit 0
+ fi
+ TEMPFILES="${TEMPFILES} ${LOCK_UNCHECKED}"
+ ;;
+ dinstall)
+ ;;
+ hourly)
+ # Only one of me should ever run.
+ FLOCKER=${FLOCKER:-""}
+ [ "${FLOCKER}" != "${configdir}/${PROGRAM}.variables" ] && exec env FLOCKER="${configdir}/${PROGRAM}.variables" flock -E 0 -en "${configdir}/${PROGRAM}.variables" "$0"
+ "$@" || :
+ ;;
+ daily)
+ ;;
+ weekly)
+ ;;
+ monthly)
+ ;;
+ *)
+ error "Unknown arg ${ARG}"
+ exit 42
+ ;;
+esac
+
+# An easy access by name for the current log
+ln -sf ${LOGFILE} ${PROGRAM}
+
+# And from here, all output to the log please
+exec >> "$LOGFILE" 2>&1
+
+# The stage function uses this directory
+# This amends the stagedir variable from "vars"
+stagedir="${stagedir}/${PROGRAM}"
+# Ensure the dir exists
+mkdir -p ${stagedir}
+
+# This loop simply wants to be fed by a list of values (see below)
+# made out of 5 columns.
+# The first four are the array values for the stage function, the
+# fifth tells us if we should background the stage call.
+#
+# - FUNC - the function name to call
+# - ARGS - Possible arguments to hand to the function. Can be the empty string
+# - TIME - The timestamp name. Can be the empty string
+# - ERR - if this is the string false, then the call will be surrounded by
+# set +e ... set -e calls, so errors in the function do not exit
+# dinstall. Can be the empty string, meaning true.
+# - BG - Background the function stage?
+#
+# ATTENTION: Spaces in arguments or timestamp names need to be escaped by \
+#
+# NOTE 1: There are two special values for the first column (FUNC).
+# STATE - do not call stage function, call the state
+# function to update the public statefile "where is dinstall"
+# NOSTAGE - do not call stage function, call the command directly.
+#
+# Note 2: If you want to hand an empty value to the stage function,
+# use the word "none" in the list below.
+while read FUNC ARGS TIME ERR BACKGROUND; do
+ debug "FUNC: $FUNC ARGS: $ARGS TIME: $TIME ERR: $ERR BG: $BACKGROUND"
+
+ # Empty values in the value list are the string "none" (or the
+ # while read loop won't work). Here we ensure that variables that
+ # can be empty, are empty if the string none is set for them.
+ for var in ARGS TIME; do
+ if [[ ${!var} == none ]]; then
+ typeset ${var}=''
+ fi
+ done
+
+ # ERR/BACKGROUND are boolean, check that they are.
+ for var in ERR BACKGROUND; do
+ if [[ ${!var} != false ]] && [[ ${!var} != true ]]; then
+ error "Illegal value ${!var} for ${var} (should be true or false), line for function ${FUNC}"
+ fi
+ done
+
+ case ${FUNC} in
+ STATE)
+ state ${ARGS}
+ ;;
+ NOSTAGE)
+ ${ARGS}
+ ;;
+ *)
+ GO=(
+ FUNC=${FUNC}
+ TIME=${TIME}
+ ARGS=${ARGS}
+ ERR=${ERR}
+ )
+ if [[ ${BACKGROUND} == true ]]; then
+ stage $GO &
+ else
+ stage $GO
+ fi
+ ;;
+ esac
+done < <(grep -v '^#' ${TASKLIST} )
+
+# we need to wait for the background processes before the end of the cron script
+wait
+
+
+# Common to all cron scripts
+log "Cron script successful, all done"
+# Redirect output to another file, as we want to compress our logfile
+# and ensure its no longer used
+exec > "$logdir/after${PROGRAM}.log" 2>&1
+
+case ${ARG,,} in
+ unchecked)
+ ;;
+ dinstall)
+ logstats ${LOGFILE}
+ state "all done"
+ touch "${DINSTALLEND}"
+ ;;
+ hourly)
+ ;;
+ daily)
+ ;;
+ weekly)
+ ;;
+ monthly)
+ ;;
+esac
+
+# Now, at the very (successful) end of this run, make sure we remove
+# our stage files, so the next dinstall run will do it all again.
+rm -f ${stagedir}/*
+bzip2 -9 ${LOGFILE}
+# FIXME: Mail the log when its non-empty
+[[ -s "${logdir}/after${PROGRAM}.log" ]] || rm "${logdir}/after${PROGRAM}.log"
diff --git a/config/debian/daily.functions b/config/debian/daily.functions
new file mode 100644
index 0000000..2d52836
--- /dev/null
+++ b/config/debian/daily.functions
@@ -0,0 +1,59 @@
+# -*- mode:sh -*-
+
+# get the latest list of wnpp bugs and their source packages
+function wnppbugs() {
+ TMPFILE=$( mktemp -p ${TMPDIR} )
+ TEMPFILES="${TEMPFILES} ${TMPFILE}"
+ wget -q -O${TMPFILE} --ca-directory=/etc/ssl/ca-debian https://qa.debian.org/data/bts/wnpp_rm
+ chmod go+r ${TMPFILE}
+ mv ${TMPFILE} /srv/ftp-master.debian.org/scripts/masterfiles/wnpp_rm
+}
+
+# Push files over to security
+function pushfilesdb() {
+ # The key over there should have the following set for the ssh key:
+ # command="/usr/bin/xzcat | /usr/bin/psql -1 -c 'DELETE FROM external_files; COPY external_files (id, filename, size, md5sum, last_used, sha1sum, sha256sum, created, modified) FROM STDIN' obscurity"
+ psql -c 'COPY files (id, filename, size, md5sum, last_used, sha1sum, sha256sum, created, modified) TO STDOUT' projectb | \
+ xz -3 | \
+ ssh -o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 \
+ -i ${base}/s3kr1t/push_external_files dak@security-master.debian.org sync
+}
+
+# Update wanna-build dump
+function wbdump() {
+ log "Update wanna-build database dump"
+ $base/dak/scripts/nfu/get-w-b-db
+}
+
+# Generate list of override disparities
+function overridedisp() {
+ dak override-disparity | gzip -9 > ${webdir}/override-disparity.gz
+}
+
+# Generate stats about the new queue
+function newstats() {
+ dak stats new ${webdir}/NEW-stats.yaml 2> /dev/null
+}
+
+# Generate the contributor data
+function contributor() {
+ log "Submitting data to contributors"
+ TMPCNTB=$( mktemp -p ${TMPDIR} )
+ TEMPFILES="${TEMPFILES} ${TMPCNTB}"
+ REQUESTS_CA_BUNDLE=/etc/ssl/ca-debian/ca-certificates.crt dc-tool --mine="${configdir}/contributor.source" --auth-token @"${base}/s3kr1t/contributor.auth" --source ftp.debian.org --json > ${TMPCNTB}
+
+ # Post with curl as a workaround for #801506
+ # See https://wiki.debian.org/ServicesSSL#curl
+ dir=/etc/ssl/ca-debian
+ test -d $dir && capath="--capath $dir"
+ curl -s $capath https://contributors.debian.org/contributors/post \
+ -F source=ftp.debian.org \
+ -F auth_token="$(cat ${base}/s3kr1t/contributor.auth)" \
+ -F data=@${TMPCNTB} > ${TMPCNTB}.result
+ cat ${TMPCNTB}.result
+ rm -f ${TMPCNTB}.result
+}
+
+function linkmorgue() {
+ ${scriptsdir}/link_morgue.sh
+}
diff --git a/config/debian/daily.tasks b/config/debian/daily.tasks
new file mode 100644
index 0000000..7bec97e
--- /dev/null
+++ b/config/debian/daily.tasks
@@ -0,0 +1,12 @@
+# FUNC ARGS TIME ERR BG
+linkmorgue none none false true
+wnppbugs none none false false
+pushfilesdb none none false false
+wbdump none none false false
+reports none none false false
+clean_debbugs none none false false
+overridedisp none none false false
+newstats none none false false
+contributor none none false false
+finalize none none false false
+
diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions
index 498999e..fcd2de3 100644
--- a/config/debian/dinstall.functions
+++ b/config/debian/dinstall.functions
@@ -5,6 +5,7 @@ function ts() {
}
# Remove daily lock
+# FIXME: Dead code when switched to cronscript
function remove_daily_lock() {
rm -f $LOCK_DAILY
}
@@ -50,7 +51,7 @@ function onerror() {
}
########################################################################
-# the actual dinstall functions follow #
+# the actual functions follow #
########################################################################
# pushing merkels QA user, part one
@@ -482,12 +483,6 @@ function bts() {
sudo -u dak-unpriv dak bts-categorize
}
-function ddaccess() {
- # Tell our dd accessible mirror to sync itself up. Including ftp dir.
- log "Trigger dd accessible parts sync including ftp dir"
- ${scriptsdir}/sync-dd dd-sync dd-sync1 dd-sync2 sync
-}
-
function mirrorpush() {
log "Checking the public archive copies..."
@@ -615,7 +610,6 @@ function logstats() {
# save timestamp when we start
function savetimestamp() {
- NOW=`date "+%Y.%m.%d-%H:%M:%S"`
echo ${NOW} > "${dbdir}/dinstallstart"
}
@@ -663,6 +657,7 @@ function changelogs() {
fi
}
+# Generate a list of extra mirror files, sha256sum em and sign that
function signotherfiles() {
log "Signing extra mirror files"
@@ -681,3 +676,13 @@ function signotherfiles() {
rm -f ${TMPLO}
done
}
+
+function startup() {
+ touch "${DINSTALLSTART}"
+ ts "startup"
+ DINSTALLBEGIN="$(date -u +"%a %b %d %T %Z %Y (%s)")"
+ lockfile -l 3600 "${LOCK_DAILY}"
+ trap onerror ERR
+ TEMPFILES="${TEMPFILES} ${LOCK_DAILY}"
+ touch "${LOCK_BRITNEY}"
+}
diff --git a/config/debian/dinstall.tasks b/config/debian/dinstall.tasks
new file mode 100644
index 0000000..2185287
--- /dev/null
+++ b/config/debian/dinstall.tasks
@@ -0,0 +1,50 @@
+# FUNC ARGS TIME ERR BG
+STATE startup none false false
+NOSTAGE startup none false false
+savetimestamp none none false false
+qa1 none init true true
+pg_timestamp predinstall pg_dump1 false false
+updates none External\ Updates false false
+i18n1 none i18n\ 1 false false
+dep11 none dep11\ 1 false false
+NOSTAGE lockaccepted none false false
+punew stable-new p-u-new false false
+opunew oldstable-new o-p-u-new false false
+backports_policy none backports-policy false false
+cruft none cruft false false
+STATE indices none false false
+dominate none dominate false false
+autocruft none autocruft false false
+fingerprints none import-keyring false false
+overrides none overrides false false
+mpfm none pkg-file-mapping false false
+STATE packages/contents none false false
+packages none apt-ftparchive false false
+STATE dists/ none false false
+pdiff none pdiff false false
+release none release\ files false false
+dakcleanup none cleanup false false
+STATE scripts none false false
+mkmaintainers none mkmaintainers false false
+copyoverrides none copyoverrides false false
+mklslar none mklslar false false
+mkfilesindices none mkfilesindices false false
+mkchecksums none mkchecksums false false
+signotherfiles none signotherfiles false false
+mirror none mirror\ hardlinks false false
+NOSTAGE remove_locks none false false
+STATE postlock none false false
+ddaccess none ddaccessible\ sync true true
+changelogs none changelogs false true
+pg_timestamp postdinstall pg_dump2 false false
+expire none expire_dumps false true
+transitionsclean none transitionsclean false true
+dm none none false true
+bts none none false true
+mirrorpush none mirrorpush false true
+mirrorpush-backports none mirrorpush-backports false true
+i18n2 none i18n\ 2 false true
+stats none stats false true
+testingsourcelist none none false true
+NOSTAGE rm\ -f\ "\${LOCK_BRITNEY}" none false false
+cleantransactions none none false false
diff --git a/config/debian/dinstall.variables b/config/debian/dinstall.variables
index 0c1db16..6f5c953 100644
--- a/config/debian/dinstall.variables
+++ b/config/debian/dinstall.variables
@@ -2,9 +2,6 @@
# usually we are not using debug logs. Set to 1 if you want them.
DEBUG=0
-# our name
-PROGRAM="dinstall"
-
# where do we want mails to go? For example log entries made with error()
if [ "x$(hostname -s)x" != "xfranckx" ]; then
# Not our ftpmaster host
@@ -22,9 +19,6 @@ DINSTALLSTART="${lockdir}/dinstallstart"
# Marker for dinstall end
DINSTALLEND="${lockdir}/dinstallend"
-# lock cron.unchecked (it immediately exits when this exists)
-LOCK_DAILY="$lockdir/daily.lock"
-
# Lock cron.unchecked from doing work
LOCK_ACCEPTED="$lockdir/unchecked.lock"
diff --git a/config/debian/hourly.functions b/config/debian/hourly.functions
new file mode 100644
index 0000000..5a43310
--- /dev/null
+++ b/config/debian/hourly.functions
@@ -0,0 +1,67 @@
+# -*- mode:sh -*-
+
+function importusers() {
+ dak import-users-from-passwd
+}
+
+function newoverview() {
+ # do not run show-new and other stuff in parallel
+ if lockfile -r16 $LOCK_UNCHECKED 2> /dev/null; then
+ TEMPFILES="${TEMPFILES} ${LOCK_UNCHECKED}"
+ do_new
+ dak show-new > /dev/null || true
+ fi
+ cleantempfiles
+}
+
+function queuereport() {
+ dak queue-report -n > ${webdir}/new.html
+ dak queue-report -n -d backports-new,backports-policy > ${webdir}/backports-new.html
+ dak queue-report -8 -d new,byhand,stable-new,oldstable-new,backports-new -r $webdir/stat
+ sudo -u dak-unpriv dak show-deferred -r ${webdir}/stat > ${webdir}/deferred.html
+ dak graph -n new,byhand,stable-new,oldstable-new,deferred -r ${webdir}/stat -i ${webdir}/stat -x $scriptsdir/rrd-release-freeze-dates
+}
+
+function removalstxt() {
+ cd ${webdir}
+ cat removals-20*.txt > removals-full.txt
+ cat removals.txt >> removals-full.txt
+ cat removals-20*.822 > removals-full.822
+ cat removals.822 >> removals-full.822
+}
+
+function rss() {
+ $base/dak/tools/queue_rss.py -q $queuedir/new -o ${webdir}/rss/ -d $base/misc -l $base/log/
+ $base/dak/tools/removals.pl $configdir/removalsrss.rc > ${webdir}/rss/removals.rss
+}
+
+function gen_di() {
+ $scriptsdir/generate-d-i
+}
+
+function backportsacl() {
+ # Update backports ACL
+ sudo -u dak-unpriv cat /srv/backports-master.debian.org/etc/acl \
+ | dak acl set-fingerprints backports \
+ | mail -a "X-Debian: DAK" -e -s "backports: acl changes" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" backports-team@debian.org
+}
+
+# do the buildd key updates
+function builddautosigning() {
+ BUILDDFUN=$(mktemp -p "${TMPDIR}" BUILDDFUN.XXXXXX)
+ TEMPFILES="${TEMPFILEs} ${BUILDDFUN}"
+ exec >> "${BUILDDFUN}" 2>&1
+ ${scriptsdir}/buildd-remove-keys
+ ${scriptsdir}/buildd-add-keys
+ ${scriptsdir}/buildd-prepare-dir
+
+ for keyring in $(dak admin k list-binary); do
+ dak import-keyring --generate-users "%s" ${keyring}
+ done
+ exec >>/dev/null 2>&1
+
+ DATE=$(date -Is)
+ cat "${BUILDDFUN}" | mail -a "X-Debian: DAK" -e -s "[$(hostname -s)] Buildd key changes ${DATE}" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" buildd-keys@ftp-master.debian.org
+
+ rm -f "${BUILDDFUN}"
+}
diff --git a/config/debian/hourly.tasks b/config/debian/hourly.tasks
new file mode 100644
index 0000000..b5417db
--- /dev/null
+++ b/config/debian/hourly.tasks
@@ -0,0 +1,10 @@
+# FUNC ARGS TIME ERR BG
+importusers none none false false
+newoverview none none false false
+queuereport none none false true
+removalstxt none none false true
+rss none none false true
+ddaccess none none true true
+gen_di none none false false
+backportsacl none none false false
+builddautosigning none none false false
\ No newline at end of file
diff --git a/config/debian/monthly.functions b/config/debian/monthly.functions
new file mode 100644
index 0000000..0241a96
--- /dev/null
+++ b/config/debian/monthly.functions
@@ -0,0 +1,19 @@
+# -*- mode:sh -*-
+
+function rotatelog() {
+ cd ${base}/log
+ touch ${DATE}
+ ln -sf ${DATE} current
+ chmod g+w ${DATE}
+ chown dak:ftpteam ${DATE}
+ xz -9 ${OLDDATE}
+}
+
+function rotatequeued() {
+ cd /srv/upload.debian.org/queued
+ sudo -u dak-unpriv /srv/upload.debian.org/queued/debianqueued -k || true
+ mv run/log log.${OLDDATE}
+ sudo -u dak-unpriv touch run/log
+ xz -9 log.${OLDDATE}
+ sudo -u dak-unpriv /srv/upload.debian.org/queued/debianqueued 2>/dev/null
+}
\ No newline at end of file
diff --git a/config/debian/monthly.tasks b/config/debian/monthly.tasks
new file mode 100644
index 0000000..619f58b
--- /dev/null
+++ b/config/debian/monthly.tasks
@@ -0,0 +1,3 @@
+# FUNC ARGS TIME ERR BG
+rotatelog none none false true
+rotatequeued none none false true
\ No newline at end of file
diff --git a/config/debian/monthly.variables b/config/debian/monthly.variables
new file mode 100644
index 0000000..da3dd1b
--- /dev/null
+++ b/config/debian/monthly.variables
@@ -0,0 +1,3 @@
+# -*- mode:sh -*-
+DATE=$(date +%Y-%m)
+OLDDATE=$(date -d yesterday +%Y-%m)
diff --git a/config/debian/unchecked.tasks b/config/debian/unchecked.tasks
new file mode 100644
index 0000000..d8f5211
--- /dev/null
+++ b/config/debian/unchecked.tasks
@@ -0,0 +1,9 @@
+# FUNC ARGS TIME ERR BG
+pg_timestamp preunchecked none false false
+punew stable-new p-u-new false false
+opunew oldstable-new o-p-u-new false false
+backports_policy none backports-policy false false
+cleanpolicy none cleanpolicy false false
+do_unchecked none unchecked false false
+scancontents none contentsscan false false
+pg_timestamp postunchecked none false false
diff --git a/config/debian/unchecked.variables b/config/debian/unchecked.variables
new file mode 100644
index 0000000..e09c696
--- /dev/null
+++ b/config/debian/unchecked.variables
@@ -0,0 +1,3 @@
+# -*- mode:sh -*-
+LOCKFILE="$lockdir/unchecked.lock"
+LOCK_BUILDD="$lockdir/buildd.lock"
diff --git a/config/debian/vars b/config/debian/vars
index aaa852f..c94ece0 100644
--- a/config/debian/vars
+++ b/config/debian/vars
@@ -40,5 +40,9 @@ umask 022
unset CDPATH
GZIP='--rsyncable' ; export GZIP
+# lock cron.unchecked (it immediately exits when this exists)
+LOCK_DAILY="$lockdir/daily.lock"
+LOCK_UNCHECKED="$lockdir/unchecked.lock"
+
# Set the database variables
eval $(dak admin config db-shell)
diff --git a/config/debian/weekly.functions b/config/debian/weekly.functions
new file mode 100644
index 0000000..55a7f23
--- /dev/null
+++ b/config/debian/weekly.functions
@@ -0,0 +1,24 @@
+# -*- mode:sh -*-
+
+function purgeempty() {
+ log "Purging empty directories in $ftpdir/pool/"
+
+ if [ ! -z "$(find $ftpdir/pool/ -type d -empty)" ]; then
+ find $ftpdir/pool/ -type d -empty | xargs rmdir;
+ fi
+}
+
+function gitcleanup() {
+ log "Doing git stuff"
+ cd ${public}/git/dak.git
+ git gc --prune
+ git update-server-info
+ # now workaround a git bug not honoring the setup in logs/*
+ # (fix in development, but until it reached backports.org.......)
+ chmod -R g+w logs/
+}
+
+function symlinks() {
+ log "Fixing symlinks in $ftpdir"
+ symlinks -d -r $ftpdir
+}
diff --git a/config/debian/weekly.tasks b/config/debian/weekly.tasks
new file mode 100644
index 0000000..4fb0d92
--- /dev/null
+++ b/config/debian/weekly.tasks
@@ -0,0 +1,4 @@
+# FUNC ARGS TIME ERR BG
+purgeempty none none false false
+gitcleanup none none false false
+symlinks none none false false
--
2.1.4
Reply to: