Re: backup strategy using rsync

To: Randall Smith <randall@tnr.cc>
Cc: debian-user@lists.debian.org
Subject: Re: backup strategy using rsync
From: Tyler MacDonald <tyler@yi.org>
Date: Fri, 19 Jan 2007 14:21:50 -0800
Message-id: <[🔎] 20070119222149.GC1383@yi.org>
In-reply-to: <[🔎] eooha7$r6t$1@sea.gmane.org>
References: <[🔎] eooha7$r6t$1@sea.gmane.org>

Randall,

> I got to thinking, how hard would it be to isolate the hardware dependent
> portions of a system, and simply backup and restore the hardware
> independent portions onto a new system using rsync?  Can someone shed some
> light on the subject?  A thinks I'm not clear on.

I have implemented this using "backup2l". I've attached a copy of my
"backup2l.conf" file and rsync shell scripts to give you an idea. :-) My
servers and those of a few friends get backed up to a central server using
rsync, and before the sync happens, the backups are encrypted using the
appropriate GPG key. My home machine then syncs all backups down to itself
for burning/archival; we try to keep all of a machine's own backups on it's
system, then update a "skip.list" file (used for rsync exclusions) to
reflect what backups have actually been burned.



	Cheers,
		Tyler

# Configuration file for backup2l                #
##################################################


# Define the backup2l version for which the configuration file is written.
# This way, future versions can automatically warn if the syntax has changed.
FOR_VERSION=1.3
TZ=GMT
BHOME="/var/backups"
GPG_HOME="$BHOME/.gpg"
GPG_KEY="backup@crackerjack.net"
SQL_TUNNEL_USER="backup@tv.knoglinger.com"
SQL_DB="backup_log"
SQL_USER="backup"

# This is the prefix for all output files;
# multiple volumes can be handled by using different configuration files
# ... or with this hack, by using one config file with all the common info :)
if test -z "$VOLNAME"; then
    VOLNAME=`hostname -f`
fi

if test "$VOLNAME" = "crackerjack"; then
    VOLNAME="crackerjack.yi.org"
fi

if test "$VOLNAME" = "barn"; then
    VOLNAME="barn.arnot"
fi

case "$VOLNAME" in
    ubookbox.yi.org)
        SYNC="sync-to-arctic.sh"
        MYSQLDUMP="0"
        PGDUMP="0"
        TRIM_KEYS="0"
        CREATE_DRIVER="DRIVER_MY_GPGBZ2"
        SQL_TUNNEL="1"
        SQL_PORT="51979"
        BTDUMP="/usr/local/bin/bt_db2xml /var/lib/mod_bt"
        ;;
    crackerjack.yi.org)
        SYNC="sync-from-arctic.sh"
        MYSQLDUMP="1"
        PGDUMP="0"
        TRIM_KEYS="0"
        CREATE_DRIVER="DRIVER_TAR_BZ2"
        SQL_TUNNEL="1"
        SQL_PORT="51979"
        BTDUMP="/usr/local/bin/bt_db2xml /var/lib/mod_bt"
        ;;
    bavarian.yi.org)
        SYNC="sync-to-arctic.sh"
        MYSQLDUMP="1"
        PGDUMP="0"
        TRIM_KEYS="0"
        CREATE_DRIVER="DRIVER_MY_GPGBZ2"
        SQL_TUNNEL="1"
        SQL_PORT="51979"
        GPG_KEY="0167415C"
        ;;
    susicivus.crackerjack.net)
        SYNC="sync-to-arctic.sh"
        MYSQLDUMP="1"
        PGDUMP="0"
        TRIM_KEYS="1"
        CREATE_DRIVER="DRIVER_MY_GPGBZ2"
        SQL_TUNNEL="1"
        SQL_PORT="51979"
        BTDUMP=""
        ;;
    libertas.crackerjack.net)
        SYNC="sync-to-arctic.sh"
        MYSQLDUMP="1"
        PGDUMP="0"
        TRIM_KEYS="0"
        CREATE_DRIVER="DRIVER_MY_GPGBZ2"
        SQL_TUNNEL="1"
        SQL_PORT="51979"
        BTDUMP=""
        ;;
    ostendo.crackerjack.net)
        SYNC="sync-to-arctic.sh"
        MYSQLDUMP="1"
        PGDUMP="0"
        TRIM_KEYS="0"
        CREATE_DRIVER="DRIVER_MY_GPGBZ2"
        SQL_TUNNEL="1"
        SQL_PORT="51979"
        BTDUMP=""
        ;;
    tv.knoglinger.com)
        MYSQLDUMP="1"
        PGDUMP="0"
        TRIM_KEYS="0"
        CREATE_DRIVER="DRIVER_MY_GPGBZ2"
        SQL_TUNNEL="0"
        GPG_KEY="0167415C"
        BTDUMP=""
        ;;
    deemfudge.yi.org)
        SYNC="sync-to-arctic.sh"
        MYSQLDUMP="1"
        PGDUMP="0"
        TRIM_KEYS="0"
        CREATE_DRIVER="DRIVER_MY_GPGBZ2"
        SQL_TUNNEL="1"
        SQL_PORT="51979"
        BTDUMP=""
        ;;
    viclog06.acd.int)
        SYNC="sync-to-arctic.sh"
        MYSQLDUMP="0"
        PGDUMP="0"
        TRIM_KEYS="0"
        CREATE_DRIVER="DRIVER_MY_GPGBZ2"
        SQL_TUNNEL="1"
        SQL_PORT="51979"
        BTDUMP=""
        ;;
    eyecandy.arnot)
        SYNC="sync-to-arctic.sh"
        MYSQLDUMP="1"
        PGDUMP="0"
        TRIM_KEYS="0"
        CREATE_DRIVER="DRIVER_MY_GPGBZ2"
        SQL_TUNNEL="1"
        SQL_PORT="51979"
        BTDUMP=""
        ;;
    barn.arnot)
        SYNC="sync-to-arctic.sh"
        MYSQLDUMP="0"
        PGDUMP="0"
        TRIM_KEYS="0"
        CREATE_DRIVER="DRIVER_MY_GPGBZ2"
        SQL_TUNNEL="1"
        SQL_PORT="51979"
        BTDUMP=""
        ;;
    knock.arnot)
        SYNC="sync-to-arctic.sh"
        MYSQLDUMP="0"
        PGDUMP="0"
        TRIM_KEYS="0"
        CREATE_DRIVER="DRIVER_MY_GPGBZ2"
        SQL_TUNNEL="1"
        SQL_PORT="51979"
        BTDUMP=""
        ;;
esac

export SQL_PORT SQL_USER SQL_DB TZ
export PATH="$BHOME/bin:$PATH"

##################################################
# Source files

# All paths MUST be absolute and start with a '/'!
SRCLIST=(/etc /root /home /var /opt /usr/local /srv)

SKIPCOND=( \
    -path "*/opt/pmx*" -o -path "*/cache/apt/*" -o -path "*/.spamassassin/*" \
    -o -path "*/lib/postgrey/*" -o -name "proclog" -o -name ".procmail.log" \
    -o -path "*.nobackup*" -o -name "*.o" -o -path "*/lib/mysql/*" \
    -o -path "*/var/run/*" -o -path "*/tmp/*" -o -path "*/cache/*" \
    -o -path "*/dump/backup/*" -o -path "*/public_html/seed/*" \
    -o -path "*/public_html/btdown/*" -o -path "*/.sheep/*" \
    -o -path "*/var/log/mysql.log*" -o -path "*/var/log/apache/*_log*" \
    -o -path "*/var/log/apache/*.log*" -o -path "*/furnace/*" \
    -o -path "*/.razor/*" -o -path "*/home/upload/*" \
    -o -path "*/var/lib/mod_bt/*" -o -path "*/var/tracker/*" \
    -o -path "*/home/*/.*/Cach*/*" -o -path "*/access_log*" \
    -o -path "*/.aMule/*" -o -path "*/access.log*" \
    -o -path "*/var/log/kismet/*" -o -path "*/urantia/shares/*" \
    -o -path "*/home/archivezelect/*" \
    -o -path "*/home/faraway/rip/*" \
    -o -path "*/home/ftp/debian/*" -o -path "*/home/ftp/CPAN/*" \
    -o -path "*/var/video/*" -o -path "*/var/music/*" \
    -o -path "*/var/lib/mythtv/*" -o -path "*/var/pictures/*" \
    -o -path "*/var/lib/mythdvd/*" -o -path "*/var/lib/mytharchive/*" \
    -o -type b -o -type c -o -type p -o -type s -o -fstype nfs \
    -o -path "*/home/morganna/*.avi" -o -path "*/home/morganna/*.mp3" \
    -o -path "*/chroot/*/sys/*" -o -path "*/chroot/*/proc/*" \
    -o -path "*/chroot/*/dev/*"
)

##################################################
# Destination
if test -z "$BACKUP_DIR"; then
	BACKUP_DIR="/dump/backup"
fi

# Mount point of backup device (optional)
# BACKUP_DEV="/dump"

##################################################
# Backup parameters

# Number of levels of differential backups (1..9)
MAX_LEVEL=3

# Maximum number of differential backups per level (1..9)
MAX_PER_LEVEL=7

# Maximum number of full backups (1..9)
MAX_FULL=2

# For differential backups: number of generations to keep per level;
# old backups are removed such that at least GENERATIONS * MAX_PER_LEVEL
# recent versions are still available for the respective level
GENERATIONS=1

# If the following variable is 1, a check file is automatically generated
CREATE_CHECK_FILE=1

##################################################
# Pre-/Post-backup functions

# This user-defined bash function is executed before a backup is made
PRE_BACKUP ()
{
    # On a Debian system, the following statements dump a machine-readable list of
    # all installed packages to a file.
    echo "  writing dpkg selections to /root/dpkg-selections.log..."
    dpkg --get-selections | diff - /root/dpkg-selections.log > /dev/null || dpkg --get-selections > /root/dpkg-selections.log

    if [ "$TRIM_KEYS" = "1" ]; then
        echo "  trimming session keys from yi"
        mysql mydns -e "delete from session_keys where expires < now()"
        echo
    fi

    if [ "$MYSQLDUMP" = "1" ]; then
        echo "  dumping MySQL database"
        mkdir -p "$BHOME/mysql"
        mysql-backup.sh "$BHOME/mysql"
        echo
    fi
    
    if test -n "$BTDUMP"; then
        echo "  dumping mod_bt database"
        $BTDUMP | diff -I ServerTime - /root/mod_bt.xml > /dev/null || $BTDUMP > /root/mod_bt.xml
        echo
    fi
}

# This user-defined bash function is executed after a backup is made
POST_BACKUP ()
{
    # e. g., restart some mail/db server if its files are to be backup'ed
    echo "  changing ownership of backup files"
    chown -R backup.backup "$BACKUP_DIR"
    chmod -R g-w,g+r,o-rwx "$BACKUP_DIR"
    echo
    
    if [ "$SQL_TUNNEL" = "1" ]; then
        echo "  establishing SSH tunnel on port $SQL_PORT"
        fuser -s -k -n tcp $SQL_PORT
        su - backup -c "ssh -NfL $SQL_PORT:127.0.0.1:3306 $SQL_TUNNEL_USER"
    fi

    if test -n "$SYNC"; then
        echo "  syncronizing with $SYNC"
        $SYNC
        echo
    fi

    if [ "$SQL_TUNNEL" = "1" ]; then
        echo "  closing SSH tunnel for SQL"
        fuser -sn tcp -k $SQL_PORT
    fi
}


##################################################
# Misc.

# Create a backup when invoked without arguments?
AUTORUN=0

# Size units
SIZE_UNITS="M"    # set to "B", "K", "M" or "G" to obtain unified units in summary list

##################################################
# User-defined archive drivers (optional)
# If you do not want to write your own archive driver, you can remove the remainder of this file.

USER_DRIVER_LIST="DRIVER_MY_GPGBZ2"  # uncomment to register the driver(s) below (optional)

GPG ()
{
    gpg --no-permission-warning --homedir "$GPG_HOME" "$@"
}

GPGE ()
{
    GPG --yes --batch --trust-model always -r "$GPG_KEY" -e "$@"
}

GPGX ()
{
    GPG --decrypt "$@"
}

DRIVER_MY_GPGBZ2 ()
{
    case $1 in
        -test)
            require_tools gpg gpgv bzip2

            if [ -z "$GPG_KEY" ]; then
                echo "The GPG_KEY setting is required for gpg encryption!"
                break
            fi

            if [ -z "$GPG_HOME" ]; then
                echo "The GPG_HOME setting is required for gpg encryption!"
                break
            fi

            if \
                echo "foo" | GPGE > /dev/null
            then
                echo "ok"
            else
                echo "gpg test failed\!"
            fi

            ;;
        -suffix)
            echo "tar.bz2.gpg"
            ;;
        -create)
            # Arguments: $2 = BID, $3 = archive file name, $4 = file list file
            # We don't encrypt the archive yet because the "-toc" step happens
            # next and needs access to it.
            outfile="$3"
            bzfile=`echo "$outfile" | sed -e "s,.gpg$,,"`
            DRIVER_TAR_BZ2 -create "$2" "$bzfile" "$4"
            ;;
        -toc)
            # Arguments: $2 = BID, $3 = archive file name
            # This function is used to validate the correct generation of an archive file.
            outfile="$3"
            bzfile=`echo "$outfile" | sed -e "s,.gpg$,,"`
            DRIVER_TAR_BZ2 -toc "$2" "$bzfile"
            GPGE "$bzfile" && rm -f "$bzfile"
            ;;
        -extract)
            # Arguments: $2 = BID, $3 = archive file name, $4 = file list file
            # It is extremely important that only those files contained in $4 are restored.
            GPGX "$3" | tar xj --same-permission --same-owner -f - -T "$4" 2>&1
            ;;
    esac
}

Attachment: sync-from-arctic.sh
Description: Bourne shell script

Attachment: sync-to-arctic.sh
Description: Bourne shell script

Reply to:

References:
- backup strategy using rsync
  - From: Randall Smith <randall@tnr.cc>

Prev by Date: Reza Sadeghi { Vaysa Donya } - Free MP3 Download
Next by Date: Re: top post fixer?
Previous by thread: Re: backup strategy using rsync
Next by thread: PPC modem not detected
Index(es):
- Date
- Thread