From 902cf37d03ebaa6f3b7a5ff645a13945f8ee2e57 Mon Sep 17 00:00:00 2001 From: Amand Tihon Date: Sun, 4 Jan 2009 22:44:28 +0100 Subject: [PATCH 1/1] First public version of the brioche backup script. --- brioche | 420 +++++++++++++++++++++++++++++++++++++++++++++++++++ brioche.conf | 44 ++++++ briochetab | 16 ++ 3 files changed, 480 insertions(+) create mode 100644 brioche create mode 100644 brioche.conf create mode 100644 briochetab diff --git a/brioche b/brioche new file mode 100644 index 0000000..2ccf1c3 --- /dev/null +++ b/brioche @@ -0,0 +1,420 @@ +#! /bin/bash + +# Brioche Backup +# Full and Incremental backup script with tar and LVM snapshots. +# +# Copyright Amand Tihon, 2008, 2009 +# Licensed under GNU GPL version 2.0. +# Inspired by +# http://blog.kagesenshi.org/2008/01/automated-tar-and-dump-incremental.html +# +# Note: This script relies on GNU tar specific options, +# do not attempt to use it as-is with other tar implementations. +# +# +# Example of use, from crontab +# # Daily incremental backup +# 30 3 * * 1-6 /usr/local/bin/brioche > /var/log/backup.`date "+%a"`.log 2>&1 +# # Weekly full backup on Sunday +# 30 3 * * 0 /usr/local/bin/brioche -f > /var/log/backup.`date "+%a"`.log 2>&1 + + +# Mandatory +CONFIG_FILE="/etc/brioche.conf" + + +####################################################################### +# Default config values. Do not change, edit $CONFIG_FILE +####################################################################### + +BACKUPTAB="/etc/briochetab" +MAILTO="root" +REPODIR="/backup" +TAR_OPTS="--one-file-system -S " +COMPRESS="gz" +COMPRESS_OPT="--gzip" +SNAPSHOT_MOUNTPOINT="/mnt/backup-snapshot" +SNAPSHOT_NAME="backup-snap" +SNAPSHOT_SIZE="5G" +USAGE_WARN="80" + +# TODO: Implement the storage on a distant FTP +USE_FTP="no" +#FTP_HOST="ftpback.example.com" +#FTP_USER="username" +#FTP_PASS="password" +#FTP_KEEP="4" + +# Ensure that we have a minimal PATH +PATH=/sbin:/bin:/usr/sbin:/usr/bin +FINAL_STATUS="SUCCESS" +SUMMARY="/tmp/backup.sumary" + + +####################################################################### +# Helpers +####################################################################### + +trap 'interrupted ctrl-C' INT +trap 'interrupted KILL' TERM + +NOW() +{ + echo -n `date "+%Y-%m-%d %H:%M:%S"` +} + +# Log a line with timestamp. +log() +{ + echo "[`NOW`] $@" +} + +# The summary that will be sent by email +summary() +{ + log $@ + echo $@ >> $SUMMARY +} + +finish() +{ + # Check free space + local usage=`df -hP "${REPODIR}" | tail -n 1 | tr -s [:space:] | cut -d" " -f5 | tr -d '%'` + log "${REPODIR} usage after backup: ${usage}%." + + if [ "$usage" -ge "$USAGE_WARN" ]; then + if [ "$FINAL_STATUS" = "SUCCESS" ]; then + FINAL_STATUS="WARNING" + fi + summary "Warning : Filesystem ${REPODIR} is ${usage}% full." + fi + + summary "Backup procedure ended on `NOW`" + + SUBJECT="Backup report for `hostname` (${FINAL_STATUS})." + mail -s "${SUBJECT}" "${MAILTO}" < "$SUMMARY" + + rm -f "$SUMMARY" + exit +} + +interrupted() +{ + FINAL_STATUS="INTERRUPTED" + summary "Backup procedure interrupted by user ($1)." + summary "Take care, an LVM snapshot may still be present !" + finish +} + +####################################################################### +# Snapshots +####################################################################### + +# Make a snapshot of a logical volume. +# Usage: make_snapshot vg lv +make_snapshot() +{ + log "Creating a snapshot volume of /dev/$1/$2" + lvcreate --snapshot -L ${SNAPSHOT_SIZE} -n ${SNAPSHOT_NAME} /dev/$1/$2 + RETVAL=$? + if [ "$RETVAL" != "0" ]; then + log "Error ${RETVAL}: Unable to create a snapshot of /dev/$1/$2" + return 1 + fi +} + +# Mount the snapshot volume. +# On error, tries to remove the snapshot. +# Usage: mount_snapshot vg +mount_snapshot() +{ + log "Mounting the snapshot volume ${SNAPSHOT_NAME}." + if [ ! -d "${SNAPSHOT_MOUNTPOINT}" ]; then + log "Creating mountpoint ${SNAPSHOT_MOUNTPOINT}." + mkdir -p "${SNAPSHOT_MOUNTPOINT}" + fi + + mount /dev/$1/${SNAPSHOT_NAME} ${SNAPSHOT_MOUNTPOINT} + RETVAL=$? + if [ "$RETVAL" != "0" ]; then + log "Error ${RETVAL}: Unable to mount /dev/$1/${SNAPSHOT_NAME} on ${SNAPSHOT_MOUNTPOINT}" + remove_snapshot $1 || return 100 + return 1 + fi +} + +# Remove a previously created snapshot. It must be unmounted. +# Usage: remove_snapshot vg +remove_snapshot() +{ + log "Removing the snapshot volume /dev/$1/${SNAPSHOT_NAME}." + lvremove -f /dev/$1/${SNAPSHOT_NAME} + RETVAL=$? + if [ "$RETVAL" != "0" ]; then + log "Error ${RETVAL}: Unable to remove the snapshot volume /dev/$1/${SNAPSHOT_NAME}" + return 100 + fi +} + +# Unmount the previously mounted snapshot. +# Usage: unmount_snapshot vg +unmount_snapshot() +{ + log "Unmounting the snapshot volume /dev/$1/${SNAPSHOT_NAME}." + umount /dev/$1/${SNAPSHOT_NAME} + RETVAL=$? + if [ "$RETVAL" != "0" ]; then + log "Error ${RETVAL}: Unable to unmount the snapshot volume /dev/$1/${SNAPSHOT_NAME}" + return 100 + fi +} + +####################################################################### +# Main backup functions +####################################################################### + +# Make a full backup of a directory, with snar file for subsequent incremental +# Usage: make_full_backup source_dir hostname volumename +# Returns 0 on success, 1 on error +make_full_backup() +{ + log "Making full backup of $2 - ${3}." + + local destdir="${REPODIR}/$2" + local today=`date "+%Y%m%d"` + local destfile="${destdir}/${3}.full.${today}.tar.${COMPRESS}" + local destsnar="${destdir}/${3}.full.snar" + + # Move previous run to the "undo" directory + if [ ! -d "${destdir}/undo" ]; then + log "Creating undo/ directory." + mkdir -p "${destdir}/undo" + fi + log "Moving old run into undo/ directory." + mv ${destdir}/${3}.* ${destdir}/undo + + # Do the actual backup. Destination file name and snar are like + # /backup/valeron/root.full.20090105.tar.bz2 + # /backup/valeron/root.full.snar + log "Running tar..." + tar -cf ${destfile} ${TAR_OPTS} ${COMPRESS_OPT} -g ${destsnar} $1 + + if [ "$?" = "0" ]; then + log "Removing undo/ directory." + rm -rf "${destdir}/undo" + else + log "Error $?: Could not archive $2 - $3" + return 1 + fi + return 0 +} + + +# Make an incremental backup of a directory, from full's snar file +# Usage: make_incr_backup source_dir hostname volumename +# Returns 0 on success, 1 on error, 2 if no previous full is found. +make_incr_backup() +{ + log "Making incremental backup of $2 - ${3}." + + local destdir="${REPODIR}/$2" + local today=`date "+%Y%m%d"` + local destfile="${destdir}/${3}.incr.${today}.tar.${COMPRESS}" + local destsnar="${destdir}/${3}.incr.${today}.snar" + local fullsnar="${destdir}/${3}.full.snar" + + # Test existence of full backup + if [ ! -e $fullsnar ]; then + log "Could not find catalog ${fullsnar}." + return 2 + fi + + # Prepare the copy of the snar file + cp $fullsnar $destsnar + + # Do the actual backup. Destination file name and snar are like + # /backup/valeron/root.incr.20090105.tar.bz2 + # /backup/valeron/root.incr.20090105.snar + log "Running tar..." + tar -caf ${destfile} ${TAR_OPTS} ${COMPRESS_OPT} -g ${destsnar} $1 + + if [ ! "$?" = "0" ]; then + log "Error $?: Could not archive $hostname - $volumename" + return 1 + fi + return 0 +} + + + +####################################################################### +# Start here +####################################################################### + +# Truncate summary file, start header blurb +echo "" > $SUMMARY +summary "Backup procedure started on `NOW`" + +if [ -r "${CONFIG_FILE}" ] +then + source "${CONFIG_FILE}" +else + summary "Error: Unable to read configuration file ${CONFIG_FILE}. Aborting." + FINAL_STATUS="CRITICAL" + finish +fi + +if [ ! -r "${BACKUPTAB}" ] +then + summary "Error: Unable to read ${BACKUPTAB}. Aborting." + FINAL_STATUS="CRITICAL" + finish +fi + +# TODO: Something cleaner, here... +DO_FULL_BACKUP="no" +case "$1" in + --full|-f) + DO_FULL_BACKUP="yes" + ;; + --help|-h) + echo "Usage: ${0} [-f]" + exit 0 + ;; +esac + +# Discover which COMPRESS_OPT to use, from COMPRESS +case "$COMPRESS" in + gz) + COMPRESS_OPT="--gzip" + ;; + bz2) + COMPRESS_OPT="--bzip2" + ;; + lzma) + COMPRESS_OPT="--lzma" + ;; + *) + summary "Unknown compression method: ${COMPRESS}. Falling back to gzip." + COMPRESS="gz" + COMPRESS_OPT="--gzip" + ;; +esac + +####################################################################### +# Parse backuptab file, call backup functions for each line +####################################################################### + +# Ignore empty and commented lines +grep -v -E '^[[:space:]]*(#.*)?$' $BACKUPTAB | tr -s [:space:]| while read line +do + # split line in fields + device=`echo $line|cut -d" " -f1` + dosnap=`echo $line|cut -d" " -f2` + group=`echo $line|cut -d" " -f3` + volume=`echo $line|cut -d" " -f4` + + # Make and mount snapshot if needed. + if [ "$dosnap" = "yes" ]; then + # Split the device to find the VG and LV + vg=`echo $device|cut -d"/" -f3` + lv=`echo $device|cut -d"/" -f4` + make_snapshot $vg $lv + if [ "$?" != "0" ]; then + summary "Could not take a snapshot of $device" + FINAL_STATUS="ERROR" + continue + # Next one + fi + + mount_snapshot $vg + RETVAL="$?" + if [ "$RETVAL" != "0" ]; then + summary "Could not mount the snapshot of $device" + FINAL_STATUS="ERROR" + if [ "$RETVAL" = "100" ]; then + summary "Could not remove the snapshot !" + FINAL_STATUS="CRITICAL" + finish + fi + continue + # Next one + fi + + BACKUP_SOURCE="${SNAPSHOT_MOUNTPOINT}" + else + BACKUP_SOURCE="$device" + fi + + # Make the backup + if [ "$DO_FULL_BACKUP" = "no" ]; then + make_incr_backup ${BACKUP_SOURCE} $group $volume + RETVAL=$? + if [ "$RETVAL" = "0" ]; then + summary "INCREMENTAL backup of $device done on `NOW`." + elif [ "$RETVAL" = "1" ]; then + summary "Error during incremental backup of $device" + FINAL_STATUS="ERROR" + elif [ "$RETVAL" = "2" ]; then + summary "Can't do an incremental backup without a full one being present." + summary "Switching to full backup for $device" + make_full_backup ${BACKUP_SOURCE} $group $volume + if [ "$?" = "0" ]; then + summary "FULL backup of $device done on `NOW`." + else + summary "Error during full backup of $device" + FINAL_STATUS="ERROR" + fi + else + summary "Unknown error during incremental backup of $device" + FINAL_STATUS="ERROR" + fi + else # Do a full backup + make_full_backup ${BACKUP_SOURCE} $group $volume + if [ "$?" = "0" ]; then + summary "FULL backup of $device done on `NOW`." + else + summary "Error during full backup of $device" + FINAL_STATUS="ERROR" + fi + fi + + # Time to unmount and destroy the snapshot, if needed. + # Any error here is critical, since the snapshot and the mountpoint are + # the same for each device... + if [ "$dosnap" = "yes" ]; then + unmount_snapshot $vg + if [ "$?" != "0" ]; then + summary "Could not unmount snapshot !" + FINAL_STATUS="CRITICAL" + finish + fi + + remove_snapshot $vg + if [ "$?" != "0" ]; then + summary "Could not destroy the snapshot !" + FINAL_STATUS="CRITICAL" + finish + fi + fi +done + +finish + +## Example Backup description table +## +## Can handle logical volumes, with snapshots, or plain mountpoints. +## Priority is not used yet. +# +## Partition or LV Snapshot Host name Volume name Priority +## ---------------------------------------------------------------------------- +#/ no cottman root 1 +#/dev/vg00/kadarin-root yes kadarin root 1 +#/dev/vg00/valeron-root yes valeron root 1 +#/dev/vg00/syrtis-root yes syrtis root 1 +#/dev/vg00/syrtis-home yes syrtis home 1 +#/dev/vg00/syrtis-usr yes syrtis usr 1 +#/dev/vg00/syrtis-var yes syrtis var 1 +#/dev/vg00/syrtis-srv yes syrtis srv 1 + + diff --git a/brioche.conf b/brioche.conf new file mode 100644 index 0000000..b191bdb --- /dev/null +++ b/brioche.conf @@ -0,0 +1,44 @@ +# Brioche backup configuration file. +# +# This file is sourced by the brioche backup script. +# It should only contain configuration directive, even if technically, more +# advanced sections are possible. + + +# BACKUPTAB: Full path to the file describing the backups to perform +BACKUPTAB="/etc/briochetab" + +# MAILTO: Who should receive the email report +MAILTO="root" + +# REPODIR: Where to store the week's backup (locally) +REPODIR="/backup" + +# USAGE_WARN: Add a warning if the REPODIR is nearly full +USAGE_WARN="80" + +# COMPRESS: What kind of compression should tar use. +# Can be "gz", "bz2" or "lzma". Make sure your version of tar understands it. +COMPRESS="gz" + +# TAR_OPTS: Additionnal options to the tar invocation. +TAR_OPTS="--one-filesystem -S" + +# SNAPSHOT_MOUNTPOINT: Where to mount the temporary LVM snaphot +SNAPSHOT_MOUNTPOINT="/mnt/backup-snapshot" + +# SNAPSHOT_NAME: How to name the snapshot logical volume +SNAPSHOT_NAME="brioche-snapshot" + +# SNAPSHOT_SIZE: The size of the snapshot. See lvcreate(8) for size suffix. +# Make sure that it will never become full, or it would be destroyed +# automatically, rendering the backup unusable. +SNAPSHOT_SIZE="5G" + +# USE_FTP: Not yet implemented. +USE_FTP="no" + +# FTP_HOST="ftpback.example.com" +# FTP_USER="username" +# FTP_PASS="password" +# FTP_KEEP="4" diff --git a/briochetab b/briochetab new file mode 100644 index 0000000..5a914ba --- /dev/null +++ b/briochetab @@ -0,0 +1,16 @@ +# Backup description table +# +# Can handle logical volumes, with snapshots +# In this example, the first partition to backup is the root of the dom0. +# The following lines are the different LVM volumes created for the domUs. + +# Partition or LV Snapshot Host name Volume name Priority +# ---------------------------------------------------------------------------- +/ no cottman root 1 +/dev/vg00/valeron-root yes valeron root 1 +/dev/vg00/kadarin-root yes kadarin root 1 +/dev/vg00/syrtis-root yes syrtis root 1 +/dev/vg00/syrtis-home yes syrtis home 1 +/dev/vg00/syrtis-usr yes syrtis usr 1 +/dev/vg00/syrtis-var yes syrtis var 1 +/dev/vg00/syrtis-srv yes syrtis srv 1 -- 2.39.5