Arhiviranje serverja s TAR - incremental in FTP na drugo lokacijo in email report - SKRIPTA

OK, tole sem sam spisal za moje potrebe ampak niti slučajno to ni 100% backup, vsaj nekaj podatkov pa tako arhiviram in jih preko FTP (LFTP zaradi stabilnosti povezav) shranjujem na drug server / lokacijo na koncu pa še mail pošlje kaj je naredil.
(napisana je v angleščini, ker sem objavil tudi na CentOS forumu)

Zadeva se dogaja dnevno (Cron.daily), izmeri čas ki se porabi za določeno operacijo in avtomatsko naredi FULL Backup v Nedeljo (ostale dni pa inkrementalno glede na Nedeljo) ker je pri meni na serverju takrat najmanj obremenjeno. Tudi pobrišem arhive starejše od XXY dni ki so lokalno narejeni v direktoriju da na samem serverju ne zavzema preveč in se hranijo na FTP lokaciji dalj časa.

Skripta dela z GNU/TAR (ker podpira incremental backup) in TXT seznamom datotek ki jih arhiviram / izločim iz arhiva:

Skripto zaganjam v CRON:
PATH=/sbin:/bin:/usr/bin:/usr/sbin:/usr/local/bin
0 1 * * * archive.sh >/dev/null # at 1.00AM every day


Ta archive.sh pa imam v /usr/local/bin , ki je v PATH.
#!/bin/bash
cd /usr/tmp/server_backup
./backup.sh # the real script to make backup

S tem archive.sh se edino prestavim v direktorij

/usr/tmp/server_backup 

kjer se arhiviranje dogaja.


Najprej seznami kaj in kaj ne:
including.txt:

/var/
/etc/
/home/

excluding.txt:

proc
*_log*
var/tmp
var/lib/bluetooth
var/lib/cs
var/lib/dav
var/lib/dbus
var/lib/dhcpv6
var/lib/dovecot
var/lib/games
var/lib/rpm
var/lib/webalizer
var/lib/yum
var/tmp/arhiv_serverja
var/log
var/run
var/www/manual
var/yp
var/lib/php/session
spool
cache
*zip
*gz
etc/rc*
home/httpd/manual
rpm

še LFTP skripta:
lftpscript.sh

#!/bin/sh
HOST='ftp.domain.com'
USER='ftpuser'
PASSWD='idontknow'
FILE=$(cat archivename.txt)

lftp -c "open $HOST && user $USER $PASSWD && cd FOLDER_NAME_FOR_STORING/backups/ && put $FILE" <<END_SCRIPT
# all in one command that connects to HOST=ftp.domain.com with Username/password 
# and changes directory to whatever you need 
# then transfers the file with the name of created archive in archivename.txt (ex. BACKUP-2009-12-15-Fri-01-15-01h.tgz)
bye
exit
END_SCRIPT
exit 0

no in končno še
backup.sh

#!/bin/sh

# DELETE archive older than -mtime +'days'
find . -name 'BACKUP*.tgz' -mtime +20 -delete
find . -name 'stopwatch*' -mtime +2 -delete

start1=$(date +"%T h   ( %s )")
start=$(date +%s)

# on SUNDAY make FULL backup
if [ $(date +"%a") = "Sun" ]; then
{
SNAPSHOTFILE="./usr-full";				# needed by TAR (GNU-TAR to be precise) which is used to compare for incremental backups
ARCHIVENAME=BACKUP-full-$(date +"%F-%a-%H-%M-%Sh").tgz;					# self explaining
rm -f "./usr-full";
}

else
{
ARCHIVENAME=BACKUP-$(date +"%F-%a-%H-%M-%Sh").tgz;
# a name of a backup file:  BACKUP-2009-12-15-Fri-01-15-01h.tgz
SNAPSHOTFILE="./usr-1";
cp "./usr-full" "./usr-1";
}
fi

echo $ARCHIVENAME >archivename.txt		# need the name to FTP transfer so store it in file archivename.txt which doesn't change (used later in lftpscript.sh ) !
# creating text to send in email
echo "-----------------------" >stopwatch-$archivename.txt
echo "Backup of $ARCHIVENAME" >>stopwatch-$archivename.txt
echo "-----------------------" >>stopwatch-$archivename.txt
echo " " >>stopwatch-$archivename.txt              # echo " " makes new line /CR or LF whatever it does
# I do not need this precise time { time tar -T including.txt -X excluding.txt -pczvRf $ARCHIVENAME; } 2>> stopwatch-$ARCHIVENAME.txt >/dev/null
{ tar -T including.txt -X excluding.txt -pczvR --listed-incremental=$SNAPSHOTFILE  -f $ARCHIVENAME; } 2>> stopwatch-$ARCHIVENAME.txt >/dev/null

stopped1=$(date +"%T h   ( %s )")
stopped=$(date +%s)
ftpstarted=$stopped

thetime=$(($stopped-$start))			# doing some math in shell that's why $()

echo " " >>stopwatch-$ARCHIVENAME.txt
echo -n "File Size (Byte-s) : " >>stopwatch-$ARCHIVENAME.txt
ls -al "$ARCHIVENAME" | cut -f 5 -d ' ' >>stopwatch-$ARCHIVENAME.txt
# this part | cut -f 5 -d ' '   is sometimes maybe 6 instead of 5, experiment which gives you only the SIZE of the file
echo " " >>stopwatch-$ARCHIVENAME.txt
echo "Started: " $start1 >>stopwatch-$ARCHIVENAME.txt
echo "Stopped: " $stopped1 >>stopwatch-$ARCHIVENAME.txt
echo "Time needed: " $(date -d "1970-01-01 $thetime sec" +"%H:%M:%S")      /  $thetime "secs" >>stopwatch-$ARCHIVENAME.txt
# outputs: Time needed: 00:03:14 / 194 secs


echo "-----------------------" >>stopwatch-$ARCHIVENAME.txt
echo " " >>stopwatch-$ARCHIVENAME.txt
echo "FTP start:" >>stopwatch-$ARCHIVENAME.txt
# again I dont need exact time procedure { time ./lftpscript.sh; } 2>> stopwatch-$ARCHIVENAME.txt
{ ./lftpscript.sh; } 2>> stopwatch-$ARCHIVENAME.txt

ftpstop1=$(date +"%T h   ( %s )")
ftpstopped=$(date +%s)

ftptime=$(($ftpstopped-$ftpstarted))

echo " " >>stopwatch-$ARCHIVENAME.txt
echo "Start of FTP: " $stopped1 >>stopwatch-$ARCHIVENAME.txt
echo "End of FTP: " $ftpstop1 >>stopwatch-$ARCHIVENAME.txt
echo "Time of FTP transfer: " $(date -d "1970-01-01 $ftptime sec" +"%H:%M:%S")     /   $ftptime "secs" >>stopwatch-$ARCHIVENAME.txt

mail -s "Backup of $ARCHIVENAME" "email address of recipient" <stopwatch-$ARCHIVENAME.txt
#finally email report :-)

Tako, upam da je kaj jasno, drugače še kaj obrazložim oziroma dobrodošli komentarji da se kaj popravi ali pa čisto tako če je komu prišlo prav ...

LP

Moja verzija backupa, ki deluje s pomočjo Duplicity.
http://github.com/orkaa/duplicity/blob/...

Za Duplicity pa nisem vedel zato pa si bom pogledal kako bi lahko izboljšal, ker piše da kopira samo spremenjen del datotek, ne pa cele datoteke ki se je spremenila (tako dela TAR)

Zakaj za hudiča bi bilo pa to slabo? :)

Duplicity al pa rdiff-backup. Brezveze je vsak dan prenašat miljavžnt giga podatkov naokoli. Pa še ena moja semi grda skripta, ki uporablja rdiff-backup za backup. Za prenos podatkov se uporablja ssh, stvari se pa odlagajo na oddaljen strežnik z navadnim userjem (rdiff-backup si beleži podatke o lastništvu in dovoljenjih nad datotekami, pa zna ob restoru pol vse pravilno nastavit).

#!/bin/bash
#
# Wrapper script for performing rdiff-backup based backups. Uses some checks and lock file to be on the safe side.
#
HELP="\n rbackup [ run | remold [older than] ] \n \n
Use run for normal backup. \n
Use remold to remove older than set in \$OLDBACKUP variable. You can overwrite default setting \n
if you specify second parameter here. See 'man rdiff-backup' --remove-older-than part for details. \n \n
"
# Set PYTHONPATH and/or PATH if needed
export PYTHONPATH="/opt/rdiff-backup-1.1.5/lib/lib.linux-i686-2.4"
#export PATH="$PATH"
# Set some variables regarding backup
CONFDIR="/root/lib"
RDIFF="/opt/rdiff-backup-1.1.5/bin/rdiff-backup"
SOURCE="/"
DEST="192.168.0.1::backup/"
LOGFILE="$CONFDIR/rdifflog-$(date +%F_%R)"
OLDBACKUP="3W"
INCLUDELIST="$CONFDIR/include-filelist"
LOCKFILE="/tmp/rbackup.lock"

# Notification variables
HOSTNAME=`hostname`
MAILHOST=""
MAILADDR=""
MAILFROM=""

# Some initial checking

if [ $EUID != "0" ]; then
	echo "This requires root privileges."
	exit 1
fi

if [ ! -x $RDIFF ];
then
    echo "No rdiff-backup installed in $RDIFF."
    exit 1
fi

if [ ! -x /usr/bin/nail ]; then
    echo "This script requires heirloom-mailx installed for mail proccesing."
    exit 1
fi

if [ ! -f $INCLUDELIST ];
then
    echo "No include file found, exiting."
    exit 1
fi

# Backup related functions

run_backup() {
    check_lockfile
    touch $LOCKFILE
    $RDIFF --print-statistics --preserve-numerical-ids --include-globbing-filelist $INCLUDELIST $SOURCE $DEST >> $LOGFILE 2>&1
    RETCODE="$?"
    if [ ! "$RETCODE" == "0" ]; then 
        send_mail_error
        exit 1
    fi
    parse_log
    send_mail
    rm -f $LOCKFILE
}

remove_old() {
    check_lockfile
    touch $LOCKFILE 
    $RDIFF --remove-older-than $OLDBACKUP --force $DEST >> $LOGFILE 2>&1
    send_mail_rotate
    rm -f $LOCKFILE
}

# Stats collection and notifications

parse_log() {
    ELT=`egrep -e "ElapsedTime" < $LOGFILE`
    MFS=`egrep -e "MirrorFileSize" < $LOGFILE`
    NFS=`egrep -e "NewFileSize" < $LOGFILE`
    CMS=`egrep -e "ChangedMirrorSize" < $LOGFILE`
    ERR=`egrep -e "Errors" < $LOGFILE`
    rm -f $LOGFILE

}

send_mail() {
    echo -e "\n$ELT\n$MFS\n$NFS\n$CMS\n$ERR\n" | nail -r $MAILFROM -s "Backup OK on $HOSTNAME at `date +%R`"  $MAILADDR

}

send_mail_error() {
    echo -e "rbackup died with exit code $RETCODE, attaching log file" | nail -r $MAILFROM -s "Backup ERROR on $HOSTNAME at `date +%R`" -a $LOGFILE $MAILADDR
    rm -f $LOGFILE
}

send_mail_rotate() {
    echo -e "Attached rotate log" | nail -r $MAILFROM -s "Older that  $OLDBACKUP removed on $HOSTNAME at `date +%R`" -a $LOGFILE $MAILADDR
    rm -f $LOGFILE

}

# Lockfile

check_lockfile() {
    if [ -f $LOCKFILE ]; then
        echo "Lockfile found, aborting! Maybe another rbackup process is running?" >> $LOGFILE
        RETCODE="1"
        send_mail_error
        rm -f $LOGFILE
        exit 1
    fi

}

# Command line argumet parsing

case $1 in
"run")
    run_backup
;;

"remold")
    if [ ! $2 ]; then
        remove_old
    else
        OLDBACKUP="$2"
        remove_old
    fi 
;;

"help")
    echo -e $HELP
;;

*)
    echo "Use 'rbackup help' to get usage tips"
;;
esac