From d5cf6335976bdb5f13687494871cb96703ed7693 Mon Sep 17 00:00:00 2001 From: James Meyer Date: Mon, 17 Dec 2012 14:15:41 -0600 Subject: LinHES-system: lh_system_backup_job. reworked the system backup job to support detection of failed remote copy. If a failed remote backup copy occurs, it will write the filename to a file and then setup a cron job to attempt to retransfer the file every hour until success. This is mostly needed in case func is down, or is the remote system is powered off when the first attemp is made. refs #885 --- abs/core/LinHES-system/PKGBUILD | 8 +- abs/core/LinHES-system/lh_system_backup_job | 161 ++++++++++++++++++++-------- 2 files changed, 123 insertions(+), 46 deletions(-) diff --git a/abs/core/LinHES-system/PKGBUILD b/abs/core/LinHES-system/PKGBUILD index 0ebf41a..61ea2f2 100755 --- a/abs/core/LinHES-system/PKGBUILD +++ b/abs/core/LinHES-system/PKGBUILD @@ -1,12 +1,12 @@ pkgname=LinHES-system pkgver=2 -pkgrel=127 +pkgrel=129 arch=('i686' 'x86_64') MVDIR=$startdir/pkg/usr/LH BINDIR=$startdir/pkg/usr/bin install=system.install pkgdesc="scripts and things related to having an automated system" -depends=(linhes-sounds xdotool tilda keylaunch python_aosd 'linhes-scripts>=7-24' 'udisks' 'LinHES-config>=2.3-59' 'sudo' 'python2-dbus' 'unclutter' 'wmctrl' 'archlinux-xdg-menu') +depends=(linhes-sounds xdotool tilda keylaunch python_aosd 'linhes-scripts>=7-24' 'udisks' 'LinHES-config>=2.3-59' 'sudo' 'python2-dbus' 'unclutter' 'wmctrl' 'archlinux-xdg-menu' 'ethtool') backup=(etc/modprobe.d/alsa-base) binfiles="LinHES-start optimize_mythdb.py myth_mtc.py myth_mtc.sh LinHES-run @@ -94,11 +94,11 @@ md5sums=('d35912d6b6b267bc6616205fa4776833' '309638acf596ce3c790aa2bf5c7c9880' '962a3e9eaba2d1466251b7ab0956705d' '1758aed160de64abfafb28a3a8f3390e' - '5510042c80283522f480e95d6242495c' + '2200f08381370f6ae553e13a9f376377' 'ceec78815ed01af733cdbca14cb0942b' '1be1d3dfc83f828eede93055713882ae' '47e093e8cfe4b5b96602358e1f540832' - 'fd7dbbf960080619ef18d6843f6374d8' + 'a85c19902f2af90931e05c839c63b62d' 'b9b0a8370f46f0b5e144a66897e8eefb' '6519d209e151cdc39ae7e1091d189c3e' '8f474e019d5fcb775497aca355d61b0b' diff --git a/abs/core/LinHES-system/lh_system_backup_job b/abs/core/LinHES-system/lh_system_backup_job index 99838be..d2c80ab 100644 --- a/abs/core/LinHES-system/lh_system_backup_job +++ b/abs/core/LinHES-system/lh_system_backup_job @@ -2,65 +2,142 @@ #process that uses this system backup script # - myth_mtc.py # - supplemental web, process.py backup -# -echo "#########################################################" -echo "Starting backup " MYTH_RUN_STATUS="1" . /etc/profile . /etc/systemconfig BACKUPDIR=/data/storage/disk0/backup/system_backups DELETE_DAYS=21 DATE=`date +%F_%H-%M` -mkdir -p $BACKUPDIR/$DATE +# +function backup(){ + echo "#########################################################" + echo "Starting backup " + mkdir -p $BACKUPDIR/$DATE -#backup db -pacman -Q mysql 2>/dev/null -if [ $? = 0 ] -then - mysqldump mythconverg > $BACKUPDIR/$DATE/mythconverg - mysqldump ncid > $BACKUPDIR/$DATE/ncid -fi + #backup db + pacman -Q mysql 2>/dev/null + if [ $? = 0 ] + then + mysqldump mythconverg > $BACKUPDIR/$DATE/mythconverg + mysqldump ncid > $BACKUPDIR/$DATE/ncid + fi -#backup etc -cp -rp /etc $BACKUPDIR/$DATE/etc -cp -rp /var/lib/oss $BACKUPDIR/$DATE/oss + #backup etc + cp -rp /etc $BACKUPDIR/$DATE/etc + cp -rp /var/lib/oss $BACKUPDIR/$DATE/oss -#backup func keys -cp -rp /data/srv/func $BACKUPDIR/$DATA/func + #backup func keys + cp -rp /etc/pki $BACKUPDIR/$DATA/pki -#make_zip file -cd $BACKUPDIR -tar -zcvf $BACKUPDIR/backup.$DATE.tgz $DATE -if [ -d $BACKUPDIR/$DATE ] -then - rm -rf $BACKUPDIR/$DATE -fi + #make_zip file + cd $BACKUPDIR + tar -zcvf $BACKUPDIR/backup.$DATE.tgz $DATE -if [ -f /home/xymon/server/ext/hbnotes.py ] -then - /home/xymon/server/ext/hbnotes.py - chown nobody:nobody /data/srv/httpd/htdocs/hobbit/notes/* 2> /dev/null >/dev/null -fi + if [ -d $BACKUPDIR/$DATE ] + then + rm -rf $BACKUPDIR/$DATE + fi -#remove old backups -find $BACKUPDIR/backup*.tgz -type f -mtime +$DELETE_DAYS -delete + if [ -f /home/xymon/server/ext/hbnotes.py ] + then + /home/xymon/server/ext/hbnotes.py + chown nobody:nobody /data/srv/httpd/htdocs/hobbit/notes/* 2> /dev/null >/dev/null + fi + echo "#########################################################" +} -#Remote copy -if [ x$RemoteBackup = x1 ] -then - localRemoteCheck=`echo $RemoteBackupDir | cut -d: -f1` - if [ x$localRemoteCheck = xdir ] +function remove_old_backups(){ + #remove old backups + find $BACKUPDIR/backup*.tgz -type f -mtime +$DELETE_DAYS -delete +} + + +function remote_backup(){ + #Remote copy + if [ x$RemoteBackup = x1 ] + then + localRemoteCheck=`echo $RemoteBackupDir | cut -d: -f1` + if [ x$localRemoteCheck = xdir ] + then + localRemotedir=`echo $RemoteBackupDir | cut -d: -f2` + echo "copying $BACKUPDIR/backup.$DATE.tgz to $localRemotedir " + cp $BACKUPDIR/backup.$DATE.tgz $localRemotedir + else + /usr/bin/func ${RemoteBackupDir} ping| grep -q "FAILED" + rc=$? + if [ $rc = 0 ] + then + #this is here to mark failed copy of the backup. + #There is a cron.hourly job that will attempt to retransfer the file + echo "Remote backup failed to ${RemoteBackupDir}" + echo backup.$DATE.tgz >> $BACKUPDIR/remote_backup_failed.txt + else + echo "copying $BACKUPDIR/backup.$DATE.tgz to ${RemoteBackupDir}:$BACKUPDIR/MBE_$DATE.tgz" + /usr/bin/func ${RemoteBackupDir} copyfile -f $BACKUPDIR/backup.$DATE.tgz --remotepath $BACKUPDIR/MBE_$DATE.tgz + fi + fi + fi +} + +function remote_transfer(){ + transfer_file=${1} + echo $transfer_file + /usr/bin/func ${RemoteBackupDir} ping| grep -q "FAILED" + rc=$? + if [ $rc = 0 ] then - localRemotedir=`echo $RemoteBackupDir | cut -d: -f2` - cmd="cp $BACKUPDIR/backup.$DATE.tgz $localRemotedir" - echo "copying$BACKUPDIR/backup.$DATE.tgz to $localRemotedir " + #this is here to mark failed copy of the backup. + #There is a cron.hourly job that will attempt to retransfer the file + echo "Remote backup failed to ${RemoteBackupDir}" + echo $transfer_file >> $BACKUPDIR/remote_backup_failed.txt else - /usr/bin/func ${RemoteBackupDir} copyfile -f $BACKUPDIR/backup.$DATE.tgz --remotepath $BACKUPDIR/backup.$DATE.tgz + echo "copying $BACKUPDIR/$transfer_file to ${RemoteBackupDir}:$BACKUPDIR/MBE_$transfer_file" + /usr/bin/func ${RemoteBackupDir} copyfile -f $BACKUPDIR/$transfer_file --remotepath $BACKUPDIR/MBE_$transfer_file + fi +} - echo "copying$BACKUPDIR/backup.$DATE.tgz to ${RemoteBackupDir}:/data/database_backup/" +function add_link(){ + if [ -f $BACKUPDIR/remote_backup_failed.txt ] + then + RETRYFILE="/etc/cron.hourly/lh_backup_retry.sh" + echo "#!/bin/bash" > $RETRYFILE + echo "#This file was autogenerated and removed by lh_system_backup_job" >> $RETRYFILE + echo "MYTH_RUN_STATUS=1">> $RETRYFILE + echo ". /etc/profile">> $RETRYFILE + echo "lh_system_backup_job retry">> $RETRYFILE + chmod 755 $RETRYFILE fi - $cmd +} + +function remove_link(){ + RETRYFILE="/etc/cron.hourly/lh_backup_retry.sh" + if [ ! -f $BACKUPDIR/remote_backup_failed.txt ] + then + rm -f $RETRYFILE + fi +} +#------------------------------------ +if [ "x$1" = "x" ] +then + remove_old_backups + backup + remote_backup + add_link +else + #this is where we attempt to transfer again as part of the cronjob + if [ -f $BACKUPDIR/remote_backup_failed.txt ] + then + mv -f $BACKUPDIR/remote_backup_failed.txt /tmp + while read line + do + echo $line + remote_transfer $line + done < /tmp/remote_backup_failed.txt + rm -f /tmp/remote_backup_failed.txt + fi + remove_link fi -echo "#########################################################" + + -- cgit v0.12