LinHES-system: lh_system_backup_job. reworked the system backup job to support detection of failed remote copy.

If a failed remote backup copy occurs, it will write the filename to a file and then setup a cron job to attempt to retransfer the file every hour until success.   This is mostly needed in case func is down, or is the remote system is powered off when the first attemp is made.

refs #885

1 files changed, 119 insertions, 42 deletions

diff --git a/abs/core/LinHES-system/lh_system_backup_job b/abs/core/LinHES-system/lh_system_backup_job
index 99838be..d2c80ab 100644
--- a/abs/core/LinHES-system/lh_system_backup_job
+++ b/abs/core/LinHES-system/lh_system_backup_job
@@ -2,65 +2,142 @@
 #process that uses this system backup script
 # - myth_mtc.py
 # - supplemental web, process.py  backup
-#
-echo "#########################################################"
-echo "Starting  backup	"
 MYTH_RUN_STATUS="1"
 . /etc/profile
 . /etc/systemconfig
 BACKUPDIR=/data/storage/disk0/backup/system_backups
 DELETE_DAYS=21
 DATE=`date +%F_%H-%M`
-mkdir -p $BACKUPDIR/$DATE
+#
+function backup(){
+    echo "#########################################################"
+    echo "Starting  backup	"
+    mkdir -p $BACKUPDIR/$DATE
 
-#backup db
-pacman -Q mysql 2>/dev/null
-if [ $? = 0 ]
-then
-	mysqldump mythconverg > $BACKUPDIR/$DATE/mythconverg
-	mysqldump ncid > $BACKUPDIR/$DATE/ncid
-fi
+    #backup db
+    pacman -Q mysql 2>/dev/null
+    if [ $? = 0 ]
+    then
+        mysqldump mythconverg > $BACKUPDIR/$DATE/mythconverg
+        mysqldump ncid > $BACKUPDIR/$DATE/ncid
+    fi
 
-#backup etc
-cp -rp /etc  $BACKUPDIR/$DATE/etc
-cp -rp /var/lib/oss $BACKUPDIR/$DATE/oss
+    #backup etc
+    cp -rp /etc  $BACKUPDIR/$DATE/etc
+    cp -rp /var/lib/oss $BACKUPDIR/$DATE/oss
 
-#backup func keys
-cp -rp /data/srv/func $BACKUPDIR/$DATA/func
+    #backup func keys
+    cp -rp /etc/pki $BACKUPDIR/$DATA/pki
 
-#make_zip file
-cd $BACKUPDIR
-tar -zcvf $BACKUPDIR/backup.$DATE.tgz  $DATE
 
-if [ -d $BACKUPDIR/$DATE ]
-then
-	rm -rf $BACKUPDIR/$DATE
-fi
+    #make_zip file
+    cd $BACKUPDIR
+    tar -zcvf $BACKUPDIR/backup.$DATE.tgz  $DATE
 
-if [ -f /home/xymon/server/ext/hbnotes.py ]
-then
-	/home/xymon/server/ext/hbnotes.py
-	chown nobody:nobody /data/srv/httpd/htdocs/hobbit/notes/* 2> /dev/null >/dev/null
-fi
+    if [ -d $BACKUPDIR/$DATE ]
+    then
+        rm -rf $BACKUPDIR/$DATE
+    fi
 
-#remove old backups
-find $BACKUPDIR/backup*.tgz -type f -mtime +$DELETE_DAYS -delete
+    if [ -f /home/xymon/server/ext/hbnotes.py ]
+    then
+        /home/xymon/server/ext/hbnotes.py
+        chown nobody:nobody /data/srv/httpd/htdocs/hobbit/notes/* 2> /dev/null >/dev/null
+    fi
+    echo "#########################################################"
+}
 
-#Remote copy
-if [ x$RemoteBackup = x1 ]
-then
-    localRemoteCheck=`echo $RemoteBackupDir | cut -d: -f1`
-    if [  x$localRemoteCheck =  xdir ]
+function remove_old_backups(){
+    #remove old backups
+    find $BACKUPDIR/backup*.tgz -type f -mtime +$DELETE_DAYS -delete
+}
+
+
+function remote_backup(){
+    #Remote copy
+    if [ x$RemoteBackup = x1 ]
+    then
+        localRemoteCheck=`echo $RemoteBackupDir | cut -d: -f1`
+        if [  x$localRemoteCheck =  xdir ]
+        then
+            localRemotedir=`echo $RemoteBackupDir | cut -d: -f2`
+            echo "copying $BACKUPDIR/backup.$DATE.tgz to  $localRemotedir  "
+            cp $BACKUPDIR/backup.$DATE.tgz  $localRemotedir
+        else
+            /usr/bin/func  ${RemoteBackupDir}  ping| grep -q "FAILED"
+            rc=$?
+            if [ $rc = 0 ]
+            then
+                #this is here to mark failed copy of the backup.
+                #There is a cron.hourly job that will attempt to retransfer the file
+                echo "Remote backup failed to ${RemoteBackupDir}"
+                echo backup.$DATE.tgz >> $BACKUPDIR/remote_backup_failed.txt
+            else
+                echo "copying $BACKUPDIR/backup.$DATE.tgz to  ${RemoteBackupDir}:$BACKUPDIR/MBE_$DATE.tgz"
+                /usr/bin/func  ${RemoteBackupDir} copyfile  -f  $BACKUPDIR/backup.$DATE.tgz  --remotepath $BACKUPDIR/MBE_$DATE.tgz
+            fi
+        fi
+    fi
+}
+
+function remote_transfer(){
+    transfer_file=${1}
+    echo $transfer_file
+    /usr/bin/func  ${RemoteBackupDir}  ping| grep -q "FAILED"
+    rc=$?
+    if [ $rc = 0 ]
     then
-        localRemotedir=`echo $RemoteBackupDir | cut -d: -f2`
-        cmd="cp $BACKUPDIR/backup.$DATE.tgz  $localRemotedir"
-        echo "copying$BACKUPDIR/backup.$DATE.tgz to  $localRemotedir  "
+        #this is here to mark failed copy of the backup.
+        #There is a cron.hourly job that will attempt to retransfer the file
+        echo "Remote backup failed to ${RemoteBackupDir}"
+        echo $transfer_file >> $BACKUPDIR/remote_backup_failed.txt
     else
-	/usr/bin/func  ${RemoteBackupDir} copyfile  -f  $BACKUPDIR/backup.$DATE.tgz  --remotepath $BACKUPDIR/backup.$DATE.tgz
+        echo "copying $BACKUPDIR/$transfer_file to  ${RemoteBackupDir}:$BACKUPDIR/MBE_$transfer_file"
+        /usr/bin/func  ${RemoteBackupDir} copyfile  -f  $BACKUPDIR/$transfer_file  --remotepath $BACKUPDIR/MBE_$transfer_file
+    fi
+}
 
-        echo "copying$BACKUPDIR/backup.$DATE.tgz to  ${RemoteBackupDir}:/data/database_backup/"
+function add_link(){
+    if [ -f $BACKUPDIR/remote_backup_failed.txt ]
+    then
+        RETRYFILE="/etc/cron.hourly/lh_backup_retry.sh"
+        echo "#!/bin/bash" > $RETRYFILE
+        echo "#This file was autogenerated and removed by lh_system_backup_job" >> $RETRYFILE
+        echo "MYTH_RUN_STATUS=1">> $RETRYFILE
+        echo ". /etc/profile">> $RETRYFILE
+        echo "lh_system_backup_job retry">> $RETRYFILE
+        chmod 755 $RETRYFILE
     fi
-    $cmd
+}
+
+function remove_link(){
+    RETRYFILE="/etc/cron.hourly/lh_backup_retry.sh"
+    if [ ! -f $BACKUPDIR/remote_backup_failed.txt ]
+    then
+        rm -f $RETRYFILE
+    fi
+}
+#------------------------------------
+if [ "x$1" = "x" ]
+then
+    remove_old_backups
+    backup
+    remote_backup
+    add_link
+else
+    #this is where we attempt to transfer again as part of the cronjob
+    if [ -f $BACKUPDIR/remote_backup_failed.txt ]
+    then
+        mv -f $BACKUPDIR/remote_backup_failed.txt /tmp
+        while read line
+        do
+            echo $line
+            remote_transfer $line
+        done < /tmp/remote_backup_failed.txt
+        rm -f /tmp/remote_backup_failed.txt
+    fi
+    remove_link
 fi
 
-echo "#########################################################"
+
+