1 files changed, 119 insertions, 42 deletions
diff --git a/abs/core/LinHES-system/lh_system_backup_job b/abs/core/LinHES-system/lh_system_backup_job
index 99838be..d2c80ab 100644
--- a/abs/core/LinHES-system/lh_system_backup_job
+++ b/abs/core/LinHES-system/lh_system_backup_job
@@ -2,65 +2,142 @@
 #process that uses this system backup script
 # - myth_mtc.py
 # - supplemental web, process.py  backup
-#
-echo "#########################################################"
-echo "Starting  backup	"
 MYTH_RUN_STATUS="1"
 . /etc/profile
 . /etc/systemconfig
 BACKUPDIR=/data/storage/disk0/backup/system_backups
 DELETE_DAYS=21
 DATE=`date +%F_%H-%M`
-mkdir -p $BACKUPDIR/$DATE
+#
+function backup(){
+    echo "#########################################################"
+    echo "Starting  backup	"
+    mkdir -p $BACKUPDIR/$DATE
 
-#backup db
-pacman -Q mysql 2>/dev/null
-if [ $? = 0 ]
-then
-	mysqldump mythconverg > $BACKUPDIR/$DATE/mythconverg
-	mysqldump ncid > $BACKUPDIR/$DATE/ncid
-fi
+    #backup db
+    pacman -Q mysql 2>/dev/null
+    if [ $? = 0 ]
+    then
+        mysqldump mythconverg > $BACKUPDIR/$DATE/mythconverg
+        mysqldump ncid > $BACKUPDIR/$DATE/ncid
+    fi
 
-#backup etc
-cp -rp /etc  $BACKUPDIR/$DATE/etc
-cp -rp /var/lib/oss $BACKUPDIR/$DATE/oss
+    #backup etc
+    cp -rp /etc  $BACKUPDIR/$DATE/etc
+    cp -rp /var/lib/oss $BACKUPDIR/$DATE/oss
 
-#backup func keys
-cp -rp /data/srv/func $BACKUPDIR/$DATA/func
+    #backup func keys
+    cp -rp /etc/pki $BACKUPDIR/$DATA/pki
 
-#make_zip file
-cd $BACKUPDIR
-tar -zcvf $BACKUPDIR/backup.$DATE.tgz  $DATE
 
-if [ -d $BACKUPDIR/$DATE ]
-then
-	rm -rf $BACKUPDIR/$DATE
-fi
+    #make_zip file
+    cd $BACKUPDIR
+    tar -zcvf $BACKUPDIR/backup.$DATE.tgz  $DATE
 
-if [ -f /home/xymon/server/ext/hbnotes.py ]
-then
-	/home/xymon/server/ext/hbnotes.py
-	chown nobody:nobody /data/srv/httpd/htdocs/hobbit/notes/* 2> /dev/null >/dev/null
-fi
+    if [ -d $BACKUPDIR/$DATE ]
+    then
+        rm -rf $BACKUPDIR/$DATE
+    fi
 
-#remove old backups
-find $BACKUPDIR/backup*.tgz -type f -mtime +$DELETE_DAYS -delete
+    if [ -f /home/xymon/server/ext/hbnotes.py ]
+    then
+        /home/xymon/server/ext/hbnotes.py
+        chown nobody:nobody /data/srv/httpd/htdocs/hobbit/notes/* 2> /dev/null >/dev/null
+    fi
+    echo "#########################################################"
+}
 
-#Remote copy
-if [ x$RemoteBackup = x1 ]
-then
-    localRemoteCheck=`echo $RemoteBackupDir | cut -d: -f1`
-    if [  x$localRemoteCheck =  xdir ]
+function remove_old_backups(){
+    #remove old backups
+    find $BACKUPDIR/backup*.tgz -type f -mtime +$DELETE_DAYS -delete
+}
+
+
+function remote_backup(){
+    #Remote copy
+    if [ x$RemoteBackup = x1 ]
+    then
+        localRemoteCheck=`echo $RemoteBackupDir | cut -d: -f1`
+        if [  x$localRemoteCheck =  xdir ]
+        then
+            localRemotedir=`echo $RemoteBackupDir | cut -d: -f2`
+            echo "copying $BACKUPDIR/backup.$DATE.tgz to  $localRemotedir  "
+            cp $BACKUPDIR/backup.$DATE.tgz  $localRemotedir
+        else
+            /usr/bin/func  ${RemoteBackupDir}  ping| grep -q "FAILED"
+            rc=$?
+            if [ $rc = 0 ]
+            then
+                #this is here to mark failed copy of the backup.
+                #There is a cron.hourly job that will attempt to retransfer the file
+                echo "Remote backup failed to ${RemoteBackupDir}"
+                echo backup.$DATE.tgz >> $BACKUPDIR/remote_backup_failed.txt
+            else
+                echo "copying $BACKUPDIR/backup.$DATE.tgz to  ${RemoteBackupDir}:$BACKUPDIR/MBE_$DATE.tgz"
+                /usr/bin/func  ${RemoteBackupDir} copyfile  -f  $BACKUPDIR/backup.$DATE.tgz  --remotepath $BACKUPDIR/MBE_$DATE.tgz
+            fi
+        fi
+    fi
+}
+
+function remote_transfer(){
+    transfer_file=${1}
+    echo $transfer_file
+    /usr/bin/func  ${RemoteBackupDir}  ping| grep -q "FAILED"
+    rc=$?
+    if [ $rc = 0 ]
     then
-        localRemotedir=`echo $RemoteBackupDir | cut -d: -f2`
-        cmd="cp $BACKUPDIR/backup.$DATE.tgz  $localRemotedir"
-        echo "copying$BACKUPDIR/backup.$DATE.tgz to  $localRemotedir  "
+        #this is here to mark failed copy of the backup.
+        #There is a cron.hourly job that will attempt to retransfer the file
+        echo "Remote backup failed to ${RemoteBackupDir}"
+        echo $transfer_file >> $BACKUPDIR/remote_backup_failed.txt
     else
-	/usr/bin/func  ${RemoteBackupDir} copyfile  -f  $BACKUPDIR/backup.$DATE.tgz  --remotepath $BACKUPDIR/backup.$DATE.tgz
+        echo "copying $BACKUPDIR/$transfer_file to  ${RemoteBackupDir}:$BACKUPDIR/MBE_$transfer_file"
+        /usr/bin/func  ${RemoteBackupDir} copyfile  -f  $BACKUPDIR/$transfer_file  --remotepath $BACKUPDIR/MBE_$transfer_file
+    fi
+}
 
-        echo "copying$BACKUPDIR/backup.$DATE.tgz to  ${RemoteBackupDir}:/data/database_backup/"
+function add_link(){
+    if [ -f $BACKUPDIR/remote_backup_failed.txt ]
+    then
+        RETRYFILE="/etc/cron.hourly/lh_backup_retry.sh"
+        echo "#!/bin/bash" > $RETRYFILE
+        echo "#This file was autogenerated and removed by lh_system_backup_job" >> $RETRYFILE
+        echo "MYTH_RUN_STATUS=1">> $RETRYFILE
+        echo ". /etc/profile">> $RETRYFILE
+        echo "lh_system_backup_job retry">> $RETRYFILE
+        chmod 755 $RETRYFILE
     fi
-    $cmd
+}
+
+function remove_link(){
+    RETRYFILE="/etc/cron.hourly/lh_backup_retry.sh"
+    if [ ! -f $BACKUPDIR/remote_backup_failed.txt ]
+    then
+        rm -f $RETRYFILE
+    fi
+}
+#------------------------------------
+if [ "x$1" = "x" ]
+then
+    remove_old_backups
+    backup
+    remote_backup
+    add_link
+else
+    #this is where we attempt to transfer again as part of the cronjob
+    if [ -f $BACKUPDIR/remote_backup_failed.txt ]
+    then
+        mv -f $BACKUPDIR/remote_backup_failed.txt /tmp
+        while read line
+        do
+            echo $line
+            remote_transfer $line
+        done < /tmp/remote_backup_failed.txt
+        rm -f /tmp/remote_backup_failed.txt
+    fi
+    remove_link
 fi
 
-echo "#########################################################"
+
+