From bd3cd46cfcbc9998150faefb71027957a5304e28 Mon Sep 17 00:00:00 2001 From: Britney Fransen Date: Wed, 16 Dec 2015 15:29:20 +0000 Subject: LinHES-system: add empty_storage_groups.py empty_storage_groups.py is a utility that will move files from the selected storage group dir to an available least full storage group dir. renamed balance_storage.py to balance_storage_groups.py. --- abs/core/LinHES-system/PKGBUILD | 8 +- abs/core/LinHES-system/balance_storage.py | 147 ----------------- abs/core/LinHES-system/balance_storage_groups.py | 153 ++++++++++++++++++ abs/core/LinHES-system/empty_storage_groups.py | 195 +++++++++++++++++++++++ 4 files changed, 353 insertions(+), 150 deletions(-) delete mode 100755 abs/core/LinHES-system/balance_storage.py create mode 100755 abs/core/LinHES-system/balance_storage_groups.py create mode 100755 abs/core/LinHES-system/empty_storage_groups.py diff --git a/abs/core/LinHES-system/PKGBUILD b/abs/core/LinHES-system/PKGBUILD index 18f1559..c700861 100755 --- a/abs/core/LinHES-system/PKGBUILD +++ b/abs/core/LinHES-system/PKGBUILD @@ -1,6 +1,6 @@ pkgname=LinHES-system pkgver=8.3 -pkgrel=21 +pkgrel=22 arch=('i686' 'x86_64') install=system.install pkgdesc="Everything that makes LinHES an automated system" @@ -17,7 +17,8 @@ binfiles="LinHES-start optimize_mythdb.py myth_mtc.py install_supplemental_service.sh get_airplay_key importfiles.sh lh_system_backup lh_system_backup_job lh_system_restore_job lh_system_host_update lh_system_all_host_update - add_storage.py balance_storage.py diskspace.sh cacheclean lh_backend_control.sh + add_storage.py balance_storage_groups.py empty_storage_groups.py + diskspace.sh cacheclean lh_backend_control.sh create_media_dirs.sh msg_client.py msg_daemon.py mythvidexport.py gen_is_xml.py gen_lib_xml.py gen_light_include.py gen_game_xml.py misc_recent_recordings.pl misc_status_config.py misc_status_info.sh @@ -98,7 +99,8 @@ md5sums=('76842f8cff548d2c1289e0992a8b84dd' '74e17d6f7453c52d56fecaed5c3f6ad5' '47e093e8cfe4b5b96602358e1f540832' '63bbc62240f46a9997eaae4a84b09b76' - 'd67a034fa1d8f253f42982f27a8d88ff' + '8ed2474fa52775769255cfb0dba00a1a' + '7753c7ce3f0db5944b7372f5d19bb374' '2c005d95312018bef80092136f80f254' 'c8db6a83ecc089ea37ab7fcb0f7a01cf' 'ca63946920ba33de1f15abda83e74e40' diff --git a/abs/core/LinHES-system/balance_storage.py b/abs/core/LinHES-system/balance_storage.py deleted file mode 100755 index 5556fd0..0000000 --- a/abs/core/LinHES-system/balance_storage.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/python2 - -import argparse, glob, operator, os, random, shutil, subprocess, sys, signal -shouldQuit = False - -def getFreeSpaceForDir(dir): - stats = os.statvfs(dir) - return (stats.f_bavail * stats.f_frsize) - -def getFreePercentForDir(dir): - stats = os.statvfs(dir) - total = (stats.f_blocks) - avail = (stats.f_bavail) - return (total - avail) / float(total) - -def getFileSize(fullPath): - return os.path.getsize(fullPath) - -def sizeof_fmt(num, suffix='B'): - for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']: - if abs(num) < 1024.0: - return "%3.1f %s%s" % (num, unit, suffix) - num /= 1024.0 - return "%.1f %s%s" % (num, 'Yi', suffix) - -def signal_handler(signal, frame): - print "\nWill quit when file has been moved.\nMoving File..." - global shouldQuit - shouldQuit = True - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('-c', '--checkonly', action='store_true', help="Check only, don't move any files.") - parser.add_argument('-p', '--percent', type=int, default=7, help="The percentage difference between the most full dir and least full dir that will stop balancing.") - cmdargs = parser.parse_args() - - SGDIRS = [] - SGgrp = "Default" - - signal.signal(signal.SIGINT, signal_handler) - - print "\nBalance MythTV Storage Group Directories\nPress Ctrl+C to quit" - - # Get Storage Groups from MythDB - try: - from MythTV import MythDB - mythDB = MythDB() - records = mythDB.getStorageGroup() - except: - print "Couldn't connect to MythTV database." - sys.exit(1) - - # Get Storage Group directories - for record in records: - if record.groupname == SGgrp: - dirname = record.dirname - SGDIRS.append(dirname) - - # If there are less than 2 directories defined bail as we can't move anything - if len(SGDIRS) < 2: - print "There are less than 2 directories defined. Exiting." - sys.exit(0) - - while not shouldQuit: - SGDIRSdata = [] - print "\n------------------------------------------------" - print "'" + SGgrp + "' Storage Group Directories - Percent Used:" - # Get percent free and size free - for directory in SGDIRS: - # Check if SG path exists - if not os.path.exists(directory): - print " " + directory + " - Not Mounted" - continue - freePcent = getFreePercentForDir(directory) - freeSize = getFreeSpaceForDir(directory) - SGDIRSdata.append([directory, freePcent, freeSize]) - print " " + directory + " - " + str(freePcent * 100) + "%" - - # Sort data on percent free - SGDIRSdata = sorted(SGDIRSdata, reverse=True, key=operator.itemgetter(1)) - #print SGDIRSdata - - # Check if SG has any mpg or nuv files - i=0 - for dir in SGDIRSdata: - mostFull = SGDIRSdata[i] - i=i+1 - if len(glob.glob1(mostFull[0],"*.mpg")) or len(glob.glob1(mostFull[0],"*.nuv")): - break - else: - if i == 1: - print "------------------------------------------------" - print " " + mostFull[0] + " - NO files to move" - - leastFull = SGDIRSdata[-1] - - print "------------------------------------------------" - print "Most Used Storage Group Directory with files to move: " - print " " + str(mostFull[0]) + " - " + str(mostFull[1] * 100) + "%" - print "Least Used Storage Group Directory: " - print " " + str(leastFull[0]) + " - " + str(leastFull[1] * 100) + "%" - - # Check if mostFull and leastFull are within the percent var of each other - if mostFull[1] - (float(cmdargs.percent) / 100) < leastFull[1]: - print "\nThe most used and least used storage group directories are\nwithin " + str(cmdargs.percent) + "% used of each other. No files will be moved." - sys.exit() - - # Get random file from most used dir - fileToMove = random.choice([f for f in os.listdir(mostFull[0]) if f.endswith(".mpg") or f.endswith(".nuv")]) - fileToMove = mostFull[0] + "/" + fileToMove - - # Check that the file isn't too big for least used dir - fileSize = getFileSize(fileToMove) - if (fileSize > getFreeSpaceForDir(leastFull[0])): - # Too big to move - print fileToMove + " is too big to move to " + leastFull[0] - sys.exit() - - print "------------------------------------------------" - print "Move File:" - print " " + fileToMove - print " Size: " + sizeof_fmt(os.path.getsize(fileToMove)) - print "To:" - print " " + leastFull[0] - print " Available: " + sizeof_fmt(getFreeSpaceForDir(leastFull[0])) - - # Move file - if cmdargs.checkonly: - print "------------------------------------------------" - print"Check Only option was used. No files were moved." - shouldQuit = True - else: - print "------------------------------------------------" - print "Checking System Status..." - if subprocess.call(["/usr/bin/python2", "/usr/LH/bin/idle.py", "-s"]): - print " System is busy. The file will not be moved." - sys.exit() - print "Moving File..." - shutil.move(fileToMove, leastFull[0]) - - # Remove png files - print "------------------------------------------------" - print "Removing png Files:" - pngFiles = glob.glob(fileToMove + "*.png") - for p in pngFiles: - os.remove(p) - print " " + p diff --git a/abs/core/LinHES-system/balance_storage_groups.py b/abs/core/LinHES-system/balance_storage_groups.py new file mode 100755 index 0000000..e088cb5 --- /dev/null +++ b/abs/core/LinHES-system/balance_storage_groups.py @@ -0,0 +1,153 @@ +#!/usr/bin/python2 + +import argparse, glob, operator, os, random, shutil, subprocess, sys, signal +shouldQuit = False + +def getFreeSpaceForDir(dir): + stats = os.statvfs(dir) + return (stats.f_bavail * stats.f_frsize) + +def getFreePercentForDir(dir): + stats = os.statvfs(dir) + total = (stats.f_blocks) + avail = (stats.f_bavail) + return (total - avail) / float(total) + +def getFileSize(fullPath): + return os.path.getsize(fullPath) + +def sizeof_fmt(num, suffix='B'): + for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']: + if abs(num) < 1024.0: + return "%3.1f %s%s" % (num, unit, suffix) + num /= 1024.0 + return "%.1f %s%s" % (num, 'Yi', suffix) + +def signal_handler(signal, frame): + print "\nWill quit when file has been moved.\nMoving File..." + global shouldQuit + shouldQuit = True + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--checkonly', action='store_true', help="Check only, don't move any files.") + parser.add_argument('-p', '--percent', type=int, default=7, help="The percentage difference between the most full dir and least full dir that will stop balancing.") + cmdargs = parser.parse_args() + + SGDIRS = [] + SGgrp = "Default" + + signal.signal(signal.SIGINT, signal_handler) + + print "\nBalance MythTV Storage Group Directories\nPress Ctrl+C to quit" + + # Get Storage Groups from MythDB + try: + from MythTV import MythDB + mythDB = MythDB() + records = mythDB.getStorageGroup() + except: + print "Couldn't connect to MythTV database." + sys.exit(1) + + # Get Storage Group directories + for record in records: + if record.groupname == SGgrp: + dirname = record.dirname + SGDIRS.append(dirname) + + # If there are less than 2 directories defined bail as we can't move anything + if len(SGDIRS) < 2: + print "There are less than 2 directories defined. Exiting." + sys.exit(0) + + while not shouldQuit: + SGDIRSdata = [] + print "\n------------------------------------------------" + print "'" + SGgrp + "' Storage Group Directories - Percent Used:" + # Get percent free and size free + for directory in SGDIRS: + # Check if SG path exists + if not os.path.exists(directory): + print " " + directory + " - Not Mounted" + continue + freePcent = getFreePercentForDir(directory) + freeSize = getFreeSpaceForDir(directory) + SGDIRSdata.append([directory, freePcent, freeSize]) + print " %s - %.2f%%" % (directory, freePcent * 100) + + # Sort data on percent free + SGDIRSdata = sorted(SGDIRSdata, reverse=True, key=operator.itemgetter(1)) + #print SGDIRSdata + + # Check if SG has any mpg or nuv files + i=0 + for dir in SGDIRSdata: + mostFull = SGDIRSdata[i] + i=i+1 + if len(glob.glob1(mostFull[0],"*.mpg")) or len(glob.glob1(mostFull[0],"*.nuv")): + break + else: + if i == 1: + print "------------------------------------------------" + print " " + mostFull[0] + " - NO files to move" + + leastFull = SGDIRSdata[-1] + + print "------------------------------------------------" + print "Most Used Storage Group Directory with files to move: " + print " %s - %.2f%%" % (mostFull[0], mostFull[1] * 100) + print "Least Used Storage Group Directory: " + print " %s - %.2f%%" % (leastFull[0], leastFull[1] * 100) + + # Check if mostFull and leastFull are within the percent var of each other + if mostFull[1] - (float(cmdargs.percent) / 100) < leastFull[1]: + print "\nThe most used and least used storage group directories are\nwithin " + str(cmdargs.percent) + "% used of each other. No files will be moved." + sys.exit() + + # Get random file from most used dir + fileToMove = random.choice([f for f in os.listdir(mostFull[0]) if f.endswith(".mpg") or f.endswith(".nuv")]) + filePathToMove = mostFull[0] + "/" + fileToMove + + # Check that the file isn't too big for least used dir + fileSize = getFileSize(filePathToMove) + if (fileSize > getFreeSpaceForDir(leastFull[0])): + # Too big to move + print filePathToMove + " is too big to move to " + leastFull[0] + sys.exit() + + print "------------------------------------------------" + print "Move File:" + print " " + filePathToMove + print " Size: " + sizeof_fmt(os.path.getsize(filePathToMove)) + print "To:" + print " " + leastFull[0] + print " Available: " + sizeof_fmt(getFreeSpaceForDir(leastFull[0])) + + # Move file + if cmdargs.checkonly: + print "------------------------------------------------" + print"Check Only option was used. No files were moved." + shouldQuit = True + else: + print "------------------------------------------------" + print "Checking System Status..." + if subprocess.call(["/usr/bin/python2", "/usr/LH/bin/idle.py", "-s"]): + print " System is busy. The file will not be moved." + sys.exit() + print "Moving File..." + try: + shutil.move(filePathToMove, leastFull[0]) + # eg. src and dest are the same file + except shutil.Error as e: + print('Error: %s' % e) + except IOError as e: + print('Error: %s' % e.strerror) + + # Remove png files + print "------------------------------------------------" + print "Removing png Files:" + pngFiles = glob.glob(filePathToMove + "*.png") + for p in pngFiles: + os.remove(p) + print " " + p diff --git a/abs/core/LinHES-system/empty_storage_groups.py b/abs/core/LinHES-system/empty_storage_groups.py new file mode 100755 index 0000000..a143d7d --- /dev/null +++ b/abs/core/LinHES-system/empty_storage_groups.py @@ -0,0 +1,195 @@ +#!/usr/bin/python2 + +import argparse, glob, operator, os, random, shutil, subprocess, sys, signal +shouldQuit = False + +def getFreeSpaceForDir(dir): + stats = os.statvfs(dir) + return (stats.f_bavail * stats.f_frsize) + +def getFreePercentForDir(dir): + stats = os.statvfs(dir) + total = (stats.f_blocks) + avail = (stats.f_bavail) + return (total - avail) / float(total) + +def getFileSize(fullPath): + return os.path.getsize(fullPath) + +def sizeof_fmt(num, suffix='B'): + for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']: + if abs(num) < 1024.0: + return "%3.1f %s%s" % (num, unit, suffix) + num /= 1024.0 + return "%.1f %s%s" % (num, 'Yi', suffix) + +def signal_handler(signal, frame): + print "\nWill quit when file has been moved.\nMoving File..." + global shouldQuit + shouldQuit = True + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--checkonly', action='store_true', help="Check only, don't move any files.") + cmdargs = parser.parse_args() + + SGs = [] + SGDIRS = [] + SGgrp = "Default" + SGselectdata = [] + + signal.signal(signal.SIGINT, signal_handler) + + print "\nEmpty a MythTV Storage Group Directory\nPress Ctrl+C to quit" + + # Get Storage Groups from MythDB + try: + from MythTV import MythDB + mythDB = MythDB() + records = mythDB.getStorageGroup() + recs = mythDB.getStorageGroup() + except: + print "Couldn't connect to MythTV database." + sys.exit(1) + + # get list of non duplicate SGs + for record in records: + SGs.append(record.groupname) + SGs=list(set(SGs)) + # Move Default to top of list + if "Default" in SGs: + SGs.remove("Default") + SGs.insert(0,"Default") + + # ask user which SG to use + print "\n------------------------------------------------" + print "Storage Groups:" + for i,sg in enumerate(SGs): + print str(i+1) + ": " + sg + + try: + SGselect=raw_input("\nEnter the number of the storage group to use (default 1): ") or 1 + SGselect=int(SGselect) + if SGselect > len(SGs) or SGselect < 1: + SGselect=int("e") + except ValueError: + print "You must enter a number between 1 and " + str(len(SGs)) + ". Exiting." + sys.exit(0) + + SGgrp=SGs[SGselect-1] + + # Get Storage Group directories + for record in recs: + if record.groupname == SGgrp: + dirname = record.dirname + SGDIRS.append(dirname) + + # If there are less than 2 directories defined bail as we can't move anything + if len(SGDIRS) < 2: + print "There are less than 2 directories defined. Exiting." + sys.exit(0) + + while not shouldQuit: + SGDIRSdata = [] + print "\n------------------------------------------------" + print "'" + SGgrp + "' Storage Group Directories - Percent Used:" + SGcnt=0 + # Get percent free and size free + for directory in SGDIRS: + # Check if SG path exists + if not os.path.exists(directory): + print " " + directory + " - Not Mounted" + continue + # Check if SG has data files to move + if len(glob.glob1(directory,"*.mpg")) or len(glob.glob1(directory,"*.nuv")) or len(glob.glob1(directory,"*.jpg")): + freePcent = getFreePercentForDir(directory) + freeSize = getFreeSpaceForDir(directory) + SGDIRSdata.append([directory, freePcent, freeSize]) + SGcnt=SGcnt+1 + print "%s: %s - %.2f%%" % (SGcnt, directory, freePcent * 100) + else: + # Check if the selected SG dir has no data files exit + if SGselectdata and SGselectdata[0] == directory: + print "\n'" + SGgrp + "' Storage Group directories have no files to move. Exiting" + sys.exit(0) + + # Exit if no SGs with data found + if SGcnt is 0: + print "\n'" + SGgrp + "' Storage Group directories have no files to move. Exiting." + sys.exit(0) + + # Ask user to select which SG to empty if not already selected + if not SGselectdata: + try: + SGDIRselect=int(raw_input("\nEnter the number of the storage group directory to empty: ")) + if SGDIRselect > SGcnt or SGDIRselect < 1: + SGDIRselect=int("e") + except ValueError: + print "You must enter a number between 1 and %s. Exiting." %SGcnt + sys.exit(0) + + SGselectdata=SGDIRSdata[SGDIRselect-1] + + # Sort data on percent free + SGDIRSdata = sorted(SGDIRSdata, reverse=True, key=operator.itemgetter(1)) + leastFull = SGDIRSdata[-1] + + # Make sure leastFull and SGselectdata are not the same dir + if leastFull[0] == SGselectdata[0]: + leastFull = SGDIRSdata[-2] + + # Get random file from user selected dir + fileToMove = random.choice([f for f in os.listdir(SGselectdata[0]) if f.endswith(".mpg") or f.endswith(".nuv") or f.endswith(".jpg")]) + filePathToMove = SGselectdata[0] + "/" + fileToMove + + # Check that the file isn't too big for least used dir + fileSize = getFileSize(filePathToMove) + if (fileSize > getFreeSpaceForDir(leastFull[0])): + # Too big to move + print filePathToMove + " is too big to move to " + leastFull[0] + sys.exit() + + print "------------------------------------------------" + print "Move File:" + print " " + filePathToMove + print " Size: " + sizeof_fmt(os.path.getsize(filePathToMove)) + print "To:" + print " " + leastFull[0] + print " Available: " + sizeof_fmt(getFreeSpaceForDir(leastFull[0])) + + # Move file + if cmdargs.checkonly: + print "------------------------------------------------" + print"Check Only option was used. No files were moved." + shouldQuit = True + else: + print "------------------------------------------------" + print "Checking System Status..." + if subprocess.call(["/usr/bin/python2", "/usr/LH/bin/idle.py", "-s"]): + print " System is busy. The file will not be moved." + sys.exit() + print "Moving File..." + try: + shutil.move(filePathToMove, leastFull[0]) + # eg. src and dest are the same file + except shutil.Error as e: + a=raw_input("\n%s. Overwrite destination (y/n)? " % e) + if a == "y" or a == "Y": + os.remove(leastFull[0] + "/" + fileToMove) + shutil.move(filePathToMove, leastFull[0]) + else: + b=raw_input("\nRemove %s (y/n)? " % filePathToMove) + if b == "y" or b == "Y": + os.remove(filePathToMove) + # eg. source or destination doesn't exist + except IOError as e: + print('Error: %s' % e.strerror) + + # Remove png files for Default & LiveTV SGs + if SGgrp == "Default" or SGgrp == "LiveTV": + print "------------------------------------------------" + print "Removing png Files:" + pngFiles = glob.glob(filePathToMove + "*.png") + for p in pngFiles: + os.remove(p) + print " " + p -- cgit v0.12