diff options
Diffstat (limited to 'abs/core/LinHES-system/add_storage.py')
| -rw-r--r-- | abs/core/LinHES-system/add_storage.py | 555 |
1 files changed, 489 insertions, 66 deletions
diff --git a/abs/core/LinHES-system/add_storage.py b/abs/core/LinHES-system/add_storage.py index 28d679c..93a67c7 100644 --- a/abs/core/LinHES-system/add_storage.py +++ b/abs/core/LinHES-system/add_storage.py @@ -11,40 +11,55 @@ import pickle import commands import sys,os import random, string +import ConfigParser +from ConfigParser import SafeConfigParser +import glob + from MythTV import MythDB, MythBE, Recorded, MythError from socket import timeout, gethostname +storage_dir = "/etc/storage.d" +pickle_file = "%s/storage.pkl" %storage_dir + -pickle_file="/usr/LH/etc/storage.pkl" +SG_MAP_BE={ 'Default' :'media/tv', +'LiveTV' :'media/tv/live', +'DB Backups' :'backup/mythtv_backups/'} +SG_MAP_FE={ + 'Videos' :'media/video', + 'Trailers' :'media/video_stuff/trailers', + 'Coverart' :'media/video_stuff/coverart', + 'Fanart' :'media/video_stuff/fanart', + 'Banners' :'media/video_stuff/banners', + 'Screenshots':'media/video_stuff/screenshots', + } -SG_MAP={ 'Default' :'tv', - 'LiveTV' :'live_tv', - 'Videos' :'video', - 'Trailers' :'video/trailers', - 'Coverart' :'video/converart', - 'Fanart' :'video/fanart', - 'Banners' :'video/banners', - 'Screenshots':'video/screenshots', - 'DB Backups' :'db_backups'} -FS_LIST=[] -for key in SG_MAP.keys(): - FS_LIST.append(SG_MAP[key]) +FS_LIST_BE=[] +for key in SG_MAP_BE.keys(): + FS_LIST_BE.append(SG_MAP_BE[key]) +FS_LIST_FE=[] +for key in SG_MAP_FE.keys(): + FS_LIST_FE.append(SG_MAP_FE[key]) class disk_device: - def __init__(self,device): + def __init__(self,device,storage_dir): device_obj = bus.get_object("org.freedesktop.UDisks", device) device_props = dbus.Interface(device_obj, dbus.PROPERTIES_IFACE) + self.storage_dir = storage_dir + self.top_mount_dir = "/data/storage" + self.config = ConfigParser.RawConfigParser() self.fs_map = self.get_fsmap() self.is_device = self.get_is_device(device_props) self.vendor = self.get_vendor(device_props) self.model = self.get_model(device_props) - self.mount_path = self.get_mounts(device_props) + self.mmount = False + self.f = self.get_mounts(device_props) self.is_mounted = self.get_is_mounted(device_props) self.parition_size = self.get_partition_size(device_props) @@ -57,7 +72,18 @@ class disk_device: self.device_file_path = self.get_device_file_path(device_props) self.block_partition="%s1" %self.block_path self.in_use = self.get_in_use() + self.uuid='' + self.new_mount_point='' + self.disk_num='' + + def set_partition(self,partition): + self.block_partition = "%s%s" %(self.block_path,partition) + def set_mmount(self,flag): + self.mmount = flag + + def set_disk_num(self,num): + self.disk_num=num def get_is_device(self,device_props): return device_props.Get('org.freedesktop.UDisks.Device', "DeviceIsDrive") @@ -74,6 +100,11 @@ class disk_device: def get_is_mounted(self,device_props): return device_props.Get('org.freedesktop.UDisks.Device', "DeviceIsMounted") + def get_name(self): + filename="%s_%s" %(self.model.replace(' ',''), + self.serial_number.replace(' ','')) + return filename + def get_in_use(self): in_use = False for i in self.fs_map: @@ -82,7 +113,6 @@ class disk_device: break return in_use - def get_partition_size(self,device_props): return device_props.Get('org.freedesktop.UDisks.Device', "PartitionSize") @@ -150,8 +180,8 @@ class disk_device: return fs_map def find_fstype(self,moutpoint): - fstype="ext3" - mp=['/myth', '/data/media'] + fstype="xfs" + mp=['/myth', '/data/storage/disk0'] for i in self.fs_map: if i[1] in mp: fstype = i[2] @@ -166,7 +196,10 @@ class disk_device: new_fstype = self.find_fstype(current_media_mount) #do format - cmd = "mkfs -t %s %s " %(new_fstype,self.block_partition) + if new_fstype == "xfs": + cmd = "mkfs -t %s -f %s " %(new_fstype,self.block_partition) + else: + cmd = "mkfs -t %s %s " %(new_fstype,self.block_partition) print " Formating %s with %s" %(self.block_partition,new_fstype) runcmd(cmd) return @@ -181,7 +214,7 @@ class disk_device: if uuid == '': print "Could not find a UUID for device: %s" %partition sys.exit(1) - + self.uuid = uuid.strip() return uuid.strip() def read_fstab(self): @@ -191,7 +224,7 @@ class disk_device: return fstab def find_options_type(self,fstab): - mp=['/myth', '/data/media'] + mp=['/myth', '/data/storage/disk0'] for i in fstab: split_line=i.split() try: @@ -206,24 +239,30 @@ class disk_device: return options,i - def add_fstab(self): + def add_fstab(self,bind=False): new_fstab_list=['UUID=', 'mount_point', 'auto', 'defaults', '0', '1'] fstab=self.read_fstab() + #determine mount_path - self.new_mount_point="/data/media/%s_%s" %(self.model.replace(' ',''),self.serial_number.replace(' ','')) + self.new_mount_point="%s/%s_%s" %(self.top_mount_dir,self.model.replace(' ',''),self.serial_number.replace(' ','')) + if bind: + new_fstab_list=["/data/storage/disk0" , self.new_mount_point , "none" , "rw,bind", '0', '0'] + uuid=self.find_uuid(self.block_partition) + else: #determine options - new_options = self.find_options_type(fstab)[0] - #find blkid - self.block_partition="%s1" %self.block_path - uuid=self.find_uuid(self.block_partition) + new_options = self.find_options_type(fstab)[0] + + #find blkid + #self.block_partition="%s1" %self.block_path + uuid=self.find_uuid(self.block_partition) - #construct new line - new_fstab_list[0]="UUID=%s" %uuid - new_fstab_list[1]=self.new_mount_point - new_fstab_list[3]=new_options + #construct new line + new_fstab_list[0]="UUID=%s" %uuid + new_fstab_list[1]=self.new_mount_point + new_fstab_list[3]=new_options new_fstab_line='\t'.join(new_fstab_list) new_fstab_line="%s\n" %new_fstab_line @@ -238,13 +277,14 @@ class disk_device: f.close() return - def mount_disk(self): + def mount_disk(self,no_mount=False): try: os.stat(self.new_mount_point) except: os.makedirs(self.new_mount_point) - cmd = "mount %s" %self.new_mount_point - runcmd(cmd) + if no_mount == False: + cmd = "mount %s" %self.new_mount_point + runcmd(cmd) return def mkdirs(self,FS_LIST): @@ -260,25 +300,77 @@ class disk_device: cmd="chmod -R 775 /%s" %self.new_mount_point runcmd(cmd) - def add_sg(self,DB,host,SG_MAP): + def add_sg(self,DB,host,SG_MAP,weight='0',install_call=False): print " Adding to storage groups" + sgweight=int(weight) for key in SG_MAP.iterkeys(): #print key," : ", SG_MAP[key] - #try: - gn=key - hn=host - dn="%s/%s" %(self.new_mount_point,SG_MAP[key]) - #print dn - #print gn - #print hn + gn=key + hn=host + dn="%s/%s" %(self.new_mount_point,SG_MAP[key]) + #print dn + #print gn + #print hn + if install_call == True : + print "Will write SG for stuff after the fact" + else: with DB as c: try: c.execute("""insert into storagegroup (groupname,hostname,dirname) values (%s,%s,%s)""",(gn,hn,dn)) print " Adding location: %s to storagegroup %s" %(dn,gn) except: - print " Error inserting %s into storage groups" %dn + print " *Error inserting %s into storage groups" %dn + + if sgweight > 0: + try: + #("SGweightPerDir:server2:/mnt/video", 99, "server2"); + sgw="SGweightPerDir:%s:%s" %(hn,dn) + #print sgw + #print sgweight + #print hn + + c.execute("""insert into settings (value,data,hostname) values (%s,%s,%s)""",(sgw,sgweight,hn)) + print " Adding storage group weight of %s for %s\n" %(sgweight,gn) + except: + print " *Error setting storage group weight %s for %s\n" %(sgweight,gn) + return + def write_config(self): + print " Writing out storage.d conf file" + self.config.add_section('storage') + self.config.set('storage','uuid',self.uuid) + self.config.set('storage','mountpoint',self.new_mount_point) + self.config.set('storage','shareable','True') + self.config.set('storage','mmount',self.mmount) + self.config.set('storage','disk_num',self.disk_num) + + + filename="%s_%s.conf" %(self.model.replace(' ',''), + self.serial_number.replace(' ','')) + + configfile="/etc/storage.d/%s" %filename + print " %s" %configfile + with open(configfile, 'wb') as configfile: + self.config.write(configfile) + return + + def symlink_disk(self): + print " Creating symlink for disk number" + disk_ln="%s/disk%s" %(self.top_mount_dir,self.disk_num) + cmd = "ln -s %s %s" %(self.new_mount_point,disk_ln) + print cmd + runcmd(cmd) + + def symlink(self): + pass + print " Creating symlink" + cmd = "ln -s %s/media /myth " %(self.new_mount_point) + runcmd(cmd) + + + +#end of class def runcmd(cmd): @@ -300,7 +392,7 @@ def scan_system(): ud_manager = dbus.Interface(ud_manager_obj, 'org.freedesktop.UDisks') current_drive_list=[] for dev in ud_manager.EnumerateDevices(): - drive = disk_device(dev) + drive = disk_device(dev,storage_dir) if drive.is_device and drive.device_size > 5000 and not drive.is_optical : current_drive_list.append(drive) @@ -333,7 +425,7 @@ def search_for_match(system_drive,known_drive_list): y_drive_hash = "%s_%s_%s" %(y.model,y.parition_size,y.device_file_path) if system_drive_hash == y_drive_hash : match_drive = True - print "No serial number was found, matched using hash method: %s" %system_drive.model + print "\n* No serial number was found, matched using hash method: %s" %system_drive.model break elif y.serial_number == system_drive.serial_number: @@ -347,7 +439,7 @@ def search_for_match(system_drive,known_drive_list): def prompt_to_add(current_drive): loop = True while loop: - str1 = raw_input(" Adding the drive will remove all contents on the drive. \n\nDo you wish enable this drive for MythTV storage(Y/N)?:") + str1 = raw_input("\n Adding the drive will remove all contents on the drive. \n Do you wish enable this drive for MythTV storage(Y/N)?:") if str1 in ['Y','N','y','n']: loop = False @@ -359,18 +451,42 @@ def prompt_to_add(current_drive): rc = False return rc +def prompt_to_continue(): + loop = True + #while loop: + str1 = raw_input("\n Ready to start, press Y to continue, anything else to abort:") + + #if str1 in ['Y','N','y','n']: + # loop = False + # break + #print "\n" + if str1 == 'Y' or str1 == 'y': + rc = True + else: + rc = False + return rc + + + def remove_pickle(): try: - print "Resetting list of known drives." - + print "* Resetting list of known drives." os.remove(pickle_file) except: pass - +def last_disk_num(): + parser = SafeConfigParser() + num_list = [] + for conf_file in glob.glob('%s/*.conf' %storage_dir): + parser.read(conf_file) + disk_num = parser.get('storage', 'disk_num') + num_list.append(int(disk_num)) + num_list.sort() + return num_list[-1] #-------------------------------------------- -def main(scan_only): +def main(scan_only, destruction, no_mount, install_call , dir_fe_only): global bus bus = dbus.SystemBus() @@ -378,7 +494,7 @@ def main(scan_only): known_drive_list=[] known_drive_list = read_known_list() process_list=[] - + no_process_list=[] for i in system_drive_list: @@ -390,23 +506,29 @@ def main(scan_only): #print "--" if search_for_match(i,known_drive_list) or i.in_use : - print " Storage is already in use or previously skipped: %s location: %s size: %s" %(i.model,i.block_path,i.device_size) + print "\n" + print " Storage is already in use or previously skipped:%s" %i.model + print " location: %s" %i.block_path + print " size: %s" %i.device_size continue else: - print "\n" if not scan_only: print "\n" print "-------------------------------------------------------------" - - print "Found new hard drive: %s location: %s size: %s \n" %(i.model,i.block_path,i.device_size) + print " Found new hard drive: %s" %i.model + print " location: %s" %i.block_path + print " size: %s " %i.device_size if prompt_to_add(i) : - print "\nDisk will be added to the storage pool!" + print "\n Disk will be added to the storage pool!" process_list.append(i) + else: + no_process_list.append(i) else: process_list.append(i) - + print " End of scan" + print "---------------------------------------" if scan_only: if len(process_list) > 0: f = open('/tmp/scan_report', 'w') @@ -416,31 +538,309 @@ def main(scan_only): f.close() sys.exit(0) + for i in no_process_list: + system_drive_list.append(i) + if len(process_list) > 0: DB = MythDB() host=gethostname() + for y in process_list: + system_drive_list.remove(y) + write_known_drive_list(system_drive_list) else: print "\nDid not find any new storage to add.\n" + write_known_drive_list(system_drive_list) #BE = MythBE(db=DB) #save new list to disk_device - write_known_drive_list(system_drive_list) + #write_known_drive_list(system_drive_list) + #disk_num = last_disk_num() + + + if len(process_list) > 0: + print "\n Adding storage: \n" + if prompt_to_continue() == True: + write_known_drive_list(system_drive_list) + disk_num = last_disk_num() + for i in process_list: + print " Drive: %s" %(i.get_name()) + disk_num = disk_num + 1 + if destruction == True: + i.partition_disk() + i.format_disk() + i.add_fstab() + i.mount_disk(no_mount) + if destruction == True: + if dir_fe_only != True: + i.mkdirs(FS_LIST_BE) + i.mkdirs(FS_LIST_FE) + i.set_disk_num(disk_num) + i.write_config() + system_drive_list.append(i) + write_known_drive_list(system_drive_list) + + i.symlink_disk() + #always create the FE groups(video)..do not need to create Be + if dir_fe_only != True: + i.add_sg(DB,host,SG_MAP_BE) + + i.add_sg(DB,host,SG_MAP_FE) + + print "-----" + cmd = "systemconfig.py -m fileshare" + runcmd(cmd) + #i.add_sg(DB,host,SG_MAP) + +def myth_main(no_mount,install_call,dir_fe_only): + global bus + bus = dbus.SystemBus() + #search for root + f = open('/etc/fstab', 'r') + fstab=f.readlines() + f.close() + for i in fstab: + split_line=i.split() + if not split_line: + continue + + try: + if split_line[1] == "/" : + uuid_device = split_line[0] + break + except: + print "Couldn't find / in fstab" + sys.exit(1) + if uuid_device.startswith("UUID"): + uuid_device = uuid_device.split("=")[1] + cmd = "blkid -U %s" %uuid_device + device = runcmd(cmd)[1] + else: + device = uuid_device.strip() + #should have something like /dev/sda1 + #remove all the digits + device = ''.join([letter for letter in device if not letter.isdigit()]) + + + system_drive_list = scan_system() + for i in system_drive_list: + if i.block_path == device: + break + else: + print "Could'nt find root device in block list" + sys.exit(1) + if not install_call == True: + DB = MythDB() + host=gethostname() + else: + DB = None + host=gethostname() + + + print " Drive: %s" %(i.get_name()) + i.set_mmount(True) + i.set_partition("7") + i.set_disk_num(0) + i.add_fstab(True) + #if not install_call: + i.mount_disk(no_mount) + i.write_config() + #always create the FE groups(video)..do not need to create Be + if dir_fe_only != True: + i.add_sg(DB,host,SG_MAP_BE,'99',install_call) - for i in process_list: - i.partition_disk() - i.format_disk() - i.add_fstab() - i.mount_disk() - i.mkdirs(FS_LIST) - i.add_sg(DB,host,SG_MAP) + i.add_sg(DB,host,SG_MAP_FE,'99',install_call) + + i.symlink() + cmd = "systemconfig.py -m fileshare" + runcmd(cmd) + +class reconstruct_path: + def __init__(self,conf_file): + self.conf_file = conf_file + parser = SafeConfigParser() + parser.read(self.conf_file) + + + self.uuid = parser.get('storage', 'uuid') + self.mount_point = parser.get('storage', 'mountpoint') + self.shareable = parser.get('storage', 'shareable') + self.myth_mount = parser.get('storage', 'mmount') + self.bind = self.myth_mount + self.disk_num = parser.get('storage', 'disk_num') + self.top_mount_dir = os.path.dirname(self.mount_point) + + def get_conf(self): + return self.conf_file + + def get_uuid(self): + return self.uuid + + def get_mount_point(self): + return self.mount_point + + def get_shareable(self): + return self.shareable + + def get_is_myth_mount(self): + return self.myth_mount + + def get_disk_num(self): + return self.disk_num + + def create_mount_point(self): + try: + os.stat(self.mount_point) + except: + os.makedirs(self.mount_point) + + def find_options_type(self,fstab): + mp=['/myth', '/data/storage/disk0'] + for i in fstab: + split_line=i.split() + try: + if split_line[1] in mp: + options = split_line[3] + break + else: + options = "defaults" + mount_point = i + except: + options = "defaults" + return options,i + + def read_fstab(self): + f = open('/etc/fstab', 'r') + fstab=f.readlines() + f.close() + return fstab + + def check_in_fstab(self,fstab,check_path): + + for line in fstab: + if line.find(check_path) > -1: + return True + return False + + def append_fstab(self,line): + new_fstab_line='\t'.join(line) + new_fstab_line="%s\n" %new_fstab_line + + f = open('/etc/fstab', 'a') + #f = open('/tmp/fstab', 'a') + f.write(new_fstab_line) + f.write("\n") + f.close() + + def symlink(self): + print " Creating symlink" + if not os.path.exists("/myth"): + cmd = "ln -s %s/media /myth " %(self.mount_point) + runcmd(cmd) + else: + print " Skipping symlink, path already present" + + def symlink_disk(self): + print " Creating symlink for disk number" + disk_ln="%s/disk%s" %(self.top_mount_dir,self.disk_num) + cmd = "ln -s %s %s" %(self.mount_point,disk_ln) + runcmd(cmd) + + + def add_fstab(self): + new_fstab_list=['UUID=', 'mount_point', 'auto', 'defaults', '0', '1'] + fstab=self.read_fstab() + + if self.bind == "True": + self.symlink() + + if self.check_in_fstab(fstab,self.uuid) == True: + print " Found storage in fstab, will not add it" + else: + print " Adding storage to fstab" + if self.bind == "True" : + print " Bind mount" + new_fstab_list=["/data/storage/disk0" , self.mount_point , "none" , "rw,bind", '0', '0'] + if self.check_in_fstab(fstab,self.mount_point) == False: + self.append_fstab(new_fstab_list) + else: + print " Found bind storage in fstab, will not add it" + + else: + #construct new line + new_options = self.find_options_type(fstab)[0] + new_fstab_list[0]="UUID=%s" %self.uuid + new_fstab_list[1]=self.mount_point + new_fstab_list[3]=new_options + self.append_fstab(new_fstab_list) + + def mount_disk(self,no_mount=False): + try: + os.stat(self.mount_point) + except: + os.makedirs(self.mount_point) + if no_mount == False : + cmd = "mount %s" %self.mount_point + runcmd(cmd) + return + +def reconstruct_mounts(no_mount): + print "Recreating devices based on contents of /etc/storage.d" + for conf_file in glob.glob('%s/*.conf' %storage_dir): + print "\n" + cf = reconstruct_path(conf_file) + + #print cf.get_conf() + #print cf.get_uuid() + print " %s" %cf.get_mount_point() + #print cf.get_shareable() + #print cf.get_is_myth_mount() + #print cf.get_disk_num() + + cf.create_mount_point() + cf.add_fstab() + cf.symlink_disk() + cf.mount_disk(no_mount) + + pass + + +def usage(): + help='''\n + Add storage is designed to find and setup new disks for mythtv usage. + It's a powerfull that could destroy data if not used correctly, so please be careful. + Normal operations include (in this order): + Partition the disk + Format the disk + Add disk to /etc/fstab + Mount the disk + Create the directory + Write out the config file to /etc/storage.d + Add new locations to mythtv storage groups + + options: + + --no_mount : Do not mount the disk, only add it to /etc/fstab and create the dir. + --no_destruction: Will not partition or format the disk + --new_init : Erase the list of new disks and rescan. + --report : will scan the disks and print out if it found new storage. + --fe_only : Will only create the storage group dir for videos..excludes tv + ''' + print help + sys.exit(0) if __name__ == "__main__": scan_only = False + myth_mount = False + no_mount = False + destruction = True + install_call = False + dir_fe_only = False + reconstruct = False try: os.remove("/tmp/scan_report") except: @@ -449,13 +849,36 @@ if __name__ == "__main__": if not os.geteuid()==0: sys.exit("\nRoot access is required to run this program\n") + if "--help" in sys.argv: + usage() + + if "--install_call" in sys.argv: + install_call = True + + if "--no_mount" in sys.argv : + no_mount = True + + if "--no_destruction" in sys.argv: + destruction = False + if "--new_init" in sys.argv : remove_pickle() if "--report" in sys.argv : scan_only = True - main(scan_only) + if "--fe_only" in sys.argv: + dir_fe_only = True + + if "--reconstruct" in sys.argv: + reconstruct = True + + if "--double_myth" in sys.argv: + myth_main(no_mount,install_call,dir_fe_only) + elif reconstruct == True: + reconstruct_mounts(no_mount) + else: + main(scan_only,destruction,no_mount, install_call, dir_fe_only) |