diff options
26 files changed, 3409 insertions, 0 deletions
diff --git a/linhes/linhes-system/10-monitor.conf b/linhes/linhes-system/10-monitor.conf new file mode 100644 index 0000000..f63cc86 --- /dev/null +++ b/linhes/linhes-system/10-monitor.conf @@ -0,0 +1,3 @@ +Section "Extensions" + Option "DPMS" "Disable" +EndSection diff --git a/linhes/linhes-system/79-cronie.hook b/linhes/linhes-system/79-cronie.hook new file mode 100644 index 0000000..59ca25e --- /dev/null +++ b/linhes/linhes-system/79-cronie.hook @@ -0,0 +1,9 @@ +[Trigger] +Operation = Install +Type = Package +Target = linhes-system + +[Action] +Description = Enable and start cronie... +When = PostTransaction +Exec = /usr/bin/systemctl enable --now cronie.service diff --git a/linhes/linhes-system/PKGBUILD b/linhes/linhes-system/PKGBUILD new file mode 100755 index 0000000..1674c61 --- /dev/null +++ b/linhes/linhes-system/PKGBUILD @@ -0,0 +1,87 @@ +pkgname=linhes-system +pkgver=9.0.0 +pkgrel=1 +arch=('x86_64') +#install=$pkgname.install +pkgdesc="Everything that makes LinHES a system" +license=('GPL2') +depends=('cronie' 'libnotify' 'linhes-templates' 'pacman-contrib' 'openssh' 'ttf-overlock') +binfiles="add_storage.py balance_storage_groups.py empty_storage_groups.py remove_storage.py + checkXFSfrag.sh enableIRWake.sh idle.py lh_systemstart.sh jobqueue_helper.py + find_orphans.py + create_media_dirs.sh be_check.py + myth2mkv myth2mp3" +source=($binfiles + 'myth_mtc.cron' 'paccache.cron' 'xfs_defrag.cron' + 'readme_is_xml' 'add_storage.readme' + '79-cronie.hook' 'fstrim.hook' 'openssh.hook' + '10-monitor.conf' + 'system-sudo.rules' + 'lh_systemstart.sh.desktop') + +package() { + cd $srcdir + + #bin files + BINDIR=$pkgdir/usr/bin + for i in $binfiles + do + item=$i + echo "installing $item to $BINDIR" + install -m755 -D $item $BINDIR/$item + done + + #startup files + install -D -m600 "$srcdir/lh_systemstart.sh.desktop" "$pkgdir/home/mythtv/.config/autostart/lh_systemstart.sh.desktop" + + #readme files + install -m644 -D $srcdir/readme_is_xml $pkgdir/etc/gen_is_xml.d/readme_is_xml + install -m644 -D $srcdir/readme_is_xml $pkgdir/etc/gen_lib_xml.d/readme_gen_xml + install -m644 -D $srcdir/readme_is_xml $pkgdir/etc/gen_game_xml.d/readme_gen_xml + install -m644 -D $srcdir/add_storage.readme $pkgdir/etc/storage.d/readme + + #cron files + install -m755 -D $srcdir/paccache.cron $pkgdir/etc/cron.weekly/paccache.cron + install -m755 -D $srcdir/xfs_defrag.cron $pkgdir/etc/cron.weekly/xfs_defrag + install -m755 -D $srcdir/myth_mtc.cron $pkgdir/etc/cron.hourly/myth_mtc + + #hooks + install -Dm0644 $srcdir/79-cronie.hook "${pkgdir}"/usr/share/libalpm/hooks/79-cronie.hook + install -Dm0644 $srcdir/fstrim.hook "${pkgdir}"/usr/share/libalpm/hooks/fstrim.hook + install -Dm0644 $srcdir/openssh.hook "${pkgdir}"/usr/share/libalpm/hooks/openssh.hook + + #sudo rules + mkdir -p $pkgdir/etc/sudoers.d/ + chmod 750 $pkgdir/etc/sudoers.d/ + chown -R root:root $pkgdir/etc/sudoers.d + install -o root -g root -m 0750 $srcdir/system-sudo.rules $pkgdir/etc/sudoers.d/system_sudo + + #disable dpms + install -m644 -D $srcdir/10-monitor.conf $pkgdir/etc/X11/xorg.conf.d/10-monitor.conf + +} +sha256sums=('96f67b5428debb7dac909893c56a7637bf6545c068732822981d4080125c53d9' + 'b0e8fe4d04e1f779d52a28156489fb51efc13e173efccba2d6d458044bf35904' + '4ab36bbabf0964f666278c225f4c2d41f4277acb42e9023163fa3a9599282466' + '632eb5073219b86667a361e567ad766fbd122f27dc5e775bba9983c417686aa9' + '11168c9cd3b117decaab6bc665c183b4aab917cf0a976bce4c1b5e4686a27bc9' + 'ae34515e144830f424d3bd3f6b1b446892d62beed20bca6f0fb19b0bbb779f27' + '23358a7bff4968eccd469613b81b1415c2ae0ebe77f14f74426697333e4d88d7' + '2cb8d676abc3b848d579aff2fe8796ca280b96abe40be03bc30055ce50d98c02' + '91bdec992bb2c933e15625c181f2195c402060b879168ebf35944cb064c904b9' + '76f023c0cde7fea269234f1b29c32b117b91769217d4b1b8a3922daceb25f9f8' + 'bffcc13e4b480f720feb2b3c781bc4247c63303250c3d885022c699573d45a33' + '0254a21644473ba7953501a223f13b9b55d7ec290c80a567724ca1ac13e02e30' + '9ea1b5583cd38f53bb79d9e4ccae91a028db0b6850162d7991b19122c564b9c9' + 'cab7724cd7c792adae79ba11cf59a0908edf197c437ae6b356ce86c6a365f0da' + '9e97b4d68c9e8988daacadd40f1de9f0b5945d870eba596a2ceb5e0c023fa9c0' + '186203d3c0520bb3d611da99d33a7713e9c1563814285f1f101097234f214b2f' + '4533e15b9141ab722f9b02c3ec7855f61c1e4d2deedd8591b1abd0ed551be581' + '5f502b1bc8815d69c802320790745e4526d5817fd8ecc7b00cf8b16078f8d440' + '12e424432bdf2d50afe3e632c018fef847e860a35a53525eccbe656b9c4118aa' + '0fbc05f521aea83157c5e6f8bd29a422873093bb6cded965cb7ffe98ff776fa4' + '4c29e0b71071ae9556cf2dbd75de560d028577fe5eb993113105112c4b445eac' + '890482242434e333024c7819e8bf3c889dc16548d0a1745479c8523930fb32f7' + 'dead17906b33a7f9d66ad13bb1c083a23438f45ece9bd5ec41ff86eda01c132a' + 'a9d0a94ff442453f0bec0b2e8afd591cf17b2845b6ae45ff300530114efd30af' + '7016c4bff7275e8d625cec418928ddeab0573df785dbaeda4ac710ddf64e9770') diff --git a/linhes/linhes-system/add_storage.py b/linhes/linhes-system/add_storage.py new file mode 100755 index 0000000..041f612 --- /dev/null +++ b/linhes/linhes-system/add_storage.py @@ -0,0 +1,1185 @@ +#!/usr/bin/python +#add_storage.py used to auto add new storage to MythTV storage groups +#If it's a new disk it will erase the entire disk and reformat. +# +#Disks that are mounted, in fstab, size < 5000 bytes, optical or +#have already been seen will not be presented as an option. +# + + +import dbus +import pickle +import subprocess +import sys,os +import random, string +import configparser +from configparser import SafeConfigParser +import glob + +from MythTV import MythDB, MythBE, Recorded, MythError +from socket import timeout, gethostname + + +storage_dir = "/etc/storage.d" +pickle_file = "%s/storage.pkl" %storage_dir + + +SG_MAP={ + 'Default' :'media/tv/', + 'LiveTV' :'media/tv/live/', + 'DB Backups' :'backup/mythtv_backups/', + 'Music' :'media/music/', + 'Streaming' :'media/streaming/', + 'Videos' :'media/video/', + 'Photographs':'media/photos/', + 'Banners' :'media/artwork/banners/', + 'Coverart' :'media/artwork/coverart/', + 'Fanart' :'media/artwork/fanart/', + 'MusicArt' :'media/artwork/musicart/', + 'Screenshots':'media/artwork/screenshots/', + 'Trailers' :'media/artwork/trailers/', + } + +FS_LIST=[] +for key in list(SG_MAP.keys()): + FS_LIST.append(SG_MAP[key]) + +class disk_device: + def __init__(self,device,storage_dir): + device_obj = bus.get_object("org.freedesktop.UDisks", device) + device_props = dbus.Interface(device_obj, dbus.PROPERTIES_IFACE) + self.storage_dir = storage_dir + self.top_mount_dir = "/data/storage" + self.config = configparser.RawConfigParser() + self.fs_map = self.get_fsmap() + self.is_device = self.get_is_device(device_props) + self.vendor = self.get_vendor(device_props) + self.model = self.get_model(device_props) + + self.mmount = False + self.dir_sg = False + self.f = self.get_mounts(device_props) + self.is_mounted = self.get_is_mounted(device_props) + + self.parition_size = self.get_partition_size(device_props) + self.device_size = self.get_device_size(device_props) + self.serial_number = self.get_serial_number(device_props) + self.read_only = self.get_is_readonly(device_props) + self.is_optical = self.get_is_optical_disc(device_props) + self.connection = self.get_connections(device_props) + self.block_path = self.get_device_file(device_props) + self.device_file_path = self.get_device_file_path(device_props) + self.block_partition="%s1" %self.block_path + self.in_use = self.get_in_use() + self.uuid='' + self.new_mount_point='' + self.disk_num='' + + def set_partition(self,partition): + self.block_partition = "%s%s" %(self.block_path,partition) + + def set_mmount(self,flag): + self.mmount = flag + + def set_dir_sg(self,flag): + self.dir_sg = flag + + def set_disk_num(self,num): + self.disk_num=num + + def get_is_device(self,device_props): + return device_props.Get('org.freedesktop.UDisks.Device', "DeviceIsDrive") + + def get_vendor(self,device_props): + return device_props.Get('org.freedesktop.UDisks.Device', "DriveVendor") + + def get_model(self,device_props): + return device_props.Get('org.freedesktop.UDisks.Device', "DriveModel") + + def get_mounts(self,device_props): + return device_props.Get('org.freedesktop.UDisks.Device', "DeviceMountPaths") + + def get_is_mounted(self,device_props): + return device_props.Get('org.freedesktop.UDisks.Device', "DeviceIsMounted") + + def get_name(self): + filename="%s_%s" %(self.model.replace(' ',''), + self.serial_number.replace(' ','')) + return filename + + def get_in_use(self): + in_use = False + for i in self.fs_map: + if self.block_path in i[0]: + in_use = True + break + return in_use + + def get_partition_size(self,device_props): + return device_props.Get('org.freedesktop.UDisks.Device', "PartitionSize") + + def get_device_size(self,device_props): + return device_props.Get('org.freedesktop.UDisks.Device', "DeviceSize") + + + def get_serial_number(self,device_props): + serial_number = device_props.Get('org.freedesktop.UDisks.Device', "DriveSerial") + random_string = os.urandom(5) + if serial_number == '': + serial_number = "".join( [random.choice(string.letters) for i in range(6)] ) + serial_number = "NSW%s" %serial_number + return serial_number + + def get_is_readonly(self,device_props): + return device_props.Get('org.freedesktop.UDisks.Device', "DeviceIsReadOnly") + + def get_is_optical_disc(self,device_props): + return device_props.Get('org.freedesktop.UDisks.Device', "DeviceIsOpticalDisc") + + def get_connections(self,device_props): + return device_props.Get('org.freedesktop.UDisks.Device', "DriveConnectionInterface") + + def get_device_file(self,device_props): + return device_props.Get('org.freedesktop.UDisks.Device', "DeviceFile") + + def get_device_file_path(self,device_props): + path = device_props.Get('org.freedesktop.UDisks.Device', "DeviceFileByPath") + try: + path = path[0] + except: + path = "None" + + return path + + + def partition_disk(self): + print(" Creating new partition table") + cmd = "parted -s -a optimal %s mklabel gpt" %self.block_path + runcmd(cmd) + cmd = "parted -s -a optimal %s mkpart primary \" 1 -1\"" %self.block_path + runcmd(cmd) + return + + def get_fsmap(self): + #block,point,fs + fs_map=[] + + f = open('/proc/mounts','r') + mounts=f.readlines() + f.close() + for line in mounts: + temp_fs=[] + split_line=line.split() + block=split_line[0] + mountp=split_line[1] + fs=split_line[2] + block=os.path.realpath(block) + if block.startswith("/dev"): + temp_fs.append(block) + temp_fs.append(mountp) + temp_fs.append(fs) + fs_map.append(temp_fs) + return fs_map + + def find_fstype(self,moutpoint): + fstype="xfs" + mp=['/myth', '/data/storage/disk0'] + for i in self.fs_map: + if i[1] in mp: + fstype = i[2] + break + return fstype + + def lookup_format(self): + fstab = self.read_fstab() + current_media_mount = self.find_options_type(fstab)[1] + new_fstype = self.find_fstype(current_media_mount) + #setting self.new_fstype so that it can be referenced when setting fstab + self.new_fstype = new_fstype + return + + def format_disk(self): + #lookup format + #self.lookup_format() + #do format + if self.new_fstype == "xfs": + cmd = "mkfs -t %s -f %s " %(self.new_fstype,self.block_partition) + else: + cmd = "mkfs -t %s %s " %(self.new_fstype,self.block_partition) + print(" Formatting %s with %s" %(self.block_partition,self.new_fstype)) + runcmd(cmd) + return + + def find_uuid(self,partition): + #logging.info("Finding the UUID for %s...", partition) + cmd = "blkid -s UUID %s" %partition + tmpuuid = runcmd(cmd)[1] + splituuid = tmpuuid.partition("=") + uuid = splituuid[2].replace('"', "") + # logging.info("The uuid is %s", uuid) + if uuid == '': + print("Could not find a UUID for device: %s" %partition) + sys.exit(1) + self.uuid = uuid.strip() + return uuid.strip() + + def read_fstab(self): + f = open('/etc/fstab', 'r') + fstab=f.readlines() + f.close() + return fstab + + def find_options_type(self,fstab): + mp=['/myth', '/data/storage/disk0'] + for i in fstab: + split_line=i.split() + try: + if split_line[1] in mp: + options = split_line[3] + break + else: + options = "defaults" + mount_point = i + except: + options = "defaults" + return options,i + + + def add_fstab(self,bind=False): + new_fstab_list=['UUID=', 'mount_point', 'auto', 'defaults', '0', '1'] + fstab=self.read_fstab() + new_fstab=[] + + #determine mount_path + self.new_mount_point="%s/%s_%s" %(self.top_mount_dir,self.model.replace(' ',''),self.serial_number.replace(' ','')) + + if bind: + new_fstab_list=["/data/storage/disk0" , self.new_mount_point , "none" , "rw,bind", '0', '0'] + uuid=self.find_uuid(self.block_partition) + else: + #check for old mount point and comment out + for line in fstab: + if not line.startswith("#"): + if line.find(self.new_mount_point) > -1: + print(" Found old mount %s in fstab, commenting out" %self.new_mount_point) + line = "#"+line + new_fstab.append(line) + fstab=new_fstab + + #determine options + new_options = self.find_options_type(fstab)[0] + + #find blkid + #self.block_partition="%s1" %self.block_path + uuid=self.find_uuid(self.block_partition) + + #construct new line + new_fstab_list[0]="UUID=%s" %uuid + new_fstab_list[1]=self.new_mount_point + new_fstab_list[3]=new_options + + new_fstab_line='\t'.join(new_fstab_list) + new_fstab_line="%s\n" %new_fstab_line + fstab.append(new_fstab_line) + + #add line to fstab + f = open('/etc/fstab', 'w') + #f = open('/tmp/fstab', 'w') + for i in fstab: + f.write(i) + #f.write("\n") + f.close() + return + + def mount_disk(self,no_mount=False): + try: + os.stat(self.new_mount_point) + except: + os.makedirs(self.new_mount_point) + if no_mount == False: + if os.path.ismount(self.new_mount_point): + print(" Disk already mounted, will not mount:\n %s" %self.new_mount_point) + pass + else: + print(" Mounting %s" %self.new_mount_point) + cmd = "mount %s" %self.new_mount_point + runcmd(cmd) + return + + def mkdirs(self,FS_LIST): + print(" Creating directory structure:") + print(" %s" %self.new_mount_point) + for y in FS_LIST: + print(" %s" %y) + new_dir="%s/%s" %(self.new_mount_point,y) + try: + os.stat(new_dir) + except: + os.makedirs(new_dir) + cmd="chown -R mythtv:mythtv /%s" %self.new_mount_point + runcmd(cmd) + cmd="chmod -R 775 /%s" %self.new_mount_point + runcmd(cmd) + + def add_sg(self,DB,host,SG_MAP,weight='0',install_call=False): + print(" Adding storage groups") + sgweight=int(weight) + for key in SG_MAP.keys(): + #print key," : ", SG_MAP[key] + gn=key + hn=host + dn="%s/%s" %(self.new_mount_point,SG_MAP[key]) + #print dn + #print gn + #print hn + if install_call == True : + print("Will write SG for stuff after the fact") + else: + with DB as c: + #delete old dir without trailing slash + c.execute("""delete from storagegroup where groupname = %s and hostname = %s and dirname = %s""", (gn,hn,dn.rstrip('/'))) + + try: + c.execute("""insert into storagegroup (groupname,hostname,dirname) values (%s,%s,%s)""",(gn,hn,dn)) + print(" Adding location: %s to storagegroup %s" %(dn,gn)) + except: + print(" *Error inserting %s into storage groups" %dn) + + if sgweight > 0: + try: + #("SGweightPerDir:server2:/mnt/video", 99, "server2"); + sgw="SGweightPerDir:%s:%s" %(hn,dn) + #print sgw + #print sgweight + #print hn + + #delete old dir without trailing slash + c.execute("""delete from settings where value = %s and data = %s and hostname = %s""", (sgw.rstrip('/'),sgweight,hn)) + + c.execute("""insert into settings (value,data,hostname) values (%s,%s,%s)""",(sgw,sgweight,hn)) + print(" Adding storage group weight of %s for %s\n" %(sgweight,gn)) + except: + print(" *Error setting storage group weight %s for %s\n" %(sgweight,gn)) + + return + + def write_config(self): + print(" Writing /etc/storage.d conf file") + self.config.add_section('storage') + self.config.set('storage','uuid',self.uuid) + self.config.set('storage','mountpoint',self.new_mount_point) + self.config.set('storage','fstype',self.new_fstype) + self.config.set('storage','shareable','True') + self.config.set('storage','mmount',self.mmount) + self.config.set('storage','storage_groups',self.dir_sg) + self.config.set('storage','disk_num',self.disk_num) + + + filename="%s_%s.conf" %(self.model.replace(' ',''), + self.serial_number.replace(' ','')) + + configfile="/etc/storage.d/%s" %filename + print(" %s" %configfile) + with open(configfile, 'wb') as configfile: + self.config.write(configfile) + return + + def symlink_disk(self): + print(" Creating symlink for disk%s" %self.disk_num) + disk_ln="%s/disk%s" %(self.top_mount_dir,self.disk_num) + cmd = "ln -s %s %s" %(self.new_mount_point,disk_ln) + runcmd(cmd) + + def symlink(self): + pass + print(" Creating symlink for /myth") + if not os.path.exists("/myth"): + cmd = "ln -s %s/media /myth " %(self.new_mount_point) + runcmd(cmd) + else: + print(" Skipping symlink, /myth already exists") + + +#end of class + + +def runcmd(cmd): + if True : + pass + else: + cmd = "echo "+cmd + #print cmd + cmdout = subprocess.getstatusoutput(cmd) + #logging.debug(" %s", cmdout) + return cmdout + + + + +def scan_system(): + + ud_manager_obj = bus.get_object("org.freedesktop.UDisks", "/org/freedesktop/UDisks") + ud_manager = dbus.Interface(ud_manager_obj, 'org.freedesktop.UDisks') + current_drive_list=[] + for dev in ud_manager.EnumerateDevices(): + drive = disk_device(dev,storage_dir) + + if drive.is_device and drive.device_size > 5000 and not drive.is_optical : + current_drive_list.append(drive) + return current_drive_list + +def read_known_list(): + #reading pickle file + known_drive_list=[] + try: + pkl_file = open(pickle_file, 'rb') + known_drive_list = pickle.load(pkl_file) + pkl_file.close() + except: + pass + return known_drive_list + + +def write_known_drive_list(known_drive_list): + output = open(pickle_file, 'wb') + pickle.dump(known_drive_list, output, -1) + output.close() + + +def search_for_match(system_drive,known_drive_list): + match_drive=False + for y in known_drive_list: + if system_drive.serial_number.startswith("NSW"): + #print "Match_test: hash" + system_drive_hash = "%s_%s_%s" %(system_drive.model,system_drive.parition_size,system_drive.device_file_path) + y_drive_hash = "%s_%s_%s" %(y.model,y.parition_size,y.device_file_path) + if system_drive_hash == y_drive_hash : + match_drive = True + print("\n* No serial number was found, matched using hash method: %s" %system_drive.model) + break + + elif y.serial_number == system_drive.serial_number: + #print "Match_test: serial number" + match_drive=True + break + + return match_drive + + +def prompt_to_add(current_drive,destruction = True): + loop = True + if destruction : + prompt = ''' + ** Adding this disk will remove all contents on the disk. ** + This disk will be partitioned and formatted. + + Enable this disk for storage (Y/N)?:''' + else: + prompt = ''' + ** Preserving existing contents on the disk. ** + This disk will NOT be partitioned or formatted. + + Enable this disk for storage (Y/N)?:''' + while loop: + str1 = input(prompt) + + if str1 in ['Y','N','y','n']: + loop = False + break + print("\n") + if str1 == 'Y' or str1 == 'y': + rc = True + else: + rc = False + return rc + +def prompt_to_continue(process_list): + loop = True + print("\n\n\n Ready to add additional storage!\n") + if destruction: + print("** WARNING: These disk(s) WILL be partitioned and formatted. **\n ** All content on these disk(s) will be erased. **") + else: + print(" ** These disk(s) will NOT be partitioned and formatted. **") + for i in process_list: + print(" %s" %(i.get_name())) + str1 = input("\n Press Y to add disk(s), any other key to cancel:") + + if str1 == 'Y' or str1 == 'y': + rc = True + else: + rc = False + print("\nCancelled: No disk(s) added to your system.") + print("-----") + return rc + +def prompt_sg(dir_sg): + #check for storage groups + print("*" * 60) + if dir_sg != True: + loop = True + prompt_string=''' + MythTV storage groups are used for artwork, database backups, + photos, music, streaming, TV recordings, and videos. + + The content on these storage groups will + only be available while the system is online. + + Enabling MythTV storage groups will create the directories + on the disk(s) and add the paths to the MythTV database. + + Enable MythTV storage groups (Y/N)?:''' + + while loop: + str1 = input(prompt_string) + if str1 in ['Y','N','y','n']: + loop = False + break + print("\n") + + if str1 == 'Y' or str1 == 'y': + dir_sg = True + print(" ** Will add MythTV storage groups!") + else: + print(" ** Will NOT add MythTV storage groups!") + dir_sg = False + else: + dir_sg = True + print("\n --add_sg option used") + print(" ** Will add MythTV storage groups!") + + return dir_sg + +def remove_pickle(): + try: + print("\n* Removing list of known disks.\n") + os.remove(pickle_file) + except: + pass + +def last_disk_num(): + parser = SafeConfigParser() + num_list = [] + for conf_file in glob.glob('%s/*.conf' %storage_dir): + parser.read(conf_file) + disk_num = parser.get('storage', 'disk_num') + num_list.append(int(disk_num)) + num_list.sort() + try: + return num_list[-1] + except: + # conf file or disk_num is missing so fallback to /data/storage/disk# links + for disk_name in glob.glob('/data/storage/disk*'): + disk_num = disk_name.strip('/data/storage/disk') + num_list.append(int(disk_num)) + num_list.sort() + try: + return num_list[-1] + except: + print("Couldn't find last disk number.") + sys.exit(1) + +#-------------------------------------------- + +def main(scan_only, destruction, no_mount, install_call, dir_sg): + global bus + bus = dbus.SystemBus() + + system_drive_list = scan_system() + known_drive_list=[] + known_drive_list = read_known_list() + process_list=[] + no_process_list=[] + + print("-" * 60) + print(" Scan for Disks") + + for i in system_drive_list: + #print i.mount_path + #print i.is_mounted + #print i.in_use + #print i.model + #print i.block_path + #print "--" + + if search_for_match(i,known_drive_list) or i.in_use : + if search_for_match(i,known_drive_list) : + dstatus = " Ignoring - Disk has been previously skipped:" + if i.in_use : + dstatus = " Ignoring - Disk is mounted:" + if search_for_match(i,known_drive_list) and i.in_use : + dstatus = " Ignoring - Disk has been previously skipped and is mounted:" + + print("\n") + print(" --------------------------------------------------------") + print(dstatus) + print(" model: %s" %i.model) + print(" location: %s" %i.block_path) + print(" size: %s" %i.device_size) + continue + + else: + if not scan_only: + print("\n") + print(" --------------------------------------------------------") + print(" Found New Disk:") + print(" model: %s" %i.model) + print(" location: %s" %i.block_path) + print(" size: %s " %i.device_size) + + if prompt_to_add(i,destruction) : + print("\n %s will be added to your system!" %i.model) + process_list.append(i) + else: + no_process_list.append(i) + else: + process_list.append(i) + print("\n") + print(" Scan Finished") + print("-" * 60) + + if scan_only: + if len(process_list) > 0: + print(" Unknown or Unmounted Disks:") + f = open('/tmp/scan_report', 'w') + for i in process_list: + f.write("disk: %s , location: %s ,size: %s \n" %(i.model,i.block_path,i.device_size)) + print("\n") + print(" ---------------------------------------------------------") + print(" Found New Disk:") + print(" model: %s" %i.model) + print(" location: %s" %i.block_path) + print(" size: %s " %i.device_size) + f.close() + sys.exit(0) + + for i in no_process_list: + system_drive_list.append(i) + + + if len(process_list) > 0: + DB = MythDB() + host=gethostname() + for y in process_list: + system_drive_list.remove(y) + write_known_drive_list(system_drive_list) + else: + print("\nThere are no disks to add to your system.\n\nFor more options: add_storage.py --help\n") + write_known_drive_list(system_drive_list) + #BE = MythBE(db=DB) + + + #save new list to disk_device + #write_known_drive_list(system_drive_list) + #disk_num = last_disk_num() + + + if len(process_list) > 0: + print("\n Will add %s disk(s) to your system." %len(process_list)) + + dir_sg = prompt_sg(dir_sg) + if prompt_to_continue(process_list) == True: + write_known_drive_list(system_drive_list) + disk_num = last_disk_num() + for i in process_list: + print(" Disk: %s" %(i.get_name())) + disk_num = disk_num + 1 + i.lookup_format() + if destruction == True: + i.partition_disk() + i.format_disk() + i.add_fstab() + i.mount_disk(no_mount) + #if destruction == True: + + if dir_sg == True: + i.mkdirs(FS_LIST) + + i.set_disk_num(disk_num) + i.set_dir_sg(dir_sg) + i.write_config() + system_drive_list.append(i) + write_known_drive_list(system_drive_list) + + i.symlink_disk() + + if dir_sg == True: + i.add_sg(DB,host,SG_MAP) + + print("-----") + cmd = "systemconfig.py -m fileshare" + runcmd(cmd) + #i.add_sg(DB,host,SG_MAP) + +def myth_main(no_mount,install_call,dir_sg): + global bus + bus = dbus.SystemBus() + #search for root + f = open('/etc/fstab', 'r') + fstab=f.readlines() + f.close() + for i in fstab: + split_line=i.split() + if not split_line: + continue + + try: + if split_line[1] == "/" : + uuid_device = split_line[0] + break + except: + print("Couldn't find / in fstab") + sys.exit(1) + if uuid_device.startswith("UUID"): + uuid_device = uuid_device.split("=")[1] + cmd = "blkid -U %s" %uuid_device + device = runcmd(cmd)[1] + else: + device = uuid_device.strip() + #should have something like /dev/sda1 + #remove all the digits + device = ''.join([letter for letter in device if not letter.isdigit()]) + + + system_drive_list = scan_system() + for i in system_drive_list: + if i.block_path == device: + break + else: + print("Couldn't find root device in block list") + sys.exit(1) + if not install_call == True: + DB = MythDB() + host=gethostname() + else: + DB = None + host=gethostname() + + + print(" Disk: %s" %(i.get_name())) + i.set_mmount(True) + i.set_dir_sg(dir_sg) + i.set_partition("7") + i.set_disk_num(0) + i.lookup_format() + i.add_fstab(True) + #if not install_call: + i.mount_disk(no_mount) + i.write_config() + #no need to make the sub directories because the install process has taken care of it. + if dir_sg == True: + i.add_sg(DB,host,SG_MAP,'99',install_call) + + i.symlink() + cmd = "systemconfig.py -m fileshare" + runcmd(cmd) + +def reconstruct_storagegroups(): + print("\nRecreating storage groups from contents of /etc/storage.d/\n") + + DB = MythDB() + host=gethostname() + + for conf_file in glob.glob('%s/*.conf' %storage_dir): + parser = SafeConfigParser() + parser.read(conf_file) + mount_point = parser.get('storage', 'mountpoint') + mmount = parser.getboolean('storage', 'mmount') + try: + removed = parser.getboolean('storage', 'removed') + except: + removed = False + if removed: + print("Skipping: " + mount_point + " - removed") + continue + if not os.path.ismount(mount_point): + print("Skipping: " + mount_point + " - not mounted") + continue + try: + dir_sg = parser.getboolean('storage', 'storage_groups') + except configparser.NoOptionError as err: + print("SG not found in conf, get setting from DB") + dir_sg = False + # Get storage sroup directories from DB + recs = DB.getStorageGroup(groupname="Default") + for record in recs: + if record.dirname.startswith(mount_point): + dir_sg = True + # Write SG usage to conf + parser.set('storage','storage_groups',str(dir_sg)) + with open(conf_file, 'wb') as conf_file: + parser.write(conf_file) + + if dir_sg is True: + print("SGs Enabled for: " + mount_point) + print(" Creating directory structure:") + print(" %s" %mount_point) + for y in FS_LIST: + new_dir="%s/%s" %(mount_point,y) + try: + os.stat(new_dir) + print(" %s - exists" %y) + except: + os.makedirs(new_dir) + cmd="chown -R mythtv:mythtv /%s" %new_dir + runcmd(cmd) + cmd="chmod -R 775 /%s" %new_dir + runcmd(cmd) + print(" %s - created" %y) + + print(" Adding storage groups to DB") + if mmount is True: + sgweight=99 + else: + sgweight=0 + + for key in SG_MAP.keys(): + gn=key + hn=host + dn="%s/%s" %(mount_point,SG_MAP[key]) + with DB as c: + #delete old dir without trailing slash + c.execute("""delete from storagegroup where groupname = %s and hostname = %s and dirname = %s""", (gn,hn,dn.rstrip('/'))) + + try: + c.execute("""insert into storagegroup (groupname,hostname,dirname) values (%s,%s,%s)""",(gn,hn,dn)) + print(" Added: %s to storagegroup %s" %(dn,gn)) + except: + print(" Skipping: %s exists" %dn) + if sgweight > 0: + try: + sgw="SGweightPerDir:%s:%s" %(hn,dn) + + #delete old dir without trailing slash + c.execute("""delete from settings where value = %s and data = %s and hostname = %s""", (sgw.rstrip('/'),sgweight,hn)) + + if DB.settings[hn][sgw] == '99': + print(" Skipping: storage group weight DB entry exists") + else: + c.execute("""insert into settings (value,data,hostname) values (%s,%s,%s)""",(sgw,sgweight,hn)) + print(" Adding storage group weight of %s for %s\n" %(sgweight,gn)) + except: + print(" *Error setting storage group weight %s for %s\n" %(sgweight,gn)) + + else: + print("SGs Disabled for: " + mount_point) + return + +class reconstruct_path: + def __init__(self,conf_file): + self.conf_file = conf_file + parser = SafeConfigParser() + parser.read(self.conf_file) + self.config = configparser.RawConfigParser() + + self.uuid = parser.get('storage', 'uuid') + self.mount_point = parser.get('storage', 'mountpoint') + self.shareable = parser.get('storage', 'shareable') + self.myth_mount = parser.get('storage', 'mmount') + self.bind = self.myth_mount + self.disk_num = parser.get('storage', 'disk_num') + self.top_mount_dir = os.path.dirname(self.mount_point) + try: + self.fstype = parser.get('storage', 'fstype') + except: + self.fstype = self.get_fstype() + try: + self.removed = parser.get('storage', 'removed') + except: + self.removed = False + + def get_fstype(self): + cmd = "fsck -N UUID=%s" %self.uuid + tmpfstype = runcmd(cmd) + tmpfstype = tmpfstype[1].split('/sbin/fsck.') + tmpfstype = tmpfstype[1].split(' ') + self.fstype = tmpfstype[0] + self.write_config() + return self.fstype + + def get_conf(self): + return self.conf_file + + def get_uuid(self): + return self.uuid + + def get_mount_point(self): + return self.mount_point + + def get_shareable(self): + return self.shareable + + def get_is_myth_mount(self): + return self.myth_mount + + def get_disk_num(self): + return self.disk_num + + def get_removed(self): + return self.removed + + def create_mount_point(self): + try: + os.stat(self.mount_point) + except: + os.makedirs(self.mount_point) + + def find_options_type(self,fstab): + mp=['/myth', '/data/storage/disk0'] + for i in fstab: + split_line=i.split() + try: + if split_line[1] in mp: + options = split_line[3] + break + else: + options = "defaults" + mount_point = i + except: + options = "defaults" + return options,i + + def read_fstab(self): + f = open('/etc/fstab', 'r') + fstab=f.readlines() + f.close() + return fstab + + def check_in_fstab(self,fstab,check_path): + for line in fstab: + if line.find(check_path) > -1: + return True + return False + + def append_fstab(self,line): + new_fstab_line='\t'.join(line) + new_fstab_line="%s\n" %new_fstab_line + + f = open('/etc/fstab', 'a') + #f = open('/tmp/fstab', 'a') + f.write(new_fstab_line) + #f.write("\n") + f.close() + + def symlink(self): + print(" Creating symlink for /myth") + if not os.path.exists("/myth"): + cmd = "ln -s %s/media /myth " %(self.mount_point) + runcmd(cmd) + else: + print(" Skipping symlink, /myth already exists") + + def symlink_disk(self): + print(" Creating symlink for disk%s" %self.disk_num) + disk_ln="%s/disk%s" %(self.top_mount_dir,self.disk_num) + cmd = "ln -s %s %s" %(self.mount_point,disk_ln) + runcmd(cmd) + + def add_fstab(self): + #new_fstab_list=['UUID=', 'mount_point', 'auto', 'defaults', '0', '1'] + new_fstab_list=['UUID=', 'mount_point', self.fstype, 'defaults', '0', '1'] + fstab=self.read_fstab() + + if self.bind == "True": + self.symlink() + + if self.check_in_fstab(fstab,self.uuid) == True: + print(" Found UUID of disk in fstab, will not add it") + else: + print(" Adding storage to fstab") + if self.bind == "True" : + print(" Bind mount") + new_fstab_list=["/data/storage/disk0" , self.mount_point , "none" , "rw,bind", '0', '0'] + if self.check_in_fstab(fstab,self.mount_point) == False: + self.append_fstab(new_fstab_list) + else: + print(" Found bind storage in fstab, will not add it") + + else: + #check for old mount point and comment out + f = open('/etc/fstab', 'w') + for line in fstab: + if not line.startswith("#"): + if line.find(self.mount_point) > -1: + print(" Found old mount %s in fstab, commenting out" %self.mount_point) + line = "#"+line + f.write(line) + f.close() + + #construct new line + new_options = self.find_options_type(fstab)[0] + new_fstab_list[0]="UUID=%s" %self.uuid + new_fstab_list[1]=self.mount_point + new_fstab_list[3]=new_options + self.append_fstab(new_fstab_list) + + def mount_disk(self,no_mount=False): + try: + os.stat(self.mount_point) + except: + os.makedirs(self.mount_point) + if no_mount == False : + if os.path.ismount(self.mount_point): + print(" Disk already mounted, will not mount:\n %s" %self.mount_point) + pass + else: + print(" Mounting %s" %self.mount_point) + cmd = "mount %s" %self.mount_point + runcmd(cmd) + return + + def write_config(self): + print(" Writing /etc/storage.d conf file") + self.config.add_section('storage') + self.config.set('storage','uuid',self.uuid) + self.config.set('storage','mountpoint',self.mount_point) + self.config.set('storage','fstype',self.fstype) + self.config.set('storage','shareable','True') + self.config.set('storage','mmount',self.myth_mount) + self.config.set('storage','storage_groups',self.dir_sg) + self.config.set('storage','disk_num',self.disk_num) + + print(" %s" %self.conf_file) + with open(self.conf_file, 'wb') as self.conf_file: + self.config.write(self.conf_file) + return + +def reconstruct_mounts(no_mount): + print("\nRecreating disks from contents of /etc/storage.d/") + for conf_file in glob.glob('%s/*.conf' %storage_dir): + print("\n") + cf = reconstruct_path(conf_file) + + # skip if the disk was removed + if cf.get_removed(): + continue + #print cf.get_conf() + #print cf.get_uuid() + print(" Recreating %s" %cf.get_mount_point()) + #print cf.get_shareable() + #print cf.get_is_myth_mount() + #print cf.get_disk_num() + + cf.create_mount_point() + cf.add_fstab() + cf.symlink_disk() + cf.mount_disk(no_mount) + + print("\n\nDone recreating disks.\n") + pass + + +def usage(): + help=''' + add_storage.py finds and sets up disks for MythTV usage. + It's a powerful tool that could destroy data if not used correctly, + please be careful. + + Scanned disks are ignored if they are mounted or have been + previously skipped by add_storage.py. + + The file system type for disks added by add_storage.py is + automatically set to the type selected for the data partition + at install. + + Normal operations without options include (in this order): + Partition the disk + Format the disk + Add disk to /etc/fstab + Mount the disk + Create the directories + (if user enables MythTV storage groups) + Write out the disk config file to /etc/storage.d/ + Create disk# symlink at /data/storage/ + Create /myth symlink (if applicable) + Create MythTV storage group paths in MythTV database + (if user enables MythTV storage groups) + + Options: + --add_sg: Create the MythTV storage group directories and + database entries for database backups, TV + recordings, photos, music, streaming, videos + and artwork. + -h, --help: Show this help message. + --new_init: Erase the list of known disks and rescan. + --no_destruction: Will not partition or format the disk. + All other normal operations will be performed. + Can be used to import disks from other systems + however, add_storage.py only works with the first + partition on a disk and ignores all others. + --no_mount: Do not mount the disk. + All other normal operations will be performed. + --reconstruct: Recreate mount point, /myth symlink, fstab entry, + /data/storage/disk# symlink, and mount the disk. + --no_mount is the only option that works with + --reconstruct. + --reconstruct_sg: Recreate the MythTV storage group directories and + database entries if they don't exist. + No other options work with --reconstruct_sg. + --report: Scan disks and print new found disks. + ''' + print(help) + sys.exit(0) + + + + +if __name__ == "__main__": + scan_only = False + myth_mount = False + no_mount = False + destruction = True + install_call = False + dir_sg = False + reconstruct = False + reconstruct_sg = False + try: + os.remove("/tmp/scan_report") + except: + pass + + if not os.geteuid()==0: + sys.exit("\nRoot access is required to run this program.\n") + + if "--help" in sys.argv or "-h" in sys.argv: + usage() + + if "--install_call" in sys.argv: + install_call = True + + if "--no_mount" in sys.argv : + no_mount = True + + if "--no_destruction" in sys.argv: + destruction = False + + if "--new_init" in sys.argv : + remove_pickle() + + if "--report" in sys.argv : + scan_only = True + + if "--add_sg" in sys.argv: + dir_sg = True + + #there is no distinction between FE and BE sg anymore + #but leaving these for backwards compatibility + if "--add_fe_sg" in sys.argv: + dir_sg = True + + if "--add_be_sg" in sys.argv: + dir_sg = True + + if "--reconstruct" in sys.argv: + reconstruct = True + + if "--reconstruct_sg" in sys.argv: + reconstruct_sg = True + + if "--double_myth" in sys.argv: + myth_main(no_mount, install_call, dir_sg) + elif reconstruct == True: + reconstruct_mounts(no_mount) + elif reconstruct_sg == True: + reconstruct_storagegroups() + else: + main(scan_only, destruction, no_mount, install_call, dir_sg) diff --git a/linhes/linhes-system/add_storage.readme b/linhes/linhes-system/add_storage.readme new file mode 100644 index 0000000..d4435d5 --- /dev/null +++ b/linhes/linhes-system/add_storage.readme @@ -0,0 +1,3 @@ +Files here are autogenerated by add_storage.py. +They will be used by systemconfig to generate nfs, smb and mountpoint recovery. + diff --git a/linhes/linhes-system/balance_storage_groups.py b/linhes/linhes-system/balance_storage_groups.py new file mode 100755 index 0000000..0dd4b01 --- /dev/null +++ b/linhes/linhes-system/balance_storage_groups.py @@ -0,0 +1,153 @@ +#!/usr/bin/python + +import argparse, glob, operator, os, random, shutil, subprocess, sys, signal +shouldQuit = False + +def getFreeSpaceForDir(dir): + stats = os.statvfs(dir) + return (stats.f_bavail * stats.f_frsize) + +def getFreePercentForDir(dir): + stats = os.statvfs(dir) + total = (stats.f_blocks) + avail = (stats.f_bavail) + return (total - avail) / float(total) + +def getFileSize(fullPath): + return os.path.getsize(fullPath) + +def sizeof_fmt(num, suffix='B'): + for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']: + if abs(num) < 1024.0: + return "%3.1f %s%s" % (num, unit, suffix) + num /= 1024.0 + return "%.1f %s%s" % (num, 'Yi', suffix) + +def signal_handler(signal, frame): + print("\nWill quit when file has been moved.\nMoving File...") + global shouldQuit + shouldQuit = True + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--checkonly', action='store_true', help="Check only, don't move any files.") + parser.add_argument('-p', '--percent', type=int, default=7, help="The percentage difference between the most full dir and least full dir that will stop balancing.") + cmdargs = parser.parse_args() + + SGDIRS = [] + SGgrp = "Default" + + signal.signal(signal.SIGINT, signal_handler) + + print("\nBalance MythTV Storage Group Directories\nPress Ctrl+C to quit") + + # Get Storage Groups from MythDB + try: + from MythTV import MythDB + mythDB = MythDB() + records = mythDB.getStorageGroup() + except: + print("Couldn't connect to MythTV database.") + sys.exit(1) + + # Get Storage Group directories + for record in records: + if record.groupname == SGgrp: + dirname = record.dirname + SGDIRS.append(dirname) + + # If there are less than 2 directories defined bail as we can't move anything + if len(SGDIRS) < 2: + print("There are less than 2 directories defined. Exiting.") + sys.exit(0) + + while not shouldQuit: + SGDIRSdata = [] + print("\n------------------------------------------------") + print("'" + SGgrp + "' Storage Group Directories - Percent Used:") + # Get percent free and size free + for directory in SGDIRS: + # Check if SG path exists + if not os.path.exists(directory): + print(" " + directory + " - Not Mounted") + continue + freePcent = getFreePercentForDir(directory) + freeSize = getFreeSpaceForDir(directory) + SGDIRSdata.append([directory, freePcent, freeSize]) + print(" %s - %.2f%%" % (directory, freePcent * 100)) + + # Sort data on percent free + SGDIRSdata = sorted(SGDIRSdata, reverse=True, key=operator.itemgetter(1)) + #print SGDIRSdata + + # Check if SG has any ts, mpg or nuv files + i=0 + for dir in SGDIRSdata: + mostFull = SGDIRSdata[i] + i=i+1 + if len(glob.glob1(mostFull[0],"*.ts")) or len(glob.glob1(mostFull[0],"*.mpg")) or len(glob.glob1(mostFull[0],"*.nuv")): + break + else: + if i == 1: + print("------------------------------------------------") + print(" " + mostFull[0] + " - NO files to move") + + leastFull = SGDIRSdata[-1] + + print("------------------------------------------------") + print("Most Used Storage Group Directory with files to move: ") + print(" %s - %.2f%%" % (mostFull[0], mostFull[1] * 100)) + print("Least Used Storage Group Directory: ") + print(" %s - %.2f%%" % (leastFull[0], leastFull[1] * 100)) + + # Check if mostFull and leastFull are within the percent var of each other + if mostFull[1] - (float(cmdargs.percent) / 100) < leastFull[1]: + print("\nThe most used and least used storage group directories are\nwithin " + str(cmdargs.percent) + "% used of each other. No files will be moved.") + sys.exit() + + # Get random file from most used dir + fileToMove = random.choice([f for f in os.listdir(mostFull[0]) if f.endswith(".ts") or f.endswith(".mpg") or f.endswith(".nuv")]) + filePathToMove = mostFull[0] + "/" + fileToMove + + # Check that the file isn't too big for least used dir + fileSize = getFileSize(filePathToMove) + if (fileSize > getFreeSpaceForDir(leastFull[0])): + # Too big to move + print(filePathToMove + " is too big to move to " + leastFull[0]) + sys.exit() + + print("------------------------------------------------") + print("Move File:") + print(" " + filePathToMove) + print(" Size: " + sizeof_fmt(os.path.getsize(filePathToMove))) + print("To:") + print(" " + leastFull[0]) + print(" Available: " + sizeof_fmt(getFreeSpaceForDir(leastFull[0]))) + + # Move file + if cmdargs.checkonly: + print("------------------------------------------------") + print("Check Only option was used. No files were moved.") + shouldQuit = True + else: + print("------------------------------------------------") + print("Checking System Status...") + if subprocess.call(["/usr/bin/python2", "/usr/LH/bin/idle.py", "-s"]): + print(" System is busy. The file will not be moved.") + sys.exit() + print("Moving File...") + try: + shutil.move(filePathToMove, leastFull[0]) + # eg. src and dest are the same file + except shutil.Error as e: + print(('Error: %s' % e)) + except IOError as e: + print(('Error: %s' % e.strerror)) + + # Remove png files + print("------------------------------------------------") + print("Removing png Files:") + pngFiles = glob.glob(filePathToMove + "*.png") + for p in pngFiles: + os.remove(p) + print(" " + p) diff --git a/linhes/linhes-system/be_check.py b/linhes/linhes-system/be_check.py new file mode 100755 index 0000000..b5f7a64 --- /dev/null +++ b/linhes/linhes-system/be_check.py @@ -0,0 +1,35 @@ +#!/usr/bin/python +#simple program to check if mythbackend is up and running +#exit code of 0 is success, anything else means it can't connect +import sys +class Logger(object): + def __init__(self, filename="/tmp/Default.log"): + #self.terminal = sys.stdout + try: + self.log = open(filename, "a") + except: + pass + + def write(self, message): + try: + #self.terminal.write(message) + self.log.write(message) + except: + pass + +sys.stdout = Logger("/tmp/be_check.log") + + +from MythTV import MythBE,MythDB +#import datetime,time,sys,subprocess + + + + + +try: + be=MythBE() + db = MythDB() +except: + sys.exit(1) +sys.exit(0) diff --git a/linhes/linhes-system/checkXFSfrag.sh b/linhes/linhes-system/checkXFSfrag.sh new file mode 100755 index 0000000..d25eaa8 --- /dev/null +++ b/linhes/linhes-system/checkXFSfrag.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# +# Bash script by Gene Alexander (http://www.eracc.com/contact) +# of ERA Computers & Consulting (www.eracc.com, blog.eracc.com, shopping.eracc.com) +# Written using vim, the BEST plain text file editor in all of Creation. +# +# Teach yourself bash scripting: http://tldp.org/LDP/abs/html/index.html +# +# Purpose: To check fragmentation on XFS with xfs_db and run xfs_fsr on XFS mount points that +# are above a specific fragmentation threshold. +# +# What is xfs_db? Use 'man xfs_db' to find out. +# What is xfs_fsr? Use 'man xfs_fsr' to find out. +# +# Any busy files, such as open logs on /var/log, will be skipped. To defragment logs one should +# wrap this script with another script to stop and restart logging. Or, even better, write +# one's own script just for defragmentation of the logs. +# +# Warranty: NONE. Use at your own discretion and be aware that data loss is on your head if +# you choose to use this script. +# +# License: GPL 2.0 http://www.gnu.org/licenses/gpl-2.0.html +# +# Suggested Usage: crontab file for root +# 0 0 * * * /root/bin/chkxfsfrag # Run at midnight +# +# Original Release: 2011 December 15 (Merry Christmas!) +# DO NOT ALTER HEADER FROM THIS LINE UP. +# +e='/usr/bin/echo -e' # Use the echo command, not built-in. +xfsfsr=/usr/bin/xfs_fsr # Set variable with the path to xfs_fsr. +xfsdb=/usr/bin/xfs_db # Set variable with the path to xfs_db. +ionice=/usr/bin/ionice # Set variable with the path to ionice. +idle='/usr/LH/bin/idle.py -s' # Set varialbe with path to idle.py. +pctmax=12 # Set maxiumum frag percent needed for defrag. + # This is zero here for testing purposes only + # a higher number should be used in production. +array=`df -T|grep xfs|cut -f 1 --delim=" "` # Array of all XFS file systems. +for i in ${array[@]}; +do + #check for idle flag + if [[ $1 == "--idle" ]] + then + while ! $idle + do + echo "System is busy. Waiting 10 minutes before trying again." + sleep 600 + done + fi + #check that the device is SATA and skip defrag on SSDs + device=`echo ${i} | cut -f 3 --delim="/" | sed 's/[0-9]//g'` + isSATA=`cat /sys/block/${device}/queue/rotational` + if [[ $isSATA -eq 1 ]] + then + percentage=`$xfsdb -c frag -r ${i}|grep factor|cut -f 7 --delim=" "` + percent2=`$e $percentage|cut -f 1 --delim=.` + if [ "$percent2" -gt "$pctmax" ] + then + $e "${i} is $percentage fragmented. Running defragment on ${i}." + # Only uncomment one of the following two lines. + #$xfsfsr -v ${i} # Uncomment for verbose defrag. + $ionice -c3 $xfsfsr ${i} # Uncomment for quiet defrag. + else + $e "${i} is $percent2% fragmented and is below the fragmentation threshold of $pctmax%. Skipping." + fi + else + echo "${i} is an SSD. Skipping." + fi +done +exit 0 diff --git a/linhes/linhes-system/create_media_dirs.sh b/linhes/linhes-system/create_media_dirs.sh new file mode 100755 index 0000000..0aa44f5 --- /dev/null +++ b/linhes/linhes-system/create_media_dirs.sh @@ -0,0 +1,74 @@ +#!/bin/bash +#script to create the media directories +#used for building ISO (go.sh) and LiveCD (pre_install.sh) +#call it like so +#create_media_dirs.sh /top_level_path + +topdir=$1 + +if [ x$topdir = "x" ] +then + echo "Top level dir is empty" + exit 1 +fi + +if [ ! -d "$topdir" ] +then + echo "$topdir is not a dir or does not exist" + exit 2 +fi + + + +while read dirname +do + mkdir -p "${topdir}/${dirname}" + touch "${topdir}/${dirname}/.media" + chown mythtv:users "${topdir}/${dirname}" + chmod 775 "${topdir}/${dirname}" + chmod 775 "${topdir}/${dirname}/.media" +done <<EOF +media/tv +media/tv/live +media/gallery +media/photos +media/music +media/games/nes/roms +media/games/nes/screens +media/games/nes +media/games/pc/screens +media/games/pc +media/games/snes/roms +media/games/snes/screens +media/games/snes +media/games/xmame/cabs +media/games/xmame/flyers +media/games/xmame/hiscores +media/games/xmame/history +media/games/xmame/roms +media/games/xmame/screens +media/games/xmame +media/games +media/video +media/tmp +media/archive +media/recordings +media/streaming +media/artwork/trailers +media/artwork/coverart +media/artwork/fanart +media/artwork/musicart +media/artwork/screenshots +media/artwork/banners +media/games/screenshots +media/games/fanart +media/games/boxart +backup +backup/system_backups +backup/user_backups +backup/mythtv_backups +EOF + +chown -R mythtv:users ${topdir} +chmod -R 775 ${topdir} +chmod 1777 "${topdir}/media/tmp" diff --git a/linhes/linhes-system/empty_storage_groups.py b/linhes/linhes-system/empty_storage_groups.py new file mode 100755 index 0000000..4b5aff8 --- /dev/null +++ b/linhes/linhes-system/empty_storage_groups.py @@ -0,0 +1,195 @@ +#!/usr/bin/python + +import argparse, glob, operator, os, random, shutil, subprocess, sys, signal +shouldQuit = False + +def getFreeSpaceForDir(dir): + stats = os.statvfs(dir) + return (stats.f_bavail * stats.f_frsize) + +def getFreePercentForDir(dir): + stats = os.statvfs(dir) + total = (stats.f_blocks) + avail = (stats.f_bavail) + return (total - avail) / float(total) + +def getFileSize(fullPath): + return os.path.getsize(fullPath) + +def sizeof_fmt(num, suffix='B'): + for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']: + if abs(num) < 1024.0: + return "%3.1f %s%s" % (num, unit, suffix) + num /= 1024.0 + return "%.1f %s%s" % (num, 'Yi', suffix) + +def signal_handler(signal, frame): + print("\nWill quit when file has been moved.\nMoving File...") + global shouldQuit + shouldQuit = True + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--checkonly', action='store_true', help="Check only, don't move any files.") + cmdargs = parser.parse_args() + + SGs = [] + SGDIRS = [] + SGgrp = "Default" + SGselectdata = [] + + signal.signal(signal.SIGINT, signal_handler) + + print("\nEmpty a MythTV Storage Group Directory\nPress Ctrl+C to quit") + + # Get Storage Groups from MythDB + try: + from MythTV import MythDB + mythDB = MythDB() + records = mythDB.getStorageGroup() + recs = mythDB.getStorageGroup() + except: + print("Couldn't connect to MythTV database.") + sys.exit(1) + + # get list of non duplicate SGs + for record in records: + SGs.append(record.groupname) + SGs=list(set(SGs)) + # Move Default to top of list + if "Default" in SGs: + SGs.remove("Default") + SGs.insert(0,"Default") + + # ask user which SG to use + print("\n------------------------------------------------") + print("Storage Groups:") + for i,sg in enumerate(SGs): + print(str(i+1) + ": " + sg) + + try: + SGselect=input("\nEnter the number of the storage group to use (default 1): ") or 1 + SGselect=int(SGselect) + if SGselect > len(SGs) or SGselect < 1: + SGselect=int("e") + except ValueError: + print("You must enter a number between 1 and " + str(len(SGs)) + ". Exiting.") + sys.exit(0) + + SGgrp=SGs[SGselect-1] + + # Get Storage Group directories + for record in recs: + if record.groupname == SGgrp: + dirname = record.dirname + SGDIRS.append(dirname) + + # If there are less than 2 directories defined bail as we can't move anything + if len(SGDIRS) < 2: + print("There are less than 2 directories defined. Exiting.") + sys.exit(0) + + while not shouldQuit: + SGDIRSdata = [] + print("\n------------------------------------------------") + print("'" + SGgrp + "' Storage Group Directories - Percent Used:") + SGcnt=0 + # Get percent free and size free + for directory in SGDIRS: + # Check if SG path exists + if not os.path.exists(directory): + print(" " + directory + " - Not Mounted") + continue + # Check if SG has data files to move + if len(glob.glob1(directory,"*.ts")) or len(glob.glob1(directory,"*.mpg")) or len(glob.glob1(directory,"*.nuv")) or len(glob.glob1(directory,"*.jpg")): + freePcent = getFreePercentForDir(directory) + freeSize = getFreeSpaceForDir(directory) + SGDIRSdata.append([directory, freePcent, freeSize]) + SGcnt=SGcnt+1 + print("%s: %s - %.2f%%" % (SGcnt, directory, freePcent * 100)) + else: + # Check if the selected SG dir has no data files exit + if SGselectdata and SGselectdata[0] == directory: + print("\n'" + SGgrp + "' Storage Group directories have no files to move. Exiting") + sys.exit(0) + + # Exit if no SGs with data found + if SGcnt is 0: + print("\n'" + SGgrp + "' Storage Group directories have no files to move. Exiting.") + sys.exit(0) + + # Ask user to select which SG to empty if not already selected + if not SGselectdata: + try: + SGDIRselect=int(input("\nEnter the number of the storage group directory to empty: ")) + if SGDIRselect > SGcnt or SGDIRselect < 1: + SGDIRselect=int("e") + except ValueError: + print("You must enter a number between 1 and %s. Exiting." %SGcnt) + sys.exit(0) + + SGselectdata=SGDIRSdata[SGDIRselect-1] + + # Sort data on percent free + SGDIRSdata = sorted(SGDIRSdata, reverse=True, key=operator.itemgetter(1)) + leastFull = SGDIRSdata[-1] + + # Make sure leastFull and SGselectdata are not the same dir + if leastFull[0] == SGselectdata[0]: + leastFull = SGDIRSdata[-2] + + # Get random file from user selected dir + fileToMove = random.choice([f for f in os.listdir(SGselectdata[0]) if f.endswith(".ts") or f.endswith(".mpg") or f.endswith(".nuv") or f.endswith(".jpg")]) + filePathToMove = SGselectdata[0] + "/" + fileToMove + + # Check that the file isn't too big for least used dir + fileSize = getFileSize(filePathToMove) + if (fileSize > getFreeSpaceForDir(leastFull[0])): + # Too big to move + print(filePathToMove + " is too big to move to " + leastFull[0]) + sys.exit() + + print("------------------------------------------------") + print("Move File:") + print(" " + filePathToMove) + print(" Size: " + sizeof_fmt(os.path.getsize(filePathToMove))) + print("To:") + print(" " + leastFull[0]) + print(" Available: " + sizeof_fmt(getFreeSpaceForDir(leastFull[0]))) + + # Move file + if cmdargs.checkonly: + print("------------------------------------------------") + print("Check Only option was used. No files were moved.") + shouldQuit = True + else: + print("------------------------------------------------") + print("Checking System Status...") + if subprocess.call(["/usr/bin/python2", "/usr/LH/bin/idle.py", "-s"]): + print(" System is busy. The file will not be moved.") + sys.exit() + print("Moving File...") + try: + shutil.move(filePathToMove, leastFull[0]) + # eg. src and dest are the same file + except shutil.Error as e: + a=input("\n%s. Overwrite destination (y/n)? " % e) + if a == "y" or a == "Y": + os.remove(leastFull[0] + "/" + fileToMove) + shutil.move(filePathToMove, leastFull[0]) + else: + b=input("\nRemove %s (y/n)? " % filePathToMove) + if b == "y" or b == "Y": + os.remove(filePathToMove) + # eg. source or destination doesn't exist + except IOError as e: + print(('Error: %s' % e.strerror)) + + # Remove png files for Default & LiveTV SGs + if SGgrp == "Default" or SGgrp == "LiveTV": + print("------------------------------------------------") + print("Removing png Files:") + pngFiles = glob.glob(filePathToMove + "*.png") + for p in pngFiles: + os.remove(p) + print(" " + p) diff --git a/linhes/linhes-system/enableIRWake.sh b/linhes/linhes-system/enableIRWake.sh new file mode 100755 index 0000000..8a9847a --- /dev/null +++ b/linhes/linhes-system/enableIRWake.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +for vendProdID in `lsusb | sed -e 's/.*ID \([a-f0-9]\+:[a-f0-9]\+\).*/\1/g'` +do + foundRemote=`grep -i "$vendProdID" /usr/share/linhes/templates/remotes/receiver_usb.id` + if [[ $? = 0 ]] + then + #echo Found: $foundRemote + vendID=`echo $vendProdID | cut -d":" -f1` + prodID=`echo $vendProdID | cut -d":" -f2` + for usbDevice in `grep . /sys/bus/usb/devices/*/power/wakeup | cut -d"/" -f6` + do + foundVendID=`cat /sys/bus/usb/devices/$usbDevice/idVendor` + foundProdID=`cat /sys/bus/usb/devices/$usbDevice/idProduct` + if [[ $foundVendID == $vendID && $foundProdID == $prodID ]] + then + echo "Enable wake for $foundRemote on $usbDevice" + sudo sh -c "echo 'enabled' > /sys/bus/usb/devices/$usbDevice/power/wakeup" + fi + done + fi +done diff --git a/linhes/linhes-system/find_orphans.py b/linhes/linhes-system/find_orphans.py new file mode 100755 index 0000000..964d6e4 --- /dev/null +++ b/linhes/linhes-system/find_orphans.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python2 + +from MythTV import MythDB, MythBE, Recorded, MythError +from socket import timeout + +import os +import sys + +def human_size(s): + s = float(s) + o = 0 + while s > 1000: + s /= 1000 + o += 1 + return str(round(s,1))+('B ','KB','MB','GB','TB')[o] + +class File( str ): + def __new__(self, host, group, path, name, size): + return str.__new__(self, name) + def __init__(self, host, group, path, name, size): + self.host = host + self.group = group + self.path = path + self.size = int(size) + def pprint(self): + name = '%s: %s' % (self.host, os.path.join(self.path, self)) + print(' {0:<90}{1:>8}'.format(name, human_size(self.size)).encode('utf-8')) + def delete(self): + be = MythBE(self.host, db=DB) + be.deleteFile(self, self.group) + +class MyRecorded( Recorded ): + _table = 'recorded' + def pprint(self): + name = '{0.hostname}: {0.title}'.format(self) + if self.subtitle: + name += ' - '+self.subtitle + print(' {0:<70}{1:>28}'.format(name,self.basename).encode('utf-8')) + +def printrecs(title, recs): + print(title) + for rec in sorted(recs, key=lambda x: x.title): + rec.pprint() + print('{0:>87}{1:>12}'.format('Count:',len(recs)).encode('utf-8')) + +def printfiles(title, files): + print(title) + for f in sorted(files, key=lambda x: x.path): + f.pprint() + size = sum([f.size for f in files]) + print('{0:>87}{1:>12}'.format('Total:',human_size(size)).encode('utf-8')) + +def populate(host=None): + unfiltered = [] + kwargs = {'livetv':True} + if host: + with DB as c: + c.execute("""SELECT count(1) FROM settings + WHERE hostname=%s AND value=%s""", + (host, 'BackendServerIP')) + if c.fetchone()[0] == 0: + raise Exception('Invalid hostname specified on command line.') + hosts = [host] + kwargs['hostname'] = host + else: + with DB as c: + c.execute("""SELECT hostname FROM settings + WHERE value='BackendServerIP'""") + hosts = [r[0] for r in c.fetchall()] + for host in hosts: + for sg in DB.getStorageGroup(): + if sg.groupname in ('Videos','Banners','Coverart','Fanart',\ + 'Music','MusicArt', 'Photographs',\ + 'Screenshots','Trailers'): + continue + try: + dirs,files,sizes = BE.getSGList(host, sg.groupname, sg.dirname) + for f,s in zip(files,sizes): + newfile = File(host, sg.groupname, sg.dirname, f, s) + if newfile not in unfiltered: + unfiltered.append(newfile) + except: + pass + + recs = list(DB.searchRecorded(**kwargs)) + + zerorecs = [] + orphvids = [] + for rec in list(recs): + if rec.basename in unfiltered: + recs.remove(rec) + i = unfiltered.index(rec.basename) + f = unfiltered.pop(i) + if f.size < 1024: + zerorecs.append(rec) + name = rec.basename.rsplit('.',1)[0] + for f in list(unfiltered): + if name in f: + unfiltered.remove(f) + for f in list(unfiltered): + if not (f.endswith('.mpg') or f.endswith('.nuv') or f.endswith('.ts')): + continue + orphvids.append(f) + unfiltered.remove(f) + + orphimgs = [] + for f in list(unfiltered): + if not f.endswith('.png'): + continue + orphimgs.append(f) + unfiltered.remove(f) + + dbbackup = [] + for f in list(unfiltered): + if 'sql' not in f: + continue + dbbackup.append(f) + unfiltered.remove(f) + + return (recs, zerorecs, orphvids, orphimgs, dbbackup, unfiltered) + +def delete_recs(recs): + printrecs('The following recordings will be deleted', recs) + print('Are you sure you want to continue? (yes/no)') + try: + res = input('> ') + while True: + if res == 'yes': + for rec in recs: + rec.delete(True, True) + break + elif res == 'no': + break + else: + res = input("'yes' or 'no' > ") + except MythError: + name = '{0.hostname}: {0.title}'.format(rec) + if rec.subtitle: + name += ' - '+rec.subtitle + print("Warning: Failed to delete '" + name + "'") + except KeyboardInterrupt: + pass + except EOFError: + sys.exit(0) + +def delete_files(files): + printfiles('The following files will be deleted', files) + print('Are you sure you want to continue? (yes/no)') + try: + res = input('> ') + while True: + if res == 'yes': + for f in files: + f.delete() + break + elif res == 'no': + break + else: + res = input("'yes' or 'no' > ") + except KeyboardInterrupt: + pass + except EOFError: + sys.exit(0) + +def main(host=None): + while True: + recs, zerorecs, orphvids, orphimgs, dbbackup, unfiltered = populate(host) + + if len(recs): + printrecs("Recordings with missing files", recs) + if len(zerorecs): + printrecs("Zero byte recordings", zerorecs) + if len(orphvids): + printfiles("Orphaned video files", orphvids) + if len(orphimgs): + printfiles("Orphaned snapshots", orphimgs) + if len(dbbackup): + printfiles("Database backups", dbbackup) + if len(unfiltered): + printfiles("Other files", unfiltered) + if not printOnly: + opts = [] + if len(recs): + opts.append(['Delete orphaned recording entries', delete_recs, recs]) + if len(zerorecs): + opts.append(['Delete zero byte recordings', delete_recs, zerorecs]) + if len(orphvids): + opts.append(['Delete orphaned video files', delete_files, orphvids]) + if len(orphimgs): + opts.append(['Delete orphaned snapshots', delete_files, orphimgs]) + if len(unfiltered): + opts.append(['Delete other files', delete_files, unfiltered]) + opts.append(['Refresh list', None, None]) + print('Please select from the following:') + for i, opt in enumerate(opts): + if opt[0] == "Refresh list": + print(' R. {1}'.format(i+1, opt[0])) + refreshNum=i+1 + else: + print(' {0}. {1}'.format(i+1, opt[0])) + + print(' Q. Quit') + try: + inner = True + res = input('> ') + while inner: + try: + res = int(res) + except: + if res == "Q" or res == "q": + sys.exit(0) + elif res == "R" or res == "r": + res = refreshNum + else: + res = input('Invalid selection > ') + continue + if (res <= 0) or (res > len(opts)): + res = input('Invalid selection > ') + continue + break + opt = opts[res-1] + if opt[1] is None: + continue + else: + opt[1](opt[2]) + + except KeyboardInterrupt: + break + except EOFError: + sys.exit(0) + else: + sys.exit(0) +DB = MythDB() +BE = MythBE(db=DB) +DB.searchRecorded.handler = MyRecorded +DB.searchRecorded.dbclass = MyRecorded + +if __name__ == '__main__': + global printOnly + if "--printonly" in sys.argv : + printOnly=True + else: + printOnly=False + + + if len(sys.argv) == 2 and sys.argv[1] != "--printonly": + main(sys.argv[1]) + else: + main() diff --git a/linhes/linhes-system/fstrim.hook b/linhes/linhes-system/fstrim.hook new file mode 100644 index 0000000..3f80657 --- /dev/null +++ b/linhes/linhes-system/fstrim.hook @@ -0,0 +1,9 @@ +[Trigger] +Operation = Install +Type = Package +Target = linhes-system + +[Action] +Description = Enable fstrim.timer... +When = PostTransaction +Exec = /usr/bin/systemctl enable fstrim.timer diff --git a/linhes/linhes-system/idle.py b/linhes/linhes-system/idle.py new file mode 100755 index 0000000..373ae24 --- /dev/null +++ b/linhes/linhes-system/idle.py @@ -0,0 +1,375 @@ +#!/usr/bin/python + +import argparse, os, re, subprocess, sys, time +from datetime import datetime, date, timedelta + +def msg(cmdargs,msg): + if cmdargs.silent is False: + print("%s" %msg) + +def mythshutdownlock_check(cmdargs,cursor): + if (cmdargs.lock): + msg(cmdargs," Checking mythshutdown for lock...") + try: + cursor.execute("select data from settings where value = 'MythShutdownLock'") + results=cursor.fetchone() + except: + return True + lock=results[0] + if int(lock) == 0 : + msg(cmdargs," mythshutdown is NOT locked.") + return True + else: + msg(cmdargs," mythshutdown is locked.") + return False + else: + return True + +def dailywake_check(cmdargs,cursor): + if (cmdargs.daily): + msg(cmdargs," Checking if in a daily wake period...") + dailyWake=False + today = date.today() + now = datetime.now() + try: + cursor.execute("select data from settings where value = 'DailyWakeupStartPeriod1'") + results=cursor.fetchone() + p1Start=datetime.strptime(' '.join([str(today), results[0]]), "%Y-%m-%d %H:%M") + cursor.execute("select data from settings where value = 'DailyWakeupEndPeriod1'") + results=cursor.fetchone() + p1End=datetime.strptime(' '.join([str(today), results[0]]), "%Y-%m-%d %H:%M") + cursor.execute("select data from settings where value = 'DailyWakeupStartPeriod2'") + results=cursor.fetchone() + p2Start=datetime.strptime(' '.join([str(today), results[0]]), "%Y-%m-%d %H:%M") + cursor.execute("select data from settings where value = 'DailyWakeupEndPeriod2'") + results=cursor.fetchone() + p2End=datetime.strptime(' '.join([str(today), results[0]]), "%Y-%m-%d %H:%M") + except: + print("error") + return True + + # Check for time periods that cross midnight + if (p1End < p1Start): + if (now > p1End): + p1End = p1End + timedelta(days=1) + else: + p1Start = p1Start + timedelta(days=-1) + if (p2End < p2Start): + if (now > p2End): + p2End = p2End + timedelta(days=1) + else: + p2Start = p2Start + timedelta(days=-1) + + #Check for one of the daily wakeup periods + if (p1Start != p1End): + if (now >= p1Start and now <= p1End): + msg(cmdargs," Currently in daily wake period 1.") + return False + if (p2Start != p2End): + if (now >= p2Start and now <= p2End): + msg(cmdargs," Currently in daily wake period 2.") + return False + + #Are we about to start a daily wakeup period using the -t TIME var + if (p1Start != p1End): + delta=p1Start-now + if (delta.seconds >= 0 and delta.seconds <= cmdargs.time * 60): + msg(cmdargs," Daily wake period 1 will start in less than %s minutes." %cmdargs.time) + return False + if (p2Start != p2End): + delta=p2Start-now + if (delta.seconds >= 0 and delta.seconds <= cmdargs.time * 60): + msg(cmdargs," Daily wake period 2 will start in less than %s minutes." %cmdargs.time) + return False + + msg(cmdargs," Currently NOT in a daily wake period.") + return True + else: + return True + +def schemalock_check(cmdargs,cursor): + msg(cmdargs," Checking if the schema is locked...") + try: + cursor.execute("select count(*) from schemalock") + results=cursor.fetchone() + except: + return True + schemalock=results[0] + if schemalock == 0: + msg(cmdargs," The schema is NOT locked.") + return True + else: + msg(cmdargs," The schema is locked.") + return False + +def in_use(cmdargs,cursor): + msg(cmdargs," Checking if programs are in use...") + try: + cursor.execute("select count(*) from inuseprograms") + results=cursor.fetchone() + except: + return True + prginuse=results[0] + if prginuse == 0 : + msg(cmdargs," Programs are NOT in use.") + return True + else: + msg(cmdargs," Programs are in use.") + return False + +def job_check(cmdargs,cursor): + msg(cmdargs," Checking jobqueue for active jobs...") + try: + cursor.execute("select count(*) from jobqueue where status between 2 and 5") + results=cursor.fetchone() + except: + return True + jobs=results[0] + if jobs == 0 : + msg(cmdargs," No jobs are active.") + return True + else: + msg(cmdargs," Jobs are active.") + return False + +def upcoming_check(cmdargs,mythBE): + msg(cmdargs," Checking for recordings in the next %s minutes..." %cmdargs.time) + try: + upcoming = mythBE.getUpcomingRecordings() + except: + msg(cmdargs," Could not get upcoming recordings.") + return True + time_diff=10000 + r=0 + for i in upcoming: + r += 1 + if r > 1: + break + show=str(i) + show=show.strip() + showtime=re.split("[-+]\d\d:\d\d",str(i.starttime))[0] + now=time.time() + rec_time=time.strptime( showtime ,"%Y-%m-%d %H:%M:%S" ) + r=time.mktime(rec_time) + time_diff = ( r - now ) / 60 + + if ( time_diff > cmdargs.time) : + msg(cmdargs," No recordings starting in %s minutes." %cmdargs.time) + return True + else: + msg(cmdargs," A recording is starting in %s minutes." %int(time_diff)) + return False + +def mfd_check(cmdargs): + msg(cmdargs," Checking if mythfilldatabase is running...") + with open(os.devnull, "w") as fnull: + mythfilldatabase_ret = subprocess.call(["pidof", "mythfilldatabase"], stdout=fnull) + if mythfilldatabase_ret == 0 : + msg(cmdargs," mythfilldatabase is running.") + return False + else: + msg(cmdargs," mythfilldatabase is NOT running.") + return True + +def mythtvsetup_check(cmdargs): + msg(cmdargs," Checking if mythtv-setup is running...") + with open(os.devnull, "w") as fnull: + mythsetup_ret = subprocess.call(["pidof", "mythtv-setup"], stdout=fnull) + if mythsetup_ret == 0 : + msg(cmdargs," mythtv-setup is running.") + return False + else: + msg(cmdargs," mythtv-setup is NOT running.") + return True + +def userlogins_check(cmdargs): + if (cmdargs.logins): + u=False + msg(cmdargs," Checking for users logged in...") + users=subprocess.check_output("who") + names=([x.split() for x in users.splitlines()]) + for i in names: + if (i[0] == "mythtv" and i[4] == "(:0)"): + msg(cmdargs," Ignoring %s %s" %(i[0],i[4])) + else: + msg(cmdargs," User logged in: %s %s" %(i[0],i[4])) + u=True + if u: + return False + else: + return True + else: + return True + +def sambafiles_check(cmdargs): + if (cmdargs.sambafiles): + msg(cmdargs," Checking if Samba files are in use...") + try: + smbstatus=subprocess.check_output(["smbstatus", "-L"]) + except: + smbstatus="No locked files" + if "No locked files" in smbstatus: + msg(cmdargs," Samba files are NOT in use.") + return True + else: + msg(cmdargs," Samba files are in use.") + return False + else: + return True + +def mythfe_check(cmdargs,cursor,mythDB): + #checks to see if a frontend is considered idle + # True means FE is idle + + if ( cmdargs.runningfe ): + msg(cmdargs," Checking for running and playing mythfrontends...") + else: + msg(cmdargs," Checking for playing mythfrontends...") + try: + cursor.execute("select distinct hostname from settings where hostname is not null;") + frontends=cursor.fetchall() + except: + return True + + for i in frontends: + try: + msg(cmdargs," Checking %s's mythfrontend status..." %i) + frontend = mythDB.getFrontend(''.join(i)) + if ( cmdargs.runningfe ): + msg(cmdargs," %s's mythfrontend is RUNNING." %i) + return False + location = frontend.sendQuery('Location') + + if location == "standbymode": + msg(cmdargs," %s's mythfrontend is in Standby Mode." %i) + continue + + if ( location.startswith('Playback ') ): + msg(cmdargs," %s's mythfrontend is PLAYING." %i) + return False + else: + msg(cmdargs," %s's mythfrontend is NOT playing." %i) + + if '.xml' in location or 'mainmenu' in location: + msg(cmdargs," %s's mythfrontend is in MENUS." %i) + else: + #FE is not in menus, so it must be active in a plugin + msg(cmdargs," %s's mythfrontend is NOT in menus." %i) + return False + except: + msg(cmdargs," Could not connect to %s's mythfrontend." %i) + + if ( cmdargs.runningfe ): + msg(cmdargs," mythfrontends are not running or playing or are in menus.") + else: + msg(cmdargs," mythfrontends are not playing or are in menus.") + + return True + +def usage(): + line = ''' + idle.py checks if the system is idle. + Use idle.py -h to see options. + + idle.py checks these parts of the system in this order to + determine if it is idle: + - (option -g) users are logged in return busy + ignores mythtv (:0) for busy + - (option -f) Samba files are in use return busy + - (option -l) mythshutdown is locked return busy + - (option -d) in a daily wake period or + about to start a daily wake period return busy + checks the next 15 minutes. -t TIME changes time + - schema is locked return busy + - there are in use programs return busy + - there are active jobs in the job queue return busy + - mythfilldatabase is running return busy + - mythtv-setup is running return busy + - there are upcoming recordings return busy + checks the next 15 minutes. -t TIME changes time + - (option -r) mythfrontends running return busy + - mythfrontends playing back a recording or video return busy + - mythfrontends not in menus return busy + + idle.py stops checking and returns false (busy) when the first busy is found. + ''' + print(line) + sys.exit(0) + +def main(args=[False]): + parser = argparse.ArgumentParser() + parser.add_argument('-d', '--daily', action='store_true', help='Include daily wake & about to start wake in system busy. (default: daily wake & about to start wake is system idle)') + parser.add_argument('-g', '--logins', action='store_true', help='Include user logins in system busy. Ignores mythtv (:0) in system busy.') + parser.add_argument('-f', '--sambafiles', action='store_true', help='Include Samba files in use in system busy.') + parser.add_argument('-l', '--lock', action='store_true', help='Include mythshutdown lock in system busy. (default: mythshutdown lock is system idle)') + parser.add_argument('-r', '--runningfe', action='store_true', help='Include running mythfrontends in system busy. (default: running mythfrontends are system idle)') + parser.add_argument('-s', '--silent', action='store_true', help='Run without printing output. Recommended for use in cron jobs or scripts.') + parser.add_argument('-t', '--time', type=int, default=15, help='Minutes of idle time needed to return idle for upcoming recordings and daily wake.') + parser.add_argument('-u', '--usage', action='store_true', help='Print usage instructions.') + if args[0] is False: + cmdargs = parser.parse_args() + else: + cmdargs = parser.parse_args(args) + + if cmdargs.usage: + usage() + idle=True + msg(cmdargs,"Checking system idle...") + + if (userlogins_check(cmdargs)): + idle = True + else: + idle = False + + if (idle and sambafiles_check(cmdargs)): + idle = True + else: + idle = False + + try: + from MythTV import MythDB + mythDB = MythDB() + cursor = mythDB.cursor() + db_conn=True + except: + msg(cmdargs,"Couldn't connect to MythTV database.") + db_conn=False + + try: + from MythTV import MythBE + mythBE = MythBE() + be_conn=True + except: + msg(cmdargs,"Couldn't connect to MythTV backend.") + be_conn=False + + if ( db_conn and idle ): + if (mythshutdownlock_check(cmdargs,cursor) and dailywake_check(cmdargs,cursor) and schemalock_check(cmdargs,cursor) and in_use(cmdargs,cursor) and job_check(cmdargs,cursor)): + idle=True + else: + idle=False + + if ( be_conn and idle ): + if (mfd_check(cmdargs) and mythtvsetup_check(cmdargs) and upcoming_check(cmdargs,mythBE)): + idle=True + else: + idle=False + + if ( db_conn and idle ): + if (mythfe_check(cmdargs,cursor,mythDB)): + idle=True + else: + idle=False + + if ( idle ): + msg(cmdargs,"System is idle.") + else: + msg(cmdargs,"System is busy.") + return idle + +if __name__ == "__main__": + idle=main() + if ( idle ): + exit(0) + else: + exit(1) diff --git a/linhes/linhes-system/jobqueue_helper.py b/linhes/linhes-system/jobqueue_helper.py new file mode 100755 index 0000000..6a567cb --- /dev/null +++ b/linhes/linhes-system/jobqueue_helper.py @@ -0,0 +1,63 @@ +#!/usr/bin/python + +import argparse, os, re, subprocess, sys, time +from MythTV import MythDB, Job + +mythDB = MythDB() +cursor = mythDB.cursor() + +def set_cmds(cmdargs,job): + #print "Setting cmds on job %s to %s" %(cmdargs.jobid,cmdargs.cmd) + cursor.execute("update jobqueue set cmds = '%s' where id = '%s'" %(cmdargs.cmd,cmdargs.jobid)) + +def set_comment(cmdargs,job): + #print "Setting comment on job %s to %s" %(cmdargs.jobid,cmdargs.comment) + job.setComment("%s" %cmdargs.comment) + +def set_status(cmdargs,job): + #print "Setting status on job %s to %s" %(cmdargs.jobid,cmdargs.status) + job.setStatus("%s" %cmdargs.status) + +def run_cursor(cmdargs): + cursor.execute("%s" %cmdargs.man_cursor) + results=cursor.fetchone() + print(results[0]) + +def usage(): + line = ''' + jobqueue_helper.py provides MythTV job queue functions + using python bindings for bash scripts. + Use jobqueue_helper.py -h to see options. + ''' + print(line) + sys.exit(0) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('-cs', '--comment_set', action='store', dest='comment', help='Set the comment of the jobid') + parser.add_argument('-cmds', '--cmd_set', type=int, default=77777, action='store', dest='cmd', help='Set the cmd of the jobid') + parser.add_argument('-ss', '--status_set', type=int, action='store', dest='status', help='Set the status of the jobid') + action = parser.add_mutually_exclusive_group(required=True) + action.add_argument('-j', '--jobid', type=int, help='jobid of the job to control') + action.add_argument('-m', '--man_cursor', action='store', dest='man_cursor', help='Manual mysql cursor command') + action.add_argument('-u', '--usage', action='store_true', help='Print usage instructions.') + + cmdargs = parser.parse_args() + + if cmdargs.usage: + usage() + + if cmdargs.jobid: + job = Job(cmdargs.jobid) + + if cmdargs.comment: + set_comment(cmdargs, job) + + if cmdargs.status: + set_status(cmdargs, job) + + if cmdargs.cmd != 77777: + set_cmds(cmdargs, job) + + if cmdargs.man_cursor: + run_cursor(cmdargs) diff --git a/linhes/linhes-system/lh_systemstart.sh b/linhes/linhes-system/lh_systemstart.sh new file mode 100755 index 0000000..cd32cdf --- /dev/null +++ b/linhes/linhes-system/lh_systemstart.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +. /etc/profile +#. /etc/systemconfig + +function msg(){ + /usr/bin/notify-send --app-name="LinHES" --icon=dialog-information "$1" "$2" +} + +function applyUIsettings(){ + plasma-apply-wallpaperimage /usr/share/linhes/templates/lights-bud-abstract-4k-cq.jpg & + plasma-apply-colorscheme BreezeDark & + kwriteconfig5 --group KDE --key SingleClick false + kwriteconfig5 --file ~/.config/kscreenlockerrc --group Daemon --key Autolock false + kwriteconfig5 --file ~/.config/kscreenlockerrc --group Daemon --key LockOnResume false + cp /usr/share/linhes/templates/plasma-org.kde.plasma.desktop-appletsrc ~/.config/ + #restart plasma + kquitapp5 plasmashell + kstart5 plasmashell + #disable file indexing + balooctl disable & +} + + + +#-------MAIN------- +/usr/bin/enableIRWake.sh & +msg "Welcome to LinHES 9!" +applyUIsettings diff --git a/linhes/linhes-system/lh_systemstart.sh.desktop b/linhes/linhes-system/lh_systemstart.sh.desktop new file mode 100644 index 0000000..478f3ab --- /dev/null +++ b/linhes/linhes-system/lh_systemstart.sh.desktop @@ -0,0 +1,7 @@ +[Desktop Entry] +Exec=/usr/bin/lh_systemstart.sh +Icon=dialog-scripts +Name=lh_systemstart.sh +Path= +Type=Application +X-KDE-AutostartScript=true diff --git a/linhes/linhes-system/myth2mkv b/linhes/linhes-system/myth2mkv new file mode 100755 index 0000000..51a0c9b --- /dev/null +++ b/linhes/linhes-system/myth2mkv @@ -0,0 +1,466 @@ +#!/bin/bash +# +# Convert video to AVC-1 / h264 +# +# version 0.27-001 +# +# Prerequisites: +# - mythtv >= 0.27 +# - handbrake-cli +# - mplayer +# - mkvtoolnix +# - jobqueue_helper.py +# +# Arguments +# $1 must be the directory/file of the recording +# $2 must be chanid +# $3 must be starttime +# $4 must be title +# $5 must be subtitle +# $6 must be jobid +# $7 must be quality of encode +# +# As a MythTV user job: +# myth2mkv "%DIR%/%FILE%" "%CHANID%" "%STARTTIME%" "%TITLE%" "%SUBTITLE%" "%JOBID%" "HP|HQ|MQ|LQ" +# Select only 1 quality setting +# HP is similar to the HandBrake built-in preset High Profile + +######################## +# # +# Adjustable variables # +# # +######################## + +OUTDIR=/myth/video +LOGPATH=/var/log/mythtv +LOGFILE=${LOGPATH}/myth2mkv-$$.log + +# TMPDIR is for large transient files +TMPDIR=/myth/tmp + +# x264 tuning: +# Tune x264 based on content. Valid options for TUNING are: +# film, animation, grain, stillimage, psnr, ssim, fastdecode, zerolatency +# Separate multiple options with a comma. DEFAULT: none +TUNING="" + +# Custom cropping. Useful if you have a 4:3 image in a HD frame or if +# HandBrake's autocrop smarts fail you. +# Crop 240 pixels off the left and right for 4:3 image in 1920x1080 frame +# Crop 160 pixels off the left and right for 4:3 image in 1280x720 frame +# <T:B:L:R> +# i.e. 0:0:240:240 +# Default: In HP and HQ: CROP="0:0:0:0" (no cropping). +# IN MQ and LQ: autocrop. +CROP="" + +# Force custom output resolution. +# Default: Keep same resolution as input file (less any cropping). +# The HP quality setting always keeps the same resolution as the input file. +WIDTH="" +HEIGHT="" + +# Force use/non-use of deinterlacing filter. Y|N|G (Yes, No, Guess) +# Default: G - Guess based on source resolution. +# If the source video width is 1920, 1440, 852, 704, 640 or 528 pixels +# "G" will deinterlace the video. Change to "Y" to force use of deinterlacing +# filter or to "N" to NOT use deinterlace filter no matter the resolution. +DEINT="G" + +############################ +# # +# End adjustable variables # +# # +############################ + +if [[ ! -d ${TMPDIR} ]] ; then + mkdir -p ${TMPDIR} +fi + +if [[ ! -d ${OUTDIR} ]] ; then + mkdir -p ${OUTDIR} +fi + +#------FUNCTIONS--------------- +update_comment() +# Arg_1 = COMMENT +{ +if [ ${NO_JOBID} -eq 0 ]; then + `jobqueue_helper.py -j ${JOBID} -cs "${1}"` +fi +} + +check_background_progress() +# check handbrake progress in background +{ +while [ ! -f ${HB_RETURN_CODE} ] +do + sleep 15 + check_myth_jobcmds + pass=`tail -1 ${STATUSFILE} | egrep -o -e 'task [0-9] of [0-9], ' | tail -1 | sed 's/task\ /Pass\ /g'` + prog_percent=`tail -1 ${STATUSFILE} | egrep -o -e '[0-9]*\.[0-9]. %' | tail -1 | sed 's/\ %//g'` + current_FPS=`tail -1 ${STATUSFILE} | egrep -o -e 'avg [0-9.]*\.[0-9]* fps' | tail -1 | sed -e 's/avg\ //g' -e 's/\ fps//g'` + current_ETA=`tail -1 ${STATUSFILE} | egrep -o -e 'ETA [0-9.][0-9.]h[0-9.][0-9.]m[0-9.][0-9.]s' | tail -1` + if [ -n "$prog_percent" ]; then + echo "${pass}${prog_percent}% @ ${current_FPS} fps. ${current_ETA}" + update_comment "${pass}${prog_percent}% @ ${current_FPS} fps. ${current_ETA}" + fi +done +} + +check_myth_jobcmds() +# check the myth database for stop pause or resume commands +{ +if [[ ${NO_JOBID} -eq 0 ]] ; then + CURRENT_CMD=`jobqueue_helper.py -m "select cmds from jobqueue where id = ${JOBID}"` + case "${CURRENT_CMD}" in + # JOB_RUN + 0) ;; + # JOB_PAUSE + 1) `jobqueue_helper.py -j ${JOBID} -ss 6` + kill -s STOP ${handbrake_pid} ;; + # JOB_RESUME + 2) `jobqueue_helper.py -j ${JOBID} -ss 4` + `jobqueue_helper.py -j ${JOBID} -cmds 0` + kill -s CONT ${handbrake_pid} ;; + # JOB_STOP + 4) `jobqueue_helper.py -j ${JOBID} -ss 5` + `jobqueue_helper.py -j ${JOBID} -cmds 0` + kill -9 ${handbrake_pid} ${command_pid} + clean_up_files + echo "Encode Cancelled" >> ${LOGFILE} + `jobqueue_helper.py -j ${JOBID} -ss 320` + exit ;; + esac +fi +} + +get_info_for_hb() { +# Collect some info about source file + +/usr/bin/mplayer -nolirc -identify -frames 0 "${HBINPUTFILE}" \ + 2>/dev/null 1>"${IDFILE}" + +VIDEOW=$( grep ID_VIDEO_WIDTH= "${IDFILE}" | awk -F= '{ print $NF }' ) +FPS=$( grep ID_VIDEO_FPS= "${IDFILE}" | awk -F= '{ print $NF }' ) + +# HandBrake does not like a framerate of 29.970, so let's drop the 0 +if [[ ${FPS} = "29.970" ]] ; then + FPS="29.97" +fi + +# HandBrake does not like a framerate of 59.940, so let's drop the 0 +if [[ ${FPS} = "59.940" ]] ; then + FPS="59.94" +fi + +# A rough guestimation that if the video width is 1920, 1440, 852, 704, 640 or +# 528 pixels it is probably interlaced. +if [[ ${DEINT} = Y ]] ; then + DEINT="-d slow" +else + if [[ ${DEINT} = N ]] ; then + DEINT="" + else + if [[ ${VIDEOW} = 1920 || ${VIDEOW} = 1440 || ${VIDEOW} = 852 || \ + ${VIDEOW} = 704 || ${VIDEOW} = 640 || ${VIDEOW} = 528 ]] ; then + DEINT="-d slow" + else + DEINT="" + fi + fi +fi + +if [[ -n ${DEINT} ]] ; then + if [[ ${QUALITY} = LQ ]] ; then + DEINT="-d fast" + fi +fi + +if [[ -n ${TUNING} ]] ; then + TUNING="--x264-tune ${TUNING}" +fi + +if [[ -n ${CROP} ]] ; then + CROP="--crop ${CROP}" +fi + +if [[ -n ${WIDTH} ]] ; then + WIDTH="-w ${WIDTH} -X ${WIDTH}" +fi + +if [[ -n ${HEIGHT} ]] ; then + HEIGHT="-l ${HEIGHT} -Y ${HEIGHT}" +fi + +if [[ ${QUALITY} = HP ]] ; then + if [[ -n ${CROP} ]] ; then + CROP="--crop ${CROP}" + else + CROP="--crop 0:0:0:0 --auto-anamorphic" + fi + HB_OPTS="-o ${TMPFILE} -e x264 ${TUNING} -q 20.0 -a 1,1 -E copy:ac3,faac -B 160,160 -6 dpl2,auto -R Auto,Auto -D 0.0,0.0 --audio-copy-mask aac,ac3,dtshd,dts,mp3 --audio-fallback ffac3 -f mkv --decomb --loose-anamorphic --modulus 2 -m --x264-preset medium --h264-profile high --h264-level 4.1 ${CROP} -s 1" +elif [[ ${QUALITY} = HQ ]] ; then + if [[ -n ${CROP} ]] ; then + CROP="--crop ${CROP}" + else + CROP="--crop 0:0:0:0 --auto-anamorphic" + fi + HB_OPTS="-o ${TMPFILE} -f mkv -m -e x264 ${TUNING} -x b-adapt=2:rc-lookahead=50 -b 5000 -2 -T ${WIDTH} ${HEIGHT} -r ${FPS} --cfr ${CROP} ${DEINT} -a 1 -E copy -s 1" +else + if [[ ${CROP} = "--crop 0:0:0:0" ]] ; then + CROP="${CROP} --auto-anamorphic" + fi + if [[ ${QUALITY} = LQ ]] ; then + HB_OPTS="-o ${TMPFILE} -f mkv -m -e x264 ${TUNING} -b 1250 ${WIDTH} ${HEIGHT} -r ${FPS} --pfr ${CROP} ${DEINT} -a 1 -E lame -B 128 -Q 8 -6 stereo -s 1" + else + # Fallback to "MQ" + HB_OPTS="-o ${TMPFILE} -f mkv -m -e x264 ${TUNING} -b 2500 -2 -T ${WIDTH} ${HEIGHT} -r ${FPS} --pfr ${CROP} ${DEINT} -a 1 -E lame -B 256 -Q 3 -6 stereo -s 1" + fi +fi +} + +get_handbrake_pid() +{ +process_name="" +i1=1 +while [ "${process_name}" != "found" ]; do + handbrake_pid=`expr ${handbrake_pid} + 1` + i1=`expr ${i1} + 1` + if [ "`ps ${handbrake_pid} | grep HandBrakeCLI | sed 's_.*\(HandBrakeCLI\).*_\1_'`" = "HandBrakeCLI" ]; then + process_name="found" + fi + if [ ${i1} -gt 20 ]; then + break + fi +done +} + +tag_file() { +DATE=`date` + +# Create a tag file here +echo "<?xml version='1.0' encoding='ISO-8859-1'?>" > "${TAG_FILE}" +echo "" >> "${TAG_FILE}" +echo "<!DOCTYPE Tags SYSTEM 'matroskatags.dtd'>" >> "${TAG_FILE}" +echo "" >> "${TAG_FILE}" +echo "<Tags>" >> "${TAG_FILE}" +echo " <Tag>" >> "${TAG_FILE}" +echo " <Simple>" >> "${TAG_FILE}" +echo " <Name>TITLE</Name>" >> "${TAG_FILE}" +echo " <String>${TITLE}</String>" >> "${TAG_FILE}" +echo " <Simple>" >> "${TAG_FILE}" +echo " <Name>SUBTITLE</Name>" >> "${TAG_FILE}" +echo " <String>${SUBTITLE}</String>" >> "${TAG_FILE}" +echo " <Simple>" >> "${TAG_FILE}" +echo " <Name>SUMMARY</Name>" >> "${TAG_FILE}" +echo " <String>${DESCR}</String>" >> "${TAG_FILE}" +echo " </Simple>" >> "${TAG_FILE}" +echo " <Simple>" >> "${TAG_FILE}" +echo " <Name>DATE_RELEASED</Name>" >> "${TAG_FILE}" +echo " <String>${OAD}</String>" >> "${TAG_FILE}" +echo " </Simple>" >> "${TAG_FILE}" +echo " <Simple>" >> "${TAG_FILE}" +echo " <Name>SEASON</Name>" >> "${TAG_FILE}" +echo " <String>${SEASON}</String>" >> "${TAG_FILE}" +echo " </Simple>" >> "${TAG_FILE}" +echo " <Simple>" >> "${TAG_FILE}" +echo " <Name>EPISODE</Name>" >> "${TAG_FILE}" +echo " <String>${EPISODE}</String>" >> "${TAG_FILE}" +echo " </Simple>" >> "${TAG_FILE}" +echo " </Simple>" >> "${TAG_FILE}" +echo " </Simple>" >> "${TAG_FILE}" +echo " <Simple>" >> "${TAG_FILE}" +echo " <Name>ENCODED_BY</Name>" >> "${TAG_FILE}" +echo " <String>HandBrakeCLI ${HBCLIVER}</String>" >> "${TAG_FILE}" +echo " </Simple>" >> "${TAG_FILE}" +echo " <Simple>" >> "${TAG_FILE}" +echo " <Name>DATE_TAGGED</Name>" >> "${TAG_FILE}" +echo " <String>${DATE}</String>" >> "${TAG_FILE}" +echo " </Simple>" >> "${TAG_FILE}" +echo " </Tag>" >> "${TAG_FILE}" +echo "</Tags>" >> "${TAG_FILE}" + +# Add tag info into MKV file +echo "Adding tag info to ${TITLE} - ${SUBTITLE} ..." >> ${LOGFILE} + +/usr/bin/mkvpropedit -r ${LOGFILE} -t all:"${TAG_FILE}" "${TMPFILE}" +} + +clean_up_files() +# clean up left over files +{ +unlink ${TMPFILE} 2> /dev/null +unlink ${TMPCUTFILE} 2> /dev/null +unlink ${TMPCUTFILE}.map 2> /dev/null +unlink ${STATUSFILE} 2> /dev/null +unlink ${IDFILE} 2> /dev/null +unlink ${HB_RETURN_CODE} 2> /dev/null +unlink ${TAG_FILE} 2> /dev/null +} + +#-------MAIN SCRIPT------------ + +# create temp filename so multiple instances won't conflict +TMPNAME=toX264-$$ +TMPFILE=${TMPDIR}/${TMPNAME}.mkv +TMPCUTFILE=${TMPDIR}/${TMPNAME}.mpg +HBINPUTFILE="${1}" +TITLE="${4}" +SUBTITLE="${5}" +JOBID="${6}" +QUALITY="${7}" +BASE=`basename ${HBINPUTFILE}` +HBCLIVER=`pacman -Q | grep handbrake-cli | awk '{ print $NF }' | awk -F"-" '{ print $1 }'` +STATUSFILE=/tmp/${TMPNAME}-status.log +HB_RETURN_CODE=/tmp/${TMPNAME}-hb_return_code +IDFILE=/tmp/${TMPNAME}-id.txt +TAG_FILE=/tmp/${TMPNAME}.xml +SEASON=`jobqueue_helper.py -m "select season from recorded where basename LIKE '${BASE}'"` +SEASON=`printf "%02d" $SEASON` +EPISODE=`jobqueue_helper.py -m "select episode from recorded where basename LIKE '${BASE}'"` +EPISODE=`printf "%02d" $EPISODE` +OAD=`jobqueue_helper.py -m "select originalairdate from recorded where basename LIKE '${BASE}'"` +DESCR=`jobqueue_helper.py -m "select description from recorded where basename LIKE '${BASE}'" | sed 's/\&/and/g'` +USER=`whoami` + +# check if %JOBID% is passed from command line +if [ -z ${JOBID} ]; then + NO_JOBID=1 +else + NO_JOBID=0 +fi + +# log file location +CDate="`date`" +echo "" >> ${LOGFILE} +echo $CDate >> ${LOGFILE} +echo "File to encode: ${HBINPUTFILE}" >> ${LOGFILE} +echo " --> Name: ${TITLE} - ${SUBTITLE}" >> ${LOGFILE} +echo " --> Temporary Files: ${TMPNAME}.*" >> ${LOGFILE} +echo "" >> ${LOGFILE} + +get_info_for_hb +ERROR=$? + +if [[ ${ERROR} != 0 ]] ; then + echo "Error parsing source file information!" >> ${LOGFILE} + cat ${IDFILE} >> ${LOGFILE} + clean_up_files + exit 1 +fi + +# start timer +beforetime="$(date +%s)" + +check_myth_jobcmds + +# If there is a cutlist, use it: +if [[ -n `mythutil --getcutlist --chanid "${2}" --starttime "${3}" | grep \ + Cutlist: | awk -F": " '{ print $NF }'` ]] ; then + echo "Applying cutlist for ${TITLE} - ${SUBTITLE} ..." >> ${LOGFILE} + mythtranscode --chanid "${2}" --starttime "${3}" -m --honorcutlist \ + -q --loglevel info --logpath "${LOGPATH}" -o "${TMPCUTFILE}" + mythtrans_pid=$! + ERROR=$? + HBINPUTFILE=${TMPCUTFILE} +fi + +if [[ ${ERROR} != 0 ]] ; then + echo "MythTranscode error!" >> ${LOGFILE} + echo "Check ${LOGPATH}/mythtranscode.date.${mythtrans_pid}.log for mythtranscode error" >> ${LOGFILE} + clean_up_files + exit 1 +fi + +# run handbrake in background to do conversion +echo "Encoding ${TITLE} - ${SUBTITLE} ..." >> ${LOGFILE} + +( /usr/bin/nice -n19 nohup /usr/bin/HandBrakeCLI -i ${HBINPUTFILE} ${HB_OPTS} \ + > ${STATUSFILE} 2>&1 ; echo $? > ${HB_RETURN_CODE} ) & +handbrake_pid=$! +command_pid=${handbrake_pid} +get_handbrake_pid + +check_background_progress + +if [[ `cat ${HB_RETURN_CODE}` != 0 ]] ; then + echo "HandBrakeCLI error!" >> ${LOGFILE} + cat ${STATUSFILE} >> ${LOGFILE} + clean_up_files + exit 1 +fi + +tag_file +ERROR=$? + +if [[ ${ERROR} != 0 ]] ; then + echo "Error creating tag file!" >> ${LOGFILE} + cat ${TAG_FILE} >> ${LOGFILE} + clean_up_files + exit 1 +fi + +# make output filename unique and do not clobber an existing file +# Build a final file name +if [[ $SEASON != "00" && $EPISODE != "00" ]]; then + FILE=$( echo "${TITLE,,} s${SEASON}e${EPISODE} ${SUBTITLE,,}" | tr -d [:punct:] | tr [:blank:] "_" | tr -s "_" ) +else + FILE=$( echo "${TITLE,,} ${OAD} ${SUBTITLE,,}" | tr -d [:punct:] | tr [:blank:] "_" | tr -s "_" ) +fi +OUTPUTFILE="${OUTDIR}/${FILE}.mkv" +i=1 +while [ -e "${OUTPUTFILE}" ] +do + OUTPUTFILE="${OUTDIR}/${FILE}-${i}.mkv" + i=`expr $i + 1` +done + +# move temp file to output location +chown -v "${USER}" "${TMPFILE}" >> ${LOGFILE} +mv -v "${TMPFILE}" "$OUTPUTFILE" >> ${LOGFILE} +ERROR=$? + +if [[ ${ERROR} != 0 ]] ; then + echo "Error moving ${TMPFILE} to ${OUTPUTFILE} !" >> ${LOGFILE} + clean_up_files + exit 1 +fi + +# stop timer +aftertime="$(date +%s)" +seconds="$(expr ${aftertime} - ${beforetime})" + +if [ ${ERROR} -eq 0 ]; then + echo "File Encoded Successfully: ${OUTPUTFILE}" >> ${LOGFILE} + hours=$((seconds / 3600)) + seconds=$((seconds % 3600)) + minutes=$((seconds / 60)) + seconds=$((seconds % 60)) + if [ $hours -eq 0 ]; then + hours="" + elif [ $hours -eq 1 ]; then + hours=" $hours hour" + else + hours=" $hours hours" + fi + if [ $minutes -eq 1 ]; then + minutes="$minutes minute" + else + minutes="$minutes minutes" + fi + if [ $seconds -eq 1 ]; then + seconds="$seconds second" + else + seconds="$seconds seconds" + fi + + echo "Encoding took${hours} ${minutes} ${seconds} @ ${current_FPS} fps." >> ${LOGFILE} + `jobqueue_helper.py -j ${JOBID} -ss 272` + update_comment "Encode Successful. Encoding Time:${hours} ${minutes} ${seconds}" +else + echo "ERROR: ${ERROR}" >> ${LOGFILE} +fi + +# Clean up +clean_up_files diff --git a/linhes/linhes-system/myth2mp3 b/linhes/linhes-system/myth2mp3 new file mode 100755 index 0000000..69d5d3f --- /dev/null +++ b/linhes/linhes-system/myth2mp3 @@ -0,0 +1,96 @@ +#!/bin/sh +# convert recordings to as mp3 audio only +# version 1.1.3 + +# usage: +# first parameter must be %DIR%/%FILE% of the recording +# second parameter must be the desired base name of the output +# third parameter must be %CHANID% if you set USECUTLIST=Y +# fourth parameter must be %STARTTIME% if you set USECUTLIST=Y +# In the mythtv setup screen invoke this script like this: +# MYTHTV User Job Command: +# /usr/LH/bin/myth2mp3 "%DIR%/%FILE%" "%TITLE% - %SUBTITLE%" "%CHANID%" "%STARTTIME%" + +# options: +BITRATE=256k #ie. 128k, 160k, 192k, 224k, 256k +USECUTLIST=Y #Y or N + +# where the converted audio is stored +OUT_DIR=/myth/music + +# create temp filename so multiple instances won't conflict +TMPNAME=toMP3-$$ +TMPFILE=/myth/tmp/$TMPNAME +TMPCUTFILE=/myth/tmp/$TMPNAME.mpg +FFINPUTFILE=$1 +TITLE=`echo $2 | sed 's/\//_/g'` + +# log file location +LOGFILE=/var/log/mythtv/myth2mp3.log +CDate="`date`" +echo "" >> $LOGFILE +echo $CDate >> $LOGFILE +echo "File to encode: $1 Name: $TITLE" >> $LOGFILE + +# start timer +beforetime="$(date +%s)" + +# check if using cutlist +if [ $USECUTLIST = Y ];then + MYTHCOMMFRAMES=`mythutil --getcutlist --chanid "$3" --starttime "$4" | grep 'Cutlist:' | cut -d \ -f 2` + if [ -n "$MYTHCOMMFRAMES" ]; then + echo "Extracting Cutlist..." >> $LOGFILE + /usr/bin/nice -n19 /usr/bin/mythtranscode --chanid "$3" --starttime "$4" --outfile "$TMPCUTFILE" --mpeg2 --honorcutlist + FFINPUTFILE=$TMPCUTFILE + fi +fi + +# run ffmpeg to do conversion to wav +echo "Encoding to intermediate wav..." >> $LOGFILE +/usr/bin/nice -n19 /usr/bin/ffmpeg -i "$FFINPUTFILE" -vn -acodec pcm_s16le -ar 44100 -ac 2 "$TMPFILE.wav" +ERROR=$? + +# Normalize the intermediate wav +echo "Normalizing intermediate wav..." >> $LOGFILE +/usr/bin/nice -n19 /usr/bin/normalize -q "$TMPFILE.wav" +ERROR=$? + +FFINPUTFILE=$TMPFILE.wav + +# Final encode of normalized wav to mp3 +echo "Encoding normalized wav to mp3..." >> $LOGFILE +/usr/bin/nice -n19 /usr/bin/ffmpeg -i "$FFINPUTFILE" -vn -acodec libmp3lame -ab $BITRATE -ar 44100 -ac 2 "$TMPFILE.mp3" +ERROR=$? + +# make output filename unique +OUTPUTFILE=$OUT_DIR/$TITLE.mp3 +i=1 +while [ -e "$OUTPUTFILE" ] +do + OUTPUTFILE=$OUT_DIR/$TITLE-$i.mp3 + i=`expr $i + 1` +done + +# move temp file to output location +chown mythtv "$TMPFILE.mp3" && mv "$TMPFILE.mp3" "$OUTPUTFILE" + +# stop timer +aftertime="$(date +%s)" +seconds="$(expr $aftertime - $beforetime)" + +if [ $ERROR -eq 0 ]; then + echo "File Encoded Sucessfully: $OUTPUTFILE" >> $LOGFILE + hours=$((seconds / 3600)) + seconds=$((seconds % 3600)) + minutes=$((seconds / 60)) + seconds=$((seconds % 60)) + echo "Encoding Time: $hours hour(s) $minutes minute(s) $seconds second(s)" >> $LOGFILE +else + echo "ERROR: $ERROR" >> $LOGFILE +fi + +# clean up left over files +unlink $TMPFILE.mp3 2> /dev/null +unlink $TMPFILE.wav 2> /dev/null +unlink $TMPCUTFILE 2> /dev/null +unlink $TMPCUTFILE.map 2> /dev/null diff --git a/linhes/linhes-system/myth_mtc.cron b/linhes/linhes-system/myth_mtc.cron new file mode 100644 index 0000000..6d70058 --- /dev/null +++ b/linhes/linhes-system/myth_mtc.cron @@ -0,0 +1,79 @@ +#!/bin/bash +MYTH_RUN_STATUS=1 +. /etc/systemconfig +. /etc/profile + +date=`date +%Y-%m-%d` +timestamp=`date +'%Y-%m-%d %H:%M'` +hostname=`hostname` +logFile="/var/log/${date}/${hostname}_myth_mtc.log" +log="logger -t myth_mtc -p local6.info" + +if [ ! -f $logFile ]; then + touch $logFile + echo "" | $log +fi + +#check logfile for Finished and if not run myth_mtc.py +if ! grep -q "Finished Maintenance" $logFile +then + + if ! grep -q "Finished checking size of MythTV home" $logFile + then + MYTHCONFDIR=/usr/share/mythtv unbuffer myth_mtc.py --check_home | $log + if [ $? = 0 ] + then + echo "" | $log + else + echo "Time Exceeded" | $log + exit + fi + fi + + if ! grep -q "Finished Optimize" $logFile + then + if [ $SystemType = Frontend_only ] + then + echo "Will not run Optimize on Frontend Only systems." | $log + echo "Finished Optimize" | $log + else + MYTHCONFDIR=/usr/share/mythtv unbuffer myth_mtc.py --optimize | $log + if [ $? = 0 ] + then + echo "" | $log + else + echo "Time Exceeded" | $log + exit + fi + fi + fi + + if ! grep -q "Finished Backup" $logFile && grep -q "Finished Optimize" $logFile + then + MYTHCONFDIR=/usr/share/mythtv unbuffer myth_mtc.py --backup | $log + if [ $? = 0 ] + then + echo "" | $log + else + echo "Time Exceeded" | $log + exit + fi + fi + + if ! grep -q "Finished Update" $logFile + then + MYTHCONFDIR=/usr/share/mythtv unbuffer myth_mtc.py --update | $log + if [ $? = 0 ] + then + echo "" | $log + else + echo "Time Exceeded" | $log + exit + fi + fi + + if grep -q "Finished checking size of MythTV home" $logFile && grep -q "Finished Optimize" $logFile && grep -q "Finished Backup" $logFile && grep -q "Finished Update" $logFile + then + echo "Finished Maintenance" | $log + fi +fi diff --git a/linhes/linhes-system/openssh.hook b/linhes/linhes-system/openssh.hook new file mode 100644 index 0000000..0f9f866 --- /dev/null +++ b/linhes/linhes-system/openssh.hook @@ -0,0 +1,9 @@ +[Trigger] +Operation = Install +Type = Package +Target = linhes-system + +[Action] +Description = Enable and start sshd... +When = PostTransaction +Exec = /usr/bin/systemctl enable --now sshd.service diff --git a/linhes/linhes-system/paccache.cron b/linhes/linhes-system/paccache.cron new file mode 100644 index 0000000..d3ca082 --- /dev/null +++ b/linhes/linhes-system/paccache.cron @@ -0,0 +1,2 @@ +#!/bin/bash +/usr/bin/paccache -r -q diff --git a/linhes/linhes-system/readme_is_xml b/linhes/linhes-system/readme_is_xml new file mode 100644 index 0000000..f0a13cb --- /dev/null +++ b/linhes/linhes-system/readme_is_xml @@ -0,0 +1 @@ +Files ending in .conf will be read by gen_is_xml.py or gen_lib_xml.py or gen_game_xml.py and output xml as appropriate diff --git a/linhes/linhes-system/remove_storage.py b/linhes/linhes-system/remove_storage.py new file mode 100755 index 0000000..15dbffc --- /dev/null +++ b/linhes/linhes-system/remove_storage.py @@ -0,0 +1,184 @@ +#!/usr/bin/python2 +#remove_storage.py removes disks that were added using add_storage.py +# +#Only disks that have a conf file in /etc/storage.d/ are presented +#as choices to remove. + + +import os,sys,subprocess +import configparser +from configparser import SafeConfigParser +import glob +from socket import gethostname +from MythTV import MythDB + +storage_dir = "/etc/storage.d" + +def runcmd(cmd): + if True : + pass + else: + cmd = "echo "+cmd + #print cmd + cmdout = subprocess.getstatusoutput(cmd) + #logging.debug(" %s", cmdout) + return cmdout + +def read_fstab(): + f = open('/etc/fstab', 'r') + fstab=f.readlines() + f.close() + return fstab + +def unmount_disk(conf): + if os.path.ismount(conf[1]): + print("Unmounting " + conf[1] + ".") + cmd = "umount %s" %conf[1] + runcmd(cmd) + +def remove_fstab_entry(conf): + fstab=read_fstab() + f = open('/etc/fstab', 'w') + for line in fstab: + if not line.startswith("#"): + if line.find(conf[1]) > -1 and line.find(conf[2]) > -1 : + print("Removing " + conf[1] + " from fstab.") + else: + f.write(line) + else: + f.write(line) + f.close() + +def remove_SG_entries(conf): + DB = MythDB() + host=gethostname() + with DB as c: + print("Removing " + conf[1] + " storage group\n paths from MythTV database.") + c.execute("""delete from storagegroup where hostname = %s and dirname like %s""", (host,conf[1] + '%')) + c.execute("""delete from settings where hostname = %s and value like %s""", (host,'SGweightPerDir:' + host + ':' + conf[1] + '%')) + +def remove_disk_link(conf): + if os.path.islink("/data/storage/disk" + str(conf[0])): + print("Removing link /data/storage/disk%s." %(conf[0])) + cmd = "rm -f /data/storage/disk%s" %(conf[0]) + runcmd(cmd) + +def remove_disk_mount(conf): + print("Removing mountpoint %s." %(conf[1])) + cmd = "rm -rf %s" %(conf[1]) + runcmd(cmd) + +def update_conf_file(conf): + print("Updating %s file\n and running systemconfig." %(conf[4])) + parser = SafeConfigParser() + parser.read(conf[4]) + parser.set('storage','shareable',"False") + parser.set('storage','removed',"True") + with open(conf[4], 'wb') as conf[4]: + parser.write(conf[4]) + cmd = "systemconfig.py -m fileshare" + runcmd(cmd) + +def usage(): + help=''' + remove_storage.py removes disks that were added using add_storage.py. + Only disks that have a conf file in /etc/storage.d/ are presented + as choices to remove. + + Disks removed from the system are not erased or formatted. + + Normal operations include (in this order): + Ask which disk to remove + Unmount the disk if mounted + Remove disk from /etc/fstab + Remove MythTV storage group paths in MythTV database + Remove disk# symlink at /data/storage/ + Remove disk name mountpoint at /data/storage/ + Make shareable = False in /etc/storage.d/DISKNAME.conf + Add removed = True in /etc/stoarge.d/DISKNAME.conf + Run systemconfig.py -m fileshare to update File Shares + + Options: + -h, --help: Show this help message. + ''' + print(help) + sys.exit(0) + +#-------------------------------------------- + +def main(): + all_confs = [] + + # get conf files that are not disk0 or not removed + for conf_file in glob.glob('%s/*.conf' %storage_dir): + this_conf = [] + parser = SafeConfigParser() + parser.read(conf_file) + mounted = "" + disk_num = parser.get('storage', 'disk_num') + try: + removed = parser.get('storage', 'removed') + except: + removed = False + if disk_num == "0" or removed: + continue + this_conf.append(int(disk_num)) #0 + this_conf.append(parser.get('storage', 'mountpoint')) #1 + this_conf.append(parser.get('storage', 'uuid')) #2 + if os.path.ismount(this_conf[1]): + this_conf.append("(mounted)") #3 + else: + this_conf.append("") #3 + this_conf.append(conf_file) #4 + all_confs.append(this_conf) + + # sort confs on disk num + all_confs.sort(key=lambda x: x[0]) + + # exit if no disks found + if len(all_confs) == 0: + print("\nThere are no disks to remove. Exiting.") + sys.exit(0) + + print("\nDisks found in /etc/storage.d/:\n") + + # print list of disks to remove + for i,conf in enumerate(all_confs): + print(str(i+1) + ": " + conf[1] + " (disk" + str(conf[0]) + ") " + conf[3]) + + # get user input of disk to remove + try: + conf_select=input("\nEnter the number of the disk to remove: ") + conf_select=int(conf_select) + if conf_select > len(all_confs) or conf_select < 1: + conf_select=int("e") + except ValueError: + print("You must enter a number between 1 and " + str(len(all_confs)) + ". Exiting.") + sys.exit(0) + + selected_conf = all_confs[conf_select-1] + + # confirm selection + confirm_select=input("\nDisk " + selected_conf[1] + " (disk" + str(selected_conf[0]) + ") will be removed.\nAre you sure you want to remove it (yes/no)? ") + if confirm_select != "yes": + print("Exiting.") + sys.exit(0) + + print("") + unmount_disk(selected_conf) + remove_fstab_entry(selected_conf) + remove_SG_entries(selected_conf) + remove_disk_link(selected_conf) + remove_disk_mount(selected_conf) + update_conf_file(selected_conf) + + print("\nDisk " + selected_conf[1] + " has been removed.") + +if __name__ == "__main__": + if not os.geteuid()==0: + sys.exit("\nRoot access is required to run this program.\n") + + if "--help" in sys.argv or "-h" in sys.argv: + usage() + else: + main() diff --git a/linhes/linhes-system/system-sudo.rules b/linhes/linhes-system/system-sudo.rules new file mode 100644 index 0000000..86e1b72 --- /dev/null +++ b/linhes/linhes-system/system-sudo.rules @@ -0,0 +1 @@ +mythtv ALL=(ALL) NOPASSWD: ALL diff --git a/linhes/linhes-system/xfs_defrag.cron b/linhes/linhes-system/xfs_defrag.cron new file mode 100644 index 0000000..792a4b4 --- /dev/null +++ b/linhes/linhes-system/xfs_defrag.cron @@ -0,0 +1,3 @@ +#!/bin/bash +#. /etc/profile +/usr/LH/bin/checkXFSfrag.sh --idle | /usr/bin/logger -t xfs_defrag |