LinHES-system, linhes-scripts,xymon: move find_orphans.py to LinHES-system and update xymon dep.

5 files changed, 296 insertions, 66 deletions

diff --git a/abs/core/LinHES-system/PKGBUILD b/abs/core/LinHES-system/PKGBUILD
index be0de10..b97034e 100755
--- a/abs/core/LinHES-system/PKGBUILD
+++ b/abs/core/LinHES-system/PKGBUILD
@@ -1,88 +1,79 @@
 pkgname=LinHES-system
 pkgver=8.0
-pkgrel=4
+pkgrel=5
 arch=('i686' 'x86_64')
-MVDIR=$startdir/pkg/usr/LH
-BINDIR=$startdir/pkg/usr/bin
 install=system.install
-pkgdesc="scripts and things related to having an  automated system"
-depends=(linhes-sounds xdotool tilda keylaunch python_aosd 'linhes-scripts>=7-24' 'udisks' 'LinHES-config>=2.3-59' 'sudo' 'python2-dbus' 'unclutter' 'wmctrl' 'archlinux-xdg-menu' 'ethtool')
-backup=(etc/modprobe.d/alsa-base)
-binfiles="LinHES-start optimize_mythdb.py
- myth_mtc.py myth_mtc.sh LinHES-run
- load-modules-mythvantage.sh
- unclutter-toggle.sh tvterm.sh
- mythfrontend-start set_windowmanager.sh
- myth_status.py myth_status.sh
+pkgdesc="Everything that makes LinHES an automated system"
+license=('GPL2')
+depends=('linhes-sounds' 'xdotool' 'tilda' 'keylaunch'
+        'python_aosd' 'linhes-scripts>=7-24' 'udisks' 
+        'LinHES-config>=2.3-59' 'sudo' 'python2-dbus' 'unclutter'
+        'wmctrl' 'archlinux-xdg-menu' 'ethtool')
+backup=('etc/modprobe.d/alsa-base')
+binfiles="LinHES-start optimize_mythdb.py myth_mtc.py myth_mtc.sh
+ LinHES-run load-modules-mythvantage.sh unclutter-toggle.sh tvterm.sh
+ mythfrontend-start set_windowmanager.sh myth_status.py myth_status.sh
  install_supplemental_service.sh
- lh_system_backup
- lh_system_backup_job
- lh_system_restore_job
- lh_system_host_update
- lh_system_all_host_update
- add_storage.py
- diskspace.sh
- cacheclean lh_backend_control.sh switch_web.sh
- create_media_dirs.sh
- msg_client.py msg_daemon.py
+ lh_system_backup lh_system_backup_job lh_system_restore_job
+ lh_system_host_update lh_system_all_host_update
+ add_storage.py diskspace.sh cacheclean lh_backend_control.sh
+ switch_web.sh create_media_dirs.sh msg_client.py msg_daemon.py
  gen_is_xml.py gen_lib_xml.py gen_light_include.py gen_game_xml.py
- misc_recent_recordings.pl
- misc_status_config.py
- misc_status_info.sh
- misc_upcoming_recordings.pl
- misc_which_recorder.pl
+ misc_recent_recordings.pl misc_status_config.py misc_status_info.sh
+ misc_upcoming_recordings.pl misc_which_recorder.pl
  change_channel.sh change_channel_wrapper.sh
- be_check.py
- checkXFSfrag.sh"
+ be_check.py checkXFSfrag.sh find_orphans.py"
 
 source=(LinHES-session
     LinHES-profile.sh $binfiles
     alsa-base myth_mtc.lr
-    diskspace cacheclean.cron xfs_defrag
+    diskspace.cron cacheclean.cron xfs_defrag.cron
     readme_is_xml readme_light add_storage.readme
     system-sudo.rules )
 
-build() {
-        cd $startdir/src
-        install -m755 -D LinHES-session $startdir/pkg/etc/X11/Sessions/LinHES
-    	install -m755 -D LinHES-profile.sh $startdir/pkg/etc/profile.d/LinHES-profile.sh
-        ####### install to /usr/LH/bin
-        for i in $binfiles
-        do
-        	item=$i
-        	install -m755 -D $item $MVDIR/bin/$item
-        	echo "installing $item to $MVDIR"
-        done
+package() {
+    cd $srcdir
 
-#         cd $startdir/bin
-#         for i in *
-#         do
-#         	item=$i
-#         	install -m755 -D $item $MVDIR/bin/$item
-#         	echo "bin stuff: $item"
-#         done
+    MVDIR=$pkgdir/usr/LH
+    BINDIR=$pkgdir/usr/bin
 
-        install -m644 -D $startdir/src/alsa-base $startdir/pkg/etc/modprobe.d/alsa-base.conf
-        install -m644 -D $startdir/src/readme_is_xml $startdir/pkg/etc/gen_is_xml.d/readme_is_xml
-        install -m644 -D $startdir/src/readme_is_xml $startdir/pkg/etc/gen_lib_xml.d/readme_gen_xml
-        install -m644 -D $startdir/src/readme_is_xml $startdir/pkg/etc/gen_game_xml.d/readme_gen_xml
-        install -m644 -D $startdir/src/readme_light  $startdir/pkg/etc/gen_light_conf.d/readme_gen_light
-        install -m644 -D $startdir/src/add_storage.readme $startdir/pkg/etc/storage.d/readme
-       # install -m755 -D $startdir/src/smolt.cron $startdir/pkg/etc/cron.weekly/smolt.cron
-        install -m755 -D $startdir/src/cacheclean.cron $startdir/pkg/etc/cron.weekly/cacheclean.cron
-        install -m755 -D $startdir/src/diskspace $startdir/pkg/etc/cron.tenminutes/diskspace
-        install -m644 -D $startdir/src/myth_mtc.lr $startdir/pkg/etc/logrotate.d/myth_mtc
-        install -m755 -D $startdir/src/xfs_defrag $startdir/pkg/etc/cron.weekly/xfs_defrag
+    install -m755 -D LinHES-session $pkgdir/etc/X11/Sessions/LinHES
+    install -m755 -D LinHES-profile.sh $pkgdir/etc/profile.d/LinHES-profile.sh
+    
+    #files install to /usr/LH/bin
+    for i in $binfiles
+    do
+        item=$i
+        install -m755 -D $item $MVDIR/bin/$item
+        echo "installing $item to $MVDIR"
+    done
 
+    install -m644 -D $srcdir/alsa-base $pkgdir/etc/modprobe.d/alsa-base.conf
 
-        #sudo rules
-        mkdir -p $startdir/pkg/etc/sudoers.d/
-        chmod 750 $startdir/pkg/etc/sudoers.d/
-        chown -R root:root $startdir/pkg/etc/sudoers.d
-        install -o root -g root -m 0750 $startdir/src/system-sudo.rules $startdir/pkg/etc/sudoers.d/system_sudo
+    #readme files
+    install -m644 -D $srcdir/readme_is_xml $pkgdir/etc/gen_is_xml.d/readme_is_xml
+    install -m644 -D $srcdir/readme_is_xml $pkgdir/etc/gen_lib_xml.d/readme_gen_xml
+    install -m644 -D $srcdir/readme_is_xml $pkgdir/etc/gen_game_xml.d/readme_gen_xml
+    install -m644 -D $srcdir/readme_light  $pkgdir/etc/gen_light_conf.d/readme_gen_light
+    install -m644 -D $srcdir/add_storage.readme $pkgdir/etc/storage.d/readme
 
+    #cron files
+    #install -m755 -D $srcdir/smolt.cron $pkgdir/etc/cron.weekly/smolt.cron
+    install -m755 -D $srcdir/cacheclean.cron $pkgdir/etc/cron.weekly/cacheclean
+    install -m755 -D $srcdir/diskspace.cron $pkgdir/etc/cron.tenminutes/diskspace
+    install -m755 -D $srcdir/xfs_defrag.cron $pkgdir/etc/cron.weekly/xfs_defrag
+
+    #logrotate files
+    install -m644 -D $srcdir/myth_mtc.lr $pkgdir/etc/logrotate.d/myth_mtc
+
+    #sudo rules
+    mkdir -p $pkgdir/etc/sudoers.d/
+    chmod 750 $pkgdir/etc/sudoers.d/
+    chown -R root:root $pkgdir/etc/sudoers.d
+    install -o root -g root -m 0750 $srcdir/system-sudo.rules $pkgdir/etc/sudoers.d/system_sudo
 
 }
+
 md5sums=('8fc4b7c1ddf8f3c4d2266ce55086b4d4'
          'de32a1c50101265dd7f6ca5037f7a26a'
          'a875ee97f86e46f34a741c2bc455f894'
@@ -124,12 +115,13 @@ md5sums=('8fc4b7c1ddf8f3c4d2266ce55086b4d4'
          'bacc813b48bafcc6fe906e5969930501'
          '8b0298f70f97cc1dc2a58b9a73c64bd3'
          '911b0fbc8d9178dac1a193346c9decaf'
+         '34fc1f58ad1eabf4eff4979d420760c0'
          'eb879fee9603a05d5420d4ce8ed9e450'
          '6782771960c495aeb597893f37d1e5ee'
          '84492954db16740f949d795b74383189'
          '02c810c2f47b7c4495fdacaf54189473'
          '474d5c6883fcfffae4f199aceb3b1356'
-         '1e2ca4dd872a50df6b70528e06df558a'
+         '36bdfa8d877cea20ca3870faf08e08d5'
          '59d0602bac7e06f11abba1894acc8bb0'
          'da432d68962e109e09beb2b71c8c8571'
          '366ccb097d1c221389709485c3b2d159')
diff --git a/abs/core/LinHES-system/diskspace b/abs/core/LinHES-system/diskspace.cron
index ab3c91b..ab3c91b 100755
--- a/abs/core/LinHES-system/diskspace
+++ b/abs/core/LinHES-system/diskspace.cron
diff --git a/abs/core/LinHES-system/find_orphans.py b/abs/core/LinHES-system/find_orphans.py
new file mode 100644
index 0000000..cdc03d4
--- /dev/null
+++ b/abs/core/LinHES-system/find_orphans.py
@@ -0,0 +1,238 @@
+#!/usr/bin/env python2
+
+from MythTV import MythDB, MythBE, Recorded, MythError
+from socket import timeout
+
+import os
+import sys
+
+def human_size(s):
+    s = float(s)
+    o = 0
+    while s > 1000:
+        s /= 1000
+        o += 1
+    return str(round(s,1))+('B ','KB','MB','GB')[o]
+
+class File( str ):
+    def __new__(self, host, group, path, name, size):
+        return str.__new__(self, name)
+    def __init__(self, host, group, path, name, size):
+        self.host = host
+        self.group = group
+        self.path = path
+        self.size = int(size)
+    def pprint(self):
+        name = u'%s: %s' % (self.host, os.path.join(self.path, self))
+        print u'  {0:<90}{1:>8}'.format(name, human_size(self.size))
+    def delete(self):
+        be = MythBE(self.host, db=DB)
+        be.deleteFile(self, self.group)
+
+class MyRecorded( Recorded ):
+    _table = 'recorded'
+    def pprint(self):
+        name = u'{0.hostname}: {0.title}'.format(self)
+        if self.subtitle:
+            name += u' - '+self.subtitle
+        print u'  {0:<70}{1:>28}'.format(name,self.basename)
+
+def printrecs(title, recs):
+    print title
+    for rec in sorted(recs, key=lambda x: x.title):
+        rec.pprint()
+    print u'{0:>88}{1:>12}'.format('Count:',len(recs))
+
+def printfiles(title, files):
+    print title
+    for f in sorted(files, key=lambda x: x.path):
+        f.pprint()
+    size = sum([f.size for f in files])
+    print u'{0:>88}{1:>12}'.format('Total:',human_size(size))
+
+def populate(host=None):
+    unfiltered = []
+    kwargs = {'livetv':True}
+    if host:
+        with DB as c:
+            c.execute("""SELECT count(1) FROM settings
+                         WHERE hostname=%s AND value=%s""",
+                        (host, 'BackendServerIP'))
+            if c.fetchone()[0] == 0:
+                raise Exception('Invalid hostname specified on command line.')
+        hosts = [host]
+        kwargs['hostname'] = host
+    else:
+        with DB as c:
+            c.execute("""SELECT hostname FROM settings
+                         WHERE value='BackendServerIP'""")
+            hosts = [r[0] for r in c.fetchall()]
+    for host in hosts:
+        for sg in DB.getStorageGroup():
+            if sg.groupname in ('Videos','Banners','Coverart',\
+                                'Fanart','Screenshots','Trailers'):
+                continue
+            try:
+                dirs,files,sizes = BE.getSGList(host, sg.groupname, sg.dirname)
+                for f,s in zip(files,sizes):
+                    newfile = File(host, sg.groupname, sg.dirname, f, s)
+                    if newfile not in unfiltered:
+                        unfiltered.append(newfile)
+            except:
+                pass
+
+    recs = list(DB.searchRecorded(**kwargs))
+
+    zerorecs = []
+    orphvids = []
+    for rec in list(recs):
+        if rec.basename in unfiltered:
+            recs.remove(rec)
+            i = unfiltered.index(rec.basename)
+            f = unfiltered.pop(i)
+            if f.size < 1024:
+                zerorecs.append(rec)
+            name = rec.basename.rsplit('.',1)[0]
+            for f in list(unfiltered):
+                if name in f:
+                    unfiltered.remove(f)
+    for f in list(unfiltered):
+        if not (f.endswith('.mpg') or f.endswith('.nuv')):
+            continue
+        orphvids.append(f)
+        unfiltered.remove(f)
+
+    orphimgs = []
+    for f in list(unfiltered):
+        if not f.endswith('.png'):
+            continue
+        orphimgs.append(f)
+        unfiltered.remove(f)
+
+    dbbackup = []
+    for f in list(unfiltered):
+        if 'sql' not in f:
+            continue
+        dbbackup.append(f)
+        unfiltered.remove(f)
+
+    return (recs, zerorecs, orphvids, orphimgs, dbbackup, unfiltered)
+
+def delete_recs(recs):
+    printrecs('The following recordings will be deleted', recs)
+    print 'Are you sure you want to continue?'
+    try:
+        res = raw_input('> ')
+        while True:
+            if res == 'yes':
+                for rec in recs:
+                    rec.delete(True, True)
+                break
+            elif res == 'no':
+                break
+            else:
+                res = raw_input("'yes' or 'no' > ")
+    except MythError:
+        name = u'{0.hostname}: {0.title}'.format(rec)
+        if rec.subtitle:
+            name += ' - '+rec.subtitle
+        print "Warning: Failed to delete '" + name + "'"
+    except KeyboardInterrupt:
+        pass
+    except EOFError:
+        sys.exit(0)
+
+def delete_files(files):
+    printfiles('The following files will be deleted', files)
+    print 'Are you sure you want to continue?'
+    try:
+        res = raw_input('> ')
+        while True:
+            if res == 'yes':
+                for f in files:
+                    f.delete()
+                break
+            elif res == 'no':
+                break
+            else:
+                res = raw_input("'yes' or 'no' > ")
+    except KeyboardInterrupt:
+        pass
+    except EOFError:
+        sys.exit(0)
+
+def main(host=None):
+   while True:
+        recs, zerorecs, orphvids, orphimgs, dbbackup, unfiltered = populate(host)
+
+        if len(recs):
+            printrecs("Recordings with missing files", recs)
+        if len(zerorecs):
+            printrecs("Zero byte recordings", zerorecs)
+        if len(orphvids):
+            printfiles("Orphaned video files", orphvids)
+        if len(orphimgs):
+            printfiles("Orphaned snapshots", orphimgs)
+        if len(dbbackup):
+            printfiles("Database backups", dbbackup)
+        if len(unfiltered):
+            printfiles("Other files", unfiltered)
+        if not printOnly:
+            opts = []
+            if len(recs):
+                opts.append(['Delete orphaned recording entries', delete_recs, recs])
+            if len(zerorecs):
+                opts.append(['Delete zero byte recordings', delete_recs, zerorecs])
+            if len(orphvids):
+                opts.append(['Delete orphaned video files', delete_files, orphvids])
+            if len(orphimgs):
+                opts.append(['Delete orphaned snapshots', delete_files, orphimgs])
+            if len(unfiltered):
+                opts.append(['Delete other files', delete_files, unfiltered])
+            opts.append(['Refresh list', None, None])
+            print 'Please select from the following'
+            for i, opt in enumerate(opts):
+                print ' {0}. {1}'.format(i+1, opt[0])
+
+            try:
+                inner = True
+                res = raw_input('> ')
+                while inner:
+                    try:
+                        res = int(res)
+                    except:
+                        res = raw_input('input number. ctrl-c to exit > ')
+                        continue
+                    if (res <= 0) or (res > len(opts)):
+                        res = raw_input('input number within range > ')
+                        continue
+                    break
+                opt = opts[res-1]
+                if opt[1] is None:
+                    continue
+                else:
+                    opt[1](opt[2])
+
+            except KeyboardInterrupt:
+                break
+            except EOFError:
+                sys.exit(0)
+        else:
+            sys.exit(0)
+DB = MythDB()
+BE = MythBE(db=DB)
+DB.searchRecorded.handler = MyRecorded
+DB.searchRecorded.dbclass = MyRecorded
+
+if __name__ == '__main__':
+    global printOnly
+    if "--printonly" in sys.argv :
+        printOnly=True
+    else:
+        printOnly=False
+
+
+    if len(sys.argv) == 2 and sys.argv[1] != "--printonly":
+        main(sys.argv[1])
+    else:
+        main()
diff --git a/abs/core/LinHES-system/readme_is_xml b/abs/core/LinHES-system/readme_is_xml
index d2af0f6..f0a13cb 100644
--- a/abs/core/LinHES-system/readme_is_xml
+++ b/abs/core/LinHES-system/readme_is_xml
@@ -1 +1 @@
-Files ending in .conf will be read by gen_is_xml.py or gen_lib_xml.py and output xml as appropriate
+Files ending in .conf will be read by gen_is_xml.py or gen_lib_xml.py or gen_game_xml.py and output xml as appropriate
diff --git a/abs/core/LinHES-system/xfs_defrag b/abs/core/LinHES-system/xfs_defrag.cron
index 4d35507..4d35507 100755
--- a/abs/core/LinHES-system/xfs_defrag
+++ b/abs/core/LinHES-system/xfs_defrag.cron