diff options
| -rwxr-xr-x | abs/core/LinHES-system/PKGBUILD | 5 | ||||
| -rwxr-xr-x | abs/core/LinHES-system/mythvidexport.py | 439 |
2 files changed, 442 insertions, 2 deletions
diff --git a/abs/core/LinHES-system/PKGBUILD b/abs/core/LinHES-system/PKGBUILD index 97d7fa0..387793b 100755 --- a/abs/core/LinHES-system/PKGBUILD +++ b/abs/core/LinHES-system/PKGBUILD @@ -1,6 +1,6 @@ pkgname=LinHES-system pkgver=8.1 -pkgrel=15 +pkgrel=16 arch=('i686' 'x86_64') install=system.install pkgdesc="Everything that makes LinHES an automated system" @@ -18,7 +18,7 @@ binfiles="LinHES-start optimize_mythdb.py myth_mtc.py lh_system_backup lh_system_backup_job lh_system_restore_job lh_system_host_update lh_system_all_host_update add_storage.py diskspace.sh cacheclean lh_backend_control.sh - create_media_dirs.sh msg_client.py msg_daemon.py + create_media_dirs.sh msg_client.py msg_daemon.py mythvidexport.py gen_is_xml.py gen_lib_xml.py gen_light_include.py gen_game_xml.py misc_recent_recordings.pl misc_status_config.py misc_status_info.sh misc_upcoming_recordings.pl misc_which_recorder.pl jobqueue_helper.py @@ -102,6 +102,7 @@ md5sums=('7ab2a2c643d2b286811d8303d08982ad' '7f7c49d859abdaa0b5fca399241d4998' '3e60b17892e5b8214d47dcfddf5215a4' '57ec994cc3964a10c00580e89ebcae35' + '7ba5e774bfebc3ec2469c9fe9a76e2ce' '0c3509b48f11af0dc1bf989721fe9ca7' 'ac61cc460d9e97ba1f5ef69e92cdfbe5' 'f3502bb7c665750da0ecdf6918f7c838' diff --git a/abs/core/LinHES-system/mythvidexport.py b/abs/core/LinHES-system/mythvidexport.py new file mode 100755 index 0000000..5fdfb6d --- /dev/null +++ b/abs/core/LinHES-system/mythvidexport.py @@ -0,0 +1,439 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +#--------------------------- +# Name: mythvidexport.py +# Python Script +# Author: Raymond Wagner +# Purpose +# This python script is intended to function as a user job, run through +# mythjobqueue, capable of exporting recordings into MythVideo. +# https://github.com/wagnerrp/mythtv-scripts/blob/master/python/mythvidexport.py +# http://www.mythtv.org/wiki/Mythvidexport.py +#--------------------------- +__title__ = "MythVidExport" +__author__ = "Raymond Wagner" +__version__= "v0.7.5" + +from MythTV import MythDB, Job, Recorded, Video, VideoGrabber,\ + MythLog, MythError, static, MythBE +from optparse import OptionParser, OptionGroup +from socket import gethostname + +import os +import re +import sys +import time +import hashlib + +def create_dummy_video(db=None): + db = MythDB(db) + +def hashfile(fd): + hasher = hashlib.sha1() + while True: + buff = fd.read(2**16) + if len(buff) == 0: + break + hasher.update(buff) + return hasher.hexdigest() + +class VIDEO: + def __init__(self, opts, jobid=None): + if jobid: + self.job = Job(jobid) + self.chanid = self.job.chanid + self.starttime = self.job.starttime + self.job.update(status=Job.STARTING) + else: + self.job = None + self.chanid = opts.chanid + self.starttime = opts.starttime + + self.opts = opts + self.db = MythDB() + self.log = MythLog(module='mythvidexport.py', db=self.db) + + # load setting strings + self.get_format() + + # prep objects + self.rec = Recorded((self.chanid,self.starttime), db=self.db) + self.log(MythLog.GENERAL, MythLog.INFO, 'Using recording', + '%s - %s' % (self.rec.title.encode('utf-8'), + self.rec.subtitle.encode('utf-8'))) + self.vid = Video(db=self.db).create({'title':'', 'filename':'', + 'host':gethostname()}) + + # process data + self.get_meta() + self.get_dest() + # bug fix to work around limitation in the bindings where DBDataRef classes + # are mapped to the filename at time of Video element creation. since the + # filename is specified as blank when the video is created, the markup + # handler is not properly initialized + self.vid.markup._refdat = (self.vid.filename,) + + # save file + self.copy() + if opts.seekdata: + self.copy_seek() + if opts.skiplist: + self.copy_markup(static.MARKUP.MARK_COMM_START, + static.MARKUP.MARK_COMM_END) + if opts.cutlist: + self.copy_markup(static.MARKUP.MARK_CUT_START, + static.MARKUP.MARK_CUT_END) + self.vid.update() + + # delete old file + if opts.delete: + self.rec.delete() + + def get_format(self): + host = self.db.gethostname() + # TV Format + if self.opts.tformat: + self.tfmt = self.opts.tformat + elif self.db.settings[host]['mythvideo.TVexportfmt']: + self.tfmt = self.db.settings[host]['mythvideo.TVexportfmt'] + else: + self.tfmt = 'Television/%TITLE%/Season %SEASON%/'+\ + '%TITLE% - S%SEASON%E%EPISODEPAD% - %SUBTITLE%' + + # Movie Format + if self.opts.mformat: + self.mfmt = self.opts.mformat + elif self.db.settings[host]['mythvideo.MOVIEexportfmt']: + self.mfmt = self.db.settings[host]['mythvideo.MOVIEexportfmt'] + else: + self.mfmt = 'Movies/%TITLE%' + + # Generic Format + if self.opts.gformat: + self.gfmt = self.opts.gformat + elif self.db.settings[host]['mythvideo.GENERICexportfmt']: + self.gfmt = self.db.settings[host]['mythvideo.GENERICexportfmt'] + else: + self.gfmt = 'Videos/%TITLE%' + + def get_meta(self): + self.vid.hostname = self.db.gethostname() + if self.rec.inetref: + # good data is available, use it + if self.rec.season > 0 or self.rec.episode > 0: + self.log(self.log.GENERAL, self.log.INFO, + 'Performing TV export with local data.') + self.type = 'TV' + grab = VideoGrabber(self.type) + metadata = grab.grabInetref(self.rec.inetref, self.rec.season, self.rec.episode) + else: + self.log(self.log.GENERAL, self.log.INFO, + 'Performing Movie export with local data.') + self.type = 'MOVIE' + grab = VideoGrabber(self.type) + metadata = grab.grabInetref(self.rec.inetref) + elif self.opts.listingonly: + # force use of local data + if self.rec.subtitle: + self.log(self.log.GENERAL, self.log.INFO, + 'Forcing TV export with local data.') + self.type = 'TV' + else: + self.log(self.log.GENERAL, self.log.INFO, + 'Forcing Movie export with local data.') + self.type = 'MOVIE' + metadata = self.rec.exportMetadata() + else: + if self.rec.subtitle: + # subtitle exists, assume tv show + self.type = 'TV' + self.log(self.log.GENERAL, self.log.INFO, + 'Attempting TV export.') + grab = VideoGrabber(self.type) + match = grab.sortedSearch(self.rec.title, self.rec.subtitle) + else: # assume movie + self.type = 'MOVIE' + self.log(self.log.GENERAL, self.log.INFO, + 'Attempting Movie export.') + grab = VideoGrabber(self.type) + match = grab.sortedSearch(self.rec.title) + + if len(match) == 0: + # no match found + self.log(self.log.GENERAL, self.log.INFO, + 'Falling back to generic export.') + self.type = 'GENERIC' + metadata = self.rec.exportMetadata() + elif (len(match) > 1) & (match[0].levenshtein > 0): + # multiple matches found, and closest is not exact + self.vid.delete() + raise MythError('Multiple metadata matches found: '\ + +self.rec.title) + else: + self.log(self.log.GENERAL, self.log.INFO, + 'Importing content from', match[0].inetref) + metadata = grab.grabInetref(match[0]) + + self.vid.importMetadata(metadata) + self.log(self.log.GENERAL, self.log.INFO, 'Import complete') + + def get_dest(self): + if self.type == 'TV': + self.vid.filename = self.process_fmt(self.tfmt) + elif self.type == 'MOVIE': + self.vid.filename = self.process_fmt(self.mfmt) + elif self.type == 'GENERIC': + self.vid.filename = self.process_fmt(self.gfmt) + + def process_fmt(self, fmt): + # replace fields from viddata + #print self.vid.data + ext = '.'+self.rec.basename.rsplit('.',1)[1] + rep = ( ('%TITLE%','title','%s'), ('%SUBTITLE%','subtitle','%s'), + ('%SEASON%','season','%d'), ('%SEASONPAD%','season','%02d'), + ('%EPISODE%','episode','%d'), ('%EPISODEPAD%','episode','%02d'), + ('%YEAR%','year','%s'), ('%DIRECTOR%','director','%s')) + for tag, data, format in rep: + if self.vid[data]: + fmt = fmt.replace(tag,format % self.vid[data]) + else: + fmt = fmt.replace(tag,'') + + # replace fields from program data + rep = ( ('%HOSTNAME%', 'hostname', '%s'), + ('%STORAGEGROUP%','storagegroup','%s')) + for tag, data, format in rep: + data = getattr(self.rec, data) + fmt = fmt.replace(tag,format % data) + +# fmt = fmt.replace('%CARDID%',self.rec.cardid) +# fmt = fmt.replace('%CARDNAME%',self.rec.cardid) +# fmt = fmt.replace('%SOURCEID%',self.rec.cardid) +# fmt = fmt.replace('%SOURCENAME%',self.rec.cardid) +# fmt = fmt.replace('%CHANNUM%',self.rec.channum) +# fmt = fmt.replace('%CHANNAME%',self.rec.cardid) + + if len(self.vid.genre): + fmt = fmt.replace('%GENRE%',self.vid.genre[0].genre) + else: + fmt = fmt.replace('%GENRE%','') +# if len(self.country): +# fmt = fmt.replace('%COUNTRY%',self.country[0]) +# else: +# fmt = fmt.replace('%COUNTRY%','') + return fmt+ext + + def copy(self): + stime = time.time() + srcsize = self.rec.filesize + htime = [stime,stime,stime,stime] + + self.log(MythLog.GENERAL|MythLog.FILE, MythLog.INFO, "Copying myth://%s@%s/%s"\ + % (self.rec.storagegroup, self.rec.hostname, self.rec.basename)\ + +" to myth://Videos@%s/%s"\ + % (self.vid.host, self.vid.filename)) + srcfp = self.rec.open('r') + dstfp = self.vid.open('w', nooverwrite=True) + + + if self.job: + self.job.setStatus(Job.RUNNING) + tsize = 2**24 + while tsize == 2**24: + tsize = min(tsize, srcsize - dstfp.tell()) + dstfp.write(srcfp.read(tsize)) + htime.append(time.time()) + rate = float(tsize*4)/(time.time()-htime.pop(0)) + remt = (srcsize-dstfp.tell())/rate + if self.job: + self.job.setComment("%02d%% complete - %d seconds remaining" %\ + (dstfp.tell()*100/srcsize, remt)) + srcfp.close() + dstfp.close() + + self.vid.hash = self.vid.getHash() + + self.log(MythLog.GENERAL|MythLog.FILE, MythLog.INFO, "Transfer Complete", + "%d seconds elapsed" % int(time.time()-stime)) + + if self.opts.reallysafe: + if self.job: + self.job.setComment("Checking file hashes") + self.log(MythLog.GENERAL|MythLog.FILE, MythLog.INFO, "Checking file hashes.") + srchash = hashfile(self.rec.open('r')) + dsthash = hashfile(self.vid.open('r')) + if srchash != dsthash: + raise MythError('Source hash (%s) does not match destination hash (%s)' \ + % (srchash, dsthash)) + elif self.opts.safe: + self.log(MythLog.GENERAL|MythLog.FILE, MythLog.INFO, "Checking file sizes.") + be = MythBE(db=self.vid._db) + try: + srcsize = be.getSGFile(self.rec.hostname, self.rec.storagegroup, \ + self.rec.basename)[1] + dstsize = be.getSGFile(self.vid.host, 'Videos', self.vid.filename)[1] + except: + raise MythError('Could not query file size from backend') + if srcsize != dstsize: + raise MythError('Source size (%d) does not match destination size (%d)' \ + % (srcsize, dstsize)) + + if self.job: + self.job.setComment("Complete - %d seconds elapsed" % \ + (int(time.time()-stime))) + self.job.setStatus(Job.FINISHED) + + def copy_seek(self): + for seek in self.rec.seek: + self.vid.markup.add(seek.mark, seek.offset, seek.type) + + def copy_markup(self, start, stop): + for mark in self.rec.markup: + if mark.type in (start, stop): + self.vid.markup.add(mark.mark, 0, mark.type) + +def usage_format(): + usagestr = """The default strings are: + Television: Television/%TITLE%/Season %SEASON%/%TITLE% - S%SEASON%E%EPISODEPAD% - %SUBTITLE% + Movie: Movies/%TITLE% + Generic: Videos/%TITLE% + +Available strings: + %TITLE%: series title + %SUBTITLE%: episode title + %SEASON%: season number + %SEASONPAD%: season number, padded to 2 digits + %EPISODE%: episode number + %EPISODEPAD%: episode number, padded to 2 digits + %YEAR%: year + %DIRECTOR%: director + %HOSTNAME%: backend used to record show + %STORAGEGROUP%: storage group containing recorded show + %GENRE%: first genre listed for recording +""" +# %CARDID%: ID of tuner card used to record show +# %CARDNAME%: name of tuner card used to record show +# %SOURCEID%: ID of video source used to record show +# %SOURCENAME%: name of video source used to record show +# %CHANNUM%: ID of channel used to record show +# %CHANNAME%: name of channel used to record show +# %COUNTRY%: first country listed for recording + print usagestr + +def print_format(): + db = MythDB() + host = gethostname() + tfmt = db.settings[host]['mythvideo.TVexportfmt'] + if not tfmt: + tfmt = 'Television/%TITLE%/Season %SEASON%/%TITLE% - S%SEASON%E%EPISODEPAD% - %SUBTITLE%' + mfmt = db.settings[host]['mythvideo.MOVIEexportfmt'] + if not mfmt: + mfmt = 'Movies/%TITLE%' + gfmt = db.settings[host]['mythvideo.GENERICexportfmt'] + if not gfmt: + gfmt = 'Videos/%TITLE%' + print "Current output formats:" + print " TV: "+tfmt + print " Movies: "+mfmt + print " Generic: "+gfmt + +def main(): + parser = OptionParser(usage="usage: %prog [options] [jobid]") + + formatgroup = OptionGroup(parser, "Formatting Options", + "These options are used to display and manipulate the output file formats.") + formatgroup.add_option("-f", "--helpformat", action="store_true", default=False, dest="fmthelp", + help="Print explination of file format string.") + formatgroup.add_option("-p", "--printformat", action="store_true", default=False, dest="fmtprint", + help="Print current file format string.") + formatgroup.add_option("--tformat", action="store", type="string", dest="tformat", + help="Use TV format for current task. If no task, store in database.") + formatgroup.add_option("--mformat", action="store", type="string", dest="mformat", + help="Use Movie format for current task. If no task, store in database.") + formatgroup.add_option("--gformat", action="store", type="string", dest="gformat", + help="Use Generic format for current task. If no task, store in database.") + formatgroup.add_option("--listingonly", action="store_true", default=False, dest="listingonly", + help="Use data from listing provider, rather than grabber") + parser.add_option_group(formatgroup) + + sourcegroup = OptionGroup(parser, "Source Definition", + "These options can be used to manually specify a recording to operate on "+\ + "in place of the job id.") + sourcegroup.add_option("--chanid", action="store", type="int", dest="chanid", + help="Use chanid for manual operation") + sourcegroup.add_option("--starttime", action="store", type="string", dest="starttime", + help="Use starttime for manual operation") + parser.add_option_group(sourcegroup) + + actiongroup = OptionGroup(parser, "Additional Actions", + "These options perform additional actions after the recording has been exported.") + actiongroup.add_option('--safe', action='store_true', default=False, dest='safe', + help='Perform quick sanity check of exported file using file size.') + actiongroup.add_option('--really-safe', action='store_true', default=False, dest='reallysafe', + help='Perform slow sanity check of exported file using SHA1 hash.') + actiongroup.add_option("--delete", action="store_true", default=False, + help="Delete source recording after successful export. Enforces use of --safe.") + parser.add_option_group(actiongroup) + + othergroup = OptionGroup(parser, "Other Data", + "These options copy additional information from the source recording.") + othergroup.add_option("--seekdata", action="store_true", default=False, dest="seekdata", + help="Copy seekdata from source recording.") + othergroup.add_option("--skiplist", action="store_true", default=False, dest="skiplist", + help="Copy commercial detection from source recording.") + othergroup.add_option("--cutlist", action="store_true", default=False, dest="cutlist", + help="Copy manual commercial cuts from source recording.") + parser.add_option_group(othergroup) + + MythLog.loadOptParse(parser) + + opts, args = parser.parse_args() + + if opts.verbose: + if opts.verbose == 'help': + print MythLog.helptext + sys.exit(0) + MythLog._setlevel(opts.verbose) + + if opts.fmthelp: + usage_format() + sys.exit(0) + + if opts.fmtprint: + print_format() + sys.exit(0) + + if opts.delete: + opts.safe = True + + if opts.chanid and opts.starttime: + export = VIDEO(opts) + elif len(args) == 1: + try: + export = VIDEO(opts,int(args[0])) + except Exception, e: + Job(int(args[0])).update({'status':Job.ERRORED, + 'comment':'ERROR: '+e.args[0]}) + MythLog(module='mythvidexport.py').logTB(MythLog.GENERAL) + sys.exit(1) + else: + if opts.tformat or opts.mformat or opts.gformat: + db = MythDB() + host = gethostname() + if opts.tformat: + print "Changing TV format to: "+opts.tformat + db.settings[host]['mythvideo.TVexportfmt'] = opts.tformat + if opts.mformat: + print "Changing Movie format to: "+opts.mformat + db.settings[host]['mythvideo.MOVIEexportfmt'] = opts.mformat + if opts.gformat: + print "Changing Generic format to: "+opts.gformat + db.settings[host]['mythvideo.GENERICexportfmt'] = opts.gformat + sys.exit(0) + else: + parser.print_help() + sys.exit(2) + +if __name__ == "__main__": + main() + |