diff options
Diffstat (limited to 'abs/core/xymon/hbfunc.py')
-rw-r--r-- | abs/core/xymon/hbfunc.py | 174 |
1 files changed, 126 insertions, 48 deletions
diff --git a/abs/core/xymon/hbfunc.py b/abs/core/xymon/hbfunc.py index edd2461..39e1680 100644 --- a/abs/core/xymon/hbfunc.py +++ b/abs/core/xymon/hbfunc.py @@ -2,11 +2,13 @@ #checks that the MBE can connect to the minions # =============================================== -import sys +import sys, subprocess import os import string import time import func.overlord.client as fc +import datetime + BBLINE = '' BBCOLOR="green" DATE = time.strftime("%a %b %d %H:%M:%S %Z %Y", time.localtime(time.time())) @@ -27,6 +29,91 @@ BBDISP="127.0.01" # MACHINE=os.environ['MACHINE'] +failed_func_hosts_file="/data/srv/httpd/htdocs/failed_func_hosts" + +def print_out(msg): + #print msg + pass + +def clear_func_cert(currentclient): + cmd="sudo /usr/bin/certmaster-ca -c " + currentclient + os.system(cmd) + line="clearing out certs via certmaster: %s" %currentclient + print_out(line) + + +def ispresent_failedfunc(host): + host=host.split(",")[0].strip() + ispresent = False + try: + f = open(failed_func_hosts_file, "r") + failed_list = f.readlines() + for line in failed_list: + print_out(host+" "+line) + if host.strip() == line.split(",")[0].strip(): + ispresent = True + break + except: + pass + print_out("host: %s is present %s" %(host,ispresent)) + return ispresent + +def remove_from_failedfunc(host): + count_fail="/tmp/%s.failed" %host + print_out("removing from failed list: %s" %host) + f = open(failed_func_hosts_file,'r') + failed_lines=f.readlines() + f.close + f = open(failed_func_hosts_file,'w') + for line in failed_lines: + if line.startswith(currentclient.strip()): + continue + f.write(line) + f.close() + if os.path.exists(count_fail): + os.remove(count_fail) + +def count_failed(host): + + + now = datetime.datetime.now() + now_str=str(now) + fc=0 + count_fail="/tmp/%s.failed" %host + with open(count_fail, "a") as f: + f.write(now_str+"\n") + + f = open(count_fail, "r") + failed_list = f.readlines() + f.close() + fc=len(failed_list) + print_out("number of failures for %s: %s" %(host,fc)) + return fc + + + +def add_to_failed_host(host): + #check number of times it's failed. If more then three, write func file + cf = count_failed(host.split(",")[0].strip()) + currentclient = host.split(",")[0].strip() + if not ispresent_failedfunc(host) and cf > 3: + with open(failed_func_hosts_file, "a") as f: + f.write(host+"\n") + print_out("adding %s to failed_func" %host) + clear_func_cert(currentclient) + else: + if cf < 3: + line="%s has less then 3 failues" %host.split(",")[0] + BBLINE="%s: %s failures (yellow) \n " %(currentclient, cf) + LINE = "status " + currentclient + ".func yellow" + " " + DATE + " " + BBLINE + cmd = BB + ' ' + BBDISP + ' "' + LINE + '"' + os.system(cmd) + print_out("sending yellow for %s" %currentclient) + else: + line="%s is already in failed func file" %host.split(",")[0] + clear_func_cert(currentclient) + print_out(line) + def readbb(): global hostlist global mythtype @@ -41,12 +128,10 @@ def readbb(): infile.close() for line in bbhostlist: cline=line.strip() - #if not cline.startswith("#") and cline != '' and cline.find("bbd") == -1 and cline.find("func") != -1 : if not cline.startswith("#") and cline != '' and cline.find("func") != -1 : host=cline.split("#") host=host[0].split() hostlist.append(host[1].strip()) - #print hostlist infile.close() try: infile = open('/etc/mythtv-releasetype') @@ -61,13 +146,16 @@ def find_local_myth_version(): local_pkg_name="not_found" pkgname="mythtv" pkgname+=mythtype - cmd="/usr/bin/pacman -Q %s " %pkgname - result = os.popen2(cmd)[1].readlines() + cmd="/usr/bin/pacman" + proc = subprocess.Popen([cmd, "-Q" , pkgname], stdout=subprocess.PIPE) + result=[] + result.append( proc.communicate()[0]) for list in result: l=list.strip() if l.startswith('mythtv') : local_pkg_name=l.strip() break + print_out("pkg name is: %s" %local_pkg_name ) return local_pkg_name @@ -117,13 +205,14 @@ hostlist = [] readbb() local_myth_version=find_local_myth_version() -for currentclient in hostlist: - #print currentclient +for currentclient_a in hostlist: + currentclient=currentclient_a.lower() + print currentclient cmd='/data/srv/xymon/server/bin/xymon 127.0.0.1 "query ' cmd+=currentclient cmd+='.conn"' bbresults=os.popen(cmd,'r' ).readline().strip().split() - + #check if host is active try: bbstate=bbresults[0] except: @@ -132,18 +221,18 @@ for currentclient in hostlist: #print bbstate if bbstate == 'green': try: - client = (fc.Client( currentclient )) + client = (fc.Client( currentclient.lower() )) except Exception as e: #couldn't find minion in certmaster - #print "couldn't find minion in certmaster:%s" %currentclient BBLINE="%s: %s (red) \n " %(currentclient,e) LINE = "status " + currentclient + ".func red" + " " + DATE + " " + BBLINE cmd = BB + ' ' + BBDISP + ' "' + LINE + '"' - cmd2= "echo " + currentclient + " >> /data/srv/httpd/htdocs/failed_func_hosts" - os.system(cmd2) os.system(cmd) BBCOLOR="red" send_myth_clear(currentclient) + add_to_failed_host(currentclient + ", func client error") + clear_func_cert(currentclient) + continue @@ -153,26 +242,24 @@ for currentclient in hostlist: #print results[currentclient] #print type(results[currentclient]) #print "----------" - if type(results[currentclient]) == str: + if "socket.error" in results[currentclient]: + print_out("socket error occured") + BBLINE="%s: %s (red) \n " %(currentclient,"socket error occured") + LINE = "status " + currentclient + ".func red" + " " + DATE + " " + BBLINE + cmd = BB + ' ' + BBDISP + ' "' + LINE + '"' + os.system(cmd) + add_to_failed_host(currentclient + ", socket error occured") + + + elif type(results[currentclient]) == str: if results[currentclient] == '12': + BBLINE="%s: success (green) \n " %currentclient LINE = "status " + currentclient + ".func green" + " " + DATE + " " + BBLINE cmd = BB + ' ' + BBDISP + ' "' + LINE + '"' os.system(cmd) #REMOVE HOST FROM FAILED_FUNC - #cmd='sed -i "/' + currentclient + '/d" /data/srv/httpd/htdocs/failed_func_hosts' - f = open("/data/srv/httpd/htdocs/failed_func_hosts") - failed_lines=f.readlines() - f.close - f = open("/data/srv/httpd/htdocs/failed_func_hosts",'w') - for line in failed_lines: - if line.strip() == currentclient: - continue - f.write(line) - f.close() - - - #os.system(cmd) + remove_from_failedfunc(currentclient) mythversion_check(currentclient,local_myth_version) else: BBLINE="%s: connected but wtf (yellow) \n " %currentclient @@ -185,33 +272,24 @@ for currentclient in hostlist: BBLINE="%s: %s (red) \n " %(currentclient,results[currentclient]) LINE = "status " + currentclient + ".func red" + " " + DATE + " " + BBLINE cmd = BB + ' ' + BBDISP + ' "' + LINE + '"' - cmd2= "echo " + currentclient + " >> /data/srv/httpd/htdocs/failed_func_hosts" - cmd3="sudo /usr/bin/certmaster-ca -c " + currentclient - os.system(cmd2) - os.system(cmd3) os.system(cmd) + add_to_failed_host(currentclient + ", test failed") + + BBCOLOR="red" + + except Exception as e: + print_out(e) + BBLINE="%s had an error : %s (red) \n " % (currentclient,str(results)) + LINE = "status " + currentclient + ".func red" + " " + DATE + " " + BBLINE + cmd = BB + ' ' + BBDISP + ' "' + LINE + '"' + os.system(cmd) + add_to_failed_host(currentclient + ", had an error") + BBCOLOR="red" - except: - BBLINE="%s had an error : %s (red) \n " % (currentclient,str(results)) - LINE = "status " + currentclient + ".func red" + " " + DATE + " " + BBLINE - cmd = BB + ' ' + BBDISP + ' "' + LINE + '"' - cmd2= "echo " + currentclient + " >> /data/srv/httpd/htdocs/failed_func_hosts" - os.system(cmd) - os.system(cmd2) - cmd3="sudo /usr/bin/certmaster-ca -c " + currentclient - os.system(cmd3) - BBCOLOR="red" else: - BBLINE+= "Not testing %s \n " % currentclient + BBLINE= "Not testing %s \n " % currentclient LINE = "status " + currentclient + ".func clear" + " " + DATE + " " + BBLINE cmd = BB + ' ' + BBDISP + ' "' + LINE + '"' os.system(cmd) BBCOLOR="clear" send_myth_clear(currentclient) - -#LINE = "status " + MACHINE + ".func green" + " " + DATE + " " + BBLINE -#cmd = BB + ' ' + BBDISP + ' "' + LINE + '"' -#os.system(cmd) - -#print BBLINE -#print BBCOLOR |