#!/usr/bin/python2 #checks that the MBE can connect to the minions # =============================================== import sys, subprocess import os import string import time import func.overlord.client as fc import datetime BBLINE = '' BBCOLOR="green" DATE = time.strftime("%a %b %d %H:%M:%S %Z %Y", time.localtime(time.time())) BB="/home/xymon/server/bin/xymon" BBDISP="127.0.01" #if os.environ['BB']: # #print os.environ['BB'] # BB=os.environ['BB'] #if os.environ['BBDISP']: # #print os.environ['BBDISP'] # BBDISP=os.environ['BBDISP'] #if os.environ['MACHINE']: # #print os.environ['MACHINE'] # MACHINE=os.environ['MACHINE'] failed_func_hosts_file="/data/srv/httpd/htdocs/failed_func_hosts" def print_out(msg): #print msg pass def clear_func_cert(currentclient): cmd="sudo /usr/bin/certmaster-ca -c " + currentclient os.system(cmd) line="clearing out certs via certmaster: %s" %currentclient print_out(line) def ispresent_failedfunc(host): host=host.split(",")[0].strip() ispresent = False try: f = open(failed_func_hosts_file, "r") failed_list = f.readlines() for line in failed_list: print_out(host+" "+line) if host.strip() == line.split(",")[0].strip(): ispresent = True break except: pass print_out("host: %s is present %s" %(host,ispresent)) return ispresent def remove_from_failedfunc(host): count_fail="/tmp/%s.failed" %host print_out("removing from failed list: %s" %host) f = open(failed_func_hosts_file,'r') failed_lines=f.readlines() f.close f = open(failed_func_hosts_file,'w') for line in failed_lines: if line.startswith(currentclient.strip()): continue f.write(line) f.close() if os.path.exists(count_fail): os.remove(count_fail) def count_failed(host): now = datetime.datetime.now() now_str=str(now) fc=0 count_fail="/tmp/%s.failed" %host with open(count_fail, "a") as f: f.write(now_str+"\n") f = open(count_fail, "r") failed_list = f.readlines() f.close() fc=len(failed_list) print_out("number of failures for %s: %s" %(host,fc)) return fc def add_to_failed_host(host): #check number of times it's failed. If more then three, write func file cf = count_failed(host.split(",")[0].strip()) currentclient = host.split(",")[0].strip() if not ispresent_failedfunc(host) and cf > 3: with open(failed_func_hosts_file, "a") as f: f.write(host+"\n") print_out("adding %s to failed_func" %host) clear_func_cert(currentclient) else: if cf < 3: line="%s has less then 3 failues" %host.split(",")[0] BBLINE="%s: %s failures (yellow) \n " %(currentclient, cf) LINE = "status " + currentclient + ".func yellow" + " " + DATE + " " + BBLINE cmd = BB + ' ' + BBDISP + ' "' + LINE + '"' os.system(cmd) print_out("sending yellow for %s" %currentclient) else: line="%s is already in failed func file" %host.split(",")[0] clear_func_cert(currentclient) print_out(line) def readbb(): global hostlist global mythtype try: infile = open('/home/xymon/etc/hosts.cfg', 'r') except(IOError), e: #print "couldn't open xymon hosts file" sys.exit(1) else: bbhostlist = infile.readlines() infile.close() for line in bbhostlist: cline=line.strip() if not cline.startswith("#") and cline != '' and cline.find("func") != -1 : host=cline.split("#") host=host[0].split() hostlist.append(host[1].strip()) infile.close() try: infile = open('/etc/mythtv-releasetype') mythtype= infile.readline() infile.close() except(IOError), e: #print "couldn't open mythfile " mythtype='' def find_local_myth_version(): global mythtype local_pkg_name="not_found" pkgname="mythtv" pkgname+=mythtype cmd="/usr/bin/pacman" proc = subprocess.Popen([cmd, "-Q" , pkgname], stdout=subprocess.PIPE) result=[] result.append( proc.communicate()[0]) for list in result: l=list.strip() if l.startswith('mythtv') : local_pkg_name=l.strip() break print_out("pkg name is: %s" %local_pkg_name ) return local_pkg_name def mythversion_check(currentclient,local_myth_version): global mythtype pkgname="mythtv" pkgname+=mythtype remote_pkg_name="remote_pkg_not_found" client = (fc.Client( currentclient )) results=client.pacman.pkgversion(pkgname) #print currentclient #print local_myth_version #print results[currentclient][1] try: for pkg in results[currentclient][1] : p=pkg.strip() if p.startswith('mythtv'): remote_pkg_name=p break if (remote_pkg_name != local_myth_version): BBLINE="Myth version MBE: %s does not match %s %s \n " %(local_myth_version,currentclient,remote_pkg_name) LINE = "status " + currentclient + ".version yellow" + " " + DATE + " " + BBLINE if (remote_pkg_name == local_myth_version): BBLINE="%s: success (green) Myth version match %s \n " %(currentclient,remote_pkg_name) LINE = "status " + currentclient + ".version green" + " " + DATE + " " + BBLINE except: BBLINE="Something went wrong! \n " LINE = "status " + currentclient + ".version red" + " " + DATE + " " + BBLINE cmd = BB + ' ' + BBDISP + ' "' + LINE + '"' os.system(cmd) return def send_myth_clear(currentclient): BBLINE="could not connect to host to check version\n " LINE = "status " + currentclient + ".version clear" + " " + DATE + " " + BBLINE cmd = BB + ' ' + BBDISP + ' "' + LINE + '"' os.system(cmd) return #------------------------- global hostlist global mythtype mythtype="" hostlist = [] readbb() local_myth_version=find_local_myth_version() for currentclient_a in hostlist: currentclient=currentclient_a.lower() #print currentclient cmd='/home/xymon/server/bin/xymon 127.0.0.1 "query ' cmd+=currentclient cmd+='.conn"' bbresults=os.popen(cmd,'r' ).readline().strip().split() #check if host is active try: bbstate=bbresults[0] except: bbstate="clear" #print bbstate if bbstate == 'green': try: client = (fc.Client( currentclient.lower() )) except Exception as e: #couldn't find minion in certmaster BBLINE="%s: %s (red) \n " %(currentclient,e) LINE = "status " + currentclient + ".func red" + " " + DATE + " " + BBLINE cmd = BB + ' ' + BBDISP + ' "' + LINE + '"' os.system(cmd) BBCOLOR="red" send_myth_clear(currentclient) add_to_failed_host(currentclient + ", func client error") clear_func_cert(currentclient) continue results=client.test.add("1","2") try: #print currentclient #print results[currentclient] #print type(results[currentclient]) #print "----------" if "socket.error" in results[currentclient]: print_out("socket error occured") BBLINE="%s: %s (red) \n " %(currentclient,"socket error occured") LINE = "status " + currentclient + ".func red" + " " + DATE + " " + BBLINE cmd = BB + ' ' + BBDISP + ' "' + LINE + '"' os.system(cmd) add_to_failed_host(currentclient + ", socket error occured") elif type(results[currentclient]) == str: if results[currentclient] == '12': BBLINE="%s: success (green) \n " %currentclient LINE = "status " + currentclient + ".func green" + " " + DATE + " " + BBLINE cmd = BB + ' ' + BBDISP + ' "' + LINE + '"' os.system(cmd) #REMOVE HOST FROM FAILED_FUNC remove_from_failedfunc(currentclient) mythversion_check(currentclient,local_myth_version) else: BBLINE="%s: connected but wtf (yellow) \n " %currentclient LINE = "status " + currentclient + ".func yellow" + " " + DATE + " " + BBLINE cmd = BB + ' ' + BBDISP + ' "' + LINE + '"' os.system(cmd) if BBCOLOR != "red": BBCOLOR="yellow" else: BBLINE="%s: %s (red) \n " %(currentclient,results[currentclient]) LINE = "status " + currentclient + ".func red" + " " + DATE + " " + BBLINE cmd = BB + ' ' + BBDISP + ' "' + LINE + '"' os.system(cmd) add_to_failed_host(currentclient + ", test failed") BBCOLOR="red" except Exception as e: print_out(e) BBLINE="%s had an error : %s (red) \n " % (currentclient,str(results)) LINE = "status " + currentclient + ".func red" + " " + DATE + " " + BBLINE cmd = BB + ' ' + BBDISP + ' "' + LINE + '"' os.system(cmd) add_to_failed_host(currentclient + ", had an error") BBCOLOR="red" else: BBLINE= "Not testing %s \n " % currentclient LINE = "status " + currentclient + ".func clear" + " " + DATE + " " + BBLINE cmd = BB + ' ' + BBDISP + ' "' + LINE + '"' os.system(cmd) BBCOLOR="clear" send_myth_clear(currentclient)