xymon: add led theme

  logrotate: send hup after rotate
  hbfunc: general rewrite
  hbnotes: better output formatting for the dot file, fix slave tuners in the dot file

1 files changed, 126 insertions, 48 deletions

diff --git a/abs/core/xymon/hbfunc.py b/abs/core/xymon/hbfunc.py
index edd2461..39e1680 100644
--- a/abs/core/xymon/hbfunc.py
+++ b/abs/core/xymon/hbfunc.py
@@ -2,11 +2,13 @@
 #checks that the MBE can connect to the minions
 
 # ===============================================
-import sys
+import sys, subprocess
 import os
 import string
 import time
 import func.overlord.client as fc
+import datetime
+
 BBLINE = ''
 BBCOLOR="green"
 DATE = time.strftime("%a %b %d %H:%M:%S %Z %Y", time.localtime(time.time()))
@@ -27,6 +29,91 @@ BBDISP="127.0.01"
 #     MACHINE=os.environ['MACHINE']
 
 
+failed_func_hosts_file="/data/srv/httpd/htdocs/failed_func_hosts"
+
+def print_out(msg):
+    #print msg
+    pass
+
+def clear_func_cert(currentclient):
+     cmd="sudo /usr/bin/certmaster-ca -c " + currentclient
+     os.system(cmd)
+     line="clearing out certs via certmaster: %s" %currentclient
+     print_out(line)
+
+
+def ispresent_failedfunc(host):
+    host=host.split(",")[0].strip()
+    ispresent = False
+    try:
+        f = open(failed_func_hosts_file, "r")
+        failed_list = f.readlines()
+        for line in failed_list:
+            print_out(host+" "+line)
+            if host.strip() == line.split(",")[0].strip():
+                ispresent = True
+                break
+    except:
+        pass
+    print_out("host: %s is present %s" %(host,ispresent))
+    return ispresent
+
+def remove_from_failedfunc(host):
+    count_fail="/tmp/%s.failed" %host
+    print_out("removing from failed list: %s" %host)
+    f = open(failed_func_hosts_file,'r')
+    failed_lines=f.readlines()
+    f.close
+    f = open(failed_func_hosts_file,'w')
+    for line in failed_lines:
+        if line.startswith(currentclient.strip()):
+            continue
+        f.write(line)
+    f.close()
+    if os.path.exists(count_fail):
+        os.remove(count_fail)
+
+def count_failed(host):
+
+
+    now = datetime.datetime.now()
+    now_str=str(now)
+    fc=0
+    count_fail="/tmp/%s.failed" %host
+    with open(count_fail, "a") as f:
+            f.write(now_str+"\n")
+
+    f = open(count_fail, "r")
+    failed_list = f.readlines()
+    f.close()
+    fc=len(failed_list)
+    print_out("number of failures for %s: %s" %(host,fc))
+    return fc
+
+
+
+def add_to_failed_host(host):
+    #check number of times it's failed.  If more then three, write func file
+    cf = count_failed(host.split(",")[0].strip())
+    currentclient = host.split(",")[0].strip()
+    if not ispresent_failedfunc(host) and cf  > 3:
+        with open(failed_func_hosts_file, "a") as f:
+            f.write(host+"\n")
+        print_out("adding %s to failed_func" %host)
+        clear_func_cert(currentclient)
+    else:
+        if cf < 3:
+            line="%s has less then 3 failues" %host.split(",")[0]
+            BBLINE="%s: %s failures  (yellow)  \n "  %(currentclient, cf)
+            LINE = "status " + currentclient  + ".func yellow" + " " + DATE + " " + BBLINE
+            cmd = BB + ' ' + BBDISP + ' "' + LINE + '"'
+            os.system(cmd)
+            print_out("sending yellow for %s" %currentclient)
+        else:
+            line="%s is already in failed func file" %host.split(",")[0]
+            clear_func_cert(currentclient)
+        print_out(line)
+
 def readbb():
         global hostlist
         global mythtype
@@ -41,12 +128,10 @@ def readbb():
                 infile.close()
         for line in bbhostlist:
             cline=line.strip()
-            #if not cline.startswith("#") and cline != '' and cline.find("bbd") == -1 and cline.find("func") != -1  :
             if not cline.startswith("#") and cline != '' and cline.find("func") != -1  :
                 host=cline.split("#")
                 host=host[0].split()
                 hostlist.append(host[1].strip())
-        #print hostlist
         infile.close()
         try:
                 infile = open('/etc/mythtv-releasetype')
@@ -61,13 +146,16 @@ def find_local_myth_version():
     local_pkg_name="not_found"
     pkgname="mythtv"
     pkgname+=mythtype
-    cmd="/usr/bin/pacman -Q %s " %pkgname
-    result = os.popen2(cmd)[1].readlines()
+    cmd="/usr/bin/pacman"
+    proc = subprocess.Popen([cmd, "-Q" , pkgname], stdout=subprocess.PIPE)
+    result=[]
+    result.append( proc.communicate()[0])
     for list in result:
         l=list.strip()
         if l.startswith('mythtv') :
             local_pkg_name=l.strip()
             break
+    print_out("pkg name is: %s" %local_pkg_name )
     return local_pkg_name
 
 
@@ -117,13 +205,14 @@ hostlist = []
 readbb()
 local_myth_version=find_local_myth_version()
 
-for currentclient in hostlist:
-    #print currentclient
+for currentclient_a in hostlist:
+    currentclient=currentclient_a.lower()
+    print currentclient
     cmd='/data/srv/xymon/server/bin/xymon 127.0.0.1  "query '
     cmd+=currentclient
     cmd+='.conn"'
     bbresults=os.popen(cmd,'r'	).readline().strip().split()
-
+    #check if host is active
     try:
         bbstate=bbresults[0]
     except:
@@ -132,18 +221,18 @@ for currentclient in hostlist:
     #print bbstate
     if bbstate == 'green':
         try:
-            client = (fc.Client( currentclient ))
+            client = (fc.Client( currentclient.lower() ))
         except Exception as e:
             #couldn't find minion in certmaster
-            #print "couldn't find minion in certmaster:%s" %currentclient
             BBLINE="%s: %s  (red)  \n "  %(currentclient,e)
             LINE = "status " + currentclient  + ".func red" + " " + DATE + " " + BBLINE
             cmd = BB + ' ' + BBDISP + ' "' + LINE + '"'
-            cmd2= "echo " + currentclient + " >> /data/srv/httpd/htdocs/failed_func_hosts"
-            os.system(cmd2)
             os.system(cmd)
             BBCOLOR="red"
             send_myth_clear(currentclient)
+            add_to_failed_host(currentclient + ", func client error")
+            clear_func_cert(currentclient)
+
             continue
 
 
@@ -153,26 +242,24 @@ for currentclient in hostlist:
             #print results[currentclient]
             #print type(results[currentclient])
             #print "----------"
-            if  type(results[currentclient]) == str:
+            if "socket.error" in results[currentclient]:
+                print_out("socket error occured")
+                BBLINE="%s: %s  (red)  \n "  %(currentclient,"socket error occured")
+                LINE = "status " + currentclient  + ".func red" + " " + DATE + " " + BBLINE
+                cmd = BB + ' ' + BBDISP + ' "' + LINE + '"'
+                os.system(cmd)
+                add_to_failed_host(currentclient + ", socket error occured")
+
+
+            elif  type(results[currentclient]) == str:
                 if results[currentclient] == '12':
+
                     BBLINE="%s:  success (green) \n " %currentclient
                     LINE = "status " + currentclient  + ".func green" + " " + DATE + " " + BBLINE
                     cmd = BB + ' ' + BBDISP + ' "' + LINE + '"'
                     os.system(cmd)
                     #REMOVE HOST FROM FAILED_FUNC
-                    #cmd='sed -i "/' +  currentclient + '/d" /data/srv/httpd/htdocs/failed_func_hosts'
-                    f = open("/data/srv/httpd/htdocs/failed_func_hosts")
-                    failed_lines=f.readlines()
-                    f.close
-                    f = open("/data/srv/httpd/htdocs/failed_func_hosts",'w')
-                    for line in failed_lines:
-                        if line.strip() == currentclient:
-                            continue
-                        f.write(line)
-                    f.close()
-
-
-                    #os.system(cmd)
+                    remove_from_failedfunc(currentclient)
                     mythversion_check(currentclient,local_myth_version)
                 else:
                     BBLINE="%s:  connected but wtf  (yellow)  \n " %currentclient
@@ -185,33 +272,24 @@ for currentclient in hostlist:
                 BBLINE="%s: %s  (red)  \n "  %(currentclient,results[currentclient])
                 LINE = "status " + currentclient  + ".func red" + " " + DATE + " " + BBLINE
                 cmd = BB + ' ' + BBDISP + ' "' + LINE + '"'
-                cmd2= "echo " + currentclient + " >> /data/srv/httpd/htdocs/failed_func_hosts"
-                cmd3="sudo /usr/bin/certmaster-ca -c " + currentclient
-                os.system(cmd2)
-                os.system(cmd3)
                 os.system(cmd)
+                add_to_failed_host(currentclient + ", test failed")
+
+                BBCOLOR="red"
+
+        except Exception as e:
+                print_out(e)
+                BBLINE="%s had an error : %s (red) \n "   % (currentclient,str(results))
+                LINE = "status " + currentclient  + ".func red" + " " + DATE + " " + BBLINE
+                cmd = BB + ' ' + BBDISP + ' "' + LINE + '"'
+                os.system(cmd)
+                add_to_failed_host(currentclient + ", had an error")
+
                 BBCOLOR="red"
-        except:
-            BBLINE="%s had an error : %s (red) \n "   % (currentclient,str(results))
-            LINE = "status " + currentclient  + ".func red" + " " + DATE + " " + BBLINE
-            cmd = BB + ' ' + BBDISP + ' "' + LINE + '"'
-            cmd2= "echo " + currentclient + " >> /data/srv/httpd/htdocs/failed_func_hosts"
-            os.system(cmd)
-            os.system(cmd2)
-            cmd3="sudo /usr/bin/certmaster-ca -c " + currentclient
-            os.system(cmd3)
-            BBCOLOR="red"
     else:
-        BBLINE+= "Not  testing %s \n "  % currentclient
+        BBLINE= "Not  testing %s \n "  % currentclient
         LINE = "status " + currentclient  + ".func clear" + " " + DATE + " " + BBLINE
         cmd = BB + ' ' + BBDISP + ' "' + LINE + '"'
         os.system(cmd)
         BBCOLOR="clear"
         send_myth_clear(currentclient)
-
-#LINE = "status " + MACHINE + ".func green" + " " + DATE + " " + BBLINE
-#cmd = BB + ' ' + BBDISP + ' "' + LINE + '"'
-#os.system(cmd)
-
-#print BBLINE
-#print BBCOLOR