From b5e92157c5caad66b4da78486f8a63aba3761b87 Mon Sep 17 00:00:00 2001 From: FernandoGarzon Date: Thu, 16 Jul 2020 18:27:16 +0200 Subject: [PATCH 1/8] Error with the xml.dom.minidom.parseString() method corrected --- AAAOps/XfedKibana/XRDFED-kibana-probe.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/AAAOps/XfedKibana/XRDFED-kibana-probe.py b/AAAOps/XfedKibana/XRDFED-kibana-probe.py index e924123..1f8948f 100755 --- a/AAAOps/XfedKibana/XRDFED-kibana-probe.py +++ b/AAAOps/XfedKibana/XRDFED-kibana-probe.py @@ -132,6 +132,8 @@ def xrd_info(redirector): [redirector, "query","1", # 1:kXR_QStats "a"]) # a_ll stats + if not out: + out = "1" if not errtext: try: dom = xml.dom.minidom.parseString(out) @@ -150,9 +152,7 @@ def run_xrd_commands(cmd,args): xrd_args = [ 'perl','-e',"alarm 180; exec @ARGV", cmd, # one-line wrapper that *actually* kills the command "-DIConnectTimeout","30", "-DITransactionTimeout","60", - "-DIRequestTimeout","60" ] + args - if not out: - out = "1" + "-DIRequestTimeout","60" ] + args try: start = time.time() proc = subprocess.Popen(xrd_args, From 6a593925d18f82d5955b262d6402596eecf94463 Mon Sep 17 00:00:00 2001 From: FernandoGarzon Date: Tue, 21 Jul 2020 23:47:56 +0200 Subject: [PATCH 2/8] Version of redirectors updated --- AAAOps/XfedKibana/XRDFED-kibana-probe.py | 36 +++++++++++++++--------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/AAAOps/XfedKibana/XRDFED-kibana-probe.py b/AAAOps/XfedKibana/XRDFED-kibana-probe.py index 1f8948f..dbf1e1c 100755 --- a/AAAOps/XfedKibana/XRDFED-kibana-probe.py +++ b/AAAOps/XfedKibana/XRDFED-kibana-probe.py @@ -128,21 +128,31 @@ def xrdcp_test(redirector,file): def xrd_info(redirector): version = "(unknown)" - (errtext,out,err,elapsed) = run_xrd_commands("xrd", + (errtext,out,err,elapsed) = run_xrd_commands("xrdfs", [redirector, - "query","1", # 1:kXR_QStats - "a"]) # a_ll stats + "query","config", # 1:kXR_QStats + "version"]) # a_ll stats + if not out: - out = "1" - if not errtext: - try: - dom = xml.dom.minidom.parseString(out) - root_node = dom.documentElement - if root_node.tagName == 'statistics': - v_attr = root_node.getAttributeNode('ver') - version = v_attr.nodeValue - except Exception,e: - errtext = "ERROR: cannot parse answer:"+str(e) + errtext = '' + os.system("xrdfs "+ redirector+" query config version > /root/aux.txt") + os.system("head -n 1 /root/aux.txt > /root/aux2.txt ") + f = open('/root/aux2.txt', 'r') + version = f.read() + if not version: + version = "(unknown)" + else: + version = version[:-1] + else: + if not errtext: + try: + dom = xml.dom.minidom.parseString(out) + root_node = dom.documentElement + if root_node.tagName == 'statistics': + v_attr = root_node.getAttributeNode('ver') + version = v_attr.nodeValue + except Exception,e: + errtext = "ERROR: cannot parse answer:"+str(e) return (errtext,version,out) def run_xrd_commands(cmd,args): From eacd8b040a50df91eb6d942593599fbb57c5fec1 Mon Sep 17 00:00:00 2001 From: Fernando Garzon Date: Tue, 18 Aug 2020 15:12:31 -0500 Subject: [PATCH 3/8] Probe working and producing Json File --- AAAOps/XfedKibana/XRDFED-kibana-probe_JSON.py | 1164 +++++++++++++++++ 1 file changed, 1164 insertions(+) create mode 100755 AAAOps/XfedKibana/XRDFED-kibana-probe_JSON.py diff --git a/AAAOps/XfedKibana/XRDFED-kibana-probe_JSON.py b/AAAOps/XfedKibana/XRDFED-kibana-probe_JSON.py new file mode 100755 index 0000000..9d614cb --- /dev/null +++ b/AAAOps/XfedKibana/XRDFED-kibana-probe_JSON.py @@ -0,0 +1,1164 @@ +#!/usr/bin/python +# functional probe and SLS extractor for the "federation" xroot services +# highlights: +# - stateless (i.e. run from cron whenever needed) +# - will try to prevent parallel runs via lockfile +# - multithreaded, one thread per service to be tested +# - overall runtime cap at 10min +# - could extract some statistics from xroot directly, but these are ever-increasing counters +# Problems: +# - need to update the code whenever a service is addded/deleted/changed +# - uses "random" files on various Xroot services all over the world, these are (for now) the same as used by the experiments but these might change.. + +import xml.dom.minidom +import subprocess +import os +import sys +import signal +import re +import time +import Lemon.XMLAPI +import socket +import atexit +import threading +import tempfile +import json +import shutil + +html_dir = '/root/ogarzonm/' # will create per-service json files here + +LOCKFILE='/var/lock/subsys/xrdfed-kibana-probe' + +class Alarm(Exception): + pass + +def alarm_handler(signum, frame): + print "ERROR: caught overall timeout after "+str(timeout_sec)+"s\n" + clear_lock() + sys.exit(2) + raise Alarm + +def clear_lock(): + try: + os.unlink(LOCKFILE) + except Exception,e: + print "could not remove lockfile:"+str(e) + +def env_setup(): + os.environ['X509_USER_CERT']='/root/.globus/slsprobe-cert.pem' + os.environ['X509_USER_KEY']='/root/.globus/slsprobe-key.pem' + os.environ['X509_USER_PROXY']='/root/.globus/slsprobe.proxy' + os.environ['KRB5CCNAME']='FILE:/dev/null' + os.environ['PATH']=os.environ['PATH']+":/opt/globus/bin/" + +def get_proxy(): + dev_null = open('/dev/null', 'rw') + (proxyfd,proxy)=tempfile.mkstemp(prefix='x509_xrdfed_',suffix='.pem') + os.close(proxyfd) + os.environ['X509_USER_PROXY']=proxy + ret = subprocess.call(['grid-proxy-init','-pwstdin'],stdin=dev_null,) + if ret > 0: + raise Exception("Cannot get X509 proxy") + dev_null.close() + +def cleanup_proxy(): + try: + os.unlink(os.environ['X509_USER_PROXY']) + except Exception,e: + print "could not remove proxy file:"+str(e) + +def try_lock(): + ret = subprocess.call(['lockfile','-5','-r2',LOCKFILE]) + if ret > 0: + print "could not create lockfile" + return False + return True + +def prepare_dictionary(servicename,redirector): + (errtext,version,out) = xrd_info(redirector) + dic={'service':servicename, 'version': version, 'host': redirector[:redirector.find(':')]} + if(errtext): + dic['status'] = 'unavailable' + dic['availinfo'] = " Error getting info from redirector: "+err_info + dic["xrdcp_below_time"] = 0 + dic["xrdcp_above_time"] = 0 + return dic + +def dnsalias_to_nodes(redirector): + (host,port) = redirector.split(':') + all_hosts = [] + data=socket.getaddrinfo(host,port,0, 0, socket.SOL_TCP ) + for addr in data: + (family, socktype, proto, canonname, sockaddr) = addr + (hostname, aliaslist, ipaddrlist) = socket.gethostbyaddr(sockaddr[0]) + if not hostname in all_hosts: + all_hosts.append(hostname) + return all_hosts + +def xrdcp_test(redirector,file): + (errtext,out,err,elapsed) = run_xrd_commands("xrdcp", + ["-d","1", + "-f", + "-DIReadCacheSize","0", + "-DIRedirCntTimeout","180", + "root://"+redirector+'/'+file, + '/dev/null']) + return (errtext,err,elapsed) + +def xrd_info(redirector): + version = "(unknown)" + (errtext,out,err,elapsed) = run_xrd_commands("xrdfs", + [redirector, + "query","config", # 1:kXR_QStats + "version"]) # a_ll stats + + if not out: + errtext = '' + os.system("xrdfs "+ redirector+" query config version > /root/aux.txt") + os.system("head -n 1 /root/aux.txt > /root/aux2.txt") + f = open('/root/aux2.txt', 'r') + version = f.read() + if not version: + version = "(unknown)" + else: + version = version[:-1] + else: + if not errtext: + try: + dom = xml.dom.minidom.parseString(out) + root_node = dom.documentElement + if root_node.tagName == 'statistics': + v_attr = root_node.getAttributeNode('ver') + version = v_attr.nodeValue + except Exception,e: + errtext = "ERROR: cannot parse answer:"+str(e) + return (errtext,version,out) + +def run_xrd_commands(cmd,args): + dev_null = open('/dev/null', 'r') + errtxt = '' + elapsed = -1.0 + xrd_args = [ 'perl','-e',"alarm 180; exec @ARGV", cmd, # one-line wrapper that *actually* kills the command + "-DIConnectTimeout","30", + "-DITransactionTimeout","60", + "-DIRequestTimeout","60" ] + args + err = '' + out = '' + try: + ran_try = True + start = time.time() + proc = subprocess.Popen(xrd_args, + stdin=dev_null, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + (out, err) = proc.communicate() + + ret = proc.returncode + elapsed = (time.time() - start) + err_redir_index = err.rfind('Received redirection to') + err_index3010 = err.rfind('(error code: 3010') # (permission denied) may be sort-of-OK - we are talking to final storage already - UK + err_index3005 = err.rfind('(error code: 3005') # (no user mapping) - INFN + if err_redir_index >= 0 and (err_index3010 >= 0 or err_index3005 >= 0): + errtxt = '' + else: + if(ret > 0): + errtxt = "client-side error - exit code "+str(ret)+"\n" + err_index = err.rfind('Last server error') + if err_index >= 0: + err_end_index=err.find("\n",err_index) + errtxt = errtxt + err[err_index:err_end_index] + except Exception,e: + errtext = errtxt + "Exception: "+str(e) + dev_null.close() + return (errtxt,out,err,elapsed) + +def test_redirector(dicci, servicename, redirector, file_below=None, file_above=None, extra_notes=""): + servicename=servicename.upper() + notes_text = "Redirector:"+redirector + availability = 'available' + availinfo = '' + c = 'No comment' + + # prepare the dictionary. + #dicci['host'] = redirector.split(':')[0] + + # run the functional tests - first some simple check to get the version, if OK look for files + #(err_info,version,dump_info) = xrd_info(redirector) + #if(err_info): + # + # availability = 'unavailable' + # availinfo=availinfo+" Error getting info from redirector "+err_info + # dicci["xrdcp_below_time"] = 0 + #dicci["status"] = "unavailable" + if 'status' in dicci and dicci['status'] == 'unavailable': + pass + else: + if (file_below): + notes_text = notes_text + "File 'below': " + file_below + (err_below,dump_below,elapsed_below) = xrdcp_test(redirector, file_below) + if err_below: + availability = 'degraded' + availinfo=availinfo+" Error below redirector "+err_below + dump_sane = re.sub('---*','__',dump_below) + c = "Detailed output for file BELOW "+redirector+":"+file_below+" "+err_below+" "+dump_sane + #dicci['comment'] = c + else: + availinfo=availinfo+" File below: OK " + dicci['xrdcp_below_time'] = str(elapsed_below) + else: + availinfo=availinfo+" File below: not tested." + if(file_above): + notes_text = notes_text + "File 'elsewhere': " + file_above + (err_above,dump_above,elapsed_above) = xrdcp_test(redirector, file_above) + if err_above : + #We've changed availability from number to string so this below won't work; Marian commented out on 2015-11-06 + #availability = availability * 0.8 # less important if some remote site is failing.. + availinfo=availinfo+" Error above redirector "+err_above + # sanitize the raw output in order to not trigger XML errors.. in a comment. + dump_sane = re.sub('---*','__',dump_above) + c = "Detailed output for file ABOVE "+redirector+":"+file_above+" "+err_above+" "+dump_sane + #dicci = {**dicci, **{'comment': c}} + #serviceUpdate.appendChild(c) + #need_xml_link=1 + else: + availinfo = availinfo+" File above: OK " + #nValue = doc.createElement("numericvalue") + #nValue.setAttribute("name", "xrdcp_above_time") + #nValue.setAttribute("desc", "Time to copy a file elsewhere in the federation") + #nValue.appendChild(doc.createTextNode(str(elapsed_above))) + dicci['xrdcp_above_time'] = str(elapsed_above) + #data.appendChild(nValue) + else: + availinfo = availinfo+" File above: not tested." + + # save functional test info to XML + #if need_xml_link: + # myhostname = socket.gethostname() + # notes_text = notes_text + "Details for failed test: http://" + myhostname + "/aaa-probe/" + servicename + ".xml
\n" + "Details for recently failed test : http://vocms039.cern.ch/aaa-probe/err/
\n" + availinfo = availinfo + " " + notes_text + dicci['status']= str(availability) + if availability == 'unavailable' or availability == 'degraded': + dicci ['availInfo'] = availinfo + dicci ['Comment'] = c + #return dicci + #dicci['availabilityinfo']=availinfo + with open(html_dir +'KIBANA_PROBES.json', 'a') as f: + json.dump(dicci, f) + f.write('\n') + + +def main(): + debug = 0 + atexit.register(clear_lock) + if len(sys.argv) > 1: + if sys.argv[1] == '-d': + debug=1 + if not try_lock(): + sys.exit(1) + if not os.path.exists(html_dir): + os.makedirs(html_dir) + env_setup() + # get a proxy cert + # get_proxy() + + timeout_sec = 10 * 60 # limit overall runtime to 10min + signal.signal(signal.SIGALRM, alarm_handler) + + ATLASLINK="%BR%Monitoring:%BR%\n http://atl-prod07.slac.stanford.edu:8080/display?page=xrd_report/aggregated/total_xrootd_lgn %BR%\n http://dashb-atlas-xrootd-transfers.cern.ch/ui %BR%\nhttp://dashb-atlas-ssb.cern.ch/dashboard/request.py/siteview#currentView=FAX+redirectors&highlight=false %BR%\n" + CMSLINK="%BR%Monitoring:%BR%\n http://xrootd.t2.ucsd.edu/dashboard/ %BR%\n http://dashb-cms-xrootd-transfers.cern.ch/ui %BR%\n" + FILEABOVE="/store/mc/SAM/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v1/00000/A64CCCF2-5C76-E711-B359-0CC47A78A3F8.root" + FILEBELOW="/store/mc/SAM/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v1/00000/A64CCCF2-5C76-E711-B359-0CC47A78A3F8.root" + + services = { + "XRDFED_CMS-GLOBAL01-NEW":{'redirector':'cms-xrd-global01.cern.ch:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + + "XRDFED_CMS-GLOBAL02-NEW":{'redirector':'cms-xrd-global02.cern.ch:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + + "XRDFED_CMS-US-FNAL":{'redirector':'cmsxrootd2.fnal.gov:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + + "XRDFED_CMS-US-UNL":{'redirector':'xrootd.unl.edu:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + + "XRDFED_CMS-EU-BARI":{'redirector':'xrootd.ba.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + + "XRDFED_CMS-EU-LLR":{'redirector':'llrxrd-redir.in2p3.fr:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + + "XRDFED_CMS-EU-PISA":{'redirector':'xrootd-redic.pi.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + + "XRDFED_CMS-GLOBAL":{'redirector':'cms-xrd-global.cern.ch:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + + "XRDFED_CMS-US":{'redirector':'cmsxrootd.fnal.gov:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + + "XRDFED_CMS-EU":{'redirector':'xrootd-cms.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + + "XRDFED_CMS-EU-IPv6":{ 'redirector':'xrootd-cms-redir-01.cr.cnaf.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-TRANSIT":{'redirector':'cms-xrd-transit.cern.ch:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + + "XRDFED_CMS-TRANSIT01":{'redirector':'vocms031.cern.ch:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + + "XRDFED_CMS-TRANSIT02":{'redirector':'vocms032.cern.ch:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + + } + signal.alarm(timeout_sec) + #shutil.copyfile(html_dir+'KIBANA_PROBES.json', html_dir+'KIBANA_PROBES_2.json') + os.remove(html_dir+'KIBANA_PROBES.json') + #shutil.copyfile(html_dir+'KIBANA_PROBES.json', html_dir+'KIBANA_PROBES_2.json') + try: + diccionaries = [] + for xrd in services: + services[xrd].update(servicename=xrd) + servicename = xrd + dicci = prepare_dictionary(servicename, services[xrd]['redirector']) + diccionaries.append(dicci) + for dicci in diccionaries: + service = dicci['service'] + argus = services[service] + argus['dicci'] = dicci + if debug: + test_redirector(** services[xrd]) + else: + t = threading.Thread(target=test_redirector, kwargs = argus) # read: "run a thread with the test function and all the parameters above as arguments" + t.start() + except Alarm: + print "ERROR: caught overall timeout after "+str(timeout_sec)+"s\n" + clear_lock() + sys.exit(2) + signal.alarm(0) + #shutil.copyfile(html_dir+'KIBANA_PROBES.json', html_dir+'KIBANA_PROBES_2.json') + # not cleaning up the proxy files (are shared via the ENV, and we don't want an extra thread to just remove that file, or wait for the individual tests to finish... + +if __name__ == '__main__': + main() + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From c8bdd3360b0f38916c2bfae059d29d5d4eba5d91 Mon Sep 17 00:00:00 2001 From: Fernando Garzon Date: Tue, 18 Aug 2020 15:13:55 -0500 Subject: [PATCH 4/8] Probe working and producing Json File. --- AAAOps/XfedKibana/XRDFED-kibana-probe_JSON.py | 792 ------------------ 1 file changed, 792 deletions(-) diff --git a/AAAOps/XfedKibana/XRDFED-kibana-probe_JSON.py b/AAAOps/XfedKibana/XRDFED-kibana-probe_JSON.py index 9d614cb..b83f242 100755 --- a/AAAOps/XfedKibana/XRDFED-kibana-probe_JSON.py +++ b/AAAOps/XfedKibana/XRDFED-kibana-probe_JSON.py @@ -370,795 +370,3 @@ def main(): if __name__ == '__main__': main() - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From ac7be82dbe36279ece63ce3ef72bc6d67d0796fa Mon Sep 17 00:00:00 2001 From: FernandoGarzon Date: Thu, 3 Sep 2020 00:55:28 +0200 Subject: [PATCH 5/8] New probe for redirectors --- .../XRDFED-kibana-probe_JSON.py | 288 ++++++++++++++++++ 1 file changed, 288 insertions(+) create mode 100755 AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON.py diff --git a/AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON.py b/AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON.py new file mode 100755 index 0000000..a62744b --- /dev/null +++ b/AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON.py @@ -0,0 +1,288 @@ +#!/usr/bin/python +# functional probe and SLS extractor for the "federation" xroot services +# highlights: +# - stateless (i.e. run from cron whenever needed) +# - will try to prevent parallel runs via lockfile +# - multithreaded, one thread per service to be tested +# - overall runtime cap at 10min +# - could extract some statistics from xroot directly, but these are ever-increasing counters +# Problems: +# - need to update the code whenever a service is addded/deleted/changed +# - uses "random" files on various Xroot services all over the world, these are (for now) the same as used by the experiments but these might change.. +import xml.dom.minidom +import subprocess +import os +import sys +import signal +import re +import time +import Lemon.XMLAPI +import socket +import atexit +import threading +import tempfile +import json +import shutil +html_dir = '/var/www/html/aaa-probe/' # will create per-service json files here +LOCKFILE='/var/lock/subsys/xrdfed-kibana-probe' +class Alarm(Exception): + pass +def alarm_handler(signum, frame): + print "ERROR: caught overall timeout after "+str(timeout_sec)+"s\n" + clear_lock() + sys.exit(2) + raise Alarm +def clear_lock(): + try: + os.unlink(LOCKFILE) + except Exception,e: + print "could not remove lockfile:"+str(e) +def env_setup(): + os.environ['X509_USER_CERT']='/root/.globus/slsprobe-cert.pem' + os.environ['X509_USER_KEY']='/root/.globus/slsprobe-key.pem' + os.environ['X509_USER_PROXY']='/root/.globus/slsprobe.proxy' + os.environ['KRB5CCNAME']='FILE:/dev/null' + os.environ['PATH']=os.environ['PATH']+":/opt/globus/bin/" +def get_proxy(): + dev_null = open('/dev/null', 'rw') + (proxyfd,proxy)=tempfile.mkstemp(prefix='x509_xrdfed_',suffix='.pem') + os.close(proxyfd) + os.environ['X509_USER_PROXY']=proxy + ret = subprocess.call(['grid-proxy-init','-pwstdin'],stdin=dev_null,) + if ret > 0: + raise Exception("Cannot get X509 proxy") + dev_null.close() +def cleanup_proxy(): + try: + os.unlink(os.environ['X509_USER_PROXY']) + except Exception,e: + print "could not remove proxy file:"+str(e) +def try_lock(): + ret = subprocess.call(['lockfile','-5','-r2',LOCKFILE]) + if ret > 0: + print "could not create lockfile" + return False + return True +def prepare_dictionary(servicename,redirector): + (errtext,version,out) = xrd_info(redirector) + dic={'service':servicename, 'version': version, 'host': redirector[:redirector.find(':')]} + if(errtext): + dic['status'] = 'unavailable' + dic['availinfo'] = " Error getting info from redirector: "+err_info + dic["xrdcp_below_time"] = 0 + dic["xrdcp_above_time"] = 0 + return dic +def xrdcp_test(redirector,file): + (errtext,out,err,elapsed) = run_xrd_commands("xrdcp", + ["-d","1", + "-f", + "-DIReadCacheSize","0", + "-DIRedirCntTimeout","180", + "root://"+redirector+'/'+file, + '/dev/null']) + return (errtext,err,elapsed) +def xrd_info(redirector): + version = "(unknown)" + (errtext,out,err,elapsed) = run_xrd_commands("xrdfs", + [redirector, + "query","config", # 1:kXR_QStats + "version"]) # a_ll stats + if not out: + errtext = '' + os.system("xrdfs "+ redirector+" query config version > /root/aux.txt") + os.system("head -n 1 /root/aux.txt > /root/aux2.txt") + f = open('/root/aux2.txt', 'r') + version = f.read() + if not version: + version = "(unknown)" + else: + version = version[:-1] + else: + if not errtext: + try: + dom = xml.dom.minidom.parseString(out) + root_node = dom.documentElement + if root_node.tagName == 'statistics': + v_attr = root_node.getAttributeNode('ver') + version = v_attr.nodeValue + except Exception,e: + errtext = "ERROR: cannot parse answer:"+str(e) + return (errtext,version,out) +def run_xrd_commands(cmd,args): + dev_null = open('/dev/null', 'r') + errtxt = '' + elapsed = -1.0 + xrd_args = [ 'perl','-e',"alarm 180; exec @ARGV", cmd, # one-line wrapper that *actually* kills the command + "-DIConnectTimeout","30", + "-DITransactionTimeout","60", + "-DIRequestTimeout","60" ] + args + err = '' + out = '' + try: + ran_try = True + start = time.time() + proc = subprocess.Popen(xrd_args, + stdin=dev_null, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + (out, err) = proc.communicate() + ret = proc.returncode + elapsed = (time.time() - start) + err_redir_index = err.rfind('Received redirection to') + err_index3010 = err.rfind('(error code: 3010') # (permission denied) may be sort-of-OK - we are talking to final storage already - UK + err_index3005 = err.rfind('(error code: 3005') # (no user mapping) - INFN + if err_redir_index >= 0 and (err_index3010 >= 0 or err_index3005 >= 0): + errtxt = '' + else: + if(ret > 0): + errtxt = "client-side error - exit code "+str(ret)+"\n" + err_index = err.rfind('Last server error') + if err_index >= 0: + err_end_index=err.find("\n",err_index) + errtxt = errtxt + err[err_index:err_end_index] + except Exception,e: + errtext = errtxt + "Exception: "+str(e) + dev_null.close() + return (errtxt,out,err,elapsed) +def test_redirector(dicci, servicename, redirector, file_below=None, file_above=None, extra_notes=""): + servicename=servicename.upper() + notes_text = "Redirector:"+redirector + availability = 'available' + availinfo = '' + c = 'No comment' + if 'status' in dicci and dicci['status'] == 'unavailable': + pass + else: + if (file_below): + notes_text = notes_text + "File 'below': " + file_below + (err_below,dump_below,elapsed_below) = xrdcp_test(redirector, file_below) + if err_below: + availability = 'degraded' + availinfo=availinfo+" Error below redirector "+err_below + dump_sane = re.sub('---*','__',dump_below) + c = "Detailed output for file BELOW "+redirector+":"+file_below+" "+err_below+" "+dump_sane + else: + availinfo=availinfo+" File below: OK " + dicci['xrdcp_below_time'] = elapsed_below + else: + availinfo=availinfo+" File below: not tested." + if(file_above): + notes_text = notes_text + "File 'elsewhere': " + file_above + (err_above,dump_above,elapsed_above) = xrdcp_test(redirector, file_above) + if err_above : + availinfo=availinfo+" Error above redirector "+err_above + dump_sane = re.sub('---*','__',dump_above) + c = "Detailed output for file ABOVE "+redirector+":"+file_above+" "+err_above+" "+dump_sane + else: + availinfo = availinfo+" File above: OK " + dicci['xrdcp_above_time'] = elapsed_above + else: + availinfo = availinfo+" File above: not tested." + availinfo = availinfo + " " + notes_text + dicci['status']= str(availability) + if availability == 'unavailable' or availability == 'degraded': + dicci ['availInfo'] = availinfo + dicci ['Comment'] = c + with open(html_dir +'KIBANA_PROBES.json', 'a') as f: + json.dump(dicci, f) + f.write('\n') +def main(): + debug = 0 + atexit.register(clear_lock) + if len(sys.argv) > 1: + if sys.argv[1] == '-d': + debug=1 + if not try_lock(): + sys.exit(1) + if not os.path.exists(html_dir): + os.makedirs(html_dir) + env_setup() + timeout_sec = 10 * 60 # limit overall runtime to 10min + signal.signal(signal.SIGALRM, alarm_handler) + ATLASLINK="%BR%Monitoring:%BR%\n http://atl-prod07.slac.stanford.edu:8080/display?page=xrd_report/aggregated/total_xrootd_lgn %BR%\n http://dashb-atlas-xrootd-transfers.cern.ch/ui %BR%\nhttp://dashb-atlas-ssb.cern.ch/dashboard/request.py/siteview#currentView=FAX+redirectors&highlight=false %BR%\n" + CMSLINK="%BR%Monitoring:%BR%\n http://xrootd.t2.ucsd.edu/dashboard/ %BR%\n http://dashb-cms-xrootd-transfers.cern.ch/ui %BR%\n" + FILEABOVE="/store/mc/SAM/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v1/00000/A64CCCF2-5C76-E711-B359-0CC47A78A3F8.root" + FILEBELOW="/store/mc/SAM/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v1/00000/A64CCCF2-5C76-E711-B359-0CC47A78A3F8.root" + services = { + "XRDFED_CMS-GLOBAL01-NEW":{'redirector':'cms-xrd-global01.cern.ch:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-GLOBAL02-NEW":{'redirector':'cms-xrd-global02.cern.ch:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-US-FNAL":{'redirector':'cmsxrootd2.fnal.gov:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-US-UNL":{'redirector':'xrootd.unl.edu:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU-BARI":{'redirector':'xrootd.ba.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU-LLR":{'redirector':'llrxrd-redir.in2p3.fr:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU-PISA":{'redirector':'xrootd-redic.pi.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-GLOBAL":{'redirector':'cms-xrd-global.cern.ch:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-US":{'redirector':'cmsxrootd.fnal.gov:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU":{'redirector':'xrootd-cms.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU-IPv6":{ 'redirector':'xrootd-cms-redir-01.cr.cnaf.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-TRANSIT":{'redirector':'cms-xrd-transit.cern.ch:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-TRANSIT01":{'redirector':'vocms031.cern.ch:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-TRANSIT02":{'redirector':'vocms032.cern.ch:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + } + signal.alarm(timeout_sec) + os.remove(html_dir+'KIBANA_PROBES.json') + try: + diccionaries = [] + for xrd in services: + services[xrd].update(servicename=xrd) + servicename = xrd + dicci = prepare_dictionary(servicename, services[xrd]['redirector']) + diccionaries.append(dicci) + for dicci in diccionaries: + service = dicci['service'] + argus = services[service] + argus['dicci'] = dicci + if debug: + test_redirector(** services[xrd]) + else: + t = threading.Thread(target=test_redirector, kwargs = argus) # read: "run a thread with the test function and all the parameters above as arguments" + t.start() + except Alarm: + print "ERROR: caught overall timeout after "+str(timeout_sec)+"s\n" + clear_lock() + sys.exit(2) + signal.alarm(0) +if __name__ == '__main__': + main() From 96b034bfaa19938bd51804f36010290b60a19e3f Mon Sep 17 00:00:00 2001 From: FernandoGarzon Date: Thu, 3 Sep 2020 00:57:29 +0200 Subject: [PATCH 6/8] New Probe added --- AAAOps/XfedKibana/XRDFED-kibana-probe_JSON.py | 372 ------------------ 1 file changed, 372 deletions(-) delete mode 100755 AAAOps/XfedKibana/XRDFED-kibana-probe_JSON.py diff --git a/AAAOps/XfedKibana/XRDFED-kibana-probe_JSON.py b/AAAOps/XfedKibana/XRDFED-kibana-probe_JSON.py deleted file mode 100755 index b83f242..0000000 --- a/AAAOps/XfedKibana/XRDFED-kibana-probe_JSON.py +++ /dev/null @@ -1,372 +0,0 @@ -#!/usr/bin/python -# functional probe and SLS extractor for the "federation" xroot services -# highlights: -# - stateless (i.e. run from cron whenever needed) -# - will try to prevent parallel runs via lockfile -# - multithreaded, one thread per service to be tested -# - overall runtime cap at 10min -# - could extract some statistics from xroot directly, but these are ever-increasing counters -# Problems: -# - need to update the code whenever a service is addded/deleted/changed -# - uses "random" files on various Xroot services all over the world, these are (for now) the same as used by the experiments but these might change.. - -import xml.dom.minidom -import subprocess -import os -import sys -import signal -import re -import time -import Lemon.XMLAPI -import socket -import atexit -import threading -import tempfile -import json -import shutil - -html_dir = '/root/ogarzonm/' # will create per-service json files here - -LOCKFILE='/var/lock/subsys/xrdfed-kibana-probe' - -class Alarm(Exception): - pass - -def alarm_handler(signum, frame): - print "ERROR: caught overall timeout after "+str(timeout_sec)+"s\n" - clear_lock() - sys.exit(2) - raise Alarm - -def clear_lock(): - try: - os.unlink(LOCKFILE) - except Exception,e: - print "could not remove lockfile:"+str(e) - -def env_setup(): - os.environ['X509_USER_CERT']='/root/.globus/slsprobe-cert.pem' - os.environ['X509_USER_KEY']='/root/.globus/slsprobe-key.pem' - os.environ['X509_USER_PROXY']='/root/.globus/slsprobe.proxy' - os.environ['KRB5CCNAME']='FILE:/dev/null' - os.environ['PATH']=os.environ['PATH']+":/opt/globus/bin/" - -def get_proxy(): - dev_null = open('/dev/null', 'rw') - (proxyfd,proxy)=tempfile.mkstemp(prefix='x509_xrdfed_',suffix='.pem') - os.close(proxyfd) - os.environ['X509_USER_PROXY']=proxy - ret = subprocess.call(['grid-proxy-init','-pwstdin'],stdin=dev_null,) - if ret > 0: - raise Exception("Cannot get X509 proxy") - dev_null.close() - -def cleanup_proxy(): - try: - os.unlink(os.environ['X509_USER_PROXY']) - except Exception,e: - print "could not remove proxy file:"+str(e) - -def try_lock(): - ret = subprocess.call(['lockfile','-5','-r2',LOCKFILE]) - if ret > 0: - print "could not create lockfile" - return False - return True - -def prepare_dictionary(servicename,redirector): - (errtext,version,out) = xrd_info(redirector) - dic={'service':servicename, 'version': version, 'host': redirector[:redirector.find(':')]} - if(errtext): - dic['status'] = 'unavailable' - dic['availinfo'] = " Error getting info from redirector: "+err_info - dic["xrdcp_below_time"] = 0 - dic["xrdcp_above_time"] = 0 - return dic - -def dnsalias_to_nodes(redirector): - (host,port) = redirector.split(':') - all_hosts = [] - data=socket.getaddrinfo(host,port,0, 0, socket.SOL_TCP ) - for addr in data: - (family, socktype, proto, canonname, sockaddr) = addr - (hostname, aliaslist, ipaddrlist) = socket.gethostbyaddr(sockaddr[0]) - if not hostname in all_hosts: - all_hosts.append(hostname) - return all_hosts - -def xrdcp_test(redirector,file): - (errtext,out,err,elapsed) = run_xrd_commands("xrdcp", - ["-d","1", - "-f", - "-DIReadCacheSize","0", - "-DIRedirCntTimeout","180", - "root://"+redirector+'/'+file, - '/dev/null']) - return (errtext,err,elapsed) - -def xrd_info(redirector): - version = "(unknown)" - (errtext,out,err,elapsed) = run_xrd_commands("xrdfs", - [redirector, - "query","config", # 1:kXR_QStats - "version"]) # a_ll stats - - if not out: - errtext = '' - os.system("xrdfs "+ redirector+" query config version > /root/aux.txt") - os.system("head -n 1 /root/aux.txt > /root/aux2.txt") - f = open('/root/aux2.txt', 'r') - version = f.read() - if not version: - version = "(unknown)" - else: - version = version[:-1] - else: - if not errtext: - try: - dom = xml.dom.minidom.parseString(out) - root_node = dom.documentElement - if root_node.tagName == 'statistics': - v_attr = root_node.getAttributeNode('ver') - version = v_attr.nodeValue - except Exception,e: - errtext = "ERROR: cannot parse answer:"+str(e) - return (errtext,version,out) - -def run_xrd_commands(cmd,args): - dev_null = open('/dev/null', 'r') - errtxt = '' - elapsed = -1.0 - xrd_args = [ 'perl','-e',"alarm 180; exec @ARGV", cmd, # one-line wrapper that *actually* kills the command - "-DIConnectTimeout","30", - "-DITransactionTimeout","60", - "-DIRequestTimeout","60" ] + args - err = '' - out = '' - try: - ran_try = True - start = time.time() - proc = subprocess.Popen(xrd_args, - stdin=dev_null, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - (out, err) = proc.communicate() - - ret = proc.returncode - elapsed = (time.time() - start) - err_redir_index = err.rfind('Received redirection to') - err_index3010 = err.rfind('(error code: 3010') # (permission denied) may be sort-of-OK - we are talking to final storage already - UK - err_index3005 = err.rfind('(error code: 3005') # (no user mapping) - INFN - if err_redir_index >= 0 and (err_index3010 >= 0 or err_index3005 >= 0): - errtxt = '' - else: - if(ret > 0): - errtxt = "client-side error - exit code "+str(ret)+"\n" - err_index = err.rfind('Last server error') - if err_index >= 0: - err_end_index=err.find("\n",err_index) - errtxt = errtxt + err[err_index:err_end_index] - except Exception,e: - errtext = errtxt + "Exception: "+str(e) - dev_null.close() - return (errtxt,out,err,elapsed) - -def test_redirector(dicci, servicename, redirector, file_below=None, file_above=None, extra_notes=""): - servicename=servicename.upper() - notes_text = "Redirector:"+redirector - availability = 'available' - availinfo = '' - c = 'No comment' - - # prepare the dictionary. - #dicci['host'] = redirector.split(':')[0] - - # run the functional tests - first some simple check to get the version, if OK look for files - #(err_info,version,dump_info) = xrd_info(redirector) - #if(err_info): - # - # availability = 'unavailable' - # availinfo=availinfo+" Error getting info from redirector "+err_info - # dicci["xrdcp_below_time"] = 0 - #dicci["status"] = "unavailable" - if 'status' in dicci and dicci['status'] == 'unavailable': - pass - else: - if (file_below): - notes_text = notes_text + "File 'below': " + file_below - (err_below,dump_below,elapsed_below) = xrdcp_test(redirector, file_below) - if err_below: - availability = 'degraded' - availinfo=availinfo+" Error below redirector "+err_below - dump_sane = re.sub('---*','__',dump_below) - c = "Detailed output for file BELOW "+redirector+":"+file_below+" "+err_below+" "+dump_sane - #dicci['comment'] = c - else: - availinfo=availinfo+" File below: OK " - dicci['xrdcp_below_time'] = str(elapsed_below) - else: - availinfo=availinfo+" File below: not tested." - if(file_above): - notes_text = notes_text + "File 'elsewhere': " + file_above - (err_above,dump_above,elapsed_above) = xrdcp_test(redirector, file_above) - if err_above : - #We've changed availability from number to string so this below won't work; Marian commented out on 2015-11-06 - #availability = availability * 0.8 # less important if some remote site is failing.. - availinfo=availinfo+" Error above redirector "+err_above - # sanitize the raw output in order to not trigger XML errors.. in a comment. - dump_sane = re.sub('---*','__',dump_above) - c = "Detailed output for file ABOVE "+redirector+":"+file_above+" "+err_above+" "+dump_sane - #dicci = {**dicci, **{'comment': c}} - #serviceUpdate.appendChild(c) - #need_xml_link=1 - else: - availinfo = availinfo+" File above: OK " - #nValue = doc.createElement("numericvalue") - #nValue.setAttribute("name", "xrdcp_above_time") - #nValue.setAttribute("desc", "Time to copy a file elsewhere in the federation") - #nValue.appendChild(doc.createTextNode(str(elapsed_above))) - dicci['xrdcp_above_time'] = str(elapsed_above) - #data.appendChild(nValue) - else: - availinfo = availinfo+" File above: not tested." - - # save functional test info to XML - #if need_xml_link: - # myhostname = socket.gethostname() - # notes_text = notes_text + "Details for failed test: http://" + myhostname + "/aaa-probe/" + servicename + ".xml
\n" + "Details for recently failed test : http://vocms039.cern.ch/aaa-probe/err/
\n" - availinfo = availinfo + " " + notes_text - dicci['status']= str(availability) - if availability == 'unavailable' or availability == 'degraded': - dicci ['availInfo'] = availinfo - dicci ['Comment'] = c - #return dicci - #dicci['availabilityinfo']=availinfo - with open(html_dir +'KIBANA_PROBES.json', 'a') as f: - json.dump(dicci, f) - f.write('\n') - - -def main(): - debug = 0 - atexit.register(clear_lock) - if len(sys.argv) > 1: - if sys.argv[1] == '-d': - debug=1 - if not try_lock(): - sys.exit(1) - if not os.path.exists(html_dir): - os.makedirs(html_dir) - env_setup() - # get a proxy cert - # get_proxy() - - timeout_sec = 10 * 60 # limit overall runtime to 10min - signal.signal(signal.SIGALRM, alarm_handler) - - ATLASLINK="%BR%Monitoring:%BR%\n http://atl-prod07.slac.stanford.edu:8080/display?page=xrd_report/aggregated/total_xrootd_lgn %BR%\n http://dashb-atlas-xrootd-transfers.cern.ch/ui %BR%\nhttp://dashb-atlas-ssb.cern.ch/dashboard/request.py/siteview#currentView=FAX+redirectors&highlight=false %BR%\n" - CMSLINK="%BR%Monitoring:%BR%\n http://xrootd.t2.ucsd.edu/dashboard/ %BR%\n http://dashb-cms-xrootd-transfers.cern.ch/ui %BR%\n" - FILEABOVE="/store/mc/SAM/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v1/00000/A64CCCF2-5C76-E711-B359-0CC47A78A3F8.root" - FILEBELOW="/store/mc/SAM/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v1/00000/A64CCCF2-5C76-E711-B359-0CC47A78A3F8.root" - - services = { - "XRDFED_CMS-GLOBAL01-NEW":{'redirector':'cms-xrd-global01.cern.ch:1094', - 'file_below': FILEABOVE, - 'file_above': FILEBELOW, - 'extra_notes':CMSLINK}, - - "XRDFED_CMS-GLOBAL02-NEW":{'redirector':'cms-xrd-global02.cern.ch:1094', - 'file_below': FILEABOVE, - 'file_above': FILEBELOW, - 'extra_notes':CMSLINK}, - - "XRDFED_CMS-US-FNAL":{'redirector':'cmsxrootd2.fnal.gov:1094', - 'file_below': FILEABOVE, - 'file_above': FILEBELOW, - 'extra_notes':CMSLINK}, - - "XRDFED_CMS-US-UNL":{'redirector':'xrootd.unl.edu:1094', - 'file_below': FILEABOVE, - 'file_above': FILEBELOW, - 'extra_notes':CMSLINK}, - - "XRDFED_CMS-EU-BARI":{'redirector':'xrootd.ba.infn.it:1094', - 'file_below': FILEBELOW, - 'file_above': FILEABOVE, - 'extra_notes':CMSLINK}, - - "XRDFED_CMS-EU-LLR":{'redirector':'llrxrd-redir.in2p3.fr:1094', - 'file_below': FILEBELOW, - 'file_above': FILEABOVE, - 'extra_notes':CMSLINK}, - - "XRDFED_CMS-EU-PISA":{'redirector':'xrootd-redic.pi.infn.it:1094', - 'file_below': FILEBELOW, - 'file_above': FILEABOVE, - 'extra_notes':CMSLINK}, - - "XRDFED_CMS-GLOBAL":{'redirector':'cms-xrd-global.cern.ch:1094', - 'file_below': FILEABOVE, - 'file_above': FILEBELOW, - 'extra_notes':CMSLINK}, - - "XRDFED_CMS-US":{'redirector':'cmsxrootd.fnal.gov:1094', - 'file_below': FILEABOVE, - 'file_above': FILEBELOW, - 'extra_notes':CMSLINK}, - - "XRDFED_CMS-EU":{'redirector':'xrootd-cms.infn.it:1094', - 'file_below': FILEBELOW, - 'file_above': FILEABOVE, - 'extra_notes':CMSLINK}, - - "XRDFED_CMS-EU-IPv6":{ 'redirector':'xrootd-cms-redir-01.cr.cnaf.infn.it:1094', - 'file_below': FILEBELOW, - 'file_above': FILEABOVE, - 'extra_notes':CMSLINK}, - "XRDFED_CMS-TRANSIT":{'redirector':'cms-xrd-transit.cern.ch:1094', - 'file_below': FILEBELOW, - 'file_above': FILEABOVE, - 'extra_notes':CMSLINK}, - - "XRDFED_CMS-TRANSIT01":{'redirector':'vocms031.cern.ch:1094', - 'file_below': FILEBELOW, - 'file_above': FILEABOVE, - 'extra_notes':CMSLINK}, - - "XRDFED_CMS-TRANSIT02":{'redirector':'vocms032.cern.ch:1094', - 'file_below': FILEBELOW, - 'file_above': FILEABOVE, - 'extra_notes':CMSLINK}, - - } - signal.alarm(timeout_sec) - #shutil.copyfile(html_dir+'KIBANA_PROBES.json', html_dir+'KIBANA_PROBES_2.json') - os.remove(html_dir+'KIBANA_PROBES.json') - #shutil.copyfile(html_dir+'KIBANA_PROBES.json', html_dir+'KIBANA_PROBES_2.json') - try: - diccionaries = [] - for xrd in services: - services[xrd].update(servicename=xrd) - servicename = xrd - dicci = prepare_dictionary(servicename, services[xrd]['redirector']) - diccionaries.append(dicci) - for dicci in diccionaries: - service = dicci['service'] - argus = services[service] - argus['dicci'] = dicci - if debug: - test_redirector(** services[xrd]) - else: - t = threading.Thread(target=test_redirector, kwargs = argus) # read: "run a thread with the test function and all the parameters above as arguments" - t.start() - except Alarm: - print "ERROR: caught overall timeout after "+str(timeout_sec)+"s\n" - clear_lock() - sys.exit(2) - signal.alarm(0) - #shutil.copyfile(html_dir+'KIBANA_PROBES.json', html_dir+'KIBANA_PROBES_2.json') - # not cleaning up the proxy files (are shared via the ENV, and we don't want an extra thread to just remove that file, or wait for the individual tests to finish... - -if __name__ == '__main__': - main() From a8b677768e63bd3cbcc5d557458e848b7c2ff10c Mon Sep 17 00:00:00 2001 From: FernandoGarzon Date: Thu, 3 Sep 2020 01:05:41 +0200 Subject: [PATCH 7/8] Old Kibana probe set back as original --- AAAOps/XfedKibana/XRDFED-kibana-probe.py | 30 +++++++----------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/AAAOps/XfedKibana/XRDFED-kibana-probe.py b/AAAOps/XfedKibana/XRDFED-kibana-probe.py index dbf1e1c..a32fb12 100755 --- a/AAAOps/XfedKibana/XRDFED-kibana-probe.py +++ b/AAAOps/XfedKibana/XRDFED-kibana-probe.py @@ -132,27 +132,15 @@ def xrd_info(redirector): [redirector, "query","config", # 1:kXR_QStats "version"]) # a_ll stats - - if not out: - errtext = '' - os.system("xrdfs "+ redirector+" query config version > /root/aux.txt") - os.system("head -n 1 /root/aux.txt > /root/aux2.txt ") - f = open('/root/aux2.txt', 'r') - version = f.read() - if not version: - version = "(unknown)" - else: - version = version[:-1] - else: - if not errtext: - try: - dom = xml.dom.minidom.parseString(out) - root_node = dom.documentElement - if root_node.tagName == 'statistics': - v_attr = root_node.getAttributeNode('ver') - version = v_attr.nodeValue - except Exception,e: - errtext = "ERROR: cannot parse answer:"+str(e) + if not errtext: + try: + dom = xml.dom.minidom.parseString(out) + root_node = dom.documentElement + if root_node.tagName == 'statistics': + v_attr = root_node.getAttributeNode('ver') + version = v_attr.nodeValue + except Exception,e: + errtext = "ERROR: cannot parse answer:"+str(e) return (errtext,version,out) def run_xrd_commands(cmd,args): From 2f873f1e89a25e52b990f26ae81f50b4011adbd4 Mon Sep 17 00:00:00 2001 From: dmielaikaite Date: Thu, 24 Sep 2020 18:13:11 +0200 Subject: [PATCH 8/8] Kibana probe with latest changes applied --- .../XRDFED-kibana-probe_JSON.py | 68 ++++++++++++------- AAAOps/XfedKibana_JSON/kibana_probe.sh | 2 + AAAOps/XfedKibana_JSON/single_quotes.sh | 2 + 3 files changed, 49 insertions(+), 23 deletions(-) create mode 100644 AAAOps/XfedKibana_JSON/kibana_probe.sh create mode 100644 AAAOps/XfedKibana_JSON/single_quotes.sh diff --git a/AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON.py b/AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON.py index a62744b..095db3f 100755 --- a/AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON.py +++ b/AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON.py @@ -23,7 +23,7 @@ import tempfile import json import shutil -html_dir = '/var/www/html/aaa-probe/' # will create per-service json files here +html_dir = '/root/ogarzonm/' # will create per-service json files here LOCKFILE='/var/lock/subsys/xrdfed-kibana-probe' class Alarm(Exception): pass @@ -65,12 +65,17 @@ def try_lock(): return True def prepare_dictionary(servicename,redirector): (errtext,version,out) = xrd_info(redirector) - dic={'service':servicename, 'version': version, 'host': redirector[:redirector.find(':')]} + dic={'service':servicename, 'host': redirector[:redirector.find(':')]} if(errtext): + dic['version'] = 'unavailable' dic['status'] = 'unavailable' - dic['availinfo'] = " Error getting info from redirector: "+err_info + errtext = errtext.replace("'", "") + errtext = errtext.replace('"', '') + dic['comment'] = "Error getting info from redirector: "+errtext dic["xrdcp_below_time"] = 0 dic["xrdcp_above_time"] = 0 + else: + dic['version'] = version return dic def xrdcp_test(redirector,file): (errtext,out,err,elapsed) = run_xrd_commands("xrdcp", @@ -146,43 +151,55 @@ def run_xrd_commands(cmd,args): return (errtxt,out,err,elapsed) def test_redirector(dicci, servicename, redirector, file_below=None, file_above=None, extra_notes=""): servicename=servicename.upper() - notes_text = "Redirector:"+redirector - availability = 'available' + notes_text = "Redirector: "+redirector + availability = 'Available' availinfo = '' - c = 'No comment' + c = '' if 'status' in dicci and dicci['status'] == 'unavailable': pass + elif file_below == None and file_above == None: + availability = 'Unavialable' + c = 'Non-existing File Above and File Below.' + dicci['xrdcp_below_time'] = 0 + dicci['xrdcp_below_time'] = 0 else: if (file_below): - notes_text = notes_text + "File 'below': " + file_below + notes_text = notes_text + "File below: " + file_below (err_below,dump_below,elapsed_below) = xrdcp_test(redirector, file_below) if err_below: - availability = 'degraded' - availinfo=availinfo+" Error below redirector "+err_below + availability = 'Degraded' + #availinfo=availinfo+" Error below redirector "+err_below dump_sane = re.sub('---*','__',dump_below) - c = "Detailed output for file BELOW "+redirector+":"+file_below+" "+err_below+" "+dump_sane + c = c+"Error for file BELOW: "+err_below+". Dumpsane: "+dump_sane+ '.' + dicci['xrdcp_below_time'] = 0 else: - availinfo=availinfo+" File below: OK " + #availinfo=availinfo+" File below: OK " dicci['xrdcp_below_time'] = elapsed_below else: - availinfo=availinfo+" File below: not tested." + c = "Error for file BELOW: Non-existing File Below. " + dicci['xrdcp_below_time'] = 0 if(file_above): - notes_text = notes_text + "File 'elsewhere': " + file_above + notes_text = notes_text + "File elsewhere: " + file_above (err_above,dump_above,elapsed_above) = xrdcp_test(redirector, file_above) if err_above : - availinfo=availinfo+" Error above redirector "+err_above + availability = 'Degraded' + #availinfo=availinfo+" Error above redirector "+err_above dump_sane = re.sub('---*','__',dump_above) - c = "Detailed output for file ABOVE "+redirector+":"+file_above+" "+err_above+" "+dump_sane + c = c+"Error for file ABOVE: "+err_above+". Dumpsane: "+dump_sane+'.' + dicci['xrdcp_above_time'] = 0 else: - availinfo = availinfo+" File above: OK " - dicci['xrdcp_above_time'] = elapsed_above + #availinfo = availinfo+" File above: OK " + dicci['xrdcp_above_time'] = elapsed_above else: - availinfo = availinfo+" File above: not tested." - availinfo = availinfo + " " + notes_text + c = c + "Error for file ABOVE: Non-existing File Above." + dicci['xrdcp_above_time'] = 0 + #availinfo = availinfo + " " + notes_text dicci['status']= str(availability) - if availability == 'unavailable' or availability == 'degraded': - dicci ['availInfo'] = availinfo - dicci ['Comment'] = c + if c == '': + c = 'N/A' + c = c.replace("\n", "") + c = c.replace("\r", "") + dicci ['Comment'] = c with open(html_dir +'KIBANA_PROBES.json', 'a') as f: json.dump(dicci, f) f.write('\n') @@ -262,7 +279,6 @@ def main(): 'extra_notes':CMSLINK}, } signal.alarm(timeout_sec) - os.remove(html_dir+'KIBANA_PROBES.json') try: diccionaries = [] for xrd in services: @@ -279,10 +295,16 @@ def main(): else: t = threading.Thread(target=test_redirector, kwargs = argus) # read: "run a thread with the test function and all the parameters above as arguments" t.start() + #t.join() + #os.system('source ~/single_quotes.sh') except Alarm: print "ERROR: caught overall timeout after "+str(timeout_sec)+"s\n" clear_lock() sys.exit(2) signal.alarm(0) if __name__ == '__main__': + for file in os.listdir(html_dir): + if file == 'KIBANA_PROBES.json': + os.remove(html_dir+'KIBANA_PROBES.json') + break main() diff --git a/AAAOps/XfedKibana_JSON/kibana_probe.sh b/AAAOps/XfedKibana_JSON/kibana_probe.sh new file mode 100644 index 0000000..3167d3e --- /dev/null +++ b/AAAOps/XfedKibana_JSON/kibana_probe.sh @@ -0,0 +1,2 @@ +python ~/XRDFED-kibana-probe_JSON.py +source ~/single_quotes.sh diff --git a/AAAOps/XfedKibana_JSON/single_quotes.sh b/AAAOps/XfedKibana_JSON/single_quotes.sh new file mode 100644 index 0000000..60b5fbe --- /dev/null +++ b/AAAOps/XfedKibana_JSON/single_quotes.sh @@ -0,0 +1,2 @@ +#!/bin/bash +sed -i "y/\"/'/" ~/ogarzonm/KIBANA_PROBES.json