Merge pull request #2 from EnginEren/master

Adding AAAOps folder and scripts

AAAOps/Federation/README.md

@@ -0,0 +1,30 @@
+# FederationInfo
+
+Cron:
+----
+Cronjob is done via "create-fedfiles.sh" script. From that script, we are calling "create_allow-list.sh" and "create_fedmaps.py" 
+
+ **create_allow-list.sh**:
+
+	-Input : 
+		Redirector names -> xrdcmsglobal01.cern.ch:1094, xrdcmsglobal02.cern.ch:1094  and cms-xrd-transit.cern.ch:1094
+
+	-Purpose : 
+		Query gloabal redirectors above and get the sites and regional redirectors who are subscribed to these global redirectors. 
+
+	-Output : 
+		Allow list of both US and EU regions are produced. 
+		Host names are produced for both production and transational federation. (prod.txt and trans.txt)
+
+  **create_fedmaps_DEV.py** :
+
+	-Input : 
+		1. Hostnames in in/prod.txt and in/trans.txt
+		2. http://dashb-cms-vo-feed.cern.ch/dashboard/request.py/cmssitemapbdii
+	-Purpose :	
+		We want to convert hostnames to "cms sitenames", e.g t2-cms-xrootd01.desy.de ---> T2_DE_DESY. 
+	-Output :
+		JSON file which consists of 3 categories :
+			1. Production Federation
+			2. Transitional Federation
+			3. Nowhere Sites. (Sites which are not subscribed to any redirector.)

AAAOps/Federation/create-fedfiles.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+PROD_hosts=/root/FederationInfo/in/prod.txt
+TRANS_hosts=/root/FederationInfo/in/trans.txt
+FED_json=/root/FederationInfo/out/federations.json
+ALLOW_EU=/root/FederationInfo/out/list_eu.allow
+
+export X509_USER_PROXY=/root/.globus/slsprobe.proxy
+
+./FederationInfo/create_allow-list.sh
+
+if [ ! -r $ALLOW_EU ]; then
+	echo "We have a problem with creating allow list.\n"
+	exit 1
+fi
+
+if [ ! -r $PROD_hosts ] || [ ! -r $TRANS_hosts ]; then
+	echo "We have problems with input files. \n"	
+	echo $CMS_topology "\t" $PROD_hosts "\t" $TRANS_prod
+	exit 1
+fi
+
+python /root/FederationInfo/create_fedmaps_DEV.py
+
+if [ ! -r $FED_json ]; then
+	echo "We have a problem creating JSON file.\n"
+	exit 1
+fi
+cp /root/FederationInfo/out/* /var/www/html/fedinfo/
+
+exit 0;

AAAOps/Federation/create_allow-list.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+BASE=/root
+FEDINFO=/root/FederationInfo
+export XRD_NETWORKSTACK=IPv4
+
+declare -a redirectors=("xrdcmsglobal01.cern.ch:1094" "xrdcmsglobal02.cern.ch:1094" "cms-xrd-transit.cern.ch:1094")
+
+for j in "${redirectors[@]}";do
+	if [ "$j" == "xrdcmsglobal01.cern.ch:1094" ] || [ "$j" == "xrdcmsglobal02.cern.ch:1094" ]; then
+		xrdmapc --list all "$j" | grep -E 'xrootd.ba.infn.it|xrootd-redic.pi.infn.it|llrxrd-redir.in2p3.fr:1094' | awk '{print $3}' | cut -d ':' -f1 > $BASE/tmp_euRED_$j
+		xrdmapc --list all "$j" | grep -E 'cmsxrootd1.fnal.gov|xrootd.unl.edu' | awk '{print $3}' | cut -d ':' -f1 > $BASE/tmp_usRED_$j
+		for i in $(cat $BASE/tmp_euRED_$j);do
+			xrdmapc --list all $i:1094 > $BASE/tmp_$i	
+			cat $BASE/tmp_$i | awk '{if($2=="Man") print $3; else print $2}' | tail -n +2 >> $BASE/tmp_total_eu_$j
+		done
+
+		for k in $(cat $BASE/tmp_usRED_$j);do
+			xrdmapc --list all $k:1094 > $BASE/tmp_us_$k	
+			cat $BASE/tmp_us_$k | awk '{if($2=="Man") print $3; else print $2}' | tail -n +2 >> $BASE/tmp_total_us_$j
+		done
+
+
+		cat $BASE/tmp_total_eu_$j | cut -d : -f1 | sort -u > $FEDINFO/in/prod_$j.txt 
+		cat $BASE/tmp_total_us_$j | cut -d : -f1 | sort -u >> $FEDINFO/in/prod_$j.txt 
+		cat $BASE/tmp_total_eu_$j | cut -d : -f1 | sort -u | awk -F. '{print "cms.allow host " "*."$(NF-1)"."$NF}' | sort -u > $FEDINFO/out/list_eu_$j.allow
+		cat $BASE/tmp_total_us_$j | cut -d : -f1 | sort -u | awk -F. '{print "cms.allow host " "*."$(NF-1)"."$NF}' | sort -u > $FEDINFO/out/list_us_$j.allow
+
+	else
+		xrdmapc --list all "$j" | tail -n +2 | awk '{if($2=="Man") print $3; else print $2}' > $BASE/tmp_total
+		cat $BASE/tmp_total | cut -d : -f1 | sort -u > $FEDINFO/in/trans.txt
+	fi	
+
+
+	rm $BASE/tmp_*
+
+done
+
+diff $FEDINFO/in/prod_xrdcmsglobal01.cern.ch\:1094.txt $FEDINFO/in/prod_xrdcmsglobal02.cern.ch\:1094.txt 
+stat=$(echo $?)
+if [ $stat == 1 ]; then
+	cat $FEDINFO/in/prod_xrdcmsglobal01.cern.ch\:1094.txt $FEDINFO/in/prod_xrdcmsglobal02.cern.ch\:1094.txt | sort -u > $FEDINFO/in/prod.txt	
+else
+	cp $FEDINFO/in/prod_xrdcmsglobal02.cern.ch\:1094.txt $FEDINFO/in/prod.txt
+fi	
+
+
+
+#cat $FEDINFO/in/prod.txt | cut -d : -f1 | sort -u | awk -F. '{if ($NF == "uk" || $NF == "fr" || $NF == "it" || $(NF-1) == "cern" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-1) == "vanderbilt" ) print $(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-1) == "mit" ) print $(NF-2)"."$(NF-1)"."$NF;  else print $(NF-1)"."$NF}' | sort -u > $FEDINFO/in/prod_domain.txt
+
+cat $FEDINFO/in/prod.txt |  sort -u | awk -F. '{if ($NF == "uk" && $(NF-2) != "rl" || $NF == "fr" || $(NF-1) == "cern" || $(NF-1) == "fnal" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-2) == "cnaf") print $(NF-4)"."$(NF-2); else if ( $NF == "it" && $(NF-2) != "cnaf" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-1) == "vanderbilt" ) print $(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-1) == "mit" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-2) == "rl" ) print $(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else if ( $NF == "kr") print $(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else if ( $NF == "be" ) print $(NF-2)"."$(NF-1)"."$NF; else print $(NF-1)"."$NF}' | sort -u > $FEDINFO/in/prod_domain.txt
+
+cat $FEDINFO/in/trans.txt | cut -d : -f1 | sort -u | awk -F. '{if ($NF == "uk" || $NF == "fr" || $NF == "kr" ) print $(NF-2)"."$(NF-1)"."$NF; else if ($NF == "it" && $(NF-2) == "cnaf" ) print $(NF-4)"."$(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-2) == "ts" ) print $(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else print $(NF-1)"."$NF}' | sort -u > $FEDINFO/in/trans_domain.txt
+
+
+exit 0;

AAAOps/Federation/create_fedmaps_DEV.py

@@ -0,0 +1,105 @@
+try: import xml.etree.ElementTree as ET
+except ImportError: from elementtree import ElementTree as ET
+try: import json
+except ImportError: import simplejson as json
+import urllib2, httplib, sys
+
+# global vars: prod, trans : we create these text files, see run.py
+#              cmsTopology : static text from dashboard team,
+#              sites       : from BDII
+#              output      : {"prod" : [...], "trans" : [...], "nowhere" : [...]}
+
+output = {"prod" : [], "trans" : [], "nowhere" : []}
+
+def getDataFromURL(url, header = {}):
+    request = urllib2.Request(url, headers=header)
+    urlObj  = urllib2.urlopen(request)
+    data    = urlObj.read()
+    return data
+
+def getSites():
+    XML   = getDataFromURL('http://dashb-cms-vo-feed.cern.ch/dashboard/request.py/cmssitemapbdii')
+    XML   = ET.fromstring(XML)
+    sites = XML.findall('atp_site')
+    ret   = {}
+    for site in sites:
+        groups   = site.findall('group')
+        siteName = None
+        for i in groups:
+            if i.attrib['type'] == 'CMS_Site':
+                siteName = groups[1].attrib['name']
+                break
+        if not siteName: 
+            continue
+        services = site.findall('service')
+        ret[siteName] = {}
+        ret[siteName]['hosts'] = []
+        ret[siteName]['name']  = site.attrib['name']
+        for service in services:
+            serviceName = service.attrib['hostname']
+            ret[siteName]['hosts'].append(serviceName)
+    return ret
+
+def parseHN(data):
+    parsedHNs = []
+    for line in data.split('\n'):
+        if not len(line): continue
+        if ':' in line: line = line[:line.find(':')]
+        parsedHNs.append(line)
+    return parsedHNs
+
+# try exception if we have a problem with URL.
+
+try :
+    sites = getSites()
+except Exception as e :
+    err={}
+    err["error"] = str(e)
+    print json.dumps(err)
+    sys.exit(1)
+
+#import expection dictionary
+with open('/root/FederationInfo/exceptions.json') as f: exc = f.read()
+exc = json.loads(exc)
+
+def exception(name):
+    ret = None
+    for i in exc.keys():
+	if i == name : return exc[i]['VOname']
+    return ret
+
+
+def siteName2CMSSiteName(dom):
+    ret = None
+    for cmsSite in sites.keys():
+        ret = exception(dom)
+	if ret : 
+		ret = str(ret)
+		return ret  
+	if sites[cmsSite]['hosts'][0].find(dom) != -1: return cmsSite
+    return ret
+
+if __name__ == "__main__":
+    # get domains
+    domains = {}
+    with open('/root/FederationInfo/in/prod_domain.txt') as f:  domains['prod']  = parseHN(f.read())
+    with open('/root/FederationInfo/in/trans_domain.txt') as f: domains['trans'] = parseHN(f.read())
+
+    # find CMS site name of prod sites
+    for federation in ['prod', 'trans']:
+        for i in domains[federation]:
+	    cmsSiteName = siteName2CMSSiteName(i)
+            #print cmsSiteName
+	    if cmsSiteName and not cmsSiteName in output[federation]:
+                output[federation].append(cmsSiteName)
+
+    # special case for nowhere sites: if a site is not placed in both
+    # federations, move it into "nowhere" array
+    for cmsSite in sites.keys():
+        if not cmsSite in output['prod'] and not cmsSite in output['trans']:
+            output["nowhere"].append(cmsSite)
+	if cmsSite == "T1_FR_CCIN2P3" and cmsSite in output['prod'] :
+	    output["prod"].append("T2_FR_CCIN2P3")
+
+    with open('/root/FederationInfo/out/federations.json', 'w') as f:
+        f.write(json.dumps(output, indent = 1))

AAAOps/Federation/exceptions.json

@@ -0,0 +1,70 @@
+{
+	"ccxrpli001.in2p3.fr" : {
+		"VOname" : "T1_FR_CCIN2P3" 
+	},
+
+	"llrpp01.in2p3.fr" : {
+		"VOname" : "T2_FR_GRIF_LLR"
+	},
+
+	"sbgse1.in2p3.fr" : {
+		"VOname" : "T2_FR_IPHC"
+	},
+
+	"polgrfs20.in2p3.fr " : {
+		"VOname" : "T2_FR_GRIF_LLR"
+	},
+
+	"unl.edu" : {
+		"VOname" : "T2_US_Nebraska"
+	},
+
+	"ds-05-01.cr.cnaf.infn.it" : {
+		"VOname" : "T3_IT_Bologna"
+	},
+
+	"ds-05-02.cr.cnaf.infn.it" : {
+		"VOname" : "T3_IT_Bologna"
+	},
+
+	"ds-202-06-33.cnaf" : {
+		"VOname" : "T1_IT_CNAF"
+	},
+
+	"ds-202-06-37.cnaf" : {
+		"VOname" : "T1_IT_CNAF"
+	},
+
+	"gridpp.rl.ac.uk" : {
+		"VOname" : "T1_UK_RAL"
+	},
+
+	"pp.rl.ac.uk" : {
+		"VOname" : "T2_UK_SGrid_RALPP"
+	},
+
+	"xrootd.knu.ac.kr" : {
+		"VOname" : "T2_KR_KNU"
+	},
+
+	"ucl.ac.be" : {
+		"VOname" : "T2_BE_UCL"
+	},
+
+	"iihe.ac.be" : {
+		"VOname" : "T2_BE_IIHE"
+	},
+
+	"cmsxrootd-site1.fnal.gov" : {
+		"VOname" : "T1_US_FNAL"
+	},
+
+	"cmsxrootd-site2.fnal.gov" : {
+		"VOname" : "T1_US_FNAL"
+	},
+
+	"cmsxrd.ts.infn.it" : {
+		"VOname" : "T3_IT_Trieste"
+	}
+
+}