diff --git a/HLTrigger/Configuration/common/utils.sh b/HLTrigger/Configuration/common/utils.sh index 61bc597daf205..080cb369d28fb 100644 --- a/HLTrigger/Configuration/common/utils.sh +++ b/HLTrigger/Configuration/common/utils.sh @@ -8,7 +8,7 @@ function parse_HLT_schema() { fi # default values - Vx="v2" + Vx="v3" DB="$1" # parse the connection string [version/]database @@ -28,8 +28,8 @@ function parse_HLT_menu() { fi # default values - Vx="v2" - DB="offline" + Vx="v3" + DB="run3" MENU="$1" # parse the connection string [[version/]database:]menu diff --git a/HLTrigger/Configuration/python/Tools/confdb.py b/HLTrigger/Configuration/python/Tools/confdb.py index c7765655c9a22..2f634d1ff5c9d 100644 --- a/HLTrigger/Configuration/python/Tools/confdb.py +++ b/HLTrigger/Configuration/python/Tools/confdb.py @@ -50,7 +50,7 @@ def __init__(self, configuration): # get the configuration from ConfdB from .confdbOfflineConverter import OfflineConverter - self.converter = OfflineConverter(version = self.config.menu.version, database = self.config.menu.database) + self.converter = OfflineConverter(version = self.config.menu.version, database = self.config.menu.database, proxy = self.config.proxy, proxyHost = self.config.proxy_host, proxyPort = self.config.proxy_port) self.buildPathList() self.buildOptions() self.getSetupConfigurationFromDB() @@ -91,7 +91,6 @@ def getRawConfigurationFromDB(self): for key, vals in self.options.items(): if vals: args.extend(('--'+key, ','.join(vals))) - data, err = self.converter.query( *args ) if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err: sys.stderr.write("%s: error while retrieving the HLT menu\n\n" % os.path.basename(sys.argv[0])) diff --git a/HLTrigger/Configuration/python/Tools/confdbOfflineConverter.py b/HLTrigger/Configuration/python/Tools/confdbOfflineConverter.py index 7c94d524b36b5..e6271d7e612c2 100644 --- a/HLTrigger/Configuration/python/Tools/confdbOfflineConverter.py +++ b/HLTrigger/Configuration/python/Tools/confdbOfflineConverter.py @@ -1,8 +1,10 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 import sys, os +import re +import hashlib import os.path import tempfile -import urllib.request +import requests import shutil import subprocess import atexit @@ -39,6 +41,14 @@ class OfflineConverter: databases['v3']['dev'] = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch', '-d', 'cms_hlt.cern.ch', '-u', 'cms_hlt_gdrdev_r', '-s', 'convertMe1!' ) databases['v3']['online'] = ( '-t', 'oracle', '-h', 'cmsonr1-s.cms', '-d', 'cms_rcms.cern.ch', '-u', 'cms_hlt_gdr_r', '-s', 'convertMe!' ) databases['v3']['adg'] = ( '-t', 'oracle', '-h', 'cmsonr1-adg1-s.cern.ch', '-d', 'cms_orcon_adg.cern.ch', '-u', 'cms_hlt_gdr_r', '-s', 'convertMe!' ) + + #ip addresses, there is a bug where we cant do dns over the socks server, sigh + ips_for_proxy = { + 'cmsr1-s.cern.ch' : '10.116.96.89', + 'cmsr2-s.cern.ch' : '10.116.96.139', + 'cmsr3-s.cern.ch' : '10.116.96.105' + } + databases['v3-beta'] = dict(databases['v3']) databases['v3-test'] = dict(databases['v3']) databases['v2'] = dict(databases['v3']) @@ -59,7 +69,8 @@ def CheckTempDirectory(dir): return dir - def __init__(self, version = 'v3', database = 'run3', url = None, verbose = False): + def __init__(self, version = 'v3', database = 'run3', url = None, verbose = False, + proxy = False, proxyHost = 'localhost', proxyPort = '8080'): self.verbose = verbose self.version = version self.baseDir = '/afs/cern.ch/user/c/confdb/www/%s/lib' % version @@ -69,6 +80,9 @@ def __init__(self, version = 'v3', database = 'run3', url = None, verbose = Fals #legacy driver for run2 gui self.jars = ( 'ojdbc6.jar', 'cmssw-evf-confdb-converter.jar' ) self.workDir = '' + self.proxy = proxy + self.proxyHost = proxyHost + self.proxyPort = proxyPort # check the schema version if version not in self.databases: @@ -84,6 +98,17 @@ def __init__(self, version = 'v3', database = 'run3', url = None, verbose = Fals sys.stderr.write( "ERROR: unknown database \"%s\" for version \"%s\"\n" % (database, version)) sys.exit(1) + if self.proxy: + self.proxy_connect_args = ('--dbproxy', '--dbproxyport', self.proxyPort, '--dbproxyhost', self.proxyHost) + temp_connect = [] + for entry in self.connect: + for key,item in self.ips_for_proxy.items(): + entry = entry.replace(key,item) + temp_connect.append(entry.replace(key,item)) + self.connect = tuple(temp_connect) + else: + self.proxy_connect_args = () + # check for a custom base URL if url is not None: self.baseUrl = url @@ -94,27 +119,38 @@ def __init__(self, version = 'v3', database = 'run3', url = None, verbose = Fals self.workDir = self.baseDir else: # try to use $CMSSW_BASE/tmp - self.workDir = OfflineConverter.CheckTempDirectory(os.environ['CMSSW_BASE'] + '/tmp/confdb') + self.workDir = OfflineConverter.CheckTempDirectory(os.path.join(os.environ['CMSSW_BASE'],'tmp','confdb',self.version)) if not self.workDir: # try to use $TMP - self.workDir = OfflineConverter.CheckTempDirectory(os.environ['TMP'] + '/confdb') + self.workDir = OfflineConverter.CheckTempDirectory(os.path.join(os.environ['TMP'],'confdb',self.version)) if not self.workDir: # create a new temporary directory, and install a cleanup callback self.workDir = tempfile.mkdtemp() atexit.register(shutil.rmtree, self.workDir) # download the .jar files - for jar in self.jars: - # check if the file is already present - if os.path.exists(self.workDir + '/' + jar): - continue - # download to a temporay name and use an atomic rename (in case an other istance is downloading the same file - handle, temp = tempfile.mkstemp(dir = self.workDir, prefix = jar + '.') - os.close(handle) - urllib.request.urlretrieve(self.baseUrl + '/' + jar, temp) - if not os.path.exists(self.workDir + '/' + jar): + version_website = requests.get(self.baseUrl+"/../confdb.version").text + jars_require_update = True + if os.path.exists(os.path.join(self.workDir,"confdb.version")): + with open(os.path.join(self.workDir,"confdb.version")) as f: + version_existing = f.read() + if version_existing==version_website: + jars_require_update = False + + if jars_require_update: + for jar in self.jars: + # download to a temporay name and use an atomic rename (in case an other istance is downloading the same file + handle, temp = tempfile.mkstemp(dir = self.workDir, prefix = jar + '.') + os.close(handle) + request = requests.get(self.baseUrl + '/' + jar) + with open(temp,'wb') as f: + f.write(request.content) os.rename(temp, self.workDir + '/' + jar) - else: - os.unlink(temp) + #jars updated, write their version + handle, temp = tempfile.mkstemp(dir = self.workDir, prefix = "confdb.version" + '.') + os.close(handle) + with open(temp,'w') as f: + f.write(version_website) + os.rename(temp,os.path.join(self.workDir,"confdb.version")) # setup the java command line and CLASSPATH if self.verbose: @@ -129,7 +165,7 @@ def __init__(self, version = 'v3', database = 'run3', url = None, verbose = Fals def query(self, *args): - args = self.javaCmd + self.connect + args + args = self.javaCmd + self.connect + self.proxy_connect_args + args if self.verbose: sys.stderr.write("\n" + ' '.join(args) + "\n\n" ) sub = subprocess.Popen( @@ -239,6 +275,20 @@ def main(): version = 'v3-test' db = 'dev' args.remove('--v3-test') + + proxy=False + proxy_host = "localhost" + proxy_port = "8080" + if '--dbproxy' in args: + proxy = True + args.remove('--dbproxy') + if '--dbproxyhost' in args: + proxy_host = args.pop(args.index('--dbproxyhost')+1) + args.remove('--dbproxyhost') + if '--dbproxyport' in args: + proxy_port = args.pop(args.index('--dbproxyport')+1) + args.remove('--dbproxyport') + _dbs = {} _dbs['v1'] = [ '--%s' % _db for _db in OfflineConverter.databases['v1'] ] + [ '--runNumber' ] @@ -264,7 +314,8 @@ def main(): sys.stderr.write( "ERROR: database version \"%s\" incompatible with specification \"%s\"\n" % (version, db) ) sys.exit(1) - converter = OfflineConverter(version = version, database = db, verbose = verbose) + converter = OfflineConverter(version = version, database = db, verbose = verbose, + proxy = proxy, proxyHost = proxy_host, proxyPort=proxy_port) out, err = converter.query( * args ) if 'ERROR' in err: sys.stderr.write( "%s: error while retriving the HLT menu\n\n%s\n\n" % (sys.argv[0], err) ) diff --git a/HLTrigger/Configuration/python/Tools/options.py b/HLTrigger/Configuration/python/Tools/options.py index 1cee33b78c390..beb65e13dab58 100644 --- a/HLTrigger/Configuration/python/Tools/options.py +++ b/HLTrigger/Configuration/python/Tools/options.py @@ -128,7 +128,9 @@ def __init__(self): self.fragment = False # prepare a configuration fragment (true) or a whole process (false) self.hilton = False # prepare a configuration for running with hilton-like modules self.setup = None # if set, downlad the setup_cff from the specified configuration and load it. - + self.proxy = False # use a socks proxy to connect + self.proxy_host = 'localhost' # host of the proxy server + self.proxy_port = '8080' # port of the proxy server # convert HLT and L1 menus to a dedicated object representation on the fly def __setattr__(self, name, value): diff --git a/HLTrigger/Configuration/scripts/hltGetConfiguration b/HLTrigger/Configuration/scripts/hltGetConfiguration index 1ff22aad45673..990fb7d3b8b3b 100755 --- a/HLTrigger/Configuration/scripts/hltGetConfiguration +++ b/HLTrigger/Configuration/scripts/hltGetConfiguration @@ -76,6 +76,24 @@ parser.add_argument('--l1-emulator', const = 'Full', help = 'Run the Full stage-2 L1T emulator.' ) +parser.add_argument('--dbproxy', + dest = 'proxy', + action = 'store_const', + const = 'True', + help = 'Use a socks proxy to connect outside CERN network (default: False)' ) +parser.add_argument('--dbproxyport', + dest = 'proxy_port', + action = 'store', + metavar = 'PROXYPORT', + default = defaults.proxy_port, + help = 'Port of the socks proxy (default: 8080)' ) +parser.add_argument('--dbproxyhost', + dest = 'proxy_host', + action = 'store', + metavar = 'PROXYHOST', + default = defaults.proxy_host, + help = 'Host of the socks proxy (default: "localhost")' ) + group = parser.add_mutually_exclusive_group() group.add_argument('--prescale', dest = 'prescale', diff --git a/HLTrigger/Configuration/scripts/hltIntegrationTests b/HLTrigger/Configuration/scripts/hltIntegrationTests index 7beb4d4382f37..e2a8357f7642e 100755 --- a/HLTrigger/Configuration/scripts/hltIntegrationTests +++ b/HLTrigger/Configuration/scripts/hltIntegrationTests @@ -44,6 +44,9 @@ Usage: --threads THREADS Run with THREADS threads when running the whole HLT (default 4) -x | --extra OPTIONS Pass OPTIONS directly to hltGetConfiguration --mc Run over MC instead of data (the default) + --dbproxy Use a socks proxy to connect to the HLT database + --dbproxyhost PROXYHOST Host of the socks proxy (default: "localhost") + --dbproxyport PROXYPORT Port of the socks proxy (default: 8080) -h | --help Print this help message and exit. @@ -84,7 +87,7 @@ Exmples: " # parse command line argument and options -OPTS=$(getopt -n "$NAME" -o "s:d:i:j:n:k:e:x:h" -l "setup:,dir:,input:,jobs:,size:,skip:,streams:,threads:,events:,mc,extra:,help" -- "$@") +OPTS=$(getopt -n "$NAME" -o "s:d:i:j:n:k:e:x:h" -l "setup:,dir:,input:,jobs:,size:,skip:,streams:,threads:,events:,mc,extra:,help,dbproxy,dbproxyhost:,dbproxyport:" -- "$@") # invalid options if [ $? != 0 ]; then @@ -109,6 +112,8 @@ STREAMS=0 WORKDIR="hltintegration" EXTRA="" DATA="--data" +HLTLISTPATHPROXY="" + SELECTION="" @@ -194,6 +199,21 @@ while true; do DATA="--mc" shift 1 ;; + "--dbproxy" ) + HLTLISTPATHPROXY="$HLTLISTPATHPROXY --dbproxy" + EXTRA="$EXTRA --dbproxy" + shift 1 + ;; + "--dbproxyhost" ) + PROXYHOST="$HLTLISTPATHPROXY --dbproxyhost $2" + EXTRA="$EXTRA --dbproxyhost $2" + shift 2 + ;; + "--dbproxyport" ) + PROXYPORT="$HLTLISTPATHPROXY --dbproxyport $2" + EXTRA="$EXTRA --dbproxyport $2" + shift 2 + ;; "--" ) # inserted by getopt to singal the end of options shift @@ -222,7 +242,7 @@ mkdir "$WORKDIR" cd "$WORKDIR" # find the list of all trigger paths -TRIGGERS=$(hltListPaths -p "$MENU" | grep -v HLTriggerFinalPath) +TRIGGERS=$(hltListPaths -p "$MENU" $HLTLISTPATHPROXY | grep -v HLTriggerFinalPath) # print some info if [ "$SELECTION" == "complex" ]; then @@ -309,8 +329,12 @@ done # if a separate setup is requested, create the setup_cff.py file and patch all dumps to use it if [ "$SETUP" ]; then log "Extracting setup_cff dump" - read SETUP_Vx SETUP_DB SETUP <<< $(parse_HLT_menu "$SETUP") - hltConfigFromDB --$SETUP_Vx --$SETUP_DB --cff --configName "$SETUP" --nopaths --services -FUShmDQMOutputService,-PrescaleService,-EvFDaqDirector,-FastMonitoringService > setup_cff.py + #we use $MENU not $SETUP here as we force the same DB / converter as the main menu + #this is the hltGetConfiguration behaviour and would be confusing if you had to + #specify converter/db on the setup menu on hltIntegration tests but not on hltGetConfiguration + read SETUP_Vx SETUP_DB SETUP <<< $(parse_HLT_menu "$MENU") + + hltConfigFromDB --$SETUP_Vx --$SETUP_DB $HLTLISTPATHPROXY --cff --configName "$SETUP" --nopaths --services -FUShmDQMOutputService,-PrescaleService,-EvFDaqDirector,-FastMonitoringService > setup_cff.py sed -i -e's/process = cms.Process(.*)/&\nprocess.load("setup_cff")/' hlt.py $(for TRIGGER in $TRIGGERS; do echo "$TRIGGER".py; done) fi diff --git a/HLTrigger/Configuration/scripts/hltListPaths b/HLTrigger/Configuration/scripts/hltListPaths index 50fe8acbe0f63..c783d8d361809 100755 --- a/HLTrigger/Configuration/scripts/hltListPaths +++ b/HLTrigger/Configuration/scripts/hltListPaths @@ -16,17 +16,19 @@ def _build_query(menu): if menu.run: return '--runNumber %s' % menu.run else: - return '--%s --configName %s' % (menu.database, menu.name) + return '--%s --%s --configName %s' % (menu.database, menu.version, menu.name) -def getPathList(menu, selection): - cmdline = 'hltConfigFromDB --cff %s --noedsources --noes --noservices --nosequences --nomodules' % _build_query(menu) +def getPathList(config): + cmdline = 'hltConfigFromDB --cff %s --noedsources --noes --noservices --nosequences --nomodules' % _build_query(config.menu) + if config.proxy: + cmdline += f" --dbproxy --dbproxyhost {config.proxy_host} --dbproxyport {config.proxy_port}" data = pipe.pipe(cmdline).decode() if 'Exhausted Resultset' in data or 'CONFIG_NOT_FOUND' in data: raise ImportError('%s is not a valid HLT menu' % menu.value) - if selection == 'paths': + if config.selection == 'paths': filter = re.compile(r' *= *cms.Path.*') - elif selection == 'endpaths': + elif config.selection == 'endpaths': filter = re.compile(r' *= *cms.EndPath.*') else: # 'all' filter = re.compile(r' *= *cms.(End)?Path.*') @@ -43,6 +45,9 @@ except: pass formatter = FixedWidthFormatter( HelpFormatterRespectNewlines, width = textwidth ) +# read defaults +defaults = options.HLTProcessOptions() + parser = argparse.ArgumentParser( description = 'List all the Paths and EndPaths from an HLT configuration in the ConfDB database.''', argument_default = argparse.SUPPRESS, @@ -55,6 +60,24 @@ parser.add_argument('menu', type = options.ConnectionHLTMenu, metavar = 'MENU', help = 'HLT menu to dump from the database. Supported formats are:\n - /path/to/configuration[/Vn]\n - [[{v1|v2|v3}/]{run3|run2|online|adg}:]/path/to/configuration[/Vn]\n - run:runnumber\nThe possible converters are "v1", "v2, and "v3" (default).\nThe possible databases are "run3" (default, used for offline development), "run2" (used for accessing run2 offline development menus), "online" (used to extract online menus within Point 5) and "adg" (used to extract the online menus outside Point 5).\nIf no menu version is specified, the latest one is automatically used.\nIf "run:" is used instead, the HLT menu used for the given run number is looked up and used.\nNote other converters and databases exist as options but they are only for expert/special use.' ) +parser.add_argument('--dbproxy', + dest = 'proxy', + action = 'store_const', + const = 'True', + help = 'Use a socks proxy to connect outside CERN network (default: False)' ) +parser.add_argument('--dbproxyport', + dest = 'proxy_port', + action = 'store', + metavar = 'PROXYPORT', + default = defaults.proxy_port, + help = 'Port of the socks proxy (default: 8080)' ) +parser.add_argument('--dbproxyhost', + dest = 'proxy_host', + action = 'store', + metavar = 'PROXYHOST', + default = defaults.proxy_host, + help = 'Host of the socks proxy (default: "localhost")' ) + # options group = parser.add_mutually_exclusive_group() @@ -82,7 +105,7 @@ parser.add_argument('-h', '--help', # parse command line arguments and options config = parser.parse_args() -paths = getPathList(config.menu, config.selection) +paths = getPathList(config) for path in paths: print(path)