Adding Proxy Support to hltGetConfiguration and fixing converter/db specification bugs with hltIntegrationTests

HLTrigger/Configuration/common/utils.sh

@@ -8,7 +8,7 @@ function parse_HLT_schema() {
   fi
 
   # default values
-  Vx="v2"
+  Vx="v3"
   DB="$1"
 
   # parse the connection string [version/]database
@@ -28,8 +28,8 @@ function parse_HLT_menu() {
   fi
 
   # default values
-  Vx="v2"
-  DB="offline"
+  Vx="v3"
+  DB="run3"
   MENU="$1"
 
   # parse the connection string [[version/]database:]menu

HLTrigger/Configuration/python/Tools/confdb.py

@@ -50,7 +50,7 @@ def __init__(self, configuration):
 
     # get the configuration from ConfdB
     from .confdbOfflineConverter import OfflineConverter
-    self.converter = OfflineConverter(version = self.config.menu.version, database = self.config.menu.database)
+    self.converter = OfflineConverter(version = self.config.menu.version, database = self.config.menu.database, proxy = self.config.proxy, proxyHost = self.config.proxy_host, proxyPort = self.config.proxy_port)
     self.buildPathList()
     self.buildOptions()
     self.getSetupConfigurationFromDB()
@@ -91,7 +91,6 @@ def getRawConfigurationFromDB(self):
     for key, vals in self.options.items():
       if vals:
         args.extend(('--'+key, ','.join(vals)))
-
     data, err = self.converter.query( *args )
     if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
         sys.stderr.write("%s: error while retrieving the HLT menu\n\n" % os.path.basename(sys.argv[0]))

HLTrigger/Configuration/python/Tools/confdbOfflineConverter.py

@@ -1,8 +1,10 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 import sys, os
+import re
+import hashlib
 import os.path
 import tempfile
-import urllib.request
+import requests
 import shutil
 import subprocess
 import atexit
@@ -39,6 +41,14 @@ class OfflineConverter:
     databases['v3']['dev'] = ( '-t', 'oracle', '-h', 'cmsr1-s.cern.ch,cmsr2-s.cern.ch,cmsr3-s.cern.ch',        '-d', 'cms_hlt.cern.ch',      '-u', 'cms_hlt_gdrdev_r',     '-s', 'convertMe1!' )
     databases['v3']['online']  = ( '-t', 'oracle', '-h', 'cmsonr1-s.cms',          '-d', 'cms_rcms.cern.ch',      '-u', 'cms_hlt_gdr_r',     '-s', 'convertMe!' )
     databases['v3']['adg']     = ( '-t', 'oracle', '-h', 'cmsonr1-adg1-s.cern.ch', '-d', 'cms_orcon_adg.cern.ch', '-u', 'cms_hlt_gdr_r',     '-s', 'convertMe!' )
+
+    #ip addresses, there is a bug where we cant do dns over the socks server, sigh
+    ips_for_proxy = {
+        'cmsr1-s.cern.ch' : '10.116.96.89',
+        'cmsr2-s.cern.ch' : '10.116.96.139',
+        'cmsr3-s.cern.ch' : '10.116.96.105'
+    }
+
     databases['v3-beta'] = dict(databases['v3'])
     databases['v3-test'] = dict(databases['v3'])
     databases['v2'] = dict(databases['v3'])
@@ -59,7 +69,8 @@ def CheckTempDirectory(dir):
         return dir
 
 
-    def __init__(self, version = 'v3', database = 'run3', url = None, verbose = False):
+    def __init__(self, version = 'v3', database = 'run3', url = None, verbose = False,
+                 proxy = False, proxyHost = 'localhost', proxyPort = '8080'): 
         self.verbose = verbose
         self.version = version
         self.baseDir = '/afs/cern.ch/user/c/confdb/www/%s/lib' % version
@@ -69,6 +80,9 @@ def __init__(self, version = 'v3', database = 'run3', url = None, verbose = Fals
             #legacy driver for run2 gui
             self.jars = ( 'ojdbc6.jar', 'cmssw-evf-confdb-converter.jar' )
         self.workDir = ''
+        self.proxy = proxy
+        self.proxyHost = proxyHost
+        self.proxyPort = proxyPort
 
         # check the schema version
         if version not in self.databases:
@@ -84,6 +98,17 @@ def __init__(self, version = 'v3', database = 'run3', url = None, verbose = Fals
             sys.stderr.write( "ERROR: unknown database \"%s\" for version \"%s\"\n" % (database, version))
             sys.exit(1)
 
+        if self.proxy:
+            self.proxy_connect_args = ('--dbproxy', '--dbproxyport', self.proxyPort, '--dbproxyhost', self.proxyHost)
+            temp_connect = []
+            for entry in self.connect:
+                for key,item in self.ips_for_proxy.items():
+                    entry = entry.replace(key,item)
+                temp_connect.append(entry.replace(key,item))
+            self.connect  = tuple(temp_connect)
+        else:
+            self.proxy_connect_args = ()
+
         # check for a custom base URL
         if url is not None:
             self.baseUrl = url
@@ -94,27 +119,38 @@ def __init__(self, version = 'v3', database = 'run3', url = None, verbose = Fals
             self.workDir = self.baseDir
         else:
             # try to use $CMSSW_BASE/tmp
-            self.workDir = OfflineConverter.CheckTempDirectory(os.environ['CMSSW_BASE'] + '/tmp/confdb')
+            self.workDir = OfflineConverter.CheckTempDirectory(os.path.join(os.environ['CMSSW_BASE'],'tmp','confdb',self.version))
             if not self.workDir:
                 # try to use $TMP
-                self.workDir = OfflineConverter.CheckTempDirectory(os.environ['TMP'] + '/confdb')
+                self.workDir = OfflineConverter.CheckTempDirectory(os.path.join(os.environ['TMP'] + 'confdb',self.version))
             if not self.workDir:
                 # create a new temporary directory, and install a cleanup callback
                 self.workDir = tempfile.mkdtemp()
                 atexit.register(shutil.rmtree, self.workDir)
             # download the .jar files
-            for jar in self.jars:
-                # check if the file is already present
-                if os.path.exists(self.workDir + '/' + jar):
-                    continue
-                # download to a temporay name and use an atomic rename (in case an other istance is downloading the same file
-                handle, temp = tempfile.mkstemp(dir = self.workDir, prefix = jar + '.')
-                os.close(handle)
-                urllib.request.urlretrieve(self.baseUrl + '/' + jar, temp)
-                if not os.path.exists(self.workDir + '/' + jar):
+            version_website = requests.get(self.baseUrl+"/../confdb.version").text
+            jars_require_update = True
+            if os.path.exists(os.path.join(self.workDir,"confdb.version")):
+                with open(os.path.join(self.workDir,"confdb.version")) as f:
+                    version_existing = f.read()                
+                    if version_existing==version_website:
+                        jars_require_update = False
+
+            if jars_require_update:
+                for jar in self.jars:
+                    # download to a temporay name and use an atomic rename (in case an other istance is downloading the same file
+                    handle, temp = tempfile.mkstemp(dir = self.workDir, prefix = jar + '.')
+                    os.close(handle)
+                    request = requests.get(self.baseUrl + '/' + jar)
+                    with open(temp,'wb') as f:
+                        f.write(request.content)
                     os.rename(temp, self.workDir + '/' + jar)
-                else:
-                    os.unlink(temp)
+                #jars updated, write their version
+                handle, temp = tempfile.mkstemp(dir = self.workDir, prefix = "confdb.version" + '.')
+                os.close(handle)
+                with open(temp,'w') as f:
+                    f.write(version_website)
+                os.rename(temp,os.path.join(self.workDir,"confdb.version"))
 
         # setup the java command line and CLASSPATH
         if self.verbose:
@@ -129,7 +165,7 @@ def __init__(self, version = 'v3', database = 'run3', url = None, verbose = Fals
 
 
     def query(self, *args):
-        args = self.javaCmd + self.connect + args
+        args = self.javaCmd + self.connect + self.proxy_connect_args + args 
         if self.verbose:
             sys.stderr.write("\n" + ' '.join(args) + "\n\n" )
         sub = subprocess.Popen(
@@ -239,6 +275,20 @@ def main():
         version = 'v3-test'
         db      = 'dev'
         args.remove('--v3-test')
+
+    proxy=False
+    proxy_host = "localhost"
+    proxy_port = "8080"
+    if '--dbproxy' in args:
+        proxy = True
+        args.remove('--dbproxy')
+    if '--dbproxyhost' in args:
+        proxy_host = args.pop(args.index('--dbproxyhost')+1)
+        args.remove('--dbproxyhost')
+    if '--dbproxyport' in args:
+        proxy_port = args.pop(args.index('--dbproxyport')+1)
+        args.remove('--dbproxyport')
+
 
     _dbs = {}
     _dbs['v1'] = [ '--%s' % _db for _db in OfflineConverter.databases['v1'] ] + [ '--runNumber' ]
@@ -264,7 +314,8 @@ def main():
             sys.stderr.write( "ERROR: database version \"%s\" incompatible with specification \"%s\"\n" % (version, db) )
             sys.exit(1)
 
-    converter = OfflineConverter(version = version, database = db, verbose = verbose)
+    converter = OfflineConverter(version = version, database = db, verbose = verbose,
+                                 proxy = proxy, proxyHost = proxy_host, proxyPort=proxy_port)
     out, err = converter.query( * args )
     if 'ERROR' in err:
         sys.stderr.write( "%s: error while retriving the HLT menu\n\n%s\n\n" % (sys.argv[0], err) )

HLTrigger/Configuration/python/Tools/options.py

@@ -128,7 +128,9 @@ def __init__(self):
     self.fragment   = False       #     prepare a configuration fragment (true) or a whole process (false)
     self.hilton     = False       #     prepare a configuration for running with hilton-like modules
     self.setup      = None        #     if set, downlad the setup_cff from the specified configuration and load it.
-
+    self.proxy      = False       #     use a socks proxy to connect
+    self.proxy_host = 'localhost' #     host of the proxy server
+    self.proxy_port = '8080'        #     port of the proxy server
 
   # convert HLT and L1 menus to a dedicated object representation on the fly
   def __setattr__(self, name, value):

HLTrigger/Configuration/scripts/hltGetConfiguration

@@ -76,6 +76,24 @@ parser.add_argument('--l1-emulator',
                     const   = 'Full',
                     help    = 'Run the Full stage-2 L1T emulator.' )
 
+parser.add_argument('--dbproxy',
+                    dest    = 'proxy',
+                    action  = 'store_const',
+                    const   = 'True',                    
+                    help    = 'Use a socks proxy to connect outside CERN network (default: False)' )
+parser.add_argument('--dbproxyport',
+                    dest    = 'proxy_port',
+                    action  = 'store',
+                    metavar = 'PROXYPORT',
+                    default = defaults.proxy_port,
+                    help    = 'Port of the socks proxy (default: 8080)' )
+parser.add_argument('--dbproxyhost',
+                    dest    = 'proxy_host',
+                    action  = 'store',
+                    metavar = 'PROXYHOST',
+                    default = defaults.proxy_host,
+                    help    = 'Host of the socks proxy (default: "localhost")' )
+
 group = parser.add_mutually_exclusive_group()
 group.add_argument('--prescale',
                     dest    = 'prescale',

HLTrigger/Configuration/scripts/hltIntegrationTests

@@ -44,6 +44,9 @@ Usage:
        --threads  THREADS     Run with THREADS threads when running the whole HLT (default 4)
   -x | --extra    OPTIONS     Pass OPTIONS directly to hltGetConfiguration
        --mc                   Run over MC instead of data (the default)
+  --dbproxy                   Use a socks proxy to connect to the HLT database
+  --dbproxyhost               Host of the socks proxy (default localhost)
+  --dbproxyport               Port of the socks proxy (default 8080)
   -h | --help                 Print this help message and exit.
 
 
@@ -84,7 +87,7 @@ Exmples:
 "
 
 # parse command line argument and options
-OPTS=$(getopt -n "$NAME" -o "s:d:i:j:n:k:e:x:h" -l "setup:,dir:,input:,jobs:,size:,skip:,streams:,threads:,events:,mc,extra:,help" -- "$@")
+OPTS=$(getopt -n "$NAME" -o "s:d:i:j:n:k:e:x:h" -l "setup:,dir:,input:,jobs:,size:,skip:,streams:,threads:,events:,mc,extra:,help,dbproxy,dbproxyhost:,dbproxyport:" -- "$@")
 
 # invalid options
 if [ $? != 0 ]; then
@@ -109,6 +112,8 @@ STREAMS=0
 WORKDIR="hltintegration"
 EXTRA=""
 DATA="--data"
+HLTLISTPATHPROXY=""
+
 
 SELECTION=""
 
@@ -194,6 +199,21 @@ while true; do
       DATA="--mc"
       shift 1
       ;;
+    "--dbproxy" )
+      HLTLISTPATHPROXY="$HLTLISTPATHPROXY --dbproxy"
+      EXTRA="$EXTRA --dbproxy"
+      shift 1
+      ;;
+    "--dbproxyhost" )
+      PROXYHOST="$HLTLISTPATHPROXY --dbproxyhost $2"
+      EXTRA="$EXTRA --dbproxyhost $2"
+      shift 2
+      ;;
+    "--dbproxyport" )
+      PROXYPORT="$HLTLISTPATHPROXY --dbproxyport $2"
+      EXTRA="$EXTRA --dbproxyport $2"
+      shift 2
+      ;;
     "--" )
       # inserted by getopt to singal the end of options
       shift
@@ -222,7 +242,7 @@ mkdir  "$WORKDIR"
 cd     "$WORKDIR"
 
 # find the list of all trigger paths
-TRIGGERS=$(hltListPaths -p "$MENU" | grep -v HLTriggerFinalPath)
+TRIGGERS=$(hltListPaths -p "$MENU" $HLTLISTPATHPROXY | grep -v HLTriggerFinalPath)
 
 # print some info
 if [ "$SELECTION" == "complex" ]; then
@@ -309,8 +329,12 @@ done
 # if a separate setup is requested, create the setup_cff.py file and patch all dumps to use it
 if [ "$SETUP" ]; then
   log "Extracting setup_cff dump"
-  read SETUP_Vx SETUP_DB SETUP <<< $(parse_HLT_menu "$SETUP")
-  hltConfigFromDB --$SETUP_Vx --$SETUP_DB --cff --configName "$SETUP" --nopaths --services -FUShmDQMOutputService,-PrescaleService,-EvFDaqDirector,-FastMonitoringService > setup_cff.py
+  #we use $MENU not $SETUP here as we force the same DB / converter as the main menu
+  #this is the hltGetConfiguration behaviour and would be confusing if you had to 
+  #specify converter/db on the setup menu on hltIntegration tests but not on hltGetConfiguration
+  read SETUP_Vx SETUP_DB SETUP <<< $(parse_HLT_menu "$MENU")
+
+  hltConfigFromDB --$SETUP_Vx --$SETUP_DB $HLTLISTPATHPROXY --cff --configName "$SETUP" --nopaths --services -FUShmDQMOutputService,-PrescaleService,-EvFDaqDirector,-FastMonitoringService > setup_cff.py
   sed -i -e's/process = cms.Process(.*)/&\nprocess.load("setup_cff")/' hlt.py $(for TRIGGER in $TRIGGERS; do echo "$TRIGGER".py; done)
 fi
 

HLTrigger/Configuration/scripts/hltListPaths

@@ -16,17 +16,19 @@ def _build_query(menu):
   if menu.run:
     return '--runNumber %s' % menu.run
   else:
-    return '--%s --configName %s' % (menu.database, menu.name)
+    return '--%s --%s --configName %s' % (menu.database, menu.version, menu.name)
 
-def getPathList(menu, selection):
-  cmdline = 'hltConfigFromDB --cff %s --noedsources --noes --noservices --nosequences --nomodules' % _build_query(menu)
+def getPathList(config):
+  cmdline = 'hltConfigFromDB --cff %s --noedsources --noes --noservices --nosequences --nomodules' % _build_query(config.menu)
+  if config.proxy:
+     cmdline += f" --dbproxy --dbproxyhost {config.proxy_host} --dbproxyport {config.proxy_port}"
   data = pipe.pipe(cmdline).decode()
   if 'Exhausted Resultset' in data or 'CONFIG_NOT_FOUND' in data:
     raise ImportError('%s is not a valid HLT menu' % menu.value)
 
-  if selection == 'paths':
+  if config.selection == 'paths':
     filter = re.compile(r' *= *cms.Path.*')
-  elif selection == 'endpaths':
+  elif config.selection == 'endpaths':
     filter = re.compile(r' *= *cms.EndPath.*')
   else: # 'all'
     filter = re.compile(r' *= *cms.(End)?Path.*')
@@ -43,6 +45,9 @@ except:
   pass
 formatter = FixedWidthFormatter( HelpFormatterRespectNewlines, width = textwidth )
 
+# read defaults
+defaults = options.HLTProcessOptions()
+
 parser = argparse.ArgumentParser(
   description       = 'List all the Paths and EndPaths from an HLT configuration in the ConfDB database.''',
   argument_default  = argparse.SUPPRESS,
@@ -55,6 +60,24 @@ parser.add_argument('menu',
                     type    = options.ConnectionHLTMenu,
                     metavar = 'MENU', 
                     help    = 'HLT menu to dump from the database. Supported formats are:\n  - /path/to/configuration[/Vn]\n  - [[{v1|v2|v3}/]{run3|run2|online|adg}:]/path/to/configuration[/Vn]\n  - run:runnumber\nThe possible converters are "v1", "v2, and "v3" (default).\nThe possible databases are "run3" (default, used for offline development), "run2" (used for accessing run2 offline development menus), "online" (used to extract online menus within Point 5) and "adg" (used to extract the online menus outside Point 5).\nIf no menu version is specified, the latest one is automatically used.\nIf "run:" is used instead, the HLT menu used for the given run number is looked up and used.\nNote other converters and databases exist as options but they are only for expert/special use.' )
+parser.add_argument('--dbproxy',
+                    dest    = 'proxy',
+                    action  = 'store_const',
+                    const   = 'True',                    
+                    help    = 'Use a socks proxy to connect outside CERN network (default: False)' )
+parser.add_argument('--dbproxyport',
+                    dest    = 'proxy_port',
+                    action  = 'store',
+                    metavar = 'PROXYPORT',
+                    default = defaults.proxy_port,
+                    help    = 'Port of the socks proxy (default: 8080)' )
+parser.add_argument('--dbproxyhost',
+                    dest    = 'proxy_host',
+                    action  = 'store',
+                    metavar = 'PROXYHOST',
+                    default = defaults.proxy_host,
+                    help    = 'Host of the socks proxy (default: "localhost")' )
+
 
 # options
 group = parser.add_mutually_exclusive_group()
@@ -82,7 +105,7 @@ parser.add_argument('-h', '--help',
 
 # parse command line arguments and options
 config = parser.parse_args()
-paths  = getPathList(config.menu, config.selection)
+paths  = getPathList(config)
 for path in paths:
   print(path)