diff --git a/bin/wmagent-mod-config b/bin/wmagent-mod-config index df0eb73e41..1f0783fab9 100755 --- a/bin/wmagent-mod-config +++ b/bin/wmagent-mod-config @@ -285,6 +285,11 @@ def modifyConfiguration(config, **args): if args.get("mspileup_url", None): config.WorkflowUpdater.msPileupUrl = args["mspileup_url"] + # custom JobSubmitter + if hasattr(config, "JobSubmitter"): + # tier0 may not start supporting tokens straight away. + if args.get("oauth_cms_token_name"): + config.JobSubmitter.oauthCMSTokenName = args["oauth_cms_token_name"] return config @@ -313,7 +318,8 @@ def main(argv=None): "reqmgr2_url=", "acdc_url=", "amq_auth_file=", "dbs3_url=", "dbs3_reader_url=", "dqm_url=", "grafana_token=", "requestcouch_url=", "central_logdb_url=", "wmarchive_url=", "amq_credentials=", - "rucio_account=", "rucio_host=", "rucio_auth=", "mspileup_url="]) + "rucio_account=", "rucio_host=", "rucio_auth=", "mspileup_url=", + "oauth_cms_token_name="]) except getopt.error as msg: raise Usage(msg) @@ -337,7 +343,9 @@ def main(argv=None): '--amq_auth_file', '--dbs3_url', '--dbs3_reader_url', '--dqm_url', '--grafana_token', '--requestcouch_url', '--central_logdb_url', '--wmarchive_url', '--amq_credentials', - '--rucio_account', '--rucio_host', '--rucio_auth', '--mspileup_url'): + '--rucio_account', '--rucio_host', '--rucio_auth', '--mspileup_url' , + '--oauth_cms_token_name' + ): parameters[option[2:]] = value diff --git a/deploy/WMAgent.production b/deploy/WMAgent.production index bc0bcd80a3..80940798e6 100644 --- a/deploy/WMAgent.production +++ b/deploy/WMAgent.production @@ -30,6 +30,7 @@ RUCIO_AUTH=https://cms-rucio-auth.cern.ch TEAMNAME= AGENT_NUMBER= MSPILEUP_URL=https://cmsweb.cern.ch/ms-pileup/data/pileup +OAUTH_CMS_TOKEN_NAME=cms_wmagent RESOURCE_OPP1=([name]=T3_US_NERSC [run]=3000 [pend]=2000 [state]=normal) RESOURCE_OPP2=([name]=T3_US_OSG [run]=3000 [pend]=2000 [state]=normal) RESOURCE_OPP3=([name]=T3_US_PSC [run]=3000 [pend]=2000 [state]=normal) diff --git a/deploy/WMAgent.testbed b/deploy/WMAgent.testbed index 52b0a20984..fa343ab5d8 100644 --- a/deploy/WMAgent.testbed +++ b/deploy/WMAgent.testbed @@ -30,6 +30,7 @@ RUCIO_AUTH=https://cms-rucio-auth-int.cern.ch TEAMNAME= AGENT_NUMBER= MSPILEUP_URL=https://cmsweb-testbed.cern.ch/ms-pileup/data/pileup +OAUTH_CMS_TOKEN_NAME=cms_wmagent RESOURCE_OPP1=([name]=T3_US_NERSC [run]=3000 [pend]=2000 [state]=normal) RESOURCE_OPP2=([name]=T3_US_OSG [run]=3000 [pend]=2000 [state]=normal) RESOURCE_OPP3=([name]=T3_US_PSC [run]=3000 [pend]=2000 [state]=normal) diff --git a/etc/WMAgentConfig.py b/etc/WMAgentConfig.py index d608788a50..2bbd55d9e1 100644 --- a/etc/WMAgentConfig.py +++ b/etc/WMAgentConfig.py @@ -221,7 +221,8 @@ config.JobSubmitter.submitScript = os.path.join(os.environ["WMCORE_ROOT"], submitScript) config.JobSubmitter.extraMemoryPerCore = 500 # in MB config.JobSubmitter.drainGraceTime = 2 * 24 * 60 * 60 # in seconds -config.JobSubmitter.useOauthToken = False +config.JobSubmitter.oauthCMSTokenName = "" # disable with: "". enable with "cms_wmagent". + # override with WMAgent.secrets variable OAUTH_CMS_TOKEN_NAME config.component_("JobTracker") config.JobTracker.namespace = "WMComponent.JobTracker.JobTracker" diff --git a/etc/submit_py3.sh b/etc/submit_py3.sh index 6081fcced3..2855b783b5 100644 --- a/etc/submit_py3.sh +++ b/etc/submit_py3.sh @@ -197,10 +197,10 @@ if [ -n "${_CONDOR_CREDS}" ]; then echo "Content under _CONDOR_CREDS: ${_CONDOR_CREDS}" ls -l ${_CONDOR_CREDS} # Now, check specifically for cms token - if [ -f "${_CONDOR_CREDS}/cms.use" ] - then - echo "CMS token found, setting BEARER_TOKEN_FILE=${_CONDOR_CREDS}/cms.use" - export BEARER_TOKEN_FILE=${_CONDOR_CREDS}/cms.use + for tokenfile in ${_CONDOR_CREDS}/*.use ; do + if [ -f ${tokenfile} ]; then + echo "CMS token found, setting BEARER_TOKEN_FILE=${tokenfile}" + export BEARER_TOKEN_FILE=${tokenfile} # Show token information # This tool requires htgettoken package in the cmssw runtime apptainer image @@ -211,12 +211,13 @@ if [ -n "${_CONDOR_CREDS}" ]; then echo "Warning: [WMAgent Token verification] httokendecode tool could not be found." echo "Warning: Token exists and can be used, but details will not be displayed." fi - else + else echo "[WMAgent token verification]: The bearer token file could not be found." # Do not fail, we still support x509 proxies # if we fail here in the future, we need to define an exit code number # exit 1106 - fi + fi + done else echo "Variable _CONDOR_CREDS is not defined, condor auth/token credentials directory not found." fi diff --git a/src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py b/src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py index 65d23a5273..584af39e44 100644 --- a/src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py +++ b/src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py @@ -140,6 +140,22 @@ def __init__(self, config): # Tier0 Case - just for the clarity (This private variable shouldn't be used self.abortedAndForceCompleteWorkflowCache = None + # log status of oauth tokens + _oauth_token_name = getattr(config.JobSubmitter, 'oauthCMSTokenName', "") + if _oauth_token_name: + logging.info("[tokens] Jobs will be submitted with tokens") + logging.info("[tokens] token available on the wmagent host with sudo at path /var/lib/condor/oauth_credentials/cmst1/%s.use", + getattr(self.config.JobSubmitter, "oauthCMSTokenName", "")) + else: + logging.info("[tokens] remote jobs will not contain oauth tokens.") + logging.info("""[tokens] enable them: +[tokens] - change config.JobSubmitter.authCMSTokenName in /data/dockerMount/srv/wmagent/current/config/config.py +[tokens] - restart the agent +[tokens] otherwise, if you can initialize the agent from scratch: +[tokens] - set OAUTH_CMS_TOKEN_NAME in WMAgent.secrets +[tokens] - initialize the new agent +""") + return def getPackageCollection(self, sandboxDir): diff --git a/src/python/WMCore/BossAir/Plugins/SimpleCondorPlugin.py b/src/python/WMCore/BossAir/Plugins/SimpleCondorPlugin.py index 3e70b49309..fa998a6b74 100644 --- a/src/python/WMCore/BossAir/Plugins/SimpleCondorPlugin.py +++ b/src/python/WMCore/BossAir/Plugins/SimpleCondorPlugin.py @@ -141,7 +141,7 @@ def __init__(self, config): self.tc = TagCollector() - self.useCMSToken = getattr(config.JobSubmitter, 'useOauthToken', False) + self.oauthCMSTokenName = getattr(config.JobSubmitter, 'oauthCMSTokenName', "") return @@ -527,8 +527,9 @@ def getJobParameters(self, jobList): ad['My.x509userproxy'] = classad.quote(self.x509userproxy) # Allow oauth based token authentication - if self.useCMSToken: - ad['use_oauth_services'] = "cms" + if self.oauthCMSTokenName: + # 2025aug: self.oauthCMSTokenName == cms_wmagent + ad['use_oauth_services'] = self.oauthCMSTokenName sites = ','.join(sorted(job.get('possibleSites'))) ad['My.DESIRED_Sites'] = classad.quote(str(sites))