From 48abc3c9074e2cb37e7a332f7769801438130d89 Mon Sep 17 00:00:00 2001 From: Dario Mapelli Date: Thu, 21 Aug 2025 18:35:13 +0200 Subject: [PATCH 1/4] tokens - added feature flag OAUTH_CMS_TOKEN_NAME in WMAgent.secrets --- bin/wmagent-mod-config | 12 ++++++++++-- etc/WMAgentConfig.py | 2 +- .../JobSubmitter/JobSubmitterPoller.py | 16 ++++++++++++++++ .../WMCore/BossAir/Plugins/SimpleCondorPlugin.py | 7 ++++--- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/bin/wmagent-mod-config b/bin/wmagent-mod-config index df0eb73e41..1f0783fab9 100755 --- a/bin/wmagent-mod-config +++ b/bin/wmagent-mod-config @@ -285,6 +285,11 @@ def modifyConfiguration(config, **args): if args.get("mspileup_url", None): config.WorkflowUpdater.msPileupUrl = args["mspileup_url"] + # custom JobSubmitter + if hasattr(config, "JobSubmitter"): + # tier0 may not start supporting tokens straight away. + if args.get("oauth_cms_token_name"): + config.JobSubmitter.oauthCMSTokenName = args["oauth_cms_token_name"] return config @@ -313,7 +318,8 @@ def main(argv=None): "reqmgr2_url=", "acdc_url=", "amq_auth_file=", "dbs3_url=", "dbs3_reader_url=", "dqm_url=", "grafana_token=", "requestcouch_url=", "central_logdb_url=", "wmarchive_url=", "amq_credentials=", - "rucio_account=", "rucio_host=", "rucio_auth=", "mspileup_url="]) + "rucio_account=", "rucio_host=", "rucio_auth=", "mspileup_url=", + "oauth_cms_token_name="]) except getopt.error as msg: raise Usage(msg) @@ -337,7 +343,9 @@ def main(argv=None): '--amq_auth_file', '--dbs3_url', '--dbs3_reader_url', '--dqm_url', '--grafana_token', '--requestcouch_url', '--central_logdb_url', '--wmarchive_url', '--amq_credentials', - '--rucio_account', '--rucio_host', '--rucio_auth', '--mspileup_url'): + '--rucio_account', '--rucio_host', '--rucio_auth', '--mspileup_url' , + '--oauth_cms_token_name' + ): parameters[option[2:]] = value diff --git a/etc/WMAgentConfig.py b/etc/WMAgentConfig.py index d608788a50..9a003f56ee 100644 --- a/etc/WMAgentConfig.py +++ b/etc/WMAgentConfig.py @@ -221,7 +221,7 @@ config.JobSubmitter.submitScript = os.path.join(os.environ["WMCORE_ROOT"], submitScript) config.JobSubmitter.extraMemoryPerCore = 500 # in MB config.JobSubmitter.drainGraceTime = 2 * 24 * 60 * 60 # in seconds -config.JobSubmitter.useOauthToken = False +config.JobSubmitter.authCMSTokenName = "none" # disable with: "none". enable with "cms_wmagent" config.component_("JobTracker") config.JobTracker.namespace = "WMComponent.JobTracker.JobTracker" diff --git a/src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py b/src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py index 65d23a5273..6d8b28a0db 100644 --- a/src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py +++ b/src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py @@ -140,6 +140,22 @@ def __init__(self, config): # Tier0 Case - just for the clarity (This private variable shouldn't be used self.abortedAndForceCompleteWorkflowCache = None + # log status of oauth tokens + _oauth_token_name = getattr(config.JobSubmitter, 'oauthCMSTokenName', "") + if _oauth_token_name and _oauth_token_name.lower() != "none": + logging.info("[tokens] Jobs will be submitted with tokens") + logging.info("[tokens] token available on the wmagent host with sudo at path /var/lib/condor/oauth_credentials/cmst1/%s.use", + getattr(self.config.JobSubmitter, "oauthCMSTokenName", "")) + else: + logging.info("[tokens] remote jobs will not contain oauth tokens.") + logging.info("""[tokens] enable them: +[tokens] - change config.JobSubmitter.authCMSTokenName in /data/dockerMount/srv/wmagent/current/config/config.py +[tokens] - restart the agent +[tokens] otherwise, if you can initialize the agent from scratch: +[tokens] - set OAUTH_CMS_TOKEN_NAME in WMAgent.secrets +[tokens] - initialize the new agent +""") + return def getPackageCollection(self, sandboxDir): diff --git a/src/python/WMCore/BossAir/Plugins/SimpleCondorPlugin.py b/src/python/WMCore/BossAir/Plugins/SimpleCondorPlugin.py index 3e70b49309..bfc56fe060 100644 --- a/src/python/WMCore/BossAir/Plugins/SimpleCondorPlugin.py +++ b/src/python/WMCore/BossAir/Plugins/SimpleCondorPlugin.py @@ -141,7 +141,7 @@ def __init__(self, config): self.tc = TagCollector() - self.useCMSToken = getattr(config.JobSubmitter, 'useOauthToken', False) + self.oauthCMSTokenName = getattr(config.JobSubmitter, 'oauthCMSTokenName', "") return @@ -527,8 +527,9 @@ def getJobParameters(self, jobList): ad['My.x509userproxy'] = classad.quote(self.x509userproxy) # Allow oauth based token authentication - if self.useCMSToken: - ad['use_oauth_services'] = "cms" + if self.oauthCMSTokenName and self.oauthCMSTokenName.lower() != "none": + # 2025aug: self.oauthCMSTokenName == cms_wmagent + ad['use_oauth_services'] = self.oauthCMSTokenName sites = ','.join(sorted(job.get('possibleSites'))) ad['My.DESIRED_Sites'] = classad.quote(str(sites)) From bbb3990965f5103c2184f3059f20c958f6bebca3 Mon Sep 17 00:00:00 2001 From: Dario Mapelli Date: Sat, 23 Aug 2025 12:02:40 +0200 Subject: [PATCH 2/4] typo --- etc/WMAgentConfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/WMAgentConfig.py b/etc/WMAgentConfig.py index 9a003f56ee..26a81fcd67 100644 --- a/etc/WMAgentConfig.py +++ b/etc/WMAgentConfig.py @@ -221,7 +221,7 @@ config.JobSubmitter.submitScript = os.path.join(os.environ["WMCORE_ROOT"], submitScript) config.JobSubmitter.extraMemoryPerCore = 500 # in MB config.JobSubmitter.drainGraceTime = 2 * 24 * 60 * 60 # in seconds -config.JobSubmitter.authCMSTokenName = "none" # disable with: "none". enable with "cms_wmagent" +config.JobSubmitter.oauthCMSTokenName = "none" # disable with: "none". enable with "cms_wmagent" config.component_("JobTracker") config.JobTracker.namespace = "WMComponent.JobTracker.JobTracker" From 778bb8694631b8f14ec4971483dc90ca4d8b9a75 Mon Sep 17 00:00:00 2001 From: Dario Mapelli Date: Thu, 28 Aug 2025 18:07:22 +0200 Subject: [PATCH 3/4] alan feedback - 1 --- deploy/WMAgent.production | 1 + deploy/WMAgent.testbed | 1 + etc/WMAgentConfig.py | 3 ++- src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py | 2 +- src/python/WMCore/BossAir/Plugins/SimpleCondorPlugin.py | 2 +- 5 files changed, 6 insertions(+), 3 deletions(-) diff --git a/deploy/WMAgent.production b/deploy/WMAgent.production index bc0bcd80a3..80940798e6 100644 --- a/deploy/WMAgent.production +++ b/deploy/WMAgent.production @@ -30,6 +30,7 @@ RUCIO_AUTH=https://cms-rucio-auth.cern.ch TEAMNAME= AGENT_NUMBER= MSPILEUP_URL=https://cmsweb.cern.ch/ms-pileup/data/pileup +OAUTH_CMS_TOKEN_NAME=cms_wmagent RESOURCE_OPP1=([name]=T3_US_NERSC [run]=3000 [pend]=2000 [state]=normal) RESOURCE_OPP2=([name]=T3_US_OSG [run]=3000 [pend]=2000 [state]=normal) RESOURCE_OPP3=([name]=T3_US_PSC [run]=3000 [pend]=2000 [state]=normal) diff --git a/deploy/WMAgent.testbed b/deploy/WMAgent.testbed index 52b0a20984..fa343ab5d8 100644 --- a/deploy/WMAgent.testbed +++ b/deploy/WMAgent.testbed @@ -30,6 +30,7 @@ RUCIO_AUTH=https://cms-rucio-auth-int.cern.ch TEAMNAME= AGENT_NUMBER= MSPILEUP_URL=https://cmsweb-testbed.cern.ch/ms-pileup/data/pileup +OAUTH_CMS_TOKEN_NAME=cms_wmagent RESOURCE_OPP1=([name]=T3_US_NERSC [run]=3000 [pend]=2000 [state]=normal) RESOURCE_OPP2=([name]=T3_US_OSG [run]=3000 [pend]=2000 [state]=normal) RESOURCE_OPP3=([name]=T3_US_PSC [run]=3000 [pend]=2000 [state]=normal) diff --git a/etc/WMAgentConfig.py b/etc/WMAgentConfig.py index 26a81fcd67..2bbd55d9e1 100644 --- a/etc/WMAgentConfig.py +++ b/etc/WMAgentConfig.py @@ -221,7 +221,8 @@ config.JobSubmitter.submitScript = os.path.join(os.environ["WMCORE_ROOT"], submitScript) config.JobSubmitter.extraMemoryPerCore = 500 # in MB config.JobSubmitter.drainGraceTime = 2 * 24 * 60 * 60 # in seconds -config.JobSubmitter.oauthCMSTokenName = "none" # disable with: "none". enable with "cms_wmagent" +config.JobSubmitter.oauthCMSTokenName = "" # disable with: "". enable with "cms_wmagent". + # override with WMAgent.secrets variable OAUTH_CMS_TOKEN_NAME config.component_("JobTracker") config.JobTracker.namespace = "WMComponent.JobTracker.JobTracker" diff --git a/src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py b/src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py index 6d8b28a0db..584af39e44 100644 --- a/src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py +++ b/src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py @@ -142,7 +142,7 @@ def __init__(self, config): # log status of oauth tokens _oauth_token_name = getattr(config.JobSubmitter, 'oauthCMSTokenName', "") - if _oauth_token_name and _oauth_token_name.lower() != "none": + if _oauth_token_name: logging.info("[tokens] Jobs will be submitted with tokens") logging.info("[tokens] token available on the wmagent host with sudo at path /var/lib/condor/oauth_credentials/cmst1/%s.use", getattr(self.config.JobSubmitter, "oauthCMSTokenName", "")) diff --git a/src/python/WMCore/BossAir/Plugins/SimpleCondorPlugin.py b/src/python/WMCore/BossAir/Plugins/SimpleCondorPlugin.py index bfc56fe060..fa998a6b74 100644 --- a/src/python/WMCore/BossAir/Plugins/SimpleCondorPlugin.py +++ b/src/python/WMCore/BossAir/Plugins/SimpleCondorPlugin.py @@ -527,7 +527,7 @@ def getJobParameters(self, jobList): ad['My.x509userproxy'] = classad.quote(self.x509userproxy) # Allow oauth based token authentication - if self.oauthCMSTokenName and self.oauthCMSTokenName.lower() != "none": + if self.oauthCMSTokenName: # 2025aug: self.oauthCMSTokenName == cms_wmagent ad['use_oauth_services'] = self.oauthCMSTokenName From fefb95bbbb35ed401c6009a46b0428db1c1c377d Mon Sep 17 00:00:00 2001 From: Dario Mapelli Date: Thu, 28 Aug 2025 18:34:21 +0200 Subject: [PATCH 4/4] tokens - update submit_py3.sh --- etc/submit_py3.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/etc/submit_py3.sh b/etc/submit_py3.sh index 6081fcced3..2855b783b5 100644 --- a/etc/submit_py3.sh +++ b/etc/submit_py3.sh @@ -197,10 +197,10 @@ if [ -n "${_CONDOR_CREDS}" ]; then echo "Content under _CONDOR_CREDS: ${_CONDOR_CREDS}" ls -l ${_CONDOR_CREDS} # Now, check specifically for cms token - if [ -f "${_CONDOR_CREDS}/cms.use" ] - then - echo "CMS token found, setting BEARER_TOKEN_FILE=${_CONDOR_CREDS}/cms.use" - export BEARER_TOKEN_FILE=${_CONDOR_CREDS}/cms.use + for tokenfile in ${_CONDOR_CREDS}/*.use ; do + if [ -f ${tokenfile} ]; then + echo "CMS token found, setting BEARER_TOKEN_FILE=${tokenfile}" + export BEARER_TOKEN_FILE=${tokenfile} # Show token information # This tool requires htgettoken package in the cmssw runtime apptainer image @@ -211,12 +211,13 @@ if [ -n "${_CONDOR_CREDS}" ]; then echo "Warning: [WMAgent Token verification] httokendecode tool could not be found." echo "Warning: Token exists and can be used, but details will not be displayed." fi - else + else echo "[WMAgent token verification]: The bearer token file could not be found." # Do not fail, we still support x509 proxies # if we fail here in the future, we need to define an exit code number # exit 1106 - fi + fi + done else echo "Variable _CONDOR_CREDS is not defined, condor auth/token credentials directory not found." fi