tokens - added feature flag OAUTH_CMS_TOKEN_NAME in WMAgent.secrets

bin/wmagent-mod-config

@@ -285,6 +285,11 @@ def modifyConfiguration(config, **args):
         if args.get("mspileup_url", None):
             config.WorkflowUpdater.msPileupUrl = args["mspileup_url"]
 
+    # custom JobSubmitter
+    if hasattr(config, "JobSubmitter"):
+        # tier0 may not start supporting tokens straight away.
+        if args.get("oauth_cms_token_name"):
+            config.JobSubmitter.oauthCMSTokenName = args["oauth_cms_token_name"]
 
     return config
 
@@ -313,7 +318,8 @@ def main(argv=None):
                                          "reqmgr2_url=", "acdc_url=", "amq_auth_file=", "dbs3_url=", "dbs3_reader_url=",
                                          "dqm_url=", "grafana_token=", "requestcouch_url=", "central_logdb_url=",
                                          "wmarchive_url=", "amq_credentials=",
-                                         "rucio_account=", "rucio_host=", "rucio_auth=", "mspileup_url="])
+                                         "rucio_account=", "rucio_host=", "rucio_auth=", "mspileup_url=",
+                                         "oauth_cms_token_name="])
 
         except getopt.error as msg:
             raise Usage(msg)
@@ -337,7 +343,9 @@ def main(argv=None):
                           '--amq_auth_file', '--dbs3_url', '--dbs3_reader_url', '--dqm_url',
                           '--grafana_token', '--requestcouch_url', '--central_logdb_url',
                           '--wmarchive_url', '--amq_credentials',
-                          '--rucio_account', '--rucio_host', '--rucio_auth', '--mspileup_url'):
+                          '--rucio_account', '--rucio_host', '--rucio_auth', '--mspileup_url' ,
+                          '--oauth_cms_token_name'
+                          ):
                 parameters[option[2:]] = value
 
 

deploy/WMAgent.production

@@ -30,6 +30,7 @@ RUCIO_AUTH=https://cms-rucio-auth.cern.ch
 TEAMNAME=
 AGENT_NUMBER=
 MSPILEUP_URL=https://cmsweb.cern.ch/ms-pileup/data/pileup
+OAUTH_CMS_TOKEN_NAME=cms_wmagent
 RESOURCE_OPP1=([name]=T3_US_NERSC [run]=3000 [pend]=2000 [state]=normal)
 RESOURCE_OPP2=([name]=T3_US_OSG [run]=3000 [pend]=2000 [state]=normal)
 RESOURCE_OPP3=([name]=T3_US_PSC [run]=3000 [pend]=2000 [state]=normal)

deploy/WMAgent.testbed

@@ -30,6 +30,7 @@ RUCIO_AUTH=https://cms-rucio-auth-int.cern.ch
 TEAMNAME=
 AGENT_NUMBER=
 MSPILEUP_URL=https://cmsweb-testbed.cern.ch/ms-pileup/data/pileup
+OAUTH_CMS_TOKEN_NAME=cms_wmagent
 RESOURCE_OPP1=([name]=T3_US_NERSC [run]=3000 [pend]=2000 [state]=normal)
 RESOURCE_OPP2=([name]=T3_US_OSG [run]=3000 [pend]=2000 [state]=normal)
 RESOURCE_OPP3=([name]=T3_US_PSC [run]=3000 [pend]=2000 [state]=normal)

etc/WMAgentConfig.py

@@ -221,7 +221,8 @@
 config.JobSubmitter.submitScript = os.path.join(os.environ["WMCORE_ROOT"], submitScript)
 config.JobSubmitter.extraMemoryPerCore = 500  # in MB
 config.JobSubmitter.drainGraceTime = 2 * 24 * 60 * 60  # in seconds
-config.JobSubmitter.useOauthToken = False
+config.JobSubmitter.oauthCMSTokenName = ""   # disable with: "". enable with "cms_wmagent". 
+                                             # override with WMAgent.secrets variable OAUTH_CMS_TOKEN_NAME
 
 config.component_("JobTracker")
 config.JobTracker.namespace = "WMComponent.JobTracker.JobTracker"

etc/submit_py3.sh

@@ -197,10 +197,10 @@ if [ -n "${_CONDOR_CREDS}" ]; then
     echo "Content under _CONDOR_CREDS: ${_CONDOR_CREDS}"
     ls -l ${_CONDOR_CREDS}
     # Now, check specifically for cms token
-    if [ -f "${_CONDOR_CREDS}/cms.use" ]
-    then
-        echo "CMS token found, setting BEARER_TOKEN_FILE=${_CONDOR_CREDS}/cms.use"
-        export BEARER_TOKEN_FILE=${_CONDOR_CREDS}/cms.use
+    for tokenfile in ${_CONDOR_CREDS}/*.use ; do
+        if [ -f ${tokenfile} ];  then
+          echo "CMS token found, setting BEARER_TOKEN_FILE=${tokenfile}"
+          export BEARER_TOKEN_FILE=${tokenfile}
 
         # Show token information
         # This tool requires htgettoken package in the cmssw runtime apptainer image
@@ -211,12 +211,13 @@ if [ -n "${_CONDOR_CREDS}" ]; then
             echo "Warning: [WMAgent Token verification] httokendecode tool could not be found."
             echo "Warning: Token exists and can be used, but details will not be displayed."
         fi
-    else
+      else
         echo "[WMAgent token verification]: The bearer token file could not be found."
         # Do not fail, we still support x509 proxies
         # if we fail here in the future, we need to define an exit code number
         # exit 1106
-    fi
+      fi
+    done
 else
     echo "Variable _CONDOR_CREDS is not defined, condor auth/token credentials directory not found."
 fi

src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py

@@ -140,6 +140,22 @@ def __init__(self, config):
             # Tier0 Case - just for the clarity (This private variable shouldn't be used
             self.abortedAndForceCompleteWorkflowCache = None
 
+        # log status of oauth tokens
+        _oauth_token_name = getattr(config.JobSubmitter, 'oauthCMSTokenName', "")
+        if _oauth_token_name:
+            logging.info("[tokens] Jobs will be submitted with tokens") 
+            logging.info("[tokens] token available on the wmagent host with sudo at path /var/lib/condor/oauth_credentials/cmst1/%s.use",
+                       getattr(self.config.JobSubmitter, "oauthCMSTokenName", ""))
+        else:
+            logging.info("[tokens] remote jobs will not contain oauth tokens.")
+            logging.info("""[tokens] enable them: 
+[tokens] - change config.JobSubmitter.authCMSTokenName in /data/dockerMount/srv/wmagent/current/config/config.py
+[tokens] - restart the agent
+[tokens] otherwise, if you can initialize the agent from scratch:
+[tokens] - set OAUTH_CMS_TOKEN_NAME in WMAgent.secrets
+[tokens] - initialize the new agent
+""")
+
         return
 
     def getPackageCollection(self, sandboxDir):

src/python/WMCore/BossAir/Plugins/SimpleCondorPlugin.py

@@ -141,7 +141,7 @@ def __init__(self, config):
 
         self.tc = TagCollector()
 
-        self.useCMSToken = getattr(config.JobSubmitter, 'useOauthToken', False)
+        self.oauthCMSTokenName = getattr(config.JobSubmitter, 'oauthCMSTokenName', "")
 
         return
 
@@ -527,8 +527,9 @@ def getJobParameters(self, jobList):
             ad['My.x509userproxy'] = classad.quote(self.x509userproxy)
 
             # Allow oauth based token authentication
-            if self.useCMSToken:
-                ad['use_oauth_services'] = "cms"
+            if self.oauthCMSTokenName:
+                # 2025aug: self.oauthCMSTokenName == cms_wmagent
+                ad['use_oauth_services'] = self.oauthCMSTokenName
 
             sites = ','.join(sorted(job.get('possibleSites')))
             ad['My.DESIRED_Sites'] = classad.quote(str(sites))