From aaa9407609e1915f633fab75753baf711913e207 Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Wed, 3 Dec 2025 11:04:40 -0600 Subject: [PATCH 1/3] Added --use-rntuple option to runTheMatrix.py This will pass --rntuple_out to ConfigBuilder. --- Configuration/PyReleaseValidation/python/WorkFlowRunner.py | 3 +++ Configuration/PyReleaseValidation/scripts/runTheMatrix.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/Configuration/PyReleaseValidation/python/WorkFlowRunner.py b/Configuration/PyReleaseValidation/python/WorkFlowRunner.py index 56a101132e6d2..2f70e056a7066 100644 --- a/Configuration/PyReleaseValidation/python/WorkFlowRunner.py +++ b/Configuration/PyReleaseValidation/python/WorkFlowRunner.py @@ -30,6 +30,7 @@ def __init__(self, wf, opt, noRun=False, dryRun=False, cafVeto=True, jobNumber=N self.recoOutput = '' self.startFrom = opt.startFrom self.recycle = opt.recycle + self.useRNTuple = opt.useRNTuple self.wfDir=str(self.wf.numId)+'_'+self.wf.nameId if jobNumber is not None: @@ -164,6 +165,8 @@ def closeCmd(i,ID): cmd += com + if self.useRNTuple: + cmd+=' --rntuple_out' if self.startFrom: steps = cmd.split("-s ")[1].split(" ")[0] if self.startFrom not in steps: diff --git a/Configuration/PyReleaseValidation/scripts/runTheMatrix.py b/Configuration/PyReleaseValidation/scripts/runTheMatrix.py index 672470df023c0..f5232e5bc2ef9 100755 --- a/Configuration/PyReleaseValidation/scripts/runTheMatrix.py +++ b/Configuration/PyReleaseValidation/scripts/runTheMatrix.py @@ -437,6 +437,12 @@ def runSelected(opt): default=None, action='store') + parser.add_argument('--use-rntuple', + help='Use RNTuple format for IO', + dest='useRNTuple', + default=False, + action='store_true') + gpugroup = parser.add_argument_group('GPU-related options','These options are only meaningful when --gpu is used, and is not set to forbidden.') gpugroup.add_argument('--gpu','--requires-gpu', From a0802ef6096c7b4905a1a6b6bc09a7ad770d0643 Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Thu, 5 Jun 2025 13:57:11 -0500 Subject: [PATCH 2/3] Handle multi-file ALCA switch to RNTUPLE --- Configuration/PyReleaseValidation/python/WorkFlowRunner.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Configuration/PyReleaseValidation/python/WorkFlowRunner.py b/Configuration/PyReleaseValidation/python/WorkFlowRunner.py index 2f70e056a7066..0bc83115c5189 100644 --- a/Configuration/PyReleaseValidation/python/WorkFlowRunner.py +++ b/Configuration/PyReleaseValidation/python/WorkFlowRunner.py @@ -220,6 +220,10 @@ def closeCmd(i,ID): m = re.search(expression, cmd) if m: cmd = re.sub(expression,r'--filein file:\1.rntpl',cmd) + expression = '--filein\s+file:([a-zA-Z0-9_]+)*\.[a-z]+, *file:([a-zA-Z0-9_]+)*\.[a-z]+' + m = re.search(expression, cmd) + if m: + cmd = re.sub(expression,r'--filein file:\1.rntpl,file:\2.rntpl',cmd) if not '--fileout' in com: cmd+=' --fileout file:step%s%s '%(istep,extension) if "RECO" in cmd: From 83a455bd04532dc94fb14d0d6e22058c26771892 Mon Sep 17 00:00:00 2001 From: iarspider Date: Wed, 18 Jun 2025 22:49:31 +0200 Subject: [PATCH 3/3] Fix replacing extensions in WorkFlowRunner.py Fixes #48360 --- .../python/WorkFlowRunner.py | 45 ++++++++++++------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/Configuration/PyReleaseValidation/python/WorkFlowRunner.py b/Configuration/PyReleaseValidation/python/WorkFlowRunner.py index 0bc83115c5189..cd9e9d0e54c90 100644 --- a/Configuration/PyReleaseValidation/python/WorkFlowRunner.py +++ b/Configuration/PyReleaseValidation/python/WorkFlowRunner.py @@ -59,6 +59,33 @@ def doCmd(self, cmd): return ret + @staticmethod + def replace_filein_extensions(command_line, extension): + # Pattern to match --filein followed by file:file.ext entries (comma-separated) + filein_pattern = re.compile( + r'(--filein\s+)((?:file:[a-zA-Z0-9_]+\.[a-z]+(?:,\s*)?)*)' + ) + + # Inner pattern to match individual file entries + file_pattern = re.compile(r'file:([a-zA-Z0-9_]+)\.[a-z]+') + + def replace_filein_match(filein_match): + filein_prefix = filein_match.group(1) + file_list_str = filein_match.group(2) + + # Replace extensions in the file list + new_file_list = file_pattern.sub( + lambda m: 'file:{0}{1}'.format(m.group(1), extension), + file_list_str + ) + + return filein_prefix + new_file_list + + # Replace the whole --filein section with updated extensions + new_command_line = filein_pattern.sub(replace_filein_match, command_line) + + return new_command_line + def run(self): startDir = os.getcwd() @@ -208,22 +235,8 @@ def closeCmd(i,ID): else: cmd+=' --filein file:step%s%s '%(istep-1,extension) elif istep!=1 and '--filein' in cmd and '--filetype' not in cmd: - #make sure correct extension is being used - #find the previous state index - expression = '--filein\s+file:step([1-9])(_[a-zA-Z]+)*\.[a-z]+' - m = re.search(expression, cmd) - if m: - cmd = re.sub(expression,r'--filein file:step\1\2'+outputExtensionForStep[int(m.group(1))],cmd) - elif extension == '.rntpl': - #some ALCA steps use special file names without step_ prefix and these are also force to use RNTuple - expression = '--filein\s+file:([a-zA-Z0-9_]+)*\.[a-z]+' - m = re.search(expression, cmd) - if m: - cmd = re.sub(expression,r'--filein file:\1.rntpl',cmd) - expression = '--filein\s+file:([a-zA-Z0-9_]+)*\.[a-z]+, *file:([a-zA-Z0-9_]+)*\.[a-z]+' - m = re.search(expression, cmd) - if m: - cmd = re.sub(expression,r'--filein file:\1.rntpl,file:\2.rntpl',cmd) + # make sure correct extension is being used + cmd = self.replace_filein_extensions(cmd, extension) if not '--fileout' in com: cmd+=' --fileout file:step%s%s '%(istep,extension) if "RECO" in cmd: