From 7ac2590586c15924af9c7c92df6d757937927fec Mon Sep 17 00:00:00 2001 From: AdrianoDee Date: Tue, 21 May 2024 16:22:24 +0200 Subject: [PATCH 1/3] New GenOnly and SimOnGen wfs: - Adding Sim step. - Introducing GenOnly and SimOnGen wfs for 2024 and D110. - Generalizing ALCA inputs. --- .../python/WorkFlowRunner.py | 10 ++++- .../python/relval_steps.py | 17 ++++++- .../python/relval_upgrade.py | 2 +- .../python/upgradeWorkflowComponents.py | 44 ++++++++++++++++--- 4 files changed, 64 insertions(+), 9 deletions(-) diff --git a/Configuration/PyReleaseValidation/python/WorkFlowRunner.py b/Configuration/PyReleaseValidation/python/WorkFlowRunner.py index e30c7d8fa000f..b01e2885d85db 100644 --- a/Configuration/PyReleaseValidation/python/WorkFlowRunner.py +++ b/Configuration/PyReleaseValidation/python/WorkFlowRunner.py @@ -25,6 +25,7 @@ def __init__(self, wf, noRun=False,dryRun=False,cafVeto=True,dasOptions="",jobRe self.nStreams=nStreams self.maxSteps=maxSteps self.nEvents=nEvents + self.recoOutput='' self.wfDir=str(self.wf.numId)+'_'+self.wf.nameId return @@ -158,9 +159,16 @@ def closeCmd(i,ID): # Disable input for premix stage2 in FastSim to allow combined stage1+stage2 workflow (in FS, stage2 does also GEN) # Ugly hack but works if istep!=1 and not '--filein' in cmd and not 'premix_stage1' in cmd and not ("--fast" in cmd and "premix_stage2" in cmd): - cmd+=' --filein file:step%s.root '%(istep-1,) + if "ALCA" not in cmd: + cmd+=' --filein file:step%s.root '%(istep-1,) + elif "ALCA" in cmd and "RECO" in cmd: + cmd+=' --filein file:step%s.root '%(istep-1,) + else: + cmd+=' --filein %s'%(self.recoOutput) if not '--fileout' in com: cmd+=' --fileout file:step%s.root '%(istep,) + if "RECO" in cmd: + self.recoOutput = "file:step%d.root"%(istep) if self.jobReport: cmd += ' --suffix "-j JobReport%s.xml " ' % istep if (self.nThreads > 1) and ('HARVESTING' not in cmd) and ('ALCAHARVEST' not in cmd): diff --git a/Configuration/PyReleaseValidation/python/relval_steps.py b/Configuration/PyReleaseValidation/python/relval_steps.py index 70d58ae732eb5..d35eae37f78ac 100644 --- a/Configuration/PyReleaseValidation/python/relval_steps.py +++ b/Configuration/PyReleaseValidation/python/relval_steps.py @@ -4222,6 +4222,7 @@ def gen2023HiMix(fragment,howMuch): defaultDataSets['2023FS']='CMSSW_13_0_11-130X_mcRun3_2023_realistic_withEarly2023BS_v1_FastSim-v' defaultDataSets['2024']='CMSSW_14_0_0_pre3-140X_mcRun3_2024_realistic_v1_STD_2024_noPU-v' defaultDataSets['2024HLTOnDigi']='CMSSW_14_0_0_pre3-140X_mcRun3_2024_realistic_v1_STD_2024_noPU-v' +defaultDataSets["2024SimOnGen"] = 'CMSSW_14_0_0_pre3-140X_mcRun3_2024_realistic_v1_STD_2024_noPU-v' defaultDataSets['2026D49']='CMSSW_12_0_0_pre4-113X_mcRun4_realistic_v7_2026D49noPU-v' defaultDataSets['2026D76']='CMSSW_12_0_0_pre4-113X_mcRun4_realistic_v7_2026D76noPU-v' defaultDataSets['2026D77']='CMSSW_12_1_0_pre2-113X_mcRun4_realistic_v7_2026D77noPU-v' @@ -4302,6 +4303,7 @@ def gen2023HiMix(fragment,howMuch): '--eventcontent': 'FEVTDEBUG', '--geometry' : geom } + if beamspot is not None: upgradeStepDict['GenSim'][k]['--beamspot']=beamspot upgradeStepDict['GenSimHLBeamSpot'][k]= {'-s' : 'GEN,SIM', @@ -4331,6 +4333,18 @@ def gen2023HiMix(fragment,howMuch): '--geometry' : geom } + upgradeStepDict['Sim'][k]= {'-s' : 'SIM', + '-n' : 10, + '--conditions' : gt, + '--beamspot' : 'Realistic25ns13TeVEarly2017Collision', + '--datatier' : 'SIM', + '--eventcontent': 'FEVTDEBUG', + '--geometry' : geom + } + + if beamspot is not None: upgradeStepDict['Sim'][k]['--beamspot']=beamspot + + upgradeStepDict['Digi'][k] = {'-s':'DIGI:pdigi_valid,L1,DIGI2RAW,HLT:%s'%(hltversion), '--conditions':gt, '--datatier':'GEN-SIM-DIGI-RAW', @@ -4461,7 +4475,6 @@ def gen2023HiMix(fragment,howMuch): '-n':'10', '--eventcontent':'ALCARECO', '--geometry' : geom, - '--filein':'file:step3.root' } upgradeStepDict['ALCAPhase2'][k] = merge([{'-s':'ALCA:SiPixelCalSingleMuonLoose+SiPixelCalSingleMuonTight+TkAlMuonIsolated+TkAlMinBias+MuAlOverlaps+EcalESAlign+TkAlZMuMu+TkAlDiMuonAndVertex+HcalCalHBHEMuonProducerFilter+TkAlUpsilonMuMu+TkAlJpsiMuMu'},upgradeStepDict['ALCA'][k]]) @@ -4545,7 +4558,7 @@ def gen2023HiMix(fragment,howMuch): for step in upgradeStepDict.keys(): # we need to do this for each fragment - if ('Sim' in step and 'Fast' not in step) or ('Premix' in step) or ('Sim' not in step and 'Gen' in step): + if ('Sim' in step and ('Fast' not in step and step != 'Sim')) or ('Premix' in step) or ('Sim' not in step and 'Gen' in step): for frag,info in upgradeFragments.items(): howMuch=info.howMuch for key in [key for year in upgradeKeys for key in upgradeKeys[year]]: diff --git a/Configuration/PyReleaseValidation/python/relval_upgrade.py b/Configuration/PyReleaseValidation/python/relval_upgrade.py index e6adaab29c36f..09a57473c2db9 100644 --- a/Configuration/PyReleaseValidation/python/relval_upgrade.py +++ b/Configuration/PyReleaseValidation/python/relval_upgrade.py @@ -30,7 +30,7 @@ def makeStepName(key,frag,step,suffix): hasHarvest = False for step in upgradeProperties[year][key]['ScenToRun']: stepMaker = makeStepName - if 'Sim' in step and 'Fast' not in step: + if 'Sim' in step and 'Fast' not in step and step != "Sim": if 'HLBeamSpot' in step: if '14TeV' in frag: step = 'GenSimHLBeamSpot14' diff --git a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py index a5b2fa489346a..bce6be1dd14c5 100644 --- a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py +++ b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py @@ -37,8 +37,9 @@ '2023HI', '2023HIRP', #RawPrime '2024HLTOnDigi', - '2024HLTOnDigiPU' - + '2024HLTOnDigiPU', + '2024GenOnly', + '2024SimOnGen', ] upgradeKeys[2026] = [ @@ -93,7 +94,9 @@ '2026D113', '2026D113PU', '2026D114', - '2026D114PU' + '2026D114PU', + '2026D110GenOnly', + '2026D110SimOnGen', ] # pre-generation of WF numbers @@ -178,7 +181,7 @@ def workflow_(self, workflows, num, fragment, stepList, key): def condition(self, fragment, stepList, key, hasHarvest): return False def preventReuse(self, stepName, stepDict, k): - if "Sim" in stepName: + if "Sim" in stepName and stepName != "Sim": stepDict[stepName][k] = None if "Gen" in stepName: stepDict[stepName][k] = None @@ -198,6 +201,7 @@ def condition(self, fragment, stepList, key, hasHarvest): upgradeWFs['baseline'] = UpgradeWorkflow_baseline( steps = [ 'Gen', + 'Sim', 'GenSim', 'GenSimHLBeamSpot', 'GenSimHLBeamSpot14', @@ -2933,7 +2937,22 @@ def condition(self, fragment, stepList, key, hasHarvest): 'Era':'Run3_pp_on_PbPb_approxSiStripClusters', 'BeamSpot': 'DBrealistic', 'ScenToRun' : ['GenSim','Digi','RecoNano','HARVESTNano','ALCA'], - } + }, + '2024GenOnly' : { + 'Geom' : 'DB:Extended', + 'GT' : 'auto:phase1_2024_realistic', + 'Era' : 'Run3', + 'BeamSpot': 'DBrealistic', + 'ScenToRun' : ['Gen'], + }, + '2024SimOnGen' : { + 'Geom' : 'DB:Extended', + 'GT' : 'auto:phase1_2024_realistic', + 'HLTmenu': '@relval2024', + 'Era' : 'Run3', + 'BeamSpot': 'DBrealistic', + 'ScenToRun' : ['Gen','Sim','Digi','RecoNano','HARVESTNano','ALCA'], + }, } # standard PU sequences @@ -3133,6 +3152,21 @@ def condition(self, fragment, stepList, key, hasHarvest): 'Era' : 'Phase2C17I13M9', 'ScenToRun' : ['GenSimHLBeamSpot','DigiTrigger','RecoGlobal', 'HARVESTGlobal', 'ALCAPhase2'], }, + '2026D110GenOnly' : { + 'Geom' : 'Extended2026D110', + 'Beamspot' : 'HLLHC', + 'GT' : 'auto:phase2_realistic_T33', + 'Era' : 'Phase2C17I13M9', + 'ScenToRun' : ['Gen'], + }, + '2026D110SimOnGen' : { + 'Geom' : 'Extended2026D110', + 'HLTmenu': '@relval2026', + 'Beamspot' : 'HLLHC', + 'GT' : 'auto:phase2_realistic_T33', + 'Era' : 'Phase2C17I13M9', + 'ScenToRun' : ['Gen','Sim','DigiTrigger','RecoGlobal', 'HARVESTGlobal', 'ALCAPhase2'], + }, } # standard PU sequences From 86a8712c03d091e947c8eaa37e0d26708f73ecea Mon Sep 17 00:00:00 2001 From: AdrianoDee Date: Fri, 31 May 2024 10:56:05 +0200 Subject: [PATCH 2/3] Restricting GenOnly wfs to baseline --- .../PyReleaseValidation/python/relval_upgrade.py | 12 ++++++++++++ .../python/upgradeWorkflowComponents.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Configuration/PyReleaseValidation/python/relval_upgrade.py b/Configuration/PyReleaseValidation/python/relval_upgrade.py index 09a57473c2db9..29e4aaef27a95 100644 --- a/Configuration/PyReleaseValidation/python/relval_upgrade.py +++ b/Configuration/PyReleaseValidation/python/relval_upgrade.py @@ -16,6 +16,10 @@ def makeStepName(key,frag,step,suffix): #just define all of them +## ... but we don't need all the flavors for the GenOnly +def notForGenOnly(key,specialType): + return "GenOnly" in key and specialType != 'baseline' + for year in upgradeKeys: for i,key in enumerate(upgradeKeys[year]): numWF=numWFAll[year][i] @@ -26,6 +30,8 @@ def makeStepName(key,frag,step,suffix): continue stepList={} for specialType in upgradeWFs.keys(): + if notForGenOnly(key,specialType): + continue stepList[specialType] = [] hasHarvest = False for step in upgradeProperties[year][key]['ScenToRun']: @@ -43,6 +49,10 @@ def makeStepName(key,frag,step,suffix): if 'HARVEST' in step: hasHarvest = True for specialType,specialWF in upgradeWFs.items(): + + if notForGenOnly(key,specialType): ## we don't need all the flavors for the GEN + continue + if (specialType != 'baseline') and ( ('PU' in step and step.replace('PU','') in specialWF.PU) or (step in specialWF.steps) ): stepList[specialType].append(stepMaker(key,frag[:-4],step,specialWF.suffix)) # hack to add an extra step @@ -71,6 +81,8 @@ def makeStepName(key,frag,step,suffix): for specialType,specialWF in upgradeWFs.items(): # remove other steps for premixS1 + if notForGenOnly(key,specialType): + continue if specialType=="PMXS1": stepList[specialType] = stepList[specialType][:1] specialWF.workflow(workflows, numWF, info.dataset, stepList[specialType], key, hasHarvest) diff --git a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py index bce6be1dd14c5..331847cf8f9c9 100644 --- a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py +++ b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py @@ -2624,7 +2624,7 @@ def condition(self, fragment, stepList, key, hasHarvest): class UpgradeWorkflow_Run3FSMBMixing(UpgradeWorkflow): def setup_(self, step, stepName, stepDict, k, properties): - if 'Gen' in step: + if 'Gen' in step and 'GenOnly' not in step: stepDict[stepName][k] = merge([{'-s':'GEN,SIM,RECOBEFMIX', '--fast':'', '--era':'Run3_FastSim', From 1250f1de38d1c113061c95a1a333c1a2b0024f64 Mon Sep 17 00:00:00 2001 From: AdrianoDee Date: Wed, 5 Jun 2024 17:49:40 +0200 Subject: [PATCH 3/3] Fix ALCA input --- Configuration/PyReleaseValidation/python/WorkFlowRunner.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Configuration/PyReleaseValidation/python/WorkFlowRunner.py b/Configuration/PyReleaseValidation/python/WorkFlowRunner.py index b01e2885d85db..931916d6ab4e9 100644 --- a/Configuration/PyReleaseValidation/python/WorkFlowRunner.py +++ b/Configuration/PyReleaseValidation/python/WorkFlowRunner.py @@ -159,10 +159,11 @@ def closeCmd(i,ID): # Disable input for premix stage2 in FastSim to allow combined stage1+stage2 workflow (in FS, stage2 does also GEN) # Ugly hack but works if istep!=1 and not '--filein' in cmd and not 'premix_stage1' in cmd and not ("--fast" in cmd and "premix_stage2" in cmd): - if "ALCA" not in cmd: + steps = cmd.split("-s ")[1].split(" ")[0] ## relying on the syntax: cmsDriver -s STEPS --otherFlags + if "ALCA" not in steps: + cmd+=' --filein file:step%s.root '%(istep-1,) + elif "ALCA" in steps and "RECO" in steps: cmd+=' --filein file:step%s.root '%(istep-1,) - elif "ALCA" in cmd and "RECO" in cmd: - cmd+=' --filein file:step%s.root '%(istep-1,) else: cmd+=' --filein %s'%(self.recoOutput) if not '--fileout' in com: