Skip to content

Fix prodlike workflows #36050

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
kskovpen wants to merge 8 commits into from
Closed

Fix prodlike workflows #36050

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
06672d1
Merge pull request #11 from cms-sw/master
kskovpen Oct 14, 2021
6263c5a
Merge pull request #12 from cms-sw/master
kskovpen Nov 9, 2021
da36f62
fix prodlike pu wfs
kskovpen Nov 9, 2021
13d052a
update
kskovpen Nov 12, 2021
7019071
update
kskovpen Nov 12, 2021
2dfd1a9
fix
kskovpen Nov 18, 2021
3adf540
update
kskovpen Nov 18, 2021
63ac30d
update
kskovpen Nov 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions Configuration/PyReleaseValidation/python/relval_steps.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3509,13 +3509,13 @@ def gen2021HiMix(fragment,howMuch):
'--filetype':'DQM',
}

upgradeStepDict['HARVESTRecNan'][k]={'-s':'HARVESTING:@standardValidation+@standardDQM+@ExtraHLT+@miniAODValidation+@miniAODDQM+@nanoAODDQM',
'--conditions':gt,
'--mc':'',
'--geometry' : geom,
'--scenario' : 'pp',
'--filetype':'DQM',
}
upgradeStepDict['HARVESTNano'][k]={'-s':'HARVESTING:@standardValidation+@standardDQM+@ExtraHLT+@miniAODValidation+@miniAODDQM+@nanoAODDQM',
'--conditions':gt,
'--mc':'',
'--geometry' : geom,
'--scenario' : 'pp',
'--filetype':'DQM',
}

upgradeStepDict['HARVESTFakeHLT'][k]={'-s':'HARVESTING:@standardValidationNoHLT+@standardDQMFakeHLT+@miniAODValidation+@miniAODDQM',
'--conditions':gt,
Expand Down Expand Up @@ -3560,7 +3560,7 @@ def gen2021HiMix(fragment,howMuch):
'--geometry' : geom
}

upgradeStepDict['RecNan'][k] = {'-s':'RAW2DIGI,L1Reco,RECO,RECOSIM,EI,PAT,NANO,VALIDATION:@standardValidation+@miniAODValidation,DQM:@standardDQM+@ExtraHLT+@miniAODDQM+@nanoAODDQM',
upgradeStepDict['RecoNano'][k] = {'-s':'RAW2DIGI,L1Reco,RECO,RECOSIM,EI,PAT,NANO,VALIDATION:@standardValidation+@miniAODValidation,DQM:@standardDQM+@ExtraHLT+@miniAODDQM+@nanoAODDQM',
'--conditions':gt,
'--datatier':'GEN-SIM-RECO,MINIAODSIM,NANOAODSIM,DQMIO',
'-n':'10',
Expand Down
8 changes: 2 additions & 6 deletions Configuration/PyReleaseValidation/python/relval_upgrade.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ def makeStepName(key,frag,step,suffix):
if 'ProdLike' in specialType:
if 'Reco' in step: # handles both Reco and RecoGlobal
stepList[specialType].append(stepMaker(key,frag[:-4],step.replace('RecoGlobal','MiniAOD').replace('Reco','MiniAOD'),specialWF.suffix))
Copy link
Contributor

@kpedro88 kpedro88 Nov 16, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@kskovpen you missed a change from my branch here: https://github.com/kpedro88/cmssw/blob/856746872c99eb3a6c180e284574c0e6fd7a088c/Configuration/PyReleaseValidation/python/relval_upgrade.py#L49 (the addition of .replace('RecoNano','MiniAOD'))

stepList[specialType].append(stepMaker(key,frag[:-4],step.replace('RecoGlobal','Nano').replace('Reco','Nano'),specialWF.suffix))
if 'RecoNano' in step:
stepList[specialType].append(stepMaker(key,frag[:-4],step.replace('RecoNano','Nano'),specialWF.suffix))
# similar hacks for premixing
if 'PMX' in specialType:
if 'GenSim' in step:
Expand All @@ -60,11 +61,6 @@ def makeStepName(key,frag,step,suffix):
else: stepList[specialType][-1] = stepMade
else:
stepList[specialType].append(stepMaker(key,frag[:-4],step,''))

if specialType in ['baseline']:
for ist, st in enumerate(stepList[specialType]):
if st.split('_')[0] == 'Reco': stepList[specialType][ist] = st.replace('Reco', 'RecNan')
elif st.split('_')[0] == 'HARVEST': stepList[specialType][ist] = st.replace('HARVEST', 'HARVESTRecNan')

for specialType,specialWF in upgradeWFs.items():
# remove other steps for premixS1
Expand Down
69 changes: 54 additions & 15 deletions Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ def setup_(self, step, stepName, stepDict, k, properties):
if cust is not None: stepDict[stepName][k]['--customise']=cust
if era is not None:
stepDict[stepName][k]['--era']=era
if 'RecNan' in stepName: stepDict[stepName][k]['--era'] += ',run3_nanoAOD_devel'
if 'RecoNano' in stepName: stepDict[stepName][k]['--era'] += ',run3_nanoAOD_devel'
if modifier is not None: stepDict[stepName][k]['--procModifier']=modifier
def condition(self, fragment, stepList, key, hasHarvest):
return True
Expand All @@ -170,14 +170,13 @@ def condition(self, fragment, stepList, key, hasHarvest):
'RecoGlobal',
'HARVEST',
Copy link
Contributor

@kpedro88 kpedro88 Nov 16, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you missed the addition of 'RecoNano' here. instead of copying changes manually from my branch, which seems error-prone, why not just use the branch/commit directly?

'HARVESTFakeHLT',
'HARVESTRecNan',
'HARVESTNano',
'FastSim',
'HARVESTFast',
'HARVESTGlobal',
'ALCA',
'Nano',
'MiniAOD',
'RecNan',
],
PU = [
'DigiTrigger',
Expand All @@ -188,11 +187,10 @@ def condition(self, fragment, stepList, key, hasHarvest):
'RecoFakeHLT',
'HARVEST',
'HARVESTFakeHLT',
'HARVESTRecNan',
'HARVESTNano',
'HARVESTGlobal',
'MiniAOD',
'Nano',
'RecNan',
],
suffix = '',
offset = 0.0,
Expand Down Expand Up @@ -221,6 +219,8 @@ def setup_(self, step, stepName, stepDict, k, properties):
'HARVEST',
'RecoGlobal',
'HARVESTGlobal',
'RecoNano',
'HARVESTNano',
],
PU = [],
suffix = '_trackingOnly',
Expand Down Expand Up @@ -294,6 +294,8 @@ def condition_(self, fragment, stepList, key, hasHarvest):
'HARVEST',
'RecoGlobal',
'HARVESTGlobal',
'RecoNano',
'HARVESTNano',
],
PU = [],
suffix = '_pixelTrackingOnly',
Expand All @@ -317,6 +319,7 @@ def condition_(self, fragment, stepList, key, hasHarvest):
'DigiTrigger',
'Reco',
'RecoGlobal',
'RecoNano',
],
PU = [],
suffix = '_trackingMkFit',
Expand Down Expand Up @@ -347,6 +350,8 @@ def condition(self, fragment, stepList, key, hasHarvest):
'HARVEST',
'RecoGlobal',
'HARVESTGlobal',
'RecoNano',
'HARVESTNano',
],
PU = [],
suffix = '_seedingDeepCore',
Expand Down Expand Up @@ -382,9 +387,11 @@ def condition(self, fragment, stepList, key, hasHarvest):
upgradeWFs['trackdnn'] = UpgradeWorkflow_trackdnn(
steps = [
'Reco',
'RecoNano',
],
PU = [
'Reco',
'RecoNano',
],
suffix = '_trackdnn',
offset = 0.91,
Expand All @@ -402,16 +409,18 @@ def condition(self, fragment, stepList, key, hasHarvest):
upgradeWFs['mlpf'] = UpgradeWorkflow_mlpf(
steps = [
'Reco',
'RecoNano',
],
PU = [
'Reco',
'RecoNano',
],
suffix = '_mlpf',
offset = 0.13,
)
upgradeWFs['mlpf'].step3 = {
'--datatier': 'GEN-SIM-RECO,RECOSIM,MINIAODSIM,DQMIO',
'--eventcontent': 'FEVTDEBUGHLT,RECOSIM,MINIAODSIM,DQM',
'--datatier': 'GEN-SIM-RECO,RECOSIM,MINIAODSIM,NANOAODSIM,DQMIO',
'--eventcontent': 'FEVTDEBUGHLT,RECOSIM,MINIAODSIM,NANOAODSIM,DQM',
'--procModifiers': 'mlpf'
}

Expand All @@ -433,6 +442,8 @@ def __init__(self, digi = {}, reco = {}, harvest = {}, **kwargs):
'HARVESTFakeHLT',
'RecoGlobal',
'HARVESTGlobal',
'RecoNano',
'HARVESTNano',
],
PU = [],
**kwargs)
Expand Down Expand Up @@ -736,10 +747,10 @@ def setup_(self, step, stepName, stepDict, k, properties):
elif 'MiniAOD' in step:
# the separate miniAOD step is used here
stepDict[stepName][k] = deepcopy(stepDict[step][k])
if 'ALCA' in step or 'HARVEST' in step:
elif 'ALCA' in step or 'HARVEST' in step:
# remove step
stepDict[stepName][k] = None
if 'Nano' in step:
elif 'Nano' in step:
stepDict[stepName][k] = merge([{'--filein':'file:step4.root','-s':'NANO','--datatier':'NANOAODSIM','--eventcontent':'NANOEDMAODSIM'}, stepDict[step][k]])
def condition(self, fragment, stepList, key, hasHarvest):
return fragment=="TTbar_14TeV" and ('2026' in key or '2021' in key)
Expand All @@ -750,8 +761,10 @@ def condition(self, fragment, stepList, key, hasHarvest):
'DigiTrigger',
'Reco',
'RecoGlobal',
'RecoNano',
'HARVEST',
'HARVESTGlobal',
'HARVESTNano',
'MiniAOD',
'ALCA',
'Nano',
Expand All @@ -762,8 +775,10 @@ def condition(self, fragment, stepList, key, hasHarvest):
'DigiTrigger',
'Reco',
'RecoGlobal',
'RecoNano',
'HARVEST',
'HARVESTGlobal',
'HARVESTNano',
'MiniAOD',
'ALCA',
'Nano',
Expand Down Expand Up @@ -815,12 +830,16 @@ def condition(self, fragment, stepList, key, hasHarvest):
'Digi',
'Reco',
'HARVEST',
'RecoNano',
'HARVESTNano',
'ALCA',
],
PU = [
'Digi',
'Reco',
'HARVEST',
'RecoNano',
'HARVESTNano',
],
suffix = '_heCollapse',
offset = 0.6,
Expand Down Expand Up @@ -863,12 +882,16 @@ def condition(self, fragment, stepList, key, hasHarvest):
'Digi',
'Reco',
'HARVEST',
'RecoNano',
'HARVESTNano',
'ALCA',
],
PU = [
'Digi',
'Reco',
'HARVEST',
'RecoNano',
'HARVESTNano',
],
suffix = '_0T',
offset = 0.24,
Expand Down Expand Up @@ -1040,6 +1063,7 @@ def workflow_(self, workflows, num, fragment, stepList, key):
'RecoLocal',
'Reco',
'RecoGlobal',
'RecoNano',
'Nano',
],
suffix = '_PMXS2',
Expand All @@ -1057,6 +1081,7 @@ def workflow_(self, workflows, num, fragment, stepList, key):
'RecoLocal',
'Reco',
'RecoGlobal',
'RecoNano',
'Nano',
],
suffix = '_PMXS1S2',
Expand Down Expand Up @@ -1123,9 +1148,11 @@ def condition(self, fragment, stepList, key, hasHarvest):
'RecoLocal',
'Reco',
'RecoGlobal',
'RecoNano',
'Nano',
'HARVEST',
'HARVESTGlobal',
'HARVESTNano',
'MiniAOD',
'ALCA',
],
Expand All @@ -1144,9 +1171,11 @@ def condition(self, fragment, stepList, key, hasHarvest):
'RecoLocal',
'Reco',
'RecoGlobal',
'RecoNano',
'Nano',
'HARVEST',
'HARVESTGlobal',
'HARVESTNano',
'MiniAOD',
'ALCA',
],
Expand All @@ -1173,8 +1202,10 @@ def condition(self, fragment, stepList, key, hasHarvest):
'DigiTrigger',
'Reco',
'RecoGlobal',
'RecoNano',
'HARVEST',
'HARVESTGlobal',
'HARVESTNano',
'ALCA',
],
PU = [],
Expand All @@ -1200,8 +1231,10 @@ def condition(self, fragment, stepList, key, hasHarvest):
'DigiTrigger',
'Reco',
'RecoGlobal',
'RecoNano',
'HARVEST',
'HARVESTGlobal',
'HARVESTNano',
'ALCA',
],
PU = [],
Expand All @@ -1225,8 +1258,10 @@ def condition(self, fragment, stepList, key, hasHarvest):
'DigiTrigger',
'Reco',
'RecoGlobal',
'RecoNano',
'HARVEST',
'HARVESTGlobal',
'HARVESTNano',
'ALCA',
],
PU = [],
Expand All @@ -1250,8 +1285,10 @@ def condition(self, fragment, stepList, key, hasHarvest):
'DigiTrigger',
'Reco',
'RecoGlobal',
'RecoNano',
'HARVEST',
'HARVESTGlobal',
'HARVESTNano',
'ALCA',
],
PU = [
Expand All @@ -1262,8 +1299,10 @@ def condition(self, fragment, stepList, key, hasHarvest):
'DigiTrigger',
'Reco',
'RecoGlobal',
'RecoNano',
'HARVEST',
'HARVESTGlobal',
'HARVESTNano',
'ALCA',
],
suffix = '_SonicTriton',
Expand Down Expand Up @@ -1317,40 +1356,40 @@ def condition(self, fragment, stepList, key, hasHarvest):
'HLTmenu': '@relval2021',
'Era' : 'Run3',
'BeamSpot': 'Run3RoundOptics25ns13TeVLowSigmaZ',
'ScenToRun' : ['GenSim','Digi','Reco','HARVEST','ALCA'],
'ScenToRun' : ['GenSim','Digi','RecoNano','HARVESTNano','ALCA'],
},
'2021Design' : {
'Geom' : 'DB:Extended',
'GT' : 'auto:phase1_2021_design',
'HLTmenu': '@relval2021',
'Era' : 'Run3',
'BeamSpot': 'GaussSigmaZ4cm',
'ScenToRun' : ['GenSim','Digi','Reco','HARVEST'],
'ScenToRun' : ['GenSim','Digi','RecoNano','HARVESTNano'],
},
'2023' : {
'Geom' : 'DB:Extended',
'GT' : 'auto:phase1_2023_realistic',
'HLTmenu': '@relval2021',
'Era' : 'Run3',
'BeamSpot': 'Run3RoundOptics25ns13TeVLowSigmaZ',
'ScenToRun' : ['GenSim','Digi','Reco','HARVEST','ALCA'],
'ScenToRun' : ['GenSim','Digi','RecoNano','HARVESTNano','ALCA'],
},
'2024' : {
'Geom' : 'DB:Extended',
'GT' : 'auto:phase1_2024_realistic',
'HLTmenu': '@relval2021',
'Era' : 'Run3',
'BeamSpot': 'Run3RoundOptics25ns13TeVLowSigmaZ',
'ScenToRun' : ['GenSim','Digi','Reco','HARVEST','ALCA'],
'ScenToRun' : ['GenSim','Digi','RecoNano','HARVESTNano','ALCA'],
},
}

# standard PU sequences
for key in list(upgradeProperties[2017].keys()):
upgradeProperties[2017][key+'PU'] = deepcopy(upgradeProperties[2017][key])
upgradeProperties[2017][key+'PU']['ScenToRun'] = ['GenSim','DigiPU'] + \
(['RecoPU','HARVESTPU'] if '202' in key else ['RecoFakeHLTPU','HARVESTFakeHLTPU']) + \
(['Nano'] if 'Design' not in key else [])
(['RecoNanoPU','HARVESTNanoPU'] if '202' in key else ['RecoFakeHLTPU','HARVESTFakeHLTPU']) + \
(['Nano'] if 'Nano' in upgradeProperties[2017][key]['ScenToRun'] else [])

upgradeProperties[2026] = {
'2026D49' : {
Expand Down