Skip to content

Update GPU workflows #35331

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cmsbuild merged 4 commits into from
Sep 24, 2021
Merged

Update GPU workflows #35331

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e0a7781
Add support for the gpu modifier at HLT
fwyzard Aug 23, 2021
84155b6
Make the Patatrack workflows apply the 'gpu' modifier to the HLT step
fwyzard Sep 8, 2021
dd84d38
Simplify the GPU pixel local reconstruction
fwyzard Sep 15, 2021
f776cab
Add Patatrack workflows running the full reconstruction
fwyzard Sep 17, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 10 additions & 6 deletions Configuration/PyReleaseValidation/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,17 @@ The offsets currently in use are:
* 0.4: LowPU tracking era, `Run2_2017_trackingLowPU`
* 0.5: Pixel tracking only + 0.1
* 0.501: Patatrack, pixel only quadruplets, on CPU
* 0.502: Patatrack, pixel only quadruplets, on GPU
* 0.502: Patatrack, pixel only quadruplets, with automatic offload to GPU if available
* 0.505: Patatrack, pixel only triplets, on CPU
* 0.506: Patatrack, pixel only triplets, on GPU
* 0.511: Patatrack, ECAL only CPU
* 0.512: Patatrack, ECAL only GPU
* 0.521: Patatrack, HCAL only CPU
* 0.522: Patatrack, HCAL only GPU
* 0.506: Patatrack, pixel only triplets, with automatic offload to GPU if available
* 0.511: Patatrack, ECAL only, on CPU
* 0.512: Patatrack, ECAL only, with automatic offload to GPU if available
* 0.521: Patatrack, HCAL only, on CPU
* 0.522: Patatrack, HCAL only, with automatic offload to GPU if available
* 0.591: Patatrack, full reco with pixel quadruplets, on CPU
* 0.592: Patatrack, full reco with pixel quadruplets, with automatic offload to GPU if available
* 0.595: Patatrack, full reco with pixel triplets, on CPU
* 0.596: Patatrack, full reco with pixel triplets, with automatic offload to GPU if available
* 0.6: HE Collapse (old depth segmentation for 2018)
* 0.7: trackingMkFit modifier
* 0.8: BPH Parking (Run-2)
Expand Down
8 changes: 8 additions & 0 deletions Configuration/PyReleaseValidation/python/relval_gpu.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,25 +20,33 @@
# (Patatrack pixel-only triplets: TTbar - on GPU, both CPU and GPU, auto)
# (Patatrack ECAL-only: TTbar - on GPU, both CPU and GPU, auto)
# (Patatrack HCAL-only: TTbar - on GPU, both CPU and GPU, auto)
# (Patatrack full reco with pixel quadruplets: TTbar - on GPU, both CPU and GPU, auto)
# (Patatrack full reco with pixel triplets: TTbar - on GPU, both CPU and GPU, auto)
# mc 2021 (Patatrack pixel-only quadruplets: ZMM - on GPU, both CPU and GPU, auto)
# (Patatrack pixel-only triplets: ZMM - on GPU, both CPU and GPU, auto)
# (Patatrack pixel-only quadruplets: TTbar - on GPU, both CPU and GPU, auto)
# (Patatrack pixel-only triplets: TTbar - on GPU, both CPU and GPU, auto)
# (Patatrack ECAL-only: TTbar - on GPU, both CPU and GPU, auto)
# (Patatrack HCAL-only: TTbar - on GPU, both CPU and GPU, auto)
# (Patatrack full reco with pixel quadruplets: TTbar - on GPU, both CPU and GPU, auto)
# (Patatrack full reco with pixel triplets: TTbar - on GPU, both CPU and GPU, auto)
numWFIB = [
10842.502, # 10842.503,10842.504,
10842.506, # 10842.507,10842.508,
10824.502, # 10824.503,10824.504,
10824.506, # 10824.507,10824.508,
10824.512, # 10824.513,10824.514,
10824.522, # 10824.523,10824.524,
10824.592, # 10824.593,10824.594,
10824.596, # 10824.597,10824.598,
11650.502, # 11650.503,11650.504,
11650.506, # 11650.507,11650.508,
11634.502, # 11634.503,11634.504,
11634.506, # 11634.507,11634.508,
11634.512, # 11634.513,11634.514,
11634.522, # 11634.523,11634.524
11634.592, # 11634.593,11634.594,
11634.596, # 11634.597,11634.598,
]
for numWF in numWFIB:
if not numWF in _upgrade_workflows: continue
Expand Down
91 changes: 89 additions & 2 deletions Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -415,10 +415,12 @@ def condition(self, fragment, stepList, key, hasHarvest):
# - 2021 conditions, TTbar
# - 2021 conditions, Z->mumu,
class PatatrackWorkflow(UpgradeWorkflow):
def __init__(self, reco, harvest, **kwargs):
def __init__(self, digi = {}, reco = {}, harvest = {}, **kwargs):
# adapt the parameters for the UpgradeWorkflow init method
super(PatatrackWorkflow, self).__init__(
steps = [
'Digi',
'DigiTrigger',
'Reco',
'HARVEST',
'RecoFakeHLT',
Expand All @@ -428,6 +430,7 @@ def __init__(self, reco, harvest, **kwargs):
],
PU = [],
**kwargs)
self.__digi = digi
self.__reco = reco
self.__reco.update({
'--datatier': 'GEN-SIM-RECO,DQMIO',
Expand All @@ -452,13 +455,18 @@ def condition(self, fragment, stepList, key, hasHarvest):
return result

def setup_(self, step, stepName, stepDict, k, properties):
if 'Reco' in step:
if 'Digi' in step:
stepDict[stepName][k] = merge([self.__digi, stepDict[step][k]])
elif 'Reco' in step:
stepDict[stepName][k] = merge([self.__reco, stepDict[step][k]])
elif 'HARVEST' in step:
stepDict[stepName][k] = merge([self.__harvest, stepDict[step][k]])


upgradeWFs['PatatrackPixelOnlyCPU'] = PatatrackWorkflow(
digi = {
# there is no customisation for enabling the Patatrack pixel quadruplets running only on the CPU
},
reco = {
'-s': 'RAW2DIGI:RawToDigi_pixelOnly,RECO:reconstruction_pixelTrackingOnly,VALIDATION:@pixelTrackingOnlyValidation,DQM:@pixelTrackingOnlyDQM',
'--procModifiers': 'pixelNtupletFit'
Expand All @@ -471,6 +479,9 @@ def setup_(self, step, stepName, stepDict, k, properties):
)

upgradeWFs['PatatrackPixelOnlyGPU'] = PatatrackWorkflow(
digi = {
'--procModifiers': 'gpu'
},
reco = {
'-s': 'RAW2DIGI:RawToDigi_pixelOnly,RECO:reconstruction_pixelTrackingOnly,VALIDATION:@pixelTrackingOnlyValidation,DQM:@pixelTrackingOnlyDQM',
'--procModifiers': 'pixelNtupletFit,gpu'
Expand All @@ -483,6 +494,9 @@ def setup_(self, step, stepName, stepDict, k, properties):
)

upgradeWFs['PatatrackPixelOnlyTripletsCPU'] = PatatrackWorkflow(
digi = {
# there is no customisation for enabling the Patatrack pixel triplets running only on the CPU
},
reco = {
'-s': 'RAW2DIGI:RawToDigi_pixelOnly,RECO:reconstruction_pixelTrackingOnly,VALIDATION:@pixelTrackingOnlyValidation,DQM:@pixelTrackingOnlyDQM',
'--procModifiers': 'pixelNtupletFit',
Expand All @@ -496,6 +510,10 @@ def setup_(self, step, stepName, stepDict, k, properties):
)

upgradeWFs['PatatrackPixelOnlyTripletsGPU'] = PatatrackWorkflow(
digi = {
'--procModifiers': 'gpu',
'--customise': 'HLTrigger/Configuration/customizeHLTforPatatrack.enablePatatrackPixelTriplets'
},
reco = {
'-s': 'RAW2DIGI:RawToDigi_pixelOnly,RECO:reconstruction_pixelTrackingOnly,VALIDATION:@pixelTrackingOnlyValidation,DQM:@pixelTrackingOnlyDQM',
'--procModifiers': 'pixelNtupletFit,gpu',
Expand All @@ -520,6 +538,9 @@ def setup_(self, step, stepName, stepDict, k, properties):
)

upgradeWFs['PatatrackECALOnlyGPU'] = PatatrackWorkflow(
digi = {
'--procModifiers': 'gpu'
},
reco = {
'-s': 'RAW2DIGI:RawToDigi_ecalOnly,RECO:reconstruction_ecalOnly,VALIDATION:@ecalOnlyValidation,DQM:@ecalOnly',
'--procModifiers': 'gpu'
Expand All @@ -543,6 +564,9 @@ def setup_(self, step, stepName, stepDict, k, properties):
)

upgradeWFs['PatatrackHCALOnlyGPU'] = PatatrackWorkflow(
digi = {
'--procModifiers': 'gpu'
},
reco = {
'-s': 'RAW2DIGI:RawToDigi_hcalOnly,RECO:reconstruction_hcalOnly,VALIDATION:@hcalOnlyValidation,DQM:@hcalOnly+@hcal2Only',
'--procModifiers': 'gpu'
Expand All @@ -554,6 +578,69 @@ def setup_(self, step, stepName, stepDict, k, properties):
offset = 0.522,
)

upgradeWFs['PatatrackCPU'] = PatatrackWorkflow(
digi = {
# there is no customisation for enabling the Patatrack pixel quadruplets running only on the CPU
},
reco = {
# skip the @pixelTrackingOnlyValidation which cannot run together with the full reconstruction
Copy link
Contributor Author

@fwyzard fwyzard Sep 18, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

see https://hypernews.cern.ch/HyperNews/CMS/get/recoTracking/1939.html

Copy link
Contributor

@mmusich mmusich Sep 19, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@sroychow @arossi83 @tsusa @connorpa please take note of this issue triggered in the phase1 pixel validation code when using the pixelonly reconstruction (by the way if the issue is only in SiPixelPhase1HitsV I suppose one could cook temporarily a lighter ad hoc validation sequence, just removing that module).

'-s': 'RAW2DIGI:RawToDigi+RawToDigi_pixelOnly,L1Reco,RECO:reconstruction+reconstruction_pixelTrackingOnly,RECOSIM,EI,PAT,VALIDATION:@standardValidation+@miniAODValidation,DQM:@standardDQM+@ExtraHLT+@miniAODDQM+@pixelTrackingOnlyDQM',
'--procModifiers': 'pixelNtupletFit'
},
harvest = {
},
suffix = 'Patatrack_CPU',
offset = 0.591,
)

upgradeWFs['PatatrackGPU'] = PatatrackWorkflow(
digi = {
'--procModifiers': 'gpu'
},
reco = {
# skip the @pixelTrackingOnlyValidation which cannot run together with the full reconstruction
'-s': 'RAW2DIGI:RawToDigi+RawToDigi_pixelOnly,L1Reco,RECO:reconstruction+reconstruction_pixelTrackingOnly,RECOSIM,EI,PAT,VALIDATION:@standardValidation+@miniAODValidation,DQM:@standardDQM+@ExtraHLT+@miniAODDQM+@pixelTrackingOnlyDQM',
'--procModifiers': 'pixelNtupletFit,gpu'
},
harvest = {
},
suffix = 'Patatrack_GPU',
offset = 0.592,
)

upgradeWFs['PatatrackTripletsCPU'] = PatatrackWorkflow(
digi = {
# there is no customisation for enabling the Patatrack pixel triplets running only on the CPU
},
reco = {
# skip the @pixelTrackingOnlyValidation which cannot run together with the full reconstruction
'-s': 'RAW2DIGI:RawToDigi+RawToDigi_pixelOnly,L1Reco,RECO:reconstruction+reconstruction_pixelTrackingOnly,RECOSIM,EI,PAT,VALIDATION:@standardValidation+@miniAODValidation,DQM:@standardDQM+@ExtraHLT+@miniAODDQM+@pixelTrackingOnlyDQM',
'--procModifiers': 'pixelNtupletFit',
'--customise' : 'RecoPixelVertexing/Configuration/customizePixelTracksForTriplets.customizePixelTracksForTriplets'
},
harvest = {
},
suffix = 'Patatrack_TripletsCPU',
offset = 0.595,
)

upgradeWFs['PatatrackTripletsGPU'] = PatatrackWorkflow(
digi = {
'--procModifiers': 'gpu',
'--customise': 'HLTrigger/Configuration/customizeHLTforPatatrack.enablePatatrackPixelTriplets'
},
reco = {
# skip the @pixelTrackingOnlyValidation which cannot run together with the full reconstruction
'-s': 'RAW2DIGI:RawToDigi+RawToDigi_pixelOnly,L1Reco,RECO:reconstruction+reconstruction_pixelTrackingOnly,RECOSIM,EI,PAT,VALIDATION:@standardValidation+@miniAODValidation,DQM:@standardDQM+@ExtraHLT+@miniAODDQM+@pixelTrackingOnlyDQM',
'--procModifiers': 'pixelNtupletFit,gpu',
'--customise': 'RecoPixelVertexing/Configuration/customizePixelTracksForTriplets.customizePixelTracksForTriplets'
},
harvest = {
},
suffix = 'Patatrack_TripletsGPU',
offset = 0.596,
)

# end of Patatrack workflows

class UpgradeWorkflow_ProdLike(UpgradeWorkflow):
Expand Down
7 changes: 7 additions & 0 deletions HLTrigger/Configuration/python/customizeHLTforCMSSW.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import FWCore.ParameterSet.Config as cms

# modifiers
from Configuration.ProcessModifiers.gpu_cff import gpu

# helper fuctions
from HLTrigger.Configuration.common import *

Expand Down Expand Up @@ -132,6 +135,10 @@ def customiseFor2018Input(process):
# CMSSW version specific customizations
def customizeHLTforCMSSW(process, menuType="GRun"):

# if the gpu modifier is enabled, make the Pixel, ECAL and HCAL reconstruction offloadable to a GPU
from HLTrigger.Configuration.customizeHLTforPatatrack import customizeHLTforPatatrack
gpu.makeProcessModifier(customizeHLTforPatatrack).apply(process)

# add call to action function in proper order: newest last!
# process = customiseFor12718(process)

Expand Down
59 changes: 29 additions & 30 deletions HLTrigger/Configuration/python/customizeHLTforPatatrack.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,51 +131,50 @@ def customisePixelLocalReconstruction(process):
src = "hltSiPixelClustersCUDA"
)

# convert the pixel digis errors to the legacy format
from EventFilter.SiPixelRawToDigi.siPixelDigiErrorsFromSoA_cfi import siPixelDigiErrorsFromSoA as _siPixelDigiErrorsFromSoA
process.hltSiPixelDigiErrors = _siPixelDigiErrorsFromSoA.clone(
digiErrorSoASrc = "hltSiPixelDigiErrorsSoA",
UsePhase1 = True
)

# copy the pixel digis (except errors) and clusters to the host
from EventFilter.SiPixelRawToDigi.siPixelDigisSoAFromCUDA_cfi import siPixelDigisSoAFromCUDA as _siPixelDigisSoAFromCUDA
process.hltSiPixelDigisSoA = _siPixelDigisSoAFromCUDA.clone(
src = "hltSiPixelClustersCUDA"
)

# convert the pixel digis (except errors) and clusters to the legacy format
from RecoLocalTracker.SiPixelClusterizer.siPixelDigisClustersFromSoA_cfi import siPixelDigisClustersFromSoA as _siPixelDigisClustersFromSoA
process.hltSiPixelDigisClusters = _siPixelDigisClustersFromSoA.clone(
src = "hltSiPixelDigisSoA"
)
# reconstruct the pixel digis on the cpu
process.hltSiPixelDigisLegacy = process.hltSiPixelDigis.clone()

# SwitchProducer wrapping the legacy pixel digis producer or an alias combining the pixel digis information converted from SoA
# SwitchProducer wrapping a subset of the legacy pixel digis producer, or the conversion of the pixel digis errors to the legacy format
from EventFilter.SiPixelRawToDigi.siPixelDigiErrorsFromSoA_cfi import siPixelDigiErrorsFromSoA as _siPixelDigiErrorsFromSoA
process.hltSiPixelDigis = SwitchProducerCUDA(
# legacy producer
cpu = process.hltSiPixelDigis,
# alias used to access products from multiple conversion modules
cuda = cms.EDAlias(
hltSiPixelDigisClusters = cms.VPSet(
cms.PSet(type = cms.string("PixelDigiedmDetSetVector"))
),
hltSiPixelDigiErrors = cms.VPSet(
cpu = cms.EDAlias(
hltSiPixelDigisLegacy = cms.VPSet(
cms.PSet(type = cms.string("DetIdedmEDCollection")),
cms.PSet(type = cms.string("SiPixelRawDataErroredmDetSetVector")),
cms.PSet(type = cms.string("PixelFEDChanneledmNewDetSetVector"))
)
),
# conversion from SoA to legacy format
cuda = _siPixelDigiErrorsFromSoA.clone(
digiErrorSoASrc = "hltSiPixelDigiErrorsSoA",
UsePhase1 = True
)
)

# SwitchProducer wrapping the legacy pixel cluster producer or an alias for the pixel clusters information converted from SoA
# reconstruct the pixel clusters on the cpu
process.hltSiPixelClustersLegacy = process.hltSiPixelClusters.clone()

# SwitchProducer wrapping a subset of the legacy pixel cluster producer, or the conversion of the pixel digis (except errors) and clusters to the legacy format
from RecoLocalTracker.SiPixelClusterizer.siPixelDigisClustersFromSoA_cfi import siPixelDigisClustersFromSoA as _siPixelDigisClustersFromSoA
process.hltSiPixelClusters = SwitchProducerCUDA(
# legacy producer
cpu = process.hltSiPixelClusters,
# alias used to access products from multiple conversion modules
cuda = cms.EDAlias(
hltSiPixelDigisClusters = cms.VPSet(
cpu = cms.EDAlias(
hltSiPixelClustersLegacy = cms.VPSet(
cms.PSet(type = cms.string("SiPixelClusteredmNewDetSetVector"))
)
),
# conversion from SoA to legacy format
cuda = _siPixelDigisClustersFromSoA.clone(
src = "hltSiPixelDigisSoA",
produceDigis = False,
storeDigis = False,
)
)

Expand All @@ -191,7 +190,7 @@ def customisePixelLocalReconstruction(process):
process.hltSiPixelRecHits = SwitchProducerCUDA(
# legacy producer
cpu = process.hltSiPixelRecHits,
# converter to legacy format
# conversion from SoA to legacy format
cuda = _siPixelRecHitFromCUDA.clone(
pixelRecHitSrc = "hltSiPixelRecHitsCUDA",
src = "hltSiPixelClusters"
Expand All @@ -206,11 +205,11 @@ def customisePixelLocalReconstruction(process):
process.hltSiPixelClustersCUDA, # reconstruct the pixel digis and clusters on the gpu
process.hltSiPixelRecHitsCUDA, # reconstruct the pixel rechits on the gpu
process.hltSiPixelDigisSoA, # copy the pixel digis (except errors) and clusters to the host
process.hltSiPixelDigisClusters, # convert the pixel digis (except errors) and clusters to the legacy format
process.hltSiPixelDigiErrorsSoA, # copy the pixel digis errors to the host
process.hltSiPixelDigiErrors, # convert the pixel digis errors to the legacy format
process.hltSiPixelDigis, # SwitchProducer wrapping the legacy pixel digis producer or an alias combining the pixel digis information converted from SoA
process.hltSiPixelClusters, # SwitchProducer wrapping the legacy pixel cluster producer or an alias for the pixel clusters information converted from SoA
process.hltSiPixelDigisLegacy, # legacy pixel digis producer
process.hltSiPixelDigis, # SwitchProducer wrapping a subset of the legacy pixel digis producer, or the conversion of the pixel digis errors from SoA
process.hltSiPixelClustersLegacy, # legacy pixel cluster producer
process.hltSiPixelClusters, # SwitchProducer wrapping a subset of the legacy pixel cluster producer, or the conversion of the pixel digis (except errors) and clusters from SoA
process.hltSiPixelClustersCache, # legacy module, used by the legacy pixel quadruplet producer
process.hltSiPixelRecHits) # SwitchProducer wrapping the legacy pixel rechit producer or the transfer of the pixel rechits to the host and the conversion from SoA

Expand Down