diff --git a/Configuration/Applications/python/ConfigBuilder.py b/Configuration/Applications/python/ConfigBuilder.py index 7b8ab3a2d04b3..433221a6f10d6 100644 --- a/Configuration/Applications/python/ConfigBuilder.py +++ b/Configuration/Applications/python/ConfigBuilder.py @@ -76,6 +76,7 @@ class Options: defaultOptions.profile = None defaultOptions.heap_profile = None defaultOptions.maxmem_profile = None +defaultOptions.alloc_monitor = None defaultOptions.isRepacked = False defaultOptions.restoreRNDSeeds = False defaultOptions.donotDropOnInput = '' diff --git a/Configuration/Applications/python/Options.py b/Configuration/Applications/python/Options.py index 65cfe9f34dac3..9930f2104bf10 100644 --- a/Configuration/Applications/python/Options.py +++ b/Configuration/Applications/python/Options.py @@ -440,6 +440,12 @@ action="store_true", dest="maxmem_profile") +expertSettings.add_argument("--alloc_monitor", + help="Add necessary LD_PRELOAD for PerfTools/AllocMonitor", + default=False, + action="store_true", + dest="alloc_monitor") + expertSettings.add_argument("--io", help="Create a json file with io informations", default=None, diff --git a/Configuration/Applications/python/cmsDriverOptions.py b/Configuration/Applications/python/cmsDriverOptions.py index 2e274c45865af..99deeadca50cd 100755 --- a/Configuration/Applications/python/cmsDriverOptions.py +++ b/Configuration/Applications/python/cmsDriverOptions.py @@ -258,6 +258,11 @@ def OptionsFromItems(items): raise Exception("--maxmem_profile and --prefix are incompatible") options.prefix = "env LD_PRELOAD=libPerfToolsAllocMonitorPreload.so:libPerfToolsMaxMemoryPreload.so " + if options.alloc_monitor: + if options.prefix: + raise Exception("--alloc_monitor and --prefix are incompatible") + options.prefix = "env LD_PRELOAD=libPerfToolsAllocMonitorPreload.so" + # If an "era" argument was supplied make sure it is one of the valid possibilities if options.era : from Configuration.StandardSequences.Eras import eras diff --git a/PerfTools/AllocMonitor/python/ModuleAllocMonitor.py b/PerfTools/AllocMonitor/python/ModuleAllocMonitor.py new file mode 100644 index 0000000000000..ad58318a1bdfb --- /dev/null +++ b/PerfTools/AllocMonitor/python/ModuleAllocMonitor.py @@ -0,0 +1,6 @@ +import FWCore.ParameterSet.Config as cms +def customise(process): + process.ModuleAllocMonitor = cms.Service("ModuleAllocMonitor", + fileName=cms.untracked.string("moduleAllocMonitor.log") + ) + return(process) diff --git a/PerfTools/AllocMonitor/scripts/edmModuleAllocJsonToCircles.py b/PerfTools/AllocMonitor/scripts/edmModuleAllocJsonToCircles.py new file mode 100755 index 0000000000000..03be4bb31f65a --- /dev/null +++ b/PerfTools/AllocMonitor/scripts/edmModuleAllocJsonToCircles.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +import json +transitionTypes = [ + "construction", + "begin job", + "begin stream", + "global begin run", + "stream begin run", + "global begin luminosity block", + "stream begin luminosity block", + "event", +] +allocTypes = ["added", "nAlloc", "nDealloc", "maxTemp", "max1Alloc"] + +def processModuleTransition(moduleLabel, moduleType, moduleInfo, transitionType, moduleTransition): + moduleTransition[moduleLabel] = {"cpptype": moduleType, "allocs": []} + for entry in moduleInfo: + if entry["transition"] == transitionType: + moduleTransition[moduleLabel]["allocs"].append(entry.get("alloc",{})) + moduleTransition[moduleLabel]["nTransitions"] = len(moduleTransition[moduleLabel]["allocs"]) + +def formatToCircles(moduleTransitions): + modules_dict = {} + doc = { + "modules": [], + "resources": [], + "total": {} + } + for transitionType in transitionTypes: + doc["resources"] += [ + { + "name": f"added {transitionType}", + "description": f"{transitionType}: added memory (average)", + "title": f"{transitionType}: Amount of memory added to the process at the end of the transition", + "unit": "kB" + }, + { + + "name": f"nAlloc {transitionType}", + "description": f"{transitionType}: num allocs (average)", + "title": f"{transitionType}: Number of allocations during the transition", + "unit": "" + }, + { + "name": f"nDealloc {transitionType}", + "description": f"{transitionType}: num deallocs (average)", + "title": f"{transitionType}: Number of deallocations during the transition", + "unit": "" + }, + { + "name": f"maxTemp {transitionType}", + "description": f"{transitionType}: maximum temporary memory (average)", + "title": f"{transitionType}: Maximum temporary memory during the transition", + "unit": "kB" + }, + { + "name": f"max1Alloc {transitionType}", + "description": f"{transitionType}: largest single allocation (average)", + "title": f"{transitionType}: Largest single allocation during the transition", + "unit": "kB" + }, + ] + # The circles code uses the "events" field to normalize the values between files with different number of events + # Here we set it to 1 for the total events because the total is already normalized per transition + doc["total"]["events"] = 1 + doc["total"]["label"] = "Job" + doc["total"]["type"] = "Job" + for allocType in allocTypes: + doc["total"][f"{allocType} {transitionType}"] = 0 + + for transitionType, moduleTransition in moduleTransitions.items(): + for label, info in moduleTransition.items(): + allocs = info.get("allocs", []) + if not label in modules_dict: + modules_dict[label] = { + "label": info.get("label", label), + "type": info.get("cpptype", "unknown") + } + added = 0 + nAlloc = 0 + nDealloc = 0 + maxTemp = 0 + max1Alloc = 0 + for alloc in allocs: + added += alloc.get("added", 0) + nAlloc += alloc.get("nAlloc", 0) + nDealloc += alloc.get("nDealloc", 0) + maxTemp += alloc.get("maxTemp", 0) + max1Alloc += alloc.get("max1Alloc", 0) + ntransitions = moduleTransitions[transitionType][label]["nTransitions"] + if ntransitions > 0: + modules_dict[label][f"nAlloc {transitionType}"] = nAlloc/ntransitions + modules_dict[label][f"added {transitionType}"] = (added/ntransitions)/1024 + modules_dict[label][f"maxTemp {transitionType}"] = (maxTemp/ntransitions)/1024 + modules_dict[label][f"nDealloc {transitionType}"] = nDealloc/ntransitions + modules_dict[label][f"max1Alloc {transitionType}"] = (max1Alloc/ntransitions)/1024 + else: + modules_dict[label][f"nAlloc {transitionType}"] = nAlloc + modules_dict[label][f"added {transitionType}"] = (added)/1024 + modules_dict[label][f"maxTemp {transitionType}"] = (maxTemp)/1024 + modules_dict[label][f"nDealloc {transitionType}"] = nDealloc + modules_dict[label][f"max1Alloc {transitionType}"] = max1Alloc/1024 + doc["total"][f"nAlloc {transitionType}"] += modules_dict[label][f"nAlloc {transitionType}"] + doc["total"][f"nDealloc {transitionType}"] += modules_dict[label][f"nDealloc {transitionType}"] + doc["total"][f"maxTemp {transitionType}"] += modules_dict[label][f"maxTemp {transitionType}"] + doc["total"][f"added {transitionType}"] += modules_dict[label][f"added {transitionType}"] + doc["total"][f"max1Alloc {transitionType}"] += modules_dict[label][f"max1Alloc {transitionType}"] + + for key in sorted(modules_dict.keys()): + module = modules_dict[key] + module["events"] = moduleTransitions['event'][key].get("nTransitions") + doc["modules"].append(module) + + return doc + +def main(args): + import sys + doc = json.load(args.filename) + moduleTypes = doc['cpptypes'] + moduleTransitions = dict() + for transition in transitionTypes: + moduleTransition = dict() + processModuleTransition("source", "PoolSource", doc["source"], transition, moduleTransition) + for moduleLabel, moduleInfo in doc["modules"].items(): + processModuleTransition(moduleLabel, moduleTypes[moduleLabel], moduleInfo, transition, moduleTransition) + moduleTransitions[transition] = moduleTransition + + json.dump(formatToCircles(moduleTransitions), sys.stdout, indent=2) + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description='Convert the JSON output of edmModuleAllocMonitorAnalyze.py to JSON for Circles') + parser.add_argument('filename', + type=argparse.FileType('r'), # open file + help='file to process') + args = parser.parse_args() + main(args) diff --git a/PerfTools/AllocMonitor/scripts/edmModuleAllocMonitorAnalyze.py b/PerfTools/AllocMonitor/scripts/edmModuleAllocMonitorAnalyze.py index e75cc2a0ea78a..20be19951bded 100755 --- a/PerfTools/AllocMonitor/scripts/edmModuleAllocMonitorAnalyze.py +++ b/PerfTools/AllocMonitor/scripts/edmModuleAllocMonitorAnalyze.py @@ -620,9 +620,9 @@ def jsonInfo(self, syncs, temp, data): start = temp.findTime("source", self.transition, self.index) #we do not know the sync yet so have to wait until the framework transition if self.transition in [ Phase.construction, Phase.getNextTransition, Phase.destruction, Phase.openFile]: - data.insert( "source" , "sourceType", start, self.time, self.transition, self.index, (0,) , Activity.process, self.allocInfo) + data.insert( "source" , "PoolSource", start, self.time, self.transition, self.index, (0,) , Activity.process, self.allocInfo) else: - data.insert( "source" , "sourceType", start, self.time, self.transition, self.index, self.index , Activity.process, self.allocInfo) + data.insert( "source" , "PoolSource", start, self.time, self.transition, self.index, self.index , Activity.process, self.allocInfo) def jsonVisInfo(self, data): index = self.index if self.transition == Phase.Event: @@ -812,7 +812,7 @@ def jsonVisInfo(self, data): return self._postJsonVis(data, self.allocInfo) def jsonInfo(self, syncs, temp, data): start = temp.findTime(self.moduleInfo._name+'source', self.transition, self.index) - data.insert( "source" , "sourceType", start, self.time, self.transition, self.index, syncs.get(self.transition, self.index) , Activity.delayedGet, self.allocInfo) + data.insert( "source" , "PoolSource", start, self.time, self.transition, self.index, syncs.get(self.transition, self.index) , Activity.delayedGet, self.allocInfo) class ESModuleTransitionParser(object): def __init__(self, payload, moduleInfos, esModuleInfos, recordNames):