CUDAService verbosity

Configuration/StandardSequences/python/Services_cff.py

@@ -10,7 +10,9 @@
 
 # load CUDA services when the "gpu" or "pixelNtupletFit" modifiers are enabled
 def _addCUDAServices(process):
-     process.load("HeterogeneousCore.CUDAServices.CUDAService_cfi")
+    process.load("HeterogeneousCore.CUDAServices.CUDAService_cfi")
+    process.load("FWCore.MessageService.MessageLogger_cfi")
+    process.MessageLogger.CUDAService = cms.untracked.PSet()
 
 from Configuration.ProcessModifiers.gpu_cff import gpu
 from Configuration.ProcessModifiers.pixelNtupletFit_cff import pixelNtupletFit

HLTrigger/Configuration/python/customizeHLTforPatatrack.py

@@ -24,8 +24,10 @@ def customiseCommon(process):
     # Services
 
     process.load("HeterogeneousCore.CUDAServices.CUDAService_cfi")
+    if 'MessageLogger' in process.__dict__:
+        process.MessageLogger.CUDAService = cms.untracked.PSet()
 
-    # NVProfilerService is broken in CMSSW 12.0,x and later
+    # NVProfilerService is broken in CMSSW 12.0.x and later
     #process.load("HeterogeneousCore.CUDAServices.NVProfilerService_cfi")
 
 

HeterogeneousCore/CUDAServices/BuildFile.xml

@@ -5,6 +5,7 @@
   <use name="FWCore/Utilities"/>
   <use name="HeterogeneousCore/CUDAUtilities"/>
   <use name="cuda"/>
+  <use name="cuda-nvml"/>
   <export>
     <lib name="1"/>
   </export>

HeterogeneousCore/CUDAServices/interface/CUDAService.h

@@ -33,6 +33,7 @@ class CUDAService {
   int numberOfDevices_ = 0;
   std::vector<std::pair<int, int>> computeCapabilities_;
   bool enabled_ = false;
+  bool verbose_ = false;
 };
 
 #endif

HeterogeneousCore/CUDAServices/test/testCUDAService.py

@@ -2,9 +2,16 @@
 
 process = cms.Process( "TEST" )
 
+process.options = cms.untracked.PSet(
+    numberOfThreads = cms.untracked.uint32( 4 ),
+    numberOfStreams = cms.untracked.uint32( 0 ),
+)
+
 process.load('FWCore.MessageService.MessageLogger_cfi')
+process.MessageLogger.CUDAService = {}
+
 process.load('HeterogeneousCore.CUDAServices.CUDAService_cfi')
-process.MessageLogger.CUDAService=dict()
+process.CUDAService.verbose = True
 
 process.source = cms.Source("EmptySource")
 

HeterogeneousCore/CUDAUtilities/BuildFile.xml

@@ -1,5 +1,6 @@
 <iftool name="cuda">
   <use name="cuda"/>
+  <use name="cuda-nvml"/>
   <use name="eigen"/>
   <use name="FWCore/Utilities"/>
   <use name="FWCore/MessageLogger"/>

HeterogeneousCore/CUDAUtilities/interface/nvmlCheck.h

@@ -0,0 +1,54 @@
+#ifndef HeterogeneousCore_CUDAUtilities_nvmlCheck_h
+#define HeterogeneousCore_CUDAUtilities_nvmlCheck_h
+
+// C++ standard headers
+#include <iostream>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+
+// CUDA headers
+#include <nvml.h>
+
+// CMSSW headers
+#include "FWCore/Utilities/interface/Likely.h"
+
+namespace cms {
+  namespace cuda {
+
+    [[noreturn]] inline void abortOnNvmlError(const char* file,
+                                              int line,
+                                              const char* cmd,
+                                              const char* error,
+                                              const char* message,
+                                              std::string_view description = std::string_view()) {
+      std::ostringstream out;
+      out << "\n";
+      out << file << ", line " << line << ":\n";
+      out << "nvmlCheck(" << cmd << ");\n";
+      out << error << ": " << message << "\n";
+      if (!description.empty())
+        out << description << "\n";
+      throw std::runtime_error(out.str());
+    }
+
+    inline bool nvmlCheck_(const char* file,
+                           int line,
+                           const char* cmd,
+                           nvmlReturn_t result,
+                           std::string_view description = std::string_view()) {
+      if (LIKELY(result == NVML_SUCCESS))
+        return true;
+
+      std::string error = "NVML Error " + std::to_string(result);
+      const char* message = nvmlErrorString(result);
+      abortOnNvmlError(file, line, cmd, error.c_str(), message, description);
+      return false;
+    }
+  }  // namespace cuda
+}  // namespace cms
+
+#define nvmlCheck(ARG, ...) (cms::cuda::nvmlCheck_(__FILE__, __LINE__, #ARG, (ARG), ##__VA_ARGS__))
+
+#endif  // HeterogeneousCore_CUDAUtilities_nvmlCheck_h