Addressing ECAL local reconstruction on GPU issues

CondFormats/EcalObjects/BuildFile.xml

@@ -3,10 +3,13 @@
 <use name="DataFormats/EcalDetId"/>
 <use name="DataFormats/EcalDigi"/>
 <use name="DataFormats/Math"/>
+<use name="HeterogeneousCore/CUDACore"/>
+<use name="HeterogeneousCore/CUDAUtilities"/>
 <use name="boost"/>
 <use name="boost_serialization"/>
 <use name="rootmath"/>
 <use name="clhep"/>
+<use name="cuda"/>
 <export>
   <lib name="1"/>
 </export>

RecoLocalCalo/EcalRecAlgos/interface/EcalGainRatiosGPU.h → CondFormats/EcalObjects/interface/EcalGainRatiosGPU.h

@@ -1,7 +1,9 @@
-#ifndef RecoLocalCalo_EcalRecAlgos_interface_EcalGainRatiosGPU_h
-#define RecoLocalCalo_EcalRecAlgos_interface_EcalGainRatiosGPU_h
+#ifndef CondFormats_EcalObjects_interface_EcalGainRatiosGPU_h
+#define CondFormats_EcalObjects_interface_EcalGainRatiosGPU_h
 
 #include "CondFormats/EcalObjects/interface/EcalGainRatios.h"
+#include "FWCore/Utilities/interface/propagate_const_array.h"
+#include "HeterogeneousCore/CUDAUtilities/interface/device_unique_ptr.h"
 
 #ifndef __CUDACC__
 #include "HeterogeneousCore/CUDAUtilities/interface/HostAllocator.h"
@@ -11,8 +13,8 @@
 class EcalGainRatiosGPU {
 public:
   struct Product {
-    ~Product();
-    float *gain12Over6 = nullptr, *gain6Over1 = nullptr;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<float[]>> gain12Over6;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<float[]>> gain6Over1;
   };
 
 #ifndef __CUDACC__
@@ -40,4 +42,4 @@ class EcalGainRatiosGPU {
 #endif  // __CUDACC__
 };
 
-#endif  // RecoLocalCalo_EcalRecAlgos_interface_EcalGainRatiosGPU_h
+#endif  // CondFormats_EcalObjects_interface_EcalGainRatiosGPU_h

RecoLocalCalo/EcalRecAlgos/interface/EcalIntercalibConstantsGPU.h → CondFormats/EcalObjects/interface/EcalIntercalibConstantsGPU.h

@@ -1,7 +1,9 @@
-#ifndef RecoLocalCalo_EcalRecAlgos_interface_EcalIntercalibConstantsGPU_h
-#define RecoLocalCalo_EcalRecAlgos_interface_EcalIntercalibConstantsGPU_h
+#ifndef CondFormats_EcalObjects_interface_EcalIntercalibConstantsGPU_h
+#define CondFormats_EcalObjects_interface_EcalIntercalibConstantsGPU_h
 
 #include "CondFormats/EcalObjects/interface/EcalIntercalibConstants.h"
+#include "FWCore/Utilities/interface/propagate_const_array.h"
+#include "HeterogeneousCore/CUDAUtilities/interface/device_unique_ptr.h"
 
 #ifndef __CUDACC__
 #include "HeterogeneousCore/CUDAUtilities/interface/HostAllocator.h"
@@ -11,8 +13,7 @@
 class EcalIntercalibConstantsGPU {
 public:
   struct Product {
-    ~Product();
-    float* values = nullptr;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<float[]>> values;
   };
 
 #ifndef __CUDACC__
@@ -27,17 +28,17 @@ class EcalIntercalibConstantsGPU {
 
   // TODO: do this centrally
   // get offset for hashes. equals number of barrel items
-  uint32_t getOffset() const { return valuesEB_.size(); }
+  uint32_t getOffset() const { return offset_; }
 
   //
   static std::string name() { return std::string{"ecalIntercalibConstantsGPU"}; }
 
 private:
-  std::vector<float> const& valuesEB_;
-  std::vector<float> const& valuesEE_;
+  std::vector<float, cms::cuda::HostAllocator<float>> values_;
+  uint32_t offset_;
 
   cms::cuda::ESProduct<Product> product_;
 #endif  // __CUDACC__
 };
 
-#endif  // RecoLocalCalo_EcalRecAlgos_interface_EcalIntercalibConstantsGPU_h
+#endif  // CondFormats_EcalObjects_interface_EcalIntercalibConstantsGPU_h

CondFormats/EcalObjects/interface/EcalLaserAPDPNRatiosGPU.h

@@ -0,0 +1,54 @@
+#ifndef CondFormats_EcalObjects_interface_EcalLaserAPDPNRatiosGPU_h
+#define CondFormats_EcalObjects_interface_EcalLaserAPDPNRatiosGPU_h
+
+#include "CondFormats/EcalObjects/interface/EcalLaserAPDPNRatios.h"
+#include "FWCore/Utilities/interface/propagate_const_array.h"
+#include "HeterogeneousCore/CUDAUtilities/interface/device_unique_ptr.h"
+
+#ifndef __CUDACC__
+#include "HeterogeneousCore/CUDAUtilities/interface/HostAllocator.h"
+#include "HeterogeneousCore/CUDACore/interface/ESProduct.h"
+#endif  // __CUDACC__
+
+class EcalLaserAPDPNRatiosGPU {
+public:
+  struct Product {
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<float[]>> p1;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<float[]>> p2;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<float[]>> p3;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<edm::TimeValue_t[]>> t1;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<edm::TimeValue_t[]>> t2;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<edm::TimeValue_t[]>> t3;
+  };
+
+#ifndef __CUDACC__
+
+  //
+  EcalLaserAPDPNRatiosGPU(EcalLaserAPDPNRatios const &);
+
+  // will call dealloation for Product thru ~Product
+  ~EcalLaserAPDPNRatiosGPU() = default;
+
+  // get device pointers
+  Product const &getProduct(cudaStream_t) const;
+
+  //
+  static std::string name() { return std::string{"ecalLaserAPDPNRatiosGPU"}; }
+
+private:
+  // in the future, we need to arrange so to avoid this copy on the host
+  // store eb first then ee
+  std::vector<float, cms::cuda::HostAllocator<float>> p1_;
+  std::vector<float, cms::cuda::HostAllocator<float>> p2_;
+  std::vector<float, cms::cuda::HostAllocator<float>> p3_;
+
+  std::vector<edm::TimeValue_t, cms::cuda::HostAllocator<edm::TimeValue_t>> t1_;
+  std::vector<edm::TimeValue_t, cms::cuda::HostAllocator<edm::TimeValue_t>> t2_;
+  std::vector<edm::TimeValue_t, cms::cuda::HostAllocator<edm::TimeValue_t>> t3_;
+
+  cms::cuda::ESProduct<Product> product_;
+
+#endif  // __CUDACC__
+};
+
+#endif  // CondFormats_EcalObjects_interface_EcalLaserAPDPNRatiosGPU_h

RecoLocalCalo/EcalRecAlgos/interface/EcalLaserAPDPNRatiosRefGPU.h → CondFormats/EcalObjects/interface/EcalLaserAPDPNRatiosRefGPU.h

@@ -1,7 +1,9 @@
-#ifndef RecoLocalCalo_EcalRecAlgos_interface_EcalLaserAPDPNRatiosRefGPU_h
-#define RecoLocalCalo_EcalRecAlgos_interface_EcalLaserAPDPNRatiosRefGPU_h
+#ifndef CondFormats_EcalObjects_interface_EcalLaserAPDPNRatiosRefGPU_h
+#define CondFormats_EcalObjects_interface_EcalLaserAPDPNRatiosRefGPU_h
 
 #include "CondFormats/EcalObjects/interface/EcalLaserAPDPNRatiosRef.h"
+#include "FWCore/Utilities/interface/propagate_const_array.h"
+#include "HeterogeneousCore/CUDAUtilities/interface/device_unique_ptr.h"
 
 #ifndef __CUDACC__
 #include "HeterogeneousCore/CUDAUtilities/interface/HostAllocator.h"
@@ -11,8 +13,7 @@
 class EcalLaserAPDPNRatiosRefGPU {
 public:
   struct Product {
-    ~Product();
-    float* values = nullptr;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<float[]>> values;
   };
 
 #ifndef __CUDACC__
@@ -27,17 +28,17 @@ class EcalLaserAPDPNRatiosRefGPU {
 
   // TODO: do this centrally
   // get offset for hashes. equals number of barrel items
-  uint32_t getOffset() const { return valuesEB_.size(); }
+  uint32_t getOffset() const { return offset_; }
+  uint32_t offset_;
 
   //
   static std::string name() { return std::string{"ecalLaserAPDPNRatiosRefGPU"}; }
 
 private:
-  std::vector<float> const& valuesEB_;
-  std::vector<float> const& valuesEE_;
+  std::vector<float, cms::cuda::HostAllocator<float>> values_;
 
   cms::cuda::ESProduct<Product> product_;
 #endif  // __CUDACC__
 };
 
-#endif  // RecoLocalCalo_EcalRecAlgos_interface_EcalLaserAPDPNRatiosRefGPU_h
+#endif  // CondFormats_EcalObjects_interface_EcalLaserAPDPNRatiosRefGPU_h

RecoLocalCalo/EcalRecAlgos/interface/EcalLaserAlphasGPU.h → CondFormats/EcalObjects/interface/EcalLaserAlphasGPU.h

@@ -1,7 +1,9 @@
-#ifndef RecoLocalCalo_EcalRecAlgos_interface_EcalLaserAlphasGPU_h
-#define RecoLocalCalo_EcalRecAlgos_interface_EcalLaserAlphasGPU_h
+#ifndef CondFormats_EcalObjects_interface_EcalLaserAlphasGPU_h
+#define CondFormats_EcalObjects_interface_EcalLaserAlphasGPU_h
 
 #include "CondFormats/EcalObjects/interface/EcalLaserAlphas.h"
+#include "FWCore/Utilities/interface/propagate_const_array.h"
+#include "HeterogeneousCore/CUDAUtilities/interface/device_unique_ptr.h"
 
 #ifndef __CUDACC__
 #include "HeterogeneousCore/CUDAUtilities/interface/HostAllocator.h"
@@ -11,8 +13,7 @@
 class EcalLaserAlphasGPU {
 public:
   struct Product {
-    ~Product();
-    float* values = nullptr;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<float[]>> values;
   };
 
 #ifndef __CUDACC__
@@ -27,17 +28,17 @@ class EcalLaserAlphasGPU {
 
   // TODO: do this centrally
   // get offset for hashes. equals number of barrel items
-  uint32_t getOffset() const { return valuesEB_.size(); }
+  uint32_t getOffset() const { return offset_; }
 
   //
   static std::string name() { return std::string{"ecalLaserAlphasGPU"}; }
 
 private:
-  std::vector<float> const& valuesEB_;
-  std::vector<float> const& valuesEE_;
+  std::vector<float, cms::cuda::HostAllocator<float>> values_;
+  uint32_t offset_;
 
   cms::cuda::ESProduct<Product> product_;
 #endif  // __CUDACC__
 };
 
-#endif  // RecoLocalCalo_EcalRecAlgos_interface_EcalLaserAlphasGPU_h
+#endif  // CondFormats_EcalObjects_interface_EcalLaserAlphasGPU_h

RecoLocalCalo/EcalRecAlgos/interface/EcalLinearCorrectionsGPU.h → CondFormats/EcalObjects/interface/EcalLinearCorrectionsGPU.h

@@ -1,7 +1,9 @@
-#ifndef RecoLocalCalo_EcalRecAlgos_interface_EcalLinearCorrectionsGPU_h
-#define RecoLocalCalo_EcalRecAlgos_interface_EcalLinearCorrectionsGPU_h
+#ifndef CondFormats_EcalObjects_interface_EcalLinearCorrectionsGPU_h
+#define CondFormats_EcalObjects_interface_EcalLinearCorrectionsGPU_h
 
 #include "CondFormats/EcalObjects/interface/EcalLinearCorrections.h"
+#include "FWCore/Utilities/interface/propagate_const_array.h"
+#include "HeterogeneousCore/CUDAUtilities/interface/device_unique_ptr.h"
 
 #ifndef __CUDACC__
 #include "HeterogeneousCore/CUDAUtilities/interface/HostAllocator.h"
@@ -11,13 +13,12 @@
 class EcalLinearCorrectionsGPU {
 public:
   struct Product {
-    ~Product();
-    float *p1 = nullptr;
-    float *p2 = nullptr;
-    float *p3 = nullptr;
-    edm::TimeValue_t *t1 = nullptr;
-    edm::TimeValue_t *t2 = nullptr;
-    edm::TimeValue_t *t3 = nullptr;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<float[]>> p1;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<float[]>> p2;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<float[]>> p3;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<edm::TimeValue_t[]>> t1;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<edm::TimeValue_t[]>> t2;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<edm::TimeValue_t[]>> t3;
   };
 
 #ifndef __CUDACC__
@@ -50,4 +51,4 @@ class EcalLinearCorrectionsGPU {
 #endif  // __CUDACC__
 };
 
-#endif  // RecoLocalCalo_EcalRecAlgos_interface_EcalLinearCorrectionsGPU_h
+#endif  // CondFormats_EcalObjects_interface_EcalLinearCorrectionsGPU_h

RecoLocalCalo/EcalRecAlgos/interface/EcalMultifitParametersGPU.h → CondFormats/EcalObjects/interface/EcalMultifitParametersGPU.h

@@ -1,9 +1,11 @@
-#ifndef RecoLocalCalo_EcalRecAlgos_interface_EcalMultifitParametersGPU_h
-#define RecoLocalCalo_EcalRecAlgos_interface_EcalMultifitParametersGPU_h
+#ifndef CondFormats_EcalObjects_interface_EcalMultifitParametersGPU_h
+#define CondFormats_EcalObjects_interface_EcalMultifitParametersGPU_h
 
 #include <array>
 
 #include "FWCore/ParameterSet/interface/ParameterSet.h"
+#include "FWCore/Utilities/interface/propagate_const_array.h"
+#include "HeterogeneousCore/CUDAUtilities/interface/device_unique_ptr.h"
 
 #ifndef __CUDACC__
 #include "HeterogeneousCore/CUDAUtilities/interface/HostAllocator.h"
@@ -13,8 +15,10 @@
 class EcalMultifitParametersGPU {
 public:
   struct Product {
-    ~Product();
-    double *amplitudeFitParametersEB, *amplitudeFitParametersEE, *timeFitParametersEB, *timeFitParametersEE;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<double[]>> amplitudeFitParametersEB;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<double[]>> amplitudeFitParametersEE;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<double[]>> timeFitParametersEB;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<double[]>> timeFitParametersEE;
   };
 
 #ifndef __CUDACC__
@@ -36,4 +40,4 @@ class EcalMultifitParametersGPU {
 #endif  // __CUDACC__
 };
 
-#endif  // RecoLocalCalo_EcalRecAlgos_interface_EcalMultifitParametersGPU_h
+#endif  // CondFormats_EcalObjects_interface_EcalMultifitParametersGPU_h

RecoLocalCalo/EcalRecAlgos/interface/EcalPedestalsGPU.h → CondFormats/EcalObjects/interface/EcalPedestalsGPU.h

@@ -1,7 +1,9 @@
-#ifndef RecoLocalCalo_EcalRecAlgos_interface_EcalPedestalsGPU_h
-#define RecoLocalCalo_EcalRecAlgos_interface_EcalPedestalsGPU_h
+#ifndef CondFormats_EcalObjects_interface_EcalPedestalsGPU_h
+#define CondFormats_EcalObjects_interface_EcalPedestalsGPU_h
 
 #include "CondFormats/EcalObjects/interface/EcalPedestals.h"
+#include "FWCore/Utilities/interface/propagate_const_array.h"
+#include "HeterogeneousCore/CUDAUtilities/interface/device_unique_ptr.h"
 
 #ifndef __CUDACC__
 #include "HeterogeneousCore/CUDAUtilities/interface/HostAllocator.h"
@@ -11,9 +13,12 @@
 class EcalPedestalsGPU {
 public:
   struct Product {
-    ~Product();
-    float *mean_x12 = nullptr, *mean_x6 = nullptr, *mean_x1 = nullptr;
-    float *rms_x12 = nullptr, *rms_x6 = nullptr, *rms_x1 = nullptr;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<float[]>> mean_x12;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<float[]>> mean_x6;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<float[]>> mean_x1;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<float[]>> rms_x12;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<float[]>> rms_x6;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<float[]>> rms_x1;
   };
 
 #ifndef __CUDACC__
@@ -44,4 +49,4 @@ class EcalPedestalsGPU {
 #endif  // __CUDACC__
 };
 
-#endif  // RecoLocalCalo_EcalRecAlgos_interface_EcalPedestalsGPU_h
+#endif  // CondFormats_EcalObjects_interface_EcalPedestalsGPU_h

RecoLocalCalo/EcalRecAlgos/interface/EcalPulseCovariancesGPU.h → CondFormats/EcalObjects/interface/EcalPulseCovariancesGPU.h

@@ -1,5 +1,5 @@
-#ifndef RecoLocalCalo_EcalRecAlgos_interface_EcalPulseCovariancesGPU_h
-#define RecoLocalCalo_EcalRecAlgos_interface_EcalPulseCovariancesGPU_h
+#ifndef CondFormats_EcalObjects_interface_EcalPulseCovariancesGPU_h
+#define CondFormats_EcalObjects_interface_EcalPulseCovariancesGPU_h
 
 #include "CondFormats/EcalObjects/interface/EcalPulseCovariances.h"
 
@@ -37,4 +37,4 @@ class EcalPulseCovariancesGPU {
 #endif  // __CUDACC__
 };
 
-#endif  // RecoLocalCalo_EcalRecAlgos_interface_EcalPulseCovariancesGPU_h
+#endif  // CondFormats_EcalObjects_interface_EcalPulseCovariancesGPU_h

RecoLocalCalo/EcalRecAlgos/interface/EcalPulseShapesGPU.h → CondFormats/EcalObjects/interface/EcalPulseShapesGPU.h

@@ -1,5 +1,5 @@
-#ifndef RecoLocalCalo_EcalRecAlgos_interface_EcalPulseShapesGPU_h
-#define RecoLocalCalo_EcalRecAlgos_interface_EcalPulseShapesGPU_h
+#ifndef CondFormats_EcalObjects_interface_EcalPulseShapesGPU_h
+#define CondFormats_EcalObjects_interface_EcalPulseShapesGPU_h
 
 #include "CondFormats/EcalObjects/interface/EcalPulseShapes.h"
 
@@ -37,4 +37,4 @@ class EcalPulseShapesGPU {
 #endif  // __CUDACC__
 };
 
-#endif  // RecoLocalCalo_EcalRecAlgos_interface_EcalPulseShapesGPU_h
+#endif  // CondFormats_EcalObjects_interface_EcalPulseShapesGPU_h

RecoLocalCalo/EcalRecAlgos/interface/EcalRecHitParametersGPU.h → CondFormats/EcalObjects/interface/EcalRecHitParametersGPU.h

@@ -1,9 +1,11 @@
-#ifndef RecoLocalCalo_EcalRecAlgos_interface_EcalRecHitParametersGPU_h
-#define RecoLocalCalo_EcalRecAlgos_interface_EcalRecHitParametersGPU_h
+#ifndef CondFormats_EcalObjects_interface_EcalRecHitParametersGPU_h
+#define CondFormats_EcalObjects_interface_EcalRecHitParametersGPU_h
 
 #include <array>
 
 #include "FWCore/ParameterSet/interface/ParameterSet.h"
+#include "FWCore/Utilities/interface/propagate_const_array.h"
+#include "HeterogeneousCore/CUDAUtilities/interface/device_unique_ptr.h"
 
 #ifndef __CUDACC__
 #include "HeterogeneousCore/CUDAUtilities/interface/HostAllocator.h"
@@ -13,9 +15,10 @@
 class EcalRecHitParametersGPU {
 public:
   struct Product {
-    ~Product();
-    int *ChannelStatusToBeExcluded, *expanded_v_DB_reco_flags;
-    uint32_t *expanded_Sizes_v_DB_reco_flags, *expanded_flagbit_v_DB_reco_flags;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<int[]>> ChannelStatusToBeExcluded;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<int[]>> expanded_v_DB_reco_flags;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<uint32_t[]>> expanded_Sizes_v_DB_reco_flags;
+    edm::propagate_const_array<cms::cuda::device::unique_ptr<uint32_t[]>> expanded_flagbit_v_DB_reco_flags;
   };
 
 #ifndef __CUDACC__
@@ -44,4 +47,4 @@ class EcalRecHitParametersGPU {
 #endif  // __CUDACC__
 };
 
-#endif  // RecoLocalCalo_EcalRecAlgos_interface_EcalRecHitParametersGPU_h
+#endif  // CondFormats_EcalObjects_interface_EcalRecHitParametersGPU_h