Update ECAL and HCAL reconstruction to run on multple GPUs

CUDADataFormats/HcalCommon/interface/Common.h → CUDADataFormats/CaloCommon/interface/Common.h

@@ -1,18 +1,20 @@
-#ifndef CUDADataFormats_HcalCommon_interface_Common_h
-#define CUDADataFormats_HcalCommon_interface_Common_h
+#ifndef CUDADataFormats_CaloCommon_interface_Common_h
+#define CUDADataFormats_CaloCommon_interface_Common_h
 
 #include <vector>
 
 #include "HeterogeneousCore/CUDAUtilities/interface/HostAllocator.h"
+#include "HeterogeneousCore/CUDAUtilities/interface/device_unique_ptr.h"
 
-namespace hcal {
+namespace calo {
   namespace common {
 
     // FIXME: not able to get enums to work with genreflex
     namespace tags {
 
       struct Vec {};
       struct Ptr {};
+      struct DevPtr {};
 
     }  // namespace tags
 
@@ -24,6 +26,11 @@ namespace hcal {
       uint32_t size;
     };
 
+    template<>
+    struct AddSize<tags::DevPtr> {
+        uint32_t size;
+    };
+
     struct ViewStoragePolicy {
       using TagType = tags::Ptr;
 
@@ -33,6 +40,15 @@ namespace hcal {
       };
     };
 
+    struct DevStoragePolicy {
+      using TagType = tags::DevPtr;
+
+      template<typename T>
+      struct StorageSelector {
+          using type = cms::cuda::device::unique_ptr<T[]>;
+      };
+    };
+
     template <template <typename> typename Allocator = std::allocator>
     struct VecStoragePolicy {
       using TagType = tags::Vec;
@@ -42,16 +58,11 @@ namespace hcal {
         using type = std::vector<T, Allocator<T>>;
       };
     };
+
+    template <typename T>
+    using CUDAHostAllocatorAlias = cms::cuda::HostAllocator<T>;
 
   }  // namespace common
-}  // namespace hcal
-
-// FIXME: move into common namespace
-namespace hcal {
-
-  template <typename T>
-  using CUDAHostAllocatorAlias = cms::cuda::HostAllocator<T>;
-
-}
+}  // namespace calo
 
-#endif  // CUDADataFormats_HcalCommon_interface_Common_h
+#endif  // CUDADataFormats_CaloCommon_interface_Common_h

CUDADataFormats/CaloCommon/src/classes.h

@@ -0,0 +1,2 @@
+#include "DataFormats/Common/interface/Wrapper.h"
+#include "CUDADataFormats/CaloCommon/interface/Common.h"

CUDADataFormats/CaloCommon/src/classes_def.xml

@@ -0,0 +1,8 @@
+<lcgdict>
+    <enum name="calo::common::tags::Ptr"/>
+    <enum name="calo::common::tags::Vec"/>
+    <enum name="calo::common::tags::DevPtr"/>
+    <class name="calo::common::AddSize<calo::common::tags::Ptr>"/>
+    <class name="calo::common::AddSize<calo::common::tags::Vec>"/>
+    <class name="calo::common::AddSize<calo::common::tags::DevPtr>"/>
+</lcgdict>

CUDADataFormats/EcalDigi/BuildFile.xml

@@ -1,4 +1,5 @@
 <use name="CUDADataFormats/Common"/>
+<use name="CUDADataFormats/CaloCommon"/>
 <use name="DataFormats/Common"/>
 <use name="HeterogeneousCore/CUDAUtilities"/>
 

CUDADataFormats/EcalDigi/interface/DigisCollection.h

@@ -1,25 +1,22 @@
 #ifndef CUDADataFormats_EcalDigi_interface_DigisCollection_h
 #define CUDADataFormats_EcalDigi_interface_DigisCollection_h
 
+#include "CUDADataFormats/CaloCommon/interface/Common.h"
+
 namespace ecal {
 
-  //
-  // this is basically a view
-  // it does not own the actual memory -> does not reclaim
-  //
-  struct DigisCollection {
+  template<typename StoragePolicy>
+  struct DigisCollection : public ::calo::common::AddSize<typename StoragePolicy::TagType> {
     DigisCollection() = default;
-    DigisCollection(uint32_t *ids, uint16_t *data, uint32_t ndigis) : ids{ids}, data{data}, ndigis{ndigis} {}
     DigisCollection(DigisCollection const &) = default;
     DigisCollection &operator=(DigisCollection const &) = default;
 
     DigisCollection(DigisCollection &&) = default;
     DigisCollection &operator=(DigisCollection &&) = default;
 
     // stride is statically known
-    uint32_t *ids = nullptr;
-    uint16_t *data = nullptr;
-    uint32_t ndigis;
+    typename StoragePolicy::template StorageSelector<uint32_t>::type ids;
+    typename StoragePolicy::template StorageSelector<uint16_t>::type data;
   };
 
 }  // namespace ecal

CUDADataFormats/EcalDigi/src/classes_def.xml

@@ -1,4 +1,6 @@
 <lcgdict>
-    <class name="cms::cuda::Product<ecal::DigisCollection>" persistent="false" />
-    <class name="edm::Wrapper<cms::cuda::Product<ecal::DigisCollection>>" persistent="false"/>
+    <class name="cms::cuda::Product<ecal::DigisCollection<calo::common::ViewStoragePolicy>>" persistent="false" />
+    <class name="cms::cuda::Product<ecal::DigisCollection<calo::common::DevStoragePolicy>>" persistent="false" />
+    <class name="edm::Wrapper<cms::cuda::Product<ecal::DigisCollection<calo::common::ViewStoragePolicy>>>" persistent="false"/>
+    <class name="edm::Wrapper<cms::cuda::Product<ecal::DigisCollection<calo::common::DevStoragePolicy>>>" persistent="false"/>
 </lcgdict>

CUDADataFormats/EcalRecHitSoA/BuildFile.xml

@@ -1,5 +1,6 @@
 <use name="cuda"/>
 <use name="CUDADataFormats/Common"/>
+<use name="CUDADataFormats/CaloCommon"/>
 <use name="DataFormats/Common"/>
 <use name="DataFormats/EcalDigi"/>
 <use name="HeterogeneousCore/CUDAUtilities"/>

CUDADataFormats/EcalRecHitSoA/interface/EcalRecHit.h

@@ -0,0 +1,45 @@
+#ifndef CUDADataFormats_EcalRecHitSoA_interface_EcalRecHit_soa_h
+#define CUDADataFormats_EcalRecHitSoA_interface_EcalRecHit_soa_h
+
+#include <vector>
+#include <array>
+
+#include "CUDADataFormats/EcalRecHitSoA/interface/RecoTypes.h"
+#include "HeterogeneousCore/CUDAUtilities/interface/HostAllocator.h"
+#include "CUDADataFormats/CaloCommon/interface/Common.h"
+
+namespace ecal {
+
+  template <typename StoragePolicy>
+  struct RecHit : public ::calo::common::AddSize<typename StoragePolicy::TagType> {
+    RecHit() = default;
+    RecHit(const RecHit&) = default;
+    RecHit& operator=(const RecHit&) = default;
+
+    RecHit(RecHit&&) = default;
+    RecHit& operator=(RecHit&&) = default;
+
+    typename StoragePolicy::template StorageSelector<reco::StorageScalarType>::type energy;
+    typename StoragePolicy::template StorageSelector<reco::StorageScalarType>::type time;
+    // should we remove the following, since already included in "extra" ?
+    typename StoragePolicy::template StorageSelector<reco::StorageScalarType>::type chi2;
+    typename StoragePolicy::template StorageSelector<uint32_t>::type extra;     // packed uint32_t for timeError, chi2, energyError
+    typename StoragePolicy::template StorageSelector<uint32_t>::type flagBits;  // store rechit condition (see Flags enum) in a bit-wise way
+
+    typename StoragePolicy::template StorageSelector<uint32_t>::type did;
+
+    template <typename U = typename StoragePolicy::TagType>
+    typename std::enable_if<std::is_same<U, ::calo::common::tags::Vec>::value, void>::type resize(size_t size) {
+      energy.resize(size);
+      time.resize(size);
+      chi2.resize(size);
+      extra.resize(size);
+      flagBits.resize(size);
+      did.resize(size);
+    }
+  };
+
+}  // namespace ecal
+
+#endif
+// RecoLocalCalo_EcalRecAlgos_interface_EcalRecHit_soa_h

CUDADataFormats/EcalRecHitSoA/interface/EcalUncalibratedRecHit.h

@@ -0,0 +1,47 @@
+#ifndef CUDADataFormats_EcalRecHitSoA_interface_EcalUncalibratedRecHit_h
+#define CUDADataFormats_EcalRecHitSoA_interface_EcalUncalibratedRecHit_h
+
+#include <vector>
+#include <array>
+
+#include "DataFormats/EcalDigi/interface/EcalDataFrame.h"
+
+#include "CUDADataFormats/CaloCommon/interface/Common.h"
+#include "CUDADataFormats/EcalRecHitSoA/interface/RecoTypes.h"
+
+namespace ecal {
+
+  template <typename StoragePolicy>
+  struct UncalibratedRecHit : public ::calo::common::AddSize<typename StoragePolicy::TagType> {
+    UncalibratedRecHit() = default;
+    UncalibratedRecHit(const UncalibratedRecHit&) = default;
+    UncalibratedRecHit& operator=(const UncalibratedRecHit&) = default;
+
+    UncalibratedRecHit(UncalibratedRecHit&&) = default;
+    UncalibratedRecHit& operator=(UncalibratedRecHit&&) = default;
+
+    typename StoragePolicy::template StorageSelector<reco::ComputationScalarType>::type amplitudesAll;
+    typename StoragePolicy::template StorageSelector<reco::StorageScalarType>::type amplitude;
+    typename StoragePolicy::template StorageSelector<reco::StorageScalarType>::type chi2;
+    typename StoragePolicy::template StorageSelector<reco::StorageScalarType>::type pedestal;
+    typename StoragePolicy::template StorageSelector<reco::StorageScalarType>::type jitter;
+    typename StoragePolicy::template StorageSelector<reco::StorageScalarType>::type jitterError;
+    typename StoragePolicy::template StorageSelector<uint32_t>::type did;
+    typename StoragePolicy::template StorageSelector<uint32_t>::type flags;
+
+    template <typename U = typename StoragePolicy::TagType>
+    typename std::enable_if<std::is_same<U, ::calo::common::tags::Vec>::value, void>::type resize(size_t size) {
+      amplitudesAll.resize(size * EcalDataFrame::MAXSAMPLES);
+      amplitude.resize(size);
+      pedestal.resize(size);
+      chi2.resize(size);
+      did.resize(size);
+      flags.resize(size);
+      jitter.resize(size);
+      jitterError.resize(size);
+    }
+  };
+
+}  // namespace ecal
+
+#endif  // RecoLocalCalo_EcalRecAlgos_interface_EcalUncalibratedRecHit_h

CUDADataFormats/EcalRecHitSoA/src/classes.h

@@ -1,4 +1,4 @@
 #include "DataFormats/Common/interface/Wrapper.h"
 #include "CUDADataFormats/Common/interface/Product.h"
-#include "CUDADataFormats/EcalRecHitSoA/interface/EcalUncalibratedRecHit_soa.h"
-#include "CUDADataFormats/EcalRecHitSoA/interface/EcalRecHit_soa.h"
+#include "CUDADataFormats/EcalRecHitSoA/interface/EcalUncalibratedRecHit.h"
+#include "CUDADataFormats/EcalRecHitSoA/interface/EcalRecHit.h"