Integration of alpaka and macro-based SoA in CMSSW

DataFormats/Portable/BuildFile.xml

@@ -0,0 +1,3 @@
+<use name="alpaka"/>
+<use name="DataFormats/SoATemplate" source_only="1"/>
+<use name="HeterogeneousCore/AlpakaInterface" source_only="1"/>

DataFormats/Portable/README.md

@@ -0,0 +1,38 @@
+## Define portable data formats that wrap SoA data structures and can be persisted to ROOT files
+
+### `PortableHostCollection<T>`
+
+`PortableHostCollection<T>` is a class template that wraps a SoA type `T` and an alpaka host buffer, which owns the
+memory where the SoA is allocated. The content of the SoA is persistent, while the buffer itself is transient.
+Specialisations of this template can be persisted, and can be read back also in "bare ROOT" mode, without any
+dictionaries.
+They have no implicit or explicit references to alpaka (neither as part of the class signature nor as part of its name).
+This could make it possible to read them back with different portability solutions in the future.
+
+### `PortableDeviceCollection<T, TDev>`
+
+`PortableDeviceCollection<T, TDev>` is a class template that wraps a SoA type `T` and an alpaka device buffer, which
+owns the memory where the SoA is allocated.
+To avoid confusion and ODR-violations, the `PortableDeviceCollection<T, TDev>` template cannot be used with the `Host`
+device type.
+Specialisations of this template are transient and cannot be persisted.
+
+### `ALPAKA_ACCELERATOR_NAMESPACE::PortableCollection<T>`
+
+`ALPAKA_ACCELERATOR_NAMESPACE::PortableCollection<T>` is a template alias that resolves to either
+`PortableHostCollection<T>` or `PortableDeviceCollection<T, ALPAKA_ACCELERATOR_NAMESPACE::Device>`, depending on the
+backend.
+
+### `PortableCollection<T, TDev>`
+
+`PortableCollection<T, TDev>` is an alias template that resolves to `ALPAKA_ACCELERATOR_NAMESPACE::PortableCollection<T>`
+for the matching device.
+
+
+## Notes
+
+Modules that are supposed to work with only host types (_e.g._ dealing with de/serialisation, data transfers, _etc._)
+should explicitly use the `PortableHostCollection<T>` types.
+
+Modules that implement portable interfaces (_e.g._ producers) should use the generic types based on
+`ALPAKA_ACCELERATOR_NAMESPACE::PortableCollection<T>` or `PortableCollection<T, TDev>`.

DataFormats/Portable/interface/PortableCollection.h

@@ -0,0 +1,16 @@
+#ifndef DataFormats_Portable_interface_PortableCollection_h
+#define DataFormats_Portable_interface_PortableCollection_h
+
+namespace traits {
+
+  // trait for a generic SoA-based product
+  template <typename T, typename TDev>
+  class PortableCollectionTrait;
+
+}  // namespace traits
+
+// type alias for a generic SoA-based product
+template <typename T, typename TDev>
+using PortableCollection = typename traits::PortableCollectionTrait<T, TDev>::CollectionType;
+
+#endif  // DataFormats_Portable_interface_PortableCollection_h

DataFormats/Portable/interface/PortableDeviceCollection.h

@@ -0,0 +1,65 @@
+#ifndef DataFormats_Portable_interface_PortableDeviceCollection_h
+#define DataFormats_Portable_interface_PortableDeviceCollection_h
+
+#include <optional>
+#include <type_traits>
+
+#include <alpaka/alpaka.hpp>
+
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
+
+// generic SoA-based product in device memory
+template <typename T, typename TDev>
+class PortableDeviceCollection {
+  static_assert(not std::is_same_v<TDev, alpaka_common::DevHost>,
+                "Use PortableHostCollection<T> instead of PortableDeviceCollection<T, DevHost>");
+
+public:
+  using Layout = T;
+  using View = typename Layout::View;
+  using ConstView = typename Layout::ConstView;
+  using Buffer = alpaka::Buf<TDev, std::byte, alpaka::DimInt<1u>, uint32_t>;
+  using ConstBuffer = alpaka::ViewConst<Buffer>;
+
+  PortableDeviceCollection() = default;
+
+  PortableDeviceCollection(int32_t elements, TDev const &device)
+      : buffer_{alpaka::allocBuf<std::byte, uint32_t>(
+            device, alpaka::Vec<alpaka::DimInt<1u>, uint32_t>{Layout::computeDataSize(elements)})},
+        layout_{buffer_->data(), elements},
+        view_{layout_} {
+    // Alpaka set to a default alignment of 128 bytes defining ALPAKA_DEFAULT_HOST_MEMORY_ALIGNMENT=128
+    assert(reinterpret_cast<uintptr_t>(buffer_->data()) % Layout::alignment == 0);
+  }
+
+  ~PortableDeviceCollection() = default;
+
+  // non-copyable
+  PortableDeviceCollection(PortableDeviceCollection const &) = delete;
+  PortableDeviceCollection &operator=(PortableDeviceCollection const &) = delete;
+
+  // movable
+  PortableDeviceCollection(PortableDeviceCollection &&other) = default;
+  PortableDeviceCollection &operator=(PortableDeviceCollection &&other) = default;
+
+  View &view() { return view_; }
+  ConstView const &view() const { return view_; }
+  ConstView const &const_view() const { return view_; }
+
+  View &operator*() { return view_; }
+  ConstView const &operator*() const { return view_; }
+
+  View *operator->() { return &view_; }
+  ConstView const *operator->() const { return &view_; }
+
+  Buffer buffer() { return *buffer_; }
+  ConstBuffer buffer() const { return *buffer_; }
+  ConstBuffer const_buffer() const { return *buffer_; }
+
+private:
+  std::optional<Buffer> buffer_;  //!
+  Layout layout_;                 //
+  View view_;                     //!
+};
+
+#endif  // DataFormats_Portable_interface_PortableDeviceCollection_h

DataFormats/Portable/interface/PortableHostCollection.h

@@ -0,0 +1,83 @@
+#ifndef DataFormats_Portable_interface_PortableHostCollection_h
+#define DataFormats_Portable_interface_PortableHostCollection_h
+
+#include <optional>
+
+#include <alpaka/alpaka.hpp>
+
+#include "DataFormats/SoATemplate/interface/SoACommon.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/host.h"
+
+// generic SoA-based product in host memory
+template <typename T>
+class PortableHostCollection {
+public:
+  using Layout = T;
+  using View = typename Layout::View;
+  using ConstView = typename Layout::ConstView;
+  using Buffer = alpaka::Buf<alpaka_common::DevHost, std::byte, alpaka::DimInt<1u>, uint32_t>;
+  using ConstBuffer = alpaka::ViewConst<Buffer>;
+
+  PortableHostCollection() = default;
+
+  PortableHostCollection(int32_t elements, alpaka_common::DevHost const &host)
+      // allocate pageable host memory
+      : buffer_{alpaka::allocBuf<std::byte, uint32_t>(
+            host, alpaka::Vec<alpaka::DimInt<1u>, uint32_t>{Layout::computeDataSize(elements)})},
+        layout_{buffer_->data(), elements},
+        view_{layout_} {
+    // Alpaka set to a default alignment of 128 bytes defining ALPAKA_DEFAULT_HOST_MEMORY_ALIGNMENT=128
+    assert(reinterpret_cast<uintptr_t>(buffer_->data()) % Layout::alignment == 0);
+  }
+
+  template <typename TDev>
+  PortableHostCollection(int32_t elements, alpaka_common::DevHost const &host, TDev const &device)
+      // allocate pinned host memory, accessible by the given device
+      : buffer_{alpaka::allocMappedBuf<std::byte, uint32_t>(
+            host, device, alpaka::Vec<alpaka::DimInt<1u>, uint32_t>{Layout::computeDataSize(elements)})},
+        layout_{buffer_->data(), elements},
+        view_{layout_} {
+    // Alpaka set to a default alignment of 128 bytes defining ALPAKA_DEFAULT_HOST_MEMORY_ALIGNMENT=128
+    assert(reinterpret_cast<uintptr_t>(buffer_->data()) % Layout::alignment == 0);
+  }
+
+  ~PortableHostCollection() = default;
+
+  // non-copyable
+  PortableHostCollection(PortableHostCollection const &) = delete;
+  PortableHostCollection &operator=(PortableHostCollection const &) = delete;
+
+  // movable
+  PortableHostCollection(PortableHostCollection &&other) = default;
+  PortableHostCollection &operator=(PortableHostCollection &&other) = default;
+
+  View &view() { return view_; }
+  ConstView const &view() const { return view_; }
+  ConstView const &const_view() const { return view_; }
+
+  View &operator*() { return view_; }
+  ConstView const &operator*() const { return view_; }
+
+  View *operator->() { return &view_; }
+  ConstView const *operator->() const { return &view_; }
+
+  Buffer buffer() { return *buffer_; }
+  ConstBuffer buffer() const { return *buffer_; }
+  ConstBuffer const_buffer() const { return *buffer_; }
+
+  // part of the ROOT read streamer
+  static void ROOTReadStreamer(PortableHostCollection *newObj, Layout const &layout) {
+    newObj->~PortableHostCollection();
+    // use the global "host" object returned by alpaka_common::host()
+    new (newObj) PortableHostCollection(layout.metadata().size(), alpaka_common::host());
+    newObj->layout_.ROOTReadStreamer(layout);
+  }
+
+private:
+  std::optional<Buffer> buffer_;  //!
+  Layout layout_;                 //
+  View view_;                     //!
+};
+
+#endif  // DataFormats_Portable_interface_PortableHostCollection_h

DataFormats/Portable/interface/alpaka/PortableCollection.h

@@ -0,0 +1,42 @@
+#ifndef DataFormats_Portable_interface_alpaka_PortableDeviceCollection_h
+#define DataFormats_Portable_interface_alpaka_PortableDeviceCollection_h
+
+#include <optional>
+
+#include <alpaka/alpaka.hpp>
+
+#include "DataFormats/Portable/interface/PortableCollection.h"
+#include "DataFormats/Portable/interface/PortableHostCollection.h"
+#include "DataFormats/Portable/interface/PortableDeviceCollection.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
+
+namespace ALPAKA_ACCELERATOR_NAMESPACE {
+
+#if defined ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED
+  // ... or any other CPU-based accelerators
+
+  // generic SoA-based product in host memory
+  template <typename T>
+  using PortableCollection = ::PortableHostCollection<T>;
+
+#else
+
+  // generic SoA-based product in device memory
+  template <typename T>
+  using PortableCollection = ::PortableDeviceCollection<T, Device>;
+
+#endif  // ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED
+
+}  // namespace ALPAKA_ACCELERATOR_NAMESPACE
+
+namespace traits {
+
+  // specialise the trait for the device provided by the ALPAKA_ACCELERATOR_NAMESPACE
+  template <typename T>
+  class PortableCollectionTrait<T, ALPAKA_ACCELERATOR_NAMESPACE::Device> {
+    using CollectionType = ALPAKA_ACCELERATOR_NAMESPACE::PortableCollection<T>;
+  };
+
+}  // namespace traits
+
+#endif  // DataFormats_Portable_interface_alpaka_PortableDeviceCollection_h

DataFormats/PortableTestObjects/BuildFile.xml

@@ -0,0 +1,10 @@
+<use name="rootcore"/>
+<use name="DataFormats/Common"/>
+<use name="DataFormats/Portable"/>
+<use name="FWCore/MessageLogger"/>
+<use name="FWCore/Utilities" source_only="1"/>
+<use name="HeterogeneousCore/AlpakaInterface"/>
+<flags ALPAKA_BACKENDS="1"/>
+<export>
+  <lib name="1"/>
+</export>

DataFormats/PortableTestObjects/README.md

@@ -0,0 +1,10 @@
+## Define the portable SoA-based data formats
+
+Notes:
+  - define a full dictionary for `portabletest::TestSoA` and `portabletest::TestHostCollection`
+  - do not define a dictionary for `alpaka_serial_sync::portabletest::TestDeviceCollection`,
+    because it is the same class as `portabletest::TestHostCollection`;
+  - define the dictionary for `alpaka_cuda_async::portabletest::TestDeviceCollection`
+    as _transient_ only;
+  - the dictionary for `alpaka_cuda_async::portabletest::TestDeviceCollection` should
+    be defined in a separate library, to factor out the CUDA dependency.

DataFormats/PortableTestObjects/interface/TestHostCollection.h

@@ -0,0 +1,14 @@
+#ifndef DataFormats_PortableTestObjects_interface_TestHostCollection_h
+#define DataFormats_PortableTestObjects_interface_TestHostCollection_h
+
+#include "DataFormats/Portable/interface/PortableHostCollection.h"
+#include "DataFormats/PortableTestObjects/interface/TestSoA.h"
+
+namespace portabletest {
+
+  // SoA with x, y, z, id fields in host memory
+  using TestHostCollection = PortableHostCollection<TestSoA>;
+
+}  // namespace portabletest
+
+#endif  // DataFormats_PortableTestObjects_interface_TestHostCollection_h

DataFormats/PortableTestObjects/interface/TestSoA.h

@@ -0,0 +1,22 @@
+#ifndef DataFormats_PortableTestObjects_interface_TestSoA_h
+#define DataFormats_PortableTestObjects_interface_TestSoA_h
+
+#include "DataFormats/SoATemplate/interface/SoACommon.h"
+#include "DataFormats/SoATemplate/interface/SoALayout.h"
+#include "DataFormats/SoATemplate/interface/SoAView.h"
+
+namespace portabletest {
+
+  // SoA layout with x, y, z, id fields
+  GENERATE_SOA_LAYOUT(TestSoALayout,
+                      // columns: one value per element
+                      SOA_COLUMN(double, x),
+                      SOA_COLUMN(double, y),
+                      SOA_COLUMN(double, z),
+                      SOA_COLUMN(int32_t, id))
+
+  using TestSoA = TestSoALayout<>;
+
+}  // namespace portabletest
+
+#endif  // DataFormats_PortableTestObjects_interface_TestSoA_h

DataFormats/PortableTestObjects/interface/alpaka/TestDeviceCollection.h

@@ -0,0 +1,22 @@
+#ifndef DataFormats_PortableTestObjects_interface_alpaka_TestDeviceCollection_h
+#define DataFormats_PortableTestObjects_interface_alpaka_TestDeviceCollection_h
+
+#include "DataFormats/Portable/interface/alpaka/PortableCollection.h"
+#include "DataFormats/PortableTestObjects/interface/TestSoA.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
+
+namespace ALPAKA_ACCELERATOR_NAMESPACE {
+
+  namespace portabletest {
+
+    // import the top-level portabletest namespace
+    using namespace ::portabletest;
+
+    // SoA with x, y, z, id fields in device global memory
+    using TestDeviceCollection = PortableCollection<TestSoA>;
+
+  }  // namespace portabletest
+
+}  // namespace ALPAKA_ACCELERATOR_NAMESPACE
+
+#endif  // DataFormats_PortableTestObjects_interface_alpaka_TestDeviceCollection_h

DataFormats/PortableTestObjects/src/alpaka/classes_cuda.h

@@ -0,0 +1,3 @@
+#include "DataFormats/Common/interface/Wrapper.h"
+#include "DataFormats/PortableTestObjects/interface/TestSoA.h"
+#include "DataFormats/PortableTestObjects/interface/alpaka/TestDeviceCollection.h"

DataFormats/PortableTestObjects/src/alpaka/classes_cuda_def.xml

@@ -0,0 +1,4 @@
+<lcgdict>
+  <class name="alpaka_cuda_async::portabletest::TestDeviceCollection" persistent="false"/>
+  <class name="edm::Wrapper<alpaka_cuda_async::portabletest::TestDeviceCollection>" persistent="false"/>
+</lcgdict>

DataFormats/PortableTestObjects/src/alpaka/classes_serial.h

@@ -0,0 +1,3 @@
+#include "DataFormats/Common/interface/Wrapper.h"
+#include "DataFormats/PortableTestObjects/interface/TestHostCollection.h"
+#include "DataFormats/PortableTestObjects/interface/TestSoA.h"

DataFormats/PortableTestObjects/src/alpaka/classes_serial_def.xml

@@ -0,0 +1,16 @@
+<lcgdict>
+  <class name="portabletest::TestHostCollection"/>
+  <read
+    sourceClass="portabletest::TestHostCollection"
+    targetClass="portabletest::TestHostCollection"
+    version="[1-]"
+    source="portabletest::TestSoA layout_;"
+    target="buffer_"
+    embed="false">
+  <![CDATA[
+    portabletest::TestHostCollection::ROOTReadStreamer(newObj, onfile.layout_);
+  ]]>
+  </read>
+
+  <class name="edm::Wrapper<portabletest::TestHostCollection>" splitLevel="0"/>
+</lcgdict>

DataFormats/PortableTestObjects/src/classes.h

@@ -0,0 +1 @@
+#include "DataFormats/PortableTestObjects/interface/TestSoA.h"

DataFormats/PortableTestObjects/src/classes_def.xml

@@ -0,0 +1,12 @@
+<lcgdict>
+  <class name="portabletest::TestSoA">
+    <field name="mem_" comment="!"/>
+    <field name="byteSize_" comment="!"/>
+    <field name="x_" comment="[nElements_]"/>
+    <field name="y_" comment="[nElements_]"/>
+    <field name="z_" comment="[nElements_]"/>
+    <field name="id_" comment="[nElements_]"/>
+    <field name="metadata()" comment="!"/>
+  </class>
+  <class name="portabletest::TestSoA::View"/>
+</lcgdict>

DataFormats/SoATemplate/BuildFile.xml

@@ -0,0 +1,2 @@
+<use name="boost_header"/>
+<use name="FWCore/Utilities" source_only="1"/>