Use Field with serializer

Rather than pass data and layouts separately, we update the serializer
to directly take the Field&. This does a better job of maintaining
consistency with what data we are operating with. Also, it reduces the
number of places we expose FieldData, which is really an internal
implementation detail that we don't want readily exposed.

Author: jacobmerson

src/pcms/coupler/field_communicator.hpp

@@ -51,8 +51,7 @@ class FieldCommunicator
     PCMS_FUNCTION_TIMER;
     PCMS_ALWAYS_ASSERT(layout_comm_.GetChannel().InSendCommunicationPhase());
     auto buffer = make_array_view(comm_buffer_);
-    serializer_->Serialize(field_.GetData(), field_.GetLayout(), buffer,
-                           layout_comm_.GetPermutationArray());
+    serializer_->Serialize(field_, buffer, layout_comm_.GetPermutationArray());
     comm_.Send(buffer.data_handle(), mode);
   }
 
@@ -64,8 +63,7 @@ class FieldCommunicator
     // mode because we make an immediate call to deserialize after a call to
     // receive.
     auto data = comm_.Recv(redev::Mode::Synchronous);
-    serializer_->Deserialize(field_.GetData(), field_.GetLayout(),
-                             make_const_array_view(data),
+    serializer_->Deserialize(field_, make_const_array_view(data),
                              layout_comm_.GetPermutationArray());
   }
 

src/pcms/coupler/field_serializer.h

@@ -14,12 +14,12 @@ template <typename T>
 class FieldSerializer
 {
 public:
-  virtual int Serialize(const FieldData<T>& field, const FieldLayout& layout,
+  virtual int Serialize(const Field<T>& field,
                         Rank1View<T, HostMemorySpace> buffer,
                         Rank1View<const LO, HostMemorySpace> permutation) const
   {
     auto data = field.GetDOFHolderDataHost();
-    auto owned = layout.GetOwnedHost();
+    auto owned = field.GetLayout().GetOwnedHost();
     // The exchange plan is per DOF holder: owned[i] and permutation[i] are
     // indexed by holder. All num_components components of a holder share its
     // location, so they occupy one contiguous block permutation[i]*num_comp in
@@ -41,10 +41,10 @@ class FieldSerializer
   }
 
   virtual void Deserialize(
-    FieldData<T>& field, const FieldLayout& layout,
-    Rank1View<const T, HostMemorySpace> buffer,
+    Field<T>& field, Rank1View<const T, HostMemorySpace> buffer,
     Rank1View<const LO, HostMemorySpace> permutation) const
   {
+    const auto& layout = field.GetLayout();
     const LO num_dof = layout.GetNumOwnedDofHolder();
     const LO num_comp = layout.GetNumComponents();
     Kokkos::View<T*, HostMemorySpace> sorted("sorted", layout.OwnedSize());

src/pcms/coupler/serializer/xgc.h

@@ -20,20 +20,21 @@ class XGCFieldSerializer : public FieldSerializer<T>
   {
   }
 
-  int Serialize(const FieldData<T>& field, const FieldLayout& layout,
-                Rank1View<T, HostMemorySpace> buffer,
+  int Serialize(const Field<T>& field, Rank1View<T, HostMemorySpace> buffer,
                 Rank1View<const LO, HostMemorySpace> permutation) const override
   {
     if (!rank_participates_) {
       return 0;
     }
 
-    auto const* xgc_field = dynamic_cast<const XGCFieldData<T>*>(&field);
+    auto const* xgc_field =
+      dynamic_cast<const XGCFieldData<T>*>(&field.GetData());
     if (!xgc_field) {
       throw pcms_error("XGCFieldSerializer::Serialize: incompatible FieldData");
     }
 
     auto data = xgc_field->GetDOFHolderDataHost();
+    const auto& layout = field.GetLayout();
     auto owned = layout.GetOwnedHost();
     // Per-holder plan: owned[i]/permutation[i] index holders; a holder's
     // num_components values form one contiguous block in the wire buffer.
@@ -54,17 +55,17 @@ class XGCFieldSerializer : public FieldSerializer<T>
   }
 
   void Deserialize(
-    FieldData<T>& field, const FieldLayout& layout,
-    Rank1View<const T, HostMemorySpace> buffer,
+    Field<T>& field, Rank1View<const T, HostMemorySpace> buffer,
     Rank1View<const LO, HostMemorySpace> permutation) const override
   {
-    auto* xgc_field = dynamic_cast<XGCFieldData<T>*>(&field);
+    auto* xgc_field = dynamic_cast<XGCFieldData<T>*>(&field.GetData());
     if (!xgc_field) {
       throw pcms_error(
         "XGCFieldSerializer::Deserialize: incompatible FieldData");
     }
 
     auto current = xgc_field->GetDOFHolderDataHost();
+    const auto& layout = field.GetLayout();
     const LO num_dof = static_cast<LO>(current.extent(0));
     const LO num_comp = static_cast<LO>(current.extent(1));
     std::vector<T> full_data(current.size());

test/CMakeLists.txt

@@ -400,7 +400,8 @@ if(Catch2_FOUND)
   message(
     STATUS "Found Catch2: ${Catch2_DIR} (found version ${Catch2_VERSION})")
   set(PCMS_UNIT_TEST_SOURCES unit_test_main.cpp test_coordinate_transform.cpp
-                             test_coordinate.cpp test_bounding_box.cpp)
+                             test_coordinate.cpp test_bounding_box.cpp
+                             test_field_serializer.cpp)
   if(PCMS_ENABLE_XGC)
     list(APPEND PCMS_UNIT_TEST_SOURCES test_xgc_reverse_classification.cpp
          test_xgc_field_data.cpp)

test/field_test_utils.h

@@ -10,7 +10,6 @@
 #include "pcms/field/evaluation_request.h"
 #include "pcms/field/point_evaluator.h"
 #include "pcms/field/out_of_bounds_policy.h"
-#include "pcms/coupler/field_serializer.h"
 #include "pcms/field/coordinate_system.h"
 #include "pcms/utility/arrays.h"
 #include "pcms/utility/memory_spaces.h"
@@ -282,39 +281,6 @@ inline void SetField(FieldData<Real>& field, const FieldLayout& layout,
   SetField<ExecutionSpace>(layout, field, func);
 }
 
-// Check that serialize followed by deserialize round-trips the data.
-// Uses an identity permutation so permutation[i] = i.
-inline void CheckSerializeDeserialize(const FieldLayout& layout,
-                                      FieldData<Real>& field)
-{
-  auto data_before = FlattenToRank1View(field.GetDOFHolderDataHost());
-  int n = static_cast<int>(data_before.size());
-
-  std::vector<Real> buffer(n);
-  std::vector<LO> perm(n);
-  for (int i = 0; i < n; ++i)
-    perm[i] = i;
-
-  Rank1View<Real, HostMemorySpace> buf_view(buffer.data(), n);
-  Rank1View<const LO, HostMemorySpace> perm_view(perm.data(), n);
-
-  FieldSerializer<Real> serializer;
-  serializer.Serialize(field, layout, buf_view, perm_view);
-  serializer.Deserialize(
-    field, layout, Rank1View<const Real, HostMemorySpace>(buf_view), perm_view);
-
-  auto data_after = FlattenToRank1View(field.GetDOFHolderDataHost());
-  REQUIRE(data_after.size() == data_before.size());
-  for (int i = 0; i < n; ++i) {
-    REQUIRE(data_after[i] == Catch::Approx(data_before[i]));
-  }
-}
-
-inline void CheckSerializeDeserialize(Field<Real>& field)
-{
-  CheckSerializeDeserialize(field.GetLayout(), field.GetData());
-}
-
 // Helper structure to hold device coordinates with proper lifetime management
 struct DeviceCoordinates
 {

test/test_field_serializer.cpp

@@ -0,0 +1,239 @@
+#include <catch2/catch_approx.hpp>
+#include <catch2/catch_test_macros.hpp>
+
+#include "pcms/configuration.h"
+#include "pcms/coupler/field_serializer.h"
+#include "pcms/utility/arrays.h"
+
+#include <memory>
+#include <numeric>
+#include <vector>
+
+#ifdef PCMS_ENABLE_OMEGA_H
+#include "field_test_utils.h"
+#include "pcms/field/function_space/lagrange.h"
+#include "pcms/field/function_space/polynomial_reconstruction.hpp"
+#include "pcms/utility/uniform_grid.h"
+#include <Omega_h_library.hpp>
+#endif
+
+#ifdef PCMS_ENABLE_XGC
+#include "pcms/coupler/serializer/xgc.h"
+#include "pcms/field/data/xgc.h"
+#include "pcms/field/function_space/xgc.h"
+#endif
+
+namespace
+{
+
+#ifdef PCMS_ENABLE_OMEGA_H
+void CheckSerializeDeserialize(pcms::Field<pcms::Real>& field)
+{
+  auto data_before = field.GetDOFHolderDataHost();
+  const auto num_dof = static_cast<pcms::LO>(data_before.extent(0));
+  const auto num_components = static_cast<pcms::LO>(data_before.extent(1));
+  const auto num_values = static_cast<size_t>(data_before.size());
+
+  std::vector<pcms::Real> expected(num_values);
+  for (pcms::LO i = 0; i < num_dof; ++i) {
+    for (pcms::LO c = 0; c < num_components; ++c) {
+      expected[static_cast<size_t>(i) * num_components + c] = data_before(i, c);
+    }
+  }
+
+  std::vector<pcms::Real> buffer(num_values);
+  std::vector<pcms::LO> permutation(num_dof);
+  std::iota(permutation.begin(), permutation.end(), pcms::LO{0});
+
+  pcms::FieldSerializer<pcms::Real> serializer;
+  REQUIRE(serializer.Serialize(field, pcms::make_array_view(buffer),
+                               pcms::make_const_array_view(permutation)) ==
+          static_cast<int>(num_values));
+
+  std::vector<pcms::Real> cleared(num_values, -1.0);
+  field.SetDOFHolderDataHost(
+    pcms::Rank2View<const pcms::Real, pcms::HostMemorySpace>(
+      cleared.data(), num_dof, num_components));
+  serializer.Deserialize(field, pcms::make_const_array_view(buffer),
+                         pcms::make_const_array_view(permutation));
+
+  auto data_after = field.GetDOFHolderDataHost();
+  REQUIRE(data_after.size() == expected.size());
+  for (pcms::LO i = 0; i < num_dof; ++i) {
+    for (pcms::LO c = 0; c < num_components; ++c) {
+      REQUIRE(
+        data_after(i, c) ==
+        Catch::Approx(expected[static_cast<size_t>(i) * num_components + c]));
+    }
+  }
+}
+#endif
+
+#ifdef PCMS_ENABLE_XGC
+pcms::ReverseClassificationVertex CreateDummyReverseClassification(int size)
+{
+  pcms::ReverseClassificationVertex rc;
+  for (int i = 0; i < size; ++i) {
+    rc.Insert(i % 4 == 0 ? pcms::DimID{0, 0} : pcms::DimID{0, 1}, i);
+  }
+  return rc;
+}
+
+bool InXGCOverlap(int, int id)
+{
+  return id == 0;
+}
+#endif
+
+} // namespace
+
+#ifdef PCMS_ENABLE_XGC
+TEST_CASE("XGCFieldSerializer preserves inactive field entries")
+{
+  static constexpr int data_size = 16;
+  auto rc = CreateDummyReverseClassification(data_size);
+  pcms::XGCFieldFactory factory(rc, InXGCOverlap, data_size);
+
+  std::vector<pcms::Real> data(data_size);
+  std::iota(data.begin(), data.end(), 0.0);
+  const auto original = data;
+  auto field = factory.CreateField<pcms::Real>(
+    "", std::make_unique<pcms::XGCFieldData<pcms::Real>>(
+          factory.GetXGCLayout(), pcms::FieldMetadata{},
+          pcms::make_array_view(data)));
+  pcms::XGCFieldSerializer<pcms::Real> serializer(MPI_COMM_SELF);
+
+  auto owned = field.GetLayout().GetOwnedHost();
+  std::vector<pcms::LO> permutation(data_size, -1);
+  int entry = 0;
+  for (int i = 0; i < data_size; ++i) {
+    if (owned[i]) {
+      permutation[i] = entry++;
+    }
+  }
+  const int num_owned = entry;
+  REQUIRE(num_owned == 4);
+
+  std::vector<pcms::Real> buffer(num_owned, -1.0);
+  REQUIRE(serializer.Serialize(field, pcms::make_array_view(buffer),
+                               pcms::make_const_array_view(permutation)) ==
+          num_owned);
+
+  for (int i = 0; i < data_size; ++i) {
+    if (owned[i]) {
+      REQUIRE(buffer[permutation[i]] == data[i]);
+      buffer[permutation[i]] += 100.0;
+    } else {
+      REQUIRE(permutation[i] == -1);
+    }
+  }
+
+  serializer.Deserialize(field, pcms::make_const_array_view(buffer),
+                         pcms::make_const_array_view(permutation));
+
+  auto after = pcms::FlattenToRank1View(field.GetDOFHolderDataHost());
+  for (int i = 0; i < data_size; ++i) {
+    if (owned[i]) {
+      REQUIRE(after[i] == Catch::Approx(original[i] + 100.0));
+    } else {
+      REQUIRE(after[i] == Catch::Approx(original[i]));
+    }
+  }
+}
+#endif
+
+#ifdef PCMS_ENABLE_OMEGA_H
+TEST_CASE("FieldSerializer round-trips a polynomial-reconstruction field")
+{
+  std::vector<pcms::Real> coords{0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0};
+  pcms::Rank2View<pcms::Real, pcms::HostMemorySpace> coords_view(coords.data(),
+                                                                 4, 2);
+  auto space = pcms::PolynomialReconstructionFunctionSpace::Create(
+    coords_view, pcms::CoordinateSystem::Cartesian);
+  auto field = space->CreateFunction<pcms::Real>();
+
+  std::vector<pcms::Real> data{5.0, 6.0, 7.0, 8.0};
+  field.SetDOFHolderDataHost(
+    pcms::Rank2View<const pcms::Real, pcms::HostMemorySpace>(data.data(), 4,
+                                                             1));
+
+  CheckSerializeDeserialize(field);
+}
+
+TEST_CASE("FieldSerializer round-trips a uniform-grid field")
+{
+  pcms::UniformGrid<2> grid;
+  grid.bot_left = {0.0, 0.0};
+  grid.edge_length = {10.0, 10.0};
+  grid.divisions = {3, 3};
+  auto space = pcms::LagrangeFunctionSpace::FromUniformGrid(
+    grid, 1, pcms::CoordinateSystem::Cartesian);
+  auto field = space->CreateFunction<pcms::Real>();
+
+  std::vector<pcms::Real> data(16);
+  for (size_t i = 0; i < data.size(); ++i) {
+    data[i] = static_cast<pcms::Real>(i * 10);
+  }
+  field.SetDOFHolderDataHost(
+    pcms::Rank2View<const pcms::Real, pcms::HostMemorySpace>(data.data(), 16,
+                                                             1));
+
+  CheckSerializeDeserialize(field);
+}
+
+TEST_CASE("FieldSerializer round-trips an order-1 Omega_h field")
+{
+  auto lib = Omega_h::Library{};
+  auto mesh = pcms::test::BuildUnitSquare(lib, 0);
+  auto space = pcms::test::MakeP1Space(mesh);
+  auto field = space->CreateFunction<pcms::Real>();
+
+  pcms::test::SetField(
+    field, OMEGA_H_LAMBDA(pcms::Real x, pcms::Real y) {
+      return pcms::test::linear_f(x, y);
+    });
+
+  CheckSerializeDeserialize(field);
+}
+
+TEST_CASE("FieldSerializer round-trips a multi-component Omega_h field")
+{
+  auto lib = Omega_h::Library{};
+  auto mesh = pcms::test::BuildUnitSquare(lib, 0);
+  constexpr int num_components = 3;
+  auto space = pcms::LagrangeFunctionSpace::FromMesh(
+    mesh, 1, num_components, pcms::CoordinateSystem::Cartesian, "global",
+    pcms::LagrangeFunctionSpace::Backend::OmegaH);
+  auto field = space->CreateFunction<pcms::Real>();
+
+  const int num_dof = field.GetLayout().GetNumOwnedDofHolder();
+  std::vector<pcms::Real> data(static_cast<size_t>(num_dof) * num_components);
+  for (int i = 0; i < num_dof; ++i) {
+    for (int c = 0; c < num_components; ++c) {
+      data[static_cast<size_t>(i) * num_components + c] = i + 0.25 * c;
+    }
+  }
+  field.SetDOFHolderDataHost(
+    pcms::Rank2View<const pcms::Real, pcms::HostMemorySpace>(
+      data.data(), num_dof, num_components));
+
+  CheckSerializeDeserialize(field);
+}
+
+TEST_CASE("FieldSerializer round-trips an order-0 Omega_h field")
+{
+  auto lib = Omega_h::Library{};
+  auto mesh = pcms::test::BuildUnitSquare(lib, 0);
+  auto space = pcms::test::MakeP0Space(mesh);
+  auto field = space->CreateFunction<pcms::Real>();
+
+  const int num_dof = mesh.nelems();
+  std::vector<pcms::Real> data(num_dof);
+  std::iota(data.begin(), data.end(), pcms::Real{0});
+  field.SetDOFHolderDataHost(
+    pcms::Rank2View<const pcms::Real, pcms::HostMemorySpace>(data.data(),
+                                                             num_dof, 1));
+
+  CheckSerializeDeserialize(field);
+}
+#endif