Make overlap masking generic and drop XGC-specific overlap handling

Author: jacobmerson

.gitattributes

@@ -0,0 +1,3 @@
+# SWIG output regenerated from client.i; collapsed in GitHub diffs
+src/pcms/fortranapi/client_wrap.c linguist-generated
+src/pcms/fortranapi/pcms.f90 linguist-generated

src/pcms/capi/client.cpp

@@ -129,10 +129,9 @@ ClientState::HandleVariant RegisterField(
   // not the arbitrary adapter name given at creation.
   registration.function_space.SetLayoutName(name);
   auto field = registration.function_space.template CreateField<T>(
-    std::move(name),
-    std::make_unique<XGCFieldData<T>>(
-      registration.function_space.GetXGCLayout(), ValueBasis{},
-      registration.data, registration.plane_comm));
+    std::move(name), std::make_unique<XGCFieldData<T>>(
+                       registration.function_space.GetXGCLayout(), ValueBasis{},
+                       registration.data, registration.plane_comm));
   return ClientState::HandleVariant{
     app.AddField(std::move(field), participates)};
 }
@@ -184,6 +183,18 @@ void pcms_destroy_reverse_classification(PcmsReverseClassificationHandle rc)
   delete reinterpret_cast<pcms::ReverseClassificationVertex*>(rc.pointer);
 }
 
+void pcms_set_overlap_mask(PcmsClientHandle client_handle, const char* name,
+                           in_overlap_function in_overlap)
+{
+  auto* app =
+    reinterpret_cast<pcms::ApplicationComm*>(client_handle.appPointer);
+  PCMS_ALWAYS_ASSERT(app != nullptr);
+  PCMS_ALWAYS_ASSERT(in_overlap != nullptr);
+  // Fields registered through the C API use their name as the layout name.
+  app->SetLayoutOverlapMask(name,
+                            std::make_unique<pcms::OverlapMask>(in_overlap));
+}
+
 PcmsFieldHandle pcms_add_field(PcmsClientHandle client_handle, const char* name,
                                PcmsFieldAdapterHandle adapter_handle,
                                int participates)
@@ -248,11 +259,10 @@ template <typename T>
 void pcms_create_xgc_field_adapter_t(
   const char* /* name */, MPI_Comm comm, void* data, int size,
   const pcms::ReverseClassificationVertex& reverse_classification,
-  in_overlap_function in_overlap, pcms::FieldAdapterVariant& field_adapter)
+  pcms::FieldAdapterVariant& field_adapter)
 {
   PCMS_ALWAYS_ASSERT((size > 0) ? (data != nullptr) : true);
-  auto function_space =
-    pcms::XGCFieldFactory(reverse_classification, in_overlap, size);
+  auto function_space = pcms::XGCFieldFactory(reverse_classification, size);
   pcms::Rank1View<T, pcms::HostMemorySpace> data_view(
     reinterpret_cast<T*>(data), size);
   field_adapter.emplace<pcms::detail::XGCFieldRegistration<T>>(
@@ -261,7 +271,7 @@ void pcms_create_xgc_field_adapter_t(
 
 PcmsFieldAdapterHandle pcms_create_xgc_field_adapter(
   const char* name, MPI_Comm comm, void* data, int size, PcmsType data_type,
-  const PcmsReverseClassificationHandle rc, in_overlap_function in_overlap)
+  const PcmsReverseClassificationHandle rc)
 {
   auto* field_adapter = new pcms::FieldAdapterVariant{};
   PCMS_ALWAYS_ASSERT(rc.pointer != nullptr);
@@ -270,24 +280,20 @@ PcmsFieldAdapterHandle pcms_create_xgc_field_adapter(
   PCMS_ALWAYS_ASSERT(reverse_classification != nullptr);
   switch (data_type) {
     case PCMS_DOUBLE:
-      pcms_create_xgc_field_adapter_t<double>(name, comm, data, size,
-                                              *reverse_classification,
-                                              in_overlap, *field_adapter);
+      pcms_create_xgc_field_adapter_t<double>(
+        name, comm, data, size, *reverse_classification, *field_adapter);
       break;
     case PCMS_FLOAT:
-      pcms_create_xgc_field_adapter_t<float>(name, comm, data, size,
-                                             *reverse_classification,
-                                             in_overlap, *field_adapter);
+      pcms_create_xgc_field_adapter_t<float>(
+        name, comm, data, size, *reverse_classification, *field_adapter);
       break;
     case PCMS_INT:
-      pcms_create_xgc_field_adapter_t<int>(name, comm, data, size,
-                                           *reverse_classification, in_overlap,
-                                           *field_adapter);
+      pcms_create_xgc_field_adapter_t<int>(
+        name, comm, data, size, *reverse_classification, *field_adapter);
       break;
     case PCMS_LONG_INT:
-      pcms_create_xgc_field_adapter_t<pcms::GO>(name, comm, data, size,
-                                                *reverse_classification,
-                                                in_overlap, *field_adapter);
+      pcms_create_xgc_field_adapter_t<pcms::GO>(
+        name, comm, data, size, *reverse_classification, *field_adapter);
       break;
     default:
       PCMS_ALWAYS_ASSERT(false, MPI_COMM_WORLD,

src/pcms/capi/client.h

@@ -65,13 +65,18 @@ void pcms_destroy_reverse_classification(PcmsReverseClassificationHandle);
 // likely to be needed for production cases
 int pcms_reverse_classification_count_verts(PcmsReverseClassificationHandle);
 
+PcmsFieldAdapterHandle pcms_create_xgc_field_adapter(
+  const char* name, MPI_Comm plane_comm, void* data, int size,
+  PcmsType data_type, const PcmsReverseClassificationHandle rc);
+
 // takes in overlap function takes a geometric dimension and a geometric id
 // C doesn't have a builtin bool type, so we use int for compatability with C++
 typedef int8_t (*in_overlap_function)(int, int);
-PcmsFieldAdapterHandle pcms_create_xgc_field_adapter(
-  const char* name, MPI_Comm plane_comm, void* data, int size,
-  PcmsType data_type, const PcmsReverseClassificationHandle rc,
-  in_overlap_function in_overlap);
+// Restricts the field registered under `name` to the DOF holders whose
+// classification satisfies in_overlap (unclassified holders are passed as
+// (-1, -1)).
+void pcms_set_overlap_mask(PcmsClientHandle client_handle, const char* name,
+                           in_overlap_function in_overlap);
 
 PcmsFieldAdapterHandle pcms_create_dummy_field_adapter();
 

src/pcms/coupler/coupler.hpp

@@ -403,7 +403,7 @@ pcms::Field<T>& pcms::ApplicationComm::GetField(const std::string& name)
 
 template <typename T>
 void pcms::ApplicationComm::RegisterFieldCommunicator(Field<T>& field_obj,
-                                                     bool participates)
+                                                      bool participates)
 {
   const std::string& name = field_obj.GetName();
   if (field_obj.GetData().GetDOFAccess() == nullptr) {
@@ -414,7 +414,8 @@ void pcms::ApplicationComm::RegisterFieldCommunicator(Field<T>& field_obj,
   FieldLayoutCommunicator& layout_communicator =
     GetOrCreateLayoutCommunicator(field_obj.GetLayout(), participates);
   FieldCommunicator2Ptr field_communicator =
-    std::make_unique<FieldCommunicator<T>>(name, layout_communicator, field_obj);
+    std::make_unique<FieldCommunicator<T>>(name, layout_communicator,
+                                           field_obj);
   auto [it, inserted] =
     field_communicators_.emplace(name, std::move(field_communicator));
   if (!inserted) {

src/pcms/coupler/field_communicator.hpp

@@ -53,7 +53,7 @@ class FieldCommunicator
     // receive.
     auto data = comm_.Recv(redev::Mode::Synchronous);
     serializer_.Deserialize(field_, make_const_array_view(data),
-                             layout_comm_.GetPermutationArray());
+                            layout_comm_.GetPermutationArray());
   }
 
 private:

src/pcms/coupler/field_layout_communicator.cpp

@@ -111,8 +111,8 @@ void FieldLayoutCommunicator::UpdateLayoutNull()
   PCMS_FUNCTION_TIMER;
   // This rank exchanges nothing, but it still unpacks on receive (backends
   // that keep replicas synchronize them there), so every holder is unmapped.
-  plan_.permutation =
-    MakeUnmappedPermutation(static_cast<size_t>(layout_.GetNumOwnedDofHolder()));
+  plan_.permutation = MakeUnmappedPermutation(
+    static_cast<size_t>(layout_.GetNumOwnedDofHolder()));
   if (redev_.GetProcessType() == redev::ProcessType::Client) {
     channel_.BeginSendCommunicationPhase();
     channel_.EndSendCommunicationPhase();

src/pcms/coupler/field_serializer.h

@@ -18,9 +18,8 @@ template <typename T>
 class FieldSerializer
 {
 public:
-  int Serialize(const Field<T>& field,
-                        Rank1View<T, HostMemorySpace> buffer,
-                        Rank1View<const LO, HostMemorySpace> permutation) const
+  int Serialize(const Field<T>& field, Rank1View<T, HostMemorySpace> buffer,
+                Rank1View<const LO, HostMemorySpace> permutation) const
   {
     const auto data = field.GetDOFHolderDataHost().GetValues();
     if (buffer.size() > 0) {
@@ -37,9 +36,8 @@ class FieldSerializer
     return static_cast<int>(buffer.size());
   }
 
-  void Deserialize(
-    Field<T>& field, Rank1View<const T, HostMemorySpace> buffer,
-    Rank1View<const LO, HostMemorySpace> permutation) const
+  void Deserialize(Field<T>& field, Rank1View<const T, HostMemorySpace> buffer,
+                   Rank1View<const LO, HostMemorySpace> permutation) const
   {
     const auto& layout = field.GetLayout();
     const LO num_dof = layout.GetNumOwnedDofHolder();

src/pcms/coupler/overlap_mask.h

@@ -15,17 +15,18 @@ struct OverlapMask
   Kokkos::View<bool*, HostMemorySpace> is_overlap_;
   std::function<int8_t(int, int)> in_overlap_func_;
 
-  // Default constructor: all DOFs are in overlap
+  // Default constructor: all DOF holders are in overlap
   OverlapMask(size_t size) : is_overlap_("overlap_info", size)
   {
     Kokkos::deep_copy(is_overlap_, true);
   }
 
-  // Construct from a function that determines overlap based on classification
-  OverlapMask(size_t size, std::function<int8_t(int, int)> in_overlap)
-    : is_overlap_("overlap_info", size), in_overlap_func_(std::move(in_overlap))
+  // Construct from a function that determines overlap from a DOF holder's
+  // classification (dimension, id). Holders without classification are passed
+  // as (-1, -1).
+  explicit OverlapMask(std::function<int8_t(int, int)> in_overlap)
+    : in_overlap_func_(std::move(in_overlap))
   {
-    Kokkos::deep_copy(is_overlap_, true);
   }
 
   // Construct from a precomputed per-DOF-holder host mask (e.g. an MFEM
@@ -56,19 +57,22 @@ struct OverlapMask
     }
   }
 
-  // Get the mask, evaluating the function if needed
-  Rank1View<const bool, HostMemorySpace> GetMask(
+  // Get the mask, evaluating the function against the layout if one was given
+  Kokkos::View<const bool*, HostMemorySpace> GetMask(
     const FieldLayout& layout) const
   {
-    if (in_overlap_func_) {
-      auto class_dims = layout.GetDOFHolderClassificationDimensionsHost();
-      auto class_ids = layout.GetDOFHolderClassificationIdsHost();
-      for (size_t i = 0; i < is_overlap_.extent(0); ++i) {
-        is_overlap_[i] =
-          static_cast<bool>(in_overlap_func_(class_dims[i], class_ids[i]));
-      }
+    if (!in_overlap_func_) {
+      return is_overlap_;
+    }
+    auto class_dims = layout.GetDOFHolderClassificationDimensionsHost();
+    auto class_ids = layout.GetDOFHolderClassificationIdsHost();
+    Kokkos::View<bool*, HostMemorySpace> is_overlap("overlap_info",
+                                                    class_dims.size());
+    for (size_t i = 0; i < is_overlap.extent(0); ++i) {
+      is_overlap[i] =
+        static_cast<bool>(in_overlap_func_(class_dims[i], class_ids[i]));
     }
-    return make_const_array_view(is_overlap_);
+    return is_overlap;
   }
 };
 

src/pcms/field/data/mfem.hpp

@@ -22,10 +22,7 @@ class MFEMVertexFieldData
   FieldValueType GetValueType() const override;
   const ValueBasis& GetValueBasis() const override;
 
-  const DOFAccess<Real>* GetDOFAccess() const noexcept override
-  {
-    return this;
-  }
+  const DOFAccess<Real>* GetDOFAccess() const noexcept override { return this; }
   DOFAccess<Real>* GetDOFAccess() noexcept override { return this; }
 
   Rank2View<const Real, DeviceMemorySpace> GetOwnedValues() const override;

src/pcms/field/data/point_cloud.h

@@ -19,10 +19,7 @@ class PointCloud
   FieldValueType GetValueType() const override;
   const ValueBasis& GetValueBasis() const override;
 
-  const DOFAccess<Real>* GetDOFAccess() const noexcept override
-  {
-    return this;
-  }
+  const DOFAccess<Real>* GetDOFAccess() const noexcept override { return this; }
   DOFAccess<Real>* GetDOFAccess() noexcept override { return this; }
 
   Rank2View<const Real, DeviceMemorySpace> GetOwnedValues() const override;