diff --git a/CMakeLists.txt b/CMakeLists.txt index 9a4677fc5..ebec41895 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -97,6 +97,7 @@ endif() # dependencies find_package(MPI REQUIRED) +find_package(mpicpp-lite 2.1 REQUIRED) find_package(HDF5 REQUIRED COMPONENTS C HL) find_package(Boost CONFIG) @@ -277,6 +278,7 @@ target_link_libraries(libopensn ${VTK_LIBRARIES} caliper ${HDF5_LIBRARIES} + mpicpp-lite::mpicpp-lite MPI::MPI_CXX ) if (OPENSN_WITH_CUDA) diff --git a/cmake/OpenSnConfig.cmake.in b/cmake/OpenSnConfig.cmake.in index 1b669b86a..f83c6aa24 100644 --- a/cmake/OpenSnConfig.cmake.in +++ b/cmake/OpenSnConfig.cmake.in @@ -5,6 +5,8 @@ set(OPENSN_VERSION @PROJECT_VERSION@) include("${CMAKE_CURRENT_LIST_DIR}/OpenSnTargets.cmake") include(FindPackageHandleStandardArgs) +find_dependency(mpicpp-lite 2.1 REQUIRED) + find_library(OPENSN_LIBRARY NAMES opensn HINTS ${PACKAGE_PREFIX_DIR}/lib NO_DEFAULT_PATH) find_path(OPENSN_INCLUDE_DIR framework/runtime.h HINTS ${PACKAGE_PREFIX_DIR}/include/opensn) diff --git a/external/mpicpp-lite/impl/Communicator.h b/external/mpicpp-lite/impl/Communicator.h deleted file mode 100644 index 1066dcfb2..000000000 --- a/external/mpicpp-lite/impl/Communicator.h +++ /dev/null @@ -1,1141 +0,0 @@ -// SPDX-FileCopyrightText: 2023 David Andrs -// SPDX-License-Identifier: MIT - -#pragma once - -#include "mpi.h" -#include -#include -#include -#include "Datatype.h" -#include "Status.h" -#include "Request.h" -#include "Operation.h" -#include "Error.h" -#include "Group.h" - -namespace mpicpp_lite { - -/// Wrapper around `MPI_Comm` -class Communicator { -public: - /// Create `MPI_COMM_WORLD` communicator - Communicator(); - - /// Create communicator from an `MPI_Comm` one - Communicator(const MPI_Comm & comm); - - /// Copy constructor - Communicator(const Communicator & comm); - - /// Determine the rank of the executing process in a communicator - /// - /// @return Rank of the executing process - int rank() const; - - /// Determine the number of processes in a communicator - /// - /// @return Number of processes - int size() const; - - /// - Communicator create(const Group & group, int tag = 0) const; - - /// Accesses the group associated with given communicator - /// - /// @return Group corresponding to communicator - Group group() const; - - /// - bool is_valid() const; - - /// Send data to another process - /// - /// @tparam T C++ type of the data - /// @param dest Destination rank - /// @param tag Message tag - /// @param value Value to send - template - void send(int dest, int tag, const T & value) const; - - /// Send data to another process - /// - /// @tparam T C++ type of the data - /// @param dest Destination rank - /// @param tag Message tag - /// @param values Values to send - /// @param n Number of values to send - template - void send(int dest, int tag, const T * values, int n) const; - - /// Send `std::vector` of data to another process - /// - /// @tparam T C++ type of the data - /// @param dest Destination rank - /// @param tag Message tag - /// @param value Vector of `T` to send - template - void send(int dest, int tag, const std::vector & value) const; - - /// Send a message to another process without any data - /// - /// @param dest Destination rank - /// @param tag Message tag - void send(int dest, int tag) const; - - /// Receive data from a remote process - /// - /// @tparam T C++ type of the data - /// @param source Source rank - /// @param tag Message tag - /// @param value Variable to recieve the data - /// @return `Status` of the operation - template - Status recv(int source, int tag, T & value) const; - - /// Receive data from a remote process - /// - /// @tparam T C++ type of the data - /// @param source Source rank - /// @param tag Message tag - /// @param values Variable to recieve the data - /// @param n Number of values to receive - /// @return `Status` of the operation - template - Status recv(int source, int tag, T * values, int n) const; - - /// Receive std::vector of data from a remote process - /// - /// @tparam T C++ type of the data - /// @param source Source rank - /// @param tag Message tag - /// @param value Variable to recieve the data - /// @return `Status` of the operation - template - Status recv(int source, int tag, std::vector & value) const; - - /// Receive a message from a remote process without any data - /// - /// @param source Source rank - /// @param tag Message tag - /// @return `Status` of the operation - Status recv(int source, int tag) const; - - /// Send a message to a remote process without blocking - /// - /// @tparam T C++ type of the data - /// @param dest Destination rank - /// @param tag Message tag - /// @param value Value to send - /// @return Communication `Request` - template - Request isend(int dest, int tag, const T & value) const; - - /// Send a std::vector of values to a remote process without blocking - /// - /// @tparam T C++ type of the data - /// @param dest Destination rank - /// @param tag Message tag - /// @param value Values to send - /// @return Communication `Request` - template - Request isend(int dest, int tag, const std::vector & values) const; - - /// Send a message to a remote process without blocking - /// - /// @tparam T C++ type of the data - /// @param dest Destination rank - /// @param tag Message tag - /// @param values Values to send - /// @param n Number of values to send - /// @return Communication `Request` - template - Request isend(int dest, int tag, const T * values, int n) const; - - /// Receive a message from a remote process without blocking - /// - /// @tparam T C++ type of the data - /// @param source Source rank - /// @param tag Message tag - /// @param value Variable to recieve the data - /// @return Communication `Request` - template - Request irecv(int source, int tag, T & value) const; - - /// Receive a message from a remote process without blocking - /// - /// @tparam T C++ type of the data - /// @param source Source rank - /// @param tag Message tag - /// @param values Variable to recieve the data - /// @param n Number of values to receive - template - Request irecv(int source, int tag, T * values, int n) const; - - /// Nonblocking test for a message - /// - /// @param source Rank of source or `ANY_SOURCE` - /// @param tag Message tag or `ANY_TAG` - /// @return `true` if a message with the specified source, and tag is available, `false` - /// otherwise - bool iprobe(int source, int tag) const; - - /// Nonblocking test for a message - /// - /// @param source Rank of source or `ANY_SOURCE` - /// @param tag Message tag or `ANY_TAG` - /// @param status Status object - /// @return `true` if a message with the specified source, and tag is available, `false` - /// otherwise - bool iprobe(int source, int tag, Status & status) const; - - /// Wait for all processes within a communicator to reach the barrier. - void barrier() const; - - /// Broadcast a value from a root process to all other processes - /// - /// @tparam T C++ type of the data - /// @param value Value to send - /// @param root Rank of the sending process - template - void broadcast(T & value, int root) const; - - /// Broadcast a std::vector of values from a root process to all other processes - /// - /// @tparam T C++ type of the data - /// @param value Value to send - /// @param root Rank of the sending process - template - void broadcast(std::vector & value, int root) const; - - /// Broadcast a value from a root process to all other processes - /// - /// @tparam T C++ type of the data - /// @param values Values to send - /// @param n Number of values to send - /// @param root Rank of the sending process - template - void broadcast(T * values, int n, int root) const; - - /// Broadcast a map from a root process to all other processes - /// - /// @tparam KEY Key in the map - /// @tparam VALUE Value in the map - /// @param map map to send - /// @param root Rank of the sending process - template - void broadcast(std::map & map, int root) const; - - /// Gather together values from a group of processes - /// - /// @tparam T C++ type of the data - /// @param in_value Value to send - /// @param out_values Receiving variable - /// @param root Rank of receiving process - template - void gather(const T & in_value, T * out_values, int root) const; - - /// Gather together values from a group of processes - /// - /// @tparam T C++ type of the data - /// @param in_value Value to send - /// @param out_values Receiving variable - /// @param root Rank of receiving process - template - void gather(const T & in_value, std::vector & out_values, int root) const; - - /// Gather together values from a group of processes - /// - /// @tparam T C++ type of the data - /// @param in_values Values to send - /// @param n Number of values to send - /// @param out_values Receiving variable - /// @param root Rank of receiving process - template - void gather(const T * in_values, int n, T * out_values, int root) const; - - /// Gather together values from a group of processes - /// - /// @tparam T C++ type of the data - /// @param in_values Values to send - /// @param n Number of values to send - /// @param out_values Receiving variable - /// @param root Rank of receiving process - template - void gather(const T * in_values, int n, std::vector & out_values, int root) const; - - /// Gather together values from a group of processes - /// - /// @tparam T C++ datatype - /// @param in_values[in] Values to send - /// @param out_values[out] Buffer to receive the data - /// @param out_counts[in] Integer array (of length group size) containing the number of elements - /// that are to be received from each process - /// @param out_offsets[in] Integer array (of length group size). Entry `i` specifies the - /// displacement (relative to out_values) at which to place the incoming data from - /// process `i` - /// @param root Rank of receiving process - template - void gather(const std::vector & in_values, - std::vector & out_values, - const std::vector & out_counts, - const std::vector & out_offsets, - int root) const; - - /// Gathers data from all tasks and distribute the combined data to all tasks - /// - /// @tparam T C++ type of the data - /// @param in_value Value to send - /// @param n Number of values to send - /// @param out_values Receiving variable - /// @param m Number of values to receive - template - void all_gather(const T * in_value, int n, T * out_values, int m) const; - - /// Gathers data from all tasks and distribute the combined data to all tasks - /// - /// @tparam T C++ type of the data - /// @param in_value Value to send - /// @param out_values Receiving variable - template - void all_gather(const T & in_value, std::vector & out_values) const; - - /// Gathers data from all tasks and distribute the combined data to all tasks - /// - /// @tparam T C++ type of the data - /// @param in_value Vector of values to send (the size of this vector can be different on every - /// process) - /// @param out_values Receiving variable - template - void all_gather(const std::vector & in_value, std::vector & out_values) const; - - /// Gathers data from all tasks and deliver the combined data to all tasks - /// - /// @tparam T C++ datatype - /// @param in_values[in] Values to send - /// @param out_values[out] Buffer to receive the data - /// @param out_counts[in] Integer array (of length group size) containing the number of elements - /// that are to be received from each process - /// @param out_offsets[in] Integer array (of length group size). Entry `i` specifies the - /// displacement (relative to out_values) at which to place the incoming data from - /// process `i` - template - void all_gather(const std::vector & in_values, - std::vector & out_values, - const std::vector & out_counts, - const std::vector & out_offsets) const; - - /// Send data from one process to all other processes in a communicator - /// - /// @tparam T C++ type of the data - /// @param in_values Values to send - /// @param out_value Receiving variable - /// @param root Rank of the sending process - template - void scatter(const T * in_values, T & out_value, int root) const; - - /// Send data from one process to all other processes in a communicator - /// - /// @tparam T C++ type of the data - /// @param in_values Values to send - /// @param out_value Receiving variable - /// @param root Rank of the sending process - template - void scatter(const std::vector & in_values, T & out_value, int root) const; - - /// Send data from one process to all other processes in a communicator - /// - /// @tparam T C++ type of the data - /// @param in_values Values to send - /// @param out_values Receiving variable - /// @param n Number of values to send - /// @param root Rank of the sending process - template - void scatter(const T * in_values, T * out_values, int n, int root) const; - - /// Send data from one process to all other processes in a communicator - /// - /// @tparam T C++ type of the data - /// @param in_values Values to send - /// @param out_values Receiving variable - /// @param n Number of values to send - /// @param root Rank of the sending process - template - void scatter(const std::vector & in_values, T * out_values, int n, int root) const; - - /// Reduce values on all processes to a single value - /// - /// @tparam T C++ type of the data - /// @tparam Op Type of the reduce operation - /// @param in_values Values to send - /// @param n Number of values to send - /// @param out_values Receiving variable - /// @param op Reduce operation - /// @param root Rank of root process - template - void reduce(const T * in_values, int n, T * out_values, Op op, int root) const; - - /// Reduce values on all processes to a single value - /// - /// @tparam T C++ type of the data - /// @tparam Op Type of the reduce operation - /// @param in_values Values to send - /// @param out_values Receiving variable - /// @param op Reduce operation - /// @param root Rank of root process - template - void - reduce(std::vector const & in_values, std::vector & out_values, Op op, int root) const; - - /// Reduce values on all processes to a single value - /// - /// @tparam T C++ type of the data - /// @tparam Op Type of the reduce operation - /// @param in_value Values to send - /// @param out_value Receiving variable - /// @param op Reduce operation - /// @param root Rank of root process - template - void reduce(const T & in_value, T & out_value, Op op, int root) const; - - /// Combine values from all processes and distributes the result back to all processes - /// - /// @tparam T C++ type of the data - /// @tparam Op Type of the reduce operation - /// @param in_values Values to send - /// @param n Number of values to send - /// @param out_values Receiving variable - /// @param op Reduce operation - template - void all_reduce(const T * in_values, int n, T * out_values, Op op) const; - - /// Combine values from all processes and distributes the result back to all processes - /// - /// @tparam T C++ type of the data - /// @tparam Op Type of the reduce operation - /// @param in_values Values to send - /// @param out_values Receiving variable - /// @param op Reduce operation - template - void all_reduce(const std::vector & in_values, std::vector & out_values, Op op) const; - - /// Combine values from all processes and distributes the result back to all processes - /// - /// @tparam T C++ type of the data - /// @tparam Op Type of the reduce operation - /// @param in_value Values to send - /// @param out_value Receiving variable - /// @param op Reduce operation - template - void all_reduce(const T & in_value, T & out_value, Op op) const; - - /// Combine values from all processes and distributes the result back to all processes - /// - /// @tparam T C++ type of the data - /// @tparam Op Type of the reduce operation - /// @param value Send/receive variable - /// @param op Reduce operation - template - void all_reduce(T & value, Op op) const; - - /// Sends data from all to all processes - /// - /// @tparam T C++ type of the data - /// @param in_values Values to send - /// @param n Number valus to send - /// @param out_values Receiving variable - /// @param m Values to receive - template - void all_to_all(const T * in_values, int n, T * out_values, int m) const; - - /// Sends data from all to all processes - /// - /// @tparam T C++ type of the data - /// @param in_values Values to send - /// @param out_values Receiving variable - template - void all_to_all(const std::vector & in_values, std::vector & out_values) const; - - /// Sends data from all to all processes - template - void all_to_all(const std::vector> & in_values, - std::vector & out_values) const; - - /// Sends data from all to all processes; each process may send a different amount of data and - /// provide displacements for the input and output data. - /// - /// @tparam T C++ datatype - /// @param in_values[in] Values to send - /// @param in_counts[in] Integer array equal to the group size specifying the number of elements - /// to send to each processor - /// @param in_offsets[in] integer array (of length group size). Entry `j` specifies the - /// displacement relative to sendbuf from which to take the outgoing data destined for - /// process `j` - /// @param out_values[out] Buffer that will receive the data - /// @param out_counts[in] Integer array equal to the group size specifying the maximum number of - /// elements that can be received from each processor - /// @param out_offsets[in] Integer array (of length group size). Entry `i` specifies the - /// displacement relative to recvbuf at which to place the incoming data from process `i` - template - void all_to_all(const std::vector & in_values, - const std::vector & in_counts, - const std::vector & in_offsets, - std::vector & out_values, - const std::vector & out_counts, - const std::vector & out_offsets) const; - - /// Abort all tasks in the group of this communicator - /// - /// @param errcode Error code to return to invoking environment - void abort(int errcode) const; - - /// Cast operator so we can pass this directly in MPI API - operator MPI_Comm() const; - - void set_error_handler(); - -private: - MPI_Comm comm; -}; - -// - -inline Communicator::Communicator() : comm(MPI_COMM_WORLD) {} - -inline Communicator::Communicator(const MPI_Comm & comm) : comm(comm) {} - -inline Communicator::Communicator(const Communicator & comm) : comm(comm.comm) {} - -inline int -Communicator::rank() const -{ - int r; - MPI_Comm_rank(this->comm, &r); - return r; -} - -inline int -Communicator::size() const -{ - int sz; - MPI_Comm_size(this->comm, &sz); - return sz; -} - -inline Communicator -Communicator::create(const Group & group, int tag) const -{ - MPI_Comm new_comm; - MPI_CHECK_SELF(MPI_Comm_create_group(this->comm, group, tag, &new_comm)); - return { new_comm }; -} - -inline Group -Communicator::group() const -{ - MPI_Group g; - MPI_CHECK_SELF(MPI_Comm_group(this->comm, &g)); - return { g }; -} - -inline bool -Communicator::is_valid() const -{ - return this->comm != MPI_COMM_NULL; -} - -// Send - -template -inline void -Communicator::send(int dest, int tag, const T & value) const -{ - send(dest, tag, &value, 1); -} - -template -inline void -Communicator::send(int dest, int tag, const T * values, int n) const -{ - MPI_CHECK_SELF( - MPI_Send(const_cast(values), n, get_mpi_datatype(), dest, tag, this->comm)); -} - -template -inline void -Communicator::send(int dest, int tag, const std::vector & value) const -{ - typename std::vector::size_type size = value.size(); - send(dest, tag, value.data(), size); -} - -inline void -Communicator::send(int dest, int tag) const -{ - MPI_CHECK_SELF(MPI_Send(MPI_BOTTOM, 0, MPI_PACKED, dest, tag, this->comm)); -} - -template <> -inline void -Communicator::send(int dest, int tag, const std::string & value) const -{ - if (size() < 2) - return; - send(dest, tag, value.data(), value.size()); -} - -// Recv - -template -inline Status -Communicator::recv(int source, int tag, T & value) const -{ - return recv(source, tag, &value, 1); -} - -template -inline Status -Communicator::recv(int source, int tag, T * values, int n) const -{ - MPI_Status status = { 0 }; - MPI_CHECK_SELF(MPI_Recv(const_cast(values), - n, - get_mpi_datatype(), - source, - tag, - this->comm, - &status)); - return { status }; -} - -template -inline Status -Communicator::recv(int source, int tag, std::vector & values) const -{ - MPI_Status status = { 0 }; - MPI_CHECK_SELF(MPI_Probe(source, tag, this->comm, &status)); - int size = 0; - MPI_Get_count(&status, get_mpi_datatype(), &size); - values.resize(size); - return recv(source, tag, values.data(), size); -} - -inline Status -Communicator::recv(int source, int tag) const -{ - MPI_Status status = { 0 }; - MPI_CHECK_SELF(MPI_Recv(MPI_BOTTOM, 0, MPI_PACKED, source, tag, this->comm, &status)); - return { status }; -} - -template <> -inline Status -Communicator::recv(int source, int tag, std::string & value) const -{ - std::vector str; - auto status = recv(source, tag, str); - value.assign(str.begin(), str.end()); - return status; -} - -// Isend - -template -inline Request -Communicator::isend(int dest, int tag, const T & value) const -{ - return isend(dest, tag, &value, 1); -} - -template -inline Request -Communicator::isend(int dest, int tag, const std::vector & values) const -{ - return isend(dest, tag, values.data(), values.size()); -} - -template -inline Request -Communicator::isend(int dest, int tag, const T * values, int n) const -{ - assert(values != nullptr); - MPI_Request request; - MPI_CHECK_SELF(MPI_Isend(const_cast(values), - n, - get_mpi_datatype(), - dest, - tag, - this->comm, - &request)); - return { request }; -} - -// Irecv - -template -inline Request -Communicator::irecv(int source, int tag, T & value) const -{ - return irecv(source, tag, &value, 1); -} - -template -inline Request -Communicator::irecv(int source, int tag, T * values, int n) const -{ - assert(values != nullptr); - MPI_Request request; - MPI_CHECK_SELF(MPI_Irecv(const_cast(values), - n, - get_mpi_datatype(), - source, - tag, - this->comm, - &request)); - return { request }; -} - -inline bool -Communicator::iprobe(int source, int tag) const -{ - int flag; - MPI_CHECK_SELF(MPI_Iprobe(source, tag, this->comm, &flag, MPI_STATUS_IGNORE)); - return flag != 0; -} - -inline bool -Communicator::iprobe(int source, int tag, Status & status) const -{ - int flag; - MPI_CHECK_SELF(MPI_Iprobe(source, tag, this->comm, &flag, status)); - return flag != 0; -} - -// Barrier - -inline void -Communicator::barrier() const -{ - MPI_CHECK_SELF(MPI_Barrier(this->comm)); -} - -// Broadcast - -template -inline void -Communicator::broadcast(T & value, int root) const -{ - broadcast(&value, 1, root); -} - -template -inline void -Communicator::broadcast(std::vector & value, int root) const -{ - if (size() < 2) - return; - - int tag = 0; - if (rank() == root) { - for (int i = 0; i < size(); ++i) { - if (i != root) - send(i, tag, value.data(), value.size()); - } - } - else - recv(root, tag, value); -} - -template -inline void -Communicator::broadcast(T * values, int n, int root) const -{ - MPI_CHECK_SELF(MPI_Bcast(values, n, get_mpi_datatype(), root, this->comm)); -} - -template -inline void -Communicator::broadcast(std::map & map, int root) const -{ - auto n = map.size(); - broadcast(n, root); - int tag = 0; - if (rank() == root) { - for (const auto & [key, value] : map) { - for (int i = 0; i < size(); ++i) { - if (i != root) { - send(i, tag, key); - send(i, tag, value); - } - } - } - } - else { - for (std::size_t i = 0; i < n; i++) { - KEY key; - recv(root, tag, key); - VALUE value; - recv(root, tag, value); - map[key] = value; - } - } -} - -template <> -inline void -Communicator::broadcast(std::string & value, int root) const -{ - if (size() < 2) - return; - - int tag = 0; - if (rank() == root) { - for (int i = 0; i < size(); ++i) { - if (i != root) - send(i, tag, value.data(), value.size()); - } - } - else { - std::vector str; - recv(root, tag, str); - value.assign(str.begin(), str.end()); - } -} - -// Gather - -template -inline void -Communicator::gather(const T & in_value, T * out_values, int root) const -{ - assert(out_values || (rank() != root)); - gather(&in_value, 1, out_values, root); -} - -template -inline void -Communicator::gather(const T & in_value, std::vector & out_values, int root) const -{ - if (rank() == root) - out_values.resize(size()); - gather(in_value, out_values.data(), root); -} - -template -inline void -Communicator::gather(const T * in_values, int n, T * out_values, int root) const -{ - auto type = get_mpi_datatype(); - MPI_CHECK_SELF( - MPI_Gather(const_cast(in_values), n, type, out_values, n, type, root, this->comm)); -} - -template -inline void -Communicator::gather(const T * in_values, int n, std::vector & out_values, int root) const -{ - if (rank() == root) - out_values.resize(size() * (std::size_t) n); - gather(in_values, n, out_values.data(), root); -} - -template -inline void -Communicator::gather(const std::vector & in_values, - std::vector & out_values, - const std::vector & out_counts, - const std::vector & out_offsets, - int root) const -{ - assert(out_counts.size() == size()); - assert(out_offsets.size() == size()); - int n_out_vals = 0; - for (std::size_t i = 0; i < out_counts.size(); i++) - n_out_vals += out_counts[i]; - out_values.resize(n_out_vals); - MPI_CHECK_SELF(MPI_Gatherv(in_values.data(), - in_values.size(), - get_mpi_datatype(), - out_values.data(), - out_counts.data(), - out_offsets.data(), - get_mpi_datatype(), - root, - this->comm)); -} - -template -inline void -Communicator::all_gather(const T * in_value, int n, T * out_values, int m) const -{ - MPI_CHECK_SELF(MPI_Allgather(in_value, - n, - get_mpi_datatype(), - out_values, - m, - get_mpi_datatype(), - this->comm)); -} - -template -inline void -Communicator::all_gather(const T & in_value, std::vector & out_values) const -{ - out_values.resize(this->size()); - all_gather(&in_value, 1, out_values.data(), 1); -} - -template -inline void -Communicator::all_gather(const std::vector & in_values, std::vector & out_values) const -{ - if (size() < 2) - out_values = in_values; - else { - std::vector n; - int sz = in_values.size(); - all_gather(sz, n); - std::vector offsets(size()); - offsets[0] = 0; - for (int i = 0; i < n.size() - 1; i++) - offsets[i + 1] = offsets[i] + n[i]; - all_gather(in_values, out_values, n, offsets); - } -} - -template -inline void -Communicator::all_gather(const std::vector & in_values, - std::vector & out_values, - const std::vector & out_counts, - const std::vector & out_offsets) const -{ - assert(out_counts.size() == size()); - assert(out_offsets.size() == size()); - int n_out_vals = 0; - for (std::size_t i = 0; i < out_counts.size(); i++) - n_out_vals += out_counts[i]; - out_values.resize(n_out_vals); - MPI_CHECK_SELF(MPI_Allgatherv(in_values.data(), - in_values.size(), - get_mpi_datatype(), - out_values.data(), - out_counts.data(), - out_offsets.data(), - get_mpi_datatype(), - this->comm)); -} - -// Scatter - -template -inline void -Communicator::scatter(const T * in_values, T & out_value, int root) const -{ - scatter(in_values, &out_value, 1, root); -} - -template -inline void -Communicator::scatter(const std::vector & in_values, T & out_value, int root) const -{ - scatter(in_values.data(), &out_value, 1, root); -} - -template -inline void -Communicator::scatter(const T * in_values, T * out_values, int n, int root) const -{ - auto type = get_mpi_datatype(); - MPI_CHECK_SELF( - MPI_Scatter(const_cast(in_values), n, type, out_values, n, type, root, this->comm)); -} - -template -inline void -Communicator::scatter(const std::vector & in_values, T * out_values, int n, int root) const -{ - scatter(in_values.data(), out_values, n, root); -} - -// Reduce - -template -inline void -Communicator::reduce(const T * in_values, int n, T * out_values, Op, int root) const -{ - MPI_Op op = mpicpp_lite::op::Operation::op(); - MPI_CHECK_SELF(MPI_Reduce(const_cast(in_values), - out_values, - n, - mpicpp_lite::get_mpi_datatype(), - op, - root, - this->comm)); -} - -template -inline void -Communicator::reduce(std::vector const & in_values, - std::vector & out_values, - Op op, - int root) const -{ - if (root == rank()) - out_values.resize(in_values.size()); - reduce(in_values.data(), in_values.size(), out_values.data(), op, root); -} - -template -inline void -Communicator::reduce(const T & in_value, T & out_value, Op op, int root) const -{ - reduce(&in_value, 1, &out_value, op, root); -} - -// All reduce - -template -inline void -Communicator::all_reduce(const T * in_values, int n, T * out_values, Op) const -{ - MPI_Op op = mpicpp_lite::op::Operation::op(); - MPI_CHECK_SELF(MPI_Allreduce(const_cast(in_values), - out_values, - n, - mpicpp_lite::get_mpi_datatype(), - op, - this->comm)); -} - -template -inline void -Communicator::all_reduce(const std::vector & in_values, std::vector & out_values, Op op) const -{ - assert(in_values.size() == out_values.size()); - all_reduce(in_values.data(), in_values.size(), out_values.data(), op); -} - -template -inline void -Communicator::all_reduce(const T & in_value, T & out_value, Op op) const -{ - all_reduce(&in_value, 1, &out_value, op); -} - -template -inline void -Communicator::all_reduce(T & in_value, Op op) const -{ - T out_value; - all_reduce(&in_value, 1, &out_value, op); - in_value = out_value; -} - -template -inline void -Communicator::all_to_all(const T * in_values, int n, T * out_values, int m) const -{ - MPI_CHECK_SELF(MPI_Alltoall(in_values, - n, - get_mpi_datatype(), - out_values, - m, - get_mpi_datatype(), - this->comm)); -} - -template -inline void -Communicator::all_to_all(const std::vector & in_values, std::vector & out_values) const -{ - assert(in_values.size() == size()); - out_values.resize(size()); - all_to_all(in_values.data(), 1, out_values.data(), 1); -} - -template -inline void -Communicator::all_to_all(const std::vector> & in_values, - std::vector & out_values) const -{ - std::vector in_count(size(), 0); - for (std::size_t i = 0; i < in_values.size(); i++) - in_count[i] = in_values[i].size(); - std::vector out_count(size(), 0); - all_to_all(in_count, out_count); - - std::vector in_offsets(size()); - in_offsets[0] = 0; - for (std::size_t i = 0; i < in_count.size() - 1; i++) - in_offsets[i + 1] = in_offsets[i] + in_count[i]; - std::size_t n_send_vals = 0; - for (std::size_t i = 0; i < in_count.size(); i++) - n_send_vals += in_count[i]; - - std::vector out_offsets(size()); - out_offsets[0] = 0; - for (std::size_t i = 0; i < out_count.size() - 1; i++) - out_offsets[i + 1] = out_offsets[i] + out_count[i]; - - std::vector in_buffer; - in_buffer.reserve(n_send_vals); - for (auto & vec : in_values) - for (auto & v : vec) - in_buffer.push_back(v); - - all_to_all(in_buffer, in_count, in_offsets, out_values, out_count, out_offsets); -} - -template -inline void -Communicator::all_to_all(const std::vector & in_values, - const std::vector & in_counts, - const std::vector & in_offsets, - std::vector & out_values, - const std::vector & out_counts, - const std::vector & out_offsets) const -{ - assert(in_counts.size() == size()); - assert(in_offsets.size() == size()); - assert(out_counts.size() == size()); - assert(out_offsets.size() == size()); - int n_receive_vals = 0; - for (std::size_t i = 0; i < out_counts.size(); i++) - n_receive_vals += out_counts[i]; - out_values.resize(n_receive_vals); - MPI_CHECK_SELF(MPI_Alltoallv(in_values.data(), - in_counts.data(), - in_offsets.data(), - get_mpi_datatype(), - out_values.data(), - out_counts.data(), - out_offsets.data(), - get_mpi_datatype(), - this->comm)); -} - -// - -inline void -Communicator::abort(int errcode) const -{ - MPI_Abort(this->comm, errcode); -} - -inline Communicator::operator MPI_Comm() const -{ - return this->comm; -} - -inline void -Communicator::set_error_handler() -{ -#if (MPI_VERSION >= 2) - MPI_Comm_set_errhandler(this->comm, MPI_ERRORS_RETURN); -#else - MPI_Errhandler_set(this->comm, MPI_ERRORS_RETURN); -#endif -} - -} // namespace mpicpp_lite diff --git a/external/mpicpp-lite/impl/Datatype.h b/external/mpicpp-lite/impl/Datatype.h deleted file mode 100644 index afc20bdc5..000000000 --- a/external/mpicpp-lite/impl/Datatype.h +++ /dev/null @@ -1,141 +0,0 @@ -// SPDX-FileCopyrightText: 2023 David Andrs -// SPDX-License-Identifier: MIT - -#pragma once - -#include "mpi.h" - -namespace mpicpp_lite { - -/// Create a new datatype for MPI communication -/// -/// @tparam T Datatype -/// @return New `MPI_Datatype` -template -inline MPI_Datatype -build_mpi_datatype() -{ - return MPI_DATATYPE_NULL; -} - -/// General template to obtain an MPI_Datatype from a C++ type -/// -/// @tparam T C++ data type -/// @return `MPI_Datatype` that is used in the MPI API -template -inline MPI_Datatype -get_mpi_datatype() -{ - return MPI_DATATYPE_NULL; -} - -template <> -inline MPI_Datatype -get_mpi_datatype() -{ - return MPI_BYTE; -} - -template <> -inline MPI_Datatype -get_mpi_datatype() -{ - return MPI_SHORT; -} - -template <> -inline MPI_Datatype -get_mpi_datatype() -{ - return MPI_INT; -} - -template <> -inline MPI_Datatype -get_mpi_datatype() -{ - return MPI_LONG; -} - -template <> -inline MPI_Datatype -get_mpi_datatype() -{ - return MPI_LONG_LONG; -} - -template <> -inline MPI_Datatype -get_mpi_datatype() -{ - return MPI_UNSIGNED_CHAR; -} - -template <> -inline MPI_Datatype -get_mpi_datatype() -{ - return MPI_UNSIGNED_SHORT; -} - -template <> -inline MPI_Datatype -get_mpi_datatype() -{ - return MPI_UNSIGNED; -} - -template <> -inline MPI_Datatype -get_mpi_datatype() -{ - return MPI_UNSIGNED_LONG; -} - -template <> -inline MPI_Datatype -get_mpi_datatype() -{ - return MPI_UNSIGNED_LONG_LONG; -} - -template <> -inline MPI_Datatype -get_mpi_datatype() -{ - return MPI_FLOAT; -} - -template <> -inline MPI_Datatype -get_mpi_datatype() -{ - return MPI_DOUBLE; -} - -template <> -inline MPI_Datatype -get_mpi_datatype() -{ - return MPI_LONG_DOUBLE; -} - -template <> -inline MPI_Datatype -get_mpi_datatype() -{ - return MPI_CXX_BOOL; -} - -#if __cplusplus >= 201703L - -template <> -inline MPI_Datatype -get_mpi_datatype() -{ - return MPI_BYTE; -} - -#endif - -} // namespace mpicpp_lite diff --git a/external/mpicpp-lite/impl/Enums.h b/external/mpicpp-lite/impl/Enums.h deleted file mode 100644 index 1eaf612aa..000000000 --- a/external/mpicpp-lite/impl/Enums.h +++ /dev/null @@ -1,8 +0,0 @@ -// SPDX-FileCopyrightText: 2023 David Andrs -// SPDX-License-Identifier: MIT - -#pragma once - -#include "mpi.h" - -enum { UNDEFINED = MPI_UNDEFINED, ANY_SOURCE = MPI_ANY_SOURCE, ANY_TAG = MPI_ANY_TAG }; diff --git a/external/mpicpp-lite/impl/Environment.h b/external/mpicpp-lite/impl/Environment.h deleted file mode 100644 index 66a21bc00..000000000 --- a/external/mpicpp-lite/impl/Environment.h +++ /dev/null @@ -1,74 +0,0 @@ -// SPDX-FileCopyrightText: 2023 David Andrs -// SPDX-License-Identifier: MIT - -#pragma once - -#include "mpi.h" -#include "Error.h" - -namespace mpicpp_lite { - -class Environment { -public: -#if (MPI_VERSION >= 2) - Environment(); -#endif - Environment(int & argc, char **& argv); - ~Environment(); - - /// Indicates whether MPI was initialized - /// - /// @return `true` if MPI is initialized, `false` otherwise - static bool is_initialized(); - - /// Indicates whether MPI was finalized - /// - /// @return `true` if MPI is finalized, `false` otherwise - static bool is_finalized(); - -private: - /// Indicates if the environment is initialized - bool initialized; -}; - -inline Environment::Environment() : initialized(false) -{ - if (!is_initialized()) { - MPI_CHECK(MPI_Init(nullptr, nullptr)); - this->initialized = true; - } -} - -inline Environment::Environment(int & argc, char **& argv) : initialized(false) -{ - if (!is_initialized()) { - MPI_CHECK(MPI_Init(&argc, &argv)); - this->initialized = true; - } -} - -inline Environment::~Environment() -{ - if (this->initialized) { - if (!is_finalized()) - MPI_CHECK(MPI_Finalize()); - } -} - -inline bool -Environment::is_initialized() -{ - int flag; - MPI_CHECK(MPI_Initialized(&flag)); - return flag != 0; -} - -inline bool -Environment::is_finalized() -{ - int flag; - MPI_CHECK(MPI_Finalized(&flag)); - return flag != 0; -} - -} // namespace mpicpp_lite diff --git a/external/mpicpp-lite/impl/Error.h b/external/mpicpp-lite/impl/Error.h deleted file mode 100644 index 653d837c1..000000000 --- a/external/mpicpp-lite/impl/Error.h +++ /dev/null @@ -1,74 +0,0 @@ -// SPDX-FileCopyrightText: 2023 David Andrs -// SPDX-License-Identifier: MIT - -#pragma once - -#ifdef MPICPP_LITE_WITH_FMT - #include "fmt/printf.h" -#else - #include -#endif -#include "mpi.h" - -namespace mpicpp_lite { - -inline int -error_class(int code) -{ - int err_class; - MPI_Error_class(code, &err_class); - return err_class; -} - -/// Return an error message for a given error code -/// -/// @param code Error code -/// @return Error message -inline std::string -error_message(int code) -{ - char buffer[MPI_MAX_ERROR_STRING]; - int len; - MPI_Error_string(code, buffer, &len); - std::string msg(buffer); - return msg; -} - -namespace internal { - -/// Terminate the run -inline void -terminate(MPI_Comm comm, int status = 1) -{ - MPI_Abort(comm, status); -} - -inline void -check_mpi_error(MPI_Comm comm, int ierr, const char * file, int line) -{ - if (ierr) { -#ifdef MPICPP_LITE_WITH_FMT - fmt::print(stderr, - "[ERROR] MPI error {} at {}:{}: {}\n", - ierr, - file, - line, - error_message(error_class(ierr))); -#else - std::cerr << "[ERROR] MPI error " << ierr << " at " << file << ":" << line << ": " - << error_message(error_class(ierr)) << std::endl; -#endif - terminate(comm, ierr); - } -} - -} // namespace internal - -/// Check that MPI call was successful. -#define MPI_CHECK_SELF(ierr) \ - mpicpp_lite::internal::check_mpi_error(this->comm, ierr, __FILE__, __LINE__) - -#define MPI_CHECK(ierr) \ - mpicpp_lite::internal::check_mpi_error(MPI_COMM_WORLD, ierr, __FILE__, __LINE__) - -} // namespace mpicpp_lite diff --git a/external/mpicpp-lite/impl/Group.h b/external/mpicpp-lite/impl/Group.h deleted file mode 100644 index fa34d84ab..000000000 --- a/external/mpicpp-lite/impl/Group.h +++ /dev/null @@ -1,195 +0,0 @@ -// SPDX-FileCopyrightText: 2023 David Andrs -// SPDX-License-Identifier: MIT - -#pragma once - -#include "mpi.h" -#include "Enums.h" -#include "Error.h" - -namespace mpicpp_lite { - -/// Wrapper around `MPI_Group` -class Group { -public: - enum ComparisonResult { IDENTICAL = MPI_IDENT, SIMILAR = MPI_SIMILAR, UNEQUAL = MPI_UNEQUAL }; - - /// Create an empty group - Group(); - - /// Create group from an `MPI_Group` - /// - /// @param group `MPI_Group` used to initialize this object - Group(const MPI_Group & group); - - /// Returns the rank of this process in the given group - /// - /// @return Rank of the calling process in group, or MPI_UNDEFINED if the process is not a - /// member - int rank() const; - - /// Returns the size of a group - /// - /// @return Number of processes in the group - int size() const; - - /// Produces a group by reordering an existing group and taking only listed members - /// - /// @param ranks Ranks of processes in group to appear in the new group - /// @return New group derived from this group, in the order defined by `ranks` - Group include(const std::vector & ranks) const; - - /// Produces a group by reordering an existing group and taking only unlisted members - /// - /// @param ranks Array of integer ranks of processes in group not to appear in the new group - /// @return New group derived from this group, preserving the order defined by group - Group exclude(const std::vector & ranks) const; - - /// Frees the group - void free(); - - /// Translates the rank of a process in one group to the one in another group - /// - /// @param in_rank Rank in this group - /// @param out_group Destination group - /// @return Rank in destination group, or `UNDEFINED` when no correspondence exists - int translate_rank(int in_rank, const Group & out_group) const; - - /// Translates the ranks of processes in one group to those in another group - /// - /// @param in_ranks Array of valid ranks in this group - /// @param out_group Destination group - /// @return Array of corresponding ranks in destination group, `UNDEFINED` when no - /// correspondence exists - std::vector translate_ranks(const std::vector & in_ranks, - const Group & out_group) const; - - /// Type cast operator so we can pass this class directly into MPI calls - operator const MPI_Group &() const { return this->group; } - -public: - /// Compares two groups - /// - /// @param g1 First group - /// @param g2 Second group - /// @return `IDENTICAL` if the order and members of the two groups are the same, `SIMILAR` if - /// only the members are the same, and `UNEQUAL` otherwise - static ComparisonResult compare(const Group & g1, const Group & g2); - - /// Produces a group by combining two groups - /// - /// @param g1 First group - /// @param g2 Second group - /// @return Union group - static Group join(const Group & g1, const Group & g2); - - /// Produces a group as the intersection of two existing groups - /// - /// @param g1 First group - /// @param g2 Second group - /// @return Intersection group - static Group intersection(const Group & g1, const Group & g2); - - /// Makes a group from the difference of two groups - /// - /// @param g1 First group - /// @param g2 Second group - /// @return Difference group - static Group difference(const Group & g1, const Group & g2); - -private: - MPI_Group group; -}; - -inline Group::Group() {} - -inline Group::Group(const MPI_Group & group) : group(group) {} - -inline Group -Group::include(const std::vector & ranks) const -{ - MPI_Group new_group; - MPI_CHECK(MPI_Group_incl(this->group, ranks.size(), ranks.data(), &new_group)); - return { new_group }; -} - -inline Group -Group::exclude(const std::vector & ranks) const -{ - MPI_Group new_group; - MPI_CHECK(MPI_Group_excl(this->group, ranks.size(), ranks.data(), &new_group)); - return { new_group }; -} - -inline void -Group::free() -{ - MPI_CHECK(MPI_Group_free(&this->group)); -} - -inline int -Group::rank() const -{ - int r; - MPI_CHECK(MPI_Group_rank(this->group, &r)); - return r; -} - -inline int -Group::size() const -{ - int sz; - MPI_CHECK(MPI_Group_size(this->group, &sz)); - return sz; -} - -inline int -Group::translate_rank(int in_rank, const Group & out_group) const -{ - int out_rank; - MPI_CHECK(MPI_Group_translate_ranks(this->group, 1, &in_rank, out_group, &out_rank)); - return out_rank; -} - -inline std::vector -Group::translate_ranks(const std::vector & in_ranks, const Group & out_group) const -{ - int n = in_ranks.size(); - std::vector ranks2(n); - MPI_CHECK(MPI_Group_translate_ranks(this->group, n, in_ranks.data(), out_group, ranks2.data())); - return ranks2; -} - -inline Group::ComparisonResult -Group::compare(const Group & g1, const Group & g2) -{ - int result; - MPI_CHECK(MPI_Group_compare(g1, g2, &result)); - return static_cast(result); -} - -inline Group -Group::join(const Group & g1, const Group & g2) -{ - MPI_Group new_group; - MPI_CHECK(MPI_Group_union(g1, g2, &new_group)); - return { new_group }; -} - -inline Group -Group::intersection(const Group & g1, const Group & g2) -{ - MPI_Group new_group; - MPI_CHECK(MPI_Group_intersection(g1, g2, &new_group)); - return { new_group }; -} - -inline Group -Group::difference(const Group & g1, const Group & g2) -{ - MPI_Group new_group; - MPI_CHECK(MPI_Group_difference(g1, g2, &new_group)); - return { new_group }; -} - -} // namespace mpicpp_lite diff --git a/external/mpicpp-lite/impl/Operation.h b/external/mpicpp-lite/impl/Operation.h deleted file mode 100644 index 8c035ae1b..000000000 --- a/external/mpicpp-lite/impl/Operation.h +++ /dev/null @@ -1,260 +0,0 @@ -// SPDX-FileCopyrightText: 2023 David Andrs -// SPDX-License-Identifier: MIT - -#pragma once - -#include "mpi.h" - -namespace mpicpp_lite { - -namespace op { - -/// Template for MPI operation `Op` on a `T` type -/// -/// @tparam Op Operation -/// @tparam T Datatype -template -struct Operation { -}; - -// Sum - -/// Template for summation operation on a `T` type -/// -/// @tparam T Datatype -template -struct sum { - /// Call operator - /// - /// @param x First operand - /// @param y Second operand - /// @return Sum of `x` and `y` - const T & - operator()(const T & x, const T & y) const - { - return x + y; - } -}; - -/// Template for summation operation on a `T` type -/// -/// @tparam T Datatype -template -struct Operation, T> { - /// Call operator - /// - /// @return MPI operation for sumation - static MPI_Op - op() - { - return MPI_SUM; - } -}; - -// Product - -/// Template for product operation on a `T` type -/// -/// @tparam T Datatype -template -struct prod { - /// Call operator - /// - /// @param x First operand - /// @param y Second operand - /// @return Product of `x` and `y` - const T & - operator()(const T & x, const T & y) const - { - return x * y; - } -}; - -/// Template for product operation on a `T` type -/// -/// @tparam T Datatype -template -struct Operation, T> { - /// Call operator - /// - /// @return MPI operation for product - static MPI_Op - op() - { - return MPI_PROD; - } -}; - -// Maximum - -/// Template for finding maximum on a `T` type -/// -/// @tparam T Datatype -template -struct max { - /// Call operator - /// - /// @param x First operand - /// @param y Second operand - /// @return The maximum of `x` and `y` - const T & - operator()(const T & x, const T & y) const - { - return x < y ? y : x; - } -}; - -/// Template for finding maximum on a `T` type -/// -/// @tparam T Datatype -template -struct Operation, T> { - /// Call operator - /// - /// @return MPI operation for finding maximum - static MPI_Op - op() - { - return MPI_MAX; - } -}; - -// Minimum - -/// Template for finding minimum on a `T` type -/// -/// @tparam T Datatype -template -struct min { - /// Call operator - /// - /// @param x First operand - /// @param y Second operand - /// @return The minimum of `x` and `y` - const T & - operator()(const T & x, const T & y) const - { - return x < y ? x : y; - } -}; - -/// Template for finding minimum on a `T` type -/// -/// @tparam T Datatype -template -struct Operation, T> { - /// Call operator - /// - /// @return MPI operation for finding minimum - static MPI_Op - op() - { - return MPI_MIN; - } -}; - -// Logical AND - -/// Template for logical AND on a `T` type -/// -/// @tparam T Datatype -template -struct logical_and { - /// Call operator - /// - /// @param x First operand - /// @param y Second operand - /// @return `x` AND `y` - const T & - operator()(const T & x, const T & y) const - { - return x && y; - } -}; - -/// Template for logical AND on a `T` type -/// -/// @tparam T Datatype -template -struct Operation, T> { - /// Call operator - /// - /// @return MPI operation logical AND - static MPI_Op - op() - { - return MPI_LAND; - } -}; - -// Logical OR - -/// Template for logical OR on a `T` type -/// -/// @tparam T Datatype -template -struct logical_or { - /// Call operator - /// - /// @param x First operand - /// @param y Second operand - /// @return `x` OR `y` - const T & - operator()(const T & x, const T & y) const - { - return x || y; - } -}; - -/// Template for logical OR on a `T` type -/// -/// @tparam T Datatype -template -struct Operation, T> { - /// Call operator - /// - /// @return MPI operation for logical OR - static MPI_Op - op() - { - return MPI_LOR; - } -}; - -// Logical XOR - -/// Template for logical XOR on a `T` type -/// -/// @tparam T Datatype -template -struct logical_xor { - /// Call operator - /// - /// @param x First operand - /// @param y Second operand - /// @return `x` XOR `y` - const T & - operator()(const T & x, const T & y) const - { - return !x != !y; - } -}; - -/// Template for logical XOR on a `T` type -/// -/// @tparam T Datatype -template -struct Operation, T> { - /// Call operator - /// - /// @return MPI operation for logical XOR - static MPI_Op - op() - { - return MPI_LXOR; - } -}; - -} // namespace op - -} // namespace mpicpp_lite diff --git a/external/mpicpp-lite/impl/Request.h b/external/mpicpp-lite/impl/Request.h deleted file mode 100644 index 39333c174..000000000 --- a/external/mpicpp-lite/impl/Request.h +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-FileCopyrightText: 2023 David Andrs -// SPDX-License-Identifier: MIT - -#pragma once - -#include "mpi.h" - -namespace mpicpp_lite { - -/// Wrapper around MPI_Request -class Request { -public: - /// Create empty request - Request(); - - /// Create request from an `MPI_Request` - /// - /// @param r `MPI_Request` used to initiliaze this object - Request(const MPI_Request & r); - - /// Type cast operator so we can pass this class directly into MPI calls - operator MPI_Request *() { return &this->request; } - - /// Type cast operator so we can pass this class directly into MPI calls - operator const MPI_Request &() const { return this->request; } - -private: - MPI_Request request; -}; - -inline Request::Request() {} - -inline Request::Request(const MPI_Request & r) : request(r) {} - -} // namespace mpicpp_lite diff --git a/external/mpicpp-lite/impl/Status.h b/external/mpicpp-lite/impl/Status.h deleted file mode 100644 index 0325df6a6..000000000 --- a/external/mpicpp-lite/impl/Status.h +++ /dev/null @@ -1,81 +0,0 @@ -// SPDX-FileCopyrightText: 2023 David Andrs -// SPDX-License-Identifier: MIT - -#pragma once - -#include "mpi.h" - -namespace mpicpp_lite { - -/// Wrapper around MPI_Status -class Status { -public: - /// Construct empty `Status` object - Status(); - - /// Construct `Status` object from `MPI_Status` structure - /// - /// @param s MPI_Status object used to initialize this object - Status(const MPI_Status & s); - - /// Get the source of the message - /// - /// @return Source of the message (i.e. rank ID) - int source() const; - - /// Get the message tag - /// - /// @return Message tag - int tag() const; - - /// Get the error code - /// - /// @return Error code - int error() const; - - /// Gets the number of "top level" elements - /// - /// @tparam T datatype of each receive buffer element - /// @return The number of "top level" elements - template - int get_count() const; - - /// Type cast operators so we can pass this class directly into the MPI API - operator MPI_Status *() { return &this->status; } - -private: - MPI_Status status; -}; - -inline Status::Status() : status({ 0 }) {} - -inline Status::Status(const MPI_Status & s) : status(s) {} - -inline int -Status::source() const -{ - return this->status.MPI_SOURCE; -} - -inline int -Status::tag() const -{ - return this->status.MPI_TAG; -} - -inline int -Status::error() const -{ - return this->status.MPI_ERROR; -} - -template -inline int -Status::get_count() const -{ - int n; - MPI_Get_count(&this->status, get_mpi_datatype(), &n); - return n; -} - -} // namespace mpicpp_lite diff --git a/external/mpicpp-lite/impl/Test.h b/external/mpicpp-lite/impl/Test.h deleted file mode 100644 index c87f1b5cd..000000000 --- a/external/mpicpp-lite/impl/Test.h +++ /dev/null @@ -1,75 +0,0 @@ -// SPDX-FileCopyrightText: 2023 David Andrs -// SPDX-License-Identifier: MIT - -#pragma once - -#include "mpi.h" -#include -#include "Request.h" -#include "Status.h" -#include "Error.h" - -namespace mpicpp_lite { - -/// Test for the completion of a request -/// -/// @param request Request to test -/// @return `true` if operation completed, `false` otherwise -inline bool -test(const Request & request) -{ - int flag; - MPI_CHECK(MPI_Test(const_cast(request), &flag, MPI_STATUS_IGNORE)); - return flag != 0; -} - -/// Test for the completion of a request with status -/// -/// @param request Request to wait for -/// @param status Status -/// @return `true` if operation completed, `false` otherwise -inline bool -test(const Request & request, Status & status) -{ - int flag; - MPI_CHECK(MPI_Test(const_cast(request), &flag, status)); - return flag != 0; -} - -/// Test for the completion of all previously initiated requests -/// -/// @param requests Requests to test -/// @return `true` only if all requests have completed, `false` otherwise -inline bool -test_all(const std::vector & requests) -{ - int n = requests.size(); - std::vector reqs; - reqs.resize(n); - for (int i = 0; i < n; i++) - reqs[i] = requests[i]; - int flag; - MPI_CHECK(MPI_Testall(n, reqs.data(), &flag, MPI_STATUSES_IGNORE)); - return flag != 0; -} - -/// Test for completion of any previously initiated requests -/// -/// @param requests Requests to test -/// @param index Index of operation that completed or `UNDEFINED` if none completed -/// @return `true` if one of the operations is complete -inline bool -test_any(const std::vector & requests, std::size_t & index) -{ - int n = requests.size(); - std::vector reqs; - reqs.resize(n); - for (int i = 0; i < n; i++) - reqs[i] = requests[i]; - int idx; - int flag; - MPI_CHECK(MPI_Testany(n, reqs.data(), &idx, &flag, MPI_STATUS_IGNORE)); - return flag != 0; -} - -} // namespace mpicpp_lite diff --git a/external/mpicpp-lite/impl/Wait.h b/external/mpicpp-lite/impl/Wait.h deleted file mode 100644 index eccab80d8..000000000 --- a/external/mpicpp-lite/impl/Wait.h +++ /dev/null @@ -1,64 +0,0 @@ -// SPDX-FileCopyrightText: 2023 David Andrs -// SPDX-License-Identifier: MIT - -#pragma once - -#include "mpi.h" -#include -#include "Request.h" -#include "Status.h" -#include "Error.h" - -namespace mpicpp_lite { - -/// Wait for a single request to complete, ignoring status -/// -/// @param request Request to wait for -inline void -wait(const Request & request) -{ - MPI_CHECK(MPI_Wait(const_cast(request), MPI_STATUS_IGNORE)); -} - -/// Wait for a single request to complete with status -/// -/// @param request Request to wait for -/// @param status Status -inline void -wait(const Request & request, Status & status) -{ - MPI_CHECK(MPI_Wait(const_cast(request), status)); -} - -/// Wait for all requests to complete -/// -/// @param requests Requests to wait for -inline void -wait_all(const std::vector & requests) -{ - int n = requests.size(); - std::vector reqs; - reqs.resize(n); - for (int i = 0; i < n; i++) - reqs[i] = requests[i]; - MPI_CHECK(MPI_Waitall(n, reqs.data(), MPI_STATUSES_IGNORE)); -} - -/// Wait for any specified request to complete -/// -/// @param requests Requests to wait for -/// @return Index of the request that completed -inline std::size_t -wait_any(const std::vector & requests) -{ - int n = requests.size(); - std::vector reqs; - reqs.resize(n); - for (int i = 0; i < n; i++) - reqs[i] = requests[i]; - int idx; - MPI_CHECK(MPI_Waitany(n, reqs.data(), &idx, MPI_STATUS_IGNORE)); - return idx; -} - -} // namespace mpicpp_lite diff --git a/external/mpicpp-lite/mpicpp-lite.h b/external/mpicpp-lite/mpicpp-lite.h deleted file mode 100644 index 16916e1f5..000000000 --- a/external/mpicpp-lite/mpicpp-lite.h +++ /dev/null @@ -1,15 +0,0 @@ -// SPDX-FileCopyrightText: 2023 David Andrs -// SPDX-License-Identifier: MIT - -#pragma once - -#include "impl/Environment.h" -#include "impl/Communicator.h" -#include "impl/Datatype.h" -#include "impl/Enums.h" -#include "impl/Group.h" -#include "impl/Operation.h" -#include "impl/Request.h" -#include "impl/Status.h" -#include "impl/Test.h" -#include "impl/Wait.h" diff --git a/modules/linear_boltzmann_solvers/discrete_ordinates_problem/sweep/communicators/cbc_async_comm.cc b/modules/linear_boltzmann_solvers/discrete_ordinates_problem/sweep/communicators/cbc_async_comm.cc index 1fe08ddd8..c1957cf3f 100644 --- a/modules/linear_boltzmann_solvers/discrete_ordinates_problem/sweep/communicators/cbc_async_comm.cc +++ b/modules/linear_boltzmann_solvers/discrete_ordinates_problem/sweep/communicators/cbc_async_comm.cc @@ -105,7 +105,7 @@ CBC_ASynchronousCommunicator::ReceiveData() mpi::Status status; if (comm.iprobe(source_rank, tag, status)) { - int num_items = status.get_count(); + int num_items = status.count(); std::vector recv_buffer(num_items); comm.recv(source_rank, status.tag(), recv_buffer.data(), num_items); ByteArray data_array(recv_buffer); diff --git a/tools/dependencies/CMakeLists.txt b/tools/dependencies/CMakeLists.txt index 2d7443cbd..f099d45db 100644 --- a/tools/dependencies/CMakeLists.txt +++ b/tools/dependencies/CMakeLists.txt @@ -22,6 +22,7 @@ set(ENV_SCRIPT "${OPENSN_ENV_DIR}/set_opensn_env.sh") find_package(MPI REQUIRED) find_package(Python3 COMPONENTS Interpreter Development REQUIRED) +find_package(mpicpp-lite 2.1 QUIET) option(ENABLE_BOOST "Install Boost" ON) option(ENABLE_PETSC "Install PETSc" ON) @@ -36,6 +37,38 @@ endif() set(INSTALLED_DEP_COUNT 0) set(ALL_DEP_TARGETS) +# mpicpp-lite install +if (TARGET mpicpp-lite::mpicpp-lite) + message(STATUS "Found mpicpp-lite: " ${MPICPP_LITE_VERSION}) + if(NOT TARGET mpicpp-lite) + add_custom_target(mpicpp-lite) + endif() + list(APPEND ALL_DEP_TARGETS mpicpp-lite) +else() + message(STATUS "No") + ExternalProject_Add(mpicpp-lite + URL https://github.com/andrsd/mpicpp-lite/archive/refs/tags/v2.1.0.tar.gz + CONFIGURE_COMMAND ${CMAKE_COMMAND} + -S + -B + -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} + -DCMAKE_BUILD_TYPE=Release + + BUILD_COMMAND "" + INSTALL_COMMAND + ${CMAKE_COMMAND} --build --target install -- + LOG_DOWNLOAD 1 + LOG_UPDATE 1 + LOG_CONFIGURE 1 + LOG_BUILD 1 + LOG_INSTALL 1 + LOG_TEST 1 + LOG_OUTPUT_ON_FAILURE 1 + ) + list(APPEND ALL_DEP_TARGETS mpicpp-lite) + math(EXPR INSTALLED_DEP_COUNT "${INSTALLED_DEP_COUNT}+1") +endif() + # Boost install (header-only) if(ENABLE_BOOST) find_package(Boost CONFIG QUIET)