diff --git a/src/serac/infrastructure/CMakeLists.txt b/src/serac/infrastructure/CMakeLists.txt index 934ec14d4..f2cb321da 100644 --- a/src/serac/infrastructure/CMakeLists.txt +++ b/src/serac/infrastructure/CMakeLists.txt @@ -21,6 +21,7 @@ set(infrastructure_headers initialize.hpp input.hpp logger.hpp + memory.hpp mpi_fstream.hpp output.hpp profiling.hpp @@ -41,9 +42,12 @@ set(infrastructure_sources terminator.cpp ) -set(infrastructure_depends axom::inlet axom::fmt axom::cli11 mfem ${serac_device_depends}) +set(infrastructure_depends axom::inlet axom::fmt axom::cli11 camp mfem ${serac_device_depends}) blt_list_append(TO infrastructure_depends ELEMENTS tribol IF TRIBOL_FOUND) blt_list_append(TO infrastructure_depends ELEMENTS caliper adiak::adiak IF SERAC_ENABLE_PROFILING) +# TODO (EBC): mfem's openmp dependency isn't being picked up on ubuntu + clang 16.0.6. +# This adds it in manually for now, but should be investigated more fully. +blt_list_append(TO infrastructure_depends ELEMENTS blt::openmp IF SERAC_ENABLE_OPENMP) list(APPEND infrastructure_depends blt::mpi) blt_add_library( diff --git a/src/serac/infrastructure/accelerator.hpp b/src/serac/infrastructure/accelerator.hpp index 6da2af89d..0169a10e4 100644 --- a/src/serac/infrastructure/accelerator.hpp +++ b/src/serac/infrastructure/accelerator.hpp @@ -55,6 +55,7 @@ #include "axom/core.hpp" #include "serac/infrastructure/logger.hpp" +#include "serac/infrastructure/memory.hpp" #include "serac/infrastructure/profiling.hpp" /** @@ -121,7 +122,7 @@ void zero_out(axom::Array& arr) /// @brief set the contents of an array to zero, byte-wise template -void zero_out(axom::ArrayView& arr) +void zero_out(axom::ArrayView& arr) { std::memset(arr.data(), 0, static_cast(arr.size()) * sizeof(T)); } diff --git a/src/serac/infrastructure/debug_print.hpp b/src/serac/infrastructure/debug_print.hpp index 7bfc1f983..3382e6138 100644 --- a/src/serac/infrastructure/debug_print.hpp +++ b/src/serac/infrastructure/debug_print.hpp @@ -13,6 +13,9 @@ #include #include +#include "serac/infrastructure/memory.hpp" +#include "serac/numerics/functional/element_restriction.hpp" + /** * @brief write an array of values out to file, in a space-separated format * @tparam T the type of each value in the array @@ -68,7 +71,7 @@ std::ostream& operator<<(std::ostream& out, DoF dof) * @param filename the name of the output file */ template -void write_to_file(axom::Array arr, std::string filename) +void write_to_file(axom::Array arr, std::string filename) { std::ofstream outfile(filename); @@ -91,7 +94,7 @@ void write_to_file(axom::Array arr, std::string f * @param filename the name of the output file */ template -void write_to_file(axom::Array arr, std::string filename) +void write_to_file(axom::Array arr, std::string filename) { std::ofstream outfile(filename); diff --git a/src/serac/infrastructure/memory.hpp b/src/serac/infrastructure/memory.hpp new file mode 100644 index 000000000..360386bce --- /dev/null +++ b/src/serac/infrastructure/memory.hpp @@ -0,0 +1,34 @@ +// Copyright (c) 2019-2024, Lawrence Livermore National Security, LLC and +// other Serac Project Developers. See the top-level LICENSE file for +// details. +// +// SPDX-License-Identifier: (BSD-3-Clause) + +/** + * @file memory.hpp + * + * @brief This file defines the host memory space + */ + +#pragma once + +#include "axom/core.hpp" + +#include "serac/serac_config.hpp" + +namespace serac { + +namespace detail { + +/** + * @brief Sets the axom memory space based on whether or not Umpire is being used + */ +#ifdef SERAC_USE_UMPIRE +constexpr axom::MemorySpace host_memory_space = axom::MemorySpace::Host; +#else +constexpr axom::MemorySpace host_memory_space = axom::MemorySpace::Dynamic; +#endif + +} // namespace detail + +} // namespace serac diff --git a/src/serac/numerics/functional/domain_integral_kernels.hpp b/src/serac/numerics/functional/domain_integral_kernels.hpp index 454884372..49164cfd9 100644 --- a/src/serac/numerics/functional/domain_integral_kernels.hpp +++ b/src/serac/numerics/functional/domain_integral_kernels.hpp @@ -10,7 +10,9 @@ #include "serac/numerics/functional/quadrature_data.hpp" #include "serac/numerics/functional/function_signature.hpp" #include "serac/numerics/functional/differentiate_wrt.hpp" +#ifdef SERAC_USE_RAJA #include "RAJA/RAJA.hpp" +#endif #include #include diff --git a/src/serac/numerics/functional/element_restriction.cpp b/src/serac/numerics/functional/element_restriction.cpp index ec57dddb9..998a4d9e9 100644 --- a/src/serac/numerics/functional/element_restriction.cpp +++ b/src/serac/numerics/functional/element_restriction.cpp @@ -216,7 +216,8 @@ std::vector > geom_local_face_dofs(int p) return output; } -axom::Array GetElementRestriction(const serac::fes_t* fes, mfem::Geometry::Type geom) +axom::Array GetElementRestriction(const serac::fes_t* fes, + mfem::Geometry::Type geom) { std::vector elem_dofs{}; mfem::Mesh* mesh = fes->GetMesh(); @@ -269,17 +270,17 @@ axom::Array GetElementRestriction(const serac:: } if (n == 0) { - return axom::Array{}; + return axom::Array{}; } else { uint64_t dofs_per_elem = elem_dofs.size() / n; - axom::Array output(n, dofs_per_elem); + axom::Array output(n, dofs_per_elem); std::memcpy(output.data(), elem_dofs.data(), sizeof(DoF) * n * dofs_per_elem); return output; } } -axom::Array GetElementDofs(const serac::fes_t* fes, mfem::Geometry::Type geom, - const std::vector& mfem_elem_ids) +axom::Array GetElementDofs(const serac::fes_t* fes, mfem::Geometry::Type geom, + const std::vector& mfem_elem_ids) { std::vector elem_dofs{}; @@ -335,17 +336,17 @@ axom::Array GetElementDofs(const serac::fes_t* } if (n == 0) { - return axom::Array{}; + return axom::Array{}; } else { uint64_t dofs_per_elem = elem_dofs.size() / n; - axom::Array output(n, dofs_per_elem); + axom::Array output(n, dofs_per_elem); std::memcpy(output.data(), elem_dofs.data(), sizeof(DoF) * n * dofs_per_elem); return output; } } -axom::Array GetFaceDofs(const serac::fes_t* fes, mfem::Geometry::Type face_geom, - FaceType type) +axom::Array GetFaceDofs(const serac::fes_t* fes, + mfem::Geometry::Type face_geom, FaceType type) { std::vector face_dofs; mfem::Mesh* mesh = fes->GetMesh(); @@ -450,17 +451,18 @@ axom::Array GetFaceDofs(const serac::fes_t* fes delete face_to_elem; if (n == 0) { - return axom::Array{}; + return axom::Array{}; } else { uint64_t dofs_per_face = face_dofs.size() / n; - axom::Array output(n, dofs_per_face); + axom::Array output(n, dofs_per_face); std::memcpy(output.data(), face_dofs.data(), sizeof(DoF) * n * dofs_per_face); return output; } } -axom::Array GetFaceDofs(const serac::fes_t* fes, mfem::Geometry::Type face_geom, - const std::vector& mfem_face_ids) +axom::Array GetFaceDofs(const serac::fes_t* fes, + mfem::Geometry::Type face_geom, + const std::vector& mfem_face_ids) { std::vector face_dofs; mfem::Mesh* mesh = fes->GetMesh(); @@ -617,10 +619,10 @@ axom::Array GetFaceDofs(const serac::fes_t* fes delete face_to_elem; if (n == 0) { - return axom::Array{}; + return axom::Array{}; } else { uint64_t dofs_per_face = face_dofs.size() / n; - axom::Array output(n, dofs_per_face); + axom::Array output(n, dofs_per_face); std::memcpy(output.data(), face_dofs.data(), sizeof(DoF) * n * dofs_per_face); return output; } diff --git a/src/serac/numerics/functional/element_restriction.hpp b/src/serac/numerics/functional/element_restriction.hpp index ccf925dc0..7f661eb54 100644 --- a/src/serac/numerics/functional/element_restriction.hpp +++ b/src/serac/numerics/functional/element_restriction.hpp @@ -7,6 +7,7 @@ #include "geometry.hpp" #include "domain.hpp" +#include "serac/infrastructure/memory.hpp" #include "serac/numerics/functional/typedefs.hpp" inline bool isH1(const mfem::FiniteElementSpace& fes) @@ -197,7 +198,7 @@ struct ElementRestriction { uint64_t nodes_per_elem; /// a 2D array (num_elements-by-nodes_per_elem) holding the dof info extracted from the finite element space - axom::Array dof_info; + axom::Array dof_info; /// whether the underlying dofs are arranged "byNodes" or "byVDim" mfem::Ordering::Type ordering; @@ -242,7 +243,8 @@ struct BlockElementRestriction { * @param fes the finite element space containing the dof information * @param geom the kind of element geometry */ -axom::Array GetElementDofs(const serac::fes_t* fes, mfem::Geometry::Type geom); +axom::Array GetElementDofs(const serac::fes_t* fes, + mfem::Geometry::Type geom); /** * @brief Get the list of dofs for each face element (of the specified geometry) from the fes_t @@ -251,9 +253,10 @@ axom::Array GetElementDofs(const serac::fes_t* * @param geom the kind of element geometry * @param type whether the face is of interior or boundary type */ -axom::Array GetFaceDofs(const serac::fes_t* fes, mfem::Geometry::Type face_geom, - FaceType type); +axom::Array GetFaceDofs(const serac::fes_t* fes, + mfem::Geometry::Type face_geom, FaceType type); /// @overload -axom::Array GetFaceDofs(const serac::fes_t* fes, mfem::Geometry::Type face_geom, - const std::vector& mfem_face_ids); +axom::Array GetFaceDofs(const serac::fes_t* fes, + mfem::Geometry::Type face_geom, + const std::vector& mfem_face_ids); diff --git a/src/serac/numerics/functional/tests/dg_restriction_operators.cpp b/src/serac/numerics/functional/tests/dg_restriction_operators.cpp index 8b1970f18..6d6e54380 100644 --- a/src/serac/numerics/functional/tests/dg_restriction_operators.cpp +++ b/src/serac/numerics/functional/tests/dg_restriction_operators.cpp @@ -229,7 +229,7 @@ mfem::Mesh generate_permuted_mesh(mfem::Geometry::Type geom, int i) return {}; } -std::ostream& operator<<(std::ostream& out, axom::Array arr) +std::ostream& operator<<(std::ostream& out, axom::Array arr) { for (int i = 0; i < arr.shape()[0]; i++) { for (int j = 0; j < arr.shape()[1]; j++) { diff --git a/src/serac/numerics/functional/tests/element_restriction_tests.cpp b/src/serac/numerics/functional/tests/element_restriction_tests.cpp index a3fb99f34..d1ec2b35e 100644 --- a/src/serac/numerics/functional/tests/element_restriction_tests.cpp +++ b/src/serac/numerics/functional/tests/element_restriction_tests.cpp @@ -5,7 +5,7 @@ using namespace serac; -std::ostream& operator<<(std::ostream& out, axom::Array arr) +std::ostream& operator<<(std::ostream& out, axom::Array arr) { for (int i = 0; i < arr.shape()[0]; i++) { for (int j = 0; j < arr.shape()[1]; j++) { diff --git a/src/serac/numerics/functional/tests/functional_basic_dg.cpp b/src/serac/numerics/functional/tests/functional_basic_dg.cpp index 652a6db93..a0943c0a5 100644 --- a/src/serac/numerics/functional/tests/functional_basic_dg.cpp +++ b/src/serac/numerics/functional/tests/functional_basic_dg.cpp @@ -39,7 +39,8 @@ void L2_test(std::string meshfile) mfem::ParFiniteElementSpace fespace(mesh.get(), &fec, dim, serac::ordering); mfem::Vector U(fespace.TrueVSize()); - U.Randomize(); + int seed = 1; + U.Randomize(seed); // Construct the new functional object using the specified test and trial spaces Functional residual(&fespace, {&fespace}); @@ -96,8 +97,8 @@ void L2_qoi_test(std::string meshfile) auto fec = mfem::L2_FECollection(p, dim, mfem::BasisType::GaussLobatto); mfem::ParFiniteElementSpace fespace(mesh.get(), &fec, dim, serac::ordering); - int seed = 0; mfem::HypreParVector U = *fespace.NewTrueDofVector(); + int seed = 2; U.Randomize(seed); // Construct the new functional object using the specified test and trial spaces @@ -162,10 +163,12 @@ void L2_scalar_valued_test(std::string meshfile) mfem::ParFiniteElementSpace fespace_1(mesh.get(), &H1fec, dim, serac::ordering); mfem::Vector U0(fespace_0.TrueVSize()); - U0.Randomize(); + int seed = 3; + U0.Randomize(seed); mfem::Vector U1(fespace_1.TrueVSize()); - U1.Randomize(); + seed = 4; + U1.Randomize(seed); // Construct the new functional object using the specified test and trial spaces Functional residual(&fespace_0, {&fespace_0, &fespace_1}); diff --git a/src/serac/numerics/functional/tests/functional_basic_h1_vector.cpp b/src/serac/numerics/functional/tests/functional_basic_h1_vector.cpp index 0ed7666df..0c53b8b49 100644 --- a/src/serac/numerics/functional/tests/functional_basic_h1_vector.cpp +++ b/src/serac/numerics/functional/tests/functional_basic_h1_vector.cpp @@ -97,7 +97,8 @@ void weird_mixed_test(std::unique_ptr& mesh) mfem::Vector U(trial_fes->TrueVSize()); Functional residual(test_fes.get(), {trial_fes.get()}); - U.Randomize(); + int seed = 5; + U.Randomize(seed); // note: this is not really an elasticity problem, it's testing source and flux // terms that have the appropriate shapes to ensure that all the differentiation @@ -122,7 +123,8 @@ void elasticity_test(std::unique_ptr& mesh) auto [test_fes, test_col] = generateParFiniteElementSpace(mesh.get()); mfem::Vector U(trial_fes->TrueVSize()); - U.Randomize(); + int seed = 6; + U.Randomize(seed); Functional residual(test_fes.get(), {trial_fes.get()}); diff --git a/src/serac/numerics/functional/tests/functional_basic_hcurl.cpp b/src/serac/numerics/functional/tests/functional_basic_hcurl.cpp index bf5dfdb32..a7a27699f 100644 --- a/src/serac/numerics/functional/tests/functional_basic_hcurl.cpp +++ b/src/serac/numerics/functional/tests/functional_basic_hcurl.cpp @@ -40,7 +40,8 @@ void hcurl_test_2D() mfem::ParFiniteElementSpace fespace(mesh.get(), &fec); mfem::Vector U(fespace.TrueVSize()); - U.Randomize(); + int seed = 7; + U.Randomize(seed); // Construct the new functional object using the specified test and trial spaces Functional residual(&fespace, {&fespace}); @@ -77,7 +78,8 @@ void hcurl_test_3D() mfem::ParFiniteElementSpace fespace(mesh.get(), &fec); mfem::Vector U(fespace.TrueVSize()); - U.Randomize(); + int seed = 8; + U.Randomize(seed); // Define the types for the test and trial spaces using the function arguments using test_space = Hcurl

; diff --git a/src/serac/numerics/functional/tests/functional_boundary_test.cpp b/src/serac/numerics/functional/tests/functional_boundary_test.cpp index 69b5cf342..d12fe6255 100644 --- a/src/serac/numerics/functional/tests/functional_boundary_test.cpp +++ b/src/serac/numerics/functional/tests/functional_boundary_test.cpp @@ -88,7 +88,8 @@ void boundary_test(mfem::ParMesh& mesh, H1

test, H1

trial, Dimension) std::unique_ptr J(B.ParallelAssemble()); mfem::Vector U(fespace->TrueVSize()); - U.Randomize(); + int seed = 9; + U.Randomize(seed); Functional residual(fespace.get(), {fespace.get()}); @@ -164,7 +165,8 @@ void boundary_test(mfem::ParMesh& mesh, L2

test, L2

trial, Dimension) std::unique_ptr J(B.ParallelAssemble()); mfem::ParGridFunction u_global(fespace.get()); - u_global.Randomize(); + int seed = 1; + u_global.Randomize(seed); mfem::FunctionCoefficient xfunc([](mfem::Vector x) { return x[0]; }); u_global.ProjectCoefficient(xfunc); diff --git a/src/serac/numerics/functional/tests/functional_comparison_L2.cpp b/src/serac/numerics/functional/tests/functional_comparison_L2.cpp index 70b54a687..14ec08322 100644 --- a/src/serac/numerics/functional/tests/functional_comparison_L2.cpp +++ b/src/serac/numerics/functional/tests/functional_comparison_L2.cpp @@ -98,7 +98,8 @@ void functional_test(mfem::ParMesh& mesh, L2

test, L2

trial, DimensionTrueVSize()); u_global.GetTrueDofs(U); diff --git a/src/serac/numerics/functional/tests/functional_comparisons.cpp b/src/serac/numerics/functional/tests/functional_comparisons.cpp index e4bc205f2..2f2f411a4 100644 --- a/src/serac/numerics/functional/tests/functional_comparisons.cpp +++ b/src/serac/numerics/functional/tests/functional_comparisons.cpp @@ -129,7 +129,8 @@ void functional_test(mfem::ParMesh& mesh, H1

test, H1

trial, DimensionTrueVSize()); u_global.GetTrueDofs(U); @@ -254,7 +255,8 @@ void functional_test(mfem::ParMesh& mesh, H1 test, H1 trial, Dim std::unique_ptr F(f.ParallelAssemble()); mfem::ParGridFunction u_global(fespace.get()); - u_global.Randomize(); + int seed = 7; + u_global.Randomize(seed); mfem::Vector U(fespace->TrueVSize()); u_global.GetTrueDofs(U); @@ -371,7 +373,8 @@ void functional_test(mfem::ParMesh& mesh, Hcurl

test, Hcurl

trial, Dimensi std::unique_ptr F(f.ParallelAssemble()); mfem::ParGridFunction u_global(fespace.get()); - u_global.Randomize(); + int seed = 8; + u_global.Randomize(seed); mfem::Vector U(fespace->TrueVSize()); u_global.GetTrueDofs(U); diff --git a/src/serac/numerics/functional/tests/functional_comparisons_cuda.cu b/src/serac/numerics/functional/tests/functional_comparisons_cuda.cu index 12d2eb262..d90093e17 100644 --- a/src/serac/numerics/functional/tests/functional_comparisons_cuda.cu +++ b/src/serac/numerics/functional/tests/functional_comparisons_cuda.cu @@ -137,7 +137,8 @@ void functional_test(H1

test, H1

trial, Dimension) // Set a random state to evaluate the residual mfem::ParGridFunction u_global(fespace.get()); - u_global.Randomize(); + int seed = 3; + u_global.Randomize(seed); mfem::Vector U(fespace->TrueVSize()); U.UseDevice(true); @@ -257,7 +258,8 @@ void functional_test(H1 test, H1 trial, Dimension) F->HostRead(); mfem::ParGridFunction u_global(fespace.get()); - u_global.Randomize(); + int seed = 4; + u_global.Randomize(seed); mfem::Vector U(fespace->TrueVSize()); U.UseDevice(true); @@ -347,7 +349,8 @@ void functional_test(Hcurl

test, Hcurl

trial, Dimension) F->UseDevice(true); mfem::ParGridFunction u_global(fespace.get()); - u_global.Randomize(); + int seed = 5; + u_global.Randomize(seed); mfem::Vector U(fespace->TrueVSize()); U.UseDevice(true); diff --git a/src/serac/numerics/functional/tests/functional_multiphysics.cpp b/src/serac/numerics/functional/tests/functional_multiphysics.cpp index e6d1acd9b..51def3976 100644 --- a/src/serac/numerics/functional/tests/functional_multiphysics.cpp +++ b/src/serac/numerics/functional/tests/functional_multiphysics.cpp @@ -41,7 +41,7 @@ TEST(FunctionalMultiphysics, NonlinearThermalTest3D) mfem::Vector U(fespace->TrueVSize()); mfem::Vector dU_dt(fespace->TrueVSize()); - int seed = 0; + int seed = 1; U.Randomize(seed); dU_dt.Randomize(seed + 1); diff --git a/src/serac/numerics/functional/tests/functional_nonlinear.cpp b/src/serac/numerics/functional/tests/functional_nonlinear.cpp index 9ebe22f23..b39642583 100644 --- a/src/serac/numerics/functional/tests/functional_nonlinear.cpp +++ b/src/serac/numerics/functional/tests/functional_nonlinear.cpp @@ -63,7 +63,8 @@ void functional_test(mfem::ParMesh& mesh, H1

test, H1

trial, DimensionTrueVSize()); - U.Randomize(); + int seed = 3; + U.Randomize(seed); // Define the types for the test and trial spaces using the function arguments using test_space = decltype(test); @@ -117,7 +118,8 @@ void functional_test(mfem::ParMesh& mesh, H1 test, H1 trial, Dim // Set a random state to evaluate the residual mfem::Vector U(fespace->TrueVSize()); - U.Randomize(); + int seed = 4; + U.Randomize(seed); // Construct the new functional object using the known test and trial spaces Functional residual(fespace.get(), {fespace.get()}); diff --git a/src/serac/numerics/functional/tests/functional_shape_derivatives.cpp b/src/serac/numerics/functional/tests/functional_shape_derivatives.cpp index c4eb57123..8319f08a5 100644 --- a/src/serac/numerics/functional/tests/functional_shape_derivatives.cpp +++ b/src/serac/numerics/functional/tests/functional_shape_derivatives.cpp @@ -278,14 +278,17 @@ void functional_test_2D(mfem::ParMesh& mesh, double tolerance) ones = 1; mfem::Vector U1(fespace1->TrueVSize()); - U1.Randomize(); + int seed = 6; + U1.Randomize(seed); mfem::Vector U2(fespace2->TrueVSize()); - U2.Randomize(); + seed = 7; + U2.Randomize(seed); U2 *= 0.1; mfem::Vector dU2(fespace2->TrueVSize()); - dU2.Randomize(); + seed = 8; + dU2.Randomize(seed); // Construct the new functional object using the known test and trial spaces ShapeAwareFunctional residual(fespace2.get(), fespace1.get(), {fespace1.get()}); @@ -323,14 +326,17 @@ void functional_test_3D(mfem::ParMesh& mesh, double tolerance) ones = 1; mfem::Vector U1(fespace1->TrueVSize()); - U1.Randomize(); + int seed = 9; + U1.Randomize(seed); mfem::Vector U2(fespace2->TrueVSize()); - U2.Randomize(); + seed = 1; + U2.Randomize(seed); U2 *= 0.1; mfem::Vector dU2(fespace2->TrueVSize()); - dU2.Randomize(); + seed = 2; + dU2.Randomize(seed); // Construct the new functional object using the known test and trial spaces ShapeAwareFunctional residual(fespace2.get(), fespace1.get(), {fespace1.get()}); diff --git a/src/serac/physics/benchmarks/physics_benchmark_functional.cpp b/src/serac/physics/benchmarks/physics_benchmark_functional.cpp index 69fa9a0f7..dcf85a77b 100644 --- a/src/serac/physics/benchmarks/physics_benchmark_functional.cpp +++ b/src/serac/physics/benchmarks/physics_benchmark_functional.cpp @@ -52,7 +52,8 @@ void functional_test(int parallel_refinement) // Set a random state to evaluate the residual mfem::ParGridFunction u_global(fespace.get()); - u_global.Randomize(); + int seed = 1; + u_global.Randomize(seed); mfem::Vector U(fespace->TrueVSize()); u_global.GetTrueDofs(U);