diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index d7f04ec3a3..ae6f9446db 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -43,6 +43,7 @@ The Axom project release numbers follow [Semantic Versioning](http://semver.org/ or when `components="all"`), all components are enabled. All components can be disabled via `components=none`. - Adds a `conduit` spack variant. Conduit was previously a required dependency in our spack package, and is now enabled by default. - Adds spack variants for `adiak` and `caliper`. These replace the previous `profiling` variant which enabled both at the same time. +- Adds the `AXOM_TEST_NUM_OMP_THREADS` configuration variable to control the default OpenMP thread count for tests. ### Changed - Treatment of materials on strided-structured Blueprint meshes has changed in `axom::mir`. diff --git a/src/axom/bump/tests/CMakeLists.txt b/src/axom/bump/tests/CMakeLists.txt index e2d8a1c7d2..7bc508daeb 100644 --- a/src/axom/bump/tests/CMakeLists.txt +++ b/src/axom/bump/tests/CMakeLists.txt @@ -43,7 +43,7 @@ foreach(test ${gtest_bump_tests}) DEPENDS_ON ${bump_tests_depends_on} FOLDER axom/bump/tests ) - blt_add_test( NAME ${test_name} - COMMAND ${test_name}_test ) + axom_add_test( NAME ${test_name} + COMMAND ${test_name}_test ) endforeach() diff --git a/src/axom/core/examples/CMakeLists.txt b/src/axom/core/examples/CMakeLists.txt index 45d44b9419..1f8ccd6244 100644 --- a/src/axom/core/examples/CMakeLists.txt +++ b/src/axom/core/examples/CMakeLists.txt @@ -73,26 +73,36 @@ if(AXOM_ENABLE_TESTS) # Run the core array performance example on N ranks for each enabled policy set(_policies "seq") if(RAJA_FOUND) - blt_list_append(TO _policies ELEMENTS "omp" IF AXOM_ENABLE_OPENMP) + blt_list_append(TO _policies ELEMENTS "omp" IF AXOM_ENABLE_OPENMP) blt_list_append(TO _policies ELEMENTS "cuda" IF AXOM_ENABLE_CUDA) - blt_list_append(TO _policies ELEMENTS "hip" IF AXOM_ENABLE_HIP) + blt_list_append(TO _policies ELEMENTS "hip" IF AXOM_ENABLE_HIP) endif() foreach(_pol ${_policies}) + # sets the number of threads for the omp policy; leave empty for non-omp policies + set(_num_threads) + if(_pol STREQUAL "omp") + set(_num_threads ${AXOM_TEST_NUM_OMP_THREADS}) + endif() + axom_add_test( - NAME "core_array_perf_1d_${_pol}" - COMMAND core_array_perf_ex --policy ${_pol} --shape 1200000 -r 10) + NAME core_array_perf_1d_${_pol} + COMMAND core_array_perf_ex --policy ${_pol} --shape 1200000 -r 10 + NUM_OMP_THREADS ${_num_threads}) axom_add_test( - NAME "core_array_perf_2d_${_pol}" - COMMAND core_array_perf_ex --policy ${_pol} --shape 1000 1200 -r 10) + NAME core_array_perf_2d_${_pol} + COMMAND core_array_perf_ex --policy ${_pol} --shape 1000 1200 -r 10 + NUM_OMP_THREADS ${_num_threads}) axom_add_test( - NAME "core_array_perf_3d_${_pol}" - COMMAND core_array_perf_ex --policy ${_pol} --shape 100 100 120 -r 10) + NAME core_array_perf_3d_${_pol} + COMMAND core_array_perf_ex --policy ${_pol} --shape 100 100 120 -r 10 + NUM_OMP_THREADS ${_num_threads}) if(NOT RAJA_FOUND) # RAJA provides support for up to 3D tiled nested loops, # so run 4D test only for non-RAJA builds. axom_add_test( - NAME "core_array_perf_4d_${_pol}" - COMMAND core_array_perf_ex --policy ${_pol} --shape 50 20 30 6 ) + NAME core_array_perf_4d_${_pol} + COMMAND core_array_perf_ex --policy ${_pol} --shape 50 20 30 6 + NUM_OMP_THREADS ${_num_threads}) endif() endforeach() endif() diff --git a/src/axom/core/tests/CMakeLists.txt b/src/axom/core/tests/CMakeLists.txt index 33658f5370..8e7f1b586c 100644 --- a/src/axom/core/tests/CMakeLists.txt +++ b/src/axom/core/tests/CMakeLists.txt @@ -140,9 +140,9 @@ if (AXOM_ENABLE_OPENMP AND RAJA_FOUND) foreach( test_suite ${core_openmp_tests} ) get_filename_component( test_suite ${test_suite} NAME_WE ) - axom_add_test( NAME ${test_suite}_openmp - COMMAND core_openmp_tests --gtest_filter=${test_suite}* - NUM_OMP_THREADS 24 ) + axom_add_test( NAME ${test_suite}_openmp + COMMAND core_openmp_tests --gtest_filter=${test_suite}* + NUM_OMP_THREADS ${AXOM_TEST_NUM_OMP_THREADS} ) endforeach() endif() @@ -183,14 +183,16 @@ axom_add_executable(NAME utils_annotations_test foreach(_mode none report counts gputx) set(_test_name utils_annotations_${_mode}_test) + + set(_num_ranks 1) if(AXOM_ENABLE_MPI) - axom_add_test(NAME ${_test_name} - COMMAND utils_annotations_test -m ${_mode} - NUM_MPI_TASKS 2) - else() - axom_add_test(NAME ${_test_name} - COMMAND utils_annotations_test -m ${_mode}) + set(_num_ranks 2) endif() + + axom_add_test(NAME ${_test_name} + COMMAND utils_annotations_test -m ${_mode} + NUM_MPI_TASKS ${_num_ranks}) + set_property(TEST ${_test_name} APPEND PROPERTY ENVIRONMENT "CALI_LOG_VERBOSITY=2") endforeach() diff --git a/src/axom/klee/tests/CMakeLists.txt b/src/axom/klee/tests/CMakeLists.txt index 15d1f76494..3c131b6da3 100644 --- a/src/axom/klee/tests/CMakeLists.txt +++ b/src/axom/klee/tests/CMakeLists.txt @@ -40,7 +40,7 @@ foreach(test ${gtest_klee_tests}) OUTPUT_DIR ${TEST_OUTPUT_DIRECTORY} DEPENDS_ON klee_test_utils FOLDER axom/klee/tests ) - blt_add_test( NAME ${test_name} + axom_add_test(NAME ${test_name} COMMAND ${test_name}_test ) endforeach() diff --git a/src/axom/mint/tests/CMakeLists.txt b/src/axom/mint/tests/CMakeLists.txt index c6fe54700d..1ea49d6fa7 100644 --- a/src/axom/mint/tests/CMakeLists.txt +++ b/src/axom/mint/tests/CMakeLists.txt @@ -7,47 +7,46 @@ #------------------------------------------------------------------------------ set( mint_tests + ## execution model tests + mint_execution_cell_traversals.cpp + mint_execution_node_traversals.cpp + mint_execution_face_traversals.cpp + mint_execution_xargs.cpp - ## execution model tests - mint_execution_cell_traversals.cpp - mint_execution_node_traversals.cpp - mint_execution_face_traversals.cpp - mint_execution_xargs.cpp - - ## mesh tests - mint_mesh.cpp - mint_mesh_cell_types.cpp - mint_mesh_blueprint.cpp - mint_mesh_connectivity_array.cpp - mint_mesh_coordinates.cpp - mint_mesh_curvilinear_mesh.cpp - mint_mesh_field.cpp - mint_mesh_field_data.cpp - mint_mesh_field_types.cpp - mint_mesh_field_variable.cpp - mint_mesh_particle_mesh.cpp - mint_mesh_rectilinear_mesh.cpp - mint_mesh_uniform_mesh.cpp - mint_mesh_unstructured_mesh.cpp - mint_mesh_face_relation.cpp - - ## fem tests - mint_fem_shape_functions.cpp - mint_fem_single_fe.cpp - - ## util tests - mint_su2_io.cpp - mint_util_write_vtk.cpp - - ## container tests - mint_deprecated_mcarray.cpp + ## mesh tests + mint_mesh.cpp + mint_mesh_cell_types.cpp + mint_mesh_blueprint.cpp + mint_mesh_connectivity_array.cpp + mint_mesh_coordinates.cpp + mint_mesh_curvilinear_mesh.cpp + mint_mesh_field.cpp + mint_mesh_field_data.cpp + mint_mesh_field_types.cpp + mint_mesh_field_variable.cpp + mint_mesh_particle_mesh.cpp + mint_mesh_rectilinear_mesh.cpp + mint_mesh_uniform_mesh.cpp + mint_mesh_unstructured_mesh.cpp + mint_mesh_face_relation.cpp + + ## fem tests + mint_fem_shape_functions.cpp + mint_fem_single_fe.cpp + + ## util tests + mint_su2_io.cpp + mint_util_write_vtk.cpp + + ## container tests + mint_deprecated_mcarray.cpp ) foreach( test ${mint_tests} ) - get_filename_component( test_name ${test} NAME_WE ) + get_filename_component( test_name ${test} NAME_WE ) - axom_add_executable( + axom_add_executable( NAME ${test_name}_test SOURCES ${test} OUTPUT_DIR ${TEST_OUTPUT_DIRECTORY} @@ -55,9 +54,15 @@ foreach( test ${mint_tests} ) FOLDER axom/mint/tests ) - axom_add_test( NAME ${test_name} - COMMAND ${test_name}_test - NUM_OMP_THREADS 4 - ) + # sets the number of threads for the omp policy; leave empty for non-omp policies + set(_num_threads) + if(AXOM_ENABLE_OPENMP) + set(_num_threads ${AXOM_TEST_NUM_OMP_THREADS}) + endif() + + axom_add_test( NAME ${test_name} + COMMAND ${test_name}_test + NUM_OMP_THREADS ${_num_threads} + ) endforeach() diff --git a/src/axom/mir/examples/concentric_circles/CMakeLists.txt b/src/axom/mir/examples/concentric_circles/CMakeLists.txt index 51edc97b5e..9079718bf4 100644 --- a/src/axom/mir/examples/concentric_circles/CMakeLists.txt +++ b/src/axom/mir/examples/concentric_circles/CMakeLists.txt @@ -39,7 +39,14 @@ if(AXOM_ENABLE_TESTS) endif() foreach(_policy ${_policies}) + # sets the number of threads for the omp policy; leave empty for non-omp policies + set(_num_threads) + if(_policy STREQUAL "omp") + set(_num_threads ${AXOM_TEST_NUM_OMP_THREADS}) + endif() + set(_testname "mir_concentric_circles_${_policy}") + axom_add_test( NAME ${_testname} COMMAND mir_concentric_circles @@ -47,7 +54,7 @@ if(AXOM_ENABLE_TESTS) --numcircles 5 --policy ${_policy} --disable-write - ) + NUM_OMP_THREADS ${_num_threads}) set_tests_properties(${_testname} PROPERTIES PASS_REGULAR_EXPRESSION "Material interface reconstruction time:") diff --git a/src/axom/mir/examples/heavily_mixed/CMakeLists.txt b/src/axom/mir/examples/heavily_mixed/CMakeLists.txt index 4729454873..79a5d8c380 100644 --- a/src/axom/mir/examples/heavily_mixed/CMakeLists.txt +++ b/src/axom/mir/examples/heavily_mixed/CMakeLists.txt @@ -28,6 +28,11 @@ if(AXOM_ENABLE_TESTS) endif() foreach(_policy ${_policies}) + set(_num_threads) + if(_policy STREQUAL "omp") + set(_num_threads ${AXOM_TEST_NUM_OMP_THREADS}) + endif() + set(_testname "mir_heavily_mixed_2D_${_policy}") axom_add_test( NAME ${_testname} @@ -37,7 +42,7 @@ if(AXOM_ENABLE_TESTS) --materials 10 --policy ${_policy} --disable-write - ) + NUM_OMP_THREADS ${_num_threads}) set_tests_properties(${_testname} PROPERTIES PASS_REGULAR_EXPRESSION "Material interface reconstruction time:") @@ -51,7 +56,7 @@ if(AXOM_ENABLE_TESTS) --materials 10 --policy ${_policy} --disable-write - ) + NUM_OMP_THREADS ${_num_threads}) set_tests_properties(${_testname} PROPERTIES PASS_REGULAR_EXPRESSION "Material interface reconstruction time:") diff --git a/src/axom/mir/examples/tutorial_simple/CMakeLists.txt b/src/axom/mir/examples/tutorial_simple/CMakeLists.txt index f88aa80a8a..5b4bfcc78b 100644 --- a/src/axom/mir/examples/tutorial_simple/CMakeLists.txt +++ b/src/axom/mir/examples/tutorial_simple/CMakeLists.txt @@ -48,6 +48,11 @@ if(AXOM_ENABLE_TESTS) endif() foreach(_policy ${_policies}) + set(_num_threads) + if(_policy STREQUAL "omp") + set(_num_threads ${AXOM_TEST_NUM_OMP_THREADS}) + endif() + foreach(_testnum ${_test_numbers}) set(_testname "mir_tutorial_simple_${_policy}_${_testnum}") axom_add_test( @@ -56,7 +61,7 @@ if(AXOM_ENABLE_TESTS) --test-case ${_testnum} --policy ${_policy} --disable-write - ) + NUM_OMP_THREADS ${_num_threads}) set_tests_properties(${_testname} PROPERTIES PASS_REGULAR_EXPRESSION "Reconstruction time:") diff --git a/src/axom/mir/tests/CMakeLists.txt b/src/axom/mir/tests/CMakeLists.txt index 0141409384..2a4869333e 100644 --- a/src/axom/mir/tests/CMakeLists.txt +++ b/src/axom/mir/tests/CMakeLists.txt @@ -37,7 +37,7 @@ foreach(test ${gtest_mir_tests}) DEPENDS_ON ${mir_tests_depends_on} FOLDER axom/mir/tests ) - blt_add_test( NAME ${test_name} - COMMAND ${test_name}_test ) + axom_add_test( NAME ${test_name} + COMMAND ${test_name}_test ) endforeach() diff --git a/src/axom/multimat/examples/CMakeLists.txt b/src/axom/multimat/examples/CMakeLists.txt index 53b3890e38..c72c8434e1 100644 --- a/src/axom/multimat/examples/CMakeLists.txt +++ b/src/axom/multimat/examples/CMakeLists.txt @@ -35,7 +35,7 @@ foreach(example_source ${multimat_example_sources}) ) unset(deps) - blt_add_test( + axom_add_test( NAME multimat_${example_name} COMMAND multimat_${example_name}_ex ) diff --git a/src/axom/multimat/tests/CMakeLists.txt b/src/axom/multimat/tests/CMakeLists.txt index e0da54c995..07e9776bf8 100644 --- a/src/axom/multimat/tests/CMakeLists.txt +++ b/src/axom/multimat/tests/CMakeLists.txt @@ -24,6 +24,6 @@ foreach(test ${gtest_multimat_tests}) OUTPUT_DIR ${TEST_OUTPUT_DIRECTORY} DEPENDS_ON multimat gtest FOLDER axom/multimat/tests ) - blt_add_test( NAME ${test_name} - COMMAND ${test_name}_test ) + axom_add_test( NAME ${test_name} + COMMAND ${test_name}_test ) endforeach() diff --git a/src/axom/primal/examples/CMakeLists.txt b/src/axom/primal/examples/CMakeLists.txt index a070685207..c44f758bb1 100644 --- a/src/axom/primal/examples/CMakeLists.txt +++ b/src/axom/primal/examples/CMakeLists.txt @@ -51,13 +51,17 @@ if (RAJA_FOUND AND UMPIRE_FOUND) blt_list_append(TO _policies ELEMENTS "hip" IF AXOM_ENABLE_HIP) foreach(_policy ${_policies}) + # sets the number of threads for the omp policy; leave empty for non-omp policies + set(_num_threads) + if(_policy STREQUAL "omp") + set(_num_threads ${AXOM_TEST_NUM_OMP_THREADS}) + endif() set(_testname "primal_hex_tet_volume_ex_${_policy}") axom_add_test( - NAME ${_testname} - COMMAND hex_tet_volume_ex - --policy ${_policy} - ) + NAME ${_testname} + COMMAND hex_tet_volume_ex --policy ${_policy} + NUM_OMP_THREADS ${_num_threads}) set_tests_properties(${_testname} PROPERTIES PASS_REGULAR_EXPRESSION "Difference between sums") diff --git a/src/axom/quest/examples/CMakeLists.txt b/src/axom/quest/examples/CMakeLists.txt index cb4880ae32..0d83df64fe 100644 --- a/src/axom/quest/examples/CMakeLists.txt +++ b/src/axom/quest/examples/CMakeLists.txt @@ -25,18 +25,13 @@ axom_add_executable( if(AXOM_ENABLE_TESTS AND AXOM_DATA_DIR AND CALIPER_FOUND) set(_input_file "${AXOM_DATA_DIR}/quest/sphere_binary.stl") - if(AXOM_ENABLE_MPI) # this test requires MPI when available - set(_num_ranks 1) # (but only 1 rank) - else() # but can otherwise be run without MPI - unset(_num_ranks) - endif() - + # this test requires MPI when available (but only 1 rank) axom_add_test( NAME quest_containment_sphere_ex COMMAND quest_containment_driver_ex --input ${_input_file} --caliper report - NUM_MPI_TASKS ${_num_ranks} + NUM_MPI_TASKS 1 ) endif() @@ -64,6 +59,11 @@ if(AXOM_ENABLE_TESTS AND AXOM_DATA_DIR) endif() foreach(_policy ${_policies}) + # sets the number of threads for the omp policy; leave empty for non-omp policies + set(_num_threads) + if(_policy STREQUAL "omp") + set(_num_threads ${AXOM_TEST_NUM_OMP_THREADS}) + endif() set(_testname "quest_bvh_two_pass_example_${_policy}") axom_add_test( @@ -71,7 +71,7 @@ if(AXOM_ENABLE_TESTS AND AXOM_DATA_DIR) COMMAND quest_bvh_two_pass_ex --file ${input_file} --exec_space ${_policy} - ) + NUM_OMP_THREADS ${_num_threads}) set_tests_properties(${_testname} PROPERTIES PASS_REGULAR_EXPRESSION "Found 4 actual collisions") @@ -117,6 +117,11 @@ if (CONDUIT_FOUND AND UMPIRE_FOUND) foreach(_method ${_methods}) foreach(_policy ${_policies}) + # sets the number of threads for the omp policy; leave empty for non-omp policies + set(_num_threads) + if(_policy STREQUAL "omp") + set(_num_threads ${AXOM_TEST_NUM_OMP_THREADS}) + endif() set(_testname "quest_candidates_example_${_method}_${_policy}") axom_add_test( @@ -126,7 +131,7 @@ if (CONDUIT_FOUND AND UMPIRE_FOUND) --queryfile ${query_file} --method ${_method} --policy ${_policy} - ) + NUM_OMP_THREADS ${_num_threads}) # Match either one comma or none for portability set_tests_properties(${_testname} PROPERTIES @@ -244,6 +249,12 @@ if(AXOM_ENABLE_MPI AND MFEM_FOUND AND MFEM_USE_MPI endif() foreach(_policy ${_policies}) + # sets the number of threads for the omp policy; leave empty for non-omp policies + set(_num_threads) + if(_policy STREQUAL "omp") + set(_num_threads ${AXOM_TEST_NUM_OMP_THREADS}) + endif() + set(_testname quest_shaping_driver_ex_semi_circle_2d_${_policy}) axom_add_test( @@ -254,7 +265,8 @@ if(AXOM_ENABLE_MPI AND MFEM_FOUND AND MFEM_USE_MPI --segments-per-knot-span 50 --method intersection inline_mesh --min 0 0 --max 3 3 --res 3 3 -d 2 - NUM_MPI_TASKS ${_nranks}) + NUM_MPI_TASKS ${_nranks} + NUM_OMP_THREADS ${_num_threads}) # steel unit semi-circle with an area of pi/2 set_tests_properties(${_testname} PROPERTIES PASS_REGULAR_EXPRESSION "Volume of material 'steel' is 1.57") @@ -270,10 +282,10 @@ if(AXOM_ENABLE_MPI AND MFEM_FOUND AND MFEM_USE_MPI COMMAND quest_shaping_driver_ex -i ${shaping_data_dir}/heroic_roses_mfem.yaml --method sampling - inline_mesh --min 0 0 --max 300 400 --res 300 400 -d 2 + inline_mesh --min 0 0 --max 300 400 --res 150 200 -d 2 NUM_MPI_TASKS ${_nranks}) set_tests_properties(${_testname} PROPERTIES - PASS_REGULAR_EXPRESSION "Volume of material 'black' is 36,?637.") + PASS_REGULAR_EXPRESSION "Volume of material 'black' is 36,?595.") # Test 2D MFEM mesh (winding number) set(_testname quest_shaping_driver_ex_heroic_roses_mfem_cp) @@ -282,10 +294,10 @@ if(AXOM_ENABLE_MPI AND MFEM_FOUND AND MFEM_USE_MPI COMMAND quest_shaping_driver_ex -i ${shaping_data_dir}/heroic_roses_mfem_cp.yaml --method sampling --sampling windingnumber - inline_mesh --min 0 0 --max 300 400 --res 300 400 -d 2 + inline_mesh --min 0 0 --max 300 400 --res 150 200 -d 2 NUM_MPI_TASKS ${_nranks}) set_tests_properties(${_testname} PROPERTIES - PASS_REGULAR_EXPRESSION "Volume of material 'black' is 36,?645.") + PASS_REGULAR_EXPRESSION "Volume of material 'black' is 36,?633.") # 3D shaping tests set(_nranks 3) @@ -351,11 +363,8 @@ if((CONDUIT_FOUND OR if(AXOM_ENABLE_TESTS AND AXOM_DATA_DIR) - if(AXOM_ENABLE_MPI) # this test requires MPI when available - set(_nranks 1) # (but only 1 rank) - else() # but can otherwise be run without MPI - unset(_nranks) - endif() + # this test requires MPI when available (but only 1 rank) + set(_nranks 1) # Run the in-memory shaping example on with each enabled policy set(_policies "seq") @@ -370,6 +379,12 @@ if((CONDUIT_FOUND OR set(_testshapes "tetmesh" "tet" "hex" "sphere" "cyl" "cone" "sor" "all" "plane") foreach(_policy ${_policies}) + # sets the number of threads for the omp policy; leave empty for non-omp policies + set(_num_threads) + if(_policy STREQUAL "omp") + set(_num_threads ${AXOM_TEST_NUM_OMP_THREADS}) + endif() + foreach(_testshape ${_testshapes}) set(_testname "quest_shape_in_memory_${_policy}_${_testshape}") axom_add_test( @@ -381,8 +396,9 @@ if((CONDUIT_FOUND OR --scale 0.75 0.75 0.75 --dir 0.2 0.4 0.8 --meshType bpSidre - inline_mesh --min -2 -2 -2 --max 2 2 2 --res 30 30 30 - NUM_MPI_TASKS ${_nranks}) + inline_mesh --min -2 -2 -2 --max 2 2 2 --res 10 10 10 + NUM_MPI_TASKS ${_nranks} + NUM_OMP_THREADS ${_num_threads}) endforeach() endforeach() @@ -391,6 +407,12 @@ if((CONDUIT_FOUND OR set(_testMeshTypes "bpConduit") blt_list_append(TO _testMeshTypes ELEMENTS "mfem" IF MFEM_FOUND) foreach(_policy ${_policies}) + # sets the number of threads for the omp policy; leave empty for non-omp policies + set(_num_threads) + if(_policy STREQUAL "omp") + set(_num_threads ${AXOM_TEST_NUM_OMP_THREADS}) + endif() + foreach(_testMeshType ${_testMeshTypes}) foreach(_testshape ${_testshapes}) set(_testname "quest_shape_in_memory_${_policy}_${_testMeshType}_${_testshape}") @@ -404,7 +426,8 @@ if((CONDUIT_FOUND OR --dir 0.2 0.4 0.8 --meshType ${_testMeshType} inline_mesh --min -2 -2 -2 --max 2 2 2 --res 8 8 8 - NUM_MPI_TASKS ${_nranks}) + NUM_MPI_TASKS ${_nranks} + NUM_OMP_THREADS ${_num_threads}) endforeach() endforeach() endforeach() @@ -427,7 +450,8 @@ if((CONDUIT_FOUND OR --dir 0.2 0.4 0.8 --meshType ${_testMeshType} inline_mesh --min -2 -2 --max 2 2 --res 2 2 - NUM_MPI_TASKS ${_nranks}) + NUM_MPI_TASKS ${_nranks} + NUM_OMP_THREADS ${_num_threads}) endforeach() endforeach() endforeach() @@ -462,6 +486,12 @@ if(AXOM_ENABLE_MPI AND AXOM_ENABLE_SIDRE AND HDF5_FOUND) # src/tools/gen-multidom-structured-mesh.py -ml=0,0 -mu=2,2 -ms=100,100 -dc=2,3 -o mdmesh.2x3 # src/tools/gen-multidom-structured-mesh.py -ml=0,0,0 -mu=2,2,2 -ms=20,20,15 -dc=2,2,1 -o mdmeshg.2x2x1 --strided foreach(_pol ${_policies}) + # sets the number of threads for the omp policy; leave empty for non-omp policies + set(_num_threads) + if(_pol STREQUAL "omp") + set(_num_threads ${AXOM_TEST_NUM_OMP_THREADS}) + endif() + foreach(_mesh ${_meshes}) # Determine problem dimension by mesh filename. # and set dimension-dependent arguments. @@ -491,10 +521,8 @@ if(AXOM_ENABLE_MPI AND AXOM_ENABLE_SIDRE AND HDF5_FOUND) --policy ${_pol} --object-file dcp_object_mesh_${_ndim}d_${_pol}_${_mesh} --distance-file dcp_closest_point_2d_${_pol}_${_mesh} - NUM_MPI_TASKS ${_nranks}) - if(${_pol} STREQUAL "omp") - set_property(TEST ${_test} APPEND PROPERTY ENVIRONMENT OMP_NUM_THREADS=4) - endif() + NUM_MPI_TASKS ${_nranks} + NUM_OMP_THREADS ${_num_threads}) endforeach() endforeach() @@ -543,6 +571,12 @@ if(CONDUIT_FOUND) # src/tools/gen-multidom-structured-mesh.py -ml=0,0,0 -mu=2,2,2 -ms=20,20,15 -dc=2,2,1 -o mdmesh.2x2x1 # src/tools/gen-multidom-structured-mesh.py -ml=0,0,0 -mu=2,2,2 -ms=20,20,15 -dc=2,2,1 -o mdmeshg.2x2x1 --strided foreach(_pol ${_policies}) + # sets the number of threads for the omp policy; leave empty for non-omp policies + set(_num_threads) + if(_pol STREQUAL "omp") + set(_num_threads ${AXOM_TEST_NUM_OMP_THREADS}) + endif() + foreach(_mesh ${_meshes}) # Determine problem dimension by mesh filename. # and set dimension-dependent arguments. @@ -573,7 +607,8 @@ if(CONDUIT_FOUND) --contourVal 1.25 --check-results --verbose - NUM_MPI_TASKS ${_nranks}) + NUM_MPI_TASKS ${_nranks} + NUM_OMP_THREADS ${_num_threads}) endforeach() endforeach() @@ -653,31 +688,22 @@ axom_add_executable( ) if(AXOM_ENABLE_TESTS AND AXOM_DATA_DIR) - # Add a test for the quest interface; Set up for MPI, when available - if (AXOM_ENABLE_MPI) - axom_add_test( - NAME quest_inout_interface_3D_mpi_test - COMMAND quest_inout_interface_ex -i ${quest_data_dir}/sphere_binary.stl - NUM_MPI_TASKS 2 - ) - if(C2C_FOUND) - axom_add_test( - NAME quest_inout_interface_2D_mpi_test - COMMAND quest_inout_interface_ex -i ${AXOM_DATA_DIR}/contours/unit_circle.contour - NUM_MPI_TASKS 2 - ) - endif() - else() + set(_num_ranks 1) + if(AXOM_ENABLE_MPI) + set(_num_ranks 2) + endif() + + axom_add_test( + NAME quest_inout_interface_3D_test + COMMAND quest_inout_interface_ex -i ${quest_data_dir}/sphere_binary.stl + NUM_MPI_TASKS ${_num_ranks} + ) + if(C2C_FOUND) axom_add_test( - NAME quest_inout_interface_3D_test - COMMAND quest_inout_interface_ex -i ${quest_data_dir}/sphere_binary.stl + NAME quest_inout_interface_2D_test + COMMAND quest_inout_interface_ex -i ${AXOM_DATA_DIR}/contours/unit_circle.contour + NUM_MPI_TASKS ${_num_ranks} ) - if(C2C_FOUND) - axom_add_test( - NAME quest_inout_interface_2D_test - COMMAND quest_inout_interface_ex -i ${AXOM_DATA_DIR}/contours/unit_circle.contour - ) - endif() endif() endif() @@ -719,16 +745,14 @@ if (ENABLE_FORTRAN) endforeach() if(AXOM_ENABLE_TESTS AND AXOM_DATA_DIR AND ${_has_inout_fortran_example}) - if (AXOM_ENABLE_MPI) - axom_add_test( - NAME quest_inout_interface_mpi_F_test - COMMAND quest_inout_interface_F_ex ${quest_data_dir}/sphere_binary.stl - NUM_MPI_TASKS 2) - else() - axom_add_test( - NAME quest_inout_interface_F_test - COMMAND quest_inout_interface_F_ex ${quest_data_dir}/sphere_binary.stl) + set(_num_ranks 1) + if(AXOM_ENABLE_MPI) + set(_num_ranks 2) endif() + axom_add_test( + NAME quest_inout_interface_F_test + COMMAND quest_inout_interface_F_ex ${quest_data_dir}/sphere_binary.stl + NUM_MPI_TASKS ${_num_ranks}) endif() endif() @@ -755,10 +779,9 @@ if(OPENCASCADE_FOUND) ) if(AXOM_ENABLE_TESTS AND AXOM_DATA_DIR) + set(_num_ranks 1) if(AXOM_ENABLE_MPI) set(_num_ranks 2) - else() - set(_num_ranks 1) endif() axom_add_test( diff --git a/src/axom/quest/tests/CMakeLists.txt b/src/axom/quest/tests/CMakeLists.txt index 14ad1e8188..c763704a00 100644 --- a/src/axom/quest/tests/CMakeLists.txt +++ b/src/axom/quest/tests/CMakeLists.txt @@ -44,19 +44,18 @@ foreach(test ${quest_tests}) FOLDER axom/quest/tests ) + # sets the number of threads for the quest_discretize test + set(_num_threads) + if(AXOM_ENABLE_OPENMP AND "${test_name}" STREQUAL "quest_discretize") + set(_num_threads ${AXOM_TEST_NUM_OMP_THREADS}) + endif() + axom_add_test( NAME ${test_name} COMMAND ${test_name}_test - ) + NUM_OMP_THREADS ${_num_threads}) endforeach() -if(TARGET quest_discretize_test) - axom_add_test( NAME quest_discretize_test_openmp - COMMAND quest_discretize_test - NUM_OMP_THREADS 8 - ) -endif() - #------------------------------------------------------------------------------ # Tests that use Conduit when available #------------------------------------------------------------------------------ @@ -75,8 +74,7 @@ if(CONDUIT_FOUND AND AXOM_DATA_DIR) if(AXOM_ENABLE_TESTS AND AXOM_DATA_DIR AND NOT WIN32) axom_add_test( NAME quest_mesh_view_util - COMMAND quest_mesh_view_util_test - --verbose) + COMMAND quest_mesh_view_util_test --verbose) unset(_nranks) unset(_test) endif() @@ -148,23 +146,19 @@ foreach(test ${quest_mpi_tests}) FOLDER axom/quest/tests ) - set(_numMPITasks 4) - if (${test_name} STREQUAL "quest_inout_interface") + # sets the number of MPI ranks; quest_inout_interface can only run w/ 1 rank + set(_numMPITasks 1) + if (AXOM_ENABLE_MPI) + set(_numMPITasks 4) + endif() + if(${test_name} STREQUAL "quest_inout_interface") set(_numMPITasks 1) endif() - if (AXOM_ENABLE_MPI) - axom_add_test( - NAME ${test_name} - COMMAND ${test_name}_test - NUM_MPI_TASKS ${_numMPITasks} - ) - else() - axom_add_test( - NAME ${test_name} - COMMAND ${test_name}_test - ) - endif() + axom_add_test( + NAME ${test_name} + COMMAND ${test_name}_test + NUM_MPI_TASKS ${_numMPITasks}) endforeach() #------------------------------------------------------------------------------ @@ -191,38 +185,48 @@ if(ENABLE_MPI AND MFEM_FOUND AND MFEM_USE_MPI endforeach() # Invoke the intersection shaper test program as many tests that run a subset of the tests. - set(_numOMPThreads 8) # limit omp parallelism - set(_policies seq) + set(_policies "seq") if(RAJA_FOUND) - blt_list_append(TO _policies ELEMENTS omp IF AXOM_ENABLE_OPENMP) + blt_list_append(TO _policies ELEMENTS "omp" IF AXOM_ENABLE_OPENMP) if(AXOM_USE_UMPIRE) - blt_list_append(TO _policies ELEMENTS cuda IF AXOM_ENABLE_CUDA) - blt_list_append(TO _policies ELEMENTS hip IF AXOM_ENABLE_HIP) + blt_list_append(TO _policies ELEMENTS "cuda" IF AXOM_ENABLE_CUDA) + blt_list_append(TO _policies ELEMENTS "hip" IF AXOM_ENABLE_HIP) endif() endif() - foreach(shapingPolicy ${_policies}) + + foreach(_pol ${_policies}) + # sets the number of threads + set(_num_threads) + if(_pol STREQUAL "omp") + set(_num_threads ${AXOM_TEST_NUM_OMP_THREADS}) + endif() + # For seq, limit the cases since the tests are somewhat slow. - if(${shapingPolicy} STREQUAL "seq") + set(caseNumbers 1 2 3 4 5) + if(_pol STREQUAL "seq") set(caseNumbers 5) - else() - set(caseNumbers 1 2 3 4 5) endif() + foreach(caseNumber ${caseNumbers}) - axom_add_test(NAME quest_intersection_shaper_${caseNumber}_${shapingPolicy} - COMMAND quest_intersection_shaper_test --casenumber ${caseNumber} --policy ${shapingPolicy} + axom_add_test(NAME quest_intersection_shaper_${caseNumber}_${_pol} + COMMAND quest_intersection_shaper_test --casenumber ${caseNumber} --policy ${_pol} NUM_MPI_TASKS 1 - NUM_OMP_THREADS ${_numOMPThreads} - ) - set_tests_properties(quest_intersection_shaper_${caseNumber}_${shapingPolicy} PROPERTIES PROCESSORS ${_numOMPThreads}) + NUM_OMP_THREADS ${_num_threads}) endforeach() endforeach() + # quest_intersection_shaper_dynamic does not take a policy argument, + # but uses OpenMP when enabled + set(_dynamic_num_threads) + if(AXOM_ENABLE_OPENMP) + set(_dynamic_num_threads ${AXOM_TEST_NUM_OMP_THREADS}) + endif() + axom_add_test(NAME quest_intersection_shaper_dynamic COMMAND quest_intersection_shaper_dynamic_test NUM_MPI_TASKS 1 - NUM_OMP_THREADS ${_numOMPThreads} + NUM_OMP_THREADS ${_dynamic_num_threads} ) - set_tests_properties(quest_intersection_shaper_dynamic PROPERTIES PROCESSORS ${_numOMPThreads}) axom_add_test(NAME quest_sampling_shaper COMMAND quest_sampling_shaper_test @@ -256,6 +260,12 @@ if(CONDUIT_FOUND AND RAJA_FOUND AND AXOM_ENABLE_SIDRE) foreach(_meshType "bpSidre" "bpConduit") foreach(_policy ${_policies}) + # sets the number of threads for the omp policy; leave empty for non-omp policies + set(_num_threads) + if(_policy STREQUAL "omp") + set(_num_threads ${AXOM_TEST_NUM_OMP_THREADS}) + endif() + foreach(_testgeom "tet") set(_testname "quest_mesh_clipper_${_meshType}_${_policy}_${_testgeom}") axom_add_test( @@ -270,7 +280,8 @@ if(CONDUIT_FOUND AND RAJA_FOUND AND AXOM_ENABLE_SIDRE) --dir 8 4 2 --meshType ${_meshType} inline_mesh --min -2 -2 -2 --max 2 2 2 --res 16 16 16 - NUM_MPI_TASKS ${_nranks}) + NUM_MPI_TASKS ${_nranks} + NUM_OMP_THREADS ${_num_threads}) endforeach() endforeach() endforeach() diff --git a/src/axom/sidre/examples/CMakeLists.txt b/src/axom/sidre/examples/CMakeLists.txt index 4e97d0222b..c129a0b25e 100644 --- a/src/axom/sidre/examples/CMakeLists.txt +++ b/src/axom/sidre/examples/CMakeLists.txt @@ -79,15 +79,13 @@ if(AXOM_ENABLE_MPI) endif() endforeach() - blt_add_test( NAME sidre_createdatastore_parallel - COMMAND sidre_createdatastore_ex - NUM_MPI_TASKS 4 - ) + axom_add_test( NAME sidre_createdatastore_parallel + COMMAND sidre_createdatastore_ex + NUM_MPI_TASKS 4) - blt_add_test( NAME sidre_generateindex_parallel - COMMAND sidre_generateindex_ex - NUM_MPI_TASKS 4 - ) + axom_add_test( NAME sidre_generateindex_parallel + COMMAND sidre_generateindex_ex + NUM_MPI_TASKS 4) endif() @@ -110,20 +108,14 @@ if(MFEM_FOUND AND AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) FOLDER axom/sidre/examples) if(AXOM_ENABLE_TESTS) - if(AXOM_ENABLE_MPI) - if(MFEM_USE_MPI) - set(_nranks 2) - else() - set(_nranks 1) - endif() - - axom_add_test( NAME ${exe_name} - COMMAND ${exe_name}_ex - NUM_MPI_TASKS ${_nranks}) - else() - axom_add_test( NAME ${exe_name} - COMMAND ${exe_name}_ex) + set(_nranks 1) + if(AXOM_ENABLE_MPI AND MFEM_USE_MPI) + set(_nranks 2) endif() + + axom_add_test( NAME ${exe_name} + COMMAND ${exe_name}_ex + NUM_MPI_TASKS ${_nranks}) endif() endforeach() diff --git a/src/axom/sidre/examples/lulesh2/CMakeLists.txt b/src/axom/sidre/examples/lulesh2/CMakeLists.txt index 6f61b64d57..6bd93d289d 100644 --- a/src/axom/sidre/examples/lulesh2/CMakeLists.txt +++ b/src/axom/sidre/examples/lulesh2/CMakeLists.txt @@ -26,14 +26,13 @@ axom_add_executable( FOLDER axom/sidre/examples ) if(AXOM_ENABLE_TESTS) - if(AXOM_ENABLE_MPI) - axom_add_test(NAME sidre_lulesh2 - COMMAND sidre_lulesh2_ex -s 5 - NUM_MPI_TASKS 1 - NUM_OMP_THREADS 2 ) - else() - axom_add_test(NAME sidre_lulesh2 -s 5 - COMMAND sidre_lulesh2_ex - NUM_OMP_THREADS 2 ) + set(_num_threads) + if(AXOM_ENABLE_OPENMP) + set(_num_threads ${AXOM_TEST_NUM_OMP_THREADS}) endif() + + axom_add_test(NAME sidre_lulesh2 + COMMAND sidre_lulesh2_ex -s 5 + NUM_MPI_TASKS 1 + NUM_OMP_THREADS ${_num_threads} ) endif() diff --git a/src/axom/spin/tests/CMakeLists.txt b/src/axom/spin/tests/CMakeLists.txt index 31da4e7fce..43d8da0333 100644 --- a/src/axom/spin/tests/CMakeLists.txt +++ b/src/axom/spin/tests/CMakeLists.txt @@ -50,10 +50,16 @@ if (AXOM_ENABLE_MINT) FOLDER axom/spin/tests ) + # sets the number of threads for the omp policy; leave empty for non-omp policies + set(_num_threads) + if(AXOM_ENABLE_OPENMP) + set(_num_threads ${AXOM_TEST_NUM_OMP_THREADS}) + endif() + axom_add_test( NAME spin_bvh COMMAND spin_bvh_test - NUM_OMP_THREADS 4 + NUM_OMP_THREADS ${_num_threads} ) endif() diff --git a/src/cmake/AxomMacros.cmake b/src/cmake/AxomMacros.cmake index 272391ac51..564c9ca037 100644 --- a/src/cmake/AxomMacros.cmake +++ b/src/cmake/AxomMacros.cmake @@ -248,16 +248,26 @@ macro(axom_add_test) NUM_OMP_THREADS ${arg_NUM_OMP_THREADS} CONFIGURATIONS ${arg_CONFIGURATIONS} ) - ########################################################################### + #-------------------------------------------------------------------------- # Newer versions of OpenMPI require OMPI_MCA_rmaps_base_oversubscribe=1 # to run with more tasks than actual cores # Since this is an OpenMPI specific env var, it shouldn't interfere # with other mpi implementations. - ########################################################################### + #-------------------------------------------------------------------------- set_property(TEST ${arg_NAME} APPEND PROPERTY ENVIRONMENT "OMPI_MCA_rmaps_base_oversubscribe=1") + #-------------------------------------------------------------------------- + # Cap OpenMP parallelism for tests that do not explicitly + # specify NUM_OMP_THREADS to avoid accidental oversubscription + #-------------------------------------------------------------------------- + if(AXOM_ENABLE_OPENMP AND (NOT arg_NUM_OMP_THREADS)) + set_property(TEST ${arg_NAME} + APPEND + PROPERTY ENVIRONMENT OMP_NUM_THREADS=1) + endif() + endmacro(axom_add_test) diff --git a/src/cmake/AxomOptions.cmake b/src/cmake/AxomOptions.cmake index cbb401bf1e..3e09484f0f 100644 --- a/src/cmake/AxomOptions.cmake +++ b/src/cmake/AxomOptions.cmake @@ -47,6 +47,16 @@ option(AXOM_ENABLE_TOOLS "Enables Axom Tools" ON) option(AXOM_ENABLE_TUTORIALS "Builds Axom tutorials as part of the Axom build" ON) mark_as_advanced(AXOM_ENABLE_TUTORIALS) +#------------------------------------------------------------------------------ +# Test execution controls +#------------------------------------------------------------------------------ +if(AXOM_ENABLE_OPENMP) + set(AXOM_TEST_NUM_OMP_THREADS 4 CACHE STRING "Default number of OpenMP threads for tests") +else() + set(AXOM_TEST_NUM_OMP_THREADS 0 CACHE STRING "Default number of OpenMP threads for tests") +endif() +mark_as_advanced(AXOM_TEST_NUM_OMP_THREADS) + #-------------------------------------------------------------------------- # Option to control whether AXOM_DEFINE compiler define is enabled # diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index 6d63017b03..24b0566644 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -133,27 +133,21 @@ if( AXOM_ENABLE_SIDRE AND AXOM_ENABLE_PRIMAL AND FOLDER axom/examples ) + set(_nranks 1) if(AXOM_ENABLE_MPI) - axom_add_test( - NAME data_collection_util_box2D - COMMAND data_collection_util --min -1 -1 --max 1 1 --res 16 16 -p 1 - NUM_MPI_TASKS 2 - ) - axom_add_test( - NAME data_collection_util_box3D - COMMAND data_collection_util --min -1 -1 -1 --max 1 1 1 --res 16 16 16 -p 1 - NUM_MPI_TASKS 2 - ) - else() - axom_add_test( - NAME data_collection_util_box2D - COMMAND data_collection_util --min -1 -1 --max 1 1 --res 16 16 -p 1 - ) - axom_add_test( - NAME data_collection_util_box3D - COMMAND data_collection_util --min -1 -1 -1 --max 1 1 1 --res 16 16 16 -p 1 - ) + set(_nranks 2) endif() + + axom_add_test( + NAME data_collection_util_box2D + COMMAND data_collection_util --min -1 -1 --max 1 1 --res 16 16 -p 1 + NUM_MPI_TASKS ${_nranks} + ) + axom_add_test( + NAME data_collection_util_box3D + COMMAND data_collection_util --min -1 -1 -1 --max 1 1 1 --res 16 16 16 -p 1 + NUM_MPI_TASKS ${_nranks} + ) install(TARGETS data_collection_util DESTINATION bin