diff --git a/packages/stk/CMakeLists.txt b/packages/stk/CMakeLists.txt index bf77c842c818..d3afaf8b720d 100644 --- a/packages/stk/CMakeLists.txt +++ b/packages/stk/CMakeLists.txt @@ -34,6 +34,8 @@ cmake_minimum_required(VERSION 3.16 FATAL_ERROR) +message("starting STK cmake configuration, CMAKE_SOURCE_DIR=${CMAKE_SOURCE_DIR}") + IF(COMMAND TRIBITS_PACKAGE_DECL) SET(HAVE_STK_Trilinos ON) TRIBITS_PACKAGE_DECL(STK) @@ -123,6 +125,28 @@ ELSE() ENDIF() ENDIF() +find_package(ArborX) +if(TARGET ArborX::ArborX) + MESSAGE("found ArborX") + SET(STK_HAS_ARBORX ON) +else() + MESSAGE("did not find ArborX. If stk_search is enabled, ArborX will not be a valid option.") +endif() + +find_package(SEACAS) +if(SEACAS_ENABLE_SEACASIoss) + message("found SEACASIoss") + SET(STK_HAS_SEACAS_IOSS ON) +endif() +if(SEACAS_ENABLE_SEACASExodus) + message("found SEACASExodus") + SET(STK_HAS_SEACAS_EXODUS ON) +endif() +if(SEACAS_ENABLE_SEACASNemesis) + message("found SEACASNemesis") + SET(STK_HAS_SEACAS_NEMESIS ON) +endif() + if(NOT HAVE_STK_Trilinos) stk_process_enables() endif() diff --git a/packages/stk/cmake/Dependencies.cmake b/packages/stk/cmake/Dependencies.cmake index 70644087957c..7e321f84db2f 100644 --- a/packages/stk/cmake/Dependencies.cmake +++ b/packages/stk/cmake/Dependencies.cmake @@ -1,26 +1,27 @@ # the order of the packages below is important. Stk packages can # depend on packages listed above them, and not below them. SET(SUBPACKAGES_DIRS_CLASSIFICATIONS_OPTREQS - Util stk_util PT OPTIONAL - Coupling stk_coupling PT OPTIONAL - Math stk_math PT OPTIONAL - Simd stk_simd PT OPTIONAL - NGP_TEST stk_ngp_test PT OPTIONAL - ExprEval stk_expreval PT OPTIONAL - Topology stk_topology PT OPTIONAL - Search stk_search PT OPTIONAL - Middle_mesh stk_middle_mesh EX OPTIONAL - Transfer stk_transfer PT OPTIONAL - Mesh stk_mesh PT OPTIONAL - SearchUtil stk_search_util EX OPTIONAL - TransferUtil stk_transfer_util EX OPTIONAL - IO stk_io PT OPTIONAL - Tools stk_tools PT OPTIONAL - Balance stk_balance PT OPTIONAL - Unit_test_utils stk_unit_test_utils PT OPTIONAL - Unit_tests stk_unit_tests PT OPTIONAL - Doc_tests stk_doc_tests PT OPTIONAL - Emend stk_emend PT OPTIONAL + Util stk_util PT OPTIONAL + Coupling stk_coupling PT OPTIONAL + Math stk_math PT OPTIONAL + Simd stk_simd PT OPTIONAL + NGP_TEST stk_ngp_test PT OPTIONAL + ExprEval stk_expreval PT OPTIONAL + Topology stk_topology PT OPTIONAL + Search stk_search PT OPTIONAL + Middle_mesh stk_middle_mesh EX OPTIONAL + Middle_mesh_util stk_middle_mesh_util EX OPTIONAL + Transfer stk_transfer PT OPTIONAL + Mesh stk_mesh PT OPTIONAL + SearchUtil stk_search_util EX OPTIONAL + TransferUtil stk_transfer_util EX OPTIONAL + IO stk_io PT OPTIONAL + Tools stk_tools PT OPTIONAL + Balance stk_balance PT OPTIONAL + Unit_test_utils stk_unit_test_utils PT OPTIONAL + Unit_tests stk_unit_tests PT OPTIONAL + Doc_tests stk_doc_tests PT OPTIONAL + Emend stk_emend PT OPTIONAL ) SET(LIB_REQUIRED_DEP_PACKAGES) diff --git a/packages/stk/cmake/STK_Trilinos_config.h.in b/packages/stk/cmake/STK_Trilinos_config.h.in index fc28dea5a613..0f5ea4683d84 100644 --- a/packages/stk/cmake/STK_Trilinos_config.h.in +++ b/packages/stk/cmake/STK_Trilinos_config.h.in @@ -52,8 +52,16 @@ #cmakedefine STK_HAS_MPI +#cmakedefine STK_HAS_ARBORX + #cmakedefine STK_VOLATILE_SIMD +#cmakedefine STK_HAS_SEACAS_IOSS + +#cmakedefine STK_HAS_SEACAS_EXODUS + +#cmakedefine STK_HAS_SEACAS_NEMESIS + /* Non-typical "not" macro for optional dependence on SEACASAprepro_lib (See * STK #17354) */ #ifndef NOT_HAVE_STK_SEACASAPREPRO_LIB diff --git a/packages/stk/cmake/stk_wrappers.cmake b/packages/stk/cmake/stk_wrappers.cmake index b501ffb7b1c1..3fc4d74cca0e 100644 --- a/packages/stk/cmake/stk_wrappers.cmake +++ b/packages/stk/cmake/stk_wrappers.cmake @@ -75,6 +75,14 @@ function(stk_process_enables) set(STK_ENABLE_STKMiddle_mesh ON CACHE BOOL "") endif() + if (DEFINED STK_ENABLE_STKMiddle_mesh_util AND NOT STK_ENABLE_STKMiddle_mesh_util) + message("STKMiddle_mesh_util explicitly disabled.") + elseif(NOT STK_ENABLE_STKMiddle_mesh) + message("STKMiddle_mesh_util disabled because STKMiddle_mesh not enabled.") + else() + set(STK_ENABLE_STKMiddle_mesh_util ON CACHE BOOL "") + endif() + if (DEFINED STK_ENABLE_STKTransfer AND NOT STK_ENABLE_STKTransfer) message("STKTransfer explicitly disabled.") elseif(NOT STK_ENABLE_STKSearch) @@ -114,7 +122,11 @@ function(stk_process_enables) elseif(NOT STK_ENABLE_STKMesh) message("STKIO disabled because STKMesh not enabled.") else() - set(STK_ENABLE_STKIO ON CACHE BOOL "") + if (STK_HAS_SEACAS_IOSS) + set(STK_ENABLE_STKIO ON CACHE BOOL "") + else() + message("STKIO disabled because SEACASIoss not available.") + endif() endif() if (DEFINED STK_ENABLE_STKTools AND NOT STK_ENABLE_STKTools) @@ -150,6 +162,7 @@ macro(STK_SUBPACKAGES) if(STK_ENABLE_STKCoupling) add_subdirectory(stk_coupling) + message("STKCoupling is enabled.") endif() if(STK_ENABLE_STKMath) @@ -187,6 +200,11 @@ macro(STK_SUBPACKAGES) message("STKMiddle_mesh is enabled.") endif() + if(STK_ENABLE_STKMiddle_mesh_util) + add_subdirectory(stk_middle_mesh_util) + message("STKMiddle_mesh_util is enabled.") + endif() + if(STK_ENABLE_STKTransfer) add_subdirectory(stk_transfer) message("STKTransfer is enabled.") diff --git a/packages/stk/stk_balance/stk_balance/internal/NodeBalancer.cpp b/packages/stk/stk_balance/stk_balance/internal/NodeBalancer.cpp index 8ea66ad5ae20..2d0939249f08 100644 --- a/packages/stk/stk_balance/stk_balance/internal/NodeBalancer.cpp +++ b/packages/stk/stk_balance/stk_balance/internal/NodeBalancer.cpp @@ -122,10 +122,10 @@ NodeBalancer::exchangeLocalSizes(const std::set& neighborProcessors, } std::vector receiveStati(receiveRequests.size()); - MPI_Waitall(receiveRequests.size(), &receiveRequests[0], &receiveStati[0]); + MPI_Waitall(receiveRequests.size(), receiveRequests.data(), receiveStati.data()); std::vector sendStati(sendRequests.size()); - MPI_Waitall(sendRequests.size(), &sendRequests[0], &sendStati[0]); + MPI_Waitall(sendRequests.size(), sendRequests.data(), sendStati.data()); int i = 0; for (int p : neighborProcessors) { diff --git a/packages/stk/stk_balance/stk_balance/internal/StkBalanceUtils.cpp b/packages/stk/stk_balance/stk_balance/internal/StkBalanceUtils.cpp index 214968528036..52003dc033da 100644 --- a/packages/stk/stk_balance/stk_balance/internal/StkBalanceUtils.cpp +++ b/packages/stk/stk_balance/stk_balance/internal/StkBalanceUtils.cpp @@ -116,7 +116,7 @@ void fillFaceBoxesWithIds(stk::mesh::BulkData &stkMeshBulkData, const BalanceSet sidesetSide, sideNodes); const double eps = balanceSettings.getToleranceForFaceSearch(stkMeshBulkData, *coord, sideNodes.data(), sideNodes.size()); - addBoxForNodes(stkMeshBulkData, sideNodes.size(), &sideNodes[0], coord, eps, + addBoxForNodes(stkMeshBulkData, sideNodes.size(), sideNodes.data(), coord, eps, stkMeshBulkData.identifier(sidesetElement), faceBoxes); } } diff --git a/packages/stk/stk_coupling/Jamfile b/packages/stk/stk_coupling/Jamfile index acbb4cf2fdd4..5cffbebd269b 100644 --- a/packages/stk/stk_coupling/Jamfile +++ b/packages/stk/stk_coupling/Jamfile @@ -157,7 +157,7 @@ exe stk_coupling_utest [ glob $(stk_coupling-root)/../stk_unit_tests/stk_coupling/*.cpp ] /sierra/stk_coupling//stk_coupling /sierra/stk_unit_test_utils//stk_unit_main - /tpl/gtest//gtest + /tpl/googletest//gtest : @sierra-exec-tag ; diff --git a/packages/stk/stk_doc_tests/stk_io/handleMissingFieldOnRead.cpp b/packages/stk/stk_doc_tests/stk_io/handleMissingFieldOnRead.cpp index c57262db384e..c3ecc5b0bc9b 100644 --- a/packages/stk/stk_doc_tests/stk_io/handleMissingFieldOnRead.cpp +++ b/packages/stk/stk_doc_tests/stk_io/handleMissingFieldOnRead.cpp @@ -38,7 +38,8 @@ #include // for operator<< #include // for StkMeshIoBroker #include // for Field -#include // for get_entities +#include +#include #include // for MetaData, put_field #include // for string #include // for vector @@ -82,24 +83,17 @@ namespace { //+ The name of the field on the database will be "temp" stkIo.add_field(fh, temperature, "temp"); - - std::vector nodes; - stk::mesh::get_entities(stkIo.bulk_data(), - stk::topology::NODE_RANK, nodes); // Add three steps to the database // For each step, the value of the field is the value 'time' for (size_t i=0; i < 3; i++) { - double time = i; + double time = i; - for(size_t inode=0; inodename()); EXPECT_EQ("disp", missing_fields[1].db_name()); EXPECT_EQ("displacement_STKFS_N", missing_fields[1].field()->name()); - + // The value of the "temperature" field at all nodes should be 2.0 - std::vector nodes; - stk::mesh::get_entities(stkIo.bulk_data(), stk::topology::NODE_RANK, - nodes); - for(size_t i=0; i(stk::mesh::field_data(*field, entity) ); - unsigned numScalars = stk::mesh::field_scalars_per_entity(*field, entity); - ASSERT_EQ(fieldLength, numScalars); + const double* fieldDataForEntity = static_cast(stk::mesh::field_data(*field, entity) ); + ASSERT_EQ(fieldLength, stk::mesh::field_scalars_per_entity(*field, entity)); ASSERT_EQ(initialValue[0], fieldDataForEntity[0]); } diff --git a/packages/stk/stk_doc_tests/stk_io/howToWriteMesh.cpp b/packages/stk/stk_doc_tests/stk_io/howToWriteMesh.cpp index f3e11aa555b2..b19a991840af 100644 --- a/packages/stk/stk_doc_tests/stk_io/howToWriteMesh.cpp +++ b/packages/stk/stk_doc_tests/stk_io/howToWriteMesh.cpp @@ -44,7 +44,7 @@ TEST(StkIoHowTo, WriteMesh) unlink(filename.c_str()); } -TEST(StkIoHowTo, generateHugeMesh) +TEST(StkIoHowTo, generateMeshWith64BitIds) { std::string meshSpec = stk::unit_test_util::simple_fields::get_option("-i", "1x1x4"); std::string fullMeshSpec = "generated:"+meshSpec; @@ -52,6 +52,8 @@ TEST(StkIoHowTo, generateHugeMesh) std::string filename = "output.exo"; stk::io::StkMeshIoBroker inputBroker; inputBroker.use_simple_fields(); + + //+ Set properties to ensure that 64-bit integers will be used inputBroker.property_add(Ioss::Property("INTEGER_SIZE_API" , 8)); inputBroker.property_add(Ioss::Property("INTEGER_SIZE_DB" , 8)); std::shared_ptr bulk = stk::mesh::MeshBuilder(MPI_COMM_WORLD).create(); diff --git a/packages/stk/stk_doc_tests/stk_io/interpolateFieldCyclic.cpp b/packages/stk/stk_doc_tests/stk_io/interpolateFieldCyclic.cpp index 4d68ee8df6f7..928a130b249f 100644 --- a/packages/stk/stk_doc_tests/stk_io/interpolateFieldCyclic.cpp +++ b/packages/stk/stk_doc_tests/stk_io/interpolateFieldCyclic.cpp @@ -39,7 +39,8 @@ #include // for InputFile, etc #include // for StkMeshIoBroker #include // for Field -#include // for get_entities +#include +#include #include // for MetaData, put_field #include // for string #include // for vector @@ -84,20 +85,12 @@ TEST(StkMeshIoBrokerHowTo, interpolateFieldCyclic) //+ The name of the field on the database will be "temp" stkIo.add_field(fh, temperature, "temp"); - std::vector nodes; - stk::mesh::get_entities(stkIo.bulk_data(), - stk::topology::NODE_RANK, nodes); - // Add three steps to the database // For each step, the value of the field is the value 'time' for (size_t i=0; i < 3; i++) { double time = i * 10.0; - for(size_t inode=0; inode nodes; - stk::mesh::get_entities(stkIo.bulk_data(), stk::topology::NODE_RANK, nodes); - // The name of the field on the database is "temp" stkIo.add_input_field(stk::io::MeshField(temperature, "temp", stk::io::MeshField::LINEAR_INTERPOLATION)); @@ -167,10 +157,12 @@ TEST(StkMeshIoBrokerHowTo, interpolateFieldCyclic) // ============================================================ //+ VERIFICATION // The value of the "temperature" field at all nodes should be 'expected' - for(size_t i=0; i= 20.0 || expected <= 0.0) { diff --git a/packages/stk/stk_doc_tests/stk_io/interpolateFieldNegativeTime.cpp b/packages/stk/stk_doc_tests/stk_io/interpolateFieldNegativeTime.cpp index 7d456b9b68de..f002b9458944 100644 --- a/packages/stk/stk_doc_tests/stk_io/interpolateFieldNegativeTime.cpp +++ b/packages/stk/stk_doc_tests/stk_io/interpolateFieldNegativeTime.cpp @@ -38,7 +38,8 @@ #include // for operator<< #include // for StkMeshIoBroker #include // for Field -#include // for get_entities +#include +#include #include // for MetaData, put_field #include // for string #include // for vector @@ -84,20 +85,12 @@ TEST(StkMeshIoBrokerHowTo, interpolateFieldNegativeTime) //+ The name of the field on the database will be "temp" stkIo.add_field(fh, temperature, "temp"); - std::vector nodes; - stk::mesh::get_entities(stkIo.bulk_data(), - stk::topology::NODE_RANK, nodes); - // Add three steps to the database // For each step, the value of the field is the value 'time' for (int i=-2; i <= 0; i++) { double time = i; - for(size_t inode=0; inode nodes; - stk::mesh::get_entities(stkIo.bulk_data(), stk::topology::NODE_RANK, nodes); - // The name of the field on the database is "temp" stkIo.add_input_field(stk::io::MeshField(temperature, "temp", stk::io::MeshField::LINEAR_INTERPOLATION)); @@ -141,10 +131,11 @@ TEST(StkMeshIoBrokerHowTo, interpolateFieldNegativeTime) // ============================================================ //+ VERIFICATION // The value of the "temperature" field at all nodes should be 'time' - for(size_t j=0; j // for operator<< #include // for StkMeshIoBroker #include // for Field -#include // for get_entities +#include +#include #include // for MetaData, put_field #include // for string #include // for vector @@ -84,19 +85,13 @@ TEST(StkMeshIoBrokerHowTo, interpolateFieldNonMonotonicTime) //+ The name of the field on the database will be "temp" stkIo.add_field(fh, temperature, "temp"); - std::vector nodes; - stk::mesh::get_entities(stkIo.bulk_data(), - stk::topology::NODE_RANK, nodes); - // Add three steps to the database // For each step, the value of the field is the value 'time' double times[] = {2.0, 0.0, 1.0}; for (int i=0; i < 3; i++) { double time = times[i]; - for(size_t inode=0; inode nodes; - stk::mesh::get_entities(stkIo.bulk_data(), stk::topology::NODE_RANK, nodes); - // The name of the field on the database is "temp" stkIo.add_input_field(stk::io::MeshField(temperature, "temp", stk::io::MeshField::LINEAR_INTERPOLATION)); @@ -141,10 +133,11 @@ TEST(StkMeshIoBrokerHowTo, interpolateFieldNonMonotonicTime) // ============================================================ //+ VERIFICATION // The value of the "temperature" field at all nodes should be 'time' - for(size_t j=0; j nodes; - stk::mesh::get_entities(stkIo.bulk_data(), - stk::topology::NODE_RANK, nodes); - // Add three steps to the database // For each step, the value of the field is the value 'time' for (size_t i=0; i < 3; i++) { double time = i; - for(size_t inode=0; inode nodes; - stk::mesh::get_entities(stkIo.bulk_data(), stk::topology::NODE_RANK, nodes); - //+ Specify that the field data are to be linear interpolated. stkIo.add_input_field(stk::io::MeshField(temperature, "temp", stk::io::MeshField::LINEAR_INTERPOLATION));/*@\label{io:input:interpolate}*/ @@ -145,10 +136,11 @@ TEST(StkMeshIoBrokerHowTo, interpolateNodalField) // ============================================================ //+ VERIFICATION // The value of the "temperature" field at all nodes should be 'time' - for(size_t j=0; j // for operator<< #include // for StkMeshIoBroker #include // for Field -#include // for get_entities +#include #include // for MetaData, put_field #include // for string #include // for vector @@ -46,6 +46,7 @@ #include "stk_io/MeshField.hpp" // for MeshField, etc #include "stk_mesh/base/Entity.hpp" // for Entity #include "stk_mesh/base/FieldBase.hpp" // for field_data +#include "stk_mesh/base/ForEachEntity.hpp" #include "stk_topology/topology.hpp" // for topology, etc #include "stk_util/parallel/Parallel.hpp" namespace { @@ -82,18 +83,11 @@ TEST(StkMeshIoBrokerHowTo, interpolateOutsideRange) //+ The name of the field on the database will be "temp" stkIo.add_field(fh, temperature, "temp"); - std::vector nodes; - stk::mesh::get_entities(stkIo.bulk_data(), stk::topology::NODE_RANK, nodes); - // Add two steps to the database for (size_t i=1; i <= 2; i++) { double time = i; - for(size_t inode=0; inode nodes; - stk::mesh::get_entities(stkIo.bulk_data(), stk::topology::NODE_RANK, nodes); - // The name of the field on the database is "temp" stkIo.add_input_field(stk::io::MeshField(temperature, "temp", stk::io::MeshField::LINEAR_INTERPOLATION)); @@ -153,10 +144,11 @@ TEST(StkMeshIoBrokerHowTo, interpolateOutsideRange) if (time >= 2.0) expected_value = 2.0; - for(size_t j=0; j // for operator<< #include // for StkMeshIoBroker #include // for Field -#include // for get_entities +#include #include // for MetaData, put_field #include // for string #include // for vector @@ -46,6 +46,7 @@ #include "stk_io/MeshField.hpp" // for MeshField, etc #include "stk_mesh/base/Entity.hpp" // for Entity #include "stk_mesh/base/FieldBase.hpp" // for field_data +#include "stk_mesh/base/FieldBLAS.hpp" #include "stk_topology/topology.hpp" // for topology, etc #include "stk_util/parallel/Parallel.hpp" @@ -82,18 +83,10 @@ TEST(StkMeshIoBrokerHowTo, interpolateSingleStep) //+ The name of the field on the database will be "temp" stkIo.add_field(fh, temperature, "temp"); - std::vector nodes; - stk::mesh::get_entities(stkIo.bulk_data(), - stk::topology::NODE_RANK, nodes); - // Add one step to the database double time = 1.0; - for(size_t inode=0; inode nodes; - stk::mesh::get_entities(stkIo.bulk_data(), stk::topology::NODE_RANK, nodes); - for (size_t i=0; i < 21; i++) { double time = i/10.0; //+ Read the field values from the database and verify that they @@ -138,10 +128,11 @@ TEST(StkMeshIoBrokerHowTo, interpolateSingleStep) // ============================================================ //+ VERIFICATION // The value of the "temperature" field at all nodes should be 1.0 - for(size_t j=0; j get_attributes_of_first_element(const stk::mesh::BulkData &b stk::mesh::get_entities(bulk, stk::topology::ELEM_RANK, *ioPart, elements); std::vector attributes; - if(!elements.empty()) - { - for(const stk::mesh::FieldBase *field : attributeFields) - { + if(!elements.empty()) { + for(const stk::mesh::FieldBase *field : attributeFields) { unsigned numAttribute = stk::mesh::field_scalars_per_entity(*field, elements[0]); double *dataForElement = static_cast (stk::mesh::field_data(*field, elements[0])); for(unsigned i=0; i nodes; - stk::mesh::get_entities(stkIo.bulk_data(), - stk::topology::NODE_RANK, nodes); - // Add three steps to the database // For each step, the value of the field is the value 'time' for (size_t i=0; i < 3; i++) { double time = i; - for(size_t inode=0; inode nodes; - stk::mesh::get_entities(stkIo.bulk_data(), stk::topology::NODE_RANK, nodes); - for(size_t i=0; i nodes; - stk::mesh::get_entities(stkIo.bulk_data(), - stk::topology::NODE_RANK, nodes); - // Add three steps to the database // For each step, the value of the field is the value 'time' for (size_t i=0; i < 3; i++) { double time = i; - for(size_t inode=0; inode nodes; - stk::mesh::get_entities(stkIo.bulk_data(), stk::topology::NODE_RANK, - nodes); - for(size_t i=0; i nodes; - stk::mesh::get_entities(stkIo.bulk_data(), - stk::topology::NODE_RANK, nodes); - // Add three steps to the database // For each step, the value of the field is the value 'time' for (size_t i=0; i < 3; i++) { double time = i; - for(size_t inode=0; inode nodes; - stk::mesh::get_entities(stkIo.bulk_data(), stk::topology::NODE_RANK, - nodes); - - //+ The value of the "temperature" field at all nodes should be 2.0 - for(size_t i=0; i // for AssertHelper, EXPECT_NE, etc #include // for size_t, NULL #include // for unlink -#include // for StkMeshIoBroker -#include // for MetaData +#include +#include +#include +#include +#include +#include +#include #include // for string #include "stk_io/DatabasePurpose.hpp" // for DatabasePurpose::READ_MESH, etc #include "stk_util/parallel/Parallel.hpp" @@ -45,7 +50,7 @@ namespace { TEST(StkMeshIoBrokerHowTo, readMesh) { - std::string mesh_name = "input_mesh_example.e"; + std::string mesh_file_name = "input_mesh_example.e"; MPI_Comm communicator = MPI_COMM_WORLD; int numProcs = stk::parallel_machine_size(communicator); if (numProcs != 1) { @@ -54,20 +59,15 @@ TEST(StkMeshIoBrokerHowTo, readMesh) { // ============================================================ - //+ INITIALIZATION: - //+ Create a basic mesh with a hex block, 3 shell blocks, 3 nodesets, and 3 sidesets. - stk::io::StkMeshIoBroker stkIo(communicator); - stkIo.use_simple_fields(); + //BeginBasicReadWrite + std::shared_ptr stkMesh = stk::mesh::MeshBuilder(communicator).create(); + stkMesh->mesh_meta_data().use_simple_fields(); + //+ Create a basic mesh with a hex block, 3 shell blocks, 3 nodesets, and 3 sidesets. const std::string generatedFileName = "generated:8x8x8|shell:xyz|nodeset:xyz|sideset:XYZ"; - size_t index = stkIo.add_mesh_database(generatedFileName, stk::io::READ_MESH); - stkIo.set_active_mesh(index); - - stkIo.create_input_mesh(); - stkIo.populate_bulk_data(); - - size_t fh = stkIo.create_output_mesh(mesh_name, stk::io::WRITE_RESULTS); - stkIo.write_output_mesh(fh); + stk::io::fill_mesh(generatedFileName, *stkMesh); + stk::io::write_mesh(mesh_file_name, *stkMesh); + //EndBasicReadWrite } { @@ -77,43 +77,45 @@ TEST(StkMeshIoBrokerHowTo, readMesh) //+ Read mesh data from the specified file. stk::io::StkMeshIoBroker stkIo(communicator); stkIo.use_simple_fields(); - stkIo.add_mesh_database(mesh_name, stk::io::READ_MESH); + stkIo.add_mesh_database(mesh_file_name, stk::io::READ_MESH); //+ Creates meta data; creates parts stkIo.create_input_mesh(); - //+ Any modifications to the meta data must be done here. - //+ This includes declaring fields. + //+ Modifications to the meta data (such as creating extra parts and fields) + //+ is generally be done here. //+ Commit the meta data and create the bulk data. - //+ populate the bulk data with data from the mesh file. + //+ Populate the bulk data with data from the mesh file. stkIo.populate_bulk_data(); // ============================================================ //+ VERIFICATION - //+ There should be: - //+ 4 parts corresponding to the 1 hex block and 3 shell blocks - stk::mesh::MetaData &meta = stkIo.meta_data(); - stk::mesh::Part *invalid = NULL; - EXPECT_NE(invalid, meta.get_part("block_1")); - EXPECT_NE(invalid, meta.get_part("block_2")); - EXPECT_NE(invalid, meta.get_part("block_3")); - EXPECT_NE(invalid, meta.get_part("block_4")); + //+ In this case we know that the mesh (specified above) contains + //+ 4 element blocks, 3 nodesets, and 3 sidesets + //+ There should be a STK Mesh Part for each of those. + std::shared_ptr meta = stkIo.meta_data_ptr(); + EXPECT_NE(nullptr, meta->get_part("block_1")); + EXPECT_NE(nullptr, meta->get_part("block_2")); + EXPECT_NE(nullptr, meta->get_part("block_3")); + EXPECT_NE(nullptr, meta->get_part("block_4")); - //+ 3 parts corresponding to the 3 nodesets. - EXPECT_NE(invalid, meta.get_part("nodelist_1")); - EXPECT_NE(invalid, meta.get_part("nodelist_2")); - EXPECT_NE(invalid, meta.get_part("nodelist_3")); + EXPECT_NE(nullptr, meta->get_part("nodelist_1")); + EXPECT_NE(nullptr, meta->get_part("nodelist_2")); + EXPECT_NE(nullptr, meta->get_part("nodelist_3")); - //+ 3 parts corresponding to the 3 sidesets. - EXPECT_NE(invalid, meta.get_part("surface_1")); - EXPECT_NE(invalid, meta.get_part("surface_2")); - EXPECT_NE(invalid, meta.get_part("surface_3")); + EXPECT_NE(nullptr, meta->get_part("surface_1")); + EXPECT_NE(nullptr, meta->get_part("surface_2")); + EXPECT_NE(nullptr, meta->get_part("surface_3")); + std::shared_ptr bulk = stkIo.bulk_data_ptr(); + stk::mesh::EntityVector shellElems; + stk::mesh::get_entities(*bulk, stk::topology::ELEM_RANK, *meta->get_part("block_2"), shellElems); + EXPECT_EQ(64u, shellElems.size()); //-END } // ============================================================ // Cleanup - unlink(mesh_name.c_str()); + unlink(mesh_file_name.c_str()); } } diff --git a/packages/stk/stk_doc_tests/stk_io/useNodesetDbVarForNodalField.cpp b/packages/stk/stk_doc_tests/stk_io/useNodesetDbVarForNodalField.cpp index 24c32f20323f..9a54df86ee5b 100644 --- a/packages/stk/stk_doc_tests/stk_io/useNodesetDbVarForNodalField.cpp +++ b/packages/stk/stk_doc_tests/stk_io/useNodesetDbVarForNodalField.cpp @@ -49,6 +49,7 @@ #include "stk_io/DatabasePurpose.hpp" // for DatabasePurpose::READ_MESH, etc #include "stk_mesh/base/Entity.hpp" // for Entity #include "stk_mesh/base/FieldBase.hpp" // for field_data +#include "stk_mesh/base/FieldBLAS.hpp" #include "stk_mesh/base/Part.hpp" // for Part #include "stk_mesh/base/Selector.hpp" // for Selector #include "stk_mesh/base/Types.hpp" // for PartVector @@ -113,20 +114,12 @@ TEST(StkMeshIoBrokerHowTo, useNodesetDbVarForNodalFields) stkIo.use_nodeset_for_block_nodes_fields(fh, true); stkIo.add_field(fh, temperature, dbFieldName); - std::vector nodes; - stk::mesh::get_entities(stkIo.bulk_data(), - stk::topology::NODE_RANK, nodes); - // Add three steps to the database // For each step, the value of the field is the value 'time' for (size_t i=0; i < 3; i++) { double time = i; - for(size_t inode=0; inode // for AssertHelper, EXPECT_EQ, etc #include // for basic_ostream::operator<< @@ -66,17 +70,12 @@ TEST(StkMeshIoBrokerHowTo, writeResults) // ============================================================ //+ INITIALIZATION: //+ Create a basic mesh with a hex block, 3 shell blocks, 3 nodesets, and 3 sidesets. - stk::io::StkMeshIoBroker stkIo(communicator); - stkIo.use_simple_fields(); + std::unique_ptr mesh = stk::mesh::MeshBuilder(communicator).create(); + mesh->mesh_meta_data().use_simple_fields(); const std::string generatedFileName = "generated:8x8x8|shell:xyz|nodeset:xyz|sideset:XYZ"; - size_t index = stkIo.add_mesh_database(generatedFileName, stk::io::READ_MESH); - stkIo.set_active_mesh(index); - stkIo.create_input_mesh(); - stkIo.populate_bulk_data(); - - size_t fh = stkIo.create_output_mesh(mesh_name, stk::io::WRITE_RESULTS); - stkIo.write_output_mesh(fh); + stk::io::fill_mesh(generatedFileName, *mesh); + stk::io::write_mesh(mesh_name, *mesh); } { @@ -110,20 +109,14 @@ TEST(StkMeshIoBrokerHowTo, writeResults) //+ The field will be output to the results file with the default field name. stkIo.add_field(fh, field); /*@\label{io:results:add_field}*/ - std::vector nodes; - stk::mesh::get_entities(stkIo.bulk_data(), stk::topology::NODE_RANK, nodes); - // Iterate the application's execute loop five times and output - // field data each iteration. + // field data each iteration (mimicing time steps). for (int step=0; step < 5; step++) { double time = step; - // Application execution... + //simulate time-varying result field... double value = 10.0 * time; - for(size_t i=0; i field_data; nb->get_field_data("disp", field_data); - double expected = 10.0 * time; + const double expectedValue = 10.0 * time; for (size_t node = 0; node < field_data.size(); node++) { - EXPECT_EQ(field_data[node], expected); + EXPECT_EQ(field_data[node], expectedValue); } results.end_state(step+1); } @@ -168,90 +161,4 @@ TEST(StkMeshIoBrokerHowTo, writeResults) unlink(results_name.c_str()); } - -TEST(StkMeshIoBrokerHowTo, DISABLED_brokenWriteResults) -{ - std::string mesh_name = "input_mesh_example.e"; - std::string results_name = "broken_results.e"; - MPI_Comm communicator = MPI_COMM_WORLD; - - { - // ============================================================ - //+ INITIALIZATION: - //+ Create a basic mesh with a hex block, 3 shell blocks, 3 nodesets, and 3 sidesets. - stk::io::StkMeshIoBroker stkIo(communicator); - stkIo.use_simple_fields(); - - const std::string generatedFileName = "generated:8x8x8|shell:xyz|nodeset:xyz|sideset:XYZ"; - size_t index = stkIo.add_mesh_database(generatedFileName, stk::io::READ_MESH); - stkIo.set_active_mesh(index); - stkIo.create_input_mesh(); - stkIo.populate_bulk_data(); - - size_t fh = stkIo.create_output_mesh(mesh_name, stk::io::WRITE_RESULTS); - stkIo.write_output_mesh(fh); - } - - { - //-BEGIN - // ============================================================ - //+ EXAMPLE: - //+ Read mesh data from the specified file. - stk::io::StkMeshIoBroker stkIo(communicator); - stkIo.use_simple_fields(); - stkIo.add_mesh_database(mesh_name, stk::io::READ_MESH); - - //+ Creates meta data; creates parts - stkIo.create_input_mesh(); - - //+ Declare a field - //+ NOTE: Fields must be declared before "populate_bulk_data()" is called - //+ since it commits the meta data. - const std::string fieldName = "disp"; - stk::mesh::Field &field = - stkIo.meta_data().declare_field(stk::topology::NODE_RANK, fieldName, 1); - stk::mesh::put_field_on_mesh(field, stkIo.meta_data().universal_part(), 3, 2, nullptr); - stk::io::set_field_output_type(field, stk::io::FieldOutputType::VECTOR_3D); - - //+ commit the meta data and create the bulk data. - //+ populate the bulk data with data from the mesh file. - stkIo.populate_bulk_data(); - - // ============================================================ - //+ Create results file. By default, all parts created from the input - //+ mesh will be written to the results output file. - size_t fh = stkIo.create_output_mesh(results_name, stk::io::WRITE_RESULTS); - - //+ The field will be output to the results file with the default field name. - stkIo.add_field(fh, field); /*@\label{io:results:add_field}*/ - - std::vector nodes; - stk::mesh::get_entities(stkIo.bulk_data(), stk::topology::NODE_RANK, nodes); - - // Iterate the application's execute loop five times and output - // field data each iteration. - for(int step = 0; step < 5; step++) - { - double time = step; - - // Application execution... - double value = 10.0 * time; - for(size_t i = 0; i < nodes.size(); i++) - { - double *node_data = stk::mesh::field_data(field, nodes[i]); - node_data[0] = value; - node_data[1] = value + 1; - node_data[2] = value + 2; - EXPECT_EQ(stk::mesh::field_scalars_per_entity(field,nodes[i]), 6u); - } - - //+ Output the field data calculated by the application. - stkIo.begin_output_step(fh, time); - stkIo.write_defined_output_fields(fh); - stkIo.end_output_step(fh); - } - //-END - } - //see output exodus -- FAIL -} } diff --git a/packages/stk/stk_doc_tests/stk_mesh/IOSidesetFaceCreation.cpp b/packages/stk/stk_doc_tests/stk_mesh/IOSidesetFaceCreation.cpp index 2c936d152999..c0f75236a6a8 100644 --- a/packages/stk/stk_doc_tests/stk_mesh/IOSidesetFaceCreation.cpp +++ b/packages/stk/stk_doc_tests/stk_mesh/IOSidesetFaceCreation.cpp @@ -52,26 +52,7 @@ namespace stk { namespace mesh { class BulkData; } } namespace { - //BEGIN2hex1sideset -bool is_positive_permutation(stk::mesh::BulkData & mesh, - stk::mesh::Entity face, - stk::mesh::Entity hex, - unsigned face_ordinal) -{ - stk::topology faceTopology = mesh.bucket(face).topology(); - stk::mesh::EntityVector face_nodes(mesh.num_nodes(face)); - for (unsigned faceNodeCount=0; faceNodeCount < mesh.num_nodes(face); ++faceNodeCount) { - face_nodes[faceNodeCount] = mesh.begin_nodes(face)[faceNodeCount]; - } - stk::EquivalentPermutation permutation = stk::mesh::side_equivalent(mesh, hex, face_ordinal, face_nodes.data()); - - bool is_a_valid_permutation = permutation.is_equivalent; - EXPECT_TRUE(is_a_valid_permutation); - bool is_positive_permutation = permutation.permutation_number < faceTopology.num_positive_permutations(); - return is_positive_permutation; -} - TEST(StkMeshHowTo, StkIO2Hex1SidesetFaceCreation) { if (stk::parallel_machine_size(MPI_COMM_WORLD) == 1) { @@ -99,13 +80,17 @@ TEST(StkMeshHowTo, StkIO2Hex1SidesetFaceCreation) ASSERT_EQ(expected_num_faces, all_faces.size()); size_t face_index = 0; stk::mesh::Entity face = all_faces[face_index]; - unsigned expected_connected_elements = 2; - ASSERT_EQ(expected_connected_elements, mesh.num_elements(face)); + stk::topology faceTopology = mesh.bucket(face).topology(); + ASSERT_EQ(stk::topology::QUAD_4, faceTopology); EXPECT_TRUE(mesh.bucket(face).member(*mesh.mesh_meta_data().get_part("surface_1"))); + unsigned expected_connected_elements = 2; + ASSERT_EQ(expected_connected_elements, mesh.num_elements(face)); + const stk::mesh::Entity * connected_elements = mesh.begin_elements(face); const stk::mesh::ConnectivityOrdinal * which_side_of_element = mesh.begin_element_ordinals(face); + const stk::mesh::Permutation* face_permutations = mesh.begin_element_permutations(face); { int element_count = 0; @@ -113,8 +98,8 @@ TEST(StkMeshHowTo, StkIO2Hex1SidesetFaceCreation) EXPECT_EQ(2u, mesh.identifier(hex_2)); unsigned expected_face_ordinal = 4; EXPECT_EQ(expected_face_ordinal, which_side_of_element[element_count]); - EXPECT_FALSE(is_positive_permutation( - mesh, face, hex_2, expected_face_ordinal)); + bool is_positive_permutation = faceTopology.is_positive_polarity(face_permutations[element_count]); + EXPECT_FALSE(is_positive_permutation); } { @@ -123,8 +108,8 @@ TEST(StkMeshHowTo, StkIO2Hex1SidesetFaceCreation) EXPECT_EQ(1u, mesh.identifier(hex_1)); unsigned expected_face_ordinal = 5; EXPECT_EQ(expected_face_ordinal, which_side_of_element[element_count]); - EXPECT_TRUE(is_positive_permutation( - mesh, face, hex_1, expected_face_ordinal)); + bool is_positive_permutation = faceTopology.is_positive_polarity(face_permutations[element_count]); + EXPECT_TRUE(is_positive_permutation); } } @@ -170,6 +155,10 @@ TEST(StkMeshHowTo, StkIO2Hex2Shell3SidesetFaceCreation) unsigned expected_num_faces = 2; ASSERT_EQ(expected_num_faces, all_faces.size()); + stk::topology faceTopology = mesh.bucket(all_faces[0]).topology(); + ASSERT_EQ(stk::topology::QUAD_4, faceTopology); + ASSERT_EQ(faceTopology, mesh.bucket(all_faces[1]).topology()); + size_t face_index = 0; { stk::mesh::Entity face = all_faces[face_index]; @@ -180,6 +169,7 @@ TEST(StkMeshHowTo, StkIO2Hex2Shell3SidesetFaceCreation) const stk::mesh::Entity * connected_elements = mesh.begin_elements(face); const stk::mesh::ConnectivityOrdinal * which_side_of_element = mesh.begin_element_ordinals(face); + const stk::mesh::Permutation* face_permutations = mesh.begin_element_permutations(face); { int element_count = 0; @@ -187,8 +177,8 @@ TEST(StkMeshHowTo, StkIO2Hex2Shell3SidesetFaceCreation) EXPECT_EQ(3u, mesh.identifier(shell_3)); unsigned expected_face_ordinal = 1; EXPECT_EQ(expected_face_ordinal, which_side_of_element[element_count]); - EXPECT_FALSE(is_positive_permutation( - mesh, face, shell_3, expected_face_ordinal)); + bool is_positive_permutation = faceTopology.is_positive_polarity(face_permutations[element_count]); + EXPECT_FALSE(is_positive_permutation); } { int element_count = 1; @@ -196,8 +186,8 @@ TEST(StkMeshHowTo, StkIO2Hex2Shell3SidesetFaceCreation) EXPECT_EQ(4u, mesh.identifier(shell_4)); unsigned expected_face_ordinal = 1; EXPECT_EQ(expected_face_ordinal, which_side_of_element[element_count]); - EXPECT_FALSE(is_positive_permutation( - mesh, face, shell_4, expected_face_ordinal)); + bool is_positive_permutation = faceTopology.is_positive_polarity(face_permutations[element_count]); + EXPECT_FALSE(is_positive_permutation); } { int element_count = 2; @@ -205,8 +195,8 @@ TEST(StkMeshHowTo, StkIO2Hex2Shell3SidesetFaceCreation) EXPECT_EQ(1u, mesh.identifier(hex_1)); unsigned expected_face_ordinal = 5; EXPECT_EQ(expected_face_ordinal, which_side_of_element[element_count]); - EXPECT_TRUE(is_positive_permutation( - mesh, face, hex_1, expected_face_ordinal)); + bool is_positive_permutation = faceTopology.is_positive_polarity(face_permutations[element_count]); + EXPECT_TRUE(is_positive_permutation); } } @@ -221,6 +211,7 @@ TEST(StkMeshHowTo, StkIO2Hex2Shell3SidesetFaceCreation) const stk::mesh::Entity * connected_elements = mesh.begin_elements(face); const stk::mesh::ConnectivityOrdinal * which_side_of_element = mesh.begin_element_ordinals(face); + const stk::mesh::Permutation* face_permutations = mesh.begin_element_permutations(face); { int element_count = 0; @@ -228,8 +219,8 @@ TEST(StkMeshHowTo, StkIO2Hex2Shell3SidesetFaceCreation) EXPECT_EQ(3u, mesh.identifier(shell_3)); unsigned expected_face_ordinal = 0; EXPECT_EQ(expected_face_ordinal, which_side_of_element[element_count]); - EXPECT_FALSE(is_positive_permutation( - mesh, face, shell_3, expected_face_ordinal)); + bool is_positive_permutation = faceTopology.is_positive_polarity(face_permutations[element_count]); + EXPECT_FALSE(is_positive_permutation); } { int element_count = 1; @@ -237,8 +228,8 @@ TEST(StkMeshHowTo, StkIO2Hex2Shell3SidesetFaceCreation) EXPECT_EQ(4u, mesh.identifier(shell_4)); unsigned expected_face_ordinal = 0; EXPECT_EQ(expected_face_ordinal, which_side_of_element[element_count]); - EXPECT_FALSE(is_positive_permutation( - mesh, face, shell_4, expected_face_ordinal)); + bool is_positive_permutation = faceTopology.is_positive_polarity(face_permutations[element_count]); + EXPECT_FALSE(is_positive_permutation); } { int element_count = 2; @@ -246,7 +237,8 @@ TEST(StkMeshHowTo, StkIO2Hex2Shell3SidesetFaceCreation) EXPECT_EQ(2u, mesh.identifier(hex_2)); unsigned expected_face_ordinal = 4; EXPECT_EQ(expected_face_ordinal, which_side_of_element[element_count]); - EXPECT_TRUE(is_positive_permutation(mesh, face, hex_2, expected_face_ordinal)); + bool is_positive_permutation = faceTopology.is_positive_polarity(face_permutations[element_count]); + EXPECT_TRUE(is_positive_permutation); } } } diff --git a/packages/stk/stk_doc_tests/stk_mesh/communicateFieldData.cpp b/packages/stk/stk_doc_tests/stk_mesh/communicateFieldData.cpp index d410b17ff4aa..d84fdeb2d58c 100644 --- a/packages/stk/stk_doc_tests/stk_mesh/communicateFieldData.cpp +++ b/packages/stk/stk_doc_tests/stk_mesh/communicateFieldData.cpp @@ -37,17 +37,14 @@ #include #include // for BulkData #include // for communicate_field_data +#include #include // for ParallelMachine #include // for string #include // for vector -#include "mpi.h" // for MPI_COMM_WORLD, etc -#include "stk_io/DatabasePurpose.hpp" // for DatabasePurpose::READ_MESH -#include "stk_io/StkMeshIoBroker.hpp" // for StkMeshIoBroker #include "stk_io/FillMesh.hpp" #include "stk_mesh/base/Bucket.hpp" // for Bucket #include "stk_mesh/base/Entity.hpp" // for Entity #include "stk_mesh/base/Field.hpp" // for Field -#include "stk_mesh/base/FieldBase.hpp" // for field_data, etc #include "stk_mesh/base/MetaData.hpp" // for MetaData, etc #include "stk_mesh/base/Part.hpp" // for Part #include "stk_mesh/base/Selector.hpp" // for operator!, Selector @@ -67,28 +64,30 @@ TEST_F(ParallelHowTo, communicateFieldDataForSharedAndAura) stk::io::fill_mesh("generated:8x8x8", get_bulk()); - const stk::mesh::BucketVector& notOwnedBuckets = get_bulk().get_buckets(stk::topology::NODE_RANK, !get_meta().locally_owned_part()); - - for(const stk::mesh::Bucket *bucket : notOwnedBuckets) - for(stk::mesh::Entity node : *bucket) + stk::mesh::Selector notOwned = !get_meta().locally_owned_part(); + stk::mesh::for_each_entity_run(get_bulk(), stk::topology::NODE_RANK, notOwned, + [&](const stk::mesh::BulkData& bulk, stk::mesh::Entity node) { *stk::mesh::field_data(field, node) = -1.2345; + }); stk::mesh::communicate_field_data(get_bulk(), {&field}); - for(const stk::mesh::Bucket *bucket : notOwnedBuckets) - for(stk::mesh::Entity node : *bucket) + stk::mesh::for_each_entity_run(get_bulk(), stk::topology::NODE_RANK, notOwned, + [&](const stk::mesh::BulkData& bulk, stk::mesh::Entity node) { EXPECT_EQ(initialValue, *stk::mesh::field_data(field, node)); + }); } //ENDCommuniateFieldData //BEGINSum -void expect_field_has_value(const stk::mesh::BucketVector& buckets, - const stk::mesh::Field &field, - double value) +void expect_nodal_field_has_value(const stk::mesh::Selector& selector, + const stk::mesh::Field &field, + const double value) { - for(const stk::mesh::Bucket *bucket : buckets) - for(stk::mesh::Entity node : *bucket) + stk::mesh::for_each_entity_run(field.get_mesh(), stk::topology::NODE_RANK, selector, + [&](const stk::mesh::BulkData& bulk, stk::mesh::Entity node) { EXPECT_EQ(value, *stk::mesh::field_data(field, node)); + }); } TEST_F(ParallelHowTo, computeParallelSum) @@ -101,14 +100,12 @@ TEST_F(ParallelHowTo, computeParallelSum) stk::io::fill_mesh("generated:8x8x8", get_bulk()); - const stk::mesh::BucketVector& shared = get_bulk().get_buckets(stk::topology::NODE_RANK, get_meta().globally_shared_part()); - const stk::mesh::BucketVector& notShared = get_bulk().get_buckets(stk::topology::NODE_RANK, !get_meta().globally_shared_part()); - expect_field_has_value(shared, field, initialValue); - expect_field_has_value(notShared, field, initialValue); + expect_nodal_field_has_value(get_meta().globally_shared_part(), field, initialValue); + expect_nodal_field_has_value(!get_meta().globally_shared_part(), field, initialValue); stk::mesh::parallel_sum(get_bulk(), {&field}); - expect_field_has_value(shared, field, 2*initialValue); - expect_field_has_value(notShared, field, initialValue); + expect_nodal_field_has_value(get_meta().globally_shared_part(), field, 2*initialValue); + expect_nodal_field_has_value(!get_meta().globally_shared_part(), field, initialValue); } //ENDSum diff --git a/packages/stk/stk_doc_tests/stk_mesh/createFacesEdgesHex.cpp b/packages/stk/stk_doc_tests/stk_mesh/createFacesEdgesHex.cpp index 8b793770dd69..4c8b84cc0512 100644 --- a/packages/stk/stk_doc_tests/stk_mesh/createFacesEdgesHex.cpp +++ b/packages/stk/stk_doc_tests/stk_mesh/createFacesEdgesHex.cpp @@ -376,5 +376,3 @@ void writeExodusFile(Iogn::GeneratedMesh *generatedMesh, const std::string &exod } - - diff --git a/packages/stk/stk_doc_tests/stk_mesh/createFacesHex.cpp b/packages/stk/stk_doc_tests/stk_mesh/createFacesHex.cpp index fa766c16f28c..c910f9f9ed7b 100644 --- a/packages/stk/stk_doc_tests/stk_mesh/createFacesHex.cpp +++ b/packages/stk/stk_doc_tests/stk_mesh/createFacesHex.cpp @@ -33,16 +33,15 @@ // #include // for AssertHelper, EXPECT_EQ, etc -#include // for StkMeshIoBroker -#include // for create_faces -#include // for count_entities +#include +#include #include // for MetaData #include // for Selector +#include // for create_faces +#include // for count_entities #include // for topology, etc #include // for string #include // for vector -#include "stk_io/DatabasePurpose.hpp" // for DatabasePurpose::READ_MESH -namespace stk { namespace mesh { class BulkData; } } namespace { @@ -52,25 +51,23 @@ TEST(StkMeshHowTo, CreateFacesHex) // ============================================================ // INITIALIZATION MPI_Comm communicator = MPI_COMM_WORLD; - if (stk::parallel_machine_size(communicator) != 1) { return; } - stk::io::StkMeshIoBroker stkIo(communicator); - stkIo.use_simple_fields(); + if (stk::parallel_machine_size(communicator) != 1) { GTEST_SKIP(); } + std::unique_ptr bulkPtr = stk::mesh::MeshBuilder(communicator).create(); + bulkPtr->mesh_meta_data().use_simple_fields(); const std::string generatedFileName = "generated:8x8x8"; - stkIo.add_mesh_database(generatedFileName, stk::io::READ_MESH); - stkIo.create_input_mesh(); - stkIo.populate_bulk_data(); + stk::io::fill_mesh(generatedFileName, *bulkPtr); // ============================================================ //+ EXAMPLE //+ Create the faces.. - stk::mesh::create_faces(stkIo.bulk_data()); + stk::mesh::create_faces(*bulkPtr); // ================================================== // VERIFICATION - stk::mesh::Selector allEntities = stkIo.meta_data().universal_part(); + stk::mesh::Selector allEntities = bulkPtr->mesh_meta_data().universal_part(); std::vector entityCounts; - stk::mesh::count_entities(allEntities, stkIo.bulk_data(), entityCounts); + stk::mesh::count_entities(allEntities, *bulkPtr, entityCounts); EXPECT_EQ( 512u, entityCounts[stk::topology::ELEMENT_RANK]); EXPECT_EQ(1728u, entityCounts[stk::topology::FACE_RANK]); diff --git a/packages/stk/stk_doc_tests/stk_mesh/createSelectedFaces.cpp b/packages/stk/stk_doc_tests/stk_mesh/createSelectedFaces.cpp index 0c1b481c1c1e..4883552ff529 100644 --- a/packages/stk/stk_doc_tests/stk_mesh/createSelectedFaces.cpp +++ b/packages/stk/stk_doc_tests/stk_mesh/createSelectedFaces.cpp @@ -34,18 +34,17 @@ #include // for AssertHelper, EXPECT_EQ, etc #include // for size_t -#include // for StkMeshIoBroker +#include +#include #include #include // for count_entities #include // for MetaData +#include // for BulkData #include // for Selector #include // for topology, etc #include // for string #include // for vector -#include "stk_io/DatabasePurpose.hpp" // for DatabasePurpose::READ_MESH #include "stk_mesh/base/Part.hpp" // for Part -#include "stk_mesh/base/Types.hpp" // for PartVector -namespace stk { namespace mesh { class BulkData; } } namespace { @@ -55,37 +54,27 @@ TEST(StkMeshHowTo, CreateSelectedFacesHex) // ============================================================ // INITIALIZATION MPI_Comm communicator = MPI_COMM_WORLD; - if (stk::parallel_machine_size(communicator) != 1) { return; } - stk::io::StkMeshIoBroker stkIo(communicator); - stkIo.use_simple_fields(); + if (stk::parallel_machine_size(communicator) != 1) { GTEST_SKIP(); } + std::unique_ptr bulkPtr = stk::mesh::MeshBuilder(communicator).create(); + bulkPtr->mesh_meta_data().use_simple_fields(); // Generate a mesh containing 1 hex part and 6 shell parts const std::string generatedFileName = "generated:8x8x8|shell:xyzXYZ"; - stkIo.add_mesh_database(generatedFileName, stk::io::READ_MESH); - stkIo.create_input_mesh(); - stkIo.populate_bulk_data(); - const stk::mesh::PartVector &all_parts = stkIo.meta_data().get_mesh_parts(); + stk::io::fill_mesh(generatedFileName, *bulkPtr); // ============================================================ //+ EXAMPLE //+ Create a selector containing just the shell parts. - stk::mesh::Selector shell_subset; - for (size_t i=0; i < all_parts.size(); i++) { - const stk::mesh::Part *part = all_parts[i]; - stk::topology topo = part->topology(); - if (topo == stk::topology::SHELL_QUAD_4) { - shell_subset |= *part; - } - } + stk::mesh::Selector shell_subset = bulkPtr->mesh_meta_data().get_topology_root_part(stk::topology::SHELL_QUAD_4); //+ Create the faces on just the selected shell parts. - stk::mesh::create_all_sides(stkIo.bulk_data(), shell_subset); + stk::mesh::create_all_sides(*bulkPtr, shell_subset); // ================================================== // VERIFICATION - stk::mesh::Selector allEntities = stkIo.meta_data().universal_part(); + stk::mesh::Selector allEntities = bulkPtr->mesh_meta_data().universal_part(); std::vector entityCounts; - stk::mesh::count_entities(allEntities, stkIo.bulk_data(), entityCounts); + stk::mesh::count_entities(allEntities, *bulkPtr, entityCounts); EXPECT_EQ( 896u, entityCounts[stk::topology::ELEMENT_RANK]); EXPECT_EQ( 768u, entityCounts[stk::topology::FACE_RANK]); diff --git a/packages/stk/stk_doc_tests/stk_mesh/createSharedNodes.cpp b/packages/stk/stk_doc_tests/stk_mesh/createSharedNodes.cpp index ec6acf44105c..ef1e66e53699 100644 --- a/packages/stk/stk_doc_tests/stk_mesh/createSharedNodes.cpp +++ b/packages/stk/stk_doc_tests/stk_mesh/createSharedNodes.cpp @@ -77,7 +77,6 @@ TEST(stkMeshHowTo, createSharedNodes) const unsigned spatialDimension = 2; stk::mesh::MeshBuilder builder(MPI_COMM_WORLD); builder.set_spatial_dimension(spatialDimension); - builder.set_entity_rank_names(stk::mesh::entity_rank_names()); std::shared_ptr bulkPtr = builder.create(); bulkPtr->mesh_meta_data().use_simple_fields(); stk::mesh::MetaData& metaData = bulkPtr->mesh_meta_data(); @@ -122,7 +121,6 @@ TEST(stkMeshHowTo, createIndependentSharedNodes) const unsigned spatialDimension = 2; stk::mesh::MeshBuilder builder(MPI_COMM_WORLD); builder.set_spatial_dimension(spatialDimension); - builder.set_entity_rank_names(stk::mesh::entity_rank_names()); std::shared_ptr bulkPtr = builder.create(); bulkPtr->mesh_meta_data().use_simple_fields(); stk::mesh::BulkData& bulkData = *bulkPtr; @@ -159,7 +157,6 @@ TEST(stkMeshHowTo, createIndependentSharedNodesThenAddDependence) const unsigned spatialDimension = 2; stk::mesh::MeshBuilder builder(MPI_COMM_WORLD); builder.set_spatial_dimension(spatialDimension); - builder.set_entity_rank_names(stk::mesh::entity_rank_names()); std::shared_ptr bulkPtr = builder.create(); bulkPtr->mesh_meta_data().use_simple_fields(); stk::mesh::BulkData& bulkData = *bulkPtr; diff --git a/packages/stk/stk_doc_tests/stk_mesh/createStkMesh.cpp b/packages/stk/stk_doc_tests/stk_mesh/createStkMesh.cpp index 455fef471502..3d7907fef40e 100644 --- a/packages/stk/stk_doc_tests/stk_mesh/createStkMesh.cpp +++ b/packages/stk/stk_doc_tests/stk_mesh/createStkMesh.cpp @@ -63,7 +63,7 @@ TEST(StkMeshHowTo, UseStkIO) meshReader.add_all_mesh_fields_as_input_fields(); meshReader.populate_bulk_data(); - unsigned numElems = stk::mesh::count_selected_entities(bulkPtr->mesh_meta_data().universal_part(), bulkPtr->buckets(stk::topology::ELEM_RANK)); + unsigned numElems = stk::mesh::count_entities(*bulkPtr, stk::topology::ELEM_RANK, bulkPtr->mesh_meta_data().universal_part()); EXPECT_EQ(512u, numElems); } } diff --git a/packages/stk/stk_doc_tests/stk_mesh/createStkMeshAlt1.cpp b/packages/stk/stk_doc_tests/stk_mesh/createStkMeshAlt1.cpp index 17767c1237a8..d46607b29eab 100644 --- a/packages/stk/stk_doc_tests/stk_mesh/createStkMeshAlt1.cpp +++ b/packages/stk/stk_doc_tests/stk_mesh/createStkMeshAlt1.cpp @@ -65,14 +65,13 @@ TEST(StkMeshHowTo, CreateStkMesh) stk::mesh::MetaData& stkMeshMetaData = stkMeshBulkDataPtr->mesh_meta_data(); stkMeshMetaData.use_simple_fields(); - // STK IO module will be described in separate chapter. - // It is used here to read the mesh data from the Exodus file and populate an STK Mesh. + // Read the mesh data from the Exodus file and populate an STK Mesh. // The order of the following lines in {} are important { stk::io::StkMeshIoBroker exodusFileReader(communicator); exodusFileReader.use_simple_fields(); - // Inform STK IO which STK Mesh objects to populate later + // Provide STK Mesh object to be populated exodusFileReader.set_bulk_data(*stkMeshBulkDataPtr); exodusFileReader.add_mesh_database(exodusFileName, stk::io::READ_MESH); @@ -84,7 +83,7 @@ TEST(StkMeshHowTo, CreateStkMesh) exodusFileReader.populate_bulk_data(); } - // Test if the STK Mesh has 512 elements. Other examples will discuss details below. + // Verify that the STK Mesh has 512 elements. stk::mesh::Selector allEntities = stkMeshMetaData.universal_part(); std::vector entityCounts; stk::mesh::count_entities(allEntities, *stkMeshBulkDataPtr, entityCounts); diff --git a/packages/stk/stk_doc_tests/stk_mesh/howToDestroyElementsInList.cpp b/packages/stk/stk_doc_tests/stk_mesh/howToDestroyElementsInList.cpp index 03ff76f723f2..0822bb0b9f53 100644 --- a/packages/stk/stk_doc_tests/stk_mesh/howToDestroyElementsInList.cpp +++ b/packages/stk/stk_doc_tests/stk_mesh/howToDestroyElementsInList.cpp @@ -3,31 +3,32 @@ #include #include #include -#include #include +#include namespace { //BEGIN_DESTROY_ELEMENTS_IN_LIST TEST(StkMeshHowTo, DestroyElementsInList) { - std::shared_ptr bulkPtr = stk::mesh::MeshBuilder(MPI_COMM_WORLD).create(); + std::unique_ptr bulkPtr = stk::mesh::MeshBuilder(MPI_COMM_WORLD).create(); bulkPtr->mesh_meta_data().use_simple_fields(); stk::mesh::BulkData& bulkData = *bulkPtr; stk::io::fill_mesh("generated:1x1x4", bulkData); - EXPECT_GT(stk::mesh::count_selected_entities(bulkPtr->mesh_meta_data().universal_part(), bulkData.buckets(stk::topology::ELEM_RANK)), 0u); + EXPECT_GT(stk::mesh::count_entities(*bulkPtr, stk::topology::ELEM_RANK, bulkPtr->mesh_meta_data().universal_part()), 0u); stk::mesh::EntityVector elementsToDestroy{bulkData.get_entity(stk::topology::ELEMENT_RANK,1)}; stk::mesh::destroy_elements(bulkData, elementsToDestroy); stk::mesh::EntityVector orphanedNodes{ bulkData.get_entity(stk::topology::NODE_RANK,1), - bulkData.get_entity(stk::topology::NODE_RANK,2), - bulkData.get_entity(stk::topology::NODE_RANK,3), - bulkData.get_entity(stk::topology::NODE_RANK,4) + bulkData.get_entity(stk::topology::NODE_RANK,2), + bulkData.get_entity(stk::topology::NODE_RANK,3), + bulkData.get_entity(stk::topology::NODE_RANK,4) }; - for(stk::mesh::Entity node : orphanedNodes) + for(stk::mesh::Entity node : orphanedNodes) { EXPECT_FALSE(bulkData.is_valid(node)); + } } //END_DESTROY_ELEMENTS_IN_LIST } diff --git a/packages/stk/stk_doc_tests/stk_mesh/howToDestroyElementsOfTopology.cpp b/packages/stk/stk_doc_tests/stk_mesh/howToDestroyElementsOfTopology.cpp index 1f0451f9efc6..ed57103269b8 100644 --- a/packages/stk/stk_doc_tests/stk_mesh/howToDestroyElementsOfTopology.cpp +++ b/packages/stk/stk_doc_tests/stk_mesh/howToDestroyElementsOfTopology.cpp @@ -1,21 +1,23 @@ #include #include #include -#include #include +#include #include -#include +#include namespace { TEST(StkMeshHowTo, DestroyElementsOfTopology) { - std::shared_ptr bulkPtr = stk::mesh::MeshBuilder(MPI_COMM_WORLD).create(); + if (stk::parallel_machine_size(MPI_COMM_WORLD) > 1) { GTEST_SKIP(); } + + std::unique_ptr bulkPtr = stk::mesh::MeshBuilder(MPI_COMM_WORLD).create(); bulkPtr->mesh_meta_data().use_simple_fields(); stk::mesh::MetaData& metaData = bulkPtr->mesh_meta_data(); - stk::mesh::BulkData& bulkData = *bulkPtr; - stk::io::fill_mesh("generated:1x1x4", bulkData); - EXPECT_GT(stk::mesh::count_selected_entities(metaData.universal_part(), bulkData.buckets(stk::topology::ELEM_RANK)), 0u); - bulkData.destroy_elements_of_topology(stk::topology::HEX_8); - EXPECT_EQ(0u, stk::mesh::count_selected_entities(metaData.universal_part(), bulkData.buckets(stk::topology::ELEM_RANK))); + stk::io::fill_mesh("generated:1x1x4", *bulkPtr); + + EXPECT_EQ(4u, stk::mesh::count_entities(*bulkPtr, stk::topology::ELEM_RANK, metaData.universal_part())); + bulkPtr->destroy_elements_of_topology(stk::topology::HEX_8); + EXPECT_EQ(0u, stk::mesh::count_entities(*bulkPtr, stk::topology::ELEM_RANK, metaData.universal_part())); } } diff --git a/packages/stk/stk_doc_tests/stk_mesh/howToIterateConnectivity.cpp b/packages/stk/stk_doc_tests/stk_mesh/howToIterateConnectivity.cpp index 6c65cfd382a7..eedaaeab28b0 100644 --- a/packages/stk/stk_doc_tests/stk_mesh/howToIterateConnectivity.cpp +++ b/packages/stk/stk_doc_tests/stk_mesh/howToIterateConnectivity.cpp @@ -34,120 +34,150 @@ #include // for AssertHelper, EXPECT_NE, etc #include // for size_t -#include // for numeric_limits -#include // for StkMeshIoBroker -#include // for BulkData +#include #include // for MetaData +#include // for BulkData +#include +#include #include // for topology, etc -#include // for string -#include "stk_io/DatabasePurpose.hpp" // for DatabasePurpose::READ_MESH #include "stk_mesh/base/Bucket.hpp" // for Bucket #include "stk_mesh/base/Entity.hpp" // for Entity #include "stk_mesh/base/Field.hpp" // for Field #include "stk_mesh/base/FieldBase.hpp" // for field_data, FieldBase #include "stk_mesh/base/Types.hpp" // for BucketVector +#include +#include namespace stk { namespace mesh { class Part; } } namespace { //-BEGIN -TEST(StkMeshHowTo, iterateConnectivityThroughBulkData) +TEST(StkMeshHowTo, iterateElemNodeConnectivity_ForEachEntityWithNodes) { - MPI_Comm communicator = MPI_COMM_WORLD; - if (stk::parallel_machine_size(communicator) != 1) { return; } - stk::io::StkMeshIoBroker stkMeshIoBroker(communicator); - stkMeshIoBroker.use_simple_fields(); - // Generate a mesh of hexes with a sideset + MPI_Comm comm = MPI_COMM_WORLD; + if (stk::parallel_machine_size(comm) != 1) { GTEST_SKIP(); } + std::unique_ptr stkMesh = stk::mesh::MeshBuilder(comm).create(); + stkMesh->mesh_meta_data().use_simple_fields(); + // Generate a mesh of unit-cube hexes with a sideset const std::string generatedMeshSpecification = "generated:2x2x2|sideset:X"; - stkMeshIoBroker.add_mesh_database(generatedMeshSpecification, stk::io::READ_MESH); - stkMeshIoBroker.create_input_mesh(); - stkMeshIoBroker.populate_bulk_data(); - - stk::mesh::MetaData &stkMeshMetaData = stkMeshIoBroker.meta_data(); - stk::mesh::BulkData &stkMeshBulkData = stkMeshIoBroker.bulk_data(); - const stk::mesh::BucketVector &elementBuckets = - stkMeshBulkData.buckets(stk::topology::ELEMENT_RANK); + stk::io::fill_mesh(generatedMeshSpecification, *stkMesh); typedef stk::mesh::Field CoordinatesField_t; CoordinatesField_t const & coord_field = - *dynamic_cast(stkMeshMetaData.coordinate_field()); + *dynamic_cast(stkMesh->mesh_meta_data().coordinate_field()); - const unsigned nodesPerHex = 8; - const unsigned spatialDim = 3; + constexpr unsigned nodesPerHex = 8; + constexpr unsigned spatialDim = 3; unsigned count = 0; - double elementNodeCoords[nodesPerHex][spatialDim]; - for (size_t bucketIndex = 0; bucketIndex < elementBuckets.size(); ++bucketIndex) - { - stk::mesh::Bucket &elemBucket = *elementBuckets[bucketIndex]; - for (size_t elemIndex = 0; elemIndex < elemBucket.size(); ++elemIndex) - { - stk::mesh::Entity elem = elemBucket[elemIndex]; - unsigned numNodes = stkMeshBulkData.num_nodes(elem); - EXPECT_EQ(numNodes, nodesPerHex); - stk::mesh::Entity const* nodes = stkMeshBulkData.begin_nodes(elem); - for (unsigned inode = 0; inode < numNodes; ++inode) - { - double *coords = stk::mesh::field_data(coord_field, nodes[inode]); + double elementNodeCoords[nodesPerHex][spatialDim] = { + {NAN,NAN,NAN}, {NAN,NAN,NAN}, {NAN,NAN,NAN}, {NAN,NAN,NAN}, + {NAN,NAN,NAN}, {NAN,NAN,NAN}, {NAN,NAN,NAN}, {NAN,NAN,NAN} }; + + stk::mesh::Selector all = stkMesh->mesh_meta_data().universal_part(); + + stk::mesh::for_each_entity_run_with_nodes(*stkMesh, stk::topology::ELEM_RANK, all, + [&](stk::mesh::Entity elem, const stk::mesh::Entity* nodes, size_t numNodesPerEntity) { + EXPECT_EQ(numNodesPerEntity, nodesPerHex); + for (unsigned inode = 0; inode < numNodesPerEntity; ++inode) { + const double *coords = stk::mesh::field_data(coord_field, nodes[inode]); elementNodeCoords[inode][0] = coords[0]; elementNodeCoords[inode][1] = coords[1]; elementNodeCoords[inode][2] = coords[2]; - EXPECT_NE(elementNodeCoords[inode][0], std::numeric_limits::max()); - EXPECT_NE(elementNodeCoords[inode][1], std::numeric_limits::max()); - EXPECT_NE(elementNodeCoords[inode][2], std::numeric_limits::max()); ++count; } - } - } - EXPECT_GE(count, 1u); + }); + + const unsigned numElems = 2*2*2; + const unsigned totalNodesVisited = numElems * nodesPerHex; + EXPECT_EQ(count, totalNodesVisited); + EXPECT_FALSE(std::isnan(elementNodeCoords[0][0])); } -TEST(StkMeshHowTo, iterateConnectivityThroughBuckets) +TEST(StkMeshHowTo, iterateConnectivity_General_BulkData) { - MPI_Comm communicator = MPI_COMM_WORLD; - if (stk::parallel_machine_size(communicator) != 1) { return; } - stk::io::StkMeshIoBroker stkMeshIoBroker(communicator); - stkMeshIoBroker.use_simple_fields(); - // Generate a mesh of hexes with a sideset + MPI_Comm comm = MPI_COMM_WORLD; + if (stk::parallel_machine_size(comm) != 1) { GTEST_SKIP(); } + std::unique_ptr stkMesh = stk::mesh::MeshBuilder(comm).create(); + stkMesh->mesh_meta_data().use_simple_fields(); + // Generate a mesh of unit-cube hexes with a sideset const std::string generatedMeshSpecification = "generated:2x2x2|sideset:X"; - stkMeshIoBroker.add_mesh_database(generatedMeshSpecification, stk::io::READ_MESH); - stkMeshIoBroker.create_input_mesh(); - stkMeshIoBroker.populate_bulk_data(); + stk::io::fill_mesh(generatedMeshSpecification, *stkMesh); - stk::mesh::MetaData &stkMeshMetaData = stkMeshIoBroker.meta_data(); - stk::mesh::BulkData &stkMeshBulkData = stkMeshIoBroker.bulk_data(); - const stk::mesh::BucketVector &elementBuckets = - stkMeshBulkData.buckets(stk::topology::ELEMENT_RANK); + typedef stk::mesh::Field CoordinatesField_t; + CoordinatesField_t const & coord_field = + *dynamic_cast(stkMesh->mesh_meta_data().coordinate_field()); + + constexpr unsigned nodesPerHex = 8; + constexpr unsigned spatialDim = 3; + unsigned count = 0; + double elementNodeCoords[nodesPerHex][spatialDim] = { + {NAN,NAN,NAN}, {NAN,NAN,NAN}, {NAN,NAN,NAN}, {NAN,NAN,NAN}, + {NAN,NAN,NAN}, {NAN,NAN,NAN}, {NAN,NAN,NAN}, {NAN,NAN,NAN} }; + + stk::mesh::for_each_entity_run(*stkMesh, stk::topology::ELEM_RANK, + [&](const stk::mesh::BulkData& bulk, stk::mesh::Entity elem) { + const stk::mesh::ConnectedEntities nodes = stkMesh->get_connected_entities(elem, stk::topology::NODE_RANK); + EXPECT_EQ(nodes.size(), nodesPerHex); + + for (unsigned inode = 0; inode < nodes.size(); ++inode) { + const double *coords = stk::mesh::field_data(coord_field, nodes[inode]); + elementNodeCoords[inode][0] = coords[0]; + elementNodeCoords[inode][1] = coords[1]; + elementNodeCoords[inode][2] = coords[2]; + ++count; + } + }); + + const unsigned numElems = 2*2*2; + const unsigned totalNodesVisited = numElems * nodesPerHex; + EXPECT_EQ(count, totalNodesVisited); + EXPECT_FALSE(std::isnan(elementNodeCoords[0][0])); +} + +TEST(StkMeshHowTo, iterateConnectivity_Buckets) +{ + MPI_Comm comm = MPI_COMM_WORLD; + if (stk::parallel_machine_size(comm) != 1) { return; } + std::unique_ptr stkMesh = stk::mesh::MeshBuilder(comm).create(); + stkMesh->mesh_meta_data().use_simple_fields(); + // Generate a mesh of unit-cube hexes with a sideset + const std::string generatedMeshSpecification = "generated:2x2x2|sideset:X"; + stk::io::fill_mesh(generatedMeshSpecification, *stkMesh); typedef stk::mesh::Field CoordinatesField_t; CoordinatesField_t const & coord_field = - *dynamic_cast(stkMeshMetaData.coordinate_field()); + *dynamic_cast(stkMesh->mesh_meta_data().coordinate_field()); + + const stk::mesh::BucketVector &elementBuckets = stkMesh->buckets(stk::topology::ELEMENT_RANK); - const unsigned nodesPerHex = 8; - const unsigned spatialDim = 3; + constexpr unsigned nodesPerHex = 8; + constexpr unsigned spatialDim = 3; unsigned count = 0; - double elementNodeCoords[nodesPerHex][spatialDim]; - for (size_t bucketIndex = 0; bucketIndex < elementBuckets.size(); ++bucketIndex) - { - stk::mesh::Bucket &elemBucket = *elementBuckets[bucketIndex]; - for (size_t elemIndex = 0; elemIndex < elemBucket.size(); ++elemIndex) - { + double elementNodeCoords[nodesPerHex][spatialDim] = { + {NAN,NAN,NAN}, {NAN,NAN,NAN}, {NAN,NAN,NAN}, {NAN,NAN,NAN}, + {NAN,NAN,NAN}, {NAN,NAN,NAN}, {NAN,NAN,NAN}, {NAN,NAN,NAN} }; + + for (size_t bucketIndex = 0; bucketIndex < elementBuckets.size(); ++bucketIndex) { + const stk::mesh::Bucket &elemBucket = *elementBuckets[bucketIndex]; + + for (size_t elemIndex = 0; elemIndex < elemBucket.size(); ++elemIndex) { unsigned numNodes = elemBucket.num_nodes(elemIndex); EXPECT_EQ(numNodes, nodesPerHex); - stk::mesh::Entity const* nodes = elemBucket.begin_nodes(elemIndex); - for (unsigned inode = 0; inode < numNodes; ++inode) - { - double *coords = stk::mesh::field_data(coord_field, nodes[inode]); + const stk::mesh::Entity* nodes = elemBucket.begin_nodes(elemIndex); + + for (unsigned inode = 0; inode < numNodes; ++inode) { + const double *coords = stk::mesh::field_data(coord_field, nodes[inode]); elementNodeCoords[inode][0] = coords[0]; elementNodeCoords[inode][1] = coords[1]; elementNodeCoords[inode][2] = coords[2]; - EXPECT_NE(elementNodeCoords[inode][0], std::numeric_limits::max()); - EXPECT_NE(elementNodeCoords[inode][1], std::numeric_limits::max()); - EXPECT_NE(elementNodeCoords[inode][2], std::numeric_limits::max()); ++count; } } } - EXPECT_GE(count, 1u); + const unsigned numElems = 2*2*2; + const unsigned totalNodesVisited = numElems * nodesPerHex; + EXPECT_EQ(count, totalNodesVisited); + EXPECT_FALSE(std::isnan(elementNodeCoords[0][0])); } //-END } diff --git a/packages/stk/stk_doc_tests/stk_mesh/howToIterateEntities.cpp b/packages/stk/stk_doc_tests/stk_mesh/howToIterateEntities.cpp index 9a2c8e0bea3c..943737502a19 100644 --- a/packages/stk/stk_doc_tests/stk_mesh/howToIterateEntities.cpp +++ b/packages/stk/stk_doc_tests/stk_mesh/howToIterateEntities.cpp @@ -35,10 +35,12 @@ #include // for TEST #include // for size_t #include // for set -#include // for StkMeshIoBroker -#include // for BulkData +#include #include // for MetaData, etc +#include // for BulkData +#include #include // for Selector +#include #include // for topology, etc #include // for string #include "stkMeshTestUtils.hpp" @@ -53,82 +55,72 @@ namespace stk { namespace mesh { class Part; } } namespace { //-BEGIN -TEST(StkMeshHowTo, iterateSidesetNodesMostEfficientlyForFieldDataAccess) +TEST(StkMeshHowTo, iterateSidesetNodes_BucketLoop_ContiguousFieldDataWithinBucket) { - MPI_Comm communicator = MPI_COMM_WORLD; - if (stk::parallel_machine_size(communicator) != 1) { return; } - stk::io::StkMeshIoBroker stkMeshIoBroker(communicator); - stkMeshIoBroker.use_simple_fields(); - // syntax creates faces for the surface on the positive 'x-side' of the 2x2x2 cube, - // this part is given the name 'surface_1' when it is created [create_input_mesh()] - const std::string generatedMeshSpecification = "generated:2x2x2|sideset:X"; - stkMeshIoBroker.add_mesh_database(generatedMeshSpecification, stk::io::READ_MESH); - stkMeshIoBroker.create_input_mesh(); + MPI_Comm comm = MPI_COMM_WORLD; + if (stk::parallel_machine_size(comm) != 1) { GTEST_SKIP(); } - stk::mesh::MetaData &stkMeshMetaData = stkMeshIoBroker.meta_data(); - stk::mesh::Field &temperatureField = stkMeshMetaData.declare_field(stk::topology::NODE_RANK, "temperature"); + std::unique_ptr stkMesh = stk::mesh::MeshBuilder(comm) + .set_spatial_dimension(3) + .create(); + stk::mesh::MetaData &stkMeshMeta = stkMesh->mesh_meta_data(); + stkMeshMeta.use_simple_fields(); + stk::mesh::Field &temperatureField = stkMeshMeta.declare_field(stk::topology::NODE_RANK, "temperature"); stk::mesh::put_field_on_entire_mesh(temperatureField); - stkMeshIoBroker.populate_bulk_data(); - stk::mesh::Part &boundaryConditionPart = *stkMeshMetaData.get_part("surface_1"); + // syntax creates faces for the surface on the positive 'x-side' of the 2x2x2 cube, + // this part is given the name 'surface_1' when it is created. + const std::string generatedMeshSpecification = "generated:2x2x2|sideset:X"; + stk::io::fill_mesh(generatedMeshSpecification, *stkMesh); + + stk::mesh::Part &boundaryConditionPart = *stkMeshMeta.get_part("surface_1"); stk::mesh::Selector boundaryNodesSelector(boundaryConditionPart); - stk::mesh::BulkData &stkMeshBulkData = stkMeshIoBroker.bulk_data(); - const stk::mesh::BucketVector &boundaryNodeBuckets = stkMeshBulkData.get_buckets(stk::topology::NODE_RANK, boundaryNodesSelector); + const stk::mesh::BucketVector &boundaryNodeBuckets = stkMesh->get_buckets(stk::topology::NODE_RANK, boundaryNodesSelector); - double prescribedTemperatureValue = 2.0; - std::set boundaryNodeIds; - for (size_t bucketIndex = 0; bucketIndex < boundaryNodeBuckets.size(); ++bucketIndex) - { - stk::mesh::Bucket &nodeBucket = *boundaryNodeBuckets[bucketIndex]; + constexpr double prescribedTemperatureValue = 2.0; + for (size_t bucketIndex = 0; bucketIndex < boundaryNodeBuckets.size(); ++bucketIndex) { + const stk::mesh::Bucket &nodeBucket = *boundaryNodeBuckets[bucketIndex]; double *temperatureValues = stk::mesh::field_data(temperatureField, nodeBucket); - for (size_t nodeIndex = 0; nodeIndex < nodeBucket.size(); ++nodeIndex) - { - stk::mesh::Entity node = nodeBucket[nodeIndex]; - boundaryNodeIds.insert(stkMeshBulkData.identifier(node)); + + for (size_t nodeIndex = 0; nodeIndex < nodeBucket.size(); ++nodeIndex) { temperatureValues[nodeIndex] = prescribedTemperatureValue; } } - testUtils::testTemperatureFieldSetCorrectly(temperatureField, prescribedTemperatureValue, boundaryNodeIds); + testUtils::testTemperatureFieldSetCorrectly(temperatureField, boundaryNodesSelector, prescribedTemperatureValue); } -TEST(StkMeshHowTo, iterateSidesetNodesWithFieldDataAccess) +TEST(StkMeshHowTo, iterateSidesetNodes_ForEachEntity_FieldDataAccess) { - MPI_Comm communicator = MPI_COMM_WORLD; - if (stk::parallel_machine_size(communicator) != 1) { return; } - stk::io::StkMeshIoBroker stkMeshIoBroker(communicator); - stkMeshIoBroker.use_simple_fields(); - // syntax creates faces for the surface on the positive 'x-side' of the 2x2x2 cube, - // this part is given the name 'surface_1' when it is created [create_input_mesh()] - const std::string generatedMeshSpecification = "generated:2x2x2|sideset:X"; - stkMeshIoBroker.add_mesh_database(generatedMeshSpecification, stk::io::READ_MESH); - stkMeshIoBroker.create_input_mesh(); + MPI_Comm comm = MPI_COMM_WORLD; + if (stk::parallel_machine_size(comm) != 1) { GTEST_SKIP(); } - stk::mesh::MetaData &stkMeshMetaData = stkMeshIoBroker.meta_data(); - stk::mesh::Field &temperatureField = stkMeshMetaData.declare_field(stk::topology::NODE_RANK, "temperature"); + std::unique_ptr stkMesh = stk::mesh::MeshBuilder(comm) + .set_spatial_dimension(3) + .create(); + stk::mesh::MetaData &stkMeshMeta = stkMesh->mesh_meta_data(); + stkMeshMeta.use_simple_fields(); + stk::mesh::Field &temperatureField = stkMeshMeta.declare_field(stk::topology::NODE_RANK, "temperature"); stk::mesh::put_field_on_entire_mesh(temperatureField); - stkMeshIoBroker.populate_bulk_data(); - - stk::mesh::Part &boundaryConditionPart = *stkMeshMetaData.get_part("surface_1"); - stk::mesh::Selector boundaryNodesSelector(boundaryConditionPart); - stk::mesh::BulkData &stkMeshBulkData = stkMeshIoBroker.bulk_data(); + // syntax creates faces for the surface on the positive 'x-side' of the 2x2x2 cube, + // this part is given the name 'surface_1' when it is created. + const std::string generatedMeshSpecification = "generated:2x2x2|sideset:X"; + stk::io::fill_mesh(generatedMeshSpecification, *stkMesh); - stk::mesh::EntityVector nodes; - stk::mesh::get_entities(stkMeshBulkData, stk::topology::NODE_RANK, boundaryNodesSelector, nodes); + stk::mesh::Part &boundaryConditionPart = *stkMeshMeta.get_part("surface_1"); + stk::mesh::Selector boundaryNodesSelector(boundaryConditionPart); - double prescribedTemperatureValue = 2.0; - std::set boundaryNodeIds; + constexpr double prescribedTemperatureValue = 2.0; - for (size_t nodeIndex = 0; nodeIndex < nodes.size(); ++nodeIndex) - { - boundaryNodeIds.insert(stkMeshBulkData.identifier(nodes[nodeIndex])); - double *temperatureValues = stk::mesh::field_data(temperatureField, nodes[nodeIndex]); - *temperatureValues = prescribedTemperatureValue; - } + stk::mesh::for_each_entity_run(*stkMesh, stk::topology::NODE_RANK, boundaryNodesSelector, + [&](const stk::mesh::BulkData& bulk, stk::mesh::Entity node) { + double *temperatureValues = stk::mesh::field_data(temperatureField, node); + *temperatureValues = prescribedTemperatureValue; + }); - testUtils::testTemperatureFieldSetCorrectly(temperatureField, prescribedTemperatureValue, boundaryNodeIds); + testUtils::testTemperatureFieldSetCorrectly(temperatureField, boundaryNodesSelector, prescribedTemperatureValue); } //-END } diff --git a/packages/stk/stk_doc_tests/stk_mesh/howToNgp.cpp b/packages/stk/stk_doc_tests/stk_mesh/howToNgp.cpp index 1be75fb6904d..e98c31647f84 100644 --- a/packages/stk/stk_doc_tests/stk_mesh/howToNgp.cpp +++ b/packages/stk/stk_doc_tests/stk_mesh/howToNgp.cpp @@ -77,6 +77,7 @@ class NgpHowTo : public stk::unit_test_util::simple_fields::MeshFixture void setup_test_mesh() { setup_empty_mesh(stk::mesh::BulkData::NO_AUTO_AURA); + extraPart = &get_meta().declare_part("extraPart"); stk::mesh::Part &shellQuadPart = get_meta().get_topology_root_part(stk::topology::SHELL_QUAD_4); auto &shellQuadField = get_meta().declare_field(stk::topology::ELEM_RANK, "myField"); stk::mesh::put_field_on_mesh(shellQuadField, shellQuadPart, nullptr); @@ -85,6 +86,7 @@ class NgpHowTo : public stk::unit_test_util::simple_fields::MeshFixture 0,2,SHELL_QUAD_4,5,6,7,8"; stk::unit_test_util::simple_fields::setup_text_mesh(get_bulk(), meshDesc); } + const stk::mesh::Part* extraPart = nullptr; }; TEST_F(NgpHowTo, loopOverSubsetOfMesh) @@ -113,25 +115,34 @@ TEST_F(NgpHowTo, loopOverSubsetOfMesh) } template -void test_mesh_up_to_date(stk::mesh::BulkData& bulk) +void test_mesh_up_to_date(stk::mesh::BulkData& bulk, const stk::mesh::Part* extraPart) { //BEGINNgpMeshUpToDate MeshType& ngpMesh = stk::mesh::get_updated_ngp_mesh(bulk); EXPECT_TRUE(ngpMesh.is_up_to_date()); bulk.modification_begin(); - //this is where modification operations would be... + stk::mesh::Entity node1 = bulk.get_entity(stk::topology::NODE_RANK, 1); + bulk.change_entity_parts(node1, stk::mesh::ConstPartVector{extraPart}); bulk.modification_end(); - MeshType& newNgpMesh = stk::mesh::get_updated_ngp_mesh(bulk); - EXPECT_TRUE(newNgpMesh.is_up_to_date()); + EXPECT_FALSE(ngpMesh.is_up_to_date()); + + { + MeshType& newNgpMesh = stk::mesh::get_updated_ngp_mesh(bulk); + EXPECT_TRUE(newNgpMesh.is_up_to_date()); + } + + EXPECT_TRUE(ngpMesh.is_up_to_date()); + //ENDNgpMeshUpToDate } TEST_F(NgpHowTo, checkIfUpToDate) { + if (stk::parallel_machine_size(MPI_COMM_WORLD) > 1) { GTEST_SKIP(); } setup_test_mesh(); - test_mesh_up_to_date(get_bulk()); + test_mesh_up_to_date(get_bulk(), extraPart); } template @@ -232,45 +243,37 @@ TEST_F(NgpHowTo, loopOverMeshFaces) void run_connected_node_test(const stk::mesh::BulkData& bulk) { - stk::topology elemTopo = stk::topology::HEX_8; - - stk::mesh::EntityVector elems; - stk::mesh::get_entities(bulk, stk::topology::ELEM_RANK, elems); - EXPECT_EQ(1u, elems.size()); - stk::mesh::Entity node0 = bulk.begin_nodes(elems[0])[0]; - stk::mesh::Entity node7 = bulk.begin_nodes(elems[0])[7]; + //BEGINNgpMeshConnectivity + stk::mesh::Entity elem1_host = bulk.get_entity(stk::topology::ELEM_RANK, 1); + stk::mesh::ConnectedEntities elem1_nodes_host = bulk.get_connected_entities(elem1_host, stk::topology::NODE_RANK); + stk::mesh::Entity node0_host = elem1_nodes_host[0]; + stk::mesh::Entity node7_host = elem1_nodes_host[7]; const stk::mesh::NgpMesh & ngpMesh = stk::mesh::get_updated_ngp_mesh(bulk); - typedef stk::ngp::TeamPolicy::member_type TeamHandleType; - const auto& teamPolicy = stk::ngp::TeamPolicy(ngpMesh.num_buckets(stk::topology::ELEM_RANK), - Kokkos::AUTO); - - Kokkos::parallel_for(teamPolicy, - KOKKOS_LAMBDA(const TeamHandleType& team) - { - const stk::mesh::NgpMesh::BucketType& bucket = ngpMesh.get_bucket(stk::topology::ELEM_RANK, - team.league_rank()); - unsigned numElems = bucket.size(); - - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, 0u, numElems), [&] (const int& i) - { - stk::mesh::Entity elem = bucket[i]; - stk::mesh::FastMeshIndex elemIndex = ngpMesh.fast_mesh_index(elem); - stk::mesh::NgpMesh::ConnectedNodes nodes = ngpMesh.get_nodes(stk::topology::ELEM_RANK, elemIndex); - stk::topology bucketTopo = bucket.topology(); - STK_NGP_ThrowRequire(elemTopo == bucketTopo); - STK_NGP_ThrowRequire(nodes.size() == bucketTopo.num_nodes()); - STK_NGP_ThrowRequire(node0 == nodes[0]); - STK_NGP_ThrowRequire(node7 == nodes[7]); - - stk::mesh::FastMeshIndex nodeIndex = ngpMesh.fast_mesh_index(nodes[0]); - stk::mesh::NgpMesh::ConnectedEntities node0_elems = ngpMesh.get_elements(stk::topology::NODE_RANK, - nodeIndex); - STK_NGP_ThrowRequire(1 == node0_elems.size()); - STK_NGP_ThrowRequire(node0_elems[0] == elem); - }); - }); + stk::mesh::Selector allElems = bulk.mesh_meta_data().universal_part(); + + stk::mesh::for_each_entity_run(ngpMesh, stk::topology::ELEM_RANK, allElems, + KOKKOS_LAMBDA(const stk::mesh::FastMeshIndex& elemIndex) { + stk::mesh::Entity elem = ngpMesh.get_entity(stk::topology::ELEM_RANK, elemIndex); + stk::mesh::EntityId elemId = ngpMesh.identifier(elem); + if (elemId == 1) { + STK_NGP_ThrowRequire(elem == elem1_host); + const stk::mesh::NgpMesh::BucketType& ngpBucket = ngpMesh.get_bucket(stk::topology::ELEM_RANK, elemIndex.bucket_id); + STK_NGP_ThrowRequire(ngpBucket.topology() == stk::topology::HEX_8); + + stk::mesh::NgpMesh::ConnectedNodes nodes = ngpMesh.get_nodes(stk::topology::ELEM_RANK, elemIndex); + STK_NGP_ThrowRequire(nodes.size() == ngpBucket.topology().num_nodes()); + STK_NGP_ThrowRequire(node0_host == nodes[0]); + STK_NGP_ThrowRequire(node7_host == nodes[7]); + + stk::mesh::FastMeshIndex nodeIndex = ngpMesh.fast_mesh_index(nodes[0]); + stk::mesh::NgpMesh::ConnectedEntities node0_elems = ngpMesh.get_elements(stk::topology::NODE_RANK, nodeIndex); + STK_NGP_ThrowRequire(1 == node0_elems.size()); + STK_NGP_ThrowRequire(node0_elems[0] == elem); + } + }); + //ENDNgpMeshConnectivity } TEST_F(NgpHowTo, loopOverElemNodes) @@ -746,8 +749,7 @@ TEST_F(NgpHowTo, ngpMeshConstruction) stk::io::fill_mesh(exodusFileName, get_bulk()); - stk::mesh::create_exposed_block_boundary_sides(get_bulk(), meta.universal_part(), {&boundaryPart}); - stk::mesh::create_interior_block_boundary_sides(get_bulk(), meta.universal_part(), {&boundaryPart}); + stk::mesh::create_all_block_boundary_sides(get_bulk(), meta.universal_part(), {&boundaryPart}); test_ngp_mesh_construction(get_bulk()); } @@ -1191,7 +1193,7 @@ NGP_TEST_F(NgpHowTo, ReuseNgpField) //now check field on host to see if FieldManager::clear_fields() (called by FieldManager dtor) //sync'd device fields back to host. //Only do this check if cuda, because if not cuda then host == device. -#ifdef KOKKOS_ENABLE_CUDA +#ifdef STK_ENABLE_GPU check_field_on_host(get_bulk(), doubleStkField, (double)3.141); #endif } diff --git a/packages/stk/stk_doc_tests/stk_mesh/howToNgpMultistateFields.cpp b/packages/stk/stk_doc_tests/stk_mesh/howToNgpMultistateFields.cpp new file mode 100644 index 000000000000..4d3f0a6cbdcd --- /dev/null +++ b/packages/stk/stk_doc_tests/stk_mesh/howToNgpMultistateFields.cpp @@ -0,0 +1,199 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +void set_field_on_host(const stk::mesh::BulkData& stkMesh, + stk::mesh::Field& stkField, + double fieldVal) +{ + stk::mesh::Selector select(stkField); + + stkField.sync_to_host(); + + stk::mesh::for_each_entity_run(stkMesh, stkField.entity_rank(), select, + [&](const stk::mesh::BulkData& bulk, stk::mesh::Entity entity) { + *stk::mesh::field_data(stkField, entity) = fieldVal; + }); + + stkField.modify_on_host(); +} + +void set_field_on_device(const stk::mesh::NgpMesh& ngpMesh, + stk::mesh::NgpField& ngpField, + double fieldVal) +{ + stk::mesh::Selector select(*ngpField.get_field_base()); + + ngpField.sync_to_device(); + + stk::mesh::for_each_entity_run(ngpMesh, ngpField.get_rank(), select, + KOKKOS_LAMBDA(const stk::mesh::FastMeshIndex& entityIndex) { + ngpField(entityIndex, 0) = fieldVal; + }); + + ngpField.modify_on_device(); +} + +void check_field_on_host(const stk::mesh::BulkData & bulk, + stk::mesh::Field & stkField, + double expectedFieldValue) +{ + stk::mesh::Selector select(stkField); + stk::mesh::for_each_entity_run(bulk, stkField.entity_rank(), select, + [&](const stk::mesh::BulkData& mesh, stk::mesh::Entity entity) { + EXPECT_NEAR(*stk::mesh::field_data(stkField, entity), expectedFieldValue, 1.e-9); + }); +} + +void check_field_on_device(const stk::mesh::NgpMesh& ngpMesh, + stk::mesh::NgpField & ngpField, + double expectedFieldValue) +{ + stk::mesh::Selector select(*ngpField.get_field_base()); + stk::mesh::for_each_entity_run(ngpMesh, ngpField.get_rank(), select, + KOKKOS_LAMBDA(const stk::mesh::FastMeshIndex& entityIndex) { + NGP_EXPECT_NEAR(ngpField(entityIndex, 0), expectedFieldValue, 1.e-9); + }); +} + +NGP_TEST(NgpMultistateField, setOnHost_swap_checkOnDevice) +{ + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 1) { GTEST_SKIP(); } + + std::unique_ptr bulkPtr = stk::mesh::MeshBuilder(MPI_COMM_WORLD) + .set_spatial_dimension(3).create(); + stk::mesh::MetaData& meta = bulkPtr->mesh_meta_data(); + meta.use_simple_fields(); + constexpr unsigned numStates = 2; + stk::mesh::Field& stkFieldNew = meta.declare_field(stk::topology::ELEM_RANK, + "myElemField", numStates); + stk::mesh::put_field_on_mesh(stkFieldNew, meta.universal_part(), 1, nullptr); + + stk::io::fill_mesh("generated:1x1x1", *bulkPtr); + + EXPECT_EQ(stk::mesh::StateNew, stkFieldNew.state()); + stk::mesh::Field& stkFieldOld = stkFieldNew.field_of_state(stk::mesh::StateOld); + EXPECT_EQ(stk::mesh::StateOld, stkFieldOld.state()); + + constexpr double oldValue = 1.0; + constexpr double newValue = 2.0; + set_field_on_host(*bulkPtr, stkFieldOld, oldValue); + set_field_on_host(*bulkPtr, stkFieldNew, newValue); + + bulkPtr->update_field_data_states(); + + stk::mesh::NgpMesh& ngpMesh = stk::mesh::get_updated_ngp_mesh(*bulkPtr); + stk::mesh::NgpField& ngpFieldOld = stk::mesh::get_updated_ngp_field(stkFieldOld); + stk::mesh::NgpField& ngpFieldNew = stk::mesh::get_updated_ngp_field(stkFieldNew); + + check_field_on_device(ngpMesh, ngpFieldOld, newValue); + check_field_on_device(ngpMesh, ngpFieldNew, oldValue); +} + +NGP_TEST(NgpMultistateField, setOnHost_swap_preExistingNgpFieldsNeedSync) +{ + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 1) { GTEST_SKIP(); } + + std::unique_ptr bulkPtr = stk::mesh::MeshBuilder(MPI_COMM_WORLD) + .set_spatial_dimension(3).create(); + stk::mesh::MetaData& meta = bulkPtr->mesh_meta_data(); + meta.use_simple_fields(); + constexpr unsigned numStates = 2; + stk::mesh::Field& stkFieldNew = meta.declare_field(stk::topology::ELEM_RANK, + "myElemField", numStates); + stk::mesh::put_field_on_mesh(stkFieldNew, meta.universal_part(), 1, nullptr); + + stk::io::fill_mesh("generated:1x1x1", *bulkPtr); + + EXPECT_EQ(stk::mesh::StateNew, stkFieldNew.state()); + stk::mesh::Field& stkFieldOld = stkFieldNew.field_of_state(stk::mesh::StateOld); + EXPECT_EQ(stk::mesh::StateOld, stkFieldOld.state()); + + constexpr double oldValue = 1.0; + constexpr double newValue = 2.0; + set_field_on_host(*bulkPtr, stkFieldOld, oldValue); + set_field_on_host(*bulkPtr, stkFieldNew, newValue); + + stk::mesh::NgpMesh& ngpMesh = stk::mesh::get_updated_ngp_mesh(*bulkPtr); + stk::mesh::NgpField& ngpFieldOld = stk::mesh::get_updated_ngp_field(stkFieldOld); + stk::mesh::NgpField& ngpFieldNew = stk::mesh::get_updated_ngp_field(stkFieldNew); + + check_field_on_device(ngpMesh, ngpFieldOld, oldValue); + check_field_on_device(ngpMesh, ngpFieldNew, newValue); + + stk::mesh::sync_to_host_and_mark_modified(meta); + bulkPtr->update_field_data_states(); + +#ifdef STK_USE_DEVICE_MESH + check_field_on_device(ngpMesh, ngpFieldOld, oldValue); + check_field_on_device(ngpMesh, ngpFieldNew, newValue); +#else + check_field_on_device(ngpMesh, ngpFieldOld, newValue); + check_field_on_device(ngpMesh, ngpFieldNew, oldValue); +#endif + + ngpFieldOld.sync_to_device(); + ngpFieldNew.sync_to_device(); + + check_field_on_device(ngpMesh, ngpFieldOld, newValue); + check_field_on_device(ngpMesh, ngpFieldNew, oldValue); +} + +NGP_TEST(NgpMultistateField, setOnDevice_swap_checkOnDevice) +{ + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 1) { GTEST_SKIP(); } + + std::unique_ptr bulkPtr = stk::mesh::MeshBuilder(MPI_COMM_WORLD) + .set_spatial_dimension(3).create(); + stk::mesh::MetaData& meta = bulkPtr->mesh_meta_data(); + meta.use_simple_fields(); + constexpr unsigned numStates = 2; + stk::mesh::Field& stkFieldNew = meta.declare_field(stk::topology::ELEM_RANK, + "myElemField", numStates); + stk::mesh::put_field_on_mesh(stkFieldNew, meta.universal_part(), 1, nullptr); + stk::io::fill_mesh("generated:1x1x1", *bulkPtr); + + EXPECT_EQ(stk::mesh::StateNew, stkFieldNew.state()); + stk::mesh::Field& stkFieldOld = stkFieldNew.field_of_state(stk::mesh::StateOld); + EXPECT_EQ(stk::mesh::StateOld, stkFieldOld.state()); + + stk::mesh::NgpMesh& ngpMesh = stk::mesh::get_updated_ngp_mesh(*bulkPtr); + stk::mesh::NgpField& ngpFieldOld = stk::mesh::get_updated_ngp_field(stkFieldOld); + stk::mesh::NgpField& ngpFieldNew = stk::mesh::get_updated_ngp_field(stkFieldNew); + + constexpr double oldValue = 1.0; + constexpr double newValue = 2.0; + + set_field_on_device(ngpMesh, ngpFieldOld, oldValue); + set_field_on_device(ngpMesh, ngpFieldNew, newValue); +#ifdef STK_USE_DEVICE_MESH + check_field_on_host(*bulkPtr, stkFieldOld, 0.0); + check_field_on_host(*bulkPtr, stkFieldNew, 0.0); +#else + check_field_on_host(*bulkPtr, stkFieldOld, oldValue); + check_field_on_host(*bulkPtr, stkFieldNew, newValue); +#endif + + const bool rotateNgpFieldViews = true; + bulkPtr->update_field_data_states(rotateNgpFieldViews); + + check_field_on_device(ngpMesh, ngpFieldOld, newValue); + check_field_on_device(ngpMesh, ngpFieldNew, oldValue); +} + +} diff --git a/packages/stk/stk_doc_tests/stk_mesh/howToSkinMesh.cpp b/packages/stk/stk_doc_tests/stk_mesh/howToSkinMesh.cpp index 264d9a76f724..53021db54387 100644 --- a/packages/stk/stk_doc_tests/stk_mesh/howToSkinMesh.cpp +++ b/packages/stk/stk_doc_tests/stk_mesh/howToSkinMesh.cpp @@ -35,6 +35,7 @@ #include // for AssertHelper, EXPECT_EQ, etc #include // for StkMeshIoBroker #include // for count_entities +#include #include // for MetaData #include // for Selector #include // for Selector @@ -42,6 +43,7 @@ #include // for string #include // for vector #include "stk_io/DatabasePurpose.hpp" // for DatabasePurpose::READ_MESH +#include "stk_io/FillMesh.hpp" namespace stk { namespace mesh { class BulkData; } } namespace @@ -53,30 +55,28 @@ TEST(StkMeshHowTo, SkinExposedHex) // INITIALIZATION MPI_Comm communicator = MPI_COMM_WORLD; if (stk::parallel_machine_size(communicator) != 1) { return; } - stk::io::StkMeshIoBroker stkIo(communicator); - stkIo.use_simple_fields(); + + std::shared_ptr bulk = stk::mesh::MeshBuilder(communicator).create(); + stk::mesh::MetaData& meta = bulk->mesh_meta_data(); + meta.use_simple_fields(); const std::string generatedFileName = "generated:1x1x1"; - stkIo.add_mesh_database(generatedFileName, stk::io::READ_MESH); - stkIo.create_input_mesh(); - stkIo.populate_bulk_data(); + stk::io::fill_mesh(generatedFileName, *bulk); // ============================================================ //+ EXAMPLE //+ Skin the mesh and create the exposed boundary sides.. - stk::mesh::MetaData &metaData = stkIo.meta_data(); - stk::mesh::BulkData &bulkData = stkIo.bulk_data(); - stk::mesh::Selector allEntities = metaData.universal_part(); - stk::mesh::Part &skinPart = metaData.declare_part("skin", metaData.side_rank()); + stk::mesh::Selector allEntities = meta.universal_part(); + stk::mesh::Part &skinPart = meta.declare_part("skin", meta.side_rank()); stk::io::put_io_part_attribute(skinPart); - stk::mesh::create_exposed_block_boundary_sides(bulkData, allEntities, {&skinPart}); + stk::mesh::create_exposed_block_boundary_sides(*bulk, allEntities, {&skinPart}); // ================================================== // VERIFICATION - EXPECT_TRUE(stk::mesh::check_exposed_block_boundary_sides(bulkData, allEntities, skinPart)); - stk::mesh::Selector skin(skinPart & metaData.locally_owned_part()); - unsigned numSkinnedSides = stk::mesh::count_selected_entities(skin, bulkData.buckets(metaData.side_rank())); + EXPECT_TRUE(stk::mesh::check_exposed_block_boundary_sides(*bulk, allEntities, skinPart)); + stk::mesh::Selector skin(skinPart & meta.locally_owned_part()); + unsigned numSkinnedSides = stk::mesh::count_entities(*bulk, meta.side_rank(), skin); EXPECT_EQ(6u, numSkinnedSides) << "in part " << skinPart.name(); } //END_CREATE_EXPOSED_BOUNDARY @@ -88,40 +88,79 @@ TEST(StkMeshHowTo, SkinInteriorHex) // INITIALIZATION MPI_Comm communicator = MPI_COMM_WORLD; if (stk::parallel_machine_size(communicator) != 1) { return; } - stk::io::StkMeshIoBroker stkIo(communicator); - stkIo.use_simple_fields(); + + std::shared_ptr bulk = stk::mesh::MeshBuilder(communicator).create(); + stk::mesh::MetaData& meta = bulk->mesh_meta_data(); + meta.use_simple_fields(); const std::string generatedFileName = "generated:1x1x2"; - stkIo.add_mesh_database(generatedFileName, stk::io::READ_MESH); - stkIo.create_input_mesh(); - stkIo.populate_bulk_data(); + stk::io::fill_mesh(generatedFileName, *bulk); // ============================================================ //+ EXAMPLE //+ Skin the mesh and create the exposed boundary sides.. - stk::mesh::MetaData &metaData = stkIo.meta_data(); - stk::mesh::BulkData &bulkData = stkIo.bulk_data(); - stk::mesh::Selector allEntities = metaData.universal_part(); - stk::mesh::Part &skinPart = metaData.declare_part("skin", metaData.side_rank()); + stk::mesh::Selector allEntities = meta.universal_part(); + stk::mesh::Part &skinPart = meta.declare_part("skin", meta.side_rank()); stk::io::put_io_part_attribute(skinPart); - stk::mesh::Entity elem2 = bulkData.get_entity(stk::topology::ELEM_RANK, 2u); - stk::mesh::Part *block_1 = metaData.get_part("block_1"); + stk::mesh::Entity elem2 = bulk->get_entity(stk::topology::ELEM_RANK, 2u); + stk::mesh::Part *block_1 = meta.get_part("block_1"); - bulkData.modification_begin(); - stk::mesh::Part &block_2 = metaData.declare_part("block_2", stk::topology::ELEM_RANK); + bulk->modification_begin(); + stk::mesh::Part &block_2 = meta.declare_part("block_2", stk::topology::ELEM_RANK); stk::io::put_io_part_attribute(block_2); - bulkData.change_entity_parts(elem2, stk::mesh::ConstPartVector{&block_2}, stk::mesh::ConstPartVector{block_1}); - bulkData.modification_end(); + bulk->change_entity_parts(elem2, stk::mesh::ConstPartVector{&block_2}, stk::mesh::ConstPartVector{block_1}); + bulk->modification_end(); - stk::mesh::create_interior_block_boundary_sides(bulkData, allEntities, {&skinPart}); + stk::mesh::create_interior_block_boundary_sides(*bulk, allEntities, {&skinPart}); // ================================================== // VERIFICATION - EXPECT_TRUE(stk::mesh::check_interior_block_boundary_sides(bulkData, allEntities, skinPart)); - stk::mesh::Selector skin(skinPart & metaData.locally_owned_part()); - unsigned numSkinnedSides = stk::mesh::count_selected_entities(skin, bulkData.buckets(metaData.side_rank())); + EXPECT_TRUE(stk::mesh::check_interior_block_boundary_sides(*bulk, allEntities, skinPart)); + stk::mesh::Selector skin(skinPart & meta.locally_owned_part()); + unsigned numSkinnedSides = stk::mesh::count_entities(*bulk, meta.side_rank(), skin); EXPECT_EQ(1u, numSkinnedSides) << "in part " << skinPart.name(); } //END_CREATE_INTERIOR_BOUNDARY + +//BEGIN_CREATE_ALL_BLOCK_BOUNDARY +TEST(StkMeshHowTo, SkinAllHexBlocks) +{ + // ============================================================ + // INITIALIZATION + MPI_Comm communicator = MPI_COMM_WORLD; + if (stk::parallel_machine_size(communicator) != 1) { return; } + + std::shared_ptr bulk = stk::mesh::MeshBuilder(communicator).create(); + stk::mesh::MetaData& meta = bulk->mesh_meta_data(); + meta.use_simple_fields(); + + const std::string generatedFileName = "generated:1x1x2"; + stk::io::fill_mesh(generatedFileName, *bulk); + + // ============================================================ + //+ EXAMPLE + //+ Skin the mesh and create all boundary sides.. + stk::mesh::Selector allEntities = meta.universal_part(); + stk::mesh::Part &skinPart = meta.declare_part("skin", meta.side_rank()); + stk::io::put_io_part_attribute(skinPart); + + stk::mesh::Entity elem2 = bulk->get_entity(stk::topology::ELEM_RANK, 2u); + stk::mesh::Part *block_1 = meta.get_part("block_1"); + + bulk->modification_begin(); + stk::mesh::Part &block_2 = meta.declare_part("block_2", stk::topology::ELEM_RANK); + stk::io::put_io_part_attribute(block_2); + bulk->change_entity_parts(elem2, stk::mesh::ConstPartVector{&block_2}, stk::mesh::ConstPartVector{block_1}); + bulk->modification_end(); + + stk::mesh::create_all_block_boundary_sides(*bulk, allEntities, {&skinPart}); + + // ================================================== + // VERIFICATION + stk::mesh::Selector skin(skinPart & meta.locally_owned_part()); + unsigned numSkinnedSides = stk::mesh::count_entities(*bulk, meta.side_rank(), skin); + EXPECT_EQ(11u, numSkinnedSides) << "in part " << skinPart.name(); +} +//END_CREATE_ALL_BLOCK_BOUNDARY } diff --git a/packages/stk/stk_doc_tests/stk_mesh/howToUseAura.cpp b/packages/stk/stk_doc_tests/stk_mesh/howToUseAura.cpp index 83cc121d129d..be8bd3f947ea 100644 --- a/packages/stk/stk_doc_tests/stk_mesh/howToUseAura.cpp +++ b/packages/stk/stk_doc_tests/stk_mesh/howToUseAura.cpp @@ -61,8 +61,7 @@ void expectNumElementsInAura(stk::mesh::BulkData::AutomaticAuraOption autoAuraOp stk::mesh::BulkData& bulk = *bulkPtr; stk::io::fill_mesh("generated:1x1x2", bulk); - EXPECT_EQ(numExpectedElementsInAura, - stk::mesh::count_selected_entities(meta.aura_part(), bulk.buckets(stk::topology::ELEMENT_RANK))); + EXPECT_EQ(numExpectedElementsInAura, stk::mesh::count_entities(bulk, stk::topology::ELEM_RANK, meta.aura_part())); } } TEST(StkMeshHowTo, useNoAura) @@ -73,19 +72,5 @@ TEST(StkMeshHowTo, useAutomaticGeneratedAura) { expectNumElementsInAura(stk::mesh::BulkData::AUTO_AURA, 1); } -TEST(StkMeshHowTo, useAuraDefaultBehavior) -{ - MPI_Comm communicator = MPI_COMM_WORLD; - if (stk::parallel_machine_size(communicator) == 2) - { - std::shared_ptr bulkPtr = stk::mesh::MeshBuilder(communicator).create(); - bulkPtr->mesh_meta_data().use_simple_fields(); - stk::mesh::MetaData& meta = bulkPtr->mesh_meta_data(); - stk::mesh::BulkData& bulk = *bulkPtr; - stk::io::fill_mesh("generated:1x1x2", bulk); - - EXPECT_EQ(1u, stk::mesh::count_selected_entities(meta.aura_part(), bulk.buckets(stk::topology::ELEMENT_RANK))); - } -} //END_AURA_EXAMPLES } diff --git a/packages/stk/stk_doc_tests/stk_mesh/howToUseGenerateNewIds.cpp b/packages/stk/stk_doc_tests/stk_mesh/howToUseGenerateNewIds.cpp index 676f27a38cd0..8970f57727eb 100644 --- a/packages/stk/stk_doc_tests/stk_mesh/howToUseGenerateNewIds.cpp +++ b/packages/stk/stk_doc_tests/stk_mesh/howToUseGenerateNewIds.cpp @@ -1,7 +1,9 @@ #include // for TEST +#include #include // for BulkData #include // for BulkData -#include // for StkMeshIoBroker +#include +#include #include #include @@ -9,24 +11,19 @@ namespace { -void test_that_ids_are_unique(stk::mesh::BulkData &bulkData, stk::topology::rank_t rank, std::vector& requestedIds) +void test_that_ids_are_unique(const stk::mesh::BulkData &bulkData, stk::mesh::EntityRank rank, std::vector& requestedIds) { std::vector ids_in_use; - const stk::mesh::BucketVector& buckets = bulkData.get_buckets(rank, bulkData.mesh_meta_data().locally_owned_part()); - for (size_t i=0;i::iterator iter1 = std::unique(requestedIds.begin(), requestedIds.end()); - STK_ThrowRequireMsg(iter1 == requestedIds.end(), "Oh no! " << __FILE__ << __LINE__); + STK_ThrowRequireMsg(iter1 == requestedIds.end(), "Ids not unique. " << __FILE__ << __LINE__); stk::CommSparse comm(bulkData.parallel()); @@ -55,18 +52,15 @@ void test_that_ids_are_unique(stk::mesh::BulkData &bulkData, stk::topology::rank } } - for(int i = 0; i < bulkData.parallel_size(); ++i) - { - if(i != bulkData.parallel_rank()) - { - while(comm.recv_buffer(i).remaining()) - { - uint64_t key; - comm.recv_buffer(i).unpack(key); - bool is_other_procs_id_on_this_proc = std::binary_search(requestedIds.begin(), requestedIds.end(), key); - STK_ThrowRequireMsg(is_other_procs_id_on_this_proc == false, "Oh no! " << __FILE__<< __LINE__); - bool is_id_already_in_use = std::binary_search(ids_in_use.begin(), ids_in_use.end(), key); - STK_ThrowRequireMsg(is_id_already_in_use == false, "Oh no! " << __FILE__ << __LINE__); + for(int i = 0; i < bulkData.parallel_size(); ++i) { + if(i != bulkData.parallel_rank()) { + while(comm.recv_buffer(i).remaining()) { + uint64_t id; + comm.recv_buffer(i).unpack(id); + bool is_other_procs_id_on_this_proc = std::binary_search(requestedIds.begin(), requestedIds.end(), id); + STK_ThrowRequireMsg(is_other_procs_id_on_this_proc == false, "Id requested on proc " << i << " also requested on proc " << bulkData.parallel_rank()<< ". " << __FILE__<< __LINE__); + bool is_id_already_in_use = std::binary_search(ids_in_use.begin(), ids_in_use.end(), id); + STK_ThrowRequireMsg(is_id_already_in_use == false, "Id requested on proc " << i << " already in use on proc " << bulkData.parallel_rank() << ". " << __FILE__ << __LINE__); } } } @@ -77,28 +71,21 @@ TEST(StkMeshHowTo, use_generate_new_ids) { MPI_Comm communicator = MPI_COMM_WORLD; - int num_procs = -1; - MPI_Comm_size(communicator, &num_procs); - std::ostringstream os; - os << "generated:1x1x" << num_procs; - const std::string generatedMeshSpecification = os.str(); - - stk::io::StkMeshIoBroker stkMeshIoBroker(communicator); - stkMeshIoBroker.use_simple_fields(); - stkMeshIoBroker.add_mesh_database(generatedMeshSpecification, stk::io::READ_MESH); - stkMeshIoBroker.create_input_mesh(); - stkMeshIoBroker.populate_bulk_data(); + int num_procs = stk::parallel_machine_size(communicator); + std::unique_ptr bulkPtr = stk::mesh::MeshBuilder(communicator).create(); + bulkPtr->mesh_meta_data().use_simple_fields(); - stk::mesh::BulkData &stkMeshBulkData = stkMeshIoBroker.bulk_data(); + const std::string generatedMeshSpecification = "generated:1x1x" + std::to_string(num_procs); + stk::io::fill_mesh(generatedMeshSpecification, *bulkPtr); // Given a mesh, request 10 unique node ids std::vector requestedIds; unsigned numRequested = 10; - stkMeshBulkData.generate_new_ids(stk::topology::NODE_RANK, numRequested, requestedIds); + bulkPtr->generate_new_ids(stk::topology::NODE_RANK, numRequested, requestedIds); - test_that_ids_are_unique(stkMeshBulkData, stk::topology::NODE_RANK, requestedIds); + test_that_ids_are_unique(*bulkPtr, stk::topology::NODE_RANK, requestedIds); } //END_TEST_1 diff --git a/packages/stk/stk_doc_tests/stk_mesh/howToUseSelectors.cpp b/packages/stk/stk_doc_tests/stk_mesh/howToUseSelectors.cpp index c8c5052652c7..35be1026d23d 100644 --- a/packages/stk/stk_doc_tests/stk_mesh/howToUseSelectors.cpp +++ b/packages/stk/stk_doc_tests/stk_mesh/howToUseSelectors.cpp @@ -35,60 +35,99 @@ #include // for AssertHelper, EXPECT_EQ, etc #include // for size_t #include // for ostringstream, etc -#include // for StkMeshIoBroker +#include +#include #include // for BulkData #include // for MetaData #include // for Selector, operator<<, etc +#include #include // for topology, etc +#include // for BucketVector, PartVector +#include #include // for string -#include "stk_io/DatabasePurpose.hpp" // for DatabasePurpose::READ_MESH -#include "stk_mesh/base/Types.hpp" // for BucketVector, PartVector namespace stk { namespace mesh { class Part; } } namespace { //-BEGIN +TEST(StkMeshHowTo, basicSelectorUsage) +{ + MPI_Comm communicator = MPI_COMM_WORLD; + if (stk::parallel_machine_size(communicator) != 1) { GTEST_SKIP(); } + std::unique_ptr bulkPtr = stk::mesh::MeshBuilder(communicator) + .set_spatial_dimension(3).create(); + stk::mesh::MetaData& meta = bulkPtr->mesh_meta_data(); + meta.use_simple_fields(); + +//create a simple shell-quad-4 mesh: +// 6 +// 3*----*----*9 +// | E2 | E4 | +// | | | +// 2*---5*----*8 +// | E1 | E3 | +// | | | +// 1*----*----*7 +// 4 +// + + std::string meshDesc = "0,1,SHELL_QUAD_4, 1,4,2,5, block_1\n" + "0,2,SHELL_QUAD_4, 2,5,6,3, block_2\n" + "0,3,SHELL_QUAD_4, 4,7,8,5, block_3\n" + "0,4,SHELL_QUAD_4, 5,8,9,6, block_4\n"; + stk::unit_test_util::simple_fields::setup_text_mesh(*bulkPtr, meshDesc); + + stk::mesh::Part& block_1 = *meta.get_part("block_1"); + stk::mesh::Part& block_2 = *meta.get_part("block_2"); + stk::mesh::Part& block_3 = *meta.get_part("block_3"); + stk::mesh::Part& block_4 = *meta.get_part("block_4"); + stk::mesh::PartVector allBlocks = {&block_1, &block_2, &block_3, &block_4}; + + stk::mesh::Selector allNodes = stk::mesh::selectUnion(allBlocks); + stk::mesh::Selector onlyCenterNode = stk::mesh::selectIntersection(allBlocks); + stk::mesh::Selector nodes456 = (block_1 | block_2) & (block_3 | block_4); + stk::mesh::Selector nodes123 = (block_1 | block_2) - nodes456; + + EXPECT_EQ(9u, stk::mesh::count_entities(*bulkPtr, stk::topology::NODE_RANK, allNodes)); + EXPECT_EQ(1u, stk::mesh::count_entities(*bulkPtr, stk::topology::NODE_RANK, onlyCenterNode)); + EXPECT_EQ(3u, stk::mesh::count_entities(*bulkPtr, stk::topology::NODE_RANK, nodes456)); + EXPECT_EQ(3u, stk::mesh::count_entities(*bulkPtr, stk::topology::NODE_RANK, nodes123)); +} + TEST(StkMeshHowTo, betterUnderstandSelectorConstruction) { MPI_Comm communicator = MPI_COMM_WORLD; - if (stk::parallel_machine_size(communicator) != 1) { return; } - stk::io::StkMeshIoBroker stkMeshIoBroker(communicator); - stkMeshIoBroker.use_simple_fields(); + if (stk::parallel_machine_size(communicator) != 1) { GTEST_SKIP(); } + std::unique_ptr bulkPtr = stk::mesh::MeshBuilder(communicator).create(); + bulkPtr->mesh_meta_data().use_simple_fields(); const std::string generatedCubeMeshSpecification = "generated:1x1x1"; - stkMeshIoBroker.add_mesh_database(generatedCubeMeshSpecification, stk::io::READ_MESH); - stkMeshIoBroker.create_input_mesh(); - stkMeshIoBroker.populate_bulk_data(); - - stk::mesh::BulkData &stkMeshBulkData = stkMeshIoBroker.bulk_data(); + stk::io::fill_mesh(generatedCubeMeshSpecification, *bulkPtr); stk::mesh::Selector nothingSelector_byDefaultConstruction; size_t expectingZeroBuckets = 0; - EXPECT_EQ(expectingZeroBuckets, stkMeshBulkData.get_buckets(stk::topology::NODE_RANK, nothingSelector_byDefaultConstruction).size()); + EXPECT_EQ(expectingZeroBuckets, bulkPtr->get_buckets(stk::topology::NODE_RANK, nothingSelector_byDefaultConstruction).size()); std::ostringstream readableSelectorDescription; readableSelectorDescription << nothingSelector_byDefaultConstruction; EXPECT_STREQ("NOTHING", readableSelectorDescription.str().c_str()); stk::mesh::Selector allSelector(!nothingSelector_byDefaultConstruction); - size_t numberOfAllNodeBuckets = stkMeshBulkData.buckets(stk::topology::NODE_RANK).size(); - EXPECT_EQ(numberOfAllNodeBuckets, stkMeshBulkData.get_buckets(stk::topology::NODE_RANK, allSelector).size()); + size_t numberOfAllNodeBuckets = bulkPtr->buckets(stk::topology::NODE_RANK).size(); + EXPECT_EQ(numberOfAllNodeBuckets, bulkPtr->get_buckets(stk::topology::NODE_RANK, allSelector).size()); } TEST(StkMeshHowTo, makeSureYouAreNotIntersectingNothingSelector) { MPI_Comm communicator = MPI_COMM_WORLD; if (stk::parallel_machine_size(communicator) != 1) { return; } - stk::io::StkMeshIoBroker stkMeshIoBroker(communicator); - stkMeshIoBroker.use_simple_fields(); + std::unique_ptr bulkPtr = stk::mesh::MeshBuilder(communicator).create(); + bulkPtr->mesh_meta_data().use_simple_fields(); // syntax creates faces for surface on the positive: 'x-side', 'y-side', and 'z-side' // of a 1x1x1 cube, these parts are given the names: 'surface_1', 'surface_2', and 'surface_3' - // automagically when it is created [create_input_mesh()] const std::string generatedCubeMeshSpecification = "generated:1x1x1|sideset:XYZ"; - stkMeshIoBroker.add_mesh_database(generatedCubeMeshSpecification, stk::io::READ_MESH); - stkMeshIoBroker.create_input_mesh(); - stkMeshIoBroker.populate_bulk_data(); + stk::io::fill_mesh(generatedCubeMeshSpecification, *bulkPtr); - stk::mesh::MetaData &stkMeshMetaData = stkMeshIoBroker.meta_data(); + stk::mesh::MetaData &stkMeshMetaData = bulkPtr->mesh_meta_data(); stk::mesh::Part *surface1Part = stkMeshMetaData.get_part("surface_1"); stk::mesh::Part *surface2Part = stkMeshMetaData.get_part("surface_2"); stk::mesh::Part *surface3Part = stkMeshMetaData.get_part("surface_3"); @@ -98,20 +137,19 @@ TEST(StkMeshHowTo, makeSureYouAreNotIntersectingNothingSelector) allSurfaces.push_back(surface3Part); stk::mesh::Selector selectorIntersectingNothing; - for (size_t surfaceIndex = 0; surfaceIndex < allSurfaces.size(); ++surfaceIndex) - { + for (size_t surfaceIndex = 0; surfaceIndex < allSurfaces.size(); ++surfaceIndex) { stk::mesh::Part &surfacePart = *(allSurfaces[surfaceIndex]); stk::mesh::Selector surfaceSelector(surfacePart); selectorIntersectingNothing &= surfacePart; } + size_t expectedNumberOfBucketsWhenIntersectingNothing = 0; - stk::mesh::BulkData &stkMeshBulkData = stkMeshIoBroker.bulk_data(); - stk::mesh::BucketVector selectedBuckets = stkMeshBulkData.get_buckets(stk::topology::NODE_RANK, selectorIntersectingNothing); + stk::mesh::BucketVector selectedBuckets = bulkPtr->get_buckets(stk::topology::NODE_RANK, selectorIntersectingNothing); EXPECT_EQ(expectedNumberOfBucketsWhenIntersectingNothing, selectedBuckets.size()); stk::mesh::Selector preferredBoundaryNodesSelector = stk::mesh::selectIntersection(allSurfaces); size_t expectedNumberOfNodeBucketsWhenIntersectingAllSurfaces = 1; - selectedBuckets = stkMeshBulkData.get_buckets(stk::topology::NODE_RANK, preferredBoundaryNodesSelector); + selectedBuckets = bulkPtr->get_buckets(stk::topology::NODE_RANK, preferredBoundaryNodesSelector); EXPECT_EQ(expectedNumberOfNodeBucketsWhenIntersectingAllSurfaces, selectedBuckets.size()); } //-END diff --git a/packages/stk/stk_doc_tests/stk_mesh/stkMeshTestUtils.hpp b/packages/stk/stk_doc_tests/stk_mesh/stkMeshTestUtils.hpp index 4a146932fbf2..78ce388573d2 100644 --- a/packages/stk/stk_doc_tests/stk_mesh/stkMeshTestUtils.hpp +++ b/packages/stk/stk_doc_tests/stk_mesh/stkMeshTestUtils.hpp @@ -36,8 +36,8 @@ #define stkMeshTestUtilsHpp #include -#include #include +#include #include #include @@ -47,30 +47,29 @@ namespace testUtils inline int get_other_proc(int myproc) { - int otherproc = 1; - if (myproc == 1) - otherproc = 0; - return otherproc; + return myproc == 0 ? 1 : 0; } inline -void testTemperatureFieldSetCorrectly(const stk::mesh::Field &temperatureField, double prescribedTemperatureValue, const std::set &boundaryNodeIds) +void testTemperatureFieldSetCorrectly(const stk::mesh::Field &temperatureField, + const stk::mesh::Selector& boundaryNodesSelector, + double prescribedTemperatureValue) { - stk::mesh::BulkData &stkMeshBulkData = temperatureField.get_mesh(); - stk::mesh::EntityVector nodes; - stk::mesh::get_entities(stkMeshBulkData, stk::topology::NODE_RANK, nodes); - for(size_t i=0; i // for MetaData, put_field, etc #include "stk_mesh/base/Entity.hpp" // for Entity #include "stk_mesh/base/FieldBase.hpp" // for field_scalars_per_entity, etc -#include "stk_mesh/base/Types.hpp" // for EntityId #include "stk_topology/topology.hpp" // for topology, etc #include "stk_io/IossBridge.hpp" -namespace stk { namespace mesh { class Part; } } +#include "stk_unit_test_utils/TextMesh.hpp" namespace { //BEGINUseAdvancedFields TEST(stkMeshHowTo, useAdvancedFields) { + if (stk::parallel_machine_size(MPI_COMM_WORLD) > 1) { GTEST_SKIP(); } + const unsigned spatialDimension = 3; stk::mesh::MeshBuilder builder(MPI_COMM_WORLD); builder.set_spatial_dimension(spatialDimension); - builder.set_entity_rank_names(stk::mesh::entity_rank_names()); - std::shared_ptr bulkPtr = builder.create(); + std::unique_ptr bulkPtr = builder.create(); bulkPtr->mesh_meta_data().use_simple_fields(); stk::mesh::MetaData& metaData = bulkPtr->mesh_meta_data(); @@ -78,16 +78,12 @@ TEST(stkMeshHowTo, useAdvancedFields) stk::mesh::put_field_on_mesh(variableSizeField, hexPart, numVectorValues, numCopies, initialVectorValue); stk::io::set_field_output_type(variableSizeField, stk::io::FieldOutputType::VECTOR_3D); - metaData.commit(); - stk::mesh::BulkData& mesh = *bulkPtr; - mesh.modification_begin(); - stk::mesh::EntityId tetId = 1; - stk::mesh::EntityIdVector tetNodes {1, 2, 3, 4}; - stk::mesh::Entity tetElem=stk::mesh::declare_element(mesh, tetPart, tetId, tetNodes); - stk::mesh::EntityId hexId = 2; - stk::mesh::EntityIdVector hexNodes {5, 6, 7, 8, 9, 10, 11, 12}; - stk::mesh::Entity hexElem=stk::mesh::declare_element(mesh, hexPart, hexId, hexNodes); - mesh.modification_end(); + std::string meshSpec = "0,1,TET_4, 1,2,3,4, tetElementPart\n" + "0,2,HEX_8, 5,6,7,8,9,10,11,12, hexElementPart"; + stk::unit_test_util::setup_text_mesh(*bulkPtr, meshSpec); + + stk::mesh::Entity tetElem = bulkPtr->get_entity(stk::topology::ELEM_RANK, 1); + stk::mesh::Entity hexElem = bulkPtr->get_entity(stk::topology::ELEM_RANK, 2); const int tensorScalarsPerTet = stk::mesh::field_scalars_per_entity(tensorField, tetElem); const int tensorScalarsPerHex = stk::mesh::field_scalars_per_entity(tensorField, hexElem); diff --git a/packages/stk/stk_doc_tests/stk_mesh/useFieldBLAS.cpp b/packages/stk/stk_doc_tests/stk_mesh/useFieldBLAS.cpp new file mode 100644 index 000000000000..415fa7146c7b --- /dev/null +++ b/packages/stk/stk_doc_tests/stk_mesh/useFieldBLAS.cpp @@ -0,0 +1,105 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + + +#include // for AssertHelper, EXPECT_EQ, etc +#include // for size_t +#include // for BulkData +#include +#include // for Field +#include +#include // for MetaData, entity_rank_names, etc +#include "stk_mesh/base/Entity.hpp" // for Entity +#include "stk_mesh/base/ForEachEntity.hpp" +#include "stk_mesh/base/Types.hpp" // for EntityId +#include "stk_topology/topology.hpp" // for topology, etc +#include "stk_io/IossBridge.hpp" +#include "stk_unit_test_utils/TextMesh.hpp" + +namespace { + +constexpr unsigned SpatialDimension = 3; + +void create_two_tet_element_mesh(stk::mesh::BulkData &bulk) +{ + std::string meshSpec = "0, 1,TET_4, 1,2,3,4\n" + "0, 2,TET_4, 2,3,4,5"; + stk::unit_test_util::simple_fields::setup_text_mesh(bulk, meshSpec); +} + +//BEGINUseFieldBLAS +TEST(stkMeshHowTo, useFieldBLAS) +{ + if (stk::parallel_machine_size(MPI_COMM_WORLD) > 1) { GTEST_SKIP(); } + stk::mesh::MeshBuilder builder(MPI_COMM_WORLD); + builder.set_spatial_dimension(SpatialDimension); + std::unique_ptr bulkPtr = builder.create(); + bulkPtr->mesh_meta_data().use_simple_fields(); + stk::mesh::MetaData& metaData = bulkPtr->mesh_meta_data(); + + typedef stk::mesh::Field DoubleField; + DoubleField& pressureField = metaData.declare_field(stk::topology::ELEM_RANK, "pressure"); + DoubleField& displacementsField = metaData.declare_field(stk::topology::NODE_RANK, "displacements"); + DoubleField& velocityField = metaData.declare_field(stk::topology::NODE_RANK, "velocity"); + + double initialPressureValue = 4.4; + constexpr unsigned numValuesPerNode = 3; + stk::mesh::put_field_on_entire_mesh_with_initial_value(pressureField, &initialPressureValue); + stk::mesh::put_field_on_mesh(displacementsField, metaData.universal_part(), numValuesPerNode, nullptr); + stk::mesh::put_field_on_mesh(velocityField, metaData.universal_part(), numValuesPerNode, nullptr); + + create_two_tet_element_mesh(*bulkPtr); + + //incompatible fields, elem-rank vs node-rank + EXPECT_ANY_THROW(stk::mesh::field_copy(pressureField, displacementsField)); + + stk::mesh::field_fill(99.0, displacementsField); + stk::mesh::field_fill(10.0, velocityField); + const double alpha = 5.0; + stk::mesh::field_axpy(alpha, displacementsField, velocityField); + + const double expectedVal = 10.0 + alpha*99.0; + + auto expectEqualVal = [&](const stk::mesh::BulkData& bulk, stk::mesh::Entity node) { + const double* velocityDataForNode = stk::mesh::field_data(velocityField, node); + for(unsigned i=0; i // for size_t #include // for BulkData #include -#include // for declare_element #include // for Field +#include #include // for MetaData, entity_rank_names, etc -#include "stk_mesh/base/Bucket.hpp" // for Bucket #include "stk_mesh/base/Entity.hpp" // for Entity -#include "stk_mesh/base/FieldBase.hpp" // for field_data, etc -#include "stk_mesh/base/Types.hpp" // for BucketVector, EntityId +#include "stk_mesh/base/ForEachEntity.hpp" +#include "stk_mesh/base/Types.hpp" // for EntityId #include "stk_topology/topology.hpp" // for topology, etc #include "stk_io/IossBridge.hpp" -namespace stk { namespace mesh { class Part; } } +#include "stk_unit_test_utils/TextMesh.hpp" namespace { -namespace SpatialDimension { const unsigned three = 3; } +constexpr unsigned SpatialDimension = 3; void create_two_tet_element_mesh(stk::mesh::BulkData &bulk) { - stk::mesh::MetaData &meta = bulk.mesh_meta_data(); - meta.use_simple_fields(); - stk::mesh::Part &tetPart = meta.declare_part_with_topology("tetElementPart", stk::topology::TET_4); - meta.commit(); - - bulk.modification_begin(); - stk::mesh::EntityId elem1Id = 1; - stk::mesh::EntityIdVector elem1Nodes {1, 2, 3, 4}; - stk::mesh::declare_element(bulk, tetPart, elem1Id, elem1Nodes); - stk::mesh::EntityId elem2Id = 2; - stk::mesh::EntityIdVector elem2Nodes {2, 3, 4, 5}; - stk::mesh::declare_element(bulk, tetPart, elem2Id, elem2Nodes); - bulk.modification_end(); + std::string meshSpec = "0, 1,TET_4, 1,2,3,4\n" + "0, 2,TET_4, 2,3,4,5"; + stk::unit_test_util::simple_fields::setup_text_mesh(bulk, meshSpec); } //BEGINUseSimpleFields TEST(stkMeshHowTo, useSimpleFields) { + if (stk::parallel_machine_size(MPI_COMM_WORLD) > 1) { GTEST_SKIP(); } + stk::mesh::MeshBuilder builder(MPI_COMM_WORLD); - builder.set_spatial_dimension(SpatialDimension::three); - builder.set_entity_rank_names(stk::mesh::entity_rank_names()); - std::shared_ptr bulkPtr = builder.create(); + builder.set_spatial_dimension(SpatialDimension); + std::unique_ptr bulkPtr = builder.create(); bulkPtr->mesh_meta_data().use_simple_fields(); stk::mesh::MetaData& metaData = bulkPtr->mesh_meta_data(); @@ -83,32 +73,34 @@ TEST(stkMeshHowTo, useSimpleFields) DoubleField& pressureField = metaData.declare_field(stk::topology::ELEM_RANK, "pressure"); DoubleField& displacementsField = metaData.declare_field(stk::topology::NODE_RANK, "displacements"); - double initialPressureValue = 4.4; + constexpr double initialPressureValue = 4.4; + constexpr unsigned vectorFieldLengthPerEntity = 3; stk::mesh::put_field_on_entire_mesh_with_initial_value(pressureField, &initialPressureValue); - stk::mesh::put_field_on_mesh(displacementsField, metaData.universal_part(), 3, nullptr); + stk::mesh::put_field_on_mesh(displacementsField, metaData.universal_part(), vectorFieldLengthPerEntity, nullptr); stk::io::set_field_output_type(displacementsField, stk::io::FieldOutputType::VECTOR_3D); stk::mesh::BulkData& mesh = *bulkPtr; create_two_tet_element_mesh(mesh); - const stk::mesh::BucketVector& nodeBuckets = mesh.buckets(stk::topology::NODE_RANK); - EXPECT_TRUE(!nodeBuckets.empty()); - for(size_t bucketIndex=0; bucketIndex bulkPtr = builder.create(); - stk::mesh::MetaData& metaData = bulkPtr->mesh_meta_data(); - - typedef stk::mesh::Field DoubleField; - DoubleField& velocities = metaData.declare_field(stk::topology::NODE_RANK, "velocities"); - DoubleField& displacements = metaData.declare_field(stk::topology::NODE_RANK, "displacements"); - - unsigned fieldLength = 3; - stk::mesh::put_field_on_mesh(velocities, metaData.universal_part(), fieldLength, nullptr); - stk::mesh::put_field_on_mesh(displacements, metaData.universal_part(), fieldLength, nullptr); - - stk::io::set_field_output_type(velocities, stk::io::FieldOutputType::VECTOR_3D); - - stk::mesh::BulkData& mesh = *bulkPtr; - create_single_tet_element(mesh); - - stk::mesh::Entity node1 = mesh.get_entity(stk::topology::NODE_RANK, 1); - EXPECT_EQ(stk::mesh::field_scalars_per_entity(velocities, node1), - stk::mesh::field_scalars_per_entity(displacements, node1)); -} -//ENDBADFIELD - } diff --git a/packages/stk/stk_doc_tests/stk_middle_mesh/CMakeLists.txt b/packages/stk/stk_doc_tests/stk_middle_mesh/CMakeLists.txt index 1d0d974af087..96070f6aeeea 100644 --- a/packages/stk/stk_doc_tests/stk_middle_mesh/CMakeLists.txt +++ b/packages/stk/stk_doc_tests/stk_middle_mesh/CMakeLists.txt @@ -17,9 +17,9 @@ if(HAVE_STK_Trilinos) else() add_executable(stk_middle_mesh_doc_tests ${SOURCES}) target_link_libraries(stk_middle_mesh_doc_tests stk_middle_mesh) - target_link_libraries(stk_coupling_doc_tests stk_unit_test_utils) - target_link_libraries(stk_coupling_doc_tests stk_util_parallel) - target_link_libraries(stk_coupling_doc_tests stk_unit_main) + target_link_libraries(stk_middle_mesh_doc_tests stk_unit_test_utils) + target_link_libraries(stk_middle_mesh_doc_tests stk_util_parallel) + target_link_libraries(stk_middle_mesh_doc_tests stk_unit_main) add_test(NAME "stk_middle_mesh_doc_tests" COMMAND stk_middle_mesh_doc_tests) endif() diff --git a/packages/stk/stk_doc_tests/stk_topology/element_topologies.cpp b/packages/stk/stk_doc_tests/stk_topology/element_topologies.cpp index 0e039487943f..849e9bb66b5a 100644 --- a/packages/stk/stk_doc_tests/stk_topology/element_topologies.cpp +++ b/packages/stk/stk_doc_tests/stk_topology/element_topologies.cpp @@ -108,7 +108,7 @@ void checkNodeOrderingAndOffsetsForFaces(const stk::topology &element, const uns element.face_node_ordinals(index, offsets); element.face_nodes(elementNodes, index, nodeIds); - checkForValidOffsets(numNodesPerFace, offsets, &expectedNodeOffsets[numNodesPerFace*index]); + checkForValidOffsets(numNodesPerFace, offsets, expectedNodeOffsets + numNodesPerFace * index); checkPermutedNodeIds(numNodesPerFace, offsets, nodeIds, elementNodes); } @@ -128,7 +128,7 @@ void checkNodeOrderingAndOffsetsForEdges(const stk::topology &element, const uns element.edge_node_ordinals(index, offsets.data()); element.edge_nodes(elementNodes, index, nodeIds.data()); - checkForValidOffsets(numNodesPerEdge, offsets.data(), &expectedNodeOffsets[numNodesPerEdge*index]); + checkForValidOffsets(numNodesPerEdge, offsets.data(), expectedNodeOffsets + numNodesPerEdge * index); checkPermutedNodeIds(numNodesPerEdge, offsets.data(), nodeIds.data(), elementNodes); } } @@ -144,7 +144,7 @@ void checkNodeOrderingAndOffsetsForPermutations(const stk::topology &element, co element.permutation_node_ordinals(index, offsets); element.permutation_nodes(elementNodes, index, nodeIds); - checkForValidOffsets(numNodes, offsets, &expectedNodeOffsets[numNodes*index]); + checkForValidOffsets(numNodes, offsets, expectedNodeOffsets + numNodes * index); checkPermutedNodeIds(numNodes, offsets, nodeIds, elementNodes); } diff --git a/packages/stk/stk_doc_tests/stk_transfer/howToUseReducedDependencyGeometricTransfer.cpp b/packages/stk/stk_doc_tests/stk_transfer/howToUseReducedDependencyGeometricTransfer.cpp index 81426607422d..4b30d2efb674 100644 --- a/packages/stk/stk_doc_tests/stk_transfer/howToUseReducedDependencyGeometricTransfer.cpp +++ b/packages/stk/stk_doc_tests/stk_transfer/howToUseReducedDependencyGeometricTransfer.cpp @@ -42,7 +42,6 @@ #include #include #include -#include #include #include #include diff --git a/packages/stk/stk_doc_tests/stk_util/AllReduceHowTo.cpp b/packages/stk/stk_doc_tests/stk_util/AllReduceHowTo.cpp index f6ebde4bccc1..011da20b34b8 100644 --- a/packages/stk/stk_doc_tests/stk_util/AllReduceHowTo.cpp +++ b/packages/stk/stk_doc_tests/stk_util/AllReduceHowTo.cpp @@ -35,12 +35,13 @@ #include "gtest/gtest.h" #include #include +#include //BEGINAllReduce TEST(AllReduce, combinedOps) { MPI_Comm comm = MPI_COMM_WORLD; - if (stk::parallel_machine_size(comm) != 2) { return; } + if (stk::parallel_machine_size(comm) != 2) { GTEST_SKIP(); } int myRank = stk::parallel_machine_rank(comm); diff --git a/packages/stk/stk_doc_tests/stk_util/CommSparseHowTo.cpp b/packages/stk/stk_doc_tests/stk_util/CommSparseHowTo.cpp index 455770fa90ee..c163dbe39471 100644 --- a/packages/stk/stk_doc_tests/stk_util/CommSparseHowTo.cpp +++ b/packages/stk/stk_doc_tests/stk_util/CommSparseHowTo.cpp @@ -41,7 +41,7 @@ #if defined ( STK_HAS_MPI ) //BEGINCommSparse -TEST(ParallelComm, HowToCommunicateOneValue) +TEST(ParallelComm, HowToCommunicateOneValue_PackAndCommunicate) { MPI_Comm comm = MPI_COMM_WORLD; stk::CommSparse commSparse(comm); @@ -51,31 +51,52 @@ TEST(ParallelComm, HowToCommunicateOneValue) double sendSomeNumber = 100-myProcId; - for(int phase = 0; phase < 2; ++phase) - { - for (int proc=0;proc(sendSomeNumber); } } - if(phase == 0) - { + }); + + for (int proc=0;proc(sendSomeNumber); + } + } + if(phase == 0) { commSparse.allocate_buffers(); } - else - { + else { commSparse.communicate(); } } - - for (int proc=0;proc(sendSomeNumber+i); } } } - if(phase == 0) - { - commSparse.allocate_buffers(); - } - else - { - commSparse.communicate(); - } - } - + }); - for (int procFromWhichDataIsReceived=0; procFromWhichDataIsReceived < numProcs; procFromWhichDataIsReceived++) - { - if ( procFromWhichDataIsReceived != myProcId ) - { - stk::CommBuffer& dataReceived = commSparse.recv_buffer(procFromWhichDataIsReceived); + for (int sourceProc=0; sourceProc < numProcs; sourceProc++) { + if ( sourceProc != myProcId ) { + stk::CommBuffer& dataReceived = commSparse.recv_buffer(sourceProc); int numItemsReceived = 0; - while ( dataReceived.remaining() ) - { + while ( dataReceived.remaining() ) { double val = -1; dataReceived.unpack(val); - EXPECT_EQ(100-procFromWhichDataIsReceived+numItemsReceived, val); + EXPECT_EQ(100-sourceProc+numItemsReceived, val); numItemsReceived++; } - int goldNumItemsReceived = procFromWhichDataIsReceived; + int goldNumItemsReceived = sourceProc; EXPECT_EQ(goldNumItemsReceived, numItemsReceived); } } diff --git a/packages/stk/stk_doc_tests/stk_util/VersionHowTo.cpp b/packages/stk/stk_doc_tests/stk_util/VersionHowTo.cpp new file mode 100644 index 000000000000..ed7064c0a8d8 --- /dev/null +++ b/packages/stk/stk_doc_tests/stk_util/VersionHowTo.cpp @@ -0,0 +1,50 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +//BEGINVersion +#include "gtest/gtest.h" +#include +#include +#include +#include + +TEST(stkHowTo, reportVersion) +{ + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 1) { GTEST_SKIP(); } + + const std::string stk_version = stk::version_string(); + std::cout << "This program is using STK version: " << stk_version << std::endl; +} +//ENDVersion + diff --git a/packages/stk/stk_emend/Jamfile b/packages/stk/stk_emend/Jamfile index 50c98caa6f67..fb9511103633 100644 --- a/packages/stk/stk_emend/Jamfile +++ b/packages/stk/stk_emend/Jamfile @@ -170,7 +170,7 @@ exe stk_emend_independent_set_utest [ glob $(stk_emend-root)/stk_emend/independent_set/unit_tests/*.cpp ] stk_independent_set_lib /sierra/stk_unit_test_utils//stk_unit_main - /tpl/gtest//gtest + /tpl/googletest//gtest : @sierra-exec-tag $(stk_emend-root)/stk_emend/independent_set/unit_tests [ ifdevbuild diff --git a/packages/stk/stk_integration_tests/cmake_install_test/README_spack_cuda_recipe b/packages/stk/stk_integration_tests/cmake_install_test/README_spack_cuda_recipe new file mode 100644 index 000000000000..2b59a81efa7e --- /dev/null +++ b/packages/stk/stk_integration_tests/cmake_install_test/README_spack_cuda_recipe @@ -0,0 +1,52 @@ + +# recipe for creating a spack env and building trilinos/stk, +# then building a stk-test-app using the spack-installed trilinos/stk +# +# Note: this is on ascicgpu057 as of March 27, 2024 +# Note2: edit user-specific paths below + +mkdir -p /fgs/william/sandbox_spack +cd /fgs/william/sandbox_spack + +git clone --depth=100 --branch=releases/v0.21 https://github.com/spack/spack.git +source ./spack/share/spack/setup-env.sh + +spack env create mystkcudaenv +spack env activate mystkcudaenv +module load aue/gcc/10.3.0 +spack compiler add +#optionally remove old compilers that spack found +spack compiler remove gcc@4.8.5 +spack compiler remove gcc@4.4.7 + +spack add hdf5@1.14.3~shared +spack add zlib +spack add openmpi@4.1.6 +spack add kokkos@4.1.00+cuda+wrapper+cuda_constexpr+cuda_lambda+cuda_relocatable_device_code~shared cuda_arch=70 +spack add trilinos@master+cuda+cuda_rdc+exodus+stk+kokkos+wrapper~shared~boost cuda_arch=70 cxxstd=17 + +spack external find +# if spack didn't find openmpi@4.1.6, add these lines in `spack config edit`: +# openmpi: +# externals: +# - spec: openmpi@4.1.6 +# modules: +# - aue/openmpi/4.1.6-gcc-10.3.0 +# buildable: false + +spack concretize -f +spack install + +spack load cmake +spack load openmpi + +# this is necessary on our ascicgpu057 machine, perhaps not in general +export OMPI_CXX=$(find $(spack location -i kokkos) -name nvcc_wrapper) + +cp -r /fgs/william/code/stk/stk_integration_tests/cmake_install_test/stk_test_app . +cd stk_test_app +source run_cmake_in_spack_cuda_env +#note: the run_cmake_in_spack_cuda_env script also did 'cd build' so you're now in build subdir +make +mpirun --np 4 ./test_stk_app + diff --git a/packages/stk/stk_integration_tests/cmake_install_test/README_spack_recipe b/packages/stk/stk_integration_tests/cmake_install_test/README_spack_recipe new file mode 100644 index 000000000000..66069fe90e48 --- /dev/null +++ b/packages/stk/stk_integration_tests/cmake_install_test/README_spack_recipe @@ -0,0 +1,34 @@ + +# recipe for creating a spack env and building trilinos/stk, +# then building a stk-test-app using the spack-installed trilinos/stk + +mkdir -p /fgs/william/sandbox_spack +cd /fgs/william/sandbox_spack + +git clone --depth=100 --branch=releases/v0.21 https://github.com/spack/spack.git +source ./spack/share/spack/setup-env.sh + +spack env create mystkenv +spack env activate mystkenv +#want to add gcc 10.2.0 compiler +module load sierra-devel +spack compiler add +#optionally remove old compilers that spack found +spack compiler remove gcc@4.8.5 +spack compiler remove gcc@4.4.7 + +spack add trilinos@master +exodus+zoltan2+stk + +spack concretize -f +spack install + +spack load cmake +spack load openmpi + +cp -r /fgs/william/code/stk/stk_integration_tests/cmake_install_test/stk_test_app . +cd stk_test_app +source run_cmake_in_spack_env +#note: the run_cmake_in_spack_env script also did 'cd build' so you're now in build subdir +make +mpirun --np 4 ./test_stk_app + diff --git a/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk b/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk index 35b276b30a43..a8e9f0f64fb5 100755 --- a/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk +++ b/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk @@ -89,6 +89,7 @@ cmake \ -DCDT_ROOT="/fgs/william/CDT/install" \ -DTrilinos_ENABLE_SEACASExodus:BOOL=ON \ -DTrilinos_ENABLE_SEACASIoss:BOOL=ON \ +-DTrilinos_ENABLE_SEACASNemesis:BOOL=ON \ -DTPL_ENABLE_Netcdf:BOOL=ON \ -DTPL_Netcdf_Enables_Netcdf4:BOOL=ON \ -DTPL_Netcdf_Enables_Pnetcdf:BOOL=ON \ diff --git a/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_standalone b/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_standalone index a44519f43126..c5c979736a44 100755 --- a/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_standalone +++ b/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_standalone @@ -43,7 +43,7 @@ cmake \ -DSEACAS_DIR=/fgs/william/Trilinos/install_gcc/lib/cmake/SEACAS \ -DShards_DIR=/fgs/william/Trilinos/install_gcc/lib/cmake/Shards \ -DZoltan2Core_DIR=/fgs/william/Trilinos/install_gcc/lib/cmake/Zoltan2Core \ --DKokkos_DIR=/fgs/william/kokkos/install/lib64/cmake/Kokkos \ +-DKokkos_DIR=/fgs/william/Trilinos/install_gcc/lib/cmake/Kokkos \ -DKokkosKernels_DIR=/fgs/william/kokkos-kernels/install/lib64/cmake/KokkosKernels \ -DGTest_DIR=/fgs/william/googletest/install/lib64/cmake/GTest \ ${stk_src_dir}/ diff --git a/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_standalone_serial b/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_standalone_serial index 482346cd0033..f1c0b827b79a 100755 --- a/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_standalone_serial +++ b/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_standalone_serial @@ -37,6 +37,7 @@ cmake \ -DSTK_ENABLE_ALL=ON \ -DSTK_ENABLE_MPI:BOOL=OFF \ -DSTK_ENABLE_STKMiddle_mesh=OFF \ +-DSTK_ENABLE_STKMiddle_mesh_util=OFF \ -DSTK_ENABLE_TESTS:BOOL=ON \ -DKokkos_DIR=/fgs/william/kokkos/install/lib64/cmake/Kokkos \ -DGTest_DIR=/fgs/william/googletest/install/lib64/cmake/GTest \ diff --git a/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/CMakeLists.txt b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/CMakeLists.txt index 0610bbdf28a4..d20d4dfc2c93 100644 --- a/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/CMakeLists.txt +++ b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/CMakeLists.txt @@ -9,7 +9,7 @@ option(ENABLE_ALL_WARNINGS "Show most warnings for most compilers" ON) option(ENABLE_WERROR "Warnings are errors" ON) option(ENABLE_OPENMP "Enable OpenMP flags" OFF) -set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_CXX_STANDARD_REQUIRED) @@ -19,8 +19,14 @@ add_library(test_stk_lib "") add_executable(${exe_name} ${CMAKE_CURRENT_SOURCE_DIR}/test_stk_app.cpp) ########################## MPI #################################### -find_package(MPI REQUIRED) -target_link_libraries(test_stk_lib PUBLIC $<$:MPI::MPI_CXX>) +find_package(MPI) +if(MPI_FOUND) + include_directories(SYSTEM ${MPI_INCLUDE_PATH}) + target_link_libraries(test_stk_lib PUBLIC ${MPI_LIBRARIES}) + message("found MPI, MPI_INCLUDE_PATH: ${MPI_INCLUDE_PATH}") +else() + message(FATAL_ERROR "MPI not found") +endif() if(ENABLE_CUDA) find_package(CUDA REQUIRED) diff --git a/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/run_cmake_in_spack_cuda_env b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/run_cmake_in_spack_cuda_env new file mode 100755 index 000000000000..4e639c6bffa1 --- /dev/null +++ b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/run_cmake_in_spack_cuda_env @@ -0,0 +1,15 @@ +spack env status + +spack find -v trilinos + +TEST_STK_APP_SOURCE_DIR=$(pwd) + +mkdir -p build + +cd build + +cmake \ +-DCMAKE_BUILD_TYPE=${BUILD_TYPE:-RELEASE} \ +-DCMAKE_CXX_COMPILER=${OMPI_CXX} \ +${TEST_STK_APP_SOURCE_DIR} + diff --git a/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/run_cmake_in_spack_env b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/run_cmake_in_spack_env new file mode 100755 index 000000000000..c42f7adf585a --- /dev/null +++ b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/run_cmake_in_spack_env @@ -0,0 +1,15 @@ +spack env status + +spack find -v trilinos + +TEST_STK_APP_SOURCE_DIR=$(pwd) + +mkdir -p build + +cd build + +cmake \ +-DCMAKE_BUILD_TYPE=${BUILD_TYPE:-RELEASE} \ +-DCMAKE_CXX_COMPILER=mpicxx \ +${TEST_STK_APP_SOURCE_DIR} + diff --git a/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_coupling.cpp b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_coupling.cpp index fff03436d4fe..a543a5c2968a 100644 --- a/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_coupling.cpp +++ b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_coupling.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -10,8 +11,10 @@ namespace test_stk_lib { void test_stk_coupling() { - std::cout << "stk_coupling installation test, STK version: " << stk::version_string() - << ", Coupling-Version: "<< stk::util::get_local_max_coupling_version() << std::endl; + if (stk::parallel_machine_rank(MPI_COMM_WORLD) == 0) { + std::cout << "stk_coupling installation test, STK version: " << stk::version_string() + << ", Coupling-Version: "<< stk::util::get_local_max_coupling_version() << std::endl; + } } } diff --git a/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_io.cpp b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_io.cpp index 98e86bbe8977..34405c7281ef 100644 --- a/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_io.cpp +++ b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_io.cpp @@ -15,6 +15,7 @@ void test_stk_io(stk::ParallelMachine comm, const std::string& meshSource, bool { std::shared_ptr bulk = stk::mesh::MeshBuilder(comm).create(); stk::mesh::MetaData& meta = bulk->mesh_meta_data(); + meta.use_simple_fields(); if (bulk->parallel_rank() == 0) { std::cout << "test_stk_io: meshSource="< +#include #include #include @@ -9,7 +10,9 @@ namespace test_stk_lib { void test_stk_search() { - std::cout << "stk_search installation test, SearchMethod: " << stk::search::SearchMethod::KDTREE << std::endl; + if (stk::parallel_machine_rank(MPI_COMM_WORLD) == 0) { + std::cout << "stk_search installation test, SearchMethod: " << stk::search::SearchMethod::KDTREE << std::endl; + } } } diff --git a/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_tools.cpp b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_tools.cpp index ad6ffb8ca3e5..8fc096035f82 100644 --- a/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_tools.cpp +++ b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_tools.cpp @@ -2,6 +2,7 @@ #include +#include #include #include #include @@ -11,7 +12,9 @@ namespace test_stk_lib { void test_stk_tools() { - std::cout << "The stk_tools test says 'Hello, World!'" << std::endl; + if (stk::parallel_machine_rank(MPI_COMM_WORLD) == 0) { + std::cout << "The stk_tools test says 'Hello, World!'" << std::endl; + } } } diff --git a/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/test_stk_app.cpp b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/test_stk_app.cpp index 3627c32ba920..7322afad6234 100644 --- a/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/test_stk_app.cpp +++ b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/test_stk_app.cpp @@ -1,8 +1,11 @@ +#include +#ifdef STK_HAVE_KOKKOS +#include +#endif #include #include #include -#include #include "src/test_stk_coupling.hpp" #include "src/test_stk_search.hpp" @@ -12,10 +15,14 @@ int main(int argc, char** argv) { - if (MPI_SUCCESS != MPI_Init(&argc, &argv)) { - std::cout << "MPI_Init failed." << std::endl; - return -1; + stk::parallel_machine_init(&argc, &argv); + +#ifdef STK_HAVE_KOKKOS + Kokkos::initialize(argc, argv); + if (stk::parallel_machine_rank(MPI_COMM_WORLD) == 0) { + std::cout << "Kokkos::DefaultExecutionSpace: " << Kokkos::DefaultExecutionSpace::device_type::execution_space::name() << std::endl; } +#endif const bool proc0 = (stk::parallel_machine_rank(MPI_COMM_WORLD) == 0); @@ -50,7 +57,11 @@ int main(int argc, char** argv) test_stk_lib::test_stk_tools(); - MPI_Finalize(); +#ifdef STK_HAVE_KOKKOS + Kokkos::finalize(); +#endif + + stk::parallel_machine_finalize(); if (proc0) { std::cout << "... exiting." << std::endl; diff --git a/packages/stk/stk_integration_tests/stk_balance/IntegrationTestLoadBalance.cpp b/packages/stk/stk_integration_tests/stk_balance/IntegrationTestLoadBalance.cpp index 47e03e581acc..ee3ee0c192d7 100644 --- a/packages/stk/stk_integration_tests/stk_balance/IntegrationTestLoadBalance.cpp +++ b/packages/stk/stk_integration_tests/stk_balance/IntegrationTestLoadBalance.cpp @@ -920,9 +920,12 @@ void gatherLoadBalanceDiagnostics(const std::vector &vertexWeights, cons int numElementsThisProc = vertexWeights.size(); diagnostics.numElementsPerProc.clear(); diagnostics.numElementsPerProc.resize(numProcs, 0); - MPI_Allgatherv(&totalVertexWeight, 1, MPI_DOUBLE, &diagnostics.sumOfVertexWeightsPerProc[0], &counts[0], &displs[0], MPI_DOUBLE, communicator); - MPI_Allgatherv(&totalEdgeWeight, 1, MPI_DOUBLE, &diagnostics.sumOfCutEdgeWeightsPerProc[0], &counts[0], &displs[0], MPI_DOUBLE, communicator); - MPI_Allgatherv(&numElementsThisProc, 1, MPI_INT, &diagnostics.numElementsPerProc[0], &counts[0], &displs[0], MPI_INT, communicator); + MPI_Allgatherv(&totalVertexWeight, 1, MPI_DOUBLE, diagnostics.sumOfVertexWeightsPerProc.data(), counts.data(), + displs.data(), MPI_DOUBLE, communicator); + MPI_Allgatherv(&totalEdgeWeight, 1, MPI_DOUBLE, diagnostics.sumOfCutEdgeWeightsPerProc.data(), counts.data(), + displs.data(), MPI_DOUBLE, communicator); + MPI_Allgatherv(&numElementsThisProc, 1, MPI_INT, diagnostics.numElementsPerProc.data(), counts.data(), displs.data(), + MPI_INT, communicator); } void printLoadBalanceDiagnostics(const struct LoadBalanceDiagnostics &loadBalanceDiagnostics) diff --git a/packages/stk/stk_integration_tests/stk_mesh/IntegrationTestElementBlockMembership.cpp b/packages/stk/stk_integration_tests/stk_mesh/IntegrationTestElementBlockMembership.cpp index 2602b29dc170..fad82aff880b 100644 --- a/packages/stk/stk_integration_tests/stk_mesh/IntegrationTestElementBlockMembership.cpp +++ b/packages/stk/stk_integration_tests/stk_mesh/IntegrationTestElementBlockMembership.cpp @@ -8,7 +8,7 @@ /*--------------------------------------------------------------------*/ #include // for AssertHelper, EXPECT_EQ, etc -#include // for Initializer +#include // for Initializer #include // for size_t, nullptr #include // for string #include "mpi.h" // for MPI_COMM_WORLD diff --git a/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestCheckExposedBoundary.cpp b/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestCheckExposedBoundary.cpp index d4ff45bd0169..c8570de9b845 100644 --- a/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestCheckExposedBoundary.cpp +++ b/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestCheckExposedBoundary.cpp @@ -8,7 +8,7 @@ /*--------------------------------------------------------------------*/ #include // for AssertHelper, EXPECT_EQ, etc -#include // for Initializer +#include // for Initializer #include // for size_t, nullptr #include // for StkMeshIoBroker #include // for string diff --git a/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestCreateAllSides.cpp b/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestCreateAllSides.cpp index c2ac7b56b9e9..4a593b2a7c40 100644 --- a/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestCreateAllSides.cpp +++ b/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestCreateAllSides.cpp @@ -8,7 +8,7 @@ /*--------------------------------------------------------------------*/ #include // for AssertHelper, EXPECT_EQ, etc -#include // for Initializer +#include // for Initializer #include // for size_t, nullptr #include // for StkMeshIoBroker #include // for string diff --git a/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestMeshChecker.cpp b/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestMeshChecker.cpp index efac60447ce8..b9d4380ae77b 100644 --- a/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestMeshChecker.cpp +++ b/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestMeshChecker.cpp @@ -9,7 +9,6 @@ #include // for AssertHelper, EXPECT_EQ, etc #include // for size_t, nullptr -#include // for string #include #include #include // for MeshTestFixture diff --git a/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestSkinExteriorBoundary.cpp b/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestSkinExteriorBoundary.cpp index e2cb2984846d..941917bd51f7 100644 --- a/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestSkinExteriorBoundary.cpp +++ b/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestSkinExteriorBoundary.cpp @@ -8,7 +8,7 @@ /*--------------------------------------------------------------------*/ #include // for AssertHelper, EXPECT_EQ, etc -#include // for Initializer +#include // for Initializer #include // for size_t, nullptr #include // for StkMeshIoBroker #include // for string diff --git a/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestSkinInteriorBoundary.cpp b/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestSkinInteriorBoundary.cpp index 664d162fe95c..95fe5e4cdcec 100644 --- a/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestSkinInteriorBoundary.cpp +++ b/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestSkinInteriorBoundary.cpp @@ -8,7 +8,7 @@ /*--------------------------------------------------------------------*/ #include // for AssertHelper, EXPECT_EQ, etc -#include // for Initializer +#include // for Initializer #include // for size_t, nullptr #include // for StkMeshIoBroker #include // for string diff --git a/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestSkinWithModifications.cpp b/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestSkinWithModifications.cpp index e4fe8a00b2ef..620792d38c17 100644 --- a/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestSkinWithModifications.cpp +++ b/packages/stk/stk_integration_tests/stk_mesh/face_creation/skin_mesh/IntegrationTestSkinWithModifications.cpp @@ -15,7 +15,7 @@ #include #include #include // for string -#include // for Initializer +#include // for Initializer #include "stk_io/DatabasePurpose.hpp" #include // for StkMeshIoBroker #include diff --git a/packages/stk/stk_integration_tests/stk_mesh/face_creation/user_created/IntegrationTestUserCreated.cpp b/packages/stk/stk_integration_tests/stk_mesh/face_creation/user_created/IntegrationTestUserCreated.cpp index c21e4cc6377e..37d5d426b430 100644 --- a/packages/stk/stk_integration_tests/stk_mesh/face_creation/user_created/IntegrationTestUserCreated.cpp +++ b/packages/stk/stk_integration_tests/stk_mesh/face_creation/user_created/IntegrationTestUserCreated.cpp @@ -8,7 +8,7 @@ /*--------------------------------------------------------------------*/ #include // for AssertHelper, EXPECT_EQ, etc -#include // for Initializer +#include // for Initializer #include // for size_t, nullptr #include // for StkMeshIoBroker #include // for string diff --git a/packages/stk/stk_integration_tests/stk_search/CMakeLists.txt b/packages/stk/stk_integration_tests/stk_search/CMakeLists.txt index 9a488692ee5b..e032acba7758 100644 --- a/packages/stk/stk_integration_tests/stk_search/CMakeLists.txt +++ b/packages/stk/stk_integration_tests/stk_search/CMakeLists.txt @@ -36,7 +36,6 @@ FILE(GLOB SOURCES *.cpp) #removing due to dependence on geometry toolkit which is not available to Trilinos -LIST(REMOVE_ITEM SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/UnitTestPerformance.cpp) LIST(REMOVE_ITEM SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/UnitTestNaluPerformance.cpp) TRIBITS_ADD_EXECUTABLE( diff --git a/packages/stk/stk_integration_tests/stk_search/UnitTestPerformance.cpp b/packages/stk/stk_integration_tests/stk_search/UnitTestPerformance.cpp deleted file mode 100644 index a210d7bcbf4a..000000000000 --- a/packages/stk/stk_integration_tests/stk_search/UnitTestPerformance.cpp +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering -// Solutions of Sandia, LLC (NTESS). Under the terms of Contract -// DE-NA0003525 with NTESS, the U.S. Government retains certain rights -// in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following -// disclaimer in the documentation and/or other materials provided -// with the distribution. -// -// * Neither the name of NTESS nor the names of its contributors -// may be used to endorse or promote products derived from this -// software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// - -#include -#include -#include - -#include -#include - -#include // for ParallelMachine, etc -#include - -namespace -{ - -void runStkSearchTestUsingStkAABoxes(stk::search::SearchMethod search); -void runStkSearchTestUsingFloatAABoxes(stk::search::SearchMethod search); -void testPerformanceOfAxisAlignedBoundingBoxes(stk::search::SearchMethod searchMethod, MPI_Comm comm); -void testStkSearchUsingStkAABoxes(MPI_Comm comm, std::vector &domainBoxes, - stk::search::SearchMethod searchMethod, SearchResults boxIdPairResults); -void testStkSearchUsingFloatAABoxes(MPI_Comm comm, std::vector &domainBoxes, - stk::search::SearchMethod searchMethod, SearchResults boxIdPairResults); - -TEST(Performance, ofAxisAlignedBoundingBoxesUsingKdtree) -{ - MPI_Comm comm = MPI_COMM_WORLD; - stk::search::SearchMethod searchMethod = stk::search::KDTREE; - testPerformanceOfAxisAlignedBoundingBoxes(searchMethod, comm); -} - -void testPerformanceOfAxisAlignedBoundingBoxes(stk::search::SearchMethod searchMethod, MPI_Comm comm) -{ - int proc = stk::parallel_machine_rank(comm); - int numProcs = stk::parallel_machine_size(comm); - - size_t numColumnsPerProcessor = 1000; - double boxSize = 1.0; - - StkBoxVector smallBoxVector; - StkBoxVector bigBoxVector; - SearchResults boxIdPairResults; - - if(proc % 2 == 0) - { - double startX = numColumnsPerProcessor * boxSize * proc / 2; - - for(size_t x = 0; x < numColumnsPerProcessor; x++) - { - for(size_t y = 0; y < numColumnsPerProcessor; y++) - { - double radius = boxSize / 2; - double centerX = startX + x * boxSize + radius; - double centerY = y * boxSize + radius; - double centerZ = radius; - - int id = x * numColumnsPerProcessor + y; - - smallBoxVector.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(centerX, - centerY, - centerZ, - radius, - id, - proc)); - } - } - } - else - { - double radius = numColumnsPerProcessor * boxSize / 2; - double startX = numColumnsPerProcessor * boxSize * (proc - 1) / 2; - double centerX = startX + radius; - double centerY = radius; - double centerZ = boxSize / 2; - int id = 1; - bigBoxVector.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(centerX, - centerY, - centerZ, - radius - boxSize / 2, - id, - proc)); - } - - double startTime = stk::wall_time(); - - stk::search::coarse_search(smallBoxVector, bigBoxVector, searchMethod, comm, boxIdPairResults); - - double elapsedTime = stk::wall_time() - startTime; - stk::unit_test_util::simple_fields::printPeformanceStats(elapsedTime, comm); - - size_t numExpectedResults = numColumnsPerProcessor * numColumnsPerProcessor; - bool lastProcessorWithOddNumberOfProcs = numProcs % 2 != 0 && proc == numProcs - 1; - if(lastProcessorWithOddNumberOfProcs) - { - numExpectedResults = 0; - } - - EXPECT_EQ(numExpectedResults, boxIdPairResults.size()); -} - -//////////////////////////////////////////////////////////// - -TEST(Performance, stkSearchUsingKdtreeUsingStkAABoxes) -{ - runStkSearchTestUsingStkAABoxes(stk::search::KDTREE); -} - -TEST(Performance, stkSearchUsingKdtreeUsingFloatAABoxes) -{ - runStkSearchTestUsingFloatAABoxes(stk::search::KDTREE); -} - -void runStkSearchTestUsingFloatAABoxes(stk::search::SearchMethod searchMethod) -{ - MPI_Comm comm = MPI_COMM_WORLD; - std::vector domainBoxes( fillDomainBoxes(comm) ); - - SearchResults boxIdPairResults; - testStkSearchUsingFloatAABoxes(comm, domainBoxes, searchMethod, boxIdPairResults); -} - -void runStkSearchTestUsingStkAABoxes(stk::search::SearchMethod searchMethod) -{ - MPI_Comm comm = MPI_COMM_WORLD; - std::vector domainBoxes( fillDomainBoxes(comm) ); - - SearchResults boxIdPairResults; - testStkSearchUsingStkAABoxes(comm, domainBoxes, searchMethod, boxIdPairResults); -} - -void testStkSearchUsingStkAABoxes(MPI_Comm comm, std::vector &domainBoxes, stk::search::SearchMethod searchMethod, SearchResults boxIdPairResults) -{ - int procId = stk::parallel_machine_rank(comm); - - StkBoxVector stkBoxes(domainBoxes.size()); - fillStkBoxesUsingFloatBoxes(domainBoxes, procId, stkBoxes); - - std::string rangeBoxComm = stk::unit_test_util::simple_fields::get_option("-rangeBoxComm", "yes"); - bool rangeResultsCommunicated = ( rangeBoxComm == "yes" ); - - double startTime = stk::wall_time(); - stk::search::coarse_search(stkBoxes, stkBoxes, searchMethod, comm, boxIdPairResults, rangeResultsCommunicated); - double elapsedTime = stk::wall_time() - startTime; - - stk::unit_test_util::simple_fields::printPeformanceStats(elapsedTime, comm); - - stk::unit_test_util::simple_fields::gatherResultstoProcZero(comm, boxIdPairResults); - size_t goldValueNumber = stk::unit_test_util::simple_fields::getGoldValueForTest(); - if ( procId == 0 ) - { - if ( goldValueNumber != 0u) - { - EXPECT_EQ(goldValueNumber, boxIdPairResults.size()); - } - std::cerr << "Number of interactions: " << boxIdPairResults.size() << std::endl; - } -} - -void testStkSearchUsingFloatAABoxes(MPI_Comm comm, std::vector &domainBoxes, - stk::search::SearchMethod searchMethod, SearchResults boxIdPairResults) -{ - int procId = stk::parallel_machine_rank(comm); - - FloatBoxVector searchBoxPairs(domainBoxes.size()); - for(size_t i = 0; i < domainBoxes.size(); i++) - { - Ident domainBoxId(i, procId); - searchBoxPairs[i] = std::make_pair(domainBoxes[i], domainBoxId); - } - - std::string rangeBoxComm = stk::unit_test_util::simple_fields::get_option("-rangeBoxComm", "yes"); - bool rangeResultsCommunicated = ( rangeBoxComm == "yes" ); - - double startTime = stk::wall_time(); - stk::search::coarse_search(searchBoxPairs, searchBoxPairs, searchMethod, comm, boxIdPairResults, rangeResultsCommunicated); - double elapsedTime = stk::wall_time() - startTime; - - stk::unit_test_util::simple_fields::printPeformanceStats(elapsedTime, comm); - - stk::unit_test_util::simple_fields::gatherResultstoProcZero(comm, boxIdPairResults); - size_t goldValueNumber = stk::unit_test_util::simple_fields::getGoldValueForTest(); - if ( procId == 0 ) - { - if ( goldValueNumber != 0u) - { - EXPECT_EQ(goldValueNumber, boxIdPairResults.size()); - } - std::cerr << "Number of interactions: " << boxIdPairResults.size() << std::endl; - } -} - -} diff --git a/packages/stk/stk_io/example/io_mesh.cpp b/packages/stk/stk_io/example/io_mesh.cpp index da8d22bc9564..aeab33f8de50 100644 --- a/packages/stk/stk_io/example/io_mesh.cpp +++ b/packages/stk/stk_io/example/io_mesh.cpp @@ -51,7 +51,6 @@ #include // for EnvData #include // for log_w... #include // for get_c... -#include #include // for paral... #include // for Reduc... #include // for Param... diff --git a/packages/stk/stk_io/stk_io/CMakeLists.txt b/packages/stk/stk_io/stk_io/CMakeLists.txt index e82ad839ddd4..b8aaed48fe06 100644 --- a/packages/stk/stk_io/stk_io/CMakeLists.txt +++ b/packages/stk/stk_io/stk_io/CMakeLists.txt @@ -45,16 +45,19 @@ if(HAVE_STK_Trilinos) SOURCES ${SOURCES} ) else() - find_package(SEACAS REQUIRED) - add_library(stk_io ${SOURCES}) - target_link_libraries(stk_io PUBLIC ${SEACAS_LIBRARIES}) - target_link_libraries(stk_io PUBLIC stk_mesh_base) + if(STK_HAS_SEACAS_IOSS) + add_library(stk_io ${SOURCES}) + target_link_libraries(stk_io PUBLIC ${SEACAS_LIBRARIES}) + target_link_libraries(stk_io PUBLIC stk_mesh_base) + endif() endif() -target_include_directories(stk_io PUBLIC - $ - $ -) +if(STK_HAS_SEACAS_IOSS) + target_include_directories(stk_io PUBLIC + $ + $ + ) +endif() INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_io/) diff --git a/packages/stk/stk_io/stk_io/Heartbeat.cpp b/packages/stk/stk_io/stk_io/Heartbeat.cpp index 6e4c873fc7ba..08acf6e40241 100644 --- a/packages/stk/stk_io/stk_io/Heartbeat.cpp +++ b/packages/stk/stk_io/stk_io/Heartbeat.cpp @@ -256,8 +256,10 @@ void impl::Heartbeat::process_output_pre_write(int step, double time) if(currentState == Ioss::STATE_DEFINE_TRANSIENT) { m_region->end_mode(Ioss::STATE_DEFINE_TRANSIENT); } + if (currentState != Ioss::STATE_TRANSIENT) { + m_region->begin_mode(Ioss::STATE_TRANSIENT); + } - m_region->begin_mode(Ioss::STATE_TRANSIENT); m_currentStep = m_region->add_state(time); m_region->begin_state(m_currentStep); } @@ -274,7 +276,6 @@ void impl::Heartbeat::process_output_post_write(int step, double time) { if (m_processor == 0) { m_region->end_state(m_currentStep); - m_region->end_mode(Ioss::STATE_TRANSIENT); } } diff --git a/packages/stk/stk_io/stk_io/IOHelpers.cpp b/packages/stk/stk_io/stk_io/IOHelpers.cpp index 5af6ebf2fad1..f0a58a5b2efa 100644 --- a/packages/stk/stk_io/stk_io/IOHelpers.cpp +++ b/packages/stk/stk_io/stk_io/IOHelpers.cpp @@ -214,7 +214,6 @@ void process_surface_entity_df(const Ioss::SideSet* sset, stk::mesh::BulkData & // is not split into homogenous side_blocks, then the topology will not necessarily // be the same and this could fail (a sideset of mixed edges and faces) int par_dimen = block->topology()->parametric_dimension(); - STKIORequire(par_dimen == 1 || par_dimen == 2); stk::mesh::EntityRank side_rank = par_dimen == 1 ? stk::topology::EDGE_RANK : stk::topology::FACE_RANK; @@ -229,18 +228,43 @@ void process_surface_entity_df(const Ioss::SideSet* sset, stk::mesh::BulkData & block->get_field_data("element_side", elemSidePairs); size_t side_count = elemSidePairs.size()/2; + // NOTE: Needed for shell side topology offset translation to IOSS + std::int64_t sideSetOffset = Ioss::Utils::get_side_offset(block); + std::vector sides; sides.reserve(side_count); for (size_t is = 0; is < side_count; ++is) { - stk::mesh::Entity side = stk::mesh::get_side_entity_for_elem_id_side_pair_of_rank(bulk, elemSidePairs[is*2], elemSidePairs[is*2+1]-1, side_rank); + INT elemId = elemSidePairs[is*2]; + INT elemSide = elemSidePairs[is*2+1] + sideSetOffset - 1; + stk::mesh::Entity const elem = bulk.get_entity(stk::topology::ELEMENT_RANK, elemId); + if (!bulk.is_valid(elem)) { continue; } + stk::topology elemTopo = bulk.bucket(elem).topology(); + if(par_dimen == 0) + { + stk::topology faceTopo = elemTopo.sub_topology(elemTopo.side_rank(), elemSide); + + Ioss::ElementTopology *ioss_topo = Ioss::ElementTopology::factory(faceTopo.name(), false); + par_dimen = ioss_topo->parametric_dimension(); + side_rank = par_dimen == 1 ? stk::topology::EDGE_RANK : stk::topology::FACE_RANK; + } + STKIORequire(par_dimen == 1 || par_dimen == 2); + + if (par_dimen == 1) { + // conversion from face ordinal to edge ordinal for shells + if (elemTopo.is_shell()) { + elemSide -= elemTopo.num_faces(); + } + } + stk::mesh::Entity side = stk::mesh::get_side_entity_for_elem_id_side_pair_of_rank(bulk, elemId, elemSide, side_rank); + if (bulk.is_valid(side)) { sides.push_back(side); } else { std::ostringstream os; os<<"P"<name() <<" does not have a corresponding face entity in the current mesh database" <<" (possibly referenced by a reduced restart file)."< & restrictions = f->restrictions(); if (restrictions.size() > 0 && f->entity_rank() == stk::topology::NODE_RANK) { - res = &restrictions[0]; + res = restrictions.data(); } if(res != nullptr) { diff --git a/packages/stk/stk_io/stk_io/IossBridge.hpp b/packages/stk/stk_io/stk_io/IossBridge.hpp index e038d9b12cd1..fa43af3aa0f2 100644 --- a/packages/stk/stk_io/stk_io/IossBridge.hpp +++ b/packages/stk/stk_io/stk_io/IossBridge.hpp @@ -58,6 +58,7 @@ #include "stk_topology/topology.hpp" // for topology #include "stk_util/util/ParameterList.hpp" // for Type #include "stk_util/util/ReportHandler.hpp" // for ThrowRequireMsg +#include "Ioss_SideBlock.h" // for SideBlock namespace Ioss { class DatabaseIO; } namespace Ioss { class ElementTopology; } namespace Ioss { class EntityBlock; } @@ -76,7 +77,6 @@ namespace stk { namespace mesh { class Part; } } namespace Ioss { class SideSet; -class SideBlock; class NodeBlock; class Field; class GroupingEntity; @@ -660,7 +660,9 @@ void fill_data_for_side_block( OutputParams ¶ms, parentElementBlock = get_parent_element_block(params.bulk_data(), params.io_region(), io.name()); } - fill_element_and_side_ids(params, part, parentElementBlock, stk_elem_topology, sides, elem_side_ids); + // An offset required to translate Ioss's interpretation of shell ordinals + INT sideOrdOffset = (io.type() == Ioss::SIDEBLOCK) ? Ioss::Utils::get_side_offset(dynamic_cast(&io)) : 0; + fill_element_and_side_ids(params, part, parentElementBlock, stk_elem_topology, sides, elem_side_ids, sideOrdOffset); } namespace impl { diff --git a/packages/stk/stk_io/stk_io/ProcessSetsOrBlocks.cpp b/packages/stk/stk_io/stk_io/ProcessSetsOrBlocks.cpp index 5ca45f026bb9..597e32274587 100644 --- a/packages/stk/stk_io/stk_io/ProcessSetsOrBlocks.cpp +++ b/packages/stk/stk_io/stk_io/ProcessSetsOrBlocks.cpp @@ -310,6 +310,8 @@ void process_surface_entity(const Ioss::SideSet* sset, stk::mesh::BulkData & bul // is not split into homogenous side_blocks, then the topology will not necessarily // be the same and this could fail (a sideset of mixed edges and faces) int par_dimen = block->topology()->parametric_dimension(); + // NOTE: Needed for shell side topology offset translation to IOSS + std::int64_t sideSetOffset = Ioss::Utils::get_side_offset(block); size_t side_count = elem_side.size() / 2; for(size_t is=0; is #include "stk_mesh/base/MetaData.hpp" diff --git a/packages/stk/stk_io/stk_io/SidesetTranslator.hpp b/packages/stk/stk_io/stk_io/SidesetTranslator.hpp index ac6e6e56e8c7..c0a635bd65c2 100644 --- a/packages/stk/stk_io/stk_io/SidesetTranslator.hpp +++ b/packages/stk/stk_io/stk_io/SidesetTranslator.hpp @@ -53,7 +53,8 @@ void fill_element_and_side_ids_from_sideset(const stk::mesh::SideSet& sset, const stk::mesh::Part *parentElementBlock, stk::topology stk_element_topology, stk::mesh::EntityVector &sides, - std::vector& elem_side_ids) + std::vector& elem_side_ids, + INT sideOrdOffset = 0) { const mesh::BulkData &bulk_data = params.bulk_data(); const stk::mesh::Selector *elemSubsetSelector = params.get_subset_selector(); @@ -104,7 +105,7 @@ void fill_element_and_side_ids_from_sideset(const stk::mesh::SideSet& sset, if (selectedByBucket && selectedByParent && selectedByOutput) { elem_side_ids.push_back(elemId); - elem_side_ids.push_back(zero_based_side_ord+1); + elem_side_ids.push_back(zero_based_side_ord - sideOrdOffset + 1); sides.push_back(side); } } @@ -144,7 +145,8 @@ void fill_element_and_side_ids_from_connectivity(stk::io::OutputParams ¶ms, const stk::mesh::Part *parentElementBlock, stk::topology stk_element_topology, stk::mesh::EntityVector &sides, - std::vector& elem_side_ids) + std::vector& elem_side_ids, + INT sideOrdOffset = 0) { const stk::mesh::BulkData &bulk_data = params.bulk_data(); const stk::mesh::Selector *subset_selector = params.get_subset_selector(); @@ -228,8 +230,9 @@ void fill_element_and_side_ids_from_connectivity(stk::io::OutputParams ¶ms, if (bulk_data.is_valid(suitable_elem)) { + int oneBasedOrdinal = suitable_ordinal - sideOrdOffset + 1; elem_side_ids.push_back(bulk_data.identifier(suitable_elem)); - elem_side_ids.push_back(suitable_ordinal + 1); // Ioss is 1-based, mesh is 0-based. + elem_side_ids.push_back(oneBasedOrdinal); sides.push_back(side); } } @@ -241,7 +244,8 @@ void fill_element_and_side_ids(stk::io::OutputParams ¶ms, const stk::mesh::Part *parentElementBlock, stk::topology stk_element_topology, stk::mesh::EntityVector &sides, - std::vector& elem_side_ids) + std::vector& elem_side_ids, + INT sideOrdOffset = 0) { const mesh::BulkData &bulk_data = params.bulk_data(); const stk::mesh::Part &parentPart = stk::mesh::get_sideset_parent(*part); @@ -250,12 +254,12 @@ void fill_element_and_side_ids(stk::io::OutputParams ¶ms, { const stk::mesh::SideSet& sset = bulk_data.get_sideset(parentPart); fill_element_and_side_ids_from_sideset(sset, params, part, parentElementBlock, - stk_element_topology, sides, elem_side_ids); + stk_element_topology, sides, elem_side_ids, sideOrdOffset); } else { fill_element_and_side_ids_from_connectivity(params, part, parentElementBlock, - stk_element_topology, sides, elem_side_ids); + stk_element_topology, sides, elem_side_ids, sideOrdOffset); } } @@ -263,7 +267,8 @@ void fill_element_and_side_ids(stk::io::OutputParams ¶ms, inline size_t get_number_sides_in_sideset(OutputParams ¶ms, const stk::mesh::Part &ssPart, stk::topology stk_element_topology, - const stk::mesh::Part *parentElementBlock = nullptr) + const stk::mesh::Part *parentElementBlock = nullptr, + int sideOrdOffset = 0) { stk::mesh::EntityVector sides; std::vector elemSideIds; @@ -273,7 +278,8 @@ inline size_t get_number_sides_in_sideset(OutputParams ¶ms, parentElementBlock, stk_element_topology, sides, - elemSideIds); + elemSideIds, + sideOrdOffset); return sides.size(); } diff --git a/packages/stk/stk_math/Jamfile b/packages/stk/stk_math/Jamfile index a9c2c54c6fba..8bc826c40059 100644 --- a/packages/stk/stk_math/Jamfile +++ b/packages/stk/stk_math/Jamfile @@ -155,7 +155,7 @@ exe stk_math_utest /sierra/stk_math//stk_math /sierra/stk_unit_test_utils//stk_unit_main /tpl/trilinos//kokkoscore - /tpl/gtest//gtest + /tpl/googletest//gtest : @sierra-exec-tag ; diff --git a/packages/stk/stk_mesh/stk_mesh/base/Bucket.cpp b/packages/stk/stk_mesh/stk_mesh/base/Bucket.cpp index 7d68d6392017..1e1c6c7e9b53 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/Bucket.cpp +++ b/packages/stk/stk_mesh/stk_mesh/base/Bucket.cpp @@ -415,20 +415,6 @@ bool Bucket::member_all( const PartVector & parts ) const return result_all ; } -bool Bucket::member_any( const PartVector & parts ) const -{ - const PartVector::const_iterator ip_end = parts.end(); - PartVector::const_iterator ip = parts.begin() ; - - bool result_none = true ; - - for ( ; result_none && ip_end != ip ; ++ip ) { - const unsigned ord = (*ip)->mesh_meta_data_ordinal(); - result_none = !member(ord); - } - return ! result_none ; -} - bool Bucket::member_any( const OrdinalVector & parts ) const { const OrdinalVector::const_iterator ip_end = parts.end(); diff --git a/packages/stk/stk_mesh/stk_mesh/base/Bucket.hpp b/packages/stk/stk_mesh/stk_mesh/base/Bucket.hpp index 139feec1ffc5..059c73a66e90 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/Bucket.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/Bucket.hpp @@ -200,7 +200,17 @@ class Bucket bool member_all( const OrdinalVector & ) const ; /** \brief Bucket is a subset of any of the given parts */ - bool member_any( const PartVector & ) const ; + template + bool member_any(const PARTVECTOR & parts) const + { + for(const Part* part : parts) { + if (member(*part)) { + return true; + } + } + return false ; + } + bool member_any( const OrdinalVector & ) const ; //-------------------------------- diff --git a/packages/stk/stk_mesh/stk_mesh/base/BulkData.cpp b/packages/stk/stk_mesh/stk_mesh/base/BulkData.cpp index 4e4ec12812b3..0ae9eda96f75 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/BulkData.cpp +++ b/packages/stk/stk_mesh/stk_mesh/base/BulkData.cpp @@ -1014,7 +1014,7 @@ void BulkData::internal_verify_and_change_entity_parts( Entity entity, OrdinalVector removePartsAndSubsetsMinusPartsInAddPartsList; impl::fill_remove_parts_and_subsets_minus_parts_in_add_parts_list(remove_parts, addPartsAndSupersets, - bucket(entity), + bucket_ptr(entity), removePartsAndSubsetsMinusPartsInAddPartsList); scratchOrdinalVec.clear(); @@ -1052,7 +1052,7 @@ void BulkData::internal_verify_and_change_entity_parts( const EntityVector& enti impl::fill_remove_parts_and_subsets_minus_parts_in_add_parts_list(remove_parts, addPartsAndSupersets, - bucket(entity), + bucket_ptr(entity), removePartsAndSubsetsMinusPartsInAddPartsList); internal_change_entity_parts(entity, @@ -1441,6 +1441,8 @@ template void BulkData::declare_entities(stk::topology::rank_t rank, const IDVECTOR& newIds, const PartVector &parts, std::vector& requested_entities) { + require_ok_to_modify(); + requested_entities.resize(newIds.size()); if (newIds.empty()) { return; @@ -2000,30 +2002,20 @@ void BulkData::reset_empty_field_data_callback(EntityRank rank, unsigned bucketI m_field_data_manager->reset_empty_field_data(rank, bucketId, bucketSize, bucketCapacity, fields); } -void BulkData::update_field_data_states(FieldBase* field) +void BulkData::update_field_data_states(FieldBase* field, bool rotateNgpFieldViews) { const int numStates = field->number_of_states(); if (numStates > 1) { - field->rotate_multistate_data(); - unsigned fieldOrdinal = field->mesh_meta_data_ordinal(); for (int s = 1; s < numStates; ++s) { m_field_data_manager->swap_fields(fieldOrdinal+s, fieldOrdinal); } - for (int state = 0; state < numStates; ++state) { - FieldBase* currentStateField = field->field_state(static_cast(state)); - - NgpFieldBase* ngpField = currentStateField->get_ngp_field(); - if (ngpField != nullptr) { - ngpField->update_bucket_pointer_view(); - ngpField->fence(); - } - } + field->rotate_multistate_data(rotateNgpFieldViews); } } -void BulkData::update_field_data_states() +void BulkData::update_field_data_states(bool rotateNgpFieldViews) { const std::vector & fields = mesh_meta_data().get_fields(); @@ -2031,7 +2023,7 @@ void BulkData::update_field_data_states() FieldBase* field = fields[i]; const int numStates = field->number_of_states(); if (numStates > 1) { - update_field_data_states(field); + update_field_data_states(field, rotateNgpFieldViews); } i += numStates ; } @@ -4159,7 +4151,10 @@ void BulkData::update_comm_list_based_on_changes_in_comm_map() void BulkData::notify_finished_mod_end() { - notifier.notify_finished_modification_end(parallel()); + bool anyModOnAnyProc = notifier.notify_finished_modification_end(parallel()); + if (!anyModOnAnyProc) { + m_meshModification.set_sync_count(synchronized_count()-1); + } } void BulkData::internal_modification_end_for_change_ghosting() diff --git a/packages/stk/stk_mesh/stk_mesh/base/BulkData.hpp b/packages/stk/stk_mesh/stk_mesh/base/BulkData.hpp index 91ec103fb417..d9d995769690 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/BulkData.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/BulkData.hpp @@ -38,7 +38,6 @@ #include // for size_t #include // for uint16_t #include // for max -#include // for less, equal_to #include // for operator<<, basic_ostream, etc #include // for list #include // for map, map<>::value_compare @@ -312,8 +311,8 @@ class BulkData { * StateNM3 <- StateNM2 * */ - void update_field_data_states(); - void update_field_data_states(FieldBase *field); + void update_field_data_states(bool rotateNgpFieldViews = false); + void update_field_data_states(FieldBase *field, bool rotateNgpFieldViews = false); /** \brief Copy field data from src entity to Dest entity * - Fields that exist on the src that don't exist on the dest will @@ -880,6 +879,8 @@ class BulkData { return hasOrphanProtection; } + bool is_mesh_consistency_check_on() const { return m_runConsistencyCheck; } + protected: //functions BulkData(std::shared_ptr mesh_meta_data, ParallelMachine parallel, @@ -2155,11 +2156,11 @@ BulkData::internal_add_node_sharing_called() const inline Bucket & BulkData::bucket(Entity entity) const { -#ifndef NDEBUG - entity_getter_debug_check(entity); -#endif + Bucket* bptr = bucket_ptr(entity); + + STK_ThrowAssertMsg(bptr != nullptr, "BulkData::bucket error, invalid bucket (nullptr) for entity with local_offset()="<* deviceFieldT = dynamic_cast*>(other); + STK_ThrowRequireMsg(deviceFieldT != nullptr, "DeviceField::swap_field_views called with class that can't dynamic_cast to DeviceField"); + swap_views(deviceData, deviceFieldT->deviceData); + } + KOKKOS_FUNCTION void swap(DeviceField &other) { @@ -521,7 +530,11 @@ class DeviceField : public NgpFieldBase if (buckets.size() > deviceView.extent(0)) { deviceView = ViewType(Kokkos::ViewAllocateWithoutInitializing(hostField->name() + suffix), impl::allocation_size(buckets.size())); if (hostView.extent(0) != deviceView.extent(0)) { +#if defined STK_USE_DEVICE_MESH && !defined(STK_ENABLE_GPU) + hostView = Kokkos::create_mirror(deviceView); +#else hostView = Kokkos::create_mirror_view(deviceView); +#endif } } } diff --git a/packages/stk/stk_mesh/stk_mesh/base/DeviceMesh.cpp b/packages/stk/stk_mesh/stk_mesh/base/DeviceMesh.cpp index b9e79df06fb5..46e6dc0d9a51 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/DeviceMesh.cpp +++ b/packages/stk/stk_mesh/stk_mesh/base/DeviceMesh.cpp @@ -47,8 +47,10 @@ void DeviceBucket::initialize_bucket_attributes(const stk::mesh::Bucket &bucket) void DeviceBucket::allocate(const stk::mesh::Bucket &bucket) { + Kokkos::Profiling::pushRegion("nodeOffsets views"); nodeOffsets = OrdinalViewType(Kokkos::view_alloc(Kokkos::WithoutInitializing, "NodeOffsets"), bucket.size()+1); hostNodeOffsets = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, Kokkos::HostSpace(), nodeOffsets); + Kokkos::Profiling::popRegion(); unsigned maxNodesPerEntity = bucketTopology.num_nodes(); unsigned totalNumNodes = bucketTopology.num_nodes()*bucketCapacity; @@ -63,18 +65,20 @@ void DeviceBucket::allocate(const stk::mesh::Bucket &bucket) const stk::mesh::PartVector& parts = bucket.supersets(); + Kokkos::Profiling::pushRegion("entities/connectivity/ordinals"); entities = EntityViewType(Kokkos::view_alloc(Kokkos::WithoutInitializing, "BucketEntities"), bucketCapacity); - hostEntities = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, Kokkos::HostSpace(), entities); + STK_ThrowRequireMsg(bucketCapacity > 0, "bucket capacity must be greater than 0"); + hostEntities = HostEntityViewType(bucket.begin(), bucketCapacity); nodeConnectivity = BucketConnectivityType(Kokkos::view_alloc(Kokkos::WithoutInitializing, "BucketConnectivity"), totalNumNodes); hostNodeConnectivity = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, Kokkos::HostSpace(), nodeConnectivity); nodeOrdinals = OrdinalViewType(Kokkos::view_alloc(Kokkos::WithoutInitializing, "NodeOrdinals"), static_cast(maxNodesPerEntity)); - hostNodeOrdinals = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, Kokkos::HostSpace(), nodeOrdinals); partOrdinals = PartOrdinalViewType(Kokkos::view_alloc(Kokkos::WithoutInitializing, "PartOrdinals"), parts.size()); - hostPartOrdinals = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, Kokkos::HostSpace(), partOrdinals); + hostPartOrdinals = HostPartOrdinalViewType(bucket.superset_part_ordinals().first, parts.size()); + Kokkos::Profiling::popRegion(); } void DeviceBucket::initialize_from_host(const stk::mesh::Bucket &bucket) @@ -87,15 +91,13 @@ void DeviceBucket::initialize_from_host(const stk::mesh::Bucket &bucket) } } - for (unsigned i = 0; i < maxNodesPerEntity; ++i) { - hostNodeOrdinals(i) = static_cast(i); - } - Kokkos::deep_copy(nodeOrdinals, hostNodeOrdinals); + OrdinalViewType& nodeOrds = nodeOrdinals; //local var to avoid implicit this capture + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const int idx) { + for (unsigned i = 0; i < maxNodesPerEntity; ++i) { + nodeOrds(i) = static_cast(i); + } + }); - const stk::mesh::PartVector& parts = bucket.supersets(); - for (unsigned i = 0; i < parts.size(); ++i) { - hostPartOrdinals(i) = parts[i]->mesh_meta_data_ordinal(); - } Kokkos::deep_copy(partOrdinals, hostPartOrdinals); update_from_host(bucket); @@ -103,11 +105,14 @@ void DeviceBucket::initialize_from_host(const stk::mesh::Bucket &bucket) void DeviceBucket::update_from_host(const stk::mesh::Bucket &bucket) { + Kokkos::Profiling::pushRegion("update_from_host"); bucketSize = bucket.size(); if (bucketSize+1 != hostNodeOffsets.size()) { + Kokkos::Profiling::pushRegion("bucket size changed"); nodeOffsets = OrdinalViewType(Kokkos::view_alloc(Kokkos::WithoutInitializing, "NodeOffsets"), bucketSize+1); hostNodeOffsets = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, Kokkos::HostSpace(), nodeOffsets); + Kokkos::Profiling::popRegion(); } unsigned totalNumNodes = bucket.topology().num_nodes()*bucketCapacity; @@ -121,26 +126,27 @@ void DeviceBucket::update_from_host(const stk::mesh::Bucket &bucket) } } + Kokkos::Profiling::pushRegion("bucket node connectivity/ordinals changed"); if (totalNumNodes != hostNodeConnectivity.size()) { nodeConnectivity = BucketConnectivityType(Kokkos::view_alloc(Kokkos::WithoutInitializing, "BucketConnectivity"), totalNumNodes); hostNodeConnectivity = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, Kokkos::HostSpace(), nodeConnectivity); } - if (maxNodesPerEntity != hostNodeOrdinals.size()) { + if (maxNodesPerEntity != nodeOrdinals.size()) { nodeOrdinals = OrdinalViewType(Kokkos::view_alloc(Kokkos::WithoutInitializing, "NodeOrdinals"), static_cast(maxNodesPerEntity)); - hostNodeOrdinals = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, Kokkos::HostSpace(), nodeOrdinals); - for (unsigned i = 0; i < maxNodesPerEntity; ++i) { - hostNodeOrdinals(i) = static_cast(i); - } - Kokkos::deep_copy(nodeOrdinals, hostNodeOrdinals); + OrdinalViewType& nodeOrds = nodeOrdinals; //local var to avoid implicit this capture + Kokkos::parallel_for(1, KOKKOS_LAMBDA(const int idx) { + for (unsigned i = 0; i < maxNodesPerEntity; ++i) { + nodeOrds(i) = static_cast(i); + } + }); } + Kokkos::Profiling::popRegion(); unsigned nodeOffset = 0; for (unsigned iEntity = 0; iEntity < bucket.size(); ++iEntity) { - stk::mesh::Entity entity = bucket[iEntity]; - hostEntities(iEntity) = entity; const unsigned nodesPerEntity = bucket.num_nodes(iEntity); const stk::mesh::Entity * elemNodes = bucket.begin_nodes(iEntity); for (unsigned iNode = 0; iNode < nodesPerEntity; ++iNode) { @@ -151,17 +157,24 @@ void DeviceBucket::update_from_host(const stk::mesh::Bucket &bucket) } hostNodeOffsets(bucket.size()) = nodeOffset; + Kokkos::Profiling::pushRegion("deep_copy entities/connectivity/offsets"); Kokkos::deep_copy(entities, hostEntities); Kokkos::deep_copy(nodeConnectivity, hostNodeConnectivity); Kokkos::deep_copy(nodeOffsets, hostNodeOffsets); + Kokkos::Profiling::popRegion(); + Kokkos::Profiling::popRegion(); } void DeviceMesh::update_mesh() { - if (is_up_to_date()) return; + if (is_up_to_date()) { + return; + } require_ngp_mesh_rank_limit(bulk->mesh_meta_data()); + Kokkos::Profiling::pushRegion("DeviceMesh::update_mesh"); + const bool anyChanges = fill_buckets(*bulk); if (anyChanges) { @@ -178,12 +191,14 @@ void DeviceMesh::update_mesh() } synchronizedCount = bulk->synchronized_count(); + Kokkos::Profiling::popRegion(); } bool DeviceMesh::fill_buckets(const stk::mesh::BulkData& bulk_in) { bool anyBucketChanges = false; + Kokkos::Profiling::pushRegion("fill_buckets"); for (stk::mesh::EntityRank rank = stk::topology::NODE_RANK; rank < endRank; ++rank) { const stk::mesh::BucketVector& stkBuckets = bulk_in.buckets(rank); unsigned numStkBuckets = stkBuckets.size(); @@ -199,14 +214,17 @@ bool DeviceMesh::fill_buckets(const stk::mesh::BulkData& bulk_in) const unsigned ngpBucketId = stkBucket.ngp_bucket_id(); if (ngpBucketId == INVALID_BUCKET_ID) { + Kokkos::Profiling::pushRegion("new bucket"); // New bucket on host new (&bucketBuffer[iBucket]) DeviceBucket(); bucketBuffer[iBucket].initialize_bucket_attributes(stkBucket); bucketBuffer[iBucket].allocate(stkBucket); bucketBuffer[iBucket].initialize_from_host(stkBucket); anyBucketChanges = true; + Kokkos::Profiling::popRegion(); } else { + Kokkos::Profiling::pushRegion("pre-existing bucket"); // Pre-existing bucket on host new (&bucketBuffer[iBucket]) DeviceBucket(buckets[rank][ngpBucketId]); if (stkBucket.is_modified()) { @@ -214,6 +232,7 @@ bool DeviceMesh::fill_buckets(const stk::mesh::BulkData& bulk_in) anyBucketChanges = true; } bucketBuffer[iBucket].bucketId = stkBucket.bucket_id(); + Kokkos::Profiling::popRegion(); } stkBucket.set_ngp_bucket_id(iBucket); @@ -227,6 +246,7 @@ bool DeviceMesh::fill_buckets(const stk::mesh::BulkData& bulk_in) buckets[rank] = bucketBuffer; } + Kokkos::Profiling::popRegion(); return anyBucketChanges; } diff --git a/packages/stk/stk_mesh/stk_mesh/base/DeviceMesh.hpp b/packages/stk/stk_mesh/stk_mesh/base/DeviceMesh.hpp index 507fc210b7be..0e0ecb15b8a4 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/DeviceMesh.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/DeviceMesh.hpp @@ -34,9 +34,9 @@ #ifndef STK_MESH_DEVICEMESH_HPP #define STK_MESH_DEVICEMESH_HPP -#include "stk_mesh/base/NgpMeshBase.hpp" #include #include +#include "stk_mesh/base/NgpMeshBase.hpp" #include "stk_mesh/base/Bucket.hpp" #include "stk_mesh/base/Entity.hpp" #include "stk_mesh/base/Types.hpp" @@ -70,7 +70,7 @@ struct DeviceBucket { : bucketId(0), entityRank(stk::topology::NODE_RANK), entities(), nodeConnectivity(), hostNodeConnectivity(), nodeOffsets(), hostNodeOffsets(), - nodeOrdinals(), hostNodeOrdinals(), + nodeOrdinals(), owningMesh(nullptr), bucketCapacity(0) {} @@ -142,7 +142,7 @@ struct DeviceBucket { stk::topology bucketTopology; EntityViewType entities; - EntityViewType::HostMirror hostEntities; + HostEntityViewType hostEntities; BucketConnectivityType nodeConnectivity; BucketConnectivityType::HostMirror hostNodeConnectivity; @@ -151,21 +151,22 @@ struct DeviceBucket { OrdinalViewType::HostMirror hostNodeOffsets; OrdinalViewType nodeOrdinals; - OrdinalViewType::HostMirror hostNodeOrdinals; PartOrdinalViewType partOrdinals; - PartOrdinalViewType::HostMirror hostPartOrdinals; + HostPartOrdinalViewType hostPartOrdinals; const stk::mesh::DeviceMesh* owningMesh; unsigned bucketCapacity; unsigned bucketSize; }; -struct DeviceMeshIndex +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after April 2024 +struct STK_DEPRECATED DeviceMeshIndex { const DeviceBucket *bucket; size_t bucketOrd; }; +#endif class DeviceMesh : public NgpMeshBase { @@ -175,7 +176,11 @@ class DeviceMesh : public NgpMeshBase using ConnectedEntities = DeviceBucket::ConnectedEntities; using ConnectedOrdinals = DeviceBucket::ConnectedOrdinals; using Permutations = DeviceBucket::Permutations; +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after April 2024 using MeshIndex = DeviceMeshIndex; +#else + using MeshIndex = FastMeshIndex; +#endif using BucketType = DeviceBucket; KOKKOS_FUNCTION @@ -244,11 +249,14 @@ class DeviceMesh : public NgpMeshBase return buckets[rank](meshIndex.bucket_id)[meshIndex.bucket_ord]; } +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after April 2024 + STK_DEPRECATED KOKKOS_FUNCTION ConnectedNodes get_nodes(const DeviceMeshIndex &entity) const { return buckets[entity.bucket->entity_rank()](entity.bucket->bucket_id()).get_nodes(entity.bucketOrd); } +#endif KOKKOS_FUNCTION ConnectedEntities get_connected_entities(stk::mesh::EntityRank rank, const stk::mesh::FastMeshIndex &entity, stk::mesh::EntityRank connectedRank) const @@ -265,7 +273,8 @@ class DeviceMesh : public NgpMeshBase ConnectedEntities connectedEntities(nullptr, 0); if (numConnected > 0) { int stride = 1; - connectedEntities = ConnectedEntities(&sparseConnectivity[rank][connectedRank](connectivityOffset), numConnected, stride); + connectedEntities = + ConnectedEntities(&(sparseConnectivity[rank][connectedRank](connectivityOffset)), numConnected, stride); } return connectedEntities; } @@ -309,7 +318,8 @@ class DeviceMesh : public NgpMeshBase if (numConnected > 0) { int stride = 1; - connectedOrdinals = ConnectedOrdinals(&sparseConnectivityOrdinals[rank][connectedRank](connectivityOffset), numConnected, stride); + connectedOrdinals = ConnectedOrdinals( + &(sparseConnectivityOrdinals[rank][connectedRank](connectivityOffset)), numConnected, stride); } return connectedOrdinals; } @@ -354,7 +364,7 @@ class DeviceMesh : public NgpMeshBase if (numConnected > 0) { int stride = 1; - permutations = Permutations(&sparsePermutations[rank][connectedRank](connectivityOffset) , numConnected, stride); + permutations = Permutations(&(sparsePermutations[rank][connectedRank](connectivityOffset)), numConnected, stride); } return permutations; } @@ -395,10 +405,12 @@ class DeviceMesh : public NgpMeshBase return deviceMeshIndices(entity.local_offset()); } - const stk::mesh::FastMeshIndex& host_mesh_index(stk::mesh::Entity entity) const +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after April 2024 +STK_DEPRECATED const stk::mesh::FastMeshIndex& host_mesh_index(stk::mesh::Entity entity) const { return hostMeshIndices(entity.local_offset()); } +#endif stk::NgpVector get_bucket_ids(stk::mesh::EntityRank rank, const stk::mesh::Selector &selector) const { diff --git a/packages/stk/stk_mesh/stk_mesh/base/EntityCommListInfo.hpp b/packages/stk/stk_mesh/stk_mesh/base/EntityCommListInfo.hpp index 224af78f2d46..0b8a0ddae4d1 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/EntityCommListInfo.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/EntityCommListInfo.hpp @@ -72,7 +72,7 @@ struct IsInvalid { bool operator()(const EntityCommListInfo& comm) const { - return comm.key == EntityKey() || comm.entity_comm == -1; + return !comm.key.is_valid() || comm.entity_comm == -1; } }; diff --git a/packages/stk/stk_mesh/stk_mesh/base/ExodusTranslator.hpp b/packages/stk/stk_mesh/stk_mesh/base/ExodusTranslator.hpp index 34c46c45743c..29474159247d 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/ExodusTranslator.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/ExodusTranslator.hpp @@ -49,11 +49,31 @@ namespace stk { namespace mesh { +namespace impl +{ + +inline bool is_elem_block(const stk::mesh::Part& part) +{ + return (part.primary_entity_rank() == stk::topology::ELEMENT_RANK && part.id() > 0); +} + +inline bool is_elem_block_ptr(const stk::mesh::Part* part) { + return part!=nullptr && is_elem_block(*part); +} + +inline bool has_subset_elem_block(const stk::mesh::Part &part) +{ + const PartVector& subsets = part.subsets(); + return std::any_of(subsets.cbegin(), subsets.cend(), is_elem_block_ptr); +} + +}//namespace impl inline bool is_element_block(const stk::mesh::Part &part) { - return (part.primary_entity_rank() == stk::topology::ELEMENT_RANK && part.id() > 0 ); + return (impl::is_elem_block(part) && !impl::has_subset_elem_block(part)); } + inline bool is_node_set(const stk::mesh::Part &part) { return (part.primary_entity_rank() == stk::topology::NODE_RANK && part.id() > 0 ); @@ -72,6 +92,7 @@ inline bool has_super_set_face_part(const stk::mesh::Part &part) } return false; } + inline bool is_side_set(const stk::mesh::Part &part) { if (part.id() > 0) { diff --git a/packages/stk/stk_mesh/stk_mesh/base/FaceCreator.cpp b/packages/stk/stk_mesh/stk_mesh/base/FaceCreator.cpp index d50e251becb9..d0f18522f971 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/FaceCreator.cpp +++ b/packages/stk/stk_mesh/stk_mesh/base/FaceCreator.cpp @@ -48,22 +48,28 @@ size_t FaceCreator::create_face_entities_per_element(size_t element_side_index, return ordinals.size(); } -void FaceCreator::create_side_entities_given_sideset(const SideSet &skinnedSideSet, const stk::mesh::PartVector& skinParts) +void FaceCreator::create_side_entities_given_sideset(const SideSet &skinnedSideSet, const stk::mesh::PartVector& skinParts, bool doLocalModCycle) { - std::vector sharedModified; + std::vector sharedModified; + if (doLocalModCycle) { m_bulkData.modification_begin(); - for(size_t i=0;i& ordinals); std::vector get_side_ordinals_of_element(size_t element_side_index, const SideSet &skinnedSideSet); diff --git a/packages/stk/stk_mesh/stk_mesh/base/FieldBLAS.hpp b/packages/stk/stk_mesh/stk_mesh/base/FieldBLAS.hpp index d338fec2b1d3..ea7ddebfe2e3 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/FieldBLAS.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/FieldBLAS.hpp @@ -433,7 +433,7 @@ template inline void field_axpy(const Scalar alpha, const FieldBase& xField, const FieldBase& yField, const Selector& selector) { - STK_ThrowAssert(is_compatible(xField, yField)); + STK_ThrowRequire(is_compatible(xField, yField)); BucketVector const& buckets = xField.get_mesh().get_buckets( xField.entity_rank(), selector ); @@ -463,7 +463,7 @@ template inline void field_axpby(const Scalar alpha, const FieldBase& xField, const Scalar beta, const FieldBase& yField, const Selector& selector) { - STK_ThrowAssert(is_compatible(xField, yField)); + STK_ThrowRequire(is_compatible(xField, yField)); BucketVector const& buckets = xField.get_mesh().get_buckets( xField.entity_rank(), selector ); @@ -524,8 +524,8 @@ void INTERNAL_field_product(const FieldBase& xField, const FieldBase& yField, co inline void field_product(const FieldBase& xField, const FieldBase& yField, const FieldBase& zField, const Selector& selector) { - STK_ThrowAssert(is_compatible(xField, yField)); - STK_ThrowAssert(is_compatible(yField, zField)); + STK_ThrowRequire(is_compatible(xField, yField)); + STK_ThrowRequire(is_compatible(yField, zField)); if (xField.data_traits().type_info == typeid(double)) { INTERNAL_field_product(xField,yField,zField,selector); @@ -594,7 +594,7 @@ void INTERNAL_field_copy(const FieldBase& xField, const FieldBase& yField, const inline void field_copy(const FieldBase& xField, const FieldBase& yField, const Selector& selector) { - STK_ThrowAssert(is_compatible(xField, yField)); + STK_ThrowRequire(is_compatible(xField, yField)); if (xField.data_traits().type_info == typeid(double)) { INTERNAL_field_copy(xField,yField,selector); @@ -622,7 +622,7 @@ template & xField, const Field & yField, const Selector& selector, const MPI_Comm comm) { - STK_ThrowAssert(is_compatible(xField, yField)); + STK_ThrowRequire(is_compatible(xField, yField)); BucketVector const& buckets = xField.get_mesh().get_buckets(xField.entity_rank(), selector & xField.mesh_meta_data().locally_owned_part()); @@ -650,7 +650,7 @@ template field_dot(const Field,T1,T2,T3,T4,T5,T6,T7>& xField, const Field,T1,T2,T3,T4,T5,T6,T7>& yField, const Selector& selector, const MPI_Comm comm) { - STK_ThrowAssert(is_compatible(xField, yField)); + STK_ThrowRequire(is_compatible(xField, yField)); BucketVector const& buckets = xField.get_mesh().get_buckets(xField.entity_rank(), selector & xField.mesh_meta_data().locally_owned_part()); @@ -700,7 +700,7 @@ template inline void field_dot(std::complex& global_result, const FieldBase& xField, const FieldBase& yField, const Selector& selector, const MPI_Comm comm) { - STK_ThrowAssert(is_compatible(xField, yField)); + STK_ThrowRequire(is_compatible(xField, yField)); BucketVector const& buckets = xField.get_mesh().get_buckets(xField.entity_rank(), selector & xField.mesh_meta_data().locally_owned_part()); @@ -734,7 +734,7 @@ template inline void field_dot(Scalar& glob_result, const FieldBase& xField, const FieldBase& yField, const Selector& selector, const MPI_Comm comm) { - STK_ThrowAssert(is_compatible(xField, yField)); + STK_ThrowRequire(is_compatible(xField, yField)); BucketVector const& buckets = xField.get_mesh().get_buckets(xField.entity_rank(), selector & xField.mesh_meta_data().locally_owned_part()); @@ -777,7 +777,7 @@ template inline void field_scale(const Scalar alpha, const FieldBase& xField, const Selector& selector) { - STK_ThrowAssert( is_compatible(xField) ); + STK_ThrowRequire( is_compatible(xField) ); BucketVector const& buckets = xField.get_mesh().get_buckets(xField.entity_rank(),selector); @@ -804,7 +804,7 @@ template inline void field_fill_component(const Scalar* alpha, const FieldBase& xField, const Selector& selector) { - STK_ThrowAssert( is_compatible(xField) ); + STK_ThrowRequire( is_compatible(xField) ); BucketVector const& buckets = xField.get_mesh().get_buckets(xField.entity_rank(),selector); @@ -837,7 +837,7 @@ template inline void field_fill(const Scalar alpha, const FieldBase& xField, const Selector& selector) { - STK_ThrowAssert( is_compatible(xField) ); + STK_ThrowRequire( is_compatible(xField) ); BucketVector const& buckets = xField.get_mesh().get_buckets(xField.entity_rank(),selector); @@ -856,7 +856,7 @@ template inline void field_fill(const Scalar alpha, const std::vector& xFields, const Selector& selector) { - STK_ThrowAssert(xFields.size() >= 1 ); + STK_ThrowRequire(xFields.size() >= 1 ); stk::mesh::EntityRank fieldEntityRank = xFields[0]->entity_rank(); BucketVector const& buckets = xFields[0]->get_mesh().get_buckets(fieldEntityRank,selector); @@ -864,8 +864,8 @@ void field_fill(const Scalar alpha, const std::vector& xFields for (auto&& bucket : buckets){ for (unsigned int i=0; i(xField) ); - STK_ThrowAssert(fieldEntityRank == xField.entity_rank()); + STK_ThrowRequire( is_compatible(xField) ); + STK_ThrowRequire(fieldEntityRank == xField.entity_rank()); BucketSpan x(xField, *bucket); FortranBLAS::fill(x.size(),alpha,x.data()); } @@ -911,7 +911,7 @@ void INTERNAL_field_swap(const FieldBase& xField, const FieldBase& yField, const inline void field_swap(const FieldBase& xField, const FieldBase& yField, const Selector& selector) { - STK_ThrowAssert(is_compatible(xField, yField)); + STK_ThrowRequire(is_compatible(xField, yField)); if (xField.data_traits().type_info == typeid(double)) { INTERNAL_field_swap(xField,yField,selector); @@ -1001,7 +1001,7 @@ template inline void field_nrm2(Scalar& glob_result, const FieldBase& xField, const Selector& selector, const MPI_Comm comm) { - STK_ThrowAssert( is_compatible(xField) ); + STK_ThrowRequire( is_compatible(xField) ); BucketVector const& buckets = xField.get_mesh().get_buckets(xField.entity_rank(), selector & xField.mesh_meta_data().locally_owned_part()); @@ -1026,7 +1026,7 @@ template inline void field_nrm2(std::complex& result, const FieldBase& xField, const Selector& selector, const MPI_Comm comm) { - STK_ThrowAssert( is_compatible>(xField) ); + STK_ThrowRequire( is_compatible>(xField) ); BucketVector const& buckets = xField.get_mesh().get_buckets(xField.entity_rank(), selector & xField.mesh_meta_data().locally_owned_part()); @@ -1129,7 +1129,7 @@ template inline void field_asum(Scalar& glob_result, const FieldBase& xField, const Selector& selector, const MPI_Comm comm) { - STK_ThrowAssert( is_compatible(xField) ); + STK_ThrowRequire( is_compatible(xField) ); BucketVector const& buckets = xField.get_mesh().get_buckets(xField.entity_rank(), selector & xField.mesh_meta_data().locally_owned_part()); @@ -1153,7 +1153,7 @@ template inline void field_asum(std::complex& result, const FieldBase& xField, const Selector& selector, const MPI_Comm comm) { - STK_ThrowAssert( is_compatible>(xField) ); + STK_ThrowRequire( is_compatible>(xField) ); BucketVector const& buckets = xField.get_mesh().get_buckets(xField.entity_rank(), selector & xField.mesh_meta_data().locally_owned_part()); @@ -1408,7 +1408,7 @@ template inline void field_amax(std::complex& result, const FieldBase& xField, const Selector& selector, const MPI_Comm comm) { - STK_ThrowAssert( is_compatible>(xField) ); + STK_ThrowRequire( is_compatible>(xField) ); BucketVector const& buckets = xField.get_mesh().get_buckets(xField.entity_rank(), selector & xField.mesh_meta_data().locally_owned_part()); @@ -1438,7 +1438,7 @@ template inline void field_amax(Scalar& result, const FieldBase& xField, const Selector& selector, const MPI_Comm comm) { - STK_ThrowAssert( is_compatible(xField) ); + STK_ThrowRequire( is_compatible(xField) ); BucketVector const& buckets = xField.get_mesh().get_buckets(xField.entity_rank(), selector & xField.mesh_meta_data().locally_owned_part()); @@ -1809,7 +1809,7 @@ template inline void field_amin(std::complex& result, const FieldBase& xField, const Selector& selector, const MPI_Comm comm) { - STK_ThrowAssert( is_compatible>(xField) ); + STK_ThrowRequire( is_compatible>(xField) ); BucketVector const& buckets = xField.get_mesh().get_buckets(xField.entity_rank(), selector & xField.mesh_meta_data().locally_owned_part()); @@ -1839,7 +1839,7 @@ template inline void field_amin(Scalar& result, const FieldBase& xField, const Selector& selector, const MPI_Comm comm) { - STK_ThrowAssert( is_compatible(xField) ); + STK_ThrowRequire( is_compatible(xField) ); BucketVector const& buckets = xField.get_mesh().get_buckets(xField.entity_rank(), selector & xField.mesh_meta_data().locally_owned_part()); diff --git a/packages/stk/stk_mesh/stk_mesh/base/FieldBase.cpp b/packages/stk/stk_mesh/stk_mesh/base/FieldBase.cpp index dff8be5771b5..13be49b8dbb8 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/FieldBase.cpp +++ b/packages/stk/stk_mesh/stk_mesh/base/FieldBase.cpp @@ -547,24 +547,42 @@ unsigned FieldBase::max_extent(unsigned dimension) const } } -void FieldBase::rotate_multistate_data() +void FieldBase::rotate_multistate_data(bool rotateNgpFieldViews) { - const unsigned numStates = number_of_states(); + const int numStates = number_of_states(); if (numStates > 1 && StateNew == state()) { - NgpFieldBase* ngpField = get_ngp_field(); - if (ngpField != nullptr) { - ngpField->rotate_multistate_data(); + bool allStatesHaveNgpFields = true; + for(int s = 0; s < numStates; ++s) { + if (field_state(static_cast(s))->get_ngp_field() == nullptr) { + allStatesHaveNgpFields = false; + } } - for (unsigned s = 1; s < numStates; ++s) { + + for (int s = 1; s < numStates; ++s) { FieldBase* sField = field_state(static_cast(s)); m_field_meta_data.swap(sField->m_field_meta_data); std::swap(m_numSyncsToDevice, sField->m_numSyncsToDevice); std::swap(m_numSyncsToHost, sField->m_numSyncsToHost); - std::swap(m_modifiedOnHost, sField->m_modifiedOnHost); std::swap(m_modifiedOnDevice, sField->m_modifiedOnDevice); } + + for(int s = 0; s < numStates; ++s) { + NgpFieldBase* ngpField = field_state(static_cast(s))->get_ngp_field(); + if (ngpField != nullptr) { + ngpField->update_bucket_pointer_view(); + ngpField->fence(); + } + } + + if (rotateNgpFieldViews && allStatesHaveNgpFields) { + for (int s = 1; s < numStates; ++s) { + NgpFieldBase* ngpField_sminus1 = field_state(static_cast(s-1))->get_ngp_field(); + NgpFieldBase* ngpField_s = field_state(static_cast(s))->get_ngp_field(); + ngpField_s->swap_field_views(ngpField_sminus1); + } + } } } diff --git a/packages/stk/stk_mesh/stk_mesh/base/FieldBase.hpp b/packages/stk/stk_mesh/stk_mesh/base/FieldBase.hpp index e924ec240e15..58aee32d04e9 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/FieldBase.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/FieldBase.hpp @@ -280,7 +280,7 @@ class FieldBase return std::any_cast(m_stkFieldSyncDebugger); } - void rotate_multistate_data(); + void rotate_multistate_data(bool rotateNgpFieldViews = false); private: unsigned legacy_field_array_rank() const; diff --git a/packages/stk/stk_mesh/stk_mesh/base/FieldParallel.cpp b/packages/stk/stk_mesh/stk_mesh/base/FieldParallel.cpp index 2a3816a7f2a6..1b1ab32d91fe 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/FieldParallel.cpp +++ b/packages/stk/stk_mesh/stk_mesh/base/FieldParallel.cpp @@ -332,7 +332,8 @@ void communicate_field_data(const BulkData& mesh , } parallel_data_exchange_nonsym_known_sizes_t(sendOffsets, send_data, - recvOffsets, recv_data, mesh.parallel()); + recvOffsets, recv_data, mesh.parallel(), + mesh.is_mesh_consistency_check_on()); //now unpack and store the recvd data for(int fi=0; fi(stk::mesh::field_data(*field, index.bucket_id, index.bucket_ord)); + T* data = static_cast(stk::mesh::field_data(*field, index.bucket->bucket_id(), index.bucketOrd)); STK_ThrowAssert(data); return data[component]; } +#endif - T& operator()(const HostMesh::MeshIndex& index, int component, + T& operator()(const stk::mesh::FastMeshIndex& index, int component, const char * fileName = HOST_DEBUG_FILE_NAME, int lineNumber = HOST_DEBUG_LINE_NUMBER) const { - T* data = static_cast(stk::mesh::field_data(*field, index.bucket->bucket_id(), index.bucketOrd)); + T *data = static_cast(stk::mesh::field_data(*field, index.bucket_id, index.bucket_ord)); STK_ThrowAssert(data); return data[component]; } @@ -263,10 +267,13 @@ class HostField : public NgpFieldBase FieldState state() const { return field->state(); } - void rotate_multistate_data() override { } +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after May 2024 + STK_DEPRECATED void rotate_multistate_data() override { } +#endif void update_bucket_pointer_view() override { } + void swap_field_views(NgpFieldBase *other) override { } void swap(HostField &other) { } stk::mesh::EntityRank get_rank() const { return field ? field->entity_rank() : stk::topology::INVALID_RANK; } diff --git a/packages/stk/stk_mesh/stk_mesh/base/HostMesh.hpp b/packages/stk/stk_mesh/stk_mesh/base/HostMesh.hpp index b22ee208665e..359b531a8cee 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/HostMesh.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/HostMesh.hpp @@ -34,9 +34,9 @@ #ifndef STK_MESH_HOSTMESH_HPP #define STK_MESH_HOSTMESH_HPP -#include "stk_mesh/base/NgpMeshBase.hpp" #include #include +#include "stk_mesh/base/NgpMeshBase.hpp" #include "stk_mesh/base/Bucket.hpp" #include "stk_mesh/baseImpl/BucketRepository.hpp" #include "stk_mesh/base/Entity.hpp" @@ -56,17 +56,23 @@ namespace stk { namespace mesh { +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after April 2024 struct HostMeshIndex { const stk::mesh::Bucket *bucket; size_t bucketOrd; }; +#endif class HostMesh : public NgpMeshBase { public: using MeshExecSpace = stk::ngp::HostExecSpace; +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after April 2024 using MeshIndex = HostMeshIndex; +#else + using MeshIndex = FastMeshIndex; +#endif using BucketType = stk::mesh::Bucket; using ConnectedNodes = util::StridedArray; using ConnectedEntities = util::StridedArray; @@ -82,7 +88,8 @@ class HostMesh : public NgpMeshBase HostMesh(const stk::mesh::BulkData& b) : NgpMeshBase(), - bulk(&b) + bulk(&b), + m_syncCountWhenUpdated(bulk->synchronized_count()) { require_ngp_mesh_rank_limit(bulk->mesh_meta_data()); } @@ -96,6 +103,7 @@ class HostMesh : public NgpMeshBase void update_mesh() override { + m_syncCountWhenUpdated = bulk->synchronized_count(); } unsigned get_spatial_dimension() const @@ -124,11 +132,14 @@ class HostMesh : public NgpMeshBase return (*(bulk->buckets(rank)[meshIndex.bucket_id]))[meshIndex.bucket_ord]; } +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after April 2024 + STK_DEPRECATED ConnectedNodes get_nodes(const MeshIndex &elem) const { const stk::mesh::Bucket& bucket = *elem.bucket; return ConnectedNodes(bucket.begin_nodes(elem.bucketOrd), bucket.num_nodes(elem.bucketOrd)); } +#endif ConnectedEntities get_connected_entities(stk::mesh::EntityRank rank, const stk::mesh::FastMeshIndex &entity, stk::mesh::EntityRank connectedRank) const { @@ -214,10 +225,12 @@ class HostMesh : public NgpMeshBase return stk::mesh::FastMeshIndex{meshIndex.bucket->bucket_id(), static_cast(meshIndex.bucket_ordinal)}; } - stk::mesh::FastMeshIndex host_mesh_index(stk::mesh::Entity entity) const +#ifndef STK_HIDE_DEPRECATED_CODE +STK_DEPRECATED stk::mesh::FastMeshIndex host_mesh_index(stk::mesh::Entity entity) const { return fast_mesh_index(entity); } +#endif stk::mesh::FastMeshIndex device_mesh_index(stk::mesh::Entity entity) const { @@ -273,10 +286,14 @@ class HostMesh : public NgpMeshBase return *bulk; } - bool is_up_to_date() const { return true; } + bool is_up_to_date() const + { + return m_syncCountWhenUpdated == bulk->synchronized_count(); + } private: const stk::mesh::BulkData *bulk; + size_t m_syncCountWhenUpdated; }; } diff --git a/packages/stk/stk_mesh/stk_mesh/base/MeshDiagnostics.cpp b/packages/stk/stk_mesh/stk_mesh/base/MeshDiagnostics.cpp index ca829c2b115c..a65a975ca40d 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/MeshDiagnostics.cpp +++ b/packages/stk/stk_mesh/stk_mesh/base/MeshDiagnostics.cpp @@ -284,7 +284,8 @@ void unpack_side_nodes_and_check_for_attached_locally_owned_elements(const stk:: nodes[i] = bulkData.get_entity(nodeKeys[i]); stk::mesh::EntityVector connectedElements; - stk::mesh::impl::find_locally_owned_elements_these_nodes_have_in_common(bulkData, nodes.size(), &nodes[0], connectedElements); + stk::mesh::impl::find_locally_owned_elements_these_nodes_have_in_common( + bulkData, nodes.size(), nodes.data(), connectedElements); sideKeyProcMap[std::make_pair(sideKey, proc_id)] = !connectedElements.empty(); } @@ -314,7 +315,8 @@ void unpack_side_nodes_and_check_for_elements(const stk::mesh::BulkData& bulkDat nodes[i] = bulkData.get_entity(nodeKeys[i]); stk::mesh::EntityVector connectedElements; - stk::mesh::impl::find_entities_these_nodes_have_in_common(bulkData, stk::topology::ELEM_RANK, nodes.size(), &nodes[0], connectedElements); + stk::mesh::impl::find_entities_these_nodes_have_in_common( + bulkData, stk::topology::ELEM_RANK, nodes.size(), nodes.data(), connectedElements); sideKeyProcMap[std::make_pair(sideKey, proc_id)] = !connectedElements.empty(); } diff --git a/packages/stk/stk_mesh/stk_mesh/base/MetaData.cpp b/packages/stk/stk_mesh/stk_mesh/base/MetaData.cpp index 71c8f54b5980..f5c58d75ddda 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/MetaData.cpp +++ b/packages/stk/stk_mesh/stk_mesh/base/MetaData.cpp @@ -42,7 +42,6 @@ #include // for BulkData #include // for CommBuffer, etc #include // for Reduce, ReduceMin, etc -#include // for equal_case #include "Shards_BasicTopologies.hpp" // for getCellTopologyData, etc #include "stk_mesh/base/Bucket.hpp" // for Bucket #include "stk_mesh/base/EntityKey.hpp" // for EntityKey @@ -787,10 +786,11 @@ class VerifyConsistency } }); + m_errStream.str(""); //clear int ok = unpack_verify(comm.recv_buffer()); stk::all_reduce(m_parallelMachine, ReduceMin<1>(&ok)); - STK_ThrowRequireMsg(ok, "[p" << m_pRank << "] MetaData parallel consistency failure"); + STK_ThrowRequireMsg(ok, "[p" << m_pRank << "] MetaData parallel consistency failure: "<name() + m_errStream << "[p" << m_pRank << "] Part name (" << localPart->name() << ") does not match Part name (" << m_rootPartName << ") on root processor" << std::endl; } return nameMatches; @@ -914,7 +915,7 @@ class VerifyPartConsistency : public VerifyConsistency bool partRankMatches = (localPartRank == m_rootPartRank); if (!partRankMatches) { - std::cerr << "[p" << m_pRank << "] Part " << localPart->name() << " rank (" << localPartRank + m_errStream << "[p" << m_pRank << "] Part " << localPart->name() << " rank (" << localPartRank << ") does not match Part " << m_rootPartName << " rank (" << m_rootPartRank << ") on root processor" << std::endl; } return partRankMatches; @@ -926,7 +927,7 @@ class VerifyPartConsistency : public VerifyConsistency bool partTopologyMatches = (localPartTopology == m_rootPartTopology); if (!partTopologyMatches) { - std::cerr << "[p" << m_pRank << "] Part " << localPart->name() << " topology (" << localPartTopology + m_errStream << "[p" << m_pRank << "] Part " << localPart->name() << " topology (" << localPartTopology << ") does not match Part " << m_rootPartName << " topology (" << stk::topology(m_rootPartTopology) << ") on root processor" << std::endl; } @@ -946,15 +947,15 @@ class VerifyPartConsistency : public VerifyConsistency } if (!subsetMatches) { - std::cerr << "[p" << m_pRank << "] Part " << localPart->name() << " subset ordinals ("; + m_errStream << "[p" << m_pRank << "] Part " << localPart->name() << " subset ordinals ("; for (const stk::mesh::Part * subsetPart : localSubsetParts) { - std::cerr << subsetPart->mesh_meta_data_ordinal() << " "; + m_errStream << subsetPart->mesh_meta_data_ordinal() << " "; } - std::cerr << ") does not match Part " << m_rootPartName << " subset ordinals ("; + m_errStream << ") does not match Part " << m_rootPartName << " subset ordinals ("; for (const unsigned rootPartSubsetOrdinal : m_rootPartSubsetOrdinals) { - std::cerr << rootPartSubsetOrdinal << " "; + m_errStream << rootPartSubsetOrdinal << " "; } - std::cerr << ") on root processor" << std::endl; + m_errStream << ") on root processor" << std::endl; } return subsetMatches; @@ -965,7 +966,7 @@ class VerifyPartConsistency : public VerifyConsistency bool noExtraParts = true; for (const stk::mesh::Part * part : unprocessedParts) { if (!stk::mesh::impl::is_internal_part(*part)) { - std::cerr << "[p" << m_pRank << "] Have extra Part (" << part->name() << ") that does not exist on root processor" << std::endl; + m_errStream << "[p" << m_pRank << "] Have extra Part (" << part->name() << ") that does not exist on root processor" << std::endl; noExtraParts = false; } } @@ -1061,7 +1062,7 @@ class VerifyFieldConsistency : public VerifyConsistency { bool localFieldValid = true; if (localField == nullptr) { - std::cerr << "[p" << m_pRank << "] Received extra Field (" << m_rootFieldName << ") from root processor" << std::endl; + m_errStream << "[p" << m_pRank << "] Received extra Field (" << m_rootFieldName << ") from root processor" << std::endl; localFieldValid = false; } return localFieldValid; @@ -1075,7 +1076,7 @@ class VerifyFieldConsistency : public VerifyConsistency bool fieldNameMatches = fieldNameLengthMatches && (strncmp(localFieldNamePtr, m_rootFieldName, localFieldNameLen-1) == 0); if (!fieldNameMatches) { - std::cerr << "[p" << m_pRank << "] Field name (" << localField->name() + m_errStream << "[p" << m_pRank << "] Field name (" << localField->name() << ") does not match Field name (" << m_rootFieldName << ") on root processor" << std::endl; } return fieldNameMatches; @@ -1087,7 +1088,7 @@ class VerifyFieldConsistency : public VerifyConsistency bool fieldRankMatches = (localFieldRank == m_rootFieldRank); if (!fieldRankMatches) { - std::cerr << "[p" << m_pRank << "] Field " << localField->name() << " rank (" << localFieldRank + m_errStream << "[p" << m_pRank << "] Field " << localField->name() << " rank (" << localFieldRank << ") does not match Field " << m_rootFieldName << " rank (" << m_rootFieldRank << ") on root processor" << std::endl; } return fieldRankMatches; @@ -1099,7 +1100,7 @@ class VerifyFieldConsistency : public VerifyConsistency bool fieldNumberOfStatesMatches = (localNumberOfStates == m_rootFieldNumberOfStates); if (!fieldNumberOfStatesMatches) { - std::cerr << "[p" << m_pRank << "] Field " << localField->name() << " number of states (" << localNumberOfStates + m_errStream << "[p" << m_pRank << "] Field " << localField->name() << " number of states (" << localNumberOfStates << ") does not match Field " << m_rootFieldName << " number of states (" << m_rootFieldNumberOfStates << ") on root processor" << std::endl; } @@ -1110,7 +1111,7 @@ class VerifyFieldConsistency : public VerifyConsistency { bool noExtraFields = true; for (const stk::mesh::FieldBase * field : unprocessedFields) { - std::cerr << "[p" << m_pRank << "] Have extra Field (" << field->name() << ") that does not exist on root processor" << std::endl; + m_errStream << "[p" << m_pRank << "] Have extra Field (" << field->name() << ") that does not exist on root processor" << std::endl; noExtraFields = false; } return noExtraFields; diff --git a/packages/stk/stk_mesh/stk_mesh/base/MetaData.hpp b/packages/stk/stk_mesh/stk_mesh/base/MetaData.hpp index 0cd9082cf484..45fc78bd8266 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/MetaData.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/MetaData.hpp @@ -1069,7 +1069,7 @@ MetaData::legacy_declare_field(stk::topology::rank_t arg_entity_rank, if (f[0] != nullptr) { for ( unsigned i = 1 ; i < number_of_states ; ++i ) { - f[i] = &f[0]->field_of_state(static_cast(i)); + f[i] = &(f[0]->field_of_state(static_cast(i))); } } else { diff --git a/packages/stk/stk_mesh/stk_mesh/base/ModificationNotifier.cpp b/packages/stk/stk_mesh/stk_mesh/base/ModificationNotifier.cpp index 479a3b75f823..151c76ff6a87 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/ModificationNotifier.cpp +++ b/packages/stk/stk_mesh/stk_mesh/base/ModificationNotifier.cpp @@ -16,13 +16,17 @@ std::vector get_global_max(const std::vector &allObserverValues, return maxValues; } -void ModificationNotifier::reduce_values_for_observers(MPI_Comm communicator) +bool ModificationNotifier::reduce_values_for_observers(MPI_Comm communicator) { - std::vector allObserverValues; - std::vector > observerValues(observers.size()); - get_values_to_reduce_from_observers(allObserverValues, observerValues); - std::vector maxValues = get_global_max(allObserverValues, communicator); - set_max_values_on_observers(maxValues, observerValues); + m_allObserverValues.clear(); + m_observerValues.resize(observers.size()); + get_values_to_reduce_from_observers(m_allObserverValues, m_observerValues); + size_t localAnyModification = anyModification ? 1 : 0; + m_allObserverValues.push_back(localAnyModification); + std::vector maxValues = get_global_max(m_allObserverValues, communicator); + bool globalAnyModification = maxValues.back()==1 ? true : false; + set_max_values_on_observers(maxValues, m_observerValues); + return globalAnyModification; } void ModificationNotifier::get_values_to_reduce_from_observers(std::vector &allObserverValues, @@ -31,7 +35,9 @@ void ModificationNotifier::get_values_to_reduce_from_observers(std::vectorfill_values_to_reduce(observerValues[i]); - allObserverValues.insert(allObserverValues.end(), observerValues[i].begin(), observerValues[i].end()); + if (!observerValues[i].empty()) { + allObserverValues.insert(allObserverValues.end(), observerValues[i].begin(), observerValues[i].end()); + } } } void ModificationNotifier::set_max_values_on_observers(const std::vector &maxValues, diff --git a/packages/stk/stk_mesh/stk_mesh/base/ModificationNotifier.hpp b/packages/stk/stk_mesh/stk_mesh/base/ModificationNotifier.hpp index b452f3db0e73..f0b67e2bfac4 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/ModificationNotifier.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/ModificationNotifier.hpp @@ -37,6 +37,7 @@ #include #include #include +#include #include #include @@ -80,6 +81,7 @@ class ModificationNotifier { observer->entity_added(entity); } + anyModification = true; } void notify_entity_deleted(stk::mesh::Entity entity) @@ -88,6 +90,7 @@ class ModificationNotifier { observer->entity_deleted(entity); } + anyModification = true; } void notify_entity_parts_added(stk::mesh::Entity entity, const stk::mesh::OrdinalVector& parts) @@ -96,6 +99,7 @@ class ModificationNotifier { observer->entity_parts_added(entity, parts); } + anyModification = true; } void notify_entity_parts_removed(stk::mesh::Entity entity, const stk::mesh::OrdinalVector& parts) @@ -104,6 +108,7 @@ class ModificationNotifier { observer->entity_parts_removed(entity, parts); } + anyModification = true; } void notify_modification_begin() @@ -122,13 +127,15 @@ class ModificationNotifier } } - void notify_finished_modification_end(MPI_Comm communicator) + bool notify_finished_modification_end(MPI_Comm communicator) { - reduce_values_for_observers(communicator); + bool globalAnyModification = reduce_values_for_observers(communicator); for(std::shared_ptr& observer : observers) { observer->finished_modification_end_notification(); } + anyModification = false; + return globalAnyModification; } void notify_elements_about_to_move_procs(const stk::mesh::EntityProcVec &elemProcPairsToMove) @@ -137,6 +144,7 @@ class ModificationNotifier { observer->elements_about_to_move_procs_notification(elemProcPairsToMove); } + anyModification = true; } void notify_elements_moved_procs(const stk::mesh::EntityProcVec &elemProcPairsToMove) @@ -145,6 +153,7 @@ class ModificationNotifier { observer->elements_moved_procs_notification(elemProcPairsToMove); } + anyModification = true; } void notify_local_entities_created_or_deleted(stk::mesh::EntityRank rank) @@ -153,6 +162,7 @@ class ModificationNotifier { observer->local_entities_created_or_deleted_notification(rank); } + anyModification = true; } void notify_local_entity_comm_info_changed(stk::mesh::EntityRank rank) @@ -161,6 +171,7 @@ class ModificationNotifier { observer->local_entity_comm_info_changed_notification(rank); } + anyModification = true; } void notify_local_buckets_changed(stk::mesh::EntityRank rank) @@ -169,6 +180,7 @@ class ModificationNotifier { observer->local_buckets_changed_notification(rank); } + anyModification = true; } template @@ -210,6 +222,7 @@ class ModificationNotifier { observer->relation_destroyed(from, to, ordinal); } + anyModification = true; } void notify_relation_declared(Entity from, Entity to, ConnectivityOrdinal ordinal) @@ -218,14 +231,19 @@ class ModificationNotifier { observer->relation_declared(from, to, ordinal); } + anyModification = true; } private: std::vector> observers; - void reduce_values_for_observers(MPI_Comm communicator); + bool reduce_values_for_observers(MPI_Comm communicator); void get_values_to_reduce_from_observers(std::vector &allObserverValues, std::vector >& observerValues); void set_max_values_on_observers(const std::vector &maxValues, std::vector >& observerValues); + + std::vector m_allObserverValues; + std::vector> m_observerValues; + bool anyModification = false; }; } diff --git a/packages/stk/stk_mesh/stk_mesh/base/NgpAtomics.hpp b/packages/stk/stk_mesh/stk_mesh/base/NgpAtomics.hpp index 4d7569013c4e..4afe077a769f 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/NgpAtomics.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/NgpAtomics.hpp @@ -43,7 +43,7 @@ namespace mesh { template KOKKOS_FUNCTION void atomic_add(T *dest, const T src) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_OPENMP) || defined(KOKKOS_ENABLE_HIP) +#if defined(STK_ENABLE_GPU) || defined(KOKKOS_ENABLE_OPENMP) Kokkos::atomic_add(dest, src); #else *dest += src; @@ -53,7 +53,7 @@ void atomic_add(T *dest, const T src) template KOKKOS_FUNCTION void atomic_sub(T *dest, const T src) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_OPENMP) || defined(KOKKOS_ENABLE_HIP) +#if defined(STK_ENABLE_GPU) || defined(KOKKOS_ENABLE_OPENMP) Kokkos::atomic_sub(dest, src); #else *dest -= src; @@ -63,7 +63,7 @@ void atomic_sub(T *dest, const T src) template KOKKOS_FUNCTION void atomic_mul(T *dest, const T src) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_OPENMP) || defined(KOKKOS_ENABLE_HIP) +#if defined(STK_ENABLE_GPU) || defined(KOKKOS_ENABLE_OPENMP) Kokkos::atomic_mul(dest, src); #else *dest *= src; diff --git a/packages/stk/stk_mesh/stk_mesh/base/NgpFieldBase.hpp b/packages/stk/stk_mesh/stk_mesh/base/NgpFieldBase.hpp index bf72b22db08e..46fbbc9f8622 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/NgpFieldBase.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/NgpFieldBase.hpp @@ -19,8 +19,11 @@ class NgpFieldBase KOKKOS_FUNCTION NgpFieldBase& operator=(const NgpFieldBase&) { return *this; } KOKKOS_FUNCTION NgpFieldBase& operator=(NgpFieldBase&&) { return *this; } KOKKOS_FUNCTION virtual ~NgpFieldBase() {} - virtual void rotate_multistate_data() = 0; +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after May 2024 + STK_DEPRECATED virtual void rotate_multistate_data() = 0; +#endif virtual void update_bucket_pointer_view() = 0; + virtual void swap_field_views(NgpFieldBase*) = 0; virtual void modify_on_host() = 0; virtual void modify_on_host(const Selector& selector) = 0; virtual void modify_on_device() = 0; diff --git a/packages/stk/stk_mesh/stk_mesh/base/NgpFieldSyncDebugger.hpp b/packages/stk/stk_mesh/stk_mesh/base/NgpFieldSyncDebugger.hpp index e85727613a81..3f42f77c62b9 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/NgpFieldSyncDebugger.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/NgpFieldSyncDebugger.hpp @@ -102,10 +102,6 @@ class EmptyNgpFieldSyncDebugger KOKKOS_INLINE_FUNCTION void device_stale_access_check(NgpField *, const stk::mesh::FastMeshIndex &, int, const char *, int) const {} - template - KOKKOS_INLINE_FUNCTION - void device_stale_access_check(NgpField *, const stk::mesh::DeviceMesh::MeshIndex &, int, const char *, int) const {} - template KOKKOS_INLINE_FUNCTION void device_stale_access_check(NgpField *, const stk::mesh::FastMeshIndex &, const char *, int) const {} @@ -230,23 +226,6 @@ class NgpFieldSyncDebugger } } - template - KOKKOS_FUNCTION - void device_stale_access_check(NgpField* ngpField, const stk::mesh::DeviceMesh::MeshIndex& index, int component, - const char* fileName, int lineNumber) const - { - anyPotentialDeviceFieldModification() = true; - - if (field_not_updated_after_mesh_mod(ngpField->synchronizedCount)) { - print_unupdated_field_warning(fileName, lineNumber); - return; - } - - if (data_is_stale_on_device(index, component)) { - print_stale_data_warning(ngpField, index.bucket->bucket_id(), index.bucketOrd, component, fileName, lineNumber); - } - } - template KOKKOS_FUNCTION void device_stale_access_check(NgpField* ngpField, const stk::mesh::FastMeshIndex& index, @@ -439,11 +418,6 @@ class NgpFieldSyncDebugger return data_is_stale_on_device(index.bucket_id, index.bucket_ord, component); } - KOKKOS_INLINE_FUNCTION - bool data_is_stale_on_device(const stk::mesh::DeviceMesh::MeshIndex & index, int component) const { - return data_is_stale_on_device(index.bucket->bucket_id(), index.bucketOrd, component); - } - KOKKOS_INLINE_FUNCTION bool data_is_stale_on_device(int bucketId, int bucketOrdinal, int component) const { return !(lastFieldModLocation(debugDeviceSelectedBucketOffset(bucketId), diff --git a/packages/stk/stk_mesh/stk_mesh/base/NgpForEachEntity.hpp b/packages/stk/stk_mesh/stk_mesh/base/NgpForEachEntity.hpp index f7aea128cbc6..8ff9ff21bd9d 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/NgpForEachEntity.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/NgpForEachEntity.hpp @@ -192,10 +192,14 @@ struct TeamFunctor { template void for_each_entity_run(Mesh &mesh, stk::topology::rank_t rank, const stk::mesh::Selector &selector, const AlgorithmPerEntity &functor) { + Kokkos::Profiling::pushRegion("for_each_entity_run with selector"); + stk::NgpVector bucketIds = mesh.get_bucket_ids(rank, selector); unsigned numBuckets = bucketIds.size(); Kokkos::parallel_for(stk::ngp::TeamPolicy(numBuckets, Kokkos::AUTO), TeamFunctor(mesh, rank, bucketIds, functor)); + + Kokkos::Profiling::popRegion(); } } diff --git a/packages/stk/stk_mesh/stk_mesh/base/NgpTypes.hpp b/packages/stk/stk_mesh/stk_mesh/base/NgpTypes.hpp index 9dd0cf7a97a1..b0bc4e188cdf 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/NgpTypes.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/NgpTypes.hpp @@ -45,12 +45,14 @@ namespace mesh { using DeviceCommMapIndices = Kokkos::View; using EntityKeyViewType = Kokkos::View; using EntityViewType = Kokkos::View; +using HostEntityViewType = Kokkos::View>; using BucketConnectivityType = Kokkos::View; using UnsignedViewType = Kokkos::View; using Unsigned2dViewType = Kokkos::View; using BoolViewType = Kokkos::View; using OrdinalViewType = Kokkos::View; using PartOrdinalViewType = Kokkos::View; +using HostPartOrdinalViewType = Kokkos::View>; using PermutationViewType = Kokkos::View; using FastSharedCommMapViewType = Kokkos::View; using HostMeshIndexType = Kokkos::View::HostMirror; @@ -66,7 +68,7 @@ using FieldDataPointerDeviceViewType = Kokkos::View using UnmanagedHostInnerView = Kokkos::View>; template using UnmanagedDevInnerView = Kokkos::View>; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) +#ifdef STK_USE_DEVICE_MESH #define ORDER_INDICES(i,j) j,i #else #define ORDER_INDICES(i,j) i,j diff --git a/packages/stk/stk_mesh/stk_mesh/base/NgpUtils.hpp b/packages/stk/stk_mesh/stk_mesh/base/NgpUtils.hpp index 0740411f38a4..98fe25d3a9b0 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/NgpUtils.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/NgpUtils.hpp @@ -70,11 +70,14 @@ inline stk::NgpVector get_bucket_ids(const stk::mesh::BulkData &bulk, stk::mesh::EntityRank rank, const stk::mesh::Selector &selector) { + Kokkos::Profiling::pushRegion("get_bucket_ids"); const stk::mesh::BucketVector &buckets = bulk.get_buckets(rank, selector); stk::NgpVector bucketIds(buckets.size()); - for(size_t i=0; ibucket_id(); + } bucketIds.copy_host_to_device(); + Kokkos::Profiling::popRegion(); return bucketIds; } diff --git a/packages/stk/stk_mesh/stk_mesh/base/Selector.cpp b/packages/stk/stk_mesh/stk_mesh/base/Selector.cpp index 3e761a3ea8f7..d92ae7446dc0 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/Selector.cpp +++ b/packages/stk/stk_mesh/stk_mesh/base/Selector.cpp @@ -729,7 +729,9 @@ Selector selectUnion( const VectorType & union_vector ) { if constexpr(std::is_same_v || std::is_same_v) { OrdinalVector partOrdinals; + partOrdinals.reserve(union_vector.size()); OrdinalVector subsetPartOrdinals; + subsetPartOrdinals.reserve(union_vector.size()); const MetaData* metaPtr = nullptr; for(const Part* part : union_vector) { if (part == nullptr) continue; @@ -749,30 +751,6 @@ Selector selectUnion( const VectorType & union_vector ) stk::util::sort_and_unique(partOrdinals); stk::util::sort_and_unique(subsetPartOrdinals); Selector selector(SelectorNodeType::PART_UNION, metaPtr, partOrdinals, subsetPartOrdinals); -// if (partOrdinals.size() > 1) { -// Selector selectorOld; -// bool foundFirstNonNullptr = false; -// for(unsigned i=0; ihas_mesh()) { -// for(EntityRank rank : {stk::topology::NODE_RANK,stk::topology::FACE_RANK,stk::topology::ELEM_RANK}) { -// const BucketVector& bkts = selector.get_buckets(rank); -// const BucketVector& bktsOld = selectorOld.get_buckets(rank); -// STK_ThrowRequireMsg(bkts.size()==bktsOld.size(),"ERROR, selector: "< skinnedSideSetVector = SkinMeshUtil::get_skinned_sideset(bulkData, blocksToSkin); + std::vector interiorSideSetVector = SkinMeshUtil::get_interior_sideset(bulkData, blocksToSkin); + + SideSet sideSet(bulkData, skinnedSideSetVector); + bulkData.modification_begin(); + bool doLocalModCycle = false; + FaceCreator(bulkData, bulkData.get_face_adjacent_element_graph()).create_side_entities_given_sideset(sideSet, partToPutSidesInto, doLocalModCycle); + + const stk::mesh::PartVector& interiorSkinPart = separatePartForInteriorSides == nullptr ? partToPutSidesInto : *separatePartForInteriorSides; + SideSet interiorSideSet(bulkData, interiorSideSetVector); + doLocalModCycle = true; + FaceCreator(bulkData, bulkData.get_face_adjacent_element_graph()).create_side_entities_given_sideset(interiorSideSet, interiorSkinPart, doLocalModCycle); +} + void create_interior_block_boundary_sides(stk::mesh::BulkData &bulkData, const stk::mesh::Selector &blocksToConsider, const stk::mesh::PartVector& partToPutSidesInto) { STK_ThrowRequireMsg(bulkData.in_synchronized_state(), "Cannot use create_interior_block_boundary_sides while in another mod cycle."); diff --git a/packages/stk/stk_mesh/stk_mesh/base/SkinBoundary.hpp b/packages/stk/stk_mesh/stk_mesh/base/SkinBoundary.hpp index 07451a0f5f21..31a42cd47c49 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/SkinBoundary.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/SkinBoundary.hpp @@ -47,6 +47,7 @@ namespace stk { namespace mesh { * API interface to skin the boundary. * */ +void create_all_block_boundary_sides(BulkData &bulkData, const Selector& blocksToSkin, const PartVector& partToPutSidesInto, const PartVector* separatePartForInteriorSides = nullptr); void create_exposed_block_boundary_sides(BulkData &bulkData, const Selector& blocksToSkin, const PartVector& partToPutSidesInto); void create_exposed_block_boundary_sides(BulkData &bulkData, const Selector& blocksToSkin, const PartVector& partToPutSidesInto, const Selector& air); void create_interior_block_boundary_sides(BulkData&, const Selector& blocksToConsider, const PartVector& partToPutSidesInto); diff --git a/packages/stk/stk_mesh/stk_mesh/base/SkinMeshUtil.cpp b/packages/stk/stk_mesh/stk_mesh/base/SkinMeshUtil.cpp index f5ef8ecec5a3..0bc9245fb316 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/SkinMeshUtil.cpp +++ b/packages/stk/stk_mesh/stk_mesh/base/SkinMeshUtil.cpp @@ -110,7 +110,7 @@ bool SkinMeshUtil::is_element_selected_and_can_have_side(const stk::mesh::Select } void SkinMeshUtil::mark_local_connections(const stk::mesh::GraphEdge &graphEdge, - std::vector &isOnlyConnectedRemotely) + std::vector &isOnlyConnectedRemotely) { if(impl::is_local_element(graphEdge.elem2())) { @@ -121,7 +121,7 @@ void SkinMeshUtil::mark_local_connections(const stk::mesh::GraphEdge &graphEdge, } void SkinMeshUtil::mark_remote_connections(const stk::mesh::GraphEdge &graphEdge, - std::vector &isConnectedToRemoteElementInBodyToSkin) + std::vector &isConnectedToRemoteElementInBodyToSkin) { if(!impl::is_local_element(graphEdge.elem2())) { @@ -132,8 +132,8 @@ void SkinMeshUtil::mark_remote_connections(const stk::mesh::GraphEdge &graphEdge } void SkinMeshUtil::mark_sides_exposed_on_other_procs(const stk::mesh::GraphEdge &graphEdge, - std::vector &isConnectedToRemoteElementInBodyToSkin, - std::vector &isOnlyConnectedRemotely) + std::vector &isConnectedToRemoteElementInBodyToSkin, + std::vector &isOnlyConnectedRemotely) { mark_local_connections(graphEdge, isOnlyConnectedRemotely); mark_remote_connections(graphEdge, isConnectedToRemoteElementInBodyToSkin); @@ -143,18 +143,18 @@ void SkinMeshUtil::get_sides_exposed_on_other_procs(stk::mesh::impl::LocalId loc int numElemSides, std::vector& exposedSides) { - std::vector isConnectedToRemoteElementInBodyToSkin(numElemSides, false); - std::vector isOnlyConnectedRemotely(numElemSides, true); + m_isConnectedToRemoteElem.assign(numElemSides, false); + m_isOnlyConnectedRemotely.assign(numElemSides, true); for(const stk::mesh::GraphEdge &graphEdge : eeGraph.get_edges_for_element(localId)) - mark_sides_exposed_on_other_procs(graphEdge, isConnectedToRemoteElementInBodyToSkin, isOnlyConnectedRemotely); + mark_sides_exposed_on_other_procs(graphEdge, m_isConnectedToRemoteElem, m_isOnlyConnectedRemotely); for(const stk::mesh::GraphEdge& graphEdge : eeGraph.get_coincident_edges_for_element(localId)) - mark_local_connections(graphEdge, isOnlyConnectedRemotely); + mark_local_connections(graphEdge, m_isOnlyConnectedRemotely); exposedSides.clear(); for(int side = 0; side < numElemSides; side++) { - if(isConnectedToRemoteElementInBodyToSkin[side] && isOnlyConnectedRemotely[side]) + if(m_isConnectedToRemoteElem[side] && m_isOnlyConnectedRemotely[side]) { exposedSides.push_back(side); } @@ -248,7 +248,7 @@ std::vector SkinMeshUtil::extract_interior_sideset() stk::mesh::impl::ParallelPartInfo parallelPartInfo; stk::mesh::impl::populate_part_ordinals_for_remote_edges(bulkData, eeGraph, parallelPartInfo); stk::mesh::EntityVector sideNodes; - std::vector scratchOrdinals1, scratchOrdinals2; + std::vector elemPartOrdinals, scratchOrdinals2; stk::mesh::Selector ownedSkinSelector = bulkData.mesh_meta_data().locally_owned_part() & skinSelector; const stk::mesh::BucketVector& buckets = bulkData.get_buckets(stk::topology::ELEM_RANK, ownedSkinSelector); @@ -257,6 +257,7 @@ std::vector SkinMeshUtil::extract_interior_sideset() for(stk::mesh::Entity element : *bucket) { impl::LocalId elementId = eeGraph.get_local_element_id(element); + stk::mesh::impl::get_element_block_part_ordinals(element, bulkData, elemPartOrdinals); for(const stk::mesh::GraphEdge & graphEdge : eeGraph.get_edges_for_element(elementId)) { @@ -271,16 +272,15 @@ std::vector SkinMeshUtil::extract_interior_sideset() { isElement2InSelector = remoteSkinSelector[graphEdge.elem2()]; if(!isElement2InSelector) continue; - should_add_side = !stk::mesh::impl::are_entity_element_blocks_equivalent(bulkData, element, parallelPartInfo[graphEdge.elem2()].elementPartOrdinals, scratchOrdinals1); + should_add_side = !(elemPartOrdinals == parallelPartInfo[graphEdge.elem2()].elementPartOrdinals); } else { otherElement = eeGraph.get_entity(graphEdge.elem2()); otherEntityId = bulkData.identifier(otherElement); - isElement2InSelector = skinSelector(bulkData.bucket(otherElement)); - if(!isElement2InSelector) continue; - if(!isParallelEdge && bulkData.identifier(element) < otherEntityId) - should_add_side = !stk::mesh::impl::are_entity_element_blocks_equivalent(bulkData, element, otherElement, scratchOrdinals1, scratchOrdinals2); + if(bulkData.identifier(element) < otherEntityId && skinSelector(bulkData.bucket(otherElement))) { + should_add_side = !stk::mesh::impl::are_entity_element_blocks_equivalent(bulkData, otherElement, elemPartOrdinals, scratchOrdinals2); + } } if(should_add_side && checkIfSideIsNotCollapsed(sideNodes, *bucket, bulkData, element, graphEdge.side1())) diff --git a/packages/stk/stk_mesh/stk_mesh/base/SkinMeshUtil.hpp b/packages/stk/stk_mesh/stk_mesh/base/SkinMeshUtil.hpp index 202981450beb..5968d61e7c9d 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/SkinMeshUtil.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/SkinMeshUtil.hpp @@ -88,14 +88,14 @@ class SkinMeshUtil { std::vector& exposedSides); void mark_local_connections(const stk::mesh::GraphEdge &graphEdge, - std::vector &isOnlyConnectedRemotely); + std::vector &isOnlyConnectedRemotely); void mark_remote_connections(const stk::mesh::GraphEdge &graphEdge, - std::vector &isConnectedToRemoteElementInBodyToSkin); + std::vector &isConnectedToRemoteElementInBodyToSkin); void mark_sides_exposed_on_other_procs(const stk::mesh::GraphEdge &graphEdge, - std::vector &isConnectedToRemoteElementInBodyToSkin, - std::vector &isOnlyConnectedRemotely); + std::vector &isConnectedToRemoteElementInBodyToSkin, + std::vector &isOnlyConnectedRemotely); stk::mesh::ElemElemGraph& eeGraph; stk::mesh::Selector skinSelector; @@ -103,6 +103,8 @@ class SkinMeshUtil { const bool useAirSelector = false; stk::mesh::Selector airSelector; stk::mesh::impl::ParallelSelectedInfo remoteAirSelector; + std::vector m_isConnectedToRemoteElem; + std::vector m_isOnlyConnectedRemotely; }; } diff --git a/packages/stk/stk_mesh/stk_mesh/baseImpl/BucketRepository.cpp b/packages/stk/stk_mesh/stk_mesh/baseImpl/BucketRepository.cpp index d7c7757165a6..9ef5c3227254 100644 --- a/packages/stk/stk_mesh/stk_mesh/baseImpl/BucketRepository.cpp +++ b/packages/stk/stk_mesh/stk_mesh/baseImpl/BucketRepository.cpp @@ -199,7 +199,7 @@ void BucketRepository::fill_key_ptr(const OrdinalVector& parts, PartOrdinal** ke const size_t keyLen = 2 + part_count; - *keyPtr = &keyTmpBuffer[0]; + *keyPtr = keyTmpBuffer; *keyEnd = *keyPtr+keyLen; if (keyLen >= maxKeyTmpBufferSize) { diff --git a/packages/stk/stk_mesh/stk_mesh/baseImpl/EquivalentEntityBlocks.hpp b/packages/stk/stk_mesh/stk_mesh/baseImpl/EquivalentEntityBlocks.hpp index 2a4b2590ba00..75b04da6e8f4 100644 --- a/packages/stk/stk_mesh/stk_mesh/baseImpl/EquivalentEntityBlocks.hpp +++ b/packages/stk/stk_mesh/stk_mesh/baseImpl/EquivalentEntityBlocks.hpp @@ -47,9 +47,19 @@ namespace stk { namespace mesh { namespace impl { -bool are_entity_element_blocks_equivalent(const stk::mesh::BulkData& bulkData, stk::mesh::Entity entity1, stk::mesh::Entity entity2, std::vector& scratchOrdinals1, std::vector& scratchOrdinals2); -bool are_entity_element_blocks_equivalent(const stk::mesh::BulkData& bulkData, stk::mesh::Entity entity1, const std::vector& other_element_part_ordinals, std::vector& scratchOrdinals); -void get_element_block_part_ordinals(stk::mesh::Entity element, const stk::mesh::BulkData& bulkData, std::vector& partOrdinalsElementBlock); +bool are_entity_element_blocks_equivalent(const stk::mesh::BulkData& bulkData, + stk::mesh::Entity entity1, stk::mesh::Entity entity2, + std::vector& scratchOrdinals1, + std::vector& scratchOrdinals2); + +bool are_entity_element_blocks_equivalent(const stk::mesh::BulkData& bulkData, + stk::mesh::Entity entity1, + const std::vector& other_element_part_ordinals, + std::vector& scratchOrdinals); + +void get_element_block_part_ordinals(stk::mesh::Entity element, + const stk::mesh::BulkData& bulkData, + std::vector& partOrdinalsElementBlock); } } diff --git a/packages/stk/stk_mesh/stk_mesh/baseImpl/MeshImplUtils.cpp b/packages/stk/stk_mesh/stk_mesh/baseImpl/MeshImplUtils.cpp index ce8da3ca485c..eaec56dc3147 100644 --- a/packages/stk/stk_mesh/stk_mesh/baseImpl/MeshImplUtils.cpp +++ b/packages/stk/stk_mesh/stk_mesh/baseImpl/MeshImplUtils.cpp @@ -1328,16 +1328,18 @@ void comm_sync_nonowned_sends( } } -EntityRank get_highest_upward_connected_rank(const Bucket& bucket, +std::pair get_highest_upward_connected_rank(const Bucket& bucket, unsigned bucketOrdinal, EntityRank entityRank, EntityRank maxRank) { - EntityRank highestRank = maxRank; - while(highestRank > entityRank && bucket.num_connectivity(bucketOrdinal, highestRank) == 0) { - highestRank = static_cast(highestRank-1); + std::pair result(maxRank,0); + while(result.first > entityRank && + (result.second = bucket.num_connectivity(bucketOrdinal, result.first)) == 0) + { + result.first = static_cast(result.first-1); } - return highestRank; + return result; } void insert_upward_relations_for_owned(const BulkData& bulk_data, @@ -1353,10 +1355,11 @@ void insert_upward_relations_for_owned(const BulkData& bulk_data, const Bucket& bucket = *idx.bucket; STK_ThrowAssert(bucket.owned()); const unsigned bucketOrd = idx.bucket_ordinal; - const EntityRank upwardRank = get_highest_upward_connected_rank(bucket, bucketOrd, entityRank, maxRank); + const std::pair rankAndNumConn = get_highest_upward_connected_rank(bucket, bucketOrd, entityRank, maxRank); + const EntityRank upwardRank = rankAndNumConn.first; if (upwardRank > entityRank) { - const int numRels = bucket.num_connectivity(bucketOrd, upwardRank); + const int numRels = rankAndNumConn.second; const Entity* rels = bucket.begin(bucketOrd, upwardRank); for (int r = 0; r < numRels; ++r) { diff --git a/packages/stk/stk_mesh/stk_mesh/baseImpl/MeshModification.cpp b/packages/stk/stk_mesh/stk_mesh/baseImpl/MeshModification.cpp index c2335a5f4c53..6a3179dd4dda 100644 --- a/packages/stk/stk_mesh/stk_mesh/baseImpl/MeshModification.cpp +++ b/packages/stk/stk_mesh/stk_mesh/baseImpl/MeshModification.cpp @@ -475,7 +475,7 @@ void MeshModification::delete_shared_entities_which_are_no_longer_in_owned_closu { Entity entity = i->entity; if (m_bulkData.is_valid(entity) && !m_bulkData.owned_closure(entity)) { - if (m_bulkData.in_shared(entity)) { + if (i->entity_comm != -1 && m_bulkData.in_shared(entity)) { destroy_dependent_ghosts(entity, entitiesToRemoveFromSharing, auraEntitiesToDestroy); } } diff --git a/packages/stk/stk_mesh/stk_mesh/baseImpl/PartVectorUtils.hpp b/packages/stk/stk_mesh/stk_mesh/baseImpl/PartVectorUtils.hpp index b66779cdb293..382f02e1a9e2 100644 --- a/packages/stk/stk_mesh/stk_mesh/baseImpl/PartVectorUtils.hpp +++ b/packages/stk/stk_mesh/stk_mesh/baseImpl/PartVectorUtils.hpp @@ -69,7 +69,7 @@ template void fill_remove_parts_and_subsets_minus_parts_in_add_parts_list( const PARTVECTOR & remove_parts, const OrdinalVector & addPartsAndSupersets, - stk::mesh::Bucket &entityBucket, + const stk::mesh::Bucket* entityBucket, OrdinalVector &removePartsAndSubsetsMinusPartsInAddPartsList) { const unsigned expected_min_num_subsets = 3; @@ -82,12 +82,12 @@ void fill_remove_parts_and_subsets_minus_parts_in_add_parts_list( rmPart->mesh_meta_data_ordinal())) { removePartsAndSubsetsMinusPartsInAddPartsList.push_back(rmPart->mesh_meta_data_ordinal()); - const PartVector& subsets = rmPart->subsets(); - for(const Part* subset : subsets) - { - if(entityBucket.member(*subset)) - { - removePartsAndSubsetsMinusPartsInAddPartsList.push_back(subset->mesh_meta_data_ordinal()); + if (entityBucket != nullptr) { + const PartVector& subsets = rmPart->subsets(); + for(const Part* subset : subsets) { + if(entityBucket->member(*subset)) { + removePartsAndSubsetsMinusPartsInAddPartsList.push_back(subset->mesh_meta_data_ordinal()); + } } } } @@ -105,20 +105,16 @@ void fill_remove_parts_and_subsets_minus_parts_in_add_parts_list( const unsigned expected_min_num_subsets = 3; const size_t expectedSizeOfRemovePartList = remove_parts.size() * expected_min_num_subsets; removePartsAndSubsetsMinusPartsInAddPartsList.reserve(expectedSizeOfRemovePartList); - for(const Part* rmPart : remove_parts) - { + for(const Part* rmPart : remove_parts) { if(!contains_ordinal(addPartsAndSupersets.begin(), addPartsAndSupersets.end(), rmPart->mesh_meta_data_ordinal())) { removePartsAndSubsetsMinusPartsInAddPartsList.push_back(rmPart->mesh_meta_data_ordinal()); const PartVector& subsets = rmPart->subsets(); - for(const Part* subset : subsets) - { - for(auto entityBucket : buckets) - { - if(entityBucket->member(*subset)) - { + for(const Part* subset : subsets) { + for(auto entityBucket : buckets) { + if(entityBucket->member(*subset)) { removePartsAndSubsetsMinusPartsInAddPartsList.push_back(subset->mesh_meta_data_ordinal()); } } diff --git a/packages/stk/stk_mesh/stk_mesh/baseImpl/elementGraph/ElemElemGraph.cpp b/packages/stk/stk_mesh/stk_mesh/baseImpl/elementGraph/ElemElemGraph.cpp index 8c9396f71b6a..892c83baaf90 100644 --- a/packages/stk/stk_mesh/stk_mesh/baseImpl/elementGraph/ElemElemGraph.cpp +++ b/packages/stk/stk_mesh/stk_mesh/baseImpl/elementGraph/ElemElemGraph.cpp @@ -1640,7 +1640,7 @@ void add_shared_side_to_element(stk::mesh::BulkData& bulkData, parts_for_creating_side); int other_proc = parallel_edge_info.get_proc_rank_of_neighbor(); - int owning_proc = std::min(other_proc, bulkData.parallel_rank()); + int owning_proc = bulkData.state(side)==Created ? std::min(other_proc, bulkData.parallel_rank()) : bulkData.parallel_owner_rank(side); shared_modified.push_back(stk::mesh::sharing_info(side, other_proc, owning_proc)); } diff --git a/packages/stk/stk_mesh/stk_mesh/baseImpl/elementGraph/ParallelInfoForGraph.cpp b/packages/stk/stk_mesh/stk_mesh/baseImpl/elementGraph/ParallelInfoForGraph.cpp index 4d2793961d5b..48bfde003ce0 100644 --- a/packages/stk/stk_mesh/stk_mesh/baseImpl/elementGraph/ParallelInfoForGraph.cpp +++ b/packages/stk/stk_mesh/stk_mesh/baseImpl/elementGraph/ParallelInfoForGraph.cpp @@ -325,11 +325,8 @@ void update_selected_values(const ElemElemGraph& graph, const stk::mesh::impl::ParallelGraphInfo& parallel_info = graph.get_parallel_graph().get_parallel_graph_info(); for(const stk::mesh::impl::ParallelGraphInfo::value_type& edgeAndParInfo : parallel_info) { - const stk::mesh::GraphEdge &graphEdge = edgeAndParInfo.first; - if(remoteSelectedValue.is_id_selected(-graphEdge.elem2())) - selInfo[graphEdge.elem2()] = true; - else - selInfo[graphEdge.elem2()] = false; + auto graphEdge_elem2 = edgeAndParInfo.first.elem2(); + selInfo[graphEdge_elem2] = remoteSelectedValue.is_id_selected(-graphEdge_elem2); } } diff --git a/packages/stk/stk_mesh/stk_mesh/baseImpl/elementGraph/SkinBoundaryErrorReporter.hpp b/packages/stk/stk_mesh/stk_mesh/baseImpl/elementGraph/SkinBoundaryErrorReporter.hpp index 8150f56c3970..0bf36963e2d9 100644 --- a/packages/stk/stk_mesh/stk_mesh/baseImpl/elementGraph/SkinBoundaryErrorReporter.hpp +++ b/packages/stk/stk_mesh/stk_mesh/baseImpl/elementGraph/SkinBoundaryErrorReporter.hpp @@ -121,7 +121,12 @@ class SkinBoundaryErrorReporter os << " List of sides in skinned sideset but not in skinned part" << std::endl; for(size_t i = 0; i < diffFromSideSetToPart.size(); ++i) { - os << " (" << i << ") " << m_bulkData.identifier(diffFromSideSetToPart[i]) << std::endl; + const stk::mesh::Bucket* bptr = m_bulkData.bucket_ptr(diffFromSideSetToPart[i]); + os << " (" << i << ") side-entity " << m_bulkData.identifier(diffFromSideSetToPart[i]) << " " << bptr->topology() << " parts: { "; + for(const stk::mesh::Part* part : bptr->supersets()) { + os << part->name() << " "; + } + os << "}" << std::endl; report_sideset_info(diffFromSideSetToPart[i], os); } os << " -----------------------------------" << std::endl; @@ -132,7 +137,7 @@ class SkinBoundaryErrorReporter int count = 0; for(const SideSetEntry & entry : setList) { - os << " Sideset Info[" << count << "] = (" << m_bulkData.identifier(entry.element) << "," << (uint32_t)entry.side << ")"<< std::endl; + os << " Sideset Info[" << count << "] = (" << m_bulkData.identifier(entry.element) << "," << m_bulkData.bucket(entry.element).topology() << "," << (uint32_t)entry.side << ")"<< std::endl; count++; } } diff --git a/packages/stk/stk_middle_mesh/Jamfile b/packages/stk/stk_middle_mesh/Jamfile index 3056169eb0c4..48b752a7ad12 100644 --- a/packages/stk/stk_middle_mesh/Jamfile +++ b/packages/stk/stk_middle_mesh/Jamfile @@ -213,8 +213,8 @@ lib stk_middle_mesh_test_util # Any parameters within this 'ifdevbuild' block apply to development # builds only and will not be present for user builds. [ glob $(stk_middle_mesh-root)/../stk_unit_tests/stk_middle_mesh/util/*.cpp ] - /tpl/gtest//gtest - /tpl/gtest//gmock + /tpl/googletest//gtest + /tpl/googletest//gmock ] stk_middle_mesh : diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/CMakeLists.txt b/packages/stk/stk_middle_mesh/stk_middle_mesh/CMakeLists.txt index 23948e962507..5bc850e979d2 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/CMakeLists.txt +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/CMakeLists.txt @@ -65,6 +65,11 @@ target_include_directories(stk_middle_mesh PUBLIC $ ) +target_include_directories(stk_middle_mesh PUBLIC + $ + $ +) + if(CDT_INCLUDE_DIRS) SET(CDT_INC_DIR "${CDT_INCLUDE_DIRS}") else() diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh.cpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh.cpp index 335630bfdd1a..d27936dd50af 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh.cpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh.cpp @@ -811,6 +811,40 @@ void apply_orientation(EntityOrientation flag, MeshEntityPtr* down, const int n) } } +void reverse_edge(MeshEntityPtr edge) +{ + assert(edge->get_type() == MeshEntityType::Edge); + + std::vector els; + get_upward(edge, 2, els); + + std::vector edgeIdxs(els.size()); + for (size_t i=0; i < els.size(); ++i) + { + for (int j=0; j < els[i]->count_down(); ++j) + { + if (els[i]->get_down(j) == edge) + { + edgeIdxs[i] = j; + } + } + } + + mesh::MeshEntityPtr v0 = edge->get_down(0); + mesh::MeshEntityPtr v1 = edge->get_down(1); + + edge->replace_down(0, v1); + edge->replace_down(1, v0); + + for (size_t i=0; i < els.size(); ++i) + { + EntityOrientation orientOld = els[i]->get_down_orientation(edgeIdxs[i]); + EntityOrientation orientNew = reverse(orientOld); + els[i]->set_down_orientation(edgeIdxs[i], orientNew); + } +} + + int get_vertices(MeshEntityPtr e, MeshEntityPtr* verts) { int nnodes; diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh.hpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh.hpp index 5bdae5ae791f..865db3d8b0a9 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh.hpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh.hpp @@ -152,6 +152,8 @@ void check_remotes_unique(std::shared_ptr mesh); void check_coordinate_field(std::shared_ptr mesh); +void reverse_edge(MeshEntityPtr edge); + void apply_orientation(EntityOrientation flag, MeshEntityPtr* down, const int n); int get_vertices(MeshEntityPtr tri, MeshEntityPtr*); diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_entity.cpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_entity.cpp index 1189755afcee..e2e32766e504 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_entity.cpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_entity.cpp @@ -41,18 +41,32 @@ MeshEntityPtr MeshEntity::get_down(const int i) EntityOrientation MeshEntity::get_down_orientation(const int i) { #ifdef NDEBUG - return m_orientation.at(i); + return m_orientation[i]; #else return m_orientation.at(i); #endif } -void MeshEntity::replace_down(const int i, MeshEntityPtr e) +void MeshEntity::set_down_orientation(int i, EntityOrientation orient) +{ +#ifdef NDEBUG + m_orientation[i] = orient; +#else + m_orientation.at(i) = orient; +#endif +} + + +void MeshEntity::replace_down(const int i, MeshEntityPtr e, EntityOrientation orient) { if (!e) throw std::invalid_argument("entity must not be null"); m_down[i] = e; + if (m_type == MeshEntityType::Edge) + { + m_orientation[i] = orient; + } } void MeshEntity::delete_down(const int i) diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_entity.hpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_entity.hpp index 932f0ac6f3c3..0d2563ad3524 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_entity.hpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_entity.hpp @@ -62,6 +62,12 @@ inline std::ostream& operator<<(std::ostream& os, EntityOrientation orient) return os; } +inline EntityOrientation reverse(EntityOrientation orient) +{ + return orient == EntityOrientation::Standard ? EntityOrientation::Reversed : + EntityOrientation::Standard; +} + struct RemoteSharedEntity { explicit RemoteSharedEntity(int remoteRank_=0, int remoteId_=0) : @@ -174,7 +180,9 @@ class MeshEntity EntityOrientation get_down_orientation(const int i); - void replace_down(const int i, MeshEntityPtr e); + void set_down_orientation(int i, EntityOrientation orient); + + void replace_down(const int i, MeshEntityPtr e, EntityOrientation orient = EntityOrientation::Standard); void delete_down(const int i); diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/variable_size_field_impl.hpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/variable_size_field_impl.hpp index 874f9c81eead..d4fbf2667a9a 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/variable_size_field_impl.hpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/variable_size_field_impl.hpp @@ -255,7 +255,7 @@ class VariableSizeFieldForDimension // int idx = idxs.start + component int start = idxs.start; int idx = start + component; - assert(size_t(idx) >= 0 && size_t(idx) < m_values.size()); + assert(idx >= 0 && size_t(idx) < m_values.size()); return idx; } @@ -324,4 +324,4 @@ class VariableSizeFieldForDimension } // namespace middle_mesh } // namespace stk -#endif \ No newline at end of file +#endif diff --git a/packages/stk/stk_middle_mesh_util/stk_middle_mesh_util/create_stk_mesh.cpp b/packages/stk/stk_middle_mesh_util/stk_middle_mesh_util/create_stk_mesh.cpp index f614736cba1e..5597a2307acc 100644 --- a/packages/stk/stk_middle_mesh_util/stk_middle_mesh_util/create_stk_mesh.cpp +++ b/packages/stk/stk_middle_mesh_util/stk_middle_mesh_util/create_stk_mesh.cpp @@ -4,7 +4,6 @@ #include "create_stk_mesh.hpp" #include "stk_middle_mesh/field.hpp" #include "constants.hpp" -#include #include #include #include diff --git a/packages/stk/stk_ngp_test/Jamfile b/packages/stk/stk_ngp_test/Jamfile index 017e05a14505..b96afd8edf4a 100644 --- a/packages/stk/stk_ngp_test/Jamfile +++ b/packages/stk/stk_ngp_test/Jamfile @@ -160,7 +160,7 @@ lib stk_ngp_test # builds only and will not be present for user builds. [ glob $(stk_ngp_test-root)/stk_ngp_test/*.cpp ] ] - /tpl/gtest//gtest + /tpl/googletest//gtest /tpl/trilinos//kokkoscore /mpi//mpi /sierra/stk_util//stk_util_ngp @@ -186,7 +186,7 @@ lib stk_ngp_test_main # builds only and will not be present for user builds. [ glob $(stk_ngp_test-root)/stk_ngp_test/main/main.cpp ] ] - /tpl/gtest//gtest + /tpl/googletest//gtest /tpl/trilinos//kokkoscore /mpi//mpi /sierra/stk_util//stk_util_ngp diff --git a/packages/stk/stk_ngp_test/stk_ngp_test/CMakeLists.txt b/packages/stk/stk_ngp_test/stk_ngp_test/CMakeLists.txt index 53db94d5e467..2fa8bdfcbb5d 100644 --- a/packages/stk/stk_ngp_test/stk_ngp_test/CMakeLists.txt +++ b/packages/stk/stk_ngp_test/stk_ngp_test/CMakeLists.txt @@ -33,7 +33,7 @@ # FILE(GLOB HEADERS *.hpp) -FILE(GLOB SOURCES ngp_test.cpp) +FILE(GLOB SOURCES *.cpp) if(HAVE_STK_Trilinos) TRIBITS_ADD_LIBRARY(stk_ngp_test diff --git a/packages/stk/stk_ngp_test/stk_ngp_test/GlobalReporter.cpp b/packages/stk/stk_ngp_test/stk_ngp_test/GlobalReporter.cpp index cee6b645c711..601c8a75ee99 100644 --- a/packages/stk/stk_ngp_test/stk_ngp_test/GlobalReporter.cpp +++ b/packages/stk/stk_ngp_test/stk_ngp_test/GlobalReporter.cpp @@ -1,99 +1,3 @@ -#ifndef _GlobalReporter_cpp_ -#define _GlobalReporter_cpp_ #include "GlobalReporter.hpp" -#include "Reporter.hpp" -#include "stk_util/ngp/NgpSpaces.hpp" - -namespace ngp_testing { -namespace global { -static constexpr int numReports = 5; -inline -HostReporter*& getHostReporter() -{ - static HostReporter* hostReporter = nullptr; - return hostReporter; -} - -inline -DeviceReporter*& getDeviceReporterOnHost() -{ - static DeviceReporter* deviceReporterOnHost = nullptr; - return deviceReporterOnHost; -} - -#ifdef KOKKOS_ENABLE_SYCL -namespace{ - sycl::ext::oneapi::experimental::device_global deviceReporterOnDevice; -} -#endif - -NGP_TEST_INLINE DeviceReporter*& getDeviceReporterOnDevice() -{ -#ifndef KOKKOS_ENABLE_SYCL - KOKKOS_IF_ON_DEVICE(( - __device__ static DeviceReporter* deviceReporterOnDevice = nullptr; - return deviceReporterOnDevice; - )) -#else - KOKKOS_IF_ON_DEVICE(( - return deviceReporterOnDevice; - )) -#endif - KOKKOS_IF_ON_HOST(( - static DeviceReporter* deviceReporterOnDevice = nullptr; - return deviceReporterOnDevice; - )) -} - -inline -DeviceReporter*& getDeviceReporterAddress() -{ - static DeviceReporter* deviceReporterAddress = nullptr; - return deviceReporterAddress; -} -} - -inline -void copy_to_device(const DeviceReporter& reporter, DeviceReporter* const addr) { - Kokkos::parallel_for(stk::ngp::DeviceRangePolicy(0, 1), KOKKOS_LAMBDA(const int){ - global::getDeviceReporterOnDevice() = addr; - new (global::getDeviceReporterOnDevice()) DeviceReporter(reporter); - }); - Kokkos::fence(); -} - -inline -void initialize_reporters() { - global::getHostReporter() = new HostReporter(global::numReports); - global::getDeviceReporterOnHost() = new DeviceReporter(global::numReports); - global::getDeviceReporterAddress() = static_cast(Kokkos::kokkos_malloc(sizeof(DeviceReporter))); - copy_to_device(dynamic_cast(*global::getDeviceReporterOnHost()), global::getDeviceReporterAddress()); -} - -inline -void finalize_reporters() { - delete global::getHostReporter(); - global::getHostReporter() = nullptr; - delete global::getDeviceReporterOnHost(); - global::getDeviceReporterOnHost() = nullptr; - Kokkos::kokkos_free(global::getDeviceReporterAddress()); -} - -NGP_TEST_INLINE HostReporter* get_host_reporter() { - return global::getHostReporter(); -} - -NGP_TEST_INLINE DeviceReporter* get_device_reporter() { - return global::getDeviceReporterOnDevice(); -} - -inline -DeviceReporter* get_device_reporter_on_host() { - return global::getDeviceReporterOnHost(); -} - -} - -#endif diff --git a/packages/stk/stk_ngp_test/stk_ngp_test/GlobalReporter.hpp b/packages/stk/stk_ngp_test/stk_ngp_test/GlobalReporter.hpp index dfdd180141d6..2081d2de1512 100644 --- a/packages/stk/stk_ngp_test/stk_ngp_test/GlobalReporter.hpp +++ b/packages/stk/stk_ngp_test/stk_ngp_test/GlobalReporter.hpp @@ -1,6 +1,8 @@ #ifndef STK_NGP_TEST_GLOBAL_REPORTER_HPP #define STK_NGP_TEST_GLOBAL_REPORTER_HPP +#include "Reporter.hpp" +#include "stk_util/ngp/NgpSpaces.hpp" #include "NgpTestDeviceMacros.hpp" namespace ngp_testing { @@ -20,6 +22,94 @@ DeviceReporter* get_device_reporter_on_host(); } -#include "GlobalReporter.cpp" +namespace ngp_testing { +namespace global { +static constexpr int numReports = 5; +inline +HostReporter*& getHostReporter() +{ + static HostReporter* hostReporter = nullptr; + return hostReporter; +} + +inline +DeviceReporter*& getDeviceReporterOnHost() +{ + static DeviceReporter* deviceReporterOnHost = nullptr; + return deviceReporterOnHost; +} + +#ifdef KOKKOS_ENABLE_SYCL +namespace{ + sycl::ext::oneapi::experimental::device_global deviceReporterOnDevice; +} +#endif + +NGP_TEST_INLINE DeviceReporter*& getDeviceReporterOnDevice() +{ +#ifndef KOKKOS_ENABLE_SYCL + KOKKOS_IF_ON_DEVICE(( + __device__ static DeviceReporter* deviceReporterOnDevice = nullptr; + return deviceReporterOnDevice; + )) +#else + KOKKOS_IF_ON_DEVICE(( + return deviceReporterOnDevice; + )) +#endif + KOKKOS_IF_ON_HOST(( + static DeviceReporter* deviceReporterOnDevice = nullptr; + return deviceReporterOnDevice; + )) +} + +inline +DeviceReporter*& getDeviceReporterAddress() +{ + static DeviceReporter* deviceReporterAddress = nullptr; + return deviceReporterAddress; +} +} + +inline +void copy_to_device(const DeviceReporter& reporter, DeviceReporter* const addr) { + Kokkos::parallel_for(stk::ngp::DeviceRangePolicy(0, 1), KOKKOS_LAMBDA(const int){ + global::getDeviceReporterOnDevice() = addr; + new (global::getDeviceReporterOnDevice()) DeviceReporter(reporter); + }); + Kokkos::fence(); +} + +inline +void initialize_reporters() { + global::getHostReporter() = new HostReporter(global::numReports); + global::getDeviceReporterOnHost() = new DeviceReporter(global::numReports); + global::getDeviceReporterAddress() = static_cast(Kokkos::kokkos_malloc(sizeof(DeviceReporter))); + copy_to_device(dynamic_cast(*global::getDeviceReporterOnHost()), global::getDeviceReporterAddress()); +} + +inline +void finalize_reporters() { + delete global::getHostReporter(); + global::getHostReporter() = nullptr; + delete global::getDeviceReporterOnHost(); + global::getDeviceReporterOnHost() = nullptr; + Kokkos::kokkos_free(global::getDeviceReporterAddress()); +} + +NGP_TEST_INLINE HostReporter* get_host_reporter() { + return global::getHostReporter(); +} + +NGP_TEST_INLINE DeviceReporter* get_device_reporter() { + return global::getDeviceReporterOnDevice(); +} + +inline +DeviceReporter* get_device_reporter_on_host() { + return global::getDeviceReporterOnHost(); +} + +} #endif diff --git a/packages/stk/stk_ngp_test/stk_ngp_test/NgpTestDeviceMacros.hpp b/packages/stk/stk_ngp_test/stk_ngp_test/NgpTestDeviceMacros.hpp index faafe9c59674..6c301b537ec7 100644 --- a/packages/stk/stk_ngp_test/stk_ngp_test/NgpTestDeviceMacros.hpp +++ b/packages/stk/stk_ngp_test/stk_ngp_test/NgpTestDeviceMacros.hpp @@ -6,7 +6,7 @@ #define NGP_TEST_FUNCTION KOKKOS_FUNCTION #define NGP_TEST_INLINE KOKKOS_INLINE_FUNCTION -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) +#ifdef STK_ENABLE_GPU #define NGP_TEST_DEVICE_ONLY __device__ #else #define NGP_TEST_DEVICE_ONLY diff --git a/packages/stk/stk_ngp_test/stk_ngp_test/Reporter.cpp b/packages/stk/stk_ngp_test/stk_ngp_test/Reporter.cpp index 34e9bd3c7820..3e6f3ab80484 100644 --- a/packages/stk/stk_ngp_test/stk_ngp_test/Reporter.cpp +++ b/packages/stk/stk_ngp_test/stk_ngp_test/Reporter.cpp @@ -1,60 +1,3 @@ -#ifndef _Reporter_cpp_ -#define _Reporter_cpp_ #include "Reporter.hpp" -namespace ngp_testing { - -NGP_TEST_INLINE void bounded_strcpy(const char* src, const int maxChar, char* dest) { - int idx = 0; - if(src) { - while(src[idx] != '\0' && idx < maxChar) { - dest[idx] = src[idx]; - ++idx; - } - } - dest[idx] = '\0'; -} - -NGP_TEST_INLINE -TruncatedString::TruncatedString(const char* src) { - const int srcLen = get_string_length(src); - if(should_truncate(srcLen)) { - src += get_truncation_offset(srcLen); - } - bounded_strcpy(src, maxNumChar, string); - if(should_truncate(srcLen)) { - prepend_truncation_indicator(string); - } -} - -NGP_TEST_INLINE -int TruncatedString::get_string_length(const char* str) const { - int len = 0; - if(str) { - while (str[len] != '\0') ++len; - } - return len; -} - -NGP_TEST_INLINE -bool TruncatedString::should_truncate(const int strLen) const { - return strLen > maxNumChar; -} - -NGP_TEST_INLINE -int TruncatedString::get_truncation_offset(const int strLen) const { - return strLen - maxNumChar + 1; -} - -NGP_TEST_INLINE -void TruncatedString::prepend_truncation_indicator(char* dest) { - dest[0] = '.'; - dest[1] = '.'; - dest[2] = '.'; -} - -} - -#endif - diff --git a/packages/stk/stk_ngp_test/stk_ngp_test/Reporter.hpp b/packages/stk/stk_ngp_test/stk_ngp_test/Reporter.hpp index 9ed460d8d245..ae72614204cd 100644 --- a/packages/stk/stk_ngp_test/stk_ngp_test/Reporter.hpp +++ b/packages/stk/stk_ngp_test/stk_ngp_test/Reporter.hpp @@ -93,6 +93,57 @@ class Reporter { } -#include "Reporter.cpp" +namespace ngp_testing { + +NGP_TEST_INLINE void bounded_strcpy(const char* src, const int maxChar, char* dest) { + int idx = 0; + if(src) { + while(src[idx] != '\0' && idx < maxChar) { + dest[idx] = src[idx]; + ++idx; + } + } + dest[idx] = '\0'; +} + +NGP_TEST_INLINE +TruncatedString::TruncatedString(const char* src) { + const int srcLen = get_string_length(src); + if(should_truncate(srcLen)) { + src += get_truncation_offset(srcLen); + } + bounded_strcpy(src, maxNumChar, string); + if(should_truncate(srcLen)) { + prepend_truncation_indicator(string); + } +} + +NGP_TEST_INLINE +int TruncatedString::get_string_length(const char* str) const { + int len = 0; + if(str) { + while (str[len] != '\0') ++len; + } + return len; +} + +NGP_TEST_INLINE +bool TruncatedString::should_truncate(const int strLen) const { + return strLen > maxNumChar; +} + +NGP_TEST_INLINE +int TruncatedString::get_truncation_offset(const int strLen) const { + return strLen - maxNumChar + 1; +} + +NGP_TEST_INLINE +void TruncatedString::prepend_truncation_indicator(char* dest) { + dest[0] = '.'; + dest[1] = '.'; + dest[2] = '.'; +} + +} #endif diff --git a/packages/stk/stk_ngp_test/stk_ngp_test/ngp_test.cpp b/packages/stk/stk_ngp_test/stk_ngp_test/ngp_test.cpp deleted file mode 100644 index c01d73d5d795..000000000000 --- a/packages/stk/stk_ngp_test/stk_ngp_test/ngp_test.cpp +++ /dev/null @@ -1,92 +0,0 @@ -#ifndef _ngp_test_cpp_ -#define _ngp_test_cpp_ - -#include "GlobalReporter.hpp" -#include "Reporter.hpp" -#include -#include - -namespace ngp_testing { - -inline -NgpTestEnvironment::NgpTestEnvironment(int* argc, char** argv) { - Kokkos::initialize(*argc, argv); - initialize_reporters(); - ::testing::InitGoogleTest(argc, argv); - Kokkos::push_finalize_hook(finalize_reporters); -} - -inline -NgpTestEnvironment::~NgpTestEnvironment() { - finalize(); -} - -inline -void NgpTestEnvironment::finalize() { - if(Kokkos::is_initialized()) Kokkos::finalize(); -} - -inline -int NgpTestEnvironment::run_all_tests() { - return RUN_ALL_TESTS(); -} - -inline -void Test::SetUp() { - internal::clear_failures(); - NGPSetUp(); -} - -inline -void Test::TearDown() { - int numReports = internal::report_failures(); - if(numReports > 0) { - set_failure_in_teardown(); - } - NGPTearDown(); -} - -inline -int get_max_failure_reports_per_test() { - return get_host_reporter()->get_capacity(); -} - -inline -void set_max_failure_reports_per_test(const int n) { - get_host_reporter()->resize(n); - get_device_reporter_on_host()->resize(n); -} - - -namespace internal { - -NGP_TEST_INLINE -void add_failure(const char* condition, const char* location) { - KOKKOS_IF_ON_HOST(( - get_host_reporter()->add_failure(condition, location); - )) - - KOKKOS_IF_ON_DEVICE(( - get_device_reporter()->add_failure(condition, location); - )) -} - -inline -int report_failures() { - int numFailures = get_host_reporter()->report_failures(); - numFailures += get_device_reporter_on_host()->report_failures(); - return numFailures; -} - -inline -void clear_failures() { - get_host_reporter()->clear(); - get_device_reporter_on_host()->clear(); -} - -} - -} - -#endif - diff --git a/packages/stk/stk_ngp_test/stk_ngp_test/ngp_test.hpp b/packages/stk/stk_ngp_test/stk_ngp_test/ngp_test.hpp index 512d59af8f6e..1e7bd867656f 100644 --- a/packages/stk/stk_ngp_test/stk_ngp_test/ngp_test.hpp +++ b/packages/stk/stk_ngp_test/stk_ngp_test/ngp_test.hpp @@ -1,6 +1,9 @@ #ifndef STK_NGP_TEST_NGP_TEST_HPP #define STK_NGP_TEST_NGP_TEST_HPP #include +#include "GlobalReporter.hpp" +#include "Reporter.hpp" +#include #include namespace ngp_testing { @@ -122,6 +125,87 @@ bool expect_near(const T a, const T b, const T tolerance) { } while (false) #endif -#include "ngp_test.cpp" +namespace ngp_testing { + +inline +NgpTestEnvironment::NgpTestEnvironment(int* argc, char** argv) { + Kokkos::initialize(*argc, argv); + initialize_reporters(); + ::testing::InitGoogleTest(argc, argv); + Kokkos::push_finalize_hook(finalize_reporters); +} + +inline +NgpTestEnvironment::~NgpTestEnvironment() { + finalize(); +} + +inline +void NgpTestEnvironment::finalize() { + if(Kokkos::is_initialized()) Kokkos::finalize(); +} + +inline +int NgpTestEnvironment::run_all_tests() { + return RUN_ALL_TESTS(); +} + +inline +void Test::SetUp() { + internal::clear_failures(); + NGPSetUp(); +} + +inline +void Test::TearDown() { + int numReports = internal::report_failures(); + if(numReports > 0) { + set_failure_in_teardown(); + } + NGPTearDown(); +} + +inline +int get_max_failure_reports_per_test() { + return get_host_reporter()->get_capacity(); +} + +inline +void set_max_failure_reports_per_test(const int n) { + get_host_reporter()->resize(n); + get_device_reporter_on_host()->resize(n); +} + + +namespace internal { + +NGP_TEST_INLINE +void add_failure(const char* condition, const char* location) { + KOKKOS_IF_ON_HOST(( + get_host_reporter()->add_failure(condition, location); + )) + + KOKKOS_IF_ON_DEVICE(( + get_device_reporter()->add_failure(condition, location); + )) +} + +inline +int report_failures() { + int numFailures = get_host_reporter()->report_failures(); + numFailures += get_device_reporter_on_host()->report_failures(); + return numFailures; +} + +inline +void clear_failures() { + get_host_reporter()->clear(); + get_device_reporter_on_host()->clear(); +} + +} + +} #endif + diff --git a/packages/stk/stk_performance_tests/Jamfile b/packages/stk/stk_performance_tests/Jamfile index 80d9e45559f0..fec7d8e3c484 100644 --- a/packages/stk/stk_performance_tests/Jamfile +++ b/packages/stk/stk_performance_tests/Jamfile @@ -134,6 +134,7 @@ exe stk_performance_tests /sierra/stk_middle_mesh_util//stk_middle_mesh_util /sierra/stk_middle_mesh//stk_middle_mesh /sierra/stk_coupling//stk_coupling + /sierra/stk_search//stk_search /sierra/stk_io//stk_io_util /sierra/stk_util//stk_util_diag /sierra/stk_tools//stk_mesh_tools_lib diff --git a/packages/stk/stk_performance_tests/stk_mesh/GatherGears.cpp b/packages/stk/stk_performance_tests/stk_mesh/GatherGears.cpp index 1cbe398105ca..01af42ce2ade 100644 --- a/packages/stk/stk_performance_tests/stk_mesh/GatherGears.cpp +++ b/packages/stk/stk_performance_tests/stk_mesh/GatherGears.cpp @@ -152,7 +152,7 @@ TEST(gather_gears, gather_gears) const double timers[NUM_TIMERS] = {mesh_create_time, gather_time, total_time}; const char* timer_names[NUM_TIMERS] = {"Create mesh", "Gather", "Total time"}; - stk::print_timers_and_memory(&timer_names[0], &timers[0], NUM_TIMERS); + stk::print_timers_and_memory(timer_names, timers, NUM_TIMERS); stk::parallel_print_time_without_output_and_hwm(MPI_COMM_WORLD, total_time); } diff --git a/packages/stk/stk_performance_tests/stk_mesh/GearsSkinning.cpp b/packages/stk/stk_performance_tests/stk_mesh/GearsSkinning.cpp index 637340b52b71..0552c1a948b6 100644 --- a/packages/stk/stk_performance_tests/stk_mesh/GearsSkinning.cpp +++ b/packages/stk/stk_performance_tests/stk_mesh/GearsSkinning.cpp @@ -50,7 +50,7 @@ #include #include -#include +#include #include #include "stk_unit_test_utils/stk_mesh_fixtures/Gear.hpp" diff --git a/packages/stk/stk_performance_tests/stk_mesh/HexRefine.cpp b/packages/stk/stk_performance_tests/stk_mesh/HexRefine.cpp index 62436fbb5c26..138c11a4563a 100644 --- a/packages/stk/stk_performance_tests/stk_mesh/HexRefine.cpp +++ b/packages/stk/stk_performance_tests/stk_mesh/HexRefine.cpp @@ -153,7 +153,7 @@ TEST( hex_refine, hex_refine) const double timers[NUM_TIMERS] = {create_time, refine_time, total_time}; const char* timer_names[NUM_TIMERS] = {"Create mesh", "Refine", "Total time"}; - stk::print_timers_and_memory(&timer_names[0], &timers[0], NUM_TIMERS); + stk::print_timers_and_memory(timer_names, timers, NUM_TIMERS); stk::parallel_print_time_without_output_and_hwm(MPI_COMM_WORLD, total_time); } diff --git a/packages/stk/stk_performance_tests/stk_mesh/HexSTKGather.cpp b/packages/stk/stk_performance_tests/stk_mesh/HexSTKGather.cpp index e94d3cd3812e..390c6a16bacd 100644 --- a/packages/stk/stk_performance_tests/stk_mesh/HexSTKGather.cpp +++ b/packages/stk/stk_performance_tests/stk_mesh/HexSTKGather.cpp @@ -180,7 +180,7 @@ TEST(hex_gather, hex_gather) const double timers[NUM_TIMERS] = {mesh_create_time, gather_time, total_time}; const char* timer_names[NUM_TIMERS] = {"Create mesh", "Gather", "Total time"}; - stk::print_timers_and_memory(&timer_names[0], &timers[0], NUM_TIMERS); + stk::print_timers_and_memory(timer_names, timers, NUM_TIMERS); stk::parallel_print_time_without_output_and_hwm(MPI_COMM_WORLD, total_time); } diff --git a/packages/stk/stk_performance_tests/stk_mesh/HexSTKedges.cpp b/packages/stk/stk_performance_tests/stk_mesh/HexSTKedges.cpp index 5f850e1c1df3..73df9a2ace99 100644 --- a/packages/stk/stk_performance_tests/stk_mesh/HexSTKedges.cpp +++ b/packages/stk/stk_performance_tests/stk_mesh/HexSTKedges.cpp @@ -126,7 +126,7 @@ TEST(hex_edges, hex_edges) const double timers[NUM_TIMERS] = {mesh_create_time, create_edges_time, total_time}; const char* timer_names[NUM_TIMERS] = {"Create mesh", "Create edges", "Total time"}; - stk::print_timers_and_memory(&timer_names[0], &timers[0], NUM_TIMERS); + stk::print_timers_and_memory(timer_names, timers, NUM_TIMERS); std::cout<<"Global HWM: "< -#include #include #include #include diff --git a/packages/stk/stk_performance_tests/stk_mesh/NgpFieldUpdate.cpp b/packages/stk/stk_performance_tests/stk_mesh/NgpFieldUpdate.cpp index 4a299906b309..a2fb15096ef4 100644 --- a/packages/stk/stk_performance_tests/stk_mesh/NgpFieldUpdate.cpp +++ b/packages/stk/stk_performance_tests/stk_mesh/NgpFieldUpdate.cpp @@ -324,7 +324,7 @@ TEST_F( NgpMeshChangeElementPartMembershipWithFields, Timing ) if (get_parallel_size() != 1) return; const unsigned NUM_RUNS = 5; - #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + #ifdef STK_ENABLE_GPU const int NUM_ITERS = 50; #else const int NUM_ITERS = 2500; @@ -353,7 +353,7 @@ TEST_F( NgpMeshCreateEntityWithFields, Timing ) if (get_parallel_size() != 1) return; const unsigned NUM_RUNS = 5; - #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + #ifdef STK_ENABLE_GPU const int numModCycles = 50; #else const int numModCycles = 4000; diff --git a/packages/stk/stk_performance_tests/stk_mesh/NgpMeshUpdate.cpp b/packages/stk/stk_performance_tests/stk_mesh/NgpMeshUpdate.cpp index 49aded329ec3..c9f70608f477 100644 --- a/packages/stk/stk_performance_tests/stk_mesh/NgpMeshUpdate.cpp +++ b/packages/stk/stk_performance_tests/stk_mesh/NgpMeshUpdate.cpp @@ -246,7 +246,7 @@ TEST_F( NgpMeshCreateEntity, Timing ) if (get_parallel_size() != 1) return; const unsigned NUM_RUNS = 5; - #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + #ifdef STK_ENABLE_GPU const int NUM_ITERS = 100; #else const int NUM_ITERS = 5000; diff --git a/packages/stk/stk_performance_tests/stk_mesh/NodeRels.cpp b/packages/stk/stk_performance_tests/stk_mesh/NodeRels.cpp index 034a2718c03a..88dab412b858 100644 --- a/packages/stk/stk_performance_tests/stk_mesh/NodeRels.cpp +++ b/packages/stk/stk_performance_tests/stk_mesh/NodeRels.cpp @@ -136,7 +136,7 @@ TEST(node_rels, node_rels) const double timers[NUM_TIMERS] = {mesh_create_time, traverse_time, total_time}; const char* timer_names[NUM_TIMERS] = {"Create mesh", "Traverse", "Total time"}; - stk::print_timers_and_memory(&timer_names[0], &timers[0], NUM_TIMERS); + stk::print_timers_and_memory(timer_names, timers, NUM_TIMERS); stk::parallel_print_time_without_output_and_hwm(MPI_COMM_WORLD, total_time); } diff --git a/packages/stk/stk_performance_tests/stk_mesh/TetSTKfaces.cpp b/packages/stk/stk_performance_tests/stk_mesh/TetSTKfaces.cpp index 9d58489c0a8b..9a44f45c53a0 100644 --- a/packages/stk/stk_performance_tests/stk_mesh/TetSTKfaces.cpp +++ b/packages/stk/stk_performance_tests/stk_mesh/TetSTKfaces.cpp @@ -107,7 +107,7 @@ TEST(tet_faces, tet_faces) const double timers[NUM_TIMERS] = {mesh_create_time, create_faces_time, total_time}; const char* timer_names[NUM_TIMERS] = {"Create mesh", "Create faces", "Total time"}; - stk::print_timers_and_memory(&timer_names[0], &timers[0], NUM_TIMERS); + stk::print_timers_and_memory(timer_names, timers, NUM_TIMERS); std::cout<<"Global HWM: "< #include #include -#include #include #include #include diff --git a/packages/stk/stk_performance_tests/stk_middle_mesh/perfMiddleMeshEntityOps.cpp b/packages/stk/stk_performance_tests/stk_middle_mesh/perfMiddleMeshEntityOps.cpp index 0da2124ff64b..01498e322077 100644 --- a/packages/stk/stk_performance_tests/stk_middle_mesh/perfMiddleMeshEntityOps.cpp +++ b/packages/stk/stk_performance_tests/stk_middle_mesh/perfMiddleMeshEntityOps.cpp @@ -147,19 +147,20 @@ TEST(MiddleMeshOps, GetUpward) if (stk::middle_mesh::utils::impl::comm_size(comm) > 1) GTEST_SKIP(); - const int NUM_RUNS = 125; + const int NUM_RUNS = 25; const int NUM_ITERS = 1; stk::middle_mesh::mesh::impl::MeshSpec spec{1000, 1000, 0, 1, 0, 1}; auto func = [&](stk::middle_mesh::utils::Point const& pt) { return pt; }; - auto mesh = stk::middle_mesh::mesh::impl::create_mesh(spec, func); std::vector edges, elems; stk::unit_test_util::BatchTimer batchTimer(comm); batchTimer.initialize_batch_timer(); + + auto mesh = stk::middle_mesh::mesh::impl::create_mesh(spec, func); + for (unsigned i = 0; i < NUM_RUNS; ++i) { - std::cout << "i = " << i << std::endl; batchTimer.start_batch_timer(); auto& verts = mesh->get_mesh_entities(VERT_RANK); @@ -182,12 +183,11 @@ TEST(MiddleMeshOps, GetDownward) if (stk::middle_mesh::utils::impl::comm_size(comm) > 1) GTEST_SKIP(); - const int NUM_RUNS = 250; + const int NUM_RUNS = 50; const int NUM_ITERS = 1; stk::middle_mesh::mesh::impl::MeshSpec spec{1000, 1000, 0, 1, 0, 1}; auto func = [&](stk::middle_mesh::utils::Point const& pt) { return pt; }; - auto mesh = stk::middle_mesh::mesh::impl::create_mesh(spec, func); std::vector edges, verts; @@ -196,6 +196,9 @@ TEST(MiddleMeshOps, GetDownward) stk::unit_test_util::BatchTimer batchTimer(comm); batchTimer.initialize_batch_timer(); + + auto mesh = stk::middle_mesh::mesh::impl::create_mesh(spec, func); + for (unsigned i = 0; i < NUM_RUNS; ++i) { batchTimer.start_batch_timer(); auto& elems = mesh->get_mesh_entities(ELEM_RANK); diff --git a/packages/stk/stk_performance_tests/stk_search/SurfaceToSurface.cpp b/packages/stk/stk_performance_tests/stk_search/SurfaceToSurface.cpp new file mode 100644 index 000000000000..c438e7dc7b1f --- /dev/null +++ b/packages/stk/stk_performance_tests/stk_search/SurfaceToSurface.cpp @@ -0,0 +1,316 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "stk_search/SearchMethod.hpp" + +namespace { + +std::string get_parallel_filename(const std::string& baseName, MPI_Comm comm) +{ + const int nProcs = stk::parallel_machine_size(comm); + const int pRank = stk::parallel_machine_rank(comm); + + return baseName + "_" + std::to_string(nProcs) + "." + std::to_string(pRank); +} + +template +void sequentially_number_all_boxes(BoxVectorType & boxVector, MPI_Comm comm) +{ + const int nProcs = stk::parallel_machine_size(comm); + const int pRank = stk::parallel_machine_rank(comm); + std::vector allNumBoxes(nProcs, 0); + const unsigned long localNumBoxes = boxVector.size(); + + MPI_Allgather(&localNumBoxes, 1, MPI_UNSIGNED_LONG, allNumBoxes.data(), 1, MPI_UNSIGNED_LONG, comm); + + unsigned long id = 0; + for (int proc = 0; proc < pRank; ++proc) { + id += allNumBoxes[proc]; + } + + for (auto & [box, ident] : boxVector) { + ident.set_proc(pRank); + ident.set_id(id++); + } +} + +template +BoxVectorType read_boxes_from_file(const std::string& baseName, MPI_Comm comm) +{ + const std::string fileName = get_parallel_filename(baseName, comm); + std::ifstream infile(fileName); + STK_ThrowRequireMsg(infile.good(), "Unable to open file " + fileName); + + BoxVectorType boxVector; + std::string line; + while (std::getline(infile, line)) { + if (line.size() == 0) continue; + if (line[0] == '#') continue; + std::istringstream iss(line); + boxVector.emplace_back(); + auto & [box, ident] = boxVector.back(); + iss >> box; + } + + sequentially_number_all_boxes(boxVector, comm); + + return boxVector; +} + +template +void run_imported_surface_to_surface_test(const std::string& boxFileBaseName, + const int numIterations, + stk::search::SearchMethod searchMethod, + bool enforceSearchResultSymmetry = true) +{ + MPI_Comm comm = MPI_COMM_WORLD; + const unsigned NUM_RUNS = 5; + stk::unit_test_util::BatchTimer batchTimer(comm); + batchTimer.initialize_batch_timer(); + + BoxVectorType diceBoxes = read_boxes_from_file(boxFileBaseName + ".txt_dice", comm); + BoxVectorType toolBoxes = read_boxes_from_file(boxFileBaseName + ".txt_tool", comm); + + for (unsigned run = 0; run < NUM_RUNS; ++run) { + SearchResults searchResults; + + batchTimer.start_batch_timer(); + for (int i = 0; i < numIterations; ++i) { + stk::search::coarse_search(diceBoxes, toolBoxes, searchMethod, comm, searchResults, enforceSearchResultSymmetry); +// std::cout << "Num intersections = " << searchResults.size() << std::endl; + } + batchTimer.stop_batch_timer(); + } + + batchTimer.print_batch_timing(numIterations); +} + +TEST(StkSearch_SurfaceToSurface, ecsl_floatBox_KDTREE) +{ + std::string boxFileBaseName = stk::unit_test_util::get_option("-m", "none-specified"); + if (boxFileBaseName == "none-specified") GTEST_SKIP(); + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 1) GTEST_SKIP(); + + const int numIterations = 4; + run_imported_surface_to_surface_test(boxFileBaseName, numIterations, stk::search::KDTREE); +} + +TEST(StkSearch_SurfaceToSurface, ecsl_floatBox_MORTON_LBVH) +{ + std::string boxFileBaseName = stk::unit_test_util::get_option("-m", "none-specified"); + if (boxFileBaseName == "none-specified") GTEST_SKIP(); + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 1) GTEST_SKIP(); + + const int numIterations = 4; + run_imported_surface_to_surface_test(boxFileBaseName, numIterations, stk::search::MORTON_LBVH); +} + +TEST(StkSearch_SurfaceToSurface, ecsl_floatBox_ARBORX) +{ + std::string boxFileBaseName = stk::unit_test_util::get_option("-m", "none-specified"); + if (boxFileBaseName == "none-specified") GTEST_SKIP(); + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 1) GTEST_SKIP(); + + const int numIterations = 4; + run_imported_surface_to_surface_test(boxFileBaseName, numIterations, stk::search::ARBORX); +} + + +TEST(StkSearch_SurfaceToSurface, newtonCradle_floatBox_KDTREE) +{ + std::string boxFileBaseName = stk::unit_test_util::get_option("-m", "none-specified"); + if (boxFileBaseName == "none-specified") GTEST_SKIP(); + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 16) GTEST_SKIP(); + + const int numIterations = 2000; + run_imported_surface_to_surface_test(boxFileBaseName, numIterations, stk::search::KDTREE); +} + +TEST(StkSearch_SurfaceToSurface, newtonCradle_floatBox_MORTON_LBVH) +{ + std::string boxFileBaseName = stk::unit_test_util::get_option("-m", "none-specified"); + if (boxFileBaseName == "none-specified") GTEST_SKIP(); + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 16) GTEST_SKIP(); + + const int numIterations = 2000; + run_imported_surface_to_surface_test(boxFileBaseName, numIterations, stk::search::MORTON_LBVH); +} + +TEST(StkSearch_SurfaceToSurface, newtonCradle_floatBox_ARBORX) +{ + std::string boxFileBaseName = stk::unit_test_util::get_option("-m", "none-specified"); + if (boxFileBaseName == "none-specified") GTEST_SKIP(); + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 16) GTEST_SKIP(); + + const int numIterations = 2000; + run_imported_surface_to_surface_test(boxFileBaseName, numIterations, stk::search::ARBORX); +} + + +TEST(StkSearch_SurfaceToSurface, b61NoseCrush_floatBox_KDTREE) +{ + std::string boxFileBaseName = stk::unit_test_util::get_option("-m", "none-specified"); + if (boxFileBaseName == "none-specified") GTEST_SKIP(); + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 32) GTEST_SKIP(); + + const int numIterations = 200; + run_imported_surface_to_surface_test(boxFileBaseName, numIterations, stk::search::KDTREE); +} + +TEST(StkSearch_SurfaceToSurface, b61NoseCrush_floatBox_MORTON_LBVH) +{ + std::string boxFileBaseName = stk::unit_test_util::get_option("-m", "none-specified"); + if (boxFileBaseName == "none-specified") GTEST_SKIP(); + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 32) GTEST_SKIP(); + + const int numIterations = 200; + run_imported_surface_to_surface_test(boxFileBaseName, numIterations, stk::search::MORTON_LBVH); +} + +TEST(StkSearch_SurfaceToSurface, b61NoseCrush_floatBox_ARBORX) +{ + std::string boxFileBaseName = stk::unit_test_util::get_option("-m", "none-specified"); + if (boxFileBaseName == "none-specified") GTEST_SKIP(); + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 32) GTEST_SKIP(); + + const int numIterations = 200; + run_imported_surface_to_surface_test(boxFileBaseName, numIterations, stk::search::ARBORX); +} + + +TEST(StkSearch_SurfaceToSurface, coneCrush_floatBox_KDTREE) +{ + std::string boxFileBaseName = stk::unit_test_util::get_option("-m", "none-specified"); + if (boxFileBaseName == "none-specified") GTEST_SKIP(); + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 32) GTEST_SKIP(); + + const int numIterations = 200; + run_imported_surface_to_surface_test(boxFileBaseName, numIterations, stk::search::KDTREE); +} + +TEST(StkSearch_SurfaceToSurface, coneCrush_floatBox_MORTON_LBVH) +{ + std::string boxFileBaseName = stk::unit_test_util::get_option("-m", "none-specified"); + if (boxFileBaseName == "none-specified") GTEST_SKIP(); + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 32) GTEST_SKIP(); + + const int numIterations = 200; + run_imported_surface_to_surface_test(boxFileBaseName, numIterations, stk::search::MORTON_LBVH); +} + +TEST(StkSearch_SurfaceToSurface, coneCrush_floatBox_ARBORX) +{ + std::string boxFileBaseName = stk::unit_test_util::get_option("-m", "none-specified"); + if (boxFileBaseName == "none-specified") GTEST_SKIP(); + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 32) GTEST_SKIP(); + + const int numIterations = 200; + run_imported_surface_to_surface_test(boxFileBaseName, numIterations, stk::search::ARBORX); +} + + +TEST(StkSearch_SurfaceToSurface, jenga_floatBox_KDTREE) +{ + std::string boxFileBaseName = stk::unit_test_util::get_option("-m", "none-specified"); + if (boxFileBaseName == "none-specified") GTEST_SKIP(); + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 32) GTEST_SKIP(); + + const int numIterations = 500; + run_imported_surface_to_surface_test(boxFileBaseName, numIterations, stk::search::KDTREE); +} + +TEST(StkSearch_SurfaceToSurface, jenga_floatBox_MORTON_LBVH) +{ + std::string boxFileBaseName = stk::unit_test_util::get_option("-m", "none-specified"); + if (boxFileBaseName == "none-specified") GTEST_SKIP(); + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 32) GTEST_SKIP(); + + const int numIterations = 500; + run_imported_surface_to_surface_test(boxFileBaseName, numIterations, stk::search::MORTON_LBVH); +} + +TEST(StkSearch_SurfaceToSurface, jenga_floatBox_ARBORX) +{ + std::string boxFileBaseName = stk::unit_test_util::get_option("-m", "none-specified"); + if (boxFileBaseName == "none-specified") GTEST_SKIP(); + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 32) GTEST_SKIP(); + + const int numIterations = 500; + run_imported_surface_to_surface_test(boxFileBaseName, numIterations, stk::search::ARBORX); +} + + +TEST(StkSearch_SurfaceToSurface, tractorTrailerCrash_floatBox_KDTREE) +{ + std::string boxFileBaseName = stk::unit_test_util::get_option("-m", "none-specified"); + if (boxFileBaseName == "none-specified") GTEST_SKIP(); + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 128) GTEST_SKIP(); + + const int numIterations = 20; + run_imported_surface_to_surface_test(boxFileBaseName, numIterations, stk::search::KDTREE); +} + +TEST(StkSearch_SurfaceToSurface, tractorTrailerCrash_floatBox_MORTON_LBVH) +{ + std::string boxFileBaseName = stk::unit_test_util::get_option("-m", "none-specified"); + if (boxFileBaseName == "none-specified") GTEST_SKIP(); + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 128) GTEST_SKIP(); + + const int numIterations = 20; + run_imported_surface_to_surface_test(boxFileBaseName, numIterations, stk::search::MORTON_LBVH); +} + +TEST(StkSearch_SurfaceToSurface, tractorTrailerCrash_floatBox_ARBORX) +{ + std::string boxFileBaseName = stk::unit_test_util::get_option("-m", "none-specified"); + if (boxFileBaseName == "none-specified") GTEST_SKIP(); + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 128) GTEST_SKIP(); + + const int numIterations = 20; + run_imported_surface_to_surface_test(boxFileBaseName, numIterations, stk::search::ARBORX); +} + +} // namespace + diff --git a/packages/stk/stk_performance_tests/stk_search/VolumeToOne.cpp b/packages/stk/stk_performance_tests/stk_search/VolumeToOne.cpp new file mode 100644 index 000000000000..e0b203e11b23 --- /dev/null +++ b/packages/stk/stk_performance_tests/stk_search/VolumeToOne.cpp @@ -0,0 +1,110 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include "stk_search/SearchMethod.hpp" + +namespace { + +template +void run_volume_to_one_test(const std::string& meshFileName, + stk::search::SearchMethod searchMethod, + bool enforceSearchResultSymmetry = true) +{ + using BoxType = typename BoxVectorType::value_type::first_type; + + MPI_Comm comm = MPI_COMM_WORLD; + const unsigned NUM_RUNS = 5; + const unsigned NUM_ITERS = 1000; + const int pRank = stk::parallel_machine_rank(comm); + stk::unit_test_util::BatchTimer batchTimer(comm); + batchTimer.initialize_batch_timer(); + + for (unsigned run = 0; run < NUM_RUNS; ++run) { + + stk::mesh::MeshBuilder builder(comm); + std::shared_ptr bulkPtr = builder.create(); + + stk::io::fill_mesh_with_auto_decomp(meshFileName, *bulkPtr); + + BoxVectorType elemBoxes; + createBoundingBoxesForEntities(*bulkPtr, stk::topology::ELEM_RANK, elemBoxes); + + BoxVectorType supersetBoxVec { elemBoxes[0] }; + auto & [supersetBox, supersetIdent] = supersetBoxVec[0]; + supersetIdent.set_id(pRank); + supersetIdent.set_proc(pRank); + + for (const auto & [box, ident] : elemBoxes) { + stk::search::add_to_box(supersetBox, box); + } + + SearchResults searchResults; + + batchTimer.start_batch_timer(); + for (unsigned i = 0; i < NUM_ITERS; ++i) { + stk::search::coarse_search(elemBoxes, supersetBoxVec, searchMethod, comm, searchResults, enforceSearchResultSymmetry); + } + batchTimer.stop_batch_timer(); + } + + batchTimer.print_batch_timing(NUM_ITERS); +} + +TEST(StkSearch_VolumeToOne, generatedMesh_floatBox_KDTREE) +{ + run_volume_to_one_test("generated:40x80x20|sideset:xXyYzZ", stk::search::KDTREE); +} + +TEST(StkSearch_VolumeToOne, generatedMesh_floatBox_MORTON_LBVH) +{ + run_volume_to_one_test("generated:40x80x20|sideset:xXyYzZ", stk::search::MORTON_LBVH); +} + +TEST(StkSearch_VolumeToOne, generatedMesh_floatBox_ARBORX) +{ + run_volume_to_one_test("generated:40x80x20|sideset:xXyYzZ", stk::search::ARBORX); +} + +} // namespace + diff --git a/packages/stk/stk_performance_tests/stk_search/VolumeToSurface.cpp b/packages/stk/stk_performance_tests/stk_search/VolumeToSurface.cpp new file mode 100644 index 000000000000..d7e0cdd1f2ad --- /dev/null +++ b/packages/stk/stk_performance_tests/stk_search/VolumeToSurface.cpp @@ -0,0 +1,287 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +template +void run_volume_to_surface_test(const std::string& meshFileName, + stk::search::SearchMethod searchMethod, + bool enforceSearchResultSymmetry = true) +{ + MPI_Comm comm = MPI_COMM_WORLD; + const unsigned NUM_RUNS = 5; + const unsigned NUM_ITERS = 100; + stk::unit_test_util::BatchTimer batchTimer(comm); + batchTimer.initialize_batch_timer(); + + for (unsigned run = 0; run < NUM_RUNS; ++run) { + + stk::mesh::MeshBuilder builder(comm); + std::shared_ptr bulkPtr = builder.create(); + + stk::io::fill_mesh_with_auto_decomp(meshFileName, *bulkPtr); + + BoxVectorType elemBoxes; + createBoundingBoxesForEntities(*bulkPtr, stk::topology::ELEM_RANK, elemBoxes); + + BoxVectorType sideBoxes; + createBoundingBoxesForEntities(*bulkPtr, stk::topology::FACE_RANK, sideBoxes); + + SearchResults searchResults; + + batchTimer.start_batch_timer(); + for (unsigned i = 0; i < NUM_ITERS; ++i) { + stk::search::coarse_search(elemBoxes, sideBoxes, searchMethod, comm, searchResults, enforceSearchResultSymmetry); + } + batchTimer.stop_batch_timer(); + } + + batchTimer.print_batch_timing(NUM_ITERS); +} + +TEST(StkSearch_VolumeToSurface, generatedMesh_floatBox_KDTREE) +{ + run_volume_to_surface_test("generated:40x80x20|sideset:xXyYzZ", stk::search::KDTREE); +} + +TEST(StkSearch_VolumeToSurface, generatedMesh_floatBox_MORTON_LBVH) +{ + run_volume_to_surface_test("generated:40x80x20|sideset:xXyYzZ", stk::search::MORTON_LBVH); +} + +TEST(StkSearch_VolumeToSurface, generatedMesh_floatBox_ARBORX) +{ + run_volume_to_surface_test("generated:40x80x20|sideset:xXyYzZ", stk::search::ARBORX); +} + + +TEST(StkSearch_VolumeToSurface, casaMesh_floatBox_KDTREE) +{ + std::string meshFileName = stk::unit_test_util::get_option("-m", "none-specified"); + if (meshFileName == "none-specified") GTEST_SKIP(); + + run_volume_to_surface_test(meshFileName, stk::search::KDTREE); +} + +TEST(StkSearch_VolumeToSurface, casaMesh_floatBox_MORTON_LBVH) +{ + std::string meshFileName = stk::unit_test_util::get_option("-m", "none-specified"); + if (meshFileName == "none-specified") GTEST_SKIP(); + + run_volume_to_surface_test(meshFileName, stk::search::MORTON_LBVH); +} + +TEST(StkSearch_VolumeToSurface, casaMesh_floatBox_ARBORX) +{ + std::string meshFileName = stk::unit_test_util::get_option("-m", "none-specified"); + if (meshFileName == "none-specified") GTEST_SKIP(); + + run_volume_to_surface_test(meshFileName, stk::search::ARBORX); +} + + +using ExecSpace = stk::ngp::ExecSpace; +using MemSpace = stk::ngp::ExecSpace::memory_space; + +inline Kokkos::View +createArborXBoundingBoxesForEntities(const stk::mesh::BulkData &bulk, + stk::mesh::EntityRank rank) +{ + stk::mesh::EntityVector entities; + const bool sortById = true; + stk::mesh::get_entities(bulk, rank, bulk.mesh_meta_data().locally_owned_part(), entities, sortById); + + size_t numberBoundingBoxes = entities.size(); + Kokkos::View boundingBoxes("ArborX BBs", numberBoundingBoxes); + auto boundingBoxesHost = Kokkos::create_mirror_view(boundingBoxes); + + stk::mesh::FieldBase const * coords = bulk.mesh_meta_data().coordinate_field(); + + std::vector boxCoordinates(6); + + for (size_t i = 0; i < entities.size(); ++i) { + unsigned num_nodes = bulk.num_nodes(entities[i]); + std::vector coordinates(3*num_nodes,0); + const stk::mesh::Entity* nodes = bulk.begin_nodes(entities[i]); + for (unsigned j = 0; j < num_nodes; ++j) { + double* data = static_cast(stk::mesh::field_data(*coords, nodes[j])); + coordinates[3*j] = data[0]; + coordinates[3*j+1] = data[1]; + coordinates[3*j+2] = data[2]; + } + findBoundingBoxCoordinates(coordinates, boxCoordinates); + ArborX::Point min_point(boxCoordinates[0], boxCoordinates[1], boxCoordinates[2]); + ArborX::Point max_point(boxCoordinates[3], boxCoordinates[4], boxCoordinates[5]); + boundingBoxesHost(i) = {min_point, max_point}; + } + + Kokkos::deep_copy(boundingBoxes, boundingBoxesHost); + + return boundingBoxes; +} + +void arborx_coarse_search(Kokkos::View elemBoxes, + Kokkos::View sideBoxes) +{ + ExecSpace execSpace; + + ArborX::BVH bvh{execSpace, elemBoxes}; + + const int numQueries = sideBoxes.extent(0); + Kokkos::View *, MemSpace> queries(Kokkos::ViewAllocateWithoutInitializing("queries"), numQueries); + + Kokkos::parallel_for("setup_queries", Kokkos::RangePolicy(0, numQueries), + KOKKOS_LAMBDA(int i) { queries(i) = ArborX::intersects(sideBoxes(i)); }); + Kokkos::fence(); + + Kokkos::View indices("Example::indices", 0); + Kokkos::View offsets("Example::offsets", 0); + + bvh.query(execSpace, queries, indices, offsets); +} + +void run_search_test_arborx(const std::string& meshFileName) +{ + const unsigned NUM_RUNS = 5; + const unsigned NUM_ITERS = 100; + stk::unit_test_util::BatchTimer batchTimer(MPI_COMM_WORLD); + batchTimer.initialize_batch_timer(); + + for (unsigned j = 0; j < NUM_RUNS; j++) { + + stk::mesh::MeshBuilder builder(MPI_COMM_WORLD); + std::shared_ptr bulkPtr = builder.create(); + + stk::io::fill_mesh_with_auto_decomp(meshFileName, *bulkPtr); + + Kokkos::View elemBoxes = createArborXBoundingBoxesForEntities(*bulkPtr, stk::topology::ELEM_RANK); + Kokkos::View sideBoxes = createArborXBoundingBoxesForEntities(*bulkPtr, stk::topology::FACE_RANK); + + batchTimer.start_batch_timer(); + for (unsigned i = 0; i < NUM_ITERS; ++i) { + arborx_coarse_search(elemBoxes, sideBoxes); + } + batchTimer.stop_batch_timer(); + } + + batchTimer.print_batch_timing(NUM_ITERS); +} + +TEST(StkSearch_VolumeToSurface, generatedMesh_floatBox_rawArborX) { + run_search_test_arborx("generated:40x80x20|sideset:xXyYzZ"); +} + +TEST(StkSearch_VolumeToSurface, casaMesh_floatBox_rawArborX) { + + std::string meshFileName = stk::unit_test_util::get_option("-m", "none-specified"); + if (meshFileName == "none-specified") GTEST_SKIP(); + + run_search_test_arborx(meshFileName); +} + +void distributed_arborx_coarse_search(Kokkos::View elemBoxes, + Kokkos::View sideBoxes, + MPI_Comm comm) +{ + + ExecSpace execSpace; + + ArborX::DistributedTree tree(comm, execSpace, elemBoxes); + + const int numQueries = sideBoxes.extent(0); + Kokkos::View *, MemSpace> queries(Kokkos::ViewAllocateWithoutInitializing("queries"), numQueries); + + Kokkos::parallel_for("setup_queries", Kokkos::RangePolicy(0, numQueries), + KOKKOS_LAMBDA(int i) { queries(i) = ArborX::intersects(sideBoxes(i)); }); + Kokkos::fence(); + + Kokkos::View values("indicesAndRanks", 0); + Kokkos::View offsets("offsets", 0); + + tree.query(execSpace, queries, values, offsets); +} + +void run_search_test_distributed_arborx(const std::string& meshFileName) +{ + const unsigned NUM_RUNS = 5; + const unsigned NUM_ITERS = 100; + stk::unit_test_util::BatchTimer batchTimer(MPI_COMM_WORLD); + batchTimer.initialize_batch_timer(); + + for (unsigned j = 0; j < NUM_RUNS; j++) { + + stk::mesh::MeshBuilder builder(MPI_COMM_WORLD); + std::shared_ptr bulkPtr = builder.create(); + + stk::io::fill_mesh_with_auto_decomp(meshFileName, *bulkPtr); + + Kokkos::View elemBoxes = createArborXBoundingBoxesForEntities(*bulkPtr, stk::topology::ELEM_RANK); + Kokkos::View sideBoxes = createArborXBoundingBoxesForEntities(*bulkPtr, stk::topology::FACE_RANK); + + batchTimer.start_batch_timer(); + for (unsigned i = 0; i < NUM_ITERS; ++i) { + distributed_arborx_coarse_search(elemBoxes, sideBoxes, MPI_COMM_WORLD); + } + batchTimer.stop_batch_timer(); + } + + batchTimer.print_batch_timing(NUM_ITERS); +} + +TEST(StkSearch_VolumeToSurface, generatedMesh_floatBox_distributed_rawArborX) { + run_search_test_distributed_arborx("generated:40x80x20|sideset:xXyYzZ"); +} + +TEST(StkSearch_VolumeToSurface, casaMesh_floatBox_distributed_rawArborX) { + + std::string meshFileName = stk::unit_test_util::get_option("-m", "none-specified"); + if (meshFileName == "none-specified") GTEST_SKIP(); + + run_search_test_distributed_arborx(meshFileName); +} + +} // namespace + diff --git a/packages/stk/stk_performance_tests/stk_search/HexMeshSearch.cpp b/packages/stk/stk_performance_tests/stk_search/VolumeToVolume.cpp similarity index 60% rename from packages/stk/stk_performance_tests/stk_search/HexMeshSearch.cpp rename to packages/stk/stk_performance_tests/stk_search/VolumeToVolume.cpp index 423ef4f9624b..87d5c511fa98 100644 --- a/packages/stk/stk_performance_tests/stk_search/HexMeshSearch.cpp +++ b/packages/stk/stk_performance_tests/stk_search/VolumeToVolume.cpp @@ -46,72 +46,66 @@ namespace { template -void run_search_test(const std::string& meshFileName, - stk::search::SearchMethod searchMethod, - bool communicateResultInfo = true) +void run_volume_to_volume_test(const std::string& meshFileName, + stk::search::SearchMethod searchMethod, + bool enforceSearchResultSymmetry = true) { + MPI_Comm comm = MPI_COMM_WORLD; const unsigned NUM_RUNS = 5; - const unsigned NUM_ITERS = 100; - stk::unit_test_util::BatchTimer batchTimer(MPI_COMM_WORLD); + const unsigned NUM_ITERS = 20; + stk::unit_test_util::BatchTimer batchTimer(comm); batchTimer.initialize_batch_timer(); - for (unsigned j = 0; j < NUM_RUNS; j++) { - stk::mesh::MeshBuilder builder(MPI_COMM_WORLD); + for (unsigned run = 0; run < NUM_RUNS; ++run) { + + stk::mesh::MeshBuilder builder(comm); std::shared_ptr bulkPtr = builder.create(); stk::io::fill_mesh_with_auto_decomp(meshFileName, *bulkPtr); BoxVectorType elemBoxes; createBoundingBoxesForEntities(*bulkPtr, stk::topology::ELEM_RANK, elemBoxes); - BoxVectorType sideBoxes; - createBoundingBoxesForEntities(*bulkPtr, stk::topology::FACE_RANK, sideBoxes); - batchTimer.start_batch_timer(); + SearchResults searchResults; - for(unsigned i=0; i("generated:40x80x20|sideset:xXyYzZ", stk::search::KDTREE); + run_volume_to_volume_test("generated:40x80x20|sideset:xXyYzZ", stk::search::KDTREE); } -TEST(StkSearch, hex_mesh_KDTREE_double) +TEST(StkSearch_VolumeToVolume, generatedMesh_doubleBox_KDTREE) { - run_search_test("generated:40x80x20|sideset:xXyYzZ", stk::search::KDTREE); + run_volume_to_volume_test("generated:40x80x20|sideset:xXyYzZ", stk::search::KDTREE); } -TEST(StkSearch, perf_mesh_KDTREE_float) +TEST(StkSearch_VolumeToVolume, generatedMesh_floatBox_MORTON_LBVH) { - std::string meshFileName = stk::unit_test_util::get_option("-m", "none-specified"); - if (meshFileName == "none-specified") { - std::cout<<"Skipping test, no mesh specified via '-m' command-line flag."<(meshFileName, stk::search::KDTREE); + run_volume_to_volume_test("generated:40x80x20|sideset:xXyYzZ", stk::search::MORTON_LBVH); } -TEST(StkSearch, perf_mesh_KDTREE_double) +TEST(StkSearch_VolumeToVolume, generatedMesh_doubleBox_MORTON_LBVH) { - std::string meshFileName = stk::unit_test_util::get_option("-m", "none-specified"); - if (meshFileName == "none-specified") { - std::cout<<"Skipping test, no mesh specified via '-m' command-line flag."<("generated:40x80x20|sideset:xXyYzZ", stk::search::MORTON_LBVH); +} - std::cout<<"running search test with '"<("generated:40x80x20|sideset:xXyYzZ", stk::search::ARBORX); +} - run_search_test(meshFileName, stk::search::KDTREE); +TEST(StkSearch_VolumeToVolume, generatedMesh_doubleBox_ARBORX) +{ + run_volume_to_volume_test("generated:40x80x20|sideset:xXyYzZ", stk::search::ARBORX); } } // namespace diff --git a/packages/stk/stk_performance_tests/stk_util/perfParallelExchange.cpp b/packages/stk/stk_performance_tests/stk_util/perfParallelExchange.cpp new file mode 100644 index 000000000000..b0c47eec71b3 --- /dev/null +++ b/packages/stk/stk_performance_tests/stk_util/perfParallelExchange.cpp @@ -0,0 +1,148 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +void fill_comm_data(int numProcs, int myProc, + unsigned BANDWIDTH, unsigned NUM_VALUES, + std::vector& sendOffsets, + std::vector& sendData, + std::vector& recvOffsets, + std::vector& recvData) +{ + sendOffsets.assign(numProcs+1, 0); + + const int halfBandwidth = BANDWIDTH/2; + const int rawFirstDestProc = myProc - halfBandwidth; + const int rawLastDestProc = myProc + halfBandwidth; + int firstDestProc = std::max(0, rawFirstDestProc); + if (firstDestProc == myProc) { + firstDestProc += 1; + } + + int lastDestProc = std::min(numProcs-1, rawLastDestProc); + if (lastDestProc == myProc) { + lastDestProc -= 1; + } + + //first, put send-sizes in sendOffsets: + for(int p=firstDestProc; p<=lastDestProc; ++p) { + if (p != myProc) { + sendOffsets[p] = NUM_VALUES; + } + } + + recvOffsets = sendOffsets; + + //the send/recv procs are 'symmetric', but data-sizes will be non-symmetric: + //add some variation to the send-sizes + for(int p=firstDestProc; p<=lastDestProc; ++p) { + if (p != myProc) { + sendOffsets[p] += p; + recvOffsets[p] += myProc; + } + } + + //now, convert sizes to offsets: + unsigned sendOffset = 0; + unsigned recvOffset = 0; + for(unsigned p=0; p +struct BoxIdent +{ + using box_type = BoxType; + using ident_type = IdentType; + using first_type = BoxType; + using second_type = IdentType; + + BoxType box; + IdentType ident; +}; + +template +struct BoxIdentProc +{ + using box_type = BoxType; + using ident_proc_type = IdentProcType; + using first_type = BoxType; + using second_type = IdentProcType; + + BoxType box; + IdentProcType identProc; +}; + +template +struct IdentIntersection +{ + using domain_ident_type = DomainIdentType; + using range_ident_type = RangeIdentType; + + DomainIdentType domainIdent; + RangeIdentType rangeIdent; + + KOKKOS_FORCEINLINE_FUNCTION bool operator==(IdentIntersection const& rhs) const + { + return domainIdent == rhs.domainIdent && rangeIdent == rhs.rangeIdent; + } + + KOKKOS_FORCEINLINE_FUNCTION bool operator<(IdentIntersection const& rhs) const + { + return domainIdent < rhs.domainIdent || + (!(rhs.domainIdent < domainIdent) && rangeIdent < rhs.rangeIdent); + } +}; + +template +struct IdentProcIntersection +{ + DomainIdentProcType domainIdentProc; + RangeIdentProcType rangeIdentProc; +}; + +} + +#endif // BOXIDENT_HPP diff --git a/packages/stk/stk_search/stk_search/CMakeLists.txt b/packages/stk/stk_search/stk_search/CMakeLists.txt index bc33712238c6..dbb061f773a8 100644 --- a/packages/stk/stk_search/stk_search/CMakeLists.txt +++ b/packages/stk/stk_search/stk_search/CMakeLists.txt @@ -34,8 +34,14 @@ SET(HEADERS "") SET(SOURCES "") -FILE(GLOB HEADERS *.hpp) -FILE(GLOB SOURCES *.cpp) +FILE(GLOB HEADERS *.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/kdtree/*.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/morton_lbvh/*.hpp +) +FILE(GLOB SOURCES *.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/kdtree/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/morton_lbvh/*.cpp +) if(HAVE_STK_Trilinos) TRIBITS_ADD_LIBRARY(stk_search @@ -48,8 +54,12 @@ else() target_link_libraries(stk_search PUBLIC stk_math) target_link_libraries(stk_search PUBLIC stk_util_parallel) - find_package(KokkosKernels REQUIRED) - target_link_libraries(stk_search PUBLIC Kokkos::kokkoskernels) + if(STK_HAS_ARBORX) + target_link_libraries(stk_search PUBLIC ArborX::ArborX) + endif() + +# find_package(KokkosKernels REQUIRED) +# target_link_libraries(stk_search PUBLIC KokkosKernels::all_libs) endif() target_include_directories(stk_search PUBLIC diff --git a/packages/stk/stk_search/stk_search/CoarseSearch.hpp b/packages/stk/stk_search/stk_search/CoarseSearch.hpp index f67d0ddefe61..c061887d0ff7 100644 --- a/packages/stk/stk_search/stk_search/CoarseSearch.hpp +++ b/packages/stk/stk_search/stk_search/CoarseSearch.hpp @@ -39,74 +39,115 @@ #include #include #include -#include +#include +#include +#ifdef STK_HAS_ARBORX +#include +#endif + #include +#include +#include #include #include +namespace stk::search { -namespace stk { namespace search { - -inline -std::ostream& operator<<(std::ostream &out, SearchMethod method) +// After providing a vector of local domain and range search box information +// (that includes both the box itself and the target entity ID and owning +// processor), this function will compute a parallel-consistent list of +// all box intersections between local entities (in the domain vector) and +// range entities from all processors. +// +// The "enforceSearchResultSymmetry" argument indicates whether you want to +// perform an additional round of parallel communication to make sure that +// no intersections with anything in the local domain or range vectors have +// been missed by only being detected on another processor. Intersections +// can be missed if, say, some local entities exist only in the range vector +// and not the domain vector. This flag will add another round of parallel +// communication that makes sure all local interactions that involve an +// off-processor entity are shared with that remote processor. Results are +// sorted if this flag is set. +// +// Search performance will generally be better if the global domain vector is +// larger than the range vector. The "autoSwapDomainAndRange" argument +// controls if a global input length comparison is made, followed by +// potentially swapping the domain and range vectors. The final results should +// be independent of this flag. +// +//BEGINcoarse_search_impl +template +void coarse_search(std::vector> const & domain, + std::vector> const & range, + SearchMethod method, + stk::ParallelMachine comm, + std::vector>& intersections, + bool enforceSearchResultSymmetry = true, + bool autoSwapDomainAndRange = true) { - switch( method ) { - case KDTREE: out << "KDTREE"; break; - case MORTON_LINEARIZED_BVH: out << "MORTON_LINEARIZED_BVH"; break; + switch (method) { + case ARBORX: { +#ifdef STK_HAS_ARBORX + coarse_search_arborx(domain, range, comm, intersections, enforceSearchResultSymmetry); +#else + STK_ThrowErrorMsg("STK(stk_search) was not configured with ARBORX enabled. Please use KDTREE or MORTON_LBVH."); +#endif + break; + } + case KDTREE: { + if (autoSwapDomainAndRange) { + coarse_search_kdtree_driver(domain, range, comm, intersections, enforceSearchResultSymmetry); + } + else { + coarse_search_kdtree(domain, range, comm, intersections, enforceSearchResultSymmetry); + } + break; + } + case MORTON_LBVH: { + coarse_search_morton_lbvh(domain, range, comm, intersections, enforceSearchResultSymmetry); + break; + } + default: { + STK_ThrowErrorMsg("Unsupported coarse_search method supplied. Choices are: KDTREE, MORTON_LVBH, or ARBORX."); + } } - return out; -} - -// THIS MIGHT BE WHAT WE ACTUALLY WANT FOR THE INTERFACE. -template -void coarse_search( - std::vector > const& domain, - std::vector > const& range, - SearchMethod method, - stk::ParallelMachine comm, - std::vector< std::pair< IdentProc, - IdentProc > > & intersections - ) -{ - std::cerr << "Future version of coarse_search called" << std::endl; - abort(); } +//ENDcoarse_search_impl -// intersections will be those of distributed domain boxes associated with this -// processor rank via get_proc(.) that intersect distributed range -// boxes. Optionally, also include intersections of distributed domain boxes -// with distributed range boxes associated with this processor rank via -// get_proc(.). -// Note that the search results that are placed in the intersections result argument -// are not sorted. If the caller needs this vector to be sorted, you must -// call std::sort(intersections.begin(), intersections.end()) or similar. -//BEGINcoarse_search_impl -template -void coarse_search(std::vector> const& domain, - std::vector> const& range, - SearchMethod method, - stk::ParallelMachine comm, - std::vector>& intersections, - bool communicateRangeBoxInfo = true, - bool determineDomainAndRange = true) +template +void coarse_search(Kokkos::View*, ExecutionSpace> const & domain, + Kokkos::View*, ExecutionSpace> const & range, + SearchMethod method, + stk::ParallelMachine comm, + Kokkos::View*, ExecutionSpace>& intersections, + ExecutionSpace const& execSpace = Kokkos::DefaultExecutionSpace{}, + bool enforceSearchResultSymmetry = true, + bool autoSwapDomainAndRange = true) { - switch( method ) - { - case KDTREE: - if (determineDomainAndRange) { - coarse_search_kdtree_driver(domain,range,comm,intersections,communicateRangeBoxInfo); + switch (method) { + case ARBORX: { +#ifdef STK_HAS_ARBORX + coarse_search_arborx(domain, range, comm, intersections, enforceSearchResultSymmetry); +#else + STK_ThrowErrorMsg("STK(stk_search) was not configured with ARBORX enabled. Please use KDTREE or MORTON_LBVH."); +#endif + break; } - else { - coarse_search_kdtree(domain,range,comm,intersections,communicateRangeBoxInfo); + case KDTREE: { + STK_ThrowErrorMsg("The KDTREE search method is not supported on GPUs. Please use MORTON_LBVH or ARBORX instead."); + break; + } + case MORTON_LBVH: { + // Parallel Morton WIP + STK_ThrowErrorMsg("The parallel Morton search on GPU is currently WIP. Please use ARBORX instead (if enabled)."); + break; + } + default: { + STK_ThrowErrorMsg("Unsupported coarse_search method supplied. Choices are: KDTREE, MORTON_LVBH, or ARBORX."); } - break; - default: - std::cerr << "coarse_search(..) interface used does not support SearchMethod " << method << std::endl; - abort(); - break; } } -//ENDcoarse_search_impl -}} // namespace stk::search + +} // namespace stk::search #endif // stk_search_CoarseSearch_hpp diff --git a/packages/stk/stk_search/stk_search/CoarseSearchKdTree.hpp b/packages/stk/stk_search/stk_search/CoarseSearchKdTree.hpp deleted file mode 100644 index db76daf90bb6..000000000000 --- a/packages/stk/stk_search/stk_search/CoarseSearchKdTree.hpp +++ /dev/null @@ -1,271 +0,0 @@ -// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering -// Solutions of Sandia, LLC (NTESS). Under the terms of Contract -// DE-NA0003525 with NTESS, the U.S. Government retains certain rights -// in this software. -// - // Redistribution and use in source and binary forms, with or without - // modification, are permitted provided that the following conditions are - // met: - // - // * Redistributions of source code must retain the above copyright - // notice, this list of conditions and the following disclaimer. - // - // * Redistributions in binary form must reproduce the above - // copyright notice, this list of conditions and the following - // disclaimer in the documentation and/or other materials provided - // with the distribution. - // -// * Neither the name of NTESS nor the names of its contributors -// may be used to endorse or promote products derived from this -// software without specific prior written permission. -// - // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -#ifndef KDTREE_STK_INTERFACE_H_ -#define KDTREE_STK_INTERFACE_H_ - -#ifdef _OPENMP -#include -#endif - -#include "stk_search/KDTree_BoundingBox.hpp" -#include "stk_util/environment/Env.hpp" -#include "stk_util/environment/WallTime.hpp" -#include "stk_util/parallel/ParallelReduce.hpp" -#include -#include -#include - - -namespace stk { - namespace search { - - // - // More general search for an arbitrary range type - // - template - inline void coarse_search_kdtree(std::vector< std::pair > const & local_domain, - std::vector< std::pair > const & local_range, - MPI_Comm comm, - std::vector >& searchResults, - bool communicateRangeBoxInfo=true) - { - - int num_procs = -1; - int proc_id = -1; - MPI_Comm_rank(comm, &proc_id); - MPI_Comm_size(comm, &num_procs); - - std::vector rangeObjs( local_range.size() ); - - std::vector rangeGhostIdentifiers; - - stk::search::ComputeRangeWithGhostsForCoarseSearch(local_domain, local_range, - num_procs, rangeObjs, rangeGhostIdentifiers, comm); - -#ifdef _OPENMP - std::vector > > - threadLocalSearchResults( omp_get_max_threads() ); -#endif - - if ((local_domain.size() > 0) && (rangeObjs.size() > 0)) { - - // - // Need to convert range objects to actual box type objects for proximity search - // - - using rangeValueType = typename RangeObjType::value_type; - using RangeBox = stk::search::Box; - - std::vector rangeBoxes; - rangeBoxes.reserve( rangeObjs.size() ); - - for(auto& p : rangeObjs) { - auto& obj = p; - rangeBoxes.emplace_back(RangeBox(obj.get_x_min(), obj.get_y_min(), obj.get_z_min(), - obj.get_x_max(), obj.get_y_max(), obj.get_z_max())); - - } - - const stk::search::ProximitySearchTree_T proxSearch(rangeBoxes); - const unsigned numBoxDomain = local_domain.size(); - -#ifdef _OPENMP -#pragma omp parallel default(shared) -#endif - { - // - // Set the known return vector sizes - // - - std::vector overlapList; -#ifdef _OPENMP - std::vector >& interList = threadLocalSearchResults[omp_get_thread_num()]; -#else - std::vector >& interList = searchResults; -#endif - // - // Create an array to store interactions returned by the recursive search routines. There are at maximum - // N interactions per object when searching N objects - // - - // - // Loop over all boxAs in group1 and search them against those objects in group2 - // -#ifdef _OPENMP -#pragma omp for -#endif - for(unsigned int iboxDomain = 0; iboxDomain < numBoxDomain; ++iboxDomain) { - proxSearch.SearchForOverlap(local_domain[iboxDomain].first, overlapList); - for(auto&& jboxRange : overlapList) { - if(intersects(local_domain[iboxDomain].first, rangeObjs[jboxRange])) { - if(jboxRange < (int)local_range.size()) { - interList.emplace_back( local_domain[iboxDomain].second, local_range[jboxRange].second ); - } else { - interList.emplace_back( local_domain[iboxDomain].second, rangeGhostIdentifiers[jboxRange-local_range.size()] ); - } - } - } - } - } - } -#ifdef _OPENMP - stk::search::ConcatenateThreadLists(threadLocalSearchResults, searchResults); -#endif - - if(communicateRangeBoxInfo) { - stk::search::communicateVector(comm, searchResults, communicateRangeBoxInfo); - std::sort(searchResults.begin(), searchResults.end()); - } - } - - - - // - // Most optimal search specific to actual box arguments - // - template - inline void coarse_search_kdtree(std::vector< std::pair > const & local_domain, - std::vector< std::pair, RangeIdentifier > > const & local_range, - MPI_Comm comm, - std::vector >& searchResults, - bool communicateRangeBoxInfo=true) - { - int num_procs = -1; - int proc_id = -1; - MPI_Comm_rank(comm, &proc_id); - MPI_Comm_size(comm, &num_procs); - -#ifdef _OPENMP - std::vector > > - threadLocalSearchResults( omp_get_max_threads() ); -#endif - - { - std::vector > rangeBoxes( local_range.size() ); - - std::vector rangeGhostIdentifiers; - - stk::search::ComputeRangeWithGhostsForCoarseSearch(local_domain, local_range, - num_procs, rangeBoxes, rangeGhostIdentifiers, comm); - - if ((local_domain.size() > 0) && (rangeBoxes.size() > 0)) { - - const stk::search::ProximitySearchTree_T > proxSearch(rangeBoxes); - const unsigned numBoxDomain = local_domain.size(); - -#ifdef _OPENMP -#pragma omp parallel default(shared) -#endif - { - // - // Set the known return vector sizes - // - - std::vector overlapList; -#ifdef _OPENMP - std::vector >& interList = threadLocalSearchResults[omp_get_thread_num()]; -#else - std::vector >& interList = searchResults; -#endif - - // - // Create an array to store interactions returned by the recursive search routines. There are at maximum - // N interactions per object when searching N objects - // - - // - // Loop over all boxAs in group1 and search them against those objects in group2 - // - interList.reserve((numBoxDomain*3)/2); -#ifdef _OPENMP -#pragma omp for -#endif - for(unsigned int iboxDomain = 0; iboxDomain < numBoxDomain; ++iboxDomain) { - proxSearch.SearchForOverlap(local_domain[iboxDomain].first, overlapList); - for(auto&& jboxRange : overlapList) { - if(jboxRange < (int)local_range.size()) { - interList.emplace_back( local_domain[iboxDomain].second, local_range[jboxRange].second ); - } else { - interList.emplace_back( local_domain[iboxDomain].second, rangeGhostIdentifiers[jboxRange-local_range.size()] ); - } - } - } - } - } - } -#ifdef _OPENMP - stk::search::ConcatenateThreadLists(threadLocalSearchResults, searchResults); -#endif - if(communicateRangeBoxInfo) { - stk::search::communicateVector(comm, searchResults, communicateRangeBoxInfo); - std::sort(searchResults.begin(), searchResults.end()); - } - } - - template - inline void coarse_search_kdtree_driver(std::vector< std::pair > const & local_domain, - std::vector< std::pair > const & local_range, - MPI_Comm comm, - std::vector >& searchResults, - bool communicateRangeBoxInfo=true) - { - const size_t local_sizes[2] = {local_domain.size(), local_range.size()}; - size_t global_sizes[2]; - all_reduce_sum(comm, local_sizes, global_sizes, 2); - const bool domain_has_more_boxes = (global_sizes[0] >= global_sizes[1]); - if(domain_has_more_boxes) - { - coarse_search_kdtree(local_domain, local_range, comm, searchResults, communicateRangeBoxInfo); - } - else - { - std::vector > tempSearchResults; - coarse_search_kdtree(local_range, local_domain, comm, tempSearchResults, communicateRangeBoxInfo); - const int p_rank = stk::parallel_machine_rank(comm); - searchResults.reserve(tempSearchResults.size()); - for(size_t i=0; i()(pair.second) == p_rank) - searchResults.emplace_back(pair.second, pair.first); - } - } - } - - } -} - -#endif diff --git a/packages/stk/stk_search/stk_search/CommonSearchUtil.hpp b/packages/stk/stk_search/stk_search/CommonSearchUtil.hpp index a53d0217742f..d5af3a9652b4 100644 --- a/packages/stk/stk_search/stk_search/CommonSearchUtil.hpp +++ b/packages/stk/stk_search/stk_search/CommonSearchUtil.hpp @@ -3,33 +3,33 @@ // DE-NA0003525 with NTESS, the U.S. Government retains certain rights // in this software. // - // Redistribution and use in source and binary forms, with or without - // modification, are permitted provided that the following conditions are - // met: - // - // * Redistributions of source code must retain the above copyright - // notice, this list of conditions and the following disclaimer. - // - // * Redistributions in binary form must reproduce the above - // copyright notice, this list of conditions and the following - // disclaimer in the documentation and/or other materials provided - // with the distribution. - // +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// // * Neither the name of NTESS nor the names of its contributors // may be used to endorse or promote products derived from this // software without specific prior written permission. // - // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef COMMON_SEARCH_UTIL_H_ #define COMMON_SEARCH_UTIL_H_ @@ -39,255 +39,123 @@ #endif #include "stk_util/environment/WallTime.hpp" +#include "stk_util/parallel/Parallel.hpp" #include "stk_util/parallel/CommSparse.hpp" #include "stk_util/parallel/ParallelComm.hpp" #include "stk_util/util/SortAndUnique.hpp" -#include "stk_search/KDTree_BoundingBox.hpp" -#include "stk_search/KDTree.hpp" -#include "mpi.h" - -namespace stk { - namespace search { - - template - inline void GlobalBoxCombine(DomainBox &box_array, MPI_Comm &communicator) - { - typedef typename DomainBox::value_type::coordinate_t coordinate_t; - - int num_boxes = static_cast(box_array.size()); - // - // Allocate a common set of arrays to perform the reductions on - // - int array_length = num_boxes * 3; - std::vector all_box_min_local (array_length); - std::vector all_box_max_local (array_length); - std::vector all_box_min_global(array_length); - std::vector all_box_max_global(array_length); - // - // Fill the local arrays - // - for(int ibox = 0; ibox < num_boxes; ++ibox) { - all_box_min_local[ibox * 3 + 0] = box_array[ibox].GetBox().get_x_min(); - all_box_min_local[ibox * 3 + 1] = box_array[ibox].GetBox().get_y_min(); - all_box_min_local[ibox * 3 + 2] = box_array[ibox].GetBox().get_z_min(); - all_box_max_local[ibox * 3 + 0] = box_array[ibox].GetBox().get_x_max(); - all_box_max_local[ibox * 3 + 1] = box_array[ibox].GetBox().get_y_max(); - all_box_max_local[ibox * 3 + 2] = box_array[ibox].GetBox().get_z_max(); - } - // - // Perform the global MPI reductions - // - MPI_Datatype floatType; - if(sizeof(coordinate_t) == sizeof(float)) { - floatType = MPI_FLOAT; - } else if (sizeof(coordinate_t) == sizeof(double)) { - floatType = MPI_DOUBLE; - } else { - floatType = MPI_DOUBLE; - } - MPI_Allreduce( all_box_min_local.data(), all_box_min_global.data(), array_length, floatType, MPI_MIN, communicator ); - MPI_Allreduce( all_box_max_local.data(), all_box_max_global.data(), array_length, floatType, MPI_MAX, communicator ); - // - // Scatter the local arrays back to the boxes - // - for(int ibox = 0; ibox < num_boxes; ++ibox) { - box_array[ibox].GetBox().set_box(all_box_min_global[ibox * 3 + 0], - all_box_min_global[ibox * 3 + 1], - all_box_min_global[ibox * 3 + 2], - all_box_max_global[ibox * 3 + 0], - all_box_max_global[ibox * 3 + 1], - all_box_max_global[ibox * 3 + 2]); - } - } - - - // - // Exchange boxes so that current proc local box is sent to the global box on all processors. - // - template - inline void AllGatherHelper(const DomainBox& localBox, std::vector &global_box_array, MPI_Comm &comm) - { - int numProc; - MPI_Comm_size(comm, &numProc); - global_box_array.clear(); - global_box_array.resize(numProc); - const char* localDataConst = reinterpret_cast(&localBox); - // NKC, hack to support old MPI version used by Goodyear - char* localData = const_cast(localDataConst); - - MPI_Allgather(localData, sizeof(DomainBox), MPI_CHAR, global_box_array.data(), sizeof(DomainBox), MPI_CHAR, comm); - } - - - - template - void ParallelComputeProcObjectBoxes(const std::vector >& local_objsWithIdents, - std::vector > &objBB_proc_box_array, - MPI_Comm &comm) - { - const unsigned numBoxDomain = local_objsWithIdents.size(); - stk::search::ObjectBoundingBox_T objBB_proc; - - #ifdef _OPENMP - std::vector > threadBoxes( omp_get_max_threads() ); - #endif - - #ifdef _OPENMP - #pragma omp parallel default(shared) - #endif - { - #ifdef _OPENMP - stk::search::ObjectBoundingBox_T& curBox = threadBoxes[omp_get_thread_num()]; - #else - stk::search::ObjectBoundingBox_T& curBox = objBB_proc; - #endif - #ifdef _OPENMP - #pragma omp for - #endif - for(unsigned ibox = 0; ibox < numBoxDomain; ++ibox) { - stk::search::add_to_box(curBox.GetBox(), local_objsWithIdents[ibox].first); - } - - } - - #ifdef _OPENMP - for(unsigned i=0; i - void - ComputeRangeWithGhostsForCoarseSearch( - const std::vector >& local_domain, - const std::vector >& local_range, - int num_procs, - std::vector& rangeBoxes, - std::vector& rangeGhostIdentifiers, MPI_Comm comm) - { - - const unsigned numBoxRange = local_range.size(); - - using domainValueType = typename DomainObjType::value_type; - using DomainBox = stk::search::Box; - +#include "stk_search/kdtree/KDTree_BoundingBox.hpp" +#include "stk_search/kdtree/KDTree.hpp" -#ifdef _OPENMP -#pragma omp parallel for default(shared) -#endif - for (size_t i = 0; i < numBoxRange; i++) { - rangeBoxes[i] = local_range[i].first; - } +namespace stk::search { - // - // Determine the total number of processors involved in the communication and the current processor number - // - int current_proc(0); - MPI_Comm_rank(comm, ¤t_proc); - if(num_procs == 0) { - return; - } - - // - // Compute the processor local bounding boxes for the box sets - // Store the boxes in unique entries in a global processor bounding box array. - // - std::vector > boxA_proc_box_array; - ParallelComputeProcObjectBoxes(local_domain, boxA_proc_box_array, comm); - - // - // Create a hierarchy of boxA processor bounding boxes. - // This hierarchy will be used to search for overlaps between processors and - // objects. - // - stk::search::ProximitySearchTree_T boxA_box_hierarchy(boxA_proc_box_array); - - // - // Determine what to ghost. If a boxB box from this processor overlaps another processor's - // processor all-boxA box, then we need to ghost the boxB's data to that other processor - // to include in the range boxes (local + global) to search against its local domain boxes. - // - - typedef typename RangeIdentifier::ident_type GlobalIdType; - typedef std::pair BoxIdPair; - std::vector > send_list(num_procs); - std::vector > recv_list(num_procs); - - std::vector proc_list(num_procs); - for(unsigned int iboxB = 0; iboxB < numBoxRange; ++iboxB) { - boxA_box_hierarchy.SearchForOverlap(local_range[iboxB].first, proc_list); - for(auto&& overlapping_proc : proc_list) { - if(overlapping_proc == current_proc) continue; - GlobalIdType id = local_range[iboxB].second.id(); - send_list[overlapping_proc].push_back(BoxIdPair(rangeBoxes[iboxB], id)); - } - } +template +inline void global_box_combine(DomainBox &box_array, MPI_Comm &communicator) +{ + typedef typename DomainBox::value_type::coordinate_t coordinate_t; + + int num_boxes = static_cast(box_array.size()); + // + // Allocate a common set of arrays to perform the reductions on + // + int array_length = num_boxes * 3; + std::vector all_box_min_local (array_length); + std::vector all_box_max_local (array_length); + std::vector all_box_min_global(array_length); + std::vector all_box_max_global(array_length); + // + // Fill the local arrays + // + for(int ibox = 0; ibox < num_boxes; ++ibox) { + all_box_min_local[ibox * 3 + 0] = box_array[ibox].GetBox().get_x_min(); + all_box_min_local[ibox * 3 + 1] = box_array[ibox].GetBox().get_y_min(); + all_box_min_local[ibox * 3 + 2] = box_array[ibox].GetBox().get_z_min(); + all_box_max_local[ibox * 3 + 0] = box_array[ibox].GetBox().get_x_max(); + all_box_max_local[ibox * 3 + 1] = box_array[ibox].GetBox().get_y_max(); + all_box_max_local[ibox * 3 + 2] = box_array[ibox].GetBox().get_z_max(); + } + // + // Perform the global MPI reductions + // + MPI_Datatype floatType; + if(sizeof(coordinate_t) == sizeof(float)) { + floatType = MPI_FLOAT; + } + else if (sizeof(coordinate_t) == sizeof(double)) { + floatType = MPI_DOUBLE; + } + else { + floatType = MPI_DOUBLE; + } + MPI_Allreduce(all_box_min_local.data(), all_box_min_global.data(), array_length, floatType, MPI_MIN, communicator); + MPI_Allreduce(all_box_max_local.data(), all_box_max_global.data(), array_length, floatType, MPI_MAX, communicator); + // + // Scatter the local arrays back to the boxes + // + for(int ibox = 0; ibox < num_boxes; ++ibox) { + box_array[ibox].GetBox().set_box(all_box_min_global[ibox * 3 + 0], + all_box_min_global[ibox * 3 + 1], + all_box_min_global[ibox * 3 + 2], + all_box_max_global[ibox * 3 + 0], + all_box_max_global[ibox * 3 + 1], + all_box_max_global[ibox * 3 + 2]); + } +} - stk::parallel_data_exchange_t(send_list, recv_list, comm); - rangeGhostIdentifiers.clear(); - for (size_t i = 0; i < recv_list.size(); i++) { - for (size_t j = 0; j < recv_list[i].size(); j++) { - const BoxIdPair& recvd_boxIdPair = recv_list[i][j]; - rangeBoxes.push_back(recvd_boxIdPair.first); - rangeGhostIdentifiers.push_back(RangeIdentifier(recvd_boxIdPair.second, i)); - } - } - } +// +// Exchange boxes so that current proc local box is sent to the global box on all processors. +// +template +inline void all_gather_helper(const DomainBox& localBox, std::vector &global_box_array, MPI_Comm &comm) +{ + int numProc; + MPI_Comm_size(comm, &numProc); + global_box_array.clear(); + global_box_array.resize(numProc); + const char* localDataConst = reinterpret_cast(&localBox); + // NKC, hack to support old MPI version used by Goodyear + char* localData = const_cast(localDataConst); + + MPI_Allgather(localData, sizeof(DomainBox), MPI_CHAR, global_box_array.data(), sizeof(DomainBox), MPI_CHAR, comm); +} - template - inline void ConcatenateThreadLists(const std::vector > &vectorIn, - std::vector &vectorOut) - { - const unsigned numThreadLists = vectorIn.size(); - std::vector offsets(numThreadLists); - unsigned totSize = 0; - for (unsigned i = 0; i < numThreadLists; ++i) { - offsets[i] = totSize; - totSize += vectorIn[i].size(); - } +template +inline void concatenate_thread_lists(const std::vector> &vectorIn, + std::vector &vectorOut) +{ + const unsigned numThreadLists = vectorIn.size(); + std::vector offsets(numThreadLists); + unsigned totSize = 0; + for (unsigned i = 0; i < numThreadLists; ++i) { + offsets[i] = totSize; + totSize += vectorIn[i].size(); + } - vectorOut.resize(totSize); + vectorOut.resize(totSize); #ifdef _OPENMP #pragma omp parallel default(shared) - { - const unsigned ithread = omp_get_thread_num(); - const std::vector &data = vectorIn[ithread]; - std::copy(data.begin(), data.end(), &vectorOut[offsets[ithread]]); - } + { + const unsigned ithread = omp_get_thread_num(); + const std::vector &data = vectorIn[ithread]; + std::copy(data.begin(), data.end(), &vectorOut[offsets[ithread]]); + } #else - for (unsigned ithread = 0; ithread < numThreadLists; ++ithread) { - const std::vector &data = vectorIn[ithread]; - std::copy(data.begin(), data.end(), &vectorOut[offsets[ithread]]); - } + for (unsigned ithread = 0; ithread < numThreadLists; ++ithread) { + const std::vector &data = vectorIn[ithread]; + std::copy(data.begin(), data.end(), &vectorOut[offsets[ithread]]); + } #endif - } +} template -void communicateVector( - stk::ParallelMachine arg_comm , - std::vector< std::pair< DomainKey, RangeKey> > & search_relations , - bool communicateRangeBoxInfo = true ) +void communicate_vector(stk::ParallelMachine arg_comm, + std::vector> & search_relations, + bool enforceSearchResultSymmetry = true) { - typedef std::pair ValueType ; + typedef std::pair ValueType; - CommSparse commSparse( arg_comm ); + CommSparse commSparse(arg_comm); const int p_rank = commSparse.parallel_rank(); const int p_size = commSparse.parallel_size(); @@ -296,50 +164,48 @@ void communicateVector( return; } - typename std::vector< ValueType >::const_iterator i ; + typename std::vector< ValueType >::const_iterator i; size_t numLocal = 0; - for ( i = search_relations.begin() ; i != search_relations.end() ; ++i ) { - const ValueType & val = *i ; - if ( static_cast(val.first.proc()) == p_rank || ( communicateRangeBoxInfo && static_cast(val.second.proc()) == p_rank) ) - { + for (i = search_relations.begin(); i != search_relations.end(); ++i) { + const ValueType & val = *i; + if (static_cast(val.first.proc()) == p_rank || + (enforceSearchResultSymmetry && static_cast(val.second.proc()) == p_rank)) { ++numLocal; - } - if ( static_cast(val.first.proc()) != p_rank ) { - CommBuffer & buf = commSparse.send_buffer( val.first.proc() ); - buf.skip( 1 ); - } - if ( communicateRangeBoxInfo ) - { - if ( static_cast(val.second.proc()) != p_rank && val.second.proc() != val.first.proc() ) { - CommBuffer & buf = commSparse.send_buffer( val.second.proc() ); - buf.skip( 1 ); - } - } + } + if (static_cast(val.first.proc()) != p_rank) { + CommBuffer & buf = commSparse.send_buffer(val.first.proc()); + buf.skip(1); + } + if (enforceSearchResultSymmetry) { + if (static_cast(val.second.proc()) != p_rank && val.second.proc() != val.first.proc()) { + CommBuffer & buf = commSparse.send_buffer(val.second.proc()); + buf.skip(1); + } + } } commSparse.allocate_buffers(); - for ( i = search_relations.begin() ; i != search_relations.end() ; ++i ) { - const ValueType & val = *i ; - if ( static_cast(val.first.proc()) != p_rank ) { - CommBuffer & buf = commSparse.send_buffer( val.first.proc() ); - buf.pack( val ); - } - if ( communicateRangeBoxInfo ) - { - if ( static_cast(val.second.proc()) != p_rank && val.second.proc() != val.first.proc() ) { - CommBuffer & buf = commSparse.send_buffer( val.second.proc() ); - buf.pack( val ); - } - } + for (i = search_relations.begin(); i != search_relations.end(); ++i) { + const ValueType & val = *i; + if (static_cast(val.first.proc()) != p_rank) { + CommBuffer & buf = commSparse.send_buffer(val.first.proc()); + buf.pack(val); + } + if (enforceSearchResultSymmetry) { + if (static_cast(val.second.proc()) != p_rank && val.second.proc() != val.first.proc()) { + CommBuffer & buf = commSparse.send_buffer(val.second.proc()); + buf.pack(val); + } + } } commSparse.communicate(); size_t numRecvd = 0; - for ( int p = 0 ; p < p_size ; ++p ) { - CommBuffer & buf = commSparse.recv_buffer( p ); + for (int p = 0; p < p_size; ++p) { + CommBuffer & buf = commSparse.recv_buffer(p); numRecvd += (buf.remaining()/sizeof(ValueType)); } search_relations.reserve(numLocal+numRecvd); @@ -347,26 +213,96 @@ void communicateVector( size_t keep = 0; for (size_t j=0; j(val.first.proc()) == p_rank || ( communicateRangeBoxInfo && static_cast(val.second.proc()) == p_rank) ) - { + if (static_cast(val.first.proc()) == p_rank || + (enforceSearchResultSymmetry && static_cast(val.second.proc()) == p_rank)) { if (j > keep) { search_relations[keep] = val; } ++keep; - } + } } - for ( int p = 0 ; p < p_size ; ++p ) { - CommBuffer & buf = commSparse.recv_buffer( p ); - while ( buf.remaining() ) { - ValueType val ; - buf.unpack( val ); + for (int p = 0; p < p_size; ++p) { + CommBuffer & buf = commSparse.recv_buffer(p); + while (buf.remaining()) { + ValueType val; + buf.unpack(val); search_relations.push_back(val); - } + } + } +} + +template +void communicate_views(stk::ParallelMachine arg_comm, + SearchRelationType& search_relations, + bool enforceSearchResultSymmetry = true) +{ + using ValueType = typename SearchRelationType::value_type; + CommSparse commSparse(arg_comm); + + const int p_rank = commSparse.parallel_rank(); + const int p_size = commSparse.parallel_size(); + + if (1 == p_size) { + return; + } + + size_t numLocal = 0; + for (unsigned i = 0; i != search_relations.extent(0); ++i) { + const ValueType & val = search_relations(i); + if (static_cast(val.domainIdent.proc()) == p_rank || + (enforceSearchResultSymmetry && static_cast(val.rangeIdent.proc()) == p_rank)) { + ++numLocal; + } + if (static_cast(val.domainIdent.proc()) != p_rank) { + CommBuffer & buf = commSparse.send_buffer(val.domainIdent.proc()); + buf.skip(1); + } + if (enforceSearchResultSymmetry) { + if (static_cast(val.rangeIdent.proc()) != p_rank && val.rangeIdent.proc() != val.domainIdent.proc()) { + CommBuffer & buf = commSparse.send_buffer(val.rangeIdent.proc()); + buf.skip(1); + } + } + } + + commSparse.allocate_buffers(); + + for (unsigned i = 0; i != search_relations.extent(0); ++i) { + const ValueType & val = search_relations(i); + if (static_cast(val.domainIdent.proc()) != p_rank) { + CommBuffer & buf = commSparse.send_buffer(val.domainIdent.proc()); + buf.pack(val); + } + if (enforceSearchResultSymmetry) { + if (static_cast(val.rangeIdent.proc()) != p_rank && val.rangeIdent.proc() != val.domainIdent.proc()) { + CommBuffer & buf = commSparse.send_buffer(val.rangeIdent.proc()); + buf.pack(val); + } + } + } + + commSparse.communicate(); + + size_t numRecvd = 0; + for (int p = 0; p < p_size; ++p) { + CommBuffer & buf = commSparse.recv_buffer(p); + numRecvd += (buf.remaining()/sizeof(ValueType)); + } + + auto keep = search_relations.extent(0); + Kokkos::resize(Kokkos::WithoutInitializing, search_relations, numLocal+numRecvd); + + for (int p = 0; p < p_size; ++p) { + CommBuffer & buf = commSparse.recv_buffer(p); + while (buf.remaining()) { + ValueType val; + buf.unpack(val); + search_relations(keep++) = val; + } } } - } // end namespace search -} // end namespace stk +} // end namespace stk::search #endif diff --git a/packages/stk/stk_search/stk_search/CommonSearchUtilsInstrumented.hpp b/packages/stk/stk_search/stk_search/CommonSearchUtilsInstrumented.hpp index 7d186382bf86..ccc8cb2eb95d 100644 --- a/packages/stk/stk_search/stk_search/CommonSearchUtilsInstrumented.hpp +++ b/packages/stk/stk_search/stk_search/CommonSearchUtilsInstrumented.hpp @@ -40,8 +40,8 @@ #include #include "stk_util/environment/WallTime.hpp" -#include "stk_search/KDTree_BoundingBox.hpp" -#include "stk_search/KDTree.hpp" +#include "stk_search/kdtree/KDTree_BoundingBox.hpp" +#include "stk_search/kdtree/KDTree.hpp" #include "stk_search/CommonSearchUtil.hpp" namespace stk { @@ -222,7 +222,7 @@ namespace stk { // std::vector boxGather; - stk::search::AllGatherHelper(boxA_proc.GetBox(), boxGather, comm); + stk::search::all_gather_helper(boxA_proc.GetBox(), boxGather, comm); diff --git a/packages/stk/stk_search/stk_search/LocalCoarseSearch.hpp b/packages/stk/stk_search/stk_search/LocalCoarseSearch.hpp new file mode 100644 index 000000000000..a3b0e8b29304 --- /dev/null +++ b/packages/stk/stk_search/stk_search/LocalCoarseSearch.hpp @@ -0,0 +1,116 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef LOCALCOARSESEARCH_HPP +#define LOCALCOARSESEARCH_HPP + +#include "stk_util/stk_config.h" +#include "stk_search/SearchMethod.hpp" +#include "stk_search/BoxIdent.hpp" +#include "stk_search/morton_lbvh/LocalCoarseSearchMortonLBVH.hpp" +#ifdef STK_HAS_ARBORX +#include "stk_search/arborx/LocalCoarseSearchArborX.hpp" +#endif +#include "stk_util/util/ReportHandler.hpp" +#include "Kokkos_Core.hpp" + +namespace stk::search { + +template +void local_coarse_search( + std::vector> const & domain, + std::vector> const & range, + SearchMethod method, + std::vector> & intersections) +{ + switch (method) { + case ARBORX: { +#ifdef STK_HAS_ARBORX + local_coarse_search_arborx(domain, range, intersections); +#else + STK_ThrowErrorMsg("STK(stk_search) was not configured with ARBORX. Please use KDTREE or MORTON_LBVH."); +#endif + break; + } + case KDTREE: { + STK_ThrowErrorMsg("The KDTREE search method is currently not supported for local_coarse_search. Please use MORTON_LBVH or ARBORX instead."); + break; + } + case MORTON_LBVH: { + local_coarse_search_morton_lbvh(domain, range, intersections); + break; + } + default: { + STK_ThrowErrorMsg("Unsupported local_coarse_search method supplied. Choices are: KDTREE, MORTON_LVBH, or ARBORX."); + } + } + +} + +template +void local_coarse_search( + Kokkos::View*, ExecutionSpace> const & domain, + Kokkos::View*, ExecutionSpace> const & range, + SearchMethod method, + Kokkos::View*, ExecutionSpace> & intersections, + ExecutionSpace const& execSpace = Kokkos::DefaultExecutionSpace{}) +{ + switch (method) { + case ARBORX: { +#ifdef STK_HAS_ARBORX + local_coarse_search_arborx(domain, range, intersections); +#else + STK_ThrowErrorMsg("STK(stk_search) was not configured with ARBORX. Please use KDTREE or MORTON_LBVH."); +#endif + break; + } + case KDTREE: { + STK_ThrowErrorMsg("The KDTREE search method is not supported on GPUs. Please use MORTON_LBVH or ARBORX instead."); + break; + } + case MORTON_LBVH: { + local_coarse_search_morton_lbvh(domain, range, intersections); + break; + } + default: { + STK_ThrowErrorMsg("Unsupported local_coarse_search method supplied. Choices are: KDTREE, MORTON_LVBH, or ARBORX."); + } + } + +} + +} + +#endif // LOCALCOARSESEARCH_HPP diff --git a/packages/stk/stk_search/stk_search/Point.hpp b/packages/stk/stk_search/stk_search/Point.hpp index e38ddd91499d..c2e50930c751 100644 --- a/packages/stk/stk_search/stk_search/Point.hpp +++ b/packages/stk/stk_search/stk_search/Point.hpp @@ -35,9 +35,9 @@ #ifndef STK_SEARCH_POINT_HPP #define STK_SEARCH_POINT_HPP -#include -#include #include +#include +#include namespace stk { namespace search { @@ -88,8 +88,8 @@ class Point KOKKOS_FUNCTION value_type get_y_max() const { return m_value[1]; } KOKKOS_FUNCTION value_type get_z_max() const { return m_value[2]; } - - KOKKOS_DEFAULTED_FUNCTION ~Point() = default; + KOKKOS_INLINE_FUNCTION const value_type* data() const { return m_value; } + KOKKOS_INLINE_FUNCTION value_type* data() { return m_value; } private: value_type m_value[Dim]; diff --git a/packages/stk/stk_search/stk_search/PrototypeSearchUtilsInstrumented.hpp b/packages/stk/stk_search/stk_search/PrototypeSearchUtilsInstrumented.hpp index 763f80304dd2..ffe70cff0209 100644 --- a/packages/stk/stk_search/stk_search/PrototypeSearchUtilsInstrumented.hpp +++ b/packages/stk/stk_search/stk_search/PrototypeSearchUtilsInstrumented.hpp @@ -52,8 +52,8 @@ #include "stk_search/BoundingBox.hpp" // for add_to_box, etc #include "stk_search/Box.hpp" // for Box #include "stk_search/CommonSearchUtil.hpp" -#include "stk_search/KDTree.hpp" -#include "stk_search/KDTree_BoundingBox.hpp" +#include "stk_search/kdtree/KDTree.hpp" +#include "stk_search/kdtree/KDTree_BoundingBox.hpp" #include "stk_util/parallel/CommSparse.hpp" // for CommSparse #include "stk_util/parallel/ParallelComm.hpp" // for CommBuffer, etc namespace stk { class CommBufferV; } @@ -901,11 +901,11 @@ void GhostingSearcher +#include "stk_util/util/ReportHandler.hpp" + +namespace stk::search { enum SearchMethod { - KDTREE, - MORTON_LINEARIZED_BVH, // Coming soon! + ARBORX, + KDTREE, + MORTON_LBVH }; -}} +inline +std::ostream& operator<<(std::ostream &out, SearchMethod method) +{ + switch (method) { + case ARBORX: out << "ARBORX"; break; + case KDTREE: out << "KDTREE"; break; + case MORTON_LBVH: out << "MORTON_LBVH"; break; + default: { + STK_ThrowErrorMsg("Unsupported coarse_search method supplied. Choices are: KDTREE, MORTON_LVBH, or ARBORX."); + } + } + return out; +} + +} #endif diff --git a/packages/stk/stk_search/stk_search/Sphere.hpp b/packages/stk/stk_search/stk_search/Sphere.hpp index c81a682d27c6..9ce51309cafd 100644 --- a/packages/stk/stk_search/stk_search/Sphere.hpp +++ b/packages/stk/stk_search/stk_search/Sphere.hpp @@ -80,6 +80,18 @@ class Sphere KOKKOS_FORCEINLINE_FUNCTION value_type get_y_max() const { return m_center[1] + m_radius; } KOKKOS_FORCEINLINE_FUNCTION value_type get_z_max() const { return m_center[2] + m_radius; } + KOKKOS_FORCEINLINE_FUNCTION + float get_expanded_radius() const { + return Kokkos::nextafter(static_cast(m_radius), Kokkos::Experimental::finite_max_v); + } + + KOKKOS_FORCEINLINE_FUNCTION float get_expanded_x_min() const { return static_cast(m_center[0]) - this->get_expanded_radius(); } + KOKKOS_FORCEINLINE_FUNCTION float get_expanded_y_min() const { return static_cast(m_center[1]) - this->get_expanded_radius(); } + KOKKOS_FORCEINLINE_FUNCTION float get_expanded_z_min() const { return static_cast(m_center[2]) - this->get_expanded_radius(); } + KOKKOS_FORCEINLINE_FUNCTION float get_expanded_x_max() const { return static_cast(m_center[0]) + this->get_expanded_radius(); } + KOKKOS_FORCEINLINE_FUNCTION float get_expanded_y_max() const { return static_cast(m_center[1]) + this->get_expanded_radius(); } + KOKKOS_FORCEINLINE_FUNCTION float get_expanded_z_max() const { return static_cast(m_center[2]) + this->get_expanded_radius(); } + friend std::ostream& operator<<(std::ostream & out, Sphere const& s) { out << "{" << s.center() << ":" << s.radius() << "}"; diff --git a/packages/stk/stk_search/stk_search/arborx/CoarseSearchArborX.hpp b/packages/stk/stk_search/stk_search/arborx/CoarseSearchArborX.hpp new file mode 100644 index 000000000000..b8539a73a995 --- /dev/null +++ b/packages/stk/stk_search/stk_search/arborx/CoarseSearchArborX.hpp @@ -0,0 +1,308 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef COARSE_SEARCH_ARBORX_HPP +#define COARSE_SEARCH_ARBORX_HPP + +#include +#include + +#include "ArborX.hpp" +#include "Kokkos_StdAlgorithms.hpp" +#include "stk_search/CommonSearchUtil.hpp" +#include "stk_search/arborx/StkToArborX.hpp" +#include "stk_util/util/ReportHandler.hpp" +#include "stk_util/util/SortAndUnique.hpp" + +namespace stk::search +{ +namespace impl +{ +template +void fill_search_result(DomainBoxPairType const& domainBox, + RangeBoxPairType const& rangeBox, + SearchResultsListType& searchResults, + [[maybe_unused]] int& searchResultIdx) +{ + constexpr bool usesSpheres = + is_stk_sphere || is_stk_sphere; + if constexpr (!Kokkos::is_view_v) { + if constexpr (usesSpheres) { + if (intersects(domainBox.first, rangeBox.first)) searchResults.emplace_back(domainBox.second, rangeBox.second); + } else { + searchResults.emplace_back(domainBox.second, rangeBox.second); + } + } else { + if constexpr (usesSpheres) { + if (intersects(domainBox.box, rangeBox.box)) + searchResults[searchResultIdx++] = {domainBox.identProc, rangeBox.identProc}; + } else { + searchResults[searchResultIdx++] = {domainBox.identProc, rangeBox.identProc}; + } + } +} + +template +void fill_search_results(LocalDomainListType const& localDomain, + LocalRangeListType const& localRange, + ValuesHostViewType valuesHost, + OffsetsHostViewType offsetsHost, + MPI_Comm comm, + SearchResultsListType& searchResults) +{ + int searchResultIdx = 0; + int myRank = 0; + int commSize = 0; + MPI_Comm_rank(comm, &myRank); + MPI_Comm_size(comm, &commSize); + + if (stk::parallel_machine_size(comm) > 1) { + stk::CommSparse commSparse(comm); + stk::CommSparse commSparse2(comm); + + for (std::size_t i = 0; i < offsetsHost.extent(0) - 1; ++i) { + int rangeIndexBegin = offsetsHost(i); + int rangeIndexEnd = offsetsHost(i + 1); + + for (auto j = rangeIndexBegin; j < rangeIndexEnd; ++j) { + auto domainIdx = i; + auto rangeIdx = valuesHost(j).index; + auto rangeProc = valuesHost(j).rank; + if (rangeProc == myRank) { + fill_search_result(localDomain[domainIdx], localRange[rangeIdx], searchResults, searchResultIdx); + } + } + } + + stk::pack_and_communicate(commSparse, [&]() { + for (std::size_t i = 0; i < offsetsHost.extent(0) - 1; ++i) { + int rangeIndexBegin = offsetsHost(i); + int rangeIndexEnd = offsetsHost(i + 1); + + for (auto j = rangeIndexBegin; j < rangeIndexEnd; ++j) { + auto domainIdx = i; + auto rangeIdx = valuesHost(j).index; + auto rangeProc = valuesHost(j).rank; + + if (rangeProc == myRank) { + continue; + } else { + commSparse.send_buffer(rangeProc).pack((int) domainIdx); + commSparse.send_buffer(rangeProc).pack(rangeIdx); + } + } + } + }); + + // domainIdx, domainProc, rangeIdx + std::vector> pairingInfo; + + for (int proc = 0; proc < commSize; ++proc) { + if (proc == myRank) { + continue; + } + while (commSparse.recv_buffer(proc).remaining()) { + int domainIdx; + int rangeIdx; + commSparse.recv_buffer(proc).unpack(domainIdx); + commSparse.recv_buffer(proc).unpack(rangeIdx); + pairingInfo.emplace_back(domainIdx, proc, rangeIdx); + } + } + + stk::pack_and_communicate(commSparse2, [&]() { + for (auto info : pairingInfo) { + commSparse2.send_buffer(std::get<1>(info)) + .pack(std::make_pair(std::get<0>(info), localRange[std::get<2>(info)])); + } + }); + + for (int proc = 0; proc < commSize; ++proc) { + if (proc == myRank) { + continue; + } + while (commSparse2.recv_buffer(proc).remaining()) { + std::pair recv; + commSparse2.recv_buffer(proc).unpack(recv); + + auto domainIdx = recv.first; + auto recvRangeBox = recv.second; + + fill_search_result(localDomain[domainIdx], recvRangeBox, searchResults, searchResultIdx); + } + } + } else { + for (std::size_t i = 0; i < offsetsHost.extent(0) - 1; ++i) { + int rangeIndexBegin = offsetsHost(i); + int rangeIndexEnd = offsetsHost(i + 1); + + for (auto j = rangeIndexBegin; j < rangeIndexEnd; ++j) { + auto domainIdx = i; + auto rangeIdx = valuesHost(j).index; + + fill_search_result(localDomain[domainIdx], localRange[rangeIdx], searchResults, searchResultIdx); + } + } + } +} +} // namespace impl + +template +inline void coarse_search_arborx(std::vector> const& localDomain, + std::vector> const& localRange, + MPI_Comm comm, + std::vector>& searchResults, + bool enforceSearchResultSymmetry = true) +{ + using ExecSpace = Kokkos::DefaultExecutionSpace; + using MemSpace = typename ExecSpace::memory_space; + using DomainValueType = typename DomainBoxType::value_type; + using RangeValueType = typename RangeBoxType::value_type; + using ArborXDomainType = typename impl::StkToArborX::ArborXType; + using ArborXRangeType = typename impl::StkToArborX::ArborXType; + + STK_ThrowRequireMsg((std::is_same_v), + "The domain and range boxes must have the same floating-point precision"); + + Kokkos::View domainBoundingBoxes( + Kokkos::view_alloc(Kokkos::WithoutInitializing, "ArborX_Domain_BBs"), localDomain.size()); + Kokkos::View rangeBoundingBoxes( + Kokkos::view_alloc(Kokkos::WithoutInitializing, "ArborX_Range_BBs"), localRange.size()); + Kokkos::View values( + Kokkos::view_alloc(Kokkos::WithoutInitializing, "Indices_And_Ranks"), 0); + Kokkos::View*, MemSpace> queries( + Kokkos::view_alloc(Kokkos::WithoutInitializing, "Queries"), localDomain.size()); + Kokkos::View offsets(Kokkos::view_alloc(Kokkos::WithoutInitializing, "Offsets"), 0); + + impl::convert_and_init_bounding_boxes(localDomain, domainBoundingBoxes); + impl::convert_and_init_bounding_boxes(localRange, rangeBoundingBoxes); + + ArborX::DistributedTree tree(comm, ExecSpace{}, rangeBoundingBoxes); + + Kokkos::parallel_for( + "setup_queries", Kokkos::RangePolicy(0, localDomain.size()), + KOKKOS_LAMBDA(int i) { queries(i) = ArborX::intersects(domainBoundingBoxes(i)); }); + tree.query(ExecSpace{}, queries, values, offsets); + + Kokkos::fence(); + + auto valuesHost = Kokkos::create_mirror_view_and_copy(ExecSpace{}, values); + auto offsetsHost = Kokkos::create_mirror_view_and_copy(ExecSpace{}, offsets); + + const int numCollisions = values.extent(0); + searchResults.clear(); + searchResults.reserve(numCollisions); + + impl::fill_search_results(localDomain, localRange, valuesHost, offsetsHost, comm, searchResults); + + if (enforceSearchResultSymmetry) { + communicate_vector(comm, searchResults, enforceSearchResultSymmetry); + std::sort(searchResults.begin(), searchResults.end()); + } +} + +template +inline void coarse_search_arborx( + Kokkos::View*, ExecutionSpace> const& localDomain, + Kokkos::View*, ExecutionSpace> const& localRange, + MPI_Comm comm, + Kokkos::View*, ExecutionSpace>& searchResults, + bool enforceSearchResultSymmetry = true) +{ + using ExecSpace = Kokkos::DefaultExecutionSpace; + using MemSpace = typename ExecSpace::memory_space; + using DomainValueType = typename DomainBoxType::value_type; + using RangeValueType = typename RangeBoxType::value_type; + using ArborXDomainType = typename impl::StkToArborX::ArborXType; + using ArborXRangeType = typename impl::StkToArborX::ArborXType; + + STK_ThrowRequireMsg((std::is_same_v), + "The domain and range boxes must have the same floating-point precision"); + + auto execSpace = ExecSpace{}; + + Kokkos::View domainBoundingBoxes( + Kokkos::view_alloc(Kokkos::WithoutInitializing, "ArborX_Domain_BBs"), localDomain.extent(0)); + Kokkos::View rangeBoundingBoxes( + Kokkos::view_alloc(Kokkos::WithoutInitializing, "ArborX_Range_BBs"), localRange.extent(0)); + Kokkos::View values( + Kokkos::view_alloc(Kokkos::WithoutInitializing, "Indices_And_Ranks"), 0); + Kokkos::View*, MemSpace> queries( + Kokkos::view_alloc(Kokkos::WithoutInitializing, "Queries"), localDomain.extent(0)); + Kokkos::View offsets(Kokkos::view_alloc(Kokkos::WithoutInitializing, "Offsets"), 0); + + impl::convert_and_init_bounding_boxes(localDomain, domainBoundingBoxes); + impl::convert_and_init_bounding_boxes(localRange, rangeBoundingBoxes); + + ArborX::DistributedTree tree(comm, execSpace, rangeBoundingBoxes); + + Kokkos::parallel_for( + "setup_queries", Kokkos::RangePolicy(0, localDomain.extent(0)), + KOKKOS_LAMBDA(int i) { queries(i) = ArborX::intersects(domainBoundingBoxes(i)); }); + tree.query(execSpace, queries, values, offsets); + + Kokkos::fence(); + + auto valuesHost = Kokkos::create_mirror_view_and_copy(execSpace, values); + auto offsetsHost = Kokkos::create_mirror_view_and_copy(execSpace, offsets); + + const int numCollisions = values.extent(0); + searchResults = Kokkos::View*, ExecSpace>( + Kokkos::ViewAllocateWithoutInitializing(searchResults.label()), numCollisions); + + auto localDomainHost = Kokkos::create_mirror_view_and_copy(execSpace, localDomain); + auto localRangeHost = Kokkos::create_mirror_view_and_copy(execSpace, localRange); + auto searchResultsHost = Kokkos::create_mirror_view_and_copy(execSpace, searchResults); + + impl::fill_search_results(localDomainHost, localRangeHost, valuesHost, offsetsHost, comm, searchResultsHost); + Kokkos::sort(searchResultsHost); + + if (enforceSearchResultSymmetry) { + communicate_views(comm, searchResultsHost, enforceSearchResultSymmetry); + Kokkos::resize(Kokkos::WithoutInitializing, searchResults, searchResultsHost.extent(0)); + } + + Kokkos::deep_copy(searchResults, searchResultsHost); +} + +} // namespace stk::search + +#endif diff --git a/packages/stk/stk_search/stk_search/arborx/LocalCoarseSearchArborX.hpp b/packages/stk/stk_search/stk_search/arborx/LocalCoarseSearchArborX.hpp new file mode 100644 index 000000000000..a2372b464578 --- /dev/null +++ b/packages/stk/stk_search/stk_search/arborx/LocalCoarseSearchArborX.hpp @@ -0,0 +1,225 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef LOCAL_COARSE_SEARCH_ARBORX_HPP +#define LOCAL_COARSE_SEARCH_ARBORX_HPP + +#include +#include +#include +#include + +#include "ArborX.hpp" +#include "Kokkos_Core.hpp" +#include "stk_search/BoxIdent.hpp" +#include "stk_search/CommonSearchUtil.hpp" +#include "stk_search/arborx/StkToArborX.hpp" +#include "stk_util/util/ReportHandler.hpp" +#include "stk_util/util/SortAndUnique.hpp" + +namespace stk::search +{ +namespace impl +{ + +template +void fill_local_search_results(const std::vector>& localDomain, + const std::vector>& localRange, + ValuesViewType values, + OffsetsViewType offsets, + std::vector>& searchResults) +{ + bool constexpr isSphere = is_stk_sphere || is_stk_sphere; + searchResults.clear(); + + for (std::size_t i = 0; i < offsets.extent(0) - 1; ++i) { + int rangeIndexBegin = offsets(i); + int rangeIndexEnd = offsets(i + 1); + + for (auto j = rangeIndexBegin; j < rangeIndexEnd; ++j) { + auto domainIdx = i; + auto rangeIdx = values(j); + + if (!(isSphere) || intersects(localDomain[domainIdx].first, localRange[rangeIdx].first)) { + searchResults.emplace_back(localDomain[domainIdx].second, localRange[rangeIdx].second); + } + } + } +} + +template +void fill_local_search_results(const Kokkos::View*, ExecutionSpace> & localDomain, + const Kokkos::View*, ExecutionSpace> & localRange, + ValuesViewType values, + OffsetsViewType offsets, + Kokkos::View*, ExecutionSpace> & searchResults) +{ + + bool constexpr isSphere = is_stk_sphere || is_stk_sphere; + + int numActualSearchResults = 0; + Kokkos::parallel_reduce(Kokkos::RangePolicy(0, 1), KOKKOS_LAMBDA(int /*index*/, int & searchResultSum) { + for (std::size_t i = 0; i < offsets.extent(0) - 1; ++i) { + int rangeIndexBegin = offsets(i); + int rangeIndexEnd = offsets(i + 1); + + for (auto j = rangeIndexBegin; j < rangeIndexEnd; ++j) { + auto domainIdx = i; + auto rangeIdx = values(j); + + if (!(isSphere) || intersects(localDomain[domainIdx].box, localRange[rangeIdx].box)) { + searchResults(searchResultSum++) = {localDomain[domainIdx].ident, localRange[rangeIdx].ident}; + } + } + } + }, + numActualSearchResults); + + Kokkos::resize(searchResults, numActualSearchResults); + +} + +} // namespace impl + +template +inline void local_coarse_search_arborx(const std::vector>& localDomain, + const std::vector>& localRange, + std::vector>& searchResults) +{ + using ExecSpace = Kokkos::DefaultHostExecutionSpace; + using MemSpace = typename ExecSpace::memory_space; + using DomainValueType = typename DomainBoxType::value_type; + using RangeValueType = typename RangeBoxType::value_type; + using ArborXDomainType = typename impl::StkToArborX::ArborXType; + using ArborXRangeType = typename impl::StkToArborX::ArborXType; + + STK_ThrowRequireMsg((std::is_same_v), + "The domain and range boxes must have the same floating-point precision"); + + auto execSpace = ExecSpace{}; + Kokkos::View domainBoundingBoxes( + Kokkos::view_alloc(Kokkos::WithoutInitializing, "ArborX_Domain_BBs"), localDomain.size()); + Kokkos::View rangeBoundingBoxes( + Kokkos::view_alloc(Kokkos::WithoutInitializing, "ArborX_Range_BBs"), localRange.size()); + + Kokkos::View values(Kokkos::view_alloc(Kokkos::WithoutInitializing, "Values"), 0); + Kokkos::View*, MemSpace> queries( + Kokkos::view_alloc(Kokkos::WithoutInitializing, "Queries"), localDomain.size()); + Kokkos::View offsets(Kokkos::view_alloc(Kokkos::WithoutInitializing, "Offsets"), 0); + + impl::convert_and_init_bounding_boxes(localDomain, domainBoundingBoxes); + impl::convert_and_init_bounding_boxes(localRange, rangeBoundingBoxes); + + ArborX::BVH bvh(execSpace, rangeBoundingBoxes); + + Kokkos::parallel_for( + "setup_queries", Kokkos::RangePolicy(0, localDomain.size()), + KOKKOS_LAMBDA(int i) { queries(i) = ArborX::intersects(domainBoundingBoxes(i)); }); + bvh.query(execSpace, queries, values, offsets); + + execSpace.fence(); + + auto valuesHost = Kokkos::create_mirror_view_and_copy(execSpace, values); + auto offsetsHost = Kokkos::create_mirror_view_and_copy(execSpace, offsets); + + const int numCollisions = values.extent(0); + searchResults.reserve(numCollisions); + + impl::fill_local_search_results(localDomain, localRange, valuesHost, offsetsHost, searchResults); +} + +template +inline void local_coarse_search_arborx( + const Kokkos::View*, ExecutionSpace>& localDomain, + const Kokkos::View*, ExecutionSpace>& localRange, + Kokkos::View*, ExecutionSpace>& searchResults) +{ + using ExecSpace = ExecutionSpace; + using MemSpace = typename ExecSpace::memory_space; + using DomainValueType = typename DomainBoxType::value_type; + using RangeValueType = typename RangeBoxType::value_type; + using ArborXDomainType = typename impl::StkToArborX::ArborXType; + using ArborXRangeType = typename impl::StkToArborX::ArborXType; + + STK_ThrowRequireMsg((std::is_same_v), + "The domain and range boxes must have the same floating-point precision"); + + auto execSpace = ExecSpace{}; + Kokkos::View domainBoundingBoxes( + Kokkos::view_alloc(Kokkos::WithoutInitializing, "ArborX_Domain_BBs"), localDomain.extent(0)); + Kokkos::View rangeBoundingBoxes( + Kokkos::view_alloc(Kokkos::WithoutInitializing, "ArborX_Range_BBs"), localRange.extent(0)); + + Kokkos::View values(Kokkos::view_alloc(Kokkos::WithoutInitializing, "Values"), 0); + Kokkos::View*, MemSpace> queries( + Kokkos::view_alloc(Kokkos::WithoutInitializing, "Queries"), localDomain.extent(0)); + Kokkos::View offsets(Kokkos::view_alloc(Kokkos::WithoutInitializing, "Offsets"), 0); + + impl::convert_and_init_bounding_boxes(localDomain, domainBoundingBoxes); + impl::convert_and_init_bounding_boxes(localRange, rangeBoundingBoxes); + + ArborX::BVH bvh(execSpace, rangeBoundingBoxes); + + Kokkos::parallel_for( + "setup_queries", Kokkos::RangePolicy(0, localDomain.extent(0)), + KOKKOS_LAMBDA(int i) { queries(i) = ArborX::intersects(domainBoundingBoxes(i)); }); + bvh.query(execSpace, queries, values, offsets); + + Kokkos::fence(); + + const int numCollisions = values.extent(0); + searchResults = Kokkos::View*, ExecutionSpace>( + Kokkos::ViewAllocateWithoutInitializing(searchResults.label()), numCollisions); + + impl::fill_local_search_results(localDomain, localRange, values, offsets, searchResults); + +} + +} // namespace stk::search + +#endif diff --git a/packages/stk/stk_search/stk_search/arborx/StkToArborX.hpp b/packages/stk/stk_search/stk_search/arborx/StkToArborX.hpp new file mode 100644 index 000000000000..7acd0857ccb9 --- /dev/null +++ b/packages/stk/stk_search/stk_search/arborx/StkToArborX.hpp @@ -0,0 +1,186 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef STK_TO_ARBORX_HPP +#define STK_TO_ARBORX_HPP + +#include +#include +#include +#include +#include + +#include "ArborX.hpp" +#include "stk_search/BoxIdent.hpp" +#include "stk_search/CommonSearchUtil.hpp" +#include "stk_util/util/ReportHandler.hpp" +#include "stk_util/util/SortAndUnique.hpp" +#include "Kokkos_Core.hpp" +#include "Kokkos_StdAlgorithms.hpp" + +namespace stk::search +{ +namespace impl +{ +template +bool constexpr is_stk_box = + std::is_same_v> || std::is_base_of_v, T>; + +template +bool constexpr is_stk_sphere = + std::is_same_v> || std::is_base_of_v, T>; + +template +bool constexpr is_stk_point = + std::is_same_v> || std::is_base_of_v, T>; + +template +struct StkToArborX { +}; + +template +struct StkToArborX>> { + using ValueType = typename T::value_type; + using ArborXType = ArborX::Point; + using StkType = Point; + + KOKKOS_INLINE_FUNCTION + StkToArborX() = delete; + + KOKKOS_INLINE_FUNCTION + StkToArborX(StkType const& src) : data(convert(src)) {} + + KOKKOS_INLINE_FUNCTION + operator ArborXType() { return data; } + + KOKKOS_INLINE_FUNCTION + auto convert(StkType const& src) + { + // ArborX only accepts single precision floating points + return ArborXType(static_cast(src[0]), static_cast(src[1]), static_cast(src[2])); + } + + ArborXType data; +}; + +template +struct StkToArborX>> { + using ValueType = typename T::value_type; + using ArborXType = ArborX::Box; + using StkType = Box; + using PointType = StkToArborX>; + + KOKKOS_INLINE_FUNCTION + StkToArborX() = delete; + + KOKKOS_INLINE_FUNCTION + StkToArborX(StkType const& src) : data(convert(src)) {} + + KOKKOS_INLINE_FUNCTION + operator ArborXType() { return data; } + + KOKKOS_INLINE_FUNCTION + auto convert(StkType const& src) + { + if (std::is_same_v) { + return ArborXType(ArborX::Point(src.get_x_min(), src.get_y_min(), src.get_z_min()), + ArborX::Point(src.get_x_max(), src.get_y_max(), src.get_z_max())); + } + else { + return ArborXType(ArborX::Point(src.get_expanded_x_min(), src.get_expanded_y_min(), src.get_expanded_z_min()), + ArborX::Point(src.get_expanded_x_max(), src.get_expanded_y_max(), src.get_expanded_z_max())); + } + } + + ArborXType data; +}; + +template +struct StkToArborX>> { + using ValueType = typename T::value_type; + using ArborXType = ArborX::Box; + using StkType = Sphere; + using PointType = StkToArborX>; + + KOKKOS_INLINE_FUNCTION + StkToArborX() = delete; + + KOKKOS_INLINE_FUNCTION + StkToArborX(StkType const& src) : data(convert(src)) {} + + KOKKOS_INLINE_FUNCTION + operator ArborXType() { return data; } + + // ArborX requires primitives to decay to either a Point or a Box + KOKKOS_INLINE_FUNCTION + auto convert(StkType const& src) + { + + if (std::is_same_v) { + return ArborXType(ArborX::Point(src.get_x_min(), src.get_y_min(), src.get_z_min()), + ArborX::Point(src.get_x_max(), src.get_y_max(), src.get_z_max())); + } + else { + return ArborXType(ArborX::Point(src.get_expanded_x_min(), src.get_expanded_y_min(), src.get_expanded_z_min()), + ArborX::Point(src.get_expanded_x_max(), src.get_expanded_y_max(), src.get_expanded_z_max())); + } + } + + ArborXType data; +}; + +template +void convert_and_init_bounding_boxes(StkSourceType const& src, ArborXViewType const& view) +{ + if constexpr (!Kokkos::is_view_v) { + auto viewHost = Kokkos::create_mirror_view(view); + + for (std::size_t i = 0; i < viewHost.extent(0); ++i) { + auto init = src[i].first; + viewHost(i) = StkToArborX(init); + } + Kokkos::deep_copy(view, viewHost); + } else { + using ExecSpace = typename StkSourceType::execution_space; + Kokkos::parallel_for( + Kokkos::RangePolicy(0, view.extent(0)), KOKKOS_LAMBDA(const int i) { + auto init = src(i).box; + view(i) = StkToArborX(init); + }); + } +} + +} // namespace impl +} // namespace stk::search + +#endif diff --git a/packages/stk/stk_search/stk_search/kdtree/CoarseSearchKdTree.hpp b/packages/stk/stk_search/stk_search/kdtree/CoarseSearchKdTree.hpp new file mode 100644 index 000000000000..95a13b5b70cb --- /dev/null +++ b/packages/stk/stk_search/stk_search/kdtree/CoarseSearchKdTree.hpp @@ -0,0 +1,272 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +#ifndef KDTREE_STK_INTERFACE_H_ +#define KDTREE_STK_INTERFACE_H_ + +#ifdef _OPENMP +#include +#endif + +#include "stk_search/kdtree/KDTree_BoundingBox.hpp" +#include "stk_search/kdtree/KDTree_ParallelConsistencyUtils.hpp" +#include "stk_util/environment/Env.hpp" +#include "stk_util/environment/WallTime.hpp" +#include "stk_util/parallel/ParallelReduce.hpp" +#include +#include +#include + +namespace stk::search { + +// +// More general search for an arbitrary range type +// +template +inline void coarse_search_kdtree(std::vector< std::pair > const & local_domain, + std::vector< std::pair > const & local_range, + MPI_Comm comm, + std::vector >& searchResults, + bool enforceSearchResultSymmetry=true) +{ + + int num_procs = -1; + int proc_id = -1; + MPI_Comm_rank(comm, &proc_id); + MPI_Comm_size(comm, &num_procs); + + searchResults.clear(); + std::vector rangeObjs( local_range.size() ); + + std::vector rangeGhostIdentifiers; + + stk::search::ComputeRangeWithGhostsForCoarseSearch(local_domain, local_range, + num_procs, rangeObjs, rangeGhostIdentifiers, comm); + +#ifdef _OPENMP + std::vector > > + threadLocalSearchResults( omp_get_max_threads() ); +#endif + + if ((local_domain.size() > 0) && (rangeObjs.size() > 0)) { + + // + // Need to convert range objects to actual box type objects for proximity search + // + + using rangeValueType = typename RangeObjType::value_type; + using RangeBox = stk::search::Box; + + std::vector rangeBoxes; + rangeBoxes.reserve( rangeObjs.size() ); + + for(auto& p : rangeObjs) { + auto& obj = p; + rangeBoxes.emplace_back(RangeBox(obj.get_x_min(), obj.get_y_min(), obj.get_z_min(), + obj.get_x_max(), obj.get_y_max(), obj.get_z_max())); + + } + + const stk::search::ProximitySearchTree_T proxSearch(rangeBoxes); + const unsigned numBoxDomain = local_domain.size(); + +#ifdef _OPENMP +#pragma omp parallel default(shared) +#endif + { + // + // Set the known return vector sizes + // + + std::vector overlapList; +#ifdef _OPENMP + std::vector >& interList = threadLocalSearchResults[omp_get_thread_num()]; +#else + std::vector >& interList = searchResults; +#endif + // + // Create an array to store interactions returned by the recursive search routines. There are at maximum + // N interactions per object when searching N objects + // + + // + // Loop over all boxAs in group1 and search them against those objects in group2 + // +#ifdef _OPENMP +#pragma omp for +#endif + for(unsigned int iboxDomain = 0; iboxDomain < numBoxDomain; ++iboxDomain) { + proxSearch.SearchForOverlap(local_domain[iboxDomain].first, overlapList); + for(auto&& jboxRange : overlapList) { + if(intersects(local_domain[iboxDomain].first, rangeObjs[jboxRange])) { + if(jboxRange < (int)local_range.size()) { + interList.emplace_back( local_domain[iboxDomain].second, local_range[jboxRange].second ); + } else { + interList.emplace_back( local_domain[iboxDomain].second, rangeGhostIdentifiers[jboxRange-local_range.size()] ); + } + } + } + } + } + } +#ifdef _OPENMP + stk::search::concatenate_thread_lists(threadLocalSearchResults, searchResults); +#endif + + if(enforceSearchResultSymmetry) { + stk::search::communicate_vector(comm, searchResults, enforceSearchResultSymmetry); + std::sort(searchResults.begin(), searchResults.end()); + } +} + + + +// +// Most optimal search specific to actual box arguments +// +template +inline void coarse_search_kdtree(std::vector< std::pair > const & local_domain, + std::vector< std::pair, RangeIdentifier > > const & local_range, + MPI_Comm comm, + std::vector >& searchResults, + bool enforceSearchResultSymmetry=true) +{ + int num_procs = -1; + int proc_id = -1; + MPI_Comm_rank(comm, &proc_id); + MPI_Comm_size(comm, &num_procs); + + searchResults.clear(); + +#ifdef _OPENMP + std::vector > > + threadLocalSearchResults( omp_get_max_threads() ); +#endif + + { + std::vector > rangeBoxes( local_range.size() ); + + std::vector rangeGhostIdentifiers; + + stk::search::ComputeRangeWithGhostsForCoarseSearch(local_domain, local_range, + num_procs, rangeBoxes, rangeGhostIdentifiers, comm); + + if ((local_domain.size() > 0) && (rangeBoxes.size() > 0)) { + + const stk::search::ProximitySearchTree_T > proxSearch(rangeBoxes); + const unsigned numBoxDomain = local_domain.size(); + +#ifdef _OPENMP +#pragma omp parallel default(shared) +#endif + { + // + // Set the known return vector sizes + // + + std::vector overlapList; +#ifdef _OPENMP + std::vector >& interList = threadLocalSearchResults[omp_get_thread_num()]; +#else + std::vector >& interList = searchResults; +#endif + + // + // Create an array to store interactions returned by the recursive search routines. There are at maximum + // N interactions per object when searching N objects + // + + // + // Loop over all boxAs in group1 and search them against those objects in group2 + // + interList.reserve((numBoxDomain*3)/2); +#ifdef _OPENMP +#pragma omp for +#endif + for(unsigned int iboxDomain = 0; iboxDomain < numBoxDomain; ++iboxDomain) { + proxSearch.SearchForOverlap(local_domain[iboxDomain].first, overlapList); + for(auto&& jboxRange : overlapList) { + if(jboxRange < (int)local_range.size()) { + interList.emplace_back( local_domain[iboxDomain].second, local_range[jboxRange].second ); + } else { + interList.emplace_back( local_domain[iboxDomain].second, rangeGhostIdentifiers[jboxRange-local_range.size()] ); + } + } + } + } + } + } +#ifdef _OPENMP + stk::search::concatenate_thread_lists(threadLocalSearchResults, searchResults); +#endif + if(enforceSearchResultSymmetry) { + stk::search::communicate_vector(comm, searchResults, enforceSearchResultSymmetry); + std::sort(searchResults.begin(), searchResults.end()); + } +} + +template +inline void coarse_search_kdtree_driver(std::vector< std::pair > const & local_domain, + std::vector< std::pair > const & local_range, + MPI_Comm comm, + std::vector >& searchResults, + bool enforceSearchResultSymmetry=true) +{ + const size_t local_sizes[2] = {local_domain.size(), local_range.size()}; + size_t global_sizes[2]; + all_reduce_sum(comm, local_sizes, global_sizes, 2); + const bool domain_has_more_boxes = (global_sizes[0] >= global_sizes[1]); + if(domain_has_more_boxes) + { + coarse_search_kdtree(local_domain, local_range, comm, searchResults, enforceSearchResultSymmetry); + } + else + { + std::vector > tempSearchResults; + coarse_search_kdtree(local_range, local_domain, comm, tempSearchResults, enforceSearchResultSymmetry); + const int p_rank = stk::parallel_machine_rank(comm); + searchResults.reserve(tempSearchResults.size()); + for(size_t i=0; i()(pair.second) == p_rank) + searchResults.emplace_back(pair.second, pair.first); + } + } +} + +} + +#endif diff --git a/packages/stk/stk_search/stk_search/KDTree.hpp b/packages/stk/stk_search/stk_search/kdtree/KDTree.hpp similarity index 95% rename from packages/stk/stk_search/stk_search/KDTree.hpp rename to packages/stk/stk_search/stk_search/kdtree/KDTree.hpp index 0fad52529054..06fc21f11faf 100644 --- a/packages/stk/stk_search/stk_search/KDTree.hpp +++ b/packages/stk/stk_search/stk_search/kdtree/KDTree.hpp @@ -36,7 +36,7 @@ #include // for vector // for AxisAlignedBB -#include +#include #ifdef _OPENMP #include "omp.h" #endif @@ -59,9 +59,6 @@ enum { MAX_TREE_LEVELS = 100 }; template class ObjectBoundingBoxHierarchy_T { public: - KOKKOS_DEFAULTED_FUNCTION ObjectBoundingBoxHierarchy_T() = default; /// Default box is null and right child offset is 0 (invalid) - KOKKOS_DEFAULTED_FUNCTION ~ObjectBoundingBoxHierarchy_T() = default; - /** * Create a hierarchy from a list of object bounding boxes. The top level of the hierarchy will contain all boxes. In the next level * each leave of the tree will contain approximiatly half of the boxes. The hierarchy continues down until the tree node contains only @@ -90,9 +87,8 @@ class ObjectBoundingBoxHierarchy_T { template class ProximitySearchTree_T { public: - // Create an empty tree - ProximitySearchTree_T(); + ProximitySearchTree_T() = default; // Create and initialize a search tree from a set of input boxes. Assumed ident based on original box order @@ -150,7 +146,7 @@ class ProximitySearchTree_T { } } -#include +#include #endif // STKSEARCH_KDTREE_BoundingBoxHierarchy_h_ diff --git a/packages/stk/stk_search/stk_search/KDTree_BoundingBox.hpp b/packages/stk/stk_search/stk_search/kdtree/KDTree_BoundingBox.hpp similarity index 100% rename from packages/stk/stk_search/stk_search/KDTree_BoundingBox.hpp rename to packages/stk/stk_search/stk_search/kdtree/KDTree_BoundingBox.hpp diff --git a/packages/stk/stk_search/stk_search/KDTree_BoundingSpheres.hpp b/packages/stk/stk_search/stk_search/kdtree/KDTree_BoundingSpheres.hpp similarity index 99% rename from packages/stk/stk_search/stk_search/KDTree_BoundingSpheres.hpp rename to packages/stk/stk_search/stk_search/kdtree/KDTree_BoundingSpheres.hpp index 30002692a221..a12b18758671 100644 --- a/packages/stk/stk_search/stk_search/KDTree_BoundingSpheres.hpp +++ b/packages/stk/stk_search/stk_search/kdtree/KDTree_BoundingSpheres.hpp @@ -38,7 +38,7 @@ #include #include #include -#include +#include namespace stk { namespace search { diff --git a/packages/stk/stk_search/stk_search/kdtree/KDTree_ParallelConsistencyUtils.hpp b/packages/stk/stk_search/stk_search/kdtree/KDTree_ParallelConsistencyUtils.hpp new file mode 100644 index 000000000000..d8ddf5d2a7fb --- /dev/null +++ b/packages/stk/stk_search/stk_search/kdtree/KDTree_ParallelConsistencyUtils.hpp @@ -0,0 +1,138 @@ +#ifndef KDTREE_PARALLELCONSISTENCYUTILS_HPP +#define KDTREE_PARALLELCONSISTENCYUTILS_HPP + +#include "stk_util/parallel/Parallel.hpp" +#include "stk_search/kdtree/KDTree_BoundingBox.hpp" +#include "stk_search/CommonSearchUtil.hpp" +#include +#include + +namespace stk::search { + +template +void ParallelComputeProcObjectBoxes(const std::vector>& local_objsWithIdents, + std::vector> &objBB_proc_box_array, + MPI_Comm &comm) +{ + const unsigned numBoxDomain = local_objsWithIdents.size(); + stk::search::ObjectBoundingBox_T objBB_proc; + +#ifdef _OPENMP + std::vector > threadBoxes( omp_get_max_threads() ); +#endif + +#ifdef _OPENMP +#pragma omp parallel default(shared) +#endif + { +#ifdef _OPENMP + stk::search::ObjectBoundingBox_T& curBox = threadBoxes[omp_get_thread_num()]; +#else + stk::search::ObjectBoundingBox_T& curBox = objBB_proc; +#endif +#ifdef _OPENMP +#pragma omp for +#endif + for(unsigned ibox = 0; ibox < numBoxDomain; ++ibox) { + stk::search::add_to_box(curBox.GetBox(), local_objsWithIdents[ibox].first); + } + } + +#ifdef _OPENMP + for(unsigned i=0; i +void +ComputeRangeWithGhostsForCoarseSearch(const std::vector>& local_domain, + const std::vector>& local_range, + int num_procs, + std::vector& rangeBoxes, + std::vector& rangeGhostIdentifiers, MPI_Comm comm) +{ + const unsigned numBoxRange = local_range.size(); + + using domainValueType = typename DomainObjType::value_type; + using DomainBox = stk::search::Box; + +#ifdef _OPENMP +#pragma omp parallel for default(shared) +#endif + for (size_t i = 0; i < numBoxRange; ++i) { + rangeBoxes[i] = local_range[i].first; + } + + // + // Determine the total number of processors involved in the communication and the current processor number + // + int current_proc(0); + MPI_Comm_rank(comm, ¤t_proc); + if(num_procs == 0) { + return; + } + + // + // Compute the processor local bounding boxes for the box sets + // Store the boxes in unique entries in a global processor bounding box array. + // + std::vector > boxA_proc_box_array; + ParallelComputeProcObjectBoxes(local_domain, boxA_proc_box_array, comm); + + // + // Create a hierarchy of boxA processor bounding boxes. + // This hierarchy will be used to search for overlaps between processors and + // objects. + // + stk::search::ProximitySearchTree_T boxA_box_hierarchy(boxA_proc_box_array); + + // + // Determine what to ghost. If a boxB box from this processor overlaps another processor's + // processor all-boxA box, then we need to ghost the boxB's data to that other processor + // to include in the range boxes (local + global) to search against its local domain boxes. + // + typedef typename RangeIdentifier::ident_type GlobalIdType; + typedef std::pair BoxIdPair; + std::vector> send_list(num_procs); + std::vector> recv_list(num_procs); + + std::vector proc_list(num_procs); + for (unsigned int iboxB = 0; iboxB < numBoxRange; ++iboxB) { + boxA_box_hierarchy.SearchForOverlap(local_range[iboxB].first, proc_list); + for (auto&& overlapping_proc : proc_list) { + if (overlapping_proc == current_proc) continue; + GlobalIdType id = local_range[iboxB].second.id(); + send_list[overlapping_proc].push_back(BoxIdPair(rangeBoxes[iboxB], id)); + } + } + + stk::parallel_data_exchange_t(send_list, recv_list, comm); + rangeGhostIdentifiers.clear(); + for (size_t i = 0; i < recv_list.size(); ++i) { + for (size_t j = 0; j < recv_list[i].size(); ++j) { + const BoxIdPair& recvd_boxIdPair = recv_list[i][j]; + rangeBoxes.push_back(recvd_boxIdPair.first); + rangeGhostIdentifiers.push_back(RangeIdentifier(recvd_boxIdPair.second, i)); + } + } +} + +} + +#endif // KDTREE_PARALLELCONSISTENCYUTILS_HPP diff --git a/packages/stk/stk_search/stk_search/KDTree_ThreadedSort.cpp b/packages/stk/stk_search/stk_search/kdtree/KDTree_ThreadedSort.cpp similarity index 100% rename from packages/stk/stk_search/stk_search/KDTree_ThreadedSort.cpp rename to packages/stk/stk_search/stk_search/kdtree/KDTree_ThreadedSort.cpp diff --git a/packages/stk/stk_search/stk_search/KDTree_Threaded_Sort.hpp b/packages/stk/stk_search/stk_search/kdtree/KDTree_Threaded_Sort.hpp similarity index 100% rename from packages/stk/stk_search/stk_search/KDTree_Threaded_Sort.hpp rename to packages/stk/stk_search/stk_search/kdtree/KDTree_Threaded_Sort.hpp diff --git a/packages/stk/stk_search/stk_search/KDTree_impl.hpp b/packages/stk/stk_search/stk_search/kdtree/KDTree_impl.hpp similarity index 99% rename from packages/stk/stk_search/stk_search/KDTree_impl.hpp rename to packages/stk/stk_search/stk_search/kdtree/KDTree_impl.hpp index 7c1710c2ea74..bea850c3cd98 100644 --- a/packages/stk/stk_search/stk_search/KDTree_impl.hpp +++ b/packages/stk/stk_search/stk_search/kdtree/KDTree_impl.hpp @@ -39,7 +39,7 @@ #include // for min #include // for assert #include // for fabs -#include // for ThreadedSort +#include // for ThreadedSort #include // for pair, make_pair #include // for stk::math::max, stk::math::min @@ -1470,13 +1470,6 @@ inline bool ProximitySearchTree_T::AnyOverlap(const DomainBox& sea } } -// -// Create empty search tree -// -template -ProximitySearchTree_T::ProximitySearchTree_T() { -} - template ProximitySearchTree_T::ProximitySearchTree_T(const std::vector& inputBoxes) { unsigned numBox = inputBoxes.size(); diff --git a/packages/stk/stk_search/stk_search/morton_lbvh/CoarseSearchMortonLBVH.hpp b/packages/stk/stk_search/stk_search/morton_lbvh/CoarseSearchMortonLBVH.hpp new file mode 100644 index 000000000000..f6c02844cce1 --- /dev/null +++ b/packages/stk/stk_search/stk_search/morton_lbvh/CoarseSearchMortonLBVH.hpp @@ -0,0 +1,126 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef COARSESEARCHMORTONLBVH_HPP +#define COARSESEARCHMORTONLBVH_HPP + +#include "stk_search/morton_lbvh/MortonLBVH_Search.hpp" +#include "stk_search/morton_lbvh/MortonLBVH_Tree.hpp" +#include "stk_search/morton_lbvh/MortonLBVH_ParallelConsistencyUtils.hpp" +#include "stk_search/IdentProc.hpp" +#include "stk_search/CommonSearchUtil.hpp" +#include "stk_search/BoundingBox.hpp" +#include "stk_util/parallel/Parallel.hpp" +#include "stk_util/parallel/ParallelReduce.hpp" +#include "stk_util/util/ReportHandler.hpp" +#include "Kokkos_Core.hpp" +#include +#include +#include + +namespace stk::search { + +template +inline void coarse_search_morton_lbvh(std::vector> const & localDomain, + std::vector> const & localRange, + MPI_Comm comm, + std::vector> & searchResults, + bool enforceSearchResultSymmetry = true) +{ + STK_ThrowRequireMsg((std::is_same_v), + "The domain and range boxes must have the same floating-point precision"); + + using ValueType = typename DomainBoxType::value_type; + + Kokkos::Profiling::pushRegion("Parallel consistency: extend range box list"); + const auto [extendedRangeBoxes, remoteRangeIdentProcs] = + morton_extend_local_range_with_remote_boxes_that_might_intersect(localDomain, localRange, comm); + Kokkos::Profiling::popRegion(); + + Kokkos::Profiling::pushRegion("Fill domain and range trees"); + stk::search::MortonAabbTree domainTree("Domain Tree", localDomain.size()); + stk::search::MortonAabbTree rangeTree("Range Tree", extendedRangeBoxes.size()); + + stk::search::export_from_box_ident_proc_vec_to_morton_tree(localDomain, domainTree); + stk::search::export_from_box_vec_to_morton_tree(extendedRangeBoxes, rangeTree); + domainTree.sync_to_device(); + rangeTree.sync_to_device(); + Kokkos::Profiling::popRegion(); + + stk::search::CollisionList collisionList("Collision List"); + stk::search::morton_lbvh_search(domainTree, rangeTree, collisionList); + collisionList.sync_from_device(); + + Kokkos::Profiling::pushRegion("Aggregate search results"); + searchResults.clear(); + const unsigned numCollisions = collisionList.hm_idx(); + searchResults.reserve(numCollisions); + + const unsigned numLocalRange = localRange.size(); + + auto insert_into_results = [&, &remoteRangeIdentProcs=remoteRangeIdentProcs](unsigned domainIdx, unsigned rangeIdx) { + if (rangeIdx < numLocalRange) { + searchResults.emplace_back(localDomain[domainIdx].second, localRange[rangeIdx].second); + } + else { + searchResults.emplace_back(localDomain[domainIdx].second, remoteRangeIdentProcs[rangeIdx - numLocalRange]); + } + }; + + for (unsigned i = 0; i < numCollisions; ++i) { + const unsigned domainIdx = collisionList.hm_data(i, 0); + const unsigned rangeIdx = collisionList.hm_data(i, 1); + + if constexpr ((std::is_same_v> || std::is_same_v>) && + (std::is_same_v> || std::is_same_v>)) { + insert_into_results(domainIdx, rangeIdx); + } + else { + if (intersects(localDomain[domainIdx].first, extendedRangeBoxes[rangeIdx])) { + insert_into_results(domainIdx, rangeIdx); + } + } + } + Kokkos::Profiling::popRegion(); + + if (enforceSearchResultSymmetry) { + Kokkos::Profiling::pushRegion("Enforce results symmetry"); + stk::search::communicate_vector(comm, searchResults, enforceSearchResultSymmetry); + std::sort(searchResults.begin(), searchResults.end()); + Kokkos::Profiling::popRegion(); + } +} + +} + +#endif // COARSESEARCHMORTONLBVH_HPP diff --git a/packages/stk/stk_search/stk_search/morton_lbvh/LocalCoarseSearchMortonLBVH.hpp b/packages/stk/stk_search/stk_search/morton_lbvh/LocalCoarseSearchMortonLBVH.hpp new file mode 100644 index 000000000000..c1c38ce4f8c7 --- /dev/null +++ b/packages/stk/stk_search/stk_search/morton_lbvh/LocalCoarseSearchMortonLBVH.hpp @@ -0,0 +1,246 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef LOCALCOARSESEARCHMORTONLBVH_HPP +#define LOCALCOARSESEARCHMORTONLBVH_HPP + +#include "stk_search/BoxIdent.hpp" +#include "stk_search/Box.hpp" +#include "stk_search/morton_lbvh/MortonLBVH_Search.hpp" +#include "stk_search/morton_lbvh/MortonLBVH_Tree.hpp" +#include "Kokkos_Core.hpp" +#include +#include + +namespace stk::search { + +template +void insert_intersections_into_results(const std::vector>& domain, + const std::vector>& range, + const stk::search::CollisionList rawIntersections, + std::vector>& intersections) +{ + const int numCollisions = rawIntersections.get_num_collisions(); + + for (int i = 0; i < numCollisions; ++i) { + const unsigned domainIdx = rawIntersections.m_data(i, 0); + const unsigned rangeIdx = rawIntersections.m_data(i, 1); + intersections.emplace_back(domain[domainIdx].second, range[rangeIdx].second); + }; +} + +template +void insert_intersections_into_results( + const Kokkos::View*, ExecutionSpace> & domain, + const Kokkos::View*, ExecutionSpace> & range, + const stk::search::CollisionList rawIntersections, + Kokkos::View*, ExecutionSpace> & intersections) +{ + const int numCollisions = rawIntersections.get_num_collisions(); + + Kokkos::parallel_for(Kokkos::RangePolicy(0, numCollisions), + KOKKOS_LAMBDA(int index) { + const unsigned domainIdx = rawIntersections.m_data(index, 0); + const unsigned rangeIdx = rawIntersections.m_data(index, 1); + intersections[index] = {domain[domainIdx].ident, range[rangeIdx].ident}; + }); +} + +template +void insert_only_confirmed_intersections_into_results( + const std::vector>& domain, + const std::vector>& range, + const stk::search::CollisionList rawIntersections, + std::vector>& intersections) +{ + constexpr DomainIdentType INVALID_DOMAIN_IDENT = std::numeric_limits::max(); + constexpr RangeIdentType INVALID_RANGE_IDENT = std::numeric_limits::max(); + const int numCollisions = rawIntersections.get_num_collisions(); + + for (int index = 0; index < numCollisions; ++index) { + const unsigned domainIdx = rawIntersections.m_data(index, 0); + const unsigned rangeIdx = rawIntersections.m_data(index, 1); + const auto& domainBoxIdent = domain[domainIdx]; + const auto& rangeBoxIdent = range[rangeIdx]; + if (intersects(domainBoxIdent.first, rangeBoxIdent.first)) { + intersections.emplace_back(domainBoxIdent.second, rangeBoxIdent.second); + } else { + intersections.emplace_back(INVALID_DOMAIN_IDENT, INVALID_RANGE_IDENT); + } + }; + + int numActualIntersections = 0; + int destIndex = 0; + for (int sourceIndex = 0; sourceIndex < numCollisions; ++sourceIndex) { + if (intersections[sourceIndex].first != INVALID_DOMAIN_IDENT) { + intersections[destIndex++] = intersections[sourceIndex]; + numActualIntersections++; + } + } + + intersections.resize(numActualIntersections); +} + +template +void insert_only_confirmed_intersections_into_results( + const Kokkos::View*, ExecutionSpace> & domain, + const Kokkos::View*, ExecutionSpace> & range, + const stk::search::CollisionList rawIntersections, + Kokkos::View*, ExecutionSpace> & intersections) +{ + constexpr DomainIdentType INVALID_DOMAIN_IDENT = std::numeric_limits::max(); + constexpr RangeIdentType INVALID_RANGE_IDENT = std::numeric_limits::max(); + const int numCollisions = rawIntersections.get_num_collisions(); + + Kokkos::parallel_for(Kokkos::RangePolicy(0, numCollisions), + KOKKOS_LAMBDA(int index) { + const unsigned domainIdx = rawIntersections.m_data(index, 0); + const unsigned rangeIdx = rawIntersections.m_data(index, 1); + const auto & domainBoxIdent = domain[domainIdx]; + const auto & rangeBoxIdent = range[rangeIdx]; + if (intersects(domainBoxIdent.box, rangeBoxIdent.box)) { + intersections[index] = {domainBoxIdent.ident, rangeBoxIdent.ident}; + } + else { + intersections[index] = {INVALID_DOMAIN_IDENT, INVALID_RANGE_IDENT}; + } + }); + + int numActualIntersections = 0; + Kokkos::parallel_reduce(Kokkos::RangePolicy(0, 1), + KOKKOS_LAMBDA(int /*index*/, int & intersectionSum) { + int destIndex = 0; + for (int sourceIndex = 0; sourceIndex < numCollisions; ++sourceIndex) { + if (intersections[sourceIndex].domainIdent != INVALID_DOMAIN_IDENT) { + intersections[destIndex++] = intersections[sourceIndex]; + } + } + intersectionSum = destIndex; + }, + numActualIntersections); + + Kokkos::resize(intersections, numActualIntersections); +} + +template +void local_coarse_search_morton_lbvh( + const std::vector> & domain, + const std::vector> & range, + std::vector> & searchResults) +{ + STK_ThrowRequireMsg((std::is_same_v), + "The domain and range boxes must have the same floating-point precision"); + + using ValueType = typename DomainBoxType::value_type; + using ExecutionSpace = Kokkos::DefaultHostExecutionSpace; + + Kokkos::Profiling::pushRegion("Fill domain and range trees"); + const bool supportHostBoxes = false; + stk::search::MortonAabbTree domainTree("Domain Tree", domain.size(), supportHostBoxes); + stk::search::MortonAabbTree rangeTree("Range Tree", range.size(), supportHostBoxes); + + stk::search::export_from_box_ident_vector_to_morton_tree(domain, domainTree); + stk::search::export_from_box_ident_vector_to_morton_tree(range, rangeTree); + Kokkos::Profiling::popRegion(); + + stk::search::CollisionList collisionList("Collision List"); + stk::search::morton_lbvh_search(domainTree, rangeTree, collisionList); + + Kokkos::Profiling::pushRegion("Aggregate search results"); + const int numCollisions = collisionList.get_num_collisions(); + searchResults.reserve(numCollisions); + + if constexpr ((std::is_same_v> || std::is_same_v>) && + (std::is_same_v> || std::is_same_v>)) + { + insert_intersections_into_results(domain, range, collisionList, searchResults); + } + else { + insert_only_confirmed_intersections_into_results(domain, range, collisionList, searchResults); + } +} + +template +void local_coarse_search_morton_lbvh( + const Kokkos::View*, ExecutionSpace> & domain, + const Kokkos::View*, ExecutionSpace> & range, + Kokkos::View*, ExecutionSpace> & searchResults) +{ + STK_ThrowRequireMsg((std::is_same_v), + "The domain and range boxes must have the same floating-point precision"); + + using ValueType = typename DomainBoxType::value_type; + + Kokkos::Profiling::pushRegion("Fill domain and range trees"); + const bool supportHostBoxes = false; + stk::search::MortonAabbTree domainTree("Domain Tree", domain.extent(0), supportHostBoxes); + stk::search::MortonAabbTree rangeTree("Range Tree", range.extent(0), supportHostBoxes); + + stk::search::export_from_box_ident_view_to_morton_tree(domain, domainTree); + stk::search::export_from_box_ident_view_to_morton_tree(range, rangeTree); + domainTree.sync_to_device(); + rangeTree.sync_to_device(); + Kokkos::Profiling::popRegion(); + + stk::search::CollisionList collisionList("Collision List"); + stk::search::morton_lbvh_search(domainTree, rangeTree, collisionList); + + Kokkos::Profiling::pushRegion("Aggregate search results"); + const int numCollisions = collisionList.get_num_collisions(); + searchResults = Kokkos::View*, ExecutionSpace>( + Kokkos::ViewAllocateWithoutInitializing(searchResults.label()), numCollisions); + + if constexpr ((std::is_same_v> || std::is_same_v>) && + (std::is_same_v> || std::is_same_v>)) + { + insert_intersections_into_results(domain, range, collisionList, searchResults); + } + else { + insert_only_confirmed_intersections_into_results(domain, range, collisionList, searchResults); + } +} + +} + +#endif // LOCALCOARSESEARCHMORTONLBVH_HPP diff --git a/packages/stk/stk_search/stk_search/morton_lbvh/MortonLBVH_BoundingBoxes.hpp b/packages/stk/stk_search/stk_search/morton_lbvh/MortonLBVH_BoundingBoxes.hpp new file mode 100644 index 000000000000..cbbf17b38d27 --- /dev/null +++ b/packages/stk/stk_search/stk_search/morton_lbvh/MortonLBVH_BoundingBoxes.hpp @@ -0,0 +1,209 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef MORTONLBVH_BOUNDINGBOXES_HPP +#define MORTONLBVH_BOUNDINGBOXES_HPP + +#include +#include +#include +#include +#include +#include + +namespace stk::search { + +template +struct MortonAABox +{ + RealType m_min[3]; + RealType m_max[3]; +}; + + +// Axis Aligned Bounding boxes. +template +struct MortonAabbList +{ + using real_type = RealType; + using kokkos_aabb_types = MortonAabbTypes; + + using aabb_points_t = typename kokkos_aabb_types::aabb_points_t; + using aabb_points_hmt = typename kokkos_aabb_types::aabb_points_hmt; + using device_type = typename aabb_points_t::device_type; + + MortonAabbList(const std::string &baseName, LocalOrdinal numBoxes = 0); + + KOKKOS_INLINE_FUNCTION + void host_set_box(LocalOrdinal boxIdx, double minX, double maxX, double minY, double maxY, double minZ, double maxZ); + + void get_bounds(double min[3], double max[3]); + + void reset(LocalOrdinal numBoxes); + void resize(LocalOrdinal numBoxes); + + void sync_from_device(); + void sync_to_device(); + + std::ostream &streamit(std::ostream &os) const; + std::ostream &streamit(std::ostream &os, size_t boxIdx) const; + + LocalOrdinal m_numBoxes; + std::string m_minsName; + std::string m_maxsName; + + aabb_points_t m_mins; + aabb_points_t m_maxs; + aabb_points_hmt hm_mins; + aabb_points_hmt hm_maxs; +}; + +template +MortonAabbList::MortonAabbList(const std::string &baseName, LocalOrdinal numBoxes) + : m_numBoxes(numBoxes), + m_minsName(compound_name(baseName, "mins")), + m_maxsName(compound_name(baseName, "maxs")), + m_mins(m_minsName, m_numBoxes), + m_maxs(m_maxsName, m_numBoxes) +{ + std::cout << "Creating a MortonAabbList object" << std::endl; + hm_mins = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_mins); + hm_maxs = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_maxs); + + // Should assert here that the views are not null... +#ifdef BANDC_DEBUG + std::cout << "Created it." << std::endl; +#endif +} + +template +KOKKOS_INLINE_FUNCTION +void MortonAabbList::host_set_box(LocalOrdinal boxIdx, + double minX, double maxX, + double minY, double maxY, + double minZ, double maxZ) +{ + hm_mins(boxIdx, 0) = minX; + hm_mins(boxIdx, 1) = minY; + hm_mins(boxIdx, 2) = minZ; + hm_maxs(boxIdx, 0) = maxX; + hm_maxs(boxIdx, 1) = maxY; + hm_maxs(boxIdx, 2) = maxZ; +} + +template +void MortonAabbList::get_bounds(double min[3], double max[3]) +{ + const LocalOrdinal count = m_numBoxes; + if (count <= 0) { + min[0] = min[1] = min[2] = -1; + max[0] = max[1] = max[2] = 1; + } + else { + for (LocalOrdinal j = 0; j < 3; ++j) { + min[j] = hm_mins(0, j); + max[j] = hm_maxs(0, j); + } + } + for (LocalOrdinal i = 1; i < count; ++i) { + for (LocalOrdinal j = 0; j < 3; ++j) { + min[j] = std::min(hm_mins(i, j), min[j]); + max[j] = std::max(hm_maxs(i, j), max[j]); + } + } +} + +template +void MortonAabbList::reset(LocalOrdinal numBoxes) +{ + if ((numBoxes >= 0) && (numBoxes != m_numBoxes)) { + m_numBoxes = numBoxes; + m_mins = aabb_points_t(m_minsName, m_numBoxes); + m_maxs = aabb_points_t(m_maxsName, m_numBoxes); + hm_mins = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_mins); + hm_maxs = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_maxs); + } +} + +template +void MortonAabbList::resize(LocalOrdinal numBoxes) +{ + if ((numBoxes >= 0) && (numBoxes != m_numBoxes)) { + m_numBoxes = numBoxes; + Kokkos::resize(m_mins, m_numBoxes); + Kokkos::resize(m_maxs, m_numBoxes); + hm_mins = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_mins); + hm_maxs = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_maxs); + } +} + +template +void MortonAabbList::sync_from_device() +{ + Kokkos::deep_copy(hm_mins, m_mins); + Kokkos::deep_copy(hm_maxs, m_maxs); +} + +template +void MortonAabbList::sync_to_device() +{ + Kokkos::deep_copy(m_mins, hm_mins); + Kokkos::deep_copy(m_maxs, hm_maxs); +} + +template +std::ostream &MortonAabbList::streamit(std::ostream &os) const +{ + LocalOrdinal num_boxes = hm_mins.extent(0); + os << "{AABBs " << num_boxes << std::endl; + + for (LocalOrdinal idx = 0; idx < num_boxes; ++idx) { + os << " {(" << hm_mins(idx, 0) << " " << hm_mins(idx, 1) << " " << hm_mins(idx, 2) + << ") (" << hm_maxs(idx, 0) << " " << hm_maxs(idx, 1) << " " << hm_maxs(idx, 2) << ")}" << std::endl; + } + + os << "}"; + return os; +} + +template +std::ostream &MortonAabbList::streamit(std::ostream &os, size_t idx) const +{ + os << " {(" << hm_mins(idx, 0) << " " << hm_mins(idx, 1) << " " << hm_mins(idx, 2) + << ") (" << hm_maxs(idx, 0) << " " << hm_maxs(idx, 1) << " " << hm_maxs(idx, 2) << ")}"; + return os; +} + +} + +#endif // MORTONLBVH_BOUNDINGBOXES_HPP diff --git a/packages/stk/stk_search/stk_search/morton_lbvh/MortonLBVH_CollisionList.hpp b/packages/stk/stk_search/stk_search/morton_lbvh/MortonLBVH_CollisionList.hpp new file mode 100644 index 000000000000..94e951731165 --- /dev/null +++ b/packages/stk/stk_search/stk_search/morton_lbvh/MortonLBVH_CollisionList.hpp @@ -0,0 +1,221 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef MORTONLBVH_COLLISIONLIST_HPP +#define MORTONLBVH_COLLISIONLIST_HPP + +#include +#include +#include +#include +#include +#include + +namespace stk::search { + +template +struct CollisionList +{ + using LVBH_types = MortonLbvhTypes; + using execution_space = typename LVBH_types::execution_space; + using memory_space = typename LVBH_types::memory_space; + using data_val_type = typename LVBH_types::local_ordinal_pairs_t; + using host_data_val_type = typename data_val_type::HostMirror; + using idx_type = Kokkos::View; + using host_idx_type = typename idx_type::HostMirror; + + struct CorruptorTag {}; + + CollisionList(const std::string &dataName, LocalOrdinal capacity = 0); + + void reset(LocalOrdinal newCapacity = -1); + + KOKKOS_INLINE_FUNCTION + void operator()(const CorruptorTag &, const int &idx) const { m_data(idx, 1) += 10; } + + KOKKOS_FORCEINLINE_FUNCTION + void set_collision(int idx, int domainIdx, int rangeIdx) const; + + KOKKOS_FORCEINLINE_FUNCTION + bool push_back(LocalOrdinal domainIdx, LocalOrdinal rangeIdx, bool flipped = false) const; + + KOKKOS_INLINE_FUNCTION + bool set_num_collisions(int numCollisions) const; + + int get_num_collisions() const; + int get_capacity() const { return m_capacity; } + + void make_contents_corrupt(); + + void sync_from_device(); + void sync_to_device(); + + std::string m_dataName; + std::string m_idxName; + LocalOrdinal m_capacity; + + data_val_type m_data; + idx_type m_idx; + host_data_val_type hm_data; + host_idx_type hm_idx; +}; + +template +CollisionList::CollisionList(const std::string &baseName, LocalOrdinal capacity) + : m_dataName(compound_name(baseName, "data")), + m_idxName(compound_name(baseName, "idx")), + m_capacity(capacity), + m_data(no_init(m_dataName, m_capacity)), + m_idx(m_idxName) +{ +#ifdef BANDC_DEBUG + std::cout << "Creating CollisionList mirror view." << std::endl; +#endif + // This should probably be done lazily, when sync_from_device() gets called. + hm_data = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_data); + hm_idx = Kokkos::create_mirror_view(m_idx); +#ifdef BANDC_DEBUG + std::cout << "Collision List Mirror View created" << std::endl; +#endif +} + +template +void CollisionList::reset(LocalOrdinal newCapacity) +{ + if ((newCapacity >= 0) && (newCapacity != m_capacity)) { + m_capacity = newCapacity; + m_data = no_init(m_dataName, m_capacity); + hm_data = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_data); + } + hm_idx() = 0; + Kokkos::deep_copy(m_idx, hm_idx); +} + +template +KOKKOS_FORCEINLINE_FUNCTION +void CollisionList::set_collision(int idx, int domainIdx, int rangeIdx) const +{ + m_data(idx, 0) = domainIdx; + m_data(idx, 1) = rangeIdx; +} + +template +KOKKOS_FORCEINLINE_FUNCTION +bool CollisionList::push_back(LocalOrdinal domainIdx, LocalOrdinal rangeIdx, bool flipped) const +{ + typename idx_type::value_type idx = Kokkos::atomic_fetch_add(&m_idx(), 1); + if (idx < static_cast(m_data.extent(0))) { + if (flipped) { + set_collision(idx, rangeIdx, domainIdx); + } + else { + set_collision(idx, domainIdx, rangeIdx); + } + return true; + } + return false; +} + +template +KOKKOS_INLINE_FUNCTION +bool CollisionList::set_num_collisions(int numCollisions) const +{ + if (numCollisions <= static_cast(m_data.extent(0))) { + m_idx() = numCollisions; + return true; + } + else { + return false; + } +} + +template +int CollisionList::get_num_collisions() const +{ + if (hm_idx() <= 0) { + Kokkos::deep_copy(hm_idx, m_idx); + } + return hm_idx(); +} + +template +void CollisionList::make_contents_corrupt() +{ + Kokkos::parallel_for(Kokkos::RangePolicy(0, m_data.extent(0)), *this); +} + +template +void CollisionList::sync_from_device() +{ + Kokkos::deep_copy(hm_data, m_data); + Kokkos::deep_copy(hm_idx, m_idx); +} + +template +void CollisionList::sync_to_device() +{ + Kokkos::deep_copy(m_data, hm_data); + Kokkos::deep_copy(m_idx, hm_idx); +} + + +template +bool SameContents(CollisionList &listA, CollisionList &listB) +{ + listA.sync_from_device(); + listB.sync_from_device(); + + int numCollisionsA = listA.get_num_collisions(); + int numCollisionsB = listB.get_num_collisions(); + + if (numCollisionsA != numCollisionsB) { + return false; + } + + using collision_pair = std::pair; + + std::set setA; + for (LocalOrdinal i = 0; i < numCollisionsA; ++i) { + setA.insert(collision_pair(listA.hm_data(i, 0), listA.hm_data(i, 1))); + } + std::set setB; + for (LocalOrdinal i = 0; i < numCollisionsB; ++i) { + setB.insert(collision_pair(listB.hm_data(i, 0), listB.hm_data(i, 1))); + } + + return (setA == setB); +} + +} + +#endif // MORTONLBVH_COLLISIONLIST_HPP diff --git a/packages/stk/stk_search/stk_search/morton_lbvh/MortonLBVH_CommonTypes.hpp b/packages/stk/stk_search/stk_search/morton_lbvh/MortonLBVH_CommonTypes.hpp new file mode 100644 index 000000000000..320ff3909b0a --- /dev/null +++ b/packages/stk/stk_search/stk_search/morton_lbvh/MortonLBVH_CommonTypes.hpp @@ -0,0 +1,156 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef STK_SEARCH_MORTON_LBVH_MORTON_COMMON_TYPES_HPP +#define STK_SEARCH_MORTON_LBVH_MORTON_COMMON_TYPES_HPP + +#include +#include + +namespace stk::search { + +// Within an MPI rank ints should suffice. LocalOrdinal MUST BE SIGNED! +using LocalOrdinal = int; + +#ifdef SMALL_MORTON +using morton_code_t = uint32_t; // 32 bit codes +#else +using morton_code_t = uint64_t; // 64 bit codes +#endif + +template +struct MortonLbvhMemorySpace +{ + using memory_space = typename ExecutionSpace::memory_space; +}; + +#ifdef KOKKOS_ENABLE_CUDA +template <> +struct MortonLbvhMemorySpace +{ + using memory_space = Kokkos::CudaSpace; +}; +#endif + +template +inline ViewT no_init(const std::string &name) +{ + return ViewT(Kokkos::ViewAllocateWithoutInitializing(name)); +} + +template +inline ViewT no_init(const std::string &name, unsigned len) +{ + return ViewT(Kokkos::ViewAllocateWithoutInitializing(name), len); +} + +template +inline ViewT with_init(const std::string &name, unsigned len) +{ + return ViewT(name, len); +} + +template +struct MortonLbvhTypes +{ + using execution_space = ExecutionSpace; + using memory_space = typename MortonLbvhMemorySpace::memory_space; + + // View of a single LocalOrdinal. + using local_ordinal_scl_t = Kokkos::View; + using local_ordinal_scl_hmt = typename local_ordinal_scl_t::HostMirror; + using local_ordinal_scl_tmt = Kokkos::View; + + // Will need a view of LocalOrdinal Scalars. + using local_ordinals_t = Kokkos::View; + using local_ordinals_hmt = typename local_ordinals_t::HostMirror; + using local_ordinals_tmt = Kokkos::View; + + // Will need a view of LocalOrdinalPairs. + using local_ordinal_pairs_t = Kokkos::View; + using local_ordinal_pairs_hmt = typename local_ordinal_pairs_t::HostMirror; + using local_ordinal_pairs_tmt = Kokkos::View; + + using aabb_morton_codes_t = Kokkos::View; + using aabb_morton_codes_hmt = typename aabb_morton_codes_t::HostMirror; + using aabb_morton_codes_tmt = Kokkos::View; +}; + +template +struct MortonAabbTypes +{ + using execution_space = typename MortonLbvhTypes::execution_space; + using memory_space = typename MortonLbvhTypes::memory_space; + using real_type = RealType; + + // Points + using aabb_points_t = Kokkos::View; + using aabb_points_hmt = typename aabb_points_t::HostMirror; + using aabb_const_points_t = Kokkos::View; + using aabb_const_points_tmt = Kokkos::View; + + // We'll use these when convert from using (min_pt, max_pt) pairs. + using bboxes_3d_view_t = Kokkos::View; + using bboxes_3d_view_hmt = typename bboxes_3d_view_t::HostMirror; + using bboxes_const_3d_view_t = Kokkos::View; + using bboxes_3d_view_amt = Kokkos::View>; +}; + +struct morton_code_id_pair +{ + morton_code_t m_code; + LocalOrdinal m_id; + + KOKKOS_FORCEINLINE_FUNCTION + bool operator<(const morton_code_id_pair &rhs) const + { + return (m_code < rhs.m_code) || ((m_code == rhs.m_code) && (m_id < rhs.m_id)); + } + + KOKKOS_FORCEINLINE_FUNCTION + bool operator>(const morton_code_id_pair &rhs) const + { + return (m_code > rhs.m_code) || ((m_code == rhs.m_code) && (m_id > rhs.m_id)); + } +}; + +inline std::string compound_name(const std::string &baseName, const std::string &componentName) +{ + std::string sep("_"); + return baseName + sep + componentName; +} + +} // namespace stk::search + +#endif // STK_SEARCH_MORTON_LBVH_MORTON_COMMON_TYPES_HPP diff --git a/packages/stk/stk_search/stk_search/morton_lbvh/MortonLBVH_ParallelConsistencyUtils.hpp b/packages/stk/stk_search/stk_search/morton_lbvh/MortonLBVH_ParallelConsistencyUtils.hpp new file mode 100644 index 000000000000..88a944bd1d6e --- /dev/null +++ b/packages/stk/stk_search/stk_search/morton_lbvh/MortonLBVH_ParallelConsistencyUtils.hpp @@ -0,0 +1,99 @@ +#ifndef MORTONLBVH_PARALLELCONSISTENCYUTILS_HPP +#define MORTONLBVH_PARALLELCONSISTENCYUTILS_HPP + +#include "stk_util/parallel/Parallel.hpp" +#include "stk_search/morton_lbvh/MortonLBVH_Tree.hpp" +#include "stk_search/morton_lbvh/MortonLBVH_Search.hpp" +#include "stk_search/Box.hpp" +#include "stk_search/BoundingBox.hpp" +#include "stk_search/CommonSearchUtil.hpp" +#include +#include + +namespace stk::search { + +template +std::vector> +gather_all_processor_superset_domain_boxes(const std::vector> & localDomain, + MPI_Comm & comm) +{ + std::vector> globalSupersetBoxes; + + Box localSupersetBox; + for (const auto & [box, ident] : localDomain) { + stk::search::add_to_box(localSupersetBox, box); + } + + stk::search::all_gather_helper(localSupersetBox, globalSupersetBoxes, comm); + + return globalSupersetBoxes; +} + + +template +std::pair, std::vector> +morton_extend_local_range_with_remote_boxes_that_might_intersect( + const std::vector> & localDomain, + const std::vector> & localRange, + MPI_Comm comm) +{ + using DomainValueType = typename DomainBoxType::value_type; + + const int numProcs = stk::parallel_machine_size(comm); + const int procId = stk::parallel_machine_rank(comm); + + const auto globalSupersetBoxes = gather_all_processor_superset_domain_boxes(localDomain, comm); + + stk::search::MortonAabbTree domainTree("Proc Domain Tree", + localRange.size()); + stk::search::MortonAabbTree rangeTree("Proc Range Tree", + globalSupersetBoxes.size()); + + export_from_box_ident_proc_vec_to_morton_tree(localRange, domainTree); + export_from_box_vec_to_morton_tree(globalSupersetBoxes, rangeTree); + domainTree.sync_to_device(); + rangeTree.sync_to_device(); + + stk::search::CollisionList collisionList("Proc Collision List"); + stk::search::morton_lbvh_search(domainTree, rangeTree, collisionList); + collisionList.sync_from_device(); + + using GlobalIdType = typename RangeIdentProcType::ident_type; + using BoxIdPair = std::pair; + std::vector> sendList(numProcs); + std::vector> recvList(numProcs); + + const unsigned numCollisions = collisionList.hm_idx(); + + for (unsigned i = 0; i < numCollisions; ++i) { + const int entityIndex = collisionList.hm_data(i, 0); + const int remoteProcId = collisionList.hm_data(i, 1); + const auto & [localRangeBox, localRangeIdentProc] = localRange[entityIndex]; + if (remoteProcId != procId) { + sendList[remoteProcId].emplace_back(localRangeBox, localRangeIdentProc.id()); + } + } + + stk::parallel_data_exchange_t(sendList, recvList, comm); + + std::pair, std::vector> result; + auto & [extendedRangeBoxes, remoteRangeIdentProcs] = result; + + extendedRangeBoxes.reserve(localRange.size()); + for (const auto & [box, identProc] : localRange) { + extendedRangeBoxes.push_back(box); + } + + for (size_t proc = 0; proc < recvList.size(); ++proc) { + for (const auto & [box, id] : recvList[proc]) { + extendedRangeBoxes.push_back(box); + remoteRangeIdentProcs.emplace_back(id, proc); + } + } + + return result; +} + +} + +#endif // MORTONLBVH_PARALLELCONSISTENCYUTILS_HPP diff --git a/packages/stk/stk_search/stk_search/morton_lbvh/MortonLBVH_Search.hpp b/packages/stk/stk_search/stk_search/morton_lbvh/MortonLBVH_Search.hpp new file mode 100644 index 000000000000..ae785ecaf952 --- /dev/null +++ b/packages/stk/stk_search/stk_search/morton_lbvh/MortonLBVH_Search.hpp @@ -0,0 +1,255 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef STK_SEARCH_MORTON_LBVH_MORTONLBVH_SEARCH_HPP +#define STK_SEARCH_MORTON_LBVH_MORTONLBVH_SEARCH_HPP + +#include +#include +#include +#include +#include +#include +#include + +#define ALWAYS_SORT_MORTON_TREES 0 + +namespace stk::search { + +template +inline void determine_mas_calling(const MortonAabbTree &partialTree1, + const MortonAabbTree &partialTree2, + bool &sortTree1, bool &sortTree2, bool &flipTrees) +{ + if (ALWAYS_SORT_MORTON_TREES) { + sortTree1 = sortTree2 = true; + flipTrees = false; + return; + } + + LocalOrdinal tree1NumLeaves = partialTree1.hm_numLeaves(); + LocalOrdinal tree2NumLeaves = partialTree2.hm_numLeaves(); + + if ((tree1NumLeaves == 0) || (tree2NumLeaves == 0)) { + // Search will shortcut out. + sortTree1 = sortTree2 = flipTrees = false; + return; + } + + flipTrees = (tree1NumLeaves < tree2NumLeaves); + + const LocalOrdinal magicThreshold = 64; + if ((tree1NumLeaves <= magicThreshold) && (tree2NumLeaves <= magicThreshold)) { + sortTree1 = flipTrees; + sortTree2 = !flipTrees; + return; + } + + const double divergenceFactor = 32.0; + const double magicRatio = (divergenceFactor - 1) * 0.2; + + if (flipTrees) { + sortTree1 = true; + double depthRatio = log(tree2NumLeaves); + if (tree1NumLeaves > 2) { + depthRatio = depthRatio / log(tree1NumLeaves); + } + sortTree2 = (depthRatio < magicRatio); + } + else { + sortTree2 = true; + double depthRatio = log(tree1NumLeaves); + if (tree2NumLeaves > 2) { + depthRatio = depthRatio / log(tree2NumLeaves); + } + sortTree1 = (depthRatio < magicRatio); + } +} + +template +inline void export_from_box_ident_proc_vec_to_morton_tree( + const std::vector> &boxIdentProcVec, + MortonAabbTree &tree) +{ + int numBoxes = static_cast(boxIdentProcVec.size()); + tree.reset(numBoxes); + + for (int i = 0; i < numBoxes; ++i) { + const auto & [box, identProc] = boxIdentProcVec[i]; + tree.host_set_box(i, box.get_x_min(), box.get_x_max(), box.get_y_min(), box.get_y_max(), box.get_z_min(), + box.get_z_max()); + } +} + +template +inline void export_from_box_ident_vector_to_morton_tree( + const std::vector> &boxIdentList, MortonAabbTree &tree) +{ + static_assert(Kokkos::SpaceAccessibility::assignable); + int numBoxes = static_cast(boxIdentList.size()); + tree.reset(numBoxes); + + Kokkos::parallel_for( + Kokkos::RangePolicy(0, numBoxes), KOKKOS_LAMBDA(int index) { + const auto &box = boxIdentList[index].first; + tree.device_set_box(index, box.get_x_min(), box.get_x_max(), box.get_y_min(), box.get_y_max(), box.get_z_min(), + box.get_z_max()); + }); +} + +template +inline void export_from_box_ident_view_to_morton_tree( + const Kokkos::View*, ExecutionSpace> & boxIdentList, + MortonAabbTree & tree) +{ + int numBoxes = static_cast(boxIdentList.extent(0)); + tree.reset(numBoxes); + + Kokkos::parallel_for(Kokkos::RangePolicy(0, numBoxes), + KOKKOS_LAMBDA(int index) { + const auto & box = boxIdentList[index].box; + tree.device_set_box(index, box.get_x_min(), box.get_x_max(), + box.get_y_min(), box.get_y_max(), + box.get_z_min(), box.get_z_max()); + }); +} + +template +inline void export_from_box_vec_to_morton_tree(const std::vector &boxVec, + MortonAabbTree &tree) +{ + int numBoxes = static_cast(boxVec.size()); + tree.reset(numBoxes); + for (int i = 0; i < numBoxes; ++i) { + const BoxType &box = boxVec[i]; + tree.host_set_box(i, box.get_x_min(), box.get_x_max(), box.get_y_min(), box.get_y_max(), box.get_z_min(), + box.get_z_max()); + } +} + +template +inline void morton_lbvh_search(MortonAabbTree &tree1, + MortonAabbTree &tree2, + CollisionList &searchResults) +{ + Kokkos::Profiling::pushRegion("Initialization"); + Kokkos::Profiling::pushRegion("Get global bounds"); + // Get total bounds + TotalBoundsFunctor::apply(tree1); + TotalBoundsFunctor::apply(tree2); + Kokkos::fence(); + Kokkos::Profiling::popRegion(); + + Kokkos::Profiling::pushRegion("Determine need to sort/flip trees"); + bool sortTree1, sortTree2, flipOrder; + determine_mas_calling(tree1, tree2, sortTree1, sortTree2, flipOrder); + Kokkos::Profiling::popRegion(); + + // Morton encode the centroids of the leaves + Kokkos::Profiling::pushRegion("Morton encoding of leaves"); + MortonEncoder::apply(tree1, sortTree1); + MortonEncoder::apply(tree2, sortTree2); + Kokkos::fence(); + Kokkos::Profiling::popRegion(); + + // Sort the leaves if appropriate + Kokkos::Profiling::pushRegion("Sort the trees"); + if (sortTree1) { + // printf("Sorting tree with %d leaves\n", tree1.hm_numLeaves()); + SortByCode::apply(tree1); + } + if (sortTree2) { + // printf("Sorting tree with %d leaves\n", tree1.hm_numLeaves()); + SortByCode::apply(tree2); + } + Kokkos::fence(); + Kokkos::Profiling::popRegion(); + + // Build the tree structures, if appropriate, following Karras's algorithm + Kokkos::Profiling::pushRegion("Build the tree structures"); + bool buildTree1 = (sortTree1 && flipOrder); + bool buildTree2 = (sortTree2 && !flipOrder); + if (buildTree1) { + BuildRadixTree::apply(tree1); + } + if (buildTree2) { + BuildRadixTree::apply(tree2); + } + Kokkos::fence(); + Kokkos::Profiling::popRegion(); + + // Augment the trees to be bounding volume (box) hierarchies + Kokkos::Profiling::pushRegion("Augment the trees to be bounding volume hierarchies"); + if (buildTree1) { + UpdateInteriorNodeBVs::apply(tree1); + } + if (buildTree2) { + UpdateInteriorNodeBVs::apply(tree2); + } + Kokkos::fence(); + Kokkos::Profiling::popRegion(); + Kokkos::Profiling::popRegion(); + + // Test the boxes from the non-tree against the tree that was built. + Kokkos::Profiling::pushRegion("Search query"); + if (flipOrder) { + Traverse_MASTB_BVH_Functor::apply_tree(tree2, tree1, searchResults, true); + } + else { + Traverse_MASTB_BVH_Functor::apply_tree(tree1, tree2, searchResults); + } + Kokkos::fence(); + Kokkos::Profiling::popRegion(); +} + +template +inline void morton_lbvh_search(const std::vector &boxA, + const std::vector &boxB, + CollisionList &searchResults) +{ + Kokkos::Profiling::pushRegion("morton_lbvh_search: export boxes to trees"); + MortonAabbTree mlbvhA("a"), mlbvhB("b"); + export_from_box_vec_to_morton_tree(boxA, mlbvhA); + export_from_box_vec_to_morton_tree(boxB, mlbvhB); + mlbvhA.sync_to_device(); + mlbvhB.sync_to_device(); + Kokkos::Profiling::popRegion(); + + Kokkos::Profiling::pushRegion("morton_lbvh_search: execute search"); + morton_lbvh_search(mlbvhA, mlbvhB, searchResults); + Kokkos::Profiling::popRegion(); +} + +} // namespace stk::search + +#endif // STK_SEARCH_MORTON_LBVH_MORTONLBVH_TREE_HPP diff --git a/packages/stk/stk_search/stk_search/morton_lbvh/MortonLBVH_Tree.hpp b/packages/stk/stk_search/stk_search/morton_lbvh/MortonLBVH_Tree.hpp new file mode 100644 index 000000000000..1dcf86cca14e --- /dev/null +++ b/packages/stk/stk_search/stk_search/morton_lbvh/MortonLBVH_Tree.hpp @@ -0,0 +1,391 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef STK_SEARCH_MORTON_LBVH_MORTONLBVH_TREE_HPP +#define STK_SEARCH_MORTON_LBVH_MORTONLBVH_TREE_HPP + +#include +#include +#include +#include +#include +#include + +//#define DEBUG_MORTON_ACCELERATED_SEARCH 1 + +namespace stk::search { + +template +struct MortonAabbTree +{ + using execution_space = ExecutionSpace; + using real_type = RealType; + using LBVH_types = MortonLbvhTypes; + using kokkos_aabb_types = MortonAabbTypes; + + using local_ordinal_scl_t = typename LBVH_types::local_ordinal_scl_t; + using local_ordinal_scl_hmt = typename LBVH_types::local_ordinal_scl_hmt; + using local_ordinal_scl_tmt = typename LBVH_types::local_ordinal_scl_tmt; + using local_ordinal_pairs_t = typename LBVH_types::local_ordinal_pairs_t; + using local_ordinals_t = typename LBVH_types::local_ordinals_t; + + using aabb_morton_codes_t = typename LBVH_types::aabb_morton_codes_t; + + using bboxes_3d_view_t = typename kokkos_aabb_types::bboxes_3d_view_t; + using bboxes_3d_view_hmt = typename kokkos_aabb_types::bboxes_3d_view_hmt; + + MortonAabbTree(const std::string &base_name, LocalOrdinal num_leaves = 0, bool support_host_boxes = true); + + void init(LocalOrdinal num_leaves); + + void reset(LocalOrdinal num_leaves) { + resize(num_leaves); + } + + void resize(LocalOrdinal num_leaves); + + KOKKOS_INLINE_FUNCTION + void host_set_box(LocalOrdinal box_idx, real_type min_x, real_type max_x, real_type min_y, real_type max_y, + real_type min_z, real_type max_z); + + KOKKOS_FORCEINLINE_FUNCTION + void device_set_box(LocalOrdinal box_idx, real_type min_x, real_type max_x, real_type min_y, real_type max_y, + real_type min_z, real_type max_z) const; + + void sync_from_device() const; + void sync_to_device() const; + + std::ostream &streamit(std::ostream &os) const; + std::ostream &streamit(std::ostream &os, size_t box_idx) const; + +#ifdef DEBUG_MORTON_ACCELERATED_SEARCH + void dump(const std::string& prefix) const; +#endif + + std::string m_baseName; + real_type m_globalMinPt[3]; + real_type m_globalMaxPt[3]; + bool m_supportHostBoxes; + bool m_verbose; + + local_ordinal_scl_t m_numLeaves; + local_ordinal_scl_hmt hm_numLeaves; + local_ordinal_scl_tmt tm_numLeaves; + local_ordinal_scl_t m_numInternalNodes; + local_ordinal_scl_hmt hm_numInternalNodes; + local_ordinal_scl_tmt tm_numInternalNodes; + + bboxes_3d_view_t m_minMaxs; + bboxes_3d_view_hmt hm_minMaxs; + + local_ordinal_pairs_t m_nodeChildren; + local_ordinals_t m_nodeParents; + local_ordinals_t m_atomicFlags; + local_ordinals_t m_leafIds; + aabb_morton_codes_t m_leafCodes; + +#ifdef DEBUG_MORTON_ACCELERATED_SEARCH + typename LBVH_types::local_ordinal_pairs_hmt hm_nodeChildren; + typename LBVH_types::local_ordinals_hmt hm_nodeParents; + typename LBVH_types::local_ordinals_hmt hm_leafIds; + typename LBVH_types::aabb_morton_codes_hmt hm_leafCodes; +#endif +}; + +template +MortonAabbTree::MortonAabbTree(const std::string &baseName, + LocalOrdinal numLeaves, + bool supportHostBoxes) + : m_baseName(baseName), + m_supportHostBoxes(supportHostBoxes), + m_verbose(false) +{ + init(numLeaves); +} + +template +void MortonAabbTree::init(LocalOrdinal numLeaves) +{ + LocalOrdinal numInternalNodes = std::max(numLeaves - 1, 0); + LocalOrdinal numNodes = numLeaves + numInternalNodes; + + // Allocate views. + Kokkos::Timer timer0; + m_numLeaves = no_init(compound_name(m_baseName, "numLeaves")); + m_numInternalNodes = no_init(compound_name(m_baseName, "numInternalNodes")); + m_minMaxs = no_init(compound_name(m_baseName, "minMaxs"), numNodes); + + m_nodeChildren = no_init(compound_name(m_baseName, "children"), numNodes); + m_nodeParents = no_init(compound_name(m_baseName, "parents"), numNodes); + + m_atomicFlags = no_init(compound_name(m_baseName, "atomicFlags"), numInternalNodes); + m_leafIds = no_init(compound_name(m_baseName, "ids"), numLeaves); + m_leafCodes = no_init(compound_name(m_baseName, "leafCodes"), numLeaves); + const double time0 = timer0.seconds(); + +#ifdef MAS_DEBUG + std::cout << "Points Views layout type is " << print_layout_name() << std::endl; + + std::cout << "Points Host Mirror Views layout type is " << print_layout_name() + << std::endl; +#endif + + // Want num_leaves, num_intnl_nodes, and num_nodes readable on both host and device. + Kokkos::Timer timer1; + hm_numLeaves = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_numLeaves); + hm_numLeaves() = numLeaves; + hm_numInternalNodes = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_numInternalNodes); + hm_numInternalNodes() = numInternalNodes; + tm_numLeaves = m_numLeaves; + tm_numInternalNodes = m_numInternalNodes; + const double time1 = timer1.seconds(); + + // Host mirror views really for debugging + Kokkos::Timer timer2; + + if (m_supportHostBoxes) { + hm_minMaxs = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_minMaxs); + } + +#ifdef DEBUG_MORTON_ACCELERATED_SEARCH + hm_nodeChildren = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_nodeChildren); + hm_nodeParents = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_nodeParents); + hm_leafIds = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_leafIds); + hm_leafCodes = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_leafCodes); +#endif + const double time2 = timer2.seconds(); + + if (m_verbose) { + std::cout << std::setw(15) << "MortonAabbTree::init " << numLeaves << " leaves " << time0 << " " + << " " << time1 << " " << time2 << std::endl; + } +} + +template +void MortonAabbTree::resize(LocalOrdinal numLeaves) +{ + if ((numLeaves < 0) || (numLeaves == hm_numLeaves())) return; + + LocalOrdinal numInternalNodes = std::max(numLeaves - 1, 0); + LocalOrdinal numNodes = numLeaves + numInternalNodes; + + // Allocate views. + Kokkos::resize(m_minMaxs, numNodes); + Kokkos::resize(m_nodeChildren, numNodes); + Kokkos::resize(m_nodeParents, numNodes); + Kokkos::resize(m_atomicFlags, numInternalNodes); + Kokkos::resize(m_leafIds, numLeaves); + Kokkos::resize(m_leafCodes, numLeaves); + + // Want num_leaves, num_intnl_nodes, and num_nodes readable on both host and device. + hm_numLeaves() = numLeaves; + hm_numInternalNodes() = numInternalNodes; + Kokkos::deep_copy(m_numLeaves, hm_numLeaves); + Kokkos::deep_copy(m_numInternalNodes, hm_numInternalNodes); + tm_numLeaves = m_numLeaves; + tm_numInternalNodes = m_numInternalNodes; + + if (m_supportHostBoxes) { + // Host mirror views really for debugging + hm_minMaxs = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_minMaxs); + } + +#ifdef DEBUG_MORTON_ACCELERATED_SEARCH + hm_nodeChildren = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_nodeChildren); + hm_nodeParents = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_nodeParents); + hm_leafIds = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_leafIds); + hm_leafCodes = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, m_leafCodes); +#endif +} + +template +KOKKOS_INLINE_FUNCTION +void MortonAabbTree::host_set_box(LocalOrdinal boxIdx, + real_type minX, real_type maxX, + real_type minY, real_type maxY, + real_type minZ, real_type maxZ) +{ + hm_minMaxs(boxIdx, 0) = minX; + hm_minMaxs(boxIdx, 1) = minY; + hm_minMaxs(boxIdx, 2) = minZ; + hm_minMaxs(boxIdx, 3) = maxX; + hm_minMaxs(boxIdx, 4) = maxY; + hm_minMaxs(boxIdx, 5) = maxZ; +} + +template +KOKKOS_FORCEINLINE_FUNCTION +void MortonAabbTree::device_set_box(LocalOrdinal boxIdx, + real_type minX, real_type maxX, + real_type minY, real_type maxY, + real_type minZ, real_type maxZ) const +{ + m_minMaxs(boxIdx, 0) = minX; + m_minMaxs(boxIdx, 1) = minY; + m_minMaxs(boxIdx, 2) = minZ; + m_minMaxs(boxIdx, 3) = maxX; + m_minMaxs(boxIdx, 4) = maxY; + m_minMaxs(boxIdx, 5) = maxZ; +} + +template +void MortonAabbTree::sync_from_device() const +{ + Kokkos::deep_copy(hm_numLeaves, m_numLeaves); + Kokkos::deep_copy(hm_numInternalNodes, m_numInternalNodes); + + if (m_supportHostBoxes) { + Kokkos::deep_copy(hm_minMaxs, m_minMaxs); + } + +#ifdef DEBUG_MORTON_ACCELERATED_SEARCH + Kokkos::deep_copy(hm_nodeChildren, m_nodeChildren); + Kokkos::deep_copy(hm_nodeParents, m_nodeParents); + + Kokkos::deep_copy(hm_leafIds, m_leafIds); + Kokkos::deep_copy(hm_leafCodes, m_leafCodes); +#endif +} + +template +void MortonAabbTree::sync_to_device() const +{ + Kokkos::deep_copy(m_numLeaves, hm_numLeaves); + Kokkos::deep_copy(m_numInternalNodes, hm_numInternalNodes); + + if (m_supportHostBoxes) { + Kokkos::deep_copy(m_minMaxs, hm_minMaxs); + } + +#ifdef DEBUG_MORTON_ACCELERATED_SEARCH + Kokkos::deep_copy(m_nodeChildren, hm_nodeChildren); + Kokkos::deep_copy(m_nodeParents, hm_nodeParents); + + Kokkos::deep_copy(m_leafIds, hm_leafIds); + Kokkos::deep_copy(m_leafCodes, hm_leafCodes); +#endif +} + +template +std::ostream &MortonAabbTree::streamit(std::ostream &os) const +{ + os << "{MortonAabbTree " << m_baseName << " " << sizeof(morton_code_t) + << " (" << m_globalMinPt[0] << " " << m_globalMinPt[1] << " " << m_globalMinPt[2] + << ") ( " << m_globalMaxPt[0] << " " << m_globalMaxPt[1] << " " << m_globalMaxPt[2] + << ") " << hm_numLeaves << " " << hm_numInternalNodes << std::endl; + + if (m_supportHostBoxes) { + for (LocalOrdinal idx = 0; idx < hm_numLeaves(); ++idx) { + os << " {(" << hm_minMaxs(idx, 0) << " " << hm_minMaxs(idx, 1) << " " << hm_minMaxs(idx, 2) + << ") (" << hm_minMaxs(idx, 4) << " " << hm_minMaxs(idx, 4) << " " << hm_minMaxs(idx, 5) << ")" + #ifdef DEBUG_MORTON_ACCELERATED_SEARCH + << " (" << hm_nodeChildren(idx, 0) << ", " << hm_nodeChildren(idx, 1) << ") " << hm_nodeParents(idx) << " " + << hm_leafIds(idx) << " 0x" << std::setfill('0') << std::setw(sizeof(morton_code_t) * 2) << std::hex + << hm_leafCodes(idx) << std::dec + #endif + << "}" << std::endl; + } + for (LocalOrdinal idx = hm_numLeaves(); idx < hm_numLeaves()+hm_numInternalNodes(); ++idx) { + os << " {(" << hm_minMaxs(idx, 0) << " " << hm_minMaxs(idx, 1) << " " << hm_minMaxs(idx, 2) + << ") (" << hm_minMaxs(idx, 3) << " " << hm_minMaxs(idx, 4) << " " << hm_minMaxs(idx, 5) << ")" + #ifdef DEBUG_MORTON_ACCELERATED_SEARCH + << " (" << hm_nodeChildren(idx, 0) << ", " << hm_nodeChildren(idx, 1) << ") " << hm_nodeParents(idx) + #endif + << "}" << std::endl; + } + } + os << "}"; + return os; +} + +template +std::ostream &MortonAabbTree::streamit(std::ostream &os, size_t idx) const +{ + if (m_supportHostBoxes) { + os << " {(" << hm_minMaxs(idx, 0) << " " << hm_minMaxs(idx, 1) << " " << hm_minMaxs(idx, 2) + << ") (" << hm_minMaxs(idx, 3) << " " << hm_minMaxs(idx, 4) << " " << hm_minMaxs(idx, 5) << ")}"; + } + else { + os << "MortonAabbTree::streamit(.) m_supportHostBoxes = 0. "; + } + return os; +} + +#ifdef DEBUG_MORTON_ACCELERATED_SEARCH +template +void MortonAabbTree::dump(const std::string& prefix) const +{ + std::cout << prefix << ": dump, numLeaves=" << hm_numLeaves() << ", numInternalNodes=" + << hm_numInternalNodes() << std::endl; + sync_from_device(); + { + std::ofstream of(prefix+"_leafCodes.txt"); + for (size_t i = 0; i < hm_leafCodes.extent(0); ++i) { + of << i << " " << hm_leafCodes(i) << std::endl; + } + } + { + std::ofstream of(prefix+"_leafIds.txt"); + for (size_t i = 0; i < hm_leafIds.extent(0); ++i) { + of << i << " " << hm_leafIds(i) << std::endl; + } + } + { + std::ofstream of(prefix+"_nodeChildren.txt"); + for (size_t i = 0; i < hm_nodeChildren.extent(0); ++i) { + of << i << " " << hm_nodeChildren(i,0) << " " << hm_nodeChildren(i,1) << std::endl; + } + } + { + std::ofstream of(prefix+"_nodeParents.txt"); + for (size_t i = 0; i < hm_nodeParents.extent(0); ++i) { + of << i << " " << hm_nodeParents(i) << std::endl; + } + } + { + std::ofstream of(prefix+"_minMaxs.txt"); + for (size_t i = 0; i < hm_minMaxs.extent(0); ++i) { + of << i; + for (size_t j = 0; j < 6; ++j) { + of << " " << std::setprecision(7) << hm_minMaxs(i,j); + } + of << std::endl; + } + } +} +#endif + +} // namespace stk::search + +#endif // STK_SEARCH_MORTON_LBVH_MORTONLBVH_TREE_HPP diff --git a/packages/stk/stk_search/stk_search/morton_lbvh/MortonLBVH_TreeManipulationUtils.hpp b/packages/stk/stk_search/stk_search/morton_lbvh/MortonLBVH_TreeManipulationUtils.hpp new file mode 100644 index 000000000000..b32c3d0883b6 --- /dev/null +++ b/packages/stk/stk_search/stk_search/morton_lbvh/MortonLBVH_TreeManipulationUtils.hpp @@ -0,0 +1,964 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// +// The device kernels in this file were ported with minimal changes from +// Sam Mish's (Sandia) code which extends work by Tero Karras. Mish's +// source file (kernels.cu) includes the following notice: +// +// Note: these CUDA kernels were written with the input and guidance from +// Tero Karras' paper, "Maximizing Parallelism in the Construction of BVHs, +// Octrees and k-d Trees", as well as his explicit sources, which can be +// found in the NVIDIA_research directory. +// +// NVIDIA CORPORATION provides those sources under the following open-source +// copyright/licence: +// +// -------------------------Begin NVIDIA Notice----------------------------- +// Copyright (c) 2013, NVIDIA CORPORATION. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// -------------------------End NVIDIA Notice------------------------------- +// + +#ifndef MORTONLBVH_TREEMANIPULATIONUTILS_HPP +#define MORTONLBVH_TREEMANIPULATIONUTILS_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef KOKKOS_ENABLE_CUDA +#include +#include + +#ifdef LENGTH +// The Thrust implementation uses this as the name of a template argument in numerous places! +#error "WHO DEFINED LENGTH IN A MACRO?" +#endif +#endif + +// +// Cuda and gcc disagree about whether the argument to clz*(.) is signed or not! +// +#if defined(__CUDACC__) && defined(__CUDA_ARCH__) +#define AABB_CLZ(x) (__clz(x)) +#define AABB_CLZLL(x) (__clzll(x)) +#define AABB_MIN(x, y) (::min(x, y)) +#define AABB_MAX(x, y) (::max(x, y)) +#else +#define AABB_CLZ(x) (__builtin_clz(x)) +#define AABB_CLZLL(x) (__builtin_clzll(x)) +#define AABB_MIN(x, y) (::std::min(x, y)) +#define AABB_MAX(x, y) (::std::max(x, y)) +#endif + +namespace stk::search { + +constexpr size_t COLLISION_SCALE_FACTOR = 16; + +template +struct TotalBoundsFunctor +{ + using execution_space = ExecutionSpace; + using size_type = typename execution_space::size_type; + + using real_type = RealType; + using value_type = MortonAABox; + using kokkos_aabb_types = MortonAabbTypes; + using bboxes_const_3d_view_t = typename kokkos_aabb_types::bboxes_const_3d_view_t; + + TotalBoundsFunctor(const MortonAabbTree &tree); + + KOKKOS_INLINE_FUNCTION + void init(value_type &update) const; + + static void apply(MortonAabbTree &tree); + + KOKKOS_INLINE_FUNCTION + void operator()(size_type idx, value_type &update) const; + + KOKKOS_INLINE_FUNCTION + void join(value_type &update, const value_type &input) const; + + bboxes_const_3d_view_t m_minMaxs; +}; + +template +TotalBoundsFunctor::TotalBoundsFunctor(const MortonAabbTree &tree) + : m_minMaxs(tree.m_minMaxs) +{} + +template +KOKKOS_INLINE_FUNCTION +void TotalBoundsFunctor::init(value_type &update) const +{ + update.m_min[0] = FLT_MAX; + update.m_min[1] = FLT_MAX; + update.m_min[2] = FLT_MAX; + + update.m_max[0] = -FLT_MAX; + update.m_max[1] = -FLT_MAX; + update.m_max[2] = -FLT_MAX; +} + +template +void TotalBoundsFunctor::apply(MortonAabbTree &tree) +{ + value_type retBox; + retBox.m_min[0] = FLT_MAX; + retBox.m_min[1] = FLT_MAX; + retBox.m_min[2] = FLT_MAX; + + retBox.m_max[0] = -FLT_MAX; + retBox.m_max[1] = -FLT_MAX; + retBox.m_max[2] = -FLT_MAX; + + if (tree.hm_numLeaves() > 0) { + const TotalBoundsFunctor tbf(tree); + const size_t numLeaves = tree.hm_numLeaves(); + Kokkos::parallel_reduce(numLeaves, tbf, retBox); + } + + tree.m_globalMinPt[0] = retBox.m_min[0]; + tree.m_globalMinPt[1] = retBox.m_min[1]; + tree.m_globalMinPt[2] = retBox.m_min[2]; + + tree.m_globalMaxPt[0] = retBox.m_max[0]; + tree.m_globalMaxPt[1] = retBox.m_max[1]; + tree.m_globalMaxPt[2] = retBox.m_max[2]; +} + +template +KOKKOS_INLINE_FUNCTION +void TotalBoundsFunctor::operator()(size_type idx, value_type &update) const +{ + update.m_min[0] = fmin(m_minMaxs(idx, 0), update.m_min[0]); + update.m_min[1] = fmin(m_minMaxs(idx, 1), update.m_min[1]); + update.m_min[2] = fmin(m_minMaxs(idx, 2), update.m_min[2]); + + update.m_max[0] = fmax(m_minMaxs(idx, 3), update.m_max[0]); + update.m_max[1] = fmax(m_minMaxs(idx, 4), update.m_max[1]); + update.m_max[2] = fmax(m_minMaxs(idx, 5), update.m_max[2]); +} + +template +KOKKOS_INLINE_FUNCTION +void TotalBoundsFunctor::join(value_type &update, const value_type &input) const +{ + update.m_min[0] = fmin(update.m_min[0], input.m_min[0]); + update.m_min[1] = fmin(update.m_min[1], input.m_min[1]); + update.m_min[2] = fmin(update.m_min[2], input.m_min[2]); + + update.m_max[0] = fmax(update.m_max[0], input.m_max[0]); + update.m_max[1] = fmax(update.m_max[1], input.m_max[1]); + update.m_max[2] = fmax(update.m_max[2], input.m_max[2]); +} + + +template +struct MortonEncoder +{ + using execution_space = ExecutionSpace; + using value_type = int; + + using real_type = RealType; + using LBVH_types = MortonLbvhTypes; + using kokkos_aabb_types = MortonAabbTypes; + using bboxes_const_3d_view_t = typename kokkos_aabb_types::bboxes_const_3d_view_t; + using bboxes_3d_view_amt = typename kokkos_aabb_types::bboxes_3d_view_amt; + + MortonEncoder(const MortonAabbTree &tree, bool reallyEncode); + + static void apply(const MortonAabbTree &tree, bool reallyEncode = true); + + KOKKOS_INLINE_FUNCTION + void operator()(unsigned leafIdx) const; + + bboxes_const_3d_view_t m_minMaxs; + typename LBVH_types::local_ordinals_t m_idsOut; + typename LBVH_types::aabb_morton_codes_t m_codesOut; + const LocalOrdinal m_numPts; + const real_type m_xWidth; + const real_type m_yWidth; + const real_type m_zWidth; + const real_type m_globalXMin; + const real_type m_globalYMin; + const real_type m_globalZMin; + const bool m_reallyDo; +}; + +template +MortonEncoder::MortonEncoder(const MortonAabbTree &tree, + bool reallyEncode) + : m_minMaxs(tree.m_minMaxs), + m_idsOut(tree.m_leafIds), + m_codesOut(tree.m_leafCodes), + m_numPts(tree.hm_numLeaves()), + m_xWidth(tree.m_globalMaxPt[0] - tree.m_globalMinPt[0]), + m_yWidth(tree.m_globalMaxPt[1] - tree.m_globalMinPt[1]), + m_zWidth(tree.m_globalMaxPt[2] - tree.m_globalMinPt[2]), + m_globalXMin(tree.m_globalMinPt[0]), + m_globalYMin(tree.m_globalMinPt[1]), + m_globalZMin(tree.m_globalMinPt[2]), + m_reallyDo(reallyEncode) +{ +} + +template +void MortonEncoder::apply(const MortonAabbTree &tree, + bool reallyEncode) +{ + const MortonEncoder op(tree, reallyEncode); + const size_t numLeaves = tree.hm_numLeaves(); + Kokkos::parallel_for(numLeaves, op); +} + +#ifdef SMALL_MORTON // 32 bit Morton code + +template +KOKKOS_INLINE_FUNCTION +void MortonEncoder::operator()(unsigned leafIdx) const +{ + real_type ctdX = 0.5 * (m_minMaxs(leafIdx, 0) + m_minMaxs(leafIdx, 3)); + real_type ctdY = 0.5 * (m_minMaxs(leafIdx, 1) + m_minMaxs(leafIdx, 4)); + real_type ctdZ = 0.5 * (m_minMaxs(leafIdx, 2) + m_minMaxs(leafIdx, 5)); + + // std::cout << "box(" << leafIdx << ") = (" << m_minMax(leafIdx, 0) << " " + // << m_minMax(leafIdx, 1) << " " << m_minMax(leafIdx, 2) + // << ") (" << m_minMax(leafIdx, 3) << " " << m_minMax(leafIdx, 4) + // << " " << m_minMax(leafIdx, 5) << ")" << std::endl; + // std::cout << "centroid(" << leafIdx << ") = (" << ctdX << " " << ctdY << " " << ctdZ + // << ")" << std::endl; + + // for a 32-bit morton code + + morton_code_t ux, uy, uz; + + // for a 32-bit morton code, each spatial dimension gets 10 bits. + // To get the most out of each bit, we normalize the coordinates + // against 1023.0f (i.e. 2^10 - 1) + + ux = static_cast((ctdX - m_globalXMin) * 1023.0f / m_xWidth); + uy = static_cast((ctdY - m_globalYMin) * 1023.0f / m_yWidth); + uz = static_cast((ctdZ - m_globalZMin) * 1023.0f / m_zWidth); + + ux = (ux * 0x00010001u) & 0xFF0000FFu; + ux = (ux * 0x00000101u) & 0x0F00F00Fu; + ux = (ux * 0x00000011u) & 0xC30C30C3u; + ux = (ux * 0x00000005u) & 0x49249249u; + + uy = (uy * 0x00010001u) & 0xFF0000FFu; + uy = (uy * 0x00000101u) & 0x0F00F00Fu; + uy = (uy * 0x00000011u) & 0xC30C30C3u; + uy = (uy * 0x00000005u) & 0x49249249u; + + uz = (uz * 0x00010001u) & 0xFF0000FFu; + uz = (uz * 0x00000101u) & 0x0F00F00Fu; + uz = (uz * 0x00000011u) & 0xC30C30C3u; + uz = (uz * 0x00000005u) & 0x49249249u; + + m_idsOut(leafIdx) = leafIdx; + m_codesOut(leafIdx) = ux * 4 + uy * 2 + uz; +} + +#else // 64 bit Morton codes + +template +KOKKOS_INLINE_FUNCTION +void MortonEncoder::operator()(unsigned leafIdx) const +{ + m_idsOut(leafIdx) = leafIdx; + + if (m_reallyDo) { + real_type ctdX = 0.5 * (m_minMaxs(leafIdx, 0) + m_minMaxs(leafIdx, 3)); + real_type ctdY = 0.5 * (m_minMaxs(leafIdx, 1) + m_minMaxs(leafIdx, 4)); + real_type ctdZ = 0.5 * (m_minMaxs(leafIdx, 2) + m_minMaxs(leafIdx, 5)); + + // std::cout << "box(" << leafIdx << ") = (" << m_minMaxs(leafIdx, 0) << " " + // << m_minMaxs(leafIdx, 1) << " " << m_minMaxs(leafIdx, 2) + // << ") (" << m_minMaxs(leafIdx, 3) << " " << m_minMaxs(leafIdx, 4) + // << " " << m_minMaxs(leafIdx, 5) << ")" << std::endl; + // std::cout << "centroid(" << leafIdx << ") = (" << ctdX << " " << ctdY << " " << ctdZ + // << ")" << std::endl; + + // for a 64-bit morton code + + morton_code_t ux, uy, uz; + + // for a 64-bit morton code, each spatial dimension gets 21 bits. + // To get the most out of each bit, we normalize the coordinates + // against 2097151.0f (i.e. 2^21 - 1) + + ux = static_cast((ctdX - m_globalXMin) * 2097151.0f / m_xWidth); + uy = static_cast((ctdY - m_globalYMin) * 2097151.0f / m_yWidth); + uz = static_cast((ctdZ = - m_globalZMin) * 2097151.0f / m_zWidth); + + ux = (ux | ux << 32) & 0x001f00000000ffff; + ux = (ux | ux << 16) & 0x001f0000ff0000ff; + ux = (ux | ux << 8) & 0x100f00f00f00f00f; + ux = (ux | ux << 4) & 0x10c30c30c30c30c3; + ux = (ux | ux << 2) & 0x1249249249249249; + + uy = (uy | uy << 32) & 0x001f00000000ffff; + uy = (uy | uy << 16) & 0x001f0000ff0000ff; + uy = (uy | uy << 8) & 0x100f00f00f00f00f; + uy = (uy | uy << 4) & 0x10c30c30c30c30c3; + uy = (uy | uy << 2) & 0x1249249249249249; + + uz = (uz | uz << 32) & 0x001f00000000ffff; + uz = (uz | uz << 16) & 0x001f0000ff0000ff; + uz = (uz | uz << 8) & 0x100f00f00f00f00f; + uz = (uz | uz << 4) & 0x10c30c30c30c30c3; + uz = (uz | uz << 2) & 0x1249249249249249; + + m_codesOut(leafIdx) = ux * 4 + uy * 2 + uz; + } +} + +#endif // 64 bit Morton code + + +// Serial sort is the default. +template +struct SortByCodeIdPair +{ + using execution_space = ExecutionSpace; + using LBVH_types = MortonLbvhTypes; + using real_type = RealType; + + SortByCodeIdPair(const MortonAabbTree &tree); + + static void apply(const MortonAabbTree &tree, bool reallyEncode = true); + + std::vector m_buffer; + typename LBVH_types::local_ordinals_hmt hm_leafIds; + typename LBVH_types::aabb_morton_codes_hmt hm_leafCodes; +}; + +template +SortByCodeIdPair::SortByCodeIdPair(const MortonAabbTree &tree) +{ + hm_leafIds = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, tree.m_leafIds); + hm_leafCodes = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, tree.m_leafCodes); + + Kokkos::deep_copy(hm_leafCodes, tree.m_leafCodes); + Kokkos::deep_copy(hm_leafIds, tree.m_leafIds); + + LocalOrdinal numLeaves = tree.hm_numLeaves(); + m_buffer.resize(numLeaves); + + for (LocalOrdinal idx = 0; idx < numLeaves; ++idx) { + m_buffer[idx] = {hm_leafCodes(idx), hm_leafIds(idx)}; + } +} + +template +void SortByCodeIdPair::apply(const MortonAabbTree &tree, + bool reallyEncode) +{ + SortByCodeIdPair tmp(tree); + std::sort(tmp.m_buffer.begin(), tmp.m_buffer.end()); + LocalOrdinal numLeaves = tree.hm_numLeaves(); + + for (LocalOrdinal idx = 0; idx < numLeaves; ++idx) { + tmp.hm_leafCodes(idx) = tmp.m_buffer[idx].m_code; + tmp.hm_leafIds(idx) = tmp.m_buffer[idx].m_id; + } + + Kokkos::deep_copy(tree.m_leafCodes, tmp.hm_leafCodes); + Kokkos::deep_copy(tree.m_leafIds, tmp.hm_leafIds); +} + + +template +struct SortByCode +{ + using real_type = RealType; + using execution_space = ExecutionSpace; + + static void apply(const MortonAabbTree &tree) + { + SortByCodeIdPair::apply(tree); + } +}; + +#ifdef KOKKOS_ENABLE_CUDA + +#define SBS_THRUST_SORT_THRESHOLD 2048 + +template +struct SortByCode { + using real_type = RealType; + using execution_space = Kokkos::Cuda; + + static void apply(const MortonAabbTree &tree) + { + if (tree.hm_numLeaves() <= SBS_THRUST_SORT_THRESHOLD) { + SortByCodeIdPair::apply(tree); + } + else { + int n = tree.m_leafIds.extent(0); + + morton_code_t *rawLeafCodes = tree.m_leafCodes.data(); + thrust::device_ptr rawLeafCodesThr = thrust::device_pointer_cast(rawLeafCodes); + LocalOrdinal *rawLeafIds = tree.m_leafIds.data(); + thrust::device_ptr rawLeafIdsThr = thrust::device_pointer_cast(rawLeafIds); + thrust::stable_sort_by_key(rawLeafCodesThr, rawLeafCodesThr + n, rawLeafIdsThr); + } + } +}; + +#endif + + +template +struct BuildRadixTree +{ + using execution_space = ExecutionSpace; + using LBVH_types = MortonLbvhTypes; + + BuildRadixTree(const MortonAabbTree &tree); + + static void apply(const MortonAabbTree &tree); + + KOKKOS_INLINE_FUNCTION + void operator()(unsigned argIdx) const; + + KOKKOS_INLINE_FUNCTION + int leaves_cpr(LocalOrdinal baseIdx, LocalOrdinal testIdx) const; + + const LocalOrdinal m_numLeaves; + const LocalOrdinal m_numInternalNodes; + typename LBVH_types::aabb_morton_codes_tmt tm_leafCodes; + typename LBVH_types::local_ordinals_tmt tm_leafIds; + typename LBVH_types::local_ordinal_pairs_t m_nodeChildren; + typename LBVH_types::local_ordinals_t m_nodeParents; + typename LBVH_types::local_ordinals_t m_atomicFlags; +}; + +template +BuildRadixTree::BuildRadixTree(const MortonAabbTree &tree) + : m_numLeaves(tree.hm_numLeaves()), + m_numInternalNodes(tree.hm_numInternalNodes()), + tm_leafCodes(tree.m_leafCodes), + tm_leafIds(tree.m_leafIds), + m_nodeChildren(tree.m_nodeChildren), + m_nodeParents(tree.m_nodeParents), + m_atomicFlags(tree.m_atomicFlags) +{} + +template +void BuildRadixTree::apply(const MortonAabbTree &tree) +{ + if (tree.hm_numLeaves() <= 0) { + return; + } + BuildRadixTree op(tree); + Kokkos::parallel_for(static_cast(tree.hm_numInternalNodes()), op); +} + +template +KOKKOS_INLINE_FUNCTION +void BuildRadixTree::operator()(unsigned argIdx) const +{ + LocalOrdinal idx = static_cast(argIdx); + + if (idx >= m_numInternalNodes) return; + + const uint32_t numLeavesU32 = static_cast(m_numLeaves); + + // Choose direction + int prefixPrev = leaves_cpr(idx, idx - 1); + int prefixNext = leaves_cpr(idx, idx + 1); + int d = (prefixNext > prefixPrev) ? 1 : -1; + int prefixMin = AABB_MIN(prefixPrev, prefixNext); + + // Find upper bound for length. + int lmax = 128 >> 2; // Shifted back upon entrance in loop. + uint32_t probe = 0; + do { + lmax <<= static_cast(2); + probe = idx + lmax * d; + } while ((probe < numLeavesU32) && (leaves_cpr(idx, probe) > prefixMin)); + + // Determine length. + int l = 0; + for (int t = lmax >> 1; t > 0; t >>= 1) { + probe = idx + (l + t) * d; + if ((probe < numLeavesU32) && (leaves_cpr(idx, probe) > prefixMin)) l += t; + } + int j = idx + l * d; + int prefixNode = leaves_cpr(idx, j); + + // Find split point. + int s = 0; + int t = l; + do { + t = (t + 1) >> 1; + probe = idx + (s + t) * d; + if ((probe < numLeavesU32) && (leaves_cpr(idx, probe) > prefixNode)) { + s += t; + } + } while (t > 1); + int k = idx + s * d + AABB_MIN(d, 0); + + // Output node. + int lo = AABB_MIN(idx, j); + int hi = AABB_MAX(idx, j); + int cx, cy; + cx = (lo == k) ? k + 0 : (k + 0 + m_numLeaves); + cy = (hi == k + 1) ? k + 1 : (k + 1 + m_numLeaves); + + // If the children are leaves, their boxes didn't get moved in the sort! + // In that case, we need to follow the m_leaf_ids map instead of using + // (k + 0) and (k + 1) directly. + if (cx < m_numLeaves) { + cx = tm_leafIds(cx); + } + if (cy < m_numLeaves) { + cy = tm_leafIds(cy); + } + + m_nodeChildren(idx + m_numLeaves, 0) = cx; + m_nodeChildren(idx + m_numLeaves, 1) = cy; + m_nodeParents(cx) = idx + m_numLeaves; + m_nodeParents(cy) = idx + m_numLeaves; + m_atomicFlags(idx) = 0; + + if (idx == 0) { + m_nodeParents(m_numLeaves) = m_numLeaves; + } +} + +#ifdef SMALL_MORTON // 32 bit Morton + +template +KOKKOS_INLINE_FUNCTION +int leaves_cpr(LocalOrdinal baseIdx, LocalOrdinal testIdx) const +{ + if (testIdx < 0 || testIdx >= m_numLeaves) { + return -1; + } + typename LBVH_types::aabb_morton_codes_tmt::value_type xorResult = + tm_leafCodes(baseIdx) ^ tm_leafCodes(testIdx); + if (xorResult != 0) { + return AABB_CLZ(xorResult); + } + else { + return 32 + AABB_CLZ(tm_leafIds(baseIdx) ^ tm_leafIds(testIdx)); + } +} + +#else // 64 bit Morton + +template +KOKKOS_INLINE_FUNCTION +int BuildRadixTree::leaves_cpr(LocalOrdinal baseIdx, LocalOrdinal testIdx) const +{ + if (testIdx < 0 || testIdx >= m_numLeaves) { + return -1; + } + typename LBVH_types::aabb_morton_codes_t::value_type xorResult = tm_leafCodes(baseIdx) ^ tm_leafCodes(testIdx); + if (xorResult != 0) { + return AABB_CLZLL(xorResult); + } + else { + return 64 + AABB_CLZ(tm_leafIds(baseIdx) ^ tm_leafIds(testIdx)); + } +} + +#endif // 64 bit Morton + + +template +struct UpdateInteriorNodeBVs +{ + using real_type = RealType; + using execution_space = ExecutionSpace; + using LBVH_types = MortonLbvhTypes; + using kokkos_aabb_types = MortonAabbTypes; + using bboxes_const_3d_view_t = typename kokkos_aabb_types::bboxes_const_3d_view_t; + + UpdateInteriorNodeBVs(const MortonAabbTree &tree); + + static void apply(const MortonAabbTree &tree); + + KOKKOS_INLINE_FUNCTION + void operator()(unsigned argIdx) const; + + template + KOKKOS_FORCEINLINE_FUNCTION + void get_box(real_type bvMinMax[6], LocalOrdinal idx, const BBox3dViewType &boxesMinMax) const; + + const LocalOrdinal m_numLeaves; + const LocalOrdinal m_numInternalNodes; + typename LBVH_types::local_ordinal_pairs_tmt tm_nodeChildren; + typename LBVH_types::local_ordinals_tmt tm_nodeParents; + bboxes_const_3d_view_t m_leafMinMaxs; + + // Will write to internal nodes bounding boxes. + typename kokkos_aabb_types::bboxes_3d_view_amt m_nodeMinMaxs; + typename LBVH_types::local_ordinals_t m_atomicFlags; +}; + +template +UpdateInteriorNodeBVs::UpdateInteriorNodeBVs(const MortonAabbTree &tree) + : m_numLeaves(tree.hm_numLeaves()), + m_numInternalNodes(tree.hm_numInternalNodes()), + tm_nodeChildren(tree.m_nodeChildren), + tm_nodeParents(tree.m_nodeParents), + m_leafMinMaxs(tree.m_minMaxs), + m_nodeMinMaxs(tree.m_minMaxs), + m_atomicFlags(tree.m_atomicFlags) +{} + +template +void UpdateInteriorNodeBVs::apply(const MortonAabbTree &tree) +{ + const UpdateInteriorNodeBVs op(tree); + const size_t numLeaves = tree.hm_numLeaves(); + + Kokkos::parallel_for(numLeaves, op); +} + +template +KOKKOS_INLINE_FUNCTION +void UpdateInteriorNodeBVs::operator()(unsigned argIdx) const +{ + if (m_numLeaves > 1) { + LocalOrdinal idx = static_cast(argIdx); + + real_type bvMinMax[6]; + get_box(bvMinMax, idx, m_leafMinMaxs); + + LocalOrdinal parent = tm_nodeParents(idx); + real_type sibMinMax[6]; + + while (Kokkos::atomic_fetch_add(&m_atomicFlags(parent - m_numLeaves), 1) == 1) { + LocalOrdinal sib = tm_nodeChildren(parent, 0); + if (sib == idx) { + sib = tm_nodeChildren(parent, 1); + } + + get_box(sibMinMax, sib, m_nodeMinMaxs); + + for (LocalOrdinal j = 0; j < 3; ++j) { + bvMinMax[j] = AABB_MIN(bvMinMax[j], sibMinMax[j]); + m_nodeMinMaxs(parent, j) = bvMinMax[j]; + } + for (LocalOrdinal j = 3; j < 6; ++j) { + bvMinMax[j] = AABB_MAX(bvMinMax[j], sibMinMax[j]); + m_nodeMinMaxs(parent, j) = bvMinMax[j]; + } + + idx = parent; + parent = tm_nodeParents(parent); + if (idx == parent) { + return; + } + } + } +} + +template +template +KOKKOS_FORCEINLINE_FUNCTION +void UpdateInteriorNodeBVs::get_box(real_type bvMinMax[6], LocalOrdinal idx, + const BBox3dViewType &boxMinMaxs) const +{ + for (LocalOrdinal j = 0; j < 6; ++j) { + bvMinMax[j] = boxMinMaxs(idx, j); + } +} + + +template +struct Traverse_MASTB_BVH_Functor +{ + using execution_space = ExecutionSpace; + using value_type = int; + using real_type = RealType; + using LBVH_types = MortonLbvhTypes; + using kokkos_aabb_types = MortonAabbTypes; + using local_ordinals_tmt = typename LBVH_types::local_ordinals_tmt; + using bboxes_3d_view_t = typename kokkos_aabb_types::bboxes_3d_view_t; + using bboxes_const_3d_view_t = typename kokkos_aabb_types::bboxes_const_3d_view_t; + using collision_list_type = CollisionList; + + Traverse_MASTB_BVH_Functor(bboxes_3d_view_t domainMinMaxs, + local_ordinals_tmt domainIds, + const MortonAabbTree &rangeTree, + collision_list_type &collisions, + bool flippedResults = false); + + KOKKOS_INLINE_FUNCTION + void init(value_type &update) const { update = 0; } + + static void apply_tree(const MortonAabbTree &domainTree, + const MortonAabbTree &rangeTree, + collision_list_type &collisions, + bool flipOutputPairs = false); + + KOKKOS_INLINE_FUNCTION + void operator()(unsigned domainIdx, value_type &update) const; + + KOKKOS_FORCEINLINE_FUNCTION + bool overlaps_range(real_type bvMinMax[6], LocalOrdinal rangeIdx) const; + + KOKKOS_FORCEINLINE_FUNCTION + bool is_range_leaf(LocalOrdinal rangeIdx) const{ return (rangeIdx < m_rangeRoot); } + + KOKKOS_FORCEINLINE_FUNCTION + void get_box(real_type bvMinMax[6], LocalOrdinal idx, const bboxes_const_3d_view_t &boxMinMaxs) const; + + KOKKOS_INLINE_FUNCTION + void join(value_type &update, const value_type &input) const { update = (input < update ? input : update); } + + std::ostream &stream_pair(LocalOrdinal domainIdx, bool overlap, LocalOrdinal rangeIdx, std::ostream &os) const; + + bboxes_const_3d_view_t m_domainMinMaxs; + typename LBVH_types::local_ordinals_tmt tm_domainIds; + + const LocalOrdinal m_rangeRoot; + bboxes_const_3d_view_t tm_rangeMinMaxs; + typename LBVH_types::local_ordinal_pairs_tmt tm_rangeNodeChildren; + typename LBVH_types::local_ordinals_tmt tm_rangeLeafIds; + + const bool m_flippedResults; + collision_list_type m_results; +}; + +template +Traverse_MASTB_BVH_Functor::Traverse_MASTB_BVH_Functor( + bboxes_3d_view_t domainMinMaxs, + local_ordinals_tmt domainIds, + const MortonAabbTree &rangeTree, + collision_list_type &collisions, + bool flippedResults) + : m_domainMinMaxs(domainMinMaxs), + tm_domainIds(domainIds), + m_rangeRoot(rangeTree.hm_numLeaves()), + tm_rangeMinMaxs(rangeTree.m_minMaxs), + tm_rangeNodeChildren(rangeTree.m_nodeChildren), + tm_rangeLeafIds(rangeTree.m_leafIds), + m_flippedResults(flippedResults), + m_results(collisions) +{} + +template +void Traverse_MASTB_BVH_Functor::apply_tree( + const MortonAabbTree &domainTree, + const MortonAabbTree &rangeTree, + collision_list_type &collisions, + bool flipOutputPairs) +{ + if ((domainTree.hm_numLeaves() == 0) || (rangeTree.hm_numLeaves() == 0)) { + return; + } + + int retCode = 0; + const int numDomainLeaves = domainTree.hm_numLeaves(); + const int numRangeLeaves = rangeTree.hm_numLeaves(); + + if (collisions.get_capacity() == 0) { + const int collisionEstimate = std::max(numDomainLeaves, numRangeLeaves) * COLLISION_SCALE_FACTOR; + collisions.reset(collisionEstimate); + } + + const Traverse_MASTB_BVH_Functor op(domainTree.m_minMaxs, domainTree.m_leafIds, rangeTree, + collisions, flipOutputPairs); + Kokkos::parallel_reduce(numDomainLeaves, op, retCode); + + int numActualCollisions = collisions.get_num_collisions(); + + if ((retCode < 0) && (numActualCollisions > collisions.get_capacity())) { + collisions.reset(numActualCollisions); + retCode = 0; + const Traverse_MASTB_BVH_Functor op2(domainTree.m_minMaxs, domainTree.m_leafIds, rangeTree, + collisions, flipOutputPairs); + Kokkos::parallel_reduce(numDomainLeaves, op2, retCode); + } +} + +template +KOKKOS_INLINE_FUNCTION void Traverse_MASTB_BVH_Functor::operator()(unsigned argDomainIdx, + value_type& update) const +{ + LocalOrdinal domainIdx = tm_domainIds(argDomainIdx); + + real_type bvMinMax[6]; + get_box(bvMinMax, domainIdx, m_domainMinMaxs); + + if (m_rangeRoot > 1) { + int ridxStack[64]; + int* stackPtr = ridxStack; + *stackPtr++ = -1; + + int result = 0; + int nodeIdx = m_rangeRoot; + do { + // Check each child node for overlap. + const int childL = tm_rangeNodeChildren(nodeIdx, 0); + const int childR = tm_rangeNodeChildren(nodeIdx, 1); + const bool overlapL = overlaps_range(bvMinMax, childL); + const bool overlapR = overlaps_range(bvMinMax, childR); + + bool traverseL = false; + + // Query overlaps a leaf node => report collision. + if (overlapL) { + if (is_range_leaf(childL)) { + result = m_results.push_back(domainIdx, childL, m_flippedResults) ? result : -1; + } + else { + traverseL = true; + nodeIdx = childL; + } + } + + // Query overlaps and internal node => traverse. + if (overlapR) { + if (is_range_leaf(childR)) { + result = m_results.push_back(domainIdx, childR, m_flippedResults) ? result : -1; + if (!traverseL) { + nodeIdx = *--stackPtr; // pop + } + } + else { + if (traverseL) { + *stackPtr++ = childR; // push + } + else { + nodeIdx = childR; + } + } + } + else if (!traverseL) { + nodeIdx = *--stackPtr; // pop + } + } while (nodeIdx >= 0); + + if (result < update) { + update = result; + } + } + else { + // Degenerate case of only one leaf node + int result = 0; + bool overlap = overlaps_range(bvMinMax, 0); + if (overlap) { + bool ok = m_results.push_back(domainIdx, 0, m_flippedResults); + result = (ok ? result : -1); + } + + if (result < update) { + update = result; + } + } +} + +template +KOKKOS_FORCEINLINE_FUNCTION +bool Traverse_MASTB_BVH_Functor::overlaps_range(real_type bvMinMax[6], + LocalOrdinal rangeIdx) const +{ + return (bvMinMax[3] < tm_rangeMinMaxs(rangeIdx, 0) || + bvMinMax[4] < tm_rangeMinMaxs(rangeIdx, 1) || + bvMinMax[5] < tm_rangeMinMaxs(rangeIdx, 2) || + bvMinMax[0] > tm_rangeMinMaxs(rangeIdx, 3) || + bvMinMax[1] > tm_rangeMinMaxs(rangeIdx, 4) || + bvMinMax[2] > tm_rangeMinMaxs(rangeIdx, 5)) ? false : true; +} + +template +KOKKOS_FORCEINLINE_FUNCTION +void Traverse_MASTB_BVH_Functor::get_box(real_type bvMinMax[6], LocalOrdinal idx, + const bboxes_const_3d_view_t &boxMinMaxs) const +{ + bvMinMax[0] = boxMinMaxs(idx, 0); + bvMinMax[1] = boxMinMaxs(idx, 1); + bvMinMax[2] = boxMinMaxs(idx, 2); + bvMinMax[3] = boxMinMaxs(idx, 3); + bvMinMax[4] = boxMinMaxs(idx, 4); + bvMinMax[5] = boxMinMaxs(idx, 5); +} + +template +std::ostream &Traverse_MASTB_BVH_Functor::stream_pair(LocalOrdinal domainIdx, bool overlap, + LocalOrdinal rangeIdx, std::ostream &os) const +{ + os << " {(" << m_domainMinMaxs(domainIdx, 0) << "," << m_domainMinMaxs(domainIdx, 1) << "," << m_domainMinMaxs(domainIdx, 2) + << ") (" << m_domainMinMaxs(domainIdx, 3) << "," << m_domainMinMaxs(domainIdx, 4) << "," << m_domainMinMaxs(domainIdx, 5) + << ")}"; + os << (overlap ? " overlaps " : " does not overlap "); + os << " {(" << tm_rangeMinMaxs(rangeIdx, 0) << "," << tm_rangeMinMaxs(rangeIdx, 1) << "," << tm_rangeMinMaxs(rangeIdx, 2) + << ") (" << tm_rangeMinMaxs(rangeIdx, 3) << "," << tm_rangeMinMaxs(rangeIdx, 4) << "," << tm_rangeMinMaxs(rangeIdx, 5) + << ")}"; + os << std::endl; + return os; +} + +} + +#endif // MORTONLBVH_TREEMANIPULATIONUTILS_HPP diff --git a/packages/stk/stk_search_util/stk_search_util/PeriodicBoundarySearch.hpp b/packages/stk/stk_search_util/stk_search_util/PeriodicBoundarySearch.hpp index 91fc3eb76fe8..448ff775dae2 100644 --- a/packages/stk/stk_search_util/stk_search_util/PeriodicBoundarySearch.hpp +++ b/packages/stk/stk_search_util/stk_search_util/PeriodicBoundarySearch.hpp @@ -622,9 +622,9 @@ class PeriodicBoundarySearch { for (auto && side_1 : side_1_vector) { - double *center = &side_1.first.center()[0]; + double *center = &(side_1.first.center()[0]); std::vector ctr(center, center+3); - rotation.transformVec(&ctr[0], center); + rotation.transformVec(ctr.data(), center); } } diff --git a/packages/stk/stk_simd/Jamfile b/packages/stk/stk_simd/Jamfile index 7662ec7146a5..7ff831f8cdfe 100644 --- a/packages/stk/stk_simd/Jamfile +++ b/packages/stk/stk_simd/Jamfile @@ -144,7 +144,7 @@ exe stk_simd_unit [ glob $(stk_simd-root)/../stk_unit_tests/stk_simd/*.cpp ] /sierra/stk_unit_test_utils//stk_unit_main /sierra/stk_math//stk_math - /tpl/gtest//gtest + /tpl/googletest//gtest : @sierra-exec-tag ; @@ -155,7 +155,7 @@ exe stk_simd_old_unit /sierra/stk_unit_test_utils//stk_unit_test_utils /sierra/stk_unit_test_utils//stk_unit_main /tpl/trilinos//kokkoscore - /tpl/gtest//gtest + /tpl/googletest//gtest : @sierra-exec-tag $(export-dynamic) ; @@ -166,7 +166,7 @@ exe stk_simd_docs_tests /sierra/stk_math//stk_math /sierra/stk_unit_test_utils//stk_unit_main /tpl/trilinos//kokkoscore - /tpl/gtest//gtest + /tpl/googletest//gtest : @sierra-exec-tag $(stk_simd-root)/../stk_doc_tests/stk_simd $(export-dynamic) diff --git a/packages/stk/stk_simd/stk_simd_view/simd_index.hpp b/packages/stk/stk_simd/stk_simd_view/simd_index.hpp index 2a5918a26f9f..3248e16ade54 100644 --- a/packages/stk/stk_simd/stk_simd_view/simd_index.hpp +++ b/packages/stk/stk_simd/stk_simd_view/simd_index.hpp @@ -65,7 +65,7 @@ struct IndexTraits { typedef float float_type; }; -#if defined(KOKKOS_ENABLE_CUDA) || defined(USE_STK_SIMD_NONE) +#if defined(STK_ENABLE_GPU) || defined(USE_STK_SIMD_NONE) typedef int DeviceIndex; template diff --git a/packages/stk/stk_simd/stk_simd_view/simd_parallel.hpp b/packages/stk/stk_simd/stk_simd_view/simd_parallel.hpp index c699077dd2d4..b4b7ce1581b6 100644 --- a/packages/stk/stk_simd/stk_simd_view/simd_parallel.hpp +++ b/packages/stk/stk_simd/stk_simd_view/simd_parallel.hpp @@ -65,7 +65,7 @@ struct DeduceFunctorExecutionSpace { template KOKKOS_INLINE_FUNCTION constexpr bool is_gpu() { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) +#ifdef STK_ENABLE_GPU using execution_space = typename internal::DeduceFunctorExecutionSpace::execution_space; return std::is_same::value; #else @@ -79,9 +79,7 @@ int get_simd_loop_size(int N) { return is_gpu() ? N : simd_pad(N) / SimdSizeTraits::simd_width; } - -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - +#ifdef STK_ENABLE_GPU template inline void parallel_for(std::string forName, const stk::ngp::RangePolicy& range, const Func& func) { diff --git a/packages/stk/stk_tools/Jamfile b/packages/stk/stk_tools/Jamfile index 7dbc11562407..d795a513187e 100644 --- a/packages/stk/stk_tools/Jamfile +++ b/packages/stk/stk_tools/Jamfile @@ -290,7 +290,7 @@ exe stk_tools_pmesh_unit_tester $(stk_tools-root)/stk_tools/pmesh_lib/UnitTest/*.cpp ] stk_pmesh_lib - /tpl/gtest//gtest + /tpl/googletest//gtest : @sierra-exec-tag ; diff --git a/packages/stk/stk_tools/cmake/Dependencies.cmake b/packages/stk/stk_tools/cmake/Dependencies.cmake index 735cc1d49c5c..fd73456e747d 100644 --- a/packages/stk/stk_tools/cmake/Dependencies.cmake +++ b/packages/stk/stk_tools/cmake/Dependencies.cmake @@ -1,5 +1,5 @@ SET(LIB_REQUIRED_DEP_PACKAGES STKIO STKMesh STKUtil STKTransfer STKTopology) -SET(LIB_OPTIONAL_DEP_PACKAGES) +SET(LIB_OPTIONAL_DEP_PACKAGES SEACASNemesis) SET(TEST_REQUIRED_DEP_PACKAGES Gtest) SET(TEST_OPTIONAL_DEP_PACKAGES) SET(LIB_REQUIRED_DEP_TPLS) diff --git a/packages/stk/stk_tools/stk_tools/CMakeLists.txt b/packages/stk/stk_tools/stk_tools/CMakeLists.txt index 36bbbb3cba4c..06f1b05a84d4 100644 --- a/packages/stk/stk_tools/stk_tools/CMakeLists.txt +++ b/packages/stk/stk_tools/stk_tools/CMakeLists.txt @@ -44,6 +44,8 @@ FILE(GLOB BLOCK_EXTRACTOR_HEADERS block_extractor/*.hpp) FILE(GLOB BLOCK_EXTRACTOR_SOURCES block_extractor/*.cpp) FILE(GLOB TRANSFER_UTILS_HEADERS transfer_utils/*.hpp) FILE(GLOB TRANSFER_UTILS_SOURCES transfer_utils/*.cpp) +FILE(GLOB PMESH_LIB_HEADERS pmesh_lib/*.hpp) +FILE(GLOB PMESH_LIB_SOURCES pmesh_lib/*.cpp) LIST(REMOVE_ITEM BLOCK_EXTRACTOR_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/block_extractor/main.cpp) # @@ -67,6 +69,12 @@ if(HAVE_STK_Trilinos) SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/block_extractor/main.cpp COMM serial mpi ) + + TRIBITS_ADD_LIBRARY(stk_pmesh_lib + NOINSTALLHEADERS ${PMESH_LIB_HEADERS} + SOURCES ${PMESH_LIB_SOURCES} + ) + else() add_library(stk_transfer_utils_lib ${TRANSFER_UTILS_SOURCES}) target_link_libraries(stk_transfer_utils_lib PUBLIC stk_io) @@ -79,6 +87,14 @@ else() add_executable(stk_block_extractor ${CMAKE_CURRENT_SOURCE_DIR}/block_extractor/main.cpp) target_link_libraries(stk_block_extractor PUBLIC stk_tools_lib) target_link_libraries(stk_block_extractor PUBLIC stk_util_command_line) + + if(STK_HAS_SEACAS_NEMESIS) + add_library(stk_pmesh_lib ${TRANSFER_UTILS_SOURCES}) + target_link_libraries(stk_pmesh_lib PUBLIC stk_io) + target_link_libraries(stk_pmesh_lib PUBLIC stk_io_util) + target_link_libraries(stk_pmesh_lib PUBLIC stk_tools_lib) + target_link_libraries(stk_pmesh_lib PUBLIC SEACASNemesis::nemesis) + endif() endif() target_include_directories(stk_transfer_utils_lib PUBLIC @@ -89,6 +105,12 @@ target_include_directories(stk_tools_lib PUBLIC $ $ ) +if(STK_HAS_SEACAS_NEMESIS) + target_include_directories(stk_pmesh_lib PUBLIC + $ + $ + ) +endif() INSTALL(FILES ${MESH_CLONE_HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_tools/mesh_clone) @@ -102,11 +124,19 @@ INSTALL(FILES ${BLOCK_EXTRACTOR_HEADERS} DESTINATION INSTALL(FILES ${TRANSFER_UTILS_HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_tools/transfer_utils) +if (STK_HAS_SEACAS_NEMESIS) + INSTALL(FILES ${PMESH_LIB_HEADERS} DESTINATION + ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_tools/pmesh_lib) +endif() + INSTALL(TARGETS stk_block_extractor COMPONENT ${PACKAGE_NAME} RUNTIME DESTINATION ${${PROJECT_NAME}_INSTALL_RUNTIME_DIR}) if(NOT HAVE_STK_Trilinos) INSTALL(TARGETS stk_transfer_utils_lib DESTINATION ${STK_INSTALL_LIBDIR}) INSTALL(TARGETS stk_tools_lib DESTINATION ${STK_INSTALL_LIBDIR}) INSTALL(TARGETS stk_block_extractor DESTINATION ${STK_INSTALL_BINDIR}) + if (STK_HAS_SEACAS_NEMESIS) + INSTALL(TARGETS stk_pmesh_lib DESTINATION ${STK_INSTALL_LIBDIR}) + endif() endif() diff --git a/packages/stk/stk_tools/stk_tools/mesh_tools/DetectHingesImpl.cpp b/packages/stk/stk_tools/stk_tools/mesh_tools/DetectHingesImpl.cpp index 2bbe32c9b592..7fc1773d64d5 100644 --- a/packages/stk/stk_tools/stk_tools/mesh_tools/DetectHingesImpl.cpp +++ b/packages/stk/stk_tools/stk_tools/mesh_tools/DetectHingesImpl.cpp @@ -323,6 +323,7 @@ stk::mesh::EntityVector get_mesh_nodes(const stk::mesh::BulkData& bulk, const st if(parts.empty()) { selector = meta.universal_part(); } + selector &= (meta.locally_owned_part() | meta.globally_shared_part()); stk::mesh::get_entities(bulk, stk::topology::NODE_RANK, selector, nodes); diff --git a/packages/stk/stk_transfer/stk_transfer/CMakeLists.txt b/packages/stk/stk_transfer/stk_transfer/CMakeLists.txt index f663facae058..e4809a4297f7 100644 --- a/packages/stk/stk_transfer/stk_transfer/CMakeLists.txt +++ b/packages/stk/stk_transfer/stk_transfer/CMakeLists.txt @@ -27,6 +27,9 @@ else() target_link_libraries(stk_transfer PUBLIC stk_search) target_link_libraries(stk_transfer PUBLIC stk_util_env) target_link_libraries(stk_transfer PUBLIC stk_util_parallel) + if (STK_ENABLE_STKMiddle_mesh) + target_link_libraries(stk_transfer PUBLIC stk_middle_mesh) + endif() endif() target_include_directories(stk_transfer PUBLIC diff --git a/packages/stk/stk_transfer/stk_transfer/TransferUtil.cpp b/packages/stk/stk_transfer/stk_transfer/TransferUtil.cpp index ec43eecd4d4e..956aab09f59c 100644 --- a/packages/stk/stk_transfer/stk_transfer/TransferUtil.cpp +++ b/packages/stk/stk_transfer/stk_transfer/TransferUtil.cpp @@ -74,7 +74,7 @@ void exchange_transfer_ids(ReducedDependencyCommData& comm_data) } std::vector receiveStati(receiveRequests.size()); - MPI_Waitall(receiveRequests.size(), &receiveRequests[0], &receiveStati[0] ); + MPI_Waitall(receiveRequests.size(), receiveRequests.data(), receiveStati.data()); if (stk::util::get_common_coupling_version() >= 10) { comm_data.m_otherTransferId.resize(comm_data.numToMeshCommunications); @@ -83,7 +83,7 @@ void exchange_transfer_ids(ReducedDependencyCommData& comm_data) } } - MPI_Waitall(sendRequests.size(), &sendRequests[0], MPI_STATUSES_IGNORE); + MPI_Waitall(sendRequests.size(), sendRequests.data(), MPI_STATUSES_IGNORE); } } diff --git a/packages/stk/stk_transfer/stk_transfer/copy_by_id/TransferCopyTranslator.hpp b/packages/stk/stk_transfer/stk_transfer/copy_by_id/TransferCopyTranslator.hpp index 8ee1741139d2..02edd92df1c3 100644 --- a/packages/stk/stk_transfer/stk_transfer/copy_by_id/TransferCopyTranslator.hpp +++ b/packages/stk/stk_transfer/stk_transfer/copy_by_id/TransferCopyTranslator.hpp @@ -37,6 +37,7 @@ #include #include "stk_util/util/ReportHandler.hpp" // for ThrowAssert, etc +#include // for std::function namespace stk { namespace transfer { diff --git a/packages/stk/stk_transfer_util/stk_transfer_util/CMakeLists.txt b/packages/stk/stk_transfer_util/stk_transfer_util/CMakeLists.txt index c9fcd1feaecc..82787aa135ad 100644 --- a/packages/stk/stk_transfer_util/stk_transfer_util/CMakeLists.txt +++ b/packages/stk/stk_transfer_util/stk_transfer_util/CMakeLists.txt @@ -51,6 +51,9 @@ else() target_link_libraries(stk_transfer_util PUBLIC stk_search_util) target_link_libraries(stk_transfer_util PUBLIC stk_util_env) + find_package(LAPACK REQUIRED) + target_link_libraries(stk_transfer_util PUBLIC LAPACK::LAPACK) + target_include_directories(stk_transfer_util PUBLIC $ $ diff --git a/packages/stk/stk_transfer_util/stk_transfer_util/EntityCentroidRecoverField.cpp b/packages/stk/stk_transfer_util/stk_transfer_util/EntityCentroidRecoverField.cpp index 7f4c695c2026..dd9d39d39dc8 100644 --- a/packages/stk/stk_transfer_util/stk_transfer_util/EntityCentroidRecoverField.cpp +++ b/packages/stk/stk_transfer_util/stk_transfer_util/EntityCentroidRecoverField.cpp @@ -127,7 +127,7 @@ void EntityCentroidLinearRecoverField::sample_patch(const std::vector // for vector #include -#include "stk_mesh/base/FieldBase.hpp" // for FieldState, StateNP1, StateNM1 #include "stk_mesh/base/Types.hpp" // for EntityRank -#include "stk_transfer_util/Patch.hpp" #include #include "stk_mesh/base/BulkData.hpp" // for BulkData #include "stk_mesh/base/Entity.hpp" // for Entity @@ -59,12 +57,8 @@ #include "stk_transfer_util/Patch.hpp" #include "stk_util/util/ReportHandler.hpp" // for ThrowRequireMsg -namespace stk { namespace mesh { class BulkData; } } -namespace stk { namespace mesh { class FieldBase; } } -namespace stk { namespace mesh { class MetaData; } } namespace stk { namespace mesh { class Part; } } namespace stk { namespace mesh { class Selector; } } -namespace stk { namespace mesh { struct Entity; } } // clang-format on // ####################### End Clang Header Tool Managed Headers ######################## diff --git a/packages/stk/stk_unit_test_utils/Jamfile b/packages/stk/stk_unit_test_utils/Jamfile index fe33a5b86f1d..beec402a2f11 100644 --- a/packages/stk/stk_unit_test_utils/Jamfile +++ b/packages/stk/stk_unit_test_utils/Jamfile @@ -149,8 +149,8 @@ lib stk_unit_test_utils /sierra/stk_topology//stk_topology /sierra/stk_mesh//stk_mesh_base /sierra/stk_balance//stk_balance_lib - /tpl/gtest//gtest - /tpl/gtest//gmock + /tpl/googletest//gtest + /tpl/googletest//gmock ] : [ ifuserbuild diff --git a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/CMakeLists.txt b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/CMakeLists.txt index c210867b680e..a2383f7ebf31 100644 --- a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/CMakeLists.txt +++ b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/CMakeLists.txt @@ -139,11 +139,17 @@ else() target_link_libraries(stk_unit_test_utils PUBLIC GTest::gtest) endif() +target_include_directories(stk_unit_test_utils PUBLIC + $ + $ +) + target_include_directories(stk_unit_test_utils PUBLIC $ $ ) + if (NOT DEFINED STK_ENABLE_UnitMain) set(STK_ENABLE_UnitMain ON) endif() diff --git a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/MeshUtilsForBoundingVolumes.hpp b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/MeshUtilsForBoundingVolumes.hpp index 8787982b927f..42c08cf6d07f 100644 --- a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/MeshUtilsForBoundingVolumes.hpp +++ b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/MeshUtilsForBoundingVolumes.hpp @@ -52,7 +52,7 @@ using stk::unit_test_util::build_mesh; inline void findBoundingBoxCoordinates(const std::vector &coordinates, std::vector& boxCoordinates) { int spatialDim = 3; - double *minCoordinates = &boxCoordinates[0]; + double *minCoordinates = boxCoordinates.data(); double *maxCoordinates = &boxCoordinates[spatialDim]; for (int j=0;j& b double zmax = boxes[i].get_z_max(); double hexCoordinates[24]; - setHexCoordinates(xmin, ymin, zmin, xmax, ymax, zmax, &hexCoordinates[0]); + setHexCoordinates(xmin, ymin, zmin, xmax, ymax, zmax, hexCoordinates); unsigned offset = i*num_nodes_per_element; for (int j=0; j& boxes, const } offset += num_elements_this_block*num_nodes_per_elem; - ex_put_elem_conn(exoid, blockId, &connect[0]); + ex_put_elem_conn(exoid, blockId, connect.data()); } ex_close(exoid); @@ -444,7 +444,7 @@ namespace simple_fields { inline void findBoundingBoxCoordinates(const std::vector &coordinates, std::vector& boxCoordinates) { int spatialDim = 3; - double *minCoordinates = &boxCoordinates[0]; + double *minCoordinates = boxCoordinates.data(); double *maxCoordinates = &boxCoordinates[spatialDim]; for (int j=0;j& b double zmax = boxes[i].get_z_max(); double hexCoordinates[24]; - setHexCoordinates(xmin, ymin, zmin, xmax, ymax, zmax, &hexCoordinates[0]); + setHexCoordinates(xmin, ymin, zmin, xmax, ymax, zmax, hexCoordinates); unsigned offset = i*num_nodes_per_element; for (int j=0; j& boxes, const } offset += num_elements_this_block*num_nodes_per_elem; - ex_put_elem_conn(exoid, blockId, &connect[0]); + ex_put_elem_conn(exoid, blockId, connect.data()); } ex_close(exoid); diff --git a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/Search_UnitTestUtils.hpp b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/Search_UnitTestUtils.hpp index 7c34bdf13e46..4bdb49d8a0da 100644 --- a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/Search_UnitTestUtils.hpp +++ b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/Search_UnitTestUtils.hpp @@ -40,6 +40,7 @@ #include #include +#include #include @@ -233,6 +234,56 @@ std::pair generateBoundingVolume(double x, double y, double z return std::make_pair(generateBoundingVolume(x,y,z,radius), Ident(id,proc)); } + +template +KOKKOS_FUNCTION +BoxType device_generateBox(double x, double y, double z, double radius); + +template<> +KOKKOS_INLINE_FUNCTION +Point device_generateBox(double x, double y, double z, double /*radius*/) +{ + return Point(x,y,z); +} + +template<> +KOKKOS_INLINE_FUNCTION +Sphere device_generateBox(double x, double y, double z, double radius) +{ + return Sphere(Point(x, y, z), radius); +} + +template<> +KOKKOS_INLINE_FUNCTION +StkBox device_generateBox(double x, double y, double z, double radius) +{ + Point min_corner(x-radius, y-radius, z-radius); + Point max_corner(x+radius, y+radius, z+radius); + return StkBox(min_corner, max_corner); +} + +template +KOKKOS_FUNCTION +stk::search::BoxIdentProc device_generateBoxIdentProc(double x, double y, double z, + double radius, int id, int proc) +{ + return stk::search::BoxIdentProc{device_generateBox(x, y, z, radius), + IdentProcType{id, proc}}; +} + +template +KOKKOS_FUNCTION +stk::search::BoxIdent device_generateBoxIdent(double x, double y, double z, + double radius, IdentType id) +{ + return stk::search::BoxIdent{device_generateBox(x, y, z, radius), id}; +} + +template +auto box_ident_to_pair(BoxIdent const& ident) { + return std::make_pair(ident.box, ident.ident); +} + //====================== inline size_t getGoldValueForTest() diff --git a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/TextMeshSidesetSkinner.hpp b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/TextMeshSidesetSkinner.hpp index e72d8adde67e..601b951b108a 100644 --- a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/TextMeshSidesetSkinner.hpp +++ b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/TextMeshSidesetSkinner.hpp @@ -24,6 +24,7 @@ #include #include #include +#include // for assert #include "TextMeshFuncs.hpp" #include "TextMeshDataTypes.hpp" diff --git a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/exampleMeshes.h b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/exampleMeshes.h index 6648698760fa..46fb91ba91ad 100644 --- a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/exampleMeshes.h +++ b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/exampleMeshes.h @@ -13,9 +13,9 @@ namespace exampleMeshes inline void fillDataForUnitCube(std::vector &coordinates) { // Nodes in Exodus Ordering for hex elements which is same as Flanagan/Belytschko paper - double *x = &coordinates[0]; - double *y = &coordinates[8]; - double *z = &coordinates[16]; + double *x = coordinates.data(); + double *y = coordinates.data() + 8; + double *z = coordinates.data() + 16; x[4] = 1; x[5] = 1; x[6] = 1; x[7] = 1; y[1] = 1; y[2] = 1; y[5] = 1; y[6] = 1; z[2] = 1; z[3] = 1; z[6] = 1; z[7] = 1; @@ -24,9 +24,9 @@ inline void fillDataForUnitCube(std::vector &coordinates) inline void fillDataForRectangloid(std::vector &coordinates) { // Nodes in Exodus Ordering for hex elements which is same as Flanagan/Belytschko paper - double *x = &coordinates[0]; - double *y = &coordinates[8]; - double *z = &coordinates[16]; + double *x = coordinates.data(); + double *y = coordinates.data() + 8; + double *z = coordinates.data() + 16; x[4] = 3; x[5] = 3; x[6] = 3; x[7] = 3; y[1] = 2; y[2] = 2; y[5] = 2; y[6] = 2; z[2] = 0.5; z[3] = 0.5; z[6] = 0.5; z[7] = 0.5; diff --git a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/meshCreationHelpers.cpp b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/meshCreationHelpers.cpp index 0ef6c3a134e0..f363ef16a57f 100644 --- a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/meshCreationHelpers.cpp +++ b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/meshCreationHelpers.cpp @@ -32,9 +32,9 @@ size_t write_mesh_data__field_1__field_2__field_3(const std::string & filename, stk::mesh::Field & field3 = stk::mesh::legacy::declare_field>(meta, stk::topology::ELEM_RANK, "field_3", 1); double fieldValues[] = {1.0, 2.0, 3.0}; - stk::mesh::put_field_on_mesh(field1, meta.universal_part(), &fieldValues[0]); - stk::mesh::put_field_on_mesh(field2, meta.universal_part(), &fieldValues[1]); - stk::mesh::put_field_on_mesh(field3, meta.universal_part(), &fieldValues[2]); + stk::mesh::put_field_on_mesh(field1, meta.universal_part(), fieldValues); + stk::mesh::put_field_on_mesh(field2, meta.universal_part(), fieldValues + 1); + stk::mesh::put_field_on_mesh(field3, meta.universal_part(), fieldValues + 2); stkIo.populate_bulk_data(); size_t results_output_index = stkIo.create_output_mesh(filename, stk::io::WRITE_RESULTS); @@ -79,9 +79,9 @@ size_t write_mesh_data__field_1__field_2__field_3(const std::string & filename, stk::mesh::Field & field3 = meta.declare_field(stk::topology::ELEM_RANK, "field_3", 1); double fieldValues[] = {1.0, 2.0, 3.0}; - stk::mesh::put_field_on_mesh(field1, meta.universal_part(), &fieldValues[0]); - stk::mesh::put_field_on_mesh(field2, meta.universal_part(), &fieldValues[1]); - stk::mesh::put_field_on_mesh(field3, meta.universal_part(), &fieldValues[2]); + stk::mesh::put_field_on_mesh(field1, meta.universal_part(), fieldValues); + stk::mesh::put_field_on_mesh(field2, meta.universal_part(), fieldValues + 1); + stk::mesh::put_field_on_mesh(field3, meta.universal_part(), fieldValues + 2); stkIo.populate_bulk_data(); size_t results_output_index = stkIo.create_output_mesh(filename, stk::io::WRITE_RESULTS); diff --git a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/stk_transfer_fixtures/CMakeLists.txt b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/stk_transfer_fixtures/CMakeLists.txt index bfd23ae7d43b..51d65af32ffb 100644 --- a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/stk_transfer_fixtures/CMakeLists.txt +++ b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/stk_transfer_fixtures/CMakeLists.txt @@ -43,12 +43,28 @@ SET(SOURCES "") FILE(GLOB HEADERS *.hpp) FILE(GLOB SOURCES *.cpp) -TRIBITS_ADD_LIBRARY( - stk_transfer_fixtures - NOINSTALLHEADERS ${HEADERS} - SOURCES ${SOURCES} - ${Gtest_NO_INSTALL_LIB_OR_HEADERS_ARG} +IF(HAVE_STK_Trilinos) + TRIBITS_ADD_LIBRARY( + stk_transfer_fixtures + NOINSTALLHEADERS ${HEADERS} + SOURCES ${SOURCES} + ${Gtest_NO_INSTALL_LIB_OR_HEADERS_ARG} + ) +ELSE() + add_library(stk_transfer_fixtures ${SOURCES}) + target_include_directories(stk_transfer_fixtures PUBLIC + $ + $ ) + target_include_directories(stk_transfer_fixtures PUBLIC + $ + $ + ) + target_include_directories(stk_transfer_fixtures PUBLIC + $ + $ + ) +ENDIF() INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_unit_test_utils/stk_transfer_fixtures) diff --git a/packages/stk/stk_unit_tests/stk_expreval/UnitTestEvaluator.cpp b/packages/stk/stk_unit_tests/stk_expreval/UnitTestEvaluator.cpp index 13205d35f41f..306f861923c9 100644 --- a/packages/stk/stk_unit_tests/stk_expreval/UnitTestEvaluator.cpp +++ b/packages/stk/stk_unit_tests/stk_expreval/UnitTestEvaluator.cpp @@ -884,7 +884,7 @@ TEST(UnitTestEvaluator, device_nonConstScalarIntAssignmentVariableBinding_modifi } // Can't properly test device-side abort -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) && !defined(KOKKOS_ENABLE_SYCL) && !defined(KOKKOS_ENABLE_OPENMP) +#if !defined(STK_ENABLE_GPU) && !defined(KOKKOS_ENABLE_OPENMP) TEST(UnitTestEvaluator, device_constScalarDoubleAssignmentVariableBinding_throws) { const double x = 2.0; @@ -1111,7 +1111,7 @@ TEST(UnitTestEvaluator, device_nonConstIntArrayAssignmentVariableBinding_modifie } // Can't properly test device-side abort -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) && !defined(KOKKOS_ENABLE_SYCL) && !defined(KOKKOS_ENABLE_OPENMP) +#if !defined(STK_ENABLE_GPU) && !defined(KOKKOS_ENABLE_OPENMP) TEST(UnitTestEvaluator, device_constDoubleArrayAssignmentVariableBinding_throws) { const double x[3] {1.0, 2.0, 3.0}; @@ -1174,7 +1174,7 @@ TEST(UnitTestEvaluator, testFunctionSyntax) EXPECT_TRUE(isInvalidFunction("gamma(1)")); } -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) && !defined(KOKKOS_ENABLE_SYCL) && !defined(KOKKOS_ENABLE_OPENMP) +#if !defined(STK_ENABLE_GPU) && !defined(KOKKOS_ENABLE_SYCL) && !defined(KOKKOS_ENABLE_OPENMP) TEST(UnitTestEvaluator, deviceVariableMap_too_small) { stk::expreval::Eval eval("x+y+z"); @@ -1745,7 +1745,7 @@ TEST(UnitTestEvaluator, defaultVector) TEST(UnitTestEvaluator, Ngp_defaultVector) { EXPECT_DOUBLE_EQ(device_evaluate("x[0]", {}, {}), 0); - #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) && !defined(KOKKOS_ENABLE_SYCL) && !defined(KOKKOS_ENABLE_OPENMP) + #if !defined(STK_ENABLE_GPU) && !defined(KOKKOS_ENABLE_SYCL) && !defined(KOKKOS_ENABLE_OPENMP) EXPECT_ANY_THROW(device_evaluate("x[0]+x[1]+x[2]", {}, {})); #endif } @@ -1785,7 +1785,7 @@ TEST(UnitTestEvaluator, Ngp_bindVector) {}, {{"a", {1, 2, 3}}, {"z", {0, 1, 2}}}), 6); EXPECT_DOUBLE_EQ(device_evaluate("a[0]=(1) ? 2 : 3", {}, {{"a", {0, 0, 0}}}), 2); - #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) && !defined(KOKKOS_ENABLE_SYCL) && !defined(KOKKOS_ENABLE_OPENMP) + #if !defined(STK_ENABLE_GPU) && !defined(KOKKOS_ENABLE_SYCL) && !defined(KOKKOS_ENABLE_OPENMP) EXPECT_ANY_THROW(device_evaluate("a[0]+a[1]+a[3]", {}, {{"a", {1, 2, 3}}})); EXPECT_ANY_THROW(device_evaluate("a[0]+a[1]+a[2]", {}, {{"a", {1, 2, 3}}}, stk::expreval::Variable::ONE_BASED_INDEX)); EXPECT_ANY_THROW(device_evaluate("a", {}, {{"a", {1, 2, 3}}})); @@ -3208,7 +3208,7 @@ void Ngp_testRandom(const char * expression) checkUniformDist(results); } -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) && !defined(KOKKOS_ENABLE_SYCL) +#if !defined(STK_ENABLE_GPU) && !defined(KOKKOS_ENABLE_SYCL) TEST(UnitTestEvaluator, Ngp_testFunction_rand) { Ngp_testRandom("rand()"); @@ -3229,7 +3229,7 @@ TEST(UnitTestEvaluator, testFunction_srand_repeatability) } } -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) && !defined(KOKKOS_ENABLE_SYCL) +#if !defined(STK_ENABLE_GPU) && !defined(KOKKOS_ENABLE_SYCL) TEST(UnitTestEvaluator, Ngp_testFunction_srand_repeatability) { std::vector result(10); @@ -3250,7 +3250,7 @@ TEST(UnitTestEvaluator, testFunction_random) testRandom("random()"); } -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) && !defined(KOKKOS_ENABLE_SYCL) +#if !defined(STK_ENABLE_GPU) && !defined(KOKKOS_ENABLE_SYCL) TEST(UnitTestEvaluator, Ngp_testFunction_random) { Ngp_testRandom("random()"); @@ -3271,7 +3271,7 @@ TEST(UnitTestEvaluator, testFunction_random1_repeatability) } } -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) && !defined(KOKKOS_ENABLE_SYCL) +#if !defined(STK_ENABLE_GPU) && !defined(KOKKOS_ENABLE_SYCL) TEST(UnitTestEvaluator, Ngp_testFunction_random1_repeatability) { std::vector result(10); @@ -3436,7 +3436,7 @@ TEST(UnitTestEvaluator, testFunction_time) EXPECT_NEAR(evaluate("time()"), std::time(nullptr), 1.1); } -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) && !defined(KOKKOS_ENABLE_SYCL) +#if !defined(STK_ENABLE_GPU) && !defined(KOKKOS_ENABLE_SYCL) TEST(UnitTestEvaluator, Ngp_testFunction_time) { EXPECT_NEAR(device_evaluate("time()"), std::time(nullptr), 1.1); diff --git a/packages/stk/stk_unit_tests/stk_io/Assembly.hpp b/packages/stk/stk_unit_tests/stk_io/Assembly.hpp index 00c56aed3be9..46b475c4f51d 100644 --- a/packages/stk/stk_unit_tests/stk_io/Assembly.hpp +++ b/packages/stk/stk_unit_tests/stk_io/Assembly.hpp @@ -66,9 +66,11 @@ namespace unit_test class Assembly : public IOMeshFixture { protected: - stk::mesh::Part& create_assembly(const std::string& assemblyName, int id) + stk::mesh::Part& create_assembly(const std::string& assemblyName, int id, + stk::mesh::EntityRank rank = stk::topology::INVALID_RANK) { - stk::mesh::Part& assemblyPart = get_meta().declare_part(assemblyName); + stk::mesh::Part& assemblyPart = rank==stk::topology::INVALID_RANK ? + get_meta().declare_part(assemblyName) : get_meta().declare_part(assemblyName, rank); stk::io::put_assembly_io_part_attribute(assemblyPart); get_meta().set_part_id(assemblyPart, id); return assemblyPart; @@ -145,6 +147,7 @@ class Assembly : public IOMeshFixture { EXPECT_TRUE(stk::io::is_part_assembly_io_part(part)); EXPECT_TRUE(stk::io::is_part_io_part(part)); + EXPECT_FALSE(stk::mesh::is_element_block(part)); } void test_assembly_part_attributes(const stk::mesh::PartVector& parts) diff --git a/packages/stk/stk_unit_tests/stk_io/CMakeLists.txt b/packages/stk/stk_unit_tests/stk_io/CMakeLists.txt index 675898aaa9b0..cc656235164b 100644 --- a/packages/stk/stk_unit_tests/stk_io/CMakeLists.txt +++ b/packages/stk/stk_unit_tests/stk_io/CMakeLists.txt @@ -68,6 +68,10 @@ else() $ ) + target_include_directories(stk_io_unit_tests PUBLIC + ${STK_TOPLEVEL_SOURCE_DIR}/stk_unit_test_utils + ) + add_test(NAME "stk_io_unit_tests" COMMAND stk_io_unit_tests) endif() diff --git a/packages/stk/stk_unit_tests/stk_io/UnitTestGetAssemblyEntities.cpp b/packages/stk/stk_unit_tests/stk_io/UnitTestGetAssemblyEntities.cpp index 4af6691d2435..6aabad8da24d 100644 --- a/packages/stk/stk_unit_tests/stk_io/UnitTestGetAssemblyEntities.cpp +++ b/packages/stk/stk_unit_tests/stk_io/UnitTestGetAssemblyEntities.cpp @@ -197,12 +197,12 @@ class GetAssemblyEntities : public stk::io::unit_test::Assembly return assemblyPart; } - stk::mesh::Part& setup_single_hex_mesh() + stk::mesh::Part& setup_single_hex_mesh(stk::mesh::EntityRank assemblyRank = stk::topology::INVALID_RANK) { setup_empty_mesh(stk::mesh::BulkData::AUTO_AURA); const std::string assemblyName("simpleAssembly"); - stk::mesh::Part& assemblyPart = create_single_hex_assembly(assemblyName, 10); + stk::mesh::Part& assemblyPart = create_single_hex_assembly(assemblyName, 10, assemblyRank); create_single_hex_mesh(); return assemblyPart; } @@ -213,11 +213,12 @@ class GetAssemblyEntities : public stk::io::unit_test::Assembly stk::io::fill_mesh(meshDesc, get_bulk()); } - stk::mesh::Part& create_single_hex_assembly(const std::string& assemblyName, int id) + stk::mesh::Part& create_single_hex_assembly(const std::string& assemblyName, int id, + stk::mesh::EntityRank assemblyRank = stk::topology::INVALID_RANK) { const std::vector partNames{"block_1"}; - stk::mesh::Part& assemblyPart = create_assembly(assemblyName, id); + stk::mesh::Part& assemblyPart = create_assembly(assemblyName, id, assemblyRank); stk::mesh::Part& block1Part = create_io_part(partNames[0], 1); declare_subsets(assemblyPart, {&block1Part}); @@ -311,9 +312,7 @@ class GetAssemblyEntities : public stk::io::unit_test::Assembly TEST_F(GetAssemblyEntities, noEntities) { - if (stk::parallel_machine_size(get_comm()) != 1) { - return; - } + if (stk::parallel_machine_size(get_comm()) != 1) { GTEST_SKIP(); } setup_empty_mesh(stk::mesh::BulkData::AUTO_AURA); @@ -329,9 +328,7 @@ TEST_F(GetAssemblyEntities, noEntities) TEST_F(GetAssemblyEntities, singleParticle_getElementBucket) { - if (stk::parallel_machine_size(get_comm()) != 1) { - return; - } + if (stk::parallel_machine_size(get_comm()) != 1) { GTEST_SKIP(); } stk::mesh::Part& assemblyPart = setup_single_particle_mesh(); @@ -341,9 +338,7 @@ TEST_F(GetAssemblyEntities, singleParticle_getElementBucket) TEST_F(GetAssemblyEntities, singleParticle_getElement) { - if (stk::parallel_machine_size(get_comm()) != 1) { - return; - } + if (stk::parallel_machine_size(get_comm()) != 1) { GTEST_SKIP(); } stk::mesh::Part& assemblyPart = setup_single_particle_mesh(); @@ -353,9 +348,7 @@ TEST_F(GetAssemblyEntities, singleParticle_getElement) TEST_F(GetAssemblyEntities, singleParticle_getNodeBucket) { - if (stk::parallel_machine_size(get_comm()) != 1) { - return; - } + if (stk::parallel_machine_size(get_comm()) != 1) { GTEST_SKIP(); } stk::mesh::Part& assemblyPart = setup_single_particle_mesh(); @@ -365,9 +358,7 @@ TEST_F(GetAssemblyEntities, singleParticle_getNodeBucket) TEST_F(GetAssemblyEntities, singleParticle_getNode) { - if (stk::parallel_machine_size(get_comm()) != 1) { - return; - } + if (stk::parallel_machine_size(get_comm()) != 1) { GTEST_SKIP(); } stk::mesh::Part& assemblyPart = setup_single_particle_mesh(); @@ -377,9 +368,7 @@ TEST_F(GetAssemblyEntities, singleParticle_getNode) TEST_F(GetAssemblyEntities, singleHex_getElementBucket) { - if (stk::parallel_machine_size(get_comm()) != 1) { - return; - } + if (stk::parallel_machine_size(get_comm()) != 1) { GTEST_SKIP(); } stk::mesh::Part& assemblyPart = setup_single_hex_mesh(); @@ -387,11 +376,19 @@ TEST_F(GetAssemblyEntities, singleHex_getElementBucket) test_get_buckets(assemblyPart, stk::topology::ELEM_RANK, BucketTestData(stk::topology::HEX_8, {1})); } +TEST_F(GetAssemblyEntities, singleHex_getElementBucket_rankedAssemblyPart) +{ + if (stk::parallel_machine_size(get_comm()) != 1) { GTEST_SKIP(); } + + stk::mesh::Part& assemblyPart = setup_single_hex_mesh(stk::topology::ELEM_RANK); + + test_assembly_part_attributes(assemblyPart); + test_get_buckets(assemblyPart, stk::topology::ELEM_RANK, BucketTestData(stk::topology::HEX_8, {1})); +} + TEST_F(GetAssemblyEntities, singleHex_getElement) { - if (stk::parallel_machine_size(get_comm()) != 1) { - return; - } + if (stk::parallel_machine_size(get_comm()) != 1) { GTEST_SKIP(); } stk::mesh::Part& assemblyPart = setup_single_hex_mesh(); @@ -401,9 +398,7 @@ TEST_F(GetAssemblyEntities, singleHex_getElement) TEST_F(GetAssemblyEntities, singleHex_getNodeBucket) { - if (stk::parallel_machine_size(get_comm()) != 1) { - return; - } + if (stk::parallel_machine_size(get_comm()) != 1) { GTEST_SKIP(); } stk::mesh::Part& assemblyPart = setup_single_hex_mesh(); @@ -413,9 +408,7 @@ TEST_F(GetAssemblyEntities, singleHex_getNodeBucket) TEST_F(GetAssemblyEntities, singleHex_getNodes) { - if (stk::parallel_machine_size(get_comm()) != 1) { - return; - } + if (stk::parallel_machine_size(get_comm()) != 1) { GTEST_SKIP(); } stk::mesh::Part& assemblyPart = setup_single_hex_mesh(); @@ -425,9 +418,7 @@ TEST_F(GetAssemblyEntities, singleHex_getNodes) TEST_F(GetAssemblyEntities, twoHexDifferentBlocks_getElementBucket) { - if (stk::parallel_machine_size(get_comm()) != 1) { - return; - } + if (stk::parallel_machine_size(get_comm()) != 1) { GTEST_SKIP(); } const std::string assemblyBlock("block_1"); stk::mesh::Part& assemblyPart = setup_two_hex_mesh_with_block_in_assembly(assemblyBlock); @@ -438,9 +429,7 @@ TEST_F(GetAssemblyEntities, twoHexDifferentBlocks_getElementBucket) TEST_F(GetAssemblyEntities, twoHexDifferentBlocks_getElement) { - if (stk::parallel_machine_size(get_comm()) != 1) { - return; - } + if (stk::parallel_machine_size(get_comm()) != 1) { GTEST_SKIP(); } const std::string assemblyBlock("block_1"); stk::mesh::Part& assemblyPart = setup_two_hex_mesh_with_block_in_assembly(assemblyBlock); diff --git a/packages/stk/stk_unit_tests/stk_io/UnitTestReadWriteSideSets.cpp b/packages/stk/stk_unit_tests/stk_io/UnitTestReadWriteSideSets.cpp index 7e5a3098af4b..d41434ffe0f5 100644 --- a/packages/stk/stk_unit_tests/stk_io/UnitTestReadWriteSideSets.cpp +++ b/packages/stk/stk_unit_tests/stk_io/UnitTestReadWriteSideSets.cpp @@ -683,7 +683,7 @@ class ShellSidesets : public stk::unit_test_util::MeshFixture stk::mesh::Part* test_surface2 = meta.get_part("surface_2"); ASSERT_TRUE(test_surface2 != nullptr); - EXPECT_EQ(stk::topology::EDGE_RANK, test_surface2->primary_entity_rank()); + EXPECT_EQ(stk::topology::FACE_RANK, test_surface2->primary_entity_rank()); unlink(fileName.c_str()); } @@ -705,6 +705,19 @@ TEST_F(ShellSidesets, testWriteThenRead) test_write_then_read(); } +TEST(ShellSideSidesets, create3DEdgesFromShellSides) +{ + if (stk::parallel_machine_size(MPI_COMM_WORLD) > 1) { GTEST_SKIP(); } + + std::string meshDesc = "textmesh: 0,1,SHELL_QUAD_4,1,2,3,4 |coordinates: 0,0,0, 1,0,0, 1,1,0, 0,1,0 |sideset:name=surface_1; data=1,3"; + + const unsigned spatialDim = 3; + std::shared_ptr bulk = build_mesh_no_simple_fields(spatialDim, MPI_COMM_WORLD, stk::mesh::BulkData::NO_AUTO_AURA); + ::testing::internal::CaptureStderr(); + EXPECT_NO_THROW(stk::io::fill_mesh(meshDesc, *bulk)); + ASSERT_TRUE(::testing::internal::GetCapturedStderr().empty()); +} + } // namespace unit_test } // namespace io } // namespace stk diff --git a/packages/stk/stk_unit_tests/stk_io/UnitTestRestart.cpp b/packages/stk/stk_unit_tests/stk_io/UnitTestRestart.cpp index c0ce20a95602..aa9cead5f74a 100644 --- a/packages/stk/stk_unit_tests/stk_io/UnitTestRestart.cpp +++ b/packages/stk/stk_unit_tests/stk_io/UnitTestRestart.cpp @@ -126,6 +126,7 @@ void create_corrupt_restart(MPI_Comm communicator, stkIo.populate_bulk_data(); + stkIo.property_add(Ioss::Property("FLUSH_INTERVAL", 1)); size_t fileIndex = stkIo.create_output_mesh(restartFilename, stk::io::WRITE_RESTART); stkIo.add_field(fileIndex, field0); diff --git a/packages/stk/stk_unit_tests/stk_mesh/UnitTestBulkData.cpp b/packages/stk/stk_unit_tests/stk_mesh/UnitTestBulkData.cpp index 7bd6f6946c72..3f7c64e51040 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/UnitTestBulkData.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/UnitTestBulkData.cpp @@ -1933,7 +1933,8 @@ void testParallelSideCreation(stk::mesh::BulkData::AutomaticAuraOption autoAuraO // Create local version of side on each proc Entity side = mesh.declare_element_side(elem, local_side_ordinal, stk::mesh::ConstPartVector{&side_part}); - stk::mesh::Permutation perm1 = mesh.find_permutation(elem_top, &nodes[0], elem_top.side_topology(local_side_ordinal), &side_nodes[0], local_side_ordinal); + stk::mesh::Permutation perm1 = mesh.find_permutation( + elem_top, nodes.data(), elem_top.side_topology(local_side_ordinal), side_nodes.data(), local_side_ordinal); ASSERT_TRUE(perm1 != stk::mesh::Permutation::INVALID_PERMUTATION); mesh.modification_end(); @@ -1965,7 +1966,8 @@ void testParallelSideCreation(stk::mesh::BulkData::AutomaticAuraOption autoAuraO } side = mesh.declare_element_side(elem, local_side_ordinal, stk::mesh::ConstPartVector{&side_part}); - stk::mesh::Permutation perm2 = mesh.find_permutation(elem_top, &nodes[0], elem_top.side_topology(local_side_ordinal), &side_nodes[0], local_side_ordinal); + stk::mesh::Permutation perm2 = mesh.find_permutation( + elem_top, nodes.data(), elem_top.side_topology(local_side_ordinal), side_nodes.data(), local_side_ordinal); ASSERT_TRUE(perm2 != stk::mesh::Permutation::INVALID_PERMUTATION); mesh.modification_end(); diff --git a/packages/stk/stk_unit_tests/stk_mesh/UnitTestBulkDataNotifications.cpp b/packages/stk/stk_unit_tests/stk_mesh/UnitTestBulkDataNotifications.cpp index ad1678ebeed0..47ff264321f0 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/UnitTestBulkDataNotifications.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/UnitTestBulkDataNotifications.cpp @@ -2,6 +2,7 @@ #include // for size_t #include // for BulkData, etc #include +#include #include // for fill_mesh_using_stk_io #include // for ParallelMachine, etc #include // for all_reduce_max @@ -50,6 +51,39 @@ class TestListener : public stk::mesh::ModificationObserver { std::vector buckets_changed; }; +TEST(BulkDataMod, synchronized_count_basic_fill_mesh) +{ + stk::ParallelMachine comm = MPI_COMM_WORLD; + if (stk::parallel_machine_size(comm) > 2) { GTEST_SKIP(); } + + std::shared_ptr bulk = stk::mesh::MeshBuilder(comm).create(); + bulk->mesh_meta_data().use_simple_fields(); + + const std::string generatedMeshSpec = "generated:1x1x2"; + stk::io::fill_mesh(generatedMeshSpec, *bulk); + + EXPECT_EQ(1u, bulk->synchronized_count()); +} + +TEST(BulkDataMod, synchronized_count_empty_mod_cycle) +{ + stk::ParallelMachine comm = MPI_COMM_WORLD; + if (stk::parallel_machine_size(comm) > 2) { GTEST_SKIP(); } + + std::shared_ptr bulk = stk::mesh::MeshBuilder(comm).create(); + bulk->mesh_meta_data().use_simple_fields(); + + const std::string generatedMeshSpec = "generated:1x1x2"; + stk::io::fill_mesh(generatedMeshSpec, *bulk); + + unsigned modCount = bulk->synchronized_count(); + + bulk->modification_begin(); + bulk->modification_end(); + + EXPECT_EQ(modCount, bulk->synchronized_count()); +} + TEST(BulkDataNotifications, test_listener_buckets_changed) { stk::ParallelMachine comm = MPI_COMM_WORLD; diff --git a/packages/stk/stk_unit_tests/stk_mesh/UnitTestBulkData_new.cpp b/packages/stk/stk_unit_tests/stk_mesh/UnitTestBulkData_new.cpp index 2e58188a24b4..afe8dd559f45 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/UnitTestBulkData_new.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/UnitTestBulkData_new.cpp @@ -919,3 +919,16 @@ TEST ( UnitTestBulkData_new , testCustomBucketCapacity ) EXPECT_EQ( bulk->bucket(node).capacity(), non_standard_bucket_capacity ); } +TEST(BulkData, bucket_debug_ThrowsWithInvalidEntity) +{ + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 1) { GTEST_SKIP(); } + +#ifndef NDEBUG + std::unique_ptr bulkPtr = stk::mesh::MeshBuilder(MPI_COMM_WORLD).create(); + + stk::mesh::Entity invalidEntity; + EXPECT_TRUE(nullptr == bulkPtr->bucket_ptr(invalidEntity)); + EXPECT_ANY_THROW(bulkPtr->bucket(invalidEntity)); +#endif +} + diff --git a/packages/stk/stk_unit_tests/stk_mesh/UnitTestCEOCommonUtils.cpp b/packages/stk/stk_unit_tests/stk_mesh/UnitTestCEOCommonUtils.cpp index d7c761f6ff32..9031c4ba1896 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/UnitTestCEOCommonUtils.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/UnitTestCEOCommonUtils.cpp @@ -608,7 +608,7 @@ void fillMeshfor3Elem4Proc1Edge3DAndTest(stk::unit_test_util::BulkDataTester &me nodes.push_back(mesh.get_entity(NODE_RANK, 13 )); mesh.declare_relation(edge, nodes[0], 0); mesh.declare_relation(edge, nodes[1], 1); - stk::mesh::impl::connectUpwardEntityToEntity(mesh, elem, edge, &nodes[0]); + stk::mesh::impl::connectUpwardEntityToEntity(mesh, elem, edge, nodes.data()); } for(int proc = 0; proc < numSharedNodeTriples; ++proc) diff --git a/packages/stk/stk_unit_tests/stk_mesh/UnitTestEntitiesNodesHaveInCommon.cpp b/packages/stk/stk_unit_tests/stk_mesh/UnitTestEntitiesNodesHaveInCommon.cpp index b7c0bc303783..4f2eb5954bef 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/UnitTestEntitiesNodesHaveInCommon.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/UnitTestEntitiesNodesHaveInCommon.cpp @@ -211,7 +211,7 @@ TEST(MeshImplUtils, do_these_nodes_have_any_shell_elements_in_common_hexshell) for (unsigned i=0 ; i<4 ; ++i) { nodes[i] = mesh.begin_nodes(shell)[i]; } - EXPECT_TRUE(stk::mesh::impl::do_these_nodes_have_any_shell_elements_in_common(mesh,4,&nodes[0])); + EXPECT_TRUE(stk::mesh::impl::do_these_nodes_have_any_shell_elements_in_common(mesh, 4, nodes.data())); } { stk::mesh::Entity hex = (*mesh.buckets(stk::topology::ELEMENT_RANK)[0])[0]; @@ -220,7 +220,7 @@ TEST(MeshImplUtils, do_these_nodes_have_any_shell_elements_in_common_hexshell) for (unsigned i=0 ; i<8 ; ++i) { nodes[i] = mesh.begin_nodes(hex)[i]; } - EXPECT_FALSE(stk::mesh::impl::do_these_nodes_have_any_shell_elements_in_common(mesh,8,&nodes[0])); + EXPECT_FALSE(stk::mesh::impl::do_these_nodes_have_any_shell_elements_in_common(mesh, 8, nodes.data())); } } @@ -243,8 +243,7 @@ TEST(MeshImplUtils, do_these_nodes_have_any_shell_elements_in_common_hexshellwra for (unsigned i=0 ; i<4 ; ++i) { nodes[i] = mesh.begin_nodes(hex)[i]; } - EXPECT_FALSE(stk::mesh::impl::do_these_nodes_have_any_shell_elements_in_common(mesh,4,&nodes[0])); - + EXPECT_FALSE(stk::mesh::impl::do_these_nodes_have_any_shell_elements_in_common(mesh, 4, nodes.data())); } } diff --git a/packages/stk/stk_unit_tests/stk_mesh/UnitTestField.cpp b/packages/stk/stk_unit_tests/stk_mesh/UnitTestField.cpp index 76732751145e..2ffd492c1d1e 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/UnitTestField.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/UnitTestField.cpp @@ -1803,6 +1803,7 @@ TEST_F(VariableCapacityBuckets, initialMeshConstruction_initialCapacity2_maxCapa { stk::mesh::EntityIdVector ids{1, 2, 3}; stk::mesh::EntityVector newNodes; + m_bulk->modification_begin(); m_bulk->declare_entities(stk::topology::NODE_RANK, ids, stk::mesh::PartVector{&block1}, newNodes); const stk::mesh::BucketVector & buckets = m_bulk->buckets(stk::topology::NODE_RANK); @@ -1813,6 +1814,7 @@ TEST_F(VariableCapacityBuckets, initialMeshConstruction_initialCapacity2_maxCapa EXPECT_EQ(buckets[1]->size(), 1u); EXPECT_EQ(buckets[1]->capacity(), 2u); + m_bulk->modification_end(); } } @@ -1827,6 +1829,7 @@ TEST_F(VariableCapacityBuckets, initialMeshConstruction_initialCapacity1_maxCapa { stk::mesh::EntityIdVector ids{1, 2, 3}; stk::mesh::EntityVector newNodes; + m_bulk->modification_begin(); m_bulk->declare_entities(stk::topology::NODE_RANK, ids, stk::mesh::PartVector{&block1}, newNodes); const stk::mesh::BucketVector & buckets = m_bulk->buckets(stk::topology::NODE_RANK); @@ -1837,6 +1840,7 @@ TEST_F(VariableCapacityBuckets, initialMeshConstruction_initialCapacity1_maxCapa EXPECT_EQ(buckets[1]->size(), 1u); EXPECT_EQ(buckets[1]->capacity(), 1u); + m_bulk->modification_end(); } } @@ -2343,6 +2347,7 @@ TEST_P(VariableCapacityFieldData, initialMeshConstruction_initialCapacity2_maxCa m_bulk->deactivate_field_updating(); stk::mesh::EntityIdVector ids{1, 2, 3}; stk::mesh::EntityVector newNodes; + m_bulk->modification_begin(); m_bulk->declare_entities(stk::topology::NODE_RANK, ids, stk::mesh::PartVector{&block1}, newNodes); m_bulk->allocate_field_data(); @@ -2351,6 +2356,7 @@ TEST_P(VariableCapacityFieldData, initialMeshConstruction_initialCapacity2_maxCa const int bytesAllocated = m_fieldDataManager->get_num_bytes_allocated_on_field(fieldOrdinal); ASSERT_EQ(bytesAllocated, expected_bytes_allocated(buckets, dataSize)); + m_bulk->modification_end(); } } @@ -2371,6 +2377,7 @@ TEST_P(VariableCapacityFieldData, initialMeshConstruction_initialCapacity1_maxCa m_bulk->deactivate_field_updating(); stk::mesh::EntityIdVector ids{1, 2, 3}; stk::mesh::EntityVector newNodes; + m_bulk->modification_begin(); m_bulk->declare_entities(stk::topology::NODE_RANK, ids, stk::mesh::PartVector{&block1}, newNodes); m_bulk->allocate_field_data(); @@ -2379,6 +2386,7 @@ TEST_P(VariableCapacityFieldData, initialMeshConstruction_initialCapacity1_maxCa const int bytesAllocated = m_fieldDataManager->get_num_bytes_allocated_on_field(fieldOrdinal); ASSERT_EQ(bytesAllocated, expected_bytes_allocated(buckets, dataSize)); + m_bulk->modification_end(); } } diff --git a/packages/stk/stk_unit_tests/stk_mesh/UnitTestFieldDataManager.cpp b/packages/stk/stk_unit_tests/stk_mesh/UnitTestFieldDataManager.cpp index 058e3547530d..2f8d7b45e363 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/UnitTestFieldDataManager.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/UnitTestFieldDataManager.cpp @@ -877,7 +877,7 @@ TEST(ContiguousFieldDataManagerTest, algorithmExploration_eraseOneEntry) } double totalTime = stk::cpu_time() - startTime; EXPECT_EQ(field.size(), numItems-numItemsToErase); - testForCheating(&field[0], numItems-numItemsToErase, valuesErased); + testForCheating(field.data(), numItems - numItemsToErase, valuesErased); std::cerr << "Time = " << totalTime << " s" << std::endl; } @@ -891,7 +891,7 @@ TEST(ContiguousFieldDataManagerTest, algorithmExploration_fasterEraseOneEntry) size_t numItems = field.size(); size_t numItemsToErase = itemsToErase.size(); - double* field_array = &field[0]; + double *field_array = field.data(); size_t field_array_length = field.size(); // Erase @@ -904,7 +904,7 @@ TEST(ContiguousFieldDataManagerTest, algorithmExploration_fasterEraseOneEntry) double totalTime = stk::cpu_time() - startTime; EXPECT_EQ(field_array_length, numItems-numItemsToErase); - testForCheating(&field[0], numItems-numItemsToErase, valuesErased); + testForCheating(field.data(), numItems - numItemsToErase, valuesErased); std::cerr << "Time = " << totalTime << " s" << std::endl; } @@ -918,7 +918,7 @@ TEST(ContiguousFieldDataManagerTest, algorithmExploration_slowestEraseOneEntry) size_t numItems = field.size(); size_t numItemsToErase = itemsToErase.size(); - double* field_array = &field[0]; + double *field_array = field.data(); size_t field_array_length = field.size(); // Erase @@ -935,7 +935,7 @@ TEST(ContiguousFieldDataManagerTest, algorithmExploration_slowestEraseOneEntry) double totalTime = stk::cpu_time() - startTime; EXPECT_EQ(field_array_length, numItems-numItemsToErase); - testForCheating(&field[0], numItems-numItemsToErase, valuesErased); + testForCheating(field.data(), numItems - numItemsToErase, valuesErased); std::cerr << "Time = " << totalTime << " s" << std::endl; } @@ -950,9 +950,9 @@ TEST(ContiguousFieldDataManagerTest, algorithmExploration_memcpyishEraseOneEntry size_t numItemsToErase = itemsToErase.size(); std::vector scratchField(numItems,0); - double* field_array = &field[0]; + double *field_array = field.data(); size_t field_array_length = field.size(); - double *scratchData = &scratchField[0]; + double *scratchData = scratchField.data(); // Erase double startTime = stk::cpu_time(); @@ -966,7 +966,7 @@ TEST(ContiguousFieldDataManagerTest, algorithmExploration_memcpyishEraseOneEntry double totalTime = stk::cpu_time() - startTime; EXPECT_EQ(field_array_length, numItems-numItemsToErase); - testForCheating(&field[0], numItems-numItemsToErase, valuesErased); + testForCheating(field.data(), numItems - numItemsToErase, valuesErased); std::cerr << "Time = " << totalTime << " s" << std::endl; } @@ -988,7 +988,7 @@ TEST(ContiguousFieldDataManagerTest, algorithmExploration_batchDeletion) std::vector itemsThisChunk(numItemsToErase+1, 0); std::vector distanceToSlideLeft(numItemsToErase+1,0); - startingPtr[0] = &field[0]; + startingPtr[0] = field.data(); itemsThisChunk[0] = itemsToErase[0]; distanceToSlideLeft[0] = 0; @@ -1013,7 +1013,7 @@ TEST(ContiguousFieldDataManagerTest, algorithmExploration_batchDeletion) double totalTime = stk::cpu_time() - startTime; EXPECT_EQ(field_array_length, numItems-numItemsToErase); - testForCheating(&field[0], numItems-numItemsToErase, valuesErased); + testForCheating(field.data(), numItems - numItemsToErase, valuesErased); std::cerr << "Time = " << totalTime << " s" << std::endl; } diff --git a/packages/stk/stk_unit_tests/stk_mesh/UnitTestGenIds.cpp b/packages/stk/stk_unit_tests/stk_mesh/UnitTestGenIds.cpp index 78e993046360..5aa5b8b544f3 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/UnitTestGenIds.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/UnitTestGenIds.cpp @@ -413,7 +413,7 @@ void respondToRootProcessorAboutIdsOwnedOnThisProc(const int root, const uint64_ } } int *rbuff = 0; - MPI_Reduce(&areIdsBeingused[0], rbuff, numIdsToGet, MPI_INT, MPI_SUM, root, comm); + MPI_Reduce(areIdsBeingused.data(), rbuff, numIdsToGet, MPI_INT, MPI_SUM, root, comm); } } @@ -440,7 +440,7 @@ void respondToRootProcessorAboutIdsOwnedOnThisProc(const int root, const uint64_ } } int *rbuff = 0; - MPI_Reduce(&areIdsBeingused[0], rbuff, numIdsToGet, MPI_INT, MPI_SUM, root, comm); + MPI_Reduce(areIdsBeingused.data(), rbuff, numIdsToGet, MPI_INT, MPI_SUM, root, comm); } } @@ -453,7 +453,7 @@ void retrieveIds(const INTMPI root, uint64_t id, MPI_Comm comm, uint64_t numIdsT { MPI_Bcast(&numIdsToGetPerProc, 1, sierra::MPI::Datatype::type(), root, comm); std::vector zeroids(numIdsToGetPerProc,0); - MPI_Reduce(&zeroids[0], &areIdsBeingUsed[0], numIdsToGetPerProc, MPI_INT, MPI_SUM, root, comm); + MPI_Reduce(zeroids.data(), areIdsBeingUsed.data(), numIdsToGetPerProc, MPI_INT, MPI_SUM, root, comm); } } @@ -533,7 +533,8 @@ void getAvailableIds_exp(const std::vector &myIds, uint64_t numIdsNeed { INTMPI numprocs = mpiInfo.getNumProcs(); std::vector receivedInfo(numprocs,0); - MPI_Allgather(&numIdsNeeded, 1, sierra::MPI::Datatype::type(), &receivedInfo[0], 1, sierra::MPI::Datatype::type(), mpiInfo.getMpiComm()); + MPI_Allgather(&numIdsNeeded, 1, sierra::MPI::Datatype::type(), receivedInfo.data(), 1, + sierra::MPI::Datatype::type(), mpiInfo.getMpiComm()); std::vector sortedIds(myIds.begin(), myIds.end()); std::sort(sortedIds.begin(), sortedIds.end()); @@ -592,7 +593,8 @@ void getAvailableIds_exp(stk::mesh::BulkData &stkMeshBulkData, uint64_t numIdsNe { INTMPI numprocs = mpiInfo.getNumProcs(); std::vector receivedInfo(numprocs,0); - MPI_Allgather(&numIdsNeeded, 1, sierra::MPI::Datatype::type(), &receivedInfo[0], 1, sierra::MPI::Datatype::type(), mpiInfo.getMpiComm()); + MPI_Allgather(&numIdsNeeded, 1, sierra::MPI::Datatype::type(), receivedInfo.data(), 1, + sierra::MPI::Datatype::type(), mpiInfo.getMpiComm()); stk::mesh::EntityId largestIdHere = 0; diff --git a/packages/stk/stk_unit_tests/stk_mesh/UnitTestMetaData.cpp b/packages/stk/stk_unit_tests/stk_mesh/UnitTestMetaData.cpp index fd0e4068df59..ac1766fd5737 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/UnitTestMetaData.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/UnitTestMetaData.cpp @@ -410,6 +410,21 @@ TEST(UnitTestMetaData, ConsistentParallelDebugCheck) EXPECT_NO_THROW(bulk.modification_begin()); } +#define STK_EXPECT_THROW_MSG(runit, myProc, msgProc, goldMsg) \ +{ \ + bool threw = false; \ + try { \ + runit; \ + } \ + catch(std::exception& e) { \ + threw = true; \ + if (myProc == msgProc) { \ + EXPECT_TRUE(std::string(e.what()).find(goldMsg) != std::string::npos)<<"failed to find '"<(stk::topology::NODE_RANK, "really_long_field_1"); } - testing::internal::CaptureStderr(); - EXPECT_THROW(bulk.modification_begin(), std::logic_error); - - std::string stderrString = testing::internal::GetCapturedStderr(); - if (stk::parallel_machine_rank(MPI_COMM_WORLD) == 1) { - EXPECT_EQ(stderrString, "[p1] Field name (really_long_field_1) does not match Field name (field_1) on root processor\n"); - } + STK_EXPECT_THROW_MSG(bulk.modification_begin(), bulk.parallel_rank(), 1, "[p1] Field name (really_long_field_1) does not match Field name (field_1) on root processor\n"); } TEST(UnitTestMetaData, InconsistentParallelDebugCheck_BadFieldNameText) @@ -631,13 +598,7 @@ TEST(UnitTestMetaData, InconsistentParallelDebugCheck_BadFieldNameText) meta.declare_field(stk::topology::NODE_RANK, "field_2"); } - testing::internal::CaptureStderr(); - EXPECT_THROW(bulk.modification_begin(), std::logic_error); - - std::string stderrString = testing::internal::GetCapturedStderr(); - if (stk::parallel_machine_rank(MPI_COMM_WORLD) == 1) { - EXPECT_EQ(stderrString, "[p1] Field name (field_2) does not match Field name (field_1) on root processor\n"); - } + STK_EXPECT_THROW_MSG(bulk.modification_begin(), bulk.parallel_rank(), 1, "[p1] Field name (field_2) does not match Field name (field_1) on root processor\n"); } TEST(UnitTestMetaData, InconsistentParallelDebugCheck_BadFieldRank) @@ -656,13 +617,7 @@ TEST(UnitTestMetaData, InconsistentParallelDebugCheck_BadFieldRank) meta.declare_field(stk::topology::ELEM_RANK, "field_1"); } - testing::internal::CaptureStderr(); - EXPECT_THROW(bulk.modification_begin(), std::logic_error); - - std::string stderrString = testing::internal::GetCapturedStderr(); - if (stk::parallel_machine_rank(MPI_COMM_WORLD) == 1) { - EXPECT_EQ(stderrString, "[p1] Field field_1 rank (ELEMENT_RANK) does not match Field field_1 rank (NODE_RANK) on root processor\n"); - } + STK_EXPECT_THROW_MSG(bulk.modification_begin(), bulk.parallel_rank(), 1, "[p1] Field field_1 rank (ELEMENT_RANK) does not match Field field_1 rank (NODE_RANK) on root processor\n"); } TEST(UnitTestMetaData, InconsistentParallelDebugCheck_BadFieldNumberOfStates) @@ -681,14 +636,7 @@ TEST(UnitTestMetaData, InconsistentParallelDebugCheck_BadFieldNumberOfStates) meta.declare_field(stk::topology::NODE_RANK, "field_1", 2); } - testing::internal::CaptureStderr(); - EXPECT_THROW(bulk.modification_begin(), std::logic_error); - - std::string stderrString = testing::internal::GetCapturedStderr(); - if (stk::parallel_machine_rank(MPI_COMM_WORLD) == 1) { - EXPECT_EQ(stderrString, "[p1] Field field_1 number of states (2) does not match Field field_1 number of states (1) on root processor\n" - "[p1] Have extra Field (field_1_STKFS_OLD) that does not exist on root processor\n"); - } + STK_EXPECT_THROW_MSG(bulk.modification_begin(), bulk.parallel_rank(), 1, "[p1] Field field_1 number of states (2) does not match Field field_1 number of states (1) on root processor\n[p1] Have extra Field (field_1_STKFS_OLD) that does not exist on root processor\n"); } TEST(UnitTestMetaData, InconsistentParallelDebugCheck_BadNumberOfFields_RootTooFew) @@ -708,13 +656,7 @@ TEST(UnitTestMetaData, InconsistentParallelDebugCheck_BadNumberOfFields_RootTooF meta.declare_field(stk::topology::NODE_RANK, "field_2"); } - testing::internal::CaptureStderr(); - EXPECT_THROW(bulk.modification_begin(), std::logic_error); - - std::string stderrString = testing::internal::GetCapturedStderr(); - if (stk::parallel_machine_rank(MPI_COMM_WORLD) == 1) { - EXPECT_EQ(stderrString, "[p1] Have extra Field (field_2) that does not exist on root processor\n"); - } + STK_EXPECT_THROW_MSG(bulk.modification_begin(), bulk.parallel_rank(), 1, "[p1] Have extra Field (field_2) that does not exist on root processor\n"); } TEST(UnitTestMetaData, InconsistentParallelDebugCheck_BadNumberOfFields_RootTooMany) @@ -734,13 +676,7 @@ TEST(UnitTestMetaData, InconsistentParallelDebugCheck_BadNumberOfFields_RootTooM meta.declare_field(stk::topology::NODE_RANK, "field_1"); } - testing::internal::CaptureStderr(); - EXPECT_THROW(bulk.modification_begin(), std::logic_error); - - std::string stderrString = testing::internal::GetCapturedStderr(); - if (stk::parallel_machine_rank(MPI_COMM_WORLD) == 1) { - EXPECT_EQ(stderrString, "[p1] Received extra Field (field_2) from root processor\n"); - } + STK_EXPECT_THROW_MSG(bulk.modification_begin(), bulk.parallel_rank(), 1, "[p1] Received extra Field (field_2) from root processor\n"); } } diff --git a/packages/stk/stk_unit_tests/stk_mesh/UnitTestModificationEnd.hpp b/packages/stk/stk_unit_tests/stk_mesh/UnitTestModificationEnd.hpp index 5a6fad2dd18e..2f2c884cf0c8 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/UnitTestModificationEnd.hpp +++ b/packages/stk/stk_unit_tests/stk_mesh/UnitTestModificationEnd.hpp @@ -523,7 +523,7 @@ void connectElementToEdge(stk::unit_test_util::BulkDataTester& stkMeshBulkData, nodes[i] = stkMeshBulkData.get_entity(nodeKey); } - stk::mesh::impl::connectUpwardEntityToEntity(stkMeshBulkData, element, edge, &nodes[0]); + stk::mesh::impl::connectUpwardEntityToEntity(stkMeshBulkData, element, edge, nodes.data()); } void create_edges(stk::unit_test_util::BulkDataTester& stkMeshBulkData, std::vector& edgeIds, diff --git a/packages/stk/stk_unit_tests/stk_mesh/UnitTestRelation.cpp b/packages/stk/stk_unit_tests/stk_mesh/UnitTestRelation.cpp index 10d96f0a7c14..d71ddbaabf6c 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/UnitTestRelation.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/UnitTestRelation.cpp @@ -372,7 +372,8 @@ TEST(UnitTestingOfRelation, testDoubleDeclareOfRelation) edge = mesh.declare_element_side(elem, local_side_id, sides_parts); stk::topology elem_top = mesh.bucket(elem).topology(); - stk::mesh::Permutation perm1 = mesh.find_permutation(elem_top, &nodes[0], elem_top.side_topology(local_side_id), &side_nodes[0], local_side_id); + stk::mesh::Permutation perm1 = mesh.find_permutation( + elem_top, nodes.data(), elem_top.side_topology(local_side_id), side_nodes.data(), local_side_id); ASSERT_TRUE(perm1 != stk::mesh::Permutation::INVALID_PERMUTATION); // Set up duplicate relations from edge to nodes diff --git a/packages/stk/stk_unit_tests/stk_mesh/UnitTestSelfContainedModCycles.cpp b/packages/stk/stk_unit_tests/stk_mesh/UnitTestSelfContainedModCycles.cpp index e4282b1277a8..356f8704bd87 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/UnitTestSelfContainedModCycles.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/UnitTestSelfContainedModCycles.cpp @@ -58,6 +58,23 @@ TEST(SelfContainedModCycle, create_exposed_block_boundary_sides_throw) stk::mesh::PartVector{})); } +TEST(SelfContainedModCycle, create_all_block_boundary_sides_throw) +{ + MPI_Comm comm = MPI_COMM_WORLD; + const int numProcs = stk::parallel_machine_size(comm); + if(numProcs > 2) { GTEST_SKIP(); } + + stk::mesh::MeshBuilder builder(comm); + std::shared_ptr bulkPtr = builder.create(); + stk::mesh::BulkData& bulk = *bulkPtr; + stk::mesh::MetaData& meta = bulk.mesh_meta_data(); + stk::io::fill_mesh("generated:1x1x2", bulk); + + bulk.modification_begin(); + EXPECT_ANY_THROW(stk::mesh::create_all_block_boundary_sides(bulk, meta.universal_part(), + stk::mesh::PartVector{})); +} + TEST(SelfContainedModCycle, create_exposed_block_boundary_sides_selector_throw) { MPI_Comm comm = MPI_COMM_WORLD; diff --git a/packages/stk/stk_unit_tests/stk_mesh/UnitTestSideSet.cpp b/packages/stk/stk_unit_tests/stk_mesh/UnitTestSideSet.cpp index 68abebee71b4..72847b60280a 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/UnitTestSideSet.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/UnitTestSideSet.cpp @@ -188,6 +188,44 @@ void create_two_elem_block_mesh_with_spanning_sidesets(const std::string& filena } } +TEST(SkinBoundary, check_interior_shared_shell_side) +{ + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 2) { GTEST_SKIP(); } + + std::unique_ptr bulkPtr = stk::mesh::MeshBuilder(MPI_COMM_WORLD) + .set_spatial_dimension(3) + .set_aura_option(stk::mesh::BulkData::NO_AUTO_AURA) + .create(); + stk::mesh::MetaData& meta = bulkPtr->mesh_meta_data(); + meta.use_simple_fields(); + stk::mesh::Part& boundaryPart = meta.declare_part("boundaryPart", meta.side_rank()); + std::string meshDesc = "1,10098,TET_4, 1,2,3,4, block_1\n" + "0,320234,SHELL_TRI_3, 1,3,4, block_2|sideset:name=surface_1; data=10098,3"; + + std::vector coords = {0,0,0, 0,1,0, 1,0,0, 1,1,1}; + + stk::unit_test_util::simple_fields::setup_text_mesh(*bulkPtr, stk::unit_test_util::simple_fields::get_full_text_mesh_desc(meshDesc, coords)); + + stk::mesh::Entity sharedSide = bulkPtr->get_entity(meta.side_rank(), 100983); + ASSERT_TRUE(bulkPtr->is_valid(sharedSide)); + ASSERT_TRUE(bulkPtr->bucket(sharedSide).shared()); + + int owner = bulkPtr->parallel_owner_rank(sharedSide); + int otherProc = 1 - owner; + stk::mesh::EntityProcVec entitiesToChange; + if (owner == bulkPtr->parallel_rank()) { + entitiesToChange = {stk::mesh::EntityProc(sharedSide, otherProc)}; + } + bulkPtr->change_entity_owner(entitiesToChange); + + stk::mesh::create_interior_block_boundary_sides(*bulkPtr, meta.universal_part(), {&boundaryPart}); + + sharedSide = bulkPtr->get_entity(meta.side_rank(), 100983); + EXPECT_TRUE(bulkPtr->bucket(sharedSide).member(boundaryPart)); + + EXPECT_TRUE(stk::mesh::check_interior_block_boundary_sides(*bulkPtr, meta.universal_part(), boundaryPart)); +} + class TestSideSet : public stk::unit_test_util::simple_fields::MeshFixture { protected: @@ -1101,6 +1139,98 @@ TEST_F(SideSetModification, nonEmptyInternalSideset_AfterRemoveOneConnectedEleme EXPECT_EQ(2u, get_bulk().num_elements(side)); } +TEST(CreateAndWrite, DISABLED_textmesh_shell_quad_4_EdgeSides) +{ + std::shared_ptr bulk = stk::mesh::MeshBuilder(MPI_COMM_WORLD).set_spatial_dimension(3).create(); +//shell-quad-4 mesh: +// 6 +// 3*----*----*9 +// | E2 | E4 | +// | | | +// 2*---5*----*8 +// | E1 | E3 | +// | | | +// 1*----*----*7 +// 4 +// + const std::string meshDesc = + "0,1,SHELL_QUAD_4, 1,4,5,2, block_1\n\ + 0,2,SHELL_QUAD_4, 2,5,6,3, block_1\n\ + 0,3,SHELL_QUAD_4, 4,7,8,5, block_1\n\ + 0,4,SHELL_QUAD_4, 5,8,9,6, block_1|sideset:name=surface_1; data=1,3,3,3, 3,4,4,4, 4,5,2,5, 2,6,1,6"; + + std::vector coords = {0,0,0, 0,1,0, 0,2,0, + 1,0,0, 1,1,0, 1,2,0, + 2,0,0, 2,1,0, 2,2,0}; + + stk::unit_test_util::simple_fields::setup_text_mesh(*bulk, stk::unit_test_util::simple_fields::get_full_text_mesh_desc(meshDesc, coords)); + + stk::io::write_mesh("shellq4_edge_sides.g", *bulk); +} + +TEST(CreateAndWrite, DISABLED_textmesh_shell_quad_4_FullExteriorSkin) +{ + std::shared_ptr bulk = stk::mesh::MeshBuilder(MPI_COMM_WORLD).set_spatial_dimension(3).create(); + const std::string meshDesc = + "0,1,SHELL_QUAD_4, 1,4,5,2, block_1\n\ + 0,2,SHELL_QUAD_4, 2,5,6,3, block_1\n\ + 0,3,SHELL_QUAD_4, 4,7,8,5, block_1\n\ + 0,4,SHELL_QUAD_4, 5,8,9,6, block_1|sideset:name=surface_1; data=1,1,2,1,3,1,4,1, 1,2,2,2,3,2,4,2, 1,3,3,3, 3,4,4,4, 4,5,2,5, 2,6,1,6"; + + std::vector coords = {0,0,0, 0,1,0, 0,2,0, + 1,0,0, 1,1,0, 1,2,0, + 2,0,0, 2,1,0, 2,2,0}; + + stk::unit_test_util::simple_fields::setup_text_mesh(*bulk, stk::unit_test_util::simple_fields::get_full_text_mesh_desc(meshDesc, coords)); + + stk::io::write_mesh("shellq4_full_exterior_skin.g", *bulk); +} + +TEST(CreateAndWrite, DISABLED_textmesh_shell_tri_3_EdgeSides) +{ + std::shared_ptr bulk = stk::mesh::MeshBuilder(MPI_COMM_WORLD).set_spatial_dimension(3).create(); +//shell-tri-3 mesh: +// 4 +// 2*---*---*6 +// |E1/|E3/| +// | / | / | +// |/E2|/E4| +// 1*---*---*5 +// 3 +// + const std::string meshDesc = + "0,1,SHELL_TRI_3, 1,4,2, block_1\n\ + 0,2,SHELL_TRI_3, 1,3,4, block_1\n\ + 0,3,SHELL_TRI_3, 3,6,4, block_1\n\ + 0,4,SHELL_TRI_3, 3,5,6, block_1|sideset:name=surface_1; data=2,3,4,3, 4,4, 3,4,1,4, 1,5"; + + std::vector coords = {0,0,0, 0,1,0, + 1,0,0, 1,1,0, + 2,0,0, 2,1,0}; + + stk::unit_test_util::simple_fields::setup_text_mesh(*bulk, stk::unit_test_util::simple_fields::get_full_text_mesh_desc(meshDesc, coords)); + + stk::io::write_mesh("shellt3_edge_sides.g", *bulk); +} + +TEST(CreateAndWrite, DISABLED_textmesh_shell_tri_3_FullExteriorSkin) +{ + std::shared_ptr bulk = stk::mesh::MeshBuilder(MPI_COMM_WORLD).set_spatial_dimension(3).create(); + const std::string meshDesc = + "0,1,SHELL_TRI_3, 1,4,2, block_1\n\ + 0,2,SHELL_TRI_3, 1,3,4, block_1\n\ + 0,3,SHELL_TRI_3, 3,6,4, block_1\n\ + 0,4,SHELL_TRI_3, 3,5,6, block_1|sideset:name=surface_1; data=1,1,2,1,3,1,4,1, 1,2,2,2,3,2,4,2, 2,3,4,3, 4,4, 3,4,1,4, 1,5"; + + std::vector coords = {0,0,0, 0,1,0, + 1,0,0, 1,1,0, + 2,0,0, 2,1,0}; + + stk::unit_test_util::simple_fields::setup_text_mesh(*bulk, stk::unit_test_util::simple_fields::get_full_text_mesh_desc(meshDesc, coords)); + + stk::io::write_mesh("shellt3_full_exterior_skin.g", *bulk); +} + class InternalSideSet : public stk::unit_test_util::simple_fields::MeshFixture { protected: @@ -1331,3 +1461,58 @@ TEST(Skinning, createSidesForBlock) EXPECT_EQ(10u, stk::mesh::count_entities(bulk, stk::topology::FACE_RANK, surface1)); } +TEST(Skinning, createAllSidesForBlock) +{ + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 1) { GTEST_SKIP(); } + + std::shared_ptr bulkPtr = stk::mesh::MeshBuilder(MPI_COMM_WORLD) + .set_spatial_dimension(3) + .create(); + stk::mesh::BulkData& bulk = *bulkPtr; + stk::mesh::MetaData& meta = bulk.mesh_meta_data(); + + stk::mesh::Part& block2 = meta.declare_part_with_topology("block_2", stk::topology::HEX_8); + stk::mesh::Part& surface1 = meta.declare_part_with_topology("surface_1", stk::topology::QUAD_4); + stk::io::put_io_part_attribute(block2); + stk::io::put_io_part_attribute(surface1); + meta.set_part_id(block2, 2); + meta.set_part_id(surface1, 1); + + stk::io::fill_mesh("generated:2x2x2", bulk); + + copy_elems_from_block_to_block(bulk, {1, 2}, "block_1", "block_2"); + + stk::mesh::create_all_block_boundary_sides(bulk, meta.universal_part(), stk::mesh::PartVector{&surface1}); + EXPECT_EQ(28u, stk::mesh::count_entities(bulk, stk::topology::FACE_RANK, surface1)); +} + +TEST(Skinning, createAllSidesForBlock_separatePartForInteriorSides) +{ + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 1) { GTEST_SKIP(); } + + std::shared_ptr bulkPtr = stk::mesh::MeshBuilder(MPI_COMM_WORLD) + .set_spatial_dimension(3) + .create(); + stk::mesh::BulkData& bulk = *bulkPtr; + stk::mesh::MetaData& meta = bulk.mesh_meta_data(); + + stk::mesh::Part& block2 = meta.declare_part_with_topology("block_2", stk::topology::HEX_8); + stk::mesh::Part& surface1 = meta.declare_part_with_topology("surface_1", stk::topology::QUAD_4); + stk::mesh::Part& interiorSkin = meta.declare_part_with_topology("interiorSkin", stk::topology::QUAD_4); + stk::io::put_io_part_attribute(block2); + stk::io::put_io_part_attribute(surface1); + stk::io::put_io_part_attribute(interiorSkin); + meta.set_part_id(block2, 2); + meta.set_part_id(surface1, 1); + meta.set_part_id(interiorSkin, 2); + + stk::io::fill_mesh("generated:2x2x2", bulk); + + copy_elems_from_block_to_block(bulk, {1, 2}, "block_1", "block_2"); + + stk::mesh::PartVector interiorSkinParts = {&interiorSkin}; + stk::mesh::create_all_block_boundary_sides(bulk, meta.universal_part(), stk::mesh::PartVector{&surface1}, &interiorSkinParts); + EXPECT_EQ(24u, stk::mesh::count_entities(bulk, stk::topology::FACE_RANK, surface1)); + EXPECT_EQ(4u, stk::mesh::count_entities(bulk, stk::topology::FACE_RANK, interiorSkin)); +} + diff --git a/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpDebugFieldSync_Fixtures.hpp b/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpDebugFieldSync_Fixtures.hpp index 703a3d21c82f..0a2ac0b77900 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpDebugFieldSync_Fixtures.hpp +++ b/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpDebugFieldSync_Fixtures.hpp @@ -598,14 +598,13 @@ class NgpDebugFieldSyncFixture : public stk::unit_test_util::simple_fields::Mesh for (unsigned i = 0; i < bucketIds.size(); ++i) { const stk::mesh::NgpMesh::BucketType & bucket = ngpMesh.get_bucket(rank, bucketIds.device_get(i)); for (unsigned j = 0; j < bucket.size(); ++j) { - stk::mesh::NgpMesh::MeshIndex meshIndex{&bucket, static_cast(j)}; stk::mesh::FastMeshIndex fastMeshIndex{bucket.bucket_id(), static_cast(j)}; const unsigned numComponents = ngpField.get_num_components_per_entity(fastMeshIndex); for (unsigned component = 0; component < numComponents; ++component) { #if defined(DEVICE_USE_LOCATION_BUILTINS) - access_for_memory_checking_tool(&ngpField(meshIndex, component)); + access_for_memory_checking_tool(&ngpField(fastMeshIndex, component)); #else - access_for_memory_checking_tool(&ngpField(meshIndex, component, __FILE__, __LINE__)); + access_for_memory_checking_tool(&ngpField(fastMeshIndex, component, __FILE__, __LINE__)); #endif } } diff --git a/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpMeshTest.cpp b/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpMeshTest.cpp index d309d71ef336..5ab49ea6f829 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpMeshTest.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/ngp/NgpMeshTest.cpp @@ -1,45 +1,36 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. // -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. // -// Questions Contact H. Carter Edwards (hcedwar@sandia.gov) +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -// ************************************************************************ -//@HEADER -*/ #include "stk_ngp_test/ngp_test.hpp" #include diff --git a/packages/stk/stk_unit_tests/stk_mesh/ngp/TestNgpMeshUpdate.cpp b/packages/stk/stk_unit_tests/stk_mesh/ngp/TestNgpMeshUpdate.cpp index 1847f2d6cd2a..dce5b47cc818 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/ngp/TestNgpMeshUpdate.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/ngp/TestNgpMeshUpdate.cpp @@ -18,35 +18,33 @@ class UpdateNgpMesh : public stk::unit_test_util::simple_fields::MeshFixture void setup_test_mesh() { setup_empty_mesh(stk::mesh::BulkData::NO_AUTO_AURA); + extraPart = &get_meta().declare_part("extraPart"); std::string meshDesc = "0,1,HEX_8,1,2,3,4,5,6,7,8\n"; stk::unit_test_util::simple_fields::setup_text_mesh(get_bulk(), meshDesc); } + const stk::mesh::Part* extraPart = nullptr; }; -TEST_F(UpdateNgpMesh, lazyAutoUpdate) +TEST_F(UpdateNgpMesh, explicitUpdate) { + if (stk::parallel_machine_size(MPI_COMM_WORLD) > 1) { GTEST_SKIP(); } setup_test_mesh(); - // Don't store persistent pointers/references if you want automatic updates - // when acquiring an NgpMesh from BulkData stk::mesh::NgpMesh * ngpMesh = &stk::mesh::get_updated_ngp_mesh(get_bulk()); get_bulk().modification_begin(); + stk::mesh::Entity node1 = get_bulk().get_entity(stk::topology::NODE_RANK, 1); + get_bulk().change_entity_parts(node1, stk::mesh::ConstPartVector{extraPart}); get_bulk().modification_end(); -#ifdef STK_USE_DEVICE_MESH EXPECT_FALSE(ngpMesh->is_up_to_date()); ngpMesh = &stk::mesh::get_updated_ngp_mesh(get_bulk()); EXPECT_TRUE(ngpMesh->is_up_to_date()); -#else - EXPECT_TRUE(ngpMesh->is_up_to_date()); - ngpMesh = &stk::mesh::get_updated_ngp_mesh(get_bulk()); - EXPECT_TRUE(ngpMesh->is_up_to_date()); -#endif } -TEST_F(UpdateNgpMesh, manualUpdate) +TEST_F(UpdateNgpMesh, referenceGetsUpdated) { + if (stk::parallel_machine_size(MPI_COMM_WORLD) > 1) { GTEST_SKIP(); } setup_test_mesh(); // If storing a persistent reference, call the get_updated_ngp_mesh() function @@ -54,17 +52,13 @@ TEST_F(UpdateNgpMesh, manualUpdate) stk::mesh::NgpMesh & ngpMesh = stk::mesh::get_updated_ngp_mesh(get_bulk()); get_bulk().modification_begin(); + stk::mesh::Entity node1 = get_bulk().get_entity(stk::topology::NODE_RANK, 1); + get_bulk().change_entity_parts(node1, stk::mesh::ConstPartVector{extraPart}); get_bulk().modification_end(); -#ifdef STK_USE_DEVICE_MESH EXPECT_FALSE(ngpMesh.is_up_to_date()); stk::mesh::get_updated_ngp_mesh(get_bulk()); // Trigger update EXPECT_TRUE(ngpMesh.is_up_to_date()); -#else - EXPECT_TRUE(ngpMesh.is_up_to_date()); - stk::mesh::get_updated_ngp_mesh(get_bulk()); // Trigger update - EXPECT_TRUE(ngpMesh.is_up_to_date()); -#endif } TEST_F(UpdateNgpMesh, OnlyOneDeviceMesh_InternalAndExternal) diff --git a/packages/stk/stk_unit_tests/stk_mesh/ngp/UnitTestNgp.cpp b/packages/stk/stk_unit_tests/stk_mesh/ngp/UnitTestNgp.cpp new file mode 100644 index 000000000000..0472a7af4b76 --- /dev/null +++ b/packages/stk/stk_unit_tests/stk_mesh/ngp/UnitTestNgp.cpp @@ -0,0 +1,70 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "NgpUnitTestUtils.hpp" + +namespace { + +using IntDualViewType = Kokkos::DualView; + +void test_view_of_fields(const stk::mesh::BulkData& bulk, + stk::mesh::Field& field1, + stk::mesh::Field& field2) +{ + using FieldViewType = Kokkos::View*,stk::ngp::MemSpace>; + + FieldViewType fields(Kokkos::ViewAllocateWithoutInitializing("fields"),2); + FieldViewType::HostMirror hostFields = Kokkos::create_mirror_view(fields); + + Kokkos::parallel_for(stk::ngp::DeviceRangePolicy(0, 2), + KOKKOS_LAMBDA(const unsigned& i) + { + new (&fields(i)) stk::mesh::NgpField(); + }); + + hostFields(0) = stk::mesh::get_updated_ngp_field(field1); + hostFields(1) = stk::mesh::get_updated_ngp_field(field2); + + Kokkos::deep_copy(fields, hostFields); + + unsigned numResults = 2; + IntDualViewType result = ngp_unit_test_utils::create_dualview("result",numResults); + + Kokkos::parallel_for(stk::ngp::DeviceRangePolicy(0, 2), + KOKKOS_LAMBDA(const unsigned& i) + { + result.d_view(i) = fields(i).get_ordinal() == i ? 1 : 0; + }); + + result.modify(); + result.sync(); + + EXPECT_EQ(1, result.h_view(0)); + EXPECT_EQ(1, result.h_view(1)); +} + +TEST(UnitTestNgp, viewOfFields) +{ + std::shared_ptr bulk = stk::mesh::MeshBuilder(MPI_COMM_WORLD).create(); + stk::mesh::MetaData& meta = bulk->mesh_meta_data(); + auto &field1 = meta.declare_field(stk::topology::NODE_RANK, "myField1"); + auto &field2 = meta.declare_field(stk::topology::NODE_RANK, "myField2"); + stk::mesh::put_field_on_mesh(field1, meta.universal_part(), nullptr); + stk::mesh::put_field_on_mesh(field2, meta.universal_part(), nullptr); + stk::io::fill_mesh("generated:1x1x4|sideset:zZ", *bulk); + + test_view_of_fields(*bulk, field1, field2); +} + +} diff --git a/packages/stk/stk_unit_tests/stk_mesh/ngp/UnitTestNgpDebugFieldSync.cpp b/packages/stk/stk_unit_tests/stk_mesh/ngp/UnitTestNgpDebugFieldSync.cpp index d6103d8be7b4..1168fb0fcd1a 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/ngp/UnitTestNgpDebugFieldSync.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/ngp/UnitTestNgpDebugFieldSync.cpp @@ -213,7 +213,11 @@ class NgpDebugFieldSync : public NgpDebugFieldSyncFixture for (unsigned i = 0; i < bucketIds.size(); ++i) { const stk::mesh::NgpMesh::BucketType & bucket = ngpMesh.get_bucket(rank, bucketIds.device_get(i)); for (unsigned j = 0; j < bucket.size(); ++j) { +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after April 2024 stk::mesh::NgpMesh::MeshIndex index{&bucket, static_cast(j)}; +#else + stk::mesh::NgpMesh::MeshIndex index{bucket.bucket_id(), static_cast(j)}; +#endif ngpField(index, component) = value; } } diff --git a/packages/stk/stk_unit_tests/stk_mesh/ngp/UnitTestNgpDebugFieldSync_PartialAllocation.cpp b/packages/stk/stk_unit_tests/stk_mesh/ngp/UnitTestNgpDebugFieldSync_PartialAllocation.cpp index ee19c2a107ae..f71a0b3386ab 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/ngp/UnitTestNgpDebugFieldSync_PartialAllocation.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/ngp/UnitTestNgpDebugFieldSync_PartialAllocation.cpp @@ -72,7 +72,11 @@ class NgpDebugFieldSync_PartialAllocation : public NgpDebugFieldSyncFixture for (unsigned i = 0; i < bucketIds.size(); ++i) { const stk::mesh::NgpMesh::BucketType & bucket = ngpMesh.get_bucket(rank, bucketIds.device_get(i)); for (unsigned j = 0; j < bucket.size(); ++j) { +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after April 2024 stk::mesh::NgpMesh::MeshIndex index{&bucket, static_cast(j)}; +#else + stk::mesh::NgpMesh::MeshIndex index{bucket.bucket_id(), static_cast(j)}; +#endif ngpField(index, component) = value; } } diff --git a/packages/stk/stk_unit_tests/stk_mesh/ngp/ngpFieldTest.cpp b/packages/stk/stk_unit_tests/stk_mesh/ngp/ngpFieldTest.cpp index fc4ef93b3b4b..294090ee5709 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/ngp/ngpFieldTest.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/ngp/ngpFieldTest.cpp @@ -1228,20 +1228,24 @@ TEST_F(NgpFieldFixture, blas_field_copy_device_to_device) EXPECT_FALSE(stkField1->need_sync_to_device()); EXPECT_FALSE(stkField2->need_sync_to_device()); - stk::mesh::NgpField ngpField1 = stk::mesh::get_updated_ngp_field(*stkField1); - stk::mesh::NgpField ngpField2 = stk::mesh::get_updated_ngp_field(*stkField2); const double myConstantValue = 97.9; stk::mesh::field_fill(myConstantValue, *stkField1, stk::ngp::ExecSpace()); +#ifdef STK_ENABLE_GPU + stk::mesh::NgpField& ngpField1 = stk::mesh::get_updated_ngp_field(*stkField1); EXPECT_TRUE(ngpField1.need_sync_to_host()); +#endif stk::mesh::field_copy(*stkField1, *stkField2); +#ifdef STK_ENABLE_GPU EXPECT_TRUE(stkField1->need_sync_to_host()); EXPECT_TRUE(stkField2->need_sync_to_host()); +#endif stk::mesh::Selector selector(*stkField2); + stk::mesh::NgpField& ngpField2 = stk::mesh::get_updated_ngp_field(*stkField2); check_field_data_on_device(ngpMesh, ngpField2, selector, myConstantValue); } @@ -1259,7 +1263,7 @@ TEST_F(NgpFieldFixture, blas_field_fill_device) EXPECT_FALSE(stkField1->need_sync_to_host()); EXPECT_FALSE(stkField1->need_sync_to_device()); - stk::mesh::NgpField ngpField1 = stk::mesh::get_updated_ngp_field(*stkField1); + stk::mesh::NgpField& ngpField1 = stk::mesh::get_updated_ngp_field(*stkField1); ngpField1.set_all(ngpMesh, 97.9); @@ -1268,10 +1272,23 @@ TEST_F(NgpFieldFixture, blas_field_fill_device) const double myConstantValue = 55.5; stk::mesh::field_fill(myConstantValue, *stkField1, stk::ngp::ExecSpace()); +#ifdef STK_ENABLE_GPU EXPECT_TRUE(stkField1->need_sync_to_host()); +#else + EXPECT_TRUE(stkField1->need_sync_to_device()); +#endif + +#if defined(STK_USE_DEVICE_MESH) && !defined(STK_ENABLE_GPU) + EXPECT_TRUE(stkField1->need_sync_to_device()); + stkField1->sync_to_device(); +#endif stk::mesh::Selector selector(*stkField1); +#ifdef STK_ENABLE_GPU check_field_data_on_device(ngpMesh, ngpField1, selector, myConstantValue); +#else + check_field_data_on_host(get_bulk(), *stkField1, selector, myConstantValue); +#endif } TEST_F(NgpFieldFixture, blas_field_fill_host_ngp) diff --git a/packages/stk/stk_unit_tests/stk_mesh/ngp/ngpMultiStateFieldTests.cpp b/packages/stk/stk_unit_tests/stk_mesh/ngp/ngpMultiStateFieldTests.cpp index e582b9ee0208..28dc60b1f8bf 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/ngp/ngpMultiStateFieldTests.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/ngp/ngpMultiStateFieldTests.cpp @@ -1,45 +1,36 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. // -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. // -// Questions Contact H. Carter Edwards (hcedwar@sandia.gov) +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -// ************************************************************************ -//@HEADER -*/ #include #include @@ -77,7 +68,7 @@ class ClassWithNgpField stk::mesh::NgpField m_ngpField; }; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) +#ifdef STK_ENABLE_GPU #define MY_LAMBDA KOKKOS_LAMBDA #else #define MY_LAMBDA [&] @@ -201,6 +192,36 @@ class NgpMultiStateFieldTest : public stk::mesh::fixtures::simple_fields::TestHe stk::mesh::Field* m_fieldOld; }; +NGP_TEST_F(NgpMultiStateFieldTest, multistateField_rotateDeviceStates_syncStatesUnchanged) +{ + if (get_parallel_size() != 1) GTEST_SKIP(); + setup_multistate_field(); + setup_mesh(2, 2, 2); + + const double valueNew = 44.4; + const double valueOld = 22.2; + stk::mesh::field_fill(valueNew, get_field_new()); + stk::mesh::field_fill(valueOld, get_field_old()); + + stk::mesh::NgpMesh& ngpMesh = stk::mesh::get_updated_ngp_mesh(get_bulk()); + stk::mesh::NgpField& ngpFieldNew = stk::mesh::get_updated_ngp_field(get_field_new()); + stk::mesh::NgpField& ngpFieldOld = stk::mesh::get_updated_ngp_field(get_field_old()); + + check_field_data_value_on_device(ngpMesh, ngpFieldNew, valueNew); + check_field_data_value_on_device(ngpMesh, ngpFieldOld, valueOld); + + const bool rotateDeviceNgpFieldStates = true; + get_bulk().update_field_data_states(rotateDeviceNgpFieldStates); + + EXPECT_FALSE(ngpFieldNew.need_sync_to_host()); + EXPECT_FALSE(ngpFieldOld.need_sync_to_host()); + EXPECT_FALSE(ngpFieldNew.need_sync_to_device()); + EXPECT_FALSE(ngpFieldOld.need_sync_to_device()); + + check_field_data_value_on_device(ngpMesh, ngpFieldNew, valueOld); + check_field_data_value_on_device(ngpMesh, ngpFieldOld, valueNew); +} + NGP_TEST_F(NgpMultiStateFieldTest, multistateField_copyHasCorrectDataAfterStateRotation) { if (get_parallel_size() != 1) GTEST_SKIP(); @@ -231,6 +252,39 @@ NGP_TEST_F(NgpMultiStateFieldTest, multistateField_copyHasCorrectDataAfterStateR check_field_data_value_on_device(ngpMesh, copyOfNgpFieldNew, valueOld); } +NGP_TEST_F(NgpMultiStateFieldTest, multistateField_copyHasWrongDataAfterDeviceStateRotation) +{ + if (get_parallel_size() != 1) GTEST_SKIP(); + setup_multistate_field(); + setup_mesh(2, 2, 2); + + const double valueNew = 44.4; + const double valueOld = 22.2; + stk::mesh::field_fill(valueNew, get_field_new()); + stk::mesh::field_fill(valueOld, get_field_old()); + + stk::mesh::NgpMesh& ngpMesh = stk::mesh::get_updated_ngp_mesh(get_bulk()); + stk::mesh::NgpField& ngpFieldNew = stk::mesh::get_updated_ngp_field(get_field_new()); + stk::mesh::NgpField& ngpFieldOld = stk::mesh::get_updated_ngp_field(get_field_old()); + stk::mesh::NgpField copyOfNgpFieldNew(ngpFieldNew); + + check_field_data_value_on_device(ngpMesh, ngpFieldNew, valueNew); + check_field_data_value_on_device(ngpMesh, ngpFieldOld, valueOld); + + const bool rotateDeviceNgpFieldStates = true; + get_bulk().update_field_data_states(rotateDeviceNgpFieldStates); + + check_field_data_value_on_device(ngpMesh, ngpFieldNew, valueOld); + check_field_data_value_on_device(ngpMesh, ngpFieldOld, valueNew); + +#ifdef STK_USE_DEVICE_MESH + const double valueNewBecauseCopyOfDeviceFieldGotDisconnectedByDeviceRotation = valueNew; + check_field_data_value_on_device(ngpMesh, copyOfNgpFieldNew, valueNewBecauseCopyOfDeviceFieldGotDisconnectedByDeviceRotation); +#else + check_field_data_value_on_device(ngpMesh, copyOfNgpFieldNew, valueOld); +#endif +} + NGP_TEST_F(NgpMultiStateFieldTest, persistentDeviceField_hasCorrectDataAfterStateRotation) { if (get_parallel_size() != 1) GTEST_SKIP(); @@ -260,6 +314,42 @@ NGP_TEST_F(NgpMultiStateFieldTest, persistentDeviceField_hasCorrectDataAfterStat delete_class_on_device(persistentDeviceClass); } +NGP_TEST_F(NgpMultiStateFieldTest, persistentDeviceField_hasWrongDataAfterDeviceStateRotation) +{ + if (get_parallel_size() != 1) GTEST_SKIP(); + setup_multistate_field(); + setup_mesh(2, 2, 2); + + const double valueNew = 44.4; + const double valueOld = 22.2; + stk::mesh::field_fill(valueNew, get_field_new()); + stk::mesh::field_fill(valueOld, get_field_old()); + + stk::mesh::NgpMesh& ngpMesh = stk::mesh::get_updated_ngp_mesh(get_bulk()); + stk::mesh::NgpField& ngpFieldNew = stk::mesh::get_updated_ngp_field(get_field_new()); + stk::mesh::NgpField& ngpFieldOld = stk::mesh::get_updated_ngp_field(get_field_old()); + + ClassWithNgpField* persistentDeviceClass = create_class_on_device(ngpFieldNew); + + check_field_data_value_on_device(ngpMesh, stk::topology::NODE_RANK, persistentDeviceClass, valueNew); + + const bool rotateDeviceNgpFieldStates = true; + get_bulk().update_field_data_states(rotateDeviceNgpFieldStates); + + ngpFieldNew.sync_to_device(); + ngpFieldOld.sync_to_device(); + + check_field_data_value_on_device(ngpMesh, ngpFieldNew, valueOld); +#ifdef STK_USE_DEVICE_MESH + const double valueNewBecausePersistentDeviceFieldGotDisconnectedByDeviceRotation = valueNew; + check_field_data_value_on_device(ngpMesh, stk::topology::NODE_RANK, persistentDeviceClass, valueNewBecausePersistentDeviceFieldGotDisconnectedByDeviceRotation); +#else + check_field_data_value_on_device(ngpMesh, stk::topology::NODE_RANK, persistentDeviceClass, valueOld); +#endif + + delete_class_on_device(persistentDeviceClass); +} + NGP_TEST_F(NgpMultiStateFieldTest, persistentSyncToDeviceCountAfterStateRotation) { if (get_parallel_size() != 1) GTEST_SKIP(); diff --git a/packages/stk/stk_unit_tests/stk_middle_mesh/CMakeLists.txt b/packages/stk/stk_unit_tests/stk_middle_mesh/CMakeLists.txt index 0f96faa56128..cd48669ad8d3 100644 --- a/packages/stk/stk_unit_tests/stk_middle_mesh/CMakeLists.txt +++ b/packages/stk/stk_unit_tests/stk_middle_mesh/CMakeLists.txt @@ -17,9 +17,9 @@ if(HAVE_STK_Trilinos) else() add_executable(stk_middle_mesh_utest ${SOURCES}) target_link_libraries(stk_middle_mesh_utest stk_middle_mesh) - target_link_libraries(stk_coupling_utest stk_unit_test_utils) - target_link_libraries(stk_coupling_utest stk_util_parallel) - target_link_libraries(stk_coupling_utest stk_unit_main) + target_link_libraries(stk_middle_mesh_utest stk_unit_test_utils) + target_link_libraries(stk_middle_mesh_utest stk_util_parallel) + target_link_libraries(stk_middle_mesh_utest stk_unit_main) add_test(NAME "stk_middle_mesh_utest" COMMAND stk_middle_mesh_utest) endif() diff --git a/packages/stk/stk_unit_tests/stk_middle_mesh/test_mesh.cpp b/packages/stk/stk_unit_tests/stk_middle_mesh/test_mesh.cpp index 5dfe30f3db19..df0ff7ac9079 100644 --- a/packages/stk/stk_unit_tests/stk_middle_mesh/test_mesh.cpp +++ b/packages/stk/stk_unit_tests/stk_middle_mesh/test_mesh.cpp @@ -52,25 +52,31 @@ GeoClassificationE get_edge_geo_classification(const MeshSpec& spec, MeshEntityP return static_cast(std::max(static_cast(g1), static_cast(g2))); } -MeshEntityPtr get_closest_vert(std::shared_ptr mesh, const utils::Point& pt) +MeshEntityPtr get_closest_entity(std::shared_ptr mesh, int dim, const utils::Point& pt, double tol=1e-13) { - MeshEntityPtr closestVert = nullptr; + MeshEntityPtr closestEntity = nullptr; double closestDistance = std::numeric_limits::max(); - for (auto vert : mesh->get_vertices()) - if (vert) + for (auto entity : mesh->get_mesh_entities(dim)) + if (entity) { - auto ptVert = vert->get_point_orig(0); + auto ptVert = mesh::compute_centroid(entity); double distSquared = dot(pt - ptVert, pt - ptVert); - if (distSquared < closestDistance) + if (distSquared < closestDistance && std::sqrt(distSquared) < tol) { closestDistance = distSquared; - closestVert = vert; + closestEntity = entity; } } - return closestVert; + return closestEntity; +} + +MeshEntityPtr get_closest_vert(std::shared_ptr mesh, const utils::Point& pt) +{ + return get_closest_entity(mesh, 0, pt, 1); } + void expect_near(const utils::Point& pt1, const utils::Point& pt2, double tol) { EXPECT_NEAR(pt1.x, pt2.x, tol); @@ -588,6 +594,114 @@ TEST(Mesh, ErrorChecking) EXPECT_NO_THROW(check_coordinate_field(mesh1)); } +TEST(Mesh, SetDownOrientation) +{ + if (utils::impl::comm_size(MPI_COMM_WORLD) != 1) + GTEST_SKIP(); + + MeshSpec spec; + spec.numelX = 1; + spec.numelY = 1; + spec.xmin = 0; + spec.xmax = 1; + spec.ymin = 0; + spec.ymax = 1; + auto func = [&](const utils::Point& pt) { return pt; }; + + std::shared_ptr mesh = create_mesh(spec, func); + + mesh::MeshEntityPtr el1 = get_closest_entity(mesh, 2, {0.5, 0.5}); + + el1->set_down_orientation(0, mesh::EntityOrientation::Standard); + EXPECT_EQ(el1->get_down_orientation(0), mesh::EntityOrientation::Standard); + + el1->set_down_orientation(0, mesh::EntityOrientation::Reversed); + EXPECT_EQ(el1->get_down_orientation(0), mesh::EntityOrientation::Reversed); +} + +TEST(Mesh, ReplaceDown) +{ + if (utils::impl::comm_size(MPI_COMM_WORLD) != 1) + GTEST_SKIP(); + + MeshSpec spec; + spec.numelX = 1; + spec.numelY = 2; + spec.xmin = 0; + spec.xmax = 1; + spec.ymin = 0; + spec.ymax = 1; + auto func = [&](const utils::Point& pt) { return pt; }; + + std::shared_ptr mesh = create_mesh(spec, func); + + mesh::MeshEntityPtr el1 = get_closest_entity(mesh, 2, {0.5, 0.25}); + + mesh::MeshEntityPtr v1 = mesh->create_vertex(0, 0.5, 0); + mesh::MeshEntityPtr v2 = mesh->create_vertex(0.5, 0.5, 0); + + mesh::MeshEntityPtr newEdge = mesh->create_edge(v1, v2); + + el1->replace_down(2, newEdge, mesh::EntityOrientation::Reversed); + EXPECT_EQ(el1->get_down(2), newEdge); + EXPECT_EQ(el1->get_down_orientation(2), mesh::EntityOrientation::Reversed); +} + +TEST(Mesh, EntityOrientationReverse) +{ + EXPECT_EQ(reverse(mesh::EntityOrientation::Standard), mesh::EntityOrientation::Reversed); + EXPECT_EQ(reverse(mesh::EntityOrientation::Reversed), mesh::EntityOrientation::Standard); +} + +TEST(Mesh, ReverseEdge) +{ + if (utils::impl::comm_size(MPI_COMM_WORLD) != 1) + GTEST_SKIP(); + + MeshSpec spec; + spec.numelX = 1; + spec.numelY = 2; + spec.xmin = 0; + spec.xmax = 1; + spec.ymin = 0; + spec.ymax = 1; + auto func = [&](const utils::Point& pt) { return pt; }; + + std::shared_ptr mesh = create_mesh(spec, func); + + mesh::MeshEntityPtr edge = get_closest_entity(mesh, 1, {0.5, 0.5}); + mesh::MeshEntityPtr el1 = get_closest_entity(mesh, 2, {0.5, 0.25}); + mesh::MeshEntityPtr el2 = get_closest_entity(mesh, 2, {0.5, 0.75}); + mesh::MeshEntityPtr v0 = edge->get_down(0); + mesh::MeshEntityPtr v1 = edge->get_down(1); + mesh::EntityOrientation orient1 = el1->get_down_orientation(2); + mesh::EntityOrientation orient2 = el2->get_down_orientation(0); + + std::array verts1, verts2; + mesh::get_downward(el1, 0, verts1.data()); + mesh::get_downward(el2, 0, verts2.data()); + + mesh::reverse_edge(edge); + + EXPECT_EQ(edge->get_down(0), v1); + EXPECT_EQ(edge->get_down(1), v0); + EXPECT_EQ(el1->get_down(2), edge); + EXPECT_EQ(el2->get_down(0), edge); + EXPECT_EQ(el1->get_down_orientation(2), reverse(orient1)); + EXPECT_EQ(el2->get_down_orientation(0), reverse(orient2)); + + + std::array verts1After, verts2After; + mesh::get_downward(el1, 0, verts1After.data()); + mesh::get_downward(el2, 0, verts2After.data()); + + for (int i=0; i < 4; ++i) + { + EXPECT_EQ(verts1[i], verts1After[i]); + EXPECT_EQ(verts2[i], verts2After[i]); + } +} + TEST(Mesh, DeleteFace) { if (utils::impl::comm_size(MPI_COMM_WORLD) != 1) diff --git a/packages/stk/stk_unit_tests/stk_search/CMakeLists.txt b/packages/stk/stk_unit_tests/stk_search/CMakeLists.txt index 5ee4b571ac61..082359a33bc5 100644 --- a/packages/stk/stk_unit_tests/stk_search/CMakeLists.txt +++ b/packages/stk/stk_unit_tests/stk_search/CMakeLists.txt @@ -56,6 +56,7 @@ else() target_link_libraries(stk_search_utest stk_unit_test_utils) target_link_libraries(stk_search_utest stk_util_diag) target_link_libraries(stk_search_utest stk_util_parallel) + target_link_libraries(stk_search_utest stk_util_util) target_link_libraries(stk_search_utest stk_unit_main) add_test(NAME "stk_search_utest" COMMAND stk_search_utest) diff --git a/packages/stk/stk_unit_tests/stk_search/UnitTestCoarseSearch.cpp b/packages/stk/stk_unit_tests/stk_search/UnitTestCoarseSearch.cpp index 403f4ffe173d..505105ccffd5 100644 --- a/packages/stk/stk_unit_tests/stk_search/UnitTestCoarseSearch.cpp +++ b/packages/stk/stk_unit_tests/stk_search/UnitTestCoarseSearch.cpp @@ -32,15 +32,18 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // +#include +#include +#include +#include +#include +#include #include - #include #include #include #include #include -#include -#include #include #include @@ -58,6 +61,44 @@ std::ostream & operator<<(std::ostream & out, std::pair +void expect_search_results_with_views(int num_procs, int proc_id, SearchResultsView const& searchResults) +{ + using IdentProc = Ident; + using IdentIntersection = stk::search::IdentIntersection; + + if (num_procs == 1) { + ASSERT_EQ(searchResults.extent(0), 2u); + EXPECT_EQ(searchResults[0], (IdentIntersection{Ident(0, 0), Ident(2, 0)}) ); + EXPECT_EQ(searchResults[1], (IdentIntersection{Ident(1, 0), Ident(3, 0)}) ); + } else { + if (proc_id == 0) { + ASSERT_EQ(searchResults.size(), 4u); + EXPECT_EQ(searchResults[0], (IdentIntersection{Ident(0, 0), Ident(2, 0)}) ); + EXPECT_EQ(searchResults[1], (IdentIntersection{Ident(1, 0), Ident(3, 0)}) ); + EXPECT_EQ(searchResults[2], (IdentIntersection{Ident(4, 1), Ident(2, 0)}) ); + EXPECT_EQ(searchResults[3], (IdentIntersection{Ident(5, 1), Ident(3, 0)}) ); + } else if (proc_id == num_procs - 1) { + ASSERT_EQ(searchResults.size(), 4u); + int prev = proc_id - 1; + EXPECT_EQ(searchResults[0], (IdentIntersection{Ident(proc_id * 4, proc_id), Ident(prev * 4 + 2, prev)}) ); + EXPECT_EQ(searchResults[1], (IdentIntersection{Ident(proc_id * 4, proc_id), Ident(proc_id * 4 + 2, proc_id)}) ); + EXPECT_EQ(searchResults[2], (IdentIntersection{Ident(proc_id * 4 + 1, proc_id), Ident(prev * 4 + 3, prev)}) ); + EXPECT_EQ(searchResults[3], (IdentIntersection{Ident(proc_id * 4 + 1, proc_id), Ident(proc_id * 4 + 3, proc_id)}) ); + } else { + ASSERT_EQ(searchResults.size(), 6u); + int prev = proc_id - 1; + int next = proc_id + 1; + EXPECT_EQ(searchResults[0], (IdentIntersection{Ident(proc_id * 4, proc_id), Ident(prev * 4 + 2, prev)}) ); + EXPECT_EQ(searchResults[1], (IdentIntersection{Ident(proc_id * 4, proc_id), Ident(proc_id * 4 + 2, proc_id)}) ); + EXPECT_EQ(searchResults[2], (IdentIntersection{Ident(proc_id * 4 + 1, proc_id), Ident(prev * 4 + 3, prev)}) ); + EXPECT_EQ(searchResults[3], (IdentIntersection{Ident(proc_id * 4 + 1, proc_id), Ident(proc_id * 4 + 3, proc_id)}) ); + EXPECT_EQ(searchResults[4], (IdentIntersection{Ident(next * 4, next), Ident(proc_id * 4 + 2, proc_id)}) ); + EXPECT_EQ(searchResults[5], (IdentIntersection{Ident(next * 4 + 1, next), Ident(proc_id * 4 + 3, proc_id)}) ); + } + } +} + void expect_search_results(int num_procs, int proc_id, const SearchResults& searchResults) { if (num_procs == 1) { @@ -92,40 +133,238 @@ void expect_search_results(int num_procs, int proc_id, const SearchResults& sea } } +template +void test_coarse_search_for_algorithm_with_views(stk::search::SearchMethod algorithm, MPI_Comm comm) +{ + int num_procs = stk::parallel_machine_size(comm); + int proc_id = stk::parallel_machine_rank(comm); + + using ExecSpace = Kokkos::DefaultExecutionSpace; + using BoxType = stk::search::Box; + using PointType = stk::search::Point; + using IdentProc = Ident; + using BoxIdentProcType = stk::search::BoxIdentProc; + using BoxIdentProcViewType = Kokkos::View; + using SearchResultsViewType = Kokkos::View*, ExecSpace>; + + BoxType box; + IdentProc identProc; + auto domain = BoxIdentProcViewType("domain test view", 2); + auto range = BoxIdentProcViewType("range test view", 2); + + auto domainHost = Kokkos::create_mirror_view_and_copy(ExecSpace{}, domain); + auto rangeHost = Kokkos::create_mirror_view_and_copy(ExecSpace{}, range); + + box = BoxType( PointType(proc_id + 0.1, 0.0, 0.0), PointType(proc_id + 0.9, 1.0, 1.0)); + identProc = IdentProc(proc_id * 4, proc_id); + domainHost(0) = BoxIdentProcType{box, identProc}; + + box = BoxType( PointType(proc_id + 0.1, 2.0, 0.0), PointType(proc_id + 0.9, 3.0, 1.0)); + identProc = IdentProc(proc_id * 4+1, proc_id); + domainHost(1) = BoxIdentProcType{box, identProc}; + + box = BoxType( PointType(proc_id + 0.6, 0.5, 0.0), PointType(proc_id + 1.4, 1.5, 1.0)); + identProc = IdentProc(proc_id * 4+2, proc_id); + rangeHost(0) = BoxIdentProcType{box, identProc}; + + box = BoxType( PointType(proc_id + 0.6, 2.5, 0.0), PointType(proc_id + 1.4, 3.5, 1.0)); + identProc = IdentProc(proc_id * 4+3, proc_id); + rangeHost(1) = BoxIdentProcType{box, identProc}; + + Kokkos::deep_copy(domain, domainHost); + Kokkos::deep_copy(range, rangeHost); + + SearchResultsViewType searchResults; + + stk::search::coarse_search(domain, range, algorithm, comm, searchResults); + + auto searchResultsHost = Kokkos::create_mirror_view_and_copy(ExecSpace{}, searchResults); + + expect_search_results_with_views(num_procs, proc_id, searchResultsHost); +} + +template void test_coarse_search_for_algorithm(stk::search::SearchMethod algorithm, MPI_Comm comm) { int num_procs = stk::parallel_machine_size(comm); int proc_id = stk::parallel_machine_rank(comm); - BoxVector local_domain, local_range; - // what if identifier is NOT unique + using BoxType = stk::search::Box; + using PointType = stk::search::Point; + using BoxIdentProcVectorType = std::vector>; - StkBox box; - Ident id; + BoxIdentProcVectorType domain, range; + BoxType box; + Ident identProc; - box = StkBox( Point(proc_id + 0.1, 0.0, 0.0), Point(proc_id + 0.9, 1.0, 1.0)); - id = Ident(proc_id * 4, proc_id); - local_domain.push_back(std::make_pair(box,id)); + box = BoxType( PointType(proc_id + 0.1, 0.0, 0.0), PointType(proc_id + 0.9, 1.0, 1.0)); + identProc = Ident(proc_id * 4, proc_id); + domain.push_back(std::make_pair(box, identProc)); - box = StkBox( Point(proc_id + 0.1, 2.0, 0.0), Point(proc_id + 0.9, 3.0, 1.0)); - id = Ident(proc_id * 4+1, proc_id); - local_domain.push_back(std::make_pair(box,id)); + box = BoxType( PointType(proc_id + 0.1, 2.0, 0.0), PointType(proc_id + 0.9, 3.0, 1.0)); + identProc = Ident(proc_id * 4+1, proc_id); + domain.push_back(std::make_pair(box, identProc)); - box = StkBox( Point(proc_id + 0.6, 0.5, 0.0), Point(proc_id + 1.4, 1.5, 1.0)); - id = Ident(proc_id * 4+2, proc_id); - local_range.push_back(std::make_pair(box,id)); + box = BoxType( PointType(proc_id + 0.6, 0.5, 0.0), PointType(proc_id + 1.4, 1.5, 1.0)); + identProc = Ident(proc_id * 4+2, proc_id); + range.push_back(std::make_pair(box, identProc)); - box = StkBox( Point(proc_id + 0.6, 2.5, 0.0), Point(proc_id + 1.4, 3.5, 1.0)); - id = Ident(proc_id * 4+3, proc_id); - local_range.push_back(std::make_pair(box,id)); + box = BoxType( PointType(proc_id + 0.6, 2.5, 0.0), PointType(proc_id + 1.4, 3.5, 1.0)); + identProc = Ident(proc_id * 4+3, proc_id); + range.push_back(std::make_pair(box, identProc)); SearchResults searchResults; - stk::search::coarse_search(local_domain, local_range, algorithm, comm, searchResults); + stk::search::coarse_search(domain, range, algorithm, comm, searchResults); expect_search_results(num_procs, proc_id, searchResults); } +TEST(stk_search, coarseSearchDoubleBoxes_KDTREE) +{ + test_coarse_search_for_algorithm(stk::search::KDTREE, MPI_COMM_WORLD); +} + +TEST(stk_search, coarseSearchFloatBoxes_KDTREE) +{ + test_coarse_search_for_algorithm(stk::search::KDTREE, MPI_COMM_WORLD); +} + +TEST(stk_search, coarseSearchDoubleBoxes_MORTON_LBVH) +{ + test_coarse_search_for_algorithm(stk::search::MORTON_LBVH, MPI_COMM_WORLD); +} + +TEST(stk_search, coarseSearchFloatBoxes_MORTON_LBVH) +{ + test_coarse_search_for_algorithm(stk::search::MORTON_LBVH, MPI_COMM_WORLD); +} + +TEST(stk_search, coarseSearchDoubleBoxes_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + test_coarse_search_for_algorithm(stk::search::ARBORX, MPI_COMM_WORLD); + test_coarse_search_for_algorithm_with_views(stk::search::ARBORX, MPI_COMM_WORLD); +} + +TEST(stk_search, coarseSearchFloatBoxes_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + test_coarse_search_for_algorithm(stk::search::ARBORX, MPI_COMM_WORLD); + test_coarse_search_for_algorithm_with_views(stk::search::ARBORX, MPI_COMM_WORLD); +} + +template +void local_expect_search_results(const HostIntersectionType & searchResults) +{ + if constexpr (Kokkos::is_view_v) { + ASSERT_EQ(searchResults.extent(0), 2u); + + EXPECT_EQ(searchResults(0).domainIdent, 0); + EXPECT_EQ(searchResults(0).rangeIdent, 2); + + EXPECT_EQ(searchResults(1).domainIdent, 1); + EXPECT_EQ(searchResults(1).rangeIdent, 3); + } else { + ASSERT_EQ(searchResults.size(), 2u); + + EXPECT_EQ(searchResults[0].first, 0); + EXPECT_EQ(searchResults[0].second, 2); + + EXPECT_EQ(searchResults[1].first, 1); + EXPECT_EQ(searchResults[1].second, 3); + } +} + +template +void host_local_test_coarse_search_for_algorithm(stk::search::SearchMethod algorithm) +{ + using BoxType = stk::search::Box; + using PointType = stk::search::Point; + using IdentType = int; + using BoxIdentType = std::pair; + using IntersectionType = std::pair; + + std::vector domain; + std::vector range; + + domain.emplace_back(BoxType(PointType(0.1, 0.0, 0.0), PointType(0.9, 1.0, 1.0)), 0); + domain.emplace_back(BoxType(PointType(0.1, 2.0, 0.0), PointType(0.9, 3.0, 1.0)), 1); + range.emplace_back(BoxType(PointType(0.6, 0.5, 0.0), PointType(1.4, 1.5, 1.0)), 2); + range.emplace_back(BoxType(PointType(0.6, 2.5, 0.0), PointType(1.4, 3.5, 1.0)), 3); + + std::vector intersections; + + stk::search::local_coarse_search(domain, range, algorithm, intersections); + + local_expect_search_results(intersections); +} + +template +void device_local_test_coarse_search_for_algorithm(stk::search::SearchMethod algorithm) +{ + using BoxType = stk::search::Box; + using PointType = stk::search::Point; + using IdentType = int; + using BoxIdentType = stk::search::BoxIdent; + using IntersectionType = stk::search::IdentIntersection; + + auto domain = Kokkos::View("domain box-ident", 2); + auto range = Kokkos::View("range box-ident", 2); + + Kokkos::parallel_for(stk::ngp::DeviceRangePolicy(0, 1), + KOKKOS_LAMBDA(const unsigned & i) { + domain[0] = {BoxType(PointType(0.1, 0.0, 0.0), PointType(0.9, 1.0, 1.0)), 0}; + domain[1] = {BoxType(PointType(0.1, 2.0, 0.0), PointType(0.9, 3.0, 1.0)), 1}; + range[0] = {BoxType(PointType(0.6, 0.5, 0.0), PointType(1.4, 1.5, 1.0)), 2}; + range[1] = {BoxType(PointType(0.6, 2.5, 0.0), PointType(1.4, 3.5, 1.0)), 3}; + }); + + auto intersections = Kokkos::View("intersections", 0); + + stk::search::local_coarse_search(domain, range, algorithm, intersections); + + Kokkos::View::HostMirror hostIntersections = Kokkos::create_mirror_view(intersections); + Kokkos::deep_copy(hostIntersections, intersections); + + local_expect_search_results(hostIntersections); +} + +TEST(stk_search, Ngp_Local_CoarseSearchDoubleBoxes_MORTON_LBVH) +{ + host_local_test_coarse_search_for_algorithm(stk::search::MORTON_LBVH); + device_local_test_coarse_search_for_algorithm(stk::search::MORTON_LBVH); +} + +TEST(stk_search, Ngp_Local_CoarseSearchFloatBoxes_MORTON_LBVH) +{ + host_local_test_coarse_search_for_algorithm(stk::search::MORTON_LBVH); + device_local_test_coarse_search_for_algorithm(stk::search::MORTON_LBVH); +} + +TEST(stk_search, Ngp_Local_CoarseSearchDoubleBoxes_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + host_local_test_coarse_search_for_algorithm(stk::search::ARBORX); + device_local_test_coarse_search_for_algorithm(stk::search::ARBORX); +} + +TEST(stk_search, Ngp_Local_CoarseSearchFloatBoxes_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + host_local_test_coarse_search_for_algorithm(stk::search::ARBORX); + device_local_test_coarse_search_for_algorithm(stk::search::ARBORX); +} + + std::pair build_range_boxes_and_nested_domain_boxes(int num_procs, int proc_id, int sizeParam=1) { BoxVector local_domain, local_range; @@ -201,11 +440,16 @@ void test_coarse_search_range_box_communication(stk::search::SearchMethod algori expect_coarse_search_range_box_communication(num_procs, proc_id, searchResultsCommunicateOn, searchResultsCommunicateOff); } -TEST(stk_search, coarse_search_range_box_communication) +TEST(stk_search, coarse_search_range_box_communication_KDTREE) { test_coarse_search_range_box_communication(stk::search::KDTREE, MPI_COMM_WORLD); } +TEST(stk_search, coarse_search_range_box_communication_MORTON_LBVH) +{ + test_coarse_search_range_box_communication(stk::search::MORTON_LBVH, MPI_COMM_WORLD); +} + void test_coarse_search_determine_domain_and_range_communicate_on(stk::search::SearchMethod algorithm, MPI_Comm comm) { @@ -227,11 +471,24 @@ void test_coarse_search_determine_domain_and_range_communicate_on(stk::search::S EXPECT_EQ(searchResultsDetermineOn, searchResultsDetermineOff); } -TEST(stk_search, coarse_search_determine_domain_and_range_communicate_on) +TEST(stk_search, coarse_search_determine_domain_and_range_communicate_on_KDTREE) { test_coarse_search_determine_domain_and_range_communicate_on(stk::search::KDTREE, MPI_COMM_WORLD); } +TEST(stk_search, coarse_search_determine_domain_and_range_communicate_on_MORTON_LBVH) +{ + test_coarse_search_determine_domain_and_range_communicate_on(stk::search::MORTON_LBVH, MPI_COMM_WORLD); +} + +TEST(stk_search, coarse_search_determine_domain_and_range_communicate_on_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + test_coarse_search_determine_domain_and_range_communicate_on(stk::search::ARBORX, MPI_COMM_WORLD); +} + void test_coarse_search_two_pass(stk::search::SearchMethod algorithm, MPI_Comm comm, int sizeParam) { BoxVector local_domain, local_range, additional_domain; @@ -261,59 +518,86 @@ void test_coarse_search_two_pass(stk::search::SearchMethod algorithm, MPI_Comm c searchResultsPassOne.insert(searchResultsPassOne.end(), searchResultsPassTwo.begin(), searchResultsPassTwo.end()); - std::sort(searchResultsPassOne.begin(), searchResultsPassOne.end()); - std::sort(searchResultsAllBoxes.begin(), searchResultsAllBoxes.end()); + stk::util::sort_and_unique(searchResultsPassOne); + stk::util::sort_and_unique(searchResultsAllBoxes); EXPECT_EQ(searchResultsPassOne, searchResultsAllBoxes); } -TEST(stk_search, coarse_search_two_pass) +TEST(stk_search, coarse_search_two_pass_KDTREE) { test_coarse_search_two_pass(stk::search::KDTREE, MPI_COMM_WORLD, 2); } -void test_coarse_search_for_algorithm_using_float_boxes(stk::search::SearchMethod algorithm, MPI_Comm comm) +TEST(stk_search, coarse_search_two_pass_MORTON_LBVH) { - int num_procs = stk::parallel_machine_size(comm); - int proc_id = stk::parallel_machine_rank(comm); + test_coarse_search_two_pass(stk::search::MORTON_LBVH, MPI_COMM_WORLD, 2); +} - FloatBoxVector local_domain, local_range; - // what if identifier is NOT unique +TEST(stk_search, coarse_search_two_pass_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + test_coarse_search_two_pass(stk::search::ARBORX, MPI_COMM_WORLD, 2); +} - FloatBox box; - Ident id; +void test_ident_proc_with_search_with_views(stk::search::SearchMethod searchMethod) +{ + using IdentProc = Ident; + using ExecSpace = Kokkos::DefaultExecutionSpace; + using BoxIdentProcType = stk::search::BoxIdentProc; + using BoxIdentProcViewType = Kokkos::View; + using SearchResultsViewType = Kokkos::View*, ExecSpace>; - box.set_box(proc_id + 0.1, 0.0, 0.0, proc_id + 0.9, 1.0, 1.0); - id = Ident(proc_id * 4, proc_id); - local_domain.push_back(std::make_pair(box,id)); + MPI_Comm comm = MPI_COMM_WORLD; + int procId = -1; + MPI_Comm_rank(comm, &procId); + int numProcs = -1; + MPI_Comm_size(comm, &numProcs); - box.set_box(proc_id + 0.1, 2.0, 0.0, proc_id + 0.9, 3.0, 1.0); - id = Ident(proc_id * 4+1, proc_id); - local_domain.push_back(std::make_pair(box,id)); + if (numProcs != 1) { + FloatBox box1(0, 0, 0, 1, 1, 1); + FloatBox box2(0.5, 0.5, 0.5, 1.5, 1.5, 1.5); + Ident id1(1, 0); + Ident id2(1, 1); - box.set_box(proc_id + 0.6, 0.5, 0.0, proc_id + 1.4, 1.5, 1.0); - id = Ident(proc_id * 4+2, proc_id); - local_range.push_back(std::make_pair(box,id)); + BoxIdentProcViewType boxes("", 1); + if (procId == 0) { + boxes(0) = {box1, id1}; + } else if (procId == 1) { + boxes(0) = {box2, id2}; + } - box.set_box(proc_id + 0.6, 2.5, 0.0, proc_id + 1.4, 3.5, 1.0); - id = Ident(proc_id * 4+3, proc_id); - local_range.push_back(std::make_pair(box,id)); + SearchResultsViewType searchResults("", 3); - SearchResults searchResults; + coarse_search(boxes, boxes, searchMethod, comm, searchResults); - stk::search::coarse_search(local_domain, local_range, algorithm, comm, searchResults); + SearchResultsViewType goldResults("", 3); - expect_search_results(num_procs, proc_id, searchResults); -} + Ident goldId1(1, 0); + Ident goldId2(1, 1); -TEST(stk_search, coarse_search_kdtree) -{ - test_coarse_search_for_algorithm(stk::search::KDTREE, MPI_COMM_WORLD); -} + if (procId == 0) { + goldResults(0) = {goldId1, goldId1}; + goldResults(1) = {goldId1, goldId2}; + goldResults(2) = {goldId2, goldId1}; + ASSERT_EQ(goldResults.extent(0), searchResults.extent(0)); + } else if (procId == 1) { + goldResults(0) = {goldId1, goldId2}; + goldResults(1) = {goldId2, goldId1}; + goldResults(2) = {goldId2, goldId2}; + ASSERT_EQ(3u, searchResults.extent(0)); + } -TEST(stk_search, coarse_search_kdtree_using_float_aa_boxes) -{ - test_coarse_search_for_algorithm_using_float_boxes(stk::search::KDTREE, MPI_COMM_WORLD); + Kokkos::sort(searchResults); + Kokkos::sort(goldResults); + + for (size_t i = 0; i < goldResults.extent(0); i++) { + EXPECT_EQ(goldResults[i], searchResults[i]) + << "Test comparison for proc " << procId << " failed for comparsion #" << i << std::endl; + } + } } void test_ident_proc_with_search(stk::search::SearchMethod searchMethod) @@ -365,11 +649,26 @@ void test_ident_proc_with_search(stk::search::SearchMethod searchMethod) } } -TEST(stk_search, coarse_search_kdtree_ident_proc_switch) +TEST(stk_search, coarse_search_kdtree_ident_proc_switch_KDTREE) { test_ident_proc_with_search(stk::search::KDTREE); } +TEST(stk_search, coarse_search_kdtree_ident_proc_switch_MORTON_LBVH) +{ + test_ident_proc_with_search(stk::search::MORTON_LBVH); +} + +TEST(stk_search, coarse_search_ident_proc_switch_ARBORX) +{ + if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) { GTEST_SKIP(); } +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + test_ident_proc_with_search(stk::search::ARBORX); + test_ident_proc_with_search_with_views(stk::search::ARBORX); +} + void test_coarse_search_one_point(stk::search::SearchMethod searchMethod) { stk::ParallelMachine comm = MPI_COMM_WORLD; @@ -421,6 +720,19 @@ TEST(stk_search, coarse_search_one_point_KDTREE) test_coarse_search_one_point(stk::search::KDTREE); } +TEST(stk_search, coarse_search_one_point_MORTON_LBVH) +{ + test_coarse_search_one_point(stk::search::MORTON_LBVH); +} + +TEST(stk_search, coarse_search_one_point_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + test_coarse_search_one_point(stk::search::ARBORX); +} + void test_coarse_search_for_determining_sharing_all_all_case(stk::search::SearchMethod searchMethod) { const stk::ParallelMachine comm = MPI_COMM_WORLD; @@ -475,6 +787,19 @@ TEST(CoarseSearch, forDeterminingSharingAllAllCase_KDTREE) test_coarse_search_for_determining_sharing_all_all_case(stk::search::KDTREE); } +TEST(CoarseSearch, forDeterminingSharingAllAllCase_MORTON_LBVH) +{ + test_coarse_search_for_determining_sharing_all_all_case(stk::search::MORTON_LBVH); +} + +TEST(CoarseSearch, forDeterminingSharingAllAllCase_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + test_coarse_search_for_determining_sharing_all_all_case(stk::search::ARBORX); +} + void test_coarse_search_for_determining_sharing_linear_adjacent_case( stk::search::SearchMethod searchMethod, int numLoops = 1) { @@ -582,4 +907,30 @@ TEST(CoarseSearchScaling, forDeterminingSharingLinearAdjacentCase_KDTREE) test_coarse_search_for_determining_sharing_linear_adjacent_case(stk::search::KDTREE, 1000); } +TEST(CoarseSearch, forDeterminingSharingLinearAdjacentCase_MORTON_LBVH) +{ + test_coarse_search_for_determining_sharing_linear_adjacent_case(stk::search::MORTON_LBVH); +} + +TEST(CoarseSearchScaling, forDeterminingSharingLinearAdjacentCase_MORTON_LBVH) +{ + test_coarse_search_for_determining_sharing_linear_adjacent_case(stk::search::MORTON_LBVH, 1000); +} + +TEST(CoarseSearch, forDeterminingSharingLinearAdjacentCase_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + test_coarse_search_for_determining_sharing_linear_adjacent_case(stk::search::ARBORX); +} + +TEST(CoarseSearchScaling, forDeterminingSharingLinearAdjacentCase_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + test_coarse_search_for_determining_sharing_linear_adjacent_case(stk::search::ARBORX, 1000); +} + } //namespace diff --git a/packages/stk/stk_unit_tests/stk_search/UnitTestCoarseSearch2.cpp b/packages/stk/stk_unit_tests/stk_search/UnitTestCoarseSearch2.cpp deleted file mode 100644 index c0876ccbde4f..000000000000 --- a/packages/stk/stk_unit_tests/stk_search/UnitTestCoarseSearch2.cpp +++ /dev/null @@ -1,287 +0,0 @@ -// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering -// Solutions of Sandia, LLC (NTESS). Under the terms of Contract -// DE-NA0003525 with NTESS, the U.S. Government retains certain rights -// in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following -// disclaimer in the documentation and/or other materials provided -// with the distribution. -// -// * Neither the name of NTESS nor the names of its contributors -// may be used to endorse or promote products derived from this -// software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// - -#include -#include -#include -#include // for ParallelMachine, etc - -#include -#include - -namespace -{ - -void runTwoSpheresTest(stk::search::SearchMethod searchMethod, const double distanceBetweenSphereCenters, const double radius, std::vector< std::pair > &boxIdPairResults) -{ - MPI_Comm comm = MPI_COMM_WORLD; - int procId = stk::parallel_machine_rank(comm); - - std::vector< std::pair > boxVector1 = { stk::unit_test_util::simple_fields::generateBoundingVolume(0, 0, 0, radius, 1, procId) }; - - std::vector< std::pair > boxVector2 = { stk::unit_test_util::simple_fields::generateBoundingVolume(distanceBetweenSphereCenters, 0, 0, radius, 2, procId) }; - - stk::search::coarse_search(boxVector1, boxVector2, searchMethod, comm, boxIdPairResults); -} - -const double radiusOfOneHalf = 0.5; - -TEST(CoarseSearchCorrectness, OverlappingSpheres_KDTREE) -{ - if (stk::parallel_machine_size(MPI_COMM_WORLD) > 1) { GTEST_SKIP(); } - - double distanceBetweenSphereCenters = 0.5; - std::vector< std::pair > boxIdPairResults; - runTwoSpheresTest(stk::search::KDTREE, distanceBetweenSphereCenters, radiusOfOneHalf, boxIdPairResults); - - ASSERT_EQ(1u, boxIdPairResults.size()); -} - -TEST(CoarseSearchCorrectness, NonOverlappingSpheres_KDTREE) -{ - double distanceBetweenSphereCenters = 2.0; - std::vector< std::pair > boxIdPairResults; - runTwoSpheresTest(stk::search::KDTREE, distanceBetweenSphereCenters, radiusOfOneHalf, boxIdPairResults); - - ASSERT_EQ(0u, boxIdPairResults.size()); -} - -TEST(CoarseSearchCorrectness, JustEdgeOverlappingSpheres_KDTREE) -{ - if (stk::parallel_machine_size(MPI_COMM_WORLD) > 1) { GTEST_SKIP(); } - - double distanceBetweenSphereCenters = 0.999999999; - std::vector< std::pair > boxIdPairResults; - runTwoSpheresTest(stk::search::KDTREE, distanceBetweenSphereCenters, radiusOfOneHalf, boxIdPairResults); - - ASSERT_EQ(1u, boxIdPairResults.size()); -} - -TEST(CoarseSearchCorrectness, NotQuiteEdgeOverlappingSpheres_KDTREE) -{ - double distanceBetweenSphereCenters = 1.0000000001; - std::vector< std::pair > boxIdPairResults; - runTwoSpheresTest(stk::search::KDTREE, distanceBetweenSphereCenters, radiusOfOneHalf, boxIdPairResults); - - ASSERT_EQ(0u, boxIdPairResults.size()); -} - -template -void runBoxOverlappingEightSurroundingBoxes(stk::search::SearchMethod searchMethod, const double radius, const unsigned numExpectedResults) -{ - MPI_Comm comm = MPI_COMM_WORLD; - int numProc = stk::parallel_machine_size(comm); - int procId = stk::parallel_machine_rank(comm); - - std::vector< std::pair > boxVector1; - if(procId == 0) - { - boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(0, 0, 0, radius, 1, procId)); - boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(1, 0, 0, radius, 2, procId)); - boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(2, 0, 0, radius, 3, procId)); - boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(0, 1, 0, radius, 4, procId)); - //skip middle one - boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(2, 1, 0, radius, 6, procId)); - boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(0, 2, 0, radius, 7, procId)); - boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(1, 2, 0, radius, 8, procId)); - boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(2, 2, 0, radius, 9, procId)); - } - - std::vector< std::pair > boxVector2; - if(procId == numProc-1) - { - boxVector2.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(1, 1, 0, radius, 5, procId)); - } - - std::vector< std::pair > boxIdPairResults; - stk::search::coarse_search(boxVector1, boxVector2, searchMethod, comm, boxIdPairResults); - - if(!boxVector1.empty() || !boxVector2.empty()) - { - if(numExpectedResults != boxIdPairResults.size()) - { - for(size_t i=0; i(stk::search::KDTREE, radius, numExpectedResults); -} - -TEST(CoarseSearchCorrectness, SphereOverlappingNoSurroundingPoints_KDTREE) -{ - const double radius = 0.99; - const unsigned numExpectedResults = 0; - runBoxOverlappingEightSurroundingBoxes(stk::search::KDTREE, radius, numExpectedResults); -} - -TEST(CoarseSearchCorrectness, SphereOverlappingFourSurroundingPoints_KDTREE) -{ - const double radius = 1.41; - const unsigned numExpectedResults = 4; - runBoxOverlappingEightSurroundingBoxes(stk::search::KDTREE, radius, numExpectedResults); -} - -TEST(CoarseSearchCorrectness, SphereOverlappingEightSurroundingPoints_KDTREE) -{ - const double radius = 1.42; - const unsigned numExpectedResults = 8; - runBoxOverlappingEightSurroundingBoxes(stk::search::KDTREE, radius, numExpectedResults); -} - -TEST(CoarseSearchCorrectness, SphereOverlappingFourOfEightSurroundingSpheres_KDTREE) -{ - const double radius = 0.706; - const unsigned numExpectedResults = 4; - runBoxOverlappingEightSurroundingBoxes(stk::search::KDTREE, radius, numExpectedResults); -} - -TEST(CoarseSearchCorrectness, BoxOverlappingNoSurroundingPoints_KDTREE) -{ - const double radius = 0.99; - const unsigned numExpectedResults = 0; - runBoxOverlappingEightSurroundingBoxes(stk::search::KDTREE, radius, numExpectedResults); -} - -TEST(CoarseSearchCorrectness, BoxOverlappingEightSurroundingPoints_KDTREE) -{ - const double radius = 1.01; - const unsigned numExpectedResults = 8; - runBoxOverlappingEightSurroundingBoxes(stk::search::KDTREE, radius, numExpectedResults); -} - -TEST(CoarseSearchCorrectness, PointOverlappingNoSurroundingBoxes_KDTREE) -{ - const double radius = 0.99; - const unsigned numExpectedResults = 0; - runBoxOverlappingEightSurroundingBoxes(stk::search::KDTREE, radius, numExpectedResults); -} - -TEST(CoarseSearchCorrectness, PointOverlappingEightSurroundingBoxes_KDTREE) -{ - const double radius = 1.01; - const unsigned numExpectedResults = 8; - runBoxOverlappingEightSurroundingBoxes(stk::search::KDTREE, radius, numExpectedResults); -} - -enum Axis -{ - xDim, yDim, zDim -}; -template -void runLineOfBoundingBoxes(stk::search::SearchMethod searchMethod, enum Axis axis) -{ - MPI_Comm comm = MPI_COMM_WORLD; - int procId = stk::parallel_machine_rank(comm); - - const double radius = 0.708; - const double distanceBetweenCenters = 1.0; - - const double paramCoord = procId * distanceBetweenCenters; - - std::vector< std::pair > boxVector1; - std::vector< std::pair > boxVector2; - if(procId % 2 == 0) - { - switch(axis) - { - case xDim: - boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(paramCoord, 0, 0, radius, 1, procId)); - break; - case yDim: - boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(0, paramCoord, 0, radius, 1, procId)); - break; - case zDim: - boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(0, 0, paramCoord, radius, 1, procId)); - break; - } - } - else - { - switch(axis) - { - case xDim: - boxVector2.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(paramCoord, 0, 0, radius, 1, procId)); - break; - case yDim: - boxVector2.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(0, paramCoord, 0, radius, 1, procId)); - break; - case zDim: - boxVector2.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(0, 0, paramCoord, radius, 1, procId)); - break; - } - } - - std::vector< std::pair > boxIdPairResults; - stk::search::coarse_search(boxVector1, boxVector2, searchMethod, comm, boxIdPairResults); - - int numProcs = stk::parallel_machine_size(comm); - - unsigned numExpectedResults = 2; - bool doOwnFirstOrLastSphereInLine = procId == 0 || procId == numProcs-1; - if(doOwnFirstOrLastSphereInLine) - { - numExpectedResults = 1; - } - if(numProcs == 1) - { - numExpectedResults = 0; - } - ASSERT_EQ(numExpectedResults, boxIdPairResults.size()) << "on proc id " << procId; -} - -TEST(CoarseSearchCorrectness, LineOfSpheres_KDTREE) -{ - runLineOfBoundingBoxes(stk::search::KDTREE, xDim); -} - -TEST(CoarseSearchCorrectness, LineOfBoxes_KDTREE) -{ - runLineOfBoundingBoxes(stk::search::KDTREE, yDim); -} - -TEST(CoarseSearchCorrectness, LineOfSpheresZDimension_KDTREE) -{ - runLineOfBoundingBoxes(stk::search::KDTREE, zDim); -} - -} diff --git a/packages/stk/stk_unit_tests/stk_search/UnitTestCoarseSearchBoxOverlappingEightSurroundingBoxes.cpp b/packages/stk/stk_unit_tests/stk_search/UnitTestCoarseSearchBoxOverlappingEightSurroundingBoxes.cpp new file mode 100644 index 000000000000..dc3baf94b821 --- /dev/null +++ b/packages/stk/stk_unit_tests/stk_search/UnitTestCoarseSearchBoxOverlappingEightSurroundingBoxes.cpp @@ -0,0 +1,652 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include +#include +#include +#include +#include +#include // for ParallelMachine, etc +#include + +#include +#include +#include "stk_search/SearchMethod.hpp" + +namespace +{ + +template +void runBoxOverlappingEightSurroundingBoxes(stk::search::SearchMethod searchMethod, const double radius, + const unsigned numExpectedResults) +{ + MPI_Comm comm = MPI_COMM_WORLD; + int numProc = stk::parallel_machine_size(comm); + int procId = stk::parallel_machine_rank(comm); + + std::vector> boxVector1; + if (procId == 0) { + boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(0, 0, 0, radius, 1, procId)); + boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(1, 0, 0, radius, 2, procId)); + boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(2, 0, 0, radius, 3, procId)); + boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(0, 1, 0, radius, 4, procId)); + //skip middle one + boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(2, 1, 0, radius, 6, procId)); + boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(0, 2, 0, radius, 7, procId)); + boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(1, 2, 0, radius, 8, procId)); + boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(2, 2, 0, radius, 9, procId)); + } + + std::vector> boxVector2; + if (procId == numProc-1) { + boxVector2.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(1, 1, 0, radius, 5, procId)); + } + + std::vector< std::pair > boxIdPairResults; + stk::search::coarse_search(boxVector1, boxVector2, searchMethod, comm, boxIdPairResults); + + if (!boxVector1.empty() || !boxVector2.empty()) { + if (numExpectedResults != boxIdPairResults.size()) { + for (size_t i = 0; i < boxIdPairResults.size(); ++i) { + std::cerr << boxIdPairResults[i].first << ", " << boxIdPairResults[i].second << std::endl; + } + } + ASSERT_EQ(numExpectedResults, boxIdPairResults.size()); + } +} + +TEST(CoarseSearchCorrectness, SphereOverlappingEightSurroundingSpheres_KDTREE) +{ + const double radius = 0.708; + const unsigned numExpectedResults = 8; + runBoxOverlappingEightSurroundingBoxes(stk::search::KDTREE, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, SphereOverlappingFourOfEightSurroundingSpheres_KDTREE) +{ + const double radius = 0.706; + const unsigned numExpectedResults = 4; + runBoxOverlappingEightSurroundingBoxes(stk::search::KDTREE, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, SphereOverlappingEightSurroundingSpheres_MORTON_LBVH) +{ + const double radius = 0.708; + const unsigned numExpectedResults = 8; + runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, SphereOverlappingFourOfEightSurroundingSpheres_MORTON_LBVH) +{ + const double radius = 0.706; + const unsigned numExpectedResults = 4; + runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); +} + + +TEST(CoarseSearchCorrectness, SphereOverlappingNoSurroundingPoints_KDTREE) +{ + const double radius = 0.99; + const unsigned numExpectedResults = 0; + runBoxOverlappingEightSurroundingBoxes(stk::search::KDTREE, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, SphereOverlappingFourSurroundingPoints_KDTREE) +{ + const double radius = 1.41; + const unsigned numExpectedResults = 4; + runBoxOverlappingEightSurroundingBoxes(stk::search::KDTREE, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, SphereOverlappingEightSurroundingPoints_KDTREE) +{ + const double radius = 1.42; + const unsigned numExpectedResults = 8; + runBoxOverlappingEightSurroundingBoxes(stk::search::KDTREE, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, SphereOverlappingNoSurroundingPoints_MORTON_LBVH) +{ + const double radius = 0.99; + const unsigned numExpectedResults = 0; + runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, SphereOverlappingFourSurroundingPoints_MORTON_LBVH) +{ + const double radius = 1.41; + const unsigned numExpectedResults = 4; + runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, SphereOverlappingEightSurroundingPoints_MORTON_LBVH) +{ + const double radius = 1.42; + const unsigned numExpectedResults = 8; + runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); +} + + +TEST(CoarseSearchCorrectness, BoxOverlappingNoSurroundingPoints_KDTREE) +{ + const double radius = 0.99; + const unsigned numExpectedResults = 0; + runBoxOverlappingEightSurroundingBoxes(stk::search::KDTREE, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, BoxOverlappingEightSurroundingPoints_KDTREE) +{ + const double radius = 1.01; + const unsigned numExpectedResults = 8; + runBoxOverlappingEightSurroundingBoxes(stk::search::KDTREE, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, BoxOverlappingNoSurroundingPoints_MORTON_LBVH) +{ + const double radius = 0.99; + const unsigned numExpectedResults = 0; + runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, BoxOverlappingEightSurroundingPoints_MORTON_LBVH) +{ + const double radius = 1.01; + const unsigned numExpectedResults = 8; + runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); +} + + +TEST(CoarseSearchCorrectness, PointOverlappingNoSurroundingBoxes_KDTREE) +{ + const double radius = 0.99; + const unsigned numExpectedResults = 0; + runBoxOverlappingEightSurroundingBoxes(stk::search::KDTREE, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, PointOverlappingEightSurroundingBoxes_KDTREE) +{ + const double radius = 1.01; + const unsigned numExpectedResults = 8; + runBoxOverlappingEightSurroundingBoxes(stk::search::KDTREE, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, PointOverlappingNoSurroundingBoxes_MORTON_LBVH) +{ + const double radius = 0.99; + const unsigned numExpectedResults = 0; + runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, PointOverlappingEightSurroundingBoxes_MORTON_LBVH) +{ + const double radius = 1.01; + const unsigned numExpectedResults = 8; + runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); +} + + +TEST(CoarseSearchCorrectness, BoxOverlappingNoSurroundingBoxes_KDTREE) +{ + const double radius = 0.49; + const unsigned numExpectedResults = 0; + runBoxOverlappingEightSurroundingBoxes(stk::search::KDTREE, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, BoxOverlappingEightSurroundingBoxes_KDTREE) +{ + const double radius = 0.51; + const unsigned numExpectedResults = 8; + runBoxOverlappingEightSurroundingBoxes(stk::search::KDTREE, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, BoxOverlappingNoSurroundingBoxes_MORTON_LBVH) +{ + const double radius = 0.49; + const unsigned numExpectedResults = 0; + runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, BoxOverlappingEightSurroundingBoxes_MORTON_LBVH) +{ + const double radius = 0.51; + const unsigned numExpectedResults = 8; + runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); +} + + +TEST(CoarseSearchCorrectness, SphereOverlappingEightSurroundingSpheres_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double radius = 0.708; + const unsigned numExpectedResults = 8; + runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, SphereOverlappingNoSurroundingPoints_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double radius = 0.99; + const unsigned numExpectedResults = 0; + runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, SphereOverlappingFourSurroundingPoints_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double radius = 1.41; + const unsigned numExpectedResults = 4; + runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, SphereOverlappingEightSurroundingPoints_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double radius = 1.42; + const unsigned numExpectedResults = 8; + runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, SphereOverlappingFourOfEightSurroundingSpheres_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double radius = 0.706; + const unsigned numExpectedResults = 4; + runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, BoxOverlappingNoSurroundingPoints_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double radius = 0.99; + const unsigned numExpectedResults = 0; + runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, BoxOverlappingEightSurroundingPoints_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double radius = 1.01; + const unsigned numExpectedResults = 8; + runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, PointOverlappingNoSurroundingBoxes_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double radius = 0.99; + const unsigned numExpectedResults = 0; + runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, PointOverlappingEightSurroundingBoxes_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double radius = 1.01; + const unsigned numExpectedResults = 8; + runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, BoxOverlappingNoSurroundingBoxes_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double radius = 0.49; + const unsigned numExpectedResults = 0; + runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, BoxOverlappingEightSurroundingBoxes_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double radius = 0.51; + const unsigned numExpectedResults = 8; + runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); +} + +template +void host_local_runBoxOverlappingEightSurroundingBoxes( + stk::search::SearchMethod searchMethod, const double radius, const unsigned numExpectedResults) +{ + using IdentType = int; + using InnerBoxIdentType = std::pair; + using OuterBoxIdentType = std::pair; + using IntersectionType = std::pair; + + std::vector domain; + std::vector range; + + domain.push_back(stk::unit_test_util::simple_fields::box_ident_to_pair( + stk::unit_test_util::simple_fields::device_generateBoxIdent(0, 0, 0, radius, 1))); + domain.push_back(stk::unit_test_util::simple_fields::box_ident_to_pair( + stk::unit_test_util::simple_fields::device_generateBoxIdent(1, 0, 0, radius, 2))); + domain.push_back(stk::unit_test_util::simple_fields::box_ident_to_pair( + stk::unit_test_util::simple_fields::device_generateBoxIdent(2, 0, 0, radius, 3))); + domain.push_back(stk::unit_test_util::simple_fields::box_ident_to_pair( + stk::unit_test_util::simple_fields::device_generateBoxIdent(0, 1, 0, radius, 4))); + // Skip middle box + domain.push_back(stk::unit_test_util::simple_fields::box_ident_to_pair( + stk::unit_test_util::simple_fields::device_generateBoxIdent(2, 1, 0, radius, 6))); + domain.push_back(stk::unit_test_util::simple_fields::box_ident_to_pair( + stk::unit_test_util::simple_fields::device_generateBoxIdent(1, 2, 0, radius, 7))); + domain.push_back(stk::unit_test_util::simple_fields::box_ident_to_pair( + stk::unit_test_util::simple_fields::device_generateBoxIdent(2, 2, 0, radius, 8))); + domain.push_back(stk::unit_test_util::simple_fields::box_ident_to_pair( + stk::unit_test_util::simple_fields::device_generateBoxIdent(0, 2, 0, radius, 9))); + + range.push_back(stk::unit_test_util::simple_fields::box_ident_to_pair( + stk::unit_test_util::simple_fields::device_generateBoxIdent(1, 1, 0, radius, 5))); + + std::vector intersections; + + stk::search::local_coarse_search(domain, range, searchMethod, intersections); + + ASSERT_EQ(intersections.size(), numExpectedResults); +} + +template +void device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::SearchMethod searchMethod, const double radius, + const unsigned numExpectedResults) +{ + using IdentType = int; + using InnerBoxIdentType = stk::search::BoxIdent; + using OuterBoxIdentType = stk::search::BoxIdent; + using IntersectionType = stk::search::IdentIntersection; + + auto domain = Kokkos::View("domain box-ident", 8); + auto range = Kokkos::View("range box-ident", 1); + + Kokkos::parallel_for(stk::ngp::DeviceRangePolicy(0, 1), + KOKKOS_LAMBDA(const unsigned & i) { + domain[0] = stk::unit_test_util::simple_fields::device_generateBoxIdent(0, 0, 0, radius, 1); + domain[1] = stk::unit_test_util::simple_fields::device_generateBoxIdent(1, 0, 0, radius, 2); + domain[2] = stk::unit_test_util::simple_fields::device_generateBoxIdent(2, 0, 0, radius, 3); + domain[3] = stk::unit_test_util::simple_fields::device_generateBoxIdent(0, 1, 0, radius, 4); + // Skip middle box + domain[4] = stk::unit_test_util::simple_fields::device_generateBoxIdent(2, 1, 0, radius, 6); + domain[5] = stk::unit_test_util::simple_fields::device_generateBoxIdent(1, 2, 0, radius, 7); + domain[6] = stk::unit_test_util::simple_fields::device_generateBoxIdent(2, 2, 0, radius, 8); + domain[7] = stk::unit_test_util::simple_fields::device_generateBoxIdent(0, 2, 0, radius, 9); + + range[0] = stk::unit_test_util::simple_fields::device_generateBoxIdent(1, 1, 0, radius, 5); + }); + + auto intersections = Kokkos::View("intersections", 0); + + stk::search::local_coarse_search(domain, range, searchMethod, intersections); + + Kokkos::View::HostMirror hostIntersections = Kokkos::create_mirror_view(intersections); + Kokkos::deep_copy(hostIntersections, intersections); + + ASSERT_EQ(intersections.extent(0), numExpectedResults); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_SphereOverlappingEightSurroundingSpheres_MORTON_LBVH) +{ + const double radius = 0.708; + const unsigned numExpectedResults = 8; + host_local_runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); + device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_SphereOverlappingFourOfEightSurroundingSpheres_MORTON_LBVH) +{ + const double radius = 0.706; + const unsigned numExpectedResults = 4; + host_local_runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); + device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); +} + + +TEST(CoarseSearchCorrectness, Ngp_Local_SphereOverlappingNoSurroundingPoints_MORTON_LBVH) +{ + const double radius = 0.99; + const unsigned numExpectedResults = 0; + host_local_runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); + device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_SphereOverlappingFourSurroundingPoints_MORTON_LBVH) +{ + const double radius = 1.41; + const unsigned numExpectedResults = 4; + host_local_runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); + device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_SphereOverlappingEightSurroundingPoints_MORTON_LBVH) +{ + const double radius = 1.42; + const unsigned numExpectedResults = 8; + host_local_runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); + device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); +} + + +TEST(CoarseSearchCorrectness, Ngp_Local_BoxOverlappingNoSurroundingPoints_MORTON_LBVH) +{ + const double radius = 0.99; + const unsigned numExpectedResults = 0; + host_local_runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); + device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_BoxOverlappingEightSurroundingPoints_MORTON_LBVH) +{ + const double radius = 1.01; + const unsigned numExpectedResults = 8; + host_local_runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); + device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); +} + + +TEST(CoarseSearchCorrectness, Ngp_Local_PointOverlappingNoSurroundingBoxes_MORTON_LBVH) +{ + const double radius = 0.99; + const unsigned numExpectedResults = 0; + host_local_runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); + device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_PointOverlappingEightSurroundingBoxes_MORTON_LBVH) +{ + const double radius = 1.01; + const unsigned numExpectedResults = 8; + host_local_runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); + device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); +} + + +TEST(CoarseSearchCorrectness, Ngp_Local_BoxOverlappingNoSurroundingBoxes_MORTON_LBVH) +{ + const double radius = 0.49; + const unsigned numExpectedResults = 0; + host_local_runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); + device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_BoxOverlappingEightSurroundingBoxes_MORTON_LBVH) +{ + const double radius = 0.51; + const unsigned numExpectedResults = 8; + host_local_runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); + device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::MORTON_LBVH, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_SphereOverlappingEightSurroundingSpheres_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double radius = 0.708; + const unsigned numExpectedResults = 8; + host_local_runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); + device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_SphereOverlappingFourOfEightSurroundingSpheres_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double radius = 0.706; + const unsigned numExpectedResults = 4; + host_local_runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); + device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); +} + + +TEST(CoarseSearchCorrectness, Ngp_Local_SphereOverlappingNoSurroundingPoints_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double radius = 0.99; + const unsigned numExpectedResults = 0; + host_local_runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); + device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_SphereOverlappingFourSurroundingPoints_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double radius = 1.41; + const unsigned numExpectedResults = 4; + host_local_runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); + device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_SphereOverlappingEightSurroundingPoints_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double radius = 1.42; + const unsigned numExpectedResults = 8; + host_local_runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); + device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); +} + + +TEST(CoarseSearchCorrectness, Ngp_Local_BoxOverlappingNoSurroundingPoints_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double radius = 0.99; + const unsigned numExpectedResults = 0; + host_local_runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); + device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_BoxOverlappingEightSurroundingPoints_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double radius = 1.01; + const unsigned numExpectedResults = 8; + host_local_runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); + device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); +} + + +TEST(CoarseSearchCorrectness, Ngp_Local_PointOverlappingNoSurroundingBoxes_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double radius = 0.99; + const unsigned numExpectedResults = 0; + host_local_runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); + device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_PointOverlappingEightSurroundingBoxes_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double radius = 1.01; + const unsigned numExpectedResults = 8; + host_local_runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); + device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); +} + + +TEST(CoarseSearchCorrectness, Ngp_Local_BoxOverlappingNoSurroundingBoxes_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double radius = 0.49; + const unsigned numExpectedResults = 0; + host_local_runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); + device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_BoxOverlappingEightSurroundingBoxes_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double radius = 0.51; + const unsigned numExpectedResults = 8; + host_local_runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); + device_local_runBoxOverlappingEightSurroundingBoxes(stk::search::ARBORX, radius, numExpectedResults); +} + +} diff --git a/packages/stk/stk_unit_tests/stk_search/UnitTestCoarseSearchLineOfBoundingBoxes.cpp b/packages/stk/stk_unit_tests/stk_search/UnitTestCoarseSearchLineOfBoundingBoxes.cpp new file mode 100644 index 000000000000..f99117edf347 --- /dev/null +++ b/packages/stk/stk_unit_tests/stk_search/UnitTestCoarseSearchLineOfBoundingBoxes.cpp @@ -0,0 +1,147 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include +#include +#include +#include +#include +#include // for ParallelMachine, etc +#include + +#include +#include +#include "stk_search/SearchMethod.hpp" + +namespace +{ + +enum Axis +{ + xDim, yDim, zDim +}; + +template +void runLineOfBoundingBoxes(stk::search::SearchMethod searchMethod, enum Axis axis) +{ + MPI_Comm comm = MPI_COMM_WORLD; + int procId = stk::parallel_machine_rank(comm); + + const double radius = 0.708; + const double distanceBetweenCenters = 1.0; + + const double paramCoord = procId * distanceBetweenCenters; + + std::vector> boxVector1; + std::vector> boxVector2; + if (procId % 2 == 0) { + switch(axis) + { + case xDim: + boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(paramCoord, 0, 0, radius, 1, procId)); + break; + case yDim: + boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(0, paramCoord, 0, radius, 1, procId)); + break; + case zDim: + boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(0, 0, paramCoord, radius, 1, procId)); + break; + } + } + else { + switch(axis) + { + case xDim: + boxVector2.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(paramCoord, 0, 0, radius, 1, procId)); + break; + case yDim: + boxVector2.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(0, paramCoord, 0, radius, 1, procId)); + break; + case zDim: + boxVector2.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(0, 0, paramCoord, radius, 1, procId)); + break; + } + } + + std::vector> boxIdPairResults; + stk::search::coarse_search(boxVector1, boxVector2, searchMethod, comm, boxIdPairResults); + + int numProcs = stk::parallel_machine_size(comm); + + unsigned numExpectedResults = 2; + bool doOwnFirstOrLastSphereInLine = procId == 0 || procId == numProcs-1; + + if (doOwnFirstOrLastSphereInLine) { + numExpectedResults = 1; + } + + if (numProcs == 1) { + numExpectedResults = 0; + } + + EXPECT_EQ(numExpectedResults, boxIdPairResults.size()) << "on proc id " << procId; +} + +TEST(CoarseSearchCorrectness, LineOfSpheres_KDTREE) +{ + runLineOfBoundingBoxes(stk::search::KDTREE, xDim); +} + +TEST(CoarseSearchCorrectness, LineOfBoxes_KDTREE) +{ + runLineOfBoundingBoxes(stk::search::KDTREE, yDim); +} + +TEST(CoarseSearchCorrectness, LineOfSpheresZDimension_KDTREE) +{ + runLineOfBoundingBoxes(stk::search::KDTREE, zDim); +} + + +TEST(CoarseSearchCorrectness, LineOfSpheres_MORTON_LBVH) +{ + runLineOfBoundingBoxes(stk::search::MORTON_LBVH, xDim); +} + +TEST(CoarseSearchCorrectness, LineOfBoxes_MORTON_LBVH) +{ + runLineOfBoundingBoxes(stk::search::MORTON_LBVH, yDim); +} + +TEST(CoarseSearchCorrectness, LineOfSpheresZDimension_MORTON_LBVH) +{ + runLineOfBoundingBoxes(stk::search::MORTON_LBVH, zDim); +} + +} diff --git a/packages/stk/stk_unit_tests/stk_search/UnitTestCoarseSearchTwoBox.cpp b/packages/stk/stk_unit_tests/stk_search/UnitTestCoarseSearchTwoBox.cpp new file mode 100644 index 000000000000..39cec3cd8cc9 --- /dev/null +++ b/packages/stk/stk_unit_tests/stk_search/UnitTestCoarseSearchTwoBox.cpp @@ -0,0 +1,331 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include +#include +#include +#include +#include +#include // for ParallelMachine, etc +#include + +#include +#include +#include "stk_search/SearchMethod.hpp" + +namespace +{ + +void runTwoBoxTest(stk::search::SearchMethod searchMethod, const double distanceBetweenBoxCenters, + const double boxSize, const unsigned expectedNumOverlap) +{ + MPI_Comm comm = MPI_COMM_WORLD; + int numProcs = stk::parallel_machine_size(comm); + int procId = stk::parallel_machine_rank(comm); + + std::vector> boxVector1; + if (procId == 0) { + boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(0, 0, 0, boxSize/2, 1, procId)); + } + + std::vector> boxVector2; + if (procId == numProcs-1) { + boxVector2.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(distanceBetweenBoxCenters, 0, 0, boxSize/2, 2, procId)); + } + + std::vector> boxIdPairResults; + stk::search::coarse_search(boxVector1, boxVector2, searchMethod, comm, boxIdPairResults); + + if (procId == 0 || (procId == numProcs-1)) { + EXPECT_EQ(expectedNumOverlap, boxIdPairResults.size()); + } + else { + EXPECT_EQ(0u, boxIdPairResults.size()); + } +} + +constexpr double boxSize = 1.0; + +TEST(CoarseSearchCorrectness, OverlappingBoxes_KDTREE) +{ + const double distanceBetweenBoxCenters = 0.5; + const unsigned expectedNumOverlap = 1; + runTwoBoxTest(stk::search::KDTREE, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, NonOverlappingBoxes_KDTREE) +{ + const double distanceBetweenBoxCenters = 2.0; + const unsigned expectedNumOverlap = 0; + runTwoBoxTest(stk::search::KDTREE, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, JustEdgeOverlappingBoxes_KDTREE) +{ + double distanceBetweenBoxCenters = 0.999999999; + const unsigned expectedNumOverlap = 1; + runTwoBoxTest(stk::search::KDTREE, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, NotQuiteEdgeOverlappingBoxes_KDTREE) +{ + double distanceBetweenBoxCenters = 1.0000000001; + const unsigned expectedNumOverlap = 0; + runTwoBoxTest(stk::search::KDTREE, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); +} + + +TEST(CoarseSearchCorrectness, OverlappingBoxes_MORTON_LBVH) +{ + const double distanceBetweenBoxCenters = 0.5; + const unsigned expectedNumOverlap = 1; + runTwoBoxTest(stk::search::MORTON_LBVH, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, NonOverlappingBoxes_MORTON_LBVH) +{ + const double distanceBetweenBoxCenters = 2.0; + const unsigned expectedNumOverlap = 0; + runTwoBoxTest(stk::search::MORTON_LBVH, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, JustEdgeOverlappingBoxes_MORTON_LBVH) +{ + double distanceBetweenBoxCenters = 0.999999999; + const unsigned expectedNumOverlap = 1; + runTwoBoxTest(stk::search::MORTON_LBVH, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, NotQuiteEdgeOverlappingBoxes_MORTON_LBVH) +{ + double distanceBetweenBoxCenters = 1.0000000001; + const unsigned expectedNumOverlap = 0; + runTwoBoxTest(stk::search::MORTON_LBVH, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, OverlappingBoxes_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double distanceBetweenBoxCenters = 0.5; + const unsigned expectedNumOverlap = 1; + runTwoBoxTest(stk::search::ARBORX, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, NonOverlappingBoxes_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double distanceBetweenBoxCenters = 2.0; + const unsigned expectedNumOverlap = 0; + runTwoBoxTest(stk::search::ARBORX, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, JustEdgeOverlappingBoxes_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + double distanceBetweenBoxCenters = 0.999999999; + const unsigned expectedNumOverlap = 1; + runTwoBoxTest(stk::search::ARBORX, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, NotQuiteEdgeOverlappingBoxes_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + double distanceBetweenBoxCenters = 1.00001; + const unsigned expectedNumOverlap = 0; + runTwoBoxTest(stk::search::ARBORX, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, NotQuiteEdgeOverlappingBoxes_FloatTruncation_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + double distanceBetweenBoxCenters = 1.0000000001; + const unsigned expectedNumOverlap = 1; + runTwoBoxTest(stk::search::ARBORX, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); +} + +void host_local_runTwoBoxTest(stk::search::SearchMethod searchMethod, + const double distanceBetweenBoxCenters, + const double boxSize, + const unsigned expectedNumOverlap) +{ + using BoxType = stk::search::Box; + using IdentType = int; + using BoxIdentType = std::pair; + using IntersectionType = std::pair; + + std::vector domain; + std::vector range; + + domain.emplace_back(stk::unit_test_util::simple_fields::box_ident_to_pair( + stk::unit_test_util::simple_fields::device_generateBoxIdent(0, 0, 0, boxSize / 2, 1))); + + range.emplace_back(stk::unit_test_util::simple_fields::box_ident_to_pair( + stk::unit_test_util::simple_fields::device_generateBoxIdent( + distanceBetweenBoxCenters, 0, 0, boxSize / 2, 2))); + + std::vector intersections; + + stk::search::local_coarse_search(domain, range, searchMethod, intersections); + + ASSERT_EQ(intersections.size(), expectedNumOverlap); + + for (unsigned i = 0; i < expectedNumOverlap; ++i) { + EXPECT_EQ(intersections[i].first, 1); + EXPECT_EQ(intersections[i].second, 2); + } +} + +void device_local_runTwoBoxTest(stk::search::SearchMethod searchMethod, const double distanceBetweenBoxCenters, + const double boxSize, const unsigned expectedNumOverlap) +{ + using BoxType = stk::search::Box; + using IdentType = int; + using BoxIdentType = stk::search::BoxIdent; + using IntersectionType = stk::search::IdentIntersection; + + auto domain = Kokkos::View("domain box-ident", 1); + auto range = Kokkos::View("range box-ident", 1); + + Kokkos::parallel_for(stk::ngp::DeviceRangePolicy(0, 1), + KOKKOS_LAMBDA(const unsigned & i) { + domain[0] = + stk::unit_test_util::simple_fields::device_generateBoxIdent(0, 0, 0, boxSize/2, 1); + + range[0] = + stk::unit_test_util::simple_fields::device_generateBoxIdent(distanceBetweenBoxCenters, 0, 0, + boxSize/2, 2); + }); + + auto intersections = Kokkos::View("intersections", 0); + + stk::search::local_coarse_search(domain, range, searchMethod, intersections); + + Kokkos::View::HostMirror hostIntersections = Kokkos::create_mirror_view(intersections); + Kokkos::deep_copy(hostIntersections, intersections); + + ASSERT_EQ(intersections.extent(0), expectedNumOverlap); + + for (unsigned i = 0; i < expectedNumOverlap; ++i) { + EXPECT_EQ(intersections(i).domainIdent, 1); + EXPECT_EQ(intersections(i).rangeIdent, 2); + } +} + +TEST(CoarseSearchCorrectness, Ngp_Local_OverlappingBoxes_MORTON_LBVH) +{ + const double distanceBetweenBoxCenters = 0.5; + const unsigned expectedNumOverlap = 1; + host_local_runTwoBoxTest(stk::search::MORTON_LBVH, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); + device_local_runTwoBoxTest(stk::search::MORTON_LBVH, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_NonOverlappingBoxes_MORTON_LBVH) +{ + const double distanceBetweenBoxCenters = 2.0; + const unsigned expectedNumOverlap = 0; + host_local_runTwoBoxTest(stk::search::MORTON_LBVH, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); + device_local_runTwoBoxTest(stk::search::MORTON_LBVH, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_JustEdgeOverlappingBoxes_MORTON_LBVH) +{ + double distanceBetweenBoxCenters = 0.999999999; + const unsigned expectedNumOverlap = 1; + host_local_runTwoBoxTest(stk::search::MORTON_LBVH, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); + device_local_runTwoBoxTest(stk::search::MORTON_LBVH, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_NotQuiteEdgeOverlappingBoxes_MORTON_LBVH) +{ + double distanceBetweenBoxCenters = 1.0000000001; + const unsigned expectedNumOverlap = 0; + host_local_runTwoBoxTest(stk::search::MORTON_LBVH, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); + device_local_runTwoBoxTest(stk::search::MORTON_LBVH, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_OverlappingBoxes_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double distanceBetweenBoxCenters = 0.5; + const unsigned expectedNumOverlap = 1; + host_local_runTwoBoxTest(stk::search::ARBORX, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); + device_local_runTwoBoxTest(stk::search::ARBORX, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_NonOverlappingBoxes_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + const double distanceBetweenBoxCenters = 2.0; + const unsigned expectedNumOverlap = 0; + host_local_runTwoBoxTest(stk::search::ARBORX, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); + device_local_runTwoBoxTest(stk::search::ARBORX, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_JustEdgeOverlappingBoxes_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + double distanceBetweenBoxCenters = 0.999999999; + const unsigned expectedNumOverlap = 1; + host_local_runTwoBoxTest(stk::search::ARBORX, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); + device_local_runTwoBoxTest(stk::search::ARBORX, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_NotQuiteEdgeOverlappingBoxes_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + double distanceBetweenBoxCenters = 1.0000000001; + const unsigned expectedNumOverlap = 1; + host_local_runTwoBoxTest(stk::search::ARBORX, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); + device_local_runTwoBoxTest(stk::search::ARBORX, distanceBetweenBoxCenters, boxSize, expectedNumOverlap); +} + +} diff --git a/packages/stk/stk_unit_tests/stk_search/UnitTestCoarseSearchTwoSpheres.cpp b/packages/stk/stk_unit_tests/stk_search/UnitTestCoarseSearchTwoSpheres.cpp new file mode 100644 index 000000000000..7f4217857728 --- /dev/null +++ b/packages/stk/stk_unit_tests/stk_search/UnitTestCoarseSearchTwoSpheres.cpp @@ -0,0 +1,405 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include +#include +#include +#include +#include +#include // for ParallelMachine, etc +#include + +#include +#include +#include "stk_search/SearchMethod.hpp" + +namespace +{ + +void runTwoSpheresTest(stk::search::SearchMethod searchMethod, const double distanceBetweenSphereCenters, + const double radius, const unsigned expectedNumOverlap) +{ + MPI_Comm comm = MPI_COMM_WORLD; + int numProcs = stk::parallel_machine_size(comm); + int procId = stk::parallel_machine_rank(comm); + + std::vector> boxVector1; + if (procId == 0) { + boxVector1.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(0, 0, 0, radius, 1, procId)); + } + + std::vector> boxVector2; + if (procId == numProcs-1) { + boxVector2.push_back(stk::unit_test_util::simple_fields::generateBoundingVolume(distanceBetweenSphereCenters, 0, 0, radius, 2, procId)); + } + + std::vector> boxIdPairResults; + stk::search::coarse_search(boxVector1, boxVector2, searchMethod, comm, boxIdPairResults); + + if (procId == 0 || (procId == numProcs-1)) { + ASSERT_EQ(expectedNumOverlap, boxIdPairResults.size()); + } + else { + ASSERT_EQ(0u, boxIdPairResults.size()); + } +} + +const double radiusOfOneHalf = 0.5; + +TEST(CoarseSearchCorrectness, OverlappingSpheres_KDTREE) +{ + double distanceBetweenSphereCenters = 0.5; + const unsigned expectedNumOverlap = 1; + runTwoSpheresTest(stk::search::KDTREE, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, NonOverlappingSpheres_KDTREE) +{ + double distanceBetweenSphereCenters = 2.0; + const unsigned expectedNumOverlap = 0; + runTwoSpheresTest(stk::search::KDTREE, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, JustEdgeOverlappingSpheres_KDTREE) +{ + double distanceBetweenSphereCenters = 0.999999999; + const unsigned expectedNumOverlap = 1; + runTwoSpheresTest(stk::search::KDTREE, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, NotQuiteEdgeOverlappingSpheres_KDTREE) +{ + double distanceBetweenSphereCenters = 1.0000000001; + const unsigned expectedNumOverlap = 0; + runTwoSpheresTest(stk::search::KDTREE, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, OverlappingSpheres_MORTON_LBVH) +{ + double distanceBetweenSphereCenters = 0.5; + const unsigned expectedNumOverlap = 1; + runTwoSpheresTest(stk::search::MORTON_LBVH, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, NonOverlappingSpheres_MORTON_LBVH) +{ + double distanceBetweenSphereCenters = 2.0; + const unsigned expectedNumOverlap = 0; + runTwoSpheresTest(stk::search::MORTON_LBVH, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, JustEdgeOverlappingSpheres_MORTON_LBVH) +{ + double distanceBetweenSphereCenters = 0.999999999; + const unsigned expectedNumOverlap = 1; + runTwoSpheresTest(stk::search::MORTON_LBVH, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, NotQuiteEdgeOverlappingSpheres_MORTON_LBVH) +{ + double distanceBetweenSphereCenters = 1.0000000001; + const unsigned expectedNumOverlap = 0; + runTwoSpheresTest(stk::search::MORTON_LBVH, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, OverlappingSpheres_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + double distanceBetweenSphereCenters = 0.5; + const unsigned expectedNumOverlap = 1; + runTwoSpheresTest(stk::search::ARBORX, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, NonOverlappingSpheres_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + double distanceBetweenSphereCenters = 2.0; + const unsigned expectedNumOverlap = 0; + runTwoSpheresTest(stk::search::KDTREE, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, JustEdgeOverlappingSpheres_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + double distanceBetweenSphereCenters = 0.999999999; + const unsigned expectedNumOverlap = 1; + runTwoSpheresTest(stk::search::ARBORX, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, NotQuiteEdgeOverlappingSpheres_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + double distanceBetweenSphereCenters = 1.00001; + const unsigned expectedNumOverlap = 0; + runTwoSpheresTest(stk::search::ARBORX, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, NotQuiteEdgeOverlappingSpheres_FloatTruncation_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + double distanceBetweenSphereCenters = 1.0000000001; + const unsigned expectedNumOverlap = 0; + runTwoSpheresTest(stk::search::ARBORX, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); +} + +void host_local_runTwoSpheresTest(stk::search::SearchMethod searchMethod, + const double distanceBetweenSphereCenters, + const double radius, + const unsigned expectedNumOverlap) +{ + using BoxType = stk::search::Sphere; + using IdentType = int; + using BoxIdentType = std::pair; + using IntersectionType = std::pair; + + std::vector domain; + std::vector range; + + domain.emplace_back(stk::unit_test_util::simple_fields::box_ident_to_pair( + stk::unit_test_util::simple_fields::device_generateBoxIdent(0, 0, 0, radius, 1))); + + const double axisOffset = distanceBetweenSphereCenters / sqrt(2.0); + range.emplace_back(stk::unit_test_util::simple_fields::box_ident_to_pair( + stk::unit_test_util::simple_fields::device_generateBoxIdent( + axisOffset, axisOffset, 0, radius, 2))); + + std::vector intersections; + + stk::search::local_coarse_search(domain, range, searchMethod, intersections); + + ASSERT_EQ(intersections.size(), expectedNumOverlap); + + for (unsigned i = 0; i < expectedNumOverlap; ++i) { + EXPECT_EQ(intersections[i].first, 1); + EXPECT_EQ(intersections[i].second, 2); + } +} + +void device_local_runTwoSpheresTest(stk::search::SearchMethod searchMethod, const double distanceBetweenSphereCenters, + const double radius, const unsigned expectedNumOverlap) +{ + using BoxType = stk::search::Sphere; + using IdentType = int; + using BoxIdentType = stk::search::BoxIdent; + using IntersectionType = stk::search::IdentIntersection; + + auto domain = Kokkos::View("domain box-ident", 1); + auto range = Kokkos::View("range box-ident", 1); + + Kokkos::parallel_for(stk::ngp::DeviceRangePolicy(0, 1), + KOKKOS_LAMBDA(const unsigned & i) { + domain[0] = + stk::unit_test_util::simple_fields::device_generateBoxIdent(0, 0, 0, radius, 1); + + const double axisOffset = distanceBetweenSphereCenters / sqrt(2.0); + range[0] = + stk::unit_test_util::simple_fields::device_generateBoxIdent(axisOffset, axisOffset, 0, + radius, 2); + }); + + auto intersections = Kokkos::View("intersections", 0); + + stk::search::local_coarse_search(domain, range, searchMethod, intersections); + + Kokkos::View::HostMirror hostIntersections = Kokkos::create_mirror_view(intersections); + Kokkos::deep_copy(hostIntersections, intersections); + + ASSERT_EQ(intersections.extent(0), expectedNumOverlap); + + for (unsigned i = 0; i < expectedNumOverlap; ++i) { + EXPECT_EQ(intersections(i).domainIdent, 1); + EXPECT_EQ(intersections(i).rangeIdent, 2); + } +} + +TEST(CoarseSearchCorrectness, Ngp_Local_OverlappingSpheres_MORTON_LBVH) +{ + double distanceBetweenSphereCenters = 0.5; + const unsigned expectedNumOverlap = 1; + host_local_runTwoSpheresTest(stk::search::MORTON_LBVH, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); + device_local_runTwoSpheresTest(stk::search::MORTON_LBVH, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_NonOverlappingSpheres_MORTON_LBVH) +{ + double distanceBetweenSphereCenters = 2.0; + const unsigned expectedNumOverlap = 0; + host_local_runTwoSpheresTest(stk::search::MORTON_LBVH, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); + device_local_runTwoSpheresTest(stk::search::MORTON_LBVH, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_JustEdgeOverlappingSpheres_MORTON_LBVH) +{ + double distanceBetweenSphereCenters = 0.999999999; + const unsigned expectedNumOverlap = 1; + host_local_runTwoSpheresTest(stk::search::MORTON_LBVH, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); + device_local_runTwoSpheresTest(stk::search::MORTON_LBVH, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_NotQuiteEdgeOverlappingSpheres_MORTON_LBVH) +{ + double distanceBetweenSphereCenters = 1.0000000001; + const unsigned expectedNumOverlap = 0; + host_local_runTwoSpheresTest(stk::search::MORTON_LBVH, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); + device_local_runTwoSpheresTest(stk::search::MORTON_LBVH, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_OverlappingSpheres_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + double distanceBetweenSphereCenters = 0.5; + const unsigned expectedNumOverlap = 1; + host_local_runTwoSpheresTest(stk::search::ARBORX, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); + device_local_runTwoSpheresTest(stk::search::ARBORX, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_NonOverlappingSpheres_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + double distanceBetweenSphereCenters = 2.0; + const unsigned expectedNumOverlap = 0; + host_local_runTwoSpheresTest(stk::search::ARBORX, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); + device_local_runTwoSpheresTest(stk::search::ARBORX, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_JustEdgeOverlappingSpheres_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + double distanceBetweenSphereCenters = 0.999999999; + const unsigned expectedNumOverlap = 1; + host_local_runTwoSpheresTest(stk::search::ARBORX, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); + device_local_runTwoSpheresTest(stk::search::ARBORX, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); +} + +TEST(CoarseSearchCorrectness, Ngp_Local_NotQuiteEdgeOverlappingSpheres_ARBORX) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + double distanceBetweenSphereCenters = 1.0000000001; + const unsigned expectedNumOverlap = 0; + host_local_runTwoSpheresTest(stk::search::ARBORX, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); + device_local_runTwoSpheresTest(stk::search::ARBORX, distanceBetweenSphereCenters, radiusOfOneHalf, expectedNumOverlap); +} + +stk::search::Box make_naive_arbor_x_box_from_double_sphere(stk::search::Sphere sphere) +{ + + auto center = sphere.center(); + float radius = static_cast(sphere.radius()); + float x_min = static_cast(center[0]) - radius; + float y_min = static_cast(center[1]) - radius; + float z_min = static_cast(center[2]) - radius; + float x_max = static_cast(center[0]) + radius; + float y_max = static_cast(center[1]) + radius; + float z_max = static_cast(center[2]) + radius; + + return stk::search::Box(x_min, y_min, z_min, x_max, y_max, z_max); +} + +TEST(CoarseSearchCorrectness, ARBORX_FLOAT_PRECISION_SPHERES) +{ +#ifndef STK_HAS_ARBORX + GTEST_SKIP(); +#endif + + if (stk::parallel_machine_size(MPI_COMM_WORLD) != 1) { GTEST_SKIP(); } + + typedef double SpherePrecision; + + SpherePrecision domainRadius = 1.400000000001; + stk::search::Point domainCenter(0.0, 0.0, 0.0); + SpherePrecision rangeRadius = 1.0; + stk::search::Point rangeCenter(2.400000000001, 0.0, 0.0); + + stk::search::Sphere domain(domainCenter, domainRadius); + std::vector, Ident>> domainVector; + domainVector.push_back({domain, Ident(0,0)}); + + stk::search::Sphere range(rangeCenter, rangeRadius); + std::vector, Ident>> rangeVector; + rangeVector.push_back({range, Ident(0,0)}); + + std::vector> searchResultsMorton; + stk::search::coarse_search(domainVector, rangeVector, stk::search::MORTON_LBVH, MPI_COMM_WORLD, searchResultsMorton); + EXPECT_EQ(1u, searchResultsMorton.size()); + + /* ArborX only uses single precision values; as a result, any double-precision StkBox will be cast down + to floats. To prevent the case where an intersection between two boxes is missed due to truncation + by ArborX, the boxes are expanded outward when converting from StkBox to ArborX. The two boxes + constructed below are representative of the "naive" StkBox to ArborX conversion that would happen + without expanding the boxes. */ + stk::search::Box naiveArborXDomain = make_naive_arbor_x_box_from_double_sphere(domain); + stk::search::Box naiveArborXRange = make_naive_arbor_x_box_from_double_sphere(range); + std::vector, Ident>> naiveArborXDomainVector; + naiveArborXDomainVector.push_back({naiveArborXDomain, Ident(0,0)}); + std::vector, Ident>> naiveArborXRangeVector; + naiveArborXRangeVector.push_back({naiveArborXRange, Ident(0,0)}); + + std::vector> searchResultsNaiveArborX; + stk::search::coarse_search(naiveArborXDomainVector, naiveArborXRangeVector, stk::search::ARBORX, MPI_COMM_WORLD, searchResultsNaiveArborX); + EXPECT_EQ(0u, searchResultsNaiveArborX.size()); + + /*This case captures the "expanded" box behavior that is now the default when converting from StkBoxes to ArborX.*/ + stk::search::Sphere domainExpanded(domainCenter, domainRadius); + std::vector, Ident>> domainExpandedVector; + domainExpandedVector.push_back({domainExpanded, Ident(0,0)}); + + stk::search::Sphere rangeExpanded(rangeCenter, rangeRadius); + std::vector, Ident>> rangeExpandedVector; + rangeExpandedVector.push_back({rangeExpanded, Ident(0,0)}); + + std::vector> searchResultsExpanded; + stk::search::coarse_search(domainExpandedVector, rangeExpandedVector, stk::search::ARBORX, MPI_COMM_WORLD, searchResultsExpanded); + EXPECT_EQ(1u, searchResultsExpanded.size()); + +} + +} diff --git a/packages/stk/stk_unit_tests/stk_search/UnitTestKDTreeSphereHierarchy.cpp b/packages/stk/stk_unit_tests/stk_search/UnitTestKDTreeSphereHierarchy.cpp index ccae7a8d1e27..61ebc1dfb418 100644 --- a/packages/stk/stk_unit_tests/stk_search/UnitTestKDTreeSphereHierarchy.cpp +++ b/packages/stk/stk_unit_tests/stk_search/UnitTestKDTreeSphereHierarchy.cpp @@ -9,7 +9,7 @@ #include #include -#include +#include namespace { diff --git a/packages/stk/stk_unit_tests/stk_simd/UnitTestCheckSimdInfo.cpp b/packages/stk/stk_unit_tests/stk_simd/UnitTestCheckSimdInfo.cpp index 71a3c88ce1fe..10bc54ffbb96 100644 --- a/packages/stk/stk_unit_tests/stk_simd/UnitTestCheckSimdInfo.cpp +++ b/packages/stk/stk_unit_tests/stk_simd/UnitTestCheckSimdInfo.cpp @@ -44,7 +44,7 @@ struct FuncWithExecPolicy { using execution_policy = Kokkos::RangePolicy; }; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef STK_ENABLE_GPU TEST( SimdInfo, checkDeviceWidths ) { EXPECT_EQ(stk::unit_test_util::get_float_width_on_device(), 1); diff --git a/packages/stk/stk_unit_tests/stk_simd/UnitTestPrintSimdInfo.cpp b/packages/stk/stk_unit_tests/stk_simd/UnitTestPrintSimdInfo.cpp index f645f554b6ec..000f845bfc5d 100644 --- a/packages/stk/stk_unit_tests/stk_simd/UnitTestPrintSimdInfo.cpp +++ b/packages/stk/stk_unit_tests/stk_simd/UnitTestPrintSimdInfo.cpp @@ -53,7 +53,7 @@ TEST( PrintSimdInfo, printTypes ) stk::unit_test_util::simple_fields::print_type(d._data); } -#ifdef KOKKOS_ENABLE_CUDA +#ifdef STK_ENABLE_GPU TEST( PrintSimdInfo, printDeviceWidths ) { std::cout << "width of stk::simd::Float on device " << stk::unit_test_util::get_float_width_on_device() << std::endl; diff --git a/packages/stk/stk_unit_tests/stk_tools/mesh_tools/DisconnectBlocksMeshConstruction.cpp b/packages/stk/stk_unit_tests/stk_tools/mesh_tools/DisconnectBlocksMeshConstruction.cpp index 18f0dfe9f843..ee0d985c3b6c 100644 --- a/packages/stk/stk_unit_tests/stk_tools/mesh_tools/DisconnectBlocksMeshConstruction.cpp +++ b/packages/stk/stk_unit_tests/stk_tools/mesh_tools/DisconnectBlocksMeshConstruction.cpp @@ -1034,18 +1034,14 @@ void print_hinge_info(const stk::mesh::BulkData& bulk, if(hingeNodes.size() > 0) { os << "PRINTING HINGE NODES on Proc " << bulk.parallel_rank() << " : " << std::endl; for(const stk::tools::impl::HingeNode& node : hingeNodes) { - if(hinge_node_is_locally_owned(bulk, node)) { - os << "\tHinge node id: " << bulk.identifier(node.get_node()) << std::endl; - } + os << "\tHinge node id: " << bulk.identifier(node.get_node()) << std::endl; } } if(hingeEdges.size() > 0) { os << "PRINTING HINGE EDGES on Proc " << bulk.parallel_rank() << " : " << std::endl; for(const stk::tools::impl::HingeEdge& edge : hingeEdges) { - if(hinge_edge_is_locally_owned(bulk, edge)) { - os << "\tHinge edge ids: " << bulk.identifier(edge.first.get_node()) - << ", " << bulk.identifier(edge.second.get_node()) << std::endl; - } + os << "\tHinge edge ids: " << bulk.identifier(edge.first.get_node()) + << ", " << bulk.identifier(edge.second.get_node()) << std::endl; } } diff --git a/packages/stk/stk_unit_tests/stk_tools/mesh_tools/UnitTestDetectHinge.cpp b/packages/stk/stk_unit_tests/stk_tools/mesh_tools/UnitTestDetectHinge.cpp index 46ef735bf8a2..d55321052b75 100644 --- a/packages/stk/stk_unit_tests/stk_tools/mesh_tools/UnitTestDetectHinge.cpp +++ b/packages/stk/stk_unit_tests/stk_tools/mesh_tools/UnitTestDetectHinge.cpp @@ -679,9 +679,16 @@ TEST(DetectHinge3D, SingleBlockTwoByTwoHexTwoEdgeHinge_Decomp1) four_elements_decomposition(bulk); + stk::io::write_mesh("output_hinge", *bulkPtr); + std::pair hingeCount = stk::tools::impl::get_hinge_count(bulk); EXPECT_EQ(0u, hingeCount.first); - EXPECT_EQ(2u, hingeCount.second); + + if (stk::parallel_machine_size(MPI_COMM_WORLD) == 4) { + EXPECT_EQ(1u, hingeCount.second); + } else { + EXPECT_EQ(2u, hingeCount.second); + } output_mesh(bulk); } @@ -697,7 +704,12 @@ TEST(DetectHinge3D, SingleBlockTwoByTwoHexTwoEdgeHinge_Decomp2) std::pair hingeCount = stk::tools::impl::get_hinge_count(bulk); EXPECT_EQ(0u, hingeCount.first); - EXPECT_EQ(2u, hingeCount.second); + + if (stk::parallel_machine_size(MPI_COMM_WORLD) == 4) { + EXPECT_EQ(1u, hingeCount.second); + } else { + EXPECT_EQ(2u, hingeCount.second); + } output_mesh(bulk); } @@ -851,7 +863,7 @@ TEST(DetectHinge3D, SingleBlockFourHexTwoNodeHingeOneEdgeHinge_Manual) std::pair hingeCount = stk::tools::impl::get_hinge_count(bulk); EXPECT_EQ(2u, hingeCount.first); - EXPECT_EQ(1u, hingeCount.second); + EXPECT_EQ(0u, hingeCount.second); output_mesh(bulk); } @@ -1348,13 +1360,16 @@ TEST(ElementGroups3D, SingleBlockTwoByTwoHexTwoEdgeHinge_Decomp2) EXPECT_EQ(1u, groupings[1].size()); EXPECT_TRUE(groupings[0] != groupings[1]); } - groupings = stk::tools::impl::get_convex_groupings(bulk, hingeEdges[1]); - if(!groupings.empty()) { - EXPECT_EQ(2u, groupings.size()); - EXPECT_EQ(2u, groupings[0].size() + groupings[1].size()); - EXPECT_EQ(1u, groupings[0].size()); - EXPECT_EQ(1u, groupings[1].size()); - EXPECT_TRUE(groupings[0] != groupings[1]); + + if (stk::parallel_machine_size(MPI_COMM_WORLD) == 1) { + groupings = stk::tools::impl::get_convex_groupings(bulk, hingeEdges[1]); + if(!groupings.empty()) { + EXPECT_EQ(2u, groupings.size()); + EXPECT_EQ(2u, groupings[0].size() + groupings[1].size()); + EXPECT_EQ(1u, groupings[0].size()); + EXPECT_EQ(1u, groupings[1].size()); + EXPECT_TRUE(groupings[0] != groupings[1]); + } } } diff --git a/packages/stk/stk_unit_tests/stk_transfer/CMakeLists.txt b/packages/stk/stk_unit_tests/stk_transfer/CMakeLists.txt index 36b7d5c22786..e648cfad5907 100644 --- a/packages/stk/stk_unit_tests/stk_transfer/CMakeLists.txt +++ b/packages/stk/stk_unit_tests/stk_transfer/CMakeLists.txt @@ -61,6 +61,10 @@ else() target_link_libraries(stk_transfer_utest stk_util_parallel) target_link_libraries(stk_transfer_utest stk_unit_main) + target_include_directories(stk_transfer_utest PUBLIC + ${STK_TOPLEVEL_SOURCE_DIR}/stk_unit_test_utils + ) + add_test("stk_transfer_utest" sh -c "mpiexec --np 2 ${CMAKE_CURRENT_BINARY_DIR}/stk_transfer_utest --gtest_filter=GeomXferImpl.coarseSearchImpl_emptyDomain") endif() diff --git a/packages/stk/stk_unit_tests/stk_util/diag/UnitTestString.cpp b/packages/stk/stk_unit_tests/stk_util/diag/UnitTestString.cpp index cf50e099414a..f160716472dc 100644 --- a/packages/stk/stk_unit_tests/stk_util/diag/UnitTestString.cpp +++ b/packages/stk/stk_unit_tests/stk_util/diag/UnitTestString.cpp @@ -131,3 +131,304 @@ TEST(StkJoinString, iterRangeNonString) const std::vector theVect{0, 1, 2}; EXPECT_EQ("0, 1, 2", stk::util::join(theVect.begin(), theVect.end(), ", ")); } + +TEST(StkConstructString, fromConstCharPtr) +{ + const char * const str = "test message"; + const sierra::String sierra_str{str}; + EXPECT_EQ(str, sierra_str); +} + +TEST(StkConstructString, fromNullPtr) +{ + const char * const str = nullptr; + const sierra::String sierra_str{str}; + EXPECT_TRUE(sierra_str.empty()); +} + +TEST(StkConstructString, fromStdString) +{ + const std::string str = "test message"; + const sierra::String sierra_str{str}; + EXPECT_EQ(str, sierra_str); +} + +TEST(StkConstructString, fromStringLiteral) +{ + const std::string str = "test message"; + const sierra::String sierra_str{"test message"}; + EXPECT_EQ(str, sierra_str); +} + +TEST(StkConstructString, fromTemporaryStdString) +{ + const std::string str = "test message"; + const sierra::String sierra_str{std::string("test message")}; + EXPECT_EQ(str, sierra_str); +} + +TEST(StkConstructString, fromStdStringView) +{ + const std::string_view str = "test message"; + const sierra::String sierra_str{str}; + EXPECT_EQ(sierra::String(str), sierra_str); +} + +TEST(StkAssignString, fromConstCharPtr) +{ + const char * const str = "test message"; + sierra::String sierra_str; + sierra_str.assign(str); + EXPECT_EQ(str, sierra_str); +} + +TEST(StkAssignString, fromNullPtr) +{ + const char * const str = nullptr; + sierra::String sierra_str; + sierra_str.assign(str); + EXPECT_TRUE(sierra_str.empty()); +} + +TEST(StkAssignString, fromStdString) +{ + const std::string str = "test message"; + sierra::String sierra_str; + sierra_str.assign(str); + EXPECT_EQ(str, sierra_str); +} + +namespace { + sierra::String func_taking_sierra_string(const sierra::String & sierra_str) + { + return sierra_str; + } +} + +// Note, following two require non explicit converting constructor for StringViewLike -> StringBase +// I think we should deprecate this behavior. +TEST(StkString, canCallSierraStringFunctionWithStandardString) +{ + const std::string str = "test message"; + + const sierra::String sierra_str = func_taking_sierra_string(str); + EXPECT_EQ(str, sierra_str); +} + +TEST(StkString, canCallSierraStringFunctionWithStringView) +{ + const std::string_view str = "test message"; + + //explicit conversion from string view + const sierra::String sierra_str = func_taking_sierra_string(sierra::String(str)); + EXPECT_EQ(sierra::String(str), sierra_str); +} + +TEST(StkAssignToString, fromStdStringView) +{ + const std::string_view str = "test message"; + sierra::String sierra_str; + sierra_str = str; + EXPECT_EQ(sierra::String(str), sierra_str); +} + +TEST(StkAssignToString, fromStdString) +{ + const std::string str = "test message"; + sierra::String sierra_str; + sierra_str = str; + EXPECT_EQ(sierra::String(str), sierra_str); +} + +TEST(StkAssignToString, fromConstCharPtr) +{ + const char * str = "test message"; + sierra::String sierra_str; + sierra_str = str; + EXPECT_EQ(sierra::String(str), sierra_str); +} + +TEST(StkAssignToString, fromNullPtr) +{ + const char * str = nullptr; + sierra::String sierra_str; + sierra_str = str; + EXPECT_EQ(sierra::String(), sierra_str); +} + +TEST(StkAppendToString, fromStdStringView) +{ + const std::string_view str = "test message"; + sierra::String sierra_str{"first: "}; + sierra_str.append(str); + EXPECT_EQ(sierra::String("first: test message"), sierra_str); +} + +TEST(StkAppendToString, fromStdString) +{ + const std::string str = "test message"; + sierra::String sierra_str{"first: "}; + sierra_str.append(str); + EXPECT_EQ(sierra::String("first: test message"), sierra_str); +} + +TEST(StkAppendToString, fromConstCharPtr) +{ + const char * str = "test message"; + sierra::String sierra_str{"first: "}; + sierra_str.append(str); + EXPECT_EQ(sierra::String("first: test message"), sierra_str); +} + +TEST(StkOperatorPlusEqToString, fromStdStringView) +{ + const std::string_view str = "test message"; + sierra::String sierra_str{"first: "}; + sierra_str += str; + EXPECT_EQ(sierra::String("first: test message"), sierra_str); +} + +TEST(StkOperatorPlusEqToString, fromStdString) +{ + const std::string str = "test message"; + sierra::String sierra_str{"first: "}; + sierra_str += str; + EXPECT_EQ(sierra::String("first: test message"), sierra_str); +} + +TEST(StkOperatorPlusEqToString, fromConstCharPtr) +{ + const char * str = "test message"; + sierra::String sierra_str{"first: "}; + sierra_str += str; + EXPECT_EQ(sierra::String("first: test message"), sierra_str); +} + +TEST(StkStringComparison, equalStrings) +{ + sierra::String str_1{"abcdefg"}; + sierra::String str_2{"AbcdEfg"}; + EXPECT_TRUE(str_1 == str_2); + EXPECT_TRUE(str_1 <= str_2); + EXPECT_TRUE(str_1 >= str_2); + EXPECT_FALSE(str_1 < str_2); + EXPECT_FALSE(str_1 > str_2); + EXPECT_FALSE(str_1 != str_2); +} + +using SierraStd = std::pair; +using StdSierra = std::pair; +using CharSierra = std::pair; +using SierraChar = std::pair; +using StringViewSierra = std::pair; +using SierraStringView = std::pair; + +template +class StringTypePairFixture : public testing::Test +{ +public: +using Type = T; +}; + +typedef ::testing::Types MyTypes; +TYPED_TEST_SUITE(StringTypePairFixture, MyTypes); +TYPED_TEST(StringTypePairFixture, str1LessThanStr2ByLength) +{ + typename TypeParam::first_type str_1{"abcdef"}; + typename TypeParam::second_type str_2{"AbcdEfg"}; + EXPECT_FALSE(str_1 == str_2); + EXPECT_TRUE(str_1 <= str_2); + EXPECT_FALSE(str_1 >= str_2); + EXPECT_TRUE(str_1 < str_2); + EXPECT_FALSE(str_1 > str_2); + EXPECT_TRUE(str_1 != str_2); +} + +TYPED_TEST(StringTypePairFixture, str1LessThanStr2ByCharacter) +{ + typename TypeParam::first_type str_1{"abbdefg"}; + typename TypeParam::second_type str_2{"AbcdEfg"}; + EXPECT_FALSE(str_1 == str_2); + EXPECT_TRUE(str_1 <= str_2); + EXPECT_FALSE(str_1 >= str_2); + EXPECT_TRUE(str_1 < str_2); + EXPECT_FALSE(str_1 > str_2); + EXPECT_TRUE(str_1 != str_2); +} + + +TYPED_TEST(StringTypePairFixture, str1GreaterThanStr2) +{ + typename TypeParam::first_type str_1{"AbcdEfg"}; + typename TypeParam::second_type str_2{"abcdef"}; + EXPECT_FALSE(str_1 == str_2); + EXPECT_FALSE(str_1 <= str_2); + EXPECT_TRUE(str_1 >= str_2); + EXPECT_FALSE(str_1 < str_2); + EXPECT_TRUE(str_1 > str_2); + EXPECT_TRUE(str_1 != str_2); +} + +TYPED_TEST(StringTypePairFixture, operatorPlus) +{ + typename TypeParam::first_type str_1{"oNe string, "}; + typename TypeParam::second_type str_2{"two string"}; + const sierra::String expected{"ONE STRING, TWO STRING"}; + EXPECT_EQ(expected, str_1 + str_2); +} + +TEST(StkStringComparison, equalIdentifiers) +{ + sierra::Identifier str_1{"abc defg"}; + sierra::Identifier str_2{"Abc_dEfg"}; + EXPECT_TRUE(str_1 == str_2); + EXPECT_TRUE(str_1 <= str_2); + EXPECT_TRUE(str_1 >= str_2); + EXPECT_FALSE(str_1 < str_2); + EXPECT_FALSE(str_1 > str_2); + EXPECT_FALSE(str_1 != str_2); +} + +TEST(StkStringComparison, compareWithNullPtr) +{ + sierra::Identifier str_1{"abc defg"}; + const char * str_2 = nullptr; + EXPECT_EQ(-1, str_1.compare(str_2)); +} + +TEST(StkStringComparison, compareEmptyWithNull) +{ + sierra::Identifier str_1{}; + const char * str_2 = nullptr; + EXPECT_EQ(0, str_1.compare(str_2)); +} + +namespace { + sierra::Identifier func_taking_sierra_identifier(const sierra::Identifier & identifier) + { + return identifier; + } +} + +TEST(StkStringIdentfierConversions, equalIdentifiers) +{ + sierra::String identifier_to_string_construction(sierra::Identifier("test")); + sierra::String string_to_identifier_construction(sierra::String("test")); + + //test implicit move conversions -- future work to deprecate these + func_taking_sierra_string(sierra::Identifier("test")); + func_taking_sierra_identifier(sierra::String("test")); + + //test implicit copy conversions -- future work to deprecate these + const sierra::Identifier sierra_identifier("test"); + const sierra::String sierra_string("test"); + func_taking_sierra_string(sierra_identifier); + func_taking_sierra_identifier(sierra_string); + + //shouldbe nondeprecated versions + func_taking_sierra_string(sierra::String(sierra_identifier)); + func_taking_sierra_identifier(sierra::Identifier(sierra_string)); + + std::string_view test = std::string_view(sierra_string); + test = ""; +} diff --git a/packages/stk/stk_unit_tests/stk_util/parallel/UnitTestGenerateIds.cpp b/packages/stk/stk_unit_tests/stk_util/parallel/UnitTestGenerateIds.cpp index 2c7e95f5217c..9526421ad82d 100644 --- a/packages/stk/stk_unit_tests/stk_util/parallel/UnitTestGenerateIds.cpp +++ b/packages/stk/stk_unit_tests/stk_util/parallel/UnitTestGenerateIds.cpp @@ -442,7 +442,8 @@ void getIdUsageAcrossAllProcs(std::vector< std::vector > &idsToComm, s MPI_Send(&numItemsToComm, 1, sierra::MPI::Datatype::type(), j, mpiInfo.getNumProcs()*i+j, mpiInfo.getMpiComm()); if ( numItemsToComm > 0 ) { - MPI_Send(&idsToComm[j][0], numItemsToComm, sierra::MPI::Datatype::type(), j, mpiInfo.getNumProcs()*i+j, mpiInfo.getMpiComm()); + MPI_Send(idsToComm[j].data(), numItemsToComm, sierra::MPI::Datatype::type(), j, + mpiInfo.getNumProcs() * i + j, mpiInfo.getMpiComm()); } } } @@ -456,7 +457,8 @@ void getIdUsageAcrossAllProcs(std::vector< std::vector > &idsToComm, s { std::vector idsFromOtherProc(numItemsToReceive,0); MPI_Request request; - MPI_Irecv(&idsFromOtherProc[0], numItemsToReceive, sierra::MPI::Datatype::type(), i, mpiInfo.getNumProcs()*i+mpiInfo.getProcId(), mpiInfo.getMpiComm(), &request); + MPI_Irecv(idsFromOtherProc.data(), numItemsToReceive, sierra::MPI::Datatype::type(), i, + mpiInfo.getNumProcs() * i + mpiInfo.getProcId(), mpiInfo.getMpiComm(), &request); MPI_Status status2; MPI_Wait(&request, &status2); idsInUseAcrossAllProcsInMyRange.insert(idsInUseAcrossAllProcsInMyRange.end(), idsFromOtherProc.begin(), idsFromOtherProc.end()); @@ -595,7 +597,7 @@ void respondToRootProcessorAboutIdsOwnedOnThisProc(const int root, const uint64_ } } int *rbuff = 0; - MPI_Reduce(&areIdsBeingused[0], rbuff, numIdsToGet, MPI_INT, MPI_SUM, root, comm); + MPI_Reduce(areIdsBeingused.data(), rbuff, numIdsToGet, MPI_INT, MPI_SUM, root, comm); } } @@ -608,7 +610,7 @@ void retrieveIds(const INTMPI root, uint64_t id, MPI_Comm comm, uint64_t numIdsT { MPI_Bcast(&numIdsToGetPerProc, 1, sierra::MPI::Datatype::type(), root, comm); std::vector zeroids(numIdsToGetPerProc,0); - MPI_Reduce(&zeroids[0], &areIdsBeingUsed[0], numIdsToGetPerProc, MPI_INT, MPI_SUM, root, comm); + MPI_Reduce(zeroids.data(), areIdsBeingUsed.data(), numIdsToGetPerProc, MPI_INT, MPI_SUM, root, comm); } } @@ -958,7 +960,8 @@ void terminateIdRequestForThisProc(INTMPI root, MPI_Comm comm) void getAvailableIds(const std::vector &myIds, uint64_t numIdsNeeded, std::vector &idsObtained, uint64_t &startingIdToSearchForNewIds, const uint64_t maxId, const MpiInfo& mpiInfo) { std::vector receivedInfo(mpiInfo.getNumProcs(),0); - MPI_Allgather(&numIdsNeeded, 1, sierra::MPI::Datatype::type(), &receivedInfo[0], 1, sierra::MPI::Datatype::type(), mpiInfo.getMpiComm()); + MPI_Allgather(&numIdsNeeded, 1, sierra::MPI::Datatype::type(), receivedInfo.data(), 1, + sierra::MPI::Datatype::type(), mpiInfo.getMpiComm()); std::vector sortedIds(myIds.begin(), myIds.end()); std::sort(sortedIds.begin(), sortedIds.end()); @@ -989,7 +992,8 @@ void getAvailableIds_exp(const std::vector &myIds, uint64_t numIdsNeed { INTMPI numprocs = mpiInfo.getNumProcs(); std::vector receivedInfo(numprocs,0); - MPI_Allgather(&numIdsNeeded, 1, sierra::MPI::Datatype::type(), &receivedInfo[0], 1, sierra::MPI::Datatype::type(), mpiInfo.getMpiComm()); + MPI_Allgather(&numIdsNeeded, 1, sierra::MPI::Datatype::type(), receivedInfo.data(), 1, + sierra::MPI::Datatype::type(), mpiInfo.getMpiComm()); std::vector sortedIds(myIds.begin(), myIds.end()); std::sort(sortedIds.begin(), sortedIds.end()); diff --git a/packages/stk/stk_unit_tests/stk_util/parallel/UnitTestParallelComm.cpp b/packages/stk/stk_unit_tests/stk_util/parallel/UnitTestParallelComm.cpp index d85e3d99a69d..64098c3ae44f 100644 --- a/packages/stk/stk_unit_tests/stk_util/parallel/UnitTestParallelComm.cpp +++ b/packages/stk/stk_unit_tests/stk_util/parallel/UnitTestParallelComm.cpp @@ -491,6 +491,140 @@ TEST_F(NeighborParallelCommTesterDouble, FuncBlocking) test_results(get_receive_lists()); } +//----------------------------------------------------------------------------- +// test parallel_data_exchange_nonsym_known_sizes_t +TEST(NonsymKnownsizes, nominal_correct) +{ + const int numProcs = stk::parallel_machine_size(MPI_COMM_WORLD); + if (numProcs != 4) { GTEST_SKIP(); } + + std::vector sendOffsets, recvOffsets; + std::vector sendData, recvData, expectedRecvData; + + const int myProc = stk::parallel_machine_rank(MPI_COMM_WORLD); + + if (myProc == 2) { + sendOffsets = { 0, 0, 0, 0, 0 }; + recvOffsets = { 0, 2, 4, 4, 6 }; //recving from procs 0, 1, 3 + expectedRecvData = { 42.0, 99.0, 42.0, 99.0, 42.0, 99.0 }; + recvData.resize(6); + } + else { + sendOffsets = { 0, 0, 0, 2, 2 }; //sending to proc 2 + sendData = { 42.0, 99.0 }; + recvOffsets = { 0, 0, 0, 0, 0 }; + } + + const bool checkInput = true; + EXPECT_NO_THROW( + stk::parallel_data_exchange_nonsym_known_sizes_t(sendOffsets.data(), sendData.data(), + recvOffsets.data(), recvData.data(), + MPI_COMM_WORLD, checkInput)); + + EXPECT_EQ(recvData.size(), expectedRecvData.size()); + if (myProc == 2) { + EXPECT_EQ(6u, recvData.size()); + } + + for(unsigned i=0; i sendOffsets, recvOffsets; + std::vector sendData, recvData, expectedRecvData; + + const int myProc = stk::parallel_machine_rank(MPI_COMM_WORLD); + + if (myProc == 2) { + sendOffsets = { 0, 0, 0, 0, 0 }; + recvOffsets = { 0, 2, 4, 4, 6 }; //recving from procs 0, 1, 3 + expectedRecvData = { 42.0, 99.0, 42.0, 99.0, 42.0, 99.0 }; + recvData.resize(6); + } + else if (myProc == 0) { + sendOffsets = { 0, 0, 1, 3, 3 }; //sending to procs 1 and 2 + sendData = { 42.0, 99.0 }; + recvOffsets = { 0, 0, 0, 0, 0 }; + } + else { + sendOffsets = { 0, 0, 0, 2, 2 }; + sendData = { 42.0, 99.0 }; + recvOffsets = { 0, 0, 0, 0, 0 }; + } + + const bool checkInput = true; + EXPECT_ANY_THROW( + stk::parallel_data_exchange_nonsym_known_sizes_t(sendOffsets.data(), sendData.data(), + recvOffsets.data(), recvData.data(), + MPI_COMM_WORLD, checkInput)); + +} + +TEST(NonsymKnownsizes, inconsistent_recv_input) +{ + const int numProcs = stk::parallel_machine_size(MPI_COMM_WORLD); + if (numProcs != 4) { GTEST_SKIP(); } + + std::vector sendOffsets, recvOffsets; + std::vector sendData, recvData, expectedRecvData; + + const int myProc = stk::parallel_machine_rank(MPI_COMM_WORLD); + + if (myProc == 2) { + sendOffsets = { 0, 0, 0, 0, 0 }; + recvOffsets = { 0, 0, 2, 2, 4 }; // only recving from procs 1 and 3 + expectedRecvData = { 42.0, 99.0, 42.0, 99.0, 42.0, 99.0 }; + recvData.resize(6); + } + else { + sendOffsets = { 0, 0, 0, 2, 2 }; //sending to proc 2 + sendData = { 42.0, 99.0 }; + recvOffsets = { 0, 0, 0, 0, 0 }; + } + + const bool checkInput = true; + EXPECT_ANY_THROW( + stk::parallel_data_exchange_nonsym_known_sizes_t(sendOffsets.data(), sendData.data(), + recvOffsets.data(), recvData.data(), + MPI_COMM_WORLD, checkInput)); + +} + +TEST(NonsymKnownsizes, inconsistent_msg_sizing) +{ + const int numProcs = stk::parallel_machine_size(MPI_COMM_WORLD); + if (numProcs != 4) { GTEST_SKIP(); } + + std::vector sendOffsets, recvOffsets; + std::vector sendData, recvData, expectedRecvData; + + const int myProc = stk::parallel_machine_rank(MPI_COMM_WORLD); + + if (myProc == 2) { + sendOffsets = { 0, 0, 0, 0, 0 }; + recvOffsets = { 0, 2, 4, 4, 6 }; //recving 2 values each from procs 0, 1, 3 + expectedRecvData = { 42.0, 99.0, 42.0, 99.0, 42.0, 99.0 }; + recvData.resize(6); + } + else { + sendOffsets = { 0, 0, 0, 1, 1 }; //sending 1 value to proc 2 + sendData = { 42.0, 99.0 }; + recvOffsets = { 0, 0, 0, 0, 0 }; + } + + const bool checkInput = true; + EXPECT_ANY_THROW( + stk::parallel_data_exchange_nonsym_known_sizes_t(sendOffsets.data(), sendData.data(), + recvOffsets.data(), recvData.data(), + MPI_COMM_WORLD, checkInput)); +} + //----------------------------------------------------------------------------- // test ParallelDataExchange class diff --git a/packages/stk/stk_unit_tests/stk_util/parallel/UnitTestParallelReduce.cpp b/packages/stk/stk_unit_tests/stk_util/parallel/UnitTestParallelReduce.cpp index 229a4475fa30..9f73e44f44a0 100644 --- a/packages/stk/stk_unit_tests/stk_util/parallel/UnitTestParallelReduce.cpp +++ b/packages/stk/stk_unit_tests/stk_util/parallel/UnitTestParallelReduce.cpp @@ -62,14 +62,14 @@ TEST(ParallelComm, AllReduceLoc) locations[n] = limit_32bit_integer + myProcId*numProcs*numProcs+n; //Want to test when outside 32-bit range for index. } - stk::all_reduce_maxloc(comm, &values[0], &locations[0], &globalValues[0], &globalLocations[0], nvalues); + stk::all_reduce_maxloc(comm, values.data(), locations.data(), globalValues.data(), globalLocations.data(), nvalues); for(int n = 0; n < nvalues; n++) { EXPECT_EQ((numProcs-1)*numProcs+n, globalValues[n]); EXPECT_EQ(limit_32bit_integer + (numProcs-1)*numProcs*numProcs+n, globalLocations[n]); } - stk::all_reduce_minloc(comm, &values[0], &locations[0], &globalValues[0], &globalLocations[0], nvalues); + stk::all_reduce_minloc(comm, values.data(), locations.data(), globalValues.data(), globalLocations.data(), nvalues); for(int n = 0; n < nvalues; n++) { EXPECT_EQ(n, globalValues[n]); @@ -93,12 +93,12 @@ TEST(ParallelComm, AllReduce) values[n] = myProcId*numProcs+n; } - stk::all_reduce_max(comm, &values[0], &globalValues[0], nvalues); + stk::all_reduce_max(comm, values.data(), globalValues.data(), nvalues); for(int n = 0; n < nvalues; n++) { EXPECT_EQ((numProcs-1)*numProcs+n, globalValues[n]); } - stk::all_reduce_min(comm, &values[0], &globalValues[0], nvalues); + stk::all_reduce_min(comm, values.data(), globalValues.data(), nvalues); for(int n = 0; n < nvalues; n++) { EXPECT_EQ(n, globalValues[n]); } @@ -108,7 +108,7 @@ TEST(ParallelComm, AllReduce) alpha += n*numProcs; } - stk::all_reduce_sum(comm, &values[0], &globalValues[0], nvalues); + stk::all_reduce_sum(comm, values.data(), globalValues.data(), nvalues); for(int n = 0; n < nvalues; n++) { EXPECT_EQ(n*numProcs + alpha, globalValues[n]); } diff --git a/packages/stk/stk_unit_tests/stk_util/util/UnitTestStridedArray.cpp b/packages/stk/stk_unit_tests/stk_util/util/UnitTestStridedArray.cpp index 3d20d9455d3a..eb5d63d7f9f5 100644 --- a/packages/stk/stk_unit_tests/stk_util/util/UnitTestStridedArray.cpp +++ b/packages/stk/stk_unit_tests/stk_util/util/UnitTestStridedArray.cpp @@ -60,3 +60,59 @@ TEST( StridedArray, pair_iter) } } +TEST( StridedArray, comparison) +{ + std::vector vec = {1, 2, 3, 4}; + stk::util::StridedArray stridedArray1(vec.data(), vec.size()); + stk::util::StridedArray stridedArray2(vec.data(), vec.size()); + + EXPECT_EQ(stridedArray1, stridedArray2); + + std::vector otherVec = {1, 4, 3, 2}; + stk::util::StridedArray otherStridedArray(otherVec.data(), otherVec.size()); + + EXPECT_NE(stridedArray1, otherStridedArray); + + std::vector smallerVec = {1, 2}; + stk::util::StridedArray smallerStridedArray(smallerVec.data(), smallerVec.size()); + + EXPECT_NE(stridedArray1, smallerStridedArray); +} + +#ifdef STK_ENABLE_GPU +void run_comparison_test_on_device() +{ + constexpr int N1 = 3; + constexpr int N2 = 5; + Kokkos::View v1("v1", N1, N2); + + Kokkos::parallel_for(N1, KOKKOS_LAMBDA(const int& i) { + for(int j=0; j sa(v1.data(), N2, stride); + localResult = static_cast(sa.size())==N2 ? 0 : 1; + localResult += sa[1]==v1(0,1) ? 0 : 3; + stk::util::StridedArray sa2(v1.data(), N2, stride); + localResult += (sa == sa2) ? 0 : 7; + + stk::util::StridedArray saWrong(v1.data(), N2, wrongStride); + localResult += (sa != saWrong) ? 0 : 15; + }, result); + + EXPECT_EQ(0, result); +} + +TEST(StridedArray, comparison_on_device) +{ + run_comparison_test_on_device(); +} + +#endif + diff --git a/packages/stk/stk_unit_tests/stk_util/util/UnitTestThrowMacros.cpp b/packages/stk/stk_unit_tests/stk_util/util/UnitTestThrowMacros.cpp index f7548bd6c929..9cfbd3aad3ea 100644 --- a/packages/stk/stk_unit_tests/stk_util/util/UnitTestThrowMacros.cpp +++ b/packages/stk/stk_unit_tests/stk_util/util/UnitTestThrowMacros.cpp @@ -260,7 +260,7 @@ void testNGPThrowRequireMsg() TEST(UnitTestingOfThrowMacros, NGP_ThrowRequireMsg) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(_OPENMP) || defined(KOKKOS_ENABLE_HIP) +#if defined(STK_ENABLE_GPU) || defined(_OPENMP) // Unable to test a device-side abort, as this eventually results in a throw // inside Kokkos::finalize(). // Also, OpenMP seems to produce an abort (in adddition to a throw?). @@ -303,7 +303,7 @@ void testNGPThrowRequire() TEST(UnitTestingOfThrowMacros, NGP_ThrowRequire) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(_OPENMP) || defined(KOKKOS_ENABLE_HIP) +#if defined(STK_ENABLE_GPU) || defined(_OPENMP) // Unable to test a device-side abort, as this eventually results in a throw // inside Kokkos::finalize(). // @@ -338,7 +338,7 @@ void testNGPThrowAssertMsg() TEST(UnitTestingOfThrowMacros, NGP_ThrowAssertMsg_debug) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(_OPENMP) || defined(KOKKOS_ENABLE_HIP) +#if defined(STK_ENABLE_GPU) || defined(_OPENMP) // Unable to test a device-side abort, as this eventually results in a throw // inside Kokkos::finalize(). // @@ -372,7 +372,7 @@ void testNGPThrowAssertMsg() TEST(UnitTestingOfThrowMacros, NGP_ThrowAssertMsg_release) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(_OPENMP) || defined(KOKKOS_ENABLE_HIP) +#if defined(STK_ENABLE_GPU) || defined(_OPENMP) // Unable to test a device-side abort, as this eventually results in a throw // inside Kokkos::finalize(). // @@ -395,7 +395,7 @@ void testNGPThrowErrorMsgIf() TEST(UnitTestingOfThrowMacros, NGP_ThrowErrorMsgIf) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(_OPENMP) || defined(KOKKOS_ENABLE_HIP) +#if defined(STK_ENABLE_GPU) || defined(_OPENMP) // Unable to test a device-side abort, as this eventually results in a throw // inside Kokkos::finalize(). // @@ -427,7 +427,7 @@ void testNGPThrowErrorIf() TEST(UnitTestingOfThrowMacros, NGP_ThrowErrorIf) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(_OPENMP) || defined(KOKKOS_ENABLE_HIP) +#if defined(STK_ENABLE_GPU) || defined(_OPENMP) // Unable to test a device-side abort, as this eventually results in a throw // inside Kokkos::finalize(). // @@ -456,7 +456,7 @@ void testNGPThrowErrorMsg() TEST(UnitTestingOfThrowMacros, NGP_ThrowErrorMsg) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(_OPENMP) || defined(KOKKOS_ENABLE_HIP) +#if defined(STK_ENABLE_GPU) || defined(_OPENMP) // Unable to test a device-side abort, as this eventually results in a throw // inside Kokkos::finalize(). // diff --git a/packages/stk/stk_util/Jamfile b/packages/stk/stk_util/Jamfile index 9f78b385cfcb..466684585b9c 100644 --- a/packages/stk/stk_util/Jamfile +++ b/packages/stk/stk_util/Jamfile @@ -210,30 +210,36 @@ lib stk_util_parallel # Run version script. This will generate stk_version.hpp, and -# return the version to this script. If it doesn't do so, fail. We don't -# want to generate a bad version for external customers. +# return the version to this script. If it doesn't do so, set the version +# string to 'unable-to-set-version'. local version ; if ( $(use_git_version) = "yes" ) || ( [ os.name ] = NT ) { if $(RTenv-arg) = "user" { - local v = [ SHELL "cat $(stk_util-root-inc)/stk_util/registry/stk_version.hpp" : exit-status ] ; - if $(v[2]) != 0 - { # Assume here that stk_version.hpp caused the error and just fail. - EXIT ; + local have_version_header = [ path.exists $(stk_util-root-inc)/stk_util/registry/stk_version.hpp ] ; + if $(have_version_header) { + local v = [ SHELL "cat $(stk_util-root-inc)/stk_util/registry/stk_version.hpp" : exit-status ] ; + if $(v[2]) != 0 { + version = "unable-to-set-version" ; + } else { + version = $(v[1]) ; } - version = $(v[1]) ; + } + else { + version = "unable-to-set-version" ; + } } else { local v = [ SHELL "$(stk_util-root-inc)/stk_util/registry/stk_version_gen.sh" : exit-status ] ; - if $(v[2]) != 0 - { # Assume here that stk_version_gen.sh reported the error and just fail. - EXIT ; - } - version = $(v[1]) ; + if $(v[2]) != 0 { + version = "unable-to-set-version" ; + } else { + version = $(v[1]) ; + } } } else { - local v = [ SHELL "echo \"// 4.59.7\" > $(stk_util-root-inc)/stk_util/registry/stk_version.hpp" : exit-status ] ; - version = 4.59.7 ; + local v = [ SHELL "echo \"// unable-to-set-version\" > $(stk_util-root-inc)/stk_util/registry/stk_version.hpp" : exit-status ] ; + version = unable-to-set-version ; } lib stk_util_environment @@ -360,5 +366,5 @@ alias stk_util_ngp : # No sources defined for header-only libraries. : # No build requirements : # No default build - : $(stk_util-root)/stk_util/ngp + : $(stk_util-root) ; diff --git a/packages/stk/stk_util/stk_util/diag/String.cpp b/packages/stk/stk_util/stk_util/diag/String.cpp index 96eefeb4ba98..82ecba8d1e39 100644 --- a/packages/stk/stk_util/stk_util/diag/String.cpp +++ b/packages/stk/stk_util/stk_util/diag/String.cpp @@ -44,7 +44,7 @@ namespace sierra { namespace { -static const char arraylower_t[] = { +static constexpr char arraylower_t[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, @@ -57,31 +57,39 @@ static const char arraylower_t[] = { 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 0x7B, 0x7C, 0x7D, 0x7E, 0x7F}; -inline int arraylower(unsigned char c) { +constexpr int arraylower(unsigned char c) noexcept { c = c > 127 ? 127 : c; return arraylower_t[c]; } -} // namespace -int to_label( int c ) +constexpr int to_label( int c ) { return isspace(c) ? '_' : c; } +} // namespace -int char_simple_traits::compare( const char * c1 , const char * c2 ) + +int char_simple_traits::compare( const char * c1 , const char * c2, size_t len ) noexcept { - if(c1 == nullptr) { - return (c2 == nullptr) ? 0 : -1; - } - if (c2 == nullptr) { - return 1; + if (len == 0) return 0; + size_t i = 0; + while (i < len - 1 && arraylower(c1[i]) == arraylower(c2[i])) + { + ++i; } + return arraylower(c1[i]) - arraylower(c2[i]); +} - for (; *c1 && arraylower(*c1) == arraylower(*c2); c1++, c2++) - ; - - return arraylower(*c1) - arraylower(*c2); +int char_label_traits::compare( const char * c1 , const char * c2, size_t len) noexcept +{ + if (len == 0) return 0; + size_t i = 0; + while (i < len - 1 && arraylower(to_label(c1[i])) == arraylower(to_label(c2[i]))) + { + ++i; + } + return arraylower(to_label(c1[i])) - arraylower(to_label(c2[i])); } void char_label_traits::convert( char * c , size_t n ) @@ -91,23 +99,6 @@ void char_label_traits::convert( char * c , size_t n ) *c = to_label(*c); } -int char_label_traits::compare( const char * c1 , const char * c2 ) -{ - if(c1 == nullptr) { - return (c2 == nullptr) ? 0 : -1; - } - - if (c2 == nullptr) { - return 1; - } - - for (; *c1 && arraylower(to_label(*c1)) == arraylower(to_label(*c2)); - c1++, c2++) - ; - - return arraylower(to_label(*c1)) - arraylower(to_label(*c2)); -} - } // namespace sierra //---------------------------------------------------------------------- diff --git a/packages/stk/stk_util/stk_util/diag/String.hpp b/packages/stk/stk_util/stk_util/diag/String.hpp index c9c729f990af..606a580fb484 100644 --- a/packages/stk/stk_util/stk_util/diag/String.hpp +++ b/packages/stk/stk_util/stk_util/diag/String.hpp @@ -36,6 +36,8 @@ #define STK_UTIL_DIAG_STRING_H #include +#include +#include #ifdef USE_CISTRING @@ -56,10 +58,9 @@ typedef cistring ParamId; namespace sierra { -/// -/// @addtogroup StringDetail -/// @{ -/// +template +using If_String_View = std::enable_if_t && + !std::is_convertible_v >; struct char_simple_traits ; @@ -119,79 +120,40 @@ bool operator>= ( const StringBase &, const StringBase & ); template -bool operator== ( const StringBase &, const std::string & ); - -template -bool operator!= ( const StringBase &, const std::string & ); - -template -bool operator< ( const StringBase &, const std::string & ); - -template -bool operator> ( const StringBase &, const std::string & ); - -template -bool operator<= ( const StringBase &, const std::string & ); - -template -bool operator>= ( const StringBase &, const std::string & ); - - -template -bool operator== ( const StringBase &, const char * ); - -template -bool operator!= ( const StringBase &, const char * ); - -template -bool operator< ( const StringBase &, const char * ); - -template -bool operator> ( const StringBase &, const char * ); - -template -bool operator<= ( const StringBase &, const char * ); +bool operator== ( const StringBase &, std::string_view ); template -bool operator>= ( const StringBase &, const char * ); - +bool operator!= ( const StringBase &, std::string_view ); template -bool operator== (const char *, const StringBase & ); +bool operator< ( const StringBase &, std::string_view ); template -bool operator!= (const char *, const StringBase & ); +bool operator> ( const StringBase &, std::string_view ); template -bool operator< (const char *, const StringBase & ); +bool operator<= ( const StringBase &, std::string_view ); template -bool operator> (const char *, const StringBase & ); +bool operator>= ( const StringBase &, std::string_view ); template -bool operator<= (const char *, const StringBase & ); +bool operator== (std::string_view, const StringBase &); template -bool operator>= (const char *, const StringBase & ); - +bool operator!= (std::string_view, const StringBase &); template -bool operator== (const std::string &, const StringBase &); +bool operator< (std::string_view, const StringBase &); template -bool operator!= (const std::string &, const StringBase &); +bool operator> (std::string_view, const StringBase &); template -bool operator< (const std::string &, const StringBase &); +bool operator<= (std::string_view, const StringBase &); template -bool operator> (const std::string &, const StringBase &); - -template -bool operator<= (const std::string &, const StringBase &); - -template -bool operator>= (const std::string &, const StringBase &); +bool operator>= (std::string_view, const StringBase &); @@ -209,8 +171,6 @@ operator>>( std::istream & is, sierra::Identifier &s ); } -//---------------------------------------------------------------------- - namespace sierra { template @@ -219,13 +179,10 @@ class StringBase { typedef const char * const_iterator; typedef char * iterator; - // Types: typedef CT traits_type ; typedef char value_type ; typedef size_t size_type ; - // Construct/copy/destroy: - StringBase() {} ~StringBase() = default; StringBase(StringBase &&) = default; @@ -233,38 +190,65 @@ class StringBase { StringBase &operator=(const StringBase &) = default; StringBase &operator=(StringBase &&) = default; - StringBase(const std::string &); - StringBase(std::string &&); + //Allow construction from things implicitly convertable to std::string_view + template > + explicit StringBase(const StringViewLike &sv) : data_(std::string_view(sv)) + { + traits_type::convert(begin(), length()); + } + + //Now we allow implicit conversion from std::string to StrinBase anywhere in code + //This can silently create copy and potential heap allocations + //This code path should be deprecated!!! + StringBase(const std::string & s) : data_(s) + { + traits_type::convert(begin(), length()); + } + + StringBase(std::string && cs) : data_(std::move(cs)) + { + traits_type::convert(begin(), length()); + } StringBase( const_iterator ); template StringBase( It, It ); StringBase( const char *, size_type ); + operator std::string_view() const noexcept {return data_;} + template StringBase( const StringBase & ); + template StringBase(StringBase &&); StringBase & operator= ( const char * ); - StringBase & operator= ( const std::string & ); + + template > + StringBase & operator= ( const StringViewLike & cs) + { + return assign( cs.data(), cs.length() ); + } StringBase &operator=(std::string &&); - template - StringBase & operator= ( const StringBase & ); template StringBase &operator=(StringBase &&); StringBase & operator+= ( const char * ); - StringBase & operator+= ( const std::string & ); - template - StringBase & operator+= ( const StringBase & ); + template > + StringBase & operator+= ( const StringViewLike & cs) + { + return append( cs.data(), cs.length() ); + } // Capacity: - size_type size() const; - size_type length() const; - bool empty() const ; + constexpr size_type size() const noexcept { return data_.size();} + constexpr size_type length() const noexcept { return data_.length();} + constexpr bool empty() const noexcept { return data_.empty();} + char * data() noexcept { return data_.data();} + const char * data() const noexcept { return data_.data();} const_iterator begin() const; iterator begin(); @@ -272,23 +256,54 @@ class StringBase { iterator end(); // Modifiers: + StringBase & assign( const char * cs) + { + if (cs == nullptr) { + data_ = std::string(); + return *this; + } + return assign(cs, strlen(cs)); + } - StringBase & assign( const char * ); - StringBase & assign( const char *, const size_type ); - StringBase &assign(const std::string &); - StringBase &assign(std::string &&); + StringBase & assign( const char * cs, const size_type n) + { + //cs must be non-null to prevent undefined behavior + data_.assign(cs, n); + traits_type::convert(begin(), length()); + return *this ; + + } + + template > + StringBase &assign(const StringViewLike & cs) + { + data_ = std::string_view(cs); + traits_type::convert(begin(), length()); + return *this; + } + StringBase &assign(std::string && cs) + { + data_ = std::move(cs); + traits_type::convert(begin(), length()); + return *this; + } - template - StringBase & assign( const StringBase & ); template - StringBase &assign(StringBase &&); + StringBase &assign(StringBase && cs) + { + data_ = std::move(cs).s_str(); + traits_type::convert(begin(), length()); + return *this; + } StringBase & append( const char * ); StringBase & append( const char *, const typename StringBase::size_type ); - StringBase & append( const std::string& ); - template - StringBase & append( const StringBase & ); + template > + StringBase & append( const StringViewLike & cs) + { + return append( cs.data(), cs.length() ); + } void swap( StringBase & ); @@ -297,14 +312,37 @@ class StringBase { const std::string &s_str() const &; std::string &&s_str() &&; - int compare( const char * ) const ; - int compare( const std::string & ) const ; + int compare(const char *cs) const + { + const auto lhs_size = length(); + if (cs == nullptr) + { + if(lhs_size) return -1; + return 0; + } + const auto rhs_size = strlen(cs); + const auto result = CT::compare(c_str(), cs, std::min(lhs_size, rhs_size)); + if (result != 0) return result; + if (lhs_size < rhs_size) return -1; + if (lhs_size > rhs_size) return 1; + + return 0; + } - template - int compare( const StringBase & ) const ; + int compare( std::string_view str ) const + { + const auto lhs_size = length(); + const auto rhs_size = str.length(); + const auto result = CT::compare(c_str(), str.data(), std::min(lhs_size, rhs_size)); + if (result != 0) return result; + if (lhs_size < rhs_size) return -1; + if (lhs_size > rhs_size) return 1; + + return 0; + } private: - std::string data; + std::string data_; }; /** @class char_simple_traits @@ -317,7 +355,8 @@ struct char_simple_traits { {} /** Compare null-terminated strings */ - static int compare( const char * c1, const char * c2 ); + // len is min of two string lengths + static int compare( const char * c1, const char * c2, size_t len ) noexcept; }; /** @class char_label_traits @@ -330,38 +369,20 @@ struct char_label_traits { static void convert( char * c, size_t n ); /** Compare null-terminated strings as per conversion */ - static int compare( const char * c1, const char * c2 ); + // len is min of two string lengths + static int compare( const char * c1, const char * c2, size_t len ) noexcept; }; -//---------------------------------------------------------------------- - -template -bool StringBase::empty() const -{ - return data.empty(); -} - -template -typename StringBase::size_type StringBase::length() const -{ - return data.size(); -} - -template -typename StringBase::size_type StringBase::size() const -{ - return data.size(); -} template typename StringBase::iterator StringBase::begin() -{ return data.data(); } +{ return data_.data(); } template typename StringBase::const_iterator StringBase::begin() const -{ return data.c_str(); } +{ return data_.c_str(); } template typename StringBase::iterator @@ -379,57 +400,45 @@ StringBase::end() const template const char* StringBase::c_str() const -{ return data.c_str(); } +{ return data_.c_str(); } template const std::string &StringBase::s_str() const & { - return data; + return data_; } template std::string &&StringBase::s_str() && { - return std::move(data); + return std::move(data_); } //---------------------------------------------------------------------- template template -StringBase::StringBase(const StringBase &cs) : data(cs.s_str()) +StringBase::StringBase(const StringBase &cs) : data_(cs.s_str()) { traits_type::convert(begin(), length()); } template template -StringBase::StringBase(StringBase &&cs) : data(std::move(cs).s_str()) -{ - traits_type::convert(begin(), length()); -} - -template -StringBase::StringBase(const std::string &cs) : data(cs) +StringBase::StringBase(StringBase &&cs) : data_(std::move(cs).s_str()) { traits_type::convert(begin(), length()); } template -StringBase::StringBase(std::string &&cs) : data(std::move(cs)) -{ - traits_type::convert(begin(), length()); -} - -template -StringBase::StringBase(const char *cs, typename StringBase::size_type n) : data(cs, n) +StringBase::StringBase(const char *cs, typename StringBase::size_type n) : data_(cs, n) { traits_type::convert(begin(), length()); } template template -StringBase::StringBase(It l_begin, It l_end) : data(l_begin, l_end) +StringBase::StringBase(It l_begin, It l_end) : data_(l_begin, l_end) { traits_type::convert(begin(), length()); } @@ -438,67 +447,11 @@ template StringBase::StringBase(const char *cs) { if (cs != nullptr) { - data = std::string(cs); + data_ = std::string(cs); traits_type::convert(begin(), length()); } } -//---------------------------------------------------------------------- - -template -StringBase & -StringBase::assign( const char * cs, const typename StringBase::size_type n ) -{ - data.assign(cs, n); - traits_type::convert(begin(), length()); - return *this ; -} - -template -StringBase & StringBase::assign( const char * cs ) { - if ( cs == nullptr ) { - return assign( nullptr, 0 ); - } - return assign( cs, strlen(cs) ); -} - -template -StringBase & StringBase::assign( const std::string & cs ) -{ - data = cs; - traits_type::convert(begin(), length()); - return *this; -} - -template -StringBase &StringBase::assign(std::string &&cs) -{ - data = std::move(cs); - traits_type::convert(begin(), length()); - return *this; -} - -template -template -StringBase & StringBase::assign( const StringBase & cs ) -{ return assign( cs.c_str(), cs.length() ); } - -template -template -StringBase &StringBase::assign(StringBase &&cs) -{ - data = std::move(cs).s_str(); - traits_type::convert(begin(), length()); - return *this; -} - -template -template -StringBase& -StringBase::operator= ( const StringBase & cs ) { - return assign(cs); -} - template template StringBase &StringBase::operator=(StringBase &&cs) @@ -510,32 +463,27 @@ template StringBase& StringBase::operator= ( const char * cs ) { if (cs == nullptr) { - return assign( nullptr, 0 ); + data_ = std::string(); + return *this; } return assign( cs, strlen(cs) ); } -template -StringBase& -StringBase::operator= ( const std::string& cs ) -{ return assign( cs.c_str(), cs.length() ); } - template StringBase &StringBase::operator=(std::string &&cs) { - data = std::move(cs); + data_ = std::move(cs); traits_type::convert(begin(), length()); return *this; } -//---------------------------------------------------------------------- template StringBase & StringBase::append( const char * cs, const typename StringBase::size_type n ) { auto orig_len = length(); - data.append(cs, n); + data_.append(cs, n); traits_type::convert(begin() + orig_len, length() - orig_len); return *this ; } @@ -546,9 +494,9 @@ inline StringBase &StringBase::append( { if (n > 0) { auto orig_len = length(); - data.reserve(orig_len + n + 1); - if (orig_len != 0) data.append("_"); - data.append(cs, n); + data_.reserve(orig_len + n + 1); + if (orig_len != 0) data_.append("_"); + data_.append(cs, n); traits_type::convert(begin() + orig_len, length() - orig_len); } return *this; @@ -558,55 +506,12 @@ template StringBase & StringBase::append( const char * cs ) { return ( cs == nullptr ) ? *this : append( cs, strlen(cs) ); } - -template -StringBase & StringBase::append( const std::string & cs ) -{ return append( cs.data(), cs.length() ); } - -template -template -StringBase & StringBase::append( const StringBase & cs ) -{ return append( cs.c_str(), cs.length() ); } - - -template -template -StringBase& -StringBase::operator+= ( const StringBase & cs ) -{ return append( cs.c_str(), cs.length() ); } - template StringBase& StringBase::operator+= ( const char * cs ) { return (cs == nullptr) ? *this : append( cs, strlen(cs) ); } -template -StringBase& -StringBase::operator+= ( const std::string& cs ) -{ return append( cs.data(), cs.length() ); } - -//---------------------------------------------------------------------- - -template -template -int StringBase::compare( const StringBase & cs ) const -{ - typedef typename Precedence::Type Traits ; - return Traits::compare( c_str(), cs.c_str() ); -} - -template -int StringBase::compare( const std::string & cs ) const -{ - return CT::compare( c_str(), cs.c_str() ); -} - -template -int StringBase::compare( const char * cs ) const -{ - return CT::compare( c_str(), cs ); -} template bool operator== ( const StringBase & lhs, @@ -641,129 +546,64 @@ bool operator>= ( const StringBase & lhs, template bool operator== ( const StringBase & lhs, - const std::string & rhs ) -{ return lhs.compare(rhs) == 0 ; } - -template -bool operator!= ( const StringBase & lhs, - const std::string & rhs ) -{ return lhs.compare(rhs) != 0 ; } - -template -bool operator< ( const StringBase & lhs, - const std::string & rhs ) -{ return lhs.compare(rhs) < 0 ; } - -template -bool operator<= ( const StringBase & lhs, - const std::string & rhs ) -{ return lhs.compare(rhs) <= 0 ; } - -template -bool operator> ( const StringBase & lhs, - const std::string & rhs ) -{ return lhs.compare(rhs) > 0 ; } - -template -bool operator>= ( const StringBase & lhs, - const std::string & rhs ) -{ return lhs.compare(rhs) >= 0 ; } - - -template -bool operator== ( const StringBase & lhs, - const char * rhs ) + std::string_view rhs ) { return lhs.compare(rhs) == 0 ; } template bool operator!= ( const StringBase & lhs, - const char * rhs ) + std::string_view rhs ) { return lhs.compare(rhs) != 0 ; } template bool operator< ( const StringBase & lhs, - const char * rhs ) + std::string_view rhs ) { return lhs.compare(rhs) < 0 ; } template bool operator<= ( const StringBase & lhs, - const char * rhs ) + std::string_view rhs ) { return lhs.compare(rhs) <= 0 ; } template bool operator> ( const StringBase & lhs, - const char * rhs ) + std::string_view rhs ) { return lhs.compare(rhs) > 0 ; } template bool operator>= ( const StringBase & lhs, - const char * rhs ) + std::string_view rhs ) { return lhs.compare(rhs) >= 0 ; } - template -bool operator== ( const std::string & lhs, +bool operator== ( std::string_view lhs, const StringBase & rhs) { return rhs.compare(lhs) == 0 ; } template -bool operator!= ( const std::string & lhs, +bool operator!= ( std::string_view lhs, const StringBase & rhs) { return rhs.compare(lhs) != 0 ; } template -bool operator< ( const std::string & lhs, +bool operator< ( std::string_view lhs, const StringBase & rhs) { return rhs.compare(lhs) > 0 ; } template -bool operator<= ( const std::string & lhs, +bool operator<= ( std::string_view lhs, const StringBase & rhs) { return rhs.compare(lhs) >= 0 ; } template -bool operator> ( const std::string & lhs, +bool operator> ( std::string_view lhs, const StringBase & rhs) { return rhs.compare(lhs) < 0 ; } template -bool operator>= ( const std::string & lhs, +bool operator>= ( std::string_view lhs, const StringBase & rhs) { return rhs.compare(lhs) <= 0 ; } - -template -bool operator== ( const char * lhs, - const StringBase & rhs) -{ return rhs.compare(lhs) == 0 ; } - -template -bool operator!= ( const char * lhs, - const StringBase & rhs) -{ return rhs.compare(lhs) != 0 ; } - -template -bool operator< ( const char * lhs, - const StringBase & rhs) -{ return rhs.compare(lhs) > 0 ; } - -template -bool operator<= ( const char * lhs, - const StringBase & rhs) -{ return rhs.compare(lhs) >= 0 ; } - -template -bool operator> ( const char * lhs, - const StringBase & rhs) -{ return rhs.compare(lhs) < 0 ; } - -template -bool operator>= ( const char * lhs, - const StringBase & rhs) -{ return rhs.compare(lhs) <= 0 ; } - -//---------------------------------------------------------------------- - template StringBase operator+( const StringBase &cs1, const StringBase &cs2) { @@ -774,31 +614,15 @@ operator+( const StringBase &cs1, const StringBase &cs2) { template StringBase -operator+( const StringBase &cs1, const char *cs2) { +operator+( const StringBase &cs1, std::string_view cs2) { StringBase t(cs1); - t.append(cs2); - return t; -} - -template -StringBase -operator+( const StringBase &cs1, const std::string &cs2) { - StringBase t(cs1); - t.append(cs2.c_str(), cs2.length()); - return t; -} - -template -StringBase -operator+ ( const char *cs1, const StringBase &cs2 ) { - StringBase t(cs1); - t.append(cs2.c_str(), cs2.length()); + t.append(cs2.data(), cs2.length()); return t; } template StringBase -operator+(const std::string & cs1, const StringBase & cs2 ) { +operator+(std::string_view cs1, const StringBase & cs2 ) { StringBase t(cs1); t.append(cs2.c_str(), cs2.length()); return t; diff --git a/packages/stk/stk_util/stk_util/parallel/CommBuffer.hpp b/packages/stk/stk_util/stk_util/parallel/CommBuffer.hpp index 9b27f3d43240..e4a8dc80be4b 100644 --- a/packages/stk/stk_util/stk_util/parallel/CommBuffer.hpp +++ b/packages/stk/stk_util/stk_util/parallel/CommBuffer.hpp @@ -134,7 +134,6 @@ class CommBuffer { /** Pointer to base of buffer. */ void * buffer() const ; - ~CommBuffer() {} CommBuffer() : m_beg(nullptr), m_ptr(nullptr), m_end(nullptr) { } void set_buffer_ptrs(unsigned char* begin, unsigned char* ptr, unsigned char* end); @@ -388,7 +387,7 @@ CommBuffer &CommBuffer::peek( std::string& value ) std::vector chars(offset+length); peek(chars.data(), chars.size()); - value.assign(&chars[offset], length); + value.assign(chars.data() + offset, length); return *this; } diff --git a/packages/stk/stk_util/stk_util/parallel/CommNeighbors.cpp b/packages/stk/stk_util/stk_util/parallel/CommNeighbors.cpp index c174f5320da7..441592d20b2e 100644 --- a/packages/stk/stk_util/stk_util/parallel/CommNeighbors.cpp +++ b/packages/stk/stk_util/stk_util/parallel/CommNeighbors.cpp @@ -177,11 +177,13 @@ void store_recvd_data(const std::vector& recvBuf, std::vector& recvBuffers) { for(size_t i=0; i 0) { + std::memcpy(recvBuffers[p].raw_buffer(), buf, len); + } } } @@ -259,7 +261,7 @@ void old_communicate(MPI_Comm comm, int numRecvProcs = 0; for(int i=0; i 0) { recv_buffers[p].resize(recv_sizes[idx]); @@ -281,10 +283,12 @@ void old_communicate(MPI_Comm comm, void CommNeighbors::communicate() { if (m_size == 1) { - int len = m_send[0].size_in_bytes(); + const int len = m_send[0].size_in_bytes(); const unsigned char* buf = m_send[0].raw_buffer(); m_recv[0].resize(len); - std::memcpy(m_recv[0].raw_buffer(), buf, len); + if (len > 0) { + std::memcpy(m_recv[0].raw_buffer(), buf, len); + } return; } diff --git a/packages/stk/stk_util/stk_util/parallel/CommSparse.hpp b/packages/stk/stk_util/stk_util/parallel/CommSparse.hpp index 740c75da05eb..463505c96291 100644 --- a/packages/stk/stk_util/stk_util/parallel/CommSparse.hpp +++ b/packages/stk/stk_util/stk_util/parallel/CommSparse.hpp @@ -271,7 +271,7 @@ class CommSparse { }; template -bool pack_and_communicate(stk::CommSparse & comm, const PACK_ALGORITHM & algorithm) +bool pack_and_communicate(stk::CommSparse & comm, const PACK_ALGORITHM & algorithm, bool deallocateSendBuffers = true) { stk::util::print_unsupported_version_warning(5, __LINE__, __FILE__); @@ -279,13 +279,13 @@ bool pack_and_communicate(stk::CommSparse & comm, const PACK_ALGORITHM & algorit algorithm(); comm.allocate_buffers(); algorithm(); - return comm.communicate(); + return comm.communicate(deallocateSendBuffers); } else { algorithm(); const bool actuallySendingOrReceiving = comm.allocate_buffers(); if (actuallySendingOrReceiving) { algorithm(); - comm.communicate(); + comm.communicate(deallocateSendBuffers); } return actuallySendingOrReceiving; } diff --git a/packages/stk/stk_util/stk_util/parallel/CouplingVersions.cpp b/packages/stk/stk_util/stk_util/parallel/CouplingVersions.cpp index bdac832426fd..827664297419 100644 --- a/packages/stk/stk_util/stk_util/parallel/CouplingVersions.cpp +++ b/packages/stk/stk_util/stk_util/parallel/CouplingVersions.cpp @@ -210,7 +210,9 @@ std::string get_deprecation_date(int version) std::make_pair(9, "1/31/2023"), std::make_pair(10, "4/12/2023"), std::make_pair(11, "4/19/2023"), - std::make_pair(12, "") + std::make_pair(12, "3/11/2024"), + std::make_pair(13, "3/28/2024"), + std::make_pair(14, "") }; return deprecationDates.at(version); diff --git a/packages/stk/stk_util/stk_util/parallel/CouplingVersions.hpp b/packages/stk/stk_util/stk_util/parallel/CouplingVersions.hpp index aa394a74677a..5f731c6f1d9d 100644 --- a/packages/stk/stk_util/stk_util/parallel/CouplingVersions.hpp +++ b/packages/stk/stk_util/stk_util/parallel/CouplingVersions.hpp @@ -8,7 +8,7 @@ #include -#define STK_MAX_COUPLING_VERSION 12 +#define STK_MAX_COUPLING_VERSION 14 #define STK_MIN_COUPLING_VERSION 0 namespace stk { diff --git a/packages/stk/stk_util/stk_util/parallel/DataExchangeKnownPatternNonBlocking.cpp b/packages/stk/stk_util/stk_util/parallel/DataExchangeKnownPatternNonBlocking.cpp index 16efdf93392a..9c2779576897 100644 --- a/packages/stk/stk_util/stk_util/parallel/DataExchangeKnownPatternNonBlocking.cpp +++ b/packages/stk/stk_util/stk_util/parallel/DataExchangeKnownPatternNonBlocking.cpp @@ -4,7 +4,6 @@ namespace stk { DataExchangeKnownPatternNonBlocking::DataExchangeKnownPatternNonBlocking(MPI_Comm comm, int tag_hint) : m_comm(comm), - m_tagHint(tag_hint), m_tag(stk::get_mpi_tag_manager().get_tag(comm, tag_hint)) { m_sendReqs.reserve(stk::parallel_machine_size(comm)); @@ -27,4 +26,4 @@ void DataExchangeKnownPatternNonBlocking::complete_sends() } -} \ No newline at end of file +} diff --git a/packages/stk/stk_util/stk_util/parallel/DataExchangeKnownPatternNonBlocking.hpp b/packages/stk/stk_util/stk_util/parallel/DataExchangeKnownPatternNonBlocking.hpp index bb7cfb525621..238affec520b 100644 --- a/packages/stk/stk_util/stk_util/parallel/DataExchangeKnownPatternNonBlocking.hpp +++ b/packages/stk/stk_util/stk_util/parallel/DataExchangeKnownPatternNonBlocking.hpp @@ -82,7 +82,6 @@ class DataExchangeKnownPatternNonBlocking void start_sends(std::vector< std::vector > &sendLists); MPI_Comm m_comm; - int m_tagHint; MPITag m_tag; std::vector m_recvRankMap; @@ -174,4 +173,4 @@ void DataExchangeKnownPatternNonBlocking::complete_receives(std::vector< std::ve } // namespace #endif -#endif \ No newline at end of file +#endif diff --git a/packages/stk/stk_util/stk_util/parallel/DataExchangeUnknownPatternNonBlocking.hpp b/packages/stk/stk_util/stk_util/parallel/DataExchangeUnknownPatternNonBlocking.hpp index c1c77832b464..6dba78fa53a2 100644 --- a/packages/stk/stk_util/stk_util/parallel/DataExchangeUnknownPatternNonBlocking.hpp +++ b/packages/stk/stk_util/stk_util/parallel/DataExchangeUnknownPatternNonBlocking.hpp @@ -146,7 +146,6 @@ class DataExchangeUnknownPatternNonBlocking void yield(); MPI_Comm m_comm; - int m_myrank = -1; int m_tagHint; MPITag m_tag; diff --git a/packages/stk/stk_util/stk_util/parallel/DeletionGroup.cpp b/packages/stk/stk_util/stk_util/parallel/DeletionGroup.cpp index ce18b2814de6..e011359c64a9 100644 --- a/packages/stk/stk_util/stk_util/parallel/DeletionGroup.cpp +++ b/packages/stk/stk_util/stk_util/parallel/DeletionGroup.cpp @@ -4,16 +4,15 @@ namespace stk { namespace impl { - -DeletionGroup::DeletionGroup(MPI_Comm comm, int barrier_tag) : - m_comm(comm), - m_req(MPI_REQUEST_NULL), - m_barrierSemanticallyInProgress(false), - m_barrierActuallyInProgress(false), - m_entryCount(0), - m_ibarrier(comm, barrier_tag) +DeletionGroup::DeletionGroup(MPI_Comm comm, int barrier_tag) + : m_comm(comm), + m_barrierSemanticallyInProgress(false), + m_barrierActuallyInProgress(false), + m_entryCount(0), + m_ibarrier(comm, barrier_tag) #ifndef NDEBUG - , m_req_debug(MPI_REQUEST_NULL) + , + m_req_debug(MPI_REQUEST_NULL) #endif { #ifndef NDEBUG diff --git a/packages/stk/stk_util/stk_util/parallel/DeletionGroup.hpp b/packages/stk/stk_util/stk_util/parallel/DeletionGroup.hpp index b030100fa8f8..e753b297179d 100644 --- a/packages/stk/stk_util/stk_util/parallel/DeletionGroup.hpp +++ b/packages/stk/stk_util/stk_util/parallel/DeletionGroup.hpp @@ -41,13 +41,12 @@ class DeletionGroup private: - MPI_Comm m_comm; - MPI_Request m_req; - bool m_barrierSemanticallyInProgress; - bool m_barrierActuallyInProgress; - EntryCountInt m_entryCount; - std::vector m_tags; - IbarrierReplacement m_ibarrier; + MPI_Comm m_comm; + bool m_barrierSemanticallyInProgress; + bool m_barrierActuallyInProgress; + EntryCountInt m_entryCount; + std::vector m_tags; + IbarrierReplacement m_ibarrier; #ifndef NDEBUG MPI_Request m_req_debug; @@ -58,4 +57,4 @@ class DeletionGroup } } -#endif \ No newline at end of file +#endif diff --git a/packages/stk/stk_util/stk_util/parallel/MPICommKey.hpp b/packages/stk/stk_util/stk_util/parallel/MPICommKey.hpp index d13a7534df9a..01b3e2080275 100644 --- a/packages/stk/stk_util/stk_util/parallel/MPICommKey.hpp +++ b/packages/stk/stk_util/stk_util/parallel/MPICommKey.hpp @@ -139,7 +139,6 @@ class MPIKeyManager void destructor(); int m_mpiAttrKey; - int m_destructorAttrKey; CommKey m_currentCommKey = 0; CallerUID m_currUID = 0; std::set m_usedCommKeys; diff --git a/packages/stk/stk_util/stk_util/parallel/Parallel.hpp b/packages/stk/stk_util/stk_util/parallel/Parallel.hpp index 8cf107954a1e..ab05a281120d 100644 --- a/packages/stk/stk_util/stk_util/parallel/Parallel.hpp +++ b/packages/stk/stk_util/stk_util/parallel/Parallel.hpp @@ -35,8 +35,8 @@ #ifndef stk_util_parallel_Parallel_hpp #define stk_util_parallel_Parallel_hpp -// stk_config.h resides in the build directory and contains the -// complete set of #define macros for build-dependent features. +// stk_config.h contains the #define macros that define +// which build-dependent features are enabled, such as 'STK_HAS_MPI'. #include "stk_util/stk_config.h" @@ -96,11 +96,11 @@ inline void parallel_machine_finalize() // Other parallel communication machines go here // as '#elif defined( STK_HAS_ )' -//---------------------------------------- -// Stub for non-parallel - #else +//---------------------------------------- +// Stub for non-parallel, no MPI + // Some needed stubs #define MPI_Comm int #define MPI_COMM_WORLD 0 diff --git a/packages/stk/stk_util/stk_util/parallel/ParallelComm.hpp b/packages/stk/stk_util/stk_util/parallel/ParallelComm.hpp index cd86cb757007..f94b23e02b0a 100644 --- a/packages/stk/stk_util/stk_util/parallel/ParallelComm.hpp +++ b/packages/stk/stk_util/stk_util/parallel/ParallelComm.hpp @@ -37,6 +37,7 @@ #include "stk_util/stk_config.h" // for STK_HAS_MPI #include "stk_util/parallel/Parallel.hpp" // for MPI_Irecv, MPI_Wait, MPI_Barrier, MPI_Send +#include "stk_util/parallel/ParallelReduce.hpp" #include "stk_util/parallel/CouplingVersions.hpp" #include "stk_util/parallel/MPITagManager.hpp" // for MPITagManager #include "stk_util/parallel/DataExchangeUnknownPatternBlocking.hpp" // for DataExchangeUnknownPatternBlocking @@ -48,7 +49,7 @@ #include // for runtime_error #include // for string #include // for vector - +#include namespace stk { @@ -219,24 +220,50 @@ void parallel_data_exchange_sym_t(std::vector< std::vector > &send_lists, #endif } +// +//The checkInput flag must have the same value on every processor, or this function will hang. +// template inline void parallel_data_exchange_nonsym_known_sizes_t(const int* sendOffsets, T* sendData, const int* recvOffsets, T* recvData, - MPI_Comm mpi_communicator ) + MPI_Comm mpi_communicator, + bool checkInput = false) { #if defined( STK_HAS_MPI) const auto msg_tag = get_mpi_tag_manager().get_tag(mpi_communicator, 10243); //arbitrary tag value, anything less than 32768 is legal const int num_procs = stk::parallel_machine_size(mpi_communicator); const int bytesPerScalar = sizeof(T); + if (checkInput && stk::util::get_common_coupling_version() > 11) { + std::vector reducedSendOffsets(num_procs+1); + const int result = MPI_Allreduce(sendOffsets, reducedSendOffsets.data(), num_procs+1, MPI_INT, MPI_SUM, mpi_communicator); + int myProc = -1; + MPI_Comm_rank(mpi_communicator, &myProc); + const int totalNumRecvs = reducedSendOffsets[myProc+1] - reducedSendOffsets[myProc]; + const int expectedNumRecvs = recvOffsets[num_procs]; + STK_ThrowRequireMsg(result == MPI_SUCCESS, "parallel_data_exchange_nonsym_known_sizes_t MPI_Allreduce return-code "< recv_handles(num_procs); - std::vector send_handles(num_procs); + std::vector send_handles; + if (stk::util::get_common_coupling_version() <= 13) { + send_handles.resize(num_procs); + } int numRecvs = 0; for(int iproc = 0; iproc < num_procs; ++iproc) { const int recvSize = recvOffsets[iproc+1]-recvOffsets[iproc]; @@ -247,7 +274,7 @@ void parallel_data_exchange_nonsym_known_sizes_t(const int* sendOffsets, } } - if (stk::util::get_common_coupling_version() <= 11) { + if (stk::util::get_common_coupling_version() <= 11 || stk::util::get_common_coupling_version() >= 13) { MPI_Barrier(mpi_communicator); } @@ -257,11 +284,18 @@ void parallel_data_exchange_nonsym_known_sizes_t(const int* sendOffsets, if(sendSize > 0) { char* sendBuffer = (char*)(&sendData[sendOffsets[iproc]]); const int sendSizeBytes = sendSize*bytesPerScalar; - MPI_Isend(sendBuffer, sendSizeBytes, MPI_CHAR, iproc, msg_tag, mpi_communicator, &send_handles[numSends++]); + if (stk::util::get_common_coupling_version() <= 13) { + MPI_Isend(sendBuffer, sendSizeBytes, MPI_CHAR, iproc, msg_tag, mpi_communicator, &send_handles[numSends++]); + } + else { + MPI_Send(sendBuffer, sendSizeBytes, MPI_CHAR, iproc, msg_tag, mpi_communicator); + } } } - MPI_Waitall(numSends, send_handles.data(), MPI_STATUSES_IGNORE); + if (stk::util::get_common_coupling_version() <= 13) { + MPI_Waitall(numSends, send_handles.data(), MPI_STATUSES_IGNORE); + } MPI_Waitall(numRecvs, recv_handles.data(), MPI_STATUSES_IGNORE); #endif } diff --git a/packages/stk/stk_util/stk_util/parallel/ReceiveCounter.cpp b/packages/stk/stk_util/stk_util/parallel/ReceiveCounter.cpp index b38b1d44a97b..0a1ae54202f2 100644 --- a/packages/stk/stk_util/stk_util/parallel/ReceiveCounter.cpp +++ b/packages/stk/stk_util/stk_util/parallel/ReceiveCounter.cpp @@ -1,5 +1,6 @@ #include "stk_util/parallel/ReceiveCounter.hpp" #include "stk_util/util/ReportHandler.hpp" // for ThrowRequireMsg +#include // for assert #include //TODO: DEBUGGING @@ -46,4 +47,4 @@ int ReceiveCounter::get_receive_count() return m_nrecv; } -} \ No newline at end of file +} diff --git a/packages/stk/stk_util/stk_util/registry/ProductRegistry.cpp b/packages/stk/stk_util/stk_util/registry/ProductRegistry.cpp index 573f50ab5f7c..dd40dcde7175 100644 --- a/packages/stk/stk_util/stk_util/registry/ProductRegistry.cpp +++ b/packages/stk/stk_util/stk_util/registry/ProductRegistry.cpp @@ -42,7 +42,7 @@ //In Sierra, STK_VERSION_STRING is provided on the compile line by bake. //For Trilinos stk snapshots, the following macro definition gets populated with //the real version string by the trilinos_snapshot.sh script. -#define STK_VERSION_STRING "5.17.6-138-g510fc684-modified" +#define STK_VERSION_STRING "5.19.2-716-g2016503a" #endif namespace stk { diff --git a/packages/stk/stk_util/stk_util/stk_config.h b/packages/stk/stk_util/stk_util/stk_config.h index 0ae153f55814..a49674f41a0f 100644 --- a/packages/stk/stk_util/stk_util/stk_config.h +++ b/packages/stk/stk_util/stk_util/stk_config.h @@ -38,11 +38,15 @@ #ifdef STK_BUILT_IN_SIERRA #define STK_HAS_MPI +#define STK_HAS_ARBORX #define STK_HAVE_BOOST #define STK_HAVE_KOKKOS #define STK_HAVE_STKMESH #define STK_HAVE_STKIO #define STK_HAVE_STKNGP_TEST +#define STK_HAS_SEACAS_IOSS +#define STK_HAS_SEACAS_EXODUS +#define STK_HAS_SEACAS_NEMESIS #else // This file gets created by cmake during a Trilinos build diff --git a/packages/stk/stk_util/stk_util/util/FArray.hpp b/packages/stk/stk_util/stk_util/util/FArray.hpp index 3447a81b963d..e2779557629b 100644 --- a/packages/stk/stk_util/stk_util/util/FArray.hpp +++ b/packages/stk/stk_util/stk_util/util/FArray.hpp @@ -445,7 +445,7 @@ class FArray : public FArrayBootstrap for (unsigned i = 0; i < NumDim; ++i) array_dimension_verify(i, index[i], m_dim[i]); - value_type *l_ptr = ArrayHelper::index(m_ptr, m_stride, m_dim, &index[0]); + value_type *l_ptr = ArrayHelper::index(m_ptr, m_stride, m_dim, index.data()); return *l_ptr; } @@ -585,7 +585,7 @@ class FArray : public FArrayBootstrap for (unsigned i = 0; i < NumDim; ++i) array_dimension_verify(i, index[i], m_dim[i]); - const value_type *l_ptr = ArrayHelper::index(m_ptr, m_stride, m_dim, &index[0]); + const value_type *l_ptr = ArrayHelper::index(m_ptr, m_stride, m_dim, index.data()); return *l_ptr; } @@ -1149,7 +1149,7 @@ class FArray : public FArrayBootstrap } void dimensions(const_iterator it, Index &index) const { - ArrayHelper::get_index(m_stride, &index[0], it - m_ptr); + ArrayHelper::get_index(m_stride, index.data(), it - m_ptr); } //---------------------------------------- diff --git a/packages/stk/stk_util/stk_util/util/Marshal.cpp b/packages/stk/stk_util/stk_util/util/Marshal.cpp index dd4cd557f5ce..36c8e1b94bb6 100644 --- a/packages/stk/stk_util/stk_util/util/Marshal.cpp +++ b/packages/stk/stk_util/stk_util/util/Marshal.cpp @@ -450,10 +450,10 @@ Marshal &operator>>(Marshal &min, std::string &s) { size_t size = 0; min >> size; std::vector c(size); - - min.stream.read(&c[0], size); - s.assign(&c[0], size); - + + min.stream.read(c.data(), size); + s.assign(c.data(), size); + return min; } diff --git a/packages/stk/stk_util/stk_util/util/ReportHandler.cpp b/packages/stk/stk_util/stk_util/util/ReportHandler.cpp index c83e423b375b..571d78d2301a 100644 --- a/packages/stk/stk_util/stk_util/util/ReportHandler.cpp +++ b/packages/stk/stk_util/stk_util/util/ReportHandler.cpp @@ -33,6 +33,7 @@ // #include "stk_util/util/ReportHandler.hpp" +#include "Kokkos_Core.hpp" #include // for cout #include // for ostringstream, operator<<, basic_ostream, basic_ostream::operator<< #include // for runtime_error, logic_error, invalid_argument @@ -265,3 +266,13 @@ std::ostream & output_stacktrace(std::ostream & os) #endif } // namespace stk + +STK_FUNCTION void ThrowMsgDevice(const char * message) +{ + Kokkos::abort(message); +} + +STK_FUNCTION void ThrowErrorMsgDevice(const char * message) +{ + Kokkos::abort(message); +} diff --git a/packages/stk/stk_util/stk_util/util/ReportHandler.hpp b/packages/stk/stk_util/stk_util/util/ReportHandler.hpp index 7b01e307bcb9..1e6e2f391596 100644 --- a/packages/stk/stk_util/stk_util/util/ReportHandler.hpp +++ b/packages/stk/stk_util/stk_util/util/ReportHandler.hpp @@ -40,9 +40,8 @@ #include // for operator+, allocator, string, char_traits #include -#include "Kokkos_Core.hpp" #include "stk_util/diag/String.hpp" -#include "stk_util/stk_kokkos_macros.h" // for STK_INLINE_FUNCTION +#include "stk_util/stk_kokkos_macros.h" // for STK_FUNCTION namespace stk { @@ -287,7 +286,7 @@ inline auto eval_test_condition(const T& val) } \ } while (false) -inline void ThrowMsgHost(bool expr, const char * exprString, const char * message, const std::string & location) +inline void ThrowMsgHost(bool /* expr */, const char* exprString, const char* message, const std::string& location) { std::ostringstream stk_util_internal_throw_require_loc_oss; stk_util_internal_throw_require_loc_oss << stk::source_relative_path(location) << "\n"; @@ -298,12 +297,9 @@ inline void ThrowMsgHost(bool expr, const char * exprString, const char * messag "Error: " + message + "\n"); } -STK_INLINE_FUNCTION void ThrowMsgDevice(const char * message) -{ - Kokkos::abort(message); -} +STK_FUNCTION void ThrowMsgDevice(const char * message); -inline void ThrowHost(bool expr, const char * exprString, const std::string & location) +inline void ThrowHost(bool /* expr */, const char* exprString, const std::string& location) { std::ostringstream stk_util_internal_throw_require_loc_oss; stk_util_internal_throw_require_loc_oss << stk::source_relative_path(location) << "\n"; @@ -323,11 +319,7 @@ inline void ThrowErrorMsgHost(const char * message, const std::string & location "Error: " + message + "\n"); } -STK_INLINE_FUNCTION void ThrowErrorMsgDevice(const char * message) -{ - Kokkos::abort(message); -} - +STK_FUNCTION void ThrowErrorMsgDevice(const char * message); // This generic macro is for unconditional throws. We pass "" as the expr // string, the handler should be smart enough to realize that this means there diff --git a/packages/stk/stk_util/stk_util/util/StkNgpVector.hpp b/packages/stk/stk_util/stk_util/util/StkNgpVector.hpp index bd579a607a0c..7e09ad6b4344 100644 --- a/packages/stk/stk_util/stk_util/util/StkNgpVector.hpp +++ b/packages/stk/stk_util/stk_util/util/StkNgpVector.hpp @@ -63,10 +63,7 @@ class NgpVector { Kokkos::deep_copy(hostVals, init); } - NgpVector(size_t s, Datatype init) : NgpVector(get_default_name(), s, init) - { - } - KOKKOS_FUNCTION ~NgpVector() {} + NgpVector(size_t s, Datatype init) : NgpVector(get_default_name(), s, init) {} std::string name() const { return hostVals.label(); } @@ -125,7 +122,7 @@ class NgpVector { return deviceVals(i); } -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) +#ifdef STK_ENABLE_GPU template KOKKOS_FUNCTION Datatype & get( typename std::enable_if< diff --git a/packages/stk/stk_util/stk_util/util/StridedArray.hpp b/packages/stk/stk_util/stk_util/util/StridedArray.hpp index 2b3336fee2e8..69881b38abeb 100644 --- a/packages/stk/stk_util/stk_util/util/StridedArray.hpp +++ b/packages/stk/stk_util/stk_util/util/StridedArray.hpp @@ -48,43 +48,43 @@ template class StridedArray { public: - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION StridedArray() : dataPointer(nullptr), length(0) -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) +#ifdef STK_ENABLE_GPU , stride(defaultStride) #endif { } - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION StridedArray(T* e, unsigned n, int stride_in=defaultStride) : dataPointer(e), length(n) -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) +#ifdef STK_ENABLE_GPU , stride(stride_in) #endif { } - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION StridedArray(PairIter data -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) +#ifdef STK_ENABLE_GPU , int stride_in=defaultStride #endif ) : dataPointer(data.begin()), length(data.size()) -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) +#ifdef STK_ENABLE_GPU , stride(stride_in) #endif { } - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION T operator[](unsigned i) const { #if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__) @@ -94,21 +94,43 @@ class StridedArray #endif } - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION T* data() { return dataPointer; } - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION const T* data() const { return dataPointer; } - KOKKOS_FUNCTION + KOKKOS_INLINE_FUNCTION unsigned size() const { return length; } + KOKKOS_INLINE_FUNCTION + bool operator==(const StridedArray& rhs) const + { + if (this->size() != rhs.size()) { + return false; + } + + for(unsigned i=0; i& rhs) const + { + return !(*this == rhs); + } + private: T* dataPointer; unsigned length; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) +#ifdef STK_ENABLE_GPU int stride; #endif };