diff --git a/Tests/Particles/AsyncIO/CMakeLists.txt b/Tests/Particles/AsyncIO/CMakeLists.txt deleted file mode 100644 index cfb0489fa11..00000000000 --- a/Tests/Particles/AsyncIO/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -foreach(D IN LISTS AMReX_SPACEDIM) - set(_sources main.cpp) - set(_input_files inputs ) - - setup_test(${D} _sources _input_files) - - unset(_sources) - unset(_input_files) -endforeach() diff --git a/Tests/Particles/AsyncIO/GNUmakefile b/Tests/Particles/AsyncIO/GNUmakefile deleted file mode 100644 index 6dddb3a2871..00000000000 --- a/Tests/Particles/AsyncIO/GNUmakefile +++ /dev/null @@ -1,32 +0,0 @@ -AMREX_HOME = ../../../ - -DEBUG = TRUE -DEBUG = FALSE - -DIM = 3 - -COMP = gcc - -TINY_PROFILE = TRUE -USE_PARTICLES = TRUE - -PRECISION = DOUBLE - -USE_MPI = TRUE -USE_OMP = FALSE - -MPI_THREAD_MULTIPLE = TRUE - -################################################### - -EBASE = main - -include $(AMREX_HOME)/Tools/GNUMake/Make.defs - -include ./Make.package -include $(AMREX_HOME)/Src/Base/Make.package -include $(AMREX_HOME)/Src/Boundary/Make.package -include $(AMREX_HOME)/Src/AmrCore/Make.package -include $(AMREX_HOME)/Src/Particle/Make.package - -include $(AMREX_HOME)/Tools/GNUMake/Make.rules diff --git a/Tests/Particles/AsyncIO/Make.package b/Tests/Particles/AsyncIO/Make.package deleted file mode 100644 index 7f43e5e87cb..00000000000 --- a/Tests/Particles/AsyncIO/Make.package +++ /dev/null @@ -1,2 +0,0 @@ -CEXE_sources += main.cpp - diff --git a/Tests/Particles/AsyncIO/inputs b/Tests/Particles/AsyncIO/inputs deleted file mode 100644 index 223575f9554..00000000000 --- a/Tests/Particles/AsyncIO/inputs +++ /dev/null @@ -1,29 +0,0 @@ - -# Domain size - -nx = 32 # number of grid points along the x axis -ny = 32 # number of grid points along the y axis -nz = 32 # number of grid points along the z axis - -#nx = 64 # number of grid points along the x axis -#ny = 64 # number of grid points along the y axis -#nz = 64 # number of grid points along the z axis - -#nx = 128 # number of grid points along the x axis -#ny = 128 # number of grid points along the y axis -#nz = 128 # number of grid points along the z axis - -# Maximum allowable size of each subdomain in the problem domain; -# this is used to decompose the domain for parallel calculations. -max_grid_size = 16 - -# Number of particles per cell -nppc = 10 - -# Verbosity -verbose = true # set to true to get more verbosity - -# Number of levels -nlevs = 2 - -amrex.async_out=1 diff --git a/Tests/Particles/AsyncIO/main.cpp b/Tests/Particles/AsyncIO/main.cpp deleted file mode 100644 index 1bc03cccda1..00000000000 --- a/Tests/Particles/AsyncIO/main.cpp +++ /dev/null @@ -1,331 +0,0 @@ -#include - -#include -#include -#include -#include -#include - -#include - -using namespace amrex; - -static constexpr int NSR = 1 + AMREX_SPACEDIM; -static constexpr int NSI = 0; -static constexpr int NAR = 0; -static constexpr int NAI = 0; - -struct TestParams { - int nx; - int ny; - int nz; - int max_grid_size; - int nppc; - int nlevs; - bool verbose; -}; - -void get_position_unit_cell(Real* r, const IntVect& nppc, int i_part) -{ - int nx = nppc[0]; -#if AMREX_SPACEDIM > 1 - int ny = nppc[1]; -#else - int ny = 1; -#endif -#if AMREX_SPACEDIM > 2 - int nz = nppc[2]; -#else - int nz = 1; -#endif - - int ix_part = i_part/(ny * nz); - int iy_part = (i_part % (ny * nz)) % ny; - int iz_part = (i_part % (ny * nz)) / ny; - - r[0] = (0.5+ix_part)/nx; - r[1] = (0.5+iy_part)/ny; - r[2] = (0.5+iz_part)/nz; -} - -class MyParticleContainer - : public amrex::ParticleContainer -{ - -public: - - MyParticleContainer (const Vector & a_geom, - const Vector & a_dmap, - const Vector & a_ba, - const Vector & a_rr) - : amrex::ParticleContainer(a_geom, a_dmap, a_ba, a_rr) - {} - - void InitParticles (const amrex::IntVect& a_num_particles_per_cell) - { - BL_PROFILE("InitParticles"); - - const int lev = 0; // only add particles on level 0 - const Real* dx = Geom(lev).CellSize(); - const Real* plo = Geom(lev).ProbLo(); - - const int num_ppc = AMREX_D_TERM( a_num_particles_per_cell[0], - *a_num_particles_per_cell[1], - *a_num_particles_per_cell[2]); - - for(MFIter mfi = MakeMFIter(lev); mfi.isValid(); ++mfi) - { - const Box& tile_box = mfi.tilebox(); - - Gpu::HostVector host_particles; - std::array, NAR> host_real; - std::array, NAI> host_int; - - std::vector > host_runtime_real(NumRuntimeRealComps()); - std::vector > host_runtime_int(NumRuntimeIntComps()); - - for (IntVect iv = tile_box.smallEnd(); iv <= tile_box.bigEnd(); tile_box.next(iv)) - { - for (int i_part=0; i_part (plo[0] + (iv[0] + r[0])*dx[0]); -#if AMREX_SPACEDIM > 1 - p.pos(1) = static_cast (plo[1] + (iv[1] + r[1])*dx[1]); -#endif -#if AMREX_SPACEDIM > 2 - p.pos(2) = static_cast (plo[2] + (iv[2] + r[2])*dx[2]); -#endif - - if constexpr (NSR > 0) { - for (int i = 0; i < NSR; ++i) { - p.rdata(i) = ParticleReal(p.id()); - } - } - if constexpr (NSI > 0) { - for (int i = 0; i < NSI; ++i) { - p.idata(i) = p.id(); - } - } - - host_particles.push_back(p); - - if constexpr (NAR > 0) { - for (int i = 0; i < NAR; ++i) { - host_real[i].push_back(ParticleReal(p.id())); - } - } - if constexpr (NAI > 0) { - for (int i = 0; i < NAI; ++i) { - host_int[i].push_back(int(p.id())); - } - } - for (int i = 0; i < NumRuntimeRealComps(); ++i) { - host_runtime_real[i].push_back(ParticleReal(p.id())); - } - for (int i = 0; i < NumRuntimeIntComps(); ++i) { - host_runtime_int[i].push_back(int(p.id())); - } - } - } - - auto& particle_tile = DefineAndReturnParticleTile(lev, mfi.index(), mfi.LocalTileIndex()); - auto old_size = particle_tile.GetArrayOfStructs().size(); - auto new_size = old_size + host_particles.size(); - particle_tile.resize(new_size); - - Gpu::copyAsync(Gpu::hostToDevice, - host_particles.begin(), - host_particles.end(), - particle_tile.GetArrayOfStructs().begin() + old_size); - - auto& soa = particle_tile.GetStructOfArrays(); - for (int i = 0; i < NAR; ++i) - { - Gpu::copyAsync(Gpu::hostToDevice, - host_real[i].begin(), - host_real[i].end(), - soa.GetRealData(i).begin() + old_size); - } - - for (int i = 0; i < NAI; ++i) - { - Gpu::copyAsync(Gpu::hostToDevice, - host_int[i].begin(), - host_int[i].end(), - soa.GetIntData(i).begin() + old_size); - } - for (int i = 0; i < NumRuntimeRealComps(); ++i) - { - Gpu::copyAsync(Gpu::hostToDevice, - host_runtime_real[i].begin(), - host_runtime_real[i].end(), - soa.GetRealData(NAR+i).begin() + old_size); - } - - for (int i = 0; i < NumRuntimeIntComps(); ++i) - { - Gpu::copyAsync(Gpu::hostToDevice, - host_runtime_int[i].begin(), - host_runtime_int[i].end(), - soa.GetIntData(NAI+i).begin() + old_size); - } - - Gpu::streamSynchronize(); - } - - Redistribute(); - } -}; - -void test_async_io(TestParams& parms) -{ - int nlevs = parms.nlevs; - - RealBox real_box; - for (int n = 0; n < BL_SPACEDIM; n++) { - real_box.setLo(n, 0.0); - real_box.setHi(n, 1.0); - } - - RealBox fine_box; - for (int n = 0; n < BL_SPACEDIM; n++) - { - fine_box.setLo(n,0.25); - fine_box.setHi(n,0.75); - } - - IntVect domain_lo(AMREX_D_DECL(0 , 0, 0)); - IntVect domain_hi(AMREX_D_DECL(parms.nx - 1, parms.ny - 1, parms.nz-1)); - const Box domain(domain_lo, domain_hi); - - // Define the refinement ratio - Vector rr(nlevs-1); - for (int lev = 1; lev < nlevs; lev++) { - rr[lev-1] = IntVect(AMREX_D_DECL(2, 2, 2)); - } - - // This sets the boundary conditions to be doubly or triply periodic - int is_per[] = {AMREX_D_DECL(1,1,1)}; - - // This defines a Geometry object which is useful for writing the plotfiles - Vector geom(nlevs); - geom[0].define(domain, &real_box, CoordSys::cartesian, is_per); - for (int lev = 1; lev < nlevs; lev++) { - geom[lev].define(amrex::refine(geom[lev-1].Domain(), rr[lev-1]), - &real_box, CoordSys::cartesian, is_per); - } - - Vector ba(nlevs); - ba[0].define(domain); - - if (nlevs > 1) { - int n_fine = parms.nx*rr[0][0]; - IntVect refined_lo(AMREX_D_DECL(n_fine/4,n_fine/4,n_fine/4)); - IntVect refined_hi(AMREX_D_DECL(3*n_fine/4-1,3*n_fine/4-1,3*n_fine/4-1)); - - // Build a box for the level 1 domain - Box refined_patch(refined_lo, refined_hi); - ba[1].define(refined_patch); - } - - // break the BoxArrays at both levels into max_grid_size^3 boxes - for (int lev = 0; lev < nlevs; lev++) { - ba[lev].maxSize(parms.max_grid_size); - } - - Vector dmap(nlevs); - - Vector > partMF(nlevs); - Vector > density(nlevs); - Vector > acceleration(nlevs); - for (int lev = 0; lev < nlevs; lev++) { - dmap[lev] = DistributionMapping{ba[lev]}; - density[lev] = std::make_unique(ba[lev], dmap[lev], 1, 0); - density[lev]->setVal(0.0); - acceleration[lev] = std::make_unique(ba[lev], dmap[lev], 3, 1); - acceleration[lev]->setVal(5.0, 1); - } - - MyParticleContainer myPC(geom, dmap, ba, rr); - myPC.SetVerbose(false); - - myPC.InitParticles(IntVect(AMREX_D_DECL(2, 2, 2))); - - for (int step = 0; step < 4000; ++step) - { - myPC.AssignDensity(0, partMF, 0, 1, nlevs-1); - - for (int lev = 0; lev < nlevs; ++lev) { - MultiFab::Copy(*density[lev], *partMF[lev], 0, 0, 1, 0); - } - - if (step % 1000 == 0) { - Vector varnames; - varnames.push_back("density"); - - Vector particle_varnames; - particle_varnames.push_back("mass"); - - Vector level_steps; - level_steps.push_back(0); - level_steps.push_back(0); - - int output_levs = nlevs; - - Vector outputMF(output_levs); - Vector outputRR(output_levs); - for (int lev = 0; lev < output_levs; ++lev) { - outputMF[lev] = density[lev].get(); - outputRR[lev] = IntVect(AMREX_D_DECL(2, 2, 2)); - } - - std::string fn = amrex::Concatenate("plt", step, 5); - - WriteMultiLevelPlotfile(fn, output_levs, outputMF, - varnames, geom, 0.0, level_steps, outputRR); - - myPC.WritePlotFile(fn, "particle0"); - } - } -} - -int main(int argc, char* argv[]) -{ - amrex::Initialize(argc,argv); - - ParmParse pp; - - TestParams parms; - - pp.get("nx", parms.nx); - pp.get("ny", parms.ny); - pp.get("nz", parms.nz); - pp.get("max_grid_size", parms.max_grid_size); - pp.get("nlevs", parms.nlevs); - pp.get("nppc", parms.nppc); - if (parms.nppc < 1 && ParallelDescriptor::IOProcessor()) { - amrex::Abort("Must specify at least one particle per cell"); - } - - parms.verbose = false; - pp.query("verbose", parms.verbose); - - if (parms.verbose && ParallelDescriptor::IOProcessor()) { - std::cout << '\n'; - std::cout << "Number of particles per cell : "; - std::cout << parms.nppc << '\n'; - std::cout << "Size of domain : "; - std::cout << "Num levels: "; - std::cout << parms.nlevs << '\n'; - std::cout << parms.nx << " " << parms.ny << " " << parms.nz << '\n'; - } - - test_async_io(parms); - - amrex::Finalize(); -} diff --git a/Tests/Particles/CheckpointRestart/CMakeLists.txt b/Tests/Particles/CheckpointRestart/CMakeLists.txt index c032392c5df..6e36bd1f8a2 100644 --- a/Tests/Particles/CheckpointRestart/CMakeLists.txt +++ b/Tests/Particles/CheckpointRestart/CMakeLists.txt @@ -1,14 +1,12 @@ -# This tests requires particle support +# This test requires particle support if (NOT AMReX_PARTICLES) return() endif () foreach(D IN LISTS AMReX_SPACEDIM) - set(_sources main.cpp) - set(_input_files inputs ) + set(_sources main.cpp) setup_test(${D} _sources _input_files) unset(_sources) - unset(_input_files) endforeach() diff --git a/Tests/Particles/CheckpointRestart/GNUmakefile b/Tests/Particles/CheckpointRestart/GNUmakefile index b3510b88d2e..d31fe5d711b 100644 --- a/Tests/Particles/CheckpointRestart/GNUmakefile +++ b/Tests/Particles/CheckpointRestart/GNUmakefile @@ -1,6 +1,5 @@ AMREX_HOME = ../../../ -# DEBUG = TRUE DEBUG = FALSE DIM = 3 @@ -18,6 +17,10 @@ TINY_PROFILE = TRUE USE_PARTICLES = TRUE +USE_HDF5 = FALSE +# Set HDF5_HOME to the root of your HDF5 installation, e.g.: +# HDF5_HOME = /path/to/hdf5 + ################################################### EBASE = main diff --git a/Tests/Particles/CheckpointRestart/inputs b/Tests/Particles/CheckpointRestart/inputs deleted file mode 100644 index 81855c5947d..00000000000 --- a/Tests/Particles/CheckpointRestart/inputs +++ /dev/null @@ -1,26 +0,0 @@ -# Domain size -ncells = 64 - -# Maximum allowable size of each subdomain in the problem domain; -# this is used to decompose the domain for parallel calculations. -max_grid_size = 8 - -# Number of levels -nlevs = 1 - -# Number of components in the multifabs -ncomp = 6 - -# Number of particles per cell -nppc = 2 - -# Number of plot files to write -nplotfile = 1 - -# Number of plot files to write -nparticlefile = 1 - -# Whether to check the correctness of Checkpoint / Restart -restart_check = 1 - -directory=. diff --git a/Tests/Particles/CheckpointRestart/main.cpp b/Tests/Particles/CheckpointRestart/main.cpp index 35e6697ae96..f3a184034d0 100644 --- a/Tests/Particles/CheckpointRestart/main.cpp +++ b/Tests/Particles/CheckpointRestart/main.cpp @@ -1,225 +1,453 @@ +/** + * Test for dual-grid particle checkpoint/restart capability.. + * + * Verifies that particles are preserved correctly when reading a + * checkpoint file into a ParticleContainer with a different mesh structure, + * covering: + * 1. Restart with fewer AMR levels than the checkpoint + * 2. Restart with more AMR levels than the checkpoint + * 3. Restart with the same number of levels but different BoxArrays + */ #include -#include -#include #include -#include +#include +#include +#include using namespace amrex; -void set_grids_nested (Vector& domains, - Vector& grids, - Vector& ref_ratio); -void test (); +// Particle type: 4 struct reals, 1 struct int, 2 array reals, 1 array int +constexpr int NStructReal = 4; +constexpr int NStructInt = 1; +constexpr int NArrayReal = 2; +constexpr int NArrayInt = 1; + +using MyPC = ParticleContainer; +using PType = typename MyPC::SuperParticleType; + +struct MeshData { + Vector domains; + Vector ba; + Vector ref_ratio; + Vector geom; + Vector dmap; +}; + +struct ParticleRecord { + std::uint64_t idcpu = 0; + std::array pos{}; + std::array real{}; + std::array idata{}; + + bool operator< (ParticleRecord const& rhs) const + { + if (idcpu != rhs.idcpu) { return idcpu < rhs.idcpu; } + if (pos != rhs.pos) { return pos < rhs.pos; } + if (real != rhs.real) { return real < rhs.real; } + return idata < rhs.idata; + } + + bool operator== (ParticleRecord const& rhs) const + { + return idcpu == rhs.idcpu && pos == rhs.pos && real == rhs.real && idata == rhs.idata; + } +}; -int main(int argc, char* argv[]) +Vector +collect_local_records (MyPC const& pc) { - amrex::Initialize(argc,argv); - test(); - amrex::Finalize(); + Vector records; + + for (int lev = 0; lev <= pc.finestLevel(); ++lev) { + for (auto const& kv : pc.GetParticles(lev)) { + auto const& ptile = kv.second; + auto const np = ptile.numParticles(); + auto const ptd = ptile.getConstParticleTileData(); + + for (int i = 0; i < np; ++i) { + auto const sp = ptd.getSuperParticle(i); + ParticleRecord rec; + rec.idcpu = sp.m_idcpu; + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + rec.pos[idim] = sp.pos(idim); + } + for (int icomp = 0; icomp < NStructReal + NArrayReal; ++icomp) { + rec.real[icomp] = sp.rdata(icomp); + } + for (int icomp = 0; icomp < NStructInt + NArrayInt; ++icomp) { + rec.idata[icomp] = sp.idata(icomp); + } + records.push_back(rec); + } + } + } + + return records; } -void test () +Vector +gather_sorted_records (MyPC const& pc) { - const int nghost = 0; - int ncells, max_grid_size, ncomp, nlevs, nppc; - int restart_check = 0, nplotfile = 1, nparticlefile = 1; - std::string directory; + auto local_records = collect_local_records(pc); + auto const local_nbytes = static_cast(local_records.size() * sizeof(ParticleRecord)); + auto const ioproc = ParallelDescriptor::IOProcessorNumber(); + auto const recv_counts = ParallelDescriptor::Gather(local_nbytes, ioproc); + + Vector recv_offsets; + Vector recv_buffer; + + if (ParallelDescriptor::IOProcessor()) { + recv_offsets.resize(recv_counts.size(), 0); + int total_nbytes = 0; + for (int i = 0, n = static_cast(recv_counts.size()); i < n; ++i) { + recv_offsets[i] = total_nbytes; + total_nbytes += recv_counts[i]; + } + recv_buffer.resize(total_nbytes); + } + + auto const* send_ptr = local_records.empty() + ? nullptr + : reinterpret_cast(local_records.data()); + auto* recv_ptr = recv_buffer.empty() ? nullptr : recv_buffer.data(); - ParmParse pp; - pp.get("ncells", ncells); - pp.get("max_grid_size", max_grid_size); - pp.get("ncomp", ncomp); - pp.get("nlevs", nlevs); - pp.get("nppc", nppc); - pp.query("nplotfile", nplotfile); - pp.query("nparticlefile", nparticlefile); - pp.query("restart_check", restart_check); - pp.query("directory", directory); + ParallelDescriptor::Gatherv(send_ptr, local_nbytes, recv_ptr, + recv_counts, recv_offsets, ioproc); - if (!directory.empty() && directory.back() != '/') { - // Include separator if one was not provided - directory += "/"; + Vector gathered_records; + if (ParallelDescriptor::IOProcessor()) { + gathered_records.resize(recv_buffer.size() / sizeof(ParticleRecord)); + if (!recv_buffer.empty()) { + std::memcpy(gathered_records.data(), recv_buffer.data(), recv_buffer.size()); + } + std::sort(gathered_records.begin(), gathered_records.end()); } - Vector domains; - Vector ba; - Vector ref_ratio; + return gathered_records; +} - set_grids_nested(domains, ba, ref_ratio); +/** + * Build a nested AMR hierarchy with `nlevs` levels over a [0,1]^d domain + * discretised by `ncells` cells in each direction. The coarse BoxArray is + * decomposed into boxes of at most `max_grid_size` cells; the fine level + * covers the central half of the domain. + */ +MeshData build_mesh (int ncells, int nlevs, int max_grid_size) +{ + AMREX_ALWAYS_ASSERT(nlevs >= 1 && nlevs <= 2); + + MeshData m; + + // Level-0 domain + IntVect lo(AMREX_D_DECL(0, 0, 0)); + IntVect hi(AMREX_D_DECL(ncells-1, ncells-1, ncells-1)); + + m.domains.resize(nlevs); + m.domains[0].setSmall(lo); + m.domains[0].setBig(hi); + + m.ref_ratio.resize(nlevs > 1 ? nlevs - 1 : 0); + for (int lev = 1; lev < nlevs; ++lev) { + m.ref_ratio[lev-1] = IntVect(AMREX_D_DECL(2, 2, 2)); + m.domains[lev] = amrex::refine(m.domains[lev-1], m.ref_ratio[lev-1]); + } + + m.ba.resize(nlevs); + m.ba[0].define(m.domains[0]); + m.ba[0].maxSize(max_grid_size); + + if (nlevs > 1) { + // Refined region: the central 1/4 of the fine-level domain + int n_fine = ncells * 2; // ref_ratio == 2 + IntVect rlo(AMREX_D_DECL(n_fine/4, n_fine/4, n_fine/4)); + IntVect rhi(AMREX_D_DECL(3*n_fine/4-1, 3*n_fine/4-1, 3*n_fine/4-1)); + m.ba[1].define(Box(rlo, rhi)); + m.ba[1].maxSize(max_grid_size); + } RealBox real_box; - for (int n = 0; n < AMREX_SPACEDIM; n++) { + for (int n = 0; n < AMREX_SPACEDIM; ++n) { real_box.setLo(n, 0.0); real_box.setHi(n, 1.0); } + int is_per[] = {AMREX_D_DECL(1, 1, 1)}; - // This sets the boundary conditions to be doubly or triply periodic - int is_per[] = {AMREX_D_DECL(1,1,1)}; + m.geom.resize(nlevs); + m.geom[0].define(m.domains[0], &real_box, CoordSys::cartesian, is_per); + for (int lev = 1; lev < nlevs; ++lev) { + m.geom[lev].define(m.domains[lev], &real_box, CoordSys::cartesian, is_per); + } - // This defines a Geometry object for each level - Vector geom(nlevs); - geom[0].define(domains[0], &real_box, CoordSys::cartesian, is_per); - for (int lev = 1; lev < nlevs; lev++) { - geom[lev].define(domains[lev], &real_box, CoordSys::cartesian, is_per); + m.dmap.resize(nlevs); + for (int lev = 0; lev < nlevs; ++lev) { + m.dmap[lev] = DistributionMapping{m.ba[lev]}; } - Vector dmap(nlevs); + return m; +} - Vector > mf(nlevs); - for (int lev = 0; lev < nlevs; lev++) { - dmap[lev] = DistributionMapping{ba[lev]}; - mf[lev] = std::make_unique(ba[lev], dmap[lev], ncomp, nghost); - mf[lev]->setVal(lev); +/** + * Assert that two ParticleContainers hold identical particle data by + * comparing exact sorted particle records gathered on the I/O rank. + */ +void verify_same (MyPC& pc_orig, MyPC& pc_new) +{ + auto orig_records = gather_sorted_records(pc_orig); + auto new_records = gather_sorted_records(pc_new); + + if (ParallelDescriptor::IOProcessor()) { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + orig_records.size() == new_records.size(), + "Particle count mismatch after restart"); + + for (Long i = 0; i < orig_records.size(); ++i) { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + orig_records[i] == new_records[i], + "Particle data mismatch after restart"); + } } +} - // these don't really matter, make something up - const Real time = 0.0; +/** + * Test 0: Write and restart with the same hierarchy and BoxArray. + * This preserves the original checkpoint/restart coverage without any + * hierarchy or decomposition changes. + */ +void test_same_hierarchy () +{ + amrex::Print() << "Test 0: restart with same hierarchy\n"; - Vector varnames; - for (int i = 0; i < ncomp; ++i) - { - varnames.push_back("component_" + std::to_string(i)); + const int ncells = 32; + const int max_grid_size = 16; + const int nppc = 2; + const int iseed = 451; + const std::string chkdir = "chk_same_hierarchy"; + + MyPC::ParticleInitData pdata = {{1.0, 2.0, 3.0, 4.0}, + {5}, + {6.0, 7.0}, + {8}}; + + Vector real_names, int_names; + for (int i = 0; i < NStructReal + NArrayReal; ++i) { + real_names.push_back("real_" + std::to_string(i)); + } + for (int i = 0; i < NStructInt + NArrayInt; ++i) { + int_names.push_back("int_" + std::to_string(i)); } - Vector level_steps(nlevs, 0); + auto mesh = build_mesh(ncells, 2, max_grid_size); + MyPC pc_write(mesh.geom, mesh.dmap, mesh.ba, mesh.ref_ratio); + pc_write.SetVerbose(false); + pc_write.InitRandom(nppc * AMREX_D_TERM(ncells, *ncells, *ncells), + iseed, pdata, /*serialize=*/false); +#ifdef AMREX_USE_HDF5 + pc_write.CheckpointHDF5(chkdir, "particles", true, real_names, int_names); +#else + pc_write.Checkpoint(chkdir, "particles", real_names, int_names); +#endif - char fname[512]; - for (int ts = 0; ts < nplotfile; ts++) { - std::snprintf(fname, sizeof fname, "%splt%05d", directory.c_str(), ts); + ParallelDescriptor::Barrier(); - amrex::Print() << "Writing plot file [" << fname << "] ..." << '\n'; + MyPC pc_read(mesh.geom, mesh.dmap, mesh.ba, mesh.ref_ratio); + pc_read.SetVerbose(false); +#ifdef AMREX_USE_HDF5 + pc_read.RestartHDF5(chkdir + "/particles", "particles"); +#else + pc_read.Restart(chkdir, "particles"); +#endif - WriteMultiLevelPlotfile(fname, nlevs, amrex::GetVecOfConstPtrs(mf), - varnames, geom, time, level_steps, ref_ratio); + verify_same(pc_write, pc_read); - amrex::Print() << " done \n"; - } + amrex::Print() << " PASSED\n"; +} - // Add some particles - constexpr int NStructReal = 4; - constexpr int NStructInt = 1; - constexpr int NArrayReal = 8; - constexpr int NArrayInt = 3; +/** + * Test 1: Write a 2-level checkpoint, restart into a 1-level container. + * Particles that resided on the fine level must be redistributed to the + * coarse level; totals and component sums must be unchanged. + */ +void test_fewer_levels () +{ + amrex::Print() << "Test 1: restart with fewer levels than checkpoint\n"; - using MyPC = ParticleContainer; - MyPC myPC(geom, dmap, ba, ref_ratio); - myPC.SetVerbose(false); + const int ncells = 32; + const int max_grid_size = 16; + const int nppc = 2; + const int iseed = 451; + const std::string chkdir = "chk_fewer_levels"; - int num_particles = nppc * AMREX_D_TERM(ncells, * ncells, * ncells); - bool serialize = false; - int iseed = 451; MyPC::ParticleInitData pdata = {{1.0, 2.0, 3.0, 4.0}, {5}, - {6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0}, - {14, 15, 16}}; + {6.0, 7.0}, + {8}}; - if (nparticlefile > 0) { - amrex::Print() << "Init particles ..." << '\n'; + Vector real_names, int_names; + for (int i = 0; i < NStructReal + NArrayReal; ++i) { + real_names.push_back("real_" + std::to_string(i)); + } + for (int i = 0; i < NStructInt + NArrayInt; ++i) { + int_names.push_back("int_" + std::to_string(i)); + } - myPC.InitRandom(num_particles, iseed, pdata, serialize); + // --- write with 2 levels --- + auto mesh2 = build_mesh(ncells, 2, max_grid_size); + MyPC pc_write(mesh2.geom, mesh2.dmap, mesh2.ba, mesh2.ref_ratio); + pc_write.SetVerbose(false); + pc_write.InitRandom(nppc * AMREX_D_TERM(ncells, *ncells, *ncells), + iseed, pdata, /*serialize=*/false); +#ifdef AMREX_USE_HDF5 + pc_write.CheckpointHDF5(chkdir, "particles", true, real_names, int_names); +#else + pc_write.Checkpoint(chkdir, "particles", real_names, int_names); +#endif - amrex::Print() << " done \n"; + ParallelDescriptor::Barrier(); - Vector particle_realnames; - for (int i = 0; i < NStructReal + NArrayReal; ++i) { - particle_realnames.push_back("particle_real_component_" + std::to_string(i)); - } + // --- restart with 1 level --- + auto mesh1 = build_mesh(ncells, 1, max_grid_size); + MyPC pc_read(mesh1.geom, mesh1.dmap, mesh1.ba, mesh1.ref_ratio); + pc_read.SetVerbose(false); +#ifdef AMREX_USE_HDF5 + pc_read.RestartHDF5(chkdir + "/particles", "particles"); +#else + pc_read.Restart(chkdir, "particles"); +#endif - Vector particle_intnames; - for (int i = 0; i < NStructInt + NArrayInt; ++i) { - particle_intnames.push_back("particle_int_component_" + std::to_string(i)); - } + verify_same(pc_write, pc_read); + + amrex::Print() << " PASSED\n"; +} - for (int ts = 0; ts < nparticlefile; ts++) { - std::snprintf(fname, sizeof fname, "%splt%05d", directory.c_str(), ts); +/** + * Test 2: Write a 1-level checkpoint, restart into a 2-level container. + * After restart, Redistribute() assigns particles inside the refined + * region to the finer level; totals and component sums must be unchanged. + */ +void test_more_levels () +{ + amrex::Print() << "Test 2: restart with more levels than checkpoint\n"; - amrex::Print() << "Writing particle file [" << fname << "] ..." << '\n'; + const int ncells = 32; + const int max_grid_size = 16; + const int nppc = 2; + const int iseed = 451; + const std::string chkdir = "chk_more_levels"; - myPC.Checkpoint(fname, "particle0", false, particle_realnames, particle_intnames); + MyPC::ParticleInitData pdata = {{1.0, 2.0, 3.0, 4.0}, + {5}, + {6.0, 7.0}, + {8}}; - amrex::Print() << " done \n"; - } + Vector real_names, int_names; + for (int i = 0; i < NStructReal + NArrayReal; ++i) { + real_names.push_back("real_" + std::to_string(i)); } + for (int i = 0; i < NStructInt + NArrayInt; ++i) { + int_names.push_back("int_" + std::to_string(i)); + } + + // --- write with 1 level --- + auto mesh1 = build_mesh(ncells, 1, max_grid_size); + MyPC pc_write(mesh1.geom, mesh1.dmap, mesh1.ba, mesh1.ref_ratio); + pc_write.SetVerbose(false); + pc_write.InitRandom(nppc * AMREX_D_TERM(ncells, *ncells, *ncells), + iseed, pdata, /*serialize=*/false); +#ifdef AMREX_USE_HDF5 + pc_write.CheckpointHDF5(chkdir, "particles", true, real_names, int_names); +#else + pc_write.Checkpoint(chkdir, "particles", real_names, int_names); +#endif ParallelDescriptor::Barrier(); - char directory_path[512]; - if (restart_check && nparticlefile > 0) - { - MyPC newPC(geom, dmap, ba, ref_ratio); - std::snprintf(directory_path, sizeof directory_path, "%s%s", directory.c_str(), "plt00000"); - newPC.Restart(directory_path, "particle0"); - - using PType = typename MyPC::SuperParticleType; - - for (int icomp=0; icomp Real - { - return p.rdata(1); - }); - - auto sm_old = amrex::ReduceSum(myPC, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> Real - { - return p.rdata(1); - }); - - ParallelDescriptor::ReduceRealSum(sm_new); - ParallelDescriptor::ReduceRealSum(sm_old); - - AMREX_ALWAYS_ASSERT(sm_old == sm_new); - } - } + // --- restart with 2 levels --- + auto mesh2 = build_mesh(ncells, 2, max_grid_size); + MyPC pc_read(mesh2.geom, mesh2.dmap, mesh2.ba, mesh2.ref_ratio); + pc_read.SetVerbose(false); +#ifdef AMREX_USE_HDF5 + pc_read.RestartHDF5(chkdir + "/particles", "particles"); +#else + pc_read.Restart(chkdir, "particles"); +#endif + + verify_same(pc_write, pc_read); + + amrex::Print() << " PASSED\n"; } -void set_grids_nested (Vector& domains, - Vector& grids, - Vector& ref_ratio) +/** + * Test 3: Write a checkpoint with one BoxArray decomposition, restart + * into a container with the same number of levels but a different BoxArray + * (different max_grid_size). This is the canonical dual-grid scenario: the + * Particle_H header written at checkpoint time differs from the BoxArray + * of the new container, so AMReX uses temporary grids while reading and + * calls Redistribute() to move particles onto the new decomposition. + */ +void test_different_boxarrays () { - int ncells, max_grid_size, nlevs; + amrex::Print() << "Test 3: restart with same levels but different BoxArrays\n"; - ParmParse pp; - pp.get("ncells", ncells); - pp.get("max_grid_size", max_grid_size); - pp.get("nlevs", nlevs); + const int ncells = 32; + const int nppc = 2; + const int iseed = 451; + const std::string chkdir = "chk_different_ba"; - AMREX_ALWAYS_ASSERT(nlevs < 2); // relax this later + MyPC::ParticleInitData pdata = {{1.0, 2.0, 3.0, 4.0}, + {5}, + {6.0, 7.0}, + {8}}; - IntVect domain_lo(AMREX_D_DECL(0, 0, 0)); - IntVect domain_hi(AMREX_D_DECL(ncells-1, ncells-1, ncells-1)); + Vector real_names, int_names; + for (int i = 0; i < NStructReal + NArrayReal; ++i) { + real_names.push_back("real_" + std::to_string(i)); + } + for (int i = 0; i < NStructInt + NArrayInt; ++i) { + int_names.push_back("int_" + std::to_string(i)); + } - domains.resize(nlevs); - domains[0].setSmall(domain_lo); - domains[0].setBig(domain_hi); + // --- write: coarse decomposition (few large boxes) --- + auto mesh_coarse = build_mesh(ncells, 1, ncells); // one box per rank at most + MyPC pc_write(mesh_coarse.geom, mesh_coarse.dmap, mesh_coarse.ba, + mesh_coarse.ref_ratio); + pc_write.SetVerbose(false); + pc_write.InitRandom(nppc * AMREX_D_TERM(ncells, *ncells, *ncells), + iseed, pdata, /*serialize=*/false); +#ifdef AMREX_USE_HDF5 + pc_write.CheckpointHDF5(chkdir, "particles", true, real_names, int_names); +#else + pc_write.Checkpoint(chkdir, "particles", real_names, int_names); +#endif - ref_ratio.resize(nlevs-1); - for (int lev = 1; lev < nlevs; lev++) { - ref_ratio[lev-1] = IntVect(AMREX_D_DECL(2, 2, 2)); - } + ParallelDescriptor::Barrier(); - grids.resize(nlevs); - grids[0].define(domains[0]); + // --- restart: finer decomposition (many smaller boxes) --- + auto mesh_fine = build_mesh(ncells, 1, ncells / 4); + MyPC pc_read(mesh_fine.geom, mesh_fine.dmap, mesh_fine.ba, + mesh_fine.ref_ratio); + pc_read.SetVerbose(false); +#ifdef AMREX_USE_HDF5 + pc_read.RestartHDF5(chkdir + "/particles", "particles"); +#else + pc_read.Restart(chkdir, "particles"); +#endif - // Now we make the refined level be the center eighth of the domain - if (nlevs > 1) { - int n_fine = ncells*ref_ratio[0][0]; - IntVect refined_lo(AMREX_D_DECL(n_fine/4,n_fine/4,n_fine/4)); - IntVect refined_hi(AMREX_D_DECL(3*n_fine/4-1,3*n_fine/4-1,3*n_fine/4-1)); + verify_same(pc_write, pc_read); - // Build a box for the level 1 domain - Box refined_patch(refined_lo, refined_hi); - grids[1].define(refined_patch); - } + amrex::Print() << " PASSED\n"; +} - // break the BoxArrays at both levels into max_grid_size^3 boxes - for (int lev = 0; lev < nlevs; lev++) { - grids[lev].maxSize(max_grid_size); - } +int main (int argc, char* argv[]) +{ + amrex::Initialize(argc, argv); - for (int lev = 1; lev < nlevs; lev++) { - domains[lev] = amrex::refine(domains[lev-1], ref_ratio[lev-1]); - } + test_same_hierarchy(); + test_fewer_levels(); + test_more_levels(); + test_different_boxarrays(); + + amrex::Print() << "All dual-grid restart tests PASSED\n"; + + amrex::Finalize(); } diff --git a/Tests/Particles/CheckpointRestartDualGrid/CMakeLists.txt b/Tests/Particles/CheckpointRestartDualGrid/CMakeLists.txt deleted file mode 100644 index 6e36bd1f8a2..00000000000 --- a/Tests/Particles/CheckpointRestartDualGrid/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# This test requires particle support -if (NOT AMReX_PARTICLES) - return() -endif () - -foreach(D IN LISTS AMReX_SPACEDIM) - set(_sources main.cpp) - - setup_test(${D} _sources _input_files) - - unset(_sources) -endforeach() diff --git a/Tests/Particles/CheckpointRestartDualGrid/GNUmakefile b/Tests/Particles/CheckpointRestartDualGrid/GNUmakefile deleted file mode 100644 index d31fe5d711b..00000000000 --- a/Tests/Particles/CheckpointRestartDualGrid/GNUmakefile +++ /dev/null @@ -1,34 +0,0 @@ -AMREX_HOME = ../../../ - -DEBUG = FALSE - -DIM = 3 - -COMP = gnu - -PRECISION = DOUBLE - -USE_MPI = TRUE -MPI_THREAD_MULTIPLE = FALSE - -USE_OMP = FALSE - -TINY_PROFILE = TRUE - -USE_PARTICLES = TRUE - -USE_HDF5 = FALSE -# Set HDF5_HOME to the root of your HDF5 installation, e.g.: -# HDF5_HOME = /path/to/hdf5 - -################################################### - -EBASE = main - -include $(AMREX_HOME)/Tools/GNUMake/Make.defs - -include ./Make.package -include $(AMREX_HOME)/Src/Base/Make.package -include $(AMREX_HOME)/Src/Particle/Make.package - -include $(AMREX_HOME)/Tools/GNUMake/Make.rules diff --git a/Tests/Particles/CheckpointRestartDualGrid/Make.package b/Tests/Particles/CheckpointRestartDualGrid/Make.package deleted file mode 100644 index 6b4b865e8fc..00000000000 --- a/Tests/Particles/CheckpointRestartDualGrid/Make.package +++ /dev/null @@ -1 +0,0 @@ -CEXE_sources += main.cpp diff --git a/Tests/Particles/CheckpointRestartDualGrid/main.cpp b/Tests/Particles/CheckpointRestartDualGrid/main.cpp deleted file mode 100644 index bba19fc0dec..00000000000 --- a/Tests/Particles/CheckpointRestartDualGrid/main.cpp +++ /dev/null @@ -1,325 +0,0 @@ -/** - * Test for dual-grid particle checkpoint/restart capability.. - * - * Verifies that particles are preserved correctly when reading a - * checkpoint file into a ParticleContainer with a different mesh structure, - * covering: - * 1. Restart with fewer AMR levels than the checkpoint - * 2. Restart with more AMR levels than the checkpoint - * 3. Restart with the same number of levels but different BoxArrays - */ -#include -#include - -using namespace amrex; - -// Particle type: 4 struct reals, 1 struct int, 2 array reals, 1 array int -constexpr int NStructReal = 4; -constexpr int NStructInt = 1; -constexpr int NArrayReal = 2; -constexpr int NArrayInt = 1; - -using MyPC = ParticleContainer; -using PType = typename MyPC::SuperParticleType; - -struct MeshData { - Vector domains; - Vector ba; - Vector ref_ratio; - Vector geom; - Vector dmap; -}; - -/** - * Build a nested AMR hierarchy with `nlevs` levels over a [0,1]^d domain - * discretised by `ncells` cells in each direction. The coarse BoxArray is - * decomposed into boxes of at most `max_grid_size` cells; the fine level - * covers the central half of the domain. - */ -MeshData build_mesh (int ncells, int nlevs, int max_grid_size) -{ - AMREX_ALWAYS_ASSERT(nlevs >= 1 && nlevs <= 2); - - MeshData m; - - // Level-0 domain - IntVect lo(AMREX_D_DECL(0, 0, 0)); - IntVect hi(AMREX_D_DECL(ncells-1, ncells-1, ncells-1)); - - m.domains.resize(nlevs); - m.domains[0].setSmall(lo); - m.domains[0].setBig(hi); - - m.ref_ratio.resize(nlevs > 1 ? nlevs - 1 : 0); - for (int lev = 1; lev < nlevs; ++lev) { - m.ref_ratio[lev-1] = IntVect(AMREX_D_DECL(2, 2, 2)); - m.domains[lev] = amrex::refine(m.domains[lev-1], m.ref_ratio[lev-1]); - } - - m.ba.resize(nlevs); - m.ba[0].define(m.domains[0]); - m.ba[0].maxSize(max_grid_size); - - if (nlevs > 1) { - // Refined region: the central 1/4 of the fine-level domain - int n_fine = ncells * 2; // ref_ratio == 2 - IntVect rlo(AMREX_D_DECL(n_fine/4, n_fine/4, n_fine/4)); - IntVect rhi(AMREX_D_DECL(3*n_fine/4-1, 3*n_fine/4-1, 3*n_fine/4-1)); - m.ba[1].define(Box(rlo, rhi)); - m.ba[1].maxSize(max_grid_size); - } - - RealBox real_box; - for (int n = 0; n < AMREX_SPACEDIM; ++n) { - real_box.setLo(n, 0.0); - real_box.setHi(n, 1.0); - } - int is_per[] = {AMREX_D_DECL(1, 1, 1)}; - - m.geom.resize(nlevs); - m.geom[0].define(m.domains[0], &real_box, CoordSys::cartesian, is_per); - for (int lev = 1; lev < nlevs; ++lev) { - m.geom[lev].define(m.domains[lev], &real_box, CoordSys::cartesian, is_per); - } - - m.dmap.resize(nlevs); - for (int lev = 0; lev < nlevs; ++lev) { - m.dmap[lev] = DistributionMapping{m.ba[lev]}; - } - - return m; -} - -/** - * Assert that two ParticleContainers hold identical particle data by - * comparing the total particle count and the component-wise sums of every - * real and integer attribute across all levels. - */ -void verify_same (MyPC& pc_orig, MyPC& pc_new) -{ - auto n_orig = pc_orig.TotalNumberOfParticles(); - auto n_new = pc_new.TotalNumberOfParticles(); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - n_orig == n_new, - "Particle count mismatch after restart"); - - for (int icomp = 0; icomp < NStructReal + NArrayReal; ++icomp) { - auto sm_orig = amrex::ReduceSum(pc_orig, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> Real { - return p.rdata(icomp); - }); - auto sm_new = amrex::ReduceSum(pc_new, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> Real { - return p.rdata(icomp); - }); - ParallelDescriptor::ReduceRealSum(sm_orig); - ParallelDescriptor::ReduceRealSum(sm_new); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - sm_orig == sm_new, - "Real component sum mismatch after restart (comp " + std::to_string(icomp) + ")"); - } - - for (int icomp = 0; icomp < NStructInt + NArrayInt; ++icomp) { - auto sm_orig = amrex::ReduceSum(pc_orig, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> Real { - return static_cast(p.idata(icomp)); - }); - auto sm_new = amrex::ReduceSum(pc_new, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> Real { - return static_cast(p.idata(icomp)); - }); - ParallelDescriptor::ReduceRealSum(sm_orig); - ParallelDescriptor::ReduceRealSum(sm_new); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - sm_orig == sm_new, - "Int component sum mismatch after restart (comp " + std::to_string(icomp) + ")"); - } -} - -/** - * Test 1: Write a 2-level checkpoint, restart into a 1-level container. - * Particles that resided on the fine level must be redistributed to the - * coarse level; totals and component sums must be unchanged. - */ -void test_fewer_levels () -{ - amrex::Print() << "Test 1: restart with fewer levels than checkpoint\n"; - - const int ncells = 32; - const int max_grid_size = 16; - const int nppc = 2; - const int iseed = 451; - const std::string chkdir = "chk_fewer_levels"; - - MyPC::ParticleInitData pdata = {{1.0, 2.0, 3.0, 4.0}, - {5}, - {6.0, 7.0}, - {8}}; - - Vector real_names, int_names; - for (int i = 0; i < NStructReal + NArrayReal; ++i) { - real_names.push_back("real_" + std::to_string(i)); - } - for (int i = 0; i < NStructInt + NArrayInt; ++i) { - int_names.push_back("int_" + std::to_string(i)); - } - - // --- write with 2 levels --- - auto mesh2 = build_mesh(ncells, 2, max_grid_size); - MyPC pc_write(mesh2.geom, mesh2.dmap, mesh2.ba, mesh2.ref_ratio); - pc_write.SetVerbose(false); - pc_write.InitRandom(nppc * AMREX_D_TERM(ncells, *ncells, *ncells), - iseed, pdata, /*serialize=*/false); -#ifdef AMREX_USE_HDF5 - pc_write.CheckpointHDF5(chkdir, "particles", true, real_names, int_names); -#else - pc_write.Checkpoint(chkdir, "particles", real_names, int_names); -#endif - - ParallelDescriptor::Barrier(); - - // --- restart with 1 level --- - auto mesh1 = build_mesh(ncells, 1, max_grid_size); - MyPC pc_read(mesh1.geom, mesh1.dmap, mesh1.ba, mesh1.ref_ratio); - pc_read.SetVerbose(false); -#ifdef AMREX_USE_HDF5 - pc_read.RestartHDF5(chkdir + "/particles", "particles"); -#else - pc_read.Restart(chkdir, "particles"); -#endif - - verify_same(pc_write, pc_read); - - amrex::Print() << " PASSED\n"; -} - -/** - * Test 2: Write a 1-level checkpoint, restart into a 2-level container. - * After restart, Redistribute() assigns particles inside the refined - * region to the finer level; totals and component sums must be unchanged. - */ -void test_more_levels () -{ - amrex::Print() << "Test 2: restart with more levels than checkpoint\n"; - - const int ncells = 32; - const int max_grid_size = 16; - const int nppc = 2; - const int iseed = 451; - const std::string chkdir = "chk_more_levels"; - - MyPC::ParticleInitData pdata = {{1.0, 2.0, 3.0, 4.0}, - {5}, - {6.0, 7.0}, - {8}}; - - Vector real_names, int_names; - for (int i = 0; i < NStructReal + NArrayReal; ++i) { - real_names.push_back("real_" + std::to_string(i)); - } - for (int i = 0; i < NStructInt + NArrayInt; ++i) { - int_names.push_back("int_" + std::to_string(i)); - } - - // --- write with 1 level --- - auto mesh1 = build_mesh(ncells, 1, max_grid_size); - MyPC pc_write(mesh1.geom, mesh1.dmap, mesh1.ba, mesh1.ref_ratio); - pc_write.SetVerbose(false); - pc_write.InitRandom(nppc * AMREX_D_TERM(ncells, *ncells, *ncells), - iseed, pdata, /*serialize=*/false); -#ifdef AMREX_USE_HDF5 - pc_write.CheckpointHDF5(chkdir, "particles", true, real_names, int_names); -#else - pc_write.Checkpoint(chkdir, "particles", real_names, int_names); -#endif - - ParallelDescriptor::Barrier(); - - // --- restart with 2 levels --- - auto mesh2 = build_mesh(ncells, 2, max_grid_size); - MyPC pc_read(mesh2.geom, mesh2.dmap, mesh2.ba, mesh2.ref_ratio); - pc_read.SetVerbose(false); -#ifdef AMREX_USE_HDF5 - pc_read.RestartHDF5(chkdir + "/particles", "particles"); -#else - pc_read.Restart(chkdir, "particles"); -#endif - - verify_same(pc_write, pc_read); - - amrex::Print() << " PASSED\n"; -} - -/** - * Test 3: Write a checkpoint with one BoxArray decomposition, restart - * into a container with the same number of levels but a different BoxArray - * (different max_grid_size). This is the canonical dual-grid scenario: the - * Particle_H header written at checkpoint time differs from the BoxArray - * of the new container, so AMReX uses temporary grids while reading and - * calls Redistribute() to move particles onto the new decomposition. - */ -void test_different_boxarrays () -{ - amrex::Print() << "Test 3: restart with same levels but different BoxArrays\n"; - - const int ncells = 32; - const int nppc = 2; - const int iseed = 451; - const std::string chkdir = "chk_different_ba"; - - MyPC::ParticleInitData pdata = {{1.0, 2.0, 3.0, 4.0}, - {5}, - {6.0, 7.0}, - {8}}; - - Vector real_names, int_names; - for (int i = 0; i < NStructReal + NArrayReal; ++i) { - real_names.push_back("real_" + std::to_string(i)); - } - for (int i = 0; i < NStructInt + NArrayInt; ++i) { - int_names.push_back("int_" + std::to_string(i)); - } - - // --- write: coarse decomposition (few large boxes) --- - auto mesh_coarse = build_mesh(ncells, 1, ncells); // one box per rank at most - MyPC pc_write(mesh_coarse.geom, mesh_coarse.dmap, mesh_coarse.ba, - mesh_coarse.ref_ratio); - pc_write.SetVerbose(false); - pc_write.InitRandom(nppc * AMREX_D_TERM(ncells, *ncells, *ncells), - iseed, pdata, /*serialize=*/false); -#ifdef AMREX_USE_HDF5 - pc_write.CheckpointHDF5(chkdir, "particles", true, real_names, int_names); -#else - pc_write.Checkpoint(chkdir, "particles", real_names, int_names); -#endif - - ParallelDescriptor::Barrier(); - - // --- restart: finer decomposition (many smaller boxes) --- - auto mesh_fine = build_mesh(ncells, 1, ncells / 4); - MyPC pc_read(mesh_fine.geom, mesh_fine.dmap, mesh_fine.ba, - mesh_fine.ref_ratio); - pc_read.SetVerbose(false); -#ifdef AMREX_USE_HDF5 - pc_read.RestartHDF5(chkdir + "/particles", "particles"); -#else - pc_read.Restart(chkdir, "particles"); -#endif - - verify_same(pc_write, pc_read); - - amrex::Print() << " PASSED\n"; -} - -int main (int argc, char* argv[]) -{ - amrex::Initialize(argc, argv); - - test_fewer_levels(); - test_more_levels(); - test_different_boxarrays(); - - amrex::Print() << "All dual-grid restart tests PASSED\n"; - - amrex::Finalize(); -} diff --git a/Tests/Particles/CheckpointRestartDualGridHDF5SOA/CMakeLists.txt b/Tests/Particles/CheckpointRestartDualGridHDF5SOA/CMakeLists.txt deleted file mode 100644 index 6e36bd1f8a2..00000000000 --- a/Tests/Particles/CheckpointRestartDualGridHDF5SOA/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# This test requires particle support -if (NOT AMReX_PARTICLES) - return() -endif () - -foreach(D IN LISTS AMReX_SPACEDIM) - set(_sources main.cpp) - - setup_test(${D} _sources _input_files) - - unset(_sources) -endforeach() diff --git a/Tests/Particles/CheckpointRestartDualGridHDF5SOA/GNUmakefile b/Tests/Particles/CheckpointRestartDualGridHDF5SOA/GNUmakefile deleted file mode 100644 index d31fe5d711b..00000000000 --- a/Tests/Particles/CheckpointRestartDualGridHDF5SOA/GNUmakefile +++ /dev/null @@ -1,34 +0,0 @@ -AMREX_HOME = ../../../ - -DEBUG = FALSE - -DIM = 3 - -COMP = gnu - -PRECISION = DOUBLE - -USE_MPI = TRUE -MPI_THREAD_MULTIPLE = FALSE - -USE_OMP = FALSE - -TINY_PROFILE = TRUE - -USE_PARTICLES = TRUE - -USE_HDF5 = FALSE -# Set HDF5_HOME to the root of your HDF5 installation, e.g.: -# HDF5_HOME = /path/to/hdf5 - -################################################### - -EBASE = main - -include $(AMREX_HOME)/Tools/GNUMake/Make.defs - -include ./Make.package -include $(AMREX_HOME)/Src/Base/Make.package -include $(AMREX_HOME)/Src/Particle/Make.package - -include $(AMREX_HOME)/Tools/GNUMake/Make.rules diff --git a/Tests/Particles/CheckpointRestartDualGridHDF5SOA/Make.package b/Tests/Particles/CheckpointRestartDualGridHDF5SOA/Make.package deleted file mode 100644 index 6b4b865e8fc..00000000000 --- a/Tests/Particles/CheckpointRestartDualGridHDF5SOA/Make.package +++ /dev/null @@ -1 +0,0 @@ -CEXE_sources += main.cpp diff --git a/Tests/Particles/CheckpointRestartDualGridHDF5SOA/main.cpp b/Tests/Particles/CheckpointRestartDualGridHDF5SOA/main.cpp deleted file mode 100644 index 320a6dd3d56..00000000000 --- a/Tests/Particles/CheckpointRestartDualGridHDF5SOA/main.cpp +++ /dev/null @@ -1,320 +0,0 @@ -/** - * Test for dual-grid particle checkpoint/restart capability using pure SoA - * particles and HDF5 I/O. - * - * Verifies that particles are preserved correctly when reading a - * checkpoint file into a ParticleContainerPureSoA with a different mesh structure, - * covering: - * 1. Restart with fewer AMR levels than the checkpoint - * 2. Restart with more AMR levels than the checkpoint - * 3. Restart with the same number of levels but different BoxArrays - */ -#include -#include - -using namespace amrex; - -// Pure SoA particle type: 6 real components (including positions), 2 int components -constexpr int NReal = 6; -constexpr int NInt = 2; - -using MyPC = ParticleContainerPureSoA; -using ConstPTDType = typename MyPC::ConstPTDType; - -struct MeshData { - Vector domains; - Vector ba; - Vector ref_ratio; - Vector geom; - Vector dmap; -}; - -/** - * Build a nested AMR hierarchy with `nlevs` levels over a [0,1]^d domain - * discretised by `ncells` cells in each direction. The coarse BoxArray is - * decomposed into boxes of at most `max_grid_size` cells; the fine level - * covers the central half of the domain. - */ -MeshData build_mesh (int ncells, int nlevs, int max_grid_size) -{ - AMREX_ALWAYS_ASSERT(nlevs >= 1 && nlevs <= 2); - - MeshData m; - - // Level-0 domain - IntVect lo(AMREX_D_DECL(0, 0, 0)); - IntVect hi(AMREX_D_DECL(ncells-1, ncells-1, ncells-1)); - - m.domains.resize(nlevs); - m.domains[0].setSmall(lo); - m.domains[0].setBig(hi); - - m.ref_ratio.resize(nlevs > 1 ? nlevs - 1 : 0); - for (int lev = 1; lev < nlevs; ++lev) { - m.ref_ratio[lev-1] = IntVect(AMREX_D_DECL(2, 2, 2)); - m.domains[lev] = amrex::refine(m.domains[lev-1], m.ref_ratio[lev-1]); - } - - m.ba.resize(nlevs); - m.ba[0].define(m.domains[0]); - m.ba[0].maxSize(max_grid_size); - - if (nlevs > 1) { - // Refined region: the central 1/4 of the fine-level domain - int n_fine = ncells * 2; // ref_ratio == 2 - IntVect rlo(AMREX_D_DECL(n_fine/4, n_fine/4, n_fine/4)); - IntVect rhi(AMREX_D_DECL(3*n_fine/4-1, 3*n_fine/4-1, 3*n_fine/4-1)); - m.ba[1].define(Box(rlo, rhi)); - m.ba[1].maxSize(max_grid_size); - } - - RealBox real_box; - for (int n = 0; n < AMREX_SPACEDIM; ++n) { - real_box.setLo(n, 0.0); - real_box.setHi(n, 1.0); - } - int is_per[] = {AMREX_D_DECL(1, 1, 1)}; - - m.geom.resize(nlevs); - m.geom[0].define(m.domains[0], &real_box, CoordSys::cartesian, is_per); - for (int lev = 1; lev < nlevs; ++lev) { - m.geom[lev].define(m.domains[lev], &real_box, CoordSys::cartesian, is_per); - } - - m.dmap.resize(nlevs); - for (int lev = 0; lev < nlevs; ++lev) { - m.dmap[lev] = DistributionMapping{m.ba[lev]}; - } - - return m; -} - -/** - * Assert that two ParticleContainerPureSoA hold identical particle data by - * comparing the total particle count and the component-wise sums of every - * real and integer attribute across all levels. - */ -void verify_same (MyPC& pc_orig, MyPC& pc_new) -{ - auto n_orig = pc_orig.TotalNumberOfParticles(); - auto n_new = pc_new.TotalNumberOfParticles(); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - n_orig == n_new, - "Particle count mismatch after restart"); - - // Skip position components (0..AMREX_SPACEDIM-1): they hold random values, - // and floating-point summation order may differ across level configurations. - for (int icomp = AMREX_SPACEDIM; icomp < NReal; ++icomp) { - auto sm_orig = amrex::ReduceSum(pc_orig, - [=] AMREX_GPU_HOST_DEVICE (const ConstPTDType& ptd, const int i) -> Real { - return ptd.rdata(icomp)[i]; - }); - auto sm_new = amrex::ReduceSum(pc_new, - [=] AMREX_GPU_HOST_DEVICE (const ConstPTDType& ptd, const int i) -> Real { - return ptd.rdata(icomp)[i]; - }); - ParallelDescriptor::ReduceRealSum(sm_orig); - ParallelDescriptor::ReduceRealSum(sm_new); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - sm_orig == sm_new, - "Real component sum mismatch after restart (comp " + std::to_string(icomp) + ")"); - } - - for (int icomp = 0; icomp < NInt; ++icomp) { - auto sm_orig = amrex::ReduceSum(pc_orig, - [=] AMREX_GPU_HOST_DEVICE (const ConstPTDType& ptd, const int i) -> Real { - return static_cast(ptd.idata(icomp)[i]); - }); - auto sm_new = amrex::ReduceSum(pc_new, - [=] AMREX_GPU_HOST_DEVICE (const ConstPTDType& ptd, const int i) -> Real { - return static_cast(ptd.idata(icomp)[i]); - }); - ParallelDescriptor::ReduceRealSum(sm_orig); - ParallelDescriptor::ReduceRealSum(sm_new); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - sm_orig == sm_new, - "Int component sum mismatch after restart (comp " + std::to_string(icomp) + ")"); - } -} - -/** - * Test 1: Write a 2-level HDF5 checkpoint, restart into a 1-level container. - * Particles that resided on the fine level must be redistributed to the - * coarse level; totals and component sums must be unchanged. - */ -void test_fewer_levels () -{ - amrex::Print() << "Test 1: restart with fewer levels than checkpoint\n"; - - const int ncells = 32; - const int max_grid_size = 16; - const int nppc = 2; - const int iseed = 451; - const std::string chkdir = "chk_fewer_levels"; - - MyPC::ParticleInitData pdata = {{}, {}, {1.0, 2.0, 3.0, 4.0, 6.0, 7.0}, {5, 8}}; - - Vector real_names, int_names; - for (int i = 0; i < NReal - AMREX_SPACEDIM; ++i) { - real_names.push_back("real_" + std::to_string(i)); - } - for (int i = 0; i < NInt; ++i) { - int_names.push_back("int_" + std::to_string(i)); - } - - // --- write with 2 levels --- - auto mesh2 = build_mesh(ncells, 2, max_grid_size); - MyPC pc_write(mesh2.geom, mesh2.dmap, mesh2.ba, mesh2.ref_ratio); - pc_write.SetVerbose(false); - pc_write.InitRandom(nppc * AMREX_D_TERM(ncells, *ncells, *ncells), - iseed, pdata, /*serialize=*/false); -#ifdef AMREX_USE_HDF5 - pc_write.CheckpointHDF5(chkdir, "particles", true, real_names, int_names); -#else - pc_write.Checkpoint(chkdir, "particles", real_names, int_names); -#endif - - AsyncOut::Finish(); - ParallelDescriptor::Barrier(); - - // --- restart with 1 level --- - auto mesh1 = build_mesh(ncells, 1, max_grid_size); - MyPC pc_read(mesh1.geom, mesh1.dmap, mesh1.ba, mesh1.ref_ratio); - pc_read.SetVerbose(false); -#ifdef AMREX_USE_HDF5 - pc_read.RestartHDF5(chkdir + "/particles", "particles"); -#else - pc_read.Restart(chkdir, "particles"); -#endif - - verify_same(pc_write, pc_read); - - amrex::Print() << " PASSED\n"; -} - -/** - * Test 2: Write a 1-level HDF5 checkpoint, restart into a 2-level container. - * After restart, Redistribute() assigns particles inside the refined - * region to the finer level; totals and component sums must be unchanged. - */ -void test_more_levels () -{ - amrex::Print() << "Test 2: restart with more levels than checkpoint\n"; - - const int ncells = 32; - const int max_grid_size = 16; - const int nppc = 2; - const int iseed = 451; - const std::string chkdir = "chk_more_levels"; - - MyPC::ParticleInitData pdata = {{}, {}, {1.0, 2.0, 3.0, 4.0, 6.0, 7.0}, {5, 8}}; - - Vector real_names, int_names; - for (int i = 0; i < NReal - AMREX_SPACEDIM; ++i) { - real_names.push_back("real_" + std::to_string(i)); - } - for (int i = 0; i < NInt; ++i) { - int_names.push_back("int_" + std::to_string(i)); - } - - // --- write with 1 level --- - auto mesh1 = build_mesh(ncells, 1, max_grid_size); - MyPC pc_write(mesh1.geom, mesh1.dmap, mesh1.ba, mesh1.ref_ratio); - pc_write.SetVerbose(false); - pc_write.InitRandom(nppc * AMREX_D_TERM(ncells, *ncells, *ncells), - iseed, pdata, /*serialize=*/false); -#ifdef AMREX_USE_HDF5 - pc_write.CheckpointHDF5(chkdir, "particles", true, real_names, int_names); -#else - pc_write.Checkpoint(chkdir, "particles", real_names, int_names); -#endif - - AsyncOut::Finish(); - ParallelDescriptor::Barrier(); - - // --- restart with 2 levels --- - auto mesh2 = build_mesh(ncells, 2, max_grid_size); - MyPC pc_read(mesh2.geom, mesh2.dmap, mesh2.ba, mesh2.ref_ratio); - pc_read.SetVerbose(false); -#ifdef AMREX_USE_HDF5 - pc_read.RestartHDF5(chkdir + "/particles", "particles"); -#else - pc_read.Restart(chkdir, "particles"); -#endif - - verify_same(pc_write, pc_read); - - amrex::Print() << " PASSED\n"; -} - -/** - * Test 3: Write an HDF5 checkpoint with one BoxArray decomposition, restart - * into a container with the same number of levels but a different BoxArray - * (different max_grid_size). This is the canonical dual-grid scenario: the - * Particle_H header written at checkpoint time differs from the BoxArray - * of the new container, so AMReX uses temporary grids while reading and - * calls Redistribute() to move particles onto the new decomposition. - */ -void test_different_boxarrays () -{ - amrex::Print() << "Test 3: restart with same levels but different BoxArrays\n"; - - const int ncells = 32; - const int nppc = 2; - const int iseed = 451; - const std::string chkdir = "chk_different_ba"; - - MyPC::ParticleInitData pdata = {{}, {}, {1.0, 2.0, 3.0, 4.0, 6.0, 7.0}, {5, 8}}; - - Vector real_names, int_names; - for (int i = 0; i < NReal - AMREX_SPACEDIM; ++i) { - real_names.push_back("real_" + std::to_string(i)); - } - for (int i = 0; i < NInt; ++i) { - int_names.push_back("int_" + std::to_string(i)); - } - - // --- write: coarse decomposition (few large boxes) --- - auto mesh_coarse = build_mesh(ncells, 1, ncells); // one box per rank at most - MyPC pc_write(mesh_coarse.geom, mesh_coarse.dmap, mesh_coarse.ba, - mesh_coarse.ref_ratio); - pc_write.SetVerbose(false); - pc_write.InitRandom(nppc * AMREX_D_TERM(ncells, *ncells, *ncells), - iseed, pdata, /*serialize=*/false); -#ifdef AMREX_USE_HDF5 - pc_write.CheckpointHDF5(chkdir, "particles", true, real_names, int_names); -#else - pc_write.Checkpoint(chkdir, "particles", real_names, int_names); -#endif - - AsyncOut::Finish(); - ParallelDescriptor::Barrier(); - - // --- restart: finer decomposition (many smaller boxes) --- - auto mesh_fine = build_mesh(ncells, 1, ncells / 4); - MyPC pc_read(mesh_fine.geom, mesh_fine.dmap, mesh_fine.ba, - mesh_fine.ref_ratio); - pc_read.SetVerbose(false); -#ifdef AMREX_USE_HDF5 - pc_read.RestartHDF5(chkdir + "/particles", "particles"); -#else - pc_read.Restart(chkdir, "particles"); -#endif - - verify_same(pc_write, pc_read); - - amrex::Print() << " PASSED\n"; -} - -int main (int argc, char* argv[]) -{ - amrex::Initialize(argc, argv); - - test_fewer_levels(); - test_more_levels(); - test_different_boxarrays(); - - amrex::Print() << "All dual-grid HDF5 SOA restart tests PASSED\n"; - - amrex::Finalize(); -} diff --git a/Tests/Particles/CheckpointRestartDualGridSOA/CMakeLists.txt b/Tests/Particles/CheckpointRestartDualGridSOA/CMakeLists.txt deleted file mode 100644 index 6e36bd1f8a2..00000000000 --- a/Tests/Particles/CheckpointRestartDualGridSOA/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# This test requires particle support -if (NOT AMReX_PARTICLES) - return() -endif () - -foreach(D IN LISTS AMReX_SPACEDIM) - set(_sources main.cpp) - - setup_test(${D} _sources _input_files) - - unset(_sources) -endforeach() diff --git a/Tests/Particles/CheckpointRestartDualGridSOA/GNUmakefile b/Tests/Particles/CheckpointRestartDualGridSOA/GNUmakefile deleted file mode 100644 index dbe9917add7..00000000000 --- a/Tests/Particles/CheckpointRestartDualGridSOA/GNUmakefile +++ /dev/null @@ -1,30 +0,0 @@ -AMREX_HOME = ../../../ - -DEBUG = FALSE - -DIM = 3 - -COMP = gnu - -PRECISION = DOUBLE - -USE_MPI = TRUE -MPI_THREAD_MULTIPLE = FALSE - -USE_OMP = FALSE - -TINY_PROFILE = TRUE - -USE_PARTICLES = TRUE - -################################################### - -EBASE = main - -include $(AMREX_HOME)/Tools/GNUMake/Make.defs - -include ./Make.package -include $(AMREX_HOME)/Src/Base/Make.package -include $(AMREX_HOME)/Src/Particle/Make.package - -include $(AMREX_HOME)/Tools/GNUMake/Make.rules diff --git a/Tests/Particles/CheckpointRestartDualGridSOA/Make.package b/Tests/Particles/CheckpointRestartDualGridSOA/Make.package deleted file mode 100644 index 6b4b865e8fc..00000000000 --- a/Tests/Particles/CheckpointRestartDualGridSOA/Make.package +++ /dev/null @@ -1 +0,0 @@ -CEXE_sources += main.cpp diff --git a/Tests/Particles/CheckpointRestartDualGridSOA/main.cpp b/Tests/Particles/CheckpointRestartDualGridSOA/main.cpp deleted file mode 100644 index 2fb1ce17e62..00000000000 --- a/Tests/Particles/CheckpointRestartDualGridSOA/main.cpp +++ /dev/null @@ -1,295 +0,0 @@ -/** - * Test for dual-grid particle checkpoint/restart capability using pure SoA particles. - * - * Verifies that particles are preserved correctly when reading a - * checkpoint file into a ParticleContainerPureSoA with a different mesh structure, - * covering: - * 1. Restart with fewer AMR levels than the checkpoint - * 2. Restart with more AMR levels than the checkpoint - * 3. Restart with the same number of levels but different BoxArrays - */ -#include -#include - -using namespace amrex; - -// Pure SoA particle type: 6 real components (including positions), 2 int components -constexpr int NReal = 6; -constexpr int NInt = 2; - -using MyPC = ParticleContainerPureSoA; -using ConstPTDType = typename MyPC::ConstPTDType; - -struct MeshData { - Vector domains; - Vector ba; - Vector ref_ratio; - Vector geom; - Vector dmap; -}; - -/** - * Build a nested AMR hierarchy with `nlevs` levels over a [0,1]^d domain - * discretised by `ncells` cells in each direction. The coarse BoxArray is - * decomposed into boxes of at most `max_grid_size` cells; the fine level - * covers the central half of the domain. - */ -MeshData build_mesh (int ncells, int nlevs, int max_grid_size) -{ - AMREX_ALWAYS_ASSERT(nlevs >= 1 && nlevs <= 2); - - MeshData m; - - // Level-0 domain - IntVect lo(AMREX_D_DECL(0, 0, 0)); - IntVect hi(AMREX_D_DECL(ncells-1, ncells-1, ncells-1)); - - m.domains.resize(nlevs); - m.domains[0].setSmall(lo); - m.domains[0].setBig(hi); - - m.ref_ratio.resize(nlevs > 1 ? nlevs - 1 : 0); - for (int lev = 1; lev < nlevs; ++lev) { - m.ref_ratio[lev-1] = IntVect(AMREX_D_DECL(2, 2, 2)); - m.domains[lev] = amrex::refine(m.domains[lev-1], m.ref_ratio[lev-1]); - } - - m.ba.resize(nlevs); - m.ba[0].define(m.domains[0]); - m.ba[0].maxSize(max_grid_size); - - if (nlevs > 1) { - // Refined region: the central 1/4 of the fine-level domain - int n_fine = ncells * 2; // ref_ratio == 2 - IntVect rlo(AMREX_D_DECL(n_fine/4, n_fine/4, n_fine/4)); - IntVect rhi(AMREX_D_DECL(3*n_fine/4-1, 3*n_fine/4-1, 3*n_fine/4-1)); - m.ba[1].define(Box(rlo, rhi)); - m.ba[1].maxSize(max_grid_size); - } - - RealBox real_box; - for (int n = 0; n < AMREX_SPACEDIM; ++n) { - real_box.setLo(n, 0.0); - real_box.setHi(n, 1.0); - } - int is_per[] = {AMREX_D_DECL(1, 1, 1)}; - - m.geom.resize(nlevs); - m.geom[0].define(m.domains[0], &real_box, CoordSys::cartesian, is_per); - for (int lev = 1; lev < nlevs; ++lev) { - m.geom[lev].define(m.domains[lev], &real_box, CoordSys::cartesian, is_per); - } - - m.dmap.resize(nlevs); - for (int lev = 0; lev < nlevs; ++lev) { - m.dmap[lev] = DistributionMapping{m.ba[lev]}; - } - - return m; -} - -/** - * Assert that two ParticleContainerPureSoA hold identical particle data by - * comparing the total particle count and the component-wise sums of every - * real and integer attribute across all levels. - */ -void verify_same (MyPC& pc_orig, MyPC& pc_new) -{ - auto n_orig = pc_orig.TotalNumberOfParticles(); - auto n_new = pc_new.TotalNumberOfParticles(); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - n_orig == n_new, - "Particle count mismatch after restart"); - - // Skip position components (0..AMREX_SPACEDIM-1): they hold random values, - // and floating-point summation order may differ across level configurations. - for (int icomp = AMREX_SPACEDIM; icomp < NReal; ++icomp) { - auto sm_orig = amrex::ReduceSum(pc_orig, - [=] AMREX_GPU_HOST_DEVICE (const ConstPTDType& ptd, const int i) -> Real { - return ptd.rdata(icomp)[i]; - }); - auto sm_new = amrex::ReduceSum(pc_new, - [=] AMREX_GPU_HOST_DEVICE (const ConstPTDType& ptd, const int i) -> Real { - return ptd.rdata(icomp)[i]; - }); - ParallelDescriptor::ReduceRealSum(sm_orig); - ParallelDescriptor::ReduceRealSum(sm_new); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - sm_orig == sm_new, - "Real component sum mismatch after restart (comp " + std::to_string(icomp) + ")"); - } - - for (int icomp = 0; icomp < NInt; ++icomp) { - auto sm_orig = amrex::ReduceSum(pc_orig, - [=] AMREX_GPU_HOST_DEVICE (const ConstPTDType& ptd, const int i) -> Real { - return static_cast(ptd.idata(icomp)[i]); - }); - auto sm_new = amrex::ReduceSum(pc_new, - [=] AMREX_GPU_HOST_DEVICE (const ConstPTDType& ptd, const int i) -> Real { - return static_cast(ptd.idata(icomp)[i]); - }); - ParallelDescriptor::ReduceRealSum(sm_orig); - ParallelDescriptor::ReduceRealSum(sm_new); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - sm_orig == sm_new, - "Int component sum mismatch after restart (comp " + std::to_string(icomp) + ")"); - } -} - -/** - * Test 1: Write a 2-level checkpoint, restart into a 1-level container. - * Particles that resided on the fine level must be redistributed to the - * coarse level; totals and component sums must be unchanged. - */ -void test_fewer_levels () -{ - amrex::Print() << "Test 1: restart with fewer levels than checkpoint\n"; - - const int ncells = 32; - const int max_grid_size = 16; - const int nppc = 2; - const int iseed = 451; - const std::string chkdir = "chk_fewer_levels"; - - MyPC::ParticleInitData pdata = {{}, {}, {1.0, 2.0, 3.0, 4.0, 6.0, 7.0}, {5, 8}}; - - Vector real_names, int_names; - for (int i = 0; i < NReal - AMREX_SPACEDIM; ++i) { - real_names.push_back("real_" + std::to_string(i)); - } - for (int i = 0; i < NInt; ++i) { - int_names.push_back("int_" + std::to_string(i)); - } - - // --- write with 2 levels --- - auto mesh2 = build_mesh(ncells, 2, max_grid_size); - MyPC pc_write(mesh2.geom, mesh2.dmap, mesh2.ba, mesh2.ref_ratio); - pc_write.SetVerbose(false); - pc_write.InitRandom(nppc * AMREX_D_TERM(ncells, *ncells, *ncells), - iseed, pdata, /*serialize=*/false); - pc_write.Checkpoint(chkdir, "particles", real_names, int_names); - - AsyncOut::Finish(); - ParallelDescriptor::Barrier(); - - // --- restart with 1 level --- - auto mesh1 = build_mesh(ncells, 1, max_grid_size); - MyPC pc_read(mesh1.geom, mesh1.dmap, mesh1.ba, mesh1.ref_ratio); - pc_read.SetVerbose(false); - pc_read.Restart(chkdir, "particles"); - - verify_same(pc_write, pc_read); - - amrex::Print() << " PASSED\n"; -} - -/** - * Test 2: Write a 1-level checkpoint, restart into a 2-level container. - * After restart, Redistribute() assigns particles inside the refined - * region to the finer level; totals and component sums must be unchanged. - */ -void test_more_levels () -{ - amrex::Print() << "Test 2: restart with more levels than checkpoint\n"; - - const int ncells = 32; - const int max_grid_size = 16; - const int nppc = 2; - const int iseed = 451; - const std::string chkdir = "chk_more_levels"; - - MyPC::ParticleInitData pdata = {{}, {}, {1.0, 2.0, 3.0, 4.0, 6.0, 7.0}, {5, 8}}; - - Vector real_names, int_names; - for (int i = 0; i < NReal - AMREX_SPACEDIM; ++i) { - real_names.push_back("real_" + std::to_string(i)); - } - for (int i = 0; i < NInt; ++i) { - int_names.push_back("int_" + std::to_string(i)); - } - - // --- write with 1 level --- - auto mesh1 = build_mesh(ncells, 1, max_grid_size); - MyPC pc_write(mesh1.geom, mesh1.dmap, mesh1.ba, mesh1.ref_ratio); - pc_write.SetVerbose(false); - pc_write.InitRandom(nppc * AMREX_D_TERM(ncells, *ncells, *ncells), - iseed, pdata, /*serialize=*/false); - pc_write.Checkpoint(chkdir, "particles", real_names, int_names); - - AsyncOut::Finish(); - ParallelDescriptor::Barrier(); - - // --- restart with 2 levels --- - auto mesh2 = build_mesh(ncells, 2, max_grid_size); - MyPC pc_read(mesh2.geom, mesh2.dmap, mesh2.ba, mesh2.ref_ratio); - pc_read.SetVerbose(false); - pc_read.Restart(chkdir, "particles"); - - verify_same(pc_write, pc_read); - - amrex::Print() << " PASSED\n"; -} - -/** - * Test 3: Write a checkpoint with one BoxArray decomposition, restart - * into a container with the same number of levels but a different BoxArray - * (different max_grid_size). This is the canonical dual-grid scenario: the - * Particle_H header written at checkpoint time differs from the BoxArray - * of the new container, so AMReX uses temporary grids while reading and - * calls Redistribute() to move particles onto the new decomposition. - */ -void test_different_boxarrays () -{ - amrex::Print() << "Test 3: restart with same levels but different BoxArrays\n"; - - const int ncells = 32; - const int nppc = 2; - const int iseed = 451; - const std::string chkdir = "chk_different_ba"; - - MyPC::ParticleInitData pdata = {{}, {}, {1.0, 2.0, 3.0, 4.0, 6.0, 7.0}, {5, 8}}; - - Vector real_names, int_names; - for (int i = 0; i < NReal - AMREX_SPACEDIM; ++i) { - real_names.push_back("real_" + std::to_string(i)); - } - for (int i = 0; i < NInt; ++i) { - int_names.push_back("int_" + std::to_string(i)); - } - - // --- write: coarse decomposition (few large boxes) --- - auto mesh_coarse = build_mesh(ncells, 1, ncells); // one box per rank at most - MyPC pc_write(mesh_coarse.geom, mesh_coarse.dmap, mesh_coarse.ba, - mesh_coarse.ref_ratio); - pc_write.SetVerbose(false); - pc_write.InitRandom(nppc * AMREX_D_TERM(ncells, *ncells, *ncells), - iseed, pdata, /*serialize=*/false); - pc_write.Checkpoint(chkdir, "particles", real_names, int_names); - - AsyncOut::Finish(); - ParallelDescriptor::Barrier(); - - // --- restart: finer decomposition (many smaller boxes) --- - auto mesh_fine = build_mesh(ncells, 1, ncells / 4); - MyPC pc_read(mesh_fine.geom, mesh_fine.dmap, mesh_fine.ba, - mesh_fine.ref_ratio); - pc_read.SetVerbose(false); - pc_read.Restart(chkdir, "particles"); - - verify_same(pc_write, pc_read); - - amrex::Print() << " PASSED\n"; -} - -int main (int argc, char* argv[]) -{ - amrex::Initialize(argc, argv); - - test_fewer_levels(); - test_more_levels(); - test_different_boxarrays(); - - amrex::Print() << "All dual-grid SOA restart tests PASSED\n"; - - amrex::Finalize(); -} diff --git a/Tests/Particles/CheckpointRestartSOA/CMakeLists.txt b/Tests/Particles/CheckpointRestartSOA/CMakeLists.txt index 087d3ad4998..6e36bd1f8a2 100644 --- a/Tests/Particles/CheckpointRestartSOA/CMakeLists.txt +++ b/Tests/Particles/CheckpointRestartSOA/CMakeLists.txt @@ -1,14 +1,12 @@ -# This tests requires particle support +# This test requires particle support if (NOT AMReX_PARTICLES) return() endif () foreach(D IN LISTS AMReX_SPACEDIM) - set(_sources main.cpp) - set(_input_files inputs) + set(_sources main.cpp) setup_test(${D} _sources _input_files) unset(_sources) - unset(_input_files) endforeach() diff --git a/Tests/Particles/CheckpointRestartSOA/GNUmakefile b/Tests/Particles/CheckpointRestartSOA/GNUmakefile index b3510b88d2e..d31fe5d711b 100644 --- a/Tests/Particles/CheckpointRestartSOA/GNUmakefile +++ b/Tests/Particles/CheckpointRestartSOA/GNUmakefile @@ -1,6 +1,5 @@ AMREX_HOME = ../../../ -# DEBUG = TRUE DEBUG = FALSE DIM = 3 @@ -18,6 +17,10 @@ TINY_PROFILE = TRUE USE_PARTICLES = TRUE +USE_HDF5 = FALSE +# Set HDF5_HOME to the root of your HDF5 installation, e.g.: +# HDF5_HOME = /path/to/hdf5 + ################################################### EBASE = main diff --git a/Tests/Particles/CheckpointRestartSOA/inputs b/Tests/Particles/CheckpointRestartSOA/inputs deleted file mode 100644 index 81855c5947d..00000000000 --- a/Tests/Particles/CheckpointRestartSOA/inputs +++ /dev/null @@ -1,26 +0,0 @@ -# Domain size -ncells = 64 - -# Maximum allowable size of each subdomain in the problem domain; -# this is used to decompose the domain for parallel calculations. -max_grid_size = 8 - -# Number of levels -nlevs = 1 - -# Number of components in the multifabs -ncomp = 6 - -# Number of particles per cell -nppc = 2 - -# Number of plot files to write -nplotfile = 1 - -# Number of plot files to write -nparticlefile = 1 - -# Whether to check the correctness of Checkpoint / Restart -restart_check = 1 - -directory=. diff --git a/Tests/Particles/CheckpointRestartSOA/main.cpp b/Tests/Particles/CheckpointRestartSOA/main.cpp index af2ce0f7a0b..850302afca4 100644 --- a/Tests/Particles/CheckpointRestartSOA/main.cpp +++ b/Tests/Particles/CheckpointRestartSOA/main.cpp @@ -1,248 +1,436 @@ +/** + * Test for dual-grid particle checkpoint/restart capability using pure SoA + * particles and HDF5 I/O. + * + * Verifies that particles are preserved correctly when reading a + * checkpoint file into a ParticleContainerPureSoA with a different mesh structure, + * covering: + * 1. Restart with fewer AMR levels than the checkpoint + * 2. Restart with more AMR levels than the checkpoint + * 3. Restart with the same number of levels but different BoxArrays + */ #include -#include -#include #include -#include +#include +#include +#include using namespace amrex; -void set_grids_nested (Vector& domains, - Vector& grids, - Vector& ref_ratio); -void test (); +// Pure SoA particle type: 6 real components (including positions), 2 int components +constexpr int NReal = 6; +constexpr int NInt = 2; + +using MyPC = ParticleContainerPureSoA; +struct MeshData { + Vector domains; + Vector ba; + Vector ref_ratio; + Vector geom; + Vector dmap; +}; + +struct ParticleRecord { + std::uint64_t idcpu = 0; + std::array real{}; + std::array idata{}; + + bool operator< (ParticleRecord const& rhs) const + { + if (idcpu != rhs.idcpu) { return idcpu < rhs.idcpu; } + if (real != rhs.real) { return real < rhs.real; } + return idata < rhs.idata; + } + + bool operator== (ParticleRecord const& rhs) const + { + return idcpu == rhs.idcpu && real == rhs.real && idata == rhs.idata; + } +}; -int main(int argc, char* argv[]) +Vector +collect_local_records (MyPC const& pc) { - amrex::Initialize(argc,argv); - test(); - amrex::Finalize(); + Vector records; + + for (int lev = 0; lev <= pc.finestLevel(); ++lev) { + for (auto const& kv : pc.GetParticles(lev)) { + auto const& ptile = kv.second; + auto const np = ptile.numParticles(); + auto const ptd = ptile.getConstParticleTileData(); + + for (int i = 0; i < np; ++i) { + ParticleRecord rec; + rec.idcpu = ptd.idcpu(i); + for (int icomp = 0; icomp < NReal; ++icomp) { + rec.real[icomp] = ptd.rdata(icomp)[i]; + } + for (int icomp = 0; icomp < NInt; ++icomp) { + rec.idata[icomp] = ptd.idata(icomp)[i]; + } + records.push_back(rec); + } + } + } + + return records; } -void test () +Vector +gather_sorted_records (MyPC const& pc) { - const int nghost = 0; - int ncells, max_grid_size, ncomp, nlevs, nppc; - int restart_check = 0, nplotfile = 1, nparticlefile = 1; - std::string directory; + auto local_records = collect_local_records(pc); + auto const local_nbytes = static_cast(local_records.size() * sizeof(ParticleRecord)); + auto const ioproc = ParallelDescriptor::IOProcessorNumber(); + auto const recv_counts = ParallelDescriptor::Gather(local_nbytes, ioproc); + + Vector recv_offsets; + Vector recv_buffer; + + if (ParallelDescriptor::IOProcessor()) { + recv_offsets.resize(recv_counts.size(), 0); + int total_nbytes = 0; + for (int i = 0, n = static_cast(recv_counts.size()); i < n; ++i) { + recv_offsets[i] = total_nbytes; + total_nbytes += recv_counts[i]; + } + recv_buffer.resize(total_nbytes); + } + + auto const* send_ptr = local_records.empty() + ? nullptr + : reinterpret_cast(local_records.data()); + auto* recv_ptr = recv_buffer.empty() ? nullptr : recv_buffer.data(); - ParmParse pp; - pp.get("ncells", ncells); - pp.get("max_grid_size", max_grid_size); - pp.get("ncomp", ncomp); - pp.get("nlevs", nlevs); - pp.get("nppc", nppc); - pp.query("nplotfile", nplotfile); - pp.query("nparticlefile", nparticlefile); - pp.query("restart_check", restart_check); - pp.query("directory", directory); + ParallelDescriptor::Gatherv(send_ptr, local_nbytes, recv_ptr, + recv_counts, recv_offsets, ioproc); - if (!directory.empty() && directory.back() != '/') { - // Include separator if one was not provided - directory += "/"; + Vector gathered_records; + if (ParallelDescriptor::IOProcessor()) { + gathered_records.resize(recv_buffer.size() / sizeof(ParticleRecord)); + if (!recv_buffer.empty()) { + std::memcpy(gathered_records.data(), recv_buffer.data(), recv_buffer.size()); + } + std::sort(gathered_records.begin(), gathered_records.end()); } - Vector domains; - Vector ba; - Vector ref_ratio; + return gathered_records; +} + +/** + * Build a nested AMR hierarchy with `nlevs` levels over a [0,1]^d domain + * discretised by `ncells` cells in each direction. The coarse BoxArray is + * decomposed into boxes of at most `max_grid_size` cells; the fine level + * covers the central half of the domain. + */ +MeshData build_mesh (int ncells, int nlevs, int max_grid_size) +{ + AMREX_ALWAYS_ASSERT(nlevs >= 1 && nlevs <= 2); + + MeshData m; + + // Level-0 domain + IntVect lo(AMREX_D_DECL(0, 0, 0)); + IntVect hi(AMREX_D_DECL(ncells-1, ncells-1, ncells-1)); - set_grids_nested(domains, ba, ref_ratio); + m.domains.resize(nlevs); + m.domains[0].setSmall(lo); + m.domains[0].setBig(hi); + + m.ref_ratio.resize(nlevs > 1 ? nlevs - 1 : 0); + for (int lev = 1; lev < nlevs; ++lev) { + m.ref_ratio[lev-1] = IntVect(AMREX_D_DECL(2, 2, 2)); + m.domains[lev] = amrex::refine(m.domains[lev-1], m.ref_ratio[lev-1]); + } + + m.ba.resize(nlevs); + m.ba[0].define(m.domains[0]); + m.ba[0].maxSize(max_grid_size); + + if (nlevs > 1) { + // Refined region: the central 1/4 of the fine-level domain + int n_fine = ncells * 2; // ref_ratio == 2 + IntVect rlo(AMREX_D_DECL(n_fine/4, n_fine/4, n_fine/4)); + IntVect rhi(AMREX_D_DECL(3*n_fine/4-1, 3*n_fine/4-1, 3*n_fine/4-1)); + m.ba[1].define(Box(rlo, rhi)); + m.ba[1].maxSize(max_grid_size); + } RealBox real_box; - for (int n = 0; n < AMREX_SPACEDIM; n++) { + for (int n = 0; n < AMREX_SPACEDIM; ++n) { real_box.setLo(n, 0.0); real_box.setHi(n, 1.0); } + int is_per[] = {AMREX_D_DECL(1, 1, 1)}; - // This sets the boundary conditions to be doubly or triply periodic - int is_per[] = {AMREX_D_DECL(1,1,1)}; + m.geom.resize(nlevs); + m.geom[0].define(m.domains[0], &real_box, CoordSys::cartesian, is_per); + for (int lev = 1; lev < nlevs; ++lev) { + m.geom[lev].define(m.domains[lev], &real_box, CoordSys::cartesian, is_per); + } - // This defines a Geometry object for each level - Vector geom(nlevs); - geom[0].define(domains[0], &real_box, CoordSys::cartesian, is_per); - for (int lev = 1; lev < nlevs; lev++) { - geom[lev].define(domains[lev], &real_box, CoordSys::cartesian, is_per); + m.dmap.resize(nlevs); + for (int lev = 0; lev < nlevs; ++lev) { + m.dmap[lev] = DistributionMapping{m.ba[lev]}; } - Vector dmap(nlevs); + return m; +} - // write some mesh data too, because tools like yt expect them to be there - Vector > mf(nlevs); - for (int lev = 0; lev < nlevs; lev++) { - dmap[lev] = DistributionMapping{ba[lev]}; - mf[lev] = std::make_unique(ba[lev], dmap[lev], ncomp, nghost); - mf[lev]->setVal(lev); +/** + * Assert that two ParticleContainerPureSoA hold identical particle data by + * comparing exact sorted particle records gathered on the I/O rank. + */ +void verify_same (MyPC& pc_orig, MyPC& pc_new) +{ + auto orig_records = gather_sorted_records(pc_orig); + auto new_records = gather_sorted_records(pc_new); + + if (ParallelDescriptor::IOProcessor()) { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + orig_records.size() == new_records.size(), + "Particle count mismatch after restart"); + + for (Long i = 0; i < orig_records.size(); ++i) { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + orig_records[i] == new_records[i], + "Particle data mismatch after restart"); + } } +} - // these don't really matter, make something up - const Real time = 0.0; +/** + * Test 0: Write and restart with the same hierarchy and BoxArray. + * This preserves the original checkpoint/restart coverage without any + * hierarchy or decomposition changes. + */ +void test_same_hierarchy () +{ + amrex::Print() << "Test 0: restart with same hierarchy\n"; - Vector varnames; - for (int i = 0; i < ncomp; ++i) - { - varnames.push_back("component_" + std::to_string(i)); + const int ncells = 32; + const int max_grid_size = 16; + const int nppc = 2; + const int iseed = 451; + const std::string chkdir = "chk_same_hierarchy"; + + MyPC::ParticleInitData pdata = {{}, {}, {1.0, 2.0, 3.0, 4.0, 6.0, 7.0}, {5, 8}}; + + Vector real_names, int_names; + for (int i = 0; i < NReal - AMREX_SPACEDIM; ++i) { + real_names.push_back("real_" + std::to_string(i)); } + for (int i = 0; i < NInt; ++i) { + int_names.push_back("int_" + std::to_string(i)); + } + + auto mesh = build_mesh(ncells, 2, max_grid_size); + MyPC pc_write(mesh.geom, mesh.dmap, mesh.ba, mesh.ref_ratio); + pc_write.SetVerbose(false); + pc_write.InitRandom(nppc * AMREX_D_TERM(ncells, *ncells, *ncells), + iseed, pdata, /*serialize=*/false); +#ifdef AMREX_USE_HDF5 + pc_write.CheckpointHDF5(chkdir, "particles", true, real_names, int_names); +#else + pc_write.Checkpoint(chkdir, "particles", real_names, int_names); +#endif - Vector level_steps(nlevs, 0); + AsyncOut::Finish(); + ParallelDescriptor::Barrier(); - char fname[512]; - for (int ts = 0; ts < nplotfile; ts++) { - std::snprintf(fname, sizeof fname, "%splt%05d", directory.c_str(), ts); + MyPC pc_read(mesh.geom, mesh.dmap, mesh.ba, mesh.ref_ratio); + pc_read.SetVerbose(false); +#ifdef AMREX_USE_HDF5 + pc_read.RestartHDF5(chkdir + "/particles", "particles"); +#else + pc_read.Restart(chkdir, "particles"); +#endif - amrex::Print() << "Writing plot file [" << fname << "] ..." << '\n'; + verify_same(pc_write, pc_read); - WriteMultiLevelPlotfile(fname, nlevs, amrex::GetVecOfConstPtrs(mf), - varnames, geom, time, level_steps, ref_ratio); + amrex::Print() << " PASSED\n"; +} - amrex::Print() << " done \n"; - } +/** + * Test 1: Write a 2-level HDF5 checkpoint, restart into a 1-level container. + * Particles that resided on the fine level must be redistributed to the + * coarse level; totals and component sums must be unchanged. + */ +void test_fewer_levels () +{ + amrex::Print() << "Test 1: restart with fewer levels than checkpoint\n"; - // Add some particles - constexpr int NReal = 12; - constexpr int NInt = 4; + const int ncells = 32; + const int max_grid_size = 16; + const int nppc = 2; + const int iseed = 451; + const std::string chkdir = "chk_fewer_levels"; - using MyPC = ParticleContainerPureSoA; - MyPC myPC(geom, dmap, ba, ref_ratio); - myPC.SetVerbose(false); + MyPC::ParticleInitData pdata = {{}, {}, {1.0, 2.0, 3.0, 4.0, 6.0, 7.0}, {5, 8}}; - int num_particles = nppc * AMREX_D_TERM(ncells, * ncells, * ncells); - bool serialize = false; - int iseed = 451; - MyPC::ParticleInitData pdata = {{}, {}, - {1.0, 2.0, 3.0, 4.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0}, - {5, 14, 15, 16}}; + Vector real_names, int_names; + for (int i = 0; i < NReal - AMREX_SPACEDIM; ++i) { + real_names.push_back("real_" + std::to_string(i)); + } + for (int i = 0; i < NInt; ++i) { + int_names.push_back("int_" + std::to_string(i)); + } - if (nparticlefile > 0) { - amrex::Print() << "Init particles ..." << '\n'; + // --- write with 2 levels --- + auto mesh2 = build_mesh(ncells, 2, max_grid_size); + MyPC pc_write(mesh2.geom, mesh2.dmap, mesh2.ba, mesh2.ref_ratio); + pc_write.SetVerbose(false); + pc_write.InitRandom(nppc * AMREX_D_TERM(ncells, *ncells, *ncells), + iseed, pdata, /*serialize=*/false); +#ifdef AMREX_USE_HDF5 + pc_write.CheckpointHDF5(chkdir, "particles", true, real_names, int_names); +#else + pc_write.Checkpoint(chkdir, "particles", real_names, int_names); +#endif - myPC.InitRandom(num_particles, iseed, pdata, serialize); + AsyncOut::Finish(); + ParallelDescriptor::Barrier(); - amrex::Print() << " done \n"; + // --- restart with 1 level --- + auto mesh1 = build_mesh(ncells, 1, max_grid_size); + MyPC pc_read(mesh1.geom, mesh1.dmap, mesh1.ba, mesh1.ref_ratio); + pc_read.SetVerbose(false); +#ifdef AMREX_USE_HDF5 + pc_read.RestartHDF5(chkdir + "/particles", "particles"); +#else + pc_read.Restart(chkdir, "particles"); +#endif - Vector particle_realnames; - for (int i = 0; i < NReal-AMREX_SPACEDIM; ++i) { - particle_realnames.push_back("particle_real_component_" + std::to_string(i)); - } + verify_same(pc_write, pc_read); - Vector particle_intnames; - for (int i = 0; i < NInt; ++i) { - particle_intnames.push_back("particle_int_component_" + std::to_string(i)); - } + amrex::Print() << " PASSED\n"; +} - for (int ts = 0; ts < nparticlefile; ts++) { - std::snprintf(fname, sizeof fname, "%splt%05d", directory.c_str(), ts); +/** + * Test 2: Write a 1-level HDF5 checkpoint, restart into a 2-level container. + * After restart, Redistribute() assigns particles inside the refined + * region to the finer level; totals and component sums must be unchanged. + */ +void test_more_levels () +{ + amrex::Print() << "Test 2: restart with more levels than checkpoint\n"; - amrex::Print() << "Writing particle file [" << fname << "] ..." << '\n'; + const int ncells = 32; + const int max_grid_size = 16; + const int nppc = 2; + const int iseed = 451; + const std::string chkdir = "chk_more_levels"; - myPC.Checkpoint(fname, "particle0", false, particle_realnames, particle_intnames); + MyPC::ParticleInitData pdata = {{}, {}, {1.0, 2.0, 3.0, 4.0, 6.0, 7.0}, {5, 8}}; - amrex::Print() << " done \n"; - } + Vector real_names, int_names; + for (int i = 0; i < NReal - AMREX_SPACEDIM; ++i) { + real_names.push_back("real_" + std::to_string(i)); } + for (int i = 0; i < NInt; ++i) { + int_names.push_back("int_" + std::to_string(i)); + } + + // --- write with 1 level --- + auto mesh1 = build_mesh(ncells, 1, max_grid_size); + MyPC pc_write(mesh1.geom, mesh1.dmap, mesh1.ba, mesh1.ref_ratio); + pc_write.SetVerbose(false); + pc_write.InitRandom(nppc * AMREX_D_TERM(ncells, *ncells, *ncells), + iseed, pdata, /*serialize=*/false); +#ifdef AMREX_USE_HDF5 + pc_write.CheckpointHDF5(chkdir, "particles", true, real_names, int_names); +#else + pc_write.Checkpoint(chkdir, "particles", real_names, int_names); +#endif AsyncOut::Finish(); ParallelDescriptor::Barrier(); - char directory_path[512]; - if (restart_check && nparticlefile > 0) - { - MyPC newPC(geom, dmap, ba, ref_ratio); - std::snprintf(directory_path, sizeof directory_path, "%s%s", directory.c_str(), "plt00000"); - newPC.Restart(directory_path, "particle0"); - - using ConstPTDType = typename MyPC::ConstPTDType; - - for (int icomp=0; icomp Real - { - return ptd.rdata(icomp)[i]; - }); - - auto sm_old = amrex::ReduceSum(myPC, - [=] AMREX_GPU_HOST_DEVICE (const ConstPTDType& ptd, const int i) -> Real - { - return ptd.rdata(icomp)[i]; - }); - - ParallelDescriptor::ReduceRealSum(sm_new); - ParallelDescriptor::ReduceRealSum(sm_old); - - amrex::Print() << sm_old << " " << sm_new << "\n"; - AMREX_ALWAYS_ASSERT(sm_old == sm_new); - } + // --- restart with 2 levels --- + auto mesh2 = build_mesh(ncells, 2, max_grid_size); + MyPC pc_read(mesh2.geom, mesh2.dmap, mesh2.ba, mesh2.ref_ratio); + pc_read.SetVerbose(false); +#ifdef AMREX_USE_HDF5 + pc_read.RestartHDF5(chkdir + "/particles", "particles"); +#else + pc_read.Restart(chkdir, "particles"); +#endif - for (int icomp=0; icomp Real - { - return ptd.idata(icomp)[i]; - }); - - auto sm_old = amrex::ReduceSum(myPC, - [=] AMREX_GPU_HOST_DEVICE (const ConstPTDType& ptd, const int i) -> Real - { - return ptd.idata(icomp)[i]; - }); - - ParallelDescriptor::ReduceRealSum(sm_new); - ParallelDescriptor::ReduceRealSum(sm_old); - - amrex::Print() << sm_old << " " << sm_new << "\n"; - AMREX_ALWAYS_ASSERT(sm_old == sm_new); - } - } + verify_same(pc_write, pc_read); + + amrex::Print() << " PASSED\n"; } -void set_grids_nested (Vector& domains, - Vector& grids, - Vector& ref_ratio) +/** + * Test 3: Write an HDF5 checkpoint with one BoxArray decomposition, restart + * into a container with the same number of levels but a different BoxArray + * (different max_grid_size). This is the canonical dual-grid scenario: the + * Particle_H header written at checkpoint time differs from the BoxArray + * of the new container, so AMReX uses temporary grids while reading and + * calls Redistribute() to move particles onto the new decomposition. + */ +void test_different_boxarrays () { - int ncells, max_grid_size, nlevs; + amrex::Print() << "Test 3: restart with same levels but different BoxArrays\n"; - ParmParse pp; - pp.get("ncells", ncells); - pp.get("max_grid_size", max_grid_size); - pp.get("nlevs", nlevs); + const int ncells = 32; + const int nppc = 2; + const int iseed = 451; + const std::string chkdir = "chk_different_ba"; - AMREX_ALWAYS_ASSERT(nlevs < 2); // relax this later + MyPC::ParticleInitData pdata = {{}, {}, {1.0, 2.0, 3.0, 4.0, 6.0, 7.0}, {5, 8}}; - IntVect domain_lo(AMREX_D_DECL(0, 0, 0)); - IntVect domain_hi(AMREX_D_DECL(ncells-1, ncells-1, ncells-1)); + Vector real_names, int_names; + for (int i = 0; i < NReal - AMREX_SPACEDIM; ++i) { + real_names.push_back("real_" + std::to_string(i)); + } + for (int i = 0; i < NInt; ++i) { + int_names.push_back("int_" + std::to_string(i)); + } - domains.resize(nlevs); - domains[0].setSmall(domain_lo); - domains[0].setBig(domain_hi); + // --- write: coarse decomposition (few large boxes) --- + auto mesh_coarse = build_mesh(ncells, 1, ncells); // one box per rank at most + MyPC pc_write(mesh_coarse.geom, mesh_coarse.dmap, mesh_coarse.ba, + mesh_coarse.ref_ratio); + pc_write.SetVerbose(false); + pc_write.InitRandom(nppc * AMREX_D_TERM(ncells, *ncells, *ncells), + iseed, pdata, /*serialize=*/false); +#ifdef AMREX_USE_HDF5 + pc_write.CheckpointHDF5(chkdir, "particles", true, real_names, int_names); +#else + pc_write.Checkpoint(chkdir, "particles", real_names, int_names); +#endif - ref_ratio.resize(nlevs-1); - for (int lev = 1; lev < nlevs; lev++) { - ref_ratio[lev-1] = IntVect(AMREX_D_DECL(2, 2, 2)); - } + AsyncOut::Finish(); + ParallelDescriptor::Barrier(); - grids.resize(nlevs); - grids[0].define(domains[0]); + // --- restart: finer decomposition (many smaller boxes) --- + auto mesh_fine = build_mesh(ncells, 1, ncells / 4); + MyPC pc_read(mesh_fine.geom, mesh_fine.dmap, mesh_fine.ba, + mesh_fine.ref_ratio); + pc_read.SetVerbose(false); +#ifdef AMREX_USE_HDF5 + pc_read.RestartHDF5(chkdir + "/particles", "particles"); +#else + pc_read.Restart(chkdir, "particles"); +#endif - // Now we make the refined level be the center eighth of the domain - if (nlevs > 1) { - int n_fine = ncells*ref_ratio[0][0]; - IntVect refined_lo(AMREX_D_DECL(n_fine/4,n_fine/4,n_fine/4)); - IntVect refined_hi(AMREX_D_DECL(3*n_fine/4-1,3*n_fine/4-1,3*n_fine/4-1)); + verify_same(pc_write, pc_read); - // Build a box for the level 1 domain - Box refined_patch(refined_lo, refined_hi); - grids[1].define(refined_patch); - } + amrex::Print() << " PASSED\n"; +} - // break the BoxArrays at both levels into max_grid_size^3 boxes - for (int lev = 0; lev < nlevs; lev++) { - grids[lev].maxSize(max_grid_size); - } +int main (int argc, char* argv[]) +{ + amrex::Initialize(argc, argv); - for (int lev = 1; lev < nlevs; lev++) { - domains[lev] = amrex::refine(domains[lev-1], ref_ratio[lev-1]); - } + test_same_hierarchy(); + test_fewer_levels(); + test_more_levels(); + test_different_boxarrays(); + + amrex::Print() << "All dual-grid HDF5 SOA restart tests PASSED\n"; + + amrex::Finalize(); } diff --git a/Tests/Particles/CheckpointRestartSOA_AsyncIO/CMakeLists.txt b/Tests/Particles/CheckpointRestartSOA_AsyncIO/CMakeLists.txt deleted file mode 100644 index 087d3ad4998..00000000000 --- a/Tests/Particles/CheckpointRestartSOA_AsyncIO/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -# This tests requires particle support -if (NOT AMReX_PARTICLES) - return() -endif () - -foreach(D IN LISTS AMReX_SPACEDIM) - set(_sources main.cpp) - set(_input_files inputs) - - setup_test(${D} _sources _input_files) - - unset(_sources) - unset(_input_files) -endforeach() diff --git a/Tests/Particles/CheckpointRestartSOA_AsyncIO/GNUmakefile b/Tests/Particles/CheckpointRestartSOA_AsyncIO/GNUmakefile deleted file mode 120000 index 402f7774f22..00000000000 --- a/Tests/Particles/CheckpointRestartSOA_AsyncIO/GNUmakefile +++ /dev/null @@ -1 +0,0 @@ -../CheckpointRestartSOA/GNUmakefile \ No newline at end of file diff --git a/Tests/Particles/CheckpointRestartSOA_AsyncIO/Make.package b/Tests/Particles/CheckpointRestartSOA_AsyncIO/Make.package deleted file mode 120000 index 6fb6dc981b7..00000000000 --- a/Tests/Particles/CheckpointRestartSOA_AsyncIO/Make.package +++ /dev/null @@ -1 +0,0 @@ -../CheckpointRestartSOA/Make.package \ No newline at end of file diff --git a/Tests/Particles/CheckpointRestartSOA_AsyncIO/inputs b/Tests/Particles/CheckpointRestartSOA_AsyncIO/inputs deleted file mode 100644 index 8efbea33ad4..00000000000 --- a/Tests/Particles/CheckpointRestartSOA_AsyncIO/inputs +++ /dev/null @@ -1,28 +0,0 @@ -# Domain size -ncells = 64 - -# Maximum allowable size of each subdomain in the problem domain; -# this is used to decompose the domain for parallel calculations. -max_grid_size = 8 - -# Number of levels -nlevs = 1 - -# Number of components in the multifabs -ncomp = 6 - -# Number of particles per cell -nppc = 2 - -# Number of plot files to write -nplotfile = 1 - -# Number of plot files to write -nparticlefile = 1 - -# Whether to check the correctness of Checkpoint / Restart -restart_check = 1 - -directory=. - -amrex.async_out=1 \ No newline at end of file diff --git a/Tests/Particles/CheckpointRestartSOA_AsyncIO/main.cpp b/Tests/Particles/CheckpointRestartSOA_AsyncIO/main.cpp deleted file mode 120000 index 36e9ca62484..00000000000 --- a/Tests/Particles/CheckpointRestartSOA_AsyncIO/main.cpp +++ /dev/null @@ -1 +0,0 @@ -../CheckpointRestartSOA/main.cpp \ No newline at end of file