diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 1e1968e5a..e79560e31 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -129,7 +129,7 @@ jobs: - name: Run working-directory: ./Exec/RegTests/EB_BackwardStepFlame/ run: | - mpirun -n 2 ./PeleLMeX2d.gnu.MPI.ex eb_bfs.inp amr.max_step=2 amr.plot_int=-1 amr.check_int=-1 amrex.abort_on_unused_inputs=1 + mpirun -n 2 ./PeleLMeX2d.gnu.MPI.ex eb_bfs.inp amr.max_step=2 amr.plot_int=-1 amr.check_int=-1 amrex.abort_on_unused_inputs=1 mac_proj.verbose=2 mac_proj.atol=5e-11 mac_proj.rtol=5e-11 # Build and Run the Plasma flamesheet with GNU9.3 and MPI support PLASMA: diff --git a/Docs/sphinx/manual/LMeXControls.rst b/Docs/sphinx/manual/LMeXControls.rst index a648016d8..402809869 100644 --- a/Docs/sphinx/manual/LMeXControls.rst +++ b/Docs/sphinx/manual/LMeXControls.rst @@ -130,6 +130,7 @@ IO parameters amr.restart = chk00100 # [OPT, DEF=""] Checkpoint from which to restart the simulation amr.initDataPlt = plt01000 # [OPT, DEF=""] Provide a plotfile from which to extract initial data peleLM.initDataPlt_reset_time = 1 # [OPT, DEF=1] Resets time and nsteps to 0 after restarting from a plot file. (Warning: plot file will be rewritten if not renamed and argument value = 0) + peleLM.initDataPlt_specname_map = spec1 spec2 # [OPT, DEF=""] If specified, lookup these entries instead of species names when populating species from the init plot file (length must match NUM_SPECIES) peleLM.initDataPlt_patch_flow_variables = false # [OPT, DEF=false] Enable user-defined flow variable patching after reading a plot solution file amr.regrid_on_restart = 1 # [OPT, DEF="0"] Trigger a regrid after the data from checkpoint are loaded amr.n_files = 64 # [OPT, DEF="min(256,NProcs)"] Number of files to write per level diff --git a/Exec/Plasma/FlameSheetIons/pelelmex_prob.H b/Exec/Plasma/FlameSheetIons/pelelmex_prob.H index c45d8c585..15dea69be 100644 --- a/Exec/Plasma/FlameSheetIons/pelelmex_prob.H +++ b/Exec/Plasma/FlameSheetIons/pelelmex_prob.H @@ -95,7 +95,7 @@ struct MyProblemSpecificFunctions : public DefaultProblemSpecificFunctions massfrac[E_ID] = 0.0; amrex::Real sum = 0.0; for (int n = 0; n < NUM_SPECIES; n++) { - massfrac[n] = std::max(0.0, massfrac[n]); + massfrac[n] = amrex::max(0.0, massfrac[n]); if (n != N2_ID) sum += massfrac[n]; } @@ -126,7 +126,7 @@ struct MyProblemSpecificFunctions : public DefaultProblemSpecificFunctions for (int n = 0; n < NUM_SPECIES; n++) { chargeDist += massfrac[n] * zk[n]; } - state(i, j, k, NE) = std::max(1.0e-24, chargeDist / elemCharge); + state(i, j, k, NE) = amrex::max(1.0e-24, chargeDist / elemCharge); AMREX_D_TERM(state(i, j, k, VELX) = 0.0; , state(i, j, k, VELY) = pmf_vals[1] * 0.01; diff --git a/Exec/Production/CounterFlowSpray/SprayParticlesInitInsert.cpp b/Exec/Production/CounterFlowSpray/SprayParticlesInitInsert.cpp index b1021e2db..3f6d19d00 100644 --- a/Exec/Production/CounterFlowSpray/SprayParticlesInitInsert.cpp +++ b/Exec/Production/CounterFlowSpray/SprayParticlesInitInsert.cpp @@ -1,49 +1,46 @@ - #include "SprayParticles.H" #include "SprayInjection.H" #include "pelelmex_prob.H" -using namespace amrex; - class CounterFlowJet : public SprayJet { public: // Use default constructor - CounterFlowJet(const std::string jet_name, const Geometry& geom) + CounterFlowJet(const std::string jet_name, const amrex::Geometry& geom) : SprayJet(jet_name, geom) { } bool get_new_particle( - const Real time, - const Real& phi_radial, - const Real& cur_radius, - Real& umag, - Real& theta_spread, - Real& phi_swirl, - Real& dia_part, - Real& T_part, - Real* Y_part) override; + const amrex::Real time, + const amrex::Real& phi_radial, + const amrex::Real& cur_radius, + amrex::Real& umag, + amrex::Real& theta_spread, + amrex::Real& phi_swirl, + amrex::Real& dia_part, + amrex::Real& T_part, + amrex::Real* Y_part) override; }; bool CounterFlowJet::get_new_particle( - const Real time, - const Real& phi_radial, - const Real& cur_radius, - Real& umag, - Real& theta_spread, - Real& phi_swirl, - Real& dia_part, - Real& T_part, - Real* Y_part) + const amrex::Real time, + const amrex::Real& phi_radial, + const amrex::Real& cur_radius, + amrex::Real& umag, + amrex::Real& theta_spread, + amrex::Real& phi_swirl, + amrex::Real& dia_part, + amrex::Real& T_part, + amrex::Real* Y_part) { umag = m_jetVel; T_part = m_jetT; dia_part = m_dropDist->get_dia(); phi_swirl = 0.; // Random spread angle - Real tan_vel_comp = (2. * amrex::Random() - 1.) * 0.05; + amrex::Real tan_vel_comp = (2. * amrex::Random() - 1.) * 0.05; theta_spread = std::asin(tan_vel_comp); for (int sp = 0; sp < SPRAY_FUEL_NUM; ++sp) { Y_part[sp] = 0.; @@ -54,7 +51,7 @@ CounterFlowJet::get_new_particle( bool SprayParticleContainer::injectParticles( - Real time, Real dt, int nstep, int lev, int finest_level) + amrex::Real time, amrex::Real dt, int nstep, int lev, int finest_level) { if (lev != 0) { return false; diff --git a/Exec/RegTests/SprayTest/compareOutput.py b/Exec/RegTests/SprayTest/compareOutput.py index fcbf6a904..828331819 100755 --- a/Exec/RegTests/SprayTest/compareOutput.py +++ b/Exec/RegTests/SprayTest/compareOutput.py @@ -95,7 +95,7 @@ def pproc(args): cer = float(line.split()[2]) if (cer > max_error): errfail += 1 - perr = "Error in {} on level {}: {} > {}".format(var,curlev,cer,maxerror) + perr = "Error in {} on level {}: {} > {}".format(var,curlev,cer,max_error) print(perr) if (varcount != len(vars)): error = "Not all variables were found in plot file" diff --git a/Exec/RegTests/Unit/pelelmex_prob_parm.H b/Exec/RegTests/Unit/pelelmex_prob_parm.H index cd7889544..7ab3018e0 100644 --- a/Exec/RegTests/Unit/pelelmex_prob_parm.H +++ b/Exec/RegTests/Unit/pelelmex_prob_parm.H @@ -3,14 +3,12 @@ #include -using namespace amrex::literals; - struct ProbParm { // Shared params int probType = 0; - amrex::Real T_mean = 298.0_rt; - amrex::Real P_mean = 101325.0_rt; + amrex::Real T_mean = 298.0; + amrex::Real P_mean = 101325.0; amrex::Real meanFlowMag = 0.0; int meanFlowDir = 1; diff --git a/Source/PeleLMeX.H b/Source/PeleLMeX.H index 4beded50e..3614ed54b 100644 --- a/Source/PeleLMeX.H +++ b/Source/PeleLMeX.H @@ -1505,6 +1505,7 @@ public: std::unique_ptr& advData, std::unique_ptr& diffData); + bool checkForNaNs(); void copyTransportOldToNew(); void copyStateNewToOld(int nGhost = 0); void copyPressNewToOld(); @@ -1900,6 +1901,7 @@ public: amrex::Vector m_evaluatePlotVars; bool m_write_hdf5_pltfile = false; bool m_do_patch_flow_variables = false; + amrex::Vector m_initDataPlt_specname_map = {}; //----------------------------------------------------------------------------- // ALGORITHM diff --git a/Source/PeleLMeX.cpp b/Source/PeleLMeX.cpp index 82b2e0419..b91b4a2ea 100644 --- a/Source/PeleLMeX.cpp +++ b/Source/PeleLMeX.cpp @@ -5,8 +5,6 @@ #include "SprayParticles.H" #endif -using namespace amrex; - pele::physics::PeleParams> @@ -30,7 +28,7 @@ PeleLM::~PeleLM() typical_values.clear(); freeProbParm(); delete prob_parm; - The_Arena()->free(prob_parm_d); + amrex::The_Arena()->free(prob_parm_d); m_initial_ba.clear(); m_regrid_ba.clear(); #ifdef PELE_USE_SPRAY @@ -56,7 +54,7 @@ PeleLM::getLevelDataPtr( m_leveldata_floating = std::make_unique( grids[lev], dmap[lev], *m_factory[lev], m_incompressible, m_has_divu, m_nAux, m_nGrowState, m_use_soret, static_cast(m_do_les)); - Real time = getTime(lev, a_time); + amrex::Real time = getTime(lev, a_time); fillpatch_state(lev, time, m_leveldata_floating->state, m_nGrowState); if (m_nAux > 0) { fillpatch_aux(lev, time, m_leveldata_floating->auxiliaries, m_nGrowState); @@ -73,22 +71,22 @@ PeleLM::getLevelDataReactPtr(int lev) return nullptr; } -Vector> +amrex::Vector> PeleLM::getStateVect(const TimeStamp& a_time) { - Vector> r; + amrex::Vector> r; r.reserve(finest_level + 1); if (a_time == AmrOldTime) { if (m_incompressible != 0) { for (int lev = 0; lev <= finest_level; ++lev) { r.push_back( - std::make_unique( + std::make_unique( m_leveldata_old[lev]->state, amrex::make_alias, 0, AMREX_SPACEDIM)); } } else { for (int lev = 0; lev <= finest_level; ++lev) { r.push_back( - std::make_unique( + std::make_unique( m_leveldata_old[lev]->state, amrex::make_alias, 0, NVAR)); } } @@ -96,13 +94,13 @@ PeleLM::getStateVect(const TimeStamp& a_time) if (m_incompressible != 0) { for (int lev = 0; lev <= finest_level; ++lev) { r.push_back( - std::make_unique( + std::make_unique( m_leveldata_new[lev]->state, amrex::make_alias, 0, AMREX_SPACEDIM)); } } else { for (int lev = 0; lev <= finest_level; ++lev) { r.push_back( - std::make_unique( + std::make_unique( m_leveldata_new[lev]->state, amrex::make_alias, 0, NVAR)); } } @@ -110,22 +108,22 @@ PeleLM::getStateVect(const TimeStamp& a_time) return r; } -Vector> +amrex::Vector> PeleLM::getVelocityVect(const TimeStamp& a_time) { - Vector> r; + amrex::Vector> r; r.reserve(finest_level + 1); if (a_time == AmrOldTime) { for (int lev = 0; lev <= finest_level; ++lev) { r.push_back( - std::make_unique( + std::make_unique( m_leveldata_old[lev]->state, amrex::make_alias, VELX, AMREX_SPACEDIM)); } } else { for (int lev = 0; lev <= finest_level; ++lev) { r.push_back( - std::make_unique( + std::make_unique( m_leveldata_new[lev]->state, amrex::make_alias, VELX, AMREX_SPACEDIM)); } @@ -133,23 +131,23 @@ PeleLM::getVelocityVect(const TimeStamp& a_time) return r; } -Vector> +amrex::Vector> PeleLM::getSpeciesVect(const TimeStamp& a_time) { AMREX_ASSERT(!m_incompressible); - Vector> r; + amrex::Vector> r; r.reserve(finest_level + 1); if (a_time == AmrOldTime) { for (int lev = 0; lev <= finest_level; ++lev) { r.push_back( - std::make_unique( + std::make_unique( m_leveldata_old[lev]->state, amrex::make_alias, FIRSTSPEC, NUM_SPECIES)); } } else { for (int lev = 0; lev <= finest_level; ++lev) { r.push_back( - std::make_unique( + std::make_unique( m_leveldata_new[lev]->state, amrex::make_alias, FIRSTSPEC, NUM_SPECIES)); } @@ -157,84 +155,85 @@ PeleLM::getSpeciesVect(const TimeStamp& a_time) return r; } -Vector> +amrex::Vector> PeleLM::getDensityVect(const TimeStamp& a_time) { AMREX_ASSERT(!m_incompressible); - Vector> r; + amrex::Vector> r; r.reserve(finest_level + 1); if (a_time == AmrOldTime) { for (int lev = 0; lev <= finest_level; ++lev) { r.push_back( - std::make_unique( + std::make_unique( m_leveldata_old[lev]->state, amrex::make_alias, DENSITY, 1)); } } else if (a_time == AmrNewTime) { for (int lev = 0; lev <= finest_level; ++lev) { r.push_back( - std::make_unique( + std::make_unique( m_leveldata_new[lev]->state, amrex::make_alias, DENSITY, 1)); } } else { for (int lev = 0; lev <= finest_level; ++lev) { - Real time = getTime(lev, a_time); + amrex::Real time = getTime(lev, a_time); r.push_back( - std::make_unique(grids[lev], dmap[lev], 1, m_nGrowState)); + std::make_unique( + grids[lev], dmap[lev], 1, m_nGrowState)); fillpatch_density(lev, time, *(r[lev]), 0, m_nGrowState); } } return r; } -Vector> +amrex::Vector> PeleLM::getTempVect(const TimeStamp& a_time) { AMREX_ASSERT(!m_incompressible); - Vector> r; + amrex::Vector> r; r.reserve(finest_level + 1); if (a_time == AmrOldTime) { for (int lev = 0; lev <= finest_level; ++lev) { r.push_back( - std::make_unique( + std::make_unique( m_leveldata_old[lev]->state, amrex::make_alias, TEMP, 1)); } } else { for (int lev = 0; lev <= finest_level; ++lev) { r.push_back( - std::make_unique( + std::make_unique( m_leveldata_new[lev]->state, amrex::make_alias, TEMP, 1)); } } return r; } -Vector> +amrex::Vector> PeleLM::getRhoHVect(const TimeStamp& a_time) { AMREX_ASSERT(!m_incompressible); - Vector> r; + amrex::Vector> r; r.reserve(finest_level + 1); if (a_time == AmrOldTime) { for (int lev = 0; lev <= finest_level; ++lev) { r.push_back( - std::make_unique( + std::make_unique( m_leveldata_old[lev]->state, amrex::make_alias, RHOH, 1)); } } else { for (int lev = 0; lev <= finest_level; ++lev) { r.push_back( - std::make_unique( + std::make_unique( m_leveldata_new[lev]->state, amrex::make_alias, RHOH, 1)); } } return r; } -Vector +amrex::Vector PeleLM::getDivUVect(const TimeStamp& a_time) { AMREX_ASSERT(!m_incompressible); - Vector r; + amrex::Vector r; r.reserve(finest_level + 1); if (a_time == AmrOldTime) { for (int lev = 0; lev <= finest_level; ++lev) { @@ -248,11 +247,11 @@ PeleLM::getDivUVect(const TimeStamp& a_time) return r; } -Vector +amrex::Vector PeleLM::getDiffusivityVect(const TimeStamp& a_time) { AMREX_ASSERT(!m_incompressible); - Vector r; + amrex::Vector r; r.reserve(finest_level + 1); if (a_time == AmrOldTime) { for (int lev = 0; lev <= finest_level; ++lev) { @@ -266,10 +265,10 @@ PeleLM::getDiffusivityVect(const TimeStamp& a_time) return r; } -Vector +amrex::Vector PeleLM::getViscosityVect(const TimeStamp& a_time) { - Vector r; + amrex::Vector r; r.reserve(finest_level + 1); if (a_time == AmrOldTime) { for (int lev = 0; lev <= finest_level; ++lev) { @@ -283,10 +282,10 @@ PeleLM::getViscosityVect(const TimeStamp& a_time) return r; } -Vector +amrex::Vector PeleLM::getIRVect() { - Vector r; + amrex::Vector r; r.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { r.push_back(&(m_leveldatareact[lev]->I_R)); @@ -294,33 +293,33 @@ PeleLM::getIRVect() return r; } -Vector> +amrex::Vector> PeleLM::getAuxVect(const TimeStamp& a_time) { AMREX_ASSERT(m_nAux > 0); - Vector> r; + amrex::Vector> r; r.reserve(finest_level + 1); if (a_time == AmrOldTime) { for (int lev = 0; lev <= finest_level; ++lev) { r.push_back( - std::make_unique( + std::make_unique( m_leveldata_old[lev]->auxiliaries, amrex::make_alias, 0, m_nAux)); } } else { for (int lev = 0; lev <= finest_level; ++lev) { r.push_back( - std::make_unique( + std::make_unique( m_leveldata_new[lev]->auxiliaries, amrex::make_alias, 0, m_nAux)); } } return r; } -Vector +amrex::Vector PeleLM::getAuxDiffusivityVect(const TimeStamp& a_time) { AMREX_ASSERT(m_nAux > 0); - Vector r; + amrex::Vector r; r.reserve(finest_level + 1); if (a_time == AmrOldTime) { for (int lev = 0; lev <= finest_level; ++lev) { @@ -445,54 +444,54 @@ PeleLM::averageDownReaction() } #ifdef PELE_USE_PLASMA -Vector> +amrex::Vector> PeleLM::getPhiVVect(const TimeStamp& a_time) { AMREX_ASSERT(!m_incompressible); - Vector> r; + amrex::Vector> r; r.reserve(finest_level + 1); if (a_time == AmrOldTime) { for (int lev = 0; lev <= finest_level; ++lev) { r.push_back( - std::make_unique( + std::make_unique( m_leveldata_old[lev]->state, amrex::make_alias, PHIV, 1)); } } else { for (int lev = 0; lev <= finest_level; ++lev) { r.push_back( - std::make_unique( + std::make_unique( m_leveldata_new[lev]->state, amrex::make_alias, PHIV, 1)); } } return r; } -Vector> +amrex::Vector> PeleLM::getnEVect(const TimeStamp& a_time) { AMREX_ASSERT(!m_incompressible); - Vector> r; + amrex::Vector> r; r.reserve(finest_level + 1); if (a_time == AmrOldTime) { for (int lev = 0; lev <= finest_level; ++lev) { r.push_back( - std::make_unique( + std::make_unique( m_leveldata_old[lev]->state, amrex::make_alias, NE, 1)); } } else { for (int lev = 0; lev <= finest_level; ++lev) { r.push_back( - std::make_unique( + std::make_unique( m_leveldata_new[lev]->state, amrex::make_alias, NE, 1)); } } return r; } -Vector +amrex::Vector PeleLM::getnEDiffusivityVect(const TimeStamp& a_time) { - Vector r; + amrex::Vector r; r.reserve(finest_level + 1); if (a_time == AmrOldTime) { for (int lev = 0; lev <= finest_level; ++lev) { diff --git a/Source/PeleLMeX_Advance.cpp b/Source/PeleLMeX_Advance.cpp index 8d5085205..48de6c9ed 100644 --- a/Source/PeleLMeX_Advance.cpp +++ b/Source/PeleLMeX_Advance.cpp @@ -3,8 +3,6 @@ #include #include -using namespace amrex; - void PeleLM::Advance(int is_initIter) { @@ -16,7 +14,7 @@ PeleLM::Advance(int is_initIter) #endif // Start timing current time step - Real strt_time = ParallelDescriptor::second(); + amrex::Real strt_time = amrex::ParallelDescriptor::second(); //---------------------------------------------------------------- BL_PROFILE_VAR("PeleLMeX::advance::setup", PLM_SETUP); @@ -160,9 +158,9 @@ PeleLM::Advance(int is_initIter) //---------------------------------------------------------------- // Scalar advance if (m_incompressible != 0) { - Real MACStart = 0.0; + amrex::Real MACStart = 0.0; if (m_verbose > 1) { - MACStart = ParallelDescriptor::second(); + MACStart = amrex::ParallelDescriptor::second(); } // Still need to get face velocities ... @@ -172,9 +170,9 @@ PeleLM::Advance(int is_initIter) macProject(AmrOldTime, advData, {}); if (m_verbose > 1) { - Real MACEnd = ParallelDescriptor::second() - MACStart; - ParallelDescriptor::ReduceRealMax( - MACEnd, ParallelDescriptor::IOProcessorNumber()); + amrex::Real MACEnd = amrex::ParallelDescriptor::second() - MACStart; + amrex::ParallelDescriptor::ReduceRealMax( + MACEnd, amrex::ParallelDescriptor::IOProcessorNumber()); amrex::Print() << " - Advance()::MACProjection() --> Time: " << MACEnd << "\n"; } @@ -217,9 +215,9 @@ PeleLM::Advance(int is_initIter) //---------------------------------------------------------------- BL_PROFILE_VAR("PeleLMeX::advance::velocity", PLM_VEL); // Velocity advance - Real VelAdvStart = 0.0; + amrex::Real VelAdvStart = 0.0; if (m_verbose > 1) { - VelAdvStart = ParallelDescriptor::second(); + VelAdvStart = amrex::ParallelDescriptor::second(); } // Re-evaluate viscosity only if scalar updated if (m_incompressible == 0) { @@ -241,9 +239,9 @@ PeleLM::Advance(int is_initIter) const TimeStamp rhoTime = AmrHalfTime; velocityProjection(is_initIter, rhoTime, m_dt); if (m_verbose > 1) { - Real VelAdvEnd = ParallelDescriptor::second() - VelAdvStart; - ParallelDescriptor::ReduceRealMax( - VelAdvEnd, ParallelDescriptor::IOProcessorNumber()); + amrex::Real VelAdvEnd = amrex::ParallelDescriptor::second() - VelAdvStart; + amrex::ParallelDescriptor::ReduceRealMax( + VelAdvEnd, amrex::ParallelDescriptor::IOProcessorNumber()); amrex::Print() << " - Advance()::VelocityAdvance --> Time: " << VelAdvEnd << "\n"; } @@ -260,9 +258,9 @@ PeleLM::Advance(int is_initIter) // Wrapup advance // Timing current time step if (m_verbose > 0) { - Real run_time = ParallelDescriptor::second() - strt_time; - ParallelDescriptor::ReduceRealMax( - run_time, ParallelDescriptor::IOProcessorNumber()); + amrex::Real run_time = amrex::ParallelDescriptor::second() - strt_time; + amrex::ParallelDescriptor::ReduceRealMax( + run_time, amrex::ParallelDescriptor::IOProcessorNumber()); amrex::Print() << " >> PeleLMeX::Advance() --> Time: " << run_time << "\n"; } } @@ -286,9 +284,9 @@ PeleLM::oneSDC( // At the first SDC, we already copied old -> new if (sdcIter > 1) { - Real UpdateStart = 0.0; + amrex::Real UpdateStart = 0.0; if (m_verbose > 1) { - UpdateStart = ParallelDescriptor::second(); + UpdateStart = amrex::ParallelDescriptor::second(); } // fillpatch the new state averageDownScalars(AmrNewTime); @@ -317,9 +315,9 @@ PeleLM::oneSDC( checkDt(AmrNewTime, m_dt); if (m_verbose > 1) { - Real UpdateEnd = ParallelDescriptor::second() - UpdateStart; - ParallelDescriptor::ReduceRealMax( - UpdateEnd, ParallelDescriptor::IOProcessorNumber()); + amrex::Real UpdateEnd = amrex::ParallelDescriptor::second() - UpdateStart; + amrex::ParallelDescriptor::ReduceRealMax( + UpdateEnd, amrex::ParallelDescriptor::IOProcessorNumber()); amrex::Print() << " - oneSDC()::Update t^{n+1,k} --> Time: " << UpdateEnd << "\n"; } @@ -330,9 +328,9 @@ PeleLM::oneSDC( // Get u MAC //---------------------------------------------------------------- BL_PROFILE_VAR("PeleLMeX::advance::mac", PLM_MAC); - Real MACStart = 0.0; + amrex::Real MACStart = 0.0; if (m_verbose > 1) { - MACStart = ParallelDescriptor::second(); + MACStart = amrex::ParallelDescriptor::second(); } // Predict face velocity with Godunov predictVelocity(advData); @@ -346,9 +344,9 @@ PeleLM::oneSDC( // MAC projection macProject(AmrOldTime, advData, GetVecOfPtrs(advData->mac_divu)); if (m_verbose > 1) { - Real MACEnd = ParallelDescriptor::second() - MACStart; - ParallelDescriptor::ReduceRealMax( - MACEnd, ParallelDescriptor::IOProcessorNumber()); + amrex::Real MACEnd = amrex::ParallelDescriptor::second() - MACStart; + amrex::ParallelDescriptor::ReduceRealMax( + MACEnd, amrex::ParallelDescriptor::IOProcessorNumber()); amrex::Print() << " - oneSDC()::MACProjection() --> Time: " << MACEnd << "\n"; } @@ -360,9 +358,9 @@ PeleLM::oneSDC( // Scalar advections //---------------------------------------------------------------- BL_PROFILE_VAR("PeleLMeX::advance::scalars_adv", PLM_SADV); - Real ScalAdvStart = 0.0; + amrex::Real ScalAdvStart = 0.0; if (m_verbose > 1) { - ScalAdvStart = ParallelDescriptor::second(); + ScalAdvStart = amrex::ParallelDescriptor::second(); } #ifdef PELE_USE_SOOT // Compute and update passive advective terms @@ -380,9 +378,9 @@ PeleLM::oneSDC( updateDensity(advData); fillPatchDensity(AmrNewTime); if (m_verbose > 1) { - Real ScalAdvEnd = ParallelDescriptor::second() - ScalAdvStart; - ParallelDescriptor::ReduceRealMax( - ScalAdvEnd, ParallelDescriptor::IOProcessorNumber()); + amrex::Real ScalAdvEnd = amrex::ParallelDescriptor::second() - ScalAdvStart; + amrex::ParallelDescriptor::ReduceRealMax( + ScalAdvEnd, amrex::ParallelDescriptor::IOProcessorNumber()); amrex::Print() << " - oneSDC()::ScalarAdvection() --> Time: " << ScalAdvEnd << "\n"; } @@ -394,9 +392,9 @@ PeleLM::oneSDC( // Scalar diffusion //---------------------------------------------------------------- BL_PROFILE_VAR("PeleLMeX::advance::diffusion", PLM_DIFF); - Real ScalDiffStart = 0.0; + amrex::Real ScalDiffStart = 0.0; if (m_verbose > 1) { - ScalDiffStart = ParallelDescriptor::second(); + ScalDiffStart = amrex::ParallelDescriptor::second(); } // Get scalar diffusion SDC RHS (stored in Forcing) getScalarDiffForce(advData, diffData); @@ -404,9 +402,10 @@ PeleLM::oneSDC( // Diffuse scalars differentialDiffusionUpdate(advData, diffData); if (m_verbose > 1) { - Real ScalDiffEnd = ParallelDescriptor::second() - ScalDiffStart; - ParallelDescriptor::ReduceRealMax( - ScalDiffEnd, ParallelDescriptor::IOProcessorNumber()); + amrex::Real ScalDiffEnd = + amrex::ParallelDescriptor::second() - ScalDiffStart; + amrex::ParallelDescriptor::ReduceRealMax( + ScalDiffEnd, amrex::ParallelDescriptor::IOProcessorNumber()); amrex::Print() << " - oneSDC()::ScalarDiffusion() --> Time: " << ScalDiffEnd << "\n"; } @@ -425,9 +424,9 @@ PeleLM::oneSDC( // Reaction //---------------------------------------------------------------- BL_PROFILE_VAR("PeleLMeX::advance::reactions", PLM_REAC); - Real ScalReacStart = 0.0; + amrex::Real ScalReacStart = 0.0; if (m_verbose > 1) { - ScalReacStart = ParallelDescriptor::second(); + ScalReacStart = amrex::ParallelDescriptor::second(); } // Get external forcing for chemistry getScalarReactForce(advData); @@ -435,9 +434,10 @@ PeleLM::oneSDC( // Integrate chemistry advanceChemistry(advData); if (m_verbose > 1) { - Real ScalReacEnd = ParallelDescriptor::second() - ScalReacStart; - ParallelDescriptor::ReduceRealMax( - ScalReacEnd, ParallelDescriptor::IOProcessorNumber()); + amrex::Real ScalReacEnd = + amrex::ParallelDescriptor::second() - ScalReacStart; + amrex::ParallelDescriptor::ReduceRealMax( + ScalReacEnd, amrex::ParallelDescriptor::IOProcessorNumber()); amrex::Print() << " - oneSDC()::ScalarReaction() --> Time: " << ScalReacEnd << "\n"; } diff --git a/Source/PeleLMeX_Advection.cpp b/Source/PeleLMeX_Advection.cpp index 15dc26bc8..68059da08 100644 --- a/Source/PeleLMeX_Advection.cpp +++ b/Source/PeleLMeX_Advection.cpp @@ -3,32 +3,31 @@ #include #include -using namespace amrex; - void PeleLM::computeVelocityAdvTerm(std::unique_ptr& advData) { - //---------------------------------------------------------------- // Create temporary containers constexpr int nGrow_force = 1; - Vector divtau(finest_level + 1); - Vector velForces(finest_level + 1); - Vector> fluxes(finest_level + 1); - Vector> faces(finest_level + 1); + amrex::Vector divtau(finest_level + 1); + amrex::Vector velForces(finest_level + 1); + amrex::Vector> fluxes( + finest_level + 1); + amrex::Vector> faces( + finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { divtau[lev].define( - grids[lev], dmap[lev], AMREX_SPACEDIM, 0, MFInfo(), Factory(lev)); + grids[lev], dmap[lev], AMREX_SPACEDIM, 0, amrex::MFInfo(), Factory(lev)); velForces[lev].define( - grids[lev], dmap[lev], AMREX_SPACEDIM, nGrow_force, MFInfo(), + grids[lev], dmap[lev], AMREX_SPACEDIM, nGrow_force, amrex::MFInfo(), Factory(lev)); for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { fluxes[lev][idim].define( - amrex::convert(grids[lev], IntVect::TheDimensionVector(idim)), - dmap[lev], AMREX_SPACEDIM, 0, MFInfo(), Factory(lev)); + amrex::convert(grids[lev], amrex::IntVect::TheDimensionVector(idim)), + dmap[lev], AMREX_SPACEDIM, 0, amrex::MFInfo(), Factory(lev)); faces[lev][idim].define( - amrex::convert(grids[lev], IntVect::TheDimensionVector(idim)), - dmap[lev], AMREX_SPACEDIM, 0, MFInfo(), Factory(lev)); + amrex::convert(grids[lev], amrex::IntVect::TheDimensionVector(idim)), + dmap[lev], AMREX_SPACEDIM, 0, amrex::MFInfo(), Factory(lev)); } } @@ -66,12 +65,12 @@ PeleLM::computeVelocityAdvTerm(std::unique_ptr& advData) //---------------------------------------------------------------- // Get divU - MultiFab divu( - grids[lev], dmap[lev], 1, m_nGrowdivu, MFInfo(), Factory(lev)); + amrex::MultiFab divu( + grids[lev], dmap[lev], 1, m_nGrowdivu, amrex::MFInfo(), Factory(lev)); if (m_incompressible != 0) { divu.setVal(0.0); } else { - const Real time = getTime(lev, AmrOldTime); + const amrex::Real time = getTime(lev, AmrOldTime); fillpatch_divu(lev, time, divu, m_nGrowdivu); } @@ -83,11 +82,12 @@ PeleLM::computeVelocityAdvTerm(std::unique_ptr& advData) //---------------------------------------------------------------- // Compute the velocity fluxes #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { - Box const& bx = mfi.tilebox(); + amrex::Box const& bx = mfi.tilebox(); AMREX_D_TERM( auto const& umac = advData->umac[lev][0].const_array(mfi); , auto const& vmac = advData->umac[lev][1].const_array(mfi); @@ -116,7 +116,7 @@ PeleLM::computeVelocityAdvTerm(std::unique_ptr& advData) ebfact, (m_useEBinflow != 0) ? getEBState(mfi, lev, VELX, AMREX_SPACEDIM, AmrOldTime).const_array() - : Array4{}, + : amrex::Array4{}, #endif m_Godunov_ppm != 0, m_Godunov_ForceInTrans != 0, is_velocity, fluxes_are_area_weighted, m_advection_type, m_Godunov_ppm_limiter); @@ -151,12 +151,12 @@ PeleLM::computeVelocityAdvTerm(std::unique_ptr& advData) // Fluxes divergence to get the velocity advection term for (int lev = 0; lev <= finest_level; ++lev) { - MultiFab divu( - grids[lev], dmap[lev], 1, m_nGrowdivu, MFInfo(), Factory(lev)); + amrex::MultiFab divu( + grids[lev], dmap[lev], 1, m_nGrowdivu, amrex::MFInfo(), Factory(lev)); if (m_incompressible != 0) { divu.setVal(0.0); } else { - Real time = getTime(lev, AmrOldTime); + amrex::Real time = getTime(lev, AmrOldTime); fillpatch_divu(lev, time, divu, m_nGrowdivu); } @@ -166,8 +166,8 @@ PeleLM::computeVelocityAdvTerm(std::unique_ptr& advData) //---------------------------------------------------------------- // Use a temporary MF to hold divergence before redistribution int nGrow_divT = 3; - MultiFab divTmp( - grids[lev], dmap[lev], AMREX_SPACEDIM, nGrow_divT, MFInfo(), + amrex::MultiFab divTmp( + grids[lev], dmap[lev], AMREX_SPACEDIM, nGrow_divT, amrex::MFInfo(), EBFactory(lev)); divTmp.setVal(0.0); if (m_useEBinflow != 0) { @@ -206,20 +206,20 @@ PeleLM::updateVelocity(std::unique_ptr& advData) { //---------------------------------------------------------------- // Compute t^n divTau - Vector divtau(finest_level + 1); + amrex::Vector divtau(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { divtau[lev].define( - grids[lev], dmap[lev], AMREX_SPACEDIM, 0, MFInfo(), Factory(lev)); + grids[lev], dmap[lev], AMREX_SPACEDIM, 0, amrex::MFInfo(), Factory(lev)); } constexpr int use_density = 0; - const Real CrankNicholsonFactor = 0.5; + const amrex::Real CrankNicholsonFactor = 0.5; computeDivTau( AmrOldTime, GetVecOfPtrs(divtau), use_density, CrankNicholsonFactor); //---------------------------------------------------------------- // Get velocity forcing at half time including lagged grad P term constexpr int nGrow_force = 1; - Vector velForces(finest_level + 1); + amrex::Vector velForces(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { velForces[lev].define(grids[lev], dmap[lev], AMREX_SPACEDIM, nGrow_force); } @@ -239,17 +239,17 @@ PeleLM::updateVelocity(std::unique_ptr& advData) // velForce holds: 1/\rho^{n+1/2} [(gravity+...)^{n+1/2} - \nabla pi^{n} + // 0.5 * divTau^{n}] #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldataOld_p->state, TilingIfNotGPU()); mfi.isValid(); - ++mfi) { + for (amrex::MFIter mfi(ldataOld_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { - Box const& bx = mfi.tilebox(); + amrex::Box const& bx = mfi.tilebox(); auto const& vel_old = ldataOld_p->state.const_array(mfi, VELX); auto const& vel_aofs = advData->AofS[lev].const_array(mfi, VELX); auto const& force = velForces[lev].const_array(mfi); auto const& vel_new = ldataNew_p->state.array(mfi, VELX); - Real dt_loc = m_dt; + const amrex::Real dt_loc = m_dt; amrex::ParallelFor( bx, AMREX_SPACEDIM, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { @@ -276,12 +276,12 @@ PeleLM::getScalarAdvForce( auto const* leosparm = eos_parms.device_parm(); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(advData->Forcing[lev], TilingIfNotGPU()); mfi.isValid(); - ++mfi) { - const Box& bx = mfi.tilebox(); - FArrayBox DummyFab(bx, 1); + for (amrex::MFIter mfi(advData->Forcing[lev], amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); + amrex::FArrayBox DummyFab(bx, 1); auto const& rho = ldata_p->state.const_array(mfi, DENSITY); auto const& rhoY = ldata_p->state.const_array(mfi, FIRSTSPEC); auto const& T = ldata_p->state.const_array(mfi, TEMP); @@ -297,11 +297,12 @@ PeleLM::getScalarAdvForce( auto const& dn_aux = (m_nAux > 0) ? diffData->Dn_aux[lev].const_array(mfi, 0) : DummyFab.const_array(); + const auto nAux = m_nAux; + const auto dp0dt = m_dp0dt; + const auto is_closed_ch = m_closed_chamber; + const auto do_react = m_do_react; amrex::ParallelFor( - bx, [rho, rhoY, T, dn, ddn, r, fY, fT, fAux, extRhoY, extRhoH, - aux_diffuse_d, dn_aux, nAux = m_nAux, dp0dt = m_dp0dt, - is_closed_ch = m_closed_chamber, do_react = m_do_react, - leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { buildAdvectionForcing( i, j, k, rho, rhoY, T, dn, ddn, r, extRhoY, extRhoH, dp0dt, is_closed_ch, do_react, fY, fT, fAux, dn_aux, aux_diffuse_d, nAux, @@ -349,18 +350,20 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) auto AdvTypeAux_d = convertToDeviceVector(AdvTypeAux); //---------------------------------------------------------------- // Create temporary containers - Vector> fluxes(finest_level + 1); - Vector> fluxes_aux(finest_level + 1); + amrex::Vector> fluxes( + finest_level + 1); + amrex::Vector> fluxes_aux( + finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { fluxes[lev][idim].define( - amrex::convert(grids[lev], IntVect::TheDimensionVector(idim)), - dmap[lev], NUM_SPECIES + 1, 0, MFInfo(), + amrex::convert(grids[lev], amrex::IntVect::TheDimensionVector(idim)), + dmap[lev], NUM_SPECIES + 1, 0, amrex::MFInfo(), Factory(lev)); // Species + RhoH if (m_nAux > 0) { fluxes_aux[lev][idim].define( - amrex::convert(grids[lev], IntVect::TheDimensionVector(idim)), - dmap[lev], m_nAux, 0, MFInfo(), + amrex::convert(grids[lev], amrex::IntVect::TheDimensionVector(idim)), + dmap[lev], m_nAux, 0, amrex::MFInfo(), Factory(lev)); // auxiliary fields } } @@ -375,16 +378,16 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) // Define edge state: Density + Species + RhoH + Temp int nGrow = 0; - Array edgeState; - Array edgeState_aux; + amrex::Array edgeState; + amrex::Array edgeState_aux; for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { edgeState[idim].define( - amrex::convert(grids[lev], IntVect::TheDimensionVector(idim)), - dmap[lev], NUM_SPECIES + 3, nGrow, MFInfo(), Factory(lev)); + amrex::convert(grids[lev], amrex::IntVect::TheDimensionVector(idim)), + dmap[lev], NUM_SPECIES + 3, nGrow, amrex::MFInfo(), Factory(lev)); if (m_nAux > 0) { edgeState_aux[idim].define( - amrex::convert(grids[lev], IntVect::TheDimensionVector(idim)), - dmap[lev], m_nAux, nGrow, MFInfo(), Factory(lev)); + amrex::convert(grids[lev], amrex::IntVect::TheDimensionVector(idim)), + dmap[lev], m_nAux, nGrow, amrex::MFInfo(), Factory(lev)); } } @@ -396,28 +399,30 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) //---------------------------------------------------------------- // Get divU - MultiFab divu( - grids[lev], dmap[lev], 1, m_nGrowdivu, MFInfo(), Factory(lev)); + amrex::MultiFab divu( + grids[lev], dmap[lev], 1, m_nGrowdivu, amrex::MFInfo(), Factory(lev)); if (m_incompressible != 0) { divu.setVal(0.0); } else { - MultiFab::Copy(divu, advData->mac_divu[lev], 0, 0, 1, m_nGrowdivu); + amrex::MultiFab::Copy(divu, advData->mac_divu[lev], 0, 0, 1, m_nGrowdivu); } //---------------------------------------------------------------- #ifdef AMREX_USE_EB // Get EBFact & areafrac const auto& ebfact = EBFactory(lev); - Array areafrac = ebfact.getAreaFrac(); + amrex::Array areafrac = + ebfact.getAreaFrac(); #endif // Get the species edge state and advection term #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { - Box const& bx = mfi.tilebox(); + amrex::Box const& bx = mfi.tilebox(); AMREX_D_TERM( auto const& umac = advData->umac[lev][0].const_array(mfi); , auto const& vmac = advData->umac[lev][1].const_array(mfi); @@ -449,7 +454,7 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) (m_useEBinflow != 0) ? getEBState(mfi, lev, FIRSTSPEC, NUM_SPECIES - NUM_IONS, AmrOldTime) .const_array() - : Array4{}, + : amrex::Array4{}, #endif m_Godunov_ppm, m_Godunov_ForceInTrans, is_velocity, fluxes_are_area_weighted, m_advection_type, m_Godunov_ppm_limiter); @@ -488,7 +493,7 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) (m_useEBinflow != 0) ? getEBState(mfi, lev, FIRSTSPEC + ion_idx, 1, AmrOldTime) .const_array() - : Array4{}, + : amrex::Array4{}, #endif m_Godunov_ppm, m_Godunov_ForceInTrans, is_velocity, fluxes_are_area_weighted, m_advection_type, m_Godunov_ppm_limiter); @@ -507,7 +512,7 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) (m_useEBinflow != 0) ? getEBState(mfi, lev, FIRSTSPEC, NUM_SPECIES, AmrOldTime) .const_array() - : Array4{}, + : amrex::Array4{}, #endif m_Godunov_ppm != 0, m_Godunov_ForceInTrans != 0, is_velocity, fluxes_are_area_weighted, m_advection_type, m_Godunov_ppm_limiter); @@ -516,11 +521,11 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) // get the edge state for auxiliary components if (m_nAux > 0) { #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->auxiliaries, TilingIfNotGPU()); mfi.isValid(); - ++mfi) { - Box const& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldata_p->auxiliaries, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + amrex::Box const& bx = mfi.tilebox(); AMREX_D_TERM( auto const& umac = advData->umac[lev][0].const_array(mfi); , auto const& vmac = advData->umac[lev][1].const_array(mfi); @@ -564,11 +569,12 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) // Get edge density by summing over the species #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { - Box const& bx = mfi.tilebox(); + amrex::Box const& bx = mfi.tilebox(); #ifdef AMREX_USE_EB auto const& flagfab = ebfact.getMultiEBCellFlagFab()[mfi]; @@ -576,21 +582,21 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) // Edge states for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - const Box& ebx = amrex::surroundingNodes(bx, idim); + const amrex::Box& ebx = amrex::surroundingNodes(bx, idim); auto const& rho_ed = edgeState[idim].array(mfi, 0); auto const& rhoY_ed = edgeState[idim].array(mfi, 1); #ifdef AMREX_USE_EB - if (flagfab.getType(ebx) == FabType::covered) { // Covered boxes + if (flagfab.getType(ebx) == amrex::FabType::covered) { // Covered boxes amrex::ParallelFor( - ebx, [rho_ed] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { rho_ed(i, j, k) = 0.0; }); - } else if (flagfab.getType(ebx) != FabType::regular) { // EB containing - // boxes + } else if ( + flagfab.getType(ebx) != amrex::FabType::regular) { // EB containing + // boxes const auto& afrac = areafrac[idim]->array(mfi); amrex::ParallelFor( - ebx, [rho_ed, rhoY_ed, - afrac] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { rho_ed(i, j, k) = 0.0; if (afrac(i, j, k) > 0.0) { // Uncovered faces pele::physics::PhysicsType::eos_type::RY2R( @@ -601,8 +607,7 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) #endif { amrex::ParallelFor( - ebx, - [rho_ed, rhoY_ed] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { pele::physics::PhysicsType::eos_type::RY2R( rhoY_ed.cellData(i, j, k), rho_ed(i, j, k)); }); @@ -612,11 +617,11 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) // Get the edge temperature #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - - Box const& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + amrex::Box const& bx = mfi.tilebox(); AMREX_D_TERM( auto const& umac = advData->umac[lev][0].const_array(mfi); , auto const& vmac = advData->umac[lev][1].const_array(mfi); @@ -647,7 +652,7 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) ebfact, (m_useEBinflow != 0) ? getEBState(mfi, lev, TEMP, 1, AmrOldTime).const_array() - : Array4{}, + : amrex::Array4{}, #endif m_Godunov_ppm != 0, m_Godunov_ForceInTrans != 0, is_velocity, fluxes_are_area_weighted, m_advection_type, m_Godunov_ppm_limiter); @@ -655,11 +660,11 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) // Get the edge RhoH states #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - - Box const& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + amrex::Box const& bx = mfi.tilebox(); auto const* leosparm = eos_parms.device_parm(); #ifdef AMREX_USE_EB @@ -667,23 +672,23 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) #endif for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - const Box& ebx = amrex::surroundingNodes(bx, idim); + const amrex::Box& ebx = amrex::surroundingNodes(bx, idim); auto const& rho = edgeState[idim].const_array(mfi, 0); auto const& rhoY = edgeState[idim].const_array(mfi, 1); auto const& T = edgeState[idim].const_array(mfi, NUM_SPECIES + 2); auto const& rhoHm = edgeState[idim].array(mfi, NUM_SPECIES + 1); #ifdef AMREX_USE_EB - if (flagfab.getType(ebx) == FabType::covered) { // Covered boxes + if (flagfab.getType(ebx) == amrex::FabType::covered) { // Covered boxes amrex::ParallelFor( - ebx, [rhoHm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { rhoHm(i, j, k) = 0.0; }); - } else if (flagfab.getType(ebx) != FabType::regular) { // EB containing - // boxes + } else if ( + flagfab.getType(ebx) != amrex::FabType::regular) { // EB containing + // boxes const auto& afrac = areafrac[idim]->array(mfi); amrex::ParallelFor( - ebx, [rho, rhoY, T, rhoHm, afrac, - leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (afrac(i, j, k) <= 0.0) { // Covered faces rhoHm(i, j, k) = 0.0; } else { @@ -694,8 +699,7 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) #endif { amrex::ParallelFor( - ebx, [rho, rhoY, T, rhoHm, - leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { getRHmixGivenTY(i, j, k, rho, rhoY, T, rhoHm, leosparm); }); } @@ -705,11 +709,11 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) // Finally get the RhoH advection term // Pass the Temp forces again here, but they aren't used. #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - - Box const& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + amrex::Box const& bx = mfi.tilebox(); AMREX_D_TERM( auto const& umac = advData->umac[lev][0].const_array(mfi); , auto const& vmac = advData->umac[lev][1].const_array(mfi); @@ -738,7 +742,7 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) ebfact, (m_useEBinflow != 0) ? getEBState(mfi, lev, RHOH, 1, AmrOldTime).const_array() - : Array4{}, + : amrex::Array4{}, #endif m_Godunov_ppm != 0, m_Godunov_ForceInTrans != 0, is_velocity, fluxes_are_area_weighted, m_advection_type, m_Godunov_ppm_limiter); @@ -798,10 +802,10 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) for (int lev = 0; lev <= finest_level; ++lev) { for (int n = 0; n < NUM_IONS; ++n) { int spec_idx = NUM_SPECIES - NUM_IONS + n; - Array, AMREX_SPACEDIM> ionFlux; + amrex::Array, AMREX_SPACEDIM> ionFlux; for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - ionFlux[idim].reset( - new MultiFab(fluxes[lev][idim], amrex::make_alias, spec_idx, 1)); + ionFlux[idim].reset(new amrex::MultiFab( + fluxes[lev][idim], amrex::make_alias, spec_idx, 1)); } average_face_to_cellcenter( *m_ionsFluxes[lev], n * AMREX_SPACEDIM, GetArrOfConstPtrs(ionFlux)); @@ -816,13 +820,12 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) fetchAdvTypeArray(FIRSTSPEC, NUM_SPECIES + 1); // Species+RhoH auto AdvTypeAll_d = convertToDeviceVector(AdvTypeAll); for (int lev = 0; lev <= finest_level; ++lev) { - - MultiFab divu( - grids[lev], dmap[lev], 1, m_nGrowdivu, MFInfo(), Factory(lev)); + amrex::MultiFab divu( + grids[lev], dmap[lev], 1, m_nGrowdivu, amrex::MFInfo(), Factory(lev)); if (m_incompressible != 0) { divu.setVal(0.0); } else { - Real time = getTime(lev, AmrOldTime); + amrex::Real time = getTime(lev, AmrOldTime); fillpatch_divu(lev, time, divu, m_nGrowdivu); } @@ -832,8 +835,8 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) //---------------------------------------------------------------- // Use a temporary MF to hold divergence before redistribution constexpr int nGrow_divTmp = 3; - MultiFab divTmp( - grids[lev], dmap[lev], NUM_SPECIES + 1, nGrow_divTmp, MFInfo(), + amrex::MultiFab divTmp( + grids[lev], dmap[lev], NUM_SPECIES + 1, nGrow_divTmp, amrex::MFInfo(), EBFactory(lev)); divTmp.setVal(0.0); if (m_useEBinflow != 0) { @@ -869,8 +872,9 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) // repeat process for auxiliaries if (m_nAux > 0) { - MultiFab divTmp_aux( - grids[lev], dmap[lev], m_nAux, nGrow_divTmp, MFInfo(), EBFactory(lev)); + amrex::MultiFab divTmp_aux( + grids[lev], dmap[lev], m_nAux, nGrow_divTmp, amrex::MFInfo(), + EBFactory(lev)); divTmp_aux.setVal(0.0); advFluxDivergence( lev, divTmp_aux, 0, divu, GetArrOfConstPtrs(fluxes_aux[lev]), 0, @@ -917,30 +921,30 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) FIRSTSPEC); }); } - Gpu::streamSynchronize(); + amrex::Gpu::streamSynchronize(); } void PeleLM::updateDensity(std::unique_ptr& advData) { for (int lev = 0; lev <= finest_level; ++lev) { - // Get MultiArrays auto const& sma_o = getLevelDataPtr(lev, AmrOldTime)->state.arrays(); auto const& sma_n = getLevelDataPtr(lev, AmrNewTime)->state.arrays(); auto aofsma = advData->AofS[lev].const_arrays(); auto extma = m_extSource[lev]->const_arrays(); + const auto dt = m_dt; amrex::ParallelFor( - advData->AofS[lev], [=, dt = m_dt] AMREX_GPU_DEVICE( - int box_no, int i, int j, int k) noexcept { + advData->AofS[lev], + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { sma_n[box_no](i, j, k, DENSITY) = sma_o[box_no](i, j, k, DENSITY) + dt * (aofsma[box_no](i, j, k, DENSITY) + extma[box_no](i, j, k, DENSITY)); }); } - Gpu::streamSynchronize(); + amrex::Gpu::streamSynchronize(); } void @@ -956,16 +960,18 @@ PeleLM::computePassiveAdvTerms( //---------------------------------------------------------------- // Create temporary containers - Vector> fluxes(finest_level + 1); - Vector> edgeState(finest_level + 1); + amrex::Vector> fluxes( + finest_level + 1); + amrex::Vector> edgeState( + finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { fluxes[lev][idim].define( - amrex::convert(grids[lev], IntVect::TheDimensionVector(idim)), - dmap[lev], ncomp, 0, MFInfo(), Factory(lev)); + amrex::convert(grids[lev], amrex::IntVect::TheDimensionVector(idim)), + dmap[lev], ncomp, 0, amrex::MFInfo(), Factory(lev)); edgeState[lev][idim].define( - amrex::convert(grids[lev], IntVect::TheDimensionVector(idim)), - dmap[lev], ncomp, 0, MFInfo(), Factory(lev)); + amrex::convert(grids[lev], amrex::IntVect::TheDimensionVector(idim)), + dmap[lev], ncomp, 0, amrex::MFInfo(), Factory(lev)); } } @@ -977,12 +983,12 @@ PeleLM::computePassiveAdvTerms( //---------------------------------------------------------------- // Get divU - MultiFab divu( - grids[lev], dmap[lev], 1, m_nGrowdivu, MFInfo(), Factory(lev)); + amrex::MultiFab divu( + grids[lev], dmap[lev], 1, m_nGrowdivu, amrex::MFInfo(), Factory(lev)); if (m_incompressible != 0) { divu.setVal(0.0); } else { - MultiFab::Copy(divu, advData->mac_divu[lev], 0, 0, 1, m_nGrowdivu); + amrex::MultiFab::Copy(divu, advData->mac_divu[lev], 0, 0, 1, m_nGrowdivu); } //---------------------------------------------------------------- @@ -993,10 +999,11 @@ PeleLM::computePassiveAdvTerms( // Get the passive variables edge state and advection term #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - Box const& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + amrex::Box const& bx = mfi.tilebox(); AMREX_D_TERM( auto const& umac = advData->umac[lev][0].const_array(mfi); , auto const& vmac = advData->umac[lev][1].const_array(mfi); @@ -1057,12 +1064,12 @@ PeleLM::computePassiveAdvTerms( auto AdvTypeAll_d = convertToDeviceVector(AdvTypeAll); for (int lev = 0; lev <= finest_level; ++lev) { - MultiFab divu( - grids[lev], dmap[lev], 1, m_nGrowdivu, MFInfo(), Factory(lev)); + amrex::MultiFab divu( + grids[lev], dmap[lev], 1, m_nGrowdivu, amrex::MFInfo(), Factory(lev)); if (m_incompressible != 0) { divu.setVal(0.0); } else { - Real time = getTime(lev, AmrOldTime); + amrex::Real time = getTime(lev, AmrOldTime); fillpatch_divu(lev, time, divu, m_nGrowdivu); } @@ -1072,8 +1079,9 @@ PeleLM::computePassiveAdvTerms( //---------------------------------------------------------------- // Use a temporary MF to hold divergence before redistribution int nGrow_divTmp = 3; - MultiFab divTmp( - grids[lev], dmap[lev], ncomp, nGrow_divTmp, MFInfo(), EBFactory(lev)); + amrex::MultiFab divTmp( + grids[lev], dmap[lev], ncomp, nGrow_divTmp, amrex::MFInfo(), + EBFactory(lev)); divTmp.setVal(0.0); advFluxDivergence( lev, divTmp, 0, divu, GetArrOfConstPtrs(fluxes[lev]), 0, @@ -1103,24 +1111,22 @@ PeleLM::updateScalarComp( std::unique_ptr& advData, int state_comp, int ncomp) { for (int lev = 0; lev <= finest_level; ++lev) { - // Get level data ptr auto* ldataOld_p = getLevelDataPtr(lev, AmrOldTime); auto* ldataNew_p = getLevelDataPtr(lev, AmrNewTime); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldataNew_p->state, TilingIfNotGPU()); mfi.isValid(); - ++mfi) { - Box const& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldataNew_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + amrex::Box const& bx = mfi.tilebox(); auto const& old_arr = ldataOld_p->state.const_array(mfi, state_comp); auto const& new_arr = ldataNew_p->state.array(mfi, state_comp); auto const& a_of_s = advData->AofS[lev].const_array(mfi, state_comp); auto const& ext = m_extSource[lev]->const_array(mfi, state_comp); + const auto dt = m_dt; amrex::ParallelFor( - bx, ncomp, - [old_arr, new_arr, a_of_s, ext, - dt = m_dt] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { new_arr(i, j, k, n) = old_arr(i, j, k, n) + dt * (a_of_s(i, j, k, n) + ext(i, j, k, n)); }); diff --git a/Source/PeleLMeX_BC.cpp b/Source/PeleLMeX_BC.cpp index 7c11dbed8..a9c21d351 100644 --- a/Source/PeleLMeX_BC.cpp +++ b/Source/PeleLMeX_BC.cpp @@ -51,9 +51,10 @@ int divu_bc[] = {amrex::BCType::int_dir, amrex::BCType::reflect_even, amrex::BCType::reflect_even, amrex::BCType::reflect_even}; // Following incflo rather than IAMR here -int force_bc[] = {BCType::int_dir, BCType::foextrap, BCType::foextrap, - BCType::foextrap, BCType::foextrap, BCType::foextrap, - BCType::foextrap, BCType::foextrap}; +int force_bc[] = {amrex::BCType::int_dir, amrex::BCType::foextrap, + amrex::BCType::foextrap, amrex::BCType::foextrap, + amrex::BCType::foextrap, amrex::BCType::foextrap, + amrex::BCType::foextrap, amrex::BCType::foextrap}; #ifdef PELE_USE_PLASMA int nE_bc[] = {amrex::BCType::int_dir, amrex::BCType::ext_dir, @@ -72,33 +73,33 @@ int soot_bc[] = {amrex::BCType::int_dir, amrex::BCType::ext_dir, amrex::BCType::ext_dir, amrex::BCType::ext_dir}; #endif -InterpBase* +amrex::InterpBase* PeleLM:: getInterpolator( // NOLINT(readability-convert-member-functions-to-static) int a_method) const { - InterpBase* mapper = nullptr; + amrex::InterpBase* mapper = nullptr; if (a_method == 0) { - mapper = &mf_pc_interp; + mapper = &amrex::mf_pc_interp; } else if (a_method == 1) { // // Get EB-aware interpolater when needed // #ifdef AMREX_USE_EB - mapper = (EBFactory(0).isAllRegular()) ? &mf_cell_cons_interp - : &eb_mf_cell_cons_interp; + mapper = (EBFactory(0).isAllRegular()) ? &amrex::mf_cell_cons_interp + : &amrex::eb_mf_cell_cons_interp; #else - mapper = &mf_cell_cons_interp; + mapper = &amrex::mf_cell_cons_interp; #endif } else if (a_method == 2) { #ifdef AMREX_USE_EB - Abort("Regrid interpolation method = 2 not available with EB !"); + amrex::Abort("Regrid interpolation method = 2 not available with EB !"); #else - mapper = &mf_linear_slope_minmax_interp; + mapper = &amrex::mf_linear_slope_minmax_interp; #endif } else { - Abort("Unknown interpolation method"); + amrex::Abort("Unknown interpolation method"); } return mapper; } @@ -109,7 +110,7 @@ PeleLM::setBoundaryConditions() // Initialize the BCRecs m_bcrec_state.resize(NVAR); - int sizeForceBC = std::max(AMREX_SPACEDIM, NUM_SPECIES + 2); + int sizeForceBC = amrex::max(AMREX_SPACEDIM, NUM_SPECIES + 2); m_bcrec_force.resize(sizeForceBC); m_bcrec_aux.resize(m_nAux); @@ -229,20 +230,20 @@ PeleLM::setBoundaryConditions() } } -Vector +amrex::Vector PeleLM::fetchBCRecArray(int scomp, int ncomp) { - Vector bc(ncomp); + amrex::Vector bc(ncomp); for (int comp = 0; comp < ncomp; comp++) { bc[comp] = m_bcrec_state[scomp + comp]; } return bc; } -Vector +amrex::Vector PeleLM::fetchBCRecAuxArray(int scomp, int ncomp) { - Vector bc(ncomp); + amrex::Vector bc(ncomp); for (int comp = 0; comp < ncomp; comp++) { bc[comp] = m_bcrec_aux[scomp + comp]; } @@ -269,7 +270,7 @@ PeleLM::fillPatchState(int lev, const TimeStamp& a_time) BL_PROFILE("PeleLMeX::fillPatchStateLev()"); auto* ldata_p = getLevelDataPtr(lev, a_time); - Real time = getTime(lev, a_time); + amrex::Real time = getTime(lev, a_time); fillpatch_state(lev, time, ldata_p->state, m_nGrowState); if (m_incompressible == 0) { @@ -286,7 +287,7 @@ PeleLM::fillPatchDensity(const TimeStamp& a_time) BL_PROFILE("PeleLMeX::fillPatchDensity()"); for (int lev = 0; lev <= finest_level; lev++) { auto* ldata_p = getLevelDataPtr(lev, a_time); - Real time = getTime(lev, a_time); + amrex::Real time = getTime(lev, a_time); fillpatch_density(lev, time, ldata_p->state, DENSITY, m_nGrowState); } } @@ -297,7 +298,7 @@ PeleLM::fillPatchSpecies(const TimeStamp& a_time) BL_PROFILE("PeleLMeX::fillPatchSpecies()"); for (int lev = 0; lev <= finest_level; lev++) { auto* ldata_p = getLevelDataPtr(lev, a_time); - Real time = getTime(lev, a_time); + amrex::Real time = getTime(lev, a_time); fillpatch_species(lev, time, ldata_p->state, FIRSTSPEC, m_nGrowState); } } @@ -308,7 +309,7 @@ PeleLM::fillPatchTemp(const TimeStamp& a_time) BL_PROFILE("PeleLMeX::fillPatchTemp()"); for (int lev = 0; lev <= finest_level; lev++) { auto* ldata_p = getLevelDataPtr(lev, a_time); - Real time = getTime(lev, a_time); + amrex::Real time = getTime(lev, a_time); fillpatch_temp(lev, time, ldata_p->state, TEMP, m_nGrowState); } } @@ -319,7 +320,7 @@ PeleLM::fillPatchAux(const TimeStamp& a_time) BL_PROFILE("PeleLMeX::fillPatchAux()"); for (int lev = 0; lev <= finest_level; lev++) { auto* ldata_p = getLevelDataPtr(lev, a_time); - Real time = getTime(lev, a_time); + amrex::Real time = getTime(lev, a_time); fillpatch_aux(lev, time, ldata_p->auxiliaries, m_nGrowState); } } @@ -331,7 +332,7 @@ PeleLM::fillPatchPhiV(const TimeStamp& a_time) BL_PROFILE("PeleLMeX::fillPatchPhiV()"); for (int lev = 0; lev <= finest_level; lev++) { auto ldata_p = getLevelDataPtr(lev, a_time); - Real time = getTime(lev, a_time); + amrex::Real time = getTime(lev, a_time); fillpatch_phiV(lev, time, ldata_p->state, PHIV, m_nGrowState); } } @@ -341,26 +342,27 @@ PeleLM::fillPatchPhiV(const TimeStamp& a_time) //----------------------------------------------------------------------------- // The following return a fillpatched MF ptr at a given level // Fill the entire state at once -std::unique_ptr -PeleLM::fillPatchState(int lev, Real a_time, int nGrow) +std::unique_ptr +PeleLM::fillPatchState(int lev, amrex::Real a_time, int nGrow) { BL_PROFILE("PeleLMeX::fillPatchState()"); - std::unique_ptr mf; + std::unique_ptr mf; if (m_incompressible != 0) { - mf = std::make_unique( - grids[lev], dmap[lev], AMREX_SPACEDIM, nGrow, MFInfo(), Factory(lev)); + mf = std::make_unique( + grids[lev], dmap[lev], AMREX_SPACEDIM, nGrow, amrex::MFInfo(), + Factory(lev)); } else { - mf = std::make_unique( - grids[lev], dmap[lev], NVAR, nGrow, MFInfo(), Factory(lev)); + mf = std::make_unique( + grids[lev], dmap[lev], NVAR, nGrow, amrex::MFInfo(), Factory(lev)); } fillpatch_state(lev, a_time, *mf, nGrow); return mf; } -std::unique_ptr -PeleLM::fillPatchReact(int lev, Real a_time, int nGrow) +std::unique_ptr +PeleLM::fillPatchReact(int lev, amrex::Real a_time, int nGrow) { BL_PROFILE("PeleLMeX::fillPatchReact()"); @@ -368,9 +370,9 @@ PeleLM::fillPatchReact(int lev, Real a_time, int nGrow) #ifdef PELE_USE_PLASMA IRsize += 1; #endif - std::unique_ptr mf; - mf = std::make_unique( - grids[lev], dmap[lev], IRsize, nGrow, MFInfo(), Factory(lev)); + std::unique_ptr mf; + mf = std::make_unique( + grids[lev], dmap[lev], IRsize, nGrow, amrex::MFInfo(), Factory(lev)); fillpatch_reaction(lev, a_time, *mf, nGrow); return mf; @@ -390,15 +392,15 @@ PeleLM::fillpatch_state( fillTurbInflow(a_state, VELX, lev, a_time); if (lev == 0) { - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> bndry_func( geom[lev], fetchBCRecArray(0, nCompState), PeleLMCCFillExtDirState{ lprobparm, lpmfdata, m_nAux, static_cast(turb_inflow.is_initialized())}); FillPatchSingleLevel( - a_state, IntVect(nGhost), a_time, + a_state, amrex::IntVect(nGhost), a_time, {&(m_leveldata_old[lev]->state), &(m_leveldata_new[lev]->state)}, {m_t_old[lev], m_t_new[lev]}, 0, 0, nCompState, geom[lev], bndry_func, 0); } else { @@ -406,22 +408,22 @@ PeleLM::fillpatch_state( // Interpolator auto* mapper = getInterpolator(); - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> crse_bndry_func( geom[lev - 1], fetchBCRecArray(0, nCompState), PeleLMCCFillExtDirState{ lprobparm, lpmfdata, m_nAux, static_cast(turb_inflow.is_initialized())}); - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> fine_bndry_func( geom[lev], fetchBCRecArray(0, nCompState), PeleLMCCFillExtDirState{ lprobparm, lpmfdata, m_nAux, static_cast(turb_inflow.is_initialized())}); FillPatchTwoLevels( - a_state, IntVect(nGhost), a_time, + a_state, amrex::IntVect(nGhost), a_time, {&(m_leveldata_old[lev - 1]->state), &(m_leveldata_new[lev - 1]->state)}, {m_t_old[lev - 1], m_t_new[lev - 1]}, {&(m_leveldata_old[lev]->state), &(m_leveldata_new[lev]->state)}, @@ -447,14 +449,14 @@ PeleLM::fillpatch_density( if (lev == 0) { // Density - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> bndry_func_rho( geom[lev], fetchBCRecArray(DENSITY, 1), PeleLMCCFillExtDirDens{ lprobparm, lpmfdata, m_nAux}); FillPatchSingleLevel( - a_density, IntVect(nGhost), a_time, + a_density, amrex::IntVect(nGhost), a_time, {&(m_leveldata_old[lev]->state), &(m_leveldata_new[lev]->state)}, {m_t_old[lev], m_t_new[lev]}, DENSITY, rho_comp, 1, geom[lev], bndry_func_rho, 0); @@ -465,20 +467,20 @@ PeleLM::fillpatch_density( auto* mapper = getInterpolator(); // Density - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> crse_bndry_func_rho( geom[lev - 1], fetchBCRecArray(DENSITY, 1), PeleLMCCFillExtDirDens{ lprobparm, lpmfdata, m_nAux}); - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> fine_bndry_func_rho( geom[lev], fetchBCRecArray(DENSITY, 1), PeleLMCCFillExtDirDens{ lprobparm, lpmfdata, m_nAux}); FillPatchTwoLevels( - a_density, IntVect(nGhost), a_time, + a_density, amrex::IntVect(nGhost), a_time, {&(m_leveldata_old[lev - 1]->state), &(m_leveldata_new[lev - 1]->state)}, {m_t_old[lev - 1], m_t_new[lev - 1]}, {&(m_leveldata_old[lev]->state), &(m_leveldata_new[lev]->state)}, @@ -502,14 +504,14 @@ PeleLM::fillpatch_species( if (lev == 0) { // Species - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> bndry_func( geom[lev], fetchBCRecArray(FIRSTSPEC, NUM_SPECIES), PeleLMCCFillExtDirSpec{ lprobparm, lpmfdata, m_nAux}); FillPatchSingleLevel( - a_species, IntVect(nGhost), a_time, + a_species, amrex::IntVect(nGhost), a_time, {&(m_leveldata_old[lev]->state), &(m_leveldata_new[lev]->state)}, {m_t_old[lev], m_t_new[lev]}, FIRSTSPEC, rhoY_comp, NUM_SPECIES, geom[lev], bndry_func, 0); @@ -519,20 +521,20 @@ PeleLM::fillpatch_species( auto* mapper = getInterpolator(); // Species - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> crse_bndry_func( geom[lev - 1], fetchBCRecArray(FIRSTSPEC, NUM_SPECIES), PeleLMCCFillExtDirSpec{ lprobparm, lpmfdata, m_nAux}); - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> fine_bndry_func( geom[lev], fetchBCRecArray(FIRSTSPEC, NUM_SPECIES), PeleLMCCFillExtDirSpec{ lprobparm, lpmfdata, m_nAux}); FillPatchTwoLevels( - a_species, IntVect(nGhost), a_time, + a_species, amrex::IntVect(nGhost), a_time, {&(m_leveldata_old[lev - 1]->state), &(m_leveldata_new[lev - 1]->state)}, {m_t_old[lev - 1], m_t_new[lev - 1]}, {&(m_leveldata_old[lev]->state), &(m_leveldata_new[lev]->state)}, @@ -554,14 +556,14 @@ PeleLM::fillpatch_temp( ProbParm const* lprobparm = prob_parm_d; auto const* lpmfdata = pmf_data.device_parm(); if (lev == 0) { - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> bndry_func( geom[lev], fetchBCRecArray(TEMP, 1), PeleLMCCFillExtDirTemp{ lprobparm, lpmfdata, m_nAux}); FillPatchSingleLevel( - a_temp, IntVect(nGhost), a_time, + a_temp, amrex::IntVect(nGhost), a_time, {&(m_leveldata_old[lev]->state), &(m_leveldata_new[lev]->state)}, {m_t_old[lev], m_t_new[lev]}, TEMP, temp_comp, 1, geom[lev], bndry_func, 0); @@ -570,20 +572,20 @@ PeleLM::fillpatch_temp( // Interpolator auto* mapper = getInterpolator(); - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> crse_bndry_func( geom[lev - 1], fetchBCRecArray(TEMP, 1), PeleLMCCFillExtDirTemp{ lprobparm, lpmfdata, m_nAux}); - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> fine_bndry_func( geom[lev], fetchBCRecArray(TEMP, 1), PeleLMCCFillExtDirTemp{ lprobparm, lpmfdata, m_nAux}); FillPatchTwoLevels( - a_temp, IntVect(nGhost), a_time, + a_temp, amrex::IntVect(nGhost), a_time, {&(m_leveldata_old[lev - 1]->state), &(m_leveldata_new[lev - 1]->state)}, {m_t_old[lev - 1], m_t_new[lev - 1]}, {&(m_leveldata_old[lev]->state), &(m_leveldata_new[lev]->state)}, @@ -603,14 +605,14 @@ PeleLM::fillpatch_aux( auto const* lpmfdata = pmf_data.device_parm(); if (lev == 0) { - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> bndry_func( geom[lev], fetchBCRecAuxArray(0, m_nAux), PeleLMCCFillExtDirAux{ lprobparm, lpmfdata, m_nAux}); FillPatchSingleLevel( - a_aux, IntVect(nGhost), a_time, + a_aux, amrex::IntVect(nGhost), a_time, {&(m_leveldata_old[lev]->auxiliaries), &(m_leveldata_new[lev]->auxiliaries)}, {m_t_old[lev], m_t_new[lev]}, 0, 0, m_nAux, geom[lev], bndry_func, 0); @@ -619,20 +621,20 @@ PeleLM::fillpatch_aux( // Interpolator auto* mapper = getInterpolator(); - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> crse_bndry_func( geom[lev - 1], fetchBCRecAuxArray(0, m_nAux), PeleLMCCFillExtDirAux{ lprobparm, lpmfdata, m_nAux}); - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> fine_bndry_func( geom[lev], fetchBCRecAuxArray(0, m_nAux), PeleLMCCFillExtDirAux{ lprobparm, lpmfdata, m_nAux}); FillPatchTwoLevels( - a_aux, IntVect(nGhost), a_time, + a_aux, amrex::IntVect(nGhost), a_time, {&(m_leveldata_old[lev - 1]->auxiliaries), &(m_leveldata_new[lev - 1]->auxiliaries)}, {m_t_old[lev - 1], m_t_new[lev - 1]}, @@ -659,14 +661,14 @@ PeleLM::fillpatch_phiV( ProbParm const* lprobparm = prob_parm_d; auto const* lpmfdata = pmf_data.device_parm(); if (lev == 0) { - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> bndry_func( geom[lev], fetchBCRecArray(PHIV, 1), PeleLMCCFillExtDirPhiV{ lprobparm, lpmfdata, m_nAux}); FillPatchSingleLevel( - a_temp, IntVect(nGhost), a_time, + a_temp, amrex::IntVect(nGhost), a_time, {&(m_leveldata_old[lev]->state), &(m_leveldata_new[lev]->state)}, {m_t_old[lev], m_t_new[lev]}, PHIV, phiV_comp, 1, geom[lev], bndry_func, 0); @@ -675,20 +677,20 @@ PeleLM::fillpatch_phiV( // Interpolator auto* mapper = getInterpolator(); - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> crse_bndry_func( geom[lev - 1], fetchBCRecArray(PHIV, 1), PeleLMCCFillExtDirPhiV{ lprobparm, lpmfdata, m_nAux}); - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> fine_bndry_func( geom[lev], fetchBCRecArray(PHIV, 1), PeleLMCCFillExtDirPhiV{ lprobparm, lpmfdata, m_nAux}); FillPatchTwoLevels( - a_temp, IntVect(nGhost), a_time, + a_temp, amrex::IntVect(nGhost), a_time, {&(m_leveldata_old[lev - 1]->state), &(m_leveldata_new[lev - 1]->state)}, {m_t_old[lev - 1], m_t_new[lev - 1]}, {&(m_leveldata_old[lev]->state), &(m_leveldata_new[lev]->state)}, @@ -705,10 +707,10 @@ PeleLM::fillpatch_divu( int lev, const amrex::Real a_time, amrex::MultiFab& a_divu, int nGhost) { if (lev == 0) { - PhysBCFunct> bndry_func( - geom[lev], {m_bcrec_divu}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + bndry_func(geom[lev], {m_bcrec_divu}, PeleLMCCFillExtDirDummy{m_nAux}); FillPatchSingleLevel( - a_divu, IntVect(nGhost), a_time, + a_divu, amrex::IntVect(nGhost), a_time, {&(m_leveldata_old[lev]->divu), &(m_leveldata_new[lev]->divu)}, {m_t_old[lev], m_t_new[lev]}, 0, 0, 1, geom[lev], bndry_func, 0); } else { @@ -716,12 +718,14 @@ PeleLM::fillpatch_divu( // Interpolator auto* mapper = getInterpolator(); - PhysBCFunct> crse_bndry_func( - geom[lev - 1], {m_bcrec_divu}, PeleLMCCFillExtDirDummy{m_nAux}); - PhysBCFunct> fine_bndry_func( - geom[lev], {m_bcrec_divu}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + crse_bndry_func( + geom[lev - 1], {m_bcrec_divu}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + fine_bndry_func( + geom[lev], {m_bcrec_divu}, PeleLMCCFillExtDirDummy{m_nAux}); FillPatchTwoLevels( - a_divu, IntVect(nGhost), a_time, + a_divu, amrex::IntVect(nGhost), a_time, {&(m_leveldata_old[lev - 1]->divu), &(m_leveldata_new[lev - 1]->divu)}, {m_t_old[lev - 1], m_t_new[lev - 1]}, {&(m_leveldata_old[lev]->divu), &(m_leveldata_new[lev]->divu)}, @@ -736,28 +740,32 @@ PeleLM::fillpatch_divu( // foextrap on domain BCs void PeleLM::fillpatch_forces( - Real a_time, Vector const& a_force, int nGrowForce) + amrex::Real a_time, + amrex::Vector const& a_force, + int nGrowForce) { AMREX_ASSERT(a_force[0]->nComp() <= m_bcrec_force.size()); const int nComp = a_force[0]->nComp(); int lev = 0; { - PhysBCFunct> bndry_func( - geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + bndry_func(geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); FillPatchSingleLevel( - *a_force[lev], IntVect(nGrowForce), a_time, {a_force[lev]}, {a_time}, 0, - 0, nComp, geom[lev], bndry_func, 0); + *a_force[lev], amrex::IntVect(nGrowForce), a_time, {a_force[lev]}, + {a_time}, 0, 0, nComp, geom[lev], bndry_func, 0); } for (lev = 1; lev <= finest_level; ++lev) { - PhysBCFunct> crse_bndry_func( - geom[lev - 1], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); - PhysBCFunct> fine_bndry_func( - geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); - Interpolater* mapper = &pc_interp; + amrex::PhysBCFunct> + crse_bndry_func( + geom[lev - 1], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + fine_bndry_func( + geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::Interpolater* mapper = &amrex::pc_interp; FillPatchTwoLevels( - *a_force[lev], IntVect(nGrowForce), a_time, {a_force[lev - 1]}, {a_time}, - {a_force[lev]}, {a_time}, 0, 0, nComp, geom[lev - 1], geom[lev], + *a_force[lev], amrex::IntVect(nGrowForce), a_time, {a_force[lev - 1]}, + {a_time}, {a_force[lev]}, {a_time}, 0, 0, nComp, geom[lev - 1], geom[lev], crse_bndry_func, 0, fine_bndry_func, 0, refRatio(lev - 1), mapper, {m_bcrec_force}, 0); } @@ -769,10 +777,10 @@ PeleLM::fillpatch_gradp( int lev, const amrex::Real a_time, amrex::MultiFab& a_gp, int nGhost) { if (lev == 0) { - PhysBCFunct> bndry_func( - geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + bndry_func(geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); FillPatchSingleLevel( - a_gp, IntVect(nGhost), a_time, + a_gp, amrex::IntVect(nGhost), a_time, {&(m_leveldata_old[lev]->gp), &(m_leveldata_new[lev]->gp)}, {m_t_old[lev], m_t_new[lev]}, 0, 0, AMREX_SPACEDIM, geom[lev], bndry_func, 0); @@ -781,12 +789,14 @@ PeleLM::fillpatch_gradp( // Interpolator auto* mapper = getInterpolator(); - PhysBCFunct> crse_bndry_func( - geom[lev - 1], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); - PhysBCFunct> fine_bndry_func( - geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + crse_bndry_func( + geom[lev - 1], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + fine_bndry_func( + geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); FillPatchTwoLevels( - a_gp, IntVect(nGhost), a_time, + a_gp, amrex::IntVect(nGhost), a_time, {&(m_leveldata_old[lev - 1]->gp), &(m_leveldata_new[lev - 1]->gp)}, {m_t_old[lev - 1], m_t_new[lev - 1]}, {&(m_leveldata_old[lev]->gp), &(m_leveldata_new[lev]->gp)}, @@ -802,25 +812,28 @@ PeleLM::fillpatch_reaction( int lev, const amrex::Real a_time, amrex::MultiFab& a_I_R, int nGhost) { if (lev == 0) { - PhysBCFunct> bndry_func( - geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + bndry_func(geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); FillPatchSingleLevel( - a_I_R, IntVect(nGhost), a_time, {&(m_leveldatareact[lev]->I_R)}, {a_time}, - 0, 0, nCompIR(), geom[lev], bndry_func, 0); + a_I_R, amrex::IntVect(nGhost), a_time, {&(m_leveldatareact[lev]->I_R)}, + {a_time}, 0, 0, nCompIR(), geom[lev], bndry_func, 0); } else { // Interpolator auto* mapper = getInterpolator(); - PhysBCFunct> crse_bndry_func( - geom[lev - 1], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); - PhysBCFunct> fine_bndry_func( - geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + crse_bndry_func( + geom[lev - 1], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + fine_bndry_func( + geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); FillPatchTwoLevels( - a_I_R, IntVect(nGhost), a_time, {&(m_leveldatareact[lev - 1]->I_R)}, - {a_time}, {&(m_leveldatareact[lev]->I_R)}, {a_time}, 0, 0, nCompIR(), - geom[lev - 1], geom[lev], crse_bndry_func, 0, fine_bndry_func, 0, - refRatio(lev - 1), mapper, {m_bcrec_force}, 0); + a_I_R, amrex::IntVect(nGhost), a_time, + {&(m_leveldatareact[lev - 1]->I_R)}, {a_time}, + {&(m_leveldatareact[lev]->I_R)}, {a_time}, 0, 0, nCompIR(), geom[lev - 1], + geom[lev], crse_bndry_func, 0, fine_bndry_func, 0, refRatio(lev - 1), + mapper, {m_bcrec_force}, 0); } } @@ -830,25 +843,29 @@ PeleLM::fillpatch_chemFunctCall( int lev, const amrex::Real a_time, amrex::MultiFab& a_fctC, int nGhost) { if (lev == 0) { - PhysBCFunct> bndry_func( - geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + bndry_func(geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); FillPatchSingleLevel( - a_fctC, IntVect(nGhost), a_time, {&(m_leveldatareact[lev]->functC)}, - {a_time}, 0, 0, 1, geom[lev], bndry_func, 0); + a_fctC, amrex::IntVect(nGhost), a_time, + {&(m_leveldatareact[lev]->functC)}, {a_time}, 0, 0, 1, geom[lev], + bndry_func, 0); } else { // Interpolator auto* mapper = getInterpolator(); - PhysBCFunct> crse_bndry_func( - geom[lev - 1], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); - PhysBCFunct> fine_bndry_func( - geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + crse_bndry_func( + geom[lev - 1], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + fine_bndry_func( + geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); FillPatchTwoLevels( - a_fctC, IntVect(nGhost), a_time, {&(m_leveldatareact[lev - 1]->functC)}, - {a_time}, {&(m_leveldatareact[lev]->functC)}, {a_time}, 0, 0, 1, - geom[lev - 1], geom[lev], crse_bndry_func, 0, fine_bndry_func, 0, - refRatio(lev - 1), mapper, {m_bcrec_force}, 0); + a_fctC, amrex::IntVect(nGhost), a_time, + {&(m_leveldatareact[lev - 1]->functC)}, {a_time}, + {&(m_leveldatareact[lev]->functC)}, {a_time}, 0, 0, 1, geom[lev - 1], + geom[lev], crse_bndry_func, 0, fine_bndry_func, 0, refRatio(lev - 1), + mapper, {m_bcrec_force}, 0); } } @@ -868,24 +885,25 @@ PeleLM::fillcoarsepatch_state( // Interpolator auto* mapper = getInterpolator(m_regrid_interp_method); - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> crse_bndry_func( geom[lev - 1], fetchBCRecArray(0, nCompState), PeleLMCCFillExtDirState{ lprobparm, lpmfdata, m_nAux, static_cast(turb_inflow.is_initialized())}); - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> fine_bndry_func( geom[lev], fetchBCRecArray(0, nCompState), PeleLMCCFillExtDirState{ lprobparm, lpmfdata, m_nAux, static_cast(turb_inflow.is_initialized())}); InterpFromCoarseLevel( - a_state, IntVect(nGhost), a_time, m_leveldata_new[lev - 1]->state, 0, 0, - nCompState, geom[lev - 1], geom[lev], crse_bndry_func, 0, fine_bndry_func, - 0, refRatio(lev - 1), mapper, fetchBCRecArray(0, nCompState), 0); + a_state, amrex::IntVect(nGhost), a_time, m_leveldata_new[lev - 1]->state, 0, + 0, nCompState, geom[lev - 1], geom[lev], crse_bndry_func, 0, + fine_bndry_func, 0, refRatio(lev - 1), mapper, + fetchBCRecArray(0, nCompState), 0); } // Fill the auxiliaries @@ -900,20 +918,23 @@ PeleLM::fillcoarsepatch_aux( // Interpolator auto* mapper = getInterpolator(m_regrid_interp_method); - PhysBCFunct>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> crse_bndry_func( geom[lev - 1], fetchBCRecAuxArray(0, m_nAux), PeleLMCCFillExtDirAux{ lprobparm, lpmfdata, m_nAux}); - PhysBCFunct>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> fine_bndry_func( geom[lev], fetchBCRecAuxArray(0, m_nAux), PeleLMCCFillExtDirAux{ lprobparm, lpmfdata, m_nAux}); InterpFromCoarseLevel( - a_aux, IntVect(nGhost), a_time, m_leveldata_new[lev - 1]->auxiliaries, 0, 0, - m_nAux, geom[lev - 1], geom[lev], crse_bndry_func, 0, fine_bndry_func, 0, - refRatio(lev - 1), mapper, fetchBCRecAuxArray(0, m_nAux), 0); + a_aux, amrex::IntVect(nGhost), a_time, + m_leveldata_new[lev - 1]->auxiliaries, 0, 0, m_nAux, geom[lev - 1], + geom[lev], crse_bndry_func, 0, fine_bndry_func, 0, refRatio(lev - 1), + mapper, fetchBCRecAuxArray(0, m_nAux), 0); } // Fill the grad P @@ -924,12 +945,14 @@ PeleLM::fillcoarsepatch_gradp( // Interpolator auto* mapper = getInterpolator(m_regrid_interp_method); - PhysBCFunct> crse_bndry_func( - geom[lev - 1], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); - PhysBCFunct> fine_bndry_func( - geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + crse_bndry_func( + geom[lev - 1], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + fine_bndry_func( + geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); InterpFromCoarseLevel( - a_gp, IntVect(nGhost), a_time, m_leveldata_new[lev - 1]->gp, 0, 0, + a_gp, amrex::IntVect(nGhost), a_time, m_leveldata_new[lev - 1]->gp, 0, 0, AMREX_SPACEDIM, geom[lev - 1], geom[lev], crse_bndry_func, 0, fine_bndry_func, 0, refRatio(lev - 1), mapper, {m_bcrec_force}, 0); } @@ -942,13 +965,14 @@ PeleLM::fillcoarsepatch_divu( // Interpolator auto* mapper = getInterpolator(m_regrid_interp_method); - PhysBCFunct> crse_bndry_func( - geom[lev - 1], {m_bcrec_divu}, PeleLMCCFillExtDirDummy{m_nAux}); - PhysBCFunct> fine_bndry_func( - geom[lev], {m_bcrec_divu}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + crse_bndry_func( + geom[lev - 1], {m_bcrec_divu}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + fine_bndry_func(geom[lev], {m_bcrec_divu}, PeleLMCCFillExtDirDummy{m_nAux}); InterpFromCoarseLevel( - a_divu, IntVect(nGhost), a_time, m_leveldata_new[lev - 1]->divu, 0, 0, 1, - geom[lev - 1], geom[lev], crse_bndry_func, 0, fine_bndry_func, 0, + a_divu, amrex::IntVect(nGhost), a_time, m_leveldata_new[lev - 1]->divu, 0, + 0, 1, geom[lev - 1], geom[lev], crse_bndry_func, 0, fine_bndry_func, 0, refRatio(lev - 1), mapper, {m_bcrec_divu}, 0); } @@ -960,12 +984,14 @@ PeleLM::fillcoarsepatch_reaction( // Interpolator auto* mapper = getInterpolator(m_regrid_interp_method); - PhysBCFunct> crse_bndry_func( - geom[lev - 1], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); - PhysBCFunct> fine_bndry_func( - geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + crse_bndry_func( + geom[lev - 1], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + fine_bndry_func( + geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); InterpFromCoarseLevel( - a_I_R, IntVect(nGhost), a_time, m_leveldatareact[lev - 1]->I_R, 0, 0, + a_I_R, amrex::IntVect(nGhost), a_time, m_leveldatareact[lev - 1]->I_R, 0, 0, nCompIR(), geom[lev - 1], geom[lev], crse_bndry_func, 0, fine_bndry_func, 0, refRatio(lev - 1), mapper, {m_bcrec_force}, 0); } @@ -978,24 +1004,26 @@ PeleLM::fillcoarsepatch_chemFunctCall( // Interpolator auto* mapper = getInterpolator(m_regrid_interp_method); - PhysBCFunct> crse_bndry_func( - geom[lev - 1], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); - PhysBCFunct> fine_bndry_func( - geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + crse_bndry_func( + geom[lev - 1], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + fine_bndry_func( + geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); InterpFromCoarseLevel( - a_fctC, IntVect(nGhost), a_time, m_leveldatareact[lev - 1]->functC, 0, 0, 1, - geom[lev - 1], geom[lev], crse_bndry_func, 0, fine_bndry_func, 0, + a_fctC, amrex::IntVect(nGhost), a_time, m_leveldatareact[lev - 1]->functC, + 0, 0, 1, geom[lev - 1], geom[lev], crse_bndry_func, 0, fine_bndry_func, 0, refRatio(lev - 1), mapper, {m_bcrec_force}, 0); } // Fill the inflow boundary of a velocity MF // used for velocity projection void -PeleLM::setInflowBoundaryVel(MultiFab& a_vel, int lev, TimeStamp a_time) +PeleLM::setInflowBoundaryVel(amrex::MultiFab& a_vel, int lev, TimeStamp a_time) { BL_PROFILE("PeleLMeX::setInflowBoundaryVel()"); - Real time = getTime(lev, a_time); + amrex::Real time = getTime(lev, a_time); // Create a dummy BCRec from Velocity BCRec keeping only Inflow and set the // other to bogus @@ -1003,15 +1031,15 @@ PeleLM::setInflowBoundaryVel(MultiFab& a_vel, int lev, TimeStamp a_time) amrex::Vector dummyVelBCRec(AMREX_SPACEDIM); for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { for (int idim2 = 0; idim2 < AMREX_SPACEDIM; idim2++) { - if (realVelBCRec[idim].lo(idim2) == BCType::ext_dir) { - dummyVelBCRec[idim].setLo(idim2, BCType::ext_dir); + if (realVelBCRec[idim].lo(idim2) == amrex::BCType::ext_dir) { + dummyVelBCRec[idim].setLo(idim2, amrex::BCType::ext_dir); } else { - dummyVelBCRec[idim].setLo(idim2, BCType::bogus); + dummyVelBCRec[idim].setLo(idim2, amrex::BCType::bogus); } - if (realVelBCRec[idim].hi(idim2) == BCType::ext_dir) { - dummyVelBCRec[idim].setHi(idim2, BCType::ext_dir); + if (realVelBCRec[idim].hi(idim2) == amrex::BCType::ext_dir) { + dummyVelBCRec[idim].setHi(idim2, amrex::BCType::ext_dir); } else { - dummyVelBCRec[idim].setHi(idim2, BCType::bogus); + dummyVelBCRec[idim].setHi(idim2, amrex::BCType::bogus); } } } @@ -1020,8 +1048,8 @@ PeleLM::setInflowBoundaryVel(MultiFab& a_vel, int lev, TimeStamp a_time) ProbParm const* lprobparm = prob_parm_d; auto const* lpmfdata = pmf_data.device_parm(); - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> bndry_func( geom[lev], dummyVelBCRec, PeleLMCCFillExtDirState{ @@ -1035,7 +1063,7 @@ PeleLM::setInflowBoundaryVel(MultiFab& a_vel, int lev, TimeStamp a_time) void PeleLM::fillTurbInflow( - MultiFab& a_vel, int vel_comp, int lev, const Real a_time) + amrex::MultiFab& a_vel, int vel_comp, int lev, const amrex::Real a_time) { if (turb_inflow.is_initialized()) { @@ -1050,11 +1078,12 @@ PeleLM::fillTurbInflow( amrex::Gpu::deviceToHost, probparmDD, probparmDD + 1, probparmDH); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(a_vel, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - Box const& bx = mfi.growntilebox(); - FArrayBox& data = a_vel[mfi]; + for (amrex::MFIter mfi(a_vel, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + amrex::Box const& bx = mfi.growntilebox(); + amrex::FArrayBox& data = a_vel[mfi]; for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) { diff --git a/Source/PeleLMeX_BCfill.H b/Source/PeleLMeX_BCfill.H index 07e3e0f0b..89b9b1a4d 100644 --- a/Source/PeleLMeX_BCfill.H +++ b/Source/PeleLMeX_BCfill.H @@ -6,8 +6,6 @@ #include #include -using namespace amrex; - template struct PeleLMCCFillExtDirState { @@ -59,7 +57,7 @@ struct PeleLMCCFillExtDirState for (int idir = 0; idir < AMREX_SPACEDIM; idir++) { // Low - for (int n = 0; n < std::min(numcomp, NVAR); n++) { + for (int n = 0; n < amrex::min(numcomp, NVAR); n++) { // bcnormal handles all the state components at once amrex::Real s_ext[NVAR] = {0.0}; @@ -93,7 +91,7 @@ struct PeleLMCCFillExtDirState } // High - for (int n = 0; n < std::min(numcomp, NVAR); n++) { + for (int n = 0; n < amrex::min(numcomp, NVAR); n++) { // bcnormal handles all the state components at once amrex::Real s_ext[NVAR] = {0.0}; diff --git a/Source/PeleLMeX_BCfillEB.H b/Source/PeleLMeX_BCfillEB.H index 399952981..1974f3e7e 100644 --- a/Source/PeleLMeX_BCfillEB.H +++ b/Source/PeleLMeX_BCfillEB.H @@ -6,8 +6,6 @@ #include #include -using namespace amrex; - template struct matchBCNormEB_sign { diff --git a/Source/PeleLMeX_Data.cpp b/Source/PeleLMeX_Data.cpp index e5c4c7b94..5cd358993 100644 --- a/Source/PeleLMeX_Data.cpp +++ b/Source/PeleLMeX_Data.cpp @@ -1,11 +1,9 @@ #include -using namespace amrex; - PeleLM::LevelData::LevelData( amrex::BoxArray const& ba, amrex::DistributionMapping const& dm, - amrex::FabFactory const& factory, + amrex::FabFactory const& factory, int a_incompressible, int a_has_divu, int a_nAux, @@ -14,77 +12,79 @@ PeleLM::LevelData::LevelData( int a_do_les) { if (a_incompressible != 0) { - state.define(ba, dm, AMREX_SPACEDIM, a_nGrowState, MFInfo(), factory); + state.define( + ba, dm, AMREX_SPACEDIM, a_nGrowState, amrex::MFInfo(), factory); } else { - state.define(ba, dm, NVAR, a_nGrowState, MFInfo(), factory); + state.define(ba, dm, NVAR, a_nGrowState, amrex::MFInfo(), factory); } - gp.define(ba, dm, AMREX_SPACEDIM, 0, MFInfo(), factory); + gp.define(ba, dm, AMREX_SPACEDIM, 0, amrex::MFInfo(), factory); press.define( - amrex::convert(ba, IntVect::TheNodeVector()), dm, 1, 1, MFInfo(), factory); - visc_cc.define(ba, dm, 1, 1, MFInfo(), factory); + amrex::convert(ba, amrex::IntVect::TheNodeVector()), dm, 1, 1, + amrex::MFInfo(), factory); + visc_cc.define(ba, dm, 1, 1, amrex::MFInfo(), factory); if (a_do_les != 0) { for (int i = 0; i < AMREX_SPACEDIM; ++i) { visc_turb_fc[i].define( - amrex::convert(ba, IntVect::TheDimensionVector(i)), dm, 1, 0, MFInfo(), - factory); + amrex::convert(ba, amrex::IntVect::TheDimensionVector(i)), dm, 1, 0, + amrex::MFInfo(), factory); if (a_incompressible == 0) { lambda_turb_fc[i].define( - amrex::convert(ba, IntVect::TheDimensionVector(i)), dm, 1, 0, - MFInfo(), factory); + amrex::convert(ba, amrex::IntVect::TheDimensionVector(i)), dm, 1, 0, + amrex::MFInfo(), factory); } } } if (a_incompressible == 0) { if (a_has_divu != 0) { - divu.define(ba, dm, 1, 1, MFInfo(), factory); + divu.define(ba, dm, 1, 1, amrex::MFInfo(), factory); } if (a_use_soret != 0) { - diff_cc.define(ba, dm, 2 * NUM_SPECIES + 2, 1, MFInfo(), factory); + diff_cc.define(ba, dm, 2 * NUM_SPECIES + 2, 1, amrex::MFInfo(), factory); } else { - diff_cc.define(ba, dm, NUM_SPECIES + 2, 1, MFInfo(), factory); + diff_cc.define(ba, dm, NUM_SPECIES + 2, 1, amrex::MFInfo(), factory); } #ifdef PELE_USE_PLASMA - diffE_cc.define(ba, dm, 1, 1, MFInfo(), factory); - mobE_cc.define(ba, dm, 1, 1, MFInfo(), factory); - mob_cc.define(ba, dm, NUM_IONS, 1, MFInfo(), factory); + diffE_cc.define(ba, dm, 1, 1, amrex::MFInfo(), factory); + mobE_cc.define(ba, dm, 1, 1, amrex::MFInfo(), factory); + mob_cc.define(ba, dm, NUM_IONS, 1, amrex::MFInfo(), factory); #endif } if (a_nAux > 0) { - auxiliaries.define(ba, dm, a_nAux, a_nGrowState, MFInfo(), factory); - diff_aux_cc.define(ba, dm, a_nAux, 1, MFInfo(), factory); + auxiliaries.define(ba, dm, a_nAux, a_nGrowState, amrex::MFInfo(), factory); + diff_aux_cc.define(ba, dm, a_nAux, 1, amrex::MFInfo(), factory); } } PeleLM::LevelDataReact::LevelDataReact( const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, - const amrex::FabFactory& factory) + const amrex::FabFactory& factory) { int IRsize = NUM_SPECIES; #ifdef PELE_USE_PLASMA IRsize += 1; #endif - I_R.define(ba, dm, IRsize, 0, MFInfo(), factory); - functC.define(ba, dm, 1, 0, MFInfo(), factory); + I_R.define(ba, dm, IRsize, 0, amrex::MFInfo(), factory); + functC.define(ba, dm, 1, 0, amrex::MFInfo(), factory); } #ifdef PELE_USE_PLASMA PeleLM::LevelDataNLSolve::LevelDataNLSolve( amrex::BoxArray const& ba, amrex::DistributionMapping const& dm, - amrex::FabFactory const& factory, + amrex::FabFactory const& factory, int a_nGrow) { - nlState.define(ba, dm, 2, a_nGrow, MFInfo(), factory); - nlResid.define(ba, dm, 2, a_nGrow, MFInfo(), factory); - backgroundCharge.define(ba, dm, 1, 0, MFInfo(), factory); + nlState.define(ba, dm, 2, a_nGrow, amrex::MFInfo(), factory); + nlResid.define(ba, dm, 2, a_nGrow, amrex::MFInfo(), factory); + backgroundCharge.define(ba, dm, 1, 0, amrex::MFInfo(), factory); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - const BoxArray& faceba = - amrex::convert(ba, IntVect::TheDimensionVector(idim)); - gPhiVOld[idim].define(faceba, dm, 1, 0, MFInfo(), factory); - uEffnE[idim].define(faceba, dm, 1, 0, MFInfo(), factory); - umac[idim].define(faceba, dm, 1, 0, MFInfo(), factory); + const amrex::BoxArray& faceba = + amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)); + gPhiVOld[idim].define(faceba, dm, 1, 0, amrex::MFInfo(), factory); + uEffnE[idim].define(faceba, dm, 1, 0, amrex::MFInfo(), factory); + umac[idim].define(faceba, dm, 1, 0, amrex::MFInfo(), factory); } } #endif @@ -93,7 +93,8 @@ PeleLM::AdvanceDiffData::AdvanceDiffData( int a_finestLevel, const amrex::Vector& ba, const amrex::Vector& dm, - const amrex::Vector>>& factory, + const amrex::Vector>>& + factory, int nGrowAdv, int a_use_wbar, int a_use_soret, @@ -107,14 +108,15 @@ PeleLM::AdvanceDiffData::AdvanceDiffData( // Define MFs for (int lev = 0; lev <= a_finestLevel; lev++) { Dnp1[lev].define( - ba[lev], dm[lev], NUM_SPECIES + 2, nGrowAdv, MFInfo(), *factory[lev]); + ba[lev], dm[lev], NUM_SPECIES + 2, nGrowAdv, amrex::MFInfo(), + *factory[lev]); } if (a_nAux > 0) { Dnp1_aux.resize(a_finestLevel + 1); for (int lev = 0; lev <= a_finestLevel; lev++) { Dnp1_aux[lev].define( - ba[lev], dm[lev], a_nAux, nGrowAdv, MFInfo(), *factory[lev]); + ba[lev], dm[lev], a_nAux, nGrowAdv, amrex::MFInfo(), *factory[lev]); } } } else { @@ -139,38 +141,43 @@ PeleLM::AdvanceDiffData::AdvanceDiffData( // Define MFs for (int lev = 0; lev <= a_finestLevel; lev++) { Dn[lev].define( - ba[lev], dm[lev], NUM_SPECIES + 2, nGrowAdv, MFInfo(), *factory[lev]); + ba[lev], dm[lev], NUM_SPECIES + 2, nGrowAdv, amrex::MFInfo(), + *factory[lev]); Dnp1[lev].define( - ba[lev], dm[lev], NUM_SPECIES + 2, nGrowAdv, MFInfo(), *factory[lev]); + ba[lev], dm[lev], NUM_SPECIES + 2, nGrowAdv, amrex::MFInfo(), + *factory[lev]); Dhat[lev].define( - ba[lev], dm[lev], NUM_SPECIES + 2, nGrowAdv, MFInfo(), *factory[lev]); + ba[lev], dm[lev], NUM_SPECIES + 2, nGrowAdv, amrex::MFInfo(), + *factory[lev]); if (a_use_wbar != 0) { Dwbar[lev].define( - ba[lev], dm[lev], NUM_SPECIES, nGrowAdv, MFInfo(), *factory[lev]); + ba[lev], dm[lev], NUM_SPECIES, nGrowAdv, amrex::MFInfo(), + *factory[lev]); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - const BoxArray& faceba = - amrex::convert(ba[lev], IntVect::TheDimensionVector(idim)); + const amrex::BoxArray& faceba = + amrex::convert(ba[lev], amrex::IntVect::TheDimensionVector(idim)); wbar_fluxes[lev][idim].define( - faceba, dm[lev], NUM_SPECIES, 0, MFInfo(), *factory[lev]); + faceba, dm[lev], NUM_SPECIES, 0, amrex::MFInfo(), *factory[lev]); } } if (a_use_soret != 0) { DT[lev].define( - ba[lev], dm[lev], NUM_SPECIES, nGrowAdv, MFInfo(), *factory[lev]); + ba[lev], dm[lev], NUM_SPECIES, nGrowAdv, amrex::MFInfo(), + *factory[lev]); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - const BoxArray& faceba = - amrex::convert(ba[lev], IntVect::TheDimensionVector(idim)); + const amrex::BoxArray& faceba = + amrex::convert(ba[lev], amrex::IntVect::TheDimensionVector(idim)); soret_fluxes[lev][idim].define( - faceba, dm[lev], NUM_SPECIES, 0, MFInfo(), *factory[lev]); + faceba, dm[lev], NUM_SPECIES, 0, amrex::MFInfo(), *factory[lev]); } } if (a_nAux > 0) { Dn_aux[lev].define( - ba[lev], dm[lev], a_nAux, nGrowAdv, MFInfo(), *factory[lev]); + ba[lev], dm[lev], a_nAux, nGrowAdv, amrex::MFInfo(), *factory[lev]); Dnp1_aux[lev].define( - ba[lev], dm[lev], a_nAux, nGrowAdv, MFInfo(), *factory[lev]); + ba[lev], dm[lev], a_nAux, nGrowAdv, amrex::MFInfo(), *factory[lev]); Dhat_aux[lev].define( - ba[lev], dm[lev], a_nAux, nGrowAdv, MFInfo(), *factory[lev]); + ba[lev], dm[lev], a_nAux, nGrowAdv, amrex::MFInfo(), *factory[lev]); } } } @@ -180,7 +187,8 @@ PeleLM::AdvanceAdvData::AdvanceAdvData( int a_finestLevel, const amrex::Vector& ba, const amrex::Vector& dm, - const amrex::Vector>>& factory, + const amrex::Vector>>& + factory, int a_incompressible, int a_nAux, int nGrowAdv, @@ -205,38 +213,39 @@ PeleLM::AdvanceAdvData::AdvanceAdvData( // Define MFs for (int lev = 0; lev <= a_finestLevel; lev++) { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - const BoxArray& faceba = - amrex::convert(ba[lev], IntVect::TheDimensionVector(idim)); + const amrex::BoxArray& faceba = + amrex::convert(ba[lev], amrex::IntVect::TheDimensionVector(idim)); umac[lev][idim].define( - faceba, dm[lev], 1, nGrowMAC, MFInfo(), *factory[lev]); + faceba, dm[lev], 1, nGrowMAC, amrex::MFInfo(), *factory[lev]); #ifdef PELE_USE_PLASMA uDrift[lev][idim].define( - faceba, dm[lev], NUM_IONS, nGrowMAC, MFInfo(), *factory[lev]); + faceba, dm[lev], NUM_IONS, nGrowMAC, amrex::MFInfo(), *factory[lev]); #endif } if (a_incompressible != 0) { AofS[lev].define( - ba[lev], dm[lev], AMREX_SPACEDIM, 0, MFInfo(), *factory[lev]); + ba[lev], dm[lev], AMREX_SPACEDIM, 0, amrex::MFInfo(), *factory[lev]); } else { - AofS[lev].define(ba[lev], dm[lev], NVAR, 0, MFInfo(), *factory[lev]); - chi[lev].define(ba[lev], dm[lev], 1, 1, MFInfo(), *factory[lev]); + AofS[lev].define( + ba[lev], dm[lev], NVAR, 0, amrex::MFInfo(), *factory[lev]); + chi[lev].define(ba[lev], dm[lev], 1, 1, amrex::MFInfo(), *factory[lev]); #ifdef PELE_USE_PLASMA Forcing[lev].define( - ba[lev], dm[lev], NUM_SPECIES + 2, nGrowAdv, MFInfo(), + ba[lev], dm[lev], NUM_SPECIES + 2, nGrowAdv, amrex::MFInfo(), *factory[lev]); // Species + TEMP + nE #else Forcing[lev].define( - ba[lev], dm[lev], NUM_SPECIES + 1, nGrowAdv, MFInfo(), + ba[lev], dm[lev], NUM_SPECIES + 1, nGrowAdv, amrex::MFInfo(), *factory[lev]); // Species + TEMP #endif mac_divu[lev].define( - ba[lev], dm[lev], 1, nGrowAdv, MFInfo(), *factory[lev]); + ba[lev], dm[lev], 1, nGrowAdv, amrex::MFInfo(), *factory[lev]); } if (a_nAux > 0) { AofS_aux[lev].define( - ba[lev], dm[lev], a_nAux, 0, MFInfo(), *factory[lev]); + ba[lev], dm[lev], a_nAux, 0, amrex::MFInfo(), *factory[lev]); Forcing_aux[lev].define( - ba[lev], dm[lev], a_nAux, nGrowAdv, MFInfo(), *factory[lev]); + ba[lev], dm[lev], a_nAux, nGrowAdv, amrex::MFInfo(), *factory[lev]); } } } @@ -247,21 +256,21 @@ PeleLM::copyStateNewToOld(int nGhost) AMREX_ASSERT(nGhost <= m_nGrowState); for (int lev = 0; lev <= finest_level; lev++) { if (m_incompressible != 0) { - MultiFab::Copy( + amrex::MultiFab::Copy( m_leveldata_old[lev]->state, m_leveldata_new[lev]->state, 0, 0, AMREX_SPACEDIM, nGhost); } else { - MultiFab::Copy( + amrex::MultiFab::Copy( m_leveldata_old[lev]->state, m_leveldata_new[lev]->state, 0, 0, NVAR, nGhost); if (m_has_divu != 0) { - MultiFab::Copy( + amrex::MultiFab::Copy( m_leveldata_old[lev]->divu, m_leveldata_new[lev]->divu, 0, 0, 1, - std::min(nGhost, 1)); + amrex::min(nGhost, 1)); } } if (m_nAux > 0) { - MultiFab::Copy( + amrex::MultiFab::Copy( m_leveldata_old[lev]->auxiliaries, m_leveldata_new[lev]->auxiliaries, 0, 0, m_nAux, nGhost); } @@ -272,9 +281,9 @@ void PeleLM::copyPressNewToOld() { for (int lev = 0; lev <= finest_level; lev++) { - MultiFab::Copy( + amrex::MultiFab::Copy( m_leveldata_old[lev]->press, m_leveldata_new[lev]->press, 0, 0, 1, 1); - MultiFab::Copy( + amrex::MultiFab::Copy( m_leveldata_old[lev]->gp, m_leveldata_new[lev]->gp, 0, 0, AMREX_SPACEDIM, 0); } @@ -286,21 +295,21 @@ PeleLM::copyStateOldToNew(int nGhost) AMREX_ASSERT(nGhost <= m_nGrowState); for (int lev = 0; lev <= finest_level; lev++) { if (m_incompressible != 0) { - MultiFab::Copy( + amrex::MultiFab::Copy( m_leveldata_new[lev]->state, m_leveldata_old[lev]->state, 0, 0, AMREX_SPACEDIM, nGhost); } else { - MultiFab::Copy( + amrex::MultiFab::Copy( m_leveldata_new[lev]->state, m_leveldata_old[lev]->state, 0, 0, NVAR, nGhost); if (m_has_divu != 0) { - MultiFab::Copy( + amrex::MultiFab::Copy( m_leveldata_new[lev]->divu, m_leveldata_old[lev]->divu, 0, 0, 1, - std::min(nGhost, 1)); + amrex::min(nGhost, 1)); } } if (m_nAux > 0) { - MultiFab::Copy( + amrex::MultiFab::Copy( m_leveldata_new[lev]->auxiliaries, m_leveldata_old[lev]->auxiliaries, 0, 0, m_nAux, nGhost); } @@ -311,26 +320,26 @@ void PeleLM::copyTransportOldToNew() { for (int lev = 0; lev <= finest_level; lev++) { - MultiFab::Copy( + amrex::MultiFab::Copy( m_leveldata_new[lev]->visc_cc, m_leveldata_old[lev]->visc_cc, 0, 0, 1, 1); if (m_incompressible == 0) { - MultiFab::Copy( + amrex::MultiFab::Copy( m_leveldata_new[lev]->diff_cc, m_leveldata_old[lev]->diff_cc, 0, 0, NUM_SPECIES + 2, 1); #ifdef PELE_USE_PLASMA - MultiFab::Copy( + amrex::MultiFab::Copy( m_leveldata_new[lev]->diffE_cc, m_leveldata_old[lev]->diffE_cc, 0, 0, 1, 1); - MultiFab::Copy( + amrex::MultiFab::Copy( m_leveldata_new[lev]->mobE_cc, m_leveldata_old[lev]->mobE_cc, 0, 0, 1, 1); - MultiFab::Copy( + amrex::MultiFab::Copy( m_leveldata_new[lev]->mob_cc, m_leveldata_old[lev]->mob_cc, 0, 0, NUM_IONS, 1); #endif } if (m_nAux > 0) { - MultiFab::Copy( + amrex::MultiFab::Copy( m_leveldata_new[lev]->diff_aux_cc, m_leveldata_old[lev]->diff_aux_cc, 0, 0, m_nAux, 1); } @@ -341,13 +350,27 @@ void PeleLM::copyDiffusionOldToNew(std::unique_ptr& diffData) { for (int lev = 0; lev <= finest_level; lev++) { - MultiFab::Copy( + amrex::MultiFab::Copy( diffData->Dnp1[lev], diffData->Dn[lev], 0, 0, NUM_SPECIES + 2, m_nGrowAdv); if (m_nAux > 0) { - MultiFab::Copy( + amrex::MultiFab::Copy( diffData->Dnp1_aux[lev], diffData->Dn_aux[lev], 0, 0, m_nAux, m_nGrowAdv); } } } + +bool +PeleLM::checkForNaNs() +{ + bool contains_nan = false; + for (int lev = 0; lev <= finest_level; lev++) { + if ( + m_leveldata_new[lev]->state.contains_nan() || + m_leveldata_new[lev]->state.contains_inf()) { + contains_nan = true; + } + } + return contains_nan; +} diff --git a/Source/PeleLMeX_Derive.cpp b/Source/PeleLMeX_Derive.cpp index d1d7a56c6..69c8ab103 100644 --- a/Source/PeleLMeX_Derive.cpp +++ b/Source/PeleLMeX_Derive.cpp @@ -1,15 +1,13 @@ #include #include -using namespace amrex; - PeleLMDeriveRec::PeleLMDeriveRec( std::string a_name, - IndexType result_type, + amrex::IndexType result_type, int nvar_derive, PeleLMDeriveFunc der_func, DeriveBoxMap box_map, - Interpolater* a_interp) + amrex::Interpolater* a_interp) : derive_name(std::move(a_name)), variable_names(), der_type(result_type), @@ -22,12 +20,12 @@ PeleLMDeriveRec::PeleLMDeriveRec( PeleLMDeriveRec::PeleLMDeriveRec( std::string a_name, - IndexType result_type, + amrex::IndexType result_type, int nvar_derive, - Vector& var_names, + amrex::Vector& var_names, PeleLMDeriveFunc der_func, DeriveBoxMap box_map, - Interpolater* a_interp) + amrex::Interpolater* a_interp) : derive_name(std::move(a_name)), variable_names(var_names), der_type(result_type), @@ -40,15 +38,14 @@ PeleLMDeriveRec::PeleLMDeriveRec( PeleLMDeriveRec::PeleLMDeriveRec( std::string a_name, - IndexType result_type, + amrex::IndexType result_type, int nvar_derive, DeriveBoxMap box_map, - Interpolater* a_interp) + amrex::Interpolater* a_interp) : derive_name(std::move(a_name)), variable_names(), der_type(result_type), n_derive(nvar_derive), - mapper(a_interp), bx_map(box_map) { @@ -56,16 +53,15 @@ PeleLMDeriveRec::PeleLMDeriveRec( PeleLMDeriveRec::PeleLMDeriveRec( std::string a_name, - IndexType result_type, + amrex::IndexType result_type, int nvar_derive, - Vector& var_names, + amrex::Vector& var_names, DeriveBoxMap box_map, - Interpolater* a_interp) + amrex::Interpolater* a_interp) : derive_name(std::move(a_name)), variable_names(var_names), der_type(result_type), n_derive(nvar_derive), - mapper(a_interp), bx_map(box_map) { @@ -84,7 +80,7 @@ PeleLMDeriveRec::name() const noexcept return derive_name; } -IndexType +amrex::IndexType PeleLMDeriveRec::deriveType() const noexcept { return der_type; @@ -102,7 +98,7 @@ PeleLMDeriveRec::boxMap() const noexcept return bx_map; } -Interpolater* +amrex::Interpolater* PeleLMDeriveRec::interp() const noexcept { return mapper; @@ -144,11 +140,11 @@ PeleLMDeriveList::PeleLMDeriveList() = default; void PeleLMDeriveList::add( const std::string& name, - IndexType result_type, + amrex::IndexType result_type, int nvar_der, PeleLMDeriveFunc der_func, PeleLMDeriveRec::DeriveBoxMap bx_map, - Interpolater* interp) + amrex::Interpolater* interp) { lst.emplace_back(name, result_type, nvar_der, der_func, bx_map, interp); } @@ -156,12 +152,12 @@ PeleLMDeriveList::add( void PeleLMDeriveList::add( const std::string& name, - IndexType result_type, + amrex::IndexType result_type, int nvar_der, - Vector& vars, + amrex::Vector& vars, PeleLMDeriveFunc der_func, PeleLMDeriveRec::DeriveBoxMap bx_map, - Interpolater* interp) + amrex::Interpolater* interp) { lst.emplace_back(name, result_type, nvar_der, vars, der_func, bx_map, interp); } @@ -169,10 +165,10 @@ PeleLMDeriveList::add( void PeleLMDeriveList::add( const std::string& name, - IndexType result_type, + amrex::IndexType result_type, int nvar_der, PeleLMDeriveRec::DeriveBoxMap bx_map, - Interpolater* interp) + amrex::Interpolater* interp) { lst.emplace_back(name, result_type, nvar_der, bx_map, interp); } @@ -180,11 +176,11 @@ PeleLMDeriveList::add( void PeleLMDeriveList::add( const std::string& name, - IndexType result_type, + amrex::IndexType result_type, int nvar_der, - Vector& vars, + amrex::Vector& vars, PeleLMDeriveRec::DeriveBoxMap bx_map, - Interpolater* interp) + amrex::Interpolater* interp) { lst.emplace_back(name, result_type, nvar_der, vars, bx_map, interp); } diff --git a/Source/PeleLMeX_DeriveFunc.cpp b/Source/PeleLMeX_DeriveFunc.cpp index a8dbf0b91..58f9c5734 100644 --- a/Source/PeleLMeX_DeriveFunc.cpp +++ b/Source/PeleLMeX_DeriveFunc.cpp @@ -11,24 +11,22 @@ #include #endif -using namespace amrex; - // // Extract temp // void pelelmex_dertemp( PeleLM* a_pelelm, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int ncomp, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& /*geom*/, - Real /*time*/, - const Vector& /*bcrec*/, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& /*geom*/, + amrex::Real /*time*/, + const amrex::Vector& /*bcrec*/, int /*level*/) { @@ -50,16 +48,16 @@ pelelmex_dertemp( void pelelmex_derheatrelease( PeleLM* a_pelelm, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int ncomp, - const FArrayBox& statefab, - const FArrayBox& reactfab, - const FArrayBox& /*pressfab*/, - const Geometry& /*geom*/, - Real /*time*/, - const Vector& /*bcrec*/, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& reactfab, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& /*geom*/, + amrex::Real /*time*/, + const amrex::Vector& /*bcrec*/, int /*level*/) { @@ -69,8 +67,8 @@ pelelmex_derheatrelease( AMREX_ASSERT(derfab.nComp() >= dcomp + ncomp); AMREX_ASSERT(!a_pelelm->m_incompressible); - FArrayBox EnthFab; - EnthFab.resize(bx, NUM_SPECIES, The_Async_Arena()); + amrex::FArrayBox EnthFab; + EnthFab.resize(bx, NUM_SPECIES, amrex::The_Async_Arena()); auto const temp = statefab.const_array(TEMP); auto const react = reactfab.const_array(0); @@ -92,16 +90,16 @@ pelelmex_derheatrelease( void pelelmex_dermassfrac( PeleLM* a_pelelm, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int ncomp, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& /*geom*/, - Real /*time*/, - const Vector& /*bcrec*/, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& /*geom*/, + amrex::Real /*time*/, + const amrex::Vector& /*bcrec*/, int /*level*/) { @@ -127,16 +125,16 @@ pelelmex_dermassfrac( void pelelmex_dermolefrac( PeleLM* a_pelelm, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int ncomp, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& /*geom*/, - Real /*time*/, - const Vector& /*bcrec*/, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& /*geom*/, + amrex::Real /*time*/, + const amrex::Vector& /*bcrec*/, int /*level*/) { amrex::ignore_unused(a_pelelm, ncomp); @@ -170,16 +168,16 @@ pelelmex_dermolefrac( void pelelmex_derrhomrhoy( PeleLM* a_pelelm, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int ncomp, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& /*geom*/, - Real /*time*/, - const Vector& /*bcrec*/, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& /*geom*/, + amrex::Real /*time*/, + const amrex::Vector& /*bcrec*/, int /*level*/) { @@ -206,23 +204,23 @@ pelelmex_derrhomrhoy( void pelelmex_deravgpress( PeleLM* /*a_pelelm*/, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int /*ncomp*/, - const FArrayBox& /*statefab*/, - const FArrayBox& /*reactfab*/, - const FArrayBox& pressfab, - const Geometry& /*geom*/, - Real /*time*/, - const Vector& /*bcrec*/, + const amrex::FArrayBox& /*statefab*/, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& pressfab, + const amrex::Geometry& /*geom*/, + amrex::Real /*time*/, + const amrex::Vector& /*bcrec*/, int /*level*/) { AMREX_ASSERT(derfab.box().contains(bx)); auto const in_dat = pressfab.array(); auto der = derfab.array(dcomp); - Real factor = 1.0 / (AMREX_D_TERM(2.0, *2.0, *2.0)); + amrex::Real factor = 1.0 / (AMREX_D_TERM(2.0, *2.0, *2.0)); amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { der(i, j, k) = factor * (in_dat(i + 1, j, k) + in_dat(i, j, k) @@ -243,16 +241,16 @@ pelelmex_deravgpress( void pelelmex_dermgvel( PeleLM* /*a_pelelm*/, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int /*ncomp*/, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& /*geom*/, - Real /*time*/, - const Vector& /*bcrec*/, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& /*geom*/, + amrex::Real /*time*/, + const amrex::Vector& /*bcrec*/, int /*level*/) { @@ -273,16 +271,16 @@ pelelmex_dermgvel( void pelelmex_dermgvort( PeleLM* /*a_pelelm*/, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int /*ncomp*/, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& geom, - Real /*time*/, - const Vector& /*bcrec*/, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& geom, + amrex::Real /*time*/, + const amrex::Vector& /*bcrec*/, int /*level*/) { @@ -295,16 +293,16 @@ pelelmex_dermgvort( auto const& vort_arr = derfab.array(dcomp); #ifdef AMREX_USE_EB - const auto& ebfab = static_cast(statefab); - const EBCellFlagFab& flags = ebfab.getEBCellFlagFab(); + const auto& ebfab = static_cast(statefab); + const amrex::EBCellFlagFab& flags = ebfab.getEBCellFlagFab(); auto typ = flags.getType(bx); - if (typ == FabType::covered) { + if (typ == amrex::FabType::covered) { amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { vort_arr(i, j, k) = 0.0; }); - } else if (typ == FabType::singlevalued) { + } else if (typ == amrex::FabType::singlevalued) { const auto& flag_fab = flags.const_array(); amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { constexpr amrex::Real c0 = -1.5; @@ -314,8 +312,9 @@ pelelmex_dermgvort( vort_arr(i, j, k) = 0.0; } else { // Define interpolation lambda - auto onesided = - [](const Real& v0, const Real& v1, const Real& v2) -> Real { + auto onesided = []( + const amrex::Real& v0, const amrex::Real& v1, + const amrex::Real& v2) -> amrex::Real { return c0 * v0 + c1 * v1 + c2 * v2; }; @@ -432,16 +431,16 @@ pelelmex_dermgvort( void pelelmex_dervort( PeleLM* /*a_pelelm*/, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int ncomp, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& geom, - Real /*time*/, - const Vector& /*bcrec*/, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& geom, + amrex::Real /*time*/, + const amrex::Vector& /*bcrec*/, int /*level*/) { @@ -458,17 +457,17 @@ pelelmex_dervort( auto const& vort_arr = derfab.array(dcomp); #ifdef AMREX_USE_EB - const auto& ebfab = static_cast(statefab); - const EBCellFlagFab& flags = ebfab.getEBCellFlagFab(); + const auto& ebfab = static_cast(statefab); + const amrex::EBCellFlagFab& flags = ebfab.getEBCellFlagFab(); auto typ = flags.getType(bx); - if (typ == FabType::covered) { + if (typ == amrex::FabType::covered) { amrex::ParallelFor( bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { vort_arr(i, j, k, n) = 0.0; }); - } else if (typ == FabType::singlevalued) { + } else if (typ == amrex::FabType::singlevalued) { const auto& flag_fab = flags.const_array(); amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { constexpr amrex::Real c0 = -1.5; @@ -480,8 +479,9 @@ pelelmex_dervort( } } else { // Define interpolation lambda - auto onesided = - [](const Real& v0, const Real& v1, const Real& v2) -> Real { + auto onesided = []( + const amrex::Real& v0, const amrex::Real& v1, + const amrex::Real& v2) -> amrex::Real { return c0 * v0 + c1 * v1 + c2 * v2; }; @@ -603,22 +603,22 @@ pelelmex_dervort( void pelelmex_dercoord( PeleLM* /*a_pelelm*/, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int ncomp, - const FArrayBox& + const amrex::FArrayBox& #ifdef AMREX_USE_EB statefab #else /*unused*/ #endif , - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& geom, - Real /*time*/, - const Vector& /*bcrec*/, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& geom, + amrex::Real /*time*/, + const amrex::Vector& /*bcrec*/, int /*level*/) { amrex::ignore_unused(ncomp); @@ -634,13 +634,13 @@ pelelmex_dercoord( #ifdef AMREX_USE_EB AMREX_ASSERT(statefab.box().contains(bx)); - const auto& ebfab = static_cast(statefab); - const EBCellFlagFab& flags = ebfab.getEBCellFlagFab(); + const auto& ebfab = static_cast(statefab); + const amrex::EBCellFlagFab& flags = ebfab.getEBCellFlagFab(); auto typ = flags.getType(bx); // Compute cell center coordinates even in covered boxes/cell. Only // modify the cell-center in cut cells - if (typ == FabType::singlevalued) { + if (typ == amrex::FabType::singlevalued) { const auto& flag_arr = flags.const_array(); const auto& ccent_fab = ebfab.getCentroidData(); const auto& ccent_arr = ccent_fab->const_array(); @@ -677,12 +677,12 @@ pelelmex_dercoord( void pelelmex_derQcrit( PeleLM* /*a_pelelm*/, - const Box& + const amrex::Box& #if AMREX_SPACEDIM == 3 bx #endif , - FArrayBox& + amrex::FArrayBox& #if AMREX_SPACEDIM == 3 derfab #endif @@ -693,20 +693,20 @@ pelelmex_derQcrit( #endif , int /*ncomp*/, - const FArrayBox& + const amrex::FArrayBox& #if AMREX_SPACEDIM == 3 statefab #endif , - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& #if AMREX_SPACEDIM == 3 geom #endif , - Real /*time*/, - const Vector& /*bcrec*/, + amrex::Real /*time*/, + const amrex::Vector& /*bcrec*/, int /*level*/) { @@ -720,16 +720,16 @@ pelelmex_derQcrit( auto const& qcrit_arr = derfab.array(dcomp); #ifdef AMREX_USE_EB - const auto& ebfab = static_cast(statefab); - const EBCellFlagFab& flags = ebfab.getEBCellFlagFab(); + const auto& ebfab = static_cast(statefab); + const amrex::EBCellFlagFab& flags = ebfab.getEBCellFlagFab(); auto typ = flags.getType(bx); - if (typ == FabType::covered) { + if (typ == amrex::FabType::covered) { amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { qcrit_arr(i, j, k) = 0.0; }); - } else if (typ == FabType::singlevalued) { + } else if (typ == amrex::FabType::singlevalued) { const auto& flag_fab = flags.const_array(); amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (flag_fab(i, j, k).isCovered()) { @@ -739,13 +739,14 @@ pelelmex_derQcrit( constexpr amrex::Real c0 = -1.5; constexpr amrex::Real c1 = 2.0; constexpr amrex::Real c2 = -0.5; - auto onesided = - [](const Real& v0, const Real& v1, const Real& v2) -> Real { + auto onesided = []( + const amrex::Real& v0, const amrex::Real& v1, + const amrex::Real& v2) -> amrex::Real { return c0 * v0 + c1 * v1 + c2 * v2; }; // Strain rate tensor - Array2D gradU; + amrex::Array2D gradU; if (!flag_fab(i, j, k).isConnected(1, 0, 0)) { gradU(0, 0) = -onesided( dat_arr(i, j, k, 0), dat_arr(i - 1, j, k, 0), @@ -858,8 +859,8 @@ pelelmex_derQcrit( qcrit_arr(i, j, k) = 0.0; for (int dim1 = 0; dim1 < AMREX_SPACEDIM; ++dim1) { for (int dim2 = 0; dim2 < AMREX_SPACEDIM; ++dim2) { - Real Ohm = 0.5 * (gradU(dim1, dim2) - gradU(dim2, dim1)); - Real Sij = 0.5 * (gradU(dim1, dim2) + gradU(dim2, dim1)); + amrex::Real Ohm = 0.5 * (gradU(dim1, dim2) - gradU(dim2, dim1)); + amrex::Real Sij = 0.5 * (gradU(dim1, dim2) + gradU(dim2, dim1)); if (dim1 == dim2) { Sij -= divU / AMREX_SPACEDIM; } @@ -873,7 +874,7 @@ pelelmex_derQcrit( { amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { // Strain rate tensor - Array2D gradU; + amrex::Array2D gradU; gradU(0, 0) = 0.5 * (dat_arr(i + 1, j, k, 0) - dat_arr(i - 1, j, k, 0)) * idx; gradU(0, 1) = @@ -902,8 +903,8 @@ pelelmex_derQcrit( qcrit_arr(i, j, k) = 0.0; for (int dim1 = 0; dim1 < AMREX_SPACEDIM; ++dim1) { for (int dim2 = 0; dim2 < AMREX_SPACEDIM; ++dim2) { - Real Ohm = 0.5 * (gradU(dim1, dim2) - gradU(dim2, dim1)); - Real Sij = 0.5 * (gradU(dim1, dim2) + gradU(dim2, dim1)); + amrex::Real Ohm = 0.5 * (gradU(dim1, dim2) - gradU(dim2, dim1)); + amrex::Real Sij = 0.5 * (gradU(dim1, dim2) + gradU(dim2, dim1)); if (dim1 == dim2) { Sij -= divU / AMREX_SPACEDIM; } @@ -921,16 +922,16 @@ pelelmex_derQcrit( void pelelmex_derkineticenergy( PeleLM* a_pelelm, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int /*ncomp*/, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& /*geom*/, - Real /*time*/, - const Vector& /*bcrec*/, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& /*geom*/, + amrex::Real /*time*/, + const amrex::Vector& /*bcrec*/, int /*level*/) { @@ -939,15 +940,14 @@ pelelmex_derkineticenergy( if (a_pelelm->m_incompressible != 0) { auto const vel = statefab.array(VELX); auto der = derfab.array(dcomp); - amrex::ParallelFor( - bx, [=, rho = a_pelelm->m_rho] AMREX_GPU_DEVICE( - int i, int j, int k) noexcept { - der(i, j, k) = 0.5 * rho * - (AMREX_D_TERM( - vel(i, j, k, 0) * vel(i, j, k, 0), - +vel(i, j, k, 1) * vel(i, j, k, 1), - +vel(i, j, k, 2) * vel(i, j, k, 2))); - }); + const auto rho = a_pelelm->m_rho; + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + der(i, j, k) = + 0.5 * rho * + (AMREX_D_TERM( + vel(i, j, k, 0) * vel(i, j, k, 0), +vel(i, j, k, 1) * vel(i, j, k, 1), + +vel(i, j, k, 2) * vel(i, j, k, 2))); + }); } else { auto const rho = statefab.array(DENSITY); auto const vel = statefab.array(VELX); @@ -968,16 +968,16 @@ pelelmex_derkineticenergy( void pelelmex_derenstrophy( PeleLM* a_pelelm, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int /*ncomp*/, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& geom, - Real /*time*/, - const Vector& /*bcrec*/, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& geom, + amrex::Real /*time*/, + const amrex::Vector& /*bcrec*/, int /*level*/) { @@ -988,78 +988,76 @@ pelelmex_derenstrophy( auto const& dat_arr = statefab.const_array(VELX); auto const& rho_arr = (a_pelelm->m_incompressible) != 0 - ? Array4{} + ? amrex::Array4{} : statefab.const_array(DENSITY); auto const& ens_arr = derfab.array(dcomp); #ifdef AMREX_USE_EB - const auto& ebfab = static_cast(statefab); - const EBCellFlagFab& flags = ebfab.getEBCellFlagFab(); + const auto& ebfab = static_cast(statefab); + const amrex::EBCellFlagFab& flags = ebfab.getEBCellFlagFab(); auto typ = flags.getType(bx); - if (typ == FabType::covered) { + if (typ == amrex::FabType::covered) { amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { ens_arr(i, j, k) = 0.0; }); - } else if (typ == FabType::singlevalued) { + } else if (typ == amrex::FabType::singlevalued) { const auto& flag_fab = flags.const_array(); - amrex::ParallelFor( - bx, - [=, incomp = a_pelelm->m_incompressible, - rho = a_pelelm->m_rho] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - constexpr amrex::Real c0 = -1.5; - constexpr amrex::Real c1 = 2.0; - constexpr amrex::Real c2 = -0.5; - if (flag_fab(i, j, k).isCovered()) { - ens_arr(i, j, k) = 0.0; - } else { - Real l_rho = rho; - if (incomp == 0) { - l_rho = rho_arr(i, j, k); - } - // Define interpolation lambda - auto onesided = - [](const Real& v0, const Real& v1, const Real& v2) -> Real { - return c0 * v0 + c1 * v1 + c2 * v2; - }; - - amrex::Real vx = 0.0; - amrex::Real uy = 0.0; + const auto incomp = a_pelelm->m_incompressible; + const auto rho = a_pelelm->m_rho; + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + constexpr amrex::Real c0 = -1.5; + constexpr amrex::Real c1 = 2.0; + constexpr amrex::Real c2 = -0.5; + if (flag_fab(i, j, k).isCovered()) { + ens_arr(i, j, k) = 0.0; + } else { + amrex::Real l_rho = rho; + if (incomp == 0) { + l_rho = rho_arr(i, j, k); + } + // Define interpolation lambda + auto onesided = []( + const amrex::Real& v0, const amrex::Real& v1, + const amrex::Real& v2) -> amrex::Real { + return c0 * v0 + c1 * v1 + c2 * v2; + }; + + amrex::Real vx = 0.0; + amrex::Real uy = 0.0; #if (AMREX_SPACEDIM == 2) - // Need to check if there are covered cells in neighbours -- - // -- if so, use one-sided difference computation (but still - // quadratic) - if (!flag_fab(i, j, k).isConnected(1, 0, 0)) { - vx = -onesided( - dat_arr(i, j, k, 1), dat_arr(i - 1, j, k, 1), - dat_arr(i - 2, j, k, 1)) * - idx; - } else if (!flag_fab(i, j, k).isConnected(-1, 0, 0)) { - vx = onesided( - dat_arr(i, j, k, 1), dat_arr(i + 1, j, k, 1), - dat_arr(i + 2, j, k, 1)) * - idx; - } else { - vx = - 0.5 * (dat_arr(i + 1, j, k, 1) - dat_arr(i - 1, j, k, 1)) * idx; - } - // Do the same in y-direction - if (!flag_fab(i, j, k).isConnected(0, 1, 0)) { - uy = -onesided( - dat_arr(i, j, k, 0), dat_arr(i, j - 1, k, 0), - dat_arr(i, j - 2, k, 0)) * - idy; - } else if (!flag_fab(i, j, k).isConnected(0, -1, 0)) { - uy = onesided( - dat_arr(i, j, k, 0), dat_arr(i, j + 1, k, 0), - dat_arr(i, j + 2, k, 0)) * - idy; - } else { - uy = - 0.5 * (dat_arr(i, j + 1, k, 0) - dat_arr(i, j - 1, k, 0)) * idy; - } - ens_arr(i, j, k) = 0.5 * l_rho * (vx - uy) * (vx - uy); + // Need to check if there are covered cells in neighbours -- + // -- if so, use one-sided difference computation (but still + // quadratic) + if (!flag_fab(i, j, k).isConnected(1, 0, 0)) { + vx = -onesided( + dat_arr(i, j, k, 1), dat_arr(i - 1, j, k, 1), + dat_arr(i - 2, j, k, 1)) * + idx; + } else if (!flag_fab(i, j, k).isConnected(-1, 0, 0)) { + vx = onesided( + dat_arr(i, j, k, 1), dat_arr(i + 1, j, k, 1), + dat_arr(i + 2, j, k, 1)) * + idx; + } else { + vx = 0.5 * (dat_arr(i + 1, j, k, 1) - dat_arr(i - 1, j, k, 1)) * idx; + } + // Do the same in y-direction + if (!flag_fab(i, j, k).isConnected(0, 1, 0)) { + uy = -onesided( + dat_arr(i, j, k, 0), dat_arr(i, j - 1, k, 0), + dat_arr(i, j - 2, k, 0)) * + idy; + } else if (!flag_fab(i, j, k).isConnected(0, -1, 0)) { + uy = onesided( + dat_arr(i, j, k, 0), dat_arr(i, j + 1, k, 0), + dat_arr(i, j + 2, k, 0)) * + idy; + } else { + uy = 0.5 * (dat_arr(i, j + 1, k, 0) - dat_arr(i, j - 1, k, 0)) * idy; + } + ens_arr(i, j, k) = 0.5 * l_rho * (vx - uy) * (vx - uy); #elif (AMREX_SPACEDIM == 3) amrex::Real wx = 0.0; @@ -1150,25 +1148,24 @@ pelelmex_derenstrophy( ((wy - vz) * (wy - vz) + (uz - wx) * (uz - wx) + (vx - uy) * (vx - uy)); #endif - } - }); + } + }); } else #endif { - amrex::ParallelFor( - bx, - [=, incomp = a_pelelm->m_incompressible, - rho = a_pelelm->m_rho] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - Real l_rho = rho; - if (incomp == 0) { - l_rho = rho_arr(i, j, k); - } + const auto incomp = a_pelelm->m_incompressible; + const auto rho = a_pelelm->m_rho; + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + amrex::Real l_rho = rho; + if (incomp == 0) { + l_rho = rho_arr(i, j, k); + } #if (AMREX_SPACEDIM == 2) - amrex::Real vx = - 0.5 * (dat_arr(i + 1, j, k, 1) - dat_arr(i - 1, j, k, 1)) * idx; - amrex::Real uy = - 0.5 * (dat_arr(i, j + 1, k, 0) - dat_arr(i, j - 1, k, 0)) * idy; - ens_arr(i, j, k) = 0.5 * l_rho * (vx - uy) * (vx - uy); + amrex::Real vx = + 0.5 * (dat_arr(i + 1, j, k, 1) - dat_arr(i - 1, j, k, 1)) * idx; + amrex::Real uy = + 0.5 * (dat_arr(i, j + 1, k, 0) - dat_arr(i, j - 1, k, 0)) * idy; + ens_arr(i, j, k) = 0.5 * l_rho * (vx - uy) * (vx - uy); #elif (AMREX_SPACEDIM == 3) amrex::Real vx = @@ -1190,7 +1187,7 @@ pelelmex_derenstrophy( ((wy - vz) * (wy - vz) + (uz - wx) * (uz - wx) + (vx - uy) * (vx - uy)); #endif - }); + }); } } @@ -1200,16 +1197,16 @@ pelelmex_derenstrophy( void pelelmex_dermixfrac( PeleLM* a_pelelm, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int ncomp, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& /*geom*/, - Real /*time*/, - const Vector& /*bcrec*/, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& /*geom*/, + amrex::Real /*time*/, + const amrex::Vector& /*bcrec*/, int /*level*/) { amrex::ignore_unused(ncomp); @@ -1233,16 +1230,14 @@ pelelmex_dermixfrac( fact_Bilger[n] = a_pelelm->spec_Bilger_fact[n]; } - amrex::ParallelFor( - bx, [density, rhoY, mixt_frac, fact_Bilger, Zox_lcl, - denom_inv] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - amrex::Real rho_inv = 1.0_rt / density(i, j, k); - mixt_frac(i, j, k) = 0.0_rt; - for (int n = 0; n < NUM_SPECIES; ++n) { - mixt_frac(i, j, k) += (rhoY(i, j, k, n) * fact_Bilger[n]) * rho_inv; - } - mixt_frac(i, j, k) = (mixt_frac(i, j, k) - Zox_lcl) * denom_inv; - }); + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + amrex::Real rho_inv = 1.0 / density(i, j, k); + mixt_frac(i, j, k) = 0.0; + for (int n = 0; n < NUM_SPECIES; ++n) { + mixt_frac(i, j, k) += (rhoY(i, j, k, n) * fact_Bilger[n]) * rho_inv; + } + mixt_frac(i, j, k) = (mixt_frac(i, j, k) - Zox_lcl) * denom_inv; + }); } // @@ -1251,16 +1246,16 @@ pelelmex_dermixfrac( void pelelmex_derprogvar( PeleLM* a_pelelm, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int ncomp, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& /*geom*/, - Real /*time*/, - const Vector& /*bcrec*/, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& /*geom*/, + amrex::Real /*time*/, + const amrex::Vector& /*bcrec*/, int /*level*/) { @@ -1286,21 +1281,20 @@ pelelmex_derprogvar( Cweights[n] = a_pelelm->m_Cweights[n]; } - amrex::ParallelFor( - bx, [=, revert = a_pelelm->m_Crevert] AMREX_GPU_DEVICE( - int i, int j, int k) noexcept { - amrex::Real rho_inv = 1.0_rt / density(i, j, k); - prog_var(i, j, k) = 0.0_rt; - for (int n = 0; n < NUM_SPECIES; ++n) { - prog_var(i, j, k) += (rhoY(i, j, k, n) * Cweights[n]) * rho_inv; - } - prog_var(i, j, k) += temp(i, j, k) * Cweights[NUM_SPECIES]; - if (revert != 0) { - prog_var(i, j, k) = 1.0 - (prog_var(i, j, k) - C0_lcl) * denom_inv; - } else { - prog_var(i, j, k) = (prog_var(i, j, k) - C0_lcl) * denom_inv; - } - }); + const auto revert = a_pelelm->m_Crevert; + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + amrex::Real rho_inv = 1.0 / density(i, j, k); + prog_var(i, j, k) = 0.0; + for (int n = 0; n < NUM_SPECIES; ++n) { + prog_var(i, j, k) += (rhoY(i, j, k, n) * Cweights[n]) * rho_inv; + } + prog_var(i, j, k) += temp(i, j, k) * Cweights[NUM_SPECIES]; + if (revert != 0) { + prog_var(i, j, k) = 1.0 - (prog_var(i, j, k) - C0_lcl) * denom_inv; + } else { + prog_var(i, j, k) = (prog_var(i, j, k) - C0_lcl) * denom_inv; + } + }); } // @@ -1309,16 +1303,16 @@ pelelmex_derprogvar( void pelelmex_dervisc( PeleLM* a_pelelm, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int ncomp, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& /*geom*/, - Real /*time*/, - const Vector& /*bcrec*/, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& /*geom*/, + amrex::Real /*time*/, + const amrex::Vector& /*bcrec*/, int /*level*/) { amrex::ignore_unused(ncomp); @@ -1327,17 +1321,15 @@ pelelmex_dervisc( AMREX_ASSERT(derfab.nComp() >= dcomp + ncomp); if (a_pelelm->m_incompressible != 0) { - derfab.setVal(a_pelelm->m_mu, bx, dcomp, 1); + derfab.setVal(a_pelelm->m_mu, bx, dcomp, 1); } else { auto const& rhoY = statefab.const_array(FIRSTSPEC); auto const& T = statefab.array(TEMP); auto der = derfab.array(dcomp); auto const* ltransparm = a_pelelm->trans_parms.device_parm(); - amrex::ParallelFor( - bx, [rhoY, T, der, - ltransparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - getVelViscosity(i, j, k, rhoY, T, der, ltransparm); - }); + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + getVelViscosity(i, j, k, rhoY, T, der, ltransparm); + }); } } @@ -1347,16 +1339,16 @@ pelelmex_dervisc( void pelelmex_derdiffc( PeleLM* a_pelelm, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int ncomp, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& /*geom*/, - Real /*time*/, - const Vector& /*bcrec*/, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& /*geom*/, + amrex::Real /*time*/, + const amrex::Vector& /*bcrec*/, int /*level*/) { amrex::ignore_unused(ncomp); @@ -1372,7 +1364,7 @@ pelelmex_derdiffc( bool do_fixed_Le = (a_pelelm->m_fixed_Le != 0); bool do_fixed_Pr = (a_pelelm->m_fixed_Pr != 0); bool do_soret = (a_pelelm->m_use_soret != 0); - FArrayBox dummies(bx, NUM_SPECIES + 2, The_Async_Arena()); + amrex::FArrayBox dummies(bx, NUM_SPECIES + 2, amrex::The_Async_Arena()); auto const& rhoY = statefab.const_array(FIRSTSPEC); auto const& T = statefab.array(TEMP); auto rhoD = derfab.array(dcomp); @@ -1384,14 +1376,11 @@ pelelmex_derdiffc( : dummies.array(2); // dummy for no soret amrex::Real LeInv = a_pelelm->m_Lewis_inv; amrex::Real PrInv = a_pelelm->m_Prandtl_inv; - amrex::ParallelFor( - bx, [do_fixed_Le, do_fixed_Pr, do_soret, LeInv, PrInv, rhoY, T, rhoD, - rhotheta, lambda, mu, ltransparm, - leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - getTransportCoeff( - i, j, k, do_fixed_Le, do_fixed_Pr, do_soret, LeInv, PrInv, rhoY, T, - rhoD, rhotheta, lambda, mu, ltransparm, leosparm); - }); + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + getTransportCoeff( + i, j, k, do_fixed_Le, do_fixed_Pr, do_soret, LeInv, PrInv, rhoY, T, rhoD, + rhotheta, lambda, mu, ltransparm, leosparm); + }); } // @@ -1400,16 +1389,16 @@ pelelmex_derdiffc( void pelelmex_derlambda( PeleLM* a_pelelm, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int ncomp, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& /*geom*/, - Real /*time*/, - const Vector& /*bcrec*/, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& /*geom*/, + amrex::Real /*time*/, + const amrex::Vector& /*bcrec*/, int /*level*/) { amrex::ignore_unused(ncomp); @@ -1419,7 +1408,7 @@ pelelmex_derlambda( bool do_fixed_Le = (a_pelelm->m_fixed_Le != 0); bool do_fixed_Pr = (a_pelelm->m_fixed_Pr != 0); bool do_soret = (a_pelelm->m_use_soret != 0); - FArrayBox dummies(bx, 2 * NUM_SPECIES + 1, The_Async_Arena()); + amrex::FArrayBox dummies(bx, 2 * NUM_SPECIES + 1, amrex::The_Async_Arena()); auto const& rhoY = statefab.const_array(FIRSTSPEC); auto const& T = statefab.array(TEMP); auto rhoD = dummies.array(1); @@ -1430,14 +1419,11 @@ pelelmex_derlambda( auto const* leosparm = a_pelelm->eos_parms.device_parm(); amrex::Real LeInv = a_pelelm->m_Lewis_inv; amrex::Real PrInv = a_pelelm->m_Prandtl_inv; - amrex::ParallelFor( - bx, [do_fixed_Le, do_fixed_Pr, do_soret, LeInv, PrInv, rhoY, T, rhoD, - rhotheta, lambda, mu, ltransparm, - leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - getTransportCoeff( - i, j, k, do_fixed_Le, do_fixed_Pr, do_soret, LeInv, PrInv, rhoY, T, - rhoD, rhotheta, lambda, mu, ltransparm, leosparm); - }); + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + getTransportCoeff( + i, j, k, do_fixed_Le, do_fixed_Pr, do_soret, LeInv, PrInv, rhoY, T, rhoD, + rhotheta, lambda, mu, ltransparm, leosparm); + }); } // @@ -1446,21 +1432,21 @@ pelelmex_derlambda( void pelelmex_derdmap( PeleLM* /*a_pelelm*/, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int /*ncomp*/, - const FArrayBox& /*statefab*/, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& /*geom*/, - Real /*time*/, - const Vector& /*bcrec*/, + const amrex::FArrayBox& /*statefab*/, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& /*geom*/, + amrex::Real /*time*/, + const amrex::Vector& /*bcrec*/, int /*level*/) { AMREX_ASSERT(derfab.box().contains(bx)); auto der = derfab.array(dcomp); - const int myrank = ParallelDescriptor::MyProc(); + const int myrank = amrex::ParallelDescriptor::MyProc(); amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { der(i, j, k) = myrank; }); @@ -1472,16 +1458,16 @@ pelelmex_derdmap( void pelelmex_derturbforcing( PeleLM* a_pelelm, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int ncomp, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& geom, - Real time, - const Vector& /*bcrec*/, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& geom, + amrex::Real time, + const amrex::Vector& /*bcrec*/, int /*level*/) { AMREX_ASSERT(derfab.box().contains(bx)); @@ -1490,17 +1476,17 @@ pelelmex_derturbforcing( AMREX_ASSERT(!a_pelelm->m_incompressible); // Need geom for forcing - GeometryData const& geomdata = geom.data(); - Array4 const& der = derfab.array(dcomp); + amrex::GeometryData const& geomdata = geom.data(); + amrex::Array4 const& der = derfab.array(dcomp); // Set derfab to zero first derfab.setVal(0.0, bx, dcomp, ncomp); - FArrayBox DummyFab(bx, 1); + amrex::FArrayBox DummyFab(bx, 1); // Declare a pointer for the density array view - Array4 rho = (a_pelelm->m_incompressible != 0) - ? DummyFab.const_array() - : statefab.const_array(DENSITY); + amrex::Array4 rho = (a_pelelm->m_incompressible != 0) + ? DummyFab.const_array() + : statefab.const_array(DENSITY); // call the function above to construct the forcing a_pelelm->turb_forcing.addTurbVelForces( @@ -1514,16 +1500,16 @@ pelelmex_derturbforcing( void pelelmex_dermaniout( PeleLM* a_pelelm, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int ncomp, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& /*geom*/, - Real /*time*/, - const Vector& /*bcrec*/, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& /*geom*/, + amrex::Real /*time*/, + const amrex::Vector& /*bcrec*/, int /*level*/) { auto& h_manf_data = diff --git a/Source/PeleLMeX_DeriveUserDefined.cpp b/Source/PeleLMeX_DeriveUserDefined.cpp index b38319017..6f843afcd 100644 --- a/Source/PeleLMeX_DeriveUserDefined.cpp +++ b/Source/PeleLMeX_DeriveUserDefined.cpp @@ -5,12 +5,10 @@ #include #include -using namespace amrex; - // // User-defined derived variables list // -Vector +amrex::Vector pelelmex_setuserderives() { return {"derUserDefine_null"}; // var_names; @@ -22,19 +20,19 @@ pelelmex_setuserderives() void pelelmex_deruserdef( PeleLM* /*a_pelelm*/, - const Box& /*bx*/, - FArrayBox& /*derfab*/, + const amrex::Box& /*bx*/, + amrex::FArrayBox& /*derfab*/, int /*dcomp*/, int /*ncomp*/, - const FArrayBox& /*statefab*/, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& /*geom*/, - Real /*time*/, - const Vector& /*bcrec*/, + const amrex::FArrayBox& /*statefab*/, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& /*geom*/, + amrex::Real /*time*/, + const amrex::Vector& /*bcrec*/, int /*level*/) { - Abort( + amrex::Abort( "Using derUserDefine derived requires providing a definition in local " "DeriveUserDefined.cpp"); } diff --git a/Source/PeleLMeX_Diagnostics.cpp b/Source/PeleLMeX_Diagnostics.cpp index 67a820b7a..781008988 100644 --- a/Source/PeleLMeX_Diagnostics.cpp +++ b/Source/PeleLMeX_Diagnostics.cpp @@ -1,15 +1,13 @@ #include -using namespace amrex; - void PeleLM::createDiagnostics() { std::string pele_prefix = "peleLM"; - ParmParse pp(pele_prefix); + amrex::ParmParse pp(pele_prefix); int n_diags = 0; n_diags = pp.countval("diagnostics"); - Vector diags; + amrex::Vector diags; if (n_diags > 0) { m_diagnostics.resize(n_diags); diags.resize(n_diags); @@ -17,7 +15,7 @@ PeleLM::createDiagnostics() for (int n = 0; n < n_diags; ++n) { pp.get("diagnostics", diags[n], n); std::string diag_prefix = pele_prefix + "." + diags[n]; - ParmParse ppd(diag_prefix); + amrex::ParmParse ppd(diag_prefix); std::string diag_type; ppd.get("type", diag_type); m_diagnostics[n] = DiagBase::create(diag_type); @@ -33,7 +31,7 @@ PeleLM::createDiagnostics() bool itexists = derive_lst.canDerive(v) || isStateVariable(v) || isReactVariable(v); if (!itexists) { - Abort("Field " + v + " is not available"); + amrex::Abort("Field " + v + " is not available"); } else { if (derive_lst.canDerive(v)) { const PeleLMDeriveRec* rec = derive_lst.get(v); @@ -41,7 +39,7 @@ PeleLM::createDiagnostics() std::string errmsg = "Diagnostics can't handle derived with more " "than 1 component at the moment.\n"; errmsg += "Add the desired components individually.\n"; - Abort(errmsg); + amrex::Abort(errmsg); } } } @@ -66,12 +64,12 @@ PeleLM::doDiagnostics() { BL_PROFILE("PeleLMeX::doDiagnostics()"); // Assemble a vector of MF containing the requested data - Vector> diagMFVec(finestLevel() + 1); + amrex::Vector> diagMFVec(finestLevel() + 1); for (int lev{0}; lev <= finestLevel(); ++lev) { - diagMFVec[lev] = - std::make_unique(grids[lev], dmap[lev], m_diagVars.size(), 1); + diagMFVec[lev] = std::make_unique( + grids[lev], dmap[lev], m_diagVars.size(), 1); for (int v{0}; v < m_diagVars.size(); ++v) { - std::unique_ptr mf; + std::unique_ptr mf; mf = derive(m_diagVars[v], m_cur_time, lev, 1); // If the variable is a derive component, get its index from the derive // multifab @@ -82,7 +80,7 @@ PeleLM::doDiagnostics() if (rec != nullptr) { mf_idx = rec->variableComp(m_diagVars[v]); } - MultiFab::Copy(*diagMFVec[lev], *mf, mf_idx, v, 1, 1); + amrex::MultiFab::Copy(*diagMFVec[lev], *mf, mf_idx, v, 1, 1); } } diff --git a/Source/PeleLMeX_Diffusion.cpp b/Source/PeleLMeX_Diffusion.cpp index b4ffa2588..c7739761e 100644 --- a/Source/PeleLMeX_Diffusion.cpp +++ b/Source/PeleLMeX_Diffusion.cpp @@ -10,8 +10,6 @@ #include #endif -using namespace amrex; - DiffusionOp* PeleLM::getDiffusionOp() { @@ -73,28 +71,30 @@ PeleLM::computeDifferentialDiffusionTerms( // [NUM_SPECIES] Temperature : - \lambda \nabla T // [NUM_SPECIES+1] DiffDiff : \sum_k ( h_k * \Flux_k ) constexpr int nGrow = 0; // No need for ghost face on fluxes - Vector> fluxes(finest_level + 1); - Vector> fluxes_aux(finest_level + 1); + amrex::Vector> fluxes( + finest_level + 1); + amrex::Vector> fluxes_aux( + finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { const auto& ba = grids[lev]; const auto& factory = Factory(lev); for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { fluxes[lev][idim].define( - amrex::convert(ba, IntVect::TheDimensionVector(idim)), dmap[lev], - NUM_SPECIES + 2, nGrow, MFInfo(), factory); + amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)), dmap[lev], + NUM_SPECIES + 2, nGrow, amrex::MFInfo(), factory); if (m_nAux > 0) { fluxes_aux[lev][idim].define( - amrex::convert(ba, IntVect::TheDimensionVector(idim)), dmap[lev], - m_nAux, nGrow, MFInfo(), factory); + amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)), + dmap[lev], m_nAux, nGrow, amrex::MFInfo(), factory); } } } #ifdef AMREX_USE_EB - Vector EBfluxes(finest_level + 1); + amrex::Vector EBfluxes(finest_level + 1); if (m_isothermalEB != 0) { for (int lev = 0; lev <= finest_level; ++lev) { EBfluxes[lev].define( - grids[lev], dmap[lev], 1, nGrow, MFInfo(), Factory(lev)); + grids[lev], dmap[lev], 1, nGrow, amrex::MFInfo(), Factory(lev)); } } #endif @@ -103,13 +103,13 @@ PeleLM::computeDifferentialDiffusionTerms( // Compute differential diffusion fluxes including correction velocity and // wbar term During initialization, don't bother getting the wbar fluxes // separately - Vector> wbarFluxVec = + amrex::Vector> wbarFluxVec = ((is_init != 0) || (m_use_wbar == 0)) - ? Vector>{} + ? amrex::Vector>{} : GetVecOfArrOfPtrs(diffData->wbar_fluxes); - Vector> soretFluxVec = + amrex::Vector> soretFluxVec = ((is_init != 0) || (m_use_soret == 0)) - ? Vector>{} + ? amrex::Vector>{} : GetVecOfArrOfPtrs(diffData->soret_fluxes); #ifdef AMREX_USE_EB @@ -130,7 +130,7 @@ PeleLM::computeDifferentialDiffusionTerms( // Factor for SDC is 0.5 is for Dn and -0.5 for Dnp1 if ( (m_sdcIter == 0 || m_sdcIter == m_nSDCmax) && (m_do_speciesBalance != 0)) { - const Real sdc_weight = (a_time == AmrOldTime) ? 0.5 : -0.5; + const amrex::Real sdc_weight = (a_time == AmrOldTime) ? 0.5 : -0.5; addRhoYFluxes(GetArrOfConstPtrs(fluxes[0]), geom[0], sdc_weight); } @@ -140,12 +140,12 @@ PeleLM::computeDifferentialDiffusionTerms( // [NUM_SPECIES] Temperature : \nabla \cdot (-\lambda \nabla T) // [NUM_SPECIES+1] Differential diff : \nabla \cdot \sum_k ( h_k * \Flux_k ) constexpr int intensiveFluxes = 1; // All the fluxes are intensive here - Vector diffTermVec = (a_time == AmrOldTime) - ? GetVecOfPtrs(diffData->Dn) - : GetVecOfPtrs(diffData->Dnp1); - Vector diffTermAuxVec = (a_time == AmrOldTime) - ? GetVecOfPtrs(diffData->Dn_aux) - : GetVecOfPtrs(diffData->Dnp1_aux); + amrex::Vector diffTermVec = + (a_time == AmrOldTime) ? GetVecOfPtrs(diffData->Dn) + : GetVecOfPtrs(diffData->Dnp1); + amrex::Vector diffTermAuxVec = + (a_time == AmrOldTime) ? GetVecOfPtrs(diffData->Dn_aux) + : GetVecOfPtrs(diffData->Dnp1_aux); #ifdef AMREX_USE_EB auto bcRecSpec = fetchBCRecArray(FIRSTSPEC, NUM_SPECIES); auto bcRecSpec_d = convertToDeviceVector(bcRecSpec); @@ -165,8 +165,9 @@ PeleLM::computeDifferentialDiffusionTerms( auto bcRecTemp = fetchBCRecArray(TEMP, 1); auto bcRecTemp_d = convertToDeviceVector(bcRecTemp); - Vector EBFluxesVec = - (m_isothermalEB) != 0 ? GetVecOfPtrs(EBfluxes) : Vector{}; + amrex::Vector EBFluxesVec = + (m_isothermalEB) != 0 ? GetVecOfPtrs(EBfluxes) + : amrex::Vector{}; fluxDivergenceRD( GetVecOfConstPtrs(getTempVect(a_time)), 0, diffTermVec, NUM_SPECIES, GetVecOfArrOfPtrs(fluxes), NUM_SPECIES, EBFluxesVec, 0, 1, intensiveFluxes, @@ -247,8 +248,9 @@ PeleLM::computeDifferentialDiffusionTerms( template void PeleLM::adjustSpeciesFluxes( - const Vector>& a_spfluxes, - Vector const& a_spec) + const amrex::Vector>& + a_spfluxes, + amrex::Vector const& a_spec) { BL_PROFILE("PeleLMeX::adjustSpeciesFluxes()"); @@ -258,7 +260,7 @@ PeleLM::adjustSpeciesFluxes( for (int lev = 0; lev <= finest_level; ++lev) { - const Box& domain = geom[lev].Domain(); + const amrex::Box& domain = geom[lev].Domain(); #ifdef AMREX_USE_EB //------------------------------------------------------------------------ @@ -267,16 +269,16 @@ PeleLM::adjustSpeciesFluxes( const auto& ba = a_spec[lev]->boxArray(); const auto& dm = a_spec[lev]->DistributionMap(); const auto& ebfact = EBFactory(lev); - Array edgstate{AMREX_D_DECL( - MultiFab( - amrex::convert(ba, IntVect::TheDimensionVector(0)), dm, NUM_SPECIES, - nGrow, MFInfo(), ebfact), - MultiFab( - amrex::convert(ba, IntVect::TheDimensionVector(1)), dm, NUM_SPECIES, - nGrow, MFInfo(), ebfact), - MultiFab( - amrex::convert(ba, IntVect::TheDimensionVector(2)), dm, NUM_SPECIES, - nGrow, MFInfo(), ebfact))}; + amrex::Array edgstate{AMREX_D_DECL( + amrex::MultiFab( + amrex::convert(ba, amrex::IntVect::TheDimensionVector(0)), dm, + NUM_SPECIES, nGrow, amrex::MFInfo(), ebfact), + amrex::MultiFab( + amrex::convert(ba, amrex::IntVect::TheDimensionVector(1)), dm, + NUM_SPECIES, nGrow, amrex::MFInfo(), ebfact), + amrex::MultiFab( + amrex::convert(ba, amrex::IntVect::TheDimensionVector(2)), dm, + NUM_SPECIES, nGrow, amrex::MFInfo(), ebfact))}; EB_interp_CellCentroid_to_FaceCentroid( *a_spec[lev], GetArrOfPtrs(edgstate), 0, 0, NUM_SPECIES, geom[lev], bcRecSpec); @@ -284,12 +286,13 @@ PeleLM::adjustSpeciesFluxes( #endif #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*a_spec[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*a_spec[lev], amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - const Box& ebx = mfi.nodaltilebox(idim); - const Box& edomain = amrex::surroundingNodes(domain, idim); + const amrex::Box& ebx = mfi.nodaltilebox(idim); + const amrex::Box& edomain = amrex::surroundingNodes(domain, idim); auto const& rhoY = a_spec[lev]->const_array(mfi); auto const& flux_dir = a_spfluxes[lev][idim]->array(mfi); @@ -299,12 +302,13 @@ PeleLM::adjustSpeciesFluxes( #ifdef AMREX_USE_EB auto const& flagfab = ebfact.getMultiEBCellFlagFab()[mfi]; - if (flagfab.getType(amrex::grow(ebx, 0)) != FabType::covered) { + if (flagfab.getType(amrex::grow(ebx, 0)) != amrex::FabType::covered) { // No cut cells in tile + nghost-cell width halo -> use non-eb routine - if (flagfab.getType(amrex::grow(ebx, nGrow)) == FabType::regular) { + if ( + flagfab.getType(amrex::grow(ebx, nGrow)) == + amrex::FabType::regular) { amrex::ParallelFor( - ebx, [idim, rhoY, flux_dir, edomain, bc_lo, - bc_hi] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { int idx[3] = {i, j, k}; bool on_lo = ((bc_lo == amrex::BCType::ext_dir) && @@ -318,9 +322,7 @@ PeleLM::adjustSpeciesFluxes( auto const& rhoYed_ar = edgstate[idim].const_array(mfi); auto const& areafrac_ar = areafrac[idim]->const_array(mfi); amrex::ParallelFor( - ebx, - [idim, rhoY, flux_dir, rhoYed_ar, areafrac_ar, edomain, bc_lo, - bc_hi] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { int idx[3] = {i, j, k}; bool on_lo = ((bc_lo == amrex::BCType::ext_dir) && @@ -336,8 +338,7 @@ PeleLM::adjustSpeciesFluxes( } #else amrex::ParallelFor( - ebx, [idim, rhoY, flux_dir, edomain, bc_lo, - bc_hi] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { int idx[3] = {i, j, k}; bool on_lo = ((bc_lo == amrex::BCType::ext_dir) && @@ -356,8 +357,9 @@ PeleLM::adjustSpeciesFluxes( template <> void PeleLM::adjustSpeciesFluxes( - const Vector>& /*a_spfluxes*/, - Vector const& /*a_spec*/) + const amrex::Vector< + amrex::Array>& /*a_spfluxes*/, + amrex::Vector const& /*a_spec*/) { // Manifold Model: "Species" don't sum to unity, so no need to adjust the // fluxes @@ -367,20 +369,23 @@ PeleLM::adjustSpeciesFluxes( void PeleLM::correctIsothermalBoundary( const TimeStamp& a_time, - const Vector& a_spec_boundary, - const Vector>& a_wbarfluxes, - const Vector>& a_soretfluxes) + const amrex::Vector& a_spec_boundary, + const amrex::Vector>& + a_wbarfluxes, + const amrex::Vector>& + a_soretfluxes) { BL_PROFILE("PeleLMeX::correctIsothermalBoundary()"); auto bcRecSpec = fetchBCRecArray(FIRSTSPEC, NUM_SPECIES); const bool need_explicit_fluxes = a_soretfluxes.empty(); - Vector> soretfluxes(finest_level + 1); + amrex::Vector> soretfluxes( + finest_level + 1); if (need_explicit_fluxes) { // need to fill the soret fluxes ourselves for (int lev = 0; lev <= finest_level; lev++) { for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - soretfluxes[lev][idim] = new MultiFab( - grids[lev], dmap[lev], NUM_SPECIES, 1, MFInfo(), Factory(lev)); + soretfluxes[lev][idim] = new amrex::MultiFab( + grids[lev], dmap[lev], NUM_SPECIES, 1, amrex::MFInfo(), Factory(lev)); soretfluxes[lev][idim]->setVal(0.0); } } @@ -390,7 +395,7 @@ PeleLM::correctIsothermalBoundary( } else { // have the lagged ones, alias to them for (int lev = 0; lev <= finest_level; lev++) { for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - soretfluxes[lev][idim] = new MultiFab( + soretfluxes[lev][idim] = new amrex::MultiFab( *a_soretfluxes[lev][idim], amrex::make_alias, 0, NUM_SPECIES); } } @@ -400,33 +405,32 @@ PeleLM::correctIsothermalBoundary( // Lets overwrite the boundaries with the fluxes for the inhomogeneous // Neumann solve // Get the edge centered diffusivities - MultiFab& ldata_beta_cc = ldata_p->diff_cc; - const Box& domain = geom[lev].Domain(); + amrex::MultiFab& ldata_beta_cc = ldata_p->diff_cc; + const amrex::Box& domain = geom[lev].Domain(); constexpr int doZeroVisc = 1; constexpr int addTurbContribution = 0; - Array beta_ec = getDiffusivity( + amrex::Array beta_ec = getDiffusivity( lev, 0, NUM_SPECIES, doZeroVisc, bcRecSpec, ldata_beta_cc, addTurbContribution); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_beta_cc, TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(ldata_beta_cc, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { const auto bc_lo = m_phys_bc.lo(idim); const auto bc_hi = m_phys_bc.hi(idim); - const Box& edomain = amrex::surroundingNodes(domain, idim); - const Box& ebx = mfi.nodaltilebox(idim); + const amrex::Box& edomain = amrex::surroundingNodes(domain, idim); + const amrex::Box& ebx = mfi.nodaltilebox(idim); auto const& flux_soret = soretfluxes[lev][idim]->const_array(mfi); auto const& rhoD_ec = beta_ec[idim].const_array(mfi); auto const& flux_wbar = (m_use_wbar != 0 && !need_explicit_fluxes) ? a_wbarfluxes[lev][idim]->const_array(mfi) : rhoD_ec; auto const& boundary_ar = a_spec_boundary[lev]->array(mfi); + const auto use_wbar = m_use_wbar; amrex::ParallelFor( - ebx, - [flux_wbar, flux_soret, rhoD_ec, boundary_ar, idim, edomain, bc_lo, - bc_hi, use_wbar = m_use_wbar, - need_explicit_fluxes] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { int idx[3] = {i, j, k}; bool on_lo = (bc_lo == BoundaryCondition::BCNoSlipWallIsotherm || bc_lo == BoundaryCondition::BCSlipWallIsotherm) && @@ -463,39 +467,43 @@ PeleLM::correctIsothermalBoundary( void PeleLM::computeDifferentialDiffusionFluxes( const TimeStamp& a_time, - const Vector>& a_fluxes, - const Vector& + const amrex::Vector>& a_fluxes, + const amrex::Vector& #ifdef AMREX_USE_EB a_EBfluxes #else /*unused*/ #endif , - const Vector>& a_wbarfluxes, - const Vector>& a_soretfluxes, - const Vector>& a_auxfluxes) + const amrex::Vector>& + a_wbarfluxes, + const amrex::Vector>& + a_soretfluxes, + const amrex::Vector>& + a_auxfluxes) { BL_PROFILE("PeleLMeX::computeDifferentialDiffusionFluxes()"); //---------------------------------------------------------------- - Vector spec_boundary; + amrex::Vector spec_boundary; if (m_soret_boundary_override != 0) { spec_boundary.resize(finest_level + 1); // this is the same regardless of lagged or not for (int lev = 0; lev <= finest_level; ++lev) { auto* ldata_p = getLevelDataPtr(lev, a_time); spec_boundary[lev].define( - grids[lev], dmap[lev], NUM_SPECIES, 1, MFInfo(), Factory(lev)); + grids[lev], dmap[lev], NUM_SPECIES, 1, amrex::MFInfo(), Factory(lev)); // if we have a mix of Dirichlet and Isothermal walls, we need to give // Dirichlet boundaries and divide by density since diffuse_scalar doesn't // touch this boundary MF - MultiFab::Copy( + amrex::MultiFab::Copy( spec_boundary[lev], ldata_p->state, FIRSTSPEC, 0, NUM_SPECIES, 1); for (int n = 0; n < NUM_SPECIES; n++) { - MultiFab::Divide(spec_boundary[lev], ldata_p->state, DENSITY, n, 1, 1); + amrex::MultiFab::Divide( + spec_boundary[lev], ldata_p->state, DENSITY, n, 1, 1); } } // correct the boundary values, pass empty soret & wbar to trigger explicit @@ -592,10 +600,11 @@ PeleLM::computeDifferentialDiffusionFluxes( if (m_isothermalEB != 0) { AMREX_ASSERT(!a_EBfluxes.empty()); // Set up EB dirichlet value and diffusivity - Vector EBdiff(finest_level + 1); + amrex::Vector EBdiff(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - EBdiff[lev].define(grids[lev], dmap[lev], 1, 0, MFInfo(), EBFactory(lev)); + EBdiff[lev].define( + grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), EBFactory(lev)); getEBDiff(lev, a_time, EBdiff[lev], NUM_SPECIES); } getDiffusionOp()->computeDiffFluxes( @@ -629,11 +638,11 @@ PeleLM::computeDifferentialDiffusionFluxes( //---------------------------------------------------------------- // Set covered faces to large dummy values to catch any usage for (int lev = 0; lev <= finest_level; ++lev) { - EB_set_covered_faces( + amrex::EB_set_covered_faces( {AMREX_D_DECL(a_fluxes[lev][0], a_fluxes[lev][1], a_fluxes[lev][2])}, 1.234e40); if (m_nAux > 0) { - EB_set_covered_faces( + amrex::EB_set_covered_faces( {AMREX_D_DECL( a_auxfluxes[lev][0], a_auxfluxes[lev][1], a_auxfluxes[lev][2])}, 1.234e40); @@ -644,12 +653,14 @@ PeleLM::computeDifferentialDiffusionFluxes( void PeleLM::addWbarTerm( - const Vector>& a_spfluxes, - const Vector>& a_spwbarfluxes, - Vector const& a_spec, - Vector const& a_rho, - Vector const& a_beta, - Vector const& a_boundary) + const amrex::Vector>& + a_spfluxes, + const amrex::Vector>& + a_spwbarfluxes, + amrex::Vector const& a_spec, + amrex::Vector const& a_rho, + amrex::Vector const& a_beta, + amrex::Vector const& a_boundary) { //------------------------------------------------------------------------ // if a container for wbar fluxes is provided, fill it @@ -658,24 +669,26 @@ PeleLM::addWbarTerm( //------------------------------------------------------------------------ // Compute Wbar on all the levels int nGrow = 1; // Need one ghost cell to compute gradWbar - Vector Wbar(finest_level + 1); - Vector Wbar_boundary; + amrex::Vector Wbar(finest_level + 1); + amrex::Vector Wbar_boundary; if (have_boundary != 0) { Wbar_boundary.resize(finest_level + 1); } auto const* leosparm = eos_parms.device_parm(); for (int lev = 0; lev <= finest_level; ++lev) { - Wbar[lev].define(grids[lev], dmap[lev], 1, nGrow, MFInfo(), Factory(lev)); + Wbar[lev].define( + grids[lev], dmap[lev], 1, nGrow, amrex::MFInfo(), Factory(lev)); if (have_boundary != 0) { Wbar_boundary[lev].define( - grids[lev], dmap[lev], 1, nGrow, MFInfo(), Factory(lev)); + grids[lev], dmap[lev], 1, nGrow, amrex::MFInfo(), Factory(lev)); } - const Box& domain = geom[lev].Domain(); + const amrex::Box& domain = geom[lev].Domain(); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(Wbar[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& gbx = mfi.growntilebox(); + for (amrex::MFIter mfi(Wbar[lev], amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + const amrex::Box& gbx = mfi.growntilebox(); auto const& rho_arr = a_rho[lev]->const_array(mfi); auto const& rhoY_arr = a_spec[lev]->const_array(mfi); auto const& Wbar_arr = Wbar[lev].array(mfi); @@ -684,11 +697,9 @@ PeleLM::addWbarTerm( auto const& Wbar_boundary_arr = (have_boundary != 0) ? Wbar_boundary[lev].array(mfi) : Wbar_arr; + const auto phys_bc = m_phys_bc; amrex::ParallelFor( - - gbx, [rho_arr, rhoY_arr, Wbar_arr, gradY_arr, Wbar_boundary_arr, domain, - have_boundary, phys_bc = m_phys_bc, - leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { getMwmixGivenRY(i, j, k, rho_arr, rhoY_arr, Wbar_arr, leosparm); if (have_boundary != 0) { // need to impose gradWbar on boundary for // computeGradient @@ -724,14 +735,15 @@ PeleLM::addWbarTerm( auto bcRecSpec = fetchBCRecArray(FIRSTSPEC, NUM_SPECIES); nGrow = 0; // No need for ghost face on fluxes - Vector> gradWbar(finest_level + 1); + amrex::Vector> gradWbar( + finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { const auto& ba = grids[lev]; const auto& factory = Factory(lev); for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { gradWbar[lev][idim].define( - amrex::convert(ba, IntVect::TheDimensionVector(idim)), dmap[lev], - NUM_SPECIES, nGrow, MFInfo(), factory); + amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)), dmap[lev], + NUM_SPECIES, nGrow, amrex::MFInfo(), factory); gradWbar[lev][idim].setVal(0.0); } } @@ -747,31 +759,31 @@ PeleLM::addWbarTerm( // Get edge diffusivity constexpr int doZeroVisc = 1; constexpr int addTurbContrib = 0; - Array beta_ec = getDiffusivity( + amrex::Array beta_ec = getDiffusivity( lev, 0, NUM_SPECIES, doZeroVisc, bcRecSpec, *a_beta[lev], addTurbContrib); - const Box& domain = geom[lev].Domain(); + const amrex::Box& domain = geom[lev].Domain(); const bool use_harmonic_avg = m_harm_avg_cen2edge != 0; #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif { - for (MFIter mfi(*a_beta[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*a_beta[lev], amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { // Get edge centered rhoYs - const Box ebx = mfi.nodaltilebox(idim); - FArrayBox rhoY_ed(ebx, NUM_SPECIES, The_Async_Arena()); + const amrex::Box ebx = mfi.nodaltilebox(idim); + amrex::FArrayBox rhoY_ed(ebx, NUM_SPECIES, amrex::The_Async_Arena()); - const Box& edomain = amrex::surroundingNodes(domain, idim); + const amrex::Box& edomain = amrex::surroundingNodes(domain, idim); auto const& rhoY_arr = a_spec[lev]->const_array(mfi); const auto& rhoYed_arr = rhoY_ed.array(0); const auto bc_lo = bcRecSpec[0].lo(idim); const auto bc_hi = bcRecSpec[0].hi(idim); amrex::ParallelFor( - ebx, [idim, bc_lo, bc_hi, use_harmonic_avg, rhoY_arr, rhoYed_arr, - edomain] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { int idx[3] = {i, j, k}; bool on_lo = ((bc_lo == amrex::BCType::ext_dir) && @@ -796,11 +808,9 @@ PeleLM::addWbarTerm( // Wbar flux is : - \rho Y_m / W_k * D_m * \nabla // \overline{W} with beta_m = \rho * D_m * overline(W) / W_k below // need to divide by \overline(W) + const auto* eosparm = leosparm; amrex::ParallelFor( - ebx, [need_wbar_fluxes, gradWbar_ar, beta_ar, rhoY, spFlux_ar, - spwbarFlux_ar, - eosparm = - leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { auto eos = pele::physics::PhysicsType::eos(eosparm); // Get Wbar from rhoYs amrex::Real rho = 0.0; @@ -834,10 +844,12 @@ PeleLM::addWbarTerm( void PeleLM::addSoretTerm( - const Vector>& a_spfluxes, - const Vector>& a_spsoretfluxes, - Vector const& a_temp, - Vector const& a_beta) + const amrex::Vector>& + a_spfluxes, + const amrex::Vector>& + a_spsoretfluxes, + amrex::Vector const& a_temp, + amrex::Vector const& a_beta) { //------------------------------------------------------------------------ // if a container for soret fluxes is provided, fill it @@ -851,14 +863,15 @@ PeleLM::addSoretTerm( auto bcRecSpec = fetchBCRecArray(FIRSTSPEC, NUM_SPECIES); constexpr int nGrow = 0; // No need for ghost face on fluxes - Vector> gradT(finest_level + 1); + amrex::Vector> gradT( + finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { const auto& ba = grids[lev]; const auto& factory = Factory(lev); for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { gradT[lev][idim].define( - amrex::convert(ba, IntVect::TheDimensionVector(idim)), dmap[lev], 1, - nGrow, MFInfo(), factory); + amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)), dmap[lev], + 1, nGrow, amrex::MFInfo(), factory); gradT[lev][idim].setVal(0.0); } } @@ -872,36 +885,35 @@ PeleLM::addSoretTerm( // Get edge diffusivity constexpr int doZeroVisc = 1; - Array beta_ec = getDiffusivity( + amrex::Array beta_ec = getDiffusivity( lev, NUM_SPECIES + 2, NUM_SPECIES, doZeroVisc, bcRecSpec, *a_beta[lev]); - const Box& domain = geom[lev].Domain(); + const amrex::Box& domain = geom[lev].Domain(); const bool use_harmonic_avg = m_harm_avg_cen2edge != 0; #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif { - FArrayBox T_ed; - for (MFIter mfi(*a_beta[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) { + amrex::FArrayBox T_ed; + for (amrex::MFIter mfi(*a_beta[lev], amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { // Get edge centered rhoYs - const Box ebx = mfi.nodaltilebox(idim); - const Box& edomain = amrex::surroundingNodes(domain, idim); + const amrex::Box ebx = mfi.nodaltilebox(idim); + const amrex::Box& edomain = amrex::surroundingNodes(domain, idim); // Get edge centered temps T_ed.resize(ebx, 1); - Elixir T_el = T_ed.elixir(); + amrex::Elixir T_el = T_ed.elixir(); auto const& T_arr = a_temp[lev]->const_array(mfi); const auto& Ted_arr = T_ed.array(0); const auto bc_lo_temp = bcRecTemp[0].lo(idim); const auto bc_hi_temp = bcRecTemp[0].hi(idim); amrex::ParallelFor( - ebx, - [idim, bc_lo_temp, bc_hi_temp, use_harmonic_avg, T_arr, Ted_arr, - edomain] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { int idx[3] = {i, j, k}; bool on_lo = ((bc_lo_temp == amrex::BCType::ext_dir) && @@ -926,9 +938,7 @@ PeleLM::addSoretTerm( // Soret flux is : - rho * D_m * chi_m * \nabla T / T // with beta_m = rho * D_m * chi_m below amrex::ParallelFor( - ebx, - [need_soret_fluxes, gradT_ar, beta_ar, T, spFlux_ar, - spsoretFlux_ar] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { for (int n = 0; n < NUM_SPECIES; n++) { spFlux_ar(i, j, k, n) -= beta_ar(i, j, k, n) * gradT_ar(i, j, k) / T(i, j, k); @@ -949,8 +959,8 @@ PeleLM::addSoretTerm( void PeleLM::computeSpeciesEnthalpyFlux( - const Vector>& a_fluxes, - Vector const& a_temp) + const amrex::Vector>& a_fluxes, + amrex::Vector const& a_temp) { BL_PROFILE("PeleLMeX::computeSpeciesEnthalpyFlux()"); @@ -967,30 +977,31 @@ PeleLM::computeSpeciesEnthalpyFlux( //------------------------------------------------------------------------ // Compute the cell-centered species enthalpies constexpr int nGrow = 1; - MultiFab Enth( - grids[lev], dmap[lev], NUM_SPECIES, nGrow, MFInfo(), Factory(lev)); + amrex::MultiFab Enth( + grids[lev], dmap[lev], NUM_SPECIES, nGrow, amrex::MFInfo(), Factory(lev)); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(Enth, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& gbx = mfi.growntilebox(); + for (amrex::MFIter mfi(Enth, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + const amrex::Box& gbx = mfi.growntilebox(); auto const& Temp_arr = a_temp[lev]->const_array(mfi); auto const& Hi_arr = Enth.array(mfi); #ifdef AMREX_USE_EB auto const& flagfab = ebfact.getMultiEBCellFlagFab()[mfi]; auto const& flag = flagfab.const_array(); - if (flagfab.getType(gbx) == FabType::covered) { // Covered boxes + if (flagfab.getType(gbx) == amrex::FabType::covered) { // Covered boxes amrex::ParallelFor( - gbx, [Hi_arr] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { Hi_arr(i, j, k) = 0.0; }); - } else if (flagfab.getType(gbx) != FabType::regular) { // EB containing - // boxes + } else if ( + flagfab.getType(gbx) != amrex::FabType::regular) { // EB containing + // boxes amrex::ParallelFor( - gbx, [Temp_arr, Hi_arr, flag, - leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (flag(i, j, k).isCovered()) { Hi_arr(i, j, k) = 0.0; } else { @@ -1001,8 +1012,7 @@ PeleLM::computeSpeciesEnthalpyFlux( #endif { amrex::ParallelFor( - gbx, [Temp_arr, Hi_arr, - leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { getHGivenT(i, j, k, Temp_arr, Hi_arr, leosparm); }); } @@ -1012,24 +1022,24 @@ PeleLM::computeSpeciesEnthalpyFlux( // Get the face-centered species enthalpies constexpr int doZeroVisc = 0; constexpr int addTurbContrib = 0; - Array Enth_ec = getDiffusivity( + amrex::Array Enth_ec = getDiffusivity( lev, 0, NUM_SPECIES, doZeroVisc, bcRecSpec, Enth, addTurbContrib); //------------------------------------------------------------------------ // Compute \sum_k { \Flux_k * h_k } #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(Enth, TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(Enth, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - const Box& ebox = mfi.nodaltilebox(idim); + const amrex::Box& ebox = mfi.nodaltilebox(idim); auto const& spflux_ar = a_fluxes[lev][idim]->const_array(mfi, 0); auto const& enthflux_ar = a_fluxes[lev][idim]->array(mfi, NUM_SPECIES + 1); auto const& enth_ar = Enth_ec[idim].const_array(mfi); amrex::ParallelFor( - ebox, [spflux_ar, enthflux_ar, - enth_ar] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebox, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { enthflux_ar(i, j, k) = 0.0; for (int n = 0; n < NUM_SPECIES; n++) { enthflux_ar(i, j, k) += @@ -1054,28 +1064,30 @@ PeleLM::differentialDiffusionUpdate( // [NUM_SPECIES] Temperature : - \lambda \nabla T // [NUM_SPECIES+1] DiffDiff : \sum_k ( h_k * \Flux_k ) constexpr int nGrow = 0; // No need for ghost face on fluxes - Vector> fluxes(finest_level + 1); - Vector> fluxes_aux(finest_level + 1); + amrex::Vector> fluxes( + finest_level + 1); + amrex::Vector> fluxes_aux( + finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { const auto& ba = grids[lev]; const auto& factory = Factory(lev); for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { fluxes[lev][idim].define( - amrex::convert(ba, IntVect::TheDimensionVector(idim)), dmap[lev], - NUM_SPECIES + 2, nGrow, MFInfo(), factory); + amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)), dmap[lev], + NUM_SPECIES + 2, nGrow, amrex::MFInfo(), factory); if (m_nAux > 0) { fluxes_aux[lev][idim].define( - amrex::convert(ba, IntVect::TheDimensionVector(idim)), dmap[lev], - m_nAux, nGrow, MFInfo(), factory); + amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)), + dmap[lev], m_nAux, nGrow, amrex::MFInfo(), factory); } } } #ifdef AMREX_USE_EB - Vector EBfluxes(finest_level + 1); + amrex::Vector EBfluxes(finest_level + 1); if (m_isothermalEB != 0) { for (int lev = 0; lev <= finest_level; ++lev) { EBfluxes[lev].define( - grids[lev], dmap[lev], 1, nGrow, MFInfo(), Factory(lev)); + grids[lev], dmap[lev], 1, nGrow, amrex::MFInfo(), Factory(lev)); } } #endif @@ -1089,12 +1101,12 @@ PeleLM::differentialDiffusionUpdate( // Get t^{n} data pointer auto* ldata_p = getLevelDataPtr(lev, AmrOldTime); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(advData->Forcing[lev], TilingIfNotGPU()); mfi.isValid(); - ++mfi) { - const Box& bx = mfi.tilebox(); - FArrayBox DummyFab(bx, 1); + for (amrex::MFIter mfi(advData->Forcing[lev], amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); + amrex::FArrayBox DummyFab(bx, 1); auto const& rhoY_o = ldata_p->state.const_array(mfi, FIRSTSPEC); auto const& fY = advData->Forcing[lev].array(mfi, 0); auto const& aux_o = (m_nAux > 0) @@ -1102,9 +1114,10 @@ PeleLM::differentialDiffusionUpdate( : DummyFab.const_array(); auto const& fAux = (m_nAux > 0) ? advData->Forcing_aux[lev].array(mfi, 0) : DummyFab.array(); + const auto dt = m_dt; + const auto nAux = m_nAux; amrex::ParallelFor( - bx, [rhoY_o, fY, aux_o, fAux, dt = m_dt, - nAux = m_nAux] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { for (int n = 0; n < NUM_SPECIES; n++) { fY(i, j, k, n) *= dt; fY(i, j, k, n) += rhoY_o(i, j, k, n); @@ -1122,23 +1135,24 @@ PeleLM::differentialDiffusionUpdate( // Get the species BCRec auto bcRecSpec = fetchBCRecArray(FIRSTSPEC, NUM_SPECIES); auto bcRecAux = fetchBCRecAuxArray(0, m_nAux); - Vector spec_boundary; + amrex::Vector spec_boundary; if (m_soret_boundary_override != 0) { spec_boundary.resize(finest_level + 1); // this is the same regardless of lagged or not for (int lev = 0; lev <= finest_level; ++lev) { auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); spec_boundary[lev].define( - grids[lev], dmap[lev], NUM_SPECIES, 1, MFInfo(), Factory(lev)); + grids[lev], dmap[lev], NUM_SPECIES, 1, amrex::MFInfo(), Factory(lev)); // if we have a mix of Dirichlet and Isothermal walls, we need to give // Dirichlet boundaries and divide by density since diffuse_scalar doesn't // touch this boundary MF - MultiFab::Copy( + amrex::MultiFab::Copy( spec_boundary[lev], ldata_p->state, FIRSTSPEC, 0, NUM_SPECIES, 1); for (int n = 0; n < NUM_SPECIES; n++) { - MultiFab::Divide(spec_boundary[lev], ldata_p->state, DENSITY, n, 1, 1); + amrex::MultiFab::Divide( + spec_boundary[lev], ldata_p->state, DENSITY, n, 1, 1); } } // correct boundary, we have lagged fluxes so lets use them @@ -1212,18 +1226,18 @@ PeleLM::differentialDiffusionUpdate( auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - const Box& ebx = mfi.nodaltilebox(idim); + const amrex::Box& ebx = mfi.nodaltilebox(idim); auto const& flux_spec = fluxes[lev][idim].array(mfi); auto const& flux_wbar = diffData->wbar_fluxes[lev][idim].const_array(mfi); amrex::ParallelFor( ebx, NUM_SPECIES, - [flux_spec, - flux_wbar] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { flux_spec(i, j, k, n) += flux_wbar(i, j, k, n); }); } @@ -1235,18 +1249,18 @@ PeleLM::differentialDiffusionUpdate( auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - const Box& ebx = mfi.nodaltilebox(idim); + const amrex::Box& ebx = mfi.nodaltilebox(idim); auto const& flux_spec = fluxes[lev][idim].array(mfi); auto const& flux_soret = diffData->soret_fluxes[lev][idim].const_array(mfi); amrex::ParallelFor( ebx, NUM_SPECIES, - [flux_spec, - flux_soret] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { flux_spec(i, j, k, n) += flux_soret(i, j, k, n); }); } @@ -1286,11 +1300,12 @@ PeleLM::differentialDiffusionUpdate( auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); - FArrayBox DummyFab(bx, 1); + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); + amrex::FArrayBox DummyFab(bx, 1); auto const& rhoY = ldata_p->state.array(mfi, FIRSTSPEC); auto const& dhat = diffData->Dhat[lev].const_array(mfi); auto const& force = advData->Forcing[lev].const_array(mfi, 0); @@ -1307,11 +1322,12 @@ PeleLM::differentialDiffusionUpdate( auto const& force_aux = (m_nAux > 0) ? advData->Forcing_aux[lev].const_array(mfi, 0) : DummyFab.const_array(); + const auto nAux = m_nAux; + const auto dt = m_dt; + const auto use_wbar = m_use_wbar; + const auto use_soret = m_use_soret; amrex::ParallelFor( - bx, [rhoY, dhat, force, dwbar, dT, aux, dhat_aux, force_aux, - nAux = m_nAux, dt = m_dt, use_wbar = m_use_wbar, - use_soret = - m_use_soret] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { for (int n = 0; n < NUM_SPECIES; n++) { rhoY(i, j, k, n) = force(i, j, k, n) + dt * dhat(i, j, k, n); if (use_wbar != 0) { @@ -1348,9 +1364,10 @@ PeleLM::differentialDiffusionUpdate( #ifdef AMREX_USE_EB if (m_isothermalEB != 0) { // Set up EB dirichlet value and diffusivity - Vector EBdiff(finest_level + 1); + amrex::Vector EBdiff(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - EBdiff[lev].define(grids[lev], dmap[lev], 1, 0, MFInfo(), EBFactory(lev)); + EBdiff[lev].define( + grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), EBFactory(lev)); getEBDiff(lev, AmrNewTime, EBdiff[lev], NUM_SPECIES); } getDiffusionOp()->computeDiffFluxes( @@ -1397,23 +1414,27 @@ PeleLM::differentialDiffusionUpdate( //------------------------------------------------------------------------ // delta(T) iterations if (m_deltaT_verbose != 0) { - Print() << " Iterative solve for deltaT \n"; + amrex::Print() << " Iterative solve for deltaT \n"; } //------------------------------------------------------------------------ // Temporary data holders - Vector rhs(finest_level + 1); // Linear deltaT solve RHS - Vector Tsave( + amrex::Vector rhs( + finest_level + 1); // Linear deltaT solve RHS + amrex::Vector Tsave( finest_level + 1); // Storage of T while working on deltaT - Vector RhoCp(finest_level + 1); // Acoeff of the linear solve + amrex::Vector RhoCp( + finest_level + 1); // Acoeff of the linear solve for (int lev = 0; lev <= finest_level; ++lev) { - rhs[lev].define(grids[lev], dmap[lev], 1, 0, MFInfo(), Factory(lev)); - Tsave[lev].define(grids[lev], dmap[lev], 1, 1, MFInfo(), Factory(lev)); - RhoCp[lev].define(grids[lev], dmap[lev], 1, 0, MFInfo(), Factory(lev)); + rhs[lev].define(grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), Factory(lev)); + Tsave[lev].define( + grids[lev], dmap[lev], 1, 1, amrex::MFInfo(), Factory(lev)); + RhoCp[lev].define( + grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), Factory(lev)); } // DeltaT norm - Real deltaT_norm = 0.0; + amrex::Real deltaT_norm = 0.0; for (int dTiter = 0; dTiter < m_deltaTIterMax && (dTiter == 0 || deltaT_norm >= m_deltaT_norm_max); ++dTiter) { @@ -1432,13 +1453,13 @@ PeleLM::differentialDiffusionUpdate( if (m_isothermalEB != 0) { // Set up EB dirichlet value and diffusivity // Dirichlet value is deltaT - Vector EBvalue(finest_level + 1); - Vector EBdiff(finest_level + 1); + amrex::Vector EBvalue(finest_level + 1); + amrex::Vector EBdiff(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { EBvalue[lev].define( - grids[lev], dmap[lev], 1, 0, MFInfo(), EBFactory(lev)); + grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), EBFactory(lev)); EBdiff[lev].define( - grids[lev], dmap[lev], 1, 0, MFInfo(), EBFactory(lev)); + grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), EBFactory(lev)); getEBDiff(lev, AmrNewTime, EBdiff[lev], NUM_SPECIES); EBvalue[lev].setVal(0.0); } @@ -1479,9 +1500,9 @@ PeleLM::differentialDiffusionUpdate( // Check for convergence failure if ((dTiter == m_deltaTIterMax - 1) && (deltaT_norm > m_deltaT_norm_max)) { if (m_crashOnDeltaTFail != 0) { - Abort("deltaT_iters not converged !"); + amrex::Abort("deltaT_iters not converged !"); } else { - Print() << "deltaT_iters not converged !\n"; + amrex::Print() << "deltaT_iters not converged !\n"; } } } @@ -1490,9 +1511,9 @@ PeleLM::differentialDiffusionUpdate( void PeleLM::deltaTIter_prepare( - const Vector& a_rhs, - const Vector& a_Tsave, - const Vector& a_rhoCp, + const amrex::Vector& a_rhs, + const amrex::Vector& a_Tsave, + const amrex::Vector& a_rhoCp, std::unique_ptr& advData, std::unique_ptr& diffData) { @@ -1503,11 +1524,11 @@ PeleLM::deltaTIter_prepare( auto* ldataNew_p = getLevelDataPtr(lev, AmrNewTime); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldataNew_p->state, TilingIfNotGPU()); mfi.isValid(); - ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldataNew_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); // RHS pieces auto const& rhoH_o = ldataOld_p->state.const_array(mfi, RHOH); auto const& rhoH_n = ldataNew_p->state.const_array(mfi, RHOH); @@ -1516,7 +1537,7 @@ PeleLM::deltaTIter_prepare( auto const& diffDiff = diffData->Dhat[lev].const_array(mfi, NUM_SPECIES + 1); auto const& rhs = a_rhs[lev]->array(mfi); - const Real dtinv = 1.0 / m_dt; + const amrex::Real dtinv = 1.0 / m_dt; // Cpmix auto const& rho = ldataNew_p->state.const_array(mfi, DENSITY); @@ -1526,8 +1547,9 @@ PeleLM::deltaTIter_prepare( // T save auto const& tsave = a_Tsave[lev]->array(mfi); + const auto dt = m_dt; amrex::ParallelFor( - bx, [=, dt = m_dt] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { // Assemble deltaT RHS rhs(i, j, k) = dt * ((rhoH_o(i, j, k) - rhoH_n(i, j, k)) * dtinv + force(i, j, k) + @@ -1551,11 +1573,11 @@ PeleLM::deltaTIter_prepare( void PeleLM::deltaTIter_update( int a_dtiter, - const Vector>& a_fluxes, - const Vector& a_ebfluxes, - const Vector& a_Tsave, + const amrex::Vector>& a_fluxes, + const amrex::Vector& a_ebfluxes, + const amrex::Vector& a_Tsave, std::unique_ptr& diffData, - Real& a_deltaT_norm) + amrex::Real& a_deltaT_norm) { #ifndef AMREX_USE_EB amrex::ignore_unused(a_ebfluxes); @@ -1567,13 +1589,13 @@ PeleLM::deltaTIter_update( for (int lev = 0; lev <= finest_level; ++lev) { auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); a_deltaT_norm = - std::max(a_deltaT_norm, ldata_p->state.norm0(TEMP, 0, false, true)); - MultiFab::Add(ldata_p->state, *a_Tsave[lev], 0, TEMP, 1, 0); + amrex::max(a_deltaT_norm, ldata_p->state.norm0(TEMP, 0, false, true)); + amrex::MultiFab::Add(ldata_p->state, *a_Tsave[lev], 0, TEMP, 1, 0); } if (m_deltaT_verbose != 0) { - Print() << " DeltaT solve norm [" << a_dtiter << "] = " << a_deltaT_norm - << "\n"; + amrex::Print() << " DeltaT solve norm [" << a_dtiter + << "] = " << a_deltaT_norm << "\n"; } // FillPatch the new temperature before going into the fluxe computation @@ -1589,9 +1611,10 @@ PeleLM::deltaTIter_update( #ifdef AMREX_USE_EB if (m_isothermalEB != 0) { // Set up EB dirichlet value and diffusivity - Vector EBdiff(finest_level + 1); + amrex::Vector EBdiff(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - EBdiff[lev].define(grids[lev], dmap[lev], 1, 0, MFInfo(), EBFactory(lev)); + EBdiff[lev].define( + grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), EBFactory(lev)); getEBDiff(lev, AmrNewTime, EBdiff[lev], NUM_SPECIES); } getDiffusionOp()->computeDiffFluxes( @@ -1644,13 +1667,13 @@ PeleLM::deltaTIter_update( ldata_p->state, [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { getRHmixGivenTY( - i, j, k, Array4(sma[box_no], DENSITY), - Array4(sma[box_no], FIRSTSPEC), - Array4(sma[box_no], TEMP), - Array4(sma[box_no], RHOH), leosparm); + i, j, k, amrex::Array4(sma[box_no], DENSITY), + amrex::Array4(sma[box_no], FIRSTSPEC), + amrex::Array4(sma[box_no], TEMP), + amrex::Array4(sma[box_no], RHOH), leosparm); }); } - Gpu::streamSynchronize(); + amrex::Gpu::streamSynchronize(); } void @@ -1668,12 +1691,12 @@ PeleLM::getScalarDiffForce( auto* ldataR_p = getLevelDataReactPtr(lev); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(advData->Forcing[lev], TilingIfNotGPU()); mfi.isValid(); - ++mfi) { - const Box& bx = mfi.tilebox(); - FArrayBox DummyFab(bx, 1); + for (amrex::MFIter mfi(advData->Forcing[lev], amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); + amrex::FArrayBox DummyFab(bx, 1); auto const& dn = diffData->Dn[lev].const_array(mfi, 0); auto const& ddn = diffData->Dn[lev].const_array(mfi, NUM_SPECIES + 1); auto const& dnp1k = diffData->Dnp1[lev].const_array(mfi, 0); @@ -1702,12 +1725,14 @@ PeleLM::getScalarDiffForce( auto const& dnp1k_aux = (m_nAux > 0) ? diffData->Dnp1_aux[lev].const_array(mfi, 0) : DummyFab.const_array(); + const auto do_react = m_do_react; + const auto use_wbar = m_use_wbar; + const auto use_soret = m_use_soret; + const auto dp0dt = m_dp0dt; + const auto is_closed_ch = m_closed_chamber; + const auto nAux = m_nAux; amrex::ParallelFor( - bx, [dn, ddn, dnp1k, ddnp1k, do_react = m_do_react, r, a, extRhoY, - extRhoH, dwbar, dT, use_wbar = m_use_wbar, use_soret = m_use_soret, - fY, fT, fAux, a_aux, dn_aux, dnp1k_aux, aux_advect_d, - aux_diffuse_d, dp0dt = m_dp0dt, is_closed_ch = m_closed_chamber, - nAux = m_nAux] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { buildDiffusionForcing( i, j, k, dn, ddn, dnp1k, ddnp1k, r, a, dp0dt, is_closed_ch, do_react, fY, fT, dwbar, dT, extRhoY, extRhoH, use_wbar, use_soret, @@ -1731,9 +1756,9 @@ PeleLM::getScalarDiffForce( void PeleLM::computeDivTau( const TimeStamp& a_time, - const Vector& a_divtau, + const amrex::Vector& a_divtau, int use_density, - Real scale) + amrex::Real scale) { BL_PROFILE("PeleLMeX::computeDivTau()"); // Get the density component BCRec to get viscosity on faces @@ -1759,7 +1784,7 @@ PeleLM::diffuseVelocity() auto bcRec = fetchBCRecArray(DENSITY, 1); // CrankNicholson 0.5 coeff - const Real dt_lcl = 0.5 * m_dt; + const amrex::Real dt_lcl = 0.5 * m_dt; if (m_incompressible != 0) { getDiffusionTensorOp()->diffuse_velocity( GetVecOfPtrs(getVelocityVect(AmrNewTime)), {}, @@ -1772,73 +1797,76 @@ PeleLM::diffuseVelocity() } } -Array -PeleLM::getDiffusionLinOpBC(Orientation::Side a_side, const BCRec& a_bc) +amrex::Array +PeleLM::getDiffusionLinOpBC( + amrex::Orientation::Side a_side, const amrex::BCRec& a_bc) { - - Array r; + amrex::Array r; for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { if (Geom(0).isPeriodic(idim)) { - r[idim] = LinOpBCType::Periodic; + r[idim] = amrex::LinOpBCType::Periodic; } else { auto amrexbc = - (a_side == Orientation::low) ? a_bc.lo(idim) : a_bc.hi(idim); + (a_side == amrex::Orientation::low) ? a_bc.lo(idim) : a_bc.hi(idim); if (amrexbc == amrex::BCType::ext_dir) { - r[idim] = LinOpBCType::Dirichlet; + r[idim] = amrex::LinOpBCType::Dirichlet; } else if ( amrexbc == amrex::BCType::foextrap || amrexbc == amrex::BCType::hoextrap || amrexbc == amrex::BCType::reflect_even) { - r[idim] = LinOpBCType::Neumann; + r[idim] = amrex::LinOpBCType::Neumann; if ( m_use_soret != 0 && amrexbc == amrex::BCType::foextrap) { // should just catch // density/species/forcing for // isothermal walls - auto phys_bc = (a_side == Orientation::low) ? m_phys_bc.lo(idim) - : m_phys_bc.hi(idim); + auto phys_bc = (a_side == amrex::Orientation::low) + ? m_phys_bc.lo(idim) + : m_phys_bc.hi(idim); if ( phys_bc == BoundaryCondition::BCNoSlipWallIsotherm || phys_bc == BoundaryCondition::BCSlipWallIsotherm) { - r[idim] = LinOpBCType::inhomogNeumann; + r[idim] = amrex::LinOpBCType::inhomogNeumann; } } } else if (amrexbc == amrex::BCType::reflect_odd) { - r[idim] = LinOpBCType::reflect_odd; + r[idim] = amrex::LinOpBCType::reflect_odd; } else { - r[idim] = LinOpBCType::bogus; + r[idim] = amrex::LinOpBCType::bogus; } } } return r; } -Vector> +amrex::Vector> PeleLM::getDiffusionTensorOpBC( - Orientation::Side a_side, const Vector a_bc) + amrex::Orientation::Side a_side, const amrex::Vector a_bc) { AMREX_ASSERT(a_bc.size() == AMREX_SPACEDIM); - Vector> r(AMREX_SPACEDIM); + amrex::Vector> r( + AMREX_SPACEDIM); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { if (Geom(0).isPeriodic(idim)) { AMREX_D_TERM( - r[0][idim] = LinOpBCType::Periodic;, r[1][idim] = LinOpBCType::Periodic; - , r[2][idim] = LinOpBCType::Periodic;); + r[0][idim] = amrex::LinOpBCType::Periodic; + , r[1][idim] = amrex::LinOpBCType::Periodic; + , r[2][idim] = amrex::LinOpBCType::Periodic;); } else { for (int dir = 0; dir < AMREX_SPACEDIM; dir++) { - auto amrexbc = (a_side == Orientation::low) ? a_bc[dir].lo(idim) - : a_bc[dir].hi(idim); + auto amrexbc = (a_side == amrex::Orientation::low) ? a_bc[dir].lo(idim) + : a_bc[dir].hi(idim); if (amrexbc == amrex::BCType::ext_dir) { - r[dir][idim] = LinOpBCType::Dirichlet; + r[dir][idim] = amrex::LinOpBCType::Dirichlet; } else if ( amrexbc == amrex::BCType::foextrap || amrexbc == amrex::BCType::hoextrap || amrexbc == amrex::BCType::reflect_even) { - r[dir][idim] = LinOpBCType::Neumann; + r[dir][idim] = amrex::LinOpBCType::Neumann; } else if (amrexbc == amrex::BCType::reflect_odd) { - r[dir][idim] = LinOpBCType::reflect_odd; + r[dir][idim] = amrex::LinOpBCType::reflect_odd; } else { - r[dir][idim] = LinOpBCType::bogus; + r[dir][idim] = amrex::LinOpBCType::bogus; } } } diff --git a/Source/PeleLMeX_DiffusionOp.cpp b/Source/PeleLMeX_DiffusionOp.cpp index d5ab45a0f..7f935523c 100644 --- a/Source/PeleLMeX_DiffusionOp.cpp +++ b/Source/PeleLMeX_DiffusionOp.cpp @@ -8,8 +8,6 @@ #include #endif -using namespace amrex; - //--------------------------------------------------------------------------------------- // Diffusion Operator @@ -20,18 +18,18 @@ DiffusionOp::DiffusionOp(PeleLM* a_pelelm, int ncomp) readParameters(); // Solve LPInfo - LPInfo info_solve; + amrex::LPInfo info_solve; info_solve.setAgglomeration(true); info_solve.setConsolidation(true); info_solve.setMaxCoarseningLevel(m_mg_max_coarsening_level); // Apply LPInfo (no coarsening) - LPInfo info_apply; + amrex::LPInfo info_apply; info_apply.setMaxCoarseningLevel(0); #ifdef AMREX_USE_EB // Get vector of EB Factory - Vector ebfactVec; + amrex::Vector ebfactVec; for (int lev = 0; lev <= m_pelelm->finestLevel(); ++lev) { ebfactVec.push_back(&(m_pelelm->EBFactory(lev))); } @@ -39,14 +37,15 @@ DiffusionOp::DiffusionOp(PeleLM* a_pelelm, int ncomp) // Scalar apply op. #ifdef AMREX_USE_EB - m_scal_apply_op = std::make_unique( + m_scal_apply_op = std::make_unique( m_pelelm->Geom(0, m_pelelm->finestLevel()), m_pelelm->boxArray(0, m_pelelm->finestLevel()), m_pelelm->DistributionMap(0, m_pelelm->finestLevel()), info_apply, ebfactVec, m_ncomp); #else - const Vector const*>& empty_factory = {}; - m_scal_apply_op = std::make_unique( + const amrex::Vector const*>& + empty_factory = {}; + m_scal_apply_op = std::make_unique( m_pelelm->Geom(0, m_pelelm->finestLevel()), m_pelelm->boxArray(0, m_pelelm->finestLevel()), m_pelelm->DistributionMap(0, m_pelelm->finestLevel()), info_apply, @@ -56,13 +55,13 @@ DiffusionOp::DiffusionOp(PeleLM* a_pelelm, int ncomp) // Scalar solve op. #ifdef AMREX_USE_EB - m_scal_solve_op = std::make_unique( + m_scal_solve_op = std::make_unique( m_pelelm->Geom(0, m_pelelm->finestLevel()), m_pelelm->boxArray(0, m_pelelm->finestLevel()), m_pelelm->DistributionMap(0, m_pelelm->finestLevel()), info_solve, ebfactVec, m_ncomp); #else - m_scal_solve_op = std::make_unique( + m_scal_solve_op = std::make_unique( m_pelelm->Geom(0, m_pelelm->finestLevel()), m_pelelm->boxArray(0, m_pelelm->finestLevel()), m_pelelm->DistributionMap(0, m_pelelm->finestLevel()), info_solve, @@ -72,13 +71,13 @@ DiffusionOp::DiffusionOp(PeleLM* a_pelelm, int ncomp) // Gradient op. : scalar/coefficient already preset #ifdef AMREX_USE_EB - m_gradient_op = std::make_unique( + m_gradient_op = std::make_unique( m_pelelm->Geom(0, m_pelelm->finestLevel()), m_pelelm->boxArray(0, m_pelelm->finestLevel()), m_pelelm->DistributionMap(0, m_pelelm->finestLevel()), info_apply, ebfactVec, m_ncomp); #else - m_gradient_op = std::make_unique( + m_gradient_op = std::make_unique( m_pelelm->Geom(0, m_pelelm->finestLevel()), m_pelelm->boxArray(0, m_pelelm->finestLevel()), m_pelelm->DistributionMap(0, m_pelelm->finestLevel()), info_apply, @@ -93,21 +92,21 @@ DiffusionOp::DiffusionOp(PeleLM* a_pelelm, int ncomp) void DiffusionOp::diffuse_scalar( - Vector const& a_phi, + amrex::Vector const& a_phi, int phi_comp, - Vector const& a_rhs, + amrex::Vector const& a_rhs, int rhs_comp, - Vector> const& a_flux, + amrex::Vector> const& a_flux, int flux_comp, - Vector const& a_acoeff, - Vector const& a_density, - Vector const& a_bcoeff, + amrex::Vector const& a_acoeff, + amrex::Vector const& a_density, + amrex::Vector const& a_bcoeff, int bcoeff_comp, - Vector a_bcrec, + amrex::Vector a_bcrec, int ncomp, int isPoissonSolve, - Real a_dt, - Vector const& a_boundary) + amrex::Real a_dt, + amrex::Vector const& a_boundary) { BL_PROFILE("DiffusionOp::diffuse_scalar()"); @@ -141,26 +140,25 @@ DiffusionOp::diffuse_scalar( // after adv., when we divide by \rho, it is inconsistent. But it only matters // if it screws up the ghost cell values 'cause interiors are just an initial // solution for the solve. - Vector phi(finest_level + 1); + amrex::Vector phi(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { phi[lev].define( - a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), ncomp, 1, MFInfo(), - a_phi[lev]->Factory()); + a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), ncomp, 1, + amrex::MFInfo(), a_phi[lev]->Factory()); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(phi[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& gbx = mfi.growntilebox(); + for (amrex::MFIter mfi(phi[lev], amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + const amrex::Box& gbx = mfi.growntilebox(); auto const& a_phi_arr = a_phi[lev]->const_array(mfi, phi_comp); auto const& a_rho_arr = - (have_density) != 0 - ? a_density[lev]->const_array(mfi) - : a_phi[lev]->const_array(mfi); // Get dummy Array4 if no density + (have_density) != 0 ? a_density[lev]->const_array(mfi) + : a_phi[lev]->const_array( + mfi); // Get dummy amrex::Array4 if no density auto const& phi_arr = phi[lev].array(mfi); amrex::ParallelFor( - gbx, ncomp, - [a_phi_arr, a_rho_arr, phi_arr, - have_density] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + gbx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { if (have_density != 0) { phi_arr(i, j, k, n) = a_phi_arr(i, j, k, n) / a_rho_arr(i, j, k); } else { @@ -176,8 +174,8 @@ DiffusionOp::diffuse_scalar( // => \alpha = 1.0, A is a_acoeff if provided, 1.0 otherwise // => \beta = a_dt, B face centered diffusivity bcoeff^{np1,k} - Real alpha = (isPoissonSolve) != 0 ? 0.0 : 1.0; - Real beta = a_dt; + amrex::Real alpha = (isPoissonSolve) != 0 ? 0.0 : 1.0; + amrex::Real beta = a_dt; m_scal_solve_op->setScalars(alpha, beta); for (int lev = 0; lev <= finest_level; ++lev) { if (have_acoeff != 0) { @@ -192,21 +190,22 @@ DiffusionOp::diffuse_scalar( for (int comp = 0; comp < ncomp; comp += m_ncomp) { // Aliases - Vector> fluxes(finest_level + 1); - Vector component; - Vector rhs; - Vector boundary; + amrex::Vector> fluxes( + finest_level + 1); + amrex::Vector component; + amrex::Vector rhs; + amrex::Vector boundary; // Allow for component specific LinOp BC m_scal_solve_op->setDomainBC( - m_pelelm->getDiffusionLinOpBC(Orientation::low, a_bcrec[comp]), - m_pelelm->getDiffusionLinOpBC(Orientation::high, a_bcrec[comp])); + m_pelelm->getDiffusionLinOpBC(amrex::Orientation::low, a_bcrec[comp]), + m_pelelm->getDiffusionLinOpBC(amrex::Orientation::high, a_bcrec[comp])); // Set aliases and bcoeff comp for (int lev = 0; lev <= finest_level; ++lev) { if (have_fluxes != 0) { for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - fluxes[lev][idim] = new MultiFab( + fluxes[lev][idim] = new amrex::MultiFab( *a_flux[lev][idim], amrex::make_alias, flux_comp + comp, m_ncomp); } } @@ -214,14 +213,16 @@ DiffusionOp::diffuse_scalar( if (have_bcoeff != 0) { int doZeroVisc = 1; int addTurbContrib = 1; - Vector subBCRec = { + amrex::Vector subBCRec = { a_bcrec.begin() + comp, a_bcrec.begin() + comp + m_ncomp}; - Array bcoeff_ec = m_pelelm->getDiffusivity( - lev, bcoeff_comp + comp, m_ncomp, doZeroVisc, subBCRec, - *a_bcoeff[lev], addTurbContrib); + amrex::Array bcoeff_ec = + m_pelelm->getDiffusivity( + lev, bcoeff_comp + comp, m_ncomp, doZeroVisc, subBCRec, + *a_bcoeff[lev], addTurbContrib); #ifdef AMREX_USE_EB m_scal_solve_op->setBCoeffs( - lev, GetArrOfConstPtrs(bcoeff_ec), MLMG::Location::FaceCentroid); + lev, GetArrOfConstPtrs(bcoeff_ec), + amrex::MLMG::Location::FaceCentroid); #else m_scal_solve_op->setBCoeffs(lev, GetArrOfConstPtrs(bcoeff_ec)); #endif @@ -242,7 +243,7 @@ DiffusionOp::diffuse_scalar( } // Setup linear solver - MLMG mlmg(*m_scal_solve_op); + amrex::MLMG mlmg(*m_scal_solve_op); // Maximum iterations mlmg.setMaxIter(m_mg_max_iter); @@ -263,9 +264,9 @@ DiffusionOp::diffuse_scalar( // Need to get the fluxes if (have_fluxes != 0) { #ifdef AMREX_USE_EB - mlmg.getFluxes(fluxes, MLMG::Location::FaceCentroid); + mlmg.getFluxes(fluxes, amrex::MLMG::Location::FaceCentroid); #else - mlmg.getFluxes(fluxes, MLMG::Location::FaceCenter); + mlmg.getFluxes(fluxes, amrex::MLMG::Location::FaceCenter); #endif for (int lev = 0; lev <= finest_level; ++lev) { @@ -282,20 +283,19 @@ DiffusionOp::diffuse_scalar( // Don't touch the ghost cells for (int lev = 0; lev <= finest_level; ++lev) { #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(phi[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(phi[lev], amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + const amrex::Box& bx = mfi.tilebox(); auto const& a_phi_arr = a_phi[lev]->array(mfi, phi_comp); auto const& a_rho_arr = - (have_density) != 0 - ? a_density[lev]->const_array(mfi) - : a_phi[lev]->const_array(mfi); // Get dummy Array4 if no density + (have_density) != 0 ? a_density[lev]->const_array(mfi) + : a_phi[lev]->const_array( + mfi); // Get dummy amrex::Array4 if no density auto const& phi_arr = phi[lev].const_array(mfi); amrex::ParallelFor( - bx, ncomp, - [a_phi_arr, a_rho_arr, phi_arr, - have_density] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { if (have_density != 0) { a_phi_arr(i, j, k, n) = phi_arr(i, j, k, n) * a_rho_arr(i, j, k); } else { @@ -309,25 +309,25 @@ DiffusionOp::diffuse_scalar( #ifdef AMREX_USE_EB void DiffusionOp::diffuse_scalar( - Vector const& a_phi, + amrex::Vector const& a_phi, int phi_comp, - Vector const& a_phiEB, + amrex::Vector const& a_phiEB, int /*phiEB_comp*/, - Vector const& a_rhs, + amrex::Vector const& a_rhs, int rhs_comp, - Vector> const& a_flux, + amrex::Vector> const& a_flux, int flux_comp, - Vector const& a_acoeff, - Vector const& a_density, - Vector const& a_bcoeff, + amrex::Vector const& a_acoeff, + amrex::Vector const& a_density, + amrex::Vector const& a_bcoeff, int bcoeff_comp, - Vector const& a_bcoeffEB, + amrex::Vector const& a_bcoeffEB, int /*bcoeffEB_comp*/, - Vector a_bcrec, + amrex::Vector a_bcrec, int ncomp, int isPoissonSolve, - Real a_dt, - Vector const& a_boundary) + amrex::Real a_dt, + amrex::Vector const& a_boundary) { BL_PROFILE("DiffusionOp::diffuse_scalar()"); @@ -361,26 +361,25 @@ DiffusionOp::diffuse_scalar( // after adv., when we divide by \rho, it is inconsistent. But it only matters // if it screws up the ghost cell values 'cause interiors are just an initial // solution for the solve. - Vector phi(finest_level + 1); + amrex::Vector phi(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { phi[lev].define( - a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), ncomp, 1, MFInfo(), - a_phi[lev]->Factory()); + a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), ncomp, 1, + amrex::MFInfo(), a_phi[lev]->Factory()); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(phi[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& gbx = mfi.growntilebox(); + for (amrex::MFIter mfi(phi[lev], amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + const amrex::Box& gbx = mfi.growntilebox(); auto const& a_phi_arr = a_phi[lev]->const_array(mfi, phi_comp); auto const& a_rho_arr = - (have_density) != 0 - ? a_density[lev]->const_array(mfi) - : a_phi[lev]->const_array(mfi); // Get dummy Array4 if no density + (have_density) != 0 ? a_density[lev]->const_array(mfi) + : a_phi[lev]->const_array( + mfi); // Get dummy amrex::Array4 if no density auto const& phi_arr = phi[lev].array(mfi); amrex::ParallelFor( - gbx, ncomp, - [a_phi_arr, a_rho_arr, phi_arr, - have_density] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + gbx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { if (have_density != 0) { phi_arr(i, j, k, n) = a_phi_arr(i, j, k, n) / a_rho_arr(i, j, k); } else { @@ -396,8 +395,8 @@ DiffusionOp::diffuse_scalar( // => \alpha = 1.0, A is a_acoeff if provided, 1.0 otherwise // => \beta = a_dt, B face centered diffusivity bcoeff^{np1,k} - Real alpha = (isPoissonSolve) != 0 ? 0.0 : 1.0; - Real beta = a_dt; + amrex::Real alpha = (isPoissonSolve) != 0 ? 0.0 : 1.0; + amrex::Real beta = a_dt; m_scal_solve_op->setScalars(alpha, beta); for (int lev = 0; lev <= finest_level; ++lev) { if (have_acoeff != 0) { @@ -412,34 +411,37 @@ DiffusionOp::diffuse_scalar( for (int comp = 0; comp < ncomp; comp += m_ncomp) { // Aliases - Vector> fluxes(finest_level + 1); - Vector component; - Vector rhs; - Vector boundary; + amrex::Vector> fluxes( + finest_level + 1); + amrex::Vector component; + amrex::Vector rhs; + amrex::Vector boundary; // Allow for component specific LinOp BC m_scal_solve_op->setDomainBC( - m_pelelm->getDiffusionLinOpBC(Orientation::low, a_bcrec[comp]), - m_pelelm->getDiffusionLinOpBC(Orientation::high, a_bcrec[comp])); + m_pelelm->getDiffusionLinOpBC(amrex::Orientation::low, a_bcrec[comp]), + m_pelelm->getDiffusionLinOpBC(amrex::Orientation::high, a_bcrec[comp])); // Set aliases and bcoeff comp for (int lev = 0; lev <= finest_level; ++lev) { if (have_fluxes != 0) { for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - fluxes[lev][idim] = new MultiFab( + fluxes[lev][idim] = new amrex::MultiFab( *a_flux[lev][idim], amrex::make_alias, flux_comp + comp, m_ncomp); } } if (have_bcoeff != 0) { int doZeroVisc = 1; - Vector subBCRec = { + amrex::Vector subBCRec = { a_bcrec.begin() + comp, a_bcrec.begin() + comp + m_ncomp}; - Array bcoeff_ec = m_pelelm->getDiffusivity( - lev, bcoeff_comp + comp, m_ncomp, doZeroVisc, subBCRec, - *a_bcoeff[lev]); + amrex::Array bcoeff_ec = + m_pelelm->getDiffusivity( + lev, bcoeff_comp + comp, m_ncomp, doZeroVisc, subBCRec, + *a_bcoeff[lev]); m_scal_solve_op->setBCoeffs( - lev, GetArrOfConstPtrs(bcoeff_ec), MLMG::Location::FaceCentroid); + lev, GetArrOfConstPtrs(bcoeff_ec), + amrex::MLMG::Location::FaceCentroid); } else { m_scal_solve_op->setBCoeffs(lev, 1.0); } @@ -458,7 +460,7 @@ DiffusionOp::diffuse_scalar( } // Setup linear solver - MLMG mlmg(*m_scal_solve_op); + amrex::MLMG mlmg(*m_scal_solve_op); // Maximum iterations mlmg.setMaxIter(m_mg_max_iter); @@ -478,7 +480,7 @@ DiffusionOp::diffuse_scalar( // Need to get the fluxes if (have_fluxes != 0) { - mlmg.getFluxes(fluxes, MLMG::Location::FaceCentroid); + mlmg.getFluxes(fluxes, amrex::MLMG::Location::FaceCentroid); for (int lev = 0; lev <= finest_level; ++lev) { for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { @@ -494,20 +496,19 @@ DiffusionOp::diffuse_scalar( // Don't touch the ghost cells for (int lev = 0; lev <= finest_level; ++lev) { #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(phi[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(phi[lev], amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + const amrex::Box& bx = mfi.tilebox(); auto const& a_phi_arr = a_phi[lev]->array(mfi, phi_comp); auto const& a_rho_arr = - (have_density) != 0 - ? a_density[lev]->const_array(mfi) - : a_phi[lev]->const_array(mfi); // Get dummy Array4 if no density + (have_density) != 0 ? a_density[lev]->const_array(mfi) + : a_phi[lev]->const_array( + mfi); // Get dummy amrex::Array4 if no density auto const& phi_arr = phi[lev].const_array(mfi); amrex::ParallelFor( - bx, ncomp, - [a_phi_arr, a_rho_arr, phi_arr, - have_density] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { if (have_density != 0) { a_phi_arr(i, j, k, n) = phi_arr(i, j, k, n) * a_rho_arr(i, j, k); } else { @@ -521,13 +522,13 @@ DiffusionOp::diffuse_scalar( void DiffusionOp::computeDiffLap( - Vector const& a_laps, + amrex::Vector const& a_laps, int lap_comp, - Vector const& a_phi, + amrex::Vector const& a_phi, int phi_comp, - Vector const& a_bcoeff, + amrex::Vector const& a_bcoeff, int bcoeff_comp, - Vector a_bcrec, + amrex::Vector a_bcrec, int ncomp) { BL_PROFILE("DiffusionOp::computeDiffLap()"); @@ -543,12 +544,12 @@ DiffusionOp::computeDiffLap( int finest_level = m_pelelm->finestLevel(); // Copy phi with 1 ghost cell - Vector phi(finest_level + 1); + amrex::Vector phi(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { phi[lev].define( - a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), ncomp, 1, MFInfo(), - a_phi[lev]->Factory()); - MultiFab::Copy(phi[lev], *a_phi[lev], phi_comp, 0, ncomp, 1); + a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), ncomp, 1, + amrex::MFInfo(), a_phi[lev]->Factory()); + amrex::MultiFab::Copy(phi[lev], *a_phi[lev], phi_comp, 0, ncomp, 1); } // LinOp is \alpha A \phi - \beta \nabla \cdot B \nabla \phi @@ -556,15 +557,15 @@ DiffusionOp::computeDiffLap( // => \beta = -1.0, B face centered diffusivity a_bcoeff // Set scalars \alpha & \beta - Real alpha = 0.0; - Real beta = -1.0; + amrex::Real alpha = 0.0; + amrex::Real beta = -1.0; m_scal_apply_op->setScalars(alpha, beta); for (int comp = 0; comp < ncomp; comp += m_ncomp) { // Component based vector of data - Vector laps; - Vector component; + amrex::Vector laps; + amrex::Vector component; for (int lev = 0; lev <= finest_level; ++lev) { laps.emplace_back( @@ -572,39 +573,40 @@ DiffusionOp::computeDiffLap( component.emplace_back(phi[lev], amrex::make_alias, comp, m_ncomp); int doZeroVisc = 0; int addTurbContrib = 0; - Vector subBCRec = { + amrex::Vector subBCRec = { a_bcrec.begin() + comp, a_bcrec.begin() + comp + m_ncomp}; - Array bcoeff_ec = m_pelelm->getDiffusivity( - lev, bcoeff_comp + comp, m_ncomp, doZeroVisc, subBCRec, *a_bcoeff[lev], - addTurbContrib); + amrex::Array bcoeff_ec = + m_pelelm->getDiffusivity( + lev, bcoeff_comp + comp, m_ncomp, doZeroVisc, subBCRec, + *a_bcoeff[lev], addTurbContrib); #ifdef AMREX_USE_EB m_scal_apply_op->setBCoeffs( - lev, GetArrOfConstPtrs(bcoeff_ec), MLMG::Location::FaceCentroid); + lev, GetArrOfConstPtrs(bcoeff_ec), amrex::MLMG::Location::FaceCentroid); #else m_scal_apply_op->setBCoeffs(lev, GetArrOfConstPtrs(bcoeff_ec)); #endif m_scal_apply_op->setLevelBC(lev, &component[lev]); } - MLMG mlmg(*m_scal_apply_op); + amrex::MLMG mlmg(*m_scal_apply_op); mlmg.apply(GetVecOfPtrs(laps), GetVecOfPtrs(component)); } } void DiffusionOp::computeDiffFluxes( - Vector> const& a_flux, + amrex::Vector> const& a_flux, int flux_comp, - Vector const& a_phi, + amrex::Vector const& a_phi, int phi_comp, - Vector const& a_density, - Vector const& a_bcoeff, + amrex::Vector const& a_density, + amrex::Vector const& a_bcoeff, int bcoeff_comp, - Vector a_bcrec, + amrex::Vector a_bcrec, int ncomp, int do_avgDown, - Vector const& a_boundary) + amrex::Vector const& a_boundary) { BL_PROFILE("DiffusionOp::computeDiffFluxes()"); @@ -623,26 +625,25 @@ DiffusionOp::computeDiffFluxes( // Duplicate phi since it is modified by the LinOp // and if have_density -> divide by density - Vector phi(finest_level + 1); + amrex::Vector phi(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { phi[lev].define( - a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), ncomp, 1, MFInfo(), - a_phi[lev]->Factory()); + a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), ncomp, 1, + amrex::MFInfo(), a_phi[lev]->Factory()); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(phi[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& gbx = mfi.growntilebox(); + for (amrex::MFIter mfi(phi[lev], amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + const amrex::Box& gbx = mfi.growntilebox(); auto const& a_phi_arr = a_phi[lev]->const_array(mfi, phi_comp); auto const& a_rho_arr = - (have_density) != 0 - ? a_density[lev]->const_array(mfi) - : a_phi[lev]->const_array(mfi); // Get dummy Array4 if no density + (have_density) != 0 ? a_density[lev]->const_array(mfi) + : a_phi[lev]->const_array( + mfi); // Get dummy amrex::Array4 if no density auto const& phi_arr = phi[lev].array(mfi); amrex::ParallelFor( - gbx, ncomp, - [a_phi_arr, a_rho_arr, phi_arr, - have_density] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + gbx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { if (have_density != 0) { phi_arr(i, j, k, n) = a_phi_arr(i, j, k, n) / a_rho_arr(i, j, k); } else { @@ -657,27 +658,28 @@ DiffusionOp::computeDiffFluxes( // => \beta = -1.0, B face centered diffusivity a_bcoeff // Set scalars \alpha & \beta - Real alpha = 0.0; - Real beta = -1.0; + amrex::Real alpha = 0.0; + amrex::Real beta = -1.0; m_scal_apply_op->setScalars(alpha, beta); // Get fluxes on a m_ncomp component(s) basis for (int comp = 0; comp < ncomp; comp += m_ncomp) { // Component based vector of data - Vector> fluxes(finest_level + 1); - Vector component; - Vector laps; - Vector boundary; + amrex::Vector> fluxes( + finest_level + 1); + amrex::Vector component; + amrex::Vector laps; + amrex::Vector boundary; // Allow for component specific LinOp BC m_scal_apply_op->setDomainBC( - m_pelelm->getDiffusionLinOpBC(Orientation::low, a_bcrec[comp]), - m_pelelm->getDiffusionLinOpBC(Orientation::high, a_bcrec[comp])); + m_pelelm->getDiffusionLinOpBC(amrex::Orientation::low, a_bcrec[comp]), + m_pelelm->getDiffusionLinOpBC(amrex::Orientation::high, a_bcrec[comp])); for (int lev = 0; lev <= finest_level; ++lev) { for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - fluxes[lev][idim] = new MultiFab( + fluxes[lev][idim] = new amrex::MultiFab( *a_flux[lev][idim], amrex::make_alias, flux_comp + comp, m_ncomp); } component.emplace_back(phi[lev], amrex::make_alias, comp, m_ncomp); @@ -690,30 +692,32 @@ DiffusionOp::computeDiffFluxes( int doZeroVisc = 1; int addTurbContrib = 1; - Vector subBCRec = { + amrex::Vector subBCRec = { a_bcrec.begin() + comp, a_bcrec.begin() + comp + m_ncomp}; - Array bcoeff_ec = m_pelelm->getDiffusivity( - lev, bcoeff_comp + comp, m_ncomp, doZeroVisc, subBCRec, *a_bcoeff[lev], - addTurbContrib); + amrex::Array bcoeff_ec = + m_pelelm->getDiffusivity( + lev, bcoeff_comp + comp, m_ncomp, doZeroVisc, subBCRec, + *a_bcoeff[lev], addTurbContrib); laps.emplace_back( a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), m_ncomp, 1, - MFInfo(), a_phi[lev]->Factory()); + amrex::MFInfo(), a_phi[lev]->Factory()); #ifdef AMREX_USE_EB m_scal_apply_op->setBCoeffs( - lev, GetArrOfConstPtrs(bcoeff_ec), MLMG::Location::FaceCentroid); + lev, GetArrOfConstPtrs(bcoeff_ec), amrex::MLMG::Location::FaceCentroid); #else m_scal_apply_op->setBCoeffs(lev, GetArrOfConstPtrs(bcoeff_ec)); #endif m_scal_apply_op->setLevelBC(lev, &boundary[lev]); } - MLMG mlmg(*m_scal_apply_op); + amrex::MLMG mlmg(*m_scal_apply_op); mlmg.apply(GetVecOfPtrs(laps), GetVecOfPtrs(component)); #ifdef AMREX_USE_EB mlmg.getFluxes( - fluxes, GetVecOfPtrs(component), MLMG::Location::FaceCentroid); + fluxes, GetVecOfPtrs(component), amrex::MLMG::Location::FaceCentroid); #else - mlmg.getFluxes(fluxes, GetVecOfPtrs(component), MLMG::Location::FaceCenter); + mlmg.getFluxes( + fluxes, GetVecOfPtrs(component), amrex::MLMG::Location::FaceCenter); #endif for (int lev = 0; lev <= finest_level; ++lev) { for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { @@ -731,21 +735,21 @@ DiffusionOp::computeDiffFluxes( #ifdef AMREX_USE_EB void DiffusionOp::computeDiffFluxes( - Vector> const& a_flux, + amrex::Vector> const& a_flux, int flux_comp, - Vector const& a_EBflux, + amrex::Vector const& a_EBflux, int ebflux_comp, - Vector const& a_phi, + amrex::Vector const& a_phi, int phi_comp, - Vector const& a_density, - Vector const& a_bcoeff, + amrex::Vector const& a_density, + amrex::Vector const& a_bcoeff, int bcoeff_comp, - Vector const& a_EBvalue, - Vector const& a_EBbcoeff, - Vector a_bcrec, + amrex::Vector const& a_EBvalue, + amrex::Vector const& a_EBbcoeff, + amrex::Vector a_bcrec, int ncomp, int do_avgDown, - Vector const& a_boundary) + amrex::Vector const& a_boundary) { BL_PROFILE("DiffusionOp::computeDiffFluxes()"); @@ -764,26 +768,25 @@ DiffusionOp::computeDiffFluxes( // Duplicate phi since it is modified by the LinOp // and if have_density -> divide by density - Vector phi(finest_level + 1); + amrex::Vector phi(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { phi[lev].define( - a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), ncomp, 1, MFInfo(), - a_phi[lev]->Factory()); + a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), ncomp, 1, + amrex::MFInfo(), a_phi[lev]->Factory()); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(phi[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& gbx = mfi.growntilebox(); + for (amrex::MFIter mfi(phi[lev], amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + const amrex::Box& gbx = mfi.growntilebox(); auto const& a_phi_arr = a_phi[lev]->const_array(mfi, phi_comp); auto const& a_rho_arr = - (have_density) != 0 - ? a_density[lev]->const_array(mfi) - : a_phi[lev]->const_array(mfi); // Get dummy Array4 if no density + (have_density) != 0 ? a_density[lev]->const_array(mfi) + : a_phi[lev]->const_array( + mfi); // Get dummy amrex::Array4 if no density auto const& phi_arr = phi[lev].array(mfi); amrex::ParallelFor( - gbx, ncomp, - [a_phi_arr, a_rho_arr, phi_arr, - have_density] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + gbx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { if (have_density != 0) { phi_arr(i, j, k, n) = a_phi_arr(i, j, k, n) / a_rho_arr(i, j, k); } else { @@ -798,33 +801,34 @@ DiffusionOp::computeDiffFluxes( // => \beta = -1.0, B face centered diffusivity a_bcoeff // Set scalars \alpha & \beta - Real alpha = 0.0; - Real beta = -1.0; + amrex::Real alpha = 0.0; + amrex::Real beta = -1.0; m_scal_apply_op->setScalars(alpha, beta); // Get fluxes on a m_ncomp component(s) basis for (int comp = 0; comp < ncomp; comp += m_ncomp) { // Component based vector of data - Vector, AMREX_SPACEDIM>> fluxes( - finest_level + 1); - Vector> ebfluxes; - Vector component; - Vector laps; - Vector boundary; + amrex::Vector< + amrex::Array, AMREX_SPACEDIM>> + fluxes(finest_level + 1); + amrex::Vector> ebfluxes; + amrex::Vector component; + amrex::Vector laps; + amrex::Vector boundary; // Allow for component specific LinOp BC m_scal_apply_op->setDomainBC( - m_pelelm->getDiffusionLinOpBC(Orientation::low, a_bcrec[comp]), - m_pelelm->getDiffusionLinOpBC(Orientation::high, a_bcrec[comp])); + m_pelelm->getDiffusionLinOpBC(amrex::Orientation::low, a_bcrec[comp]), + m_pelelm->getDiffusionLinOpBC(amrex::Orientation::high, a_bcrec[comp])); for (int lev = 0; lev <= finest_level; ++lev) { for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - fluxes[lev][idim] = std::make_unique( + fluxes[lev][idim] = std::make_unique( *a_flux[lev][idim], amrex::make_alias, flux_comp + comp, m_ncomp); } ebfluxes.push_back( - std::make_unique( + std::make_unique( *a_EBflux[lev], amrex::make_alias, ebflux_comp + comp, m_ncomp)); component.emplace_back(phi[lev], amrex::make_alias, comp, m_ncomp); if (have_boundary != 0) { @@ -834,24 +838,26 @@ DiffusionOp::computeDiffFluxes( boundary.emplace_back(phi[lev], amrex::make_alias, comp, m_ncomp); } int doZeroVisc = 1; - Vector subBCRec = { + amrex::Vector subBCRec = { a_bcrec.begin() + comp, a_bcrec.begin() + comp + m_ncomp}; - Array bcoeff_ec = m_pelelm->getDiffusivity( - lev, bcoeff_comp + comp, m_ncomp, doZeroVisc, subBCRec, *a_bcoeff[lev]); + amrex::Array bcoeff_ec = + m_pelelm->getDiffusivity( + lev, bcoeff_comp + comp, m_ncomp, doZeroVisc, subBCRec, + *a_bcoeff[lev]); laps.emplace_back( a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), m_ncomp, 1, - MFInfo(), a_phi[lev]->Factory()); + amrex::MFInfo(), a_phi[lev]->Factory()); m_scal_apply_op->setBCoeffs( - lev, GetArrOfConstPtrs(bcoeff_ec), MLMG::Location::FaceCentroid); + lev, GetArrOfConstPtrs(bcoeff_ec), amrex::MLMG::Location::FaceCentroid); m_scal_apply_op->setLevelBC(lev, &boundary[lev]); m_scal_apply_op->setEBDirichlet(lev, *a_EBvalue[lev], *a_EBbcoeff[lev]); } - MLMG mlmg(*m_scal_apply_op); + amrex::MLMG mlmg(*m_scal_apply_op); mlmg.apply(GetVecOfPtrs(laps), GetVecOfPtrs(component)); mlmg.getFluxes( GetVecOfArrOfPtrs(fluxes), GetVecOfPtrs(component), - MLMG::Location::FaceCentroid); + amrex::MLMG::Location::FaceCentroid); mlmg.getEBFluxes(GetVecOfPtrs(ebfluxes), GetVecOfPtrs(component)); } @@ -864,11 +870,11 @@ DiffusionOp::computeDiffFluxes( void DiffusionOp::computeGradient( - const Vector>& a_grad, - const Vector& a_laps, - const Vector& a_phi, - const Vector& a_boundary, - const BCRec& a_bcrec, + const amrex::Vector>& a_grad, + const amrex::Vector& a_laps, + const amrex::Vector& a_phi, + const amrex::Vector& a_boundary, + const amrex::BCRec& a_bcrec, int do_avgDown, int comp) const { @@ -890,28 +896,28 @@ DiffusionOp::computeGradient( // Set domainBCs m_gradient_op->setDomainBC( - m_pelelm->getDiffusionLinOpBC(Orientation::low, a_bcrec), - m_pelelm->getDiffusionLinOpBC(Orientation::high, a_bcrec)); + m_pelelm->getDiffusionLinOpBC(amrex::Orientation::low, a_bcrec), + m_pelelm->getDiffusionLinOpBC(amrex::Orientation::high, a_bcrec)); // Duplicate phi since it is modified by the LinOp // and setup level BCs - Vector phi(finest_level + 1); - Vector boundary(finest_level + 1); - Vector laps; + amrex::Vector phi(finest_level + 1); + amrex::Vector boundary(finest_level + 1); + amrex::Vector laps; for (int lev = 0; lev <= finest_level; ++lev) { phi[lev].define( - a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), 1, 1, MFInfo(), - a_phi[lev]->Factory()); + a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), 1, 1, + amrex::MFInfo(), a_phi[lev]->Factory()); boundary[lev].define( - a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), 1, 1, MFInfo(), - a_phi[lev]->Factory()); + a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), 1, 1, + amrex::MFInfo(), a_phi[lev]->Factory()); - MultiFab::Copy(phi[lev], *a_phi[lev], comp, 0, 1, 1); + amrex::MultiFab::Copy(phi[lev], *a_phi[lev], comp, 0, 1, 1); if (have_boundary != 0) { - MultiFab::Copy(boundary[lev], *a_boundary[lev], 0, 0, 1, 1); + amrex::MultiFab::Copy(boundary[lev], *a_boundary[lev], 0, 0, 1, 1); } else { - MultiFab::Copy(boundary[lev], *a_phi[lev], comp, 0, 1, 1); + amrex::MultiFab::Copy(boundary[lev], *a_phi[lev], comp, 0, 1, 1); } m_gradient_op->setLevelBC(lev, &boundary[lev]); @@ -919,17 +925,18 @@ DiffusionOp::computeGradient( laps.emplace_back(*a_laps[lev], amrex::make_alias, 0, 1); } else { laps.emplace_back( - a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), 1, 1, MFInfo(), - a_phi[lev]->Factory()); + a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), 1, 1, + amrex::MFInfo(), a_phi[lev]->Factory()); } } - MLMG mlmg(*m_gradient_op); + amrex::MLMG mlmg(*m_gradient_op); mlmg.apply(GetVecOfPtrs(laps), GetVecOfPtrs(phi)); #ifdef AMREX_USE_EB - mlmg.getFluxes(a_grad, GetVecOfPtrs(phi), MLMG::Location::FaceCentroid); + mlmg.getFluxes( + a_grad, GetVecOfPtrs(phi), amrex::MLMG::Location::FaceCentroid); #else - mlmg.getFluxes(a_grad, GetVecOfPtrs(phi), MLMG::Location::FaceCenter); + mlmg.getFluxes(a_grad, GetVecOfPtrs(phi), amrex::MLMG::Location::FaceCenter); #endif if (do_avgDown != 0) { avgDownFluxes(a_grad, 0, 1); @@ -938,7 +945,7 @@ DiffusionOp::computeGradient( void DiffusionOp::avgDownFluxes( - const Vector>& a_fluxes, + const amrex::Vector>& a_fluxes, int flux_comp, int ncomp) const { @@ -947,12 +954,12 @@ DiffusionOp::avgDownFluxes( for (int lev = finest_level; lev > 0; --lev) { // Get the requested components only - Array flux_fine; - Array flux_crse; + amrex::Array flux_fine; + amrex::Array flux_crse; for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - flux_fine[idim] = - new MultiFab(*a_fluxes[lev][idim], amrex::make_alias, flux_comp, ncomp); - flux_crse[idim] = new MultiFab( + flux_fine[idim] = new amrex::MultiFab( + *a_fluxes[lev][idim], amrex::make_alias, flux_comp, ncomp); + flux_crse[idim] = new amrex::MultiFab( *a_fluxes[lev - 1][idim], amrex::make_alias, flux_comp, ncomp); } #ifdef AMREX_USE_EB @@ -974,7 +981,7 @@ DiffusionOp::avgDownFluxes( void DiffusionOp::readParameters() { - ParmParse pp("diffusion"); + amrex::ParmParse pp("diffusion"); pp.query("verbose", m_mg_verbose); pp.query("atol", m_mg_atol); @@ -997,54 +1004,55 @@ DiffusionTensorOp::DiffusionTensorOp(PeleLM* a_pelelm) : m_pelelm(a_pelelm) auto bcRecVel = m_pelelm->fetchBCRecArray(VELX, AMREX_SPACEDIM); // Solve LPInfo - LPInfo info_solve; + amrex::LPInfo info_solve; info_solve.setMaxCoarseningLevel(m_mg_max_coarsening_level); #ifdef AMREX_USE_EB // Get vector of EB Factory - Vector ebfactVec; + amrex::Vector ebfactVec; for (int lev = 0; lev <= finest_level; ++lev) { ebfactVec.push_back(&(m_pelelm->EBFactory(lev))); } #endif #ifdef AMREX_USE_EB - m_solve_op = std::make_unique( + m_solve_op = std::make_unique( m_pelelm->Geom(0, finest_level), m_pelelm->boxArray(0, finest_level), m_pelelm->DistributionMap(0, finest_level), info_solve, ebfactVec); #else - m_solve_op = std::make_unique( + m_solve_op = std::make_unique( m_pelelm->Geom(0, finest_level), m_pelelm->boxArray(0, finest_level), m_pelelm->DistributionMap(0, finest_level), info_solve); #endif m_solve_op->setMaxOrder(m_mg_maxorder); m_solve_op->setDomainBC( - m_pelelm->getDiffusionTensorOpBC(Orientation::low, bcRecVel), - m_pelelm->getDiffusionTensorOpBC(Orientation::high, bcRecVel)); + m_pelelm->getDiffusionTensorOpBC(amrex::Orientation::low, bcRecVel), + m_pelelm->getDiffusionTensorOpBC(amrex::Orientation::high, bcRecVel)); // Apply LPInfo (no coarsening) - LPInfo info_apply; + amrex::LPInfo info_apply; info_apply.setMaxCoarseningLevel(0); #ifdef AMREX_USE_EB - m_apply_op = std::make_unique( + m_apply_op = std::make_unique( m_pelelm->Geom(0, finest_level), m_pelelm->boxArray(0, finest_level), m_pelelm->DistributionMap(0, finest_level), info_apply, ebfactVec); #else - m_apply_op = std::make_unique( + m_apply_op = std::make_unique( m_pelelm->Geom(0, finest_level), m_pelelm->boxArray(0, finest_level), m_pelelm->DistributionMap(0, finest_level), info_apply); #endif m_apply_op->setMaxOrder(m_mg_maxorder); m_apply_op->setDomainBC( - m_pelelm->getDiffusionTensorOpBC(Orientation::low, bcRecVel), - m_pelelm->getDiffusionTensorOpBC(Orientation::high, bcRecVel)); + m_pelelm->getDiffusionTensorOpBC(amrex::Orientation::low, bcRecVel), + m_pelelm->getDiffusionTensorOpBC(amrex::Orientation::high, bcRecVel)); } void DiffusionTensorOp::computeGradientTensor( - Vector> const& a_velgrad, - Vector const& a_vel) + amrex::Vector> const& + a_velgrad, + amrex::Vector const& a_vel) { // This function returns the velocity gradient tensor at faces // @@ -1057,12 +1065,12 @@ DiffusionTensorOp::computeGradientTensor( int finest_level = m_pelelm->finestLevel(); // Duplicate vel since it may be modified by the TensorOp - Vector vel(finest_level + 1); + amrex::Vector vel(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { vel[lev].define( a_vel[lev]->boxArray(), a_vel[lev]->DistributionMap(), AMREX_SPACEDIM, 1, - MFInfo(), a_vel[lev]->Factory()); - MultiFab::Copy(vel[lev], *a_vel[lev], 0, 0, AMREX_SPACEDIM, 1); + amrex::MFInfo(), a_vel[lev]->Factory()); + amrex::MultiFab::Copy(vel[lev], *a_vel[lev], 0, 0, AMREX_SPACEDIM, 1); } // Set up some parameters @@ -1081,49 +1089,49 @@ DiffusionTensorOp::computeGradientTensor( for (int lev = 0; lev <= finest_level; ++lev) { divtau.emplace_back( a_vel[lev]->boxArray(), a_vel[lev]->DistributionMap(), AMREX_SPACEDIM, 1, - MFInfo(), a_vel[lev]->Factory()); + amrex::MFInfo(), a_vel[lev]->Factory()); } // Create MLMG object and apply to setup BCs, etc - MLMG mlmg(*m_apply_op); + amrex::MLMG mlmg(*m_apply_op); mlmg.apply(GetVecOfPtrs(divtau), GetVecOfPtrs(vel)); // Compute the velocity gradient on faces for (int lev = 0; lev <= finest_level; ++lev) { m_apply_op->compVelGrad( - lev, a_velgrad[lev], vel[lev], MLLinOp::Location::FaceCentroid); + lev, a_velgrad[lev], vel[lev], amrex::MLLinOp::Location::FaceCentroid); } } void DiffusionTensorOp::compute_divtau( - Vector const& a_divtau, - Vector const& a_vel, - Vector const& a_density, - Vector const& a_beta, - const BCRec& a_bcrec, - Real scale) + amrex::Vector const& a_divtau, + amrex::Vector const& a_vel, + amrex::Vector const& a_density, + amrex::Vector const& a_beta, + const amrex::BCRec& a_bcrec, + amrex::Real scale) { int finest_level = m_pelelm->finestLevel(); int have_density = (a_density.empty()) ? 0 : 1; // Duplicate vel since it is modified by the TensorOp - Vector vel(finest_level + 1); + amrex::Vector vel(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { vel[lev].define( a_vel[lev]->boxArray(), a_vel[lev]->DistributionMap(), AMREX_SPACEDIM, 2, - MFInfo(), a_vel[lev]->Factory()); - MultiFab::Copy(vel[lev], *a_vel[lev], 0, 0, AMREX_SPACEDIM, 2); + amrex::MFInfo(), a_vel[lev]->Factory()); + amrex::MultiFab::Copy(vel[lev], *a_vel[lev], 0, 0, AMREX_SPACEDIM, 2); } #ifdef AMREX_USE_EB // Need a temporary divTau to apply redistribution - Vector divtau_tmp(finest_level + 1); + amrex::Vector divtau_tmp(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { divtau_tmp[lev].define( a_divtau[lev]->boxArray(), a_divtau[lev]->DistributionMap(), - AMREX_SPACEDIM, 2, MFInfo(), a_divtau[lev]->Factory()); + AMREX_SPACEDIM, 2, amrex::MFInfo(), a_divtau[lev]->Factory()); divtau_tmp[lev].setVal(0.0); } @@ -1135,10 +1143,11 @@ DiffusionTensorOp::compute_divtau( } int doZeroVisc = 0; int addTurbContrib = 1; - Array beta_ec = m_pelelm->getDiffusivity( - lev, 0, 1, doZeroVisc, {a_bcrec}, *a_beta[lev], addTurbContrib); + amrex::Array beta_ec = + m_pelelm->getDiffusivity( + lev, 0, 1, doZeroVisc, {a_bcrec}, *a_beta[lev], addTurbContrib); m_apply_op->setShearViscosity( - lev, GetArrOfConstPtrs(beta_ec), MLMG::Location::FaceCentroid); + lev, GetArrOfConstPtrs(beta_ec), amrex::MLMG::Location::FaceCentroid); if (m_pelelm->m_useEBinflow != 0) { m_apply_op->setEBShearViscosityWithInflow( lev, *a_beta[lev], @@ -1150,7 +1159,7 @@ DiffusionTensorOp::compute_divtau( m_apply_op->setLevelBC(lev, &vel[lev]); } - MLMG mlmg(*m_apply_op); + amrex::MLMG mlmg(*m_apply_op); mlmg.apply(GetVecOfPtrs(divtau_tmp), GetVecOfPtrs(vel)); // Flux redistribute explicit diffusion fluxes into outgoing a_divtau @@ -1168,28 +1177,30 @@ DiffusionTensorOp::compute_divtau( } int doZeroVisc = 0; int addTurbContrib = 1; - Array beta_ec = m_pelelm->getDiffusivity( - lev, 0, 1, doZeroVisc, {a_bcrec}, *a_beta[lev], addTurbContrib); + amrex::Array beta_ec = + m_pelelm->getDiffusivity( + lev, 0, 1, doZeroVisc, {a_bcrec}, *a_beta[lev], addTurbContrib); m_apply_op->setShearViscosity(lev, GetArrOfConstPtrs(beta_ec)); m_apply_op->setLevelBC(lev, &vel[lev]); } - MLMG mlmg(*m_apply_op); + amrex::MLMG mlmg(*m_apply_op); mlmg.apply(a_divtau, GetVecOfPtrs(vel)); #endif if (have_density != 0) { for (int lev = 0; lev <= finest_level; ++lev) { #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*a_divtau[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) { - Box const& bx = mfi.tilebox(); + for (amrex::MFIter mfi(*a_divtau[lev], amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + amrex::Box const& bx = mfi.tilebox(); auto const& divtau_arr = a_divtau[lev]->array(mfi); auto const& rho_arr = a_density[lev]->const_array(mfi); amrex::ParallelFor( bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - Real rhoinv = 1.0 / rho_arr(i, j, k); + amrex::Real rhoinv = 1.0 / rho_arr(i, j, k); AMREX_D_TERM(divtau_arr(i, j, k, 0) *= rhoinv; , divtau_arr(i, j, k, 1) *= rhoinv; , divtau_arr(i, j, k, 2) *= rhoinv;); @@ -1201,11 +1212,11 @@ DiffusionTensorOp::compute_divtau( void DiffusionTensorOp::diffuse_velocity( - Vector const& a_vel, - Vector const& a_density, - Vector const& a_beta, - const BCRec& a_bcrec, - Real a_dt) + amrex::Vector const& a_vel, + amrex::Vector const& a_density, + amrex::Vector const& a_beta, + const amrex::BCRec& a_bcrec, + amrex::Real a_dt) { const int finest_level = m_pelelm->finestLevel(); @@ -1225,11 +1236,12 @@ DiffusionTensorOp::diffuse_velocity( } int doZeroVisc = 0; int addTurbContrib = 1; - Array beta_ec = m_pelelm->getDiffusivity( - lev, 0, 1, doZeroVisc, {a_bcrec}, *a_beta[lev], addTurbContrib); + amrex::Array beta_ec = + m_pelelm->getDiffusivity( + lev, 0, 1, doZeroVisc, {a_bcrec}, *a_beta[lev], addTurbContrib); #ifdef AMREX_USE_EB m_solve_op->setShearViscosity( - lev, GetArrOfConstPtrs(beta_ec), MLMG::Location::FaceCentroid); + lev, GetArrOfConstPtrs(beta_ec), amrex::MLMG::Location::FaceCentroid); if (m_pelelm->m_useEBinflow != 0) { m_solve_op->setEBShearViscosityWithInflow( lev, *a_beta[lev], @@ -1244,25 +1256,26 @@ DiffusionTensorOp::diffuse_velocity( m_solve_op->setLevelBC(lev, a_vel[lev]); } - Vector rhs(finest_level + 1); + amrex::Vector rhs(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { rhs[lev].define( a_vel[lev]->boxArray(), a_vel[lev]->DistributionMap(), AMREX_SPACEDIM, 0); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(rhs[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) { - Box const& bx = mfi.tilebox(); + for (amrex::MFIter mfi(rhs[lev], amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + amrex::Box const& bx = mfi.tilebox(); auto const& rhs_a = rhs[lev].array(mfi); auto const& vel_a = a_vel[lev]->const_array(mfi); - auto const& rho_a = (have_density) != 0 ? a_density[lev]->const_array(mfi) - : Array4{}; + auto const& rho_a = (have_density) != 0 + ? a_density[lev]->const_array(mfi) + : amrex::Array4{}; + const auto rho_incomp = m_pelelm->m_rho; + const auto is_incomp = m_pelelm->m_incompressible; amrex::ParallelFor( bx, AMREX_SPACEDIM, - [=, rho_incomp = m_pelelm->m_rho, - is_incomp = - m_pelelm - ->m_incompressible] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { if (is_incomp != 0) { rhs_a(i, j, k, n) = rho_incomp * vel_a(i, j, k, n); } else { @@ -1272,7 +1285,7 @@ DiffusionTensorOp::diffuse_velocity( } } - MLMG mlmg(*m_solve_op); + amrex::MLMG mlmg(*m_solve_op); // Maximum iterations for MultiGrid / ConjugateGradients mlmg.setMaxIter(m_mg_max_iter); @@ -1292,7 +1305,7 @@ DiffusionTensorOp::diffuse_velocity( void DiffusionTensorOp::readParameters() { - ParmParse pp("tensor_diffusion"); + amrex::ParmParse pp("tensor_diffusion"); pp.query("verbose", m_mg_verbose); pp.query("atol", m_mg_atol); diff --git a/Source/PeleLMeX_EB.cpp b/Source/PeleLMeX_EB.cpp index ace760822..2d451c58a 100644 --- a/Source/PeleLMeX_EB.cpp +++ b/Source/PeleLMeX_EB.cpp @@ -10,8 +10,6 @@ #include #include -using namespace amrex; - void PeleLM::makeEBGeometry() { @@ -20,7 +18,7 @@ PeleLM::makeEBGeometry() int req_coarsening_level = static_cast(geom.size()) - 1; // Read the geometry type and act accordingly - ParmParse ppeb2("eb2"); + amrex::ParmParse ppeb2("eb2"); std::string geom_type; ppeb2.get("geom_type", geom_type); @@ -47,28 +45,28 @@ PeleLM::makeEBGeometry() geom[prev_max_lvl_eb], req_coarsening_level, max_coarsening_level); } else { // If geom_type is not an AMReX recognized type, it'll crash. - EB2::Build( + amrex::EB2::Build( geom[prev_max_lvl_eb], req_coarsening_level, max_coarsening_level); } // Add finer level, might be inconsistent with the coarser level created // above. - EB2::addFineLevels(max_level - prev_max_lvl_eb); + amrex::EB2::addFineLevels(max_level - prev_max_lvl_eb); } void PeleLM::redistributeAofS( int a_lev, - Real& a_dt, - MultiFab& a_tmpDiv, + amrex::Real& a_dt, + amrex::MultiFab& a_tmpDiv, int div_comp, - MultiFab& a_AofS, + amrex::MultiFab& a_AofS, int aofs_comp, - MultiFab& a_state, + amrex::MultiFab& a_state, int state_comp, int ncomp, - const BCRec* d_bc, - const Geometry& a_geom) const + const amrex::BCRec* d_bc, + const amrex::Geometry& a_geom) const { BL_PROFILE("PeleLMeX::redistributeAofS()"); AMREX_ASSERT(a_tmpDiv.nComp() >= div_comp + ncomp); @@ -79,34 +77,37 @@ PeleLM::redistributeAofS( const auto& ebfact = EBFactory(a_lev); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(a_tmpDiv, TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(a_tmpDiv, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { - Box const& bx = mfi.tilebox(); + amrex::Box const& bx = mfi.tilebox(); auto const& flagfab = ebfact.getMultiEBCellFlagFab()[mfi]; auto const& flag = flagfab.const_array(); auto const& divT_ar = a_tmpDiv.array(mfi, div_comp); auto const& aofs_ar = a_AofS.array(mfi, aofs_comp); - if (flagfab.getType(bx) != FabType::covered) { - if (flagfab.getType(grow(bx, 4)) != FabType::regular) { + if (flagfab.getType(bx) != amrex::FabType::covered) { + if (flagfab.getType(grow(bx, 4)) != amrex::FabType::regular) { AMREX_D_TERM( auto apx = ebfact.getAreaFrac()[0]->const_array(mfi); , auto apy = ebfact.getAreaFrac()[1]->const_array(mfi); , auto apz = ebfact.getAreaFrac()[2]->const_array(mfi);); AMREX_D_TERM( - Array4 fcx = ebfact.getFaceCent()[0]->const_array(mfi); - , Array4 fcy = ebfact.getFaceCent()[1]->const_array(mfi); - , - Array4 fcz = ebfact.getFaceCent()[2]->const_array(mfi);); + amrex::Array4 fcx = + ebfact.getFaceCent()[0]->const_array(mfi); + , amrex::Array4 fcy = + ebfact.getFaceCent()[1]->const_array(mfi); + , amrex::Array4 fcz = + ebfact.getFaceCent()[2]->const_array(mfi);); auto const& ccc = ebfact.getCentroid().const_array(mfi); auto const& vfrac_arr = ebfact.getVolFrac().const_array(mfi); // This is scratch space if calling StateRedistribute, // but is used as the weights (here set to 1) if calling // FluxRedistribute - Box gbx = bx; + amrex::Box gbx = bx; if (m_adv_redist_type == "StateRedist") { gbx.grow(3); @@ -114,11 +115,12 @@ PeleLM::redistributeAofS( gbx.grow(2); } - FArrayBox tmpfab(gbx, ncomp, The_Async_Arena()); - Array4 scratch = tmpfab.array(0); + amrex::FArrayBox tmpfab(gbx, ncomp, amrex::The_Async_Arena()); + amrex::Array4 scratch = tmpfab.array(0); if (m_adv_redist_type == "FluxRedist") { amrex::ParallelFor( - Box(scratch), [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + amrex::Box(scratch), + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { scratch(i, j, k) = 1.; }); } @@ -139,7 +141,7 @@ PeleLM::redistributeAofS( } void -PeleLM::getCoveredIMask(int a_lev, iMultiFab& a_imask) const +PeleLM::getCoveredIMask(int a_lev, amrex::iMultiFab& a_imask) const { const auto& ebfact = EBFactory(a_lev); const auto& flags = ebfact.getMultiEBCellFlagFab(); @@ -149,12 +151,13 @@ PeleLM::getCoveredIMask(int a_lev, iMultiFab& a_imask) const (a_imask.DistributionMap() == flags.DistributionMap())) { #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(a_imask, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(a_imask, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + const amrex::Box& bx = mfi.tilebox(); const auto& flagarr = flags.const_array(mfi); - Array4 const& arr = a_imask.array(mfi); + amrex::Array4 const& arr = a_imask.array(mfi); AMREX_HOST_DEVICE_PARALLEL_FOR_3D(bx, i, j, k, { if (flagarr(i, j, k).isCovered()) { arr(i, j, k) = -1; @@ -165,15 +168,16 @@ PeleLM::getCoveredIMask(int a_lev, iMultiFab& a_imask) const } } else { - iMultiFab mask_tmp(flags.boxArray(), flags.DistributionMap(), 1, 0); + amrex::iMultiFab mask_tmp(flags.boxArray(), flags.DistributionMap(), 1, 0); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(mask_tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(mask_tmp, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + const amrex::Box& bx = mfi.tilebox(); const auto& flagarr = flags.const_array(mfi); - Array4 const& arr = mask_tmp.array(mfi); + amrex::Array4 const& arr = mask_tmp.array(mfi); AMREX_HOST_DEVICE_PARALLEL_FOR_3D(bx, i, j, k, { if (flagarr(i, j, k).isCovered()) { arr(i, j, k) = -1; @@ -189,16 +193,16 @@ PeleLM::getCoveredIMask(int a_lev, iMultiFab& a_imask) const void PeleLM::redistributeDiff( int a_lev, - const Real& a_dt, - MultiFab& a_tmpDiv, + const amrex::Real& a_dt, + amrex::MultiFab& a_tmpDiv, int div_comp, - MultiFab& a_diff, + amrex::MultiFab& a_diff, int diff_comp, - const MultiFab& a_state, + const amrex::MultiFab& a_state, int state_comp, int ncomp, - const BCRec* d_bc, - const Geometry& a_geom) const + const amrex::BCRec* d_bc, + const amrex::Geometry& a_geom) const { BL_PROFILE("PeleLMeX::redistributeDiff()"); AMREX_ASSERT(a_tmpDiv.nComp() >= div_comp + ncomp); @@ -209,34 +213,37 @@ PeleLM::redistributeDiff( const auto& ebfact = EBFactory(a_lev); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(a_tmpDiv, TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(a_tmpDiv, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { - Box const& bx = mfi.tilebox(); + amrex::Box const& bx = mfi.tilebox(); auto const& flagfab = ebfact.getMultiEBCellFlagFab()[mfi]; auto const& flag = flagfab.const_array(); auto const& divT_ar = a_tmpDiv.array(mfi, div_comp); auto const& diff_ar = a_diff.array(mfi, diff_comp); - if (flagfab.getType(bx) != FabType::covered) { - if (flagfab.getType(grow(bx, 4)) != FabType::regular) { + if (flagfab.getType(bx) != amrex::FabType::covered) { + if (flagfab.getType(grow(bx, 4)) != amrex::FabType::regular) { AMREX_D_TERM( auto apx = ebfact.getAreaFrac()[0]->const_array(mfi); , auto apy = ebfact.getAreaFrac()[1]->const_array(mfi); , auto apz = ebfact.getAreaFrac()[2]->const_array(mfi);); AMREX_D_TERM( - Array4 fcx = ebfact.getFaceCent()[0]->const_array(mfi); - , Array4 fcy = ebfact.getFaceCent()[1]->const_array(mfi); - , - Array4 fcz = ebfact.getFaceCent()[2]->const_array(mfi);); + amrex::Array4 fcx = + ebfact.getFaceCent()[0]->const_array(mfi); + , amrex::Array4 fcy = + ebfact.getFaceCent()[1]->const_array(mfi); + , amrex::Array4 fcz = + ebfact.getFaceCent()[2]->const_array(mfi);); auto const& ccc = ebfact.getCentroid().const_array(mfi); auto const& vfrac_arr = ebfact.getVolFrac().const_array(mfi); // This is scratch space if calling StateRedistribute, // but is used as the weights (here set to 1) if calling // FluxRedistribute - Box gbx = bx; + amrex::Box gbx = bx; if (m_diff_redist_type == "StateRedist") { gbx.grow(3); @@ -244,11 +251,12 @@ PeleLM::redistributeDiff( gbx.grow(2); } - FArrayBox tmpfab(gbx, ncomp, The_Async_Arena()); - Array4 scratch = tmpfab.array(0); + amrex::FArrayBox tmpfab(gbx, ncomp, amrex::The_Async_Arena()); + amrex::Array4 scratch = tmpfab.array(0); if (m_diff_redist_type == "FluxRedist") { amrex::ParallelFor( - Box(scratch), [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + amrex::Box(scratch), + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { scratch(i, j, k) = 1.; }); } @@ -278,8 +286,8 @@ PeleLM::initCoveredState() coveredState_h[0] = 0.0;, coveredState_h[1] = 0.0; , coveredState_h[2] = 0.0;) coveredState_d.resize(AMREX_SPACEDIM); - Gpu::copy( - Gpu::hostToDevice, coveredState_h.begin(), coveredState_h.end(), + amrex::Gpu::copy( + amrex::Gpu::hostToDevice, coveredState_h.begin(), coveredState_h.end(), coveredState_d.begin()); } else { coveredState_h.resize(NVAR); @@ -295,8 +303,8 @@ PeleLM::initCoveredState() coveredState_h[RHORT] = typical_values[RHORT]; coveredState_d.resize(NVAR); - Gpu::copy( - Gpu::hostToDevice, coveredState_h.begin(), coveredState_h.end(), + amrex::Gpu::copy( + amrex::Gpu::hostToDevice, coveredState_h.begin(), coveredState_h.end(), coveredState_d.begin()); } } @@ -334,7 +342,7 @@ PeleLM::initialRedistribution() for (int lev = 0; lev <= finest_level; ++lev) { // New time - Real timeNew = getTime(lev, AmrNewTime); + amrex::Real timeNew = getTime(lev, AmrNewTime); // Jungle with Old/New states: fillPatch old and redistribute // from Old->New to end up with proper New state @@ -345,34 +353,35 @@ PeleLM::initialRedistribution() // State if (m_incompressible != 0) { - Vector stateCovered(AMREX_SPACEDIM, 0.0); + amrex::Vector stateCovered(AMREX_SPACEDIM, 0.0); EB_set_covered(ldataNew_p->state, 0, AMREX_SPACEDIM, stateCovered); ldataNew_p->state.FillBoundary(geom[lev].periodicity()); - MultiFab::Copy( + amrex::MultiFab::Copy( ldataOld_p->state, ldataNew_p->state, 0, 0, AMREX_SPACEDIM, m_nGrowState); } else { - Vector stateCovered(NVAR, 0.0); + amrex::Vector stateCovered(NVAR, 0.0); EB_set_covered(ldataNew_p->state, 0, NVAR, stateCovered); ldataNew_p->state.FillBoundary(geom[lev].periodicity()); - MultiFab::Copy( + amrex::MultiFab::Copy( ldataOld_p->state, ldataNew_p->state, 0, 0, NVAR, m_nGrowState); } fillpatch_state(lev, timeNew, ldataOld_p->state, m_nGrowState); - for (MFIter mfi(ldataNew_p->state, TilingIfNotGPU()); mfi.isValid(); - ++mfi) { + for (amrex::MFIter mfi(ldataNew_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { - const Box& bx = mfi.validbox(); + const amrex::Box& bx = mfi.validbox(); - EBCellFlagFab const& flagfab = fact.getMultiEBCellFlagFab()[mfi]; - Array4 const& flag = flagfab.const_array(); + amrex::EBCellFlagFab const& flagfab = fact.getMultiEBCellFlagFab()[mfi]; + amrex::Array4 const& flag = + flagfab.const_array(); if ( - (flagfab.getType(bx) != FabType::covered) && - (flagfab.getType(amrex::grow(bx, 4)) != FabType::regular)) { - Array4 AMREX_D_DECL(fcx, fcy, fcz), ccc, vfrac, - AMREX_D_DECL(apx, apy, apz); + (flagfab.getType(bx) != amrex::FabType::covered) && + (flagfab.getType(amrex::grow(bx, 4)) != amrex::FabType::regular)) { + amrex::Array4 AMREX_D_DECL(fcx, fcy, fcz), ccc, + vfrac, AMREX_D_DECL(apx, apy, apz); AMREX_D_TERM( fcx = fact.getFaceCent()[0]->const_array(mfi); , fcy = fact.getFaceCent()[1]->const_array(mfi); @@ -411,47 +420,49 @@ PeleLM::initialRedistribution() } void -PeleLM::getEBDistance(int a_lev, MultiFab& a_signDistLev) +PeleLM::getEBDistance(int a_lev, amrex::MultiFab& a_signDistLev) { BL_PROFILE("PeleLMeX::getEBDistance()"); if (a_lev == 0) { - MultiFab::Copy(a_signDistLev, *m_signedDist0, 0, 0, 1, 0); + amrex::MultiFab::Copy(a_signDistLev, *m_signedDist0, 0, 0, 1, 0); return; } // A pair of MF to hold crse & fine dist data - Array, 2> MFpair; + amrex::Array, 2> MFpair; // Interpolate on successive levels up to a_lev for (int lev = 1; lev <= a_lev; ++lev) { // Use MF EB interp - auto& interpolater = eb_mf_lincc_interp; + auto& interpolater = amrex::eb_mf_lincc_interp; // Get signDist on coarsen fineBA - BoxArray coarsenBA(grids[lev].size()); + amrex::BoxArray coarsenBA(grids[lev].size()); for (int j = 0, N = static_cast(coarsenBA.size()); j < N; ++j) { coarsenBA.set( j, interpolater.CoarseBox(grids[lev][j], refRatio(lev - 1))); } - MultiFab coarsenSignDist(coarsenBA, dmap[lev], 1, 0); + amrex::MultiFab coarsenSignDist(coarsenBA, dmap[lev], 1, 0); coarsenSignDist.setVal(0.0); - MultiFab* crseSignDist = (lev == 1) ? m_signedDist0.get() : MFpair[0].get(); + amrex::MultiFab* crseSignDist = + (lev == 1) ? m_signedDist0.get() : MFpair[0].get(); coarsenSignDist.ParallelCopy(*crseSignDist, 0, 0, 1); // Interpolate on current lev - MultiFab* currentSignDist; + amrex::MultiFab* currentSignDist; if (lev < a_lev) { - MFpair[1] = std::make_unique( - grids[lev], dmap[lev], 1, 0, MFInfo(), EBFactory(lev)); + MFpair[1] = std::make_unique( + grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), EBFactory(lev)); } currentSignDist = (lev == a_lev) ? &a_signDistLev : MFpair[1].get(); interpolater.interp( - coarsenSignDist, 0, *currentSignDist, 0, 1, IntVect(0), Geom(lev - 1), - Geom(lev), Geom(lev).Domain(), refRatio(lev - 1), {m_bcrec_force}, 0); + coarsenSignDist, 0, *currentSignDist, 0, 1, amrex::IntVect(0), + Geom(lev - 1), Geom(lev), Geom(lev).Domain(), refRatio(lev - 1), + {m_bcrec_force}, 0); // Swap MFpair if (lev < a_lev) { @@ -460,38 +471,40 @@ PeleLM::getEBDistance(int a_lev, MultiFab& a_signDistLev) } } -Vector> +amrex::Vector> PeleLM::getEBState(int first_comp, int ncomp, const PeleLM::TimeStamp& a_time) { AMREX_ASSERT(first_comp >= VELX); AMREX_ASSERT(first_comp + ncomp <= NVAR); - Vector> r; + amrex::Vector> r; r.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { r.push_back( - std::make_unique( - grids[lev], dmap[lev], ncomp, m_nGrowState, MFInfo(), Factory(lev))); + std::make_unique( + grids[lev], dmap[lev], ncomp, m_nGrowState, amrex::MFInfo(), + Factory(lev))); getEBState(lev, a_time, *r[lev], first_comp, ncomp); } return r; } -std::unique_ptr +std::unique_ptr PeleLM::getEBState( int a_lev, int first_comp, int ncomp, const PeleLM::TimeStamp& a_time) { AMREX_ASSERT(first_comp >= VELX); AMREX_ASSERT(first_comp + ncomp <= NVAR); - std::unique_ptr r = std::make_unique( - grids[a_lev], dmap[a_lev], ncomp, m_nGrowState, MFInfo(), Factory(a_lev)); + std::unique_ptr r = std::make_unique( + grids[a_lev], dmap[a_lev], ncomp, m_nGrowState, amrex::MFInfo(), + Factory(a_lev)); getEBState(a_lev, a_time, *r, first_comp, ncomp); return r; } -FArrayBox +amrex::FArrayBox PeleLM::getEBState( - MFIter const& a_mfi, + amrex::MFIter const& a_mfi, int a_lev, int first_comp, int ncomp, @@ -507,11 +520,11 @@ PeleLM::getEBState( auto* ldata_p = getLevelDataPtr(a_lev, a_time); const auto bx = a_mfi.growntilebox(m_nGrowState); - FArrayBox r(bx, ncomp, The_Async_Arena()); + amrex::FArrayBox r(bx, ncomp, amrex::The_Async_Arena()); auto ebscal_arr = r.array(); const auto& ebfact = EBFactory(a_lev); auto const& flagfab = ebfact.getMultiEBCellFlagFab()[a_mfi]; - if (flagfab.getType(bx) == FabType::singlevalued) { + if (flagfab.getType(bx) == amrex::FabType::singlevalued) { const PeleLMFillBCStateEB< ProblemSpecificFunctions, hasBCNormalEB::value> @@ -547,7 +560,7 @@ void PeleLM::getEBState( int a_lev, const PeleLM::TimeStamp& a_time, - MultiFab& a_EBstate, + amrex::MultiFab& a_EBstate, int stateComp, int nComp) { @@ -557,28 +570,29 @@ PeleLM::getEBState( ProbParm const* lprobparm = prob_parm_d; const auto geomdata = geom[a_lev].data(); const auto& ebfact = EBFactory(a_lev); - Array faceCentroid = ebfact.getFaceCent(); + amrex::Array faceCentroid = + ebfact.getFaceCent(); auto time = getTime(a_lev, a_time); auto* ldata_p = getLevelDataPtr(a_lev, a_time); - MFItInfo mfi_info; - if (Gpu::notInLaunchRegion()) { + amrex::MFItInfo mfi_info; + if (amrex::Gpu::notInLaunchRegion()) { mfi_info.EnableTiling().SetDynamic(true); } #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(a_EBstate, mfi_info); mfi.isValid(); ++mfi) { - const Box& bx = mfi.growntilebox(); + for (amrex::MFIter mfi(a_EBstate, mfi_info); mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.growntilebox(); auto const& flagfab = ebfact.getMultiEBCellFlagFab()[mfi]; auto const& flag = flagfab.const_array(); const auto& ebState = a_EBstate.array(mfi); - if (flagfab.getType(bx) == FabType::covered) { // Set to zero + if (flagfab.getType(bx) == amrex::FabType::covered) { // Set to zero AMREX_PARALLEL_FOR_4D( bx, nComp, i, j, k, n, { ebState(i, j, k, n) = 0.0; }); - } else if (flagfab.getType(bx) == FabType::regular) { // Set to zero + } else if (flagfab.getType(bx) == amrex::FabType::regular) { // Set to zero AMREX_PARALLEL_FOR_4D( bx, nComp, i, j, k, n, { ebState(i, j, k, n) = 0.0; }); } else { @@ -611,35 +625,36 @@ PeleLM::getEBState( void PeleLM::getEBDiff( - int a_lev, const TimeStamp& a_time, MultiFab& a_EBDiff, int diffComp) + int a_lev, const TimeStamp& a_time, amrex::MultiFab& a_EBDiff, int diffComp) { // Get Geom / EB data ProbParm const* lprobparm = prob_parm_d; const auto geomdata = geom[a_lev].data(); const auto& ebfact = EBFactory(a_lev); - Array faceCentroid = ebfact.getFaceCent(); + amrex::Array faceCentroid = + ebfact.getFaceCent(); // Get diffusivity cell-centered auto* ldata_p = getLevelDataPtr(a_lev, a_time); - MFItInfo mfi_info; - if (Gpu::notInLaunchRegion()) { + amrex::MFItInfo mfi_info; + if (amrex::Gpu::notInLaunchRegion()) { mfi_info.EnableTiling().SetDynamic(true); } #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(a_EBDiff, mfi_info); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(a_EBDiff, mfi_info); mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); auto const& flagfab = ebfact.getMultiEBCellFlagFab()[mfi]; auto const& flag = flagfab.const_array(); const auto& ebdiff = a_EBDiff.array(mfi); const auto& diff_cc = ldata_p->diff_cc.const_array(mfi, diffComp); - if (flagfab.getType(bx) == FabType::covered) { // Set to zero + if (flagfab.getType(bx) == amrex::FabType::covered) { // Set to zero AMREX_PARALLEL_FOR_3D(bx, i, j, k, { ebdiff(i, j, k) = 0.0; }); - } else if (flagfab.getType(bx) == FabType::regular) { // Set to zero + } else if (flagfab.getType(bx) == amrex::FabType::regular) { // Set to zero AMREX_PARALLEL_FOR_3D(bx, i, j, k, { ebdiff(i, j, k) = 0.0; }); } else { const PeleLMFillBCTypeEB< @@ -657,7 +672,7 @@ PeleLM::getEBDiff( ebdiff(i, j, k) = 0.0; } else { // cut-cells int ebflagtype = pelelmex::BCTypeEB::wall_adiab; - Real ebfacefrac = 0.0; + amrex::Real ebfacefrac = 0.0; EBTypfiller( i, j, k, ebflagtype, ebfacefrac, AMREX_D_DECL(ebfc_x, ebfc_y, ebfc_z), geomdata, *lprobparm); @@ -675,20 +690,23 @@ PeleLM::getEBDiff( void PeleLM::correct_vel_small_cells( - Vector const& a_vel, - Vector> const& a_umac) + amrex::Vector const& a_vel, + amrex::Vector> const& + a_umac) { BL_PROFILE("PeleLMeX::correct_vel_small_cells"); for (int lev = 0; lev <= finest_level; lev++) { #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*a_vel[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*a_vel[lev], amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { // Tilebox - const Box bx = mfi.tilebox(); + const amrex::Box bx = mfi.tilebox(); - EBCellFlagFab const& flags = EBFactory(lev).getMultiEBCellFlagFab()[mfi]; + amrex::EBCellFlagFab const& flags = + EBFactory(lev).getMultiEBCellFlagFab()[mfi]; // Face-centered velocity components AMREX_D_TERM( @@ -698,8 +716,8 @@ PeleLM::correct_vel_small_cells( // No cut cells in this FAB if ( - (flags.getType(amrex::grow(bx, 0)) == FabType::covered) || - (flags.getType(amrex::grow(bx, 1)) == FabType::regular)) { + (flags.getType(amrex::grow(bx, 0)) == amrex::FabType::covered) || + (flags.getType(amrex::grow(bx, 1)) == amrex::FabType::regular)) { // do nothing } else { // Cut cells in this FAB // Face-centered areas @@ -718,21 +736,21 @@ PeleLM::correct_vel_small_cells( // This FAB has cut cells -- we define the centroid value in terms of // the MAC velocities onfaces amrex::ParallelFor( - bx, [vfrac_fab, AMREX_D_DECL(apx_fab, apy_fab, apz_fab), ccvel_fab, - AMREX_D_DECL( - umac_fab, vmac_fab, - wmac_fab)] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (vfrac_fab(i, j, k) > 0.0 && vfrac_fab(i, j, k) < 5.e-3) { AMREX_D_TERM( - Real u_avg = (apx_fab(i, j, k) * umac_fab(i, j, k) + - apx_fab(i + 1, j, k) * umac_fab(i + 1, j, k)) / - (apx_fab(i, j, k) + apx_fab(i + 1, j, k)); - , Real v_avg = (apy_fab(i, j, k) * vmac_fab(i, j, k) + - apy_fab(i, j + 1, k) * vmac_fab(i, j + 1, k)) / - (apy_fab(i, j, k) + apy_fab(i, j + 1, k)); - , Real w_avg = (apz_fab(i, j, k) * wmac_fab(i, j, k) + - apz_fab(i, j, k + 1) * wmac_fab(i, j, k + 1)) / - (apz_fab(i, j, k) + apz_fab(i, j, k + 1));); + amrex::Real u_avg = + (apx_fab(i, j, k) * umac_fab(i, j, k) + + apx_fab(i + 1, j, k) * umac_fab(i + 1, j, k)) / + (apx_fab(i, j, k) + apx_fab(i + 1, j, k)); + , amrex::Real v_avg = + (apy_fab(i, j, k) * vmac_fab(i, j, k) + + apy_fab(i, j + 1, k) * vmac_fab(i, j + 1, k)) / + (apy_fab(i, j, k) + apy_fab(i, j + 1, k)); + , amrex::Real w_avg = + (apz_fab(i, j, k) * wmac_fab(i, j, k) + + apz_fab(i, j, k + 1) * wmac_fab(i, j, k + 1)) / + (apz_fab(i, j, k) + apz_fab(i, j, k + 1));); AMREX_D_TERM(ccvel_fab(i, j, k, 0) = u_avg; , ccvel_fab(i, j, k, 1) = v_avg; @@ -753,10 +771,10 @@ PeleLM::getRestartEBMaxLevel() const // Go and parse the line we need std::string File(m_restart_chkfile + "/Header"); - VisMF::IO_Buffer io_buffer(VisMF::GetIOBufferSize()); + amrex::VisMF::IO_Buffer io_buffer(amrex::VisMF::GetIOBufferSize()); - Vector fileCharPtr; - ParallelDescriptor::ReadAndBcastFile(File, fileCharPtr); + amrex::Vector fileCharPtr; + amrex::ParallelDescriptor::ReadAndBcastFile(File, fileCharPtr); std::string fileCharPtrString(fileCharPtr.dataPtr()); std::istringstream is(fileCharPtrString, std::istringstream::in); @@ -786,20 +804,21 @@ void PeleLM::checkEBInflowFunctions() { if (!hasBCNormalEB::value) { - Abort( + amrex::Abort( "Provided ProblemSpecificFunctions doesn't have a viable bcnormal_eb " "function"); } if (!hasBCTypeEB::value) { - Abort( + amrex::Abort( "Provided ProblemSpecificFunctions doesn't have a viable bctype_eb " "function"); } if (m_verbose != 0 && m_useEBinflow != 0) { - Print() << "WARNING: EB-inflow capability is experimental. Scalar " - "diffusion is not supported at these boundaries and future " - "interface changes are possible!" - << std::endl; + amrex::Print() + << "WARNING: EB-inflow capability is experimental. Scalar " + "diffusion is not supported at these boundaries and future " + "interface changes are possible!" + << std::endl; } } #endif diff --git a/Source/PeleLMeX_Eos.cpp b/Source/PeleLMeX_Eos.cpp index 18d43c876..06d9314d9 100644 --- a/Source/PeleLMeX_Eos.cpp +++ b/Source/PeleLMeX_Eos.cpp @@ -2,8 +2,6 @@ #include #include -using namespace amrex; - void PeleLM::setThermoPress(const TimeStamp& a_time) { @@ -30,12 +28,12 @@ PeleLM::setThermoPress(int lev, const TimeStamp& a_time) ldata_p->state, [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { getPGivenRTY( - i, j, k, Array4(sma[box_no], DENSITY), - Array4(sma[box_no], FIRSTSPEC), - Array4(sma[box_no], TEMP), Array4(sma[box_no], RHORT), - leosparm); + i, j, k, amrex::Array4(sma[box_no], DENSITY), + amrex::Array4(sma[box_no], FIRSTSPEC), + amrex::Array4(sma[box_no], TEMP), + amrex::Array4(sma[box_no], RHORT), leosparm); }); - Gpu::streamSynchronize(); + amrex::Gpu::streamSynchronize(); } void @@ -63,13 +61,13 @@ PeleLM::calcDivU( auto* ldata_p = getLevelDataPtr(lev, a_time); - MultiFab RhoYdot; + amrex::MultiFab RhoYdot; if ((m_do_react != 0) && (m_skipInstantRR == 0)) { if (is_init != 0) { // Either pre-divU, divU or press initial iterations if (m_dt > 0.0) { // divU ite -> use I_R auto* ldataR_p = getLevelDataReactPtr(lev); RhoYdot.define(grids[lev], dmap[lev], nCompIR(), 0); - MultiFab::Copy(RhoYdot, ldataR_p->I_R, 0, 0, nCompIR(), 0); + amrex::MultiFab::Copy(RhoYdot, ldataR_p->I_R, 0, 0, nCompIR(), 0); } else { // press ite -> set to zero RhoYdot.define(grids[lev], dmap[lev], nCompIR(), 0); RhoYdot.setVal(0.0); @@ -91,10 +89,11 @@ PeleLM::calcDivU( #endif #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->divu, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldata_p->divu, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); #ifdef AMREX_USE_EB auto const& flagfab = ebfact.getMultiEBCellFlagFab()[mfi]; @@ -125,17 +124,16 @@ PeleLM::calcDivU( auto const* leosparm = eos_parms.device_parm(); #ifdef AMREX_USE_EB - if (flagfab.getType(bx) == FabType::covered) { // Covered boxes + if (flagfab.getType(bx) == amrex::FabType::covered) { // Covered boxes amrex::ParallelFor( - bx, [divu] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { divu(i, j, k) = 0.0; }); - } else if (flagfab.getType(bx) != FabType::regular) { // EB containing - // boxes + } else if (flagfab.getType(bx) != amrex::FabType::regular) { // EB + // containing + // boxes amrex::ParallelFor( - bx, [rhoY, T, SpecD, Fourier, DiffDiff, r, extRhoY, extRhoH, divu, - use_react, flag, - leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (flag(i, j, k).isCovered()) { divu(i, j, k) = 0.0; } else { @@ -148,9 +146,7 @@ PeleLM::calcDivU( #endif { amrex::ParallelFor( - bx, - [rhoY, T, SpecD, Fourier, DiffDiff, r, extRhoY, extRhoH, divu, - use_react, leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { compute_divu( i, j, k, rhoY, T, SpecD, Fourier, DiffDiff, r, extRhoY, extRhoH, divu, use_react, leosparm); @@ -176,7 +172,7 @@ PeleLM::calcDivU( // fillPatch a_time divu to get properly filled ghost cells for (int lev = 0; lev <= finest_level; ++lev) { - Real time = getTime(lev, a_time); + amrex::Real time = getTime(lev, a_time); auto* ldata_p = getLevelDataPtr(lev, a_time); fillpatch_divu(lev, time, ldata_p->divu, m_nGrowdivu); } @@ -198,7 +194,7 @@ PeleLM::setRhoToSumRhoY(int lev, const TimeStamp& a_time) sma[box_no].cellData(i, j, k), sma[box_no](i, j, k, DENSITY), FIRSTSPEC); }); - Gpu::streamSynchronize(); + amrex::Gpu::streamSynchronize(); } void @@ -211,7 +207,7 @@ PeleLM::setTemperature(const TimeStamp& a_time) for (int lev = 0; lev <= finest_level; ++lev) { setTemperature(lev, a_time); } - Gpu::streamSynchronize(); + amrex::Gpu::streamSynchronize(); } void @@ -228,16 +224,17 @@ PeleLM::setTemperature(int lev, const TimeStamp& a_time) ldata_p->state, [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { getTfromHY( - i, j, k, Array4(sma[box_no], DENSITY), - Array4(sma[box_no], FIRSTSPEC), - Array4(sma[box_no], RHOH), Array4(sma[box_no], TEMP), - leosparm); + i, j, k, amrex::Array4(sma[box_no], DENSITY), + amrex::Array4(sma[box_no], FIRSTSPEC), + amrex::Array4(sma[box_no], RHOH), + amrex::Array4(sma[box_no], TEMP), leosparm); }); - Gpu::streamSynchronize(); + amrex::Gpu::streamSynchronize(); } void -PeleLM::calc_dPdt(const TimeStamp& a_time, const Vector& a_dPdt) +PeleLM::calc_dPdt( + const TimeStamp& a_time, const amrex::Vector& a_dPdt) { BL_PROFILE("PeleLMeX::calc_dPdt()"); @@ -257,61 +254,63 @@ PeleLM::calc_dPdt(const TimeStamp& a_time, const Vector& a_dPdt) } void -PeleLM::calc_dPdt(int lev, const TimeStamp& a_time, MultiFab* a_dPdt) +PeleLM::calc_dPdt(int lev, const TimeStamp& a_time, amrex::MultiFab* a_dPdt) { auto const& sma = getLevelDataPtr(lev, a_time)->state.arrays(); auto const& dPdtma = a_dPdt->arrays(); // Use new ambient pressure to compute dPdt - Real p_amb = m_pNew; - + const amrex::Real p_amb = m_pNew; + const auto dt = m_dt; + const auto dpdt_fac = m_dpdtFactor; amrex::ParallelFor( - *a_dPdt, [=, dt = m_dt, dpdt_fac = m_dpdtFactor] AMREX_GPU_DEVICE( - int box_no, int i, int j, int k) noexcept { + *a_dPdt, [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { auto dPdta = dPdtma[box_no]; auto sa = sma[box_no]; dPdta(i, j, k) = (sa(i, j, k, RHORT) - p_amb) / (dt * sa(i, j, k, RHORT)) * dpdt_fac; }); - Gpu::streamSynchronize(); + amrex::Gpu::streamSynchronize(); } -Real +amrex::Real PeleLM::adjustPandDivU(std::unique_ptr& advData) { BL_PROFILE("PeleLMeX::adjustPandDivU()"); - Vector> ThetaHalft(finest_level + 1); + amrex::Vector> ThetaHalft(finest_level + 1); // Get theta = 1 / (\Gamma * P_amb) at half time for (int lev = 0; lev <= finest_level; ++lev) { - ThetaHalft[lev] = std::make_unique( - grids[lev], dmap[lev], 1, 0, MFInfo(), *m_factory[lev]); + ThetaHalft[lev] = std::make_unique( + grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), *m_factory[lev]); auto const& tma = ThetaHalft[lev]->arrays(); auto const& sma_o = getLevelDataPtr(lev, AmrOldTime)->state.const_arrays(); auto const& sma_n = getLevelDataPtr(lev, AmrNewTime)->state.const_arrays(); auto const* leosparm = eos_parms.device_parm(); + const auto pOld = m_pOld; + const auto pNew = m_pNew; amrex::ParallelFor( - *ThetaHalft[lev], [=, pOld = m_pOld, pNew = m_pNew] AMREX_GPU_DEVICE( - int box_no, int i, int j, int k) noexcept { + *ThetaHalft[lev], + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { auto theta = tma[box_no]; - Real gammaInv_o = getGammaInv( - i, j, k, Array4(sma_o[box_no], FIRSTSPEC), - Array4(sma_o[box_no], TEMP), leosparm); - Real gammaInv_n = getGammaInv( - i, j, k, Array4(sma_n[box_no], FIRSTSPEC), - Array4(sma_n[box_no], TEMP), leosparm); + amrex::Real gammaInv_o = getGammaInv( + i, j, k, amrex::Array4(sma_o[box_no], FIRSTSPEC), + amrex::Array4(sma_o[box_no], TEMP), leosparm); + amrex::Real gammaInv_n = getGammaInv( + i, j, k, amrex::Array4(sma_n[box_no], FIRSTSPEC), + amrex::Array4(sma_n[box_no], TEMP), leosparm); theta(i, j, k) = 0.5 * (gammaInv_o / pOld + gammaInv_n / pNew); }); } - Gpu::streamSynchronize(); + amrex::Gpu::streamSynchronize(); // Get the mean mac_divu (Sbar) and mean theta - Real Sbar = MFSum(GetVecOfConstPtrs(advData->mac_divu), 0); + amrex::Real Sbar = MFSum(GetVecOfConstPtrs(advData->mac_divu), 0); Sbar /= m_uncoveredVol; - Real Thetabar = MFSum(GetVecOfConstPtrs(ThetaHalft), 0); + amrex::Real Thetabar = MFSum(GetVecOfConstPtrs(ThetaHalft), 0); Thetabar /= m_uncoveredVol; // Adjust @@ -327,7 +326,7 @@ PeleLM::adjustPandDivU(std::unique_ptr& advData) m_domainUmacFlux[0] + m_domainUmacFlux[1], +m_domainUmacFlux[2] + m_domainUmacFlux[3], +m_domainUmacFlux[4] + m_domainUmacFlux[5]); - Real divu_vol = umacFluxBalance / m_uncoveredVol; + amrex::Real divu_vol = umacFluxBalance / m_uncoveredVol; // Advance the ambient pressure m_pNew = m_pOld + m_dt * (Sbar - divu_vol) / Thetabar; @@ -346,13 +345,13 @@ PeleLM::adjustPandDivU(std::unique_ptr& advData) divu_vol * (1 + theta(i, j, k) / Thetabar)); }); } - Gpu::streamSynchronize(); + amrex::Gpu::streamSynchronize(); if (m_verbose > 2) { - Print() << " >> Closed chamber pOld: " << m_pOld << ", pNew: " << m_pNew - << ", dp0dt: " << m_dp0dt << "\n"; - Print() << " >> Total mass old: " << m_massOld - << ", mass new: " << m_massNew << std::endl; + amrex::Print() << " >> Closed chamber pOld: " << m_pOld + << ", pNew: " << m_pNew << ", dp0dt: " << m_dp0dt << "\n"; + amrex::Print() << " >> Total mass old: " << m_massOld + << ", mass new: " << m_massNew << std::endl; } // Return Sbar so that we'll add it back to mac_divu after the MAC projection diff --git a/Source/PeleLMeX_Evaluate.cpp b/Source/PeleLMeX_Evaluate.cpp index 9c98ec92c..5939da65d 100644 --- a/Source/PeleLMeX_Evaluate.cpp +++ b/Source/PeleLMeX_Evaluate.cpp @@ -2,8 +2,6 @@ #include #include -using namespace amrex; - void PeleLM::Evaluate() { @@ -13,7 +11,7 @@ PeleLM::Evaluate() // Check that requested evaluate entries exist and determine the size // of the container and entries names int ncomp = 0; - Vector plt_VarsName; + amrex::Vector plt_VarsName; for (int ivar = 0; ivar < m_evaluatePlotVarCount; ivar++) { bool itexists = derive_lst.canDerive(m_evaluatePlotVars[ivar]) || evaluate_lst.canDerive(m_evaluatePlotVars[ivar]) || @@ -42,9 +40,10 @@ PeleLM::Evaluate() //---------------------------------------------------------------- // Define the outgoing container - Vector mf_plt(finest_level + 1); + amrex::Vector mf_plt(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - mf_plt[lev].define(grids[lev], dmap[lev], ncomp, 0, MFInfo(), Factory(lev)); + mf_plt[lev].define( + grids[lev], dmap[lev], ncomp, 0, amrex::MFInfo(), Factory(lev)); } //---------------------------------------------------------------- @@ -59,7 +58,7 @@ PeleLM::Evaluate() for (int ivar = 0; ivar < m_evaluatePlotVarCount; ivar++) { int cntIncr = 0; - Print() << " --> Evaluating " << m_evaluatePlotVars[ivar] << "\n"; + amrex::Print() << " --> Evaluating " << m_evaluatePlotVars[ivar] << "\n"; // Evaluate function calls actual PeleLM::Evolve pieces and may require // the entire multi-level hierarchy @@ -72,9 +71,9 @@ PeleLM::Evaluate() derive_lst.canDerive(m_evaluatePlotVars[ivar]) || isStateVariable(m_evaluatePlotVars[ivar])) { for (int lev = 0; lev <= finest_level; ++lev) { - std::unique_ptr mf; + std::unique_ptr mf; mf = derive(m_evaluatePlotVars[ivar], m_cur_time, lev, 0); - MultiFab::Copy(mf_plt[lev], *mf, 0, cnt, mf->nComp(), 0); + amrex::MultiFab::Copy(mf_plt[lev], *mf, 0, cnt, mf->nComp(), 0); cntIncr = mf->nComp(); } } @@ -83,7 +82,7 @@ PeleLM::Evaluate() //---------------------------------------------------------------- // Write the evaluated variables to disc - Vector istep(finest_level + 1, 0); + amrex::Vector istep(finest_level + 1, 0); // Override m_cur_time to store the dt in pltEvaluate m_cur_time = m_dt; @@ -96,7 +95,7 @@ PeleLM::Evaluate() void PeleLM::MLevaluate( - const Vector& a_MFVec, + const amrex::Vector& a_MFVec, int a_comp, int& nComp, const std::string& a_var) @@ -120,7 +119,7 @@ PeleLM::MLevaluate( diffData); for (int lev = 0; lev <= finest_level; ++lev) { auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); - MultiFab::Copy(*a_MFVec[lev], ldata_p->divu, 0, a_comp, 1, 0); + amrex::MultiFab::Copy(*a_MFVec[lev], ldata_p->divu, 0, a_comp, 1, 0); } nComp = 1; } else if (a_var == "velProj") { @@ -144,16 +143,17 @@ PeleLM::MLevaluate( // Copy into outgoing data holder for (int lev = 0; lev <= finest_level; ++lev) { auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); - MultiFab::Copy( + amrex::MultiFab::Copy( *a_MFVec[lev], ldata_p->state, VELX, a_comp, AMREX_SPACEDIM, 0); } nComp = AMREX_SPACEDIM; } else if (a_var == "divTau") { // Velocity tensor components int use_density = 0; - Vector> aliasDivTau(finest_level + 1); + amrex::Vector> aliasDivTau( + finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - aliasDivTau[lev] = std::make_unique( + aliasDivTau[lev] = std::make_unique( *a_MFVec[lev], amrex::make_alias, a_comp, AMREX_SPACEDIM); } computeDivTau(AmrNewTime, GetVecOfPtrs(aliasDivTau), use_density); @@ -177,14 +177,14 @@ PeleLM::MLevaluate( } computeDifferentialDiffusionTerms(AmrNewTime, diffData); for (int lev = 0; lev <= finest_level; ++lev) { - MultiFab::Copy( + amrex::MultiFab::Copy( *a_MFVec[lev], diffData->Dnp1[lev], 0, a_comp, NUM_SPECIES + 2, 0); } nComp = NUM_SPECIES + 2; } else if (a_var == "advTerm") { - Vector> aliasMF(finest_level + 1); + amrex::Vector> aliasMF(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - aliasMF[lev] = std::make_unique( + aliasMF[lev] = std::make_unique( *a_MFVec[lev], amrex::make_alias, a_comp, NVAR - 2); } evaluateAdvectionTerms(GetVecOfPtrs(aliasMF)); @@ -194,16 +194,16 @@ PeleLM::MLevaluate( // integration Replicate most of the advance function Copy the state for (int lev = 0; lev <= finest_level; ++lev) { auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); - MultiFab::Copy( + amrex::MultiFab::Copy( *a_MFVec[lev], ldata_p->state, FIRSTSPEC, a_comp, NUM_SPECIES + 2, 0); } // Initial velocity projection if (m_restart_chkfile.empty()) { projectInitSolution(); } - Vector> aliasMF(finest_level + 1); + amrex::Vector> aliasMF(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - aliasMF[lev] = std::make_unique( + aliasMF[lev] = std::make_unique( *a_MFVec[lev], amrex::make_alias, a_comp + NUM_SPECIES + 2, NUM_SPECIES + 1); } @@ -211,7 +211,7 @@ PeleLM::MLevaluate( nComp = 2 * (NUM_SPECIES + 1) + 1; } else if (a_var == "instRR") { for (int lev = 0; lev <= finest_level; ++lev) { - std::unique_ptr I_RR = std::make_unique( + std::unique_ptr I_RR = std::make_unique( *a_MFVec[lev], amrex::make_alias, a_comp, NUM_SPECIES); computeInstantaneousReactionRate(lev, AmrNewTime, I_RR.get()); } @@ -224,12 +224,12 @@ PeleLM::MLevaluate( calcDiffusivity(AmrNewTime); for (int lev = 0; lev <= finest_level; ++lev) { auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); - MultiFab::Copy( + amrex::MultiFab::Copy( *a_MFVec[lev], ldata_p->diff_cc, 0, a_comp, NUM_SPECIES + 1, 0); - MultiFab::Copy( + amrex::MultiFab::Copy( *a_MFVec[lev], ldata_p->visc_cc, 0, a_comp + NUM_SPECIES + 1, 1, 0); if (m_use_soret != 0) { - MultiFab::Copy( + amrex::MultiFab::Copy( *a_MFVec[lev], ldata_p->diff_cc, NUM_SPECIES + 2, a_comp + NUM_SPECIES + 2, NUM_SPECIES, 0); } @@ -242,9 +242,10 @@ PeleLM::MLevaluate( } else if (a_var == "velForce") { // Velocity forces used in computing the velocity advance int add_gradP = 0; - Vector> aliasMFVec(finest_level + 1); + amrex::Vector> aliasMFVec( + finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - aliasMFVec[lev] = std::make_unique( + aliasMFVec[lev] = std::make_unique( *a_MFVec[lev], amrex::make_alias, a_comp, AMREX_SPACEDIM); } getVelForces(AmrNewTime, {}, GetVecOfPtrs(aliasMFVec), 0, add_gradP); @@ -350,7 +351,7 @@ PeleLM::evaluateChemExtForces( // Copy external forcing for chemistry into outgoing container for (int lev = 0; lev <= finest_level; ++lev) { - MultiFab::Copy( + amrex::MultiFab::Copy( *a_chemForces[lev], advData->Forcing[lev], 0, 0, NUM_SPECIES + 1, 0); } @@ -449,7 +450,8 @@ PeleLM::evaluateAdvectionTerms( // Copy AofS into outgoing container, skip Temperature and RhoRT for (int lev = 0; lev <= finest_level; ++lev) { - MultiFab::Copy(*a_advTerms[lev], advData->AofS[lev], 0, 0, NVAR - 2, 0); + amrex::MultiFab::Copy( + *a_advTerms[lev], advData->AofS[lev], 0, 0, NVAR - 2, 0); } // Reset state diff --git a/Source/PeleLMeX_Evolve.cpp b/Source/PeleLMeX_Evolve.cpp index 0712c416a..3ac3a71d7 100644 --- a/Source/PeleLMeX_Evolve.cpp +++ b/Source/PeleLMeX_Evolve.cpp @@ -1,7 +1,5 @@ #include -using namespace amrex; - void PeleLM::Evolve() { @@ -12,6 +10,7 @@ PeleLM::Evolve() int plt_justDidIt = 0; int chk_justDidIt = 0; + bool nans_in_solution = false; while (!do_not_evolve) { @@ -90,8 +89,9 @@ PeleLM::Evolve() // Check for the end of the simulation bool over_max_wall_time = false; if (m_max_wall_time > 0.0) { - amrex::Real t_elapsed = ParallelDescriptor::second() - m_wall_start; - ParallelDescriptor::ReduceRealMax(t_elapsed); + amrex::Real t_elapsed = + amrex::ParallelDescriptor::second() - m_wall_start; + amrex::ParallelDescriptor::ReduceRealMax(t_elapsed); if (t_elapsed >= (m_max_wall_time * 3600)) { over_max_wall_time = true; if (m_verbose > 0) { @@ -101,10 +101,12 @@ PeleLM::Evolve() } } } + nans_in_solution = checkForNaNs(); do_not_evolve = ((m_max_step >= 0 && m_nstep >= m_max_step) || (m_stop_time >= 0.0 && m_cur_time >= m_stop_time - 1.0e-12 * m_dt) || - (m_dt < m_min_dt) || over_max_wall_time || dump_and_stop); + (m_dt < m_min_dt) || over_max_wall_time || dump_and_stop || + nans_in_solution); } if (m_verbose > 0) { @@ -120,6 +122,10 @@ PeleLM::Evolve() m_nstep > 0) { WriteCheckPointFile(); } + + if (nans_in_solution) { + amrex::Error("Stopped early because NaNs detected in solution"); + } } bool @@ -142,9 +148,9 @@ PeleLM::writePlotNow() const // within machine epsilon of the next interval. In that case, increment // the counter, because we have indeed reached the next plot_per interval // at this point. - const Real eps = - std::numeric_limits::epsilon() * 10.0_rt * std::abs(m_cur_time); - const Real next_plot_time = (num_per_old + 1) * m_plot_per_approx; + const amrex::Real eps = + std::numeric_limits::epsilon() * 10.0 * std::abs(m_cur_time); + const amrex::Real next_plot_time = (num_per_old + 1) * m_plot_per_approx; if ( (num_per_new == num_per_old) && std::abs(m_cur_time - next_plot_time) <= eps) { @@ -184,9 +190,9 @@ PeleLM::writeCheckNow() const // within machine epsilon of the next interval. In that case, increment // the counter, because we have indeed reached the next plot_per interval // at this point. - const Real eps = - std::numeric_limits::epsilon() * 10.0_rt * std::abs(m_cur_time); - const Real next_check_time = (num_per_old + 1) * m_check_per; + const amrex::Real eps = + std::numeric_limits::epsilon() * 10.0 * std::abs(m_cur_time); + const amrex::Real next_check_time = (num_per_old + 1) * m_check_per; if ( (num_per_new == num_per_old) && std::abs(m_cur_time - next_check_time) <= eps) { @@ -234,12 +240,12 @@ PeleLM::checkMessage(const std::string& a_action) const } else if (a_action == "chk_and_continue") { action_file = "chk_and_continue"; } else { - Abort("Unknown action in checkMessage()"); + amrex::Abort("Unknown action in checkMessage()"); } if (m_nstep % m_message_int == 0) { int action_flag = 0; - if (ParallelDescriptor::IOProcessor()) { + if (amrex::ParallelDescriptor::IOProcessor()) { FILE* fp = fopen(action_file.c_str(), "r"); if (fp != nullptr) { remove(action_file.c_str()); @@ -249,8 +255,8 @@ PeleLM::checkMessage(const std::string& a_action) const } int packed_data[1]; packed_data[0] = action_flag; - ParallelDescriptor::Bcast( - packed_data, 1, ParallelDescriptor::IOProcessorNumber()); + amrex::ParallelDescriptor::Bcast( + packed_data, 1, amrex::ParallelDescriptor::IOProcessorNumber()); take_action = (packed_data[0] != 0); } return take_action; diff --git a/Source/PeleLMeX_FlowController.cpp b/Source/PeleLMeX_FlowController.cpp index 19d4d5f67..a476f19a5 100644 --- a/Source/PeleLMeX_FlowController.cpp +++ b/Source/PeleLMeX_FlowController.cpp @@ -3,12 +3,10 @@ #include -using namespace amrex; - void PeleLM::initActiveControl() { - ParmParse pp("active_control"); + amrex::ParmParse pp("active_control"); pp.query("on", m_ctrl_active); @@ -16,7 +14,7 @@ PeleLM::initActiveControl() return; } - Print() << " Initialization of active control \n"; + amrex::Print() << " Initialization of active control \n"; pp.query("use_temp", m_ctrl_useTemp); pp.query("temperature", m_ctrl_temperature); @@ -53,7 +51,8 @@ PeleLM::initActiveControl() // Initialize flow controller if (!hasFlowControllerData::value) { - Abort("ProbParm doesn't have a FCData FlowControllerData member variable"); + amrex::Abort( + "ProbParm doesn't have a FCData FlowControllerData member variable"); } // Get FlowControllerData from ProbParm if it exists @@ -70,38 +69,37 @@ PeleLM::initActiveControl() ProbParm const* lprobparm = prob_parm_d; auto const* lpmfdata = pmf_data.device_parm(); - Gpu::DeviceVector s_ext_v(NVAR); - Real* s_ext_d = s_ext_v.data(); - Real x[AMREX_SPACEDIM] = { + amrex::Gpu::DeviceVector s_ext_v(NVAR); + amrex::Real* s_ext_d = s_ext_v.data(); + amrex::Real x[AMREX_SPACEDIM] = { AMREX_D_DECL(Geom(0).ProbLo(0), Geom(0).ProbLo(1), Geom(0).ProbLo(2))}; x[m_ctrl_flameDir] -= 1.0; const int ctrl_flameDir_l = m_ctrl_flameDir; const amrex::Real time_l = -1.0; const auto geomdata = Geom(0).data(); - auto fake_state = Array4{}; + auto fake_state = amrex::Array4{}; - Box dumbx({AMREX_D_DECL(0, 0, 0)}, {AMREX_D_DECL(0, 0, 0)}); + amrex::Box dumbx({AMREX_D_DECL(0, 0, 0)}, {AMREX_D_DECL(0, 0, 0)}); amrex::ParallelFor( - dumbx, - [fake_state, x, s_ext_d, ctrl_flameDir_l, time_l, geomdata, lprobparm, - lpmfdata] AMREX_GPU_DEVICE(int /*i*/, int /*j*/, int /*k*/) noexcept { + dumbx, [=] AMREX_GPU_DEVICE(int /*i*/, int /*j*/, int /*k*/) noexcept { const auto s_in = fake_state.cellData(0, 0, 0); ProblemSpecificFunctions::bcnormal( x, s_in, s_ext_d, ctrl_flameDir_l, 1, time_l, geomdata, *lprobparm, lpmfdata); }); - Vector s_ext(NVAR); - Gpu::copy(Gpu::deviceToHost, s_ext_v.begin(), s_ext_v.end(), s_ext.begin()); + amrex::Vector s_ext(NVAR); + amrex::Gpu::copy( + amrex::Gpu::deviceToHost, s_ext_v.begin(), s_ext_v.end(), s_ext.begin()); if (m_ctrl_useTemp == 0) { // Get the fuel rhoY if (fuelID < 0) { - Abort( + amrex::Abort( "Using activeControl based on fuel mass requires peleLM.fuelName !"); } // Compute some active control parameters - Real area_tot = 1.0; + amrex::Real area_tot = 1.0; for (int idim{0}; idim < AMREX_SPACEDIM; idim++) { if (idim != m_ctrl_flameDir) { area_tot *= (Geom(0).ProbHi(idim) - Geom(0).ProbLo(idim)); @@ -126,13 +124,14 @@ PeleLM::initActiveControl() if (m_ctrl_verbose != 0) { if (m_ctrl_useTemp != 0) { - Print() << " Active control based on temperature iso-level activated." - << " Maintaining the flame at " << m_ctrl_h << " in " - << m_ctrl_flameDir << " direction. \n"; + amrex::Print() + << " Active control based on temperature iso-level activated." + << " Maintaining the flame at " << m_ctrl_h << " in " + << m_ctrl_flameDir << " direction. \n"; } else { - Print() << " Active control based on fuel mass activated." - << " Maintaining the flame at " << m_ctrl_h << " in " - << m_ctrl_flameDir << " direction. \n"; + amrex::Print() << " Active control based on fuel mass activated." + << " Maintaining the flame at " << m_ctrl_h << " in " + << m_ctrl_flameDir << " direction. \n"; } } } @@ -148,7 +147,7 @@ PeleLM::activeControl(int is_restart) // ------------------------------------------- // Get the current target state (either amount of fuel or T-iso position) // ------------------------------------------- - Real coft = 0.0; + amrex::Real coft = 0.0; if (m_ctrl_useTemp == 0) { // Compute the integral of the fuel mass in the domain coft = MFSum(GetVecOfConstPtrs(getSpeciesVect(AmrNewTime)), fuelID); @@ -179,8 +178,8 @@ PeleLM::activeControl(int is_restart) m_ctrl_V_in += m_dt * m_ctrl_dV; } - Real slocal = 0.5 * (m_ctrl_V_in_old + m_ctrl_V_in) - - (coft - m_ctrl_coftOld) / (m_dt * m_ctrl_scale); + amrex::Real slocal = 0.5 * (m_ctrl_V_in_old + m_ctrl_V_in) - + (coft - m_ctrl_coftOld) / (m_dt * m_ctrl_scale); // ------------------------------------------- // Get s_est averaged from previous N steps @@ -202,7 +201,7 @@ PeleLM::activeControl(int is_restart) } if (m_ctrl_nfilled <= 0) { - Real velIntegral = 0.0; + amrex::Real velIntegral = 0.0; for (int n = 1; n <= m_ctrl_NavgPts; n++) { // Piecewise constant velocity over NavgPts last steps velIntegral += 0.5 * (m_ctrl_velo_pts[n - 1] + m_ctrl_velo_pts[n]) * @@ -223,39 +222,42 @@ PeleLM::activeControl(int is_restart) // ------------------------------------------- // Compute Vnew // ------------------------------------------- - Real Vnew = 0.0; + amrex::Real Vnew = 0.0; if (m_ctrl_method == 1) { // linear - Real vslope = 2.0 * - ((m_ctrl_cfix - coft) / (m_ctrl_scale * m_ctrl_tauControl) + - m_ctrl_sest - m_ctrl_V_in) / - m_ctrl_tauControl; + amrex::Real vslope = + 2.0 * + ((m_ctrl_cfix - coft) / (m_ctrl_scale * m_ctrl_tauControl) + m_ctrl_sest - + m_ctrl_V_in) / + m_ctrl_tauControl; Vnew = m_ctrl_V_in + m_dt * vslope; } else if (m_ctrl_method == 2) { // Quadratic 1 - Real vslope = 3.0 * - ((m_ctrl_cfix - coft) / (m_ctrl_scale * m_ctrl_tauControl) + - m_ctrl_sest - m_ctrl_V_in) / - m_ctrl_tauControl; + amrex::Real vslope = + 3.0 * + ((m_ctrl_cfix - coft) / (m_ctrl_scale * m_ctrl_tauControl) + m_ctrl_sest - + m_ctrl_V_in) / + m_ctrl_tauControl; Vnew = m_ctrl_V_in + (m_dt - 0.5 * m_dt * m_dt / m_ctrl_tauControl) * vslope; } else if (m_ctrl_method == 3) { // Quadratic 2 - Real rhs2 = 2.0 * - ((m_ctrl_cfix - coft) / (m_ctrl_scale * m_ctrl_tauControl) + - m_ctrl_sest - m_ctrl_V_in) / - m_ctrl_tauControl; - Real rhs1 = (m_ctrl_sest - m_ctrl_V_in) / m_ctrl_tauControl; - Real vt_tay = 3.0 * rhs2 - 2.0 * rhs1; - Real vtt_tay = 6.0 * (rhs1 - rhs2) / m_ctrl_tauControl; + amrex::Real rhs2 = + 2.0 * + ((m_ctrl_cfix - coft) / (m_ctrl_scale * m_ctrl_tauControl) + m_ctrl_sest - + m_ctrl_V_in) / + m_ctrl_tauControl; + amrex::Real rhs1 = (m_ctrl_sest - m_ctrl_V_in) / m_ctrl_tauControl; + amrex::Real vt_tay = 3.0 * rhs2 - 2.0 * rhs1; + amrex::Real vtt_tay = 6.0 * (rhs1 - rhs2) / m_ctrl_tauControl; Vnew = m_ctrl_V_in + m_dt * vt_tay + 0.5 * m_dt * m_dt * vtt_tay; } // Limit Vnew - Real dVmax = m_ctrl_changeMax * 1.0; - Real dVmin = m_ctrl_changeMax * std::max(1.0, m_ctrl_V_in); - Vnew = std::max(Vnew, 0.0); - Vnew = std::min(std::max(Vnew, m_ctrl_V_in - dVmin), m_ctrl_V_in + dVmax); + amrex::Real dVmax = m_ctrl_changeMax * 1.0; + amrex::Real dVmin = m_ctrl_changeMax * amrex::max(1.0, m_ctrl_V_in); + Vnew = amrex::max(Vnew, 0.0); + Vnew = amrex::min(amrex::max(Vnew, m_ctrl_V_in - dVmin), m_ctrl_V_in + dVmax); if (m_ctrl_velMax > 0.0) { // Only limit Vnew to velMax if velMax > 0.0 - Vnew = std::min(Vnew, m_ctrl_velMax); + Vnew = amrex::min(Vnew, m_ctrl_velMax); } if ((is_restart == 0) && m_nstep > 0) { @@ -280,25 +282,28 @@ PeleLM::activeControl(int is_restart) fcdata_h->ctrl_tBase = m_ctrl_tBase; // Update Device version - Gpu::copy(Gpu::hostToDevice, fcdata_h, fcdata_h + 1, fcdata_d); + amrex::Gpu::copy( + amrex::Gpu::hostToDevice, fcdata_h, fcdata_h + 1, fcdata_d); } if ((m_ctrl_verbose != 0) && (is_restart == 0)) { - Print() + amrex::Print() << "\n------------------------AC CONTROL------------------------ \n"; - Print() << " | Time: " << m_cur_time << " - dt: " << m_dt << "\n"; - Print() << " | Position: " << coft / m_ctrl_scale - << " - Target: " << m_ctrl_cfix / m_ctrl_scale << "\n"; - Print() << " | V_new: " << Vnew << " - V_in: " << m_ctrl_V_in - << " - dV: " << m_ctrl_dV << "\n"; - Print() << "---------------------------------------------------------- \n"; + amrex::Print() << " | Time: " << m_cur_time << " - dt: " << m_dt + << "\n"; + amrex::Print() << " | Position: " << coft / m_ctrl_scale + << " - Target: " << m_ctrl_cfix / m_ctrl_scale << "\n"; + amrex::Print() << " | V_new: " << Vnew << " - V_in: " << m_ctrl_V_in + << " - dV: " << m_ctrl_dV << "\n"; + amrex::Print() + << "---------------------------------------------------------- \n"; } // Append to (or create) AC history file if (is_restart == 0) { std::ofstream ACfile( m_ctrl_AChistory.c_str(), std::ofstream::out | std::ofstream::app); - Print(ACfile).SetPrecision(15) + amrex::Print(ACfile).SetPrecision(15) << m_nstep << " " << m_cur_time << " " << m_ctrl_V_in << " " << slocal << " " << m_ctrl_dV << " " << m_ctrl_sest << " " << m_ctrl_coftOld << "\n"; @@ -306,7 +311,7 @@ PeleLM::activeControl(int is_restart) } void -PeleLM::getActiveControlLowT(Real& a_coft) +PeleLM::getActiveControlLowT(amrex::Real& a_coft) { for (int lev = 0; lev <= finest_level; lev++) { @@ -316,40 +321,39 @@ PeleLM::getActiveControlLowT(Real& a_coft) // local FC data int AC_FlameDir = m_ctrl_flameDir; - Real AC_Tcross = m_ctrl_temperature; + amrex::Real AC_Tcross = m_ctrl_temperature; - Real lowT = 1.e37; + amrex::Real lowT = 1.e37; if (lev != finest_level) { lowT = amrex::ReduceMin( ldata_p->state, *m_coveredMask[lev], 0, [=] AMREX_GPU_HOST_DEVICE( - Box const& bx, Array4 const& T_arr, - Array4 const& covered_arr) -> Real { - using namespace amrex::literals; + amrex::Box const& bx, amrex::Array4 const& T_arr, + amrex::Array4 const& covered_arr) -> amrex::Real { const auto lo = amrex::lbound(bx); const auto hi = amrex::ubound(bx); - Real tmp_pos = 1.e37_rt; - const Real* prob_lo = geomdata.ProbLo(); - const Real* dx = geomdata.CellSize(); + amrex::Real tmp_pos = 1.e37; + const amrex::Real* prob_lo = geomdata.ProbLo(); + const amrex::Real* dx = geomdata.CellSize(); for (int k = lo.z; k <= hi.z; ++k) { for (int j = lo.y; j <= hi.y; ++j) { for (int i = lo.x; i <= hi.x; ++i) { - Real lcl_pos = 1.e37_rt; + amrex::Real lcl_pos = 1.e37; if ( T_arr(i, j, k, TEMP) > AC_Tcross && covered_arr(i, j, k) > 0) { int idx[3] = {i, j, k}; idx[AC_FlameDir] -= 1; if (T_arr(idx[0], idx[1], idx[2], TEMP) < AC_Tcross) { - Real coor[3] = {0.0}; + amrex::Real coor[3] = {0.0}; AMREX_D_TERM( coor[0] = prob_lo[0] + (i + 0.5) * dx[0]; , coor[1] = prob_lo[1] + (j + 0.5) * dx[1]; , coor[2] = prob_lo[2] + (k + 0.5) * dx[2];); - Real slope = ((T_arr(i, j, k, TEMP)) - - T_arr(idx[0], idx[1], idx[2], TEMP)) / - dx[AC_FlameDir]; + amrex::Real slope = ((T_arr(i, j, k, TEMP)) - + T_arr(idx[0], idx[1], idx[2], TEMP)) / + dx[AC_FlameDir]; lcl_pos = coor[AC_FlameDir] - dx[AC_FlameDir] + (AC_Tcross - T_arr(idx[0], idx[1], idx[2], TEMP)) / slope; @@ -365,29 +369,29 @@ PeleLM::getActiveControlLowT(Real& a_coft) lowT = amrex::ReduceMin( ldata_p->state, 0, [=] AMREX_GPU_HOST_DEVICE( - Box const& bx, Array4 const& T_arr) -> Real { - using namespace amrex::literals; + amrex::Box const& bx, + amrex::Array4 const& T_arr) -> amrex::Real { const auto lo = amrex::lbound(bx); const auto hi = amrex::ubound(bx); - Real tmp_pos = 1.e37_rt; - const Real* prob_lo = geomdata.ProbLo(); - const Real* dx = geomdata.CellSize(); + amrex::Real tmp_pos = 1.e37; + const amrex::Real* prob_lo = geomdata.ProbLo(); + const amrex::Real* dx = geomdata.CellSize(); for (int k = lo.z; k <= hi.z; ++k) { for (int j = lo.y; j <= hi.y; ++j) { for (int i = lo.x; i <= hi.x; ++i) { - Real lcl_pos = 1.e37_rt; + amrex::Real lcl_pos = 1.e37; if (T_arr(i, j, k, TEMP) > AC_Tcross) { int idx[3] = {i, j, k}; idx[AC_FlameDir] -= 1; if (T_arr(idx[0], idx[1], idx[2], TEMP) < AC_Tcross) { - Real coor[3] = {0.0}; + amrex::Real coor[3] = {0.0}; AMREX_D_TERM( coor[0] = prob_lo[0] + (i + 0.5) * dx[0]; , coor[1] = prob_lo[1] + (j + 0.5) * dx[1]; , coor[2] = prob_lo[2] + (k + 0.5) * dx[2];); - Real slope = ((T_arr(i, j, k, TEMP)) - - T_arr(idx[0], idx[1], idx[2], TEMP)) / - dx[AC_FlameDir]; + amrex::Real slope = ((T_arr(i, j, k, TEMP)) - + T_arr(idx[0], idx[1], idx[2], TEMP)) / + dx[AC_FlameDir]; lcl_pos = coor[AC_FlameDir] - dx[AC_FlameDir] + (AC_Tcross - T_arr(idx[0], idx[1], idx[2], TEMP)) / slope; @@ -402,7 +406,7 @@ PeleLM::getActiveControlLowT(Real& a_coft) } a_coft = amrex::min(a_coft, lowT); } - ParallelDescriptor::ReduceRealMin(a_coft); + amrex::ParallelDescriptor::ReduceRealMin(a_coft); } void @@ -413,13 +417,14 @@ PeleLM::loadActiveControlHistory() bool have_history = (stat(m_ctrl_AChistory.c_str(), &buffer) == 0); if (have_history) { if (m_ctrl_verbose != 0) { - Print() << " Setting AC from history from " << m_ctrl_AChistory << "\n"; + amrex::Print() << " Setting AC from history from " << m_ctrl_AChistory + << "\n"; } std::fstream ACfile(m_ctrl_AChistory.c_str(), std::fstream::in); if (ACfile.is_open()) { while (ACfile.good()) { int step_io; - Real time_io, vel_io, slocal_io, dV_io, s_est_io, coft_old_io; + amrex::Real time_io, vel_io, slocal_io, dV_io, s_est_io, coft_old_io; ACfile >> step_io >> time_io >> vel_io >> slocal_io >> dV_io >> s_est_io >> coft_old_io; if ( @@ -443,17 +448,17 @@ PeleLM::loadActiveControlHistory() ACfile.close(); } if (m_ctrl_verbose != 0) { - Print() << " AC history arrays: \n"; + amrex::Print() << " AC history arrays: \n"; for (long int n = 0; n < m_ctrl_time_pts.size(); n++) { - Print() << " [" << n << "] time: " << m_ctrl_time_pts[n] - << ", velo: " << m_ctrl_velo_pts[n] - << ", coft: " << m_ctrl_cntl_pts[n] << "\n"; + amrex::Print() << " [" << n << "] time: " << m_ctrl_time_pts[n] + << ", velo: " << m_ctrl_velo_pts[n] + << ", coft: " << m_ctrl_cntl_pts[n] << "\n"; } } } else { if (m_ctrl_verbose != 0) { - Print() << " AC history file " << m_ctrl_AChistory - << " not found, restarting from scratch \n"; + amrex::Print() << " AC history file " << m_ctrl_AChistory + << " not found, restarting from scratch \n"; } } } diff --git a/Source/PeleLMeX_Forces.cpp b/Source/PeleLMeX_Forces.cpp index 5ec411e2e..d8c836ddd 100644 --- a/Source/PeleLMeX_Forces.cpp +++ b/Source/PeleLMeX_Forces.cpp @@ -1,16 +1,14 @@ #include #include -using namespace amrex; - // Return velocity forces scaled by rhoInv // including grapP term if add_gradP = 1 -// including divTau if input Vector not empty +// including divTau if input amrex::Vector not empty void PeleLM::getVelForces( const TimeStamp& a_time, - const Vector& a_divTau, - const Vector& a_velForce, + const amrex::Vector& a_divTau, + const amrex::Vector& a_velForce, int nGrowForce, int add_gradP) { @@ -35,8 +33,8 @@ void PeleLM::getVelForces( const TimeStamp& a_time, int lev, - MultiFab* a_divTau, - MultiFab* a_velForce, + amrex::MultiFab* a_divTau, + amrex::MultiFab* a_velForce, int add_gradP) { @@ -49,16 +47,17 @@ PeleLM::getVelForces( auto* ldataGP_p = (m_t_old[lev] < 0.0) ? getLevelDataPtr(lev, AmrNewTime) : getLevelDataPtr(lev, AmrOldTime); - Real time = getTime(lev, a_time); + amrex::Real time = getTime(lev, a_time); int has_divTau = static_cast(a_divTau != nullptr); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*a_velForce, TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*a_velForce, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { const auto& bx = mfi.tilebox(); - FArrayBox DummyFab(bx, 1); + amrex::FArrayBox DummyFab(bx, 1); const auto& vel_arr = ldata_p->state.const_array(mfi, VELX); const auto& rho_arr = (m_incompressible) != 0 ? DummyFab.array() @@ -98,16 +97,14 @@ PeleLM::getVelForces( // Add pressure gradient and viscous forces (if req.) and scale by density. int is_incomp = m_incompressible; - Real incomp_rho_inv = 1.0 / m_rho; + amrex::Real incomp_rho_inv = 1.0 / m_rho; if ((add_gradP != 0) || (has_divTau != 0)) { const auto& gp_arr = (add_gradP) != 0 ? ldataGP_p->gp.const_array(mfi) : DummyFab.array(); const auto& divTau_arr = (has_divTau) != 0 ? a_divTau->const_array(mfi) : DummyFab.array(); amrex::ParallelFor( - bx, - [incomp_rho_inv, is_incomp, add_gradP, has_divTau, rho_arr, gp_arr, - divTau_arr, force_arr] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (is_incomp != 0) { for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { if (add_gradP != 0) { @@ -132,8 +129,7 @@ PeleLM::getVelForces( }); } else { amrex::ParallelFor( - bx, [incomp_rho_inv, is_incomp, rho_arr, - force_arr] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (is_incomp != 0) { for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { force_arr(i, j, k, idim) *= incomp_rho_inv; @@ -151,34 +147,31 @@ PeleLM::getVelForces( void PeleLM::getVelForces( int lev, - const Box& bx, - const Real& a_time, - Array4 const& force, - Array4 const& vel, - Array4 const& rho, - Array4 const& rhoY, - Array4 const& rhoh, - Array4 const& temp, - Array4 const& extMom, - Array4 const& extRho) + const amrex::Box& bx, + const amrex::Real& a_time, + amrex::Array4 const& force, + amrex::Array4 const& vel, + amrex::Array4 const& rho, + amrex::Array4 const& rhoY, + amrex::Array4 const& rhoh, + amrex::Array4 const& temp, + amrex::Array4 const& extMom, + amrex::Array4 const& extRho) { const auto dx = geom[lev].CellSizeArray(); - - // Get non-static info for the pseudo gravity forcing - int pseudo_gravity = m_ctrl_pseudoGravity; - const Real dV_control = m_ctrl_dV; - - int is_incomp = m_incompressible; - Real rho_incomp = m_rho; - - amrex::ParallelFor( - bx, - [=, grav = m_gravity, gp0 = m_background_gp, - ps_dir = m_ctrl_flameDir] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - makeVelForce( - i, j, k, is_incomp, rho_incomp, pseudo_gravity, ps_dir, a_time, grav, - gp0, dV_control, dx, vel, rho, rhoY, rhoh, temp, extMom, extRho, force); - }); + const int pseudo_gravity = m_ctrl_pseudoGravity; + const amrex::Real dV_control = m_ctrl_dV; + const int is_incomp = m_incompressible; + const amrex::Real rho_incomp = m_rho; + const auto grav = m_gravity; + const auto gp0 = m_background_gp; + const int ps_dir = m_ctrl_flameDir; + + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + makeVelForce( + i, j, k, is_incomp, rho_incomp, pseudo_gravity, ps_dir, a_time, grav, gp0, + dV_control, dx, vel, rho, rhoY, rhoh, temp, extMom, extRho, force); + }); } void @@ -187,51 +180,52 @@ PeleLM::addSpark(const TimeStamp& a_timestamp) for (int lev = 0; lev <= finest_level; lev++) { for (int n = 0; n < m_n_sparks; n++) { // Do the checks first - Real time = getTime(lev, a_timestamp); + amrex::Real time = getTime(lev, a_timestamp); bool verb = m_spark_verbose > 1 && lev == 0; if ( time < m_spark_time[n] || time > m_spark_time[n] + m_spark_duration[n]) { if (verb) { - Print() << m_spark[n] << " not active" << std::endl; + amrex::Print() << m_spark[n] << " not active" << std::endl; } continue; } - const Real* probLo = geom[lev].ProbLo(); + const amrex::Real* probLo = geom[lev].ProbLo(); auto const dx = geom[lev].CellSizeArray(); - IntVect spark_idx; + amrex::IntVect spark_idx; for (int d = 0; d < AMREX_SPACEDIM; d++) { spark_idx[d] = (int)((m_spark_location[n][d] - probLo[d]) / dx[d]); } - Box domainBox = geom[lev].Domain(); + amrex::Box domainBox = geom[lev].Domain(); // just a check if (!domainBox.contains(spark_idx)) { - Warning(m_spark[n] + " not in domain!"); + amrex::Warning(m_spark[n] + " not in domain!"); continue; } if (verb) { - Print() << m_spark[n] << " active" << std::endl; + amrex::Print() << m_spark[n] << " active" << std::endl; } auto statema = getLevelDataPtr(lev, a_timestamp)->state.const_arrays(); auto extma = m_extSource[lev]->arrays(); auto const* leosparm = eos_parms.device_parm(); + const auto spark_duration = m_spark_duration[n]; + const auto spark_temp = m_spark_temp[n]; + const auto* eosparm = leosparm; + const auto spark_radius = m_spark_radius[n]; amrex::ParallelFor( *m_extSource[lev], - [=, spark_duration = m_spark_duration[n], spark_temp = m_spark_temp[n], - eosparm = leosparm, - spark_radius = m_spark_radius - [n]] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { auto eos = pele::physics::PhysicsType::eos(eosparm); - Real dist_to_center = std::sqrt(AMREX_D_TERM( + amrex::Real dist_to_center = std::sqrt(AMREX_D_TERM( (i - spark_idx[0]) * (i - spark_idx[0]) * dx[0] * dx[0], +(j - spark_idx[1]) * (j - spark_idx[1]) * dx[1] * dx[1], +(k - spark_idx[2]) * (k - spark_idx[2]) * dx[2] * dx[2])); if (dist_to_center < spark_radius) { - Real rhoh_src_loc = 0; - Real rho = statema[box_no](i, j, k, DENSITY); - Real Y[NUM_SPECIES]; + amrex::Real rhoh_src_loc = 0; + amrex::Real rho = statema[box_no](i, j, k, DENSITY); + amrex::Real Y[NUM_SPECIES]; for (int ns = 0; ns < NUM_SPECIES; ns++) { Y[ns] = statema[box_no](i, j, k, FIRSTSPEC + ns) / rho; } @@ -240,7 +234,7 @@ PeleLM::addSpark(const TimeStamp& a_timestamp) extma[box_no](i, j, k, RHOH) = rhoh_src_loc; } }); - Gpu::streamSynchronize(); + amrex::Gpu::streamSynchronize(); } } } @@ -284,14 +278,15 @@ PeleLM::addScalarVarianceSources(const TimeStamp& a_timestamp) int do_avgDown = 0; auto bcRecScalar = fetchBCRecArray(var_of_scalar, 1); int nGrow = 0; // No need for ghost face on fluxes - Vector> grad_fc(finest_level + 1); + amrex::Vector> grad_fc( + finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { const auto& ba = grids[lev]; const auto& factory = Factory(lev); for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { grad_fc[lev][idim].define( - amrex::convert(ba, IntVect::TheDimensionVector(idim)), dmap[lev], 1, - nGrow, MFInfo(), factory); + amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)), + dmap[lev], 1, nGrow, amrex::MFInfo(), factory); grad_fc[lev][idim].setVal(0.0); // Required? } } @@ -381,7 +376,7 @@ PeleLM::addScalarVarianceSources(const TimeStamp& a_timestamp) }); } } - Gpu::streamSynchronize(); + amrex::Gpu::streamSynchronize(); } } } diff --git a/Source/PeleLMeX_Init.cpp b/Source/PeleLMeX_Init.cpp index 95f29e554..fd3d2efc8 100644 --- a/Source/PeleLMeX_Init.cpp +++ b/Source/PeleLMeX_Init.cpp @@ -5,8 +5,6 @@ #include #endif -using namespace amrex; - void PeleLM::Init() { @@ -36,7 +34,7 @@ PeleLM::MakeNewLevelFromScratch( << std::endl; if (m_verbose > 2 && lev > 0) { auto const dx = geom[lev].CellSizeArray(); - Real vol = AMREX_D_TERM(dx[0], *dx[1], *dx[2]); + amrex::Real vol = AMREX_D_TERM(dx[0], *dx[1], *dx[2]); amrex::Print() << " with " << ba.numPts() << " cells," << ba.size() << " boxes," << " over " @@ -56,9 +54,9 @@ PeleLM::MakeNewLevelFromScratch( // Define the FAB Factory #ifdef AMREX_USE_EB m_factory[lev] = makeEBFabFactory( - geom[lev], grids[lev], dmap[lev], {6, 6, 6}, EBSupport::full); + geom[lev], grids[lev], dmap[lev], {6, 6, 6}, amrex::EBSupport::full); #else - m_factory[lev] = std::make_unique(); + m_factory[lev] = std::make_unique(); #endif // Initialize the LevelData @@ -71,7 +69,7 @@ PeleLM::MakeNewLevelFromScratch( if (max_level > 0 && lev != max_level) { m_coveredMask[lev] = - std::make_unique(grids[lev], dmap[lev], 1, 0); + std::make_unique(grids[lev], dmap[lev], 1, 0); } m_resetCoveredMask = 1; @@ -89,7 +87,7 @@ PeleLM::MakeNewLevelFromScratch( new LevelDataNLSolve(grids[lev], dmap[lev], *m_factory[lev], m_nGrowState)); if (m_do_extraEFdiags) { m_ionsFluxes[lev].reset( - new MultiFab(grids[lev], dmap[lev], NUM_IONS * AMREX_SPACEDIM, 0)); + new amrex::MultiFab(grids[lev], dmap[lev], NUM_IONS * AMREX_SPACEDIM, 0)); } #endif @@ -111,55 +109,56 @@ PeleLM::MakeNewLevelFromScratch( m_t_old[lev] = time - 1.0e200; // Load balance - m_costs[lev] = std::make_unique>(ba, dm); + m_costs[lev] = std::make_unique>(ba, dm); // Mac projector #ifdef AMREX_USE_EB macproj = std::make_unique( Geom(0, finest_level), - MLMG::Location::FaceCentroid, // Location of mac velocity - MLMG::Location::FaceCentroid, // Location of beta - MLMG::Location::CellCenter); // Location of solution variable phi + amrex::MLMG::Location::FaceCentroid, // Location of mac velocity + amrex::MLMG::Location::FaceCentroid, // Location of beta + amrex::MLMG::Location::CellCenter); // Location of solution variable phi #else macproj = std::make_unique(Geom(0, finest_level)); #endif m_macProjOldSize = finest_level + 1; - m_extSource[lev] = std::make_unique( - grids[lev], dmap[lev], NVAR, amrex::max(m_nGrowAdv, m_nGrowMAC), MFInfo(), - *m_factory[lev]); + m_extSource[lev] = std::make_unique( + grids[lev], dmap[lev], NVAR, amrex::max(m_nGrowAdv, m_nGrowMAC), + amrex::MFInfo(), *m_factory[lev]); m_extSource[lev]->setVal(0.); #ifdef AMREX_USE_EB if (lev == 0 && (m_signDistNeeded != 0)) { // Set up CC signed distance container to control EB refinement - m_signedDist0 = std::make_unique( - grids[lev], dmap[lev], 1, 1, MFInfo(), *m_factory[lev]); + m_signedDist0 = std::make_unique( + grids[lev], dmap[lev], 1, 1, amrex::MFInfo(), *m_factory[lev]); // Estimate the maximum distance we need in terms of level 0 dx: - Real extentFactor = static_cast(nErrorBuf(0)); + auto extentFactor = static_cast(nErrorBuf(0)); for (int ilev = 1; ilev <= max_level; ++ilev) { - extentFactor += - static_cast(nErrorBuf(ilev)) / - std::pow( - static_cast(refRatio(ilev - 1)[0]), static_cast(ilev)); + extentFactor += static_cast(nErrorBuf(ilev)) / + std::pow( + static_cast(refRatio(ilev - 1)[0]), + static_cast(ilev)); } extentFactor *= std::sqrt(2.0) * m_derefineEBBuffer; // Account for diagonals - MultiFab signDist( - convert(grids[0], IntVect::TheUnitVector()), dmap[0], 1, 1, MFInfo(), - EBFactory(0)); + amrex::MultiFab signDist( + convert(grids[0], amrex::IntVect::TheUnitVector()), dmap[0], 1, 1, + amrex::MFInfo(), EBFactory(0)); FillSignedDistance(signDist, true); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*m_signedDist0, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.growntilebox(); + for (amrex::MFIter mfi(*m_signedDist0, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.growntilebox(); auto const& sd_cc = m_signedDist0->array(mfi); auto const& sd_nd = signDist.const_array(mfi); amrex::ParallelFor( - bx, [sd_cc, sd_nd] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { amrex::Real fac = AMREX_D_PICK(0.5, 0.25, 0.125); sd_cc(i, j, k) = AMREX_D_TERM( sd_nd(i, j, k) + sd_nd(i + 1, j, k), @@ -262,7 +261,7 @@ PeleLM::initData() WriteCheckPointFile(); } - Print() << PrettyLine; + amrex::Print() << PrettyLine; // Diagnostics doDiagnostics(); @@ -304,8 +303,8 @@ PeleLM::initData() } m_dt = -1.0; int is_init = 1; - Real dtInit = computeDt(is_init, AmrNewTime); - Print() << " Initial dt: " << dtInit << "\n"; + amrex::Real dtInit = computeDt(is_init, AmrNewTime); + amrex::Print() << " Initial dt: " << dtInit << "\n"; } // Let's write the initial condition @@ -317,7 +316,7 @@ PeleLM::initData() // Regrid after restart if requested if (m_regrid_on_restart != 0) { - Print() << " Regriding on restart \n"; + amrex::Print() << " Regriding on restart \n"; for (int lev{finest_level}; lev < max_level; ++lev) { regrid(0, m_cur_time); // Need to fill the old state to enable regrid on higher levels @@ -360,11 +359,12 @@ PeleLM::initLevelData(int lev) auto const local_m_incompressible = m_incompressible; #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); - FArrayBox DummyFab(bx, 1); + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); + amrex::FArrayBox DummyFab(bx, 1); auto const& state_arr = ldata_p->state.array(mfi); auto const& aux_arr = (m_nAux > 0) ? ldata_p->auxiliaries.array(mfi) : DummyFab.array(); @@ -395,12 +395,12 @@ PeleLM::projectInitSolution() #endif // Post data Init time step estimate - Real dtInit = computeDt(is_init, AmrNewTime); - Print() << " Initial dt: " << dtInit << "\n"; + amrex::Real dtInit = computeDt(is_init, AmrNewTime); + amrex::Print() << " Initial dt: " << dtInit << "\n"; if (m_do_init_proj != 0) { - Print() << "\n Doing initial projection(s) \n\n"; + amrex::Print() << "\n Doing initial projection(s) \n\n"; // Subcycling IAMR/PeleLM first does a projection with no reaction divU // which can make the dt for evaluating I_R better if (m_has_divu != 0) { @@ -427,7 +427,7 @@ PeleLM::projectInitSolution() // Post data init time step estimate m_dt = computeDt(is_init, AmrNewTime); - Print() << " Initial dt: " << m_dt << "\n"; + amrex::Print() << " Initial dt: " << m_dt << "\n"; //---------------------------------------------------------------- // Initial velocity projection iterations @@ -439,7 +439,7 @@ PeleLM::projectInitSolution() for (int lev = finest_level; lev >= 0; --lev) { // Setup empty forcing - MultiFab Forcing(grids[lev], dmap[lev], nCompForcing(), 0); + amrex::MultiFab Forcing(grids[lev], dmap[lev], nCompForcing(), 0); Forcing.setVal(0.0); if (lev != finest_level) { @@ -481,7 +481,7 @@ PeleLM::projectInitSolution() ldataR_p->I_R.setVal(0.0); } } - Print() << PrettyLine; + amrex::Print() << PrettyLine; } else { // If we didn't do the projection, initialize press/gp(/I_R) for (int lev = 0; lev <= finest_level; ++lev) { @@ -533,13 +533,13 @@ PeleLM::InitFromGridFile(amrex::Real time) { { const amrex::BoxArray& ba = MakeBaseGrids(); - DistributionMapping dm(ba); + amrex::DistributionMapping dm(ba); MakeNewLevelFromScratch(0, time, ba, dm); } finest_level = static_cast(m_initial_ba.size()); for (int lev = 1; lev <= finest_level; lev++) { const amrex::BoxArray ba = m_initial_ba[lev - 1]; - DistributionMapping dm(ba); + amrex::DistributionMapping dm(ba); MakeNewLevelFromScratch(lev, time, ba, dm); } } @@ -549,13 +549,13 @@ PeleLM::checkRunParams() { #ifdef AMREX_USE_EB if (geom[0].IsRZ()) { - Abort("RZ geometry is not available with EB"); + amrex::Abort("RZ geometry is not available with EB"); } #endif #if (AMREX_SPACEDIM == 2) if (geom[0].IsRZ() && m_phys_bc.lo(0) != 3) { - Abort("x-low must be 'Symmetry' when using RZ coordinate system"); + amrex::Abort("x-low must be 'Symmetry' when using RZ coordinate system"); } #endif } diff --git a/Source/PeleLMeX_K.H b/Source/PeleLMeX_K.H index 8567ee637..81d8e93b3 100644 --- a/Source/PeleLMeX_K.H +++ b/Source/PeleLMeX_K.H @@ -30,31 +30,29 @@ getTransportCoeff( pele::physics::eos::EosParm const* eosparm) noexcept { - using namespace amrex::literals; - auto eos = pele::physics::PhysicsType::eos(eosparm); amrex::Real mwtinv[NUM_SPECIES] = {0.0}; eos.inv_molecular_weight(mwtinv); // Get rho & Y from rhoY - amrex::Real rho = 0.0_rt; + amrex::Real rho = 0.0; for (int n = 0; n < NUM_SPECIES; n++) { rho += rhoY(i, j, k, n); } - amrex::Real rhoinv = 1.0_rt / rho; + amrex::Real rhoinv = 1.0 / rho; amrex::Real y[NUM_SPECIES] = {0.0}; for (int n = 0; n < NUM_SPECIES; n++) { y[n] = rhoY(i, j, k, n) * rhoinv; } - amrex::Real Wbar = 0.0_rt; + amrex::Real Wbar = 0.0; eos.Y2WBAR(y, Wbar); amrex::Real rho_cgs = m2c::Rho(rho); // MKS -> CGS conversion amrex::Real rhoDi_cgs[NUM_SPECIES] = {0.0}; - amrex::Real lambda_cgs = 0.0_rt; - amrex::Real mu_cgs = 0.0_rt; - amrex::Real dummy_xi = 0.0_rt; + amrex::Real lambda_cgs = 0.0; + amrex::Real mu_cgs = 0.0; + amrex::Real dummy_xi = 0.0; amrex::Real chi_loc[NUM_SPECIES] = {0.0}; amrex::Real Tloc = T(i, j, k); @@ -72,7 +70,7 @@ getTransportCoeff( mu(i, j, k) = c2m::Mu(mu_cgs); if (do_fixed_Pr && do_fixed_Le) { // fixed Pr and Le, transport all dependent // on visc - amrex::Real cpmix_cgs = 0.0_rt; + amrex::Real cpmix_cgs = 0.0; eos.TY2Cp(T(i, j, k), y, cpmix_cgs); lambda(i, j, k) = PrInv * c2m::Cp(cpmix_cgs) * mu(i, j, k); amrex::Real ScInv = PrInv * LeInv; @@ -80,25 +78,25 @@ getTransportCoeff( rhoDi(i, j, k, n) = ScInv * mu(i, j, k); } } else if (do_fixed_Pr) { // fixed Pr, still use species diffs - amrex::Real cpmix_cgs = 0.0_rt; + amrex::Real cpmix_cgs = 0.0; eos.TY2Cp(T(i, j, k), y, cpmix_cgs); lambda(i, j, k) = PrInv * c2m::Cp(cpmix_cgs) * mu(i, j, k); for (int n = 0; n < NUM_SPECIES; n++) { - rhoDi(i, j, k, n) = rhoDi_cgs[n] * Wbar * mwtinv[n] * 1.0e-1_rt; + rhoDi(i, j, k, n) = rhoDi_cgs[n] * Wbar * mwtinv[n] * 1.0e-1; } } else if (do_fixed_Le) { // fixed Le, still use thermal cond lambda(i, j, k) = c2m::Lambda(lambda_cgs); - amrex::Real cpmix_cgs = 0.0_rt; + amrex::Real cpmix_cgs = 0.0; eos.TY2Cp(T(i, j, k), y, cpmix_cgs); for (int n = 0; n < NUM_SPECIES; n++) { - rhoDi(i, j, k, n) = lambda_cgs * 1.0e-1_rt * LeInv / cpmix_cgs; + rhoDi(i, j, k, n) = lambda_cgs * 1.0e-1 * LeInv / cpmix_cgs; } } else { // full MA model, with Soret if needed lambda(i, j, k) = c2m::Lambda(lambda_cgs); for (int n = 0; n < NUM_SPECIES; n++) { - rhoDi(i, j, k, n) = rhoDi_cgs[n] * Wbar * mwtinv[n] * 1.0e-1_rt; + rhoDi(i, j, k, n) = rhoDi_cgs[n] * Wbar * mwtinv[n] * 1.0e-1; if (do_soret) { - rhotheta(i, j, k, n) = -rhoDi_cgs[n] * chi_loc[n] * 1.0e-1_rt; + rhotheta(i, j, k, n) = -rhoDi_cgs[n] * chi_loc[n] * 1.0e-1; } } if (do_soret) { @@ -138,8 +136,6 @@ getTransportCoeff( const pele::physics::eos::EosParm* eosparm) noexcept { - using namespace amrex::literals; - // No Soret with manifold moels AMREX_ASSERT(!do_soret); @@ -155,9 +151,9 @@ getTransportCoeff( amrex::Real rho_cgs = m2c::Rho(rho); // MKS -> CGS conversion amrex::Real rhoDi_cgs[NUM_SPECIES] = {0.0}; - amrex::Real lambda_cgs = 0.0_rt; - amrex::Real mu_cgs = 0.0_rt; - amrex::Real dummy_xi = 0.0_rt; + amrex::Real lambda_cgs = 0.0; + amrex::Real mu_cgs = 0.0; + amrex::Real dummy_xi = 0.0; amrex::Real* dummy_chi = nullptr; amrex::Real Tloc = T(i, j, k); @@ -173,7 +169,7 @@ getTransportCoeff( // Do CGS -> MKS conversions for (int n = 0; n < NUM_SPECIES; n++) { - rhoDi(i, j, k, n) = rhoDi_cgs[n] * 1.0e-1_rt; + rhoDi(i, j, k, n) = rhoDi_cgs[n] * 1.0e-1; } lambda(i, j, k) = 0.0; // No need to carry lambda for manifold mu(i, j, k) = c2m::Mu(mu_cgs); @@ -193,23 +189,21 @@ getVelViscosity( pele::physics::PhysicsType::eos_type, pele::physics::PhysicsType::transport_type> const* trans_parm) noexcept { - using namespace amrex::literals; - // Get rho & Y from rhoY amrex::Real massdens[NUM_SPECIES]; for (int n = 0; n < NUM_SPECIES; n++) { massdens[n] = rhoY(i, j, k, n); } - amrex::Real rho = 0.0_rt, rhoinv = 0.0_rt, y[NUM_SPECIES] = {0.0}; + amrex::Real rho = 0.0, rhoinv = 0.0, y[NUM_SPECIES] = {0.0}; pele::physics::PhysicsType::eos_type::RY2RRinvY(massdens, rho, rhoinv, y); amrex::Real rho_cgs = m2c::Rho(rho); // MKS -> CGS conversion amrex::Real temp = T(i, j, k); amrex::Real dummy_rhoDi[NUM_SPECIES] = {0.0}; amrex::Real dummy_chi[NUM_SPECIES] = {0.0}; - amrex::Real dummy_lambda = 0.0_rt; - amrex::Real mu_cgs = 0.0_rt; - amrex::Real dummy_xi = 0.0_rt; + amrex::Real dummy_lambda = 0.0; + amrex::Real mu_cgs = 0.0; + amrex::Real dummy_xi = 0.0; constexpr bool get_xi = false; constexpr bool get_mu = true; @@ -239,13 +233,11 @@ getPGivenRTY( pele::physics::eos::EosParm const* eosparm) noexcept { - using namespace amrex::literals; - auto eos = pele::physics::PhysicsType::eos(eosparm); - amrex::Real rhoinv = 1.0_rt / rho(i, j, k); + amrex::Real rhoinv = 1.0 / rho(i, j, k); amrex::Real rho_cgs = m2c::Rho(rho(i, j, k)); - amrex::Real y[NUM_SPECIES] = {0.0_rt}; + amrex::Real y[NUM_SPECIES] = {0.0}; for (int n = 0; n < NUM_SPECIES; n++) { y[n] = rhoY(i, j, k, n) * rhoinv; } @@ -273,26 +265,24 @@ compute_divu( pele::physics::eos::EosParm const* eosparm) noexcept { - using namespace amrex::literals; - auto eos = pele::physics::PhysicsType::eos(eosparm); amrex::Real mwtinv[NUM_SPECIES] = {0.0}; eos.inv_molecular_weight(mwtinv); // Get rho & Y from rhoY - amrex::Real rho = 0.0_rt; + amrex::Real rho = 0.0; for (int n = 0; n < NUM_SPECIES; n++) { rho += rhoY(i, j, k, n); } - amrex::Real rhoinv = 1.0_rt / rho; + amrex::Real rhoinv = 1.0 / rho; amrex::Real y[NUM_SPECIES] = {0.0}; for (int n = 0; n < NUM_SPECIES; n++) { y[n] = rhoY(i, j, k, n) * rhoinv; } - amrex::Real cpmix_cgs = 0.0_rt; + amrex::Real cpmix_cgs = 0.0; eos.TY2Cp(T(i, j, k), y, cpmix_cgs); - amrex::Real Wbar = 0.0_rt; + amrex::Real Wbar = 0.0; eos.Y2WBAR(y, Wbar); amrex::Real hi_cgs[NUM_SPECIES] = {0.0}; eos.T2Hi(T(i, j, k), hi_cgs); @@ -303,7 +293,7 @@ compute_divu( } // Note: divu does not depend on extRho. See docs and PR #428 for details. - amrex::Real denominv = 1.0_rt / (rho * cpmix * T(i, j, k)); + amrex::Real denominv = 1.0 / (rho * cpmix * T(i, j, k)); divu(i, j, k) = (specEnthDiff(i, j, k) + tempDiff(i, j, k) + extRhoH(i, j, k)) * denominv; for (int n = 0; n < NUM_SPECIES; n++) { @@ -334,12 +324,10 @@ compute_divu( pele::physics::eos::EosParm const* eosparm) noexcept { - using namespace amrex::literals; - // Get rho & Ys from rhoYs. amrex::Real rho, rhoinv; - amrex::Real y[NUM_SPECIES] = {0.0_rt}; - amrex::Real massdens[NUM_SPECIES] = {0.0_rt}; + amrex::Real y[NUM_SPECIES] = {0.0}; + amrex::Real massdens[NUM_SPECIES] = {0.0}; for (int n = 0; n < NUM_SPECIES; n++) { massdens[n] = rhoY(i, j, k, n); } @@ -383,8 +371,6 @@ extFluxDivergence_K( const amrex::Real& scaling, amrex::Array4 const& div) noexcept { - using namespace amrex::literals; - amrex::Real factor = scaling / vol(i, j, k); for (int n = 0; n < ncomp; n++) { @@ -417,8 +403,6 @@ EB_intFluxDivergence_K( const amrex::Real& scaling, amrex::Array4 const& div) noexcept { - using namespace amrex::literals; - amrex::Real factor = scaling / vol(i, j, k); for (int n = 0; n < ncomp; n++) { @@ -457,8 +441,6 @@ EB_intFluxDivergence_K( const amrex::Real& scaling, amrex::Array4 const& div) noexcept { - using namespace amrex::literals; - amrex::Real factor = scaling / vol(i, j, k); amrex::Real eb_area = AMREX_D_TERM(1.0, *a_dx, *a_dx) * ebAreaFrac(i, j, k); @@ -492,8 +474,6 @@ intFluxDivergence_K( const amrex::Real& scaling, amrex::Array4 const& div) noexcept { - using namespace amrex::literals; - amrex::Real factor = scaling / vol(i, j, k); for (int n = 0; n < ncomp; n++) { @@ -524,8 +504,6 @@ intFluxDivergence_rz_K( const amrex::Real& scaling, amrex::Array4 const& div) noexcept { - using namespace amrex::literals; - amrex::Real factor = scaling / vol(i, j, k); for (int n = 0; n < ncomp; n++) { @@ -551,17 +529,15 @@ getMwmixGivenRY( pele::physics::eos::EosParm const* eosparm) noexcept { - using namespace amrex::literals; - auto eos = pele::physics::PhysicsType::eos(eosparm); - amrex::Real rhoinv = 1.0_rt / rho(i, j, k); - amrex::Real y[NUM_SPECIES] = {0.0_rt}; + amrex::Real rhoinv = 1.0 / rho(i, j, k); + amrex::Real y[NUM_SPECIES] = {0.0}; for (int n = 0; n < NUM_SPECIES; n++) { y[n] = rhoY(i, j, k, n) * rhoinv; } eos.Y2WBAR(y, Mwmix(i, j, k)); - Mwmix(i, j, k) = Mwmix(i, j, k) * 0.001_rt; // CGS -> MKS conversion + Mwmix(i, j, k) = Mwmix(i, j, k) * 0.001; // CGS -> MKS conversion } AMREX_GPU_DEVICE @@ -577,10 +553,8 @@ getGradMwmixGivengradYMwmix( pele::physics::eos::EosParm const* eosparm) noexcept { - using namespace amrex::literals; - auto eos = pele::physics::PhysicsType::eos(eosparm); - amrex::Real imw[NUM_SPECIES] = {0.0_rt}; + amrex::Real imw[NUM_SPECIES] = {0.0}; eos.inv_molecular_weight(imw); gradMwmix(i, j, k) = 0.0; for (int n = 0; n < NUM_SPECIES; n++) { @@ -606,8 +580,6 @@ cen2edg_cpp( amrex::Array4 const& cfab, amrex::Array4 const& efab) noexcept { - using namespace amrex::literals; - // Default behavior: id_l -1 in dir // Handle the BCs // Need -1 in id_l and id_h in dir on low Dirichlet BC @@ -626,20 +598,20 @@ cen2edg_cpp( for (int n = 0; n < ncomp; n++) { if ( (cfab(id_l[0], id_l[1], id_l[2], n) * - cfab(id_h[0], id_h[1], id_h[2], n)) > 0.0_rt) { - efab(i, j, k, n) = 2.0_rt * + cfab(id_h[0], id_h[1], id_h[2], n)) > 0.0) { + efab(i, j, k, n) = 2.0 * (cfab(id_l[0], id_l[1], id_l[2], n) * cfab(id_h[0], id_h[1], id_h[2], n)) / (cfab(id_l[0], id_l[1], id_l[2], n) + cfab(id_h[0], id_h[1], id_h[2], n)); } else { - efab(i, j, k, n) = 0.0_rt; + efab(i, j, k, n) = 0.0; } } } else { for (int n = 0; n < ncomp; n++) { - efab(i, j, k, n) = 0.5_rt * (cfab(id_l[0], id_l[1], id_l[2], n) + - cfab(id_h[0], id_h[1], id_h[2], n)); + efab(i, j, k, n) = 0.5 * (cfab(id_l[0], id_l[1], id_l[2], n) + + cfab(id_h[0], id_h[1], id_h[2], n)); } } } @@ -657,8 +629,6 @@ repair_flux( amrex::Array4 const& rhoY, amrex::Array4 const& flux) noexcept { - using namespace amrex::literals; - // Handle the BCs : need a -1 on id_l in dir away from the BC // Need -1 in id_l and id_h in dir on low Dirichlet BC // Need nothing on high Dirichlet BC @@ -673,16 +643,16 @@ repair_flux( id_h[dir] -= 1; } - amrex::Real sumFlux = 0.0_rt; - amrex::Real sumRhoYe = 0.0_rt; + amrex::Real sumFlux = 0.0; + amrex::Real sumRhoYe = 0.0; amrex::Real RhoYe[NUM_SPECIES] = {0.0}; for (int n = 0; n < NUM_SPECIES; n++) { sumFlux += flux(i, j, k, n); - RhoYe[n] = 0.5_rt * (rhoY(id_l[0], id_l[1], id_l[2], n) + - rhoY(id_h[0], id_h[1], id_h[2], n)); + RhoYe[n] = 0.5 * (rhoY(id_l[0], id_l[1], id_l[2], n) + + rhoY(id_h[0], id_h[1], id_h[2], n)); sumRhoYe += RhoYe[n]; } - sumRhoYe = 1.0_rt / sumRhoYe; + sumRhoYe = 1.0 / sumRhoYe; for (int n = 0; n < NUM_SPECIES; n++) { flux(i, j, k, n) -= sumFlux * RhoYe[n] * sumRhoYe; } @@ -703,14 +673,12 @@ repair_flux_eb( amrex::Array4 const& areafrac, amrex::Array4 const& flux) noexcept { - using namespace amrex::literals; - // Handle the BCs : need a -1 on id_l in dir on Dirichlet BC int id_l[3] = {i, j, k}; id_l[dir] -= 1; - if (areafrac(i, j, k) > 0.0_rt) { - amrex::Real sumFlux = 0.0_rt; - amrex::Real sumRhoYe = 0.0_rt; + if (areafrac(i, j, k) > 0.0) { + amrex::Real sumFlux = 0.0; + amrex::Real sumRhoYe = 0.0; amrex::Real RhoYe[NUM_SPECIES] = {0.0}; for (int n = 0; n < NUM_SPECIES; n++) { sumFlux += flux(i, j, k, n); @@ -723,13 +691,13 @@ repair_flux_eb( } sumRhoYe += RhoYe[n]; } - sumRhoYe = 1.0_rt / sumRhoYe; + sumRhoYe = 1.0 / sumRhoYe; for (int n = 0; n < NUM_SPECIES; n++) { flux(i, j, k, n) -= sumFlux * RhoYe[n] * sumRhoYe; } } else { for (int n = 0; n < NUM_SPECIES; n++) { - flux(i, j, k, n) = 0.0_rt; + flux(i, j, k, n) = 0.0; } } } @@ -746,13 +714,11 @@ getHGivenT( pele::physics::eos::EosParm const* eosparm) noexcept { - using namespace amrex::literals; - auto eos = pele::physics::PhysicsType::eos(eosparm); - amrex::Real hi_spec[NUM_SPECIES] = {0.0_rt}; + amrex::Real hi_spec[NUM_SPECIES] = {0.0}; eos.T2Hi(T(i, j, k), hi_spec); for (int n = 0; n < NUM_SPECIES; n++) { - Hi(i, j, k, n) = hi_spec[n] * 0.0001_rt; // CGS -> MKS conversion + Hi(i, j, k, n) = hi_spec[n] * 0.0001; // CGS -> MKS conversion } } @@ -781,8 +747,6 @@ makeVelForce( amrex::Array4 const& extrho, amrex::Array4 const& force) noexcept { - using namespace amrex::literals; - // Switch between incompressible/low-Mach rhos amrex::Real rho_lcl = 0.0; if (is_incomp != 0) { @@ -822,11 +786,9 @@ getRHmixGivenTY( pele::physics::eos::EosParm const* eosparm) noexcept { - using namespace amrex::literals; - auto eos = pele::physics::PhysicsType::eos(eosparm); - amrex::Real rhoinv = 1.0_rt / rho(i, j, k); - amrex::Real y[NUM_SPECIES] = {0.0_rt}; + amrex::Real rhoinv = 1.0 / rho(i, j, k); + amrex::Real y[NUM_SPECIES] = {0.0}; for (int n = 0; n < NUM_SPECIES; n++) { y[n] = rhoY(i, j, k, n) * rhoinv; } @@ -849,10 +811,8 @@ getTfromHY( pele::physics::eos::EosParm const* eosparm) noexcept { - using namespace amrex::literals; - auto eos = pele::physics::PhysicsType::eos(eosparm); - amrex::Real rhoinv = 1.0_rt / rho(i, j, k); + amrex::Real rhoinv = 1.0 / rho(i, j, k); amrex::Real y[NUM_SPECIES] = {0.0}; for (int n = 0; n < NUM_SPECIES; n++) { y[n] = rhoY(i, j, k, n) * rhoinv; @@ -877,10 +837,8 @@ getCpmixGivenRYT( pele::physics::eos::EosParm const* eosparm) noexcept { - using namespace amrex::literals; - auto eos = pele::physics::PhysicsType::eos(eosparm); - amrex::Real rhoinv = 1.0_rt / rho(i, j, k); + amrex::Real rhoinv = 1.0 / rho(i, j, k); amrex::Real y[NUM_SPECIES] = {0.0}; for (int n = 0; n < NUM_SPECIES; n++) { y[n] = rhoY(i, j, k, n) * rhoinv; @@ -917,15 +875,13 @@ buildAdvectionForcing( pele::physics::eos::EosParm const* eosparm) noexcept { - using namespace amrex::literals; - auto eos = pele::physics::PhysicsType::eos(eosparm); // Get species enthalpy amrex::Real hi_spec_cgs[NUM_SPECIES] = {0.0}; eos.T2Hi(T(i, j, k), hi_spec_cgs); // For species mass fractions - amrex::Real rhoinv = 1.0_rt / rho(i, j, k); + amrex::Real rhoinv = 1.0 / rho(i, j, k); amrex::Real y[NUM_SPECIES] = {0.0}; forceT(i, j, k) = dn(i, j, k, NUM_SPECIES) + ddn(i, j, k); @@ -956,9 +912,9 @@ buildAdvectionForcing( // TO DO: source terms for Aux } - amrex::Real cpmix_cgs = 0.0_rt; + amrex::Real cpmix_cgs = 0.0; eos.TY2Cp(T(i, j, k), y, cpmix_cgs); - amrex::Real cpmixinv = 1.0_rt / c2m::Cp(cpmix_cgs); // CGS -> MKS conversion + amrex::Real cpmixinv = 1.0 / c2m::Cp(cpmix_cgs); // CGS -> MKS conversion forceT(i, j, k) *= rhoinv * cpmixinv; } @@ -994,11 +950,9 @@ buildDiffusionForcing( const int* aux_diffuse, int nAux) noexcept { - using namespace amrex::literals; - for (int n = 0; n < NUM_SPECIES; n++) { forceY(i, j, k, n) = a(i, j, k, n) + - 0.5_rt * (dn(i, j, k, n) - dnp1k(i, j, k, n)) + + 0.5 * (dn(i, j, k, n) - dnp1k(i, j, k, n)) + extRhoY(i, j, k, n); if (do_react != 0) { forceY(i, j, k, n) += r(i, j, k, n); @@ -1011,8 +965,8 @@ buildDiffusionForcing( } } forceT(i, j, k) = a(i, j, k, NUM_SPECIES) + - 0.5_rt * (dn(i, j, k, NUM_SPECIES) + ddn(i, j, k) - - dnp1k(i, j, k, NUM_SPECIES) - ddnp1k(i, j, k)) + + 0.5 * (dn(i, j, k, NUM_SPECIES) + ddn(i, j, k) - + dnp1k(i, j, k, NUM_SPECIES) - ddnp1k(i, j, k)) + extRhoH(i, j, k); if (closed_chamber == 1) { @@ -1023,7 +977,7 @@ buildDiffusionForcing( forceAux(i, j, k, n) = 0.0; if (aux_diffuse[n] != 0) { forceAux(i, j, k, n) += - 0.5_rt * (dn_aux(i, j, k, n) - dnp1k_aux(i, j, k, n)); + 0.5 * (dn_aux(i, j, k, n) - dnp1k_aux(i, j, k, n)); } if (aux_advect[n] != 0) { forceAux(i, j, k, n) += a_aux(i, j, k, n); @@ -1046,8 +1000,6 @@ reactionRateRhoY( pele::physics::eos::EosParm const* eosparm) noexcept { - using namespace amrex::literals; - auto eos = pele::physics::PhysicsType::eos(eosparm); // Get rho & Ys from rhoYs. @@ -1062,10 +1014,10 @@ reactionRateRhoY( // Get wdot. amrex::Real rho_cgs = m2c::Rho(rho); // rho MKS -> CGS - amrex::Real wdot[NUM_SPECIES] = {0.0_rt}; + amrex::Real wdot[NUM_SPECIES] = {0.0}; eos.RTY2WDOT(rho_cgs, Tloc, y, wdot); for (int n = 0; n < NUM_SPECIES; n++) { - rhoYdot(i, j, k, n) = wdot[n] * 1000.0_rt; // CGS -> MKS conversion + rhoYdot(i, j, k, n) = wdot[n] * 1000.0; // CGS -> MKS conversion } } @@ -1081,8 +1033,6 @@ fabMinMax( amrex::Real fmax, amrex::Array4 const& fab) noexcept { - using namespace amrex::literals; - for (int n = 0; n < ncomp; n++) { fab(i, j, k, n) = amrex::max(fmin, amrex::min(fmax, fab(i, j, k, n))); } @@ -1102,14 +1052,12 @@ est_divu_dt_1( amrex::Array4 const& rho, amrex::Array4 const& divu) noexcept { - using namespace amrex::literals; - - amrex::Real dtcell = 1.0e20_rt; + amrex::Real dtcell = 1.0e20; const bool rho_is_okay = (rho(i, j, k) > rhomin); - if (divu(i, j, k) > 0.0_rt) { - dtcell = (rho_is_okay) ? (1.0_rt - rhomin / rho(i, j, k)) / divu(i, j, k) - : 1.0_rt / divu(i, j, k); + if (divu(i, j, k) > 0.0) { + dtcell = (rho_is_okay) ? (1.0 - rhomin / rho(i, j, k)) / divu(i, j, k) + : 1.0 / divu(i, j, k); dtcell *= dtfactor; } return dtcell; @@ -1131,10 +1079,8 @@ est_divu_dt_2( amrex::Array4 const& vel, amrex::Array4 const& divu) noexcept { - using namespace amrex::literals; - - amrex::Real dtcell = 1.0e20_rt; - amrex::Real denom = 0.0_rt; + amrex::Real dtcell = 1.0e20; + amrex::Real denom = 0.0; bool rho_is_okay = (rho(i, j, k) > rhomin); denom = @@ -1143,7 +1089,7 @@ est_divu_dt_2( vel(i, j, k, 0) * (rho(i + 1, j, k) - rho(i - 1, j, k)) * dxinv[0], +vel(i, j, k, 1) * (rho(i, j + 1, k) - rho(i, j - 1, k)) * dxinv[1], +vel(i, j, k, 2) * (rho(i, j, k + 1) - rho(i, j, k - 1)) * dxinv[2]); - if (denom > 0.0_rt) { + if (denom > 0.0) { dtcell = (rho_is_okay) ? (rho(i, j, k) - rhomin) / denom : std::abs(rho(i, j, k)) / denom; dtcell *= dtfactor; @@ -1167,16 +1113,14 @@ check_divu_dt( amrex::Array4 const& divu, amrex::Real const& a_dt) noexcept { - using namespace amrex::literals; - - amrex::Real dtcell = 1.0e12_rt; - amrex::Real denom = 0.0_rt; + amrex::Real dtcell = 1.0e12; + amrex::Real denom = 0.0; bool rho_is_okay = (rho(i, j, k) > rhomin); if (check_type == 1) { // Check based on divU - if (divu(i, j, k) > 0.0_rt) { - dtcell = (rho_is_okay) ? (1.0_rt - rhomin / rho(i, j, k)) / divu(i, j, k) - : 1.0_rt / divu(i, j, k); + if (divu(i, j, k) > 0.0) { + dtcell = (rho_is_okay) ? (1.0 - rhomin / rho(i, j, k)) / divu(i, j, k) + : 1.0 / divu(i, j, k); } } else if (check_type == 2) { // Check based rho * divU + u \cdot grad rho denom = @@ -1185,7 +1129,7 @@ check_divu_dt( vel(i, j, k, 0) * (rho(i + 1, j, k) - rho(i - 1, j, k)) * dxinv[0], +vel(i, j, k, 1) * (rho(i, j + 1, k) - rho(i, j - 1, k)) * dxinv[1], +vel(i, j, k, 2) * (rho(i, j, k + 1) - rho(i, j, k - 1)) * dxinv[2]); - if (denom > 0.0_rt) { + if (denom > 0.0) { dtcell = (rho_is_okay) ? (rho(i, j, k) - rhomin) / denom : std::abs(rho(i, j, k)) / denom; } @@ -1209,23 +1153,21 @@ getGammaInv( pele::physics::eos::EosParm const* eosparm) noexcept { - using namespace amrex::literals; - auto eos = pele::physics::PhysicsType::eos(eosparm); // Get rho & Y from rhoY - amrex::Real rho = 0.0_rt; + amrex::Real rho = 0.0; for (int n = 0; n < NUM_SPECIES; n++) { rho += rhoY(i, j, k, n); } - amrex::Real rhoinv = 1.0_rt / rho; + amrex::Real rhoinv = 1.0 / rho; amrex::Real y[NUM_SPECIES] = {0.0}; for (int n = 0; n < NUM_SPECIES; n++) { y[n] = rhoY(i, j, k, n) * rhoinv; } - amrex::Real cpmix = 0.0_rt; + amrex::Real cpmix = 0.0; eos.TY2Cp(T(i, j, k), y, cpmix); - amrex::Real cvmix = 0.0_rt; + amrex::Real cvmix = 0.0; eos.TY2Cv(T(i, j, k), y, cvmix); amrex::Real gammainv = cvmix / cpmix; @@ -1304,8 +1246,6 @@ getTurbViscSmagorinsky( amrex::Array4 const& rho, amrex::Array4 const& mu_t) noexcept { - using namespace amrex::literals; - // mu_t = rho * Cs * Delta^2 * |S| where |S| = sqrt(2 SijSij) amrex::Real Smag = std::sqrt(2.0 * getSijSij(i, j, k, velgrad)); @@ -1324,8 +1264,6 @@ getTurbViscWALE( amrex::Array4 const& rho, amrex::Array4 const& mu_t) noexcept { - using namespace amrex::literals; - // From Ducros, Nicoud, and Poinsot 1999 // mu_t = rho * Cs * Delta^2 * (S^d_ijS^d_ij)^(3/2) // / (SijSij^(5/2) + (S^d_ijS^d_ij)^(5/4) + smallnum) @@ -1357,8 +1295,6 @@ getTurbViscSigma( amrex::ignore_unused(i, j, k, prefactor, velgrad, rho, mu_t); amrex::Abort("getTurbViscSigma not implemented in 2D"); #elif (AMREX_SPACEDIM == 3) - using namespace amrex::literals; - // From Nicoud, et al . 2011 // mu_t = rho * Cs * Delta^2 * \sig_3(\sig_1-\sig_2)(\sig_2-\sig_3) // / (\sig_1*\sig_1 + smallnum) @@ -1396,30 +1332,30 @@ getTurbViscSigma( amrex::Real alpha2 = 0.0; amrex::Real alpha3 = 0.0; - alpha1 = std::max(alpha1, (I[0] / 3.0) * (I[0] / 3.0) - I[1] / 3.0); + alpha1 = amrex::max(alpha1, (I[0] / 3.0) * (I[0] / 3.0) - I[1] / 3.0); if (alpha1 != 0.0) { alpha2 = (I[0] / 3.0) * (I[0] / 3.0) * (I[0] / 3.0) - I[0] * I[1] / 6.0 + I[2] / 2.0; amrex::Real arcCosArg = - std::min(1.0, std::max(-1.0, alpha2 / std::pow(alpha1, 1.5))); + amrex::min(1.0, amrex::max(-1.0, alpha2 / std::pow(alpha1, 1.5))); alpha3 = std::acos(arcCosArg) / 3.0; // Singular values: amrex::Real sigma1 = std::sqrt( - std::max(0.0, I[0] / 3.0 + 2.0 * std::sqrt(alpha1) * std::cos(alpha3))); + amrex::max(0.0, I[0] / 3.0 + 2.0 * std::sqrt(alpha1) * std::cos(alpha3))); amrex::Real sigma2 = std::sqrt( - std::max( + amrex::max( 0.0, I[0] / 3.0 - 2.0 * std::sqrt(alpha1) * std::cos(Pi / 3.0 + alpha3))); amrex::Real sigma3 = std::sqrt( - std::max( + amrex::max( 0.0, I[0] / 3.0 - 2.0 * std::sqrt(alpha1) * std::cos(Pi / 3.0 - alpha3))); // Sort - sigma2 = std::max(sigma3, sigma2); - sigma1 = std::max(sigma2, sigma1); - sigma1 = std::max(1e-16, sigma1); + sigma2 = amrex::max(sigma3, sigma2); + sigma1 = amrex::max(sigma2, sigma1); + sigma1 = amrex::max(1e-16, sigma1); D_sig = sigma3 * (sigma1 - sigma2) * (sigma2 - sigma3) / (sigma1 * sigma1); } diff --git a/Source/PeleLMeX_ODEQty.cpp b/Source/PeleLMeX_ODEQty.cpp index 44bd75df2..aec597af0 100644 --- a/Source/PeleLMeX_ODEQty.cpp +++ b/Source/PeleLMeX_ODEQty.cpp @@ -1,8 +1,6 @@ #include #include -using namespace amrex; - #if NUM_ODE > 0 void PeleLM::predictODEQty() @@ -12,16 +10,18 @@ PeleLM::predictODEQty() for (int lev = 0; lev <= finest_level; lev++) { auto const& state_arrs = getLevelDataPtr(lev, AmrNewTime)->state.arrays(); auto const& ext_src_arrs = m_extSource[lev]->arrays(); + const auto dt = m_dt; ParallelFor( - *m_extSource[lev], [state_arrs, ext_src_arrs, dt = m_dt] AMREX_GPU_DEVICE( - int box_no, int i, int j, int k) noexcept { + *m_extSource[lev], + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { for (int n = 0; n < NUM_ODE; n++) { - Real const& B_n = state_arrs[box_no](i, j, k, FIRSTODE + n); - Real const& S_ext_n = ext_src_arrs[box_no](i, j, k, FIRSTODE + n); + amrex::Real const& B_n = state_arrs[box_no](i, j, k, FIRSTODE + n); + amrex::Real const& S_ext_n = + ext_src_arrs[box_no](i, j, k, FIRSTODE + n); state_arrs[box_no](i, j, k, FIRSTODE + n) = B_n + dt * S_ext_n; } }); - Gpu::streamSynchronize(); + amrex::Gpu::streamSynchronize(); } } -#endif \ No newline at end of file +#endif diff --git a/Source/PeleLMeX_Plot.cpp b/Source/PeleLMeX_Plot.cpp index 61b110a49..80ea88b35 100644 --- a/Source/PeleLMeX_Plot.cpp +++ b/Source/PeleLMeX_Plot.cpp @@ -21,8 +21,6 @@ #include "PeleLMRad.H" #endif -using namespace amrex; - namespace { const std::string level_prefix{"Level_"}; } @@ -36,14 +34,14 @@ GotoNextLine(std::istream& is) void PeleLM::WriteDebugPlotFile( - const Vector& a_MF, const std::string& pltname) + const amrex::Vector& a_MF, const std::string& pltname) { int nComp = a_MF[0]->nComp(); - Vector names(nComp); + amrex::Vector names(nComp); for (int n = 0; n < nComp; n++) { names[n] = "comp" + std::to_string(n); } - Vector istep(finest_level + 1, m_nstep); + amrex::Vector istep(finest_level + 1, m_nstep); #ifdef AMREX_USE_HDF5 if (m_write_hdf5_pltfile) { amrex::WriteMultiLevelPlotfileHDF5( @@ -80,7 +78,7 @@ PeleLM::WritePlotFile() } } - VisMF::SetNOutFiles(m_nfiles); + amrex::VisMF::SetNOutFiles(m_nfiles); //---------------------------------------------------------------- // Average down the state @@ -174,18 +172,19 @@ PeleLM::WritePlotFile() //---------------------------------------------------------------- // Plot MultiFabs - Vector mf_plt(finest_level + 1); + amrex::Vector mf_plt(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - mf_plt[lev].define(grids[lev], dmap[lev], ncomp, 0, MFInfo(), Factory(lev)); + mf_plt[lev].define( + grids[lev], dmap[lev], ncomp, 0, amrex::MFInfo(), Factory(lev)); } //---------------------------------------------------------------- // Components names - Vector names; + amrex::Vector names; pele::physics::eos::speciesNames( names, &(eos_parms.host_parm())); - Vector plt_VarsName; + amrex::Vector plt_VarsName; AMREX_D_TERM(plt_VarsName.push_back("x_velocity"); , plt_VarsName.push_back("y_velocity"); , plt_VarsName.push_back("z_velocity")); @@ -311,87 +310,94 @@ PeleLM::WritePlotFile() for (int lev = 0; lev <= finest_level; ++lev) { int cnt = 0; if (m_incompressible != 0) { - MultiFab::Copy( + amrex::MultiFab::Copy( mf_plt[lev], m_leveldata_new[lev]->state, 0, cnt, AMREX_SPACEDIM, 0); cnt += AMREX_SPACEDIM; } else { // Velocity and density - MultiFab::Copy( + amrex::MultiFab::Copy( mf_plt[lev], m_leveldata_new[lev]->state, 0, cnt, AMREX_SPACEDIM + 1, 0); cnt += AMREX_SPACEDIM + 1; // Species only if requested if (m_plotStateSpec != 0) { - MultiFab::Copy( + amrex::MultiFab::Copy( mf_plt[lev], m_leveldata_new[lev]->state, FIRSTSPEC, cnt, NUM_SPECIES, 0); cnt += NUM_SPECIES; } - MultiFab::Copy(mf_plt[lev], m_leveldata_new[lev]->state, RHOH, cnt, 3, 0); + amrex::MultiFab::Copy( + mf_plt[lev], m_leveldata_new[lev]->state, RHOH, cnt, 3, 0); cnt += 3; #ifdef PELE_USE_PLASMA - MultiFab::Copy(mf_plt[lev], m_leveldata_new[lev]->state, NE, cnt, 2, 0); + amrex::MultiFab::Copy( + mf_plt[lev], m_leveldata_new[lev]->state, NE, cnt, 2, 0); cnt += 2; #endif #ifdef PELE_USE_SOOT - MultiFab::Copy( + amrex::MultiFab::Copy( mf_plt[lev], m_leveldata_new[lev]->state, FIRSTSOOT, cnt, NUMSOOTVAR, 0); cnt += NUMSOOTVAR; #endif #ifdef PELE_USE_RADIATION if (do_rad_solve) { - MultiFab::Copy(mf_plt[lev], rad_model->G()[lev], 0, cnt, 1, 0); + amrex::MultiFab::Copy(mf_plt[lev], rad_model->G()[lev], 0, cnt, 1, 0); cnt += 1; - MultiFab::Copy(mf_plt[lev], rad_model->kappa()[lev], 0, cnt, 1, 0); + amrex::MultiFab::Copy( + mf_plt[lev], rad_model->kappa()[lev], 0, cnt, 1, 0); cnt += 1; - MultiFab::Copy(mf_plt[lev], rad_model->emis()[lev], 0, cnt, 1, 0); + amrex::MultiFab::Copy( + mf_plt[lev], rad_model->emis()[lev], 0, cnt, 1, 0); cnt += 1; } #endif if (m_has_divu != 0) { - MultiFab::Copy(mf_plt[lev], m_leveldata_new[lev]->divu, 0, cnt, 1, 0); + amrex::MultiFab::Copy( + mf_plt[lev], m_leveldata_new[lev]->divu, 0, cnt, 1, 0); cnt += 1; } } if (m_plot_grad_p != 0) { - MultiFab::Copy( + amrex::MultiFab::Copy( mf_plt[lev], m_leveldata_new[lev]->gp, 0, cnt, AMREX_SPACEDIM, 0); cnt += AMREX_SPACEDIM; } if (m_nAux > 0) { - MultiFab::Copy( + amrex::MultiFab::Copy( mf_plt[lev], m_leveldata_new[lev]->auxiliaries, 0, cnt, m_nAux, 0); cnt += m_nAux; } if ((m_do_react != 0) && (m_skipInstantRR == 0) && (m_plot_react != 0)) { - MultiFab::Copy( + amrex::MultiFab::Copy( mf_plt[lev], m_leveldatareact[lev]->I_R, 0, cnt, nCompIR(), 0); cnt += nCompIR(); - MultiFab::Copy(mf_plt[lev], m_leveldatareact[lev]->functC, 0, cnt, 1, 0); + amrex::MultiFab::Copy( + mf_plt[lev], m_leveldatareact[lev]->functC, 0, cnt, 1, 0); cnt += 1; if (m_plotHeatRelease != 0) { - std::unique_ptr mf; - mf = std::make_unique(grids[lev], dmap[lev], 1, 0); + std::unique_ptr mf; + mf = std::make_unique(grids[lev], dmap[lev], 1, 0); getHeatRelease(lev, mf.get()); - MultiFab::Copy(mf_plt[lev], *mf, 0, cnt, 1, 0); + amrex::MultiFab::Copy(mf_plt[lev], *mf, 0, cnt, 1, 0); cnt += 1; } } #ifdef AMREX_USE_EB - MultiFab::Copy(mf_plt[lev], EBFactory(lev).getVolFrac(), 0, cnt, 1, 0); + amrex::MultiFab::Copy( + mf_plt[lev], EBFactory(lev).getVolFrac(), 0, cnt, 1, 0); cnt += 1; #endif for (int ivar = 0; ivar < m_derivePlotVarCount; ivar++) { - std::unique_ptr mf; + std::unique_ptr mf; mf = derive(m_derivePlotVars[ivar], m_cur_time, lev, 0); - MultiFab::Copy(mf_plt[lev], *mf, 0, cnt, mf->nComp(), 0); + amrex::MultiFab::Copy(mf_plt[lev], *mf, 0, cnt, mf->nComp(), 0); cnt += mf->nComp(); } #ifdef PELE_USE_SPRAY @@ -400,26 +406,27 @@ PeleLM::WritePlotFile() mf_plt[lev].setVal(0., cnt, num_spray_derive); SprayPC->computeDerivedVars(mf_plt[lev], lev, cnt); if (lev < finest_level) { - MultiFab tmp_plt( - grids[lev], dmap[lev], num_spray_derive, 0, MFInfo(), Factory(lev)); + amrex::MultiFab tmp_plt( + grids[lev], dmap[lev], num_spray_derive, 0, amrex::MFInfo(), + Factory(lev)); tmp_plt.setVal(0.); VirtPC->computeDerivedVars(tmp_plt, lev, 0); - MultiFab::Add(mf_plt[lev], tmp_plt, 0, cnt, num_spray_derive, 0); + amrex::MultiFab::Add(mf_plt[lev], tmp_plt, 0, cnt, num_spray_derive, 0); } cnt += num_spray_derive; } if (do_spray_particles && SprayParticleContainer::plot_spray_src) { SprayComps scomps = SprayParticleContainer::getSprayComps(); - MultiFab::Copy( + amrex::MultiFab::Copy( mf_plt[lev], *m_spraysource[lev], scomps.rhoSrcIndx, cnt++, 1, 0); - MultiFab::Copy( + amrex::MultiFab::Copy( mf_plt[lev], *m_spraysource[lev], scomps.engSrcIndx, cnt++, 1, 0); - MultiFab::Copy( + amrex::MultiFab::Copy( mf_plt[lev], *m_spraysource[lev], scomps.momSrcIndx, cnt, AMREX_SPACEDIM, 0); cnt += AMREX_SPACEDIM; for (int spf = 0; spf < SPRAY_FUEL_NUM; ++spf) { - MultiFab::Copy( + amrex::MultiFab::Copy( mf_plt[lev], *m_spraysource[lev], scomps.specSrcIndx + spf, cnt++, 1, 0); } @@ -427,7 +434,7 @@ PeleLM::WritePlotFile() #endif #ifdef PELE_USE_PLASMA if (m_do_extraEFdiags) { - MultiFab::Copy( + amrex::MultiFab::Copy( mf_plt[lev], *m_ionsFluxes[lev], 0, cnt, m_ionsFluxes[lev]->nComp(), 0); cnt += m_ionsFluxes[lev]->nComp(); } @@ -446,8 +453,7 @@ PeleLM::WritePlotFile() // interpolate turbulent viscosity from faces to centers amrex::ParallelFor( mf_plt[lev], - [plot_arr, AMREX_D_DECL(mut_arr_x, mut_arr_y, mut_arr_z), - cnt] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { plot_arr[box_no](i, j, k, cnt) = fact * (AMREX_D_TERM( @@ -455,18 +461,18 @@ PeleLM::WritePlotFile() +mut_arr_y[box_no](i, j, k) + mut_arr_y[box_no](i, j + 1, k), +mut_arr_z[box_no](i, j, k) + mut_arr_z[box_no](i, j, k + 1))); }); - Gpu::streamSynchronize(); + amrex::Gpu::streamSynchronize(); cnt += 1; } #if NUM_ODE > 0 - MultiFab::Copy( + amrex::MultiFab::Copy( mf_plt[lev], m_leveldata_new[lev]->state, FIRSTODE, cnt, NUM_ODE, 0); cnt += NUM_ODE; #endif if (m_plot_extSource) { - MultiFab::Copy(mf_plt[lev], *m_extSource[lev], 0, cnt, NVAR, 0); + amrex::MultiFab::Copy(mf_plt[lev], *m_extSource[lev], 0, cnt, NVAR, 0); } #ifdef AMREX_USE_EB @@ -477,7 +483,7 @@ PeleLM::WritePlotFile() } // No SubCycling, all levels the same step. - Vector istep(finest_level + 1, m_nstep); + amrex::Vector istep(finest_level + 1, m_nstep); #ifdef AMREX_USE_HDF5 if (m_write_hdf5_pltfile) { @@ -507,9 +513,9 @@ PeleLM::WritePlotFile() void PeleLM::WriteHeader(const std::string& name, bool is_checkpoint) const { - if (ParallelDescriptor::IOProcessor()) { + if (amrex::ParallelDescriptor::IOProcessor()) { std::string HeaderFileName(name + "/Header"); - VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size); + amrex::VisMF::IO_Buffer io_buffer(amrex::VisMF::IO_Buffer_Size); std::ofstream HeaderFile; HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size()); @@ -588,7 +594,7 @@ PeleLM::WriteCheckPointFile() } } - VisMF::SetNOutFiles(m_nfiles); + amrex::VisMF::SetNOutFiles(m_nfiles); amrex::PreBuildDirectorHierarchy( checkpointname, level_prefix, finest_level + 1, true); @@ -598,21 +604,21 @@ PeleLM::WriteCheckPointFile() WriteJobInfo(checkpointname); for (int lev = 0; lev <= finest_level; ++lev) { - VisMF::Write( + amrex::VisMF::Write( m_leveldata_new[lev]->state, amrex::MultiFabFileFullPrefix( lev, checkpointname, level_prefix, "state")); - VisMF::Write( + amrex::VisMF::Write( m_leveldata_new[lev]->gp, amrex::MultiFabFileFullPrefix( lev, checkpointname, level_prefix, "gradp")); - VisMF::Write( + amrex::VisMF::Write( m_leveldata_new[lev]->press, amrex::MultiFabFileFullPrefix(lev, checkpointname, level_prefix, "p")); if (m_nAux > 0) { - VisMF::Write( + amrex::VisMF::Write( m_leveldata_new[lev]->auxiliaries, amrex::MultiFabFileFullPrefix( lev, checkpointname, level_prefix, "aux")); @@ -620,14 +626,14 @@ PeleLM::WriteCheckPointFile() if (m_incompressible == 0) { if (m_has_divu != 0) { - VisMF::Write( + amrex::VisMF::Write( m_leveldata_new[lev]->divu, amrex::MultiFabFileFullPrefix( lev, checkpointname, level_prefix, "divU")); } if (m_do_react != 0) { - VisMF::Write( + amrex::VisMF::Write( m_leveldatareact[lev]->I_R, amrex::MultiFabFileFullPrefix( lev, checkpointname, level_prefix, "I_R")); @@ -651,8 +657,8 @@ PeleLM::ReadCheckPointFile() amrex::Print() << "Restarting from checkpoint " << m_restart_chkfile << "\n"; - Real prob_lo[AMREX_SPACEDIM]; - Real prob_hi[AMREX_SPACEDIM]; + amrex::Real prob_lo[AMREX_SPACEDIM]; + amrex::Real prob_hi[AMREX_SPACEDIM]; /*************************************************************************** ** Load header: set up problem domain (including BoxArray) * @@ -662,10 +668,10 @@ PeleLM::ReadCheckPointFile() std::string File(m_restart_chkfile + "/Header"); - VisMF::IO_Buffer io_buffer(VisMF::GetIOBufferSize()); + amrex::VisMF::IO_Buffer io_buffer(amrex::VisMF::GetIOBufferSize()); - Vector fileCharPtr; - ParallelDescriptor::ReadAndBcastFile(File, fileCharPtr); + amrex::Vector fileCharPtr; + amrex::ParallelDescriptor::ReadAndBcastFile(File, fileCharPtr); std::string fileCharPtrString(fileCharPtr.dataPtr()); std::istringstream is(fileCharPtrString, std::istringstream::in); @@ -680,7 +686,7 @@ PeleLM::ReadCheckPointFile() int chk_finest_level = 0; is >> chk_finest_level; GotoNextLine(is); - finest_level = std::min(chk_finest_level, max_level); + finest_level = amrex::min(chk_finest_level, max_level); // Step count is >> m_nstep; @@ -735,29 +741,29 @@ PeleLM::ReadCheckPointFile() } // Set up problem domain - RealBox rb(prob_lo, prob_hi); - Geometry::ResetDefaultProbDomain(rb); + amrex::RealBox rb(prob_lo, prob_hi); + amrex::Geometry::ResetDefaultProbDomain(rb); for (int lev = 0; lev <= max_level; ++lev) { SetGeometry( lev, - Geometry( + amrex::Geometry( Geom(lev).Domain(), rb, Geom(lev).CoordInt(), Geom(lev).isPeriodic())); } for (int lev = 0; lev <= finest_level; ++lev) { // read in level 'lev' BoxArray from Header - BoxArray ba; + amrex::BoxArray ba; ba.readFrom(is); GotoNextLine(is); // Create distribution mapping - DistributionMapping dm{ba, ParallelDescriptor::NProcs()}; + amrex::DistributionMapping dm{ba, amrex::ParallelDescriptor::NProcs()}; MakeNewLevelFromScratch(lev, m_cur_time, ba, dm); } for (int lev = finest_level + 1; lev <= chk_finest_level; ++lev) { // read dummy level 'lev' BoxArray if restarting with reduced levels - BoxArray ba; + amrex::BoxArray ba; ba.readFrom(is); } @@ -778,36 +784,36 @@ PeleLM::ReadCheckPointFile() for (int lev = 0; lev <= finest_level; ++lev) { #ifdef PELE_USE_PLASMA if (!m_restart_nonEF) { - VisMF::Read( + amrex::VisMF::Read( m_leveldata_new[lev]->state, amrex::MultiFabFileFullPrefix( lev, m_restart_chkfile, level_prefix, "state")); } else { // The chk state is 2 component shorter since phiV and nE aren't in it - MultiFab stateTemp(grids[lev], dmap[lev], NVAR - 2, m_nGrowState); - VisMF::Read( + amrex::MultiFab stateTemp(grids[lev], dmap[lev], NVAR - 2, m_nGrowState); + amrex::VisMF::Read( stateTemp, amrex::MultiFabFileFullPrefix( lev, m_restart_chkfile, level_prefix, "state")); - MultiFab::Copy( + amrex::MultiFab::Copy( m_leveldata_new[lev]->state, stateTemp, 0, 0, NVAR - 2, m_nGrowState); } #else - VisMF::Read( + amrex::VisMF::Read( m_leveldata_new[lev]->state, amrex::MultiFabFileFullPrefix( lev, m_restart_chkfile, level_prefix, "state")); #endif - VisMF::Read( + amrex::VisMF::Read( m_leveldata_new[lev]->gp, amrex::MultiFabFileFullPrefix( lev, m_restart_chkfile, level_prefix, "gradp")); - VisMF::Read( + amrex::VisMF::Read( m_leveldata_new[lev]->press, amrex::MultiFabFileFullPrefix(lev, m_restart_chkfile, level_prefix, "p")); if (m_nAux > 0) { - VisMF::Read( + amrex::VisMF::Read( m_leveldata_new[lev]->auxiliaries, amrex::MultiFabFileFullPrefix( lev, m_restart_chkfile, level_prefix, "aux")); @@ -815,7 +821,7 @@ PeleLM::ReadCheckPointFile() if (m_incompressible == 0) { if (m_has_divu != 0) { - VisMF::Read( + amrex::VisMF::Read( m_leveldata_new[lev]->divu, amrex::MultiFabFileFullPrefix( lev, m_restart_chkfile, level_prefix, "divU")); @@ -824,7 +830,7 @@ PeleLM::ReadCheckPointFile() #ifdef PELE_USE_PLASMA if (!m_restart_nonEF) { if (m_do_react) { - VisMF::Read( + amrex::VisMF::Read( m_leveldatareact[lev]->I_R, amrex::MultiFabFileFullPrefix( lev, m_restart_chkfile, level_prefix, "I_R")); @@ -833,11 +839,11 @@ PeleLM::ReadCheckPointFile() // I_R for non-EF simulation is one component shorted, need to account // for that. if (m_do_react) { - MultiFab I_Rtemp(grids[lev], dmap[lev], NUM_SPECIES, 0); - VisMF::Read( + amrex::MultiFab I_Rtemp(grids[lev], dmap[lev], NUM_SPECIES, 0); + amrex::VisMF::Read( I_Rtemp, amrex::MultiFabFileFullPrefix( lev, m_restart_chkfile, level_prefix, "I_R")); - MultiFab::Copy( + amrex::MultiFab::Copy( m_leveldatareact[lev]->I_R, I_Rtemp, 0, 0, NUM_SPECIES, 0); } @@ -846,7 +852,7 @@ PeleLM::ReadCheckPointFile() } #else if (m_do_react != 0) { - VisMF::Read( + amrex::VisMF::Read( m_leveldatareact[lev]->I_R, amrex::MultiFabFileFullPrefix( lev, m_restart_chkfile, level_prefix, "I_R")); @@ -863,33 +869,35 @@ void PeleLM::initLevelDataFromPlt(int a_lev, const std::string& a_dataPltFile) { if (m_incompressible != 0) { - Abort( + amrex::Abort( " initializing data from a pltfile only available for low-Mach " "simulations"); } if (m_nAux > 0) { - Warning( + amrex::Warning( " restarting from plotfile with auxiliaries not currently " "implemented, and will not be captured"); } - amrex::Print() << " initData on level " << a_lev << " from pltfile " - << a_dataPltFile << "\n"; - if (pltfileSource == "LM") { - amrex::Print() << " Assuming pltfile was generated in LM/LMeX \n"; - } else if (pltfileSource == "C") { - amrex::Print() << " Assuming pltfile was generated in PeleC \n"; + if (m_verbose > 0) { + amrex::Print() << " initData on level " << a_lev << " from pltfile " + << a_dataPltFile << "\n"; + if (pltfileSource == "LM") { + amrex::Print() << " Assuming pltfile was generated in LM/LMeX \n"; + } else if (pltfileSource == "C") { + amrex::Print() << " Assuming pltfile was generated in PeleC \n"; + } } // Use PelePhysics PltFileManager pele::physics::pltfilemanager::PltFileManager pltData(a_dataPltFile); - Vector plt_vars = pltData.getVariableList(); + amrex::Vector plt_vars = pltData.getVariableList(); if (m_do_reset_time == 0) { m_cur_time = pltData.getTime(); m_nstep = pltData.getNsteps(); } // Find required data in pltfile - Vector spec_names; + amrex::Vector spec_names; pele::physics::eos::speciesNames( spec_names, &(eos_parms.host_parm())); int idT = -1, idV = -1, idY = -1, nSpecPlt = 0; @@ -934,12 +942,15 @@ PeleLM::initLevelDataFromPlt(int a_lev, const std::string& a_dataPltFile) #endif } if (idY < 0) { - Abort("Couldn't find species mass fractions in pltfile"); + amrex::Abort("Couldn't find species mass fractions in pltfile"); } else if (idT < 0) { - Abort("Couldn't find temperature in pltfile"); + amrex::Abort("Couldn't find temperature in pltfile"); + } + if (m_verbose > 0) { + amrex::Print() << " " << nSpecPlt + << " species found in pltfile, starting with " + << plt_vars[idY] << "\n"; } - Print() << " " << nSpecPlt << " species found in pltfile, starting with " - << plt_vars[idY] << "\n"; // Get level data auto* ldata_p = getLevelDataPtr(a_lev, AmrNewTime); @@ -954,30 +965,61 @@ PeleLM::initLevelDataFromPlt(int a_lev, const std::string& a_dataPltFile) // Species // Hold the species in temporary MF before copying to level data // in case the number of species differs. - MultiFab speciesPlt(grids[a_lev], dmap[a_lev], nSpecPlt, 0); + amrex::MultiFab speciesPlt(grids[a_lev], dmap[a_lev], nSpecPlt, 0); pltData.fillPatchFromPlt(a_lev, geom[a_lev], idY, 0, nSpecPlt, speciesPlt); for (int i = 0; i < NUM_SPECIES; i++) { - std::string specString = "Y(" + spec_names[i] + ")"; + std::string specName = "Y(" + spec_names[i] + ")"; + std::string specString = specName; + if (!m_initDataPlt_specname_map.empty()) { + specString = m_initDataPlt_specname_map[i]; + } int foundSpec = 0; for (int iplt = 0; iplt < nSpecPlt; iplt++) { if (specString == plt_vars[idY + iplt]) { - MultiFab::Copy(ldata_p->state, speciesPlt, iplt, FIRSTSPEC + i, 1, 0); + amrex::MultiFab::Copy( + ldata_p->state, speciesPlt, iplt, FIRSTSPEC + i, 1, 0); foundSpec = 1; + if (m_verbose > 0) { + amrex::Print() << "Loading species " << specName + << " from plotfile species " << specString + << std::endl; + } + } + } + if (foundSpec == 0) { + for (int iplt = 0; iplt < plt_vars.size(); iplt++) { + if (specString == plt_vars[iplt]) { + foundSpec = 1; + if (m_verbose > 0) { + amrex::Print() << "Loading species " << specName + << " from plotfile entry " << specString + << std::endl; + } + pltData.fillPatchFromPlt( + a_lev, geom[a_lev], iplt, FIRSTSPEC + i, 1, ldata_p->state); + } } } if (foundSpec == 0) { ldata_p->state.setVal(0.0, FIRSTSPEC + i, 1); + if (m_verbose > 0) { + amrex::Print() << "For species " << specName << " entry " << specString + << " not found in plot file, setting to 0 " << std::endl; + } } } // Converting units when pltfile is coming from PeleC solution if (pltfileSource == "C") { - amrex::Print() << " Converting CGS to MKS units... \n"; + if (m_verbose > 0) { + amrex::Print() << " Converting CGS to MKS units... \n"; + } #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); auto const& vel_arr = ldata_p->state.array(mfi, VELX); amrex::ParallelFor( bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { @@ -1005,11 +1047,11 @@ PeleLM::initLevelDataFromPlt(int a_lev, const std::string& a_dataPltFile) amrex::Real* momV = sc.MomOrderV.data(); amrex::Real* momS = sc.MomOrderS.data(); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); - ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); auto const& soot_arr = ldata_p->state.array(mfi, FIRSTSOOT); amrex::ParallelFor( bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { @@ -1049,48 +1091,48 @@ PeleLM::initLevelDataFromPlt(int a_lev, const std::string& a_dataPltFile) // The above handles species mapping (to some extent), but nothing enforce // sum of Ys = 1 -> use N2 in the following if N2 is present #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); auto const& rho_arr = ldata_p->state.array(mfi, DENSITY); auto const& rhoY_arr = ldata_p->state.array(mfi, FIRSTSPEC); auto const& rhoH_arr = ldata_p->state.array(mfi, RHOH); auto const& temp_arr = ldata_p->state.array(mfi, TEMP); - amrex::ParallelFor( - bx, - [=, eosparm = leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - auto eos = pele::physics::PhysicsType::eos(eosparm); - Real massfrac[NUM_SPECIES] = {0.0}; - Real sumYs = 0.0; - for (int n = 0; n < NUM_SPECIES; n++) { - massfrac[n] = rhoY_arr(i, j, k, n); + const auto* eosparm = leosparm; + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + auto eos = pele::physics::PhysicsType::eos(eosparm); + amrex::Real massfrac[NUM_SPECIES] = {0.0}; + amrex::Real sumYs = 0.0; + for (int n = 0; n < NUM_SPECIES; n++) { + massfrac[n] = rhoY_arr(i, j, k, n); #ifdef N2_ID - if (n != N2_ID) { - sumYs += massfrac[n]; - } -#endif + if (n != N2_ID) { + sumYs += massfrac[n]; } +#endif + } #ifdef N2_ID - massfrac[N2_ID] = 1.0 - sumYs; + massfrac[N2_ID] = 1.0 - sumYs; #endif - // Get density - Real P_cgs = lprobparm->P_mean * 10.0; - Real rho_cgs = 0.0; - eos.PYT2R(P_cgs, massfrac, temp_arr(i, j, k), rho_cgs); - rho_arr(i, j, k) = rho_cgs * 1.0e3; + // Get density + amrex::Real P_cgs = lprobparm->P_mean * 10.0; + amrex::Real rho_cgs = 0.0; + eos.PYT2R(P_cgs, massfrac, temp_arr(i, j, k), rho_cgs); + rho_arr(i, j, k) = rho_cgs * 1.0e3; - // Get enthalpy - Real h_cgs = 0.0; - eos.TY2H(temp_arr(i, j, k), massfrac, h_cgs); - rhoH_arr(i, j, k) = h_cgs * 1.0e-4 * rho_arr(i, j, k); + // Get enthalpy + amrex::Real h_cgs = 0.0; + eos.TY2H(temp_arr(i, j, k), massfrac, h_cgs); + rhoH_arr(i, j, k) = h_cgs * 1.0e-4 * rho_arr(i, j, k); - // Fill rhoYs - for (int n = 0; n < NUM_SPECIES; n++) { - rhoY_arr(i, j, k, n) = massfrac[n] * rho_arr(i, j, k); - } - }); + // Fill rhoYs + for (int n = 0; n < NUM_SPECIES; n++) { + rhoY_arr(i, j, k, n) = massfrac[n] * rho_arr(i, j, k); + } + }); } // Initialize thermodynamic pressure @@ -1103,19 +1145,21 @@ PeleLM::initLevelDataFromPlt(int a_lev, const std::string& a_dataPltFile) void PeleLM::addLevelVelocityDataFromPlt(int a_lev, const std::string& a_velPltFile) { - amrex::Print() << " init velocity data on level " << a_lev << " from pltfile " - << a_velPltFile << "\n"; + if (m_verbose > 0) { + amrex::Print() << " init velocity data on level " << a_lev + << " from pltfile " << a_velPltFile << "\n"; + } // Use PelePhysics PltFileManager pele::physics::pltfilemanager::PltFileManager pltData(a_velPltFile); - Vector plt_vars = pltData.getVariableList(); + amrex::Vector plt_vars = pltData.getVariableList(); // do some compatibility checks if (pltData.getNlev() < a_lev) { - Abort("USE_VELOCITY: not enough levels in plotfile"); + amrex::Abort("USE_VELOCITY: not enough levels in plotfile"); } if (pltData.getGeom(a_lev).Domain() != geom[a_lev].Domain()) { - Abort("USE_VELOCITY: problem domains do not match"); + amrex::Abort("USE_VELOCITY: problem domains do not match"); } // find velocity in the plotfile @@ -1126,28 +1170,30 @@ PeleLM::addLevelVelocityDataFromPlt(int a_lev, const std::string& a_velPltFile) } } if (idXvel == -1) { - Abort("Could not find velocity fields in supplied velocity_plotfile"); + amrex::Abort( + "Could not find velocity fields in supplied velocity_plotfile"); } // Get level data auto* ldata_p = getLevelDataPtr(a_lev, AmrNewTime); // load data from plot file - BoxArray tmpVelBA(ldata_p->state.boxArray()); - DistributionMapping tmpVelDM(tmpVelBA); + amrex::BoxArray tmpVelBA(ldata_p->state.boxArray()); + amrex::DistributionMapping tmpVelDM(tmpVelBA); int nGrow0(0), sComp0(0); - MultiFab tmpVel(tmpVelBA, tmpVelDM, AMREX_SPACEDIM, nGrow0); + amrex::MultiFab tmpVel(tmpVelBA, tmpVelDM, AMREX_SPACEDIM, nGrow0); pltData.fillPatchFromPlt( a_lev, geom[a_lev], idXvel, sComp0, AMREX_SPACEDIM, tmpVel); // scale the velocity tmpVel.mult(m_velocity_plotfile_scale); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); - FArrayBox DummyFab(bx, 1); + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); + amrex::FArrayBox DummyFab(bx, 1); auto const& state_arr = ldata_p->state.array(mfi); auto const& tmpVel_arr = tmpVel.array(mfi); amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { @@ -1161,7 +1207,7 @@ PeleLM::addLevelVelocityDataFromPlt(int a_lev, const std::string& a_velPltFile) void PeleLM::WriteJobInfo(const std::string& path) const { - if (ParallelDescriptor::IOProcessor()) { + if (amrex::ParallelDescriptor::IOProcessor()) { // job_info file with details about the run std::ofstream jobInfoFile; std::string FullPathJobInfoFile = path; @@ -1174,8 +1220,8 @@ PeleLM::WriteJobInfo(const std::string& path) const jobInfoFile << " PeleLMeX Job Information\n"; jobInfoFile << PrettyLine; - jobInfoFile << "number of MPI processes: " << ParallelDescriptor::NProcs() - << "\n"; + jobInfoFile << "number of MPI processes: " + << amrex::ParallelDescriptor::NProcs() << "\n"; #ifdef AMREX_USE_OMP jobInfoFile << "number of threads: " << omp_get_max_threads() << "\n"; #endif @@ -1187,24 +1233,26 @@ PeleLM::WriteJobInfo(const std::string& path) const jobInfoFile << " Build Information\n"; jobInfoFile << PrettyLine; - jobInfoFile << "build date: " << buildInfoGetBuildDate() << "\n"; - jobInfoFile << "build machine: " << buildInfoGetBuildMachine() << "\n"; - jobInfoFile << "build dir: " << buildInfoGetBuildDir() << "\n"; - jobInfoFile << "AMReX dir: " << buildInfoGetAMReXDir() << "\n"; + jobInfoFile << "build date: " << amrex::buildInfoGetBuildDate() << "\n"; + jobInfoFile << "build machine: " << amrex::buildInfoGetBuildMachine() + << "\n"; + jobInfoFile << "build dir: " << amrex::buildInfoGetBuildDir() << "\n"; + jobInfoFile << "AMReX dir: " << amrex::buildInfoGetAMReXDir() << "\n"; jobInfoFile << "\n"; - jobInfoFile << "COMP: " << buildInfoGetComp() << "\n"; - jobInfoFile << "COMP version: " << buildInfoGetCompVersion() << "\n"; - jobInfoFile << "C++ compiler: " << buildInfoGetCXXName() << "\n"; - jobInfoFile << "C++ flags: " << buildInfoGetCXXFlags() << "\n"; + jobInfoFile << "COMP: " << amrex::buildInfoGetComp() << "\n"; + jobInfoFile << "COMP version: " << amrex::buildInfoGetCompVersion() + << "\n"; + jobInfoFile << "C++ compiler: " << amrex::buildInfoGetCXXName() << "\n"; + jobInfoFile << "C++ flags: " << amrex::buildInfoGetCXXFlags() << "\n"; jobInfoFile << "\n"; - const char* githash1 = buildInfoGetGitHash(1); - const char* githash2 = buildInfoGetGitHash(2); - const char* githash3 = buildInfoGetGitHash(3); - const char* githash4 = buildInfoGetGitHash(4); + const char* githash1 = amrex::buildInfoGetGitHash(1); + const char* githash2 = amrex::buildInfoGetGitHash(2); + const char* githash3 = amrex::buildInfoGetGitHash(3); + const char* githash4 = amrex::buildInfoGetGitHash(4); if (strlen(githash1) > 0) { jobInfoFile << "PeleLMeX git describe: " << githash1 << "\n"; @@ -1243,7 +1291,7 @@ PeleLM::WriteJobInfo(const std::string& path) const jobInfoFile << " Inputs File Parameters\n"; jobInfoFile << PrettyLine; - ParmParse::dumpTable(jobInfoFile, true); + amrex::ParmParse::dumpTable(jobInfoFile, true); jobInfoFile.close(); } diff --git a/Source/PeleLMeX_ProblemSpecificFunctions.H b/Source/PeleLMeX_ProblemSpecificFunctions.H index cc64d7bc9..176dd1e9e 100644 --- a/Source/PeleLMeX_ProblemSpecificFunctions.H +++ b/Source/PeleLMeX_ProblemSpecificFunctions.H @@ -221,12 +221,12 @@ struct DefaultProblemSpecificFunctions * *ext_src, * [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { * for (int n = 0; n < NUM_ODE; n++) { - * Real B_n = state_old_arr[box_no](i, j, k, FIRSTODE + n); - * Real src = -1.0 * pow(10.0, n + 1) * B_n; + * amrex::Real B_n = state_old_arr[box_no](i, j, k, FIRSTODE + n); + * amrex::Real src = -1.0 * pow(10.0, n + 1) * B_n; * ext_src_arr[box_no](i, j, k, FIRSTODE + n) += src; * } * }); - * Gpu::streamSynchronize(); + * amrex::Gpu::streamSynchronize(); */ } diff --git a/Source/PeleLMeX_Projection.cpp b/Source/PeleLMeX_Projection.cpp index 04599206c..1f76627f3 100644 --- a/Source/PeleLMeX_Projection.cpp +++ b/Source/PeleLMeX_Projection.cpp @@ -1,15 +1,13 @@ #include #include -using namespace amrex; - void PeleLM::initialProjection() { BL_PROFILE("PeleLMeX::initialProjection()"); if (m_verbose != 0) { - Vector velMax(AMREX_SPACEDIM); + amrex::Vector velMax(AMREX_SPACEDIM); velMax = MLNorm0( GetVecOfConstPtrs(getVelocityVect(AmrNewTime)), 0, AMREX_SPACEDIM); amrex::Print() << " Initial velocity projection: "; @@ -18,30 +16,30 @@ PeleLM::initialProjection() " W: " << velMax[2] <<) "\n"; } - Real dummy_dt = 1.0; + amrex::Real dummy_dt = 1.0; int incremental = 0; int nGhost = 0; // Get sigma : density if not incompressible - Vector> sigma(finest_level + 1); + amrex::Vector> sigma(finest_level + 1); if (m_incompressible == 0) { for (int lev = 0; lev <= finest_level; ++lev) { - sigma[lev] = std::make_unique( - grids[lev], dmap[lev], 1, nGhost, MFInfo(), *m_factory[lev]); + sigma[lev] = std::make_unique( + grids[lev], dmap[lev], 1, nGhost, amrex::MFInfo(), *m_factory[lev]); auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - Box const& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + amrex::Box const& bx = mfi.tilebox(); auto const& rho_arr = ldata_p->state.const_array(mfi, DENSITY); auto const& sig_arr = sigma[lev]->array(mfi); amrex::ParallelFor( - bx, [rho_arr, sig_arr, - dummy_dt] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { sig_arr(i, j, k) = dummy_dt / rho_arr(i, j, k); }); } @@ -50,10 +48,10 @@ PeleLM::initialProjection() } // Get velocity - Vector> vel; + amrex::Vector> vel; for (int lev = 0; lev <= finest_level; ++lev) { vel.push_back( - std::make_unique( + std::make_unique( m_leveldata_new[lev]->state, amrex::make_alias, VELX, AMREX_SPACEDIM)); vel[lev]->setBndry(0.0); setInflowBoundaryVel(*vel[lev], lev, AmrNewTime); @@ -61,8 +59,8 @@ PeleLM::initialProjection() } // Get RHS cc: - divU (- \int{divU}) - Real Sbar = 0.0; - Vector rhs_cc(finest_level + 1); + amrex::Real Sbar = 0.0; + amrex::Vector rhs_cc(finest_level + 1); if ((m_incompressible == 0) && (m_has_divu != 0)) { // Ensure integral of RHS is zero for closed chamber if (m_closed_chamber != 0) { @@ -72,7 +70,7 @@ PeleLM::initialProjection() for (int lev = 0; lev <= finest_level; ++lev) { rhs_cc[lev].define( grids[lev], dmap[lev], 1, m_leveldata_new[lev]->divu.nGrow()); - MultiFab::Copy( + amrex::MultiFab::Copy( rhs_cc[lev], m_leveldata_new[lev]->divu, 0, 0, 1, m_leveldata_new[lev]->divu.nGrow()); if (m_closed_chamber != 0) { @@ -111,7 +109,7 @@ PeleLM::initialProjection() } if (m_verbose != 0) { - Vector velMax(AMREX_SPACEDIM); + amrex::Vector velMax(AMREX_SPACEDIM); velMax = MLNorm0( GetVecOfConstPtrs(getVelocityVect(AmrNewTime)), 0, AMREX_SPACEDIM); amrex::Print() << " >> After initial velocity projection: "; @@ -130,30 +128,30 @@ PeleLM::initialPressProjection() amrex::Print() << " Initial pressure projection \n"; } - Real dummy_dt = 1.0; + amrex::Real dummy_dt = 1.0; int incremental = 0; int nGhost = 1; // Get sigma : density if not incompressible - Vector> sigma(finest_level + 1); + amrex::Vector> sigma(finest_level + 1); if (m_incompressible == 0) { for (int lev = 0; lev <= finest_level; ++lev) { - sigma[lev] = std::make_unique( - grids[lev], dmap[lev], 1, nGhost, MFInfo(), *m_factory[lev]); + sigma[lev] = std::make_unique( + grids[lev], dmap[lev], 1, nGhost, amrex::MFInfo(), *m_factory[lev]); auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - Box const& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + amrex::Box const& bx = mfi.tilebox(); auto const& rho_arr = ldata_p->state.const_array(mfi, DENSITY); auto const& sig_arr = sigma[lev]->array(mfi); amrex::ParallelFor( - bx, [rho_arr, sig_arr, - dummy_dt] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { sig_arr(i, j, k) = dummy_dt / rho_arr(i, j, k); }); } @@ -162,10 +160,11 @@ PeleLM::initialPressProjection() } // Set the velocity to the gravity field - Vector vel(finest_level + 1); + amrex::Vector vel(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { vel[lev].define( - grids[lev], dmap[lev], AMREX_SPACEDIM, nGhost, MFInfo(), *m_factory[lev]); + grids[lev], dmap[lev], AMREX_SPACEDIM, nGhost, amrex::MFInfo(), + *m_factory[lev]); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { vel[lev].setVal(m_gravity[idim], idim, 1, 1); } @@ -181,7 +180,7 @@ PeleLM::initialPressProjection() void PeleLM::velocityProjection( - int is_initIter, const TimeStamp& a_rhoTime, const Real& a_dt) + int is_initIter, const TimeStamp& a_rhoTime, const amrex::Real& a_dt) { BL_PROFILE("PeleLMeX::velocityProjection()"); @@ -189,25 +188,25 @@ PeleLM::velocityProjection( int incremental = (is_initIter) != 0 ? 1 : 0; // Get sigma : scaled density inv. if not incompressible - Vector> sigma(finest_level + 1); + amrex::Vector> sigma(finest_level + 1); if (m_incompressible == 0) { - Vector> rhoHalf(finest_level + 1); + amrex::Vector> rhoHalf(finest_level + 1); rhoHalf = getDensityVect(a_rhoTime); for (int lev = 0; lev <= finest_level; ++lev) { - sigma[lev] = std::make_unique( - grids[lev], dmap[lev], 1, nGhost, MFInfo(), *m_factory[lev]); + sigma[lev] = std::make_unique( + grids[lev], dmap[lev], 1, nGhost, amrex::MFInfo(), *m_factory[lev]); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*rhoHalf[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) { - Box const& bx = mfi.tilebox(); + for (amrex::MFIter mfi(*rhoHalf[lev], amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + amrex::Box const& bx = mfi.tilebox(); auto const& rho_arr = rhoHalf[lev]->const_array(mfi); auto const& sig_arr = sigma[lev]->array(mfi); amrex::ParallelFor( - bx, [rho_arr, sig_arr, - a_dt] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { sig_arr(i, j, k) = a_dt / rho_arr(i, j, k); }); } @@ -219,7 +218,7 @@ PeleLM::velocityProjection( } if (incremental == 0) { - Vector> rhoHalf(finest_level + 1); + amrex::Vector> rhoHalf(finest_level + 1); if (m_incompressible == 0) { rhoHalf = getDensityVect(a_rhoTime); } @@ -229,21 +228,21 @@ PeleLM::velocityProjection( auto* ldataNew_p = getLevelDataPtr(lev, AmrNewTime); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldataNew_p->state, TilingIfNotGPU()); mfi.isValid(); - ++mfi) { - Box const& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldataNew_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + amrex::Box const& bx = mfi.tilebox(); auto const& vel_arr = ldataNew_p->state.array(mfi, VELX); auto const& gp_arr = ldataOld_p->gp.const_array(mfi); auto const& rho_arr = (m_incompressible) != 0 - ? Array4() + ? amrex::Array4() : rhoHalf[lev]->const_array(mfi); + const auto incompressible = m_incompressible; + const auto rho = m_rho; amrex::ParallelFor( - bx, - [vel_arr, gp_arr, rho_arr, a_dt, incompressible = m_incompressible, - rho = m_rho] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - Real soverrho = + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + amrex::Real soverrho = (incompressible) != 0 ? a_dt / rho : a_dt / rho_arr(i, j, k); AMREX_D_TERM(vel_arr(i, j, k, 0) += gp_arr(i, j, k, 0) * soverrho; , vel_arr(i, j, k, 1) += gp_arr(i, j, k, 1) * soverrho; @@ -260,16 +259,16 @@ PeleLM::velocityProjection( for (int lev = 0; lev <= finest_level; ++lev) { auto* ldataOld_p = getLevelDataPtr(lev, AmrOldTime); auto* ldataNew_p = getLevelDataPtr(lev, AmrNewTime); - MultiFab::Subtract( + amrex::MultiFab::Subtract( ldataNew_p->state, ldataOld_p->state, VELX, VELX, AMREX_SPACEDIM, 0); } } // Get velocity - Vector> vel; + amrex::Vector> vel; for (int lev = 0; lev <= finest_level; ++lev) { vel.push_back( - std::make_unique( + std::make_unique( m_leveldata_new[lev]->state, amrex::make_alias, VELX, AMREX_SPACEDIM)); #ifdef AMREX_USE_EB EB_set_covered(*vel[lev], 0.0); @@ -282,8 +281,8 @@ PeleLM::velocityProjection( } // To ensure integral of RHS is zero for closed chamber, get mean divU - Real SbarOld = 0.0; - Real SbarNew = 0.0; + amrex::Real SbarOld = 0.0; + amrex::Real SbarNew = 0.0; if ((m_closed_chamber != 0) && (m_incompressible == 0)) { SbarNew = MFSum(GetVecOfConstPtrs(getDivUVect(AmrNewTime)), 0); SbarNew /= m_uncoveredVol; // Transform in Mean. @@ -294,16 +293,16 @@ PeleLM::velocityProjection( } // Get RHS cc - Vector rhs_cc; + amrex::Vector rhs_cc; if ((m_incompressible == 0) && (m_has_divu != 0)) { rhs_cc.resize(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { if (incremental == 0) { auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); rhs_cc[lev].define( - grids[lev], dmap[lev], 1, ldata_p->divu.nGrow(), MFInfo(), + grids[lev], dmap[lev], 1, ldata_p->divu.nGrow(), amrex::MFInfo(), *m_factory[lev]); - MultiFab::Copy( + amrex::MultiFab::Copy( rhs_cc[lev], ldata_p->divu, 0, 0, 1, ldata_p->divu.nGrow()); if (m_closed_chamber != 0) { rhs_cc[lev].plus(-SbarNew, 0, 1); @@ -313,21 +312,20 @@ PeleLM::velocityProjection( auto* ldataOld_p = getLevelDataPtr(lev, AmrOldTime); auto* ldataNew_p = getLevelDataPtr(lev, AmrNewTime); rhs_cc[lev].define( - grids[lev], dmap[lev], 1, ldataOld_p->divu.nGrow(), MFInfo(), + grids[lev], dmap[lev], 1, ldataOld_p->divu.nGrow(), amrex::MFInfo(), *m_factory[lev]); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(rhs_cc[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& gbx = mfi.growntilebox(); + for (amrex::MFIter mfi(rhs_cc[lev], amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& gbx = mfi.growntilebox(); const auto& divu_o = ldataOld_p->divu.const_array(mfi); const auto& divu_n = ldataNew_p->divu.const_array(mfi); const auto& rhs = rhs_cc[lev].array(mfi); + const auto is_closed_ch = m_closed_chamber; amrex::ParallelFor( - gbx, - [divu_o, divu_n, rhs, SbarNew, SbarOld, - is_closed_ch = - m_closed_chamber] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { rhs(i, j, k) = -(divu_n(i, j, k) - divu_o(i, j, k)); if (is_closed_ch != 0) { rhs(i, j, k) += @@ -356,7 +354,7 @@ PeleLM::velocityProjection( auto* ldataNew_p = getLevelDataPtr(lev, AmrNewTime); unscaleProj_RZ( lev, *vel[lev]); // Unscaling New vel before adding back old one - MultiFab::Add( + amrex::MultiFab::Add( ldataNew_p->state, ldataOld_p->state, VELX, VELX, AMREX_SPACEDIM, 0); } } else { @@ -375,12 +373,12 @@ PeleLM::velocityProjection( void PeleLM::doNodalProject( - const Vector& a_vel, - const Vector& a_sigma, - const Vector& rhs_cc, - const Vector& rhs_nd, + const amrex::Vector& a_vel, + const amrex::Vector& a_sigma, + const amrex::Vector& rhs_cc, + const amrex::Vector& rhs_nd, int incremental, - Real scaling_factor) + amrex::Real scaling_factor) { // Asserts AMREX_ASSERT(a_vel.size() == a_sigma.size()); @@ -388,29 +386,29 @@ PeleLM::doNodalProject( AMREX_ASSERT(rhs_nd.empty() || (a_vel.size() == rhs_nd.size())); AMREX_ASSERT(a_vel[0]->nComp() == AMREX_SPACEDIM); - LPInfo info; + amrex::LPInfo info; info.setMaxCoarseningLevel(m_nodal_mg_max_coarsening_level); // BCs - std::array lobc; - std::array hibc; + std::array lobc; + std::array hibc; for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { if (Geom(0).isPeriodic(idim)) { - lobc[idim] = hibc[idim] = LinOpBCType::Periodic; + lobc[idim] = hibc[idim] = amrex::LinOpBCType::Periodic; } else { if (m_phys_bc.lo(idim) == amrex::PhysBCType::outflow) { - lobc[idim] = LinOpBCType::Dirichlet; + lobc[idim] = amrex::LinOpBCType::Dirichlet; } else if (m_phys_bc.lo(idim) == amrex::PhysBCType::inflow) { - lobc[idim] = LinOpBCType::inflow; + lobc[idim] = amrex::LinOpBCType::inflow; } else { - lobc[idim] = LinOpBCType::Neumann; + lobc[idim] = amrex::LinOpBCType::Neumann; } if (m_phys_bc.hi(idim) == amrex::PhysBCType::outflow) { - hibc[idim] = LinOpBCType::Dirichlet; + hibc[idim] = amrex::LinOpBCType::Dirichlet; } else if (m_phys_bc.hi(idim) == amrex::PhysBCType::inflow) { - hibc[idim] = LinOpBCType::inflow; + hibc[idim] = amrex::LinOpBCType::inflow; } else { - hibc[idim] = LinOpBCType::Neumann; + hibc[idim] = amrex::LinOpBCType::Neumann; } } } @@ -419,7 +417,7 @@ PeleLM::doNodalProject( std::unique_ptr nodal_projector; if (m_incompressible != 0) { - Real constant_sigma = scaling_factor / m_rho; + amrex::Real constant_sigma = scaling_factor / m_rho; nodal_projector = std::make_unique( a_vel, constant_sigma, Geom(0, finest_level), info); } else { @@ -459,11 +457,12 @@ PeleLM::doNodalProject( auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->gp, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - Box const& tbx = mfi.tilebox(); - Box const& nbx = mfi.nodaltilebox(); + for (amrex::MFIter mfi(ldata_p->gp, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + amrex::Box const& tbx = mfi.tilebox(); + amrex::Box const& nbx = mfi.nodaltilebox(); auto const& p_lev_arr = ldata_p->press.array(mfi); auto const& gp_lev_arr = ldata_p->gp.array(mfi); auto const& p_proj_arr = phi[lev]->const_array(mfi); @@ -471,25 +470,21 @@ PeleLM::doNodalProject( if (incremental != 0) { amrex::ParallelFor( tbx, AMREX_SPACEDIM, - [gp_lev_arr, - gp_proj_arr] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { gp_lev_arr(i, j, k, n) += gp_proj_arr(i, j, k, n); }); amrex::ParallelFor( - nbx, [p_lev_arr, - p_proj_arr] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + nbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { p_lev_arr(i, j, k) += p_proj_arr(i, j, k); }); } else { amrex::ParallelFor( tbx, AMREX_SPACEDIM, - [gp_lev_arr, - gp_proj_arr] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { gp_lev_arr(i, j, k, n) = gp_proj_arr(i, j, k, n); }); amrex::ParallelFor( - nbx, [p_lev_arr, - p_proj_arr] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + nbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { p_lev_arr(i, j, k) = p_proj_arr(i, j, k); }); } @@ -513,31 +508,31 @@ PeleLM::doNodalProject( void PeleLM::scaleProj_RZ( // NOLINT(readability-convert-member-functions-to-static) int a_lev, - MultiFab& a_mf) + amrex::MultiFab& a_mf) { #if AMREX_SPACEDIM == 2 // Scale nodal projection cell-centered mfs by radius if (geom[a_lev].IsRZ()) { - Box domain = geom[a_lev].Domain(); + amrex::Box domain = geom[a_lev].Domain(); auto BCRecVel = fetchBCRecArray(VELX, 1); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (BCRecVel[0].lo(idim) == BCType::ext_dir) { + if (BCRecVel[0].lo(idim) == amrex::BCType::ext_dir) { domain.growLo(idim, 1); } - if (BCRecVel[0].hi(idim) == BCType::ext_dir) { + if (BCRecVel[0].hi(idim) == amrex::BCType::ext_dir) { domain.growHi(idim, 1); } } - const Real dr = geom[a_lev].CellSize()[0]; + const amrex::Real dr = geom[a_lev].CellSize()[0]; auto const& mf_ma = a_mf.arrays(); + const auto ncomp = a_mf.nComp(); amrex::ParallelFor( a_mf, a_mf.nGrowVect(), - [=, ncomp = a_mf.nComp()] AMREX_GPU_DEVICE( - int box_no, int i, int j, int k) noexcept { + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { auto mf = mf_ma[box_no]; if (domain.contains(i, j, k)) { for (int n = 0; n < ncomp; ++n) { - mf(i, j, k, n) *= (static_cast(i) + 0.5) * dr; + mf(i, j, k, n) *= (static_cast(i) + 0.5) * dr; } } else { for (int n = 0; n < ncomp; ++n) { @@ -545,7 +540,7 @@ PeleLM::scaleProj_RZ( // NOLINT(readability-convert-member-functions-to-static) } } }); - Gpu::streamSynchronize(); + amrex::Gpu::streamSynchronize(); } #else amrex::ignore_unused(a_lev, a_mf); @@ -556,22 +551,22 @@ void PeleLM:: unscaleProj_RZ( // NOLINT(readability-convert-member-functions-to-static) int a_lev, - MultiFab& a_mf) + amrex::MultiFab& a_mf) { #if AMREX_SPACEDIM == 2 // Unscale nodal projection cell-centered mfs by radius if (geom[a_lev].IsRZ()) { - const Box& domain = geom[a_lev].Domain(); - const Real dr = geom[a_lev].CellSize()[0]; + const amrex::Box& domain = geom[a_lev].Domain(); + const amrex::Real dr = geom[a_lev].CellSize()[0]; auto const& mf_ma = a_mf.arrays(); + const auto ncomp = a_mf.nComp(); amrex::ParallelFor( a_mf, a_mf.nGrowVect(), - [=, ncomp = a_mf.nComp()] AMREX_GPU_DEVICE( - int box_no, int i, int j, int k) noexcept { + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { auto mf = mf_ma[box_no]; if (domain.contains(i, j, k)) { for (int n = 0; n < ncomp; ++n) { - mf(i, j, k, n) /= (static_cast(i) + 0.5) * dr; + mf(i, j, k, n) /= (static_cast(i) + 0.5) * dr; } } else { for (int n = 0; n < ncomp; ++n) { diff --git a/Source/PeleLMeX_Radiation.cpp b/Source/PeleLMeX_Radiation.cpp index 50f449dab..cef12e10d 100644 --- a/Source/PeleLMeX_Radiation.cpp +++ b/Source/PeleLMeX_Radiation.cpp @@ -45,7 +45,7 @@ PeleLM::computeRadSource(const PeleLM::TimeStamp& a_timestamp) for (int lev = 0; lev <= finest_level; lev++) { auto* ldata_p = PeleLM::getLevelDataPtr(lev, a_timestamp); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif for (amrex::MFIter mfi(*(m_extSource[lev]), amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { @@ -74,7 +74,7 @@ PeleLM::computeRadSource(const PeleLM::TimeStamp& a_timestamp) for (int lev = 0; lev <= finest_level; lev++) { #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif for (amrex::MFIter mfi(*(m_extSource[lev]), amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { diff --git a/Source/PeleLMeX_Reactions.cpp b/Source/PeleLMeX_Reactions.cpp index f0998767b..e1e93c852 100644 --- a/Source/PeleLMeX_Reactions.cpp +++ b/Source/PeleLMeX_Reactions.cpp @@ -4,8 +4,6 @@ #include #endif -using namespace amrex; - void PeleLM::advanceChemistry(std::unique_ptr& advData) { @@ -29,7 +27,8 @@ PeleLM::advanceChemistry(std::unique_ptr& advData) // This advanceChemistry is called on the finest level // It works with the AmrCore BoxArray and do not involve ParallelCopy void -PeleLM::advanceChemistry(int lev, const Real& a_dt, MultiFab& a_extForcing) +PeleLM::advanceChemistry( + int lev, const amrex::Real& a_dt, amrex::MultiFab& a_extForcing) { BL_PROFILE("PeleLMeX::advanceChemistry_Lev" + std::to_string(lev) + "()"); @@ -38,22 +37,22 @@ PeleLM::advanceChemistry(int lev, const Real& a_dt, MultiFab& a_extForcing) auto* ldataR_p = getLevelDataReactPtr(lev); // Setup EB-covered cells mask - iMultiFab mask(grids[lev], dmap[lev], 1, 0); + amrex::iMultiFab mask(grids[lev], dmap[lev], 1, 0); #ifdef AMREX_USE_EB getCoveredIMask(lev, mask); #else mask.setVal(1); #endif - MFItInfo mfi_info; - if (Gpu::notInLaunchRegion()) { + amrex::MFItInfo mfi_info; + if (amrex::Gpu::notInLaunchRegion()) { mfi_info.EnableTiling().SetDynamic(true); } #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldataNew_p->state, mfi_info); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldataNew_p->state, mfi_info); mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); auto const& rhoY_o = ldataOld_p->state.const_array(mfi, FIRSTSPEC); auto const& rhoH_o = ldataOld_p->state.const_array(mfi, RHOH); auto const& temp_o = ldataOld_p->state.const_array(mfi, TEMP); @@ -66,17 +65,15 @@ PeleLM::advanceChemistry(int lev, const Real& a_dt, MultiFab& a_extForcing) auto const& mask_arr = mask.array(mfi); // Reset new to old and convert MKS -> CGS - ParallelFor( - bx, [rhoY_o, rhoH_o, temp_o, rhoY_n, rhoH_n, temp_n, extF_rhoY, - extF_rhoH] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - for (int n = 0; n < NUM_SPECIES; n++) { - rhoY_n(i, j, k, n) = rhoY_o(i, j, k, n) * 1.0e-3; - extF_rhoY(i, j, k, n) *= 1.0e-3; - } - temp_n(i, j, k) = temp_o(i, j, k); - rhoH_n(i, j, k) = rhoH_o(i, j, k) * 10.0; - extF_rhoH(i, j, k) *= 10.0; - }); + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + for (int n = 0; n < NUM_SPECIES; n++) { + rhoY_n(i, j, k, n) = rhoY_o(i, j, k, n) * 1.0e-3; + extF_rhoY(i, j, k, n) *= 1.0e-3; + } + temp_n(i, j, k) = temp_o(i, j, k); + rhoH_n(i, j, k) = rhoH_o(i, j, k) * 10.0; + extF_rhoH(i, j, k) *= 10.0; + }); #ifdef PELE_USE_PLASMA // Pass nE -> rhoY_e & FnE -> FrhoY_e @@ -85,18 +82,16 @@ PeleLM::advanceChemistry(int lev, const Real& a_dt, MultiFab& a_extForcing) auto const& rhoYe_n = ldataNew_p->state.array(mfi, FIRSTSPEC + E_ID); auto const& FrhoYe = a_extForcing.array(mfi, E_ID); auto eos = pele::physics::PhysicsType::eos(&eos_parms.host_parm()); - Real mwt[NUM_SPECIES] = {0.0}; + amrex::Real mwt[NUM_SPECIES] = {0.0}; eos.molecular_weight(mwt); - ParallelFor( - bx, [mwt, nE_o, FnE, rhoYe_n, - FrhoYe] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - rhoYe_n(i, j, k) = nE_o(i, j, k) / Na * mwt[E_ID] * 1.0e-6; - FrhoYe(i, j, k) = FnE(i, j, k) / Na * mwt[E_ID] * 1.0e-6; - }); + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + rhoYe_n(i, j, k) = nE_o(i, j, k) / Na * mwt[E_ID] * 1.0e-6; + FrhoYe(i, j, k) = FnE(i, j, k) / Na * mwt[E_ID] * 1.0e-6; + }); #endif - Real dt_incr = a_dt; - Real time_chem = 0; + amrex::Real dt_incr = a_dt; + amrex::Real time_chem = 0; /* Solve */ m_reactor->react( bx, rhoY_n, extF_rhoY, temp_n, rhoH_n, extF_rhoH, fcl, mask_arr, dt_incr, @@ -108,52 +103,47 @@ PeleLM::advanceChemistry(int lev, const Real& a_dt, MultiFab& a_extForcing) ); // Convert CGS -> MKS - ParallelFor( - bx, [rhoY_n, rhoH_n, extF_rhoY, - extF_rhoH] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - for (int n = 0; n < NUM_SPECIES; n++) { - rhoY_n(i, j, k, n) *= 1.0e3; - extF_rhoY(i, j, k, n) *= 1.0e3; - } - rhoH_n(i, j, k) *= 0.1; - extF_rhoH(i, j, k) *= 0.1; - }); + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + for (int n = 0; n < NUM_SPECIES; n++) { + rhoY_n(i, j, k, n) *= 1.0e3; + extF_rhoY(i, j, k, n) *= 1.0e3; + } + rhoH_n(i, j, k) *= 0.1; + extF_rhoH(i, j, k) *= 0.1; + }); #ifdef PELE_USE_PLASMA // rhoY_e -> nE and set rhoY_e to zero auto const& nE_n = ldataNew_p->state.array(mfi, NE); - Real invmwt[NUM_SPECIES] = {0.0}; + amrex::Real invmwt[NUM_SPECIES] = {0.0}; eos.inv_molecular_weight(invmwt); - ParallelFor( - bx, [invmwt, nE_n, rhoYe_n, - extF_rhoY] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - nE_n(i, j, k) = rhoYe_n(i, j, k) * Na * invmwt[E_ID] * 1.0e3; - rhoYe_n(i, j, k) = 0.0; - extF_rhoY(i, j, k, E_ID) = 0.0; - }); + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + nE_n(i, j, k) = rhoYe_n(i, j, k) * Na * invmwt[E_ID] * 1.0e3; + rhoYe_n(i, j, k) = 0.0; + extF_rhoY(i, j, k, E_ID) = 0.0; + }); #endif #ifdef AMREX_USE_GPU - Gpu::Device::streamSynchronize(); + amrex::Gpu::Device::streamSynchronize(); #endif } // Set reaction term #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldataNew_p->state, amrex::TilingIfNotGPU()); mfi.isValid(); - ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldataNew_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); auto const& rhoY_o = ldataOld_p->state.const_array(mfi, FIRSTSPEC); auto const& rhoY_n = ldataNew_p->state.const_array(mfi, FIRSTSPEC); auto const& extF_rhoY = a_extForcing.const_array(mfi, 0); auto const& rhoYdot = ldataR_p->I_R.array(mfi, 0); - Real dt_inv = 1.0 / a_dt; - ParallelFor( + amrex::Real dt_inv = 1.0 / a_dt; + amrex::ParallelFor( bx, NUM_SPECIES, - [rhoY_o, rhoY_n, extF_rhoY, rhoYdot, - dt_inv] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { rhoYdot(i, j, k, n) = -(rhoY_o(i, j, k, n) - rhoY_n(i, j, k, n)) * dt_inv - extF_rhoY(i, j, k, n); @@ -164,12 +154,9 @@ PeleLM::advanceChemistry(int lev, const Real& a_dt, MultiFab& a_extForcing) auto const& nE_n = ldataNew_p->state.const_array(mfi, NE); auto const& FnE = a_extForcing.const_array(mfi, NUM_SPECIES + 1); auto const& nEdot = ldataR_p->I_R.array(mfi, NUM_SPECIES); - ParallelFor( - bx, [nE_o, nE_n, FnE, nEdot, - dt_inv] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - nEdot(i, j, k) = - -(nE_o(i, j, k) - nE_n(i, j, k)) * dt_inv - FnE(i, j, k); - }); + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + nEdot(i, j, k) = -(nE_o(i, j, k) - nE_n(i, j, k)) * dt_inv - FnE(i, j, k); + }); #endif } } @@ -179,7 +166,7 @@ PeleLM::advanceChemistry(int lev, const Real& a_dt, MultiFab& a_extForcing) // on uncovered boxes. void PeleLM::advanceChemistryBAChem( - int lev, const Real& a_dt, MultiFab& a_extForcing) + int lev, const amrex::Real& a_dt, amrex::MultiFab& a_extForcing) { BL_PROFILE("PeleLMeX::advanceChemistry_Lev" + std::to_string(lev) + "()"); @@ -188,15 +175,17 @@ PeleLM::advanceChemistryBAChem( auto* ldataR_p = getLevelDataReactPtr(lev); // Set chemistry MFs based on baChem and dmapChem - MultiFab chemState(*m_baChem[lev], *m_dmapChem[lev], NUM_SPECIES + 3, 0); - MultiFab chemForcing(*m_baChem[lev], *m_dmapChem[lev], nCompForcing(), 0); - MultiFab functC(*m_baChem[lev], *m_dmapChem[lev], 1, 0); + amrex::MultiFab chemState( + *m_baChem[lev], *m_dmapChem[lev], NUM_SPECIES + 3, 0); + amrex::MultiFab chemForcing( + *m_baChem[lev], *m_dmapChem[lev], nCompForcing(), 0); + amrex::MultiFab functC(*m_baChem[lev], *m_dmapChem[lev], 1, 0); #ifdef PELE_USE_PLASMA - MultiFab chemnE(*m_baChem[lev], *m_dmapChem[lev], 1, 0); + amrex::MultiFab chemnE(*m_baChem[lev], *m_dmapChem[lev], 1, 0); #endif // Setup EB covered cells mask - iMultiFab mask(*m_baChem[lev], *m_dmapChem[lev], 1, 0); + amrex::iMultiFab mask(*m_baChem[lev], *m_dmapChem[lev], 1, 0); #ifdef AMREX_USE_EB getCoveredIMask(lev, mask); #else @@ -210,15 +199,15 @@ PeleLM::advanceChemistryBAChem( chemnE.ParallelCopy(ldataOld_p->state, NE, 0, 1); #endif - MFItInfo mfi_info; - if (Gpu::notInLaunchRegion()) { + amrex::MFItInfo mfi_info; + if (amrex::Gpu::notInLaunchRegion()) { mfi_info.EnableTiling().SetDynamic(true); } #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(chemState, mfi_info); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(chemState, mfi_info); mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); auto const& rhoY_o = chemState.array(mfi, 0); auto const& rhoH_o = chemState.array(mfi, NUM_SPECIES); auto const& temp_o = chemState.array(mfi, NUM_SPECIES + 1); @@ -228,16 +217,14 @@ PeleLM::advanceChemistryBAChem( auto const& mask_arr = mask.array(mfi); // Convert MKS -> CGS - ParallelFor( - bx, [rhoY_o, rhoH_o, extF_rhoY, - extF_rhoH] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - for (int n = 0; n < NUM_SPECIES; n++) { - rhoY_o(i, j, k, n) *= 1.0e-3; - extF_rhoY(i, j, k, n) *= 1.0e-3; - } - rhoH_o(i, j, k) *= 10.0; - extF_rhoH(i, j, k) *= 10.0; - }); + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + for (int n = 0; n < NUM_SPECIES; n++) { + rhoY_o(i, j, k, n) *= 1.0e-3; + extF_rhoY(i, j, k, n) *= 1.0e-3; + } + rhoH_o(i, j, k) *= 10.0; + extF_rhoH(i, j, k) *= 10.0; + }); #ifdef PELE_USE_PLASMA // Pass nE -> rhoY_e & FnE -> FrhoY_e @@ -246,14 +233,12 @@ PeleLM::advanceChemistryBAChem( auto const& rhoYe_o = chemState.array(mfi, E_ID); auto const& FrhoYe = chemForcing.array(mfi, E_ID); auto eos = pele::physics::PhysicsType::eos(&eos_parms.host_parm()); - Real mwt[NUM_SPECIES] = {0.0}; + amrex::Real mwt[NUM_SPECIES] = {0.0}; eos.molecular_weight(mwt); - ParallelFor( - bx, [mwt, nE_o, FnE, rhoYe_o, - FrhoYe] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - rhoYe_o(i, j, k) = nE_o(i, j, k) / Na * mwt[E_ID] * 1.0e-6; - FrhoYe(i, j, k) = FnE(i, j, k) / Na * mwt[E_ID] * 1.0e-6; - }); + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + rhoYe_o(i, j, k) = nE_o(i, j, k) / Na * mwt[E_ID] * 1.0e-6; + FrhoYe(i, j, k) = FnE(i, j, k) / Na * mwt[E_ID] * 1.0e-6; + }); #endif // Do reaction only on uncovered box @@ -261,8 +246,8 @@ PeleLM::advanceChemistryBAChem( if (do_reactionBox != 0) { // Do reaction as usual using PelePhysics chemistry integrator - Real dt_incr = a_dt; - Real time_chem = 0; + amrex::Real dt_incr = a_dt; + amrex::Real time_chem = 0; /* Solve */ m_reactor->react( bx, rhoY_o, extF_rhoY, temp_o, rhoH_o, extF_rhoH, fcl, mask_arr, @@ -274,54 +259,52 @@ PeleLM::advanceChemistryBAChem( ); } else { // Just set the function call to 0.0 - ParallelFor(bx, [fcl] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - fcl(i, j, k) = 0.0; - }); + amrex::ParallelFor( + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + fcl(i, j, k) = 0.0; + }); } // Convert CGS -> MKS - ParallelFor( - bx, [rhoY_o, rhoH_o] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - for (int n = 0; n < NUM_SPECIES; n++) { - rhoY_o(i, j, k, n) *= 1.0e3; - } - rhoH_o(i, j, k) *= 0.1; - }); + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + for (int n = 0; n < NUM_SPECIES; n++) { + rhoY_o(i, j, k, n) *= 1.0e3; + } + rhoH_o(i, j, k) *= 0.1; + }); #ifdef PELE_USE_PLASMA // rhoY_e -> nE and set rhoY_e to zero - Real invmwt[NUM_SPECIES] = {0.0}; + amrex::Real invmwt[NUM_SPECIES] = {0.0}; eos.inv_molecular_weight(invmwt); - ParallelFor( - bx, - [invmwt, nE_o, rhoYe_o] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - nE_o(i, j, k) = rhoYe_o(i, j, k) * Na * invmwt[E_ID] * 1.0e3; - rhoYe_o(i, j, k) = 0.0; - }); + ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + nE_o(i, j, k) = rhoYe_o(i, j, k) * Na * invmwt[E_ID] * 1.0e3; + rhoYe_o(i, j, k) = 0.0; + }); #endif #ifdef AMREX_USE_GPU - Gpu::Device::streamSynchronize(); + amrex::Gpu::Device::streamSynchronize(); #endif } // ParallelCopy into newstate MFs // Get the entire new state - MultiFab StateTemp(grids[lev], dmap[lev], NUM_SPECIES + 3, 0); + amrex::MultiFab StateTemp(grids[lev], dmap[lev], NUM_SPECIES + 3, 0); StateTemp.ParallelCopy(chemState, 0, 0, NUM_SPECIES + 3); ldataR_p->functC.ParallelCopy(functC, 0, 0, 1); #ifdef PELE_USE_PLASMA - MultiFab nETemp(grids[lev], dmap[lev], 1, 0); + amrex::MultiFab nETemp(grids[lev], dmap[lev], 1, 0); nETemp.ParallelCopy(chemnE, 0, 0, 1); #endif // Pass from temp state MF to leveldata and set reaction term #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldataNew_p->state, amrex::TilingIfNotGPU()); mfi.isValid(); - ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldataNew_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); auto const& state_arr = StateTemp.const_array(mfi); auto const& rhoY_o = ldataOld_p->state.const_array(mfi, FIRSTSPEC); auto const& rhoY_n = ldataNew_p->state.array(mfi, FIRSTSPEC); @@ -329,23 +312,21 @@ PeleLM::advanceChemistryBAChem( auto const& temp_n = ldataNew_p->state.array(mfi, TEMP); auto const& extF_rhoY = a_extForcing.const_array(mfi, 0); auto const& rhoYdot = ldataR_p->I_R.array(mfi, 0); - Real dt_inv = 1.0 / a_dt; - ParallelFor( - bx, [state_arr, rhoY_o, rhoY_n, rhoH_n, temp_n, extF_rhoY, rhoYdot, - dt_inv] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - // Pass into leveldata_new - for (int n = 0; n < NUM_SPECIES; n++) { - rhoY_n(i, j, k, n) = state_arr(i, j, k, n); - } - rhoH_n(i, j, k) = state_arr(i, j, k, NUM_SPECIES); - temp_n(i, j, k) = state_arr(i, j, k, NUM_SPECIES + 1); - // Compute I_R - for (int n = 0; n < NUM_SPECIES; n++) { - rhoYdot(i, j, k, n) = - -(rhoY_o(i, j, k, n) - rhoY_n(i, j, k, n)) * dt_inv - - extF_rhoY(i, j, k, n); - } - }); + amrex::Real dt_inv = 1.0 / a_dt; + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + // Pass into leveldata_new + for (int n = 0; n < NUM_SPECIES; n++) { + rhoY_n(i, j, k, n) = state_arr(i, j, k, n); + } + rhoH_n(i, j, k) = state_arr(i, j, k, NUM_SPECIES); + temp_n(i, j, k) = state_arr(i, j, k, NUM_SPECIES + 1); + // Compute I_R + for (int n = 0; n < NUM_SPECIES; n++) { + rhoYdot(i, j, k, n) = + -(rhoY_o(i, j, k, n) - rhoY_n(i, j, k, n)) * dt_inv - + extF_rhoY(i, j, k, n); + } + }); #ifdef PELE_USE_PLASMA auto const& nE_arr = nETemp.const_array(mfi); @@ -353,22 +334,19 @@ PeleLM::advanceChemistryBAChem( auto const& nE_n = ldataNew_p->state.array(mfi, NE); auto const& FnE = a_extForcing.const_array(mfi, NUM_SPECIES + 1); auto const& nEdot = ldataR_p->I_R.array(mfi, NUM_SPECIES); - ParallelFor( - bx, [nE_arr, nE_o, nE_n, FnE, nEdot, - dt_inv] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - // Pass into leveldata_new - nE_n(i, j, k) = nE_arr(i, j, k); - // Compute I_R - nEdot(i, j, k) = - -(nE_o(i, j, k) - nE_n(i, j, k)) * dt_inv - FnE(i, j, k); - }); + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + // Pass into leveldata_new + nE_n(i, j, k) = nE_arr(i, j, k); + // Compute I_R + nEdot(i, j, k) = -(nE_o(i, j, k) - nE_n(i, j, k)) * dt_inv - FnE(i, j, k); + }); #endif } } void PeleLM::computeInstantaneousReactionRate( - const Vector& I_R, const TimeStamp& a_time) + const amrex::Vector& I_R, const TimeStamp& a_time) { for (int lev = 0; lev <= finest_level; ++lev) { #ifdef PELE_USE_PLASMA @@ -381,7 +359,7 @@ PeleLM::computeInstantaneousReactionRate( void PeleLM::computeInstantaneousReactionRate( - int lev, const TimeStamp& a_time, MultiFab* a_I_R) + int lev, const TimeStamp& a_time, amrex::MultiFab* a_I_R) { BL_PROFILE("PeleLMeX::computeInstantaneousReactionRate()"); auto* ldata_p = getLevelDataPtr(lev, a_time); @@ -392,10 +370,11 @@ PeleLM::computeInstantaneousReactionRate( #endif #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); auto const& rhoY = ldata_p->state.const_array(mfi, FIRSTSPEC); auto const& rhoH = ldata_p->state.const_array(mfi, RHOH); auto const& T = ldata_p->state.const_array(mfi, TEMP); @@ -404,16 +383,17 @@ PeleLM::computeInstantaneousReactionRate( #ifdef AMREX_USE_EB auto const& flagfab = ebfact.getMultiEBCellFlagFab()[mfi]; auto const& flag = flagfab.const_array(); - if (flagfab.getType(bx) == FabType::covered) { // Covered boxes + if (flagfab.getType(bx) == amrex::FabType::covered) { // Covered boxes amrex::ParallelFor( bx, NUM_SPECIES, - [rhoYdot] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { rhoYdot(i, j, k, n) = 0.0; }); - } else if (flagfab.getType(bx) != FabType::regular) { // EB containing boxes + } else if (flagfab.getType(bx) != amrex::FabType::regular) { // EB + // containing + // boxes amrex::ParallelFor( - bx, [rhoY, rhoH, T, rhoYdot, flag, - leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (flag(i, j, k).isCovered()) { for (int n = 0; n < NUM_SPECIES; n++) { rhoYdot(i, j, k, n) = 0.0; @@ -426,8 +406,7 @@ PeleLM::computeInstantaneousReactionRate( #endif { amrex::ParallelFor( - bx, [rhoY, rhoH, T, rhoYdot, - leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { reactionRateRhoY(i, j, k, rhoY, rhoH, T, rhoYdot, leosparm); }); } @@ -447,11 +426,11 @@ PeleLM::getScalarReactForce(std::unique_ptr& advData) auto* ldataR_p = getLevelDataReactPtr(lev); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(advData->Forcing[lev], TilingIfNotGPU()); mfi.isValid(); - ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(advData->Forcing[lev], amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); auto const& rhoY_o = ldataOld_p->state.const_array(mfi, FIRSTSPEC); auto const& rhoH_o = ldataOld_p->state.const_array(mfi, RHOH); auto const& rhoY_n = ldataNew_p->state.const_array(mfi, FIRSTSPEC); @@ -461,8 +440,7 @@ PeleLM::getScalarReactForce(std::unique_ptr& advData) auto const& extF_rhoH = advData->Forcing[lev].array(mfi, NUM_SPECIES); amrex::Real dtinv = 1.0 / m_dt; amrex::ParallelFor( - bx, [rhoY_o, rhoH_o, rhoY_n, rhoH_n, react, extF_rhoY, extF_rhoH, - dtinv] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { for (int n = 0; n < NUM_SPECIES; n++) { extF_rhoY(i, j, k, n) = (rhoY_n(i, j, k, n) - rhoY_o(i, j, k, n)) * dtinv - @@ -475,26 +453,26 @@ PeleLM::getScalarReactForce(std::unique_ptr& advData) } void -PeleLM::getHeatRelease(int a_lev, MultiFab* a_HR) +PeleLM::getHeatRelease(int a_lev, amrex::MultiFab* a_HR) { auto* ldataNew_p = getLevelDataPtr(a_lev, AmrNewTime); auto* ldataR_p = getLevelDataReactPtr(a_lev); auto const* leosparm = eos_parms.device_parm(); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif { - for (MFIter mfi(*a_HR, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); - FArrayBox EnthFab(bx, NUM_SPECIES, The_Async_Arena()); + for (amrex::MFIter mfi(*a_HR, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + const amrex::Box& bx = mfi.tilebox(); + amrex::FArrayBox EnthFab(bx, NUM_SPECIES, amrex::The_Async_Arena()); auto const& react = ldataR_p->I_R.const_array(mfi, 0); auto const& T = ldataNew_p->state.const_array(mfi, TEMP); auto const& Hi = EnthFab.array(); auto const& HRR = a_HR->array(mfi); amrex::ParallelFor( - bx, [T, Hi, HRR, react, - leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { getHGivenT(i, j, k, T, Hi, leosparm); HRR(i, j, k) = 0.0; for (int n = 0; n < NUM_SPECIES; n++) { diff --git a/Source/PeleLMeX_Regrid.cpp b/Source/PeleLMeX_Regrid.cpp index 0aa9d8f59..24a13dcc0 100644 --- a/Source/PeleLMeX_Regrid.cpp +++ b/Source/PeleLMeX_Regrid.cpp @@ -1,8 +1,6 @@ #include #include -using namespace amrex; - void PeleLM::regrid(int lbase, amrex::Real time, bool initial) { @@ -21,7 +19,7 @@ PeleLM::regrid(int lbase, amrex::Real time, bool initial) if (lbase == 0 && (m_doLoadBalance != 0) && !initial) { if (m_verbose > 0) { - Print() << " Load balancing level 0 \n"; + amrex::Print() << " Load balancing level 0 \n"; } int remakeLevel = 0; @@ -32,64 +30,65 @@ PeleLM::regrid(int lbase, amrex::Real time, bool initial) amrex::Real currentEfficiency = 0.0; amrex::Real testEfficiency = 0.0; - DistributionMapping test_dmap; + amrex::DistributionMapping test_dmap; // Build the test dmap, w/o braodcasting if (m_loadBalanceMethod == LoadBalanceMethod::SFC) { - test_dmap = DistributionMapping::makeSFC( + test_dmap = amrex::DistributionMapping::makeSFC( *m_costs[0], currentEfficiency, testEfficiency, false, - ParallelDescriptor::IOProcessorNumber()); + amrex::ParallelDescriptor::IOProcessorNumber()); } else if (m_loadBalanceMethod == LoadBalanceMethod::Knapsack) { const amrex::Real navg = - static_cast(grids[0].size()) / - static_cast(ParallelDescriptor::NProcs()); - const int nmax = static_cast( - std::max(std::round(m_loadBalanceKSfactor * navg), std::ceil(navg))); - test_dmap = DistributionMapping::makeKnapSack( + static_cast(grids[0].size()) / + static_cast(amrex::ParallelDescriptor::NProcs()); + const int nmax = static_cast(amrex::max( + std::round(m_loadBalanceKSfactor * navg), std::ceil(navg))); + test_dmap = amrex::DistributionMapping::makeKnapSack( *m_costs[0], currentEfficiency, testEfficiency, nmax, false, - ParallelDescriptor::IOProcessorNumber()); + amrex::ParallelDescriptor::IOProcessorNumber()); } // IO proc determine if the test dmap offers significant improvements if ( (m_loadBalanceEffRatioThreshold > 0.0) && - (ParallelDescriptor::MyProc() == - ParallelDescriptor::IOProcessorNumber())) { + (amrex::ParallelDescriptor::MyProc() == + amrex::ParallelDescriptor::IOProcessorNumber())) { remakeLevel = static_cast( (remakeLevel != 0) || (testEfficiency > m_loadBalanceEffRatioThreshold * currentEfficiency)); } - ParallelDescriptor::Bcast( - &remakeLevel, 1, ParallelDescriptor::IOProcessorNumber()); + amrex::ParallelDescriptor::Bcast( + &remakeLevel, 1, amrex::ParallelDescriptor::IOProcessorNumber()); if (m_verbose > 1 && (remakeLevel != 0)) { - Print() << " Current LoadBalancing efficiency: " << currentEfficiency - << "\n" - << " Test LoadBalancing efficiency: " << testEfficiency - << " \n"; + amrex::Print() << " Current LoadBalancing efficiency: " + << currentEfficiency << "\n" + << " Test LoadBalancing efficiency: " << testEfficiency + << " \n"; } // Bcast the test dmap and remake level if (remakeLevel != 0) { - Vector pmap; + amrex::Vector pmap; if ( - ParallelDescriptor::MyProc() == - ParallelDescriptor::IOProcessorNumber()) { + amrex::ParallelDescriptor::MyProc() == + amrex::ParallelDescriptor::IOProcessorNumber()) { pmap = test_dmap.ProcessorMap(); } else { #pragma GCC diagnostic ignored "-Wnull-dereference" pmap.resize(static_cast(grids[0].size())); } - ParallelDescriptor::Bcast( - pmap.data(), pmap.size(), ParallelDescriptor::IOProcessorNumber()); + amrex::ParallelDescriptor::Bcast( + pmap.data(), pmap.size(), + amrex::ParallelDescriptor::IOProcessorNumber()); if ( - ParallelDescriptor::MyProc() != - ParallelDescriptor::IOProcessorNumber()) { - test_dmap = DistributionMapping(pmap); + amrex::ParallelDescriptor::MyProc() != + amrex::ParallelDescriptor::IOProcessorNumber()) { + test_dmap = amrex::DistributionMapping(pmap); } RemakeLevel(0, time, grids[0], test_dmap); @@ -102,7 +101,7 @@ PeleLM::regrid(int lbase, amrex::Real time, bool initial) } int new_finest; - Vector new_grids(finest_level + 2); + amrex::Vector new_grids(finest_level + 2); MakeNewGrids(lbase, time, new_finest, new_grids); BL_ASSERT(new_finest <= finest_level + 1); @@ -117,11 +116,11 @@ PeleLM::regrid(int lbase, amrex::Real time, bool initial) // remake level only if the grid changed using the default DMap strategy if (initial || (m_doLoadBalance == 0)) { if (ba_changed || coarse_ba_changed) { - BoxArray level_grids = grids[lev]; - DistributionMapping level_dmap = dmap[lev]; + amrex::BoxArray level_grids = grids[lev]; + amrex::DistributionMapping level_dmap = dmap[lev]; if (ba_changed) { level_grids = new_grids[lev]; - level_dmap = DistributionMapping(level_grids); + level_dmap = amrex::DistributionMapping(level_grids); } const auto old_num_setdm = num_setdm; RemakeLevel(lev, time, level_grids, level_dmap); @@ -137,8 +136,8 @@ PeleLM::regrid(int lbase, amrex::Real time, bool initial) int remakeLevel = 0; - BoxArray new_ba; - DistributionMapping new_dmap; + amrex::BoxArray new_ba; + amrex::DistributionMapping new_dmap; // If the grid changed, let's build a new dmap if (ba_changed) { @@ -146,51 +145,53 @@ PeleLM::regrid(int lbase, amrex::Real time, bool initial) remakeLevel = 1; new_ba = new_grids[lev]; - new_dmap = DistributionMapping(new_ba); + new_dmap = amrex::DistributionMapping(new_ba); // Get the cost on a LayoutData associated with the new grid - LayoutData new_cost(new_ba, new_dmap); + amrex::LayoutData new_cost(new_ba, new_dmap); computeCosts(lev, new_cost, m_loadBalanceCost); if (m_loadBalanceMethod == LoadBalanceMethod::SFC) { - Vector costsVec(new_ba.size()); - ParallelDescriptor::GatherLayoutDataToVector( - new_cost, costsVec, ParallelContext::IOProcessorNumberSub()); - ParallelDescriptor::Bcast( + amrex::Vector costsVec(new_ba.size()); + amrex::ParallelDescriptor::GatherLayoutDataToVector( + new_cost, costsVec, + amrex::ParallelContext::IOProcessorNumberSub()); + amrex::ParallelDescriptor::Bcast( costsVec.data(), costsVec.size(), - ParallelContext::IOProcessorNumberSub()); - Real efficiency; - new_dmap = - DistributionMapping::makeSFC(costsVec, new_ba, efficiency); + amrex::ParallelContext::IOProcessorNumberSub()); + amrex::Real efficiency; + new_dmap = amrex::DistributionMapping::makeSFC( + costsVec, new_ba, efficiency); } else if (m_loadBalanceMethod == LoadBalanceMethod::Knapsack) { const amrex::Real navg = - static_cast(new_ba.size()) / - static_cast(ParallelDescriptor::NProcs()); - const int nmax = static_cast(std::max( + static_cast(new_ba.size()) / + static_cast(amrex::ParallelDescriptor::NProcs()); + const int nmax = static_cast(amrex::max( std::round(m_loadBalanceKSfactor * navg), std::ceil(navg))); - Vector costsVec(new_ba.size()); - ParallelDescriptor::GatherLayoutDataToVector( - new_cost, costsVec, ParallelContext::IOProcessorNumberSub()); - ParallelDescriptor::Bcast( + amrex::Vector costsVec(new_ba.size()); + amrex::ParallelDescriptor::GatherLayoutDataToVector( + new_cost, costsVec, + amrex::ParallelContext::IOProcessorNumberSub()); + amrex::ParallelDescriptor::Bcast( costsVec.data(), costsVec.size(), - ParallelContext::IOProcessorNumberSub()); - Real efficiency; - new_dmap = - DistributionMapping::makeKnapSack(costsVec, efficiency, nmax); + amrex::ParallelContext::IOProcessorNumberSub()); + amrex::Real efficiency; + new_dmap = amrex::DistributionMapping::makeKnapSack( + costsVec, efficiency, nmax); } // Let's see if we can get a better dmap } else { if (m_verbose > 1) { - Print() << " Load balancing level " << lev << "\n"; + amrex::Print() << " Load balancing level " << lev << "\n"; } // Try to build a new dmap with the same old grid new_ba = grids[lev]; - DistributionMapping test_dmap; + amrex::DistributionMapping test_dmap; computeCosts(lev); @@ -201,62 +202,63 @@ PeleLM::regrid(int lbase, amrex::Real time, bool initial) // Build the test dmap, w/o braodcasting if (m_loadBalanceMethod == LoadBalanceMethod::SFC) { - test_dmap = DistributionMapping::makeSFC( + test_dmap = amrex::DistributionMapping::makeSFC( *m_costs[lev], currentEfficiency, testEfficiency, false, - ParallelDescriptor::IOProcessorNumber()); + amrex::ParallelDescriptor::IOProcessorNumber()); } else if (m_loadBalanceMethod == LoadBalanceMethod::Knapsack) { const amrex::Real navg = - static_cast(new_ba.size()) / - static_cast(ParallelDescriptor::NProcs()); - const int nmax = static_cast(std::max( + static_cast(new_ba.size()) / + static_cast(amrex::ParallelDescriptor::NProcs()); + const int nmax = static_cast(amrex::max( std::round(m_loadBalanceKSfactor * navg), std::ceil(navg))); - test_dmap = DistributionMapping::makeKnapSack( + test_dmap = amrex::DistributionMapping::makeKnapSack( *m_costs[lev], currentEfficiency, testEfficiency, nmax, false, - ParallelDescriptor::IOProcessorNumber()); + amrex::ParallelDescriptor::IOProcessorNumber()); } // IO proc determine if the test dmap offers significant // improvements if ( (m_loadBalanceEffRatioThreshold > 0.0) && - (ParallelDescriptor::MyProc() == - ParallelDescriptor::IOProcessorNumber())) { + (amrex::ParallelDescriptor::MyProc() == + amrex::ParallelDescriptor::IOProcessorNumber())) { remakeLevel = static_cast( (remakeLevel != 0) || (testEfficiency > m_loadBalanceEffRatioThreshold * currentEfficiency)); } - ParallelDescriptor::Bcast( - &remakeLevel, 1, ParallelDescriptor::IOProcessorNumber()); + amrex::ParallelDescriptor::Bcast( + &remakeLevel, 1, amrex::ParallelDescriptor::IOProcessorNumber()); if (m_verbose > 1 && (remakeLevel != 0)) { - Print() << " Current LoadBalancing efficiency: " - << currentEfficiency << "\n" - << " Test LoadBalancing efficiency: " << testEfficiency - << " \n"; + amrex::Print() + << " Current LoadBalancing efficiency: " << currentEfficiency + << "\n" + << " Test LoadBalancing efficiency: " << testEfficiency + << " \n"; } // Bcast the test dmap if we plan on remaking the level if (remakeLevel != 0) { - Vector pmap; + amrex::Vector pmap; if ( - ParallelDescriptor::MyProc() == - ParallelDescriptor::IOProcessorNumber()) { + amrex::ParallelDescriptor::MyProc() == + amrex::ParallelDescriptor::IOProcessorNumber()) { pmap = test_dmap.ProcessorMap(); } else { #pragma GCC diagnostic ignored "-Wnull-dereference" pmap.resize(static_cast(new_ba.size())); } - ParallelDescriptor::Bcast( + amrex::ParallelDescriptor::Bcast( pmap.data(), pmap.size(), - ParallelDescriptor::IOProcessorNumber()); + amrex::ParallelDescriptor::IOProcessorNumber()); if ( - ParallelDescriptor::MyProc() != - ParallelDescriptor::IOProcessorNumber()) { - test_dmap = DistributionMapping(pmap); + amrex::ParallelDescriptor::MyProc() != + amrex::ParallelDescriptor::IOProcessorNumber()) { + test_dmap = amrex::DistributionMapping(pmap); } new_dmap = test_dmap; } @@ -273,7 +275,7 @@ PeleLM::regrid(int lbase, amrex::Real time, bool initial) } coarse_ba_changed = ba_changed; } else { // a new level, use default DMap strategy - DistributionMapping new_dmap(new_grids[lev]); + amrex::DistributionMapping new_dmap(new_grids[lev]); const auto old_num_setdm = num_setdm; MakeNewLevelFromCoarse(lev, time, new_grids[lev], new_dmap); SetBoxArray(lev, new_grids[lev]); @@ -313,10 +315,10 @@ PeleLM::MakeNewLevelFromCoarse( BL_PROFILE("PeleLMeX::MakeNewLevelFromCoarse()"); if (m_verbose > 0) { - Print() << " Making new level " << lev << " from coarse\n"; + amrex::Print() << " Making new level " << lev << " from coarse\n"; if (m_verbose > 2) { auto const dx = geom[lev].CellSizeArray(); - Real vol = AMREX_D_TERM(dx[0], *dx[1], *dx[2]); + amrex::Real vol = AMREX_D_TERM(dx[0], *dx[1], *dx[2]); amrex::Print() << " with " << ba.numPts() << " cells, " << ba.size() << " boxes," << " over " @@ -331,10 +333,11 @@ PeleLM::MakeNewLevelFromCoarse( // New level factory #ifdef AMREX_USE_EB - std::unique_ptr> new_fact = - makeEBFabFactory(geom[lev], ba, dm, {6, 6, 6}, EBSupport::full); + std::unique_ptr> new_fact = + makeEBFabFactory(geom[lev], ba, dm, {6, 6, 6}, amrex::EBSupport::full); #else - std::unique_ptr> new_fact(new FArrayBoxFactory()); + std::unique_ptr> new_fact( + new amrex::FArrayBoxFactory()); #endif // New leveldatas @@ -388,7 +391,7 @@ PeleLM::MakeNewLevelFromCoarse( } if (max_level > 0 && lev != max_level) { - m_coveredMask[lev] = std::make_unique(ba, dm, 1, 0); + m_coveredMask[lev] = std::make_unique(ba, dm, 1, 0); } m_resetCoveredMask = 1; @@ -396,13 +399,14 @@ PeleLM::MakeNewLevelFromCoarse( m_leveldatanlsolve[lev].reset( new LevelDataNLSolve(ba, dm, *m_factory[lev], m_nGrowState)); if (m_do_extraEFdiags) { - m_ionsFluxes[lev].reset(new MultiFab(ba, dm, NUM_IONS * AMREX_SPACEDIM, 0)); + m_ionsFluxes[lev].reset( + new amrex::MultiFab(ba, dm, NUM_IONS * AMREX_SPACEDIM, 0)); } m_precond_op.reset(); #endif // Load balance - m_costs[lev] = std::make_unique>(ba, dm); + m_costs[lev] = std::make_unique>(ba, dm); // DiffusionOp will be recreated m_diffusion_op.reset(); @@ -411,8 +415,8 @@ PeleLM::MakeNewLevelFromCoarse( // Trigger MacProj reset m_macProjNeedReset = 1; - m_extSource[lev] = std::make_unique( - ba, dm, NVAR, amrex::max(m_nGrowAdv, m_nGrowMAC), MFInfo(), + m_extSource[lev] = std::make_unique( + ba, dm, NVAR, amrex::max(m_nGrowAdv, m_nGrowMAC), amrex::MFInfo(), *m_factory[lev]); m_extSource[lev]->setVal(0.); } @@ -427,10 +431,10 @@ PeleLM::RemakeLevel( BL_PROFILE("PeleLMeX::RemakeLevel()"); if (m_verbose > 0) { - Print() << " Remaking level " << lev << "\n"; + amrex::Print() << " Remaking level " << lev << "\n"; if (m_verbose > 2) { auto const dx = geom[lev].CellSizeArray(); - Real vol = AMREX_D_TERM(dx[0], *dx[1], *dx[2]); + amrex::Real vol = AMREX_D_TERM(dx[0], *dx[1], *dx[2]); amrex::Print() << " with " << ba.numPts() << " cells," << ba.size() << " boxes," << " over " @@ -445,10 +449,11 @@ PeleLM::RemakeLevel( // New level factory #ifdef AMREX_USE_EB - std::unique_ptr> new_fact = - makeEBFabFactory(geom[lev], ba, dm, {6, 6, 6}, EBSupport::full); + std::unique_ptr> new_fact = + makeEBFabFactory(geom[lev], ba, dm, {6, 6, 6}, amrex::EBSupport::full); #else - std::unique_ptr> new_fact(new FArrayBoxFactory()); + std::unique_ptr> new_fact( + new amrex::FArrayBoxFactory()); #endif // New leveldatas @@ -488,7 +493,7 @@ PeleLM::RemakeLevel( } if (max_level > 0 && lev != max_level) { - m_coveredMask[lev] = std::make_unique(ba, dm, 1, 0); + m_coveredMask[lev] = std::make_unique(ba, dm, 1, 0); } m_resetCoveredMask = 1; @@ -510,13 +515,14 @@ PeleLM::RemakeLevel( m_leveldatanlsolve[lev].reset( new LevelDataNLSolve(ba, dm, *m_factory[lev], m_nGrowState)); if (m_do_extraEFdiags) { - m_ionsFluxes[lev].reset(new MultiFab(ba, dm, NUM_IONS * AMREX_SPACEDIM, 0)); + m_ionsFluxes[lev].reset( + new amrex::MultiFab(ba, dm, NUM_IONS * AMREX_SPACEDIM, 0)); } m_precond_op.reset(); #endif // Load balance - m_costs[lev] = std::make_unique>(ba, dm); + m_costs[lev] = std::make_unique>(ba, dm); // DiffusionOp will be recreated m_diffusion_op.reset(); @@ -525,8 +531,8 @@ PeleLM::RemakeLevel( // Trigger MacProj reset m_macProjNeedReset = 1; - m_extSource[lev] = std::make_unique( - ba, dm, NVAR, amrex::max(m_nGrowAdv, m_nGrowMAC), MFInfo(), + m_extSource[lev] = std::make_unique( + ba, dm, NVAR, amrex::max(m_nGrowAdv, m_nGrowMAC), amrex::MFInfo(), *m_factory[lev]); m_extSource[lev]->setVal(0.); } @@ -564,52 +570,53 @@ PeleLM::ClearLevel(int lev) } void -PeleLM::computeCosts(int a_lev, LayoutData& a_costs, int a_costMethod) +PeleLM::computeCosts( + int a_lev, amrex::LayoutData& a_costs, int a_costMethod) { if (a_costMethod == LoadBalanceCost::Ncell) { - for (MFIter mfi(a_costs, false); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(a_costs, false); mfi.isValid(); ++mfi) { a_costs[mfi] = static_cast(mfi.validbox().numPts()); } } else if (a_costMethod == LoadBalanceCost::ChemFunctCallAvg) { - MultiFab costMF(a_costs.boxArray(), a_costs.DistributionMap(), 1, 0); + amrex::MultiFab costMF(a_costs.boxArray(), a_costs.DistributionMap(), 1, 0); fillpatch_chemFunctCall(a_lev, m_cur_time, costMF, 0); - for (MFIter mfi(costMF, false); mfi.isValid(); ++mfi) { - a_costs[mfi] = costMF[mfi].sum(mfi.validbox(), 0) / + for (amrex::MFIter mfi(costMF, false); mfi.isValid(); ++mfi) { + a_costs[mfi] = costMF[mfi].sum(mfi.validbox(), 0) / static_cast(mfi.validbox().numPts()); } } else if (a_costMethod == LoadBalanceCost::ChemFunctCallMax) { - MultiFab costMF(a_costs.boxArray(), a_costs.DistributionMap(), 1, 0); + amrex::MultiFab costMF(a_costs.boxArray(), a_costs.DistributionMap(), 1, 0); fillpatch_chemFunctCall(a_lev, m_cur_time, costMF, 0); - for (MFIter mfi(costMF, false); mfi.isValid(); ++mfi) { - a_costs[mfi] = costMF[mfi].max(mfi.validbox(), 0); + for (amrex::MFIter mfi(costMF, false); mfi.isValid(); ++mfi) { + a_costs[mfi] = costMF[mfi].max(mfi.validbox(), 0); } } else if (a_costMethod == LoadBalanceCost::ChemFunctCallSum) { - MultiFab costMF(a_costs.boxArray(), a_costs.DistributionMap(), 1, 0); + amrex::MultiFab costMF(a_costs.boxArray(), a_costs.DistributionMap(), 1, 0); fillpatch_chemFunctCall(a_lev, m_cur_time, costMF, 0); - for (MFIter mfi(costMF, false); mfi.isValid(); ++mfi) { - a_costs[mfi] = costMF[mfi].sum(mfi.validbox(), 0); + for (amrex::MFIter mfi(costMF, false); mfi.isValid(); ++mfi) { + a_costs[mfi] = costMF[mfi].sum(mfi.validbox(), 0); } } else if (a_costMethod == LoadBalanceCost::UserDefinedDerivedAvg) { - MultiFab costMF(a_costs.boxArray(), a_costs.DistributionMap(), 1, 0); + amrex::MultiFab costMF(a_costs.boxArray(), a_costs.DistributionMap(), 1, 0); costMF.setVal(0.0); - std::unique_ptr mf; + std::unique_ptr mf; mf = derive("derUserDefined", m_cur_time, a_lev, 0); costMF.ParallelCopy(*mf, 0, 0, 1); - for (MFIter mfi(costMF, false); mfi.isValid(); ++mfi) { - a_costs[mfi] = costMF[mfi].sum(mfi.validbox(), 0) / + for (amrex::MFIter mfi(costMF, false); mfi.isValid(); ++mfi) { + a_costs[mfi] = costMF[mfi].sum(mfi.validbox(), 0) / static_cast(mfi.validbox().numPts()); } } else if (a_costMethod == LoadBalanceCost::UserDefinedDerivedSum) { - MultiFab costMF(a_costs.boxArray(), a_costs.DistributionMap(), 1, 0); + amrex::MultiFab costMF(a_costs.boxArray(), a_costs.DistributionMap(), 1, 0); costMF.setVal(0.0); - std::unique_ptr mf; + std::unique_ptr mf; mf = derive("derUserDefined", m_cur_time, a_lev, 0); costMF.ParallelCopy(*mf, 0, 0, 1); - for (MFIter mfi(costMF, false); mfi.isValid(); ++mfi) { - a_costs[mfi] = costMF[mfi].sum(mfi.validbox(), 0); + for (amrex::MFIter mfi(costMF, false); mfi.isValid(); ++mfi) { + a_costs[mfi] = costMF[mfi].sum(mfi.validbox(), 0); } } else { - Abort(" Unknown cost estimate method !"); + amrex::Abort(" Unknown cost estimate method !"); } } @@ -631,9 +638,9 @@ PeleLM::resetMacProjector() #ifdef AMREX_USE_EB macproj = std::make_unique( Geom(0, finest_level), - MLMG::Location::FaceCentroid, // Location of mac velocity - MLMG::Location::FaceCentroid, // Location of beta - MLMG::Location::CellCenter); // Location of solution variable phi + amrex::MLMG::Location::FaceCentroid, // Location of mac velocity + amrex::MLMG::Location::FaceCentroid, // Location of beta + amrex::MLMG::Location::CellCenter); // Location of solution variable phi #else macproj = std::make_unique(Geom(0, finest_level)); #endif @@ -647,7 +654,7 @@ void PeleLM::regridFromGridFile(int lbase, amrex::Real time, bool /*initial*/) { const int new_finest = static_cast(m_regrid_ba.size()); - Vector new_grids(finest_level + 2); + amrex::Vector new_grids(finest_level + 2); BL_ASSERT(new_finest <= finest_level + 1); bool coarse_ba_changed = false; @@ -656,11 +663,11 @@ PeleLM::regridFromGridFile(int lbase, amrex::Real time, bool /*initial*/) if (lev <= finest_level) { // an old level bool ba_changed = (new_grids[lev] != grids[lev]); if (ba_changed || coarse_ba_changed) { - BoxArray level_grids = grids[lev]; - DistributionMapping level_dmap = dmap[lev]; + amrex::BoxArray level_grids = grids[lev]; + amrex::DistributionMapping level_dmap = dmap[lev]; if (ba_changed) { level_grids = new_grids[lev]; - level_dmap = DistributionMapping(level_grids); + level_dmap = amrex::DistributionMapping(level_grids); } const auto old_num_setdm = num_setdm; RemakeLevel(lev, time, level_grids, level_dmap); @@ -672,7 +679,7 @@ PeleLM::regridFromGridFile(int lbase, amrex::Real time, bool /*initial*/) coarse_ba_changed = ba_changed; ; } else { // a new level - DistributionMapping new_dmap(new_grids[lev]); + amrex::DistributionMapping new_dmap(new_grids[lev]); const auto old_num_setdm = num_setdm; MakeNewLevelFromCoarse(lev, time, new_grids[lev], new_dmap); SetBoxArray(lev, new_grids[lev]); diff --git a/Source/PeleLMeX_Setup.cpp b/Source/PeleLMeX_Setup.cpp index 4ee8e98e4..524659422 100644 --- a/Source/PeleLMeX_Setup.cpp +++ b/Source/PeleLMeX_Setup.cpp @@ -17,22 +17,20 @@ #include "SootModel.H" #endif -using namespace amrex; - -static Box -the_same_box(const Box& b) +static amrex::Box +the_same_box(const amrex::Box& b) { return b; } #ifdef PELE_USE_PLASMA -static Box -grow_box_by_one(const Box& b) +static amrex::Box +grow_box_by_one(const amrex::Box& b) { return amrex::grow(b, 1); } #endif -static Box -grow_box_by_two(const Box& b) +static amrex::Box +grow_box_by_two(const amrex::Box& b) { return amrex::grow(b, 2); } @@ -52,10 +50,10 @@ PeleLM::Setup() &&amrex::almostEqual(dx[1], dx[2], 10))); } // Print build info to screen - const char* githash1 = buildInfoGetGitHash(1); - const char* githash2 = buildInfoGetGitHash(2); - const char* githash3 = buildInfoGetGitHash(3); - const char* githash4 = buildInfoGetGitHash(4); + const char* githash1 = amrex::buildInfoGetGitHash(1); + const char* githash2 = amrex::buildInfoGetGitHash(2); + const char* githash3 = amrex::buildInfoGetGitHash(3); + const char* githash4 = amrex::buildInfoGetGitHash(4); amrex::Print() << "\n ================= Build infos =================\n"; amrex::Print() << " PeleLMeX git hash: " << githash1 << "\n"; amrex::Print() << " AMReX git hash: " << githash2 << "\n"; @@ -139,7 +137,7 @@ PeleLM::Setup() int ncells_chem = 1; amrex::Print() << " Initialization of chemical reactor ... \n"; m_chem_integrator = "ReactorNull"; - ParmParse pp("peleLM"); + amrex::ParmParse pp("peleLM"); pp.query("chem_integrator", m_chem_integrator); m_reactor = pele::physics::reactions::ReactorBase::create(m_chem_integrator); @@ -208,7 +206,7 @@ PeleLM::Setup() // Problem parameters prob_parm = new ProbParm{}; - prob_parm_d = (ProbParm*)The_Arena()->alloc(sizeof(ProbParm)); + prob_parm_d = (ProbParm*)amrex::The_Arena()->alloc(sizeof(ProbParm)); // Problem parameters readProbParm(); @@ -219,7 +217,8 @@ PeleLM::Setup() m_pNew = prob_parm->P_mean; // Copy problem parameters into device copy - Gpu::copy(Gpu::hostToDevice, prob_parm, prob_parm + 1, prob_parm_d); + amrex::Gpu::copy( + amrex::Gpu::hostToDevice, prob_parm, prob_parm + 1, prob_parm_d); // Initialize active control initActiveControl(); @@ -235,7 +234,7 @@ PeleLM::readParameters() readIOParameters(); - ParmParse pp("peleLM"); + amrex::ParmParse pp("peleLM"); // ----------------------------------------- // Misc @@ -266,15 +265,16 @@ PeleLM::readParameters() m_closed_chamber = (isOpenDomain) != 0 ? 0 : 1; pp.query("closed_chamber", m_closed_chamber); if ((verbose != 0) && (m_closed_chamber != 0)) { - Print() << " Simulation performed with the closed chamber algorithm \n"; + amrex::Print() + << " Simulation performed with the closed chamber algorithm \n"; } #ifdef PELE_USE_PLASMA - ParmParse ppef("ef"); + amrex::ParmParse ppef("ef"); // Get the phiV bc - Vector lo_bc_char(AMREX_SPACEDIM); - Vector hi_bc_char(AMREX_SPACEDIM); + amrex::Vector lo_bc_char(AMREX_SPACEDIM); + amrex::Vector hi_bc_char(AMREX_SPACEDIM); ppef.getarr("phiV_lo_bc", lo_bc_char, 0, AMREX_SPACEDIM); ppef.getarr("phiV_hi_bc", hi_bc_char, 0, AMREX_SPACEDIM); for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { @@ -347,9 +347,9 @@ PeleLM::readParameters() AMREX_ASSERT_WITH_MESSAGE( m_mu > 0.0, "peleLM.mu is needed when running incompressible"); } - Vector grav(AMREX_SPACEDIM, 0); + amrex::Vector grav(AMREX_SPACEDIM, 0); pp.queryarr("gravity", grav, 0, AMREX_SPACEDIM); - Vector gp0(AMREX_SPACEDIM, 0); + amrex::Vector gp0(AMREX_SPACEDIM, 0); pp.queryarr("gradP0", gp0, 0, AMREX_SPACEDIM); for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { m_background_gp[idim] = gp0[idim]; @@ -376,7 +376,7 @@ PeleLM::readParameters() for (int n = 0; n < m_n_sparks; n++) { pp.get("sparks", m_spark[n], n); std::string spark_prefix = "peleLM." + m_spark[n]; - ParmParse pps(spark_prefix); + amrex::ParmParse pps(spark_prefix); pps.get("time", m_spark_time[n]); pps.get("duration", m_spark_duration[n]); m_spark_location[n].resize(AMREX_SPACEDIM); @@ -385,21 +385,22 @@ PeleLM::readParameters() pps.get("radius", m_spark_radius[n]); } if (m_spark_verbose > 0) { - Print() << "Spark list:" << std::endl; + amrex::Print() << "Spark list:" << std::endl; for (int n = 0; n < m_n_sparks; n++) { - Print() << "Spark " << n << " name: " << m_spark[n] << std::endl; - Print() << "Spark " << n << " time: " << m_spark_time[n] << std::endl; - Print() << "Spark " << n << " duration: " << m_spark_duration[n] - << std::endl; - Print() << "Spark " << n << " location: "; + amrex::Print() << "Spark " << n << " name: " << m_spark[n] << std::endl; + amrex::Print() << "Spark " << n << " time: " << m_spark_time[n] + << std::endl; + amrex::Print() << "Spark " << n << " duration: " << m_spark_duration[n] + << std::endl; + amrex::Print() << "Spark " << n << " location: "; for (int d = 0; d < AMREX_SPACEDIM; d++) { - Print() << m_spark_location[n][d] << " "; + amrex::Print() << m_spark_location[n][d] << " "; } - Print() << std::endl; - Print() << "Spark " << n << " temperature: " << m_spark_temp[n] - << std::endl; - Print() << "Spark " << n << " radius: " << m_spark_radius[n] - << std::endl; + amrex::Print() << std::endl; + amrex::Print() << "Spark " << n << " temperature: " << m_spark_temp[n] + << std::endl; + amrex::Print() << "Spark " << n << " radius: " << m_spark_radius[n] + << std::endl; } } } @@ -414,7 +415,7 @@ PeleLM::readParameters() for (int n = 0; n < m_nAux; n++) { pp.get("aux_vars", m_aux_names[n], n); std::string aux_prefix = "peleLM." + m_aux_names[n]; - ParmParse ppa(aux_prefix); + amrex::ParmParse ppa(aux_prefix); m_aux_advect[n] = 1; ppa.query("advect", m_aux_advect[n]); m_AdvTypeAux[n] = 1; @@ -462,7 +463,7 @@ PeleLM::readParameters() // ----------------------------------------- // diffusion - ParmParse pptrans("transport"); + amrex::ParmParse pptrans("transport"); pptrans.query("use_soret", m_use_soret); pp.query("use_wbar", m_use_wbar); if (m_use_soret != 0) { @@ -545,6 +546,13 @@ PeleLM::readParameters() pp.query("do_init_proj", m_do_init_proj); pp.query("num_init_iter", m_init_iter); pp.query("initDataPlt_patch_flow_variables", m_do_patch_flow_variables); + pp.queryarr("initDataPlt_specname_map", m_initDataPlt_specname_map); + if ( + !m_initDataPlt_specname_map.empty() && + m_initDataPlt_specname_map.size() != NUM_SPECIES) { + amrex::Abort( + "If specifying a species name map, length must equal number of species"); + } pp.query("initDataPlt_reset_time", m_do_reset_time); // ----------------------------------------- @@ -584,13 +592,14 @@ PeleLM::readParameters() m_max_grid_size_chem[0] = mgsc;, m_max_grid_size_chem[1] = mgsc; , m_max_grid_size_chem[2] = mgsc); } else if (mgsc_size == AMREX_SPACEDIM) { - Vector mgsc; + amrex::Vector mgsc; pp.getarr("max_grid_size_chem", mgsc, 0, AMREX_SPACEDIM); AMREX_D_TERM( m_max_grid_size_chem[0] = mgsc[0];, m_max_grid_size_chem[1] = mgsc[1]; , m_max_grid_size_chem[2] = mgsc[2]); } else { - Abort("peleLM.max_grid_size_chem should have 1 or AMREX_SPACEDIM values"); + amrex::Abort( + "peleLM.max_grid_size_chem should have 1 or AMREX_SPACEDIM values"); } } @@ -617,7 +626,7 @@ PeleLM::readParameters() // Deactivate load balancing for serial runs #ifdef AMREX_USE_MPI - if (ParallelContext::NProcsSub() == 1) { + if (amrex::ParallelContext::NProcsSub() == 1) { m_doLoadBalance = 0; } #else @@ -631,31 +640,31 @@ PeleLM::readParameters() if (m_advection_key == "Godunov_PLM") { m_advection_type = "Godunov"; m_Godunov_ppm = 0; - ParmParse ppg("godunov"); + amrex::ParmParse ppg("godunov"); ppg.query("use_forceInTrans", m_Godunov_ForceInTrans); } else if (m_advection_key == "Godunov_PPM") { m_advection_type = "Godunov"; m_Godunov_ppm = 1; m_Godunov_ppm_limiter = PPM::VanLeer; - ParmParse ppg("godunov"); + amrex::ParmParse ppg("godunov"); ppg.query("use_forceInTrans", m_Godunov_ForceInTrans); } else if (m_advection_key == "Godunov_PPM_WENOZ") { m_advection_type = "Godunov"; m_Godunov_ppm = 1; m_Godunov_ppm_limiter = PPM::WENOZ; - ParmParse ppg("godunov"); + amrex::ParmParse ppg("godunov"); ppg.query("use_forceInTrans", m_Godunov_ForceInTrans); } else if (m_advection_key == "Godunov_PPM_NOLIM") { m_advection_type = "Godunov"; m_Godunov_ppm = 1; m_Godunov_ppm_limiter = PPM::NoLimiter; - ParmParse ppg("godunov"); + amrex::ParmParse ppg("godunov"); ppg.query("use_forceInTrans", m_Godunov_ForceInTrans); } else if (m_advection_key == "Godunov_BDS") { m_advection_type = "BDS"; m_Godunov_ppm = 0; } else { - Abort( + amrex::Abort( "Unknown 'advection_scheme'. Recognized options are: Godunov_PLM, " "Godunov_PPM or Godunov_BDS"); } @@ -666,13 +675,13 @@ PeleLM::readParameters() // ----------------------------------------- // Linear solvers tols // ----------------------------------------- - ParmParse ppnproj("nodal_proj"); + amrex::ParmParse ppnproj("nodal_proj"); ppnproj.query("mg_max_coarsening_level", m_nodal_mg_max_coarsening_level); ppnproj.query("atol", m_nodal_mg_atol); ppnproj.query("rtol", m_nodal_mg_rtol); ppnproj.query("hypre_namespace", m_hypre_namespace_nodal); - ParmParse ppmacproj("mac_proj"); + amrex::ParmParse ppmacproj("mac_proj"); ppmacproj.query("mg_max_coarsening_level", m_mac_mg_max_coarsening_level); ppmacproj.query("atol", m_mac_mg_atol); ppmacproj.query("rtol", m_mac_mg_rtol); @@ -693,7 +702,7 @@ PeleLM::readParameters() // ----------------------------------------- // Time stepping control // ----------------------------------------- - ParmParse ppa("amr"); + amrex::ParmParse ppa("amr"); ppa.query("max_wall_time", m_max_wall_time); // hours ppa.query("max_step", m_max_step); ppa.query("stop_time", m_stop_time); @@ -705,7 +714,8 @@ PeleLM::readParameters() ppa.query("dt_change_max", m_dtChangeMax); ppa.query("max_dt", m_max_dt); ppa.query("min_dt", m_min_dt); - m_nfiles = std::max(1, std::min(ParallelDescriptor::NProcs(), 256)); + m_nfiles = + amrex::max(1, amrex::min(amrex::ParallelDescriptor::NProcs(), 256)); ppa.query("n_files", m_nfiles); if (max_level > 0 || (m_doLoadBalance != 0)) { @@ -718,7 +728,7 @@ PeleLM::readParameters() // Default EB refine type is Static pp.query("refine_EB_type", m_EB_refine_type); if (m_EB_refine_type != "Static" && m_EB_refine_type != "Adaptive") { - Abort("refine_EB_type can only be 'Static' or 'Adaptive'"); + amrex::Abort("refine_EB_type can only be 'Static' or 'Adaptive'"); } // Default EB refinement level is max_level m_EB_refine_LevMax = max_level; @@ -796,7 +806,7 @@ PeleLM::readParameters() do_soot_solve = true; pp.query("do_soot_solve", do_soot_solve); if ((m_verbose != 0) && do_soot_solve) { - Print() << "Simulation performed with soot modeling \n"; + amrex::Print() << "Simulation performed with soot modeling \n"; } soot_model->readSootParams(); #endif @@ -804,7 +814,7 @@ PeleLM::readParameters() do_rad_solve = false; pp.query("do_rad_solve", do_rad_solve); if ((m_verbose != 0) && do_rad_solve) { - Print() << "Simulation performed with radiation modeling \n"; + amrex::Print() << "Simulation performed with radiation modeling \n"; } #endif @@ -868,7 +878,7 @@ PeleLM::readIOParameters() { BL_PROFILE_VAR("PeleLMeX::readIOParameters()", readIOParameters); - ParmParse pp("amr"); + amrex::ParmParse pp("amr"); pp.query("check_file", m_check_file); pp.query("check_int", m_check_int); @@ -919,65 +929,65 @@ PeleLM::variablesSetup() // Variables ordering is defined through compiler macro in PeleLMeX_Index.H // Simply print on screen the state layout and append to the stateComponents // list - Print() << "\n"; - Print() << PrettyLine; - Print() << " State components \n"; - Print() << PrettyLine; + amrex::Print() << "\n"; + amrex::Print() << PrettyLine; + amrex::Print() << " State components \n"; + amrex::Print() << PrettyLine; - Print() << " Velocity X: " << VELX; + amrex::Print() << " Velocity X: " << VELX; stateComponents.emplace_back(VELX, "x_velocity"); #if AMREX_SPACEDIM > 1 - Print() << ", Velocity Y: " << VELY; + amrex::Print() << ", Velocity Y: " << VELY; stateComponents.emplace_back(VELY, "y_velocity"); #if AMREX_SPACEDIM > 2 - Print() << ", Velocity Z: " << VELZ; + amrex::Print() << ", Velocity Z: " << VELZ; stateComponents.emplace_back(VELZ, "z_velocity"); #endif #endif - Print() << " \n"; + amrex::Print() << " \n"; if (m_incompressible == 0) { - Print() << " Density: " << DENSITY << "\n"; + amrex::Print() << " Density: " << DENSITY << "\n"; stateComponents.emplace_back(DENSITY, "density"); - Print() << " First species: " << FIRSTSPEC << "\n"; - Vector names; + amrex::Print() << " First species: " << FIRSTSPEC << "\n"; + amrex::Vector names; pele::physics::eos::speciesNames( names, &(eos_parms.host_parm())); for (int n = 0; n < NUM_SPECIES; n++) { stateComponents.emplace_back(FIRSTSPEC + n, "rho.Y(" + names[n] + ")"); reactComponents.emplace_back(n, "I_R(" + names[n] + ")"); } - Print() << " Enthalpy: " << RHOH << "\n"; + amrex::Print() << " Enthalpy: " << RHOH << "\n"; stateComponents.emplace_back(RHOH, "rhoh"); - Print() << " Temperature: " << TEMP << "\n"; + amrex::Print() << " Temperature: " << TEMP << "\n"; stateComponents.emplace_back(TEMP, "temp"); - Print() << " thermo. pressure: " << RHORT << "\n"; + amrex::Print() << " thermo. pressure: " << RHORT << "\n"; stateComponents.emplace_back(RHORT, "RhoRT"); #ifdef PELE_USE_PLASMA - Print() << " nE: " << NE << "\n"; + amrex::Print() << " nE: " << NE << "\n"; stateComponents.emplace_back(NE, "nE"); - Print() << " PhiV: " << PHIV << "\n"; + amrex::Print() << " PhiV: " << PHIV << "\n"; stateComponents.emplace_back(PHIV, "PhiV"); #endif #ifdef PELE_USE_SOOT for (int mom = 0; mom < NUMSOOTVAR; mom++) { std::string sootname = soot_model->sootVariableName(mom); - Print() << " " << sootname << ": " << FIRSTSOOT + mom << "\n"; + amrex::Print() << " " << sootname << ": " << FIRSTSOOT + mom << "\n"; stateComponents.emplace_back(FIRSTSOOT + mom, sootname); } setSootIndx(); #endif #if NUM_ODE > 0 - Print() << " First ODE: " << FIRSTODE << "\n"; + amrex::Print() << " First ODE: " << FIRSTODE << "\n"; ProblemSpecificFunctions::set_ode_names(m_ode_names); if (m_ode_names.size() != NUM_ODE) { - Abort( + amrex::Abort( "ODEQty names improperly set. Adjust set_ode_names in " "ProblemSpecificFunctions or NUM_ODE in GNUMakefile"); } for (int n = 0; n < NUM_ODE; ++n) { if (m_ode_names[n].empty()) { - Abort( + amrex::Abort( "ODEQty names improperly set. Adjust set_ode_names in " "ProblemSpecificFunctions or NUM_ODE in GNUMakefile"); } @@ -987,27 +997,28 @@ PeleLM::variablesSetup() } if (m_incompressible != 0) { - Print() << " => Total number of state variables: " << AMREX_SPACEDIM - << "\n"; + amrex::Print() << " => Total number of state variables: " << AMREX_SPACEDIM + << "\n"; } else { - Print() << " => Total number of state variables: " << NVAR << "\n"; + amrex::Print() << " => Total number of state variables: " << NVAR << "\n"; } if (m_nAux > 0) { for (int n = 0; n < m_nAux; n++) { - Print() << " Auxiliary " + std::to_string(n + 1) + ": " << m_aux_names[n] - << "\n"; - Print() << " Advective: " << m_aux_advect[n] << "\n"; - Print() << " Conservative: " << m_AdvTypeAux[n] << "\n"; - Print() << " Diffusive: " << m_DiffTypeAux[n]; + amrex::Print() << " Auxiliary " + std::to_string(n + 1) + ": " + << m_aux_names[n] << "\n"; + amrex::Print() << " Advective: " << m_aux_advect[n] << "\n"; + amrex::Print() << " Conservative: " << m_AdvTypeAux[n] << "\n"; + amrex::Print() << " Diffusive: " << m_DiffTypeAux[n]; if (m_aux_Schmidt[n] > 0) { - Print() << " - Schmidt number: " << m_aux_Schmidt[n]; + amrex::Print() << " - Schmidt number: " << m_aux_Schmidt[n]; } - Print() << "\n"; + amrex::Print() << "\n"; } - Print() << " => Total number of auxiliary variables: " << m_nAux << "\n"; + amrex::Print() << " => Total number of auxiliary variables: " << m_nAux + << "\n"; } - Print() << PrettyLine; - Print() << "\n"; + amrex::Print() << PrettyLine; + amrex::Print() << "\n"; //---------------------------------------------------------------- // Set advection/diffusion types @@ -1064,7 +1075,7 @@ PeleLM::variablesSetup() // ----------------------------------------- // Combustion // ----------------------------------------- - ParmParse pp("peleLM"); + amrex::ParmParse pp("peleLM"); std::string fuel_name; pp.query("fuel_name", fuel_name); fuel_name = "rho.Y(" + fuel_name + ")"; @@ -1113,11 +1124,11 @@ PeleLM::readGridFile( } for (int lev = 1; lev <= in_finest; lev++) { - BoxList bl; + amrex::BoxList bl; is >> ngrid; STRIP; for (int i = 0; i < ngrid; i++) { - Box bx; + amrex::Box bx; is >> bx; STRIP; bx.refine(ref_ratio[lev - 1]); @@ -1138,24 +1149,24 @@ PeleLM::derivedSetup() if (m_incompressible == 0) { // Get species names - Vector spec_names; + amrex::Vector spec_names; pele::physics::eos::speciesNames( spec_names, &(eos_parms.host_parm())); // Set species mass fractions - Vector var_names_massfrac(NUM_SPECIES); + amrex::Vector var_names_massfrac(NUM_SPECIES); for (int n = 0; n < NUM_SPECIES; n++) { var_names_massfrac[n] = "Y(" + spec_names[n] + ")"; } derive_lst.add( - "mass_fractions", IndexType::TheCellType(), NUM_SPECIES, + "mass_fractions", amrex::IndexType::TheCellType(), NUM_SPECIES, var_names_massfrac, pelelmex_dermassfrac, the_same_box); for (int n = 0; n < NUM_SPECIES; n++) { var_names_massfrac[n] = "X(" + spec_names[n] + ")"; } derive_lst.add( - "mole_fractions", IndexType::TheCellType(), NUM_SPECIES, + "mole_fractions", amrex::IndexType::TheCellType(), NUM_SPECIES, var_names_massfrac, pelelmex_dermolefrac, the_same_box); // Species diffusion coefficients @@ -1168,74 +1179,77 @@ PeleLM::derivedSetup() var_names_massfrac[n + NUM_SPECIES] = "theta_" + spec_names[n]; } derive_lst.add( - "diffcoeff", IndexType::TheCellType(), 2 * NUM_SPECIES, + "diffcoeff", amrex::IndexType::TheCellType(), 2 * NUM_SPECIES, var_names_massfrac, pelelmex_derdiffc, the_same_box); } else { derive_lst.add( - "diffcoeff", IndexType::TheCellType(), NUM_SPECIES, var_names_massfrac, - pelelmex_derdiffc, the_same_box); + "diffcoeff", amrex::IndexType::TheCellType(), NUM_SPECIES, + var_names_massfrac, pelelmex_derdiffc, the_same_box); } // Rho - sum rhoYs derive_lst.add( - "rhominsumrhoY", IndexType::TheCellType(), 1, pelelmex_derrhomrhoy, + "rhominsumrhoY", amrex::IndexType::TheCellType(), 1, pelelmex_derrhomrhoy, the_same_box); // Heat Release derive_lst.add( - "HeatRelease", IndexType::TheCellType(), 1, pelelmex_derheatrelease, - the_same_box); + "HeatRelease", amrex::IndexType::TheCellType(), 1, + pelelmex_derheatrelease, the_same_box); // Thermal diffusivity derive_lst.add( - "lambda", IndexType::TheCellType(), 1, pelelmex_derlambda, the_same_box); + "lambda", amrex::IndexType::TheCellType(), 1, pelelmex_derlambda, + the_same_box); // Mixture fraction derive_lst.add( - "mixture_fraction", IndexType::TheCellType(), 1, pelelmex_dermixfrac, - the_same_box); + "mixture_fraction", amrex::IndexType::TheCellType(), 1, + pelelmex_dermixfrac, the_same_box); // Progress variable derive_lst.add( - "progress_variable", IndexType::TheCellType(), 1, pelelmex_derprogvar, - the_same_box); + "progress_variable", amrex::IndexType::TheCellType(), 1, + pelelmex_derprogvar, the_same_box); } // Distribution Map derive_lst.add( - "DistributionMap", IndexType::TheCellType(), 1, pelelmex_derdmap, + "DistributionMap", amrex::IndexType::TheCellType(), 1, pelelmex_derdmap, the_same_box); // Turbulent Forcing Terms - Vector var_names_turbforcing = { + amrex::Vector var_names_turbforcing = { AMREX_D_DECL("forcex", "forcey", "forcez")}; derive_lst.add( - "turbforces", IndexType::TheCellType(), AMREX_SPACEDIM, + "turbforces", amrex::IndexType::TheCellType(), AMREX_SPACEDIM, var_names_turbforcing, pelelmex_derturbforcing, the_same_box); // Cell average pressure derive_lst.add( - "avg_pressure", IndexType::TheCellType(), 1, pelelmex_deravgpress, + "avg_pressure", amrex::IndexType::TheCellType(), 1, pelelmex_deravgpress, the_same_box); // Viscosity derive_lst.add( - "viscosity", IndexType::TheCellType(), 1, pelelmex_dervisc, the_same_box); + "viscosity", amrex::IndexType::TheCellType(), 1, pelelmex_dervisc, + the_same_box); // Velocity magnitude derive_lst.add( - "mag_vel", IndexType::TheCellType(), 1, pelelmex_dermgvel, the_same_box); + "mag_vel", amrex::IndexType::TheCellType(), 1, pelelmex_dermgvel, + the_same_box); // Vorticity magnitude derive_lst.add( - "mag_vort", IndexType::TheCellType(), 1, pelelmex_dermgvort, + "mag_vort", amrex::IndexType::TheCellType(), 1, pelelmex_dermgvort, grow_box_by_two); // Spatial coordinates { - Vector var_names({AMREX_D_DECL("x", "y", "z")}); + amrex::Vector var_names({AMREX_D_DECL("x", "y", "z")}); derive_lst.add( - "coordinates", IndexType::TheCellType(), AMREX_SPACEDIM, var_names, + "coordinates", amrex::IndexType::TheCellType(), AMREX_SPACEDIM, var_names, pelelmex_dercoord, the_same_box); } @@ -1243,20 +1257,20 @@ PeleLM::derivedSetup() { const int vort_ncomp = 2 * AMREX_SPACEDIM - 3; #if (AMREX_SPACEDIM == 2) - Vector var_names({"VortZ"}); + amrex::Vector var_names({"VortZ"}); #elif (AMREX_SPACEDIM == 3) - Vector var_names({"VortX", "VortY", "VortZ"}); + amrex::Vector var_names({"VortX", "VortY", "VortZ"}); #endif derive_lst.add( - "vorticity", IndexType::TheCellType(), vort_ncomp, var_names, + "vorticity", amrex::IndexType::TheCellType(), vort_ncomp, var_names, pelelmex_dervort, grow_box_by_two); } // UserDefined derived { - Vector var_names = pelelmex_setuserderives(); + amrex::Vector var_names = pelelmex_setuserderives(); derive_lst.add( - "derUserDefined", IndexType::TheCellType(), + "derUserDefined", amrex::IndexType::TheCellType(), static_cast(var_names.size()), var_names, pelelmex_deruserdef, the_same_box); } @@ -1264,61 +1278,65 @@ PeleLM::derivedSetup() #if (AMREX_SPACEDIM == 3) // Q-criterion derive_lst.add( - "Qcrit", IndexType::TheCellType(), 1, pelelmex_derQcrit, grow_box_by_two); + "Qcrit", amrex::IndexType::TheCellType(), 1, pelelmex_derQcrit, + grow_box_by_two); #endif // Kinetic energy derive_lst.add( - "kinetic_energy", IndexType::TheCellType(), 1, pelelmex_derkineticenergy, - the_same_box); + "kinetic_energy", amrex::IndexType::TheCellType(), 1, + pelelmex_derkineticenergy, the_same_box); // Enstrophy derive_lst.add( - "enstrophy", IndexType::TheCellType(), 1, pelelmex_derenstrophy, + "enstrophy", amrex::IndexType::TheCellType(), 1, pelelmex_derenstrophy, grow_box_by_two); #ifdef USE_MANIFOLD_EOS auto& mani_data = eos_parms.host_only_parm().manfunc_par->host_parm(); const int nmanivar = mani_data.Nvar; - Vector var_names_maniout(nmanivar); + amrex::Vector var_names_maniout(nmanivar); for (int n = 0; n < nmanivar; n++) { std::string nametmp = std::string( &(mani_data.varnames)[n * mani_data.len_str], mani_data.len_str); var_names_maniout[n] = "MANI_" + amrex::trim(nametmp); } derive_lst.add( - "maniout", IndexType::TheCellType(), nmanivar, var_names_maniout, + "maniout", amrex::IndexType::TheCellType(), nmanivar, var_names_maniout, pelelmex_dermaniout, the_same_box); #endif #ifdef PELE_USE_PLASMA // Charge distribution derive_lst.add( - "chargedistrib", IndexType::TheCellType(), 1, pelelmex_derchargedist, + "chargedistrib", amrex::IndexType::TheCellType(), 1, pelelmex_derchargedist, the_same_box); // Electric field derive_lst.add( - "efieldx", IndexType::TheCellType(), 1, pelelmex_derefx, grow_box_by_one); + "efieldx", amrex::IndexType::TheCellType(), 1, pelelmex_derefx, + grow_box_by_one); #if (AMREX_SPACEDIM > 1) derive_lst.add( - "efieldy", IndexType::TheCellType(), 1, pelelmex_derefy, grow_box_by_one); + "efieldy", amrex::IndexType::TheCellType(), 1, pelelmex_derefy, + grow_box_by_one); #if (AMREX_SPACEDIM > 2) derive_lst.add( - "efieldz", IndexType::TheCellType(), 1, pelelmex_derefz, grow_box_by_one); + "efieldz", amrex::IndexType::TheCellType(), 1, pelelmex_derefz, + grow_box_by_one); #endif #endif // Lorentz forces derive_lst.add( - "LorentzFx", IndexType::TheCellType(), 1, pelelmex_derLorentzx, + "LorentzFx", amrex::IndexType::TheCellType(), 1, pelelmex_derLorentzx, grow_box_by_one); #if (AMREX_SPACEDIM > 1) derive_lst.add( - "LorentzFy", IndexType::TheCellType(), 1, pelelmex_derLorentzy, + "LorentzFy", amrex::IndexType::TheCellType(), 1, pelelmex_derLorentzy, grow_box_by_one); #if (AMREX_SPACEDIM > 2) derive_lst.add( - "LorentzFz", IndexType::TheCellType(), 1, pelelmex_derLorentzz, + "LorentzFz", amrex::IndexType::TheCellType(), 1, pelelmex_derLorentzz, grow_box_by_one); #endif #endif @@ -1340,56 +1358,56 @@ PeleLM::evaluateSetup() BL_PROFILE("PeleLMeX::evaluateSetup()"); // Get species names - Vector spec_names; + amrex::Vector spec_names; pele::physics::eos::speciesNames( spec_names, &(eos_parms.host_parm())); // divU - evaluate_lst.add("divU", IndexType::TheCellType(), 1, the_same_box); + evaluate_lst.add("divU", amrex::IndexType::TheCellType(), 1, the_same_box); // projected velocity field { - Vector var_names = { + amrex::Vector var_names = { AMREX_D_DECL("x_velProj", "y_velProj", "z_velProj")}; evaluate_lst.add( - "velProj", IndexType::TheCellType(), AMREX_SPACEDIM, var_names, + "velProj", amrex::IndexType::TheCellType(), AMREX_SPACEDIM, var_names, the_same_box); } // velocity force { - Vector var_names = { + amrex::Vector var_names = { AMREX_D_DECL("x_velForce", "y_velForce", "z_velForce")}; evaluate_lst.add( - "velForce", IndexType::TheCellType(), AMREX_SPACEDIM, var_names, + "velForce", amrex::IndexType::TheCellType(), AMREX_SPACEDIM, var_names, the_same_box); } // divTau { - Vector var_names = { + amrex::Vector var_names = { AMREX_D_DECL("x_divTau", "y_divTau", "z_divTau")}; evaluate_lst.add( - "divTau", IndexType::TheCellType(), AMREX_SPACEDIM, var_names, + "divTau", amrex::IndexType::TheCellType(), AMREX_SPACEDIM, var_names, the_same_box); } // scalar diffusion term { - Vector var_names(NUM_SPECIES + 2); + amrex::Vector var_names(NUM_SPECIES + 2); for (int n = 0; n < NUM_SPECIES; n++) { var_names[n] = "D(" + spec_names[n] + ")"; } var_names[NUM_SPECIES] = "D(RhoH)"; var_names[NUM_SPECIES + 1] = "D(Temp)"; evaluate_lst.add( - "diffTerm", IndexType::TheCellType(), NUM_SPECIES + 2, var_names, + "diffTerm", amrex::IndexType::TheCellType(), NUM_SPECIES + 2, var_names, the_same_box); } // advection terms { - Vector var_names( + amrex::Vector var_names( NVAR - 2); // Skip temperature and RhoRT, unused AMREX_D_TERM( var_names[VELX] = "A(VELX)";, var_names[VELY] = "A(VELY)"; @@ -1400,12 +1418,13 @@ PeleLM::evaluateSetup() } var_names[RHOH] = "A(RhoH)"; evaluate_lst.add( - "advTerm", IndexType::TheCellType(), NVAR - 2, var_names, the_same_box); + "advTerm", amrex::IndexType::TheCellType(), NVAR - 2, var_names, + the_same_box); } // Chemical state and external chem. forcing (used in ReactEval) { - Vector var_names(2 * (NUM_SPECIES + 1) + 1); + amrex::Vector var_names(2 * (NUM_SPECIES + 1) + 1); for (int n = 0; n < NUM_SPECIES; n++) { var_names[n] = "rhoY(" + spec_names[n] + ")"; } @@ -1416,31 +1435,32 @@ PeleLM::evaluateSetup() } var_names[2 * NUM_SPECIES + 2] = "F_rhoH"; evaluate_lst.add( - "chemTest", IndexType::TheCellType(), 2 * (NUM_SPECIES + 1) + 1, + "chemTest", amrex::IndexType::TheCellType(), 2 * (NUM_SPECIES + 1) + 1, var_names, the_same_box); } // instantaneous reaction rate { - Vector var_names(NUM_SPECIES); + amrex::Vector var_names(NUM_SPECIES); for (int n = 0; n < NUM_SPECIES; n++) { var_names[n] = "I_R(" + spec_names[n] + ")"; } evaluate_lst.add( - "instRR", IndexType::TheCellType(), NUM_SPECIES, var_names, the_same_box); + "instRR", amrex::IndexType::TheCellType(), NUM_SPECIES, var_names, + the_same_box); } // cell-centered transport coefficients { - Vector var_names(NUM_SPECIES + 2); + amrex::Vector var_names(NUM_SPECIES + 2); for (int n = 0; n < NUM_SPECIES; n++) { var_names[n] = "rhoD(" + spec_names[n] + ")"; } var_names[NUM_SPECIES] = "Lamdba"; var_names[NUM_SPECIES + 1] = "Mu"; evaluate_lst.add( - "transportCC", IndexType::TheCellType(), NUM_SPECIES + 2, var_names, - the_same_box); + "transportCC", amrex::IndexType::TheCellType(), NUM_SPECIES + 2, + var_names, the_same_box); } } @@ -1450,40 +1470,40 @@ PeleLM::taggingSetup() BL_PROFILE("PeleLMeX::taggingSetup()"); std::string amr_prefix = "amr"; - ParmParse ppamr(amr_prefix); + amrex::ParmParse ppamr(amr_prefix); - Vector refinement_indicators; + amrex::Vector refinement_indicators; ppamr.queryarr( "refinement_indicators", refinement_indicators, 0, ppamr.countval("refinement_indicators")); for (const auto& refinement_indicator : refinement_indicators) { std::string ref_prefix = amr_prefix + "." + refinement_indicator; - ParmParse ppr(ref_prefix); + amrex::ParmParse ppr(ref_prefix); // Tag a given box - RealBox realbox; + amrex::RealBox realbox; if (ppr.countval("in_box_lo") != 0) { - Vector box_lo(AMREX_SPACEDIM); - Vector box_hi(AMREX_SPACEDIM); + amrex::Vector box_lo(AMREX_SPACEDIM); + amrex::Vector box_hi(AMREX_SPACEDIM); ppr.getarr("in_box_lo", box_lo, 0, static_cast(box_lo.size())); ppr.getarr("in_box_hi", box_hi, 0, static_cast(box_hi.size())); - realbox = RealBox(box_lo.data(), box_hi.data()); + realbox = amrex::RealBox(box_lo.data(), box_hi.data()); } - AMRErrorTagInfo info; + amrex::AMRErrorTagInfo info; if (realbox.ok()) { info.SetRealBox(realbox); } if (ppr.countval("start_time") > 0) { - Real min_time; + amrex::Real min_time; ppr.get("start_time", min_time); info.SetMinTime(min_time); } if (ppr.countval("end_time") > 0) { - Real max_time; + amrex::Real max_time; ppr.get("end_time", max_time); info.SetMaxTime(max_time); } @@ -1496,41 +1516,45 @@ PeleLM::taggingSetup() bool itexists = false; if (ppr.countval("value_greater") != 0) { - Real value; + amrex::Real value; ppr.get("value_greater", value); std::string field; ppr.get("field_name", field); - errTags.push_back(AMRErrorTag(value, AMRErrorTag::GREATER, field, info)); + errTags.push_back( + amrex::AMRErrorTag(value, amrex::AMRErrorTag::GREATER, field, info)); itexists = derive_lst.canDerive(field) || isStateVariable(field) || isReactVariable(field); } else if (ppr.countval("value_less") != 0) { - Real value; + amrex::Real value; ppr.get("value_less", value); std::string field; ppr.get("field_name", field); - errTags.push_back(AMRErrorTag(value, AMRErrorTag::LESS, field, info)); + errTags.push_back( + amrex::AMRErrorTag(value, amrex::AMRErrorTag::LESS, field, info)); itexists = derive_lst.canDerive(field) || isStateVariable(field) || isReactVariable(field); } else if (ppr.countval("vorticity_greater") != 0) { - Real value; + amrex::Real value; ppr.get("vorticity_greater", value); const std::string field = "mag_vort"; - errTags.push_back(AMRErrorTag(value, AMRErrorTag::VORT, field, info)); + errTags.push_back( + amrex::AMRErrorTag(value, amrex::AMRErrorTag::VORT, field, info)); itexists = derive_lst.canDerive(field) || isStateVariable(field) || isReactVariable(field); } else if (ppr.countval("adjacent_difference_greater") != 0) { - Real value; + amrex::Real value; ppr.get("adjacent_difference_greater", value); std::string field; ppr.get("field_name", field); - errTags.push_back(AMRErrorTag(value, AMRErrorTag::GRAD, field, info)); + errTags.push_back( + amrex::AMRErrorTag(value, amrex::AMRErrorTag::GRAD, field, info)); itexists = derive_lst.canDerive(field) || isStateVariable(field) || isReactVariable(field); } else if (realbox.ok()) { - errTags.push_back(AMRErrorTag(info)); + errTags.push_back(amrex::AMRErrorTag(info)); itexists = true; } else { - Abort( + amrex::Abort( std::string( "Unrecognized refinement indicator for " + refinement_indicator) .c_str()); @@ -1549,7 +1573,8 @@ PeleLM::resizeArray() { if (m_verbose != 0) { - Print() << " Initializing data for " << max_level + 1 << " levels \n"; + amrex::Print() << " Initializing data for " << max_level + 1 + << " levels \n"; } // State data diff --git a/Source/PeleLMeX_Soot.cpp b/Source/PeleLMeX_Soot.cpp index 04e96d9fc..f32a601e2 100644 --- a/Source/PeleLMeX_Soot.cpp +++ b/Source/PeleLMeX_Soot.cpp @@ -1,11 +1,8 @@ - #ifdef PELE_USE_SOOT #include #include #include "SootModel.H" -using namespace amrex; - void PeleLM::setSootIndx() { @@ -32,18 +29,19 @@ PeleLM::cleanupSootModel() } void -PeleLM::computeSootSource(const PeleLM::TimeStamp& a_timestamp, const Real a_dt) +PeleLM::computeSootSource( + const PeleLM::TimeStamp& a_timestamp, const amrex::Real a_dt) { bool pres_term = false; // Do not include change in pressure in energy for (int lev = 0; lev <= finest_level; lev++) { auto* ldata_p = getLevelDataPtr(lev, a_timestamp); - Real time = getTime(lev, a_timestamp); + amrex::Real time = getTime(lev, a_timestamp); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*(m_extSource[lev]), TilingIfNotGPU()); mfi.isValid(); - ++mfi) { - Box const& gbx = mfi.growntilebox(); + for (amrex::MFIter mfi(*(m_extSource[lev]), amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + amrex::Box const& gbx = mfi.growntilebox(); auto const& state_arr = ldata_p->state.const_array(mfi, DENSITY); auto const& mu = ldata_p->visc_cc.const_array(mfi, 0); auto const& source_arr = m_extSource[lev]->array(mfi, DENSITY); @@ -59,15 +57,16 @@ PeleLM::clipSootMoments() for (int lev = 0; lev <= finest_level; lev++) { auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - Box const& gbx = mfi.tilebox(); + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + amrex::Box const& gbx = mfi.tilebox(); auto const& state_arr = ldata_p->state.array(mfi, FIRSTSOOT); SootData* sd = soot_model->getSootData_d(); amrex::ParallelFor( gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - GpuArray moments; + amrex::GpuArray moments; for (int mom = 0; mom < NUM_SOOT_MOMENTS + 1; mom++) { moments[mom] = state_arr(i, j, k, mom); } @@ -86,13 +85,13 @@ void PeleLM::addSootDerivePlotVars(PeleLMDeriveList& derive_lst) { // Add in soot variables - Vector sootNames = {"rho_soot", "sum_rho_soot"}; + amrex::Vector sootNames = {"rho_soot", "sum_rho_soot"}; derive_lst.add( "soot_vars", IndexType::TheCellType(), sootNames.size(), sootNames, soot_genvars, PeleLMDeriveRec::TheSameBox); // Variables associated with the second mode (large particles) - Vector large_part_names = {"NL", "soot_V_L", "soot_S_L"}; + amrex::Vector large_part_names = {"NL", "soot_V_L", "soot_S_L"}; derive_lst.add( "soot_large_particles", IndexType::TheCellType(), large_part_names.size(), large_part_names, soot_largeparticledata, PeleLMDeriveRec::TheSameBox); diff --git a/Source/PeleLMeX_SprayParticles.cpp b/Source/PeleLMeX_SprayParticles.cpp index 76c73a74a..76604cf18 100644 --- a/Source/PeleLMeX_SprayParticles.cpp +++ b/Source/PeleLMeX_SprayParticles.cpp @@ -1,4 +1,3 @@ - #include #ifdef PELE_USE_SPRAY @@ -6,20 +5,18 @@ #include #include -using namespace amrex; - namespace { // Indices for spray source MultiFab int sprayMomSrcIndx = VELX; int sprayRhoSrcIndx = DENSITY; int spraySpecSrcIndx = FIRSTSPEC; int sprayEngSrcIndx = FIRSTSPEC + SPRAY_FUEL_NUM; -Vector spray_cfl; -Vector spray_state_ghosts; -Vector spray_source_ghosts; -Vector spray_ghost_num; -Vector prev_state; -Vector prev_source; +amrex::Vector spray_cfl; +amrex::Vector spray_state_ghosts; +amrex::Vector spray_source_ghosts; +amrex::Vector spray_ghost_num; +amrex::Vector prev_state; +amrex::Vector prev_source; bool mesh_regrid = true; int spray_verbose = 0; @@ -32,21 +29,21 @@ bool PeleLM::do_spray_particles = true; // momentum + density + fuel species + enthalpy int PeleLM::num_spray_src = AMREX_SPACEDIM + 2 + SPRAY_FUEL_NUM; -Real +amrex::Real PeleLM::SprayEstDt() { - Real estdt = 1.0e200; + amrex::Real estdt = 1.0e200; if (!do_spray_particles || SprayPC == nullptr) { return estdt; } BL_PROFILE("PeleLMeX::SprayEstDt()"); for (int lev = 0; lev <= finest_level; ++lev) { - Real estdt_lev = SprayPC->estTimestep(lev); + amrex::Real estdt_lev = SprayPC->estTimestep(lev); if (estdt_lev > 0. && estdt_lev < estdt) { estdt = estdt_lev; } else if (lev > 0) { // Limit time step as if particles are on finest level - estdt /= Real(refRatio(lev - 1)[0]); + estdt /= amrex::Real(refRatio(lev - 1)[0]); } } return estdt; @@ -55,7 +52,7 @@ PeleLM::SprayEstDt() void PeleLM::SprayReadParameters() { - ParmParse pp("peleLM"); + amrex::ParmParse pp("peleLM"); pp.query("do_spray_particles", do_spray_particles); if (!do_spray_particles) { @@ -75,7 +72,7 @@ PeleLM::SpraySetup() // There must be at least as many fuel species in the spray as // there are species in the fluid if (SPRAY_FUEL_NUM > NUM_SPECIES) { - Abort("Cannot have more spray fuel species than fluid species"); + amrex::Abort("Cannot have more spray fuel species than fluid species"); } SprayParticleContainer::spraySetup(m_gravity.data()); SprayComps scomps; @@ -160,27 +157,28 @@ PeleLM::SprayInit() std::string restart_file = (m_restart_chkfile.empty()) ? m_restart_pltfile : m_restart_chkfile; std::string restart_partfile = restart_file + "/particles"; - if (!FileSystem::Exists(restart_partfile)) { + if (!amrex::FileSystem::Exists(restart_partfile)) { restart_file = ""; if (!m_restart_chkfile.empty() || !m_restart_pltfile.empty()) { std::string warn_msg = "Restart file does not contain particles. " "Particles are being initialized from scratch."; - Warning(warn_msg); + amrex::Warning(warn_msg); } } SprayPC->SprayInitialize(restart_file); SprayPostRegrid(); SprayInjectRedist(); if (spray_verbose >= 1) { - Print() << "Total number of initial particles " - << SprayPC->TotalNumberOfParticles(false, false) << std::endl; + amrex::Print() << "Total number of initial particles " + << SprayPC->TotalNumberOfParticles(false, false) + << std::endl; } } // Sets the number of ghost cells for the state, source, and ghost particles // Also creates and fills the state data void -PeleLM::SpraySetState(const Real& a_flow_dt) +PeleLM::SpraySetState(const amrex::Real& a_flow_dt) { if (!do_spray_particles) { return; @@ -195,14 +193,14 @@ PeleLM::SpraySetState(const Real& a_flow_dt) // Determine the max velocity of particles at each level // This is necessary because not every level will be aware of the fastest // particle for when ghost and virtual particles are present - Real max_vel = 0.; + amrex::Real max_vel = 0.; spray_state_ghosts[0] = 0; for (int lev = 0; lev <= finest_level; ++lev) { auto const dx = geom[lev].CellSizeArray(); // Extract velocity and CFL from a given spray CFL - Real cur_spray_cfl = SprayParticleContainer::spray_cfl; - Real spraydt_lev = SprayPC->estTimestep(lev); - Real vel_lev = cur_spray_cfl * dx[0] / spraydt_lev; + amrex::Real cur_spray_cfl = SprayParticleContainer::spray_cfl; + amrex::Real spraydt_lev = SprayPC->estTimestep(lev); + amrex::Real vel_lev = cur_spray_cfl * dx[0] / spraydt_lev; max_vel = amrex::max(max_vel, vel_lev); if (spraydt_lev > 0.) { spray_cfl[lev] = cur_spray_cfl / spraydt_lev * a_flow_dt; @@ -230,10 +228,11 @@ PeleLM::SpraySetState(const Real& a_flow_dt) if ( mesh_regrid || prev_state[lev] != state_ghosts || prev_source[lev] != source_ghosts) { - m_spraystate[lev] = std::make_unique( - grids[lev], dmap[lev], NVAR, state_ghosts, MFInfo(), *m_factory[lev]); - m_spraysource[lev] = std::make_unique( - grids[lev], dmap[lev], num_spray_src, source_ghosts, MFInfo(), + m_spraystate[lev] = std::make_unique( + grids[lev], dmap[lev], NVAR, state_ghosts, amrex::MFInfo(), + *m_factory[lev]); + m_spraysource[lev] = std::make_unique( + grids[lev], dmap[lev], num_spray_src, source_ghosts, amrex::MFInfo(), *m_factory[lev]); } fillpatch_state(lev, m_cur_time, *(m_spraystate[lev]), state_ghosts); @@ -245,40 +244,41 @@ PeleLM::SpraySetState(const Real& a_flow_dt) void PeleLM::SprayAddSource(const int level) { - MultiFab& source = *(m_spraysource[level]); - MultiFab& extsource = *(m_extSource[level]); + amrex::MultiFab& source = *(m_spraysource[level]); + amrex::MultiFab& extsource = *(m_extSource[level]); const int eghosts = extsource.nGrow(); SprayComps scomps = SprayParticleContainer::getSprayComps(); - MultiFab::Add( + amrex::MultiFab::Add( extsource, source, scomps.rhoSrcIndx, scomps.rhoIndx, 1, eghosts); - MultiFab::Add( + amrex::MultiFab::Add( extsource, source, scomps.engSrcIndx, scomps.engIndx, 1, eghosts); - MultiFab::Add( + amrex::MultiFab::Add( extsource, source, scomps.momSrcIndx, scomps.momIndx, AMREX_SPACEDIM, eghosts); for (int n = 0; n < SPRAY_FUEL_NUM; ++n) { const int dstcomp = scomps.specIndx + SprayParticleContainer::getFuelIndx(n); - MultiFab::Add( + amrex::MultiFab::Add( extsource, source, scomps.specSrcIndx + n, dstcomp, 1, eghosts); } } void -PeleLM::SprayMKD(const Real time, const Real dt) +PeleLM::SprayMKD(const amrex::Real time, const amrex::Real dt) { if (!do_spray_particles) { return; } if (spray_verbose != 0) { - Print() << "moveKickDrift ... updating particle positions and velocity\n"; + amrex::Print() + << "moveKickDrift ... updating particle positions and velocity\n"; } BL_PROFILE("PeleLMeX::SprayMKD()"); // Setup the virtual particles that represent particles on finer levels setupVirtualParticles(0); for (int lev = 0; lev <= finest_level; ++lev) { if (spray_verbose > 1) { - Print() << "SprayMKDLevel " << lev << std::endl; + amrex::Print() << "SprayMKDLevel " << lev << std::endl; } m_spraysource[lev]->setVal(0.); SprayMKDLevel(lev, time, dt); @@ -289,7 +289,8 @@ PeleLM::SprayMKD(const Real time, const Real dt) } void -PeleLM::SprayMKDLevel(const int level, const Real time, const Real dt) +PeleLM::SprayMKDLevel( + const int level, const amrex::Real time, const amrex::Real dt) { if (level < finest_level) { // Make a copy of the particles on this level into ghost particles @@ -302,8 +303,8 @@ PeleLM::SprayMKDLevel(const int level, const Real time, const Real dt) // the new time auto const* ltransparm = PeleLM::trans_parms.device_parm(); - MultiFab& state = *(m_spraystate[level]); - MultiFab& source = *(m_spraysource[level]); + amrex::MultiFab& state = *(m_spraystate[level]); + amrex::MultiFab& source = *(m_spraysource[level]); const int state_ghosts = spray_state_ghosts[level]; const int source_ghosts = spray_source_ghosts[level]; bool isVirt = false; @@ -333,8 +334,8 @@ PeleLM::SprayMKDLevel(const int level, const Real time, const Real dt) void PeleLM::SprayPostRegrid() { - static Vector ba_spray; - static Vector dm_spray; + static amrex::Vector ba_spray; + static amrex::Vector dm_spray; bool changed = false; if (ba_spray.size() != finest_level + 1) { ba_spray.resize(finest_level + 1); @@ -369,15 +370,16 @@ void PeleLM::SprayInjectRedist() { BL_PROFILE("PeleLMeX::SprayInjectRedist"); - Long prev_count = 0; + amrex::Long prev_count = 0; if (spray_verbose >= 3) { prev_count = SprayPC->TotalNumberOfParticles(true, false); } bool injected = false; for (int lev = 0; lev <= finest_level; ++lev) { - int nstep = 0; // Unused - Real cur_time = m_t_new[lev]; // Still the time from the last time step - Real dt = m_dt; + int nstep = 0; // Unused + amrex::Real cur_time = + m_t_new[lev]; // Still the time from the last time step + amrex::Real dt = m_dt; bool lev_injected = SprayPC->injectParticles(cur_time, dt, nstep, lev, finest_level); if (lev_injected) { @@ -387,10 +389,10 @@ PeleLM::SprayInjectRedist() // We must redistribute after each time step SprayPC->Redistribute(); if (spray_verbose >= 3 && injected) { - Long new_count = SprayPC->TotalNumberOfParticles(true, false); - Long num_inj = new_count - prev_count; - Print() << "Injected " << num_inj << " particles at time " << m_t_new[0] - << std::endl; + amrex::Long new_count = SprayPC->TotalNumberOfParticles(true, false); + amrex::Long num_inj = new_count - prev_count; + amrex::Print() << "Injected " << num_inj << " particles at time " + << m_t_new[0] << std::endl; } } diff --git a/Source/PeleLMeX_Tagging.cpp b/Source/PeleLMeX_Tagging.cpp index 637651015..e1ace5a1c 100644 --- a/Source/PeleLMeX_Tagging.cpp +++ b/Source/PeleLMeX_Tagging.cpp @@ -4,10 +4,9 @@ #include #endif -using namespace amrex; - void -PeleLM::ErrorEst(int lev, TagBoxArray& tags, Real time, int /*ng*/) +PeleLM::ErrorEst( + int lev, amrex::TagBoxArray& tags, amrex::Real time, int /*ng*/) { BL_PROFILE("PeleLMeX::ErrorEst()"); @@ -17,31 +16,33 @@ PeleLM::ErrorEst(int lev, TagBoxArray& tags, Real time, int /*ng*/) if ( (m_EB_refine_type == "Static" && lev < m_EB_refine_LevMax) || (m_EB_refine_type == "Adaptive" && lev < m_EB_refine_LevAdapt)) { - const MultiFab& state = (getLevelDataPtr(lev, AmrNewTime))->state; + const amrex::MultiFab& state = (getLevelDataPtr(lev, AmrNewTime))->state; TagCutCells(tags, state); } #endif for (const auto& errTag : errTags) { - std::unique_ptr mf; + std::unique_ptr mf; if (!errTag.Field().empty()) { mf = deriveComp(errTag.Field(), time, lev, errTag.NGrow()); } - errTag(tags, mf.get(), TagBox::CLEAR, TagBox::SET, time, lev, geom[lev]); + errTag( + tags, mf.get(), amrex::TagBox::CLEAR, amrex::TagBox::SET, time, lev, + geom[lev]); } #ifdef AMREX_USE_EB // Untag covered cells #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(tags, TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(tags, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { const auto& bx = mfi.tilebox(); auto tag = tags.array(mfi); auto vfrac = EBFactory(lev).getVolFrac().const_array(mfi); amrex::ParallelFor(bx, [=] AMREX_GPU_HOST_DEVICE(int i, int j, int k) { if (vfrac(i, j, k) <= 0.0) { - tag(i, j, k) = TagBox::CLEAR; + tag(i, j, k) = amrex::TagBox::CLEAR; } }); } @@ -49,30 +50,32 @@ PeleLM::ErrorEst(int lev, TagBoxArray& tags, Real time, int /*ng*/) // Untag cell close to EB if (m_EB_refine_type == "Static" && lev >= m_EB_refine_LevMax) { // Get distance function at current level - MultiFab signDist(grids[lev], dmap[lev], 1, 0, MFInfo(), EBFactory(lev)); + amrex::MultiFab signDist( + grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), EBFactory(lev)); getEBDistance(lev, signDist); // Estimate how far I need to derefine - Real diagFac = std::sqrt(2.0) * m_derefineEBBuffer; - Real clearTagDist = Geom(m_EB_refine_LevMax).CellSize(0) * - static_cast(nErrorBuf(m_EB_refine_LevMax)) * - diagFac; + amrex::Real diagFac = std::sqrt(2.0) * m_derefineEBBuffer; + amrex::Real clearTagDist = + Geom(m_EB_refine_LevMax).CellSize(0) * + static_cast(nErrorBuf(m_EB_refine_LevMax)) * diagFac; for (int ilev = m_EB_refine_LevMax + 1; ilev <= finest_level; ++ilev) { - clearTagDist += static_cast(nErrorBuf(ilev)) * + clearTagDist += static_cast(nErrorBuf(ilev)) * Geom(m_EB_refine_LevMax).CellSize(0) * diagFac; } // Untag cells too close to EB #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(tags, TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(tags, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { const auto& bx = mfi.tilebox(); const auto& dist = signDist.const_array(mfi); auto tag = tags.array(mfi); amrex::ParallelFor(bx, [=] AMREX_GPU_HOST_DEVICE(int i, int j, int k) { if (dist(i, j, k) < clearTagDist) { - tag(i, j, k) = TagBox::CLEAR; + tag(i, j, k) = amrex::TagBox::CLEAR; } }); } diff --git a/Source/PeleLMeX_Temporals.cpp b/Source/PeleLMeX_Temporals.cpp index b407c7333..7af899344 100644 --- a/Source/PeleLMeX_Temporals.cpp +++ b/Source/PeleLMeX_Temporals.cpp @@ -1,8 +1,6 @@ #include #include -using namespace amrex; - void PeleLM::initTemporals(const PeleLM::TimeStamp& a_time) { @@ -42,8 +40,8 @@ PeleLM::massBalance() { // Compute the mass balance on the computational domain m_massNew = MFSum(GetVecOfConstPtrs(getDensityVect(AmrNewTime)), 0); - Real dmdt = (m_massNew - m_massOld) / m_dt; - Real massFluxBalance = AMREX_D_TERM( + amrex::Real dmdt = (m_massNew - m_massOld) / m_dt; + amrex::Real massFluxBalance = AMREX_D_TERM( m_domainMassFlux[0] + m_domainMassFlux[1], +m_domainMassFlux[2] + m_domainMassFlux[3], +m_domainMassFlux[4] + m_domainMassFlux[5]); @@ -74,9 +72,9 @@ void PeleLM::speciesBalance() { // Compute the species rhoY balance on the computational domain - Array dmYdt; - Array massYFluxBalance; - Array rhoYdots; + amrex::Array dmYdt; + amrex::Array massYFluxBalance; + amrex::Array rhoYdots; for (int n = 0; n < NUM_SPECIES; n++) { m_RhoYNew[n] = MFSum(GetVecOfConstPtrs(getSpeciesVect(AmrNewTime)), n); rhoYdots[n] = MFSum(GetVecOfConstPtrs(getIRVect()), n); @@ -106,8 +104,8 @@ PeleLM::speciesBalance() void PeleLM::addMassFluxes( - const Array& a_fluxes, - const Geometry& a_geom) + const amrex::Array& a_fluxes, + const amrex::Geometry& a_geom) { // Do when m_nstep is -1 since m_nstep is increased by one before @@ -117,8 +115,8 @@ PeleLM::addMassFluxes( } // Get the face areas - const Real* dx = a_geom.CellSize(); - Array area; + const amrex::Real* dx = a_geom.CellSize(); + amrex::Array area; #if (AMREX_SPACEDIM == 1) area[0] = 1.0; #elif (AMREX_SPACEDIM == 2) @@ -132,36 +130,37 @@ PeleLM::addMassFluxes( for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { auto faceDomain = - amrex::convert(a_geom.Domain(), IntVect::TheDimensionVector(idim)); + amrex::convert(a_geom.Domain(), amrex::IntVect::TheDimensionVector(idim)); auto const& fma = a_fluxes[idim]->const_arrays(); - Real sumLo = 0.0; - Real sumHi = 0.0; + amrex::Real sumLo = 0.0; + amrex::Real sumHi = 0.0; #if (AMREX_SPACEDIM == 2) if (geom[0].IsRZ()) { - MultiFab mf_a; + amrex::MultiFab mf_a; geom[0].GetFaceArea(mf_a, grids[0], dmap[0], idim, 0); auto const& ama = mf_a.const_arrays(); auto r = amrex::ParReduce( - TypeList{}, TypeList{}, - *a_fluxes[idim], IntVect(0), - [=] AMREX_GPU_DEVICE( - int box_no, int i, int j, int k) noexcept -> GpuTuple { - Array4 const& flux = fma[box_no]; - Array4 const& area_ar = ama[box_no]; + amrex::TypeList{}, + amrex::TypeList{}, *a_fluxes[idim], + amrex::IntVect(0), + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept + -> amrex::GpuTuple { + amrex::Array4 const& flux = fma[box_no]; + amrex::Array4 const& area_ar = ama[box_no]; int idx = (idim == 0) ? i : ((idim == 1) ? j : k); // low - Real low = 0.0; + amrex::Real low = 0.0; if (idx == faceDomain.smallEnd(idim)) { for (int n = 0; n < NUM_SPECIES; n++) { low += flux(i, j, k, n) * area_ar(i, j, k); } } // high - Real high = 0.0; + amrex::Real high = 0.0; if (idx == faceDomain.bigEnd(idim)) { for (int n = 0; n < NUM_SPECIES; n++) { high += flux(i, j, k, n) * area_ar(i, j, k); @@ -175,22 +174,23 @@ PeleLM::addMassFluxes( #endif { auto r = amrex::ParReduce( - TypeList{}, TypeList{}, - *a_fluxes[idim], IntVect(0), - [=] AMREX_GPU_DEVICE( - int box_no, int i, int j, int k) noexcept -> GpuTuple { - Array4 const& flux = fma[box_no]; + amrex::TypeList{}, + amrex::TypeList{}, *a_fluxes[idim], + amrex::IntVect(0), + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept + -> amrex::GpuTuple { + amrex::Array4 const& flux = fma[box_no]; int idx = (idim == 0) ? i : ((idim == 1) ? j : k); // low - Real low = 0.0; + amrex::Real low = 0.0; if (idx == faceDomain.smallEnd(idim)) { for (int n = 0; n < NUM_SPECIES; n++) { low += flux(i, j, k, n) * area[idim]; } } // high - Real high = 0.0; + amrex::Real high = 0.0; if (idx == faceDomain.bigEnd(idim)) { for (int n = 0; n < NUM_SPECIES; n++) { high += flux(i, j, k, n) * area[idim]; @@ -201,8 +201,8 @@ PeleLM::addMassFluxes( sumLo = amrex::get<0>(r); sumHi = amrex::get<1>(r); } - ParallelAllReduce::Sum( - {sumLo, sumHi}, ParallelContext::CommunicatorSub()); + amrex::ParallelAllReduce::Sum( + {sumLo, sumHi}, amrex::ParallelContext::CommunicatorSub()); m_domainMassFlux[2 * idim] += sumLo; m_domainMassFlux[2 * idim + 1] -= sumHi; // Outflow, negate flux } @@ -210,11 +210,11 @@ PeleLM::addMassFluxes( void PeleLM::addUmacFluxes( - std::unique_ptr& advData, const Geometry& a_geom) + std::unique_ptr& advData, const amrex::Geometry& a_geom) { // Get the face areas - const Real* dx = a_geom.CellSize(); - Array area; + const amrex::Real* dx = a_geom.CellSize(); + amrex::Array area; #if (AMREX_SPACEDIM == 1) area[0] = 1.0; #elif (AMREX_SPACEDIM == 2) @@ -231,34 +231,35 @@ PeleLM::addUmacFluxes( for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { auto faceDomain = - amrex::convert(a_geom.Domain(), IntVect::TheDimensionVector(idim)); + amrex::convert(a_geom.Domain(), amrex::IntVect::TheDimensionVector(idim)); auto const& fma = advData->umac[lev][idim].const_arrays(); - Real sumLo = 0.0; - Real sumHi = 0.0; + amrex::Real sumLo = 0.0; + amrex::Real sumHi = 0.0; #if (AMREX_SPACEDIM == 2) if (geom[0].IsRZ()) { - MultiFab mf_a; + amrex::MultiFab mf_a; geom[0].GetFaceArea(mf_a, grids[0], dmap[0], idim, 0); auto const& ama = mf_a.const_arrays(); auto r = amrex::ParReduce( - TypeList{}, TypeList{}, - advData->umac[lev][idim], IntVect(0), - [=] AMREX_GPU_DEVICE( - int box_no, int i, int j, int k) noexcept -> GpuTuple { - Array4 const& flux = fma[box_no]; - Array4 const& area_ar = ama[box_no]; + amrex::TypeList{}, + amrex::TypeList{}, advData->umac[lev][idim], + amrex::IntVect(0), + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept + -> amrex::GpuTuple { + amrex::Array4 const& flux = fma[box_no]; + amrex::Array4 const& area_ar = ama[box_no]; int idx = (idim == 0) ? i : ((idim == 1) ? j : k); // low - Real low = 0.0; + amrex::Real low = 0.0; if (idx == faceDomain.smallEnd(idim)) { low += flux(i, j, k) * area_ar(i, j, k); } // high - Real high = 0.0; + amrex::Real high = 0.0; if (idx == faceDomain.bigEnd(idim)) { high += flux(i, j, k) * area_ar(i, j, k); } @@ -270,20 +271,21 @@ PeleLM::addUmacFluxes( #endif { auto r = amrex::ParReduce( - TypeList{}, TypeList{}, - advData->umac[lev][idim], IntVect(0), - [=] AMREX_GPU_DEVICE( - int box_no, int i, int j, int k) noexcept -> GpuTuple { - Array4 const& flux = fma[box_no]; + amrex::TypeList{}, + amrex::TypeList{}, advData->umac[lev][idim], + amrex::IntVect(0), + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept + -> amrex::GpuTuple { + amrex::Array4 const& flux = fma[box_no]; int idx = (idim == 0) ? i : ((idim == 1) ? j : k); // low - Real low = 0.0; + amrex::Real low = 0.0; if (idx == faceDomain.smallEnd(idim)) { low += flux(i, j, k) * area[idim]; } // high - Real high = 0.0; + amrex::Real high = 0.0; if (idx == faceDomain.bigEnd(idim)) { high += flux(i, j, k) * area[idim]; } @@ -292,8 +294,8 @@ PeleLM::addUmacFluxes( sumLo = amrex::get<0>(r); sumHi = amrex::get<1>(r); } - ParallelAllReduce::Sum( - {sumLo, sumHi}, ParallelContext::CommunicatorSub()); + amrex::ParallelAllReduce::Sum( + {sumLo, sumHi}, amrex::ParallelContext::CommunicatorSub()); m_domainUmacFlux[2 * idim] += sumLo; m_domainUmacFlux[2 * idim + 1] -= sumHi; // Outflow, negate flux } @@ -304,8 +306,8 @@ PeleLM::rhoHBalance() { // Compute the enthalpy balance on the computational domain (rho*h) m_RhoHNew = MFSum(GetVecOfConstPtrs(getRhoHVect(AmrNewTime)), 0); - Real dRhoHdt = (m_RhoHNew - m_RhoHOld) / m_dt; - Real rhoHFluxBalance = AMREX_D_TERM( + amrex::Real dRhoHdt = (m_RhoHNew - m_RhoHOld) / m_dt; + amrex::Real rhoHFluxBalance = AMREX_D_TERM( m_domainRhoHFlux[0] + m_domainRhoHFlux[1], +m_domainRhoHFlux[2] + m_domainRhoHFlux[3], +m_domainRhoHFlux[4] + m_domainRhoHFlux[5]); @@ -320,8 +322,8 @@ PeleLM::rhoHBalance() void PeleLM::addRhoHFluxes( - const Array& a_fluxes, - const Geometry& a_geom) + const amrex::Array& a_fluxes, + const amrex::Geometry& a_geom) { // Do when m_nstep is -1 since m_nstep is increased by one before @@ -331,8 +333,8 @@ PeleLM::addRhoHFluxes( } // Get the face areas - const Real* dx = a_geom.CellSize(); - Array area; + const amrex::Real* dx = a_geom.CellSize(); + amrex::Array area; #if (AMREX_SPACEDIM == 1) area[0] = 1.0; #elif (AMREX_SPACEDIM == 2) @@ -346,34 +348,35 @@ PeleLM::addRhoHFluxes( for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { auto faceDomain = - amrex::convert(a_geom.Domain(), IntVect::TheDimensionVector(idim)); + amrex::convert(a_geom.Domain(), amrex::IntVect::TheDimensionVector(idim)); auto const& fma = a_fluxes[idim]->const_arrays(); - Real sumLo = 0.0; - Real sumHi = 0.0; + amrex::Real sumLo = 0.0; + amrex::Real sumHi = 0.0; #if (AMREX_SPACEDIM == 2) if (geom[0].IsRZ()) { - MultiFab mf_a; + amrex::MultiFab mf_a; geom[0].GetFaceArea(mf_a, grids[0], dmap[0], idim, 0); auto const& ama = mf_a.const_arrays(); auto r = amrex::ParReduce( - TypeList{}, TypeList{}, - *a_fluxes[idim], IntVect(0), - [=] AMREX_GPU_DEVICE( - int box_no, int i, int j, int k) noexcept -> GpuTuple { - Array4 const& flux = fma[box_no]; - Array4 const& area_ar = ama[box_no]; + amrex::TypeList{}, + amrex::TypeList{}, *a_fluxes[idim], + amrex::IntVect(0), + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept + -> amrex::GpuTuple { + amrex::Array4 const& flux = fma[box_no]; + amrex::Array4 const& area_ar = ama[box_no]; int idx = (idim == 0) ? i : ((idim == 1) ? j : k); // low - Real low = 0.0; + amrex::Real low = 0.0; if (idx == faceDomain.smallEnd(idim)) { low += flux(i, j, k, NUM_SPECIES) * area_ar(i, j, k); } // high - Real high = 0.0; + amrex::Real high = 0.0; if (idx == faceDomain.bigEnd(idim)) { high += flux(i, j, k, NUM_SPECIES) * area_ar(i, j, k); } @@ -385,20 +388,21 @@ PeleLM::addRhoHFluxes( #endif { auto r = amrex::ParReduce( - TypeList{}, TypeList{}, - *a_fluxes[idim], IntVect(0), - [=] AMREX_GPU_DEVICE( - int box_no, int i, int j, int k) noexcept -> GpuTuple { - Array4 const& flux = fma[box_no]; + amrex::TypeList{}, + amrex::TypeList{}, *a_fluxes[idim], + amrex::IntVect(0), + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept + -> amrex::GpuTuple { + amrex::Array4 const& flux = fma[box_no]; int idx = (idim == 0) ? i : ((idim == 1) ? j : k); // low - Real low = 0.0; + amrex::Real low = 0.0; if (idx == faceDomain.smallEnd(idim)) { low += flux(i, j, k, NUM_SPECIES) * area[idim]; } // high - Real high = 0.0; + amrex::Real high = 0.0; if (idx == faceDomain.bigEnd(idim)) { high += flux(i, j, k, NUM_SPECIES) * area[idim]; } @@ -407,8 +411,8 @@ PeleLM::addRhoHFluxes( sumLo = amrex::get<0>(r); sumHi = amrex::get<1>(r); } - ParallelAllReduce::Sum( - {sumLo, sumHi}, ParallelContext::CommunicatorSub()); + amrex::ParallelAllReduce::Sum( + {sumLo, sumHi}, amrex::ParallelContext::CommunicatorSub()); m_domainRhoHFlux[2 * idim] += sumLo; m_domainRhoHFlux[2 * idim + 1] -= sumHi; // Outflow, negate flux } @@ -416,9 +420,9 @@ PeleLM::addRhoHFluxes( void PeleLM::addRhoYFluxes( - const Array& a_fluxes, - const Geometry& a_geom, - const Real& a_factor) + const amrex::Array& a_fluxes, + const amrex::Geometry& a_geom, + const amrex::Real& a_factor) { // Do when m_nstep is -1 since m_nstep is increased by one before @@ -428,8 +432,8 @@ PeleLM::addRhoYFluxes( } // Get the face areas - const Real* dx = a_geom.CellSize(); - Array area; + const amrex::Real* dx = a_geom.CellSize(); + amrex::Array area; #if (AMREX_SPACEDIM == 1) area[0] = 1.0; #elif (AMREX_SPACEDIM == 2) @@ -445,35 +449,36 @@ PeleLM::addRhoYFluxes( for (int n = 0; n < NUM_SPECIES; n++) { // Inner loop over dimensions for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - auto faceDomain = - amrex::convert(a_geom.Domain(), IntVect::TheDimensionVector(idim)); + auto faceDomain = amrex::convert( + a_geom.Domain(), amrex::IntVect::TheDimensionVector(idim)); auto const& fma = a_fluxes[idim]->const_arrays(); - Real sumLo = 0.0; - Real sumHi = 0.0; + amrex::Real sumLo = 0.0; + amrex::Real sumHi = 0.0; #if (AMREX_SPACEDIM == 2) if (geom[0].IsRZ()) { - MultiFab mf_a; + amrex::MultiFab mf_a; geom[0].GetFaceArea(mf_a, grids[0], dmap[0], idim, 0); auto const& ama = mf_a.const_arrays(); auto r = amrex::ParReduce( - TypeList{}, TypeList{}, - *a_fluxes[idim], IntVect(0), - [=] AMREX_GPU_DEVICE( - int box_no, int i, int j, int k) noexcept -> GpuTuple { - Array4 const& flux = fma[box_no]; - Array4 const& area_ar = ama[box_no]; + amrex::TypeList{}, + amrex::TypeList{}, *a_fluxes[idim], + amrex::IntVect(0), + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept + -> amrex::GpuTuple { + amrex::Array4 const& flux = fma[box_no]; + amrex::Array4 const& area_ar = ama[box_no]; int idx = (idim == 0) ? i : ((idim == 1) ? j : k); // low - Real low = 0.0; + amrex::Real low = 0.0; if (idx == faceDomain.smallEnd(idim)) { low += flux(i, j, k, n) * area_ar(i, j, k); } // high - Real high = 0.0; + amrex::Real high = 0.0; if (idx == faceDomain.bigEnd(idim)) { high += flux(i, j, k, n) * area_ar(i, j, k); } @@ -485,20 +490,21 @@ PeleLM::addRhoYFluxes( #endif { auto r = amrex::ParReduce( - TypeList{}, TypeList{}, - *a_fluxes[idim], IntVect(0), - [=] AMREX_GPU_DEVICE( - int box_no, int i, int j, int k) noexcept -> GpuTuple { - Array4 const& flux = fma[box_no]; + amrex::TypeList{}, + amrex::TypeList{}, *a_fluxes[idim], + amrex::IntVect(0), + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept + -> amrex::GpuTuple { + amrex::Array4 const& flux = fma[box_no]; int idx = (idim == 0) ? i : ((idim == 1) ? j : k); // low - Real low = 0.0; + amrex::Real low = 0.0; if (idx == faceDomain.smallEnd(idim)) { low += flux(i, j, k, n) * area[idim]; } // high - Real high = 0.0; + amrex::Real high = 0.0; if (idx == faceDomain.bigEnd(idim)) { high += flux(i, j, k, n) * area[idim]; } @@ -507,8 +513,8 @@ PeleLM::addRhoYFluxes( sumLo = amrex::get<0>(r); sumHi = amrex::get<1>(r); } - ParallelAllReduce::Sum( - {sumLo, sumHi}, ParallelContext::CommunicatorSub()); + amrex::ParallelAllReduce::Sum( + {sumLo, sumHi}, amrex::ParallelContext::CommunicatorSub()); m_domainRhoYFlux[2 * idim + n * 2 * AMREX_SPACEDIM] += a_factor * sumLo; m_domainRhoYFlux[2 * idim + n * 2 * AMREX_SPACEDIM + 1] -= a_factor * sumHi; // Outflow, negate flux @@ -517,14 +523,14 @@ PeleLM::addRhoYFluxes( } void -PeleLM::initBPatches(Geometry& a_geom) +PeleLM::initBPatches(amrex::Geometry& a_geom) { std::string pele_prefix = "peleLM.bpatch"; - ParmParse pp(pele_prefix); + amrex::ParmParse pp(pele_prefix); int num_bPatches = 0; num_bPatches = pp.countval("patchnames"); - Vector bpatch_name; + amrex::Vector bpatch_name; if (num_bPatches > 0) { m_bPatches.resize(num_bPatches); @@ -534,17 +540,17 @@ PeleLM::initBPatches(Geometry& a_geom) pp.get("patchnames", bpatch_name[n], n); m_bPatches[n] = std::make_unique(bpatch_name[n], a_geom); if (m_verbose > 0) { - Print() << " Initializing boundary patch: " << bpatch_name[n] - << std::endl; + amrex::Print() << " Initializing boundary patch: " << bpatch_name[n] + << std::endl; } } } void PeleLM::addRhoYFluxesPatch( - const Array& a_fluxes, - const Geometry& a_geom, - const Real& a_factor) + const amrex::Array& a_fluxes, + const amrex::Geometry& a_geom, + const amrex::Real& a_factor) { if (!(m_nstep % m_temp_int == m_temp_int - 1)) { @@ -559,7 +565,7 @@ PeleLM::addRhoYFluxesPatch( area[0] = 1.0; #elif (AMREX_SPACEDIM == 2) if (geom[0].IsRZ() && m_bPatches.size() > 0) { - Abort("Bpatches not supported in RZ coordinates"); + amrex::Abort("Bpatches not supported in RZ coordinates"); } area[0] = dx[1]; area[1] = dx[0]; @@ -578,21 +584,22 @@ PeleLM::addRhoYFluxesPatch( const int idim = bphost->m_boundary_dir; auto faceDomain = - amrex::convert(a_geom.Domain(), IntVect::TheDimensionVector(idim)); + amrex::convert(a_geom.Domain(), amrex::IntVect::TheDimensionVector(idim)); auto const& fma = a_fluxes[idim]->const_arrays(); // Loop through species specified by user for (int m = 0; m < bphost->num_species; m++) { - Real sum_species_flux_global = 0.0; + amrex::Real sum_species_flux_global = 0.0; { auto r = amrex::ParReduce( - TypeList{}, TypeList{}, - *a_fluxes[idim], IntVect(0), - [=] AMREX_GPU_DEVICE( - int box_no, int i, int j, int k) noexcept -> GpuTuple { - Array4 const& flux = fma[box_no]; + amrex::TypeList{}, + amrex::TypeList{}, *a_fluxes[idim], + amrex::IntVect(0), + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept + -> amrex::GpuTuple { + amrex::Array4 const& flux = fma[box_no]; int idx = (bpdevice->m_boundary_dir == 0 ? i @@ -606,8 +613,8 @@ PeleLM::addRhoYFluxesPatch( prob_lo[0] + (i + 0.5) * dx[0], prob_lo[1] + (j + 0.5) * dx[1], prob_lo[2] + (k + 0.5) * dx[2])}; - Real sum_species_flux = 0.0; - Real dummy = 0.0; + amrex::Real sum_species_flux = 0.0; + amrex::Real dummy = 0.0; const bool ifinside = bpdevice->CheckifPointInside(point_coordinates, dx[0]); @@ -619,8 +626,8 @@ PeleLM::addRhoYFluxesPatch( }); sum_species_flux_global = amrex::get<0>(r); - ParallelAllReduce::Sum( - {sum_species_flux_global}, ParallelContext::CommunicatorSub()); + amrex::ParallelAllReduce::Sum( + {sum_species_flux_global}, amrex::ParallelContext::CommunicatorSub()); bphost->speciesFlux[m] = a_factor * sum_species_flux_global; } } @@ -650,18 +657,18 @@ PeleLM::writeTemporals() //---------------------------------------------------------------- // State // Get kinetic energy and enstrophy - Vector> kinEnergy(finest_level + 1); - Vector> enstrophy(finest_level + 1); + amrex::Vector> kinEnergy(finest_level + 1); + amrex::Vector> enstrophy(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { kinEnergy[lev] = derive("kinetic_energy", m_cur_time, lev, 0); enstrophy[lev] = derive("enstrophy", m_cur_time, lev, 0); } - Real kinenergy_int = MFSum(GetVecOfConstPtrs(kinEnergy), 0); - Real enstrophy_int = MFSum(GetVecOfConstPtrs(enstrophy), 0); + amrex::Real kinenergy_int = MFSum(GetVecOfConstPtrs(kinEnergy), 0); + amrex::Real enstrophy_int = MFSum(GetVecOfConstPtrs(enstrophy), 0); // Combustion - Real fuelConsumptionInt = 0.0; - Real heatReleaseRateInt = 0.0; + amrex::Real fuelConsumptionInt = 0.0; + amrex::Real heatReleaseRateInt = 0.0; if (fuelID >= 0 && !(m_chem_integrator == "ReactorNull")) { fuelConsumptionInt = MFSum(GetVecOfConstPtrs(getIRVect()), fuelID); for (int lev = 0; lev <= finest_level; ++lev) { @@ -712,9 +719,9 @@ PeleLM::openTempFile() } // Create the temporal directory - UtilCreateDirectory("temporals", 0755); + amrex::UtilCreateDirectory("temporals", 0755); - if (ParallelDescriptor::IOProcessor()) { + if (amrex::ParallelDescriptor::IOProcessor()) { std::string tempFileName = "temporals/tempState"; tmpStateFile.open( tempFileName.c_str(), @@ -800,7 +807,7 @@ PeleLM::closeTempFile() return; } - if (ParallelDescriptor::IOProcessor()) { + if (amrex::ParallelDescriptor::IOProcessor()) { tmpStateFile.flush(); tmpStateFile.close(); if (m_do_massBalance != 0) { diff --git a/Source/PeleLMeX_Timestep.cpp b/Source/PeleLMeX_Timestep.cpp index 7a33fd72e..de67079cb 100644 --- a/Source/PeleLMeX_Timestep.cpp +++ b/Source/PeleLMeX_Timestep.cpp @@ -1,14 +1,12 @@ #include #include -using namespace amrex; - -Real +amrex::Real PeleLM::computeDt(int is_init, const TimeStamp& a_time) { BL_PROFILE("PeleLMeX::computeDt()"); - Real estdt = 1.0e200; + amrex::Real estdt = 1.0e200; //---------------------------------------------------------------- // Store prev dt(s) @@ -22,30 +20,31 @@ PeleLM::computeDt(int is_init, const TimeStamp& a_time) if (((is_init != 0) || m_nstep == 0) && m_init_dt > 0.0) { estdt = m_init_dt; } else { - Real dtconv = estConvectiveDt(a_time); - estdt = std::min(estdt, dtconv); - Real dtdivU = 1.0e200; + amrex::Real dtconv = estConvectiveDt(a_time); + estdt = amrex::min(estdt, dtconv); + amrex::Real dtdivU = 1.0e200; if ((m_incompressible == 0) && (m_has_divu != 0)) { dtdivU = estDivUDt(a_time); - estdt = std::min(estdt, dtdivU); + estdt = amrex::min(estdt, dtdivU); } #ifdef PELE_USE_PLASMA - Real dtions = estEFIonsDt(a_time); - estdt = std::min(estdt, dtions); + amrex::Real dtions = estEFIonsDt(a_time); + estdt = amrex::min(estdt, dtions); #endif #ifdef PELE_USE_SPRAY - Real dtspray = SprayEstDt(); - estdt = std::min(estdt, dtspray); + amrex::Real dtspray = SprayEstDt(); + estdt = amrex::min(estdt, dtspray); #endif if (m_verbose != 0) { - Print() << " Est. time step - Conv: " << dtconv << ", divu: " << dtdivU + amrex::Print() << " Est. time step - Conv: " << dtconv + << ", divu: " << dtdivU #ifdef PELE_USE_PLASMA - << ", ions: " << dtions + << ", ions: " << dtions #endif #ifdef PELE_USE_SPRAY - << ", sprays: " << dtspray + << ", sprays: " << dtspray #endif - << "\n"; + << "\n"; } } } @@ -55,19 +54,19 @@ PeleLM::computeDt(int is_init, const TimeStamp& a_time) if ((is_init != 0) || m_nstep == 0) { estdt *= m_dtshrink; } else { - estdt = std::min(estdt, m_prev_dt * m_dtChangeMax); - estdt = std::min(estdt, m_max_dt); + estdt = amrex::min(estdt, m_prev_dt * m_dtChangeMax); + estdt = amrex::min(estdt, m_max_dt); // Shorten the dt to output plt file at exact req. time if (m_plot_per_exact > 0.0) { // Ensure ~O(dt) step by checking a little in advance - Real timeToNextPlot = + amrex::Real timeToNextPlot = (std::floor(m_cur_time / m_plot_per_exact) + 1) * m_plot_per_exact - m_cur_time; if (2.0 * estdt > timeToNextPlot && timeToNextPlot > estdt) { - estdt = Real(0.5) * timeToNextPlot; + estdt = amrex::Real(0.5) * timeToNextPlot; } else { if (timeToNextPlot > 1.e-12) { - estdt = std::min(estdt, timeToNextPlot); + estdt = amrex::min(estdt, timeToNextPlot); } } } @@ -75,37 +74,37 @@ PeleLM::computeDt(int is_init, const TimeStamp& a_time) // too if (m_stop_time >= 0.0) { // Ensure ~O(dt) last step by checking a little in advance - Real timeLeft = (m_stop_time - m_cur_time); + amrex::Real timeLeft = (m_stop_time - m_cur_time); if (2.0 * estdt > timeLeft && timeLeft > estdt) { estdt = 0.5 * timeLeft; } else { - estdt = std::min(estdt, timeLeft); + estdt = amrex::min(estdt, timeLeft); } } } if (estdt < m_min_dt) { - Print() << "\n"; - Print() << " ###################################### \n"; - Print() << " Estimated dt " << estdt << " is below allowed dt_min " - << m_min_dt << ": the simulation will stop ! \n"; - Print() << " ###################################### \n"; - Print() << "\n"; + amrex::Print() << "\n"; + amrex::Print() << " ###################################### \n"; + amrex::Print() << " Estimated dt " << estdt << " is below allowed dt_min " + << m_min_dt << ": the simulation will stop ! \n"; + amrex::Print() << " ###################################### \n"; + amrex::Print() << "\n"; } return estdt; } -Real +amrex::Real PeleLM::estConvectiveDt(const TimeStamp& a_time) { - Real estdt = 1.0e200; - constexpr Real small = 1.0e-8; + amrex::Real estdt = 1.0e200; + constexpr amrex::Real small = 1.0e-8; for (int lev = 0; lev <= finest_level; ++lev) { - Real estdt_lev = 1.0e200; + amrex::Real estdt_lev = 1.0e200; //---------------------------------------------------------------- // Get level data ptr @@ -116,8 +115,8 @@ PeleLM::estConvectiveDt(const TimeStamp& a_time) //---------------------------------------------------------------- // Get velocity forces int nGrow_force = 0; - MultiFab velForces( - grids[lev], dmap[lev], AMREX_SPACEDIM, nGrow_force, MFInfo(), + amrex::MultiFab velForces( + grids[lev], dmap[lev], AMREX_SPACEDIM, nGrow_force, amrex::MFInfo(), Factory(lev)); int add_gradP = 1; @@ -125,11 +124,11 @@ PeleLM::estConvectiveDt(const TimeStamp& a_time) //---------------------------------------------------------------- // Get max forces - Vector f_max(AMREX_SPACEDIM); + amrex::Vector f_max(AMREX_SPACEDIM); f_max = velForces.norm0({AMREX_D_DECL(0, 1, 2)}, 0, true, true); // Get max velocity - Vector u_max(AMREX_SPACEDIM); + amrex::Vector u_max(AMREX_SPACEDIM); u_max = ldata_p->state.norm0({AMREX_D_DECL(VELX, VELY, VELZ)}, 0, true, true); @@ -137,29 +136,29 @@ PeleLM::estConvectiveDt(const TimeStamp& a_time) // Est. min time step on lev for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { if (u_max[idim] > small) { - estdt_lev = std::min(estdt_lev, dx[idim] / u_max[idim]); + estdt_lev = amrex::min(estdt_lev, dx[idim] / u_max[idim]); } if (f_max[idim] > small) { estdt_lev = - std::min(estdt_lev, std::sqrt(2.0 * dx[idim] / f_max[idim])); + amrex::min(estdt_lev, std::sqrt(2.0 * dx[idim] / f_max[idim])); } } //---------------------------------------------------------------- // Set overall convective dt - estdt = std::min(estdt, estdt_lev * m_cfl); + estdt = amrex::min(estdt, estdt_lev * m_cfl); } - ParallelDescriptor::ReduceRealMin(estdt); + amrex::ParallelDescriptor::ReduceRealMin(estdt); return estdt; } -Real +amrex::Real PeleLM::estDivUDt(const TimeStamp& a_time) { - Real estdt = 1.0e200; + amrex::Real estdt = 1.0e200; // Note: only methods 1 & 2 of PeleLM are available here AMREX_ASSERT(m_divu_checkFlag >= 0 && m_divu_checkFlag <= 2); @@ -167,50 +166,50 @@ PeleLM::estDivUDt(const TimeStamp& a_time) for (int lev = 0; lev <= finest_level; ++lev) { auto* ldata_p = getLevelDataPtr(lev, a_time); - std::unique_ptr density = - std::make_unique(ldata_p->state, amrex::make_alias, DENSITY, 1); + std::unique_ptr density = + std::make_unique( + ldata_p->state, amrex::make_alias, DENSITY, 1); auto dtfac = m_divu_dtFactor; auto rhoMin = m_divu_rhoMin; if (m_divu_checkFlag == 1) { - Real divu_dt = amrex::ReduceMin( + amrex::Real divu_dt = amrex::ReduceMin( *density, ldata_p->divu, 0, [dtfac, rhoMin] AMREX_GPU_HOST_DEVICE( - Box const& bx, Array4 const& rho, - Array4 const& divu) -> Real { - using namespace amrex::literals; + amrex::Box const& bx, amrex::Array4 const& rho, + amrex::Array4 const& divu) -> amrex::Real { const auto lo = amrex::lbound(bx); const auto hi = amrex::ubound(bx); - amrex::Real dt = 1.e37_rt; + amrex::Real dt = 1.e37; for (int k = lo.z; k <= hi.z; ++k) { for (int j = lo.y; j <= hi.y; ++j) { for (int i = lo.x; i <= hi.x; ++i) { - Real dtcell = est_divu_dt_1(i, j, k, dtfac, rhoMin, rho, divu); + amrex::Real dtcell = + est_divu_dt_1(i, j, k, dtfac, rhoMin, rho, divu); dt = amrex::min(dt, dtcell); } } } return dt; }); - estdt = std::min(divu_dt, estdt); + estdt = amrex::min(divu_dt, estdt); } else if (m_divu_checkFlag == 2) { const auto& dxinv = geom[lev].InvCellSizeArray(); - std::unique_ptr velo = std::make_unique( + std::unique_ptr velo = std::make_unique( ldata_p->state, amrex::make_alias, VELX, AMREX_SPACEDIM); - Real divu_dt = amrex::ReduceMin( + amrex::Real divu_dt = amrex::ReduceMin( *density, ldata_p->divu, *velo, 0, [dtfac, rhoMin, dxinv] AMREX_GPU_HOST_DEVICE( - Box const& bx, Array4 const& rho, - Array4 const& vel, - Array4 const& divu) -> Real { - using namespace amrex::literals; + amrex::Box const& bx, amrex::Array4 const& rho, + amrex::Array4 const& vel, + amrex::Array4 const& divu) -> amrex::Real { const auto lo = amrex::lbound(bx); const auto hi = amrex::ubound(bx); - amrex::Real dt = 1.e37_rt; + amrex::Real dt = 1.e37; for (int k = lo.z; k <= hi.z; ++k) { for (int j = lo.y; j <= hi.y; ++j) { for (int i = lo.x; i <= hi.x; ++i) { - Real dtcell = + amrex::Real dtcell = est_divu_dt_2(i, j, k, dtfac, rhoMin, dxinv, rho, vel, divu); dt = amrex::min(dt, dtcell); } @@ -218,17 +217,17 @@ PeleLM::estDivUDt(const TimeStamp& a_time) } return dt; }); - estdt = std::min(divu_dt, estdt); + estdt = amrex::min(divu_dt, estdt); } } - ParallelDescriptor::ReduceRealMin(estdt); + amrex::ParallelDescriptor::ReduceRealMin(estdt); return estdt; } void -PeleLM::checkDt(const TimeStamp& a_time, const Real& a_dt) +PeleLM::checkDt(const TimeStamp& a_time, const amrex::Real& a_dt) { BL_PROFILE("PeleLMeX::checkDt()"); @@ -243,10 +242,11 @@ PeleLM::checkDt(const TimeStamp& a_time, const Real& a_dt) const auto dxinv = geom[lev].InvCellSizeArray(); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); auto const& rho = ldata_p->state.const_array(mfi, DENSITY); auto const& vel = ldata_p->state.const_array(mfi, VELX); auto const& divu = ldata_p->divu.const_array(mfi); @@ -254,8 +254,7 @@ PeleLM::checkDt(const TimeStamp& a_time, const Real& a_dt) auto dtfac = m_divu_dtFactor; auto rhoMin = m_divu_rhoMin; amrex::ParallelFor( - bx, [rho, vel, divu, divu_checkFlag, dtfac, rhoMin, dxinv, - a_dt] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { check_divu_dt( i, j, k, divu_checkFlag, dtfac, rhoMin, dxinv, rho, vel, divu, a_dt); diff --git a/Source/PeleLMeX_TransportProp.cpp b/Source/PeleLMeX_TransportProp.cpp index b58519d42..91bc5a15b 100644 --- a/Source/PeleLMeX_TransportProp.cpp +++ b/Source/PeleLMeX_TransportProp.cpp @@ -6,8 +6,6 @@ #include #endif -using namespace amrex; - void PeleLM::calcTurbViscosity(const TimeStamp& a_time) { @@ -33,8 +31,8 @@ PeleLM::calcTurbViscosity(const TimeStamp& a_time) constexpr int ncomp = AMREX_SPACEDIM * AMREX_SPACEDIM; for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { GradVel[lev][idim].define( - amrex::convert(ba, IntVect::TheDimensionVector(idim)), dm, ncomp, 0, - MFInfo(), factory); + amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)), dm, ncomp, + 0, amrex::MFInfo(), factory); } } @@ -57,27 +55,28 @@ PeleLM::calcTurbViscosity(const TimeStamp& a_time) // just set density as a constant; don't need to worry about cp for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { dens_fc[idim].define( - amrex::convert(ba, IntVect::TheDimensionVector(idim)), dm, 1, 0, - MFInfo(), factory); + amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)), dm, 1, + 0, amrex::MFInfo(), factory); dens_fc[idim].setVal(m_rho); } } else { // get cp_cc (valid in 1 grow cell for interpolation to FCs) int ngrow = 1; auto const* leosparm = eos_parms.device_parm(); - cp_cc.define(ba, dm, 1, ngrow, MFInfo(), factory); + cp_cc.define(ba, dm, 1, ngrow, amrex::MFInfo(), factory); auto const& state_arr = ldata_p->state.const_arrays(); auto const& cp_arr = cp_cc.arrays(); amrex::ParallelFor( cp_cc, cp_cc.nGrowVect(), [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { getCpmixGivenRYT( - i, j, k, Array4(state_arr[box_no], DENSITY), - Array4(state_arr[box_no], FIRSTSPEC), - Array4(state_arr[box_no], TEMP), - Array4(cp_arr[box_no]), leosparm); + i, j, k, + amrex::Array4(state_arr[box_no], DENSITY), + amrex::Array4(state_arr[box_no], FIRSTSPEC), + amrex::Array4(state_arr[box_no], TEMP), + amrex::Array4(cp_arr[box_no]), leosparm); }); - Gpu::streamSynchronize(); + amrex::Gpu::streamSynchronize(); // this function really just interpolates CCs to FCs in this case int doZeroVisc = 0; @@ -108,9 +107,10 @@ PeleLM::calcTurbViscosity(const TimeStamp& a_time) ldata_p->visc_turb_fc[idim], ldata_p->visc_turb_fc[idim].nGrowVect(), [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { getTurbViscSmagorinsky( - i, j, k, prefact, Array4(velgrad_arr[box_no]), - Array4(dens_arr[box_no]), - Array4(mut_arr[box_no])); + i, j, k, prefact, + amrex::Array4(velgrad_arr[box_no]), + amrex::Array4(dens_arr[box_no]), + amrex::Array4(mut_arr[box_no])); #ifdef AMREX_USE_EB if (idim == 0) { const amrex::Real vfr_m = @@ -152,9 +152,10 @@ PeleLM::calcTurbViscosity(const TimeStamp& a_time) ldata_p->visc_turb_fc[idim], ldata_p->visc_turb_fc[idim].nGrowVect(), [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { getTurbViscWALE( - i, j, k, prefact, Array4(velgrad_arr[box_no]), - Array4(dens_arr[box_no]), - Array4(mut_arr[box_no])); + i, j, k, prefact, + amrex::Array4(velgrad_arr[box_no]), + amrex::Array4(dens_arr[box_no]), + amrex::Array4(mut_arr[box_no])); #ifdef AMREX_USE_EB if (idim == 0) { const amrex::Real vfr_m = @@ -196,9 +197,10 @@ PeleLM::calcTurbViscosity(const TimeStamp& a_time) ldata_p->visc_turb_fc[idim], ldata_p->visc_turb_fc[idim].nGrowVect(), [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { getTurbViscSigma( - i, j, k, prefact, Array4(velgrad_arr[box_no]), - Array4(dens_arr[box_no]), - Array4(mut_arr[box_no])); + i, j, k, prefact, + amrex::Array4(velgrad_arr[box_no]), + amrex::Array4(dens_arr[box_no]), + amrex::Array4(mut_arr[box_no])); #ifdef AMREX_USE_EB if (idim == 0) { const amrex::Real vfr_m = @@ -234,7 +236,7 @@ PeleLM::calcTurbViscosity(const TimeStamp& a_time) #endif }); } - Gpu::streamSynchronize(); + amrex::Gpu::streamSynchronize(); // Compute lambda_turb = alpha_t * cp = mu_t / Pr_t * cp if (m_incompressible == 0) { @@ -273,13 +275,13 @@ PeleLM::calcViscosity(const TimeStamp& a_time) ldata_p->visc_cc, ldata_p->visc_cc.nGrowVect(), [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { getVelViscosity( - i, j, k, Array4(sma[box_no], FIRSTSPEC), - Array4(sma[box_no], TEMP), Array4(vma[box_no], 0), - ltransparm); + i, j, k, amrex::Array4(sma[box_no], FIRSTSPEC), + amrex::Array4(sma[box_no], TEMP), + amrex::Array4(vma[box_no], 0), ltransparm); }); } } - Gpu::streamSynchronize(); + amrex::Gpu::streamSynchronize(); } void @@ -299,7 +301,7 @@ PeleLM::calcDiffusivity(const TimeStamp& a_time) auto const* ltransparm = trans_parms.device_parm(); auto const* leosparm = eos_parms.device_parm(); #ifdef PELE_USE_PLASMA - GpuArray mwt{0.0}; + amrex::GpuArray mwt{0.0}; { auto eos = pele::physics::PhysicsType::eos(leosparm); eos.molecular_weight(mwt.arr); @@ -322,29 +324,33 @@ PeleLM::calcDiffusivity(const TimeStamp& a_time) [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { getTransportCoeff( i, j, k, do_fixed_Le, do_fixed_Pr, do_soret, Le_inv, Pr_inv, - Array4(sma[box_no], FIRSTSPEC), - Array4(sma[box_no], TEMP), Array4(dma[box_no], 0), - Array4(dma[box_no], NUM_SPECIES + 1 + soret_idx), - Array4(dma[box_no], NUM_SPECIES), - Array4(dma[box_no], NUM_SPECIES + 1), ltransparm, leosparm); + amrex::Array4(sma[box_no], FIRSTSPEC), + amrex::Array4(sma[box_no], TEMP), + amrex::Array4(dma[box_no], 0), + amrex::Array4(dma[box_no], NUM_SPECIES + 1 + soret_idx), + amrex::Array4(dma[box_no], NUM_SPECIES), + amrex::Array4(dma[box_no], NUM_SPECIES + 1), ltransparm, + leosparm); #ifdef PELE_USE_PLASMA getKappaSp( - i, j, k, mwt.arr, zk, Array4(sma[box_no], FIRSTSPEC), - Array4(dma[box_no], 0), Array4(sma[box_no], TEMP), - Array4(kma[box_no], 0)); + i, j, k, mwt.arr, zk, + amrex::Array4(sma[box_no], FIRSTSPEC), + amrex::Array4(dma[box_no], 0), + amrex::Array4(sma[box_no], TEMP), + amrex::Array4(kma[box_no], 0)); #endif }); // Fill the diff_aux MF with specified Schmidt number for (int n = 0; n < m_nAux; n++) { if (m_aux_Schmidt[n] > 0) { - MultiFab::Copy( + amrex::MultiFab::Copy( ldata_p->diff_aux_cc, ldata_p->diff_cc, NUM_SPECIES + 1, n, 1, ldata_p->diff_cc.nGrowVect()); ldata_p->diff_aux_cc.mult( 1.0 / m_aux_Schmidt[n], n, 1, ldata_p->diff_cc.nGrow()); } else { - MultiFab::Copy( + amrex::MultiFab::Copy( ldata_p->diff_aux_cc, ldata_p->diff_cc, NUM_SPECIES, n, 1, ldata_p->diff_cc.nGrowVect()); // lambda @@ -354,34 +360,35 @@ PeleLM::calcDiffusivity(const TimeStamp& a_time) amrex::MultiFab cp_cc; int ngrow = ldata_p->diff_cc.nGrow(); - cp_cc.define(ba, dm, 1, ngrow, MFInfo(), factory); + cp_cc.define(ba, dm, 1, ngrow, amrex::MFInfo(), factory); auto const& state_arr = ldata_p->state.const_arrays(); auto const& cp_arr = cp_cc.arrays(); amrex::ParallelFor( cp_cc, cp_cc.nGrowVect(), [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { getCpmixGivenRYT( - i, j, k, Array4(state_arr[box_no], DENSITY), - Array4(state_arr[box_no], FIRSTSPEC), - Array4(state_arr[box_no], TEMP), - Array4(cp_arr[box_no]), leosparm); + i, j, k, + amrex::Array4(state_arr[box_no], DENSITY), + amrex::Array4(state_arr[box_no], FIRSTSPEC), + amrex::Array4(state_arr[box_no], TEMP), + amrex::Array4(cp_arr[box_no]), leosparm); }); ldata_p->diff_aux_cc.divide(cp_cc, n, 1, ldata_p->diff_cc.nGrow()); } } } - Gpu::streamSynchronize(); + amrex::Gpu::streamSynchronize(); } -Array +amrex::Array PeleLM::getDiffusivity( int lev, int beta_comp, int ncomp, int doZeroVisc, - Vector bcrec, - MultiFab const& beta_cc, + amrex::Vector bcrec, + amrex::MultiFab const& beta_cc, int addTurbContrib) { BL_PROFILE("PeleLMeX::getDiffusivity()"); @@ -392,18 +399,18 @@ PeleLM::getDiffusivity( const auto& ba = beta_cc.boxArray(); const auto& dm = beta_cc.DistributionMap(); const auto& factory = beta_cc.Factory(); - Array beta_ec{AMREX_D_DECL( - MultiFab( - amrex::convert(ba, IntVect::TheDimensionVector(0)), dm, ncomp, 0, - MFInfo(), factory), - MultiFab( - amrex::convert(ba, IntVect::TheDimensionVector(1)), dm, ncomp, 0, - MFInfo(), factory), - MultiFab( - amrex::convert(ba, IntVect::TheDimensionVector(2)), dm, ncomp, 0, - MFInfo(), factory))}; - - const Box& domain = geom[lev].Domain(); + amrex::Array beta_ec{AMREX_D_DECL( + amrex::MultiFab( + amrex::convert(ba, amrex::IntVect::TheDimensionVector(0)), dm, ncomp, 0, + amrex::MFInfo(), factory), + amrex::MultiFab( + amrex::convert(ba, amrex::IntVect::TheDimensionVector(1)), dm, ncomp, 0, + amrex::MFInfo(), factory), + amrex::MultiFab( + amrex::convert(ba, amrex::IntVect::TheDimensionVector(2)), dm, ncomp, 0, + amrex::MFInfo(), factory))}; + + const amrex::Box& domain = geom[lev].Domain(); #ifdef AMREX_USE_EB // EB : use EB CCentroid -> FCentroid @@ -415,19 +422,19 @@ PeleLM::getDiffusivity( bool use_harmonic_avg = m_harm_avg_cen2edge != 0; #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(beta_cc, TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(beta_cc, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - const Box ebx = mfi.nodaltilebox(idim); - const Box& edomain = amrex::surroundingNodes(domain, idim); + const amrex::Box ebx = mfi.nodaltilebox(idim); + const amrex::Box& edomain = amrex::surroundingNodes(domain, idim); const auto& diff_c = beta_cc.const_array(mfi, beta_comp); const auto& diff_ec = beta_ec[idim].array(mfi); const auto bc_lo = bcrec[0].lo(idim); const auto bc_hi = bcrec[0].hi(idim); amrex::ParallelFor( - ebx, [idim, ncomp, bc_lo, bc_hi, use_harmonic_avg, diff_c, diff_ec, - edomain] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { int idx[3] = {i, j, k}; bool on_lo = ((bc_lo == amrex::BCType::ext_dir) && @@ -494,12 +501,13 @@ PeleLM::getDiffusivity( ProbParm const* lprobparm = prob_parm_d; const auto geomdata = geom[lev].data(); for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - const Box& edomain = amrex::surroundingNodes(domain, idim); + const amrex::Box& edomain = amrex::surroundingNodes(domain, idim); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(beta_ec[idim], TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box ebx = mfi.tilebox(); + for (amrex::MFIter mfi(beta_ec[idim], amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box ebx = mfi.tilebox(); const auto& diff_ec = beta_ec[idim].array(mfi); amrex::ParallelFor( ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { diff --git a/Source/PeleLMeX_UMac.cpp b/Source/PeleLMeX_UMac.cpp index 914933602..d85a8a450 100644 --- a/Source/PeleLMeX_UMac.cpp +++ b/Source/PeleLMeX_UMac.cpp @@ -4,8 +4,6 @@ #include #include -using namespace amrex; - void PeleLM::predictVelocity(std::unique_ptr& advData) { @@ -23,13 +21,13 @@ PeleLM::predictVelocity(std::unique_ptr& advData) //---------------------------------------------------------------- // Get viscous forces int nGrow_force = 1; - Vector divtau(finest_level + 1); - Vector velForces(finest_level + 1); + amrex::Vector divtau(finest_level + 1); + amrex::Vector velForces(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { divtau[lev].define( - grids[lev], dmap[lev], AMREX_SPACEDIM, 0, MFInfo(), Factory(lev)); + grids[lev], dmap[lev], AMREX_SPACEDIM, 0, amrex::MFInfo(), Factory(lev)); velForces[lev].define( - grids[lev], dmap[lev], AMREX_SPACEDIM, nGrow_force, MFInfo(), + grids[lev], dmap[lev], AMREX_SPACEDIM, nGrow_force, amrex::MFInfo(), Factory(lev)); } int use_density = 0; @@ -78,7 +76,7 @@ PeleLM::createMACRHS(std::unique_ptr& advData) BL_PROFILE("PeleLMeX::createMACRHS()"); for (int lev = 0; lev <= finest_level; ++lev) { - Real halftime = 0.5 * (m_t_old[lev] + m_t_new[lev]); + amrex::Real halftime = 0.5 * (m_t_old[lev] + m_t_new[lev]); fillpatch_divu(lev, halftime, advData->mac_divu[lev], m_nGrowAdv); } } @@ -92,10 +90,10 @@ PeleLM::addChiIncrement( BL_PROFILE("PeleLMeX::addChiIncrement()"); int nGrow = m_nGrowAdv; - Vector chiIncr(finest_level + 1); + amrex::Vector chiIncr(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { chiIncr[lev].define( - grids[lev], dmap[lev], 1, nGrow, MFInfo(), Factory(lev)); + grids[lev], dmap[lev], 1, nGrow, amrex::MFInfo(), Factory(lev)); } // Update the thermodynamic pressure @@ -109,18 +107,17 @@ PeleLM::addChiIncrement( // grownbox for (int lev = 0; lev <= finest_level; ++lev) { #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(advData->chi[lev], TilingIfNotGPU()); mfi.isValid(); - ++mfi) { - const Box& gbx = mfi.growntilebox(); + for (amrex::MFIter mfi(advData->chi[lev], amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& gbx = mfi.growntilebox(); auto const& chiInc_ar = chiIncr[lev].const_array(mfi); auto const& chi_ar = advData->chi[lev].array(mfi); auto const& mac_divu_ar = advData->mac_divu[lev].array(mfi); if (m_chi_correction_type == ChiCorrectionType::DivuFirstIter) { amrex::ParallelFor( - gbx, [chi_ar, chiInc_ar, mac_divu_ar, - a_sdcIter] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (a_sdcIter == 1) { chi_ar(i, j, k) = chiInc_ar(i, j, k) + mac_divu_ar(i, j, k); } else { @@ -130,8 +127,7 @@ PeleLM::addChiIncrement( }); } else if (m_chi_correction_type == ChiCorrectionType::NoDivu) { amrex::ParallelFor( - gbx, [chi_ar, chiInc_ar, mac_divu_ar, - a_sdcIter] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (a_sdcIter == 1) { chi_ar(i, j, k) = chiInc_ar(i, j, k); } else { @@ -141,8 +137,7 @@ PeleLM::addChiIncrement( }); } else { // Default: use updated divu every iteration amrex::ParallelFor( - gbx, [chi_ar, chiInc_ar, mac_divu_ar, - a_sdcIter] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (a_sdcIter == 1) { chi_ar(i, j, k) = chiInc_ar(i, j, k); } else { @@ -165,7 +160,7 @@ void PeleLM::macProject( const TimeStamp& a_time, std::unique_ptr& advData, - const Vector& a_divu) + const amrex::Vector& a_divu) { BL_PROFILE("PeleLMeX::macProject()"); @@ -173,14 +168,15 @@ PeleLM::macProject( // Get face rho inv auto bcRec = fetchBCRecArray(DENSITY, 1); - Vector> rho_inv(finest_level + 1); + amrex::Vector> rho_inv( + finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { if (m_incompressible != 0) { - Real rhoInv = m_dt / (2.0 * m_rho); + amrex::Real rhoInv = m_dt / (2.0 * m_rho); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { rho_inv[lev][idim].define( - amrex::convert(grids[lev], IntVect::TheDimensionVector(idim)), - dmap[lev], 1, 0, MFInfo(), Factory(lev)); + amrex::convert(grids[lev], amrex::IntVect::TheDimensionVector(idim)), + dmap[lev], 1, 0, amrex::MFInfo(), Factory(lev)); rho_inv[lev][idim].setVal(rhoInv); } } else { @@ -196,18 +192,18 @@ PeleLM::macProject( } // For closed chamber, compute change in chamber pressure - Real Sbar = 0.0; + amrex::Real Sbar = 0.0; if ((m_closed_chamber != 0) && (m_incompressible == 0)) { Sbar = adjustPandDivU(advData); } if (macproj->needInitialization()) { - LPInfo lpInfo; + amrex::LPInfo lpInfo; lpInfo.setMaxCoarseningLevel(m_mac_mg_max_coarsening_level); macproj->initProjector(lpInfo, GetVecOfArrOfConstPtrs(rho_inv)); macproj->setDomainBC( - getMACProjectionBC(Orientation::low), - getMACProjectionBC(Orientation::high)); + getMACProjectionBC(amrex::Orientation::low), + getMACProjectionBC(amrex::Orientation::high)); #ifdef AMREX_USE_HYPRE macproj->getMLMG().setHypreOptionsNamespace(m_hypre_namespace_mac); #endif @@ -246,7 +242,8 @@ PeleLM::macProject( if (lev > 0) { // We need to fill the MAC velocities outside the fine region so we can // use them in the Godunov method - IntVect rr = geom[lev].Domain().size() / geom[lev - 1].Domain().size(); + amrex::IntVect rr = + geom[lev].Domain().size() / geom[lev - 1].Domain().size(); create_constrained_umac_grown( m_nGrowMAC, &geom[lev - 1], &geom[lev], GetArrOfPtrs(advData->umac[lev - 1]), GetArrOfPtrs(advData->umac[lev]), @@ -263,63 +260,67 @@ PeleLM::macProject( void PeleLM::create_constrained_umac_grown( int a_nGrow, - const Geometry* crse_geom, - const Geometry* fine_geom, - Array u_mac_crse, - Array u_mac_fine, - const IntVect& crse_ratio) + const amrex::Geometry* crse_geom, + const amrex::Geometry* fine_geom, + amrex::Array u_mac_crse, + amrex::Array u_mac_fine, + const amrex::IntVect& crse_ratio) { // Divergence preserving interp - Interpolater* mapper = &face_divfree_interp; + amrex::Interpolater* mapper = &amrex::face_divfree_interp; // Set BCRec for Umac - Vector bcrec(1); + amrex::Vector bcrec(1); for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { if (crse_geom->isPeriodic(idim)) { - bcrec[0].setLo(idim, BCType::int_dir); - bcrec[0].setHi(idim, BCType::int_dir); + bcrec[0].setLo(idim, amrex::BCType::int_dir); + bcrec[0].setHi(idim, amrex::BCType::int_dir); } else { - bcrec[0].setLo(idim, BCType::foextrap); - bcrec[0].setHi(idim, BCType::foextrap); + bcrec[0].setLo(idim, amrex::BCType::foextrap); + bcrec[0].setHi(idim, amrex::BCType::foextrap); } } - Array, AMREX_SPACEDIM> bcrecArr = { + amrex::Array, AMREX_SPACEDIM> bcrecArr = { AMREX_D_DECL(bcrec, bcrec, bcrec)}; - PhysBCFunct> crse_bndry_func( + amrex::PhysBCFunct> crse_bndry_func( *crse_geom, bcrec, umacFill{}); - Array>, AMREX_SPACEDIM> cbndyFuncArr = { - AMREX_D_DECL(crse_bndry_func, crse_bndry_func, crse_bndry_func)}; + amrex::Array< + amrex::PhysBCFunct>, AMREX_SPACEDIM> + cbndyFuncArr = { + AMREX_D_DECL(crse_bndry_func, crse_bndry_func, crse_bndry_func)}; - PhysBCFunct> fine_bndry_func( + amrex::PhysBCFunct> fine_bndry_func( *fine_geom, bcrec, umacFill{}); - Array>, AMREX_SPACEDIM> fbndyFuncArr = { - AMREX_D_DECL(fine_bndry_func, fine_bndry_func, fine_bndry_func)}; + amrex::Array< + amrex::PhysBCFunct>, AMREX_SPACEDIM> + fbndyFuncArr = { + AMREX_D_DECL(fine_bndry_func, fine_bndry_func, fine_bndry_func)}; // Use piecewise constant interpolation in time, so create dummy variable for // time - Real dummy = 0.; + amrex::Real dummy = 0.; FillPatchTwoLevels( - u_mac_fine, IntVect(a_nGrow), dummy, {u_mac_crse}, {dummy}, {u_mac_fine}, - {dummy}, 0, 0, 1, *crse_geom, *fine_geom, cbndyFuncArr, 0, fbndyFuncArr, 0, - crse_ratio, mapper, bcrecArr, 0); + u_mac_fine, amrex::IntVect(a_nGrow), dummy, {u_mac_crse}, {dummy}, + {u_mac_fine}, {dummy}, 0, 0, 1, *crse_geom, *fine_geom, cbndyFuncArr, 0, + fbndyFuncArr, 0, crse_ratio, mapper, bcrecArr, 0); } -Array -PeleLM::getMACProjectionBC(Orientation::Side a_side) +amrex::Array +PeleLM::getMACProjectionBC(amrex::Orientation::Side a_side) { - Array r; + amrex::Array r; for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { if (Geom(0).isPeriodic(idim)) { - r[idim] = LinOpBCType::Periodic; + r[idim] = amrex::LinOpBCType::Periodic; } else { - auto physbc = - (a_side == Orientation::low) ? m_phys_bc.lo(idim) : m_phys_bc.hi(idim); + auto physbc = (a_side == amrex::Orientation::low) ? m_phys_bc.lo(idim) + : m_phys_bc.hi(idim); if (physbc == amrex::PhysBCType::outflow) { - r[idim] = LinOpBCType::Dirichlet; + r[idim] = amrex::LinOpBCType::Dirichlet; } else { - r[idim] = LinOpBCType::Neumann; + r[idim] = amrex::LinOpBCType::Neumann; } } } diff --git a/Source/PeleLMeX_Utils.cpp b/Source/PeleLMeX_Utils.cpp index 7ad46486b..0b5c26afc 100644 --- a/Source/PeleLMeX_Utils.cpp +++ b/Source/PeleLMeX_Utils.cpp @@ -7,8 +7,6 @@ #include #endif -using namespace amrex; - void writeBuildInfo() { @@ -17,38 +15,38 @@ writeBuildInfo() std::cout << " PeleLMeX Build Information\n"; std::cout << PrettyLine; - std::cout << "build date: " << buildInfoGetBuildDate() << "\n"; - std::cout << "build machine: " << buildInfoGetBuildMachine() << "\n"; - std::cout << "build dir: " << buildInfoGetBuildDir() << "\n"; - std::cout << "AMReX dir: " << buildInfoGetAMReXDir() << "\n"; + std::cout << "build date: " << amrex::buildInfoGetBuildDate() << "\n"; + std::cout << "build machine: " << amrex::buildInfoGetBuildMachine() << "\n"; + std::cout << "build dir: " << amrex::buildInfoGetBuildDir() << "\n"; + std::cout << "AMReX dir: " << amrex::buildInfoGetAMReXDir() << "\n"; std::cout << "\n"; - std::cout << "COMP: " << buildInfoGetComp() << "\n"; - std::cout << "COMP version: " << buildInfoGetCompVersion() << "\n"; + std::cout << "COMP: " << amrex::buildInfoGetComp() << "\n"; + std::cout << "COMP version: " << amrex::buildInfoGetCompVersion() << "\n"; std::cout << "\n"; - std::cout << "C++ compiler: " << buildInfoGetCXXName() << "\n"; - std::cout << "C++ flags: " << buildInfoGetCXXFlags() << "\n"; + std::cout << "C++ compiler: " << amrex::buildInfoGetCXXName() << "\n"; + std::cout << "C++ flags: " << amrex::buildInfoGetCXXFlags() << "\n"; std::cout << "\n"; - std::cout << "Link flags: " << buildInfoGetLinkFlags() << "\n"; - std::cout << "Libraries: " << buildInfoGetLibraries() << "\n"; + std::cout << "Link flags: " << amrex::buildInfoGetLinkFlags() << "\n"; + std::cout << "Libraries: " << amrex::buildInfoGetLibraries() << "\n"; std::cout << "\n"; - for (int n = 1; n <= buildInfoGetNumModules(); n++) { - std::cout << buildInfoGetModuleName(n) << ": " << buildInfoGetModuleVal(n) - << "\n"; + for (int n = 1; n <= amrex::buildInfoGetNumModules(); n++) { + std::cout << amrex::buildInfoGetModuleName(n) << ": " + << amrex::buildInfoGetModuleVal(n) << "\n"; } std::cout << "\n"; - const char* githash1 = buildInfoGetGitHash(1); - const char* githash2 = buildInfoGetGitHash(2); - const char* githash3 = buildInfoGetGitHash(3); + const char* githash1 = amrex::buildInfoGetGitHash(1); + const char* githash2 = amrex::buildInfoGetGitHash(2); + const char* githash3 = amrex::buildInfoGetGitHash(3); if (strlen(githash1) > 0) { std::cout << "PeleLMeX git describe: " << githash1 << "\n"; @@ -60,8 +58,8 @@ writeBuildInfo() std::cout << "PelePhysics git describe: " << githash3 << "\n"; } - const char* buildgithash = buildInfoGetBuildGitHash(); - const char* buildgitname = buildInfoGetBuildGitName(); + const char* buildgithash = amrex::buildInfoGetBuildGitHash(); + const char* buildgitname = amrex::buildInfoGetBuildGitName(); if (strlen(buildgithash) > 0) { std::cout << buildgitname << " git describe: " << buildgithash << "\n"; } @@ -71,13 +69,13 @@ writeBuildInfo() void PeleLM::fluxDivergence( - const Vector& a_divergence, + const amrex::Vector& a_divergence, int div_comp, - const Vector>& a_fluxes, + const amrex::Vector>& a_fluxes, int flux_comp, int ncomp, int intensiveFluxes, - Real scale) + amrex::Real scale) { BL_PROFILE("PeleLMeX::fluxDivergence()"); if (intensiveFluxes != 0) { // Fluxes are intensive -> need area scaling in @@ -98,15 +96,15 @@ PeleLM::fluxDivergence( void PeleLM::fluxDivergence( - const Vector& a_divergence, + const amrex::Vector& a_divergence, int div_comp, - const Vector>& a_fluxes, + const amrex::Vector>& a_fluxes, int flux_comp, - const Vector& a_EBfluxes, + const amrex::Vector& a_EBfluxes, int ebflux_comp, int ncomp, int intensiveFluxes, - Real scale) + amrex::Real scale) { BL_PROFILE("PeleLMeX::fluxDivergence()"); @@ -128,19 +126,19 @@ PeleLM::fluxDivergence( void PeleLM::fluxDivergenceRD( - const Vector& a_state, + const amrex::Vector& a_state, int state_comp, - const Vector& a_divergence, + const amrex::Vector& a_divergence, int div_comp, - const Vector>& a_fluxes, + const amrex::Vector>& a_fluxes, int flux_comp, - const Vector& a_EBfluxes, + const amrex::Vector& a_EBfluxes, int ebflux_comp, int ncomp, int intensiveFluxes, - const BCRec* state_bc_d, - const Real& scale, - const Real& a_dt) + const amrex::BCRec* state_bc_d, + const amrex::Real& scale, + const amrex::Real& a_dt) { BL_PROFILE("PeleLMeX::fluxDivergenceRD()"); #ifdef AMREX_USE_EB @@ -149,8 +147,9 @@ PeleLM::fluxDivergenceRD( //---------------------------------------------------------------- // Use a temporary MF to hold divergence before redistribution int nGrow_divTmp = 3; - MultiFab divTmp( - grids[lev], dmap[lev], ncomp, nGrow_divTmp, MFInfo(), EBFactory(lev)); + amrex::MultiFab divTmp( + grids[lev], dmap[lev], ncomp, nGrow_divTmp, amrex::MFInfo(), + EBFactory(lev)); divTmp.setVal(0.0); if (intensiveFluxes != 0) { // Fluxes are intensive -> need area scaling in // div @@ -190,18 +189,18 @@ PeleLM::fluxDivergenceRD( void PeleLM::extFluxDivergenceLevel( int lev, - MultiFab& a_divergence, + amrex::MultiFab& a_divergence, int div_comp, - const Array& a_fluxes, + const amrex::Array& a_fluxes, int flux_comp, int ncomp, - Real scale) + amrex::Real scale) { AMREX_ASSERT(a_divergence.nComp() >= div_comp + ncomp); // Get the volume - MultiFab volume(grids[lev], dmap[lev], 1, 0); + amrex::MultiFab volume(grids[lev], dmap[lev], 1, 0); geom[lev].GetVolume(volume); #ifdef AMREX_USE_EB @@ -209,10 +208,11 @@ PeleLM::extFluxDivergenceLevel( #endif #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(a_divergence, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(a_divergence, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + const amrex::Box& bx = mfi.tilebox(); AMREX_D_TERM( auto const& fluxX = a_fluxes[0]->const_array(mfi, flux_comp); , auto const& fluxY = a_fluxes[1]->const_array(mfi, flux_comp); @@ -226,17 +226,17 @@ PeleLM::extFluxDivergenceLevel( #endif #ifdef AMREX_USE_EB - if (flagfab.getType(bx) == FabType::covered) { // Covered boxes + if (flagfab.getType(bx) == amrex::FabType::covered) { // Covered boxes amrex::ParallelFor( - bx, ncomp, - [divergence] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { divergence(i, j, k, n) = 0.0; }); - } else if (flagfab.getType(bx) != FabType::regular) { // EB containing boxes + } else if (flagfab.getType(bx) != amrex::FabType::regular) { // EB + // containing + // boxes auto vfrac = ebfact.getVolFrac().const_array(mfi); amrex::ParallelFor( - bx, [ncomp, flag, vfrac, divergence, AMREX_D_DECL(fluxX, fluxY, fluxZ), - vol, scale] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (flag(i, j, k).isCovered()) { for (int n = 0; n < ncomp; n++) { divergence(i, j, k, n) = 0.0; @@ -246,7 +246,7 @@ PeleLM::extFluxDivergenceLevel( i, j, k, ncomp, AMREX_D_DECL(fluxX, fluxY, fluxZ), vol, scale, divergence); } else { - Real vfracinv = 1.0 / vfrac(i, j, k); + amrex::Real vfracinv = 1.0 / vfrac(i, j, k); extFluxDivergence_K( i, j, k, ncomp, AMREX_D_DECL(fluxX, fluxY, fluxZ), vol, scale, divergence); @@ -259,8 +259,7 @@ PeleLM::extFluxDivergenceLevel( #endif { amrex::ParallelFor( - bx, [ncomp, divergence, AMREX_D_DECL(fluxX, fluxY, fluxZ), vol, - scale] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { extFluxDivergence_K( i, j, k, ncomp, AMREX_D_DECL(fluxX, fluxY, fluxZ), vol, scale, divergence); @@ -272,48 +271,49 @@ PeleLM::extFluxDivergenceLevel( void PeleLM::intFluxDivergenceLevel( int lev, - MultiFab& a_divergence, + amrex::MultiFab& a_divergence, int div_comp, - const Array& a_fluxes, + const amrex::Array& a_fluxes, int flux_comp, int ncomp, - Real scale) + amrex::Real scale) { AMREX_ASSERT(a_divergence.nComp() >= div_comp + ncomp); // Get the volume - MultiFab volume(grids[lev], dmap[lev], 1, 0); + amrex::MultiFab volume(grids[lev], dmap[lev], 1, 0); geom[lev].GetVolume(volume); // Get areas - const Real* dx = Geom(lev).CellSize(); + const amrex::Real* dx = Geom(lev).CellSize(); #if (AMREX_SPACEDIM == 2) - MultiFab mf_ax, mf_ay; + amrex::MultiFab mf_ax, mf_ay; if (geom[lev].IsRZ()) { geom[lev].GetFaceArea(mf_ax, grids[lev], dmap[lev], 0, 0); geom[lev].GetFaceArea(mf_ay, grids[lev], dmap[lev], 1, 0); } - Real areax = dx[1]; - Real areay = dx[0]; + amrex::Real areax = dx[1]; + amrex::Real areay = dx[0]; #elif (AMREX_SPACEDIM == 3) - Real areax = dx[1] * dx[2]; - Real areay = dx[0] * dx[2]; - Real areaz = dx[0] * dx[1]; + amrex::Real areax = dx[1] * dx[2]; + amrex::Real areay = dx[0] * dx[2]; + amrex::Real areaz = dx[0] * dx[1]; #endif // Get areafrac if EB #ifdef AMREX_USE_EB auto const& ebfact = EBFactory(lev); - Array areafrac; + amrex::Array areafrac; areafrac = ebfact.getAreaFrac(); #endif #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(a_divergence, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(a_divergence, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + const amrex::Box& bx = mfi.tilebox(); AMREX_D_TERM( auto const& fluxX = a_fluxes[0]->const_array(mfi, flux_comp); , auto const& fluxY = a_fluxes[1]->const_array(mfi, flux_comp); @@ -325,23 +325,21 @@ PeleLM::intFluxDivergenceLevel( auto const& flagfab = ebfact.getMultiEBCellFlagFab()[mfi]; auto const& flag = flagfab.const_array(); - if (flagfab.getType(bx) == FabType::covered) { // Covered boxes + if (flagfab.getType(bx) == amrex::FabType::covered) { // Covered boxes amrex::ParallelFor( - bx, ncomp, - [divergence] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { divergence(i, j, k, n) = 0.0; }); - } else if (flagfab.getType(bx) != FabType::regular) { // EB containing boxes + } else if (flagfab.getType(bx) != amrex::FabType::regular) { // EB + // containing + // boxes auto vfrac = ebfact.getVolFrac().const_array(mfi); AMREX_D_TERM( const auto& afrac_x = areafrac[0]->array(mfi); , const auto& afrac_y = areafrac[1]->array(mfi); , const auto& afrac_z = areafrac[2]->array(mfi);); amrex::ParallelFor( - bx, [ncomp, flag, vfrac, divergence, AMREX_D_DECL(fluxX, fluxY, fluxZ), - AMREX_D_DECL(afrac_x, afrac_y, afrac_z), - AMREX_D_DECL(areax, areay, areaz), vol, - scale] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (flag(i, j, k).isCovered()) { for (int n = 0; n < ncomp; n++) { divergence(i, j, k, n) = 0.0; @@ -351,7 +349,7 @@ PeleLM::intFluxDivergenceLevel( i, j, k, ncomp, AMREX_D_DECL(fluxX, fluxY, fluxZ), AMREX_D_DECL(areax, areay, areaz), vol, scale, divergence); } else { - Real vfracinv = 1.0 / vfrac(i, j, k); + amrex::Real vfracinv = 1.0 / vfrac(i, j, k); EB_intFluxDivergence_K( i, j, k, ncomp, AMREX_D_DECL(fluxX, fluxY, fluxZ), AMREX_D_DECL(afrac_x, afrac_y, afrac_z), @@ -366,11 +364,10 @@ PeleLM::intFluxDivergenceLevel( { #if (AMREX_SPACEDIM == 2) if (geom[lev].IsRZ()) { - Array4 const& ax = mf_ax.const_array(mfi); - Array4 const& ay = mf_ay.const_array(mfi); + amrex::Array4 const& ax = mf_ax.const_array(mfi); + amrex::Array4 const& ay = mf_ay.const_array(mfi); amrex::ParallelFor( - bx, [ncomp, divergence, AMREX_D_DECL(fluxX, fluxY, fluxZ), ax, ay, - vol, scale] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { intFluxDivergence_rz_K( i, j, k, ncomp, AMREX_D_DECL(fluxX, fluxY, fluxZ), ax, ay, vol, scale, divergence); @@ -379,9 +376,7 @@ PeleLM::intFluxDivergenceLevel( #endif { amrex::ParallelFor( - bx, [ncomp, divergence, AMREX_D_DECL(fluxX, fluxY, fluxZ), - AMREX_D_DECL(areax, areay, areaz), vol, - scale] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { intFluxDivergence_K( i, j, k, ncomp, AMREX_D_DECL(fluxX, fluxY, fluxZ), AMREX_D_DECL(areax, areay, areaz), vol, scale, divergence); @@ -394,37 +389,38 @@ PeleLM::intFluxDivergenceLevel( void PeleLM::intFluxDivergenceLevelEB( int lev, - MultiFab& a_divergence, + amrex::MultiFab& a_divergence, int div_comp, - const Array& a_fluxes, + const amrex::Array& a_fluxes, int flux_comp, - const MultiFab* a_EBfluxes, + const amrex::MultiFab* a_EBfluxes, int ebflux_comp, int ncomp, - Real scale) + amrex::Real scale) { AMREX_ASSERT(a_divergence.nComp() >= div_comp + ncomp); // Get the volume - MultiFab volume(grids[lev], dmap[lev], 1, 0); + amrex::MultiFab volume(grids[lev], dmap[lev], 1, 0); geom[lev].GetVolume(volume); // Get area - const GpuArray dx = Geom(lev).CellSizeArray(); + const amrex::GpuArray dx = + Geom(lev).CellSizeArray(); #if (AMREX_SPACEDIM == 2) - Real areax = dx[1]; - Real areay = dx[0]; + amrex::Real areax = dx[1]; + amrex::Real areay = dx[0]; #elif (AMREX_SPACEDIM == 3) - Real areax = dx[1] * dx[2]; - Real areay = dx[0] * dx[2]; - Real areaz = dx[0] * dx[1]; + amrex::Real areax = dx[1] * dx[2]; + amrex::Real areay = dx[0] * dx[2]; + amrex::Real areaz = dx[0] * dx[1]; #endif // Get areafrac if EB #ifdef AMREX_USE_EB auto const& ebfact = EBFactory(lev); - Array areafrac; + amrex::Array areafrac; areafrac = ebfact.getAreaFrac(); const auto* eb_area = &(ebfact.getBndryArea()); #else @@ -432,10 +428,11 @@ PeleLM::intFluxDivergenceLevelEB( #endif #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(a_divergence, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(a_divergence, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + const amrex::Box& bx = mfi.tilebox(); AMREX_D_TERM( auto const& fluxX = a_fluxes[0]->const_array(mfi, flux_comp); , auto const& fluxY = a_fluxes[1]->const_array(mfi, flux_comp); @@ -448,13 +445,14 @@ PeleLM::intFluxDivergenceLevelEB( auto const& flagfab = ebfact.getMultiEBCellFlagFab()[mfi]; auto const& flag = flagfab.const_array(); - if (flagfab.getType(bx) == FabType::covered) { // Covered boxes + if (flagfab.getType(bx) == amrex::FabType::covered) { // Covered boxes amrex::ParallelFor( - bx, ncomp, - [divergence] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { divergence(i, j, k, n) = 0.0; }); - } else if (flagfab.getType(bx) != FabType::regular) { // EB containing boxes + } else if (flagfab.getType(bx) != amrex::FabType::regular) { // EB + // containing + // boxes auto vfrac = ebfact.getVolFrac().const_array(mfi); AMREX_D_TERM( const auto& afrac_x = areafrac[0]->array(mfi); @@ -472,7 +470,7 @@ PeleLM::intFluxDivergenceLevelEB( i, j, k, ncomp, AMREX_D_DECL(fluxX, fluxY, fluxZ), AMREX_D_DECL(areax, areay, areaz), vol, scale, divergence); } else { - Real vfracinv = 1.0 / vfrac(i, j, k); + amrex::Real vfracinv = 1.0 / vfrac(i, j, k); EB_intFluxDivergence_K( i, j, k, ncomp, AMREX_D_DECL(fluxX, fluxY, fluxZ), AMREX_D_DECL(afrac_x, afrac_y, afrac_z), @@ -487,9 +485,7 @@ PeleLM::intFluxDivergenceLevelEB( #endif { amrex::ParallelFor( - bx, [ncomp, divergence, AMREX_D_DECL(fluxX, fluxY, fluxZ), - AMREX_D_DECL(areax, areay, areaz), vol, - scale] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { intFluxDivergence_K( i, j, k, ncomp, AMREX_D_DECL(fluxX, fluxY, fluxZ), AMREX_D_DECL(areax, areay, areaz), vol, scale, divergence); @@ -502,16 +498,16 @@ void PeleLM:: advFluxDivergence( // NOLINT(readability-convert-member-functions-to-static) int a_lev, - MultiFab& a_divergence, + amrex::MultiFab& a_divergence, int div_comp, - MultiFab& a_divu, - const Array& a_fluxes, + amrex::MultiFab& a_divu, + const amrex::Array& a_fluxes, int flux_comp, - const Array& a_faceState, + const amrex::Array& a_faceState, int face_comp, int ncomp, int const* l_conserv_d, - const Geometry& a_geom, + const amrex::Geometry& a_geom, amrex::Real scale, bool fluxes_are_area_weighted) const { @@ -522,7 +518,7 @@ PeleLM:: AMREX_ASSERT(a_faceState[0]->nComp() >= face_comp + ncomp); #if (AMREX_SPACEDIM == 2) - MultiFab mf_ax, mf_ay; + amrex::MultiFab mf_ax, mf_ay; if (geom[a_lev].IsRZ()) { geom[a_lev].GetFaceArea(mf_ax, grids[a_lev], dmap[a_lev], 0, 0); geom[a_lev].GetFaceArea(mf_ay, grids[a_lev], dmap[a_lev], 1, 0); @@ -536,11 +532,12 @@ PeleLM:: #endif #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(a_divergence, TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(a_divergence, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { - Box const& bx = mfi.tilebox(); + amrex::Box const& bx = mfi.tilebox(); // Get the divergence auto const& div_arr = a_divergence.array(mfi, div_comp); @@ -552,11 +549,11 @@ PeleLM:: #ifdef AMREX_USE_EB auto const& flagfab = ebfact.getMultiEBCellFlagFab()[mfi]; auto const& vfrac_arr = ebfact.getVolFrac().const_array(mfi); - if (flagfab.getType(bx) == FabType::singlevalued) { + if (flagfab.getType(bx) == amrex::FabType::singlevalued) { HydroUtils::EB_ComputeDivergence( bx, div_arr, AMREX_D_DECL(fx, fy, fz), vfrac_arr, ncomp, a_geom, scale, fluxes_are_area_weighted); - } else if (flagfab.getType(bx) == FabType::regular) + } else if (flagfab.getType(bx) == amrex::FabType::regular) #endif { HydroUtils::ComputeDivergence( @@ -573,25 +570,25 @@ PeleLM:: , auto const& facez = a_faceState[2]->const_array(mfi, face_comp);) #ifdef AMREX_USE_EB - if (flagfab.getType(bx) == FabType::covered) { + if (flagfab.getType(bx) == amrex::FabType::covered) { AMREX_PARALLEL_FOR_4D( bx, ncomp, i, j, k, n, { div_arr(i, j, k, n) = 0.0; }); - } else if (flagfab.getType(bx) == FabType::singlevalued) { + } else if (flagfab.getType(bx) == amrex::FabType::singlevalued) { AMREX_D_TERM( auto const& apx_arr = ebfact.getAreaFrac()[0]->const_array(mfi); , auto const& apy_arr = ebfact.getAreaFrac()[1]->const_array(mfi); , auto const& apz_arr = ebfact.getAreaFrac()[2]->const_array(mfi);); - ParallelFor( + amrex::ParallelFor( bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { if ((l_conserv_d[n] == 0) && vfrac_arr(i, j, k) > 0.) { - Real qwsum = AMREX_D_TERM( + amrex::Real qwsum = AMREX_D_TERM( apx_arr(i, j, k) * facex(i, j, k, n) + apx_arr(i + 1, j, k) * facex(i + 1, j, k, n), +apy_arr(i, j, k) * facey(i, j, k, n) + apy_arr(i, j + 1, k) * facey(i, j + 1, k, n), +apz_arr(i, j, k) * facez(i, j, k, n) + apz_arr(i, j, k + 1) * facez(i, j, k + 1, n)); - Real areasum = AMREX_D_TERM( + amrex::Real areasum = AMREX_D_TERM( apx_arr(i, j, k) + apx_arr(i + 1, j, k), +apy_arr(i, j, k) + apy_arr(i, j + 1, k), +apz_arr(i, j, k) + apz_arr(i, j, k + 1)); @@ -605,18 +602,18 @@ PeleLM:: { #if (AMREX_SPACEDIM == 2) if (geom[a_lev].IsRZ()) { - Array4 const& ax = mf_ax.const_array(mfi); - Array4 const& ay = mf_ay.const_array(mfi); - ParallelFor( + amrex::Array4 const& ax = mf_ax.const_array(mfi); + amrex::Array4 const& ay = mf_ay.const_array(mfi); + amrex::ParallelFor( bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { if (l_conserv_d[n] == 0) { - Real qavg = AMREX_D_TERM( + amrex::Real qavg = AMREX_D_TERM( ax(i, j, k) * facex(i, j, k, n) + ax(i + 1, j, k) * facex(i + 1, j, k, n), +ay(i, j, k) * facey(i, j, k, n) + ay(i, j + 1, k) * facey(i, j + 1, k, n), +0.0); - Real areasum = + amrex::Real areasum = ax(i, j, k) + ax(i + 1, j, k) + ay(i, j, k) + ay(i, j + 1, k); qavg /= areasum; // Note that because we define adv update as MINUS div(u q), here @@ -627,10 +624,10 @@ PeleLM:: } else #endif { - ParallelFor( + amrex::ParallelFor( bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { if (l_conserv_d[n] == 0) { - Real qavg = AMREX_D_TERM( + amrex::Real qavg = AMREX_D_TERM( facex(i, j, k, n) + facex(i + 1, j, k, n), +facey(i, j, k, n) + facey(i, j + 1, k, n), +facez(i, j, k, n) + facez(i, j, k + 1, n)); @@ -649,18 +646,18 @@ PeleLM:: void PeleLM::advFluxDivergence( int a_lev, - MultiFab& a_divergence, + amrex::MultiFab& a_divergence, int div_comp, - MultiFab& a_divu, - const Array& a_fluxes, + amrex::MultiFab& a_divu, + const amrex::Array& a_fluxes, int flux_comp, - const Array& a_faceState, + const amrex::Array& a_faceState, int face_comp, - const MultiFab* a_EBvelocity, - const MultiFab* a_EBvalue, + const amrex::MultiFab* a_EBvelocity, + const amrex::MultiFab* a_EBvalue, int ncomp, int const* l_conserv_d, - const Geometry& a_geom, + const amrex::Geometry& a_geom, amrex::Real scale, bool fluxes_are_area_weighted) const { @@ -673,11 +670,12 @@ PeleLM::advFluxDivergence( auto const& ebfact = EBFactory(a_lev); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(a_divergence, TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(a_divergence, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { - Box const& bx = mfi.tilebox(); + amrex::Box const& bx = mfi.tilebox(); // Get the divergence auto const& div_arr = a_divergence.array(mfi, div_comp); @@ -690,14 +688,14 @@ PeleLM::advFluxDivergence( auto const& vfrac_arr = ebfact.getVolFrac().const_array(mfi); auto const& ebvel_arr = a_EBvelocity->const_array(mfi); auto const& ebval_arr = a_EBvalue->const_array(mfi); - if (flagfab.getType(bx) == FabType::singlevalued) { + if (flagfab.getType(bx) == amrex::FabType::singlevalued) { HydroUtils::EB_ComputeDivergence( bx, div_arr, AMREX_D_DECL(fx, fy, fz), vfrac_arr, ncomp, a_geom, scale, fluxes_are_area_weighted, ebvel_arr, ebval_arr, ebfact.getMultiEBCellFlagFab().const_array(mfi), ebfact.getBndryArea().const_array(mfi), ebfact.getBndryNormal().const_array(mfi)); - } else if (flagfab.getType(bx) == FabType::regular) { + } else if (flagfab.getType(bx) == amrex::FabType::regular) { HydroUtils::ComputeDivergence( bx, div_arr, AMREX_D_DECL(fx, fy, fz), ncomp, a_geom, scale, fluxes_are_area_weighted); @@ -711,25 +709,25 @@ PeleLM::advFluxDivergence( , auto const& facey = a_faceState[1]->const_array(mfi, face_comp); , auto const& facez = a_faceState[2]->const_array(mfi, face_comp);) - if (flagfab.getType(bx) == FabType::covered) { + if (flagfab.getType(bx) == amrex::FabType::covered) { AMREX_PARALLEL_FOR_4D( bx, ncomp, i, j, k, n, { div_arr(i, j, k, n) = 0.0; }); - } else if (flagfab.getType(bx) == FabType::singlevalued) { + } else if (flagfab.getType(bx) == amrex::FabType::singlevalued) { AMREX_D_TERM( auto const& apx_arr = ebfact.getAreaFrac()[0]->const_array(mfi); , auto const& apy_arr = ebfact.getAreaFrac()[1]->const_array(mfi); , auto const& apz_arr = ebfact.getAreaFrac()[2]->const_array(mfi);); - ParallelFor( + amrex::ParallelFor( bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { if (l_conserv_d[n] == 0 && vfrac_arr(i, j, k) > 0.) { - Real qwsum = AMREX_D_TERM( + amrex::Real qwsum = AMREX_D_TERM( apx_arr(i, j, k) * facex(i, j, k, n) + apx_arr(i + 1, j, k) * facex(i + 1, j, k, n), +apy_arr(i, j, k) * facey(i, j, k, n) + apy_arr(i, j + 1, k) * facey(i, j + 1, k, n), +apz_arr(i, j, k) * facez(i, j, k, n) + apz_arr(i, j, k + 1) * facez(i, j, k + 1, n)); - Real areasum = AMREX_D_TERM( + amrex::Real areasum = AMREX_D_TERM( apx_arr(i, j, k) + apx_arr(i + 1, j, k), +apy_arr(i, j, k) + apy_arr(i, j + 1, k), +apz_arr(i, j, k) + apz_arr(i, j, k + 1)); @@ -739,10 +737,10 @@ PeleLM::advFluxDivergence( } }); } else { - ParallelFor( + amrex::ParallelFor( bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { if (l_conserv_d[n] == 0) { - Real qavg = AMREX_D_TERM( + amrex::Real qavg = AMREX_D_TERM( facex(i, j, k, n) + facex(i + 1, j, k, n), +facey(i, j, k, n) + facey(i, j + 1, k, n), +facez(i, j, k, n) + facez(i, j, k + 1, n)); @@ -777,15 +775,16 @@ PeleLM::floorSpecies(const TimeStamp& a_time) [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { fabMinMax( i, j, k, NUM_SPECIES, 0.0, AMREX_REAL_MAX, - Array4(sma[box_no], FIRSTSPEC)); + amrex::Array4(sma[box_no], FIRSTSPEC)); #ifdef PELE_USE_PLASMA fabMinMax( - i, j, k, 1, 0.0, AMREX_REAL_MAX, Array4(sma[box_no], NE)); + i, j, k, 1, 0.0, AMREX_REAL_MAX, + amrex::Array4(sma[box_no], NE)); #endif // Update density and RhoH accordingly ... - Real massfrac[NUM_SPECIES] = {0.0}; - Real massdens[NUM_SPECIES] = {0.0}; - Real rhoinv, h_cgs = 0.0; + amrex::Real massfrac[NUM_SPECIES] = {0.0}; + amrex::Real massdens[NUM_SPECIES] = {0.0}; + amrex::Real rhoinv, h_cgs = 0.0; for (int n = 0; n < NUM_SPECIES; n++) { massdens[n] = sma[box_no](i, j, k, FIRSTSPEC + n); } @@ -796,7 +795,7 @@ PeleLM::floorSpecies(const TimeStamp& a_time) sma[box_no](i, j, k, RHOH) = h_cgs * 1.0e-4 * sma[box_no](i, j, k, DENSITY); }); - Gpu::streamSynchronize(); + amrex::Gpu::streamSynchronize(); } } @@ -807,26 +806,26 @@ PeleLM::resetCoveredMask() if (m_resetCoveredMask != 0) { if (m_verbose != 0) { - Print() << " Resetting fine-covered cells mask \n"; + amrex::Print() << " Resetting fine-covered cells mask \n"; } for (int lev = 0; lev < finest_level; ++lev) { // Set a fine-covered mask - BoxArray baf = grids[lev + 1]; + amrex::BoxArray baf = grids[lev + 1]; baf.coarsen(ref_ratio[lev]); m_coveredMask[lev]->setVal(1); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif { - std::vector> isects; - for (MFIter mfi(*m_coveredMask[lev], TilingIfNotGPU()); mfi.isValid(); - ++mfi) { + std::vector> isects; + for (amrex::MFIter mfi(*m_coveredMask[lev], amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { auto const& mask = m_coveredMask[lev]->array(mfi); baf.intersections(grids[lev][mfi.index()], isects); for (const auto& is : isects) { amrex::ParallelFor( - is.second, [mask] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + is.second, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { mask(i, j, k) = 0; }); } @@ -837,15 +836,15 @@ PeleLM::resetCoveredMask() // Setup a BoxArray for the chemistry // Get an uncovered BoxArray - BoxArray baCompDom = complementIn(geom[lev].Domain(), baf); - BoxArray baUnCovered = intersect(baCompDom, grids[lev]); + amrex::BoxArray baCompDom = complementIn(geom[lev].Domain(), baf); + amrex::BoxArray baUnCovered = intersect(baCompDom, grids[lev]); // Chop in smaller boxes if triggered if (m_max_grid_size_chem.min() > 0) { baUnCovered.maxSize(m_max_grid_size_chem); } // Assemble a BoxArray with covered and uncovered ones + flags - BoxList bl(grids[lev].ixType()); + amrex::BoxList bl(grids[lev].ixType()); bl.reserve(baUnCovered.size() + baf.size()); m_baChemFlag[lev].resize(baUnCovered.size() + baf.size()); int bxIdx = 0; @@ -861,8 +860,9 @@ PeleLM::resetCoveredMask() m_baChemFlag[lev][bxIdx] = 0; bxIdx += 1; } - m_baChem[lev] = std::make_unique(std::move(bl)); - m_dmapChem[lev] = std::make_unique(*m_baChem[lev]); + m_baChem[lev] = std::make_unique(std::move(bl)); + m_dmapChem[lev] = + std::make_unique(*m_baChem[lev]); // Load balancing of the chemistry DMap if (m_doLoadBalance != 0) { @@ -871,7 +871,8 @@ PeleLM::resetCoveredMask() } // Set a BoxArray for the chemistry on the finest level too - m_baChem[finest_level] = std::make_unique(grids[finest_level]); + m_baChem[finest_level] = + std::make_unique(grids[finest_level]); if (m_max_grid_size_chem.min() > 0) { m_baChem[finest_level]->maxSize(m_max_grid_size_chem); } @@ -879,7 +880,7 @@ PeleLM::resetCoveredMask() std::fill( m_baChemFlag[finest_level].begin(), m_baChemFlag[finest_level].end(), 1); m_dmapChem[finest_level] = - std::make_unique(*m_baChem[finest_level]); + std::make_unique(*m_baChem[finest_level]); if ((m_doLoadBalance != 0) && m_max_grid_size_chem.min() > 0) { loadBalanceChemLev(finest_level); @@ -898,9 +899,10 @@ PeleLM::resetCoveredMask() //---------------------------------------------------------------------------- // Need to compute the uncovered volume if (m_uncoveredVol < 0.0) { - Vector dummy(finest_level + 1); + amrex::Vector dummy(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - dummy[lev].define(grids[lev], dmap[lev], 1, 0, MFInfo(), *m_factory[lev]); + dummy[lev].define( + grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), *m_factory[lev]); dummy[lev].setVal(1.0); } m_uncoveredVol = MFSum(GetVecOfConstPtrs(dummy), 0); @@ -924,80 +926,84 @@ PeleLM::loadBalanceChem() void PeleLM::loadBalanceChemLev(int a_lev) { - - LayoutData new_cost(*m_baChem[a_lev], *m_dmapChem[a_lev]); + amrex::LayoutData new_cost(*m_baChem[a_lev], *m_dmapChem[a_lev]); computeCosts(a_lev, new_cost, m_loadBalanceCostChem); // Use efficiency: average MPI rank cost / max cost amrex::Real currentEfficiency = 0.0; amrex::Real testEfficiency = 0.0; - DistributionMapping test_dmap; + amrex::DistributionMapping test_dmap; // Build the test dmap, w/o braodcasting if (m_loadBalanceMethodChem == LoadBalanceMethod::SFC) { - test_dmap = DistributionMapping::makeSFC( + test_dmap = amrex::DistributionMapping::makeSFC( new_cost, currentEfficiency, testEfficiency, false, - ParallelDescriptor::IOProcessorNumber()); + amrex::ParallelDescriptor::IOProcessorNumber()); } else if (m_loadBalanceMethodChem == LoadBalanceMethod::Knapsack) { - const amrex::Real navg = static_cast(m_baChem[a_lev]->size()) / - static_cast(ParallelDescriptor::NProcs()); + const amrex::Real navg = + static_cast(m_baChem[a_lev]->size()) / + static_cast(amrex::ParallelDescriptor::NProcs()); const int nmax = static_cast( - std::max(std::round(m_loadBalanceKSfactor * navg), std::ceil(navg))); - test_dmap = DistributionMapping::makeKnapSack( + amrex::max(std::round(m_loadBalanceKSfactor * navg), std::ceil(navg))); + test_dmap = amrex::DistributionMapping::makeKnapSack( new_cost, currentEfficiency, testEfficiency, nmax, false, - ParallelDescriptor::IOProcessorNumber()); + amrex::ParallelDescriptor::IOProcessorNumber()); } // IO proc determine if the test dmap offers significant improvements int updateDmap = 0; if ( (m_loadBalanceEffRatioThreshold > 0.0) && - (ParallelDescriptor::MyProc() == ParallelDescriptor::IOProcessorNumber())) { + (amrex::ParallelDescriptor::MyProc() == + amrex::ParallelDescriptor::IOProcessorNumber())) { updateDmap = static_cast( testEfficiency > m_loadBalanceEffRatioThreshold * currentEfficiency); } - ParallelDescriptor::Bcast( - &updateDmap, 1, ParallelDescriptor::IOProcessorNumber()); + amrex::ParallelDescriptor::Bcast( + &updateDmap, 1, amrex::ParallelDescriptor::IOProcessorNumber()); if (m_verbose > 2 && (updateDmap != 0)) { - Print() << " Old Chem LoadBalancing efficiency on a_lev " << a_lev << ": " - << currentEfficiency << "\n" - << " New Chem LoadBalancing efficiency: " << testEfficiency - << " \n"; + amrex::Print() << " Old Chem LoadBalancing efficiency on a_lev " << a_lev + << ": " << currentEfficiency << "\n" + << " New Chem LoadBalancing efficiency: " << testEfficiency + << " \n"; } // Bcast the test dmap if better if (updateDmap != 0) { - Vector pmap; + amrex::Vector pmap; if ( - ParallelDescriptor::MyProc() == ParallelDescriptor::IOProcessorNumber()) { + amrex::ParallelDescriptor::MyProc() == + amrex::ParallelDescriptor::IOProcessorNumber()) { pmap = test_dmap.ProcessorMap(); } else { #pragma GCC diagnostic ignored "-Wnull-dereference" pmap.resize(static_cast(m_baChem[a_lev]->size())); } - ParallelDescriptor::Bcast( - pmap.data(), pmap.size(), ParallelDescriptor::IOProcessorNumber()); + amrex::ParallelDescriptor::Bcast( + pmap.data(), pmap.size(), amrex::ParallelDescriptor::IOProcessorNumber()); if ( - ParallelDescriptor::MyProc() != ParallelDescriptor::IOProcessorNumber()) { - test_dmap = DistributionMapping(pmap); + amrex::ParallelDescriptor::MyProc() != + amrex::ParallelDescriptor::IOProcessorNumber()) { + test_dmap = amrex::DistributionMapping(pmap); } - m_dmapChem[a_lev] = std::make_unique(test_dmap); + m_dmapChem[a_lev] = std::make_unique(test_dmap); } } // Return a unique_ptr with the entire derive -std::unique_ptr -PeleLM::derive(const std::string& a_name, Real a_time, int lev, int nGrow) +std::unique_ptr +PeleLM::derive( + const std::string& a_name, amrex::Real a_time, int lev, int nGrow) { BL_PROFILE("PeleLMeX::derive()"); AMREX_ASSERT(nGrow >= 0); - std::unique_ptr mf; + std::unique_ptr mf; bool itexists = derive_lst.canDerive(a_name) || isStateVariable(a_name) || isReactVariable(a_name); @@ -1009,57 +1015,62 @@ PeleLM::derive(const std::string& a_name, Real a_time, int lev, int nGrow) const PeleLMDeriveRec* rec = derive_lst.get(a_name); if (rec != nullptr) { // This is a derived variable - mf = std::make_unique( - grids[lev], dmap[lev], rec->numDerive(), nGrow, MFInfo(), Factory(lev)); - std::unique_ptr statemf = + mf = std::make_unique( + grids[lev], dmap[lev], rec->numDerive(), nGrow, amrex::MFInfo(), + Factory(lev)); + std::unique_ptr statemf = fillPatchState(lev, a_time, m_nGrowState); // Get pressure: TODO no fillpatch for pressure just yet, simply get new // state auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); - std::unique_ptr reactmf; + std::unique_ptr reactmf; if (m_do_react != 0) { reactmf = fillPatchReact(lev, a_time, nGrow); } auto stateBCs = fetchBCRecArray(VELX, NVAR); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.growntilebox(nGrow); - FArrayBox& derfab = (*mf)[mfi]; - FArrayBox const& statefab = (*statemf)[mfi]; - FArrayBox const& reactfab = + for (amrex::MFIter mfi(*mf, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + const amrex::Box& bx = mfi.growntilebox(nGrow); + amrex::FArrayBox& derfab = (*mf)[mfi]; + amrex::FArrayBox const& statefab = (*statemf)[mfi]; + amrex::FArrayBox const& reactfab = (m_incompressible) != 0 ? ldata_p->press[mfi] : (*reactmf)[mfi]; - FArrayBox const& pressfab = ldata_p->press[mfi]; + amrex::FArrayBox const& pressfab = ldata_p->press[mfi]; rec->derFunc()( this, bx, derfab, 0, rec->numDerive(), statefab, reactfab, pressfab, geom[lev], a_time, stateBCs, lev); } } else if (isStateVariable(a_name)) { // This is a state variable - mf = std::make_unique( - grids[lev], dmap[lev], 1, nGrow, MFInfo(), Factory(lev)); + mf = std::make_unique( + grids[lev], dmap[lev], 1, nGrow, amrex::MFInfo(), Factory(lev)); int idx = stateVariableIndex(a_name); - std::unique_ptr statemf = fillPatchState(lev, a_time, nGrow); - MultiFab::Copy(*mf, *statemf, idx, 0, 1, nGrow); + std::unique_ptr statemf = + fillPatchState(lev, a_time, nGrow); + amrex::MultiFab::Copy(*mf, *statemf, idx, 0, 1, nGrow); } else { // This is a reaction variable - mf = std::make_unique( - grids[lev], dmap[lev], 1, nGrow, MFInfo(), Factory(lev)); + mf = std::make_unique( + grids[lev], dmap[lev], 1, nGrow, amrex::MFInfo(), Factory(lev)); int idx = reactVariableIndex(a_name); - std::unique_ptr reactmf = fillPatchReact(lev, a_time, nGrow); - MultiFab::Copy(*mf, *reactmf, idx, 0, 1, nGrow); + std::unique_ptr reactmf = + fillPatchReact(lev, a_time, nGrow); + amrex::MultiFab::Copy(*mf, *reactmf, idx, 0, 1, nGrow); } return mf; } // Return a unique_ptr with only the required component of a derive -std::unique_ptr -PeleLM::deriveComp(const std::string& a_name, Real a_time, int lev, int nGrow) +std::unique_ptr +PeleLM::deriveComp( + const std::string& a_name, amrex::Real a_time, int lev, int nGrow) { BL_PROFILE("PeleLMeX::derive()"); AMREX_ASSERT(nGrow >= 0); - std::unique_ptr mf; + std::unique_ptr mf; bool itexists = derive_lst.canDerive(a_name) || isStateVariable(a_name) || isReactVariable(a_name); @@ -1071,31 +1082,32 @@ PeleLM::deriveComp(const std::string& a_name, Real a_time, int lev, int nGrow) const PeleLMDeriveRec* rec = derive_lst.get(a_name); if (rec != nullptr) { // This is a derived variable - mf = std::make_unique( - grids[lev], dmap[lev], 1, nGrow, MFInfo(), Factory(lev)); - std::unique_ptr statemf = + mf = std::make_unique( + grids[lev], dmap[lev], 1, nGrow, amrex::MFInfo(), Factory(lev)); + std::unique_ptr statemf = fillPatchState(lev, a_time, m_nGrowState); // Get pressure: TODO no fillpatch for pressure just yet, simply get new // state auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); - std::unique_ptr reactmf; + std::unique_ptr reactmf; if (m_do_react != 0) { reactmf = fillPatchReact(lev, a_time, nGrow); } auto stateBCs = fetchBCRecArray(VELX, NVAR); // Temp MF for all the derive components - MultiFab derTemp(grids[lev], dmap[lev], rec->numDerive(), nGrow); + amrex::MultiFab derTemp(grids[lev], dmap[lev], rec->numDerive(), nGrow); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.growntilebox(nGrow); - FArrayBox& derfab = derTemp[mfi]; - FArrayBox const& statefab = (*statemf)[mfi]; - FArrayBox const& reactfab = + for (amrex::MFIter mfi(*mf, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + const amrex::Box& bx = mfi.growntilebox(nGrow); + amrex::FArrayBox& derfab = derTemp[mfi]; + amrex::FArrayBox const& statefab = (*statemf)[mfi]; + amrex::FArrayBox const& reactfab = (m_incompressible) != 0 ? ldata_p->press[mfi] : (*reactmf)[mfi]; - FArrayBox const& pressfab = ldata_p->press[mfi]; + amrex::FArrayBox const& pressfab = ldata_p->press[mfi]; rec->derFunc()( this, bx, derfab, 0, rec->numDerive(), statefab, reactfab, pressfab, geom[lev], a_time, stateBCs, lev); @@ -1107,19 +1119,21 @@ PeleLM::deriveComp(const std::string& a_name, Real a_time, int lev, int nGrow) "PeleLM::deriveComp(): unknown derive component: " + a_name + " of " + rec->variableName(1000)); } - MultiFab::Copy(*mf, derTemp, derComp, 0, 1, nGrow); + amrex::MultiFab::Copy(*mf, derTemp, derComp, 0, 1, nGrow); } else if (isStateVariable(a_name)) { // This is a state variable - mf = std::make_unique( - grids[lev], dmap[lev], 1, nGrow, MFInfo(), Factory(lev)); + mf = std::make_unique( + grids[lev], dmap[lev], 1, nGrow, amrex::MFInfo(), Factory(lev)); int idx = stateVariableIndex(a_name); - std::unique_ptr statemf = fillPatchState(lev, a_time, nGrow); - MultiFab::Copy(*mf, *statemf, idx, 0, 1, nGrow); + std::unique_ptr statemf = + fillPatchState(lev, a_time, nGrow); + amrex::MultiFab::Copy(*mf, *statemf, idx, 0, 1, nGrow); } else { // This is a reaction variable - mf = std::make_unique( - grids[lev], dmap[lev], 1, nGrow, MFInfo(), Factory(lev)); + mf = std::make_unique( + grids[lev], dmap[lev], 1, nGrow, amrex::MFInfo(), Factory(lev)); int idx = reactVariableIndex(a_name); - std::unique_ptr reactmf = fillPatchReact(lev, a_time, nGrow); - MultiFab::Copy(*mf, *reactmf, idx, 0, 1, nGrow); + std::unique_ptr reactmf = + fillPatchReact(lev, a_time, nGrow); + amrex::MultiFab::Copy(*mf, *reactmf, idx, 0, 1, nGrow); } return mf; @@ -1128,12 +1142,12 @@ PeleLM::deriveComp(const std::string& a_name, Real a_time, int lev, int nGrow) void PeleLM::initProgressVariable() { - Vector varNames; + amrex::Vector varNames; pele::physics::eos::speciesNames( varNames, &(eos_parms.host_parm())); varNames.push_back("temp"); - ParmParse pp("peleLM"); + amrex::ParmParse pp("peleLM"); std::string Cformat; int hasUserC = static_cast(pp.contains("progressVariable.format")); if (hasUserC != 0) { @@ -1141,20 +1155,20 @@ PeleLM::initProgressVariable() if (Cformat == "Cantera") { // use a Cantera-like format with // :, default to 0.0 // Weights - Vector stringIn; - Vector weightsIn(NUM_SPECIES + 1, 0.0); + amrex::Vector stringIn; + amrex::Vector weightsIn(NUM_SPECIES + 1, 0.0); int entryCount = pp.countval("progressVariable.weights"); stringIn.resize(entryCount); pp.getarr("progressVariable.weights", stringIn, 0, entryCount); parseVars(varNames, stringIn, weightsIn); // Cold side/Hot side - Vector coldState(NUM_SPECIES + 1, 0.0); + amrex::Vector coldState(NUM_SPECIES + 1, 0.0); entryCount = pp.countval("progressVariable.coldState"); stringIn.resize(entryCount); pp.getarr("progressVariable.coldState", stringIn, 0, entryCount); parseVars(varNames, stringIn, coldState); - Vector hotState(NUM_SPECIES + 1, 0.0); + amrex::Vector hotState(NUM_SPECIES + 1, 0.0); entryCount = pp.countval("progressVariable.hotState"); stringIn.resize(entryCount); pp.getarr("progressVariable.hotState", stringIn, 0, entryCount); @@ -1166,11 +1180,11 @@ PeleLM::initProgressVariable() m_C0 += coldState[i] * m_Cweights[i]; m_C1 += hotState[i] * m_Cweights[i]; } - } else if (Cformat == "RealList") { // use a list of Real. MUST + } else if (Cformat == "RealList") { // use a list of amrex::Real. MUST // contains an entry // for each species+Temp // Weights - Vector weightsIn; + amrex::Vector weightsIn; int entryCount = pp.countval("progressVariable.weights"); AMREX_ALWAYS_ASSERT(entryCount == NUM_SPECIES + 1); weightsIn.resize(entryCount); @@ -1181,11 +1195,11 @@ PeleLM::initProgressVariable() // Cold side/Hot side entryCount = pp.countval("progressVariable.coldState"); AMREX_ALWAYS_ASSERT(entryCount == NUM_SPECIES + 1); - Vector coldState(entryCount); + amrex::Vector coldState(entryCount); pp.getarr("progressVariable.coldState", coldState, 0, entryCount); entryCount = pp.countval("progressVariable.hotState"); AMREX_ALWAYS_ASSERT(entryCount == NUM_SPECIES + 1); - Vector hotState(entryCount); + amrex::Vector hotState(entryCount); pp.getarr("progressVariable.hotState", hotState, 0, entryCount); m_C0 = 0.0; m_C1 = 0.0; @@ -1194,7 +1208,7 @@ PeleLM::initProgressVariable() m_C1 += hotState[i] * m_Cweights[i]; } } else { - Abort( + amrex::Abort( "Unknown progressVariable.format ! Should be 'Cantera' or 'RealList'"); } pp.query("progressVariable.revert", m_Crevert); @@ -1203,9 +1217,9 @@ PeleLM::initProgressVariable() void PeleLM::parseVars( - const Vector& a_varsNames, - const Vector& a_stringIn, - Vector& a_rVars) + const amrex::Vector& a_varsNames, + const amrex::Vector& a_stringIn, + amrex::Vector& a_rVars) { const int varCountIn = static_cast(a_stringIn.size()); @@ -1214,11 +1228,11 @@ PeleLM::parseVars( for (int i = 0; i < varCountIn; i++) { long unsigned sep = a_stringIn[i].find(delimiter); if (sep == std::string::npos) { - Abort( + amrex::Abort( "Error parsing '" + a_stringIn[i] + "' --> unable to find delimiter :"); } std::string varNameIn = a_stringIn[i].substr(0, sep); - Real value = + amrex::Real value = std::stod(a_stringIn[i].substr(sep + 1, a_stringIn[i].length())); int foundIt = 0; for (int k = 0; k < a_varsNames.size(); k++) { @@ -1228,51 +1242,52 @@ PeleLM::parseVars( } } if (foundIt == 0) { - Abort( + amrex::Abort( "Error parsing '" + a_stringIn[i] + "' --> unable to match to any provided variable name"); } } } -Real -PeleLM::MLNorm0(const Vector& a_MF) +amrex::Real +PeleLM::MLNorm0(const amrex::Vector& a_MF) { BL_PROFILE("PeleLMeX::MLNorm0()"); - Real r = 0.0; + amrex::Real r = 0.0; for (int lev = 0; lev < a_MF.size(); ++lev) { if (lev != finest_level) { - r = std::max(r, a_MF[lev]->norm0(*m_coveredMask[lev], 0, 0, true)); + r = amrex::max(r, a_MF[lev]->norm0(*m_coveredMask[lev], 0, 0, true)); } else { - r = std::max(r, a_MF[lev]->norm0(0, 0, true, true)); + r = amrex::max(r, a_MF[lev]->norm0(0, 0, true, true)); } } - ParallelDescriptor::ReduceRealMax(r); + amrex::ParallelDescriptor::ReduceRealMax(r); return r; } -Vector -PeleLM::MLNorm0(const Vector& a_MF, int startcomp, int ncomp) +amrex::Vector +PeleLM::MLNorm0( + const amrex::Vector& a_MF, int startcomp, int ncomp) { BL_PROFILE("PeleLMeX::MLNorm0()"); AMREX_ASSERT(a_MF[0]->nComp() >= startcomp + ncomp); - Vector r(ncomp); + amrex::Vector r(ncomp); for (int n = 0; n < ncomp; n++) { r[n] = 0.0; } for (int lev = 0; lev < a_MF.size(); ++lev) { if (lev != finest_level) { for (int n = 0; n < ncomp; n++) { - r[n] = std::max( + r[n] = amrex::max( r[n], a_MF[lev]->norm0(*m_coveredMask[lev], startcomp + n, 0, true)); } } else { for (int n = 0; n < ncomp; n++) { - r[n] = std::max(r[n], a_MF[lev]->norm0(startcomp + n, 0, true, true)); + r[n] = amrex::max(r[n], a_MF[lev]->norm0(startcomp + n, 0, true, true)); } } } - ParallelDescriptor::ReduceRealMax(r.data(), ncomp); + amrex::ParallelDescriptor::ReduceRealMax(r.data(), ncomp); return r; } @@ -1350,77 +1365,78 @@ PeleLM::reactVariableIndex(std::string_view a_name) return idx; } -Vector +amrex::Vector PeleLM::fetchAdvTypeArray(int scomp, int ncomp) { - Vector types(ncomp); + amrex::Vector types(ncomp); for (int comp = 0; comp < ncomp; comp++) { types[comp] = m_AdvTypeState[scomp + comp]; } return types; } -Vector +amrex::Vector PeleLM::fetchDiffTypeArray(int scomp, int ncomp) { - Vector types(ncomp); + amrex::Vector types(ncomp); for (int comp = 0; comp < ncomp; comp++) { types[comp] = m_DiffTypeState[scomp + comp]; } return types; } -Vector +amrex::Vector PeleLM::fetchAdvTypeAuxArray(int scomp, int ncomp) { - Vector types(ncomp); + amrex::Vector types(ncomp); for (int comp = 0; comp < ncomp; comp++) { types[comp] = m_AdvTypeAux[scomp + comp]; } return types; } -Vector +amrex::Vector PeleLM::fetchDiffTypeAuxArray(int scomp, int ncomp) { - Vector types(ncomp); + amrex::Vector types(ncomp); for (int comp = 0; comp < ncomp; comp++) { types[comp] = m_DiffTypeAux[scomp + comp]; } return types; } -Real -PeleLM::MFSum(const Vector& a_mf, int comp) +amrex::Real +PeleLM::MFSum(const amrex::Vector& a_mf, int comp) { BL_PROFILE("PeleLMeX::MFSum()"); // Get the integral of the MF, not including the fine-covered and // EB-covered cells - Real volwgtsum = 0.0; + amrex::Real volwgtsum = 0.0; for (int lev = 0; lev <= finest_level; ++lev) { #ifdef AMREX_USE_EB // For EB, use constant vol - const Real* dx = geom[lev].CellSize(); - Real vol = AMREX_D_TERM(dx[0], *dx[1], *dx[2]); + const amrex::Real* dx = geom[lev].CellSize(); + amrex::Real vol = AMREX_D_TERM(dx[0], *dx[1], *dx[2]); // Use amrex::ReduceSum - auto const& ebfact = dynamic_cast(Factory(lev)); + auto const& ebfact = + dynamic_cast(Factory(lev)); auto const& vfrac = ebfact.getVolFrac(); - Real sm = 0.0; + amrex::Real sm = 0.0; if (lev != finest_level) { sm = amrex::ReduceSum( *a_mf[lev], vfrac, *m_coveredMask[lev], 0, [vol, comp] AMREX_GPU_HOST_DEVICE( - Box const& bx, Array4 const& mf_arr, - Array4 const& vf_arr, - Array4 const& covered_arr) -> Real { - Real sum = 0.0; + amrex::Box const& bx, amrex::Array4 const& mf_arr, + amrex::Array4 const& vf_arr, + amrex::Array4 const& covered_arr) -> amrex::Real { + amrex::Real sum = 0.0; AMREX_LOOP_3D(bx, i, j, k, { sum += mf_arr(i, j, k, comp) * vf_arr(i, j, k) * vol * - static_cast(covered_arr(i, j, k)); + static_cast(covered_arr(i, j, k)); }); return sum; }); @@ -1428,9 +1444,9 @@ PeleLM::MFSum(const Vector& a_mf, int comp) sm = amrex::ReduceSum( *a_mf[lev], vfrac, 0, [vol, comp] AMREX_GPU_HOST_DEVICE( - Box const& bx, Array4 const& mf_arr, - Array4 const& vf_arr) -> Real { - Real sum = 0.0; + amrex::Box const& bx, amrex::Array4 const& mf_arr, + amrex::Array4 const& vf_arr) -> amrex::Real { + amrex::Real sum = 0.0; AMREX_LOOP_3D(bx, i, j, k, { sum += mf_arr(i, j, k, comp) * vf_arr(i, j, k) * vol; }); @@ -1439,21 +1455,21 @@ PeleLM::MFSum(const Vector& a_mf, int comp) } #else // Get the geometry volume to account for 2D-RZ - MultiFab volume(grids[lev], dmap[lev], 1, 0); + amrex::MultiFab volume(grids[lev], dmap[lev], 1, 0); geom[lev].GetVolume(volume); - Real sm = 0.0; + amrex::Real sm = 0.0; if (lev != finest_level) { sm = amrex::ReduceSum( *a_mf[lev], volume, *m_coveredMask[lev], 0, [comp] AMREX_GPU_HOST_DEVICE( - Box const& bx, Array4 const& mf_arr, - Array4 const& vol_arr, - Array4 const& covered_arr) -> Real { - Real sum = 0.0; + amrex::Box const& bx, amrex::Array4 const& mf_arr, + amrex::Array4 const& vol_arr, + amrex::Array4 const& covered_arr) -> amrex::Real { + amrex::Real sum = 0.0; AMREX_LOOP_3D(bx, i, j, k, { sum += mf_arr(i, j, k, comp) * vol_arr(i, j, k) * - static_cast(covered_arr(i, j, k)); + static_cast(covered_arr(i, j, k)); }); return sum; }); @@ -1461,9 +1477,9 @@ PeleLM::MFSum(const Vector& a_mf, int comp) sm = amrex::ReduceSum( *a_mf[lev], volume, 0, [comp] AMREX_GPU_HOST_DEVICE( - Box const& bx, Array4 const& mf_arr, - Array4 const& vol_arr) -> Real { - Real sum = 0.0; + amrex::Box const& bx, amrex::Array4 const& mf_arr, + amrex::Array4 const& vol_arr) -> amrex::Real { + amrex::Real sum = 0.0; AMREX_LOOP_3D( bx, i, j, k, { sum += mf_arr(i, j, k, comp) * vol_arr(i, j, k); }); return sum; @@ -1474,14 +1490,14 @@ PeleLM::MFSum(const Vector& a_mf, int comp) volwgtsum += sm; } // lev - ParallelDescriptor::ReduceRealSum(volwgtsum); + amrex::ParallelDescriptor::ReduceRealSum(volwgtsum); return volwgtsum; } /* -Array -PeleLM::MFStat (const Vector &a_mf, int comp) +amrex::Array +PeleLM::MFStat (const amrex::Vector &a_mf, int comp) { // Get the min/max/mean of a given component, not including the fine-covered cells @@ -1504,7 +1520,7 @@ PeleLM::setTypicalValues(const TimeStamp& a_time, int is_init) // Fill typical values vector for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { typical_values[idim] = - std::max(stateMax[VELX + idim], std::abs(stateMin[VELX + idim])); + amrex::max(stateMax[VELX + idim], std::abs(stateMin[VELX + idim])); } if (m_incompressible == 0) { @@ -1534,40 +1550,42 @@ PeleLM::setTypicalValues(const TimeStamp& a_time, int is_init) } if ((is_init != 0) || m_verbose > 1) { - Print() << PrettyLine; - Print() << " Typical values: " << '\n'; - Print() << "\tVelocity: "; + amrex::Print() << PrettyLine; + amrex::Print() << " Typical values: " << '\n'; + amrex::Print() << "\tVelocity: "; for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - Print() << typical_values[idim] << ' '; + amrex::Print() << typical_values[idim] << ' '; } - Print() << '\n'; + amrex::Print() << '\n'; if (m_incompressible == 0) { - Print() << "\tDensity: " << typical_values[DENSITY] << '\n'; - Print() << "\tTemp: " << typical_values[TEMP] << '\n'; - Print() << "\tH: " << typical_values[RHOH] << '\n'; - Vector spec_names; + amrex::Print() << "\tDensity: " << typical_values[DENSITY] << '\n'; + amrex::Print() << "\tTemp: " << typical_values[TEMP] << '\n'; + amrex::Print() << "\tH: " << typical_values[RHOH] << '\n'; + amrex::Vector spec_names; pele::physics::eos::speciesNames( spec_names, &(eos_parms.host_parm())); for (int n = 0; n < NUM_SPECIES; n++) { - Print() << "\tY_" << spec_names[n] - << std::setw( - std::max(0, static_cast(8 - spec_names[n].length()))) - << std::left << ":" << typical_values[FIRSTSPEC + n] << '\n'; + amrex::Print() << "\tY_" << spec_names[n] + << std::setw( + amrex::max( + 0, static_cast(8 - spec_names[n].length()))) + << std::left << ":" << typical_values[FIRSTSPEC + n] + << '\n'; } #ifdef PELE_USE_PLASMA - Print() << "\tnE: " << typical_values[NE] << '\n'; + amrex::Print() << "\tnE: " << typical_values[NE] << '\n'; #endif #if NUM_ODE > 0 for (int n = 0; n < NUM_ODE; n++) { - Print() << "\t" << m_ode_names[n] - << std::setw( - std::max( - 0, static_cast(10 - m_ode_names[n].length()))) - << std::left << ":" << typical_values[FIRSTODE + n] << '\n'; + amrex::Print() + << "\t" << m_ode_names[n] + << std::setw( + amrex::max(0, static_cast(10 - m_ode_names[n].length()))) + << std::left << ":" << typical_values[FIRSTODE + n] << '\n'; } #endif } - Print() << PrettyLine; + amrex::Print() << PrettyLine; } } @@ -1576,13 +1594,13 @@ PeleLM::updateTypicalValuesChem() { if ((m_useTypValChem != 0) && (m_do_react != 0)) { if (m_verbose > 2) { - Print() << " Update chemistry typical values \n"; + amrex::Print() << " Update chemistry typical values \n"; } #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif { - Vector typical_values_chem; + amrex::Vector typical_values_chem; typical_values_chem.resize(NUM_SPECIES + 1); for (int i = 0; i < NUM_SPECIES; ++i) { typical_values_chem[i] = amrex::max( @@ -1594,7 +1612,7 @@ PeleLM::updateTypicalValuesChem() #ifdef PELE_USE_PLASMA auto const* leosparm = &eos_parms.host_parm(); auto eos = pele::physics::PhysicsType::eos(leosparm); - Real mwt[NUM_SPECIES] = {0.0}; + amrex::Real mwt[NUM_SPECIES] = {0.0}; eos.molecular_weight(mwt); typical_values_chem[E_ID] = typical_values[NE] / Na * mwt[E_ID] * 1.0e-6 * 1.0e-2; @@ -1604,27 +1622,29 @@ PeleLM::updateTypicalValuesChem() } } -// MultiFab max, excluding EB-covered/fine-covered cells, local -Real -PeleLM::MFmax(const MultiFab* a_MF, const iMultiFab& a_mask, int comp) +// amrex::MultiFab max, excluding EB-covered/fine-covered cells, local +amrex::Real +PeleLM::MFmax( + const amrex::MultiFab* a_MF, const amrex::iMultiFab& a_mask, int comp) { BL_PROFILE("PeleLMeX::MFmax()"); - Real mx = std::numeric_limits::lowest(); + amrex::Real mx = std::numeric_limits::lowest(); #ifdef AMREX_USE_EB if (a_MF->hasEBFabFactory()) { const auto& ebfactory = - dynamic_cast(a_MF->Factory()); + dynamic_cast(a_MF->Factory()); auto const& flags = ebfactory.getMultiEBCellFlagFab(); #ifdef AMREX_USE_GPU - if (Gpu::inLaunchRegion()) { + if (amrex::Gpu::inLaunchRegion()) { auto const& flagsma = flags.const_arrays(); auto const& ma = a_MF->const_arrays(); auto const& mask = a_mask.const_arrays(); mx = ParReduce( - TypeList{}, TypeList{}, *a_MF, IntVect(0), - [=] AMREX_GPU_DEVICE( - int box_no, int i, int j, int k) noexcept -> GpuTuple { + amrex::TypeList{}, amrex::TypeList{}, + *a_MF, amrex::IntVect(0), + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept + -> amrex::GpuTuple { if (flagsma[box_no](i, j, k).isCovered() || !mask[box_no](i, j, k)) { return AMREX_REAL_LOWEST; } else { @@ -1637,15 +1657,15 @@ PeleLM::MFmax(const MultiFab* a_MF, const iMultiFab& a_mask, int comp) #ifdef AMREX_USE_OMP #pragma omp parallel reduction(max : mx) #endif - for (MFIter mfi(*a_MF, true); mfi.isValid(); ++mfi) { - Box const& bx = mfi.tilebox(); - if (flags[mfi].getType(bx) != FabType::covered) { + for (amrex::MFIter mfi(*a_MF, true); mfi.isValid(); ++mfi) { + amrex::Box const& bx = mfi.tilebox(); + if (flags[mfi].getType(bx) != amrex::FabType::covered) { auto const& flag = flags.const_array(mfi); auto const& a = a_MF->const_array(mfi); auto const& mask = a_mask.const_array(mfi); AMREX_LOOP_3D(bx, i, j, k, { if (!flag(i, j, k).isCovered() && mask(i, j, k)) { - mx = std::max(mx, a(i, j, k, comp)); + mx = amrex::max(mx, a(i, j, k, comp)); } }); } @@ -1655,13 +1675,14 @@ PeleLM::MFmax(const MultiFab* a_MF, const iMultiFab& a_mask, int comp) #endif { #ifdef AMREX_USE_GPU - if (Gpu::inLaunchRegion()) { + if (amrex::Gpu::inLaunchRegion()) { auto const& ma = a_MF->const_arrays(); auto const& mask = a_mask.const_arrays(); mx = ParReduce( - TypeList{}, TypeList{}, *a_MF, IntVect(0), - [=] AMREX_GPU_DEVICE( - int box_no, int i, int j, int k) noexcept -> GpuTuple { + amrex::TypeList{}, amrex::TypeList{}, + *a_MF, amrex::IntVect(0), + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept + -> amrex::GpuTuple { if (!mask[box_no](i, j, k)) { return AMREX_REAL_LOWEST; } else { @@ -1674,13 +1695,13 @@ PeleLM::MFmax(const MultiFab* a_MF, const iMultiFab& a_mask, int comp) #ifdef AMREX_USE_OMP #pragma omp parallel reduction(max : mx) #endif - for (MFIter mfi(*a_MF, true); mfi.isValid(); ++mfi) { - Box const& bx = mfi.tilebox(); + for (amrex::MFIter mfi(*a_MF, true); mfi.isValid(); ++mfi) { + amrex::Box const& bx = mfi.tilebox(); auto const& a = a_MF->const_array(mfi); auto const& mask = a_mask.const_array(mfi); AMREX_LOOP_3D(bx, i, j, k, { if (mask(i, j, k)) { - mx = std::max(mx, a(i, j, k, comp)); + mx = amrex::max(mx, a(i, j, k, comp)); } }); } @@ -1690,27 +1711,29 @@ PeleLM::MFmax(const MultiFab* a_MF, const iMultiFab& a_mask, int comp) return mx; } -// MultiFab min, excluding EB-covered/fine-covered cells, local -Real -PeleLM::MFmin(const MultiFab* a_MF, const iMultiFab& a_mask, int comp) +// amrex::MultiFab min, excluding EB-covered/fine-covered cells, local +amrex::Real +PeleLM::MFmin( + const amrex::MultiFab* a_MF, const amrex::iMultiFab& a_mask, int comp) { BL_PROFILE("PeleLMeX::MFmin()"); - Real mn = std::numeric_limits::max(); + amrex::Real mn = std::numeric_limits::max(); #ifdef AMREX_USE_EB if (a_MF->hasEBFabFactory()) { const auto& ebfactory = - dynamic_cast(a_MF->Factory()); + dynamic_cast(a_MF->Factory()); auto const& flags = ebfactory.getMultiEBCellFlagFab(); #ifdef AMREX_USE_GPU - if (Gpu::inLaunchRegion()) { + if (amrex::Gpu::inLaunchRegion()) { auto const& flagsma = flags.const_arrays(); auto const& ma = a_MF->const_arrays(); auto const& mask = a_mask.const_arrays(); mn = ParReduce( - TypeList{}, TypeList{}, *a_MF, IntVect(0), - [=] AMREX_GPU_DEVICE( - int box_no, int i, int j, int k) noexcept -> GpuTuple { + amrex::TypeList{}, amrex::TypeList{}, + *a_MF, amrex::IntVect(0), + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept + -> amrex::GpuTuple { if (flagsma[box_no](i, j, k).isCovered() || !mask[box_no](i, j, k)) { return AMREX_REAL_MAX; } else { @@ -1723,15 +1746,15 @@ PeleLM::MFmin(const MultiFab* a_MF, const iMultiFab& a_mask, int comp) #ifdef AMREX_USE_OMP #pragma omp parallel reduction(min : mn) #endif - for (MFIter mfi(*a_MF, true); mfi.isValid(); ++mfi) { - Box const& bx = mfi.tilebox(); - if (flags[mfi].getType(bx) != FabType::covered) { + for (amrex::MFIter mfi(*a_MF, true); mfi.isValid(); ++mfi) { + amrex::Box const& bx = mfi.tilebox(); + if (flags[mfi].getType(bx) != amrex::FabType::covered) { auto const& flag = flags.const_array(mfi); auto const& a = a_MF->const_array(mfi); auto const& mask = a_mask.const_array(mfi); AMREX_LOOP_3D(bx, i, j, k, { if (!flag(i, j, k).isCovered() && mask(i, j, k)) { - mn = std::min(mn, a(i, j, k, comp)); + mn = amrex::min(mn, a(i, j, k, comp)); } }); } @@ -1741,13 +1764,14 @@ PeleLM::MFmin(const MultiFab* a_MF, const iMultiFab& a_mask, int comp) #endif { #ifdef AMREX_USE_GPU - if (Gpu::inLaunchRegion()) { + if (amrex::Gpu::inLaunchRegion()) { auto const& ma = a_MF->const_arrays(); auto const& mask = a_mask.const_arrays(); mn = ParReduce( - TypeList{}, TypeList{}, *a_MF, IntVect(0), - [=] AMREX_GPU_DEVICE( - int box_no, int i, int j, int k) noexcept -> GpuTuple { + amrex::TypeList{}, amrex::TypeList{}, + *a_MF, amrex::IntVect(0), + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept + -> amrex::GpuTuple { if (!mask[box_no](i, j, k)) { return AMREX_REAL_MAX; } else { @@ -1760,13 +1784,13 @@ PeleLM::MFmin(const MultiFab* a_MF, const iMultiFab& a_mask, int comp) #ifdef AMREX_USE_OMP #pragma omp parallel reduction(min : mn) #endif - for (MFIter mfi(*a_MF, true); mfi.isValid(); ++mfi) { - Box const& bx = mfi.tilebox(); + for (amrex::MFIter mfi(*a_MF, true); mfi.isValid(); ++mfi) { + amrex::Box const& bx = mfi.tilebox(); auto const& a = a_MF->const_array(mfi); auto const& mask = a_mask.const_array(mfi); AMREX_LOOP_3D(bx, i, j, k, { if (mask(i, j, k)) { - mn = std::min(mn, a(i, j, k, comp)); + mn = amrex::min(mn, a(i, j, k, comp)); } }); } @@ -1777,54 +1801,56 @@ PeleLM::MFmin(const MultiFab* a_MF, const iMultiFab& a_mask, int comp) } // MultiLevel max, exlucing EB-covered/fine-covered cells -Vector -PeleLM::MLmax(const Vector& a_MF, int scomp, int ncomp) +amrex::Vector +PeleLM::MLmax( + const amrex::Vector& a_MF, int scomp, int ncomp) { BL_PROFILE("PeleLMeX::MLmax()"); AMREX_ASSERT(a_MF[0]->nComp() >= scomp + ncomp); - Vector nmax(ncomp, AMREX_REAL_LOWEST); + amrex::Vector nmax(ncomp, AMREX_REAL_LOWEST); for (int lev = 0; lev < a_MF.size(); ++lev) { if (lev != finest_level) { for (int n = 0; n < ncomp; n++) { nmax[n] = - std::max(nmax[n], MFmax(a_MF[lev], *m_coveredMask[lev], scomp + n)); + amrex::max(nmax[n], MFmax(a_MF[lev], *m_coveredMask[lev], scomp + n)); } } else { for (int n = 0; n < ncomp; n++) { - nmax[n] = std::max(nmax[n], a_MF[lev]->max(scomp + n, 0, true)); + nmax[n] = amrex::max(nmax[n], a_MF[lev]->max(scomp + n, 0, true)); } } } - ParallelDescriptor::ReduceRealMax(nmax.data(), ncomp); + amrex::ParallelDescriptor::ReduceRealMax(nmax.data(), ncomp); return nmax; } // MultiLevel min, exlucing EB-covered/fine-covered cells -Vector -PeleLM::MLmin(const Vector& a_MF, int scomp, int ncomp) +amrex::Vector +PeleLM::MLmin( + const amrex::Vector& a_MF, int scomp, int ncomp) { BL_PROFILE("PeleLMeX::MLmin()"); AMREX_ASSERT(a_MF[0]->nComp() >= scomp + ncomp); - Vector nmin(ncomp, AMREX_REAL_MAX); + amrex::Vector nmin(ncomp, AMREX_REAL_MAX); for (int lev = 0; lev < a_MF.size(); ++lev) { if (lev != finest_level) { for (int n = 0; n < ncomp; n++) { nmin[n] = - std::min(nmin[n], MFmin(a_MF[lev], *m_coveredMask[lev], scomp + n)); + amrex::min(nmin[n], MFmin(a_MF[lev], *m_coveredMask[lev], scomp + n)); } } else { for (int n = 0; n < ncomp; n++) { - nmin[n] = std::min(nmin[n], a_MF[lev]->min(scomp + n, 0, true)); + nmin[n] = amrex::min(nmin[n], a_MF[lev]->min(scomp + n, 0, true)); } } } - ParallelDescriptor::ReduceRealMin(nmin.data(), ncomp); + amrex::ParallelDescriptor::ReduceRealMin(nmin.data(), ncomp); return nmin; } @@ -1835,19 +1861,20 @@ PeleLM::checkMemory(const std::string& a_message) const return; } - const int IOProc = ParallelDescriptor::IOProcessorNumber(); + const int IOProc = amrex::ParallelDescriptor::IOProcessorNumber(); #ifdef AMREX_USE_GPU - Long free_mem_avail = Gpu::Device::freeMemAvailable() / (1024 * 1024); - ParallelDescriptor::ReduceLongMin(free_mem_avail, IOProc); - Print() << " [" << a_message << "] GPU mem. avail. (MB) " - << free_mem_avail << "\n"; + amrex::Long free_mem_avail = + amrex::Gpu::Device::freeMemAvailable() / (1024 * 1024); + amrex::ParallelDescriptor::ReduceLongMin(free_mem_avail, IOProc); + amrex::Print() << " [" << a_message << "] GPU mem. avail. (MB) " + << free_mem_avail << "\n"; #else - // MultiFab memory usage - Long max_fab_megabytes = + // amrex::MultiFab memory usage + amrex::Long max_fab_megabytes = amrex::TotalBytesAllocatedInFabsHWM() / (1024 * 1024); - ParallelDescriptor::ReduceLongMax(max_fab_megabytes, IOProc); - Print() << " [" << a_message << "] MFs mem. allocated (MB) " - << max_fab_megabytes << "\n"; + amrex::ParallelDescriptor::ReduceLongMax(max_fab_megabytes, IOProc); + amrex::Print() << " [" << a_message << "] MFs mem. allocated (MB) " + << max_fab_megabytes << "\n"; #endif } @@ -1857,10 +1884,10 @@ PeleLM::initMixtureFraction() // set up a few variables auto const* leosparm = &eos_parms.host_parm(); auto eos = pele::physics::PhysicsType::eos(leosparm); - Vector specNames; + amrex::Vector specNames; pele::physics::eos::speciesNames( specNames, leosparm); - ParmParse pp("peleLM"); + amrex::ParmParse pp("peleLM"); // Do simpler things for some EOS if (pele::physics::PhysicsType::eos_type::identifier() == "GammaLaw") { @@ -1929,7 +1956,7 @@ PeleLM::initMixtureFraction() // :, default in 0.0 std::string MFCompoType; pp.query("mixtureFraction.type", MFCompoType); - Vector compositionIn; + amrex::Vector compositionIn; int entryCount = pp.countval("mixtureFraction.oxidTank"); compositionIn.resize(entryCount); pp.getarr("mixtureFraction.oxidTank", compositionIn, 0, entryCount); @@ -1938,7 +1965,7 @@ PeleLM::initMixtureFraction() compositionIn.resize(entryCount); pp.getarr("mixtureFraction.fuelTank", compositionIn, 0, entryCount); parseComposition(compositionIn, MFCompoType, YF); - } else if (MFformat == "RealList") { // use a list of Real. MUST + } else if (MFformat == "RealList") { // use a list of amrex::Real. MUST // contains an entry // for each species in the mixture std::string MFCompoType; @@ -1946,7 +1973,7 @@ PeleLM::initMixtureFraction() if (MFCompoType == "mass") { int entryCount = pp.countval("mixtureFraction.oxidTank"); AMREX_ALWAYS_ASSERT(entryCount == NUM_SPECIES); - Vector compositionIn(NUM_SPECIES); + amrex::Vector compositionIn(NUM_SPECIES); pp.getarr("mixtureFraction.oxidTank", compositionIn, 0, NUM_SPECIES); for (int i = 0; i < NUM_SPECIES; ++i) { YO[i] = compositionIn[i]; @@ -1961,7 +1988,7 @@ PeleLM::initMixtureFraction() amrex::Real XF[NUM_SPECIES], XO[NUM_SPECIES]; int entryCount = pp.countval("mixtureFraction.oxidTank"); AMREX_ALWAYS_ASSERT(entryCount == NUM_SPECIES); - Vector compositionIn(NUM_SPECIES); + amrex::Vector compositionIn(NUM_SPECIES); pp.getarr("mixtureFraction.oxidTank", compositionIn, 0, NUM_SPECIES); for (int i = 0; i < NUM_SPECIES; ++i) { XO[i] = compositionIn[i]; @@ -1975,16 +2002,18 @@ PeleLM::initMixtureFraction() eos.X2Y(XO, YO); eos.X2Y(XF, YF); } else { - Abort("Unknown mixtureFraction.type ! Should be 'mass' or 'mole'"); + amrex::Abort( + "Unknown mixtureFraction.type ! Should be 'mass' or 'mole'"); } } else { - Abort( + amrex::Abort( "Unknown mixtureFraction.format ! Should be 'Cantera' or 'RealList'"); } } if (fuelID < 0 && (hasUserMF != 0)) { - Print() << " Mixture fraction definition lacks fuelID: consider using " - "peleLM.fuel_name keyword \n"; + amrex::Print() + << " Mixture fraction definition lacks fuelID: consider using " + "peleLM.fuel_name keyword \n"; } // Detailed chem - compute Bilger coefficients @@ -2019,14 +2048,14 @@ PeleLM::initMixtureFraction() void PeleLM::parseComposition( - Vector compositionIn, + amrex::Vector compositionIn, std::string compositionType, - Real* massFrac) + amrex::Real* massFrac) { - Real compoIn[NUM_SPECIES] = {0.0}; + amrex::Real compoIn[NUM_SPECIES] = {0.0}; // Get species names - Vector specNames; + amrex::Vector specNames; pele::physics::eos::speciesNames( specNames, &(eos_parms.host_parm())); @@ -2036,12 +2065,12 @@ PeleLM::parseComposition( for (int i = 0; i < specCountIn; i++) { long unsigned sep = compositionIn[i].find(delimiter); if (sep == std::string::npos) { - Abort( + amrex::Abort( "Error parsing '" + compositionIn[i] + "' --> unable to find delimiter :"); } std::string specNameIn = compositionIn[i].substr(0, sep); - Real value = + amrex::Real value = std::stod(compositionIn[i].substr(sep + 1, compositionIn[i].length())); int foundIt = 0; for (int k = 0; k < NUM_SPECIES; k++) { @@ -2051,14 +2080,14 @@ PeleLM::parseComposition( } } if (foundIt == 0) { - Abort( + amrex::Abort( "Error parsing '" + compositionIn[i] + "' --> unable to match to any species name"); } } // Ensure that it sums to 1.0: - Real sum = 0.0; + amrex::Real sum = 0.0; for (double k : compoIn) { sum += k; } @@ -2076,36 +2105,38 @@ PeleLM::parseComposition( auto eos = pele::physics::PhysicsType::eos(leosparm); eos.X2Y(compoIn, massFrac); } else { - Abort("Unknown mixtureFraction.type ! Should be 'mass' or 'mole'"); + amrex::Abort("Unknown mixtureFraction.type ! Should be 'mass' or 'mole'"); } } #ifdef AMREX_USE_EB // Extend the cell-centered based signed distance function void -PeleLM::extendSignedDistance(MultiFab* a_signDist, Real a_extendFactor) +PeleLM::extendSignedDistance( + amrex::MultiFab* a_signDist, amrex::Real a_extendFactor) { BL_PROFILE("PeleLMeX::extendSignedDistance()"); // This is a not-so-pretty piece of code that'll take AMReX cell-averaged // signed distance and propagates it manually up to the point where we need to // have it for derefining. const auto geomdata = geom[0].data(); - Real maxSignedDist = a_signDist->max(0); + amrex::Real maxSignedDist = a_signDist->max(0); const auto& ebfactory = - dynamic_cast(a_signDist->Factory()); + dynamic_cast(a_signDist->Factory()); const auto& flags = ebfactory.getMultiEBCellFlagFab(); int nGrowFac = flags.nGrow() + 1; // First set the region far away at the max value we need #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*a_signDist, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.growntilebox(); + for (amrex::MFIter mfi(*a_signDist, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + const amrex::Box& bx = mfi.growntilebox(); auto const& sd_cc = a_signDist->array(mfi); - ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (sd_cc(i, j, k) >= maxSignedDist - 1e-12) { - const Real* dx = geomdata.CellSize(); + const amrex::Real* dx = geomdata.CellSize(); sd_cc(i, j, k) = nGrowFac * dx[0] * a_extendFactor; } }); @@ -2117,47 +2148,49 @@ PeleLM::extendSignedDistance(MultiFab* a_signDist, Real a_extendFactor) int nMaxLoop = 4; for (int dloop = 1; dloop <= nMaxLoop; dloop++) { #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*a_signDist, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); - const Box& gbx = grow(bx, 1); - if (flags[mfi].getType(gbx) == FabType::covered) { + for (amrex::MFIter mfi(*a_signDist, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + const amrex::Box& bx = mfi.tilebox(); + const amrex::Box& gbx = grow(bx, 1); + if (flags[mfi].getType(gbx) == amrex::FabType::covered) { continue; } auto const& sd_cc = a_signDist->array(mfi); - ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - const auto glo = amrex::lbound(gbx); - const auto ghi = amrex::ubound(gbx); - const Real* dx = geomdata.CellSize(); - Real extendedDist = dx[0] * a_extendFactor; - if (sd_cc(i, j, k) >= maxSignedDist - 1e-12) { - Real closestEBDist = 1e12; - for (int kk = glo.z; kk <= ghi.z; ++kk) { - for (int jj = glo.y; jj <= ghi.y; ++jj) { - for (int ii = glo.x; ii <= ghi.x; ++ii) { - if ((i != ii) || (j != jj) || (k != kk)) { - if (sd_cc(ii, jj, kk) > 0.0) { - Real distToCell = std::sqrt(AMREX_D_TERM( - ((i - ii) * dx[0] * (i - ii) * dx[0]), - +((j - jj) * dx[1] * (j - jj) * dx[1]), - +((k - kk) * dx[2] * (k - kk) * dx[2]))); - Real distToEB = distToCell + sd_cc(ii, jj, kk); - if (distToEB < closestEBDist) { - closestEBDist = distToEB; + amrex::ParallelFor( + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + const auto glo = amrex::lbound(gbx); + const auto ghi = amrex::ubound(gbx); + const amrex::Real* dx = geomdata.CellSize(); + amrex::Real extendedDist = dx[0] * a_extendFactor; + if (sd_cc(i, j, k) >= maxSignedDist - 1e-12) { + amrex::Real closestEBDist = 1e12; + for (int kk = glo.z; kk <= ghi.z; ++kk) { + for (int jj = glo.y; jj <= ghi.y; ++jj) { + for (int ii = glo.x; ii <= ghi.x; ++ii) { + if ((i != ii) || (j != jj) || (k != kk)) { + if (sd_cc(ii, jj, kk) > 0.0) { + amrex::Real distToCell = std::sqrt(AMREX_D_TERM( + ((i - ii) * dx[0] * (i - ii) * dx[0]), + +((j - jj) * dx[1] * (j - jj) * dx[1]), + +((k - kk) * dx[2] * (k - kk) * dx[2]))); + amrex::Real distToEB = distToCell + sd_cc(ii, jj, kk); + if (distToEB < closestEBDist) { + closestEBDist = distToEB; + } } } } } } + if (closestEBDist < 1e10) { + sd_cc(i, j, k) = closestEBDist; + } else { + sd_cc(i, j, k) = extendedDist; + } } - if (closestEBDist < 1e10) { - sd_cc(i, j, k) = closestEBDist; - } else { - sd_cc(i, j, k) = extendedDist; - } - } - }); + }); } a_signDist->FillBoundary(geom[0].periodicity()); } diff --git a/Source/Plasma/GMRES/MLGMRES.cpp b/Source/Plasma/GMRES/MLGMRES.cpp index f7c536a7f..0b6efb3a8 100644 --- a/Source/Plasma/GMRES/MLGMRES.cpp +++ b/Source/Plasma/GMRES/MLGMRES.cpp @@ -1,8 +1,6 @@ #include #include -using namespace amrex; - MLGMRESSolver::MLGMRESSolver() { m_verbose = 0; @@ -99,8 +97,8 @@ MLGMRESSolver::setNorm(MLNormFunc a_norm) int MLGMRESSolver::solve( - const Vector& a_sol, - const Vector& a_rhs, + const amrex::Vector& a_sol, + const amrex::Vector& a_rhs, amrex::Real a_abs_tol, amrex::Real a_rel_tol) { @@ -111,10 +109,10 @@ MLGMRESSolver::solve( AMREX_ALWAYS_ASSERT( a_sol[0]->nComp() == m_nComp && a_rhs[0]->nComp() == m_nComp); if (m_prec == nullptr) - Print() << " Using unpreconditioned GMRES ! Might take a while to " - "converge ...\n"; + amrex::Print() << " Using unpreconditioned GMRES ! Might take a while to " + "converge ...\n"; - Real rhsNorm = computeMLNorm(a_rhs); // RHS norm + amrex::Real rhsNorm = computeMLNorm(a_rhs); // RHS norm initResNorm = computeMLResidualNorm(a_sol, a_rhs); // Initial resisual norm target_relResNorm = initResNorm * a_rel_tol; // Target relative tolerance target_absResNorm = a_abs_tol; // Target absolute tolerance @@ -138,7 +136,7 @@ MLGMRESSolver::solve( restart_count++; } while (!m_converged && restart_count < m_restart); - Real finalResNorm = + amrex::Real finalResNorm = computeMLResidualNorm(a_sol, a_rhs); // Final resisual norm if (m_verbose > 0) amrex::Print() << " GMRES: [" << iter_count @@ -164,12 +162,13 @@ MLGMRESSolver::prepareForSolve() void MLGMRESSolver::one_restart( - const Vector& a_x, const Vector& a_rhs) + const amrex::Vector& a_x, + const amrex::Vector& a_rhs) { int finest_level = m_pelelm->finestLevel(); computeMLResidual(a_x, a_rhs, GetVecOfPtrs(res)); - Real resNorm_0 = computeMLNorm(GetVecOfPtrs(res)); + amrex::Real resNorm_0 = computeMLNorm(GetVecOfPtrs(res)); if (m_verbose > 1) amrex::Print() << " [Restart:" << restart_count << "] initial relative res: " << resNorm_0 / initResNorm @@ -179,11 +178,11 @@ MLGMRESSolver::one_restart( g[0] = resNorm_0; for (int lev = 0; lev <= finest_level; ++lev) { res[lev].mult(1.0 / resNorm_0); - MultiFab::Copy(KspBase[0][lev], res[lev], 0, 0, m_nComp, 0); + amrex::MultiFab::Copy(KspBase[0][lev], res[lev], 0, 0, m_nComp, 0); } int k_end; - Real resNorm = resNorm_0; + amrex::Real resNorm = resNorm_0; for (int k = 0; k < m_krylovSize; ++k) { // Do one GMRES iteration, update the residual norm @@ -207,7 +206,7 @@ MLGMRESSolver::one_restart( // Solve the minimization problem H.y = g y[k_end] = g[k_end] / H[k_end][k_end]; for (int k = k_end - 1; k >= 0; --k) { - Real sum_tmp = 0.0; + amrex::Real sum_tmp = 0.0; for (int j = k + 1; j <= k_end; ++j) { sum_tmp += H[k][j] * y[j]; } @@ -219,7 +218,7 @@ MLGMRESSolver::one_restart( } void -MLGMRESSolver::one_iter(const int iter, Real& resNorm) +MLGMRESSolver::one_iter(const int iter, amrex::Real& resNorm) { BL_PROFILE("MLGMRESSolver::one_iter()"); if (m_verbose > 1) @@ -231,23 +230,25 @@ MLGMRESSolver::one_iter(const int iter, Real& resNorm) } void -MLGMRESSolver::updateSolution(int k_end, const Vector& a_x) +MLGMRESSolver::updateSolution( + int k_end, const amrex::Vector& a_x) { int finest_level = m_pelelm->finestLevel(); for (int lev = 0; lev <= finest_level; ++lev) { - MultiFab update(m_grids[lev], m_dmap[lev], m_nComp, 0); + amrex::MultiFab update(m_grids[lev], m_dmap[lev], m_nComp, 0); update.setVal(0.0); for (int i = k_end; i >= 0; --i) { - MultiFab::Saxpy(update, y[i], KspBase[i][lev], 0, 0, m_nComp, 0); + amrex::MultiFab::Saxpy(update, y[i], KspBase[i][lev], 0, 0, m_nComp, 0); } - MultiFab::Add(*a_x[lev], update, 0, 0, m_nComp, 0); + amrex::MultiFab::Add(*a_x[lev], update, 0, 0, m_nComp, 0); } // TODO Do an average down ? } void -MLGMRESSolver::appendBasisVector(const int iter, Vector>& Base) +MLGMRESSolver::appendBasisVector( + const int iter, amrex::Vector>& Base) { if (m_prec == nullptr) { MEMBER_FUNC_PTR(*m_pelelm, m_jtv) @@ -261,19 +262,20 @@ MLGMRESSolver::appendBasisVector(const int iter, Vector>& Base) } void -MLGMRESSolver::gramSchmidtOrtho(const int iter, Vector>& Base) +MLGMRESSolver::gramSchmidtOrtho( + const int iter, amrex::Vector>& Base) { int finest_level = m_pelelm->finestLevel(); for (int row = 0; row <= iter; ++row) { H[row][iter] = MFVecDot( GetVecOfConstPtrs(Base[iter + 1]), 0, GetVecOfConstPtrs(Base[row]), 0, m_nComp, 0); - Real GS_corr = -H[row][iter]; + amrex::Real GS_corr = -H[row][iter]; MFVecSaxpy( GetVecOfPtrs(Base[iter + 1]), GS_corr, GetVecOfConstPtrs(Base[row]), 0, 0, m_nComp, 0); if (check_GramSchmidtOrtho) { - Real Hcorr = MFVecDot( + amrex::Real Hcorr = MFVecDot( GetVecOfConstPtrs(Base[iter + 1]), 0, GetVecOfConstPtrs(Base[row]), 0, m_nComp, 0); if (std::fabs(Hcorr) > 1.0e-14) { @@ -285,7 +287,7 @@ MLGMRESSolver::gramSchmidtOrtho(const int iter, Vector>& Base) } } } - Real normNewVec = computeMLNorm(GetVecOfPtrs(Base[iter + 1])); + amrex::Real normNewVec = computeMLNorm(GetVecOfPtrs(Base[iter + 1])); H[iter + 1][iter] = normNewVec; if (normNewVec > 0) { for (int lev = 0; lev <= finest_level; ++lev) { @@ -295,16 +297,18 @@ MLGMRESSolver::gramSchmidtOrtho(const int iter, Vector>& Base) // m_pelelm->WriteDebugPlotFile(GetVecOfConstPtrs(Base[iter+1]),"KspVec_"+std::to_string(iter_count)); } -Real +amrex::Real MLGMRESSolver::givensRotation(const int iter) { for (int row = 0; row < iter; ++row) { - Real v1 = givens[row][0] * H[row][iter] - givens[row][1] * H[row + 1][iter]; - Real v2 = givens[row][1] * H[row][iter] + givens[row][0] * H[row + 1][iter]; + amrex::Real v1 = + givens[row][0] * H[row][iter] - givens[row][1] * H[row + 1][iter]; + amrex::Real v2 = + givens[row][1] * H[row][iter] + givens[row][0] * H[row + 1][iter]; H[row][iter] = v1; H[row + 1][iter] = v2; } - Real norm_Hlast = std::sqrt( + amrex::Real norm_Hlast = std::sqrt( H[iter][iter] * H[iter][iter] + H[iter + 1][iter] * H[iter + 1][iter]); if (norm_Hlast > 0.0) { givens[iter][0] = H[iter][iter] / norm_Hlast; @@ -312,33 +316,35 @@ MLGMRESSolver::givensRotation(const int iter) H[iter][iter] = givens[iter][0] * H[iter][iter] - givens[iter][1] * H[iter + 1][iter]; H[iter + 1][iter] = 0.0; - Real v1 = givens[iter][0] * g[iter] - givens[iter][1] * g[iter + 1]; - Real v2 = givens[iter][1] * g[iter] + givens[iter][0] * g[iter + 1]; + amrex::Real v1 = givens[iter][0] * g[iter] - givens[iter][1] * g[iter + 1]; + amrex::Real v2 = givens[iter][1] * g[iter] + givens[iter][0] * g[iter + 1]; g[iter] = v1; g[iter + 1] = v2; } return std::fabs(g[iter + 1]); } -Real +amrex::Real MLGMRESSolver::computeMLResidualNorm( - const Vector& a_x, const Vector& a_rhs) + const amrex::Vector& a_x, + const amrex::Vector& a_rhs) { computeMLResidual(a_x, a_rhs, GetVecOfPtrs(res)); return computeMLNorm(GetVecOfPtrs(res)); } -Real -MLGMRESSolver::computeMLNorm(const Vector& a_vec) +amrex::Real +MLGMRESSolver::computeMLNorm(const amrex::Vector& a_vec) { - Real r = 0.0; + amrex::Real r = 0.0; if (m_norm != nullptr) { MEMBER_FUNC_PTR(*m_pelelm, m_norm)(a_vec, r); } else { for (int comp = 0; comp < m_nComp; comp++) { - Real norm = 0.0; + amrex::Real norm = 0.0; for (int lev = 0; lev <= a_vec.size(); ++lev) { - norm += MultiFab::Dot(*a_vec[lev], comp, *a_vec[lev], comp, 1, 0); + norm += + amrex::MultiFab::Dot(*a_vec[lev], comp, *a_vec[lev], comp, 1, 0); } r += norm; } @@ -349,45 +355,45 @@ MLGMRESSolver::computeMLNorm(const Vector& a_vec) void MLGMRESSolver::computeMLResidual( - const Vector& a_x, - const Vector& a_rhs, - const Vector& a_res) + const amrex::Vector& a_x, + const amrex::Vector& a_rhs, + const amrex::Vector& a_res) { BL_PROFILE("MLGMRESSolver::computeMLResidual()"); int finest_level = m_pelelm->finestLevel(); MEMBER_FUNC_PTR(*m_pelelm, m_jtv)(a_x, GetVecOfPtrs(Ax)); if (m_prec != nullptr) { for (int lev = 0; lev <= finest_level; ++lev) { - MultiFab::Xpay(Ax[lev], -1.0, *a_rhs[lev], 0, 0, m_nComp, 0); + amrex::MultiFab::Xpay(Ax[lev], -1.0, *a_rhs[lev], 0, 0, m_nComp, 0); } MEMBER_FUNC_PTR(*m_pelelm, m_prec)(GetVecOfPtrs(Ax), a_res); } else { for (int lev = 0; lev <= finest_level; ++lev) { - MultiFab::LinComb( + amrex::MultiFab::LinComb( *a_res[lev], 1.0, Ax[lev], 0, -1.0, *a_rhs[lev], 0, 0, m_nComp, 0); } } } -Real +amrex::Real MLGMRESSolver::MFVecDot( - const Vector& a_mf1, + const amrex::Vector& a_mf1, int mf1comp, - const Vector& a_mf2, + const amrex::Vector& a_mf2, int mf2comp, int nComp, int nGrow) { int finest_level = m_pelelm->finestLevel(); - Real r = 0.0; + amrex::Real r = 0.0; for (int lev = 0; lev <= finest_level; ++lev) { if (lev != finest_level) { - r += MultiFab::Dot( + r += amrex::MultiFab::Dot( *(m_pelelm->m_coveredMask[lev]), *a_mf1[lev], mf1comp, *a_mf2[lev], mf2comp, nComp, nGrow); } else { - r += - MultiFab::Dot(*a_mf1[lev], mf1comp, *a_mf2[lev], mf2comp, nComp, nGrow); + r += amrex::MultiFab::Dot( + *a_mf1[lev], mf1comp, *a_mf2[lev], mf2comp, nComp, nGrow); } } return r; @@ -395,9 +401,9 @@ MLGMRESSolver::MFVecDot( void MLGMRESSolver::MFVecSaxpy( - const Vector& a_mfdest, - Real a_a, - const Vector& a_mfsrc, + const amrex::Vector& a_mfdest, + amrex::Real a_a, + const amrex::Vector& a_mfsrc, int destComp, int srcComp, int nComp, @@ -405,7 +411,7 @@ MLGMRESSolver::MFVecSaxpy( { int finest_level = m_pelelm->finestLevel(); for (int lev = 0; lev <= finest_level; ++lev) { - MultiFab::Saxpy( + amrex::MultiFab::Saxpy( *a_mfdest[lev], a_a, *a_mfsrc[lev], destComp, srcComp, nComp, nGrow); } } @@ -413,7 +419,7 @@ MLGMRESSolver::MFVecSaxpy( void MLGMRESSolver::readParameters() { - ParmParse pp("gmres"); + amrex::ParmParse pp("gmres"); pp.query("krylovBasis_size", m_krylovSize); pp.query("max_restart", m_restart); diff --git a/Source/Plasma/LinOps/PrecondOp.cpp b/Source/Plasma/LinOps/PrecondOp.cpp index a3b56d7f8..a5f7377f5 100644 --- a/Source/Plasma/LinOps/PrecondOp.cpp +++ b/Source/Plasma/LinOps/PrecondOp.cpp @@ -7,8 +7,6 @@ #include #endif -using namespace amrex; - //--------------------------------------------------------------------------------------- // Preconditioner Operator PrecondOp::PrecondOp(PeleLM* a_pelelm) : m_pelelm(a_pelelm) @@ -16,152 +14,156 @@ PrecondOp::PrecondOp(PeleLM* a_pelelm) : m_pelelm(a_pelelm) readParameters(); // Solve LPInfo - LPInfo info_diff; + amrex::LPInfo info_diff; info_diff.setMaxCoarseningLevel(m_mg_max_coarsening_level_diff); - LPInfo info_Stilda; + amrex::LPInfo info_Stilda; info_Stilda.setMaxCoarseningLevel(m_mg_max_coarsening_level_Stilda); // Apply LPInfo (no coarsening) - LPInfo info_apply; + amrex::LPInfo info_apply; info_apply.setMaxCoarseningLevel(0); // Diff/Drift Op - m_diff.reset(new MLABecCecLaplacian( + m_diff.reset(new amrex::MLABecCecLaplacian( m_pelelm->Geom(0, m_pelelm->finestLevel()), m_pelelm->boxArray(0, m_pelelm->finestLevel()), m_pelelm->DistributionMap(0, m_pelelm->finestLevel()), info_diff)); m_diff->setMaxOrder(m_mg_maxorder); // Drift Op - m_drift.reset(new MLABecLaplacian( + m_drift.reset(new amrex::MLABecLaplacian( m_pelelm->Geom(0, m_pelelm->finestLevel()), m_pelelm->boxArray(0, m_pelelm->finestLevel()), m_pelelm->DistributionMap(0, m_pelelm->finestLevel()), info_apply)); m_drift->setMaxOrder(m_mg_maxorder); // Stilda Op - m_Stilda.reset(new MLABecLaplacian( + m_Stilda.reset(new amrex::MLABecLaplacian( m_pelelm->Geom(0, m_pelelm->finestLevel()), m_pelelm->boxArray(0, m_pelelm->finestLevel()), m_pelelm->DistributionMap(0, m_pelelm->finestLevel()), info_Stilda)); m_Stilda->setMaxOrder(m_mg_maxorder); // MLMGs - m_mlmg_diff = std::make_unique(*m_diff); - m_mlmg_drift = std::make_unique(*m_drift); - m_mlmg_Stilda = std::make_unique(*m_Stilda); + m_mlmg_diff = std::make_unique(*m_diff); + m_mlmg_drift = std::make_unique(*m_drift); + m_mlmg_Stilda = std::make_unique(*m_Stilda); } void PrecondOp::setDiffOpScalars( - const Real& aScal, const Real& bScal, const Real& cScal) + const amrex::Real& aScal, const amrex::Real& bScal, const amrex::Real& cScal) { m_diff->setScalars(aScal, bScal, cScal); } void -PrecondOp::setDiffOpRelaxation(const Real& a_omega) +PrecondOp::setDiffOpRelaxation(const amrex::Real& a_omega) { m_diff->setRelaxation(a_omega); } void -PrecondOp::setDriftOpScalars(const Real& aScal, const Real& bScal) +PrecondOp::setDriftOpScalars(const amrex::Real& aScal, const amrex::Real& bScal) { m_drift->setScalars(aScal, bScal); } void -PrecondOp::setStildaOpScalars(const Real& aScal, const Real& bScal) +PrecondOp::setStildaOpScalars( + const amrex::Real& aScal, const amrex::Real& bScal) { m_Stilda->setScalars(aScal, bScal); } void -PrecondOp::setDiffOpBCs(const BCRec& a_bcrec) +PrecondOp::setDiffOpBCs(const amrex::BCRec& a_bcrec) { m_diff->setDomainBC( - getOpBC(Orientation::low, a_bcrec), getOpBC(Orientation::high, a_bcrec)); + getOpBC(amrex::Orientation::low, a_bcrec), + getOpBC(amrex::Orientation::high, a_bcrec)); } void -PrecondOp::setDriftOpBCs(const BCRec& a_bcrec) +PrecondOp::setDriftOpBCs(const amrex::BCRec& a_bcrec) { m_drift->setDomainBC( - getOpBC(Orientation::low, a_bcrec), getOpBC(Orientation::high, a_bcrec)); + getOpBC(amrex::Orientation::low, a_bcrec), + getOpBC(amrex::Orientation::high, a_bcrec)); } void -PrecondOp::setStildaOpBCs(const BCRec& a_bcrec) +PrecondOp::setStildaOpBCs(const amrex::BCRec& a_bcrec) { m_Stilda->setDomainBC( - getOpBC(Orientation::low, a_bcrec), getOpBC(Orientation::high, a_bcrec)); + getOpBC(amrex::Orientation::low, a_bcrec), + getOpBC(amrex::Orientation::high, a_bcrec)); } void -PrecondOp::setDiffOpLevelBC(int lev, MultiFab* a_MF) +PrecondOp::setDiffOpLevelBC(int lev, amrex::MultiFab* a_MF) { m_diff->setLevelBC(lev, a_MF); } void -PrecondOp::setDriftOpLevelBC(int lev, MultiFab* a_MF) +PrecondOp::setDriftOpLevelBC(int lev, amrex::MultiFab* a_MF) { m_drift->setLevelBC(lev, a_MF); } void -PrecondOp::setStildaOpLevelBC(int lev, MultiFab* a_MF) +PrecondOp::setStildaOpLevelBC(int lev, amrex::MultiFab* a_MF) { m_Stilda->setLevelBC(lev, a_MF); } void -PrecondOp::setDiffOpACoeff(int lev, const Real& acoeff) +PrecondOp::setDiffOpACoeff(int lev, const amrex::Real& acoeff) { m_diff->setACoeffs(lev, acoeff); } void PrecondOp::setDiffOpBCoeff( - int lev, const Array& a_bCoeff) + int lev, const amrex::Array& a_bCoeff) { m_diff->setBCoeffs(lev, a_bCoeff); } void PrecondOp::setDiffOpCCoeff( - int lev, const Array& a_cCoeff) + int lev, const amrex::Array& a_cCoeff) { m_diff->setCCoeffs(lev, a_cCoeff); } void -PrecondOp::getDiffOpDiagonal(int lev, MultiFab& a_diffOpDiag) +PrecondOp::getDiffOpDiagonal(int lev, amrex::MultiFab& a_diffOpDiag) { m_diff->getDiagonal(lev, a_diffOpDiag); } void PrecondOp::setDriftOpBCoeff( - int lev, const Array& a_bCoeff) + int lev, const amrex::Array& a_bCoeff) { m_drift->setBCoeffs(lev, a_bCoeff); } void PrecondOp::setStildaOpBCoeff( - int lev, const Array& a_bCoeff) + int lev, const amrex::Array& a_bCoeff) { m_Stilda->setBCoeffs(lev, a_bCoeff); } void PrecondOp::diffOpSolve( - const Vector& a_sol, - const Vector& a_rhs, - const Real& rtol, - const Real& atol) + const amrex::Vector& a_sol, + const amrex::Vector& a_rhs, + const amrex::Real& rtol, + const amrex::Real& atol) { // TODO set all the MLMG options m_mlmg_diff->setVerbose(m_diff_verbose); @@ -175,10 +177,10 @@ PrecondOp::diffOpSolve( void PrecondOp::StildaOpSolve( - const Vector& a_sol, - const Vector& a_rhs, - const Real& rtol, - const Real& atol) + const amrex::Vector& a_sol, + const amrex::Vector& a_rhs, + const amrex::Real& rtol, + const amrex::Real& atol) { // TODO set all the MLMG options m_mlmg_Stilda->setVerbose(m_Stilda_verbose); @@ -190,34 +192,35 @@ PrecondOp::StildaOpSolve( void PrecondOp::driftOpApply( - const Vector& a_Ax, const Vector& a_x) + const amrex::Vector& a_Ax, + const amrex::Vector& a_x) { // TODO set all the MLMG options m_mlmg_drift->setVerbose(m_drift_verbose); m_mlmg_drift->apply(a_Ax, a_x); } -Array -PrecondOp::getOpBC(Orientation::Side a_side, const BCRec& a_bc) +amrex::Array +PrecondOp::getOpBC(amrex::Orientation::Side a_side, const amrex::BCRec& a_bc) { - Array r; + amrex::Array r; for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { if (m_pelelm->Geom(0).isPeriodic(idim)) { - r[idim] = LinOpBCType::Periodic; + r[idim] = amrex::LinOpBCType::Periodic; } else { auto amrexbc = - (a_side == Orientation::low) ? a_bc.lo(idim) : a_bc.hi(idim); + (a_side == amrex::Orientation::low) ? a_bc.lo(idim) : a_bc.hi(idim); if (amrexbc == amrex::BCType::ext_dir) { - r[idim] = LinOpBCType::Dirichlet; + r[idim] = amrex::LinOpBCType::Dirichlet; } else if ( amrexbc == amrex::BCType::foextrap || amrexbc == amrex::BCType::hoextrap || amrexbc == amrex::BCType::reflect_even) { - r[idim] = LinOpBCType::Neumann; + r[idim] = amrex::LinOpBCType::Neumann; } else if (amrexbc == amrex::BCType::reflect_odd) { - r[idim] = LinOpBCType::reflect_odd; + r[idim] = amrex::LinOpBCType::reflect_odd; } else { - r[idim] = LinOpBCType::bogus; + r[idim] = amrex::LinOpBCType::bogus; } } } @@ -227,7 +230,7 @@ PrecondOp::getOpBC(Orientation::Side a_side, const BCRec& a_bc) void PrecondOp::readParameters() { - ParmParse pp("ef.precond"); + amrex::ParmParse pp("ef.precond"); pp.query("diff_verbose", m_diff_verbose); pp.query("Stilda_verbose", m_Stilda_verbose); diff --git a/Source/Plasma/PeleLMeX_EFDeriveFunc.cpp b/Source/Plasma/PeleLMeX_EFDeriveFunc.cpp index 1e98c10fb..ddca235a0 100644 --- a/Source/Plasma/PeleLMeX_EFDeriveFunc.cpp +++ b/Source/Plasma/PeleLMeX_EFDeriveFunc.cpp @@ -5,21 +5,19 @@ #include #include -using namespace amrex; - void pelelmex_derchargedist( PeleLM* /*a_pelelm*/, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int ncomp, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& /*geomdata*/, - Real /*time*/, - const Vector& /*bcrec*/, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& /*geomdata*/, + amrex::Real /*time*/, + const amrex::Vector& /*bcrec*/, int /*level*/) { AMREX_ASSERT(derfab.box().contains(bx)); @@ -43,16 +41,16 @@ pelelmex_derchargedist( void pelelmex_derefx( PeleLM* /*a_pelelm*/, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int ncomp, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& geomdata, - Real /*time*/, - const Vector& bcrec, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& geomdata, + amrex::Real /*time*/, + const amrex::Vector& bcrec, int /*level*/) { AMREX_ASSERT(derfab.box().contains(bx)); @@ -83,16 +81,16 @@ pelelmex_derefx( void pelelmex_derLorentzx( PeleLM* /*a_pelelm*/, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int ncomp, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& geomdata, - Real /*time*/, - const Vector& bcrec, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& geomdata, + amrex::Real /*time*/, + const amrex::Vector& bcrec, int /*level*/) { AMREX_ASSERT(derfab.box().contains(bx)); @@ -137,16 +135,16 @@ pelelmex_derLorentzx( void pelelmex_derefy( PeleLM* /*a_pelelm*/, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int ncomp, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& geomdata, - Real /*time*/, - const Vector& bcrec, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& geomdata, + amrex::Real /*time*/, + const amrex::Vector& bcrec, int /*level*/) { AMREX_ASSERT(derfab.box().contains(bx)); @@ -177,16 +175,16 @@ pelelmex_derefy( void pelelmex_derLorentzy( PeleLM* /*a_pelelm*/, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int ncomp, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& geomdata, - Real /*time*/, - const Vector& bcrec, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& geomdata, + amrex::Real /*time*/, + const amrex::Vector& bcrec, int /*level*/) { AMREX_ASSERT(derfab.box().contains(bx)); @@ -231,16 +229,16 @@ pelelmex_derLorentzy( void pelelmex_derefz( PeleLM* /*a_pelelm*/, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int ncomp, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& geomdata, - Real /*time*/, - const Vector& bcrec, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& geomdata, + amrex::Real /*time*/, + const amrex::Vector& bcrec, int /*level*/) { AMREX_ASSERT(derfab.box().contains(bx)); @@ -271,16 +269,16 @@ pelelmex_derefz( void pelelmex_derLorentzz( PeleLM* /*a_pelelm*/, - const Box& bx, - FArrayBox& derfab, + const amrex::Box& bx, + amrex::FArrayBox& derfab, int dcomp, int ncomp, - const FArrayBox& statefab, - const FArrayBox& /*reactfab*/, - const FArrayBox& /*pressfab*/, - const Geometry& geomdata, - Real /*time*/, - const Vector& bcrec, + const amrex::FArrayBox& statefab, + const amrex::FArrayBox& /*reactfab*/, + const amrex::FArrayBox& /*pressfab*/, + const amrex::Geometry& geomdata, + amrex::Real /*time*/, + const amrex::Vector& bcrec, int /*level*/) { AMREX_ASSERT(derfab.box().contains(bx)); diff --git a/Source/Plasma/PeleLMeX_EFIonDrift.cpp b/Source/Plasma/PeleLMeX_EFIonDrift.cpp index 895a3f6a5..01d4fadb3 100644 --- a/Source/Plasma/PeleLMeX_EFIonDrift.cpp +++ b/Source/Plasma/PeleLMeX_EFIonDrift.cpp @@ -4,8 +4,6 @@ #include #include -using namespace amrex; - void PeleLM::ionDriftVelocity(std::unique_ptr& advData) { @@ -21,19 +19,21 @@ PeleLM::ionDriftVelocity(std::unique_ptr& advData) //---------------------------------------------------------------- // Get the gradient of Old and New phiV - Vector> gphiVOld(finest_level + 1); - Vector> gphiVNew(finest_level + 1); + amrex::Vector> gphiVOld( + finest_level + 1); + amrex::Vector> gphiVNew( + finest_level + 1); int nGrow = 0; // No need for ghost face on gphiV for (int lev = 0; lev <= finest_level; ++lev) { const auto& ba = grids[lev]; const auto& factory = Factory(lev); for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { gphiVOld[lev][idim].define( - amrex::convert(ba, IntVect::TheDimensionVector(idim)), dmap[lev], 1, - nGrow, MFInfo(), factory); + amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)), dmap[lev], + 1, nGrow, amrex::MFInfo(), factory); gphiVNew[lev][idim].define( - amrex::convert(ba, IntVect::TheDimensionVector(idim)), dmap[lev], 1, - nGrow, MFInfo(), factory); + amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)), dmap[lev], + 1, nGrow, amrex::MFInfo(), factory); } } @@ -57,43 +57,43 @@ PeleLM::ionDriftVelocity(std::unique_ptr& advData) auto ldataOld_p = getLevelDataPtr(lev, AmrOldTime); auto ldataNew_p = getLevelDataPtr(lev, AmrNewTime); - MultiFab mobH_cc(grids[lev], dmap[lev], NUM_IONS, 1); + amrex::MultiFab mobH_cc(grids[lev], dmap[lev], NUM_IONS, 1); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(mobH_cc, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& gbx = mfi.growntilebox(); + for (amrex::MFIter mfi(mobH_cc, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + const amrex::Box& gbx = mfi.growntilebox(); const auto& mob_o = ldataOld_p->mob_cc.const_array(mfi); const auto& mob_n = ldataNew_p->mob_cc.const_array(mfi); const auto& mob_h = mobH_cc.array(mfi); amrex::ParallelFor( gbx, NUM_IONS, - [mob_o, mob_n, - mob_h] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { mob_h(i, j, k, n) = 0.5 * (mob_o(i, j, k, n) + mob_n(i, j, k, n)); }); } // Get the face centered ions mobility int doZeroVisc = 0; - Array mobH_ec = + amrex::Array mobH_ec = getDiffusivity(lev, 0, NUM_IONS, doZeroVisc, bcRecIons, mobH_cc); // Assemble the ions drift velocity for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(mobH_ec[idim], TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box bx = mfi.tilebox(); + for (amrex::MFIter mfi(mobH_ec[idim], amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box bx = mfi.tilebox(); const auto& mob_h = mobH_ec[idim].const_array(mfi); const auto& gp_o = gphiVOld[lev][idim].const_array(mfi); const auto& gp_n = gphiVNew[lev][idim].const_array(mfi); const auto& Ud_Sp = advData->uDrift[lev][idim].array(mfi); amrex::ParallelFor( bx, NUM_IONS, - [mob_h, gp_o, gp_n, - Ud_Sp] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { Ud_Sp(i, j, k, n) = mob_h(i, j, k, n) * -0.5 * (gp_o(i, j, k) + gp_n(i, j, k)); }); @@ -118,40 +118,43 @@ PeleLM::ionDriftVelocity(std::unique_ptr& advData) // FillPatch Udrift on levels > 0 for (int lev = 0; lev <= finest_level; ++lev) { if (lev > 0) { - IntVect rr = geom[lev].Domain().size() / geom[lev - 1].Domain().size(); - Interpolater* mapper = &face_linear_interp; + amrex::IntVect rr = + geom[lev].Domain().size() / geom[lev - 1].Domain().size(); + amrex::Interpolater* mapper = &amrex::face_linear_interp; // Set BCRec for Umac - Vector bcrec(NUM_IONS); + amrex::Vector bcrec(NUM_IONS); for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { for (int ion = 0; ion < NUM_IONS; ion++) { if (geom[lev - 1].isPeriodic(idim)) { - bcrec[ion].setLo(idim, BCType::int_dir); - bcrec[ion].setHi(idim, BCType::int_dir); + bcrec[ion].setLo(idim, amrex::BCType::int_dir); + bcrec[ion].setHi(idim, amrex::BCType::int_dir); } else { - bcrec[ion].setLo(idim, BCType::foextrap); - bcrec[ion].setHi(idim, BCType::foextrap); + bcrec[ion].setLo(idim, amrex::BCType::foextrap); + bcrec[ion].setHi(idim, amrex::BCType::foextrap); } } } - Array, AMREX_SPACEDIM> bcrecArr = { + amrex::Array, AMREX_SPACEDIM> bcrecArr = { AMREX_D_DECL(bcrec, bcrec, bcrec)}; - PhysBCFunct> crse_bndry_func( + amrex::PhysBCFunct> crse_bndry_func( geom[lev - 1], bcrec, umacFill{}); - Array>, AMREX_SPACEDIM> + amrex::Array< + amrex::PhysBCFunct>, AMREX_SPACEDIM> cbndyFuncArr = { AMREX_D_DECL(crse_bndry_func, crse_bndry_func, crse_bndry_func)}; - PhysBCFunct> fine_bndry_func( + amrex::PhysBCFunct> fine_bndry_func( geom[lev], bcrec, umacFill{}); - Array>, AMREX_SPACEDIM> + amrex::Array< + amrex::PhysBCFunct>, AMREX_SPACEDIM> fbndyFuncArr = { AMREX_D_DECL(fine_bndry_func, fine_bndry_func, fine_bndry_func)}; - Real dummy = 0.; + amrex::Real dummy = 0.0; FillPatchTwoLevels( - GetArrOfPtrs(advData->uDrift[lev]), IntVect(1), dummy, + GetArrOfPtrs(advData->uDrift[lev]), amrex::IntVect(1), dummy, {GetArrOfPtrs(advData->uDrift[lev - 1])}, {dummy}, {GetArrOfPtrs(advData->uDrift[lev])}, {dummy}, 0, 0, NUM_IONS, geom[lev - 1], geom[lev - 1], cbndyFuncArr, 0, fbndyFuncArr, 0, rr, @@ -171,16 +174,16 @@ PeleLM::ionDriftAddUmac(int lev, std::unique_ptr& advData) // Add umac to the ions drift velocity to get the effective velocity for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(advData->umac[lev][idim], TilingIfNotGPU()); mfi.isValid(); - ++mfi) { - const Box gbx = mfi.growntilebox(); + for (amrex::MFIter mfi(advData->umac[lev][idim], amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box gbx = mfi.growntilebox(); const auto& umac = advData->umac[lev][idim].const_array(mfi); const auto& Ud_Sp = advData->uDrift[lev][idim].array(mfi); amrex::ParallelFor( gbx, NUM_IONS, - [umac, Ud_Sp] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { Ud_Sp(i, j, k, n) += umac(i, j, k); }); } diff --git a/Source/Plasma/PeleLMeX_EFNLSolve.cpp b/Source/Plasma/PeleLMeX_EFNLSolve.cpp index 25d9be7b1..ec5103d55 100644 --- a/Source/Plasma/PeleLMeX_EFNLSolve.cpp +++ b/Source/Plasma/PeleLMeX_EFNLSolve.cpp @@ -6,8 +6,6 @@ #include #include -using namespace amrex; - PrecondOp* PeleLM::getPrecondOp() { @@ -19,13 +17,13 @@ PeleLM::getPrecondOp() void PeleLM::implicitNonLinearSolve( int sdcIter, - const Real& a_dt, + const amrex::Real& a_dt, std::unique_ptr& diffData, std::unique_ptr& advData) { BL_PROFILE_VAR("PeleLMeX::implicitNonLinearSolve()", implicitNonLinearSolve); - const Real strt_time = ParallelDescriptor::second(); + const amrex::Real strt_time = amrex::ParallelDescriptor::second(); //------------------------------------------------------------------------ // Pre-solve @@ -43,8 +41,9 @@ PeleLM::implicitNonLinearSolve( auto ldata_p = getLevelDataPtr(lev, AmrOldTime); // Get nl solve data pointer auto ldataNLs_p = getLevelDataNLSolvePtr(lev); - MultiFab::Copy(ldataNLs_p->nlState, ldata_p->state, NE, 0, 1, m_nGrowState); - MultiFab::Copy( + amrex::MultiFab::Copy( + ldataNLs_p->nlState, ldata_p->state, NE, 0, 1, m_nGrowState); + amrex::MultiFab::Copy( ldataNLs_p->nlState, ldata_p->state, PHIV, 1, 1, m_nGrowState); } @@ -60,7 +59,7 @@ PeleLM::implicitNonLinearSolve( // Get nl solve data pointer auto ldataNLs_p = getLevelDataNLSolvePtr(lev); for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - MultiFab::Copy( + amrex::MultiFab::Copy( ldataNLs_p->umac[idim], advData->umac[lev][idim], 0, 0, 1, 0); } } @@ -81,10 +80,9 @@ PeleLM::implicitNonLinearSolve( //------------------------------------------------------------------------ // Outer subcycling loop if (ef_substep > 1) - Abort("Non-linear solve sub-stepping not re-implemented yet"); + amrex::Abort("Non-linear solve sub-stepping not re-implemented yet"); int NK_tot_count = 0; for (int sstep = 0; sstep < ef_substep; sstep++) { - curtime = getTime(0, AmrOldTime) + (sstep + 1) * dtsub; // ----------------- @@ -118,9 +116,9 @@ PeleLM::implicitNonLinearSolve( // WriteDebugPlotFile(GetVecOfConstPtrs(getNLresidVect()),"NLResInit"); // Check for convergence - Real scaledResnE, scaledResphiV; + amrex::Real scaledResnE, scaledResphiV; getNLResidScaling(scaledResnE, scaledResphiV); - Real max_nlres = std::max(scaledResnE, scaledResphiV); + amrex::Real max_nlres = std::max(scaledResnE, scaledResphiV); if (max_nlres <= m_ef_newtonTol) { if (ef_verbose) { amrex::Print() << " No Newton iteration needed, exiting. \n"; @@ -143,21 +141,21 @@ PeleLM::implicitNonLinearSolve( } // Solve for Newton direction - Vector newtonDir(finest_level + 1); + amrex::Vector newtonDir(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { newtonDir[lev].define( - grids[lev], dmap[lev], 2, 1, MFInfo(), Factory(lev)); + grids[lev], dmap[lev], 2, 1, amrex::MFInfo(), Factory(lev)); newtonDir[lev].setVal(0.0, 0, 2, 1); } if (!m_ef_use_PETSC_direct) { - const Real S_tol = m_ef_GMRES_reltol; - const Real S_tol_abs = m_ef_GMRES_abstol; + const amrex::Real S_tol = m_ef_GMRES_reltol; + const amrex::Real S_tol_abs = m_ef_GMRES_abstol; GMRES_tot_count += gmres.solve( GetVecOfPtrs(newtonDir), getNLresidVect(), S_tol_abs, S_tol); } else { } // WriteDebugPlotFile(GetVecOfConstPtrs(newtonDir),"newtonDir_"+std::to_string(NK_ite)); - Real newtonDir_Norm; + amrex::Real newtonDir_Norm; nlSolveNorm(GetVecOfPtrs(newtonDir), newtonDir_Norm); // Print() << " newtonDir_Norm " << newtonDir_Norm << "\n"; @@ -198,28 +196,30 @@ PeleLM::implicitNonLinearSolve( // Get nl solve data pointer auto ldataNLs_p = getLevelDataNLSolvePtr(lev); int nGrowNL = 0; - MultiFab::Copy(ldata_p->state, ldataNLs_p->nlState, 0, NE, 1, nGrowNL); - MultiFab::Copy(ldata_p->state, ldataNLs_p->nlState, 1, PHIV, 1, nGrowNL); + amrex::MultiFab::Copy( + ldata_p->state, ldataNLs_p->nlState, 0, NE, 1, nGrowNL); + amrex::MultiFab::Copy( + ldata_p->state, ldataNLs_p->nlState, 1, PHIV, 1, nGrowNL); } if (ef_verbose) { - Real run_time = ParallelDescriptor::second() - strt_time; - ParallelDescriptor::ReduceRealMax( - run_time, ParallelDescriptor::IOProcessorNumber()); + amrex::Real run_time = amrex::ParallelDescriptor::second() - strt_time; + amrex::ParallelDescriptor::ReduceRealMax( + run_time, amrex::ParallelDescriptor::IOProcessorNumber()); if (!m_ef_use_PETSC_direct) { - Real avgGMRES = (float)GMRES_tot_count / (float)NK_tot_count; + amrex::Real avgGMRES = (float)GMRES_tot_count / (float)NK_tot_count; amrex::Print() << " [" << sdcIter << "] dt: " << a_dt << " - Avg GMRES/Newton: " << avgGMRES << "\n"; } amrex::Print() << " >> PeleLMeX::implicitNLSolve() " << run_time << "\n"; } - // VisMF::Write(advData->Forcing[0],"ForcingNE"); + // amrex::VisMF::Write(advData->Forcing[0],"ForcingNE"); } int PeleLM::testExitNewton( - int newtonIter, const Real& max_res, const Real& norm_NewtonDir) + int newtonIter, const amrex::Real& max_res, const amrex::Real& norm_NewtonDir) { int exit = 0; if (max_res <= m_ef_newtonTol || norm_NewtonDir <= 1e-11) { @@ -244,7 +244,7 @@ PeleLM::testExitNewton( } void -PeleLM::updateNLState(const Vector& a_update) +PeleLM::updateNLState(const amrex::Vector& a_update) { // AverageDown the newton direction /* @@ -297,8 +297,8 @@ PeleLM::updateNLState(const Vector& a_update) } // FillPatch - Vector nEState; - Vector phiVState; + amrex::Vector nEState; + amrex::Vector phiVState; for (int lev = 0; lev <= finest_level; ++lev) { auto ldataNLs_p = getLevelDataNLSolvePtr(lev); // NL data nEState.emplace_back(ldataNLs_p->nlState, amrex::make_alias, 0, 1); @@ -322,19 +322,19 @@ PeleLM::incrementElectronForcing( auto ldataNLs_p = getLevelDataNLSolvePtr(lev); // NL data #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); auto const& nE_o = ldata_p->state.const_array(mfi, NE); auto const& nE_n = ldataNLs_p->nlState.const_array(mfi); auto const& I_R_nE = ldataR_p->I_R.const_array(mfi, NUM_SPECIES); auto const& FnE = advData->Forcing[lev].array(mfi, NUM_SPECIES + 1); - Real scaling = nE_scale; - Real dtinv = 1.0 / dtsub; + amrex::Real scaling = nE_scale; + amrex::Real dtinv = 1.0 / dtsub; amrex::ParallelFor( - bx, [nE_o, nE_n, I_R_nE, FnE, dtinv, scaling, - a_sstep] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (a_sstep == 0) { FnE(i, j, k) = (nE_n(i, j, k) * scaling - nE_o(i, j, k)) * dtinv - I_R_nE(i, j, k); @@ -349,26 +349,26 @@ PeleLM::incrementElectronForcing( void PeleLM::computeBGcharge( - const Real& a_time, + const amrex::Real& a_time, std::unique_ptr& diffData, std::unique_ptr& advData) { // Get integration dt - Real dt_int = a_time - getTime(0, AmrOldTime); + amrex::Real dt_int = a_time - getTime(0, AmrOldTime); for (int lev = 0; lev <= finest_level; ++lev) { - // Get data pointers auto ldata_p = getLevelDataPtr(lev, AmrOldTime); // Old time species auto ldataR_p = getLevelDataReactPtr(lev); // Reaction auto ldataNLs_p = getLevelDataNLSolvePtr(lev); // NL data #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldataNLs_p->backgroundCharge, TilingIfNotGPU()); + for (amrex::MFIter mfi( + ldataNLs_p->backgroundCharge, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + const amrex::Box& bx = mfi.tilebox(); auto const& rhoYold = ldata_p->state.const_array(mfi, FIRSTSPEC); auto const& adv_arr = advData->AofS[lev].const_array(mfi, FIRSTSPEC); auto const& dn_arr = diffData->Dn[lev].const_array(mfi); @@ -376,14 +376,12 @@ PeleLM::computeBGcharge( auto const& dhat_arr = diffData->Dhat[lev].const_array(mfi); auto const& rhoYdot = ldataR_p->I_R.const_array(mfi); auto const& charge = ldataNLs_p->backgroundCharge.array(mfi); - Real factor = 1.0 / elemCharge; + amrex::Real factor = 1.0 / elemCharge; amrex::ParallelFor( - bx, - [dt_int, rhoYold, adv_arr, dn_arr, dnp1_arr, dhat_arr, rhoYdot, charge, - factor, zk = zk] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { charge(i, j, k) = 0.0; for (int n = 0; n < NUM_SPECIES; n++) { - Real rhoYprov = + amrex::Real rhoYprov = rhoYold(i, j, k, n) + dt_int * (adv_arr(i, j, k, n) + 0.5 * (dn_arr(i, j, k, n) - dnp1_arr(i, j, k, n)) + @@ -399,23 +397,23 @@ PeleLM::computeBGcharge( void PeleLM::nonLinearResidual( - const Real& a_dt, - const Vector& a_nlstate, - const Vector& a_nlresid, + const amrex::Real& a_dt, + const amrex::Vector& a_nlstate, + const amrex::Vector& a_nlresid, int updateScaling, int updatePrecond) { // Get unscaled copy of the NL state - Vector nE(finest_level + 1); - Vector phiV(finest_level + 1); + amrex::Vector nE(finest_level + 1); + amrex::Vector phiV(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { nE[lev].define( - grids[lev], dmap[lev], 1, m_nGrowState, MFInfo(), Factory(lev)); + grids[lev], dmap[lev], 1, m_nGrowState, amrex::MFInfo(), Factory(lev)); phiV[lev].define( - grids[lev], dmap[lev], 1, m_nGrowState, MFInfo(), Factory(lev)); - MultiFab::Copy(nE[lev], *a_nlstate[lev], 0, 0, 1, m_nGrowState); + grids[lev], dmap[lev], 1, m_nGrowState, amrex::MFInfo(), Factory(lev)); + amrex::MultiFab::Copy(nE[lev], *a_nlstate[lev], 0, 0, 1, m_nGrowState); nE[lev].mult(nE_scale, 0, 1, m_nGrowState); - MultiFab::Copy(phiV[lev], *a_nlstate[lev], 1, 0, 1, m_nGrowState); + amrex::MultiFab::Copy(phiV[lev], *a_nlstate[lev], 1, 0, 1, m_nGrowState); phiV[lev].mult(phiV_scale, 0, 1, m_nGrowState); } @@ -423,15 +421,17 @@ PeleLM::nonLinearResidual( fillPatchNLnE(m_cur_time, GetVecOfPtrs(nE), m_nGrowState); // Get L(phiV) and Grad(phiV) - Vector laplacian(finest_level + 1); - Vector> gradPhiVCur(finest_level + 1); + amrex::Vector laplacian(finest_level + 1); + amrex::Vector> gradPhiVCur( + finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - laplacian[lev].define(grids[lev], dmap[lev], 1, 0, MFInfo(), Factory(lev)); + laplacian[lev].define( + grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), Factory(lev)); for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { const auto& fba = - amrex::convert(grids[lev], IntVect::TheDimensionVector(idim)); + amrex::convert(grids[lev], amrex::IntVect::TheDimensionVector(idim)); gradPhiVCur[lev][idim].define( - fba, dmap[lev], 1, 0, MFInfo(), Factory(lev)); + fba, dmap[lev], 1, 0, amrex::MFInfo(), Factory(lev)); } } int do_avgDown = 0; // TODO or should I ? @@ -441,22 +441,24 @@ PeleLM::nonLinearResidual( GetVecOfConstPtrs(phiV), {}, bcRecPhiV[0], do_avgDown); // Get nE diffusion term - Vector diffnE(finest_level + 1); + amrex::Vector diffnE(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - diffnE[lev].define(grids[lev], dmap[lev], 1, 0, MFInfo(), Factory(lev)); + diffnE[lev].define( + grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), Factory(lev)); } auto bcRecnE = fetchBCRecArray(NE, 1); getDiffusionOp()->computeDiffLap( GetVecOfPtrs(diffnE), 0, GetVecOfConstPtrs(nE), 0, GetVecOfConstPtrs(getnEDiffusivityVect(AmrNewTime)), 0, bcRecnE, 1); // WriteDebugPlotFile(GetVecOfConstPtrs(diffnE),"diffnE"); - // VisMF::Write(nE[0],"nEForDiffnlResid"); - // VisMF::Write(diffnE[0],"diffnEnlResid"); + // amrex::VisMF::Write(nE[0],"nEForDiffnlResid"); + // amrex::VisMF::Write(diffnE[0],"diffnEnlResid"); // Get nE advection term - Vector advnE(finest_level + 1); + amrex::Vector advnE(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - advnE[lev].define(grids[lev], dmap[lev], 1, 0, MFInfo(), Factory(lev)); + advnE[lev].define( + grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), Factory(lev)); } getAdvectionTerm( GetVecOfConstPtrs(nE), GetVecOfPtrs(advnE), @@ -467,7 +469,6 @@ PeleLM::nonLinearResidual( // res(ne(:)) = dt * ( diff(:) + conv(:) + I_R(:) ) - ( ne(:) - ne_old(:) ) // res(phiv(:)) = \Sum z_k * \tilde Y_k / q_e - ne + Lapl_PhiV for (int lev = 0; lev <= finest_level; ++lev) { - auto ldataOld_p = getLevelDataPtr(lev, AmrOldTime); // Old time auto ldataR_p = getLevelDataReactPtr(lev); // Reaction @@ -478,11 +479,11 @@ PeleLM::nonLinearResidual( a_nlresid[lev]->setVal(0.0); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldataNLs_p->nlResid, TilingIfNotGPU()); mfi.isValid(); - ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldataNLs_p->nlResid, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); auto const& I_R_nE = ldataR_p->I_R.const_array(mfi, NUM_SPECIES); auto const& lapPhiV = laplacian[lev].const_array(mfi); auto const& ne_diff = diffnE[lev].const_array(mfi); @@ -492,11 +493,9 @@ PeleLM::nonLinearResidual( auto const& charge = ldataNLs_p->backgroundCharge.const_array(mfi); auto const& res_nE = a_nlresid[lev]->array(mfi, 0); auto const& res_phiV = a_nlresid[lev]->array(mfi, 1); - Real scalLap = eps0 * epsr / elemCharge; + amrex::Real scalLap = eps0 * epsr / elemCharge; amrex::ParallelFor( - bx, [ne_curr, ne_old, lapPhiV, I_R_nE, ne_diff, ne_adv, charge, res_nE, - res_phiV, a_dt, - scalLap] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { res_nE(i, j, k) = ne_old(i, j, k) - ne_curr(i, j, k) + a_dt * (ne_diff(i, j, k) + ne_adv(i, j, k) + I_R_nE(i, j, k)); @@ -543,21 +542,22 @@ PeleLM::nonLinearResidual( void PeleLM::getAdvectionTerm( - const Vector& a_nE, - const Vector& a_advTerm, - const Vector>& a_gPhiVCur) + const amrex::Vector& a_nE, + const amrex::Vector& a_advTerm, + const amrex::Vector>& + a_gPhiVCur) { - // Get advection fluxes on all levels int nGrow = 0; - Vector> fluxes(finest_level + 1); + amrex::Vector> fluxes( + finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { const auto& ba = grids[lev]; const auto& factory = Factory(lev); for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { fluxes[lev][idim].define( - amrex::convert(ba, IntVect::TheDimensionVector(idim)), dmap[lev], 1, - nGrow, MFInfo(), factory); + amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)), dmap[lev], + 1, nGrow, amrex::MFInfo(), factory); } } @@ -574,25 +574,24 @@ PeleLM::getAdvectionTerm( // Get the face centered electron mobility int doZeroVisc = 0; - Array mobE_ec = + amrex::Array mobE_ec = getDiffusivity(lev, 0, 1, doZeroVisc, bcRecnE, ldata_p->mobE_cc); // Get the electron effective velocity for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldataNLs_p->uEffnE[idim], TilingIfNotGPU()); + for (amrex::MFIter mfi(ldataNLs_p->uEffnE[idim], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + const amrex::Box& bx = mfi.tilebox(); auto const& ueff = ldataNLs_p->uEffnE[idim].array(mfi); auto const& umac = ldataNLs_p->umac[idim].const_array(mfi); auto const& gphi_c = a_gPhiVCur[lev][idim]->const_array(mfi); auto const& gphi_o = ldataNLs_p->gPhiVOld[idim].const_array(mfi); auto const& kappa_e = mobE_ec[idim].const_array(mfi); amrex::ParallelFor( - bx, [umac, ueff, gphi_c, gphi_o, - kappa_e] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { ueff(i, j, k) = umac(i, j, k) - kappa_e(i, j, k) * -1.0 * 0.5 * (gphi_c(i, j, k) + gphi_o(i, j, k)); @@ -627,7 +626,7 @@ PeleLM::getAdvectionTerm( GetArrOfConstPtrs(ldataNLs_p->uEffnE), bcRecnE[0]); // getAdvectionFluxesMOL(lev, GetArrOfPtrs(fluxes[lev]), *a_nE[lev], // GetArrOfConstPtrs(ldataNLs_p->uEffnE), bcRecnE[0]); - // VisMF::Write(fluxes[lev][1],"FluxYNewbefAvgDown_lev"+std::to_string(lev)); + // amrex::VisMF::Write(fluxes[lev][1],"FluxYNewbefAvgDown_lev"+std::to_string(lev)); } // Average down the fluxes @@ -652,27 +651,28 @@ PeleLM::getAdvectionTerm( void PeleLM::getAdvectionFluxesMOL( int lev, - const Array& a_fluxes, - const MultiFab& a_nE, - const Array& a_ueff, - BCRec bcrec) + const amrex::Array& a_fluxes, + const amrex::MultiFab& a_nE, + const amrex::Array& a_ueff, + amrex::BCRec bcrec) { - auto bcRec_d = convertToDeviceVector(Vector(1, bcrec)); + auto bcRec_d = convertToDeviceVector(amrex::Vector(1, bcrec)); auto AdvType = fetchAdvTypeArray(FIRSTSPEC, 1); auto AdvType_d = convertToDeviceVector(AdvType); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif { - FArrayBox edgstate[AMREX_SPACEDIM]; - for (MFIter mfi(a_nE, TilingIfNotGPU()); mfi.isValid(); ++mfi) { + amrex::FArrayBox edgstate[AMREX_SPACEDIM]; + for (amrex::MFIter mfi(a_nE, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { - Box const& bx = mfi.tilebox(); + amrex::Box const& bx = mfi.tilebox(); AMREX_D_TERM( - const Box& xbx = mfi.grownnodaltilebox(0, 0); - , const Box& ybx = mfi.grownnodaltilebox(1, 0); - , const Box& zbx = mfi.grownnodaltilebox(2, 0)); + const amrex::Box& xbx = mfi.grownnodaltilebox(0, 0); + , const amrex::Box& ybx = mfi.grownnodaltilebox(1, 0); + , const amrex::Box& zbx = mfi.grownnodaltilebox(2, 0)); AMREX_D_TERM( auto const& fx = a_fluxes[0]->array(mfi, 0); @@ -686,13 +686,13 @@ PeleLM::getAdvectionFluxesMOL( edgstate[0].resize(xbx, 1);, edgstate[1].resize(ybx, 1); , edgstate[2].resize(zbx, 1)); AMREX_D_TERM( - Array4 xstate = edgstate[0].array(); - , Array4 ystate = edgstate[1].array(); - , Array4 zstate = edgstate[2].array()); + amrex::Array4 xstate = edgstate[0].array(); + , amrex::Array4 ystate = edgstate[1].array(); + , amrex::Array4 zstate = edgstate[2].array()); AMREX_D_TERM( - Elixir eli_edgex = edgstate[0].elixir(); - , Elixir eli_edgey = edgstate[1].elixir(); - , Elixir eli_edgez = edgstate[2].elixir()); + amrex::Elixir eli_edgex = edgstate[0].elixir(); + , amrex::Elixir eli_edgey = edgstate[1].elixir(); + , amrex::Elixir eli_edgez = edgstate[2].elixir()); auto const& nE_arr = a_nE.const_array(mfi); auto const& divu_arr = a_nE.const_array(mfi); @@ -718,54 +718,55 @@ PeleLM::getAdvectionFluxesMOL( void PeleLM::getAdvectionFluxes( int lev, - const Array& a_fluxes, - const MultiFab& a_nE, - const Array& a_ueff, - BCRec bcrec) + const amrex::Array& a_fluxes, + const amrex::MultiFab& a_nE, + const amrex::Array& a_ueff, + amrex::BCRec bcrec) { - const Box& domain = geom[lev].Domain(); + const amrex::Box& domain = geom[lev].Domain(); int order = m_nEAdvOrder; #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif { - FArrayBox edgstate[AMREX_SPACEDIM]; - for (MFIter mfi(a_nE, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - Box const& bx = mfi.tilebox(); + amrex::FArrayBox edgstate[AMREX_SPACEDIM]; + for (amrex::MFIter mfi(a_nE, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + amrex::Box const& bx = mfi.tilebox(); AMREX_D_TERM( - const Box& xbx = surroundingNodes(bx, 0); - , const Box& ybx = surroundingNodes(bx, 1); - , const Box& zbx = surroundingNodes(bx, 2)); + const amrex::Box& xbx = surroundingNodes(bx, 0); + , const amrex::Box& ybx = surroundingNodes(bx, 1); + , const amrex::Box& zbx = surroundingNodes(bx, 2)); // data arrays auto const& ne_arr = a_nE.const_array(mfi); AMREX_D_TERM( - Array4 xflux = a_fluxes[0]->array(mfi); - , Array4 yflux = a_fluxes[1]->array(mfi); - , Array4 zflux = a_fluxes[2]->array(mfi)); + amrex::Array4 xflux = a_fluxes[0]->array(mfi); + , amrex::Array4 yflux = a_fluxes[1]->array(mfi); + , amrex::Array4 zflux = a_fluxes[2]->array(mfi)); AMREX_D_TERM( - Array4 u = a_ueff[0]->const_array(mfi); - , Array4 v = a_ueff[1]->const_array(mfi); - , Array4 w = a_ueff[2]->const_array(mfi);); + amrex::Array4 u = a_ueff[0]->const_array(mfi); + , amrex::Array4 v = a_ueff[1]->const_array(mfi); + , amrex::Array4 w = a_ueff[2]->const_array(mfi);); AMREX_D_TERM( edgstate[0].resize(xbx, 1);, edgstate[1].resize(ybx, 1); , edgstate[2].resize(zbx, 1)); AMREX_D_TERM( - Array4 xstate = edgstate[0].array(); - , Array4 ystate = edgstate[1].array(); - , Array4 zstate = edgstate[2].array()); + amrex::Array4 xstate = edgstate[0].array(); + , amrex::Array4 ystate = edgstate[1].array(); + , amrex::Array4 zstate = edgstate[2].array()); AMREX_D_TERM( - Elixir xstate_eli = edgstate[0].elixir(); - , Elixir ystate_eli = edgstate[1].elixir(); - , Elixir zstate_eli = edgstate[2].elixir()); + amrex::Elixir xstate_eli = edgstate[0].elixir(); + , amrex::Elixir ystate_eli = edgstate[1].elixir(); + , amrex::Elixir zstate_eli = edgstate[2].elixir()); // Predict edge states // X { // BCs - const Box& edomain = surroundingNodes(domain, 0); + const amrex::Box& edomain = surroundingNodes(domain, 0); const auto bc_lo = bcrec.lo(0); const auto bc_hi = bcrec.hi(0); @@ -773,17 +774,19 @@ PeleLM::getAdvectionFluxes( xbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { int idx[3] = {i, j, k}; bool on_lo = - ((bc_lo == BCType::ext_dir) && (idx[0] <= edomain.smallEnd(0))); + ((bc_lo == amrex::BCType::ext_dir) && + (idx[0] <= edomain.smallEnd(0))); bool on_hi = - ((bc_hi == BCType::ext_dir) && (idx[0] >= edomain.bigEnd(0))); + ((bc_hi == amrex::BCType::ext_dir) && + (idx[0] >= edomain.bigEnd(0))); if (order == 1) { xstate(i, j, k) = ef_edge_state_extdir(i, j, k, 0, on_lo, on_hi, ne_arr, u); } else if (order == 2) { - bool extdir_or_ho_lo = - (bc_lo == BCType::ext_dir) || (bc_lo == BCType::hoextrap); - bool extdir_or_ho_hi = - (bc_hi == BCType::ext_dir) || (bc_hi == BCType::hoextrap); + bool extdir_or_ho_lo = (bc_lo == amrex::BCType::ext_dir) || + (bc_lo == amrex::BCType::hoextrap); + bool extdir_or_ho_hi = (bc_hi == amrex::BCType::ext_dir) || + (bc_hi == amrex::BCType::hoextrap); xstate(i, j, k) = ef_edge_state_2ndO_extdir( i, j, k, 0, on_lo, on_hi, extdir_or_ho_lo, extdir_or_ho_hi, domain.smallEnd(0), domain.bigEnd(0), ne_arr, u, xbx); @@ -794,7 +797,7 @@ PeleLM::getAdvectionFluxes( // Y { // BCs - const Box& edomain = surroundingNodes(domain, 1); + const amrex::Box& edomain = surroundingNodes(domain, 1); const auto bc_lo = bcrec.lo(1); const auto bc_hi = bcrec.hi(1); @@ -802,17 +805,19 @@ PeleLM::getAdvectionFluxes( ybx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { int idx[3] = {i, j, k}; bool on_lo = - ((bc_lo == BCType::ext_dir) && (idx[1] <= edomain.smallEnd(1))); + ((bc_lo == amrex::BCType::ext_dir) && + (idx[1] <= edomain.smallEnd(1))); bool on_hi = - ((bc_hi == BCType::ext_dir) && (idx[1] >= edomain.bigEnd(1))); + ((bc_hi == amrex::BCType::ext_dir) && + (idx[1] >= edomain.bigEnd(1))); if (order == 1) { ystate(i, j, k) = ef_edge_state_extdir(i, j, k, 1, on_lo, on_hi, ne_arr, v); } else if (order == 2) { - bool extdir_or_ho_lo = - (bc_lo == BCType::ext_dir) || (bc_lo == BCType::hoextrap); - bool extdir_or_ho_hi = - (bc_hi == BCType::ext_dir) || (bc_hi == BCType::hoextrap); + bool extdir_or_ho_lo = (bc_lo == amrex::BCType::ext_dir) || + (bc_lo == amrex::BCType::hoextrap); + bool extdir_or_ho_hi = (bc_hi == amrex::BCType::ext_dir) || + (bc_hi == amrex::BCType::hoextrap); ystate(i, j, k) = ef_edge_state_2ndO_extdir( i, j, k, 1, on_lo, on_hi, extdir_or_ho_lo, extdir_or_ho_hi, domain.smallEnd(1), domain.bigEnd(1), ne_arr, v, ybx); @@ -823,7 +828,7 @@ PeleLM::getAdvectionFluxes( // Z { // BCs - const Box& edomain = surroundingNodes(domain, 2); + const amrex::Box& edomain = surroundingNodes(domain, 2); const auto bc_lo = bcrec.lo(2); const auto bc_hi = bcrec.hi(2); @@ -831,17 +836,19 @@ PeleLM::getAdvectionFluxes( zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { int idx[3] = {i, j, k}; bool on_lo = - ((bc_lo == BCType::ext_dir) && (idx[2] <= edomain.smallEnd(2))); + ((bc_lo == amrex::BCType::ext_dir) && + (idx[2] <= edomain.smallEnd(2))); bool on_hi = - ((bc_hi == BCType::ext_dir) && (idx[2] >= edomain.bigEnd(2))); + ((bc_hi == amrex::BCType::ext_dir) && + (idx[2] >= edomain.bigEnd(2))); if (order == 1) { zstate(i, j, k) = ef_edge_state_extdir(i, j, k, 2, on_lo, on_hi, ne_arr, w); } else if (order == 2) { - bool extdir_or_ho_lo = - (bc_lo == BCType::ext_dir) || (bc_lo == BCType::hoextrap); - bool extdir_or_ho_hi = - (bc_hi == BCType::ext_dir) || (bc_hi == BCType::hoextrap); + bool extdir_or_ho_lo = (bc_lo == amrex::BCType::ext_dir) || + (bc_lo == amrex::BCType::hoextrap); + bool extdir_or_ho_hi = (bc_hi == amrex::BCType::ext_dir) || + (bc_hi == amrex::BCType::hoextrap); zstate(i, j, k) = ef_edge_state_2ndO_extdir( i, j, k, 2, on_lo, on_hi, extdir_or_ho_lo, extdir_or_ho_hi, domain.smallEnd(2), domain.bigEnd(2), ne_arr, w, zbx); @@ -853,17 +860,17 @@ PeleLM::getAdvectionFluxes( // Computing fluxes amrex::ParallelFor( - xbx, [u, xstate, xflux] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + xbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { xflux(i, j, k) = u(i, j, k) * xstate(i, j, k); }); #if (AMREX_SPACEDIM > 1) amrex::ParallelFor( - ybx, [v, ystate, yflux] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ybx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { yflux(i, j, k) = v(i, j, k) * ystate(i, j, k); }); #if (AMREX_SPACEDIM == 3) amrex::ParallelFor( - zbx, [w, zstate, zflux] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { zflux(i, j, k) = w(i, j, k) * zstate(i, j, k); }); #endif @@ -873,7 +880,8 @@ PeleLM::getAdvectionFluxes( } void -PeleLM::setUpPrecond(const Real& a_dt, const Vector& a_nE) +PeleLM::setUpPrecond( + const amrex::Real& a_dt, const amrex::Vector& a_nE) { BL_PROFILE("PeleLMeX::setUpPrecond()"); @@ -889,7 +897,7 @@ PeleLM::setUpPrecond(const Real& a_dt, const Vector& a_nE) getPrecondOp()->setDiffOpScalars( -nE_scale / FnE_scale, -a_dt * nE_scale / FnE_scale, a_dt * nE_scale / FnE_scale); - Real omega = m_ABecCecOmega; + amrex::Real omega = m_ABecCecOmega; getPrecondOp()->setDiffOpRelaxation(omega); getPrecondOp()->setDiffOpBCs(bcRecnE[0]); @@ -903,7 +911,7 @@ PeleLM::setUpPrecond(const Real& a_dt, const Vector& a_nE) auto ldataNLs_p = getLevelDataNLSolvePtr(lev); int doZeroVisc = 0; - Array diffE_ec = + amrex::Array diffE_ec = getDiffusivity(lev, 0, 1, doZeroVisc, bcRecnE, ldata_p->diffE_cc); getPrecondOp()->setDiffOpACoeff(lev, 1.0); @@ -912,7 +920,7 @@ PeleLM::setUpPrecond(const Real& a_dt, const Vector& a_nE) } // Stilda approx first-order - Vector diagDiffOp(finest_level + 1); + amrex::Vector diagDiffOp(finest_level + 1); if (m_ef_PC_approx == 2) { for (int lev = 0; lev <= finest_level; ++lev) { diagDiffOp[lev].define(grids[lev], dmap[lev], 1, 1); @@ -945,10 +953,10 @@ PeleLM::setUpPrecond(const Real& a_dt, const Vector& a_nE) auto ldata_p = getLevelDataPtr(lev, AmrNewTime); // CC neKe values - MultiFab nEKe(grids[lev], dmap[lev], 1, 1); + amrex::MultiFab nEKe(grids[lev], dmap[lev], 1, 1); // CC Schur neKe values - MultiFab Schur_nEKe; + amrex::MultiFab Schur_nEKe; if (m_ef_PC_approx == 2) { Schur_nEKe.define(grids[lev], dmap[lev], 1, 1); } @@ -956,8 +964,9 @@ PeleLM::setUpPrecond(const Real& a_dt, const Vector& a_nE) #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(nEKe, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& gbx = mfi.growntilebox(); + for (amrex::MFIter mfi(nEKe, amrex::TilingIfNotGPU()); mfi.isValid(); + ++mfi) { + const amrex::Box& gbx = mfi.growntilebox(); auto const& neke = nEKe.array(mfi); auto const& kappaE = ldata_p->mobE_cc.array(mfi); auto const& ne_arr = a_nE[lev]->const_array(mfi); @@ -967,8 +976,7 @@ PeleLM::setUpPrecond(const Real& a_dt, const Vector& a_nE) (m_ef_PC_approx == 2) ? diagDiffOp[lev].array(mfi) : nEKe.array(mfi); int do_Schur = (m_ef_PC_approx == 2) ? 1 : 0; amrex::ParallelFor( - gbx, [neke, ne_arr, kappaE, Schur, diffOp_diag, a_dt, - do_Schur] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { neke(i, j, k) = kappaE(i, j, k) * ne_arr(i, j, k); if (do_Schur) { Schur(i, j, k) = -a_dt * 0.5 * neke(i, j, k) / diffOp_diag(i, j, k); @@ -977,7 +985,7 @@ PeleLM::setUpPrecond(const Real& a_dt, const Vector& a_nE) } // Upwinded edge neKe values - Array neKe_ec = getUpwindedEdge( + amrex::Array neKe_ec = getUpwindedEdge( lev, 0, 1, bcRecnE, nEKe, GetArrOfConstPtrs(ldataNLs_p->uEffnE)); // Set drift Op coefficients @@ -986,7 +994,7 @@ PeleLM::setUpPrecond(const Real& a_dt, const Vector& a_nE) // Set Stilda Op coefficients if (m_ef_PC_approx == 1) { // Assuming identity of the inverse of DiffOp // Add Stilda pieces - Real scalLap = eps0 * epsr / elemCharge; + amrex::Real scalLap = eps0 * epsr / elemCharge; for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { neKe_ec[idim].mult(0.5 * a_dt, 0, 1); neKe_ec[idim].plus(scalLap, 0, 1); @@ -994,63 +1002,64 @@ PeleLM::setUpPrecond(const Real& a_dt, const Vector& a_nE) getPrecondOp()->setStildaOpBCoeff(lev, GetArrOfConstPtrs(neKe_ec)); } else if (m_ef_PC_approx == 2) { // Assuming inverse of the diag of DiffOp // Upwinded Schur edge neKe values - Array Schur_neKe_ec = getUpwindedEdge( - lev, 0, 1, bcRecnE, Schur_nEKe, GetArrOfConstPtrs(ldataNLs_p->uEffnE)); - Real scalLap = eps0 * epsr / elemCharge; + amrex::Array Schur_neKe_ec = + getUpwindedEdge( + lev, 0, 1, bcRecnE, Schur_nEKe, + GetArrOfConstPtrs(ldataNLs_p->uEffnE)); + amrex::Real scalLap = eps0 * epsr / elemCharge; for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { Schur_neKe_ec[idim].plus(scalLap, 0, 1); } getPrecondOp()->setStildaOpBCoeff(lev, GetArrOfConstPtrs(Schur_neKe_ec)); } else { - Abort("Preconditioner option /= 1 or 2 not available yet"); + amrex::Abort("Preconditioner option /= 1 or 2 not available yet"); } } } -Array +amrex::Array PeleLM::getUpwindedEdge( int lev, int edge_comp, int ncomp, - Vector bcrec, - const MultiFab& ccMF, - const Array& ecVel) + amrex::Vector bcrec, + const amrex::MultiFab& ccMF, + const amrex::Array& ecVel) { AMREX_ASSERT(bcrec.size() >= ncomp); AMREX_ASSERT(ccMF.nComp() >= edge_comp + ncomp); const auto& ba = ccMF.boxArray(); const auto& dm = ccMF.DistributionMap(); const auto& factory = ccMF.Factory(); - Array ecMFs{AMREX_D_DECL( - MultiFab( - amrex::convert(ba, IntVect::TheDimensionVector(0)), dm, ncomp, 0, - MFInfo(), factory), - MultiFab( - amrex::convert(ba, IntVect::TheDimensionVector(1)), dm, ncomp, 0, - MFInfo(), factory), - MultiFab( - amrex::convert(ba, IntVect::TheDimensionVector(2)), dm, ncomp, 0, - MFInfo(), factory))}; + amrex::Array ecMFs{AMREX_D_DECL( + amrex::MultiFab( + amrex::convert(ba, amrex::IntVect::TheDimensionVector(0)), dm, ncomp, 0, + amrex::MFInfo(), factory), + amrex::MultiFab( + amrex::convert(ba, amrex::IntVect::TheDimensionVector(1)), dm, ncomp, 0, + amrex::MFInfo(), factory), + amrex::MultiFab( + amrex::convert(ba, amrex::IntVect::TheDimensionVector(2)), dm, ncomp, 0, + amrex::MFInfo(), factory))}; #ifdef AMREX_USE_EB // TODO cen2edg_upwind EB #else - const Box& domain = geom[lev].Domain(); + const amrex::Box& domain = geom[lev].Domain(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ccMF, TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(ccMF, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - const Box ebx = mfi.nodaltilebox(idim); - const Box& edomain = amrex::surroundingNodes(domain, idim); + const amrex::Box ebx = mfi.nodaltilebox(idim); + const amrex::Box& edomain = amrex::surroundingNodes(domain, idim); const auto& ccVal = ccMF.const_array(mfi, edge_comp); const auto& ecUeff = ecVel[idim]->const_array(mfi); const auto& ecVal = ecMFs[idim].array(mfi); const auto bc_lo = bcrec[0].lo(idim); const auto bc_hi = bcrec[0].hi(idim); amrex::ParallelFor( - ebx, [idim, bc_lo, bc_hi, ccVal, ecUeff, ecVal, - edomain] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { int idx[3] = {i, j, k}; bool on_lo = ((bc_lo == amrex::BCType::ext_dir) && @@ -1068,9 +1077,11 @@ PeleLM::getUpwindedEdge( } void -PeleLM::jTimesV(const Vector& a_v, const Vector& a_Jv) +PeleLM::jTimesV( + const amrex::Vector& a_v, + const amrex::Vector& a_Jv) { - Real vNorm; + amrex::Real vNorm; nlSolveNorm(a_v, vNorm); // v is zero, Jv is zero and return @@ -1082,21 +1093,21 @@ PeleLM::jTimesV(const Vector& a_v, const Vector& a_Jv) } // TODO: only one-sided difference for now - Real delta_pert = + amrex::Real delta_pert = m_ef_lambda_jfnk * (m_ef_lambda_jfnk + nl_stateNorm / vNorm); if (m_ef_diffT_jfnk == 1) { - Vector statePert(finest_level + 1); - Vector residPert(finest_level + 1); + amrex::Vector statePert(finest_level + 1); + amrex::Vector residPert(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { auto ldataNLs_p = getLevelDataNLSolvePtr(lev); statePert[lev].define( - grids[lev], dmap[lev], 2, m_nGrowState, MFInfo(), Factory(lev)); + grids[lev], dmap[lev], 2, m_nGrowState, amrex::MFInfo(), Factory(lev)); residPert[lev].define( - grids[lev], dmap[lev], 2, m_nGrowState, MFInfo(), Factory(lev)); - MultiFab::Copy( + grids[lev], dmap[lev], 2, m_nGrowState, amrex::MFInfo(), Factory(lev)); + amrex::MultiFab::Copy( statePert[lev], ldataNLs_p->nlState, 0, 0, 2, m_nGrowState); - MultiFab::Saxpy(statePert[lev], delta_pert, *a_v[lev], 0, 0, 2, 0); + amrex::MultiFab::Saxpy(statePert[lev], delta_pert, *a_v[lev], 0, 0, 2, 0); } int update_scaling = 0; @@ -1107,32 +1118,34 @@ PeleLM::jTimesV(const Vector& a_v, const Vector& a_Jv) for (int lev = 0; lev <= finest_level; ++lev) { auto ldataNLs_p = getLevelDataNLSolvePtr(lev); - MultiFab::LinComb( + amrex::MultiFab::LinComb( *a_Jv[lev], 1.0, residPert[lev], 0, -1.0, ldataNLs_p->nlResid, 0, 0, 2, 0); a_Jv[lev]->mult(-1.0 / delta_pert); } } else if (m_ef_diffT_jfnk == 2) { - Vector statePertPls(finest_level + 1); - Vector residPertPls(finest_level + 1); - Vector statePertMns(finest_level + 1); - Vector residPertMns(finest_level + 1); + amrex::Vector statePertPls(finest_level + 1); + amrex::Vector residPertPls(finest_level + 1); + amrex::Vector statePertMns(finest_level + 1); + amrex::Vector residPertMns(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { auto ldataNLs_p = getLevelDataNLSolvePtr(lev); statePertPls[lev].define( - grids[lev], dmap[lev], 2, m_nGrowState, MFInfo(), Factory(lev)); + grids[lev], dmap[lev], 2, m_nGrowState, amrex::MFInfo(), Factory(lev)); residPertPls[lev].define( - grids[lev], dmap[lev], 2, m_nGrowState, MFInfo(), Factory(lev)); + grids[lev], dmap[lev], 2, m_nGrowState, amrex::MFInfo(), Factory(lev)); statePertMns[lev].define( - grids[lev], dmap[lev], 2, m_nGrowState, MFInfo(), Factory(lev)); + grids[lev], dmap[lev], 2, m_nGrowState, amrex::MFInfo(), Factory(lev)); residPertMns[lev].define( - grids[lev], dmap[lev], 2, m_nGrowState, MFInfo(), Factory(lev)); - MultiFab::Copy( + grids[lev], dmap[lev], 2, m_nGrowState, amrex::MFInfo(), Factory(lev)); + amrex::MultiFab::Copy( statePertPls[lev], ldataNLs_p->nlState, 0, 0, 2, m_nGrowState); - MultiFab::Copy( + amrex::MultiFab::Copy( statePertMns[lev], ldataNLs_p->nlState, 0, 0, 2, m_nGrowState); - MultiFab::Saxpy(statePertPls[lev], delta_pert, *a_v[lev], 0, 0, 2, 0); - MultiFab::Saxpy(statePertMns[lev], -delta_pert, *a_v[lev], 0, 0, 2, 0); + amrex::MultiFab::Saxpy( + statePertPls[lev], delta_pert, *a_v[lev], 0, 0, 2, 0); + amrex::MultiFab::Saxpy( + statePertMns[lev], -delta_pert, *a_v[lev], 0, 0, 2, 0); } int update_scaling = 0; @@ -1146,7 +1159,7 @@ PeleLM::jTimesV(const Vector& a_v, const Vector& a_Jv) for (int lev = 0; lev <= finest_level; ++lev) { auto ldataNLs_p = getLevelDataNLSolvePtr(lev); - MultiFab::LinComb( + amrex::MultiFab::LinComb( *a_Jv[lev], 1.0, residPertPls[lev], 0, -1.0, residPertMns[lev], 0, 0, 2, 0); a_Jv[lev]->mult(-0.5 / delta_pert); @@ -1156,17 +1169,18 @@ PeleLM::jTimesV(const Vector& a_v, const Vector& a_Jv) void PeleLM::applyPrecond( - const Vector& a_v, const Vector& a_Pv) + const amrex::Vector& a_v, + const amrex::Vector& a_Pv) { BL_PROFILE("PeleLMeX::applyPrecond()"); // Setup aliases and temps - Vector nE_al; - Vector phiV_al; - Vector PnE_al; - Vector PphiV_al; - Vector> temp(finest_level + 1); - Vector> temp2(finest_level + 1); + amrex::Vector nE_al; + amrex::Vector phiV_al; + amrex::Vector PnE_al; + amrex::Vector PphiV_al; + amrex::Vector> temp(finest_level + 1); + amrex::Vector> temp2(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { // Aliases nE_al.emplace_back(*a_v[lev], amrex::make_alias, 0, 1); @@ -1177,11 +1191,13 @@ PeleLM::applyPrecond( PphiV_al[lev].setVal(0.0, 0, 1, 1); // Temporary data holder - temp[lev].reset(new MultiFab( - grids[lev], dmap[lev], 1, nE_al[lev].nGrow(), MFInfo(), Factory(lev))); + temp[lev].reset(new amrex::MultiFab( + grids[lev], dmap[lev], 1, nE_al[lev].nGrow(), amrex::MFInfo(), + Factory(lev))); temp[lev]->setVal(0.0); - temp2[lev].reset(new MultiFab( - grids[lev], dmap[lev], 1, nE_al[lev].nGrow(), MFInfo(), Factory(lev))); + temp2[lev].reset(new amrex::MultiFab( + grids[lev], dmap[lev], 1, nE_al[lev].nGrow(), amrex::MFInfo(), + Factory(lev))); temp2[lev]->setVal(0.0); } @@ -1193,17 +1209,17 @@ PeleLM::applyPrecond( } // Most inner matrix - Real S_tol = m_ef_PC_MG_Tol; - Real S_tol_abs = MLNorm0(GetVecOfConstPtrs(nE_al)) * m_ef_PC_MG_Tol; + amrex::Real S_tol = m_ef_PC_MG_Tol; + amrex::Real S_tol_abs = MLNorm0(GetVecOfConstPtrs(nE_al)) * m_ef_PC_MG_Tol; getPrecondOp()->diffOpSolve( GetVecOfPtrs(PnE_al), GetVecOfConstPtrs(nE_al), S_tol, S_tol_abs); for (int lev = 0; lev <= finest_level; ++lev) { - MultiFab::Copy(PphiV_al[lev], phiV_al[lev], 0, 0, 1, 0); + amrex::MultiFab::Copy(PphiV_al[lev], phiV_al[lev], 0, 0, 1, 0); } // Pivot second matrix for (int lev = 0; lev <= finest_level; ++lev) { - MultiFab::Saxpy( + amrex::MultiFab::Saxpy( PphiV_al[lev], nE_scale / FphiV_scale, PnE_al[lev], 0, 0, 1, 0); } @@ -1216,7 +1232,7 @@ PeleLM::applyPrecond( getPrecondOp()->StildaOpSolve( GetVecOfPtrs(temp), GetVecOfConstPtrs(PphiV_al), S_tol, S_tol_abs); for (int lev = 0; lev <= finest_level; ++lev) { - MultiFab::Copy(PphiV_al[lev], *temp[lev], 0, 0, 1, 0); + amrex::MultiFab::Copy(PphiV_al[lev], *temp[lev], 0, 0, 1, 0); } // Final matrix @@ -1225,24 +1241,24 @@ PeleLM::applyPrecond( getPrecondOp()->diffOpSolve( GetVecOfPtrs(temp2), GetVecOfConstPtrs(temp), S_tol, S_tol_abs); for (int lev = 0; lev <= finest_level; ++lev) { - MultiFab::Saxpy(PnE_al[lev], -1.0, *temp2[lev], 0, 0, 1, 0); + amrex::MultiFab::Saxpy(PnE_al[lev], -1.0, *temp2[lev], 0, 0, 1, 0); } } void -PeleLM::nlSolveNorm(const Vector& a_MF, Real& r) +PeleLM::nlSolveNorm(const amrex::Vector& a_MF, amrex::Real& r) { r = 0.0; int nComp = a_MF[0]->nComp(); for (int comp = 0; comp < nComp; comp++) { - Real norm = 0.0; + amrex::Real norm = 0.0; for (int lev = 0; lev < a_MF.size(); ++lev) { // TODO : norm not weighted by cell size, should it ? if (lev != a_MF.size() - 1) { - norm += MultiFab::Dot( + norm += amrex::MultiFab::Dot( *m_coveredMask[lev], *a_MF[lev], comp, *a_MF[lev], comp, 1, 0); } else { - norm += MultiFab::Dot(*a_MF[lev], comp, *a_MF[lev], comp, 1, 0); + norm += amrex::MultiFab::Dot(*a_MF[lev], comp, *a_MF[lev], comp, 1, 0); } } r += norm; diff --git a/Source/Plasma/PeleLMeX_EFPoisson.cpp b/Source/Plasma/PeleLMeX_EFPoisson.cpp index 065a71c4a..5f41ba613 100644 --- a/Source/Plasma/PeleLMeX_EFPoisson.cpp +++ b/Source/Plasma/PeleLMeX_EFPoisson.cpp @@ -3,14 +3,12 @@ #include #include -using namespace amrex; - void PeleLM::poissonSolveEF(const TimeStamp& a_time) { BL_PROFILE("PeleLMeX::poissonSolveEF()"); if (ef_verbose) { - Print() << " EF Poisson solve \n"; + amrex::Print() << " EF Poisson solve \n"; } // Get the phiV BCRec @@ -18,24 +16,24 @@ PeleLM::poissonSolveEF(const TimeStamp& a_time) // Build Poisson RHS: charge distribution int nGhost = 0; - Vector> rhsPoisson(finest_level + 1); + amrex::Vector> rhsPoisson(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - rhsPoisson[lev].reset(new MultiFab( - grids[lev], dmap[lev], 1, nGhost, MFInfo(), *m_factory[lev])); + rhsPoisson[lev].reset(new amrex::MultiFab( + grids[lev], dmap[lev], 1, nGhost, amrex::MFInfo(), *m_factory[lev])); auto ldata_p = getLevelDataPtr(lev, a_time); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*rhsPoisson[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(*rhsPoisson[lev], amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); auto const& rhoY = ldata_p->state.const_array(mfi, FIRSTSPEC); auto const& nE = ldata_p->state.const_array(mfi, NE); auto const& rhs = rhsPoisson[lev]->array(mfi); - Real factor = -1.0; // / ( eps0 * epsr); + amrex::Real factor = -1.0; // / ( eps0 * epsr); amrex::ParallelFor( - bx, [rhs, rhoY, nE, factor, - zk = zk] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { rhs(i, j, k) = -nE(i, j, k) * elemCharge * factor; for (int n = 0; n < NUM_SPECIES; n++) { rhs(i, j, k) += zk[n] * rhoY(i, j, k, n) * factor; diff --git a/Source/Plasma/PeleLMeX_EFReactions.cpp b/Source/Plasma/PeleLMeX_EFReactions.cpp index c0aa9e7aa..193e665fb 100644 --- a/Source/Plasma/PeleLMeX_EFReactions.cpp +++ b/Source/Plasma/PeleLMeX_EFReactions.cpp @@ -1,19 +1,18 @@ #include #include -using namespace amrex; - void PeleLM::computeInstantaneousReactionRateEF( - int lev, const TimeStamp& a_time, MultiFab* a_I_R) + int lev, const TimeStamp& a_time, amrex::MultiFab* a_I_R) { auto ldata_p = getLevelDataPtr(lev, a_time); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); auto const& rhoY = ldata_p->state.const_array(mfi, FIRSTSPEC); auto const& rhoH = ldata_p->state.const_array(mfi, RHOH); auto const& nE = ldata_p->state.const_array(mfi, NE); @@ -21,10 +20,8 @@ PeleLM::computeInstantaneousReactionRateEF( auto const& rhoYdot = a_I_R->array(mfi); auto const& nEdot = a_I_R->array(mfi, NUM_SPECIES); - amrex::ParallelFor( - bx, [rhoY, rhoH, nE, T, rhoYdot, - nEdot] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - reactionRateRhoY_EF(i, j, k, rhoY, rhoH, T, nE, rhoYdot, nEdot); - }); + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + reactionRateRhoY_EF(i, j, k, rhoY, rhoH, T, nE, rhoYdot, nEdot); + }); } } diff --git a/Source/Plasma/PeleLMeX_EFTimeStep.cpp b/Source/Plasma/PeleLMeX_EFTimeStep.cpp index 019ea5825..159cf2761 100644 --- a/Source/Plasma/PeleLMeX_EFTimeStep.cpp +++ b/Source/Plasma/PeleLMeX_EFTimeStep.cpp @@ -1,12 +1,10 @@ #include -using namespace amrex; - -Real +amrex::Real PeleLM::estEFIonsDt(const TimeStamp& a_time) { - Real estdt = 1.0e200; - constexpr Real small = 1.0e-8; + amrex::Real estdt = 1.0e200; + constexpr amrex::Real small = 1.0e-8; auto bcRecPhiV = fetchBCRecArray(PHIV, 1); @@ -15,21 +13,22 @@ PeleLM::estEFIonsDt(const TimeStamp& a_time) for (int lev = 0; lev <= finest_level; ++lev) { - Real estdt_lev = 1.0e200; + amrex::Real estdt_lev = 1.0e200; // Get cell centered gradient of phiV auto ldata_p = getLevelDataPtr(lev, a_time); - MultiFab efield_cc(grids[lev], dmap[lev], AMREX_SPACEDIM, 0); - MultiFab driftVelMax_cc(grids[lev], dmap[lev], 1, 0); + amrex::MultiFab efield_cc(grids[lev], dmap[lev], AMREX_SPACEDIM, 0); + amrex::MultiFab driftVelMax_cc(grids[lev], dmap[lev], 1, 0); const auto dxinv = Geom(lev).InvCellSizeArray(); const auto domain = Geom(lev).Domain(); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); auto const& phiV = ldata_p->state.const_array(mfi, PHIV); auto const& efield = efield_cc.array(mfi, 0); @@ -98,20 +97,21 @@ PeleLM::estEFIonsDt(const TimeStamp& a_time) // Get cell centered max effective velocities across // all dimension/ions #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); auto const& vel = ldata_p->state.const_array(mfi, VELX); auto const& efield = efield_cc.const_array(mfi); auto const& mob_cc = ldata_p->mob_cc.const_array(mfi); auto const& uDrMax = driftVelMax_cc.array(mfi); amrex::ParallelFor( bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - Real maxVel = 0.0; + amrex::Real maxVel = 0.0; for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { for (int n = 0; n < NUM_IONS; n++) { - Real ueff = + amrex::Real ueff = vel(i, j, k, idim) + mob_cc(i, j, k, n) * efield(i, j, k, idim); maxVel = amrex::max(maxVel, std::abs(ueff)); } @@ -125,15 +125,15 @@ PeleLM::estEFIonsDt(const TimeStamp& a_time) estdt_lev = amrex::ReduceMin( driftVelMax_cc, 0, [dx, cfl_lcl] AMREX_GPU_HOST_DEVICE( - Box const& bx, Array4 const& ueffm) noexcept -> Real { - using namespace amrex::literals; + amrex::Box const& bx, + amrex::Array4 const& ueffm) noexcept -> amrex::Real { const auto lo = amrex::lbound(bx); const auto hi = amrex::ubound(bx); #if !defined(__CUDACC__) || (__CUDACC_VER_MAJOR__ != 9) || \ (__CUDACC_VER_MINOR__ != 2) amrex::Real velmax = std::numeric_limits::min(); #else - amrex::Real velmax = -1.e37_rt; + amrex::Real velmax = -1.e37; #endif for (int k = lo.z; k <= hi.z; ++k) { for (int j = lo.y; j <= hi.y; ++j) { @@ -146,11 +146,11 @@ PeleLM::estEFIonsDt(const TimeStamp& a_time) }); // Min across levels - estdt = std::min(estdt, estdt_lev); + estdt = amrex::min(estdt, estdt_lev); } // Min across processors - ParallelDescriptor::ReduceRealMin(estdt); + amrex::ParallelDescriptor::ReduceRealMin(estdt); return estdt; } diff --git a/Source/Plasma/PeleLMeX_EFTransport.cpp b/Source/Plasma/PeleLMeX_EFTransport.cpp index 4a35479a7..9aa18fa5f 100644 --- a/Source/Plasma/PeleLMeX_EFTransport.cpp +++ b/Source/Plasma/PeleLMeX_EFTransport.cpp @@ -1,35 +1,30 @@ #include #include -using namespace amrex; - void PeleLM::calcEFTransport(const TimeStamp& a_time) { BL_PROFILE("PeleLMeX::calcEFTransport()"); for (int lev = 0; lev <= finest_level; ++lev) { - auto ldata_p = getLevelDataPtr(lev, a_time); auto dxinv = Geom(lev).InvCellSizeArray(); - #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->diffE_cc, TilingIfNotGPU()); mfi.isValid(); - ++mfi) { - const Box& gbx = mfi.growntilebox(); + for (amrex::MFIter mfi(ldata_p->diffE_cc, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& gbx = mfi.growntilebox(); auto const& mobE = ldata_p->mobE_cc.array(mfi); auto const& diffE = ldata_p->diffE_cc.array(mfi); auto const& rhoY = ldata_p->state.const_array(mfi, FIRSTSPEC); auto const& phiV = ldata_p->state.const_array(mfi, PHIV); auto const& T = ldata_p->state.const_array(mfi, TEMP); - Real factor = PP_RU_MKS / (Na * elemCharge); + amrex::Real factor = PP_RU_MKS / (Na * elemCharge); + const auto useTab = m_electronKappaTab; + const auto fixedKe = m_fixedKappaE; amrex::ParallelFor( - gbx, - [mobE, diffE, rhoY, phiV, T, factor, dxinv, useTab = m_electronKappaTab, - fixedKe = - m_fixedKappaE] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { getKappaE(i, j, k, useTab, fixedKe, dxinv, rhoY, phiV, T, mobE); getDiffE(i, j, k, factor, T, mobE, diffE); }); diff --git a/Source/Plasma/PeleLMeX_EFUtils.cpp b/Source/Plasma/PeleLMeX_EFUtils.cpp index c6b3a36c1..4ccf6fdf2 100644 --- a/Source/Plasma/PeleLMeX_EFUtils.cpp +++ b/Source/Plasma/PeleLMeX_EFUtils.cpp @@ -4,12 +4,10 @@ #include #include -using namespace amrex; - -Vector> +amrex::Vector> PeleLM::getNLgradPhiVVect() { - Vector> r; + amrex::Vector> r; r.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { r.push_back(GetArrOfPtrs(m_leveldatanlsolve[lev]->gPhiVOld)); @@ -17,10 +15,10 @@ PeleLM::getNLgradPhiVVect() return r; } -Vector> +amrex::Vector> PeleLM::getUeffVect() { - Vector> r; + amrex::Vector> r; r.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { r.push_back(GetArrOfPtrs(m_leveldatanlsolve[lev]->uEffnE)); @@ -28,10 +26,10 @@ PeleLM::getUeffVect() return r; } -Vector +amrex::Vector PeleLM::getNLresidVect() { - Vector r; + amrex::Vector r; r.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { r.push_back(&(m_leveldatanlsolve[lev]->nlResid)); @@ -39,10 +37,10 @@ PeleLM::getNLresidVect() return r; } -Vector +amrex::Vector PeleLM::getNLstateVect() { - Vector r; + amrex::Vector r; r.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { r.push_back(&(m_leveldatanlsolve[lev]->nlState)); @@ -50,10 +48,10 @@ PeleLM::getNLstateVect() return r; } -Vector +amrex::Vector PeleLM::getNLBGChargeVect() { - Vector r; + amrex::Vector r; r.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { r.push_back(&(m_leveldatanlsolve[lev]->backgroundCharge)); @@ -62,49 +60,49 @@ PeleLM::getNLBGChargeVect() } void -PeleLM::getNLStateScaling(Real& nEScale, Real& phiVScale) +PeleLM::getNLStateScaling(amrex::Real& nEScale, amrex::Real& phiVScale) { - Array r = {0.0, 0.0}; + amrex::Array r = {0.0, 0.0}; for (int comp = 0; comp < 2; comp++) { for (int lev = 0; lev <= finest_level; ++lev) { if (lev != finest_level) { - r[comp] = std::max( + r[comp] = amrex::max( r[comp], m_leveldatanlsolve[lev]->nlState.norm0( *m_coveredMask[lev], comp, 0, true)); } else { - r[comp] = std::max( + r[comp] = amrex::max( r[comp], m_leveldatanlsolve[lev]->nlState.norm0(comp, 0, true, true)); } } - ParallelDescriptor::ReduceRealMax(r[comp]); + amrex::ParallelDescriptor::ReduceRealMax(r[comp]); } nEScale = r[0]; phiVScale = r[1]; } void -PeleLM::getNLResidScaling(Real& nEScale, Real& phiVScale) +PeleLM::getNLResidScaling(amrex::Real& nEScale, amrex::Real& phiVScale) { - Array r = {0.0, 0.0}; + amrex::Array r = {0.0, 0.0}; for (int comp = 0; comp < 2; comp++) { for (int lev = 0; lev <= finest_level; ++lev) { if (lev != finest_level) { - r[comp] = std::max( + r[comp] = amrex::max( r[comp], m_leveldatanlsolve[lev]->nlResid.norm0( *m_coveredMask[lev], comp, 0, true)); } else { - r[comp] = std::max( + r[comp] = amrex::max( r[comp], m_leveldatanlsolve[lev]->nlResid.norm0(comp, 0, true)); } } - ParallelDescriptor::ReduceRealMax(r[comp]); + amrex::ParallelDescriptor::ReduceRealMax(r[comp]); } nEScale = r[0]; phiVScale = r[1]; } void -PeleLM::scaleNLState(const Real& nEScale, const Real& phiVScale) +PeleLM::scaleNLState(const amrex::Real& nEScale, const amrex::Real& phiVScale) { for (int lev = 0; lev <= finest_level; ++lev) { m_leveldatanlsolve[lev]->nlState.mult(1.0 / nE_scale, 0, 1, m_nGrowState); @@ -114,7 +112,9 @@ PeleLM::scaleNLState(const Real& nEScale, const Real& phiVScale) void PeleLM::scaleNLResid( - const Vector& a_resid, const Real& nEScale, const Real& phiVScale) + const amrex::Vector& a_resid, + const amrex::Real& nEScale, + const amrex::Real& phiVScale) { for (int lev = 0; lev <= finest_level; ++lev) { a_resid[lev]->mult(1.0 / FnE_scale, 0, 1, 1); @@ -122,17 +122,16 @@ PeleLM::scaleNLResid( } } -BCRec -PeleLM::hackBCChargedParticle(const Real& charge, const BCRec& bc_in) +amrex::BCRec +PeleLM::hackBCChargedParticle( + const amrex::Real& charge, const amrex::BCRec& bc_in) { - - BCRec bc_hacked; + amrex::BCRec bc_hacked; const int* lo_bc = bc_in.lo(); const int* hi_bc = bc_in.hi(); for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - int lo = lo_bc[idim]; int hi = hi_bc[idim]; // Spec is In/Out and it's cathode (neg electrode) @@ -183,22 +182,20 @@ PeleLM::hackBCChargedParticle(const Real& charge, const BCRec& bc_in) void PeleLM::addLorentzVelForces( int lev, - const Box& bx, - const Real& a_time, - Array4 const& force, - Array4 const& rhoY, - Array4 const& phiV, - Array4 const& nE) + const amrex::Box& bx, + const amrex::Real& a_time, + amrex::Array4 const& force, + amrex::Array4 const& rhoY, + amrex::Array4 const& phiV, + amrex::Array4 const& nE) { const auto dx = geom[lev].CellSizeArray(); - GpuArray blo = bx.loVect3d(); - GpuArray bhi = bx.hiVect3d(); + amrex::GpuArray blo = bx.loVect3d(); + amrex::GpuArray bhi = bx.hiVect3d(); - amrex::ParallelFor( - bx, [force, rhoY, phiV, nE, a_time, dx, blo, bhi, - zk = zk] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - addLorentzForce(i, j, k, blo, bhi, a_time, dx, zk, rhoY, nE, phiV, force); - }); + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + addLorentzForce(i, j, k, blo, bhi, a_time, dx, zk, rhoY, nE, phiV, force); + }); } void @@ -208,23 +205,22 @@ PeleLM::initializeElectronNeutral() ProbParm const* lprobparm = prob_parm_d; for (int lev = 0; lev <= finest_level; ++lev) { - // Get level data new time pointer auto ldata_p = getLevelDataPtr(lev, AmrNewTime); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldata_p->state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); auto const& rho = ldata_p->state.array(mfi, DENSITY); auto const& rhoY = ldata_p->state.array(mfi, FIRSTSPEC); auto const& rhoH = ldata_p->state.array(mfi, RHOH); auto const& temp = ldata_p->state.array(mfi, TEMP); auto const& nE = ldata_p->state.array(mfi, NE); amrex::ParallelFor( - bx, [rho, rhoY, rhoH, temp, nE, - lprobparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { initElecNeutral(i, j, k, rho, rhoY, rhoH, temp, nE, *lprobparm); }); } @@ -232,18 +228,18 @@ PeleLM::initializeElectronNeutral() // Convert I_R(Y_nE) into I_R(nE) and set I_R(Y_nE) to zero auto ldataR_p = getLevelDataReactPtr(lev); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(ldataR_p->I_R, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); + for (amrex::MFIter mfi(ldataR_p->I_R, amrex::TilingIfNotGPU()); + mfi.isValid(); ++mfi) { + const amrex::Box& bx = mfi.tilebox(); auto const& YnEdot = ldataR_p->I_R.array(mfi, E_ID); auto const& nEdot = ldataR_p->I_R.array(mfi, NUM_SPECIES); auto eos = pele::physics::PhysicsType::eos(); - Real invmwt[NUM_SPECIES] = {0.0}; + amrex::Real invmwt[NUM_SPECIES] = {0.0}; eos.inv_molecular_weight(invmwt); - ParallelFor( - bx, - [YnEdot, nEdot, invmwt] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + amrex::ParallelFor( + bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { nEdot(i, j, k) = YnEdot(i, j, k) * Na * invmwt[E_ID] * 1.0e3; YnEdot(i, j, k) = 0.0; }); @@ -257,7 +253,8 @@ PeleLM::initializeElectronFromMassFraction() } void -PeleLM::fillPatchExtrap(Real a_time, Vector const& a_MF, int a_nGrow) +PeleLM::fillPatchExtrap( + amrex::Real a_time, amrex::Vector const& a_MF, int a_nGrow) { AMREX_ASSERT(a_MF[0]->nComp() <= m_bcrec_force.size()); const int nComp = a_MF[0]->nComp(); @@ -265,20 +262,22 @@ PeleLM::fillPatchExtrap(Real a_time, Vector const& a_MF, int a_nGrow) int lev = 0; { - PhysBCFunct> bndry_func( - geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + bndry_func(geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); FillPatchSingleLevel( - *a_MF[lev], IntVect(a_nGrow), a_time, {a_MF[lev]}, {a_time}, 0, 0, nComp, - geom[lev], bndry_func, 0); + *a_MF[lev], amrex::IntVect(a_nGrow), a_time, {a_MF[lev]}, {a_time}, 0, 0, + nComp, geom[lev], bndry_func, 0); } for (lev = 1; lev <= finest_level; ++lev) { - PhysBCFunct> crse_bndry_func( - geom[lev - 1], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); - PhysBCFunct> fine_bndry_func( - geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + crse_bndry_func( + geom[lev - 1], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); + amrex::PhysBCFunct> + fine_bndry_func( + geom[lev], {m_bcrec_force}, PeleLMCCFillExtDirDummy{m_nAux}); auto* mapper = getInterpolator(); FillPatchTwoLevels( - *a_MF[lev], IntVect(a_nGrow), a_time, {a_MF[lev - 1]}, {a_time}, + *a_MF[lev], amrex::IntVect(a_nGrow), a_time, {a_MF[lev - 1]}, {a_time}, {a_MF[lev]}, {a_time}, 0, 0, nComp, geom[lev - 1], geom[lev], crse_bndry_func, 0, fine_bndry_func, 0, refRatio(lev - 1), mapper, {m_bcrec_force}, 0); @@ -286,29 +285,33 @@ PeleLM::fillPatchExtrap(Real a_time, Vector const& a_MF, int a_nGrow) } void -PeleLM::fillPatchNLnE(Real a_time, Vector const& a_nE, int a_nGrow) +PeleLM::fillPatchNLnE( + amrex::Real a_time, amrex::Vector const& a_nE, int a_nGrow) { ProbParm const* lprobparm = prob_parm_d; auto const* lpmfdata = pmf_data.device_parm(); int lev = 0; { - PhysBCFunct>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> bndry_func( geom[lev], fetchBCRecArray(NE, 1), PeleLMCCFillExtDirnE{ lprobparm, lpmfdata, m_nAux}); FillPatchSingleLevel( - *a_nE[lev], IntVect(a_nGrow), a_time, {a_nE[lev]}, {a_time}, 0, 0, 1, - geom[lev], bndry_func, 0); + *a_nE[lev], amrex::IntVect(a_nGrow), a_time, {a_nE[lev]}, {a_time}, 0, 0, + 1, geom[lev], bndry_func, 0); } for (lev = 1; lev <= finest_level; ++lev) { - PhysBCFunct>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> crse_bndry_func( geom[lev - 1], fetchBCRecArray(NE, 1), PeleLMCCFillExtDirnE{ lprobparm, lpmfdata, m_nAux}); - PhysBCFunct>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> fine_bndry_func( geom[lev], fetchBCRecArray(NE, 1), PeleLMCCFillExtDirnE{ @@ -316,7 +319,7 @@ PeleLM::fillPatchNLnE(Real a_time, Vector const& a_nE, int a_nGrow) auto* mapper = getInterpolator(); FillPatchTwoLevels( - *a_nE[lev], IntVect(a_nGrow), a_time, {a_nE[lev - 1]}, {a_time}, + *a_nE[lev], amrex::IntVect(a_nGrow), a_time, {a_nE[lev - 1]}, {a_time}, {a_nE[lev]}, {a_time}, 0, 0, 1, geom[lev - 1], geom[lev], crse_bndry_func, 0, fine_bndry_func, 0, refRatio(lev - 1), mapper, fetchBCRecArray(NE, 1), 0); @@ -325,32 +328,34 @@ PeleLM::fillPatchNLnE(Real a_time, Vector const& a_nE, int a_nGrow) void PeleLM::fillPatchNLphiV( - Real a_time, Vector const& a_phiV, int a_nGrow) + amrex::Real a_time, + amrex::Vector const& a_phiV, + int a_nGrow) { ProbParm const* lprobparm = prob_parm_d; auto const* lpmfdata = pmf_data.device_parm(); int lev = 0; { - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> bndry_func( geom[lev], fetchBCRecArray(PHIV, 1), PeleLMCCFillExtDirPhiV{ lprobparm, lpmfdata, m_nAux}); FillPatchSingleLevel( - *a_phiV[lev], IntVect(a_nGrow), a_time, {a_phiV[lev]}, {a_time}, 0, 0, 1, - geom[lev], bndry_func, 0); + *a_phiV[lev], amrex::IntVect(a_nGrow), a_time, {a_phiV[lev]}, {a_time}, 0, + 0, 1, geom[lev], bndry_func, 0); } for (lev = 1; lev <= finest_level; ++lev) { - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> crse_bndry_func( geom[lev - 1], fetchBCRecArray(PHIV, 1), PeleLMCCFillExtDirPhiV{ lprobparm, lpmfdata, m_nAux}); - PhysBCFunct< - GpuBndryFuncFab>> + amrex::PhysBCFunct< + amrex::GpuBndryFuncFab>> fine_bndry_func( geom[lev], fetchBCRecArray(PHIV, 1), PeleLMCCFillExtDirPhiV{ @@ -358,8 +363,8 @@ PeleLM::fillPatchNLphiV( auto* mapper = getInterpolator(); FillPatchTwoLevels( - *a_phiV[lev], IntVect(a_nGrow), a_time, {a_phiV[lev - 1]}, {a_time}, - {a_phiV[lev]}, {a_time}, 0, 0, 1, geom[lev - 1], geom[lev], + *a_phiV[lev], amrex::IntVect(a_nGrow), a_time, {a_phiV[lev - 1]}, + {a_time}, {a_phiV[lev]}, {a_time}, 0, 0, 1, geom[lev - 1], geom[lev], crse_bndry_func, 0, fine_bndry_func, 0, refRatio(lev - 1), mapper, fetchBCRecArray(PHIV, 1), 0); } @@ -369,7 +374,7 @@ void PeleLM::ionsBalance() { // Compute the sum of ions on the domain boundaries - Array ionsCurrent{0.0}; + amrex::Array ionsCurrent{0.0}; for (int n = NUM_SPECIES - NUM_IONS; n < NUM_SPECIES; n++) { for (int i = 0; i < 2 * AMREX_SPACEDIM; i++) { ionsCurrent[i] += m_domainRhoYFlux[2 * n * AMREX_SPACEDIM + i] * zk[n]; diff --git a/Source/Plasma/PeleLMeX_EF_K.H b/Source/Plasma/PeleLMeX_EF_K.H index 68ccac394..1669a70fa 100644 --- a/Source/Plasma/PeleLMeX_EF_K.H +++ b/Source/Plasma/PeleLMeX_EF_K.H @@ -22,26 +22,24 @@ getKappaE( amrex::Array4 const& T, amrex::Array4 const& Ke) noexcept { - using namespace amrex::literals; - if (!isKeTabulated) { Ke(i, j, k) = cstKe; } else { auto eos = pele::physics::PhysicsType::eos(); // Get rho & Y from rhoY - amrex::Real rho = 0.0_rt; + amrex::Real rho = 0.0; for (int n = 0; n < NUM_SPECIES; n++) { rho += rhoY(i, j, k, n); } - amrex::Real rhoinv = 1.0_rt / rho; + amrex::Real rhoinv = 1.0 / rho; amrex::Real y[NUM_SPECIES] = {0.0}; for (int n = 0; n < NUM_SPECIES; n++) { y[n] = rhoY(i, j, k, n) * rhoinv; } // Get the reduced electric field - amrex::Real Wbar = 0.0_rt; + amrex::Real Wbar = 0.0; eos.Y2WBAR(y, Wbar); amrex::Real Ngas = rho * Na / Wbar * 1000.0; // CGS -> MKS conversion amrex::Real EnormSq = AMREX_D_TERM( @@ -78,7 +76,6 @@ getKappaSp( amrex::Array4 const& T, amrex::Array4 const& Ksp) noexcept { - using namespace amrex::literals; amrex::Real rho = 0.0; for (int n = 0; n < NUM_SPECIES; n++) { rho += rhoY(i, j, k, n); @@ -108,8 +105,6 @@ getDiffE( amrex::Array4 const& Ke, amrex::Array4 const& De) noexcept { - using namespace amrex::literals; - De(i, j, k) = Ke(i, j, k) * T(i, j, k) * factor; } @@ -126,8 +121,6 @@ ef_edge_state_extdir( amrex::Array4 const& state, amrex::Array4 const& effvel) noexcept { - using namespace amrex::literals; - // Default behavior: id_l -1 in dir // Handle the BCs // Need -1 in id_l and id_h in dir on low Dirichlet BC @@ -149,8 +142,8 @@ ef_edge_state_extdir( } else if (effvel(i, j, k) < -smallVel) { state_ed = state(id_h[0], id_h[1], id_h[2]); } else { - state_ed = 0.5_rt * (state(id_l[0], id_l[1], id_l[2]) + - state(id_h[0], id_h[1], id_h[2])); + state_ed = 0.5 * (state(id_l[0], id_l[1], id_l[2]) + + state(id_h[0], id_h[1], id_h[2])); } return state_ed; } @@ -173,8 +166,6 @@ ef_edge_state_2ndO_extdir( amrex::Array4 const& effvel, const amrex::Box& box) noexcept { - using namespace amrex::literals; - // Default behavior: id_l -1 in dir // Handle the BCs // Need -1 in id_l and id_h in dir on low Dirichlet BC @@ -235,7 +226,7 @@ ef_edge_state_2ndO_extdir( } else if (effvel(i, j, k) < -smallVel) { state_ed = state_pls; } else { - state_ed = 0.5_rt * (state_pls + state_mns); + state_ed = 0.5 * (state_pls + state_mns); } } return state_ed; @@ -256,8 +247,6 @@ cen2edg_upwind( amrex::Array4 const& cfab, amrex::Array4 const& efab) noexcept { - using namespace amrex::literals; - // Default behavior: id_l -1 in dir // Handle the BCs // Need -1 in id_l and id_h in dir on low Dirichlet BC @@ -279,8 +268,8 @@ cen2edg_upwind( } else if (ueff(i, j, k) < -smallVel) { efab(i, j, k, n) = cfab(id_h[0], id_h[1], id_h[2], n); } else { - efab(i, j, k, n) = 0.5_rt * (cfab(id_l[0], id_l[1], id_l[2], n) + - cfab(id_h[0], id_h[1], id_h[2], n)); + efab(i, j, k, n) = 0.5 * (cfab(id_l[0], id_l[1], id_l[2], n) + + cfab(id_h[0], id_h[1], id_h[2], n)); } } } @@ -299,14 +288,12 @@ reactionRateRhoY_EF( amrex::Array4 const& rhoYdot, amrex::Array4 const& nEdot) noexcept { - using namespace amrex::literals; - // Get rho from rhoY. Is there a better way ? - amrex::Real rho = 0.0_rt; + amrex::Real rho = 0.0; for (int n = 0; n < NUM_SPECIES; n++) { rho += rhoY(i, j, k, n); } - amrex::Real rhoinv = 1.0_rt / rho; + amrex::Real rhoinv = 1.0 / rho; amrex::Real y[NUM_SPECIES] = {0.0}; for (int n = 0; n < NUM_SPECIES; n++) { y[n] = rhoY(i, j, k, n) * rhoinv; @@ -321,15 +308,15 @@ reactionRateRhoY_EF( // Get T from Y/H. Was done like that before. amrex::Real Tloc = T(i, j, k); amrex::Real H = - rhoH(i, j, k) * rhoinv * 1.0e4_rt; // Include MKS -> CGS conversion + rhoH(i, j, k) * rhoinv * 1.0e4; // Include MKS -> CGS conversion eos.HY2T(H, y, Tloc); // Get wdot. Don't like the temporary wdot there ... - rho = rho * 0.001_rt; // rho MKS -> CGS + rho = rho * 0.001; // rho MKS -> CGS amrex::Real wdot[NUM_SPECIES] = {0.0}; eos.RTY2WDOT(rho, Tloc, y, wdot); for (int n = 0; n < NUM_SPECIES; n++) { - rhoYdot(i, j, k, n) = wdot[n] * 1000.0_rt; // CGS -> MKS conversion + rhoYdot(i, j, k, n) = wdot[n] * 1000.0; // CGS -> MKS conversion } nEdot(i, j, k) = wdot[E_ID] * Na * 1.0e6; rhoYdot(i, j, k, E_ID) = 0.0; @@ -352,32 +339,31 @@ addLorentzForce( amrex::Array4 const& phiV, amrex::Array4 const& force) noexcept { - using namespace amrex::literals; // Electric field. Use 1-sided stencil in the force ghost cells. amrex::Real EFx; if (i == bxlo[0]) { - EFx = -1.0_rt / dx[0] * (phiV(i + 1, j, k) - phiV(i, j, k)); + EFx = -1.0 / dx[0] * (phiV(i + 1, j, k) - phiV(i, j, k)); } else if (i == bxhi[0]) { - EFx = -1.0_rt / dx[0] * (phiV(i, j, k) - phiV(i - 1, j, k)); + EFx = -1.0 / dx[0] * (phiV(i, j, k) - phiV(i - 1, j, k)); } else { - EFx = -0.5_rt / dx[0] * (phiV(i + 1, j, k) - phiV(i - 1, j, k)); + EFx = -0.5 / dx[0] * (phiV(i + 1, j, k) - phiV(i - 1, j, k)); } amrex::Real EFy; if (j == bxlo[1]) { - EFy = -1.0_rt / dx[1] * (phiV(i, j + 1, k) - phiV(i, j, k)); + EFy = -1.0 / dx[1] * (phiV(i, j + 1, k) - phiV(i, j, k)); } else if (j == bxhi[1]) { - EFy = -1.0_rt / dx[1] * (phiV(i, j, k) - phiV(i, j - 1, k)); + EFy = -1.0 / dx[1] * (phiV(i, j, k) - phiV(i, j - 1, k)); } else { - EFy = -0.5_rt / dx[1] * (phiV(i, j + 1, k) - phiV(i, j - 1, k)); + EFy = -0.5 / dx[1] * (phiV(i, j + 1, k) - phiV(i, j - 1, k)); } #if (AMREX_SPACEDIM == 3) amrex::Real EFz; if (k == bxlo[2]) { - EFz = -1.0_rt / dx[2] * (phiV(i, j, k + 1) - phiV(i, j, k)); + EFz = -1.0 / dx[2] * (phiV(i, j, k + 1) - phiV(i, j, k)); } else if (k == bxhi[2]) { - EFz = -1.0_rt / dx[2] * (phiV(i, j, k) - phiV(i, j, k - 1)); + EFz = -1.0 / dx[2] * (phiV(i, j, k) - phiV(i, j, k - 1)); } else { - EFz = -0.5_rt / dx[2] * (phiV(i, j, k + 1) - phiV(i, j, k - 1)); + EFz = -0.5 / dx[2] * (phiV(i, j, k + 1) - phiV(i, j, k - 1)); } #endif for (int n = 0; n < NUM_SPECIES; n++) { diff --git a/Source/main.cpp b/Source/main.cpp index 7f26308c1..b06bdeb93 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -6,13 +6,9 @@ #include #endif -using namespace amrex; - int main(int argc, char* argv[]) { - - // check to see if it contains --describe if (argc >= 2) { for (auto i = 1; i < argc; i++) { if (std::string(argv[i]) == "--describe") { @@ -22,8 +18,7 @@ main(int argc, char* argv[]) } } - // in AMReX.cpp - Initialize(argc, argv); + amrex::Initialize(argc, argv); // Defined and initialized when in gnumake, but not defined in cmake and // initialization done manually @@ -31,53 +26,36 @@ main(int argc, char* argv[]) amrex::sundials::Initialize(amrex::OpenMP::get_max_threads()); #endif - // Refuse to continue if we did not provide an inputs file. if (argc <= 1) { - Abort("Error: no inputs file provided on command line."); + amrex::Abort("Error: no inputs file provided on command line."); } - // timer for profiling BL_PROFILE_VAR("PeleLMeX::main()", main); - // wallclock time - const Real strt_total = ParallelDescriptor::second(); + const amrex::Real strt_total = amrex::ParallelDescriptor::second(); { - // declare an PeleLMeX object to manage multilevel data PeleLM pelelmex; - - // Description Setup pelelmex.Setup(); - - // Description Init pelelmex.Init(); - // Switch between Evolve and UnitTest mode if (pelelmex.runMode() == "normal") { - - // Advance solution to final time pelelmex.Evolve(); - } else if (pelelmex.runMode() == "evaluate") { - - // pelelmex.Evaluate(); - } else { - Abort( + amrex::Abort( " Wrong peleLM.run_mode ! It can only be 'normal' (D) or 'evaluate'"); } - // wallclock time - Real end_total = ParallelDescriptor::second() - strt_total; + amrex::Real end_total = amrex::ParallelDescriptor::second() - strt_total; + + amrex::ParallelDescriptor::ReduceRealMax( + end_total, amrex::ParallelDescriptor::IOProcessorNumber()); - // print wallclock time - ParallelDescriptor::ReduceRealMax( - end_total, ParallelDescriptor::IOProcessorNumber()); - Print() << "\nTotal Time: " << end_total << '\n'; + amrex::Print() << "\nTotal Time: " << end_total << '\n'; } - // destroy timer for profiling BL_PROFILE_VAR_STOP(main); // Defined and finalized when in gnumake, but not defined in cmake and @@ -86,6 +64,5 @@ main(int argc, char* argv[]) amrex::sundials::Finalize(); #endif - // in AMReX.cpp - Finalize(); + amrex::Finalize(); } diff --git a/Submodules/PelePhysics b/Submodules/PelePhysics index 7b7fb9144..80155c899 160000 --- a/Submodules/PelePhysics +++ b/Submodules/PelePhysics @@ -1 +1 @@ -Subproject commit 7b7fb91445313bfcb6949cacb7f2fe02e05da395 +Subproject commit 80155c899bdc5c2fddca38615d4790b84d746dcb