diff --git a/.clang-tidy b/.clang-tidy index fe56c6aff..d357cce9c 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -7,6 +7,7 @@ Checks: 'bugprone-*, clang-diagnostic-*, clang-analyzer-*, corecppguidelines-*, + cppcoreguidelines-avoid-capture-default, modernize-*, -modernize-avoid-c-arrays, -modernize-use-trailing-return-type, diff --git a/Exec/RegTests/EB_BackwardStepFlame/eb_bfs.inp b/Exec/RegTests/EB_BackwardStepFlame/eb_bfs.inp index d9dc7cbe9..0aa340ca3 100644 --- a/Exec/RegTests/EB_BackwardStepFlame/eb_bfs.inp +++ b/Exec/RegTests/EB_BackwardStepFlame/eb_bfs.inp @@ -29,6 +29,9 @@ prob.T_wall = 300 prob.meanFlowMag = 10.0 prob.Y_fuel = 0.0445269 prob.Y_oxid = 0.2226345 +mac_proj.atol = 5e-11 +mac_proj.rtol = 5e-11 +mac_proj.verbose = 2 #---------------------- PeleLM CONTROL --------------------------- peleLM.v = 1 # PeleLMeX verbose peleLM.use_wbar = 1 # Include Wbar term in species diffusion fluxes diff --git a/Exec/RegTests/EB_BackwardStepFlame/eb_bfs_pp.inp b/Exec/RegTests/EB_BackwardStepFlame/eb_bfs_pp.inp index f47cb0cad..a5a8358b9 100644 --- a/Exec/RegTests/EB_BackwardStepFlame/eb_bfs_pp.inp +++ b/Exec/RegTests/EB_BackwardStepFlame/eb_bfs_pp.inp @@ -31,7 +31,9 @@ prob.Y_fuel = 0.0 prob.Y_oxid = 0.23 prob.EBinflow_Yfuel = 0.1 peleLM.EBinflow = 1 - +mac_proj.atol = 5e-11 +mac_proj.rtol = 5e-11 +mac_proj.verbose = 2 #---------------------- PeleLM CONTROL --------------------------- peleLM.v = 1 # PeleLMeX verbose peleLM.use_wbar = 1 # Include Wbar term in species diffusion fluxes diff --git a/Source/PeleLMeX.H b/Source/PeleLMeX.H index 3614ed54b..e0718e168 100644 --- a/Source/PeleLMeX.H +++ b/Source/PeleLMeX.H @@ -57,7 +57,7 @@ class PeleLM : public amrex::AmrCore { public: - enum TimeStamp { AmrOldTime, AmrHalfTime, AmrNewTime }; + enum TimeStamp : std::uint8_t { AmrOldTime, AmrHalfTime, AmrNewTime }; // constructor PeleLM(); @@ -78,34 +78,40 @@ public: void Evaluate(); // Time advance function - void Advance(int is_init); + void Advance(const int is_init); //----------------------------------------------------------------------------- // Virtual AmrCore functions - void regrid(int lbase, amrex::Real time, bool initial = false) override; + void regrid( + const int lbase, + const amrex::Real time, + const bool initial = false) override; void MakeNewLevelFromScratch( - int lev, - amrex::Real time, + const int lev, + const amrex::Real time, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm) override; void ErrorEst( - int lev, amrex::TagBoxArray& tags, amrex::Real time, int ng) override; + const int lev, + amrex::TagBoxArray& tags, + const amrex::Real time, + const int ng) override; void MakeNewLevelFromCoarse( - int lev, - amrex::Real time, + const int lev, + const amrex::Real time, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm) override; void RemakeLevel( - int lev, - amrex::Real time, + const int lev, + const amrex::Real time, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm) override; - void ClearLevel(int lev) override; + void ClearLevel(const int lev) override; //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- @@ -122,8 +128,8 @@ public: void freeProbParm(); // ReadGridFile - void - readGridFile(std::string grid_file, amrex::Vector& input_ba); + void readGridFile( + const std::string& grid_file, amrex::Vector& input_ba); #ifdef PELE_USE_SPRAY // ReadSprayParameters @@ -166,12 +172,12 @@ public: amrex::BoxArray const& ba, amrex::DistributionMapping const& dm, amrex::FabFactory const& factory, - int a_incompressible, - int a_has_divu, - int a_nAux, - int a_nGrowState, - int a_use_soret, - int a_do_les); + const int a_incompressible, + const int a_has_divu, + const int a_nAux, + const int a_nGrowState, + const int a_use_soret, + const int a_do_les); // cell-centered state multifabs amrex::MultiFab @@ -225,16 +231,16 @@ public: { AdvanceDiffData() = default; AdvanceDiffData( - int a_finestLevel, + const int a_finestLevel, const amrex::Vector& ba, const amrex::Vector& dm, const amrex::Vector>>& factory, - int a_nGrowAdv, - int a_use_wbar, - int a_use_soret, - int a_nAux, - int is_init = 0); + const int a_nGrowAdv, + const int a_use_wbar, + const int a_use_soret, + const int a_nAux, + const int is_init = 0); amrex::Vector Dn; // Diffusion term t^n amrex::Vector Dnp1; // Diffusion term t^(n+1,k) amrex::Vector Dhat; // Diffusion term t^(n+1,k+1) @@ -259,15 +265,15 @@ public: { AdvanceAdvData() = default; AdvanceAdvData( - int a_finestLevel, + const int a_finestLevel, const amrex::Vector& ba, const amrex::Vector& dm, const amrex::Vector>>& factory, - int a_incompressible, - int a_nAux, - int a_nGrowAdv, - int a_nGrowMAC); + const int a_incompressible, + const int a_nAux, + const int a_nGrowAdv, + const int a_nGrowMAC); amrex::Vector> umac; // MAC face velocity amrex::Vector AofS; // Scalar advection term at t^(n+1/2) @@ -300,14 +306,14 @@ public: * using the user defined init function * \param lev the target level index */ - void initLevelData(int lev); + void initLevelData(const int lev); /** * \brief Fill the initial level data container * using the data from a pltfile * \param a_pltFile path to PeleLMeX plot file */ - void initLevelDataFromPlt(int a_lev, const std::string& a_dataPltFile); + void initLevelDataFromPlt(const int a_lev, const std::string& a_dataPltFile); /** * \brief Adds the velocity level data to state @@ -330,10 +336,11 @@ public: void initialIterations(); // initFromGridFile - void InitFromGridFile(amrex::Real time); + void InitFromGridFile(const amrex::Real time); // regridFromFile - void regridFromGridFile(int lbase, amrex::Real time, bool initial); + void regridFromGridFile( + const int lbase, const amrex::Real time, const bool initial); /** * \brief Check the consistency of the run @@ -350,7 +357,7 @@ public: * filling the PeleLM class LayoutData with default method * \param a_lev target level */ - void computeCosts(int a_lev); + void computeCosts(const int a_lev); /** * \brief Compute load balancing cost estimate on a given level @@ -359,7 +366,7 @@ public: * \param a_costMethod cost method */ void computeCosts( - int a_lev, amrex::LayoutData& a_costs, int a_costMethod); + const int a_lev, amrex::LayoutData& a_costs, int a_costMethod); /** * \brief Create/update the DMap used for chemistry on all levels @@ -370,7 +377,7 @@ public: * \brief Create/update the DMap used for chemistry on a given level * \param a_lev level of interest */ - void loadBalanceChemLev(int a_lev); + void loadBalanceChemLev(const int a_lev); //----------------------------------------------------------------------------- @@ -399,7 +406,9 @@ public: * \param a_dt time time step size */ void velocityProjection( - int is_init, const PeleLM::TimeStamp& a_rhoTime, const amrex::Real& a_dt); + const int is_init, + const PeleLM::TimeStamp a_rhoTime, + const amrex::Real a_dt); /** * \brief Actual nodal projection call function. @@ -415,23 +424,24 @@ public: const amrex::Vector& a_sigma, const amrex::Vector& rhs_cc, const amrex::Vector& rhs_nd, - int incremental, - amrex::Real scaling_factor); - + const int incremental, + const amrex::Real scaling_factor); +#if AMREX_SPACEDIM == 2 /** * \brief For 2D-RZ, scale multifab components by radius * including 1 ghost cell on Dirichlet BC * \param a_lev level index * \param a_mf MultiFab to act upon */ - void scaleProj_RZ(int a_lev, amrex::MultiFab& a_mf); + void scaleProj_RZ(const int a_lev, amrex::MultiFab& a_mf); /** * \brief For 2D-RZ, unscale multifab components by radius * \param a_lev level index * \param a_mf MultiFab to act upon */ - void unscaleProj_RZ(int a_lev, amrex::MultiFab& a_mf); + void unscaleProj_RZ(const int a_lev, amrex::MultiFab& a_mf); +#endif //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- @@ -448,14 +458,14 @@ public: * from cell-centered ones * \param advData container for the face-centered velocity umac */ - void predictVelocity(std::unique_ptr& advData); + void predictVelocity(const std::unique_ptr& advData); /** * \brief Create MAC projection RHS by fillpatching divU * between old and new time * \param advData container for mac_divu */ - void createMACRHS(std::unique_ptr& advData); + void createMACRHS(const std::unique_ptr& advData); /** * \brief Create MAC projection RHS by fillpatching divU @@ -465,9 +475,9 @@ public: * \param advData container for chi and incremented mac_divu */ void addChiIncrement( - int a_sdcIter, - const PeleLM::TimeStamp& a_time, - std::unique_ptr& advData); + const int a_sdcIter, + const PeleLM::TimeStamp a_time, + const std::unique_ptr& advData); /** * \brief Perform MAC projection to obtained divergence constrained @@ -477,8 +487,8 @@ public: * \param a_divu velocity divergence constraint (can be empty) */ void macProject( - const PeleLM::TimeStamp& a_time, - std::unique_ptr& advData, + const PeleLM::TimeStamp a_time, + const std::unique_ptr& advData, const amrex::Vector& a_divu); /** @@ -495,11 +505,11 @@ public: * crse_ratio refinement ratio with the next coarser level */ void create_constrained_umac_grown( - int a_nGrow, + const int a_nGrow, const amrex::Geometry* crse_geom, const amrex::Geometry* fine_geom, - amrex::Array u_mac_crse, - amrex::Array u_mac_fine, + const amrex::Array& u_mac_crse, + const amrex::Array& u_mac_fine, const amrex::IntVect& crse_ratio); /** @@ -519,21 +529,21 @@ public: * filling the specified container of Old or New LevelData * \param a_time either Old or New stamp */ - void calcViscosity(const PeleLM::TimeStamp& a_time); + void calcViscosity(const PeleLM::TimeStamp a_time); /** * \brief Compute the face-centered turbulent viscosity on all levels * filling the specified container of Old or New LevelData * \param a_time either Old or New stamp */ - void calcTurbViscosity(const PeleLM::TimeStamp& a_time); + void calcTurbViscosity(const PeleLM::TimeStamp a_time); /** * \brief Compute the cell-centereds diffusivity on all levels * filling the specified container of Old or New LevelData * \param a_time either Old or New stamp */ - void calcDiffusivity(const PeleLM::TimeStamp& a_time); + void calcDiffusivity(const PeleLM::TimeStamp a_time); // get edge-centered diffusivity on a per level / per comp basis /** @@ -550,13 +560,13 @@ public: * \param addTurbContrib turbulent contribution increment trigger */ amrex::Array getDiffusivity( - int lev, - int beta_comp, - int ncomp, - int doZeroVisc, - amrex::Vector bcrec, + const int lev, + const int beta_comp, + const int ncomp, + const int doZeroVisc, + const amrex::Vector& bcrec, const amrex::MultiFab& beta_cc, - int addTurbContrib = 0); + const int addTurbContrib = 0); /** * \brief Compute the explicit face-centered diffusion fluxes on all levels @@ -569,7 +579,7 @@ public: * container */ void computeDifferentialDiffusionFluxes( - const PeleLM::TimeStamp& a_time, + const PeleLM::TimeStamp a_time, const amrex::Vector>& a_fluxes, const amrex::Vector& a_EBfluxes, @@ -581,7 +591,7 @@ public: a_auxfluxes); void correctIsothermalBoundary( - const TimeStamp& a_time, + const PeleLM::TimeStamp a_time, const amrex::Vector& a_spec_boundary, const amrex::Vector>& a_wbarfluxes, @@ -597,9 +607,9 @@ public: * \param is_init initial computation flag */ void computeDifferentialDiffusionTerms( - const PeleLM::TimeStamp& a_time, - std::unique_ptr& diffData, - int is_init = 0); + const PeleLM::TimeStamp a_time, + const std::unique_ptr& diffData, + const int is_init = 0); /** * \brief Add the Wbar contribution to the face-centered species diffusion @@ -666,8 +676,8 @@ public: * \param diffData container for the returning implicit diffusion term */ void differentialDiffusionUpdate( - std::unique_ptr& advData, - std::unique_ptr& diffData); + const std::unique_ptr& advData, + const std::unique_ptr& diffData); /** * \brief Setup the implicit deltaT linear solve @@ -681,8 +691,8 @@ public: const amrex::Vector& a_rhs, const amrex::Vector& a_Tsave, const amrex::Vector& a_rhoCp, - std::unique_ptr& advData, - std::unique_ptr& diffData); + const std::unique_ptr& advData, + const std::unique_ptr& diffData); /** * \brief Restore state, update enthalpy and compute fluxes after deltaT @@ -694,12 +704,12 @@ public: * \param a_deltaT_norm norm of deltaT residual */ void deltaTIter_update( - int a_dtiter, + const int a_dtiter, const amrex::Vector>& a_fluxes, const amrex::Vector& a_ebfluxes, const amrex::Vector& a_Tsave, - std::unique_ptr& diffData, + const std::unique_ptr& diffData, amrex::Real& a_deltaT_norm); /** @@ -711,10 +721,10 @@ public: * scheme property (e.g. 0.5 for Crank Nicholson) */ void computeDivTau( - const PeleLM::TimeStamp& a_time, + const PeleLM::TimeStamp a_time, const amrex::Vector& a_divtau, - int use_density, - amrex::Real scale = 1.0); + const int use_density, + const amrex::Real scale = 1.0); /** * \brief Implicit diffusion of the of velocity field using AMReX @@ -737,7 +747,8 @@ public: */ amrex::Vector> getDiffusionTensorOpBC( - amrex::Orientation::Side a_side, const amrex::Vector a_bc); + const amrex::Orientation::Side a_side, + const amrex::Vector& a_bc); //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- @@ -749,7 +760,7 @@ public: * the advection/diffusion component of the advance function in the SDC * context */ - void advanceChemistry(std::unique_ptr& advData); + void advanceChemistry(const std::unique_ptr& advData); /** * \brief Performing the chemistry integration on a given level, for a fixed @@ -758,7 +769,7 @@ public: * a_extForcing advection/diffusion forcing */ void advanceChemistry( - int lev, const amrex::Real& a_dt, amrex::MultiFab& a_extForcing); + const int lev, const amrex::Real a_dt, amrex::MultiFab& a_extForcing); /** * \brief Performing the chemistry integration on a given level, for a fixed @@ -769,7 +780,7 @@ public: * \param a_extForcing advection/diffusion forcing */ void advanceChemistryBAChem( - int lev, const amrex::Real& a_dt, amrex::MultiFab& a_extForcing); + const int lev, const amrex::Real a_dt, amrex::MultiFab& a_extForcing); /** * \brief Top-level instantaneous reaction rate function, acting on all levels @@ -778,7 +789,7 @@ public: */ void computeInstantaneousReactionRate( const amrex::Vector& a_I_R, - const PeleLM::TimeStamp& a_time); + const PeleLM::TimeStamp a_time); /** * \brief Compute instantaneous reaction rate on a given level @@ -787,7 +798,7 @@ public: * \param a_I_R outgoing MultiFab for inst. RR container */ void computeInstantaneousReactionRate( - int lev, const PeleLM::TimeStamp& a_time, amrex::MultiFab* a_I_R); + const int lev, const PeleLM::TimeStamp a_time, amrex::MultiFab* a_I_R); /** * \brief Compute the heat release rate on a given level, using the @@ -795,7 +806,7 @@ public: * \param lev level of interest * \param a_HR outgoing MultiFab */ - void getHeatRelease(int lev, amrex::MultiFab* a_HR); + void getHeatRelease(const int lev, amrex::MultiFab* a_HR); //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- @@ -806,38 +817,38 @@ public: * \param is_init flag to trigger init-only dt factors * \param a_time state time to work on, either Old or New */ - amrex::Real computeDt(int is_init, const PeleLM::TimeStamp& a_time); + amrex::Real computeDt(const int is_init, const PeleLM::TimeStamp a_time); /** * \brief Compute the CFL step size * \param a_time state time to work on, either Old or New */ - amrex::Real estConvectiveDt(const PeleLM::TimeStamp& a_time); + amrex::Real estConvectiveDt(const PeleLM::TimeStamp a_time); /** * \brief Compute the divU step size * \param a_time state time to work on, either Old or New */ - amrex::Real estDivUDt(const PeleLM::TimeStamp& a_time); - void checkDt(const PeleLM::TimeStamp& a_time, const amrex::Real& a_dt); + amrex::Real estDivUDt(const PeleLM::TimeStamp a_time); + void checkDt(const PeleLM::TimeStamp a_time, const amrex::Real a_dt); //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- // FORCES void getVelForces( - const TimeStamp& a_time, + const PeleLM::TimeStamp a_time, const amrex::Vector& a_divTau, const amrex::Vector& a_velForce, - int nGrowForce, - int add_gradP); + const int nGrowForce, + const int add_gradP); void getVelForces( - const TimeStamp& a_time, - int lev, + const PeleLM::TimeStamp a_time, + const int lev, amrex::MultiFab* a_divTau, amrex::MultiFab* a_velForce, - int add_gradP); + const int add_gradP); void getVelForces( int lev, @@ -852,9 +863,9 @@ public: amrex::Array4 const& extMom, amrex::Array4 const& extRho); - void addSpark(const PeleLM::TimeStamp& a_time); + void addSpark(const PeleLM::TimeStamp a_time); - void addScalarVarianceSources(const PeleLM::TimeStamp& a_time); + void addScalarVarianceSources(const PeleLM::TimeStamp a_time); //----------------------------------------------------------------------------- @@ -898,7 +909,7 @@ public: // Compute the soot source terms void computeSootSource( - const PeleLM::TimeStamp& a_timestamp, const amrex::Real a_dt); + const PeleLM::TimeStamp a_timestamp, const amrex::Real a_dt); // Clip soot moments void clipSootMoments(); @@ -911,7 +922,7 @@ public: void RadInit(); // Compute the radiation source term - void computeRadSource(const PeleLM::TimeStamp& a_timestamp); + void computeRadSource(const PeleLM::TimeStamp a_timestamp); #endif //----------------------------------------------------------------------------- @@ -921,9 +932,9 @@ public: bool m_ext_sources_SDC; bool m_add_variance_sources; void getExternalSources( - int is_initIter, - const PeleLM::TimeStamp& a_timestamp_old, - const PeleLM::TimeStamp& a_timestamp_new); + const int is_initIter, + const PeleLM::TimeStamp a_timestamp_old, + const PeleLM::TimeStamp a_timestamp_new); //----------------------------------------------------------------------------- // EOS @@ -932,34 +943,34 @@ public: * \brief Compute thermodynamic pressure (RhoRT) from state on all levels * \param a_time state time to work on, either Old or New */ - void setThermoPress(const PeleLM::TimeStamp& a_time); + void setThermoPress(const PeleLM::TimeStamp a_time); /** * \brief Compute thermodynamic pressure (RhoRT) from state on a single level * \param lev level of interest * \param a_time state time to work on, either Old or New */ - void setThermoPress(int lev, const PeleLM::TimeStamp& a_time); + void setThermoPress(const int lev, const PeleLM::TimeStamp a_time); /** * \brief Enforce rho = \sum rhoYs on a given level * \param lev level of interest * \param a_time state time to work on, either Old or New */ - void setRhoToSumRhoY(int lev, const PeleLM::TimeStamp& a_time); + void setRhoToSumRhoY(const int lev, const PeleLM::TimeStamp a_time); /** * \brief Compute temperature from rhoY and rhoH on all levels * \param a_time state time to work on, either Old or New */ - void setTemperature(const PeleLM::TimeStamp& a_time); + void setTemperature(const PeleLM::TimeStamp a_time); /** * \brief Compute temperature from rhoY and rhoH on a given level * \param lev level of interest * \param a_time state time to work on, either Old or New */ - void setTemperature(int lev, const PeleLM::TimeStamp& a_time); + void setTemperature(const int lev, const PeleLM::TimeStamp a_time); /** * \brief Compute the divergence constraint from reaction/diffusion @@ -971,11 +982,11 @@ public: * \param diffData container for the diffusion terms */ void calcDivU( - int is_init, - int computeDiff, - int do_avgDown, - const PeleLM::TimeStamp& a_time, - std::unique_ptr& diffData); + const int is_init, + const int computeDiff, + const int do_avgDown, + const PeleLM::TimeStamp a_time, + const std::unique_ptr& diffData); /** * \brief Compute the pressure drift term using RhoRT and @@ -984,7 +995,7 @@ public: * \param a_dPdt outgoing container */ void calc_dPdt( - const PeleLM::TimeStamp& a_time, + const PeleLM::TimeStamp a_time, const amrex::Vector& a_dPdt); /** @@ -994,8 +1005,8 @@ public: * \param a_time state time to work on, either Old or New * \param a_dPdt outgoing container */ - void - calc_dPdt(int lev, const PeleLM::TimeStamp& a_time, amrex::MultiFab* a_dPdt); + void calc_dPdt( + const int lev, const PeleLM::TimeStamp a_time, amrex::MultiFab* a_dPdt); /** * \brief For the closed chamber algorithm, compute averaged @@ -1003,7 +1014,7 @@ public: * thermodynamic pressure. Return the mean divU. * \param advData container including mac_divu */ - amrex::Real adjustPandDivU(std::unique_ptr& advData); + amrex::Real adjustPandDivU(const std::unique_ptr& advData); //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- @@ -1015,7 +1026,7 @@ public: void ReadCheckPointFile(); bool writeCheckNow() const; void WriteJobInfo(const std::string& path) const; - void WriteHeader(const std::string& name, bool is_checkpoint) const; + void WriteHeader(const std::string& name, const bool is_checkpoint) const; void WriteDebugPlotFile( const amrex::Vector& a_MF, const std::string& pltname); @@ -1034,10 +1045,10 @@ public: void massBalance(); void speciesBalance(); void speciesBalancePatch(); - void initBPatches(amrex::Geometry& a_geom); + void initBPatches(const amrex::Geometry& a_geom); void rhoHBalance(); - void initTemporals(const PeleLM::TimeStamp& a_time = AmrOldTime); + void initTemporals(const PeleLM::TimeStamp a_time = AmrOldTime); void writeTemporals(); void addMassFluxes( const amrex::Array& a_fluxes, @@ -1054,7 +1065,8 @@ public: const amrex::Geometry& a_geom, const amrex::Real& a_factor = 1.0); void addUmacFluxes( - std::unique_ptr& advData, const amrex::Geometry& a_geom); + const std::unique_ptr& advData, + const amrex::Geometry& a_geom); void openTempFile(); void closeTempFile(); bool doTemporalsNow() const; @@ -1062,7 +1074,7 @@ public: //----------------------------------------------------------------------------- // BOUNDARY CONDITIONS / FILLPATCH - amrex::InterpBase* getInterpolator(int a_method = 1) const; + amrex::InterpBase* getInterpolator(const int a_method = 1) const; // Interpolation method for regrid (default to cell cons interp) int m_regrid_interp_method = 1; @@ -1071,113 +1083,152 @@ public: void setBoundaryConditions(); // FillPatch the entire class state - void fillPatchState(const PeleLM::TimeStamp& a_time); - void fillPatchState(int lev, const PeleLM::TimeStamp& a_time); + void fillPatchState(const PeleLM::TimeStamp a_time); + void fillPatchState(const int lev, const PeleLM::TimeStamp a_time); // FillPatch the various state components - void fillPatchDensity(const PeleLM::TimeStamp& a_time); - void fillPatchSpecies(const PeleLM::TimeStamp& a_time); - void fillPatchTemp(const PeleLM::TimeStamp& a_time); - void fillPatchAux(const PeleLM::TimeStamp& a_time); + void fillPatchDensity(const PeleLM::TimeStamp a_time); + void fillPatchSpecies(const PeleLM::TimeStamp a_time); + void fillPatchTemp(const PeleLM::TimeStamp a_time); + void fillPatchAux(const PeleLM::TimeStamp a_time); #ifdef PELE_USE_PLASMA - void fillPatchPhiV(const PeleLM::TimeStamp& a_time); + void fillPatchPhiV(const PeleLM::TimeStamp a_time); #endif // FillPatch and return a MF std::unique_ptr - fillPatchState(int lev, amrex::Real a_time, int nGrow); + fillPatchState(const int lev, const amrex::Real a_time, const int nGrow); std::unique_ptr - fillPatchReact(int lev, amrex::Real a_time, int nGrow); + fillPatchReact(const int lev, const amrex::Real a_time, const int nGrow); // FillPatch state components void fillpatch_state( - int lev, amrex::Real a_time, amrex::MultiFab& a_state, int nGhost); + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_state, + const int nGhost); void fillpatch_density( - int lev, - amrex::Real a_time, + const int lev, + const amrex::Real a_time, amrex::MultiFab& a_density, - int rho_comp, - int nGhost); + const int rho_comp, + const int nGhost); void fillpatch_species( - int lev, - amrex::Real a_time, + const int lev, + const amrex::Real a_time, amrex::MultiFab& a_species, - int rhoY_comp, - int nGhost); + const int rhoY_comp, + const int nGhost); void fillpatch_temp( - int lev, - amrex::Real a_time, + const int lev, + const amrex::Real a_time, amrex::MultiFab& a_temp, - int temp_comp, - int nGhost); + const int temp_comp, + const int nGhost); void fillpatch_aux( - int lev, amrex::Real a_time, amrex::MultiFab& a_aux, int nGhost); + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_aux, + const int nGhost); void fillpatch_divu( - int lev, amrex::Real a_time, amrex::MultiFab& a_divu, int nGhost); + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_divu, + const int nGhost); void fillpatch_gradp( - int lev, amrex::Real a_time, amrex::MultiFab& a_gp, int nGhost); + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_gp, + const int nGhost); void fillpatch_reaction( - int lev, amrex::Real a_time, amrex::MultiFab& a_I_R, int nGhost); + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_I_R, + const int nGhost); void fillpatch_forces( - amrex::Real a_time, + const amrex::Real a_time, amrex::Vector const& a_force, - int nGrowForce); + const int nGrowForce); void fillpatch_chemFunctCall( - int lev, amrex::Real a_time, amrex::MultiFab& a_fctC, int nGhost); + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_fctC, + const int nGhost); #ifdef PELE_USE_PLASMA void fillpatch_phiV( - int lev, - amrex::Real a_time, + const int lev, + const amrex::Real a_time, amrex::MultiFab& a_phiV, - int phiV_comp, - int nGhost); + const int phiV_comp, + const int nGhost); void fillPatchExtrap( - amrex::Real a_time, + const amrex::Real a_time, amrex::Vector const& a_MF, - int a_nGrow); + const int a_nGrow); void fillPatchNLnE( - amrex::Real a_time, + const amrex::Real a_time, amrex::Vector const& a_nE, - int a_nGrow); + const int a_nGrow); void fillPatchNLphiV( - amrex::Real a_time, + const amrex::Real a_time, amrex::Vector const& a_phiV, - int a_nGrow); + const int a_nGrow); #endif // FillCoarsePatch state components void fillcoarsepatch_state( - int lev, amrex::Real a_time, amrex::MultiFab& a_state, int nGhost); + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_state, + const int nGhost); void fillcoarsepatch_aux( - int lev, amrex::Real a_time, amrex::MultiFab& a_aux, int nGhost); + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_aux, + const int nGhost); void fillcoarsepatch_divu( - int lev, amrex::Real a_time, amrex::MultiFab& a_divu, int nGhost); + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_divu, + const int nGhost); void fillcoarsepatch_gradp( - int lev, amrex::Real a_time, amrex::MultiFab& a_gp, int nGhost); + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_gp, + const int nGhost); void fillcoarsepatch_reaction( - int lev, amrex::Real a_time, amrex::MultiFab& a_I_R, int nGhost); + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_I_R, + const int nGhost); void fillcoarsepatch_chemFunctCall( - int lev, amrex::Real a_time, amrex::MultiFab& a_fctC, int nGhost); + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_fctC, + const int nGhost); // Fill physical boundaries void setInflowBoundaryVel( - amrex::MultiFab& a_vel, int lev, PeleLM::TimeStamp a_time); + amrex::MultiFab& a_vel, const int lev, const PeleLM::TimeStamp a_time); void fillTurbInflow( - amrex::MultiFab& a_vel, int vel_comp, int lev, const amrex::Real a_time); + amrex::MultiFab& a_vel, + const int vel_comp, + const int lev, + const amrex::Real a_time); // Average down operations - void averageDownState(const PeleLM::TimeStamp& a_time); - void averageDownScalars(const PeleLM::TimeStamp& a_time); - void averageDownVelocity(const PeleLM::TimeStamp& a_time); - void averageDownAux(const PeleLM::TimeStamp& a_time); + void averageDownState(const PeleLM::TimeStamp a_time); + void averageDownScalars(const PeleLM::TimeStamp a_time); + void averageDownVelocity(const PeleLM::TimeStamp a_time); + void averageDownAux(const PeleLM::TimeStamp a_time); void averageDown( - const PeleLM::TimeStamp& a_time, const int state_comp, const int ncomp); + const PeleLM::TimeStamp a_time, const int state_comp, const int ncomp); void averageDownReaction(); #ifdef PELE_USE_PLASMA - void averageDownnE(const PeleLM::TimeStamp& a_time); - void averageDownPhiV(const PeleLM::TimeStamp& a_time); + void averageDownnE(const PeleLM::TimeStamp a_time); + void averageDownPhiV(const PeleLM::TimeStamp a_time); #endif //----------------------------------------------------------------------------- @@ -1236,144 +1287,158 @@ public: // flux divergence void fluxDivergence( const amrex::Vector& a_divergence, - int div_comp, + const int div_comp, const amrex::Vector>& a_fluxes, - int flux_comp, - int ncomp, - int intensiveFluxes, - amrex::Real scale); + const int flux_comp, + const int ncomp, + const int intensiveFluxes, + const amrex::Real scale); void fluxDivergence( const amrex::Vector& a_divergence, - int div_comp, + const int div_comp, const amrex::Vector>& a_fluxes, - int flux_comp, + const int flux_comp, const amrex::Vector& a_EBfluxes, - int ebflux_comp, - int ncomp, - int intensiveFluxes, - amrex::Real scale); + const int ebflux_comp, + const int ncomp, + const int intensiveFluxes, + const amrex::Real scale); void fluxDivergenceRD( const amrex::Vector& a_state, - int state_comp, + const int state_comp, const amrex::Vector& a_divergence, - int div_comp, + const int div_comp, const amrex::Vector>& a_fluxes, - int flux_comp, + const int flux_comp, const amrex::Vector& a_EBfluxes, - int ebflux_comp, - int ncomp, - int intensiveFluxes, + const int ebflux_comp, + const int ncomp, + const int intensiveFluxes, const amrex::BCRec* state_bc_d, const amrex::Real& scale, const amrex::Real& a_dt); void advFluxDivergence( - int a_lev, + const int a_lev, amrex::MultiFab& a_divergence, - int div_comp, + const int div_comp, amrex::MultiFab& a_divu, const amrex::Array& a_fluxes, - int flux_comp, + const int flux_comp, const amrex::Array& a_faceState, - int face_comp, - int ncomp, + const int face_comp, + const int ncomp, int const* l_conserv_d, const amrex::Geometry& a_geom, - amrex::Real scale, - bool fluxes_are_area_weighted) const; + const amrex::Real scale, + const bool fluxes_are_area_weighted) const; #ifdef AMREX_USE_EB void advFluxDivergence( - int a_lev, + const int a_lev, amrex::MultiFab& a_divergence, - int div_comp, + const int div_comp, amrex::MultiFab& a_divu, const amrex::Array& a_fluxes, - int flux_comp, + const int flux_comp, const amrex::Array& a_faceState, - int face_comp, + const int face_comp, const amrex::MultiFab* a_EBvelocity, const amrex::MultiFab* a_EBvalue, - int ncomp, + const int ncomp, int const* l_conserv_d, const amrex::Geometry& a_geom, - amrex::Real scale, - bool fluxes_are_area_weighted) const; + const amrex::Real scale, + const bool fluxes_are_area_weighted) const; #endif void extFluxDivergenceLevel( - int lev, + const int lev, amrex::MultiFab& a_divergence, - int div_comp, + const int div_comp, const amrex::Array& a_fluxes, - int flux_comp, - int ncomp, - amrex::Real scale); + const int flux_comp, + const int ncomp, + const amrex::Real scale); void intFluxDivergenceLevel( - int lev, + const int lev, amrex::MultiFab& a_divergence, - int div_comp, + const int div_comp, const amrex::Array& a_fluxes, - int flux_comp, - int ncomp, - amrex::Real scale); + const int flux_comp, + const int ncomp, + const amrex::Real scale); void intFluxDivergenceLevelEB( - int lev, + const int lev, amrex::MultiFab& a_divergence, - int div_comp, + const int div_comp, const amrex::Array& a_fluxes, - int flux_comp, + const int flux_comp, const amrex::MultiFab* a_EBfluxes, - int ebflux_comp, - int ncomp, - amrex::Real scale); + const int ebflux_comp, + const int ncomp, + const amrex::Real scale); - void floorSpecies(const PeleLM::TimeStamp& a_time); + void floorSpecies(const PeleLM::TimeStamp a_time); amrex::Real MLNorm0(const amrex::Vector& a_MF); amrex::Vector MLNorm0( const amrex::Vector& a_MF, - int startcomp, - int ncomp); + const int startcomp, + const int ncomp); amrex::Real - MFSum(const amrex::Vector& a_MF, int comp); - static amrex::Real - MFmax(const amrex::MultiFab* a_MF, const amrex::iMultiFab& a_mask, int comp); - static amrex::Real - MFmin(const amrex::MultiFab* a_MF, const amrex::iMultiFab& a_mask, int comp); + MFSum(const amrex::Vector& a_MF, const int comp); + static amrex::Real MFmax( + const amrex::MultiFab* a_MF, + const amrex::iMultiFab& a_mask, + const int comp); + static amrex::Real MFmin( + const amrex::MultiFab* a_MF, + const amrex::iMultiFab& a_mask, + const int comp); amrex::Vector MLmax( - const amrex::Vector& a_MF, int scomp, int ncomp); + const amrex::Vector& a_MF, + const int scomp, + const int ncomp); amrex::Vector MLmin( - const amrex::Vector& a_MF, int scomp, int ncomp); + const amrex::Vector& a_MF, + const int scomp, + const int ncomp); void resetCoveredMask(); // Active control void initActiveControl(); - void activeControl(int is_restart); + void activeControl(const int is_restart); void loadActiveControlHistory(); void getActiveControlLowT(amrex::Real& a_coft); // Derived variable evaluation - std::unique_ptr - derive(const std::string& a_name, amrex::Real a_time, int lev, int nGrow); - std::unique_ptr - deriveComp(const std::string& a_name, amrex::Real a_time, int lev, int nGrow); + std::unique_ptr derive( + const std::string& a_name, + const amrex::Real a_time, + const int lev, + const int nGrow); + std::unique_ptr deriveComp( + const std::string& a_name, + const amrex::Real a_time, + const int lev, + const int nGrow); // Evaluate function void MLevaluate( const amrex::Vector& a_MFVec, - int a_comp, + const int a_comp, int& nComp, const std::string& a_var); @@ -1390,7 +1455,7 @@ public: std::string stateVariableName(int a_index); // Typical values - void setTypicalValues(const PeleLM::TimeStamp& a_time, int is_init = 0); + void setTypicalValues(const PeleLM::TimeStamp a_time, const int is_init = 0); void updateTypicalValuesChem(); void checkMemory(const std::string& a_message) const; @@ -1398,8 +1463,8 @@ public: // Mixture fraction & Progress variable void initMixtureFraction(); static void parseComposition( - amrex::Vector compositionIn, - std::string compositionType, + const amrex::Vector& compositionIn, + const std::string& compositionType, amrex::Real* massFrac); static void parseVars( const amrex::Vector& a_varsNames, @@ -1419,70 +1484,73 @@ public: void initialRedistribution(); void initCoveredState(); - void setCoveredState(const PeleLM::TimeStamp& a_time); - void setCoveredState(int lev, const PeleLM::TimeStamp& a_time); + void setCoveredState(const PeleLM::TimeStamp a_time); + void setCoveredState(const int lev, const PeleLM::TimeStamp a_time); void redistributeAofS( - int a_lev, - amrex::Real& a_dt, + const int a_lev, + amrex::Real a_dt, amrex::MultiFab& a_tmpDiv, - int div_comp, + const int div_comp, amrex::MultiFab& a_AofS, - int aofs_comp, + const int aofs_comp, amrex::MultiFab& a_state, - int state_comp, - int ncomp, + const int state_comp, + const int ncomp, const amrex::BCRec* d_bc, const amrex::Geometry& a_geom) const; void redistributeDiff( - int a_lev, - const amrex::Real& a_dt, + const int a_lev, + const amrex::Real a_dt, amrex::MultiFab& a_tmpDiv, - int div_comp, + const int div_comp, amrex::MultiFab& a_diff, - int diff_comp, + const int diff_comp, const amrex::MultiFab& a_state, - int state_comp, - int ncomp, + const int state_comp, + const int ncomp, const amrex::BCRec* d_bc, const amrex::Geometry& a_geom) const; - void getCoveredIMask(int a_lev, amrex::iMultiFab& a_imask) const; + void getCoveredIMask(const int a_lev, amrex::iMultiFab& a_imask) const; - void getEBDistance(int a_lev, amrex::MultiFab& a_signDistLev); + void getEBDistance(const int a_lev, amrex::MultiFab& a_signDistLev); - void - extendSignedDistance(amrex::MultiFab* a_signDist, amrex::Real a_extendFactor); + void extendSignedDistance( + amrex::MultiFab* a_signDist, const amrex::Real a_extendFactor); void getEBState( - int a_lev, - const PeleLM::TimeStamp& a_time, + const int a_lev, + const PeleLM::TimeStamp a_time, amrex::MultiFab& a_EBstate, - int stateComp, - int nComp); + const int stateComp, + const int nComp); - amrex::Vector> - getEBState(int first_comp, int ncomp, const PeleLM::TimeStamp& a_time); + amrex::Vector> getEBState( + const int first_comp, const int ncomp, const PeleLM::TimeStamp a_time); std::unique_ptr getEBState( - int a_lev, int first_comp, int ncomp, const PeleLM::TimeStamp& a_time); + const int a_lev, + const int first_comp, + const int ncomp, + const PeleLM::TimeStamp a_time); amrex::FArrayBox getEBState( const amrex::MFIter& a_mfi, - int a_lev, - int first_comp, - int ncomp, - const PeleLM::TimeStamp& a_time); + const int a_lev, + const int first_comp, + const int ncomp, + const PeleLM::TimeStamp a_time); void getEBDiff( - int a_lev, - const TimeStamp& a_time, + const int a_lev, + const TimeStamp a_time, amrex::MultiFab& a_EBDiff, - int diffComp); + const int diffComp); // static_cast factory into ebfactory - amrex::EBFArrayBoxFactory const& EBFactory(int lev) const noexcept + amrex::EBFArrayBoxFactory const& EBFactory(const int lev) const noexcept { return static_cast(*m_factory[lev]); } @@ -1501,35 +1569,39 @@ public: void advanceSetup(); void oneSDC( - int sdcIter, - std::unique_ptr& advData, - std::unique_ptr& diffData); + const int sdcIter, + const std::unique_ptr& advData, + const std::unique_ptr& diffData); bool checkForNaNs(); void copyTransportOldToNew(); - void copyStateNewToOld(int nGhost = 0); + void copyStateNewToOld(const int nGhost = 0); void copyPressNewToOld(); - void copyStateOldToNew(int nGhost = 0); - void copyDiffusionOldToNew(std::unique_ptr& diffData); + void copyStateOldToNew(const int nGhost = 0); + void copyDiffusionOldToNew(const std::unique_ptr& diffData); - void computeVelocityAdvTerm(std::unique_ptr& advData); - void updateVelocity(std::unique_ptr& advData); + void computeVelocityAdvTerm(const std::unique_ptr& advData); + void updateVelocity(const std::unique_ptr& advData); void getScalarAdvForce( - std::unique_ptr& advData, - std::unique_ptr& diffData); - void computeScalarAdvTerms(std::unique_ptr& advData); + const std::unique_ptr& advData, + const std::unique_ptr& diffData); + void computeScalarAdvTerms(const std::unique_ptr& advData); void computePassiveAdvTerms( - std::unique_ptr& advData, int state_comp, int ncomp); - void updateDensity(std::unique_ptr& advData); + const std::unique_ptr& advData, + const int state_comp, + const int ncomp); + void updateDensity(const std::unique_ptr& advData); void updateScalarComp( - std::unique_ptr& advData, int state_comp, int ncomp); + const std::unique_ptr& advData, + const int state_comp, + const int ncomp); void getScalarDiffForce( - std::unique_ptr& advData, - std::unique_ptr& diffData); + const std::unique_ptr& advData, + const std::unique_ptr& diffData); - void getScalarReactForce(std::unique_ptr& advData); + void getScalarReactForce(const std::unique_ptr& advData); //----------------------------------------------------------------------------- #ifdef PELE_USE_PLASMA @@ -1537,29 +1609,29 @@ public: //----------------------------------------------------------------------------- // PLASMA - void poissonSolveEF(const PeleLM::TimeStamp& a_time); + void poissonSolveEF(const PeleLM::TimeStamp a_time); - void calcEFTransport(const PeleLM::TimeStamp& a_time); + void calcEFTransport(const PeleLM::TimeStamp a_time); void initializeElectronNeutral(); void initializeElectronFromMassFraction(); void implicitNonLinearSolve( - int sdcIter, - const amrex::Real& a_dt, - std::unique_ptr& diffData, - std::unique_ptr& advData); + const int sdcIter, + const amrex::Real a_dt, + const std::unique_ptr& diffData, + const std::unique_ptr& advData); void computeBGcharge( - const amrex::Real& a_time, - std::unique_ptr& diffData, - std::unique_ptr& advData); + const amrex::Real a_time, + const std::unique_ptr& diffData, + const std::unique_ptr& advData); void nonLinearResidual( - const amrex::Real& a_dt, + const amrex::Real a_dt, const amrex::Vector& a_nlstate, const amrex::Vector& a_nlresid, - int updateScaling, - int updatePrecond); + const int updateScaling, + const int updatePrecond); void getAdvectionTerm( const amrex::Vector& a_nE, @@ -1568,34 +1640,35 @@ public: a_gPhiVCur); void getAdvectionFluxes( - int lev, + const int lev, const amrex::Array& a_fluxes, const amrex::MultiFab& a_nE, const amrex::Array& a_ueff, amrex::BCRec bcrec); void getAdvectionFluxesMOL( - int lev, + const int lev, const amrex::Array& a_fluxes, const amrex::MultiFab& a_nE, const amrex::Array& a_ueff, amrex::BCRec bcrec); - void ionDriftVelocity(std::unique_ptr& advData); - void ionDriftAddUmac(int lev, std::unique_ptr& advData); + void ionDriftVelocity(const std::unique_ptr& advData); + void ionDriftAddUmac( + const int lev, const std::unique_ptr& advData); void computeInstantaneousReactionRateEF( - int lev, const PeleLM::TimeStamp& a_time, amrex::MultiFab* a_I_R); + const int lev, const PeleLM::TimeStamp a_time, amrex::MultiFab* a_I_R); amrex::BCRec - hackBCChargedParticle(const amrex::Real& zk, const amrex::BCRec& bc_in); + hackBCChargedParticle(const amrex::Real zk, const amrex::BCRec& bc_in); - amrex::Real estEFIonsDt(const PeleLM::TimeStamp& a_time); + amrex::Real estEFIonsDt(const PeleLM::TimeStamp a_time); void addLorentzVelForces( - int lev, + const int lev, const amrex::Box& bx, - const amrex::Real& a_time, + const amrex::Real a_time, amrex::Array4 const& force, amrex::Array4 const& rhoY, amrex::Array4 const& phiV, @@ -1605,29 +1678,29 @@ public: nlSolveNorm(const amrex::Vector& a_MF, amrex::Real& norm); int testExitNewton( - int newtonIter, - const amrex::Real& max_res, - const amrex::Real& norm_newtonDir); + const int newtonIter, + const amrex::Real max_res, + const amrex::Real norm_newtonDir); void updateNLState(const amrex::Vector& a_update); void incrementElectronForcing( - int a_sstep, std::unique_ptr& advData); + const int a_sstep, const std::unique_ptr& advData); void getNLStateScaling(amrex::Real& nEScale, amrex::Real& phiVScale); void getNLResidScaling(amrex::Real& nEScale, amrex::Real& phiVScale); - void scaleNLState(const amrex::Real& nEScale, const amrex::Real& phiVScale); + void scaleNLState(const amrex::Real nEScale, const amrex::Real phiVScale); void scaleNLResid( const amrex::Vector& a_resid, - const amrex::Real& nEScale, - const amrex::Real& phiVScale); + const amrex::Real nEScale, + const amrex::Real phiVScale); // get upwinded edge-centered MF from cc MF and ec Vel amrex::Array getUpwindedEdge( - int lev, - int edge_comp, - int ncomp, - amrex::Vector a_bcrec, + const int lev, + const int edge_comp, + const int ncomp, + const amrex::Vector& a_bcrec, const amrex::MultiFab& ccMF, const amrex::Array& ecVel); @@ -1636,7 +1709,7 @@ public: const amrex::Vector& a_Ax); void setUpPrecond( - const amrex::Real& a_dt, const amrex::Vector& a_nE); + const amrex::Real a_dt, const amrex::Vector& a_nE); void applyPrecond( const amrex::Vector& a_v, const amrex::Vector& a_Pv); @@ -1649,7 +1722,7 @@ public: // Accessors std::string runMode() const { return m_run_mode; } - static int nCompIR() + static constexpr int nCompIR() { #ifdef PELE_USE_PLASMA return NUM_SPECIES + 1; @@ -1658,7 +1731,7 @@ public: #endif } - static int nCompForcing() + static constexpr int nCompForcing() { #ifdef PELE_USE_PLASMA return NUM_SPECIES + 2; @@ -1672,11 +1745,11 @@ public: return *m_factory[lev]; } - LevelData* - getLevelDataPtr(int lev, const PeleLM::TimeStamp& a_time, int useUmac = 0); - LevelDataReact* getLevelDataReactPtr(int lev); + LevelData* getLevelDataPtr( + const int lev, const PeleLM::TimeStamp a_time, const int useUmac = 0); + LevelDataReact* getLevelDataReactPtr(const int lev); - amrex::Real getTime(int lev, const PeleLM::TimeStamp& a_time) const + amrex::Real getTime(const int lev, const PeleLM::TimeStamp a_time) const { AMREX_ASSERT( a_time == AmrOldTime || a_time == AmrNewTime || a_time == AmrHalfTime); @@ -1690,45 +1763,46 @@ public: } amrex::Vector> - getStateVect(const PeleLM::TimeStamp& a_time); + getStateVect(const PeleLM::TimeStamp a_time); amrex::Vector> - getVelocityVect(const PeleLM::TimeStamp& a_time); + getVelocityVect(const PeleLM::TimeStamp a_time); amrex::Vector> - getDensityVect(const PeleLM::TimeStamp& a_time); + getDensityVect(const PeleLM::TimeStamp a_time); amrex::Vector> - getSpeciesVect(const PeleLM::TimeStamp& a_time); + getSpeciesVect(const PeleLM::TimeStamp a_time); amrex::Vector> - getRhoHVect(const PeleLM::TimeStamp& a_time); + getRhoHVect(const PeleLM::TimeStamp a_time); amrex::Vector> - getTempVect(const PeleLM::TimeStamp& a_time); - amrex::Vector getDivUVect(const PeleLM::TimeStamp& a_time); + getTempVect(const PeleLM::TimeStamp a_time); + amrex::Vector getDivUVect(const PeleLM::TimeStamp a_time); amrex::Vector - getDiffusivityVect(const PeleLM::TimeStamp& a_time); + getDiffusivityVect(const PeleLM::TimeStamp a_time); amrex::Vector - getViscosityVect(const PeleLM::TimeStamp& a_time); + getViscosityVect(const PeleLM::TimeStamp a_time); amrex::Vector getIRVect(); amrex::Vector> - getAuxVect(const PeleLM::TimeStamp& a_time); + getAuxVect(const PeleLM::TimeStamp a_time); amrex::Vector - getAuxDiffusivityVect(const PeleLM::TimeStamp& a_time); + getAuxDiffusivityVect(const PeleLM::TimeStamp a_time); #ifdef PELE_USE_PLASMA amrex::Vector> - getPhiVVect(const PeleLM::TimeStamp& a_time); + getPhiVVect(const PeleLM::TimeStamp a_time); amrex::Vector> - getnEVect(const PeleLM::TimeStamp& a_time); + getnEVect(const PeleLM::TimeStamp a_time); amrex::Vector - getnEDiffusivityVect(const PeleLM::TimeStamp& a_time); + getnEDiffusivityVect(const PeleLM::TimeStamp a_time); #endif - amrex::Vector fetchBCRecArray(int scomp, int ncomp); - amrex::Vector fetchBCRecAuxArray(int scomp, int ncomp); - amrex::Vector fetchAdvTypeArray(int scomp, int ncomp); - amrex::Vector fetchDiffTypeArray(int scomp, int ncomp); - amrex::Vector fetchAdvTypeAuxArray(int scomp, int ncomp); - amrex::Vector fetchDiffTypeAuxArray(int scomp, int ncomp); + amrex::Vector fetchBCRecArray(const int scomp, const int ncomp); + amrex::Vector + fetchBCRecAuxArray(const int scomp, const int ncomp); + amrex::Vector fetchAdvTypeArray(const int scomp, const int ncomp); + amrex::Vector fetchDiffTypeArray(const int scomp, const int ncomp); + amrex::Vector fetchAdvTypeAuxArray(const int scomp, const int ncomp); + amrex::Vector fetchDiffTypeAuxArray(const int scomp, const int ncomp); DiffusionOp* getDiffusionOp(); - DiffusionOp* getMCDiffusionOp(int ncomp = 1); + DiffusionOp* getMCDiffusionOp(const int ncomp = 1); DiffusionTensorOp* getDiffusionTensorOp(); //----------------------------------------------------------------------------- @@ -2085,13 +2159,13 @@ public: // Number of ghost cells #ifdef AMREX_USE_EB - int m_nGrowState = 4; + static constexpr int m_nGrowState = 4; #else - int m_nGrowState = 3; + static constexpr int m_nGrowState = 3; #endif - int m_nGrowAdv = 1; - int m_nGrowdivu = 1; - int m_nGrowMAC = 1; + static constexpr int m_nGrowAdv = 1; + static constexpr int m_nGrowdivu = 1; + static constexpr int m_nGrowMAC = 1; // Initialization int m_do_init_proj = 1; diff --git a/Source/PeleLMeX.cpp b/Source/PeleLMeX.cpp index b91b4a2ea..3a38d21c1 100644 --- a/Source/PeleLMeX.cpp +++ b/Source/PeleLMeX.cpp @@ -41,7 +41,7 @@ PeleLM::~PeleLM() PeleLM::LevelData* PeleLM::getLevelDataPtr( - int lev, const PeleLM::TimeStamp& a_time, int /*useUMac*/) + const int lev, const TimeStamp a_time, const int /*useUMac*/) { AMREX_ASSERT( a_time == AmrOldTime || a_time == AmrNewTime || a_time == AmrHalfTime); @@ -54,7 +54,7 @@ PeleLM::getLevelDataPtr( m_leveldata_floating = std::make_unique( grids[lev], dmap[lev], *m_factory[lev], m_incompressible, m_has_divu, m_nAux, m_nGrowState, m_use_soret, static_cast(m_do_les)); - amrex::Real time = getTime(lev, a_time); + const amrex::Real time = getTime(lev, a_time); fillpatch_state(lev, time, m_leveldata_floating->state, m_nGrowState); if (m_nAux > 0) { fillpatch_aux(lev, time, m_leveldata_floating->auxiliaries, m_nGrowState); @@ -63,7 +63,7 @@ PeleLM::getLevelDataPtr( } PeleLM::LevelDataReact* -PeleLM::getLevelDataReactPtr(int lev) +PeleLM::getLevelDataReactPtr(const int lev) { if (m_do_react != 0) { return m_leveldatareact[lev].get(); @@ -72,7 +72,7 @@ PeleLM::getLevelDataReactPtr(int lev) } amrex::Vector> -PeleLM::getStateVect(const TimeStamp& a_time) +PeleLM::getStateVect(const TimeStamp a_time) { amrex::Vector> r; r.reserve(finest_level + 1); @@ -109,7 +109,7 @@ PeleLM::getStateVect(const TimeStamp& a_time) } amrex::Vector> -PeleLM::getVelocityVect(const TimeStamp& a_time) +PeleLM::getVelocityVect(const TimeStamp a_time) { amrex::Vector> r; r.reserve(finest_level + 1); @@ -132,9 +132,9 @@ PeleLM::getVelocityVect(const TimeStamp& a_time) } amrex::Vector> -PeleLM::getSpeciesVect(const TimeStamp& a_time) +PeleLM::getSpeciesVect(const TimeStamp a_time) { - AMREX_ASSERT(!m_incompressible); + AMREX_ASSERT(m_incompressible == 0); amrex::Vector> r; r.reserve(finest_level + 1); if (a_time == AmrOldTime) { @@ -156,9 +156,9 @@ PeleLM::getSpeciesVect(const TimeStamp& a_time) } amrex::Vector> -PeleLM::getDensityVect(const TimeStamp& a_time) +PeleLM::getDensityVect(const TimeStamp a_time) { - AMREX_ASSERT(!m_incompressible); + AMREX_ASSERT(m_incompressible == 0); amrex::Vector> r; r.reserve(finest_level + 1); if (a_time == AmrOldTime) { @@ -175,7 +175,7 @@ PeleLM::getDensityVect(const TimeStamp& a_time) } } else { for (int lev = 0; lev <= finest_level; ++lev) { - amrex::Real time = getTime(lev, a_time); + const amrex::Real time = getTime(lev, a_time); r.push_back( std::make_unique( grids[lev], dmap[lev], 1, m_nGrowState)); @@ -186,9 +186,9 @@ PeleLM::getDensityVect(const TimeStamp& a_time) } amrex::Vector> -PeleLM::getTempVect(const TimeStamp& a_time) +PeleLM::getTempVect(const TimeStamp a_time) { - AMREX_ASSERT(!m_incompressible); + AMREX_ASSERT(m_incompressible == 0); amrex::Vector> r; r.reserve(finest_level + 1); if (a_time == AmrOldTime) { @@ -208,9 +208,9 @@ PeleLM::getTempVect(const TimeStamp& a_time) } amrex::Vector> -PeleLM::getRhoHVect(const TimeStamp& a_time) +PeleLM::getRhoHVect(const TimeStamp a_time) { - AMREX_ASSERT(!m_incompressible); + AMREX_ASSERT(m_incompressible == 0); amrex::Vector> r; r.reserve(finest_level + 1); if (a_time == AmrOldTime) { @@ -230,9 +230,9 @@ PeleLM::getRhoHVect(const TimeStamp& a_time) } amrex::Vector -PeleLM::getDivUVect(const TimeStamp& a_time) +PeleLM::getDivUVect(const TimeStamp a_time) { - AMREX_ASSERT(!m_incompressible); + AMREX_ASSERT(m_incompressible == 0); amrex::Vector r; r.reserve(finest_level + 1); if (a_time == AmrOldTime) { @@ -248,9 +248,9 @@ PeleLM::getDivUVect(const TimeStamp& a_time) } amrex::Vector -PeleLM::getDiffusivityVect(const TimeStamp& a_time) +PeleLM::getDiffusivityVect(const TimeStamp a_time) { - AMREX_ASSERT(!m_incompressible); + AMREX_ASSERT(m_incompressible == 0); amrex::Vector r; r.reserve(finest_level + 1); if (a_time == AmrOldTime) { @@ -266,7 +266,7 @@ PeleLM::getDiffusivityVect(const TimeStamp& a_time) } amrex::Vector -PeleLM::getViscosityVect(const TimeStamp& a_time) +PeleLM::getViscosityVect(const TimeStamp a_time) { amrex::Vector r; r.reserve(finest_level + 1); @@ -294,7 +294,7 @@ PeleLM::getIRVect() } amrex::Vector> -PeleLM::getAuxVect(const TimeStamp& a_time) +PeleLM::getAuxVect(const TimeStamp a_time) { AMREX_ASSERT(m_nAux > 0); amrex::Vector> r; @@ -316,7 +316,7 @@ PeleLM::getAuxVect(const TimeStamp& a_time) } amrex::Vector -PeleLM::getAuxDiffusivityVect(const TimeStamp& a_time) +PeleLM::getAuxDiffusivityVect(const TimeStamp a_time) { AMREX_ASSERT(m_nAux > 0); amrex::Vector r; @@ -334,9 +334,9 @@ PeleLM::getAuxDiffusivityVect(const TimeStamp& a_time) } void -PeleLM::averageDownState(const PeleLM::TimeStamp& a_time) +PeleLM::averageDownState(const TimeStamp a_time) { - int nCompState = (m_incompressible) != 0 ? AMREX_SPACEDIM : NVAR; + const int nCompState = (m_incompressible != 0) ? AMREX_SPACEDIM : NVAR; for (int lev = finest_level; lev > 0; --lev) { auto* ldataFine_p = getLevelDataPtr(lev, a_time); auto* ldataCrse_p = getLevelDataPtr(lev - 1, a_time); @@ -351,11 +351,12 @@ PeleLM::averageDownState(const PeleLM::TimeStamp& a_time) } void -PeleLM::averageDownScalars(const PeleLM::TimeStamp& a_time) +PeleLM::averageDownScalars(const TimeStamp a_time) { - int nScal = NUM_SPECIES + 3; // rho, rhoYs, rhoH, Temp #ifdef PELE_USE_PLASMA - nScal += 2; // rhoRT, nE + constexpr int nScal = NUM_SPECIES + 5; +#else + constexpr int nScal = NUM_SPECIES + 3; #endif for (int lev = finest_level; lev > 0; --lev) { auto* ldataFine_p = getLevelDataPtr(lev, a_time); @@ -373,7 +374,7 @@ PeleLM::averageDownScalars(const PeleLM::TimeStamp& a_time) } void -PeleLM::averageDownAux(const PeleLM::TimeStamp& a_time) +PeleLM::averageDownAux(const TimeStamp a_time) { for (int lev = finest_level; lev > 0; --lev) { auto* ldataFine_p = getLevelDataPtr(lev, a_time); @@ -392,7 +393,7 @@ PeleLM::averageDownAux(const PeleLM::TimeStamp& a_time) void PeleLM::averageDown( - const PeleLM::TimeStamp& a_time, const int state_comp, const int ncomp) + const TimeStamp a_time, const int state_comp, const int ncomp) { for (int lev = finest_level; lev > 0; --lev) { auto* ldataFine_p = getLevelDataPtr(lev, a_time); @@ -410,7 +411,7 @@ PeleLM::averageDown( } void -PeleLM::averageDownVelocity(const PeleLM::TimeStamp& a_time) +PeleLM::averageDownVelocity(const TimeStamp a_time) { for (int lev = finest_level; lev > 0; --lev) { auto* ldataFine_p = getLevelDataPtr(lev, a_time); @@ -445,9 +446,9 @@ PeleLM::averageDownReaction() #ifdef PELE_USE_PLASMA amrex::Vector> -PeleLM::getPhiVVect(const TimeStamp& a_time) +PeleLM::getPhiVVect(const TimeStamp a_time) { - AMREX_ASSERT(!m_incompressible); + AMREX_ASSERT(m_incompressible == 0); amrex::Vector> r; r.reserve(finest_level + 1); if (a_time == AmrOldTime) { @@ -467,9 +468,9 @@ PeleLM::getPhiVVect(const TimeStamp& a_time) } amrex::Vector> -PeleLM::getnEVect(const TimeStamp& a_time) +PeleLM::getnEVect(const TimeStamp a_time) { - AMREX_ASSERT(!m_incompressible); + AMREX_ASSERT(m_incompressible == 0); amrex::Vector> r; r.reserve(finest_level + 1); if (a_time == AmrOldTime) { @@ -489,7 +490,7 @@ PeleLM::getnEVect(const TimeStamp& a_time) } amrex::Vector -PeleLM::getnEDiffusivityVect(const TimeStamp& a_time) +PeleLM::getnEDiffusivityVect(const TimeStamp a_time) { amrex::Vector r; r.reserve(finest_level + 1); diff --git a/Source/PeleLMeX_Advance.cpp b/Source/PeleLMeX_Advance.cpp index 48de6c9ed..bb3e3c158 100644 --- a/Source/PeleLMeX_Advance.cpp +++ b/Source/PeleLMeX_Advance.cpp @@ -4,17 +4,17 @@ #include void -PeleLM::Advance(int is_initIter) +PeleLM::Advance(const int is_initIter) { BL_PROFILE("PeleLMeX::Advance()"); #ifdef AMREX_MEM_PROFILING // Memory profiler if compiled - MemProfiler::report("STEP [" + std::to_string(m_nstep) + "]"); + amrex::MemProfiler::report("STEP [" + std::to_string(m_nstep) + "]"); #endif // Start timing current time step - amrex::Real strt_time = amrex::ParallelDescriptor::second(); + const amrex::Real strt_time = amrex::ParallelDescriptor::second(); //---------------------------------------------------------------- BL_PROFILE_VAR("PeleLMeX::advance::setup", PLM_SETUP); @@ -47,7 +47,7 @@ PeleLM::Advance(int is_initIter) #endif // Update time vectors - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { m_t_old[lev] = m_cur_time; m_t_new[lev] = m_cur_time + m_dt; } @@ -71,7 +71,7 @@ PeleLM::Advance(int is_initIter) finest_level, grids, dmap, m_factory, m_incompressible, m_nAux, m_nGrowAdv, m_nGrowMAC); - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { m_extSource[lev]->setVal(0.); } //---------------------------------------------------------------- @@ -86,7 +86,7 @@ PeleLM::Advance(int is_initIter) // Reset velocity flux on boundary faces if doing closed chamber if (m_closed_chamber != 0) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_domainUmacFlux[2 * idim] = 0.0; m_domainUmacFlux[2 * idim + 1] = 0.0; } @@ -201,10 +201,10 @@ PeleLM::Advance(int is_initIter) #endif if (m_has_divu != 0) { - int is_initialization = 0; // Not here - int computeDiffusionTerm = + constexpr int is_initialization = 0; // Not here + constexpr int computeDiffusionTerm = 1; // Yes, re-evaluate the diffusion term after the last chemistry solve - int do_avgDown = 1; // Always + constexpr int do_avgDown = 1; // Always calcDivU( is_initialization, computeDiffusionTerm, do_avgDown, AmrNewTime, diffData); @@ -267,9 +267,9 @@ PeleLM::Advance(int is_initIter) void PeleLM::oneSDC( - int sdcIter, - std::unique_ptr& advData, - std::unique_ptr& diffData) + const int sdcIter, + const std::unique_ptr& advData, + const std::unique_ptr& diffData) { BL_PROFILE("PeleLMeX::oneSDC()"); m_sdcIter = sdcIter; @@ -300,9 +300,9 @@ PeleLM::oneSDC( calcDiffusivity(AmrNewTime); computeDifferentialDiffusionTerms(AmrNewTime, diffData); if (m_has_divu != 0) { - int is_initialization = 0; // Not here - int computeDiffusionTerm = 0; // Nope, we just did that - int do_avgDown = 1; // Always + constexpr int is_initialization = 0; // Not here + constexpr int computeDiffusionTerm = 0; // Nope, we just did that + constexpr int do_avgDown = 1; // Always calcDivU( is_initialization, computeDiffusionTerm, do_avgDown, AmrNewTime, diffData); diff --git a/Source/PeleLMeX_Advection.cpp b/Source/PeleLMeX_Advection.cpp index 68059da08..b04177c79 100644 --- a/Source/PeleLMeX_Advection.cpp +++ b/Source/PeleLMeX_Advection.cpp @@ -4,24 +4,26 @@ #include void -PeleLM::computeVelocityAdvTerm(std::unique_ptr& advData) +PeleLM::computeVelocityAdvTerm(const std::unique_ptr& advData) { //---------------------------------------------------------------- // Create temporary containers constexpr int nGrow_force = 1; - amrex::Vector divtau(finest_level + 1); - amrex::Vector velForces(finest_level + 1); + amrex::Vector divtau; + divtau.reserve(finest_level + 1); + amrex::Vector velForces; + velForces.reserve(finest_level + 1); amrex::Vector> fluxes( finest_level + 1); amrex::Vector> faces( finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - divtau[lev].define( + divtau.emplace_back( grids[lev], dmap[lev], AMREX_SPACEDIM, 0, amrex::MFInfo(), Factory(lev)); - velForces[lev].define( + velForces.emplace_back( grids[lev], dmap[lev], AMREX_SPACEDIM, nGrow_force, amrex::MFInfo(), Factory(lev)); - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { fluxes[lev][idim].define( amrex::convert(grids[lev], amrex::IntVect::TheDimensionVector(idim)), dmap[lev], AMREX_SPACEDIM, 0, amrex::MFInfo(), Factory(lev)); @@ -156,7 +158,7 @@ PeleLM::computeVelocityAdvTerm(std::unique_ptr& advData) if (m_incompressible != 0) { divu.setVal(0.0); } else { - amrex::Real time = getTime(lev, AmrOldTime); + const amrex::Real time = getTime(lev, AmrOldTime); fillpatch_divu(lev, time, divu, m_nGrowdivu); } @@ -165,7 +167,7 @@ PeleLM::computeVelocityAdvTerm(std::unique_ptr& advData) auto* ldata_p = getLevelDataPtr(lev, AmrOldTime); //---------------------------------------------------------------- // Use a temporary MF to hold divergence before redistribution - int nGrow_divT = 3; + constexpr int nGrow_divT = 3; amrex::MultiFab divTmp( grids[lev], dmap[lev], AMREX_SPACEDIM, nGrow_divT, amrex::MFInfo(), EBFactory(lev)); @@ -202,26 +204,28 @@ PeleLM::computeVelocityAdvTerm(std::unique_ptr& advData) } void -PeleLM::updateVelocity(std::unique_ptr& advData) +PeleLM::updateVelocity(const std::unique_ptr& advData) { //---------------------------------------------------------------- // Compute t^n divTau - amrex::Vector divtau(finest_level + 1); + amrex::Vector divtau; + divtau.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - divtau[lev].define( + divtau.emplace_back( grids[lev], dmap[lev], AMREX_SPACEDIM, 0, amrex::MFInfo(), Factory(lev)); } constexpr int use_density = 0; - const amrex::Real CrankNicholsonFactor = 0.5; + constexpr amrex::Real CrankNicholsonFactor = 0.5; computeDivTau( AmrOldTime, GetVecOfPtrs(divtau), use_density, CrankNicholsonFactor); //---------------------------------------------------------------- // Get velocity forcing at half time including lagged grad P term constexpr int nGrow_force = 1; - amrex::Vector velForces(finest_level + 1); + amrex::Vector velForces; + velForces.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - velForces[lev].define(grids[lev], dmap[lev], AMREX_SPACEDIM, nGrow_force); + velForces.emplace_back(grids[lev], dmap[lev], AMREX_SPACEDIM, nGrow_force); } constexpr int add_gradP = 1; getVelForces( @@ -263,18 +267,18 @@ PeleLM::updateVelocity(std::unique_ptr& advData) void PeleLM::getScalarAdvForce( - std::unique_ptr& advData, - std::unique_ptr& diffData) + const std::unique_ptr& advData, + const std::unique_ptr& diffData) { - int* aux_diffuse_d = convertToDeviceVector(m_DiffTypeAux).dataPtr(); + const int* aux_diffuse_d = convertToDeviceVector(m_DiffTypeAux).dataPtr(); + auto const* leosparm = eos_parms.device_parm(); + for (int lev = 0; lev <= finest_level; ++lev) { // Get t^{n} data pointer auto* ldata_p = getLevelDataPtr(lev, AmrOldTime); auto* ldataR_p = getLevelDataReactPtr(lev); - auto const* leosparm = eos_parms.device_parm(); - #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -310,7 +314,6 @@ PeleLM::getScalarAdvForce( }); } } - // Fill forcing ghost cells if (advData->Forcing[0].nGrow() > 0) { fillpatch_forces( @@ -327,7 +330,7 @@ PeleLM::getScalarAdvForce( } void -PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) +PeleLM::computeScalarAdvTerms(const std::unique_ptr& advData) { //---------------------------------------------------------------- @@ -355,7 +358,7 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) amrex::Vector> fluxes_aux( finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { fluxes[lev][idim].define( amrex::convert(grids[lev], amrex::IntVect::TheDimensionVector(idim)), dmap[lev], NUM_SPECIES + 1, 0, amrex::MFInfo(), @@ -377,10 +380,10 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) auto* ldata_p = getLevelDataPtr(lev, AmrOldTime); // Define edge state: Density + Species + RhoH + Temp - int nGrow = 0; + constexpr int nGrow = 0; amrex::Array edgeState; amrex::Array edgeState_aux; - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { edgeState[idim].define( amrex::convert(grids[lev], amrex::IntVect::TheDimensionVector(idim)), dmap[lev], NUM_SPECIES + 3, nGrow, amrex::MFInfo(), Factory(lev)); @@ -460,7 +463,7 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) fluxes_are_area_weighted, m_advection_type, m_Godunov_ppm_limiter); // Ions one by one - for (int n = 0; n < NUM_IONS; n++) { + for (int n = 0; n < NUM_IONS; ++n) { const int ion_idx = NUM_SPECIES - NUM_IONS + n; auto bcRecIons = fetchBCRecArray(FIRSTSPEC + ion_idx, 1); auto bcRecIons_d = convertToDeviceVector(bcRecIons); @@ -558,9 +561,9 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) fluxes_are_area_weighted, m_advection_type, m_Godunov_ppm_limiter); } // Zero out fluxes for non-advected auxiliaries - for (int n = 0; n < m_nAux; n++) { + for (int n = 0; n < m_nAux; ++n) { if (m_aux_advect[n] == 0) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { fluxes_aux[lev][idim].setVal(0.0, n, 1); } } @@ -581,14 +584,14 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) #endif // Edge states - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { const amrex::Box& ebx = amrex::surroundingNodes(bx, idim); auto const& rho_ed = edgeState[idim].array(mfi, 0); auto const& rhoY_ed = edgeState[idim].array(mfi, 1); #ifdef AMREX_USE_EB if (flagfab.getType(ebx) == amrex::FabType::covered) { // Covered boxes amrex::ParallelFor( - ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [rho_ed] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { rho_ed(i, j, k) = 0.0; }); } else if ( @@ -596,7 +599,8 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) // boxes const auto& afrac = areafrac[idim]->array(mfi); amrex::ParallelFor( - ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [rho_ed, afrac, + rhoY_ed] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { rho_ed(i, j, k) = 0.0; if (afrac(i, j, k) > 0.0) { // Uncovered faces pele::physics::PhysicsType::eos_type::RY2R( @@ -607,7 +611,8 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) #endif { amrex::ParallelFor( - ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, + [rhoY_ed, rho_ed] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { pele::physics::PhysicsType::eos_type::RY2R( rhoY_ed.cellData(i, j, k), rho_ed(i, j, k)); }); @@ -671,7 +676,7 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) auto const& flagfab = ebfact.getMultiEBCellFlagFab()[mfi]; #endif - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { const amrex::Box& ebx = amrex::surroundingNodes(bx, idim); auto const& rho = edgeState[idim].const_array(mfi, 0); auto const& rhoY = edgeState[idim].const_array(mfi, 1); @@ -680,7 +685,7 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) #ifdef AMREX_USE_EB if (flagfab.getType(ebx) == amrex::FabType::covered) { // Covered boxes amrex::ParallelFor( - ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [rhoHm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { rhoHm(i, j, k) = 0.0; }); } else if ( @@ -688,7 +693,8 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) // boxes const auto& afrac = areafrac[idim]->array(mfi); amrex::ParallelFor( - ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [rhoHm, afrac, rhoY, T, rho, + leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (afrac(i, j, k) <= 0.0) { // Covered faces rhoHm(i, j, k) = 0.0; } else { @@ -699,7 +705,8 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) #endif { amrex::ParallelFor( - ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [rho, rhoY, T, rhoHm, + leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { getRHmixGivenTY(i, j, k, rho, rhoY, T, rhoHm, leosparm); }); } @@ -801,9 +808,9 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) if (m_do_extraEFdiags) { for (int lev = 0; lev <= finest_level; ++lev) { for (int n = 0; n < NUM_IONS; ++n) { - int spec_idx = NUM_SPECIES - NUM_IONS + n; + const int spec_idx = NUM_SPECIES - NUM_IONS + n; amrex::Array, AMREX_SPACEDIM> ionFlux; - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { ionFlux[idim].reset(new amrex::MultiFab( fluxes[lev][idim], amrex::make_alias, spec_idx, 1)); } @@ -825,7 +832,7 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) if (m_incompressible != 0) { divu.setVal(0.0); } else { - amrex::Real time = getTime(lev, AmrOldTime); + const amrex::Real time = getTime(lev, AmrOldTime); fillpatch_divu(lev, time, divu, m_nGrowdivu); } @@ -912,10 +919,10 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) //---------------------------------------------------------------- // Sum over the species AofS to get the density advection term for (int lev = 0; lev <= finest_level; ++lev) { - auto aofsma = advData->AofS[lev].arrays(); + auto const& aofsma = advData->AofS[lev].arrays(); amrex::ParallelFor( advData->AofS[lev], - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + [aofsma] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { pele::physics::PhysicsType::eos_type::RY2R( aofsma[box_no].cellData(i, j, k), aofsma[box_no](i, j, k, DENSITY), FIRSTSPEC); @@ -925,19 +932,19 @@ PeleLM::computeScalarAdvTerms(std::unique_ptr& advData) } void -PeleLM::updateDensity(std::unique_ptr& advData) +PeleLM::updateDensity(const std::unique_ptr& advData) { for (int lev = 0; lev <= finest_level; ++lev) { // Get MultiArrays auto const& sma_o = getLevelDataPtr(lev, AmrOldTime)->state.arrays(); auto const& sma_n = getLevelDataPtr(lev, AmrNewTime)->state.arrays(); - auto aofsma = advData->AofS[lev].const_arrays(); - auto extma = m_extSource[lev]->const_arrays(); + auto const& aofsma = advData->AofS[lev].const_arrays(); + auto const& extma = m_extSource[lev]->const_arrays(); const auto dt = m_dt; amrex::ParallelFor( - advData->AofS[lev], - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + advData->AofS[lev], [sma_o, sma_n, aofsma, extma, dt] AMREX_GPU_DEVICE( + int box_no, int i, int j, int k) noexcept { sma_n[box_no](i, j, k, DENSITY) = sma_o[box_no](i, j, k, DENSITY) + dt * (aofsma[box_no](i, j, k, DENSITY) + @@ -949,7 +956,9 @@ PeleLM::updateDensity(std::unique_ptr& advData) void PeleLM::computePassiveAdvTerms( - std::unique_ptr& advData, int state_comp, int ncomp) + const std::unique_ptr& advData, + const int state_comp, + const int ncomp) { //---------------------------------------------------------------- // Get the BCRecs and AdvectionTypes @@ -965,7 +974,7 @@ PeleLM::computePassiveAdvTerms( amrex::Vector> edgeState( finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { fluxes[lev][idim].define( amrex::convert(grids[lev], amrex::IntVect::TheDimensionVector(idim)), dmap[lev], ncomp, 0, amrex::MFInfo(), Factory(lev)); @@ -1021,9 +1030,9 @@ PeleLM::computePassiveAdvTerms( // TODO: Find way to include diffusive forces for passive scalars that // diffuse auto const& force_arr = m_extSource[lev]->const_array(mfi, state_comp); - bool is_velocity = false; - bool fluxes_are_area_weighted = false; - bool knownEdgeState = false; + constexpr bool is_velocity = false; + constexpr bool fluxes_are_area_weighted = false; + constexpr bool knownEdgeState = false; HydroUtils::ComputeFluxesOnBoxFromState( bx, ncomp, mfi, pass_arr, AMREX_D_DECL(fx, fy, fz), AMREX_D_DECL(edgex, edgey, edgez), knownEdgeState, @@ -1069,7 +1078,7 @@ PeleLM::computePassiveAdvTerms( if (m_incompressible != 0) { divu.setVal(0.0); } else { - amrex::Real time = getTime(lev, AmrOldTime); + const amrex::Real time = getTime(lev, AmrOldTime); fillpatch_divu(lev, time, divu, m_nGrowdivu); } @@ -1078,7 +1087,7 @@ PeleLM::computePassiveAdvTerms( auto* ldata_p = getLevelDataPtr(lev, AmrOldTime); //---------------------------------------------------------------- // Use a temporary MF to hold divergence before redistribution - int nGrow_divTmp = 3; + constexpr int nGrow_divTmp = 3; amrex::MultiFab divTmp( grids[lev], dmap[lev], ncomp, nGrow_divTmp, amrex::MFInfo(), EBFactory(lev)); @@ -1108,12 +1117,15 @@ PeleLM::computePassiveAdvTerms( void PeleLM::updateScalarComp( - std::unique_ptr& advData, int state_comp, int ncomp) + const std::unique_ptr& advData, + const int state_comp, + const int ncomp) { for (int lev = 0; lev <= finest_level; ++lev) { // Get level data ptr auto* ldataOld_p = getLevelDataPtr(lev, AmrOldTime); auto* ldataNew_p = getLevelDataPtr(lev, AmrNewTime); + #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif diff --git a/Source/PeleLMeX_BC.cpp b/Source/PeleLMeX_BC.cpp index a9c21d351..d39c3949a 100644 --- a/Source/PeleLMeX_BC.cpp +++ b/Source/PeleLMeX_BC.cpp @@ -10,97 +10,106 @@ // Components are Interior, Inflow, Outflow, Symmetry, & // SlipWallAdiab, NoSlipWallAdiab, SlipWallIsoTherm, NoSlipWallIsoTherm. -int norm_vel_bc[] = {amrex::BCType::int_dir, amrex::BCType::ext_dir, - amrex::BCType::foextrap, amrex::BCType::reflect_odd, - amrex::BCType::ext_dir, amrex::BCType::ext_dir, - amrex::BCType::ext_dir, amrex::BCType::ext_dir}; - -int tang_vel_bc[] = {amrex::BCType::int_dir, amrex::BCType::ext_dir, - amrex::BCType::foextrap, amrex::BCType::reflect_even, - amrex::BCType::hoextrap, amrex::BCType::ext_dir, - amrex::BCType::hoextrap, amrex::BCType::ext_dir}; - -int density_bc[] = {amrex::BCType::int_dir, amrex::BCType::ext_dir, - amrex::BCType::foextrap, amrex::BCType::reflect_even, - amrex::BCType::foextrap, amrex::BCType::foextrap, - amrex::BCType::foextrap, amrex::BCType::foextrap}; - -int species_bc[] = {amrex::BCType::int_dir, amrex::BCType::ext_dir, - amrex::BCType::foextrap, amrex::BCType::reflect_even, - amrex::BCType::foextrap, amrex::BCType::foextrap, - amrex::BCType::ext_dir, amrex::BCType::ext_dir}; - -int rhoh_bc[] = {amrex::BCType::int_dir, amrex::BCType::ext_dir, - amrex::BCType::foextrap, amrex::BCType::reflect_even, - amrex::BCType::foextrap, amrex::BCType::foextrap, - amrex::BCType::ext_dir, amrex::BCType::ext_dir}; - -int temp_bc[] = {amrex::BCType::int_dir, amrex::BCType::ext_dir, - amrex::BCType::foextrap, amrex::BCType::reflect_even, - amrex::BCType::foextrap, amrex::BCType::foextrap, - amrex::BCType::ext_dir, amrex::BCType::ext_dir}; - -int aux_bc[] = {amrex::BCType::int_dir, amrex::BCType::ext_dir, - amrex::BCType::foextrap, amrex::BCType::reflect_even, - amrex::BCType::foextrap, amrex::BCType::foextrap, - amrex::BCType::ext_dir, amrex::BCType::ext_dir}; - -int divu_bc[] = {amrex::BCType::int_dir, amrex::BCType::reflect_even, - amrex::BCType::reflect_even, amrex::BCType::reflect_even, - amrex::BCType::reflect_even, amrex::BCType::reflect_even, - amrex::BCType::reflect_even, amrex::BCType::reflect_even}; +constexpr int norm_vel_bc[] = { + amrex::BCType::int_dir, amrex::BCType::ext_dir, amrex::BCType::foextrap, + amrex::BCType::reflect_odd, amrex::BCType::ext_dir, amrex::BCType::ext_dir, + amrex::BCType::ext_dir, amrex::BCType::ext_dir}; + +constexpr int tang_vel_bc[] = { + amrex::BCType::int_dir, amrex::BCType::ext_dir, amrex::BCType::foextrap, + amrex::BCType::reflect_even, amrex::BCType::hoextrap, amrex::BCType::ext_dir, + amrex::BCType::hoextrap, amrex::BCType::ext_dir}; + +constexpr int density_bc[] = { + amrex::BCType::int_dir, amrex::BCType::ext_dir, amrex::BCType::foextrap, + amrex::BCType::reflect_even, amrex::BCType::foextrap, amrex::BCType::foextrap, + amrex::BCType::foextrap, amrex::BCType::foextrap}; + +constexpr int species_bc[] = { + amrex::BCType::int_dir, amrex::BCType::ext_dir, amrex::BCType::foextrap, + amrex::BCType::reflect_even, amrex::BCType::foextrap, amrex::BCType::foextrap, + amrex::BCType::foextrap, amrex::BCType::foextrap}; + +constexpr int rhoh_bc[] = {amrex::BCType::int_dir, amrex::BCType::ext_dir, + amrex::BCType::foextrap, amrex::BCType::reflect_even, + amrex::BCType::foextrap, amrex::BCType::foextrap, + amrex::BCType::ext_dir, amrex::BCType::ext_dir}; + +constexpr int temp_bc[] = {amrex::BCType::int_dir, amrex::BCType::ext_dir, + amrex::BCType::foextrap, amrex::BCType::reflect_even, + amrex::BCType::foextrap, amrex::BCType::foextrap, + amrex::BCType::ext_dir, amrex::BCType::ext_dir}; + +constexpr int aux_bc[] = {amrex::BCType::int_dir, amrex::BCType::ext_dir, + amrex::BCType::foextrap, amrex::BCType::reflect_even, + amrex::BCType::foextrap, amrex::BCType::foextrap, + amrex::BCType::ext_dir, amrex::BCType::ext_dir}; + +constexpr int divu_bc[] = { + amrex::BCType::int_dir, amrex::BCType::reflect_even, + amrex::BCType::reflect_even, amrex::BCType::reflect_even, + amrex::BCType::reflect_even, amrex::BCType::reflect_even, + amrex::BCType::reflect_even, amrex::BCType::reflect_even}; // Following incflo rather than IAMR here -int force_bc[] = {amrex::BCType::int_dir, amrex::BCType::foextrap, - amrex::BCType::foextrap, amrex::BCType::foextrap, - amrex::BCType::foextrap, amrex::BCType::foextrap, - amrex::BCType::foextrap, amrex::BCType::foextrap}; +constexpr int force_bc[] = {amrex::BCType::int_dir, amrex::BCType::foextrap, + amrex::BCType::foextrap, amrex::BCType::foextrap, + amrex::BCType::foextrap, amrex::BCType::foextrap, + amrex::BCType::foextrap, amrex::BCType::foextrap}; #ifdef PELE_USE_PLASMA -int nE_bc[] = {amrex::BCType::int_dir, amrex::BCType::ext_dir, - amrex::BCType::foextrap, amrex::BCType::reflect_even, - amrex::BCType::reflect_even, amrex::BCType::reflect_even, - amrex::BCType::ext_dir, amrex::BCType::ext_dir}; +constexpr int nE_bc[] = { + amrex::BCType::int_dir, amrex::BCType::ext_dir, + amrex::BCType::foextrap, amrex::BCType::reflect_even, + amrex::BCType::reflect_even, amrex::BCType::reflect_even, + amrex::BCType::ext_dir, amrex::BCType::ext_dir}; -int phiV_bc[] = { +constexpr int phiV_bc[] = { amrex::BCType::int_dir, amrex::BCType::ext_dir, amrex::BCType::reflect_even}; #endif #ifdef PELE_USE_SOOT -int soot_bc[] = {amrex::BCType::int_dir, amrex::BCType::ext_dir, - amrex::BCType::foextrap, amrex::BCType::reflect_even, - amrex::BCType::reflect_even, amrex::BCType::reflect_even, - amrex::BCType::ext_dir, amrex::BCType::ext_dir}; +constexpr int soot_bc[] = { + amrex::BCType::int_dir, amrex::BCType::ext_dir, + amrex::BCType::foextrap, amrex::BCType::reflect_even, + amrex::BCType::reflect_even, amrex::BCType::reflect_even, + amrex::BCType::ext_dir, amrex::BCType::ext_dir}; #endif amrex::InterpBase* PeleLM:: getInterpolator( // NOLINT(readability-convert-member-functions-to-static) - int a_method) const + const int a_method) const { amrex::InterpBase* mapper = nullptr; - if (a_method == 0) { + switch (a_method) { + case 0: mapper = &amrex::mf_pc_interp; - } else if (a_method == 1) { -// -// Get EB-aware interpolater when needed -// + break; + + case 1: #ifdef AMREX_USE_EB + // Get EB-aware interpolator when needed mapper = (EBFactory(0).isAllRegular()) ? &amrex::mf_cell_cons_interp : &amrex::eb_mf_cell_cons_interp; #else mapper = &amrex::mf_cell_cons_interp; #endif - } else if (a_method == 2) { + break; + + case 2: #ifdef AMREX_USE_EB amrex::Abort("Regrid interpolation method = 2 not available with EB !"); #else mapper = &amrex::mf_linear_slope_minmax_interp; #endif - } else { + break; + + default: amrex::Abort("Unknown interpolation method"); } + return mapper; } @@ -110,7 +119,7 @@ PeleLM::setBoundaryConditions() // Initialize the BCRecs m_bcrec_state.resize(NVAR); - int sizeForceBC = amrex::max(AMREX_SPACEDIM, NUM_SPECIES + 2); + constexpr int sizeForceBC = amrex::max(AMREX_SPACEDIM, NUM_SPECIES + 2); m_bcrec_force.resize(sizeForceBC); m_bcrec_aux.resize(m_nAux); @@ -120,8 +129,8 @@ PeleLM::setBoundaryConditions() const int* hi_bc = m_phys_bc.hi(); // Velocity - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - for (int idim2 = 0; idim2 < AMREX_SPACEDIM; idim2++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + for (int idim2 = 0; idim2 < AMREX_SPACEDIM; ++idim2) { if (idim == idim2) { m_bcrec_state[VELX + idim].setLo(idim2, norm_vel_bc[lo_bc[idim2]]); m_bcrec_state[VELX + idim].setHi(idim2, norm_vel_bc[hi_bc[idim2]]); @@ -133,8 +142,8 @@ PeleLM::setBoundaryConditions() } // General forces: use int_dir in interior and foextrap otherwise - for (int i = 0; i < sizeForceBC; i++) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int i = 0; i < sizeForceBC; ++i) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_bcrec_force[i].setLo(idim, force_bc[lo_bc[idim]]); m_bcrec_force[i].setHi(idim, force_bc[hi_bc[idim]]); } @@ -142,47 +151,47 @@ PeleLM::setBoundaryConditions() if (m_incompressible == 0) { // Density - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_bcrec_state[DENSITY].setLo(idim, density_bc[lo_bc[idim]]); m_bcrec_state[DENSITY].setHi(idim, density_bc[hi_bc[idim]]); } // Species - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - for (int n = 0; n < NUM_SPECIES; n++) { - m_bcrec_state[FIRSTSPEC + n].setLo(idim, density_bc[lo_bc[idim]]); - m_bcrec_state[FIRSTSPEC + n].setHi(idim, density_bc[hi_bc[idim]]); + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + for (int n = 0; n < NUM_SPECIES; ++n) { + m_bcrec_state[FIRSTSPEC + n].setLo(idim, species_bc[lo_bc[idim]]); + m_bcrec_state[FIRSTSPEC + n].setHi(idim, species_bc[hi_bc[idim]]); } } // Enthalpy - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_bcrec_state[RHOH].setLo(idim, rhoh_bc[lo_bc[idim]]); m_bcrec_state[RHOH].setHi(idim, rhoh_bc[hi_bc[idim]]); } // Temperature - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_bcrec_state[TEMP].setLo(idim, temp_bc[lo_bc[idim]]); m_bcrec_state[TEMP].setHi(idim, temp_bc[hi_bc[idim]]); } // rhoRT: reflect even on all but interior bndy - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_bcrec_state[RHORT].setLo(idim, divu_bc[lo_bc[idim]]); m_bcrec_state[RHORT].setHi(idim, divu_bc[hi_bc[idim]]); } // divU if (m_has_divu != 0) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_bcrec_divu.setLo(idim, divu_bc[lo_bc[idim]]); m_bcrec_divu.setHi(idim, divu_bc[hi_bc[idim]]); } } // auxiliaries - assumed to be the same as species - for (int n = 0; n < m_nAux; n++) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int n = 0; n < m_nAux; ++n) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_bcrec_aux[n].setLo(idim, aux_bc[lo_bc[idim]]); m_bcrec_aux[n].setHi(idim, aux_bc[hi_bc[idim]]); } @@ -190,7 +199,7 @@ PeleLM::setBoundaryConditions() #ifdef PELE_USE_PLASMA // nE - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_bcrec_state[NE].setLo(idim, nE_bc[lo_bc[idim]]); m_bcrec_state[NE].setHi(idim, nE_bc[hi_bc[idim]]); } @@ -198,7 +207,7 @@ PeleLM::setBoundaryConditions() // Get m_phiV_bc const int* lo_phibc = m_phiV_bc.lo(); const int* hi_phibc = m_phiV_bc.hi(); - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_bcrec_state[PHIV].setLo(idim, phiV_bc[lo_phibc[idim]]); m_bcrec_state[PHIV].setHi(idim, phiV_bc[hi_phibc[idim]]); } @@ -206,22 +215,22 @@ PeleLM::setBoundaryConditions() // Hack charged species BCs int FIRSTIONinVar = FIRSTSPEC + NUM_SPECIES - NUM_IONS; int FIRSTIONinSpec = NUM_SPECIES - NUM_IONS; - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - for (int n = 0; n < NUM_IONS; n++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + for (int n = 0; n < NUM_IONS; ++n) { auto const bcIonSave = m_bcrec_state[FIRSTIONinVar + n]; m_bcrec_state[FIRSTIONinVar + n] = hackBCChargedParticle(zk[FIRSTIONinSpec + n], bcIonSave); } } // Need to hack nE too actually ... - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { auto const bcnESave = m_bcrec_state[NE]; m_bcrec_state[NE] = hackBCChargedParticle(-1.0, bcnESave); } #endif #ifdef PELE_USE_SOOT - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - for (int mom = 0; mom < NUMSOOTVAR; mom++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + for (int mom = 0; mom < NUMSOOTVAR; ++mom) { m_bcrec_state[FIRSTSOOT + mom].setLo(idim, soot_bc[lo_bc[idim]]); m_bcrec_state[FIRSTSOOT + mom].setHi(idim, soot_bc[hi_bc[idim]]); } @@ -231,20 +240,20 @@ PeleLM::setBoundaryConditions() } amrex::Vector -PeleLM::fetchBCRecArray(int scomp, int ncomp) +PeleLM::fetchBCRecArray(const int scomp, const int ncomp) { amrex::Vector bc(ncomp); - for (int comp = 0; comp < ncomp; comp++) { + for (int comp = 0; comp < ncomp; ++comp) { bc[comp] = m_bcrec_state[scomp + comp]; } return bc; } amrex::Vector -PeleLM::fetchBCRecAuxArray(int scomp, int ncomp) +PeleLM::fetchBCRecAuxArray(const int scomp, const int ncomp) { amrex::Vector bc(ncomp); - for (int comp = 0; comp < ncomp; comp++) { + for (int comp = 0; comp < ncomp; ++comp) { bc[comp] = m_bcrec_aux[scomp + comp]; } return bc; @@ -252,25 +261,24 @@ PeleLM::fetchBCRecAuxArray(int scomp, int ncomp) //----------------------------------------------------------------------------- // The following work directly on the leveldata - // Fill the entire class state at once void -PeleLM::fillPatchState(const TimeStamp& a_time) +PeleLM::fillPatchState(const TimeStamp a_time) { BL_PROFILE("PeleLMeX::fillPatchState()"); - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { fillPatchState(lev, a_time); } } // Fill the a given level class state void -PeleLM::fillPatchState(int lev, const TimeStamp& a_time) +PeleLM::fillPatchState(const int lev, const TimeStamp a_time) { BL_PROFILE("PeleLMeX::fillPatchStateLev()"); auto* ldata_p = getLevelDataPtr(lev, a_time); - amrex::Real time = getTime(lev, a_time); + const amrex::Real time = getTime(lev, a_time); fillpatch_state(lev, time, ldata_p->state, m_nGrowState); if (m_incompressible == 0) { @@ -282,57 +290,57 @@ PeleLM::fillPatchState(int lev, const TimeStamp& a_time) // Fill a state components void -PeleLM::fillPatchDensity(const TimeStamp& a_time) +PeleLM::fillPatchDensity(const TimeStamp a_time) { BL_PROFILE("PeleLMeX::fillPatchDensity()"); - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { auto* ldata_p = getLevelDataPtr(lev, a_time); - amrex::Real time = getTime(lev, a_time); + const amrex::Real time = getTime(lev, a_time); fillpatch_density(lev, time, ldata_p->state, DENSITY, m_nGrowState); } } void -PeleLM::fillPatchSpecies(const TimeStamp& a_time) +PeleLM::fillPatchSpecies(const TimeStamp a_time) { BL_PROFILE("PeleLMeX::fillPatchSpecies()"); - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { auto* ldata_p = getLevelDataPtr(lev, a_time); - amrex::Real time = getTime(lev, a_time); + const amrex::Real time = getTime(lev, a_time); fillpatch_species(lev, time, ldata_p->state, FIRSTSPEC, m_nGrowState); } } void -PeleLM::fillPatchTemp(const TimeStamp& a_time) +PeleLM::fillPatchTemp(const TimeStamp a_time) { BL_PROFILE("PeleLMeX::fillPatchTemp()"); - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { auto* ldata_p = getLevelDataPtr(lev, a_time); - amrex::Real time = getTime(lev, a_time); + const amrex::Real time = getTime(lev, a_time); fillpatch_temp(lev, time, ldata_p->state, TEMP, m_nGrowState); } } void -PeleLM::fillPatchAux(const TimeStamp& a_time) +PeleLM::fillPatchAux(const TimeStamp a_time) { BL_PROFILE("PeleLMeX::fillPatchAux()"); - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { auto* ldata_p = getLevelDataPtr(lev, a_time); - amrex::Real time = getTime(lev, a_time); + const amrex::Real time = getTime(lev, a_time); fillpatch_aux(lev, time, ldata_p->auxiliaries, m_nGrowState); } } #ifdef PELE_USE_PLASMA void -PeleLM::fillPatchPhiV(const TimeStamp& a_time) +PeleLM::fillPatchPhiV(const TimeStamp a_time) { BL_PROFILE("PeleLMeX::fillPatchPhiV()"); - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { auto ldata_p = getLevelDataPtr(lev, a_time); - amrex::Real time = getTime(lev, a_time); + const amrex::Real time = getTime(lev, a_time); fillpatch_phiV(lev, time, ldata_p->state, PHIV, m_nGrowState); } } @@ -343,7 +351,7 @@ PeleLM::fillPatchPhiV(const TimeStamp& a_time) // The following return a fillpatched MF ptr at a given level // Fill the entire state at once std::unique_ptr -PeleLM::fillPatchState(int lev, amrex::Real a_time, int nGrow) +PeleLM::fillPatchState(const int lev, const amrex::Real a_time, const int nGrow) { BL_PROFILE("PeleLMeX::fillPatchState()"); @@ -362,13 +370,14 @@ PeleLM::fillPatchState(int lev, amrex::Real a_time, int nGrow) } std::unique_ptr -PeleLM::fillPatchReact(int lev, amrex::Real a_time, int nGrow) +PeleLM::fillPatchReact(const int lev, const amrex::Real a_time, const int nGrow) { BL_PROFILE("PeleLMeX::fillPatchReact()"); - int IRsize = NUM_SPECIES; #ifdef PELE_USE_PLASMA - IRsize += 1; + constexpr int IRsize = NUM_SPECIES + 1; +#else + constexpr int IRsize = NUM_SPECIES; #endif std::unique_ptr mf; mf = std::make_unique( @@ -382,12 +391,15 @@ PeleLM::fillPatchReact(int lev, amrex::Real a_time, int nGrow) // Fill the state void PeleLM::fillpatch_state( - int lev, const amrex::Real a_time, amrex::MultiFab& a_state, int nGhost) + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_state, + const int nGhost) { ProbParm const* lprobparm = prob_parm_d; auto const* lpmfdata = pmf_data.device_parm(); - int nCompState = (m_incompressible) != 0 ? AMREX_SPACEDIM : NVAR; + const int nCompState = (m_incompressible) != 0 ? AMREX_SPACEDIM : NVAR; fillTurbInflow(a_state, VELX, lev, a_time); @@ -438,11 +450,11 @@ PeleLM::fillpatch_state( // Fill the density void PeleLM::fillpatch_density( - int lev, + const int lev, const amrex::Real a_time, amrex::MultiFab& a_density, - int rho_comp, - int nGhost) + const int rho_comp, + const int nGhost) { ProbParm const* lprobparm = prob_parm_d; auto const* lpmfdata = pmf_data.device_parm(); @@ -493,11 +505,11 @@ PeleLM::fillpatch_density( // Fill the mass fractions void PeleLM::fillpatch_species( - int lev, + const int lev, const amrex::Real a_time, amrex::MultiFab& a_species, - int rhoY_comp, - int nGhost) + const int rhoY_comp, + const int nGhost) { ProbParm const* lprobparm = prob_parm_d; auto const* lpmfdata = pmf_data.device_parm(); @@ -547,11 +559,11 @@ PeleLM::fillpatch_species( // Fill temperature void PeleLM::fillpatch_temp( - int lev, + const int lev, const amrex::Real a_time, amrex::MultiFab& a_temp, - int temp_comp, - int nGhost) + const int temp_comp, + const int nGhost) { ProbParm const* lprobparm = prob_parm_d; auto const* lpmfdata = pmf_data.device_parm(); @@ -598,7 +610,10 @@ PeleLM::fillpatch_temp( // Fill the auxiliaries void PeleLM::fillpatch_aux( - int lev, const amrex::Real a_time, amrex::MultiFab& a_aux, int nGhost) + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_aux, + const int nGhost) { ProbParm const* lprobparm = prob_parm_d; @@ -652,11 +667,11 @@ PeleLM::fillpatch_aux( // Fill electro-static potential void PeleLM::fillpatch_phiV( - int lev, + const int lev, const amrex::Real a_time, amrex::MultiFab& a_temp, - int phiV_comp, - int nGhost) + const int phiV_comp, + const int nGhost) { ProbParm const* lprobparm = prob_parm_d; auto const* lpmfdata = pmf_data.device_parm(); @@ -704,7 +719,10 @@ PeleLM::fillpatch_phiV( // Fill the divU void PeleLM::fillpatch_divu( - int lev, const amrex::Real a_time, amrex::MultiFab& a_divu, int nGhost) + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_divu, + const int nGhost) { if (lev == 0) { amrex::PhysBCFunct> @@ -740,9 +758,9 @@ PeleLM::fillpatch_divu( // foextrap on domain BCs void PeleLM::fillpatch_forces( - amrex::Real a_time, + const amrex::Real a_time, amrex::Vector const& a_force, - int nGrowForce) + const int nGrowForce) { AMREX_ASSERT(a_force[0]->nComp() <= m_bcrec_force.size()); const int nComp = a_force[0]->nComp(); @@ -774,7 +792,10 @@ PeleLM::fillpatch_forces( // Fill the gradp void PeleLM::fillpatch_gradp( - int lev, const amrex::Real a_time, amrex::MultiFab& a_gp, int nGhost) + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_gp, + const int nGhost) { if (lev == 0) { amrex::PhysBCFunct> @@ -809,7 +830,10 @@ PeleLM::fillpatch_gradp( // Fill the reaction data void PeleLM::fillpatch_reaction( - int lev, const amrex::Real a_time, amrex::MultiFab& a_I_R, int nGhost) + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_I_R, + const int nGhost) { if (lev == 0) { amrex::PhysBCFunct> @@ -840,7 +864,10 @@ PeleLM::fillpatch_reaction( // Fill functC void PeleLM::fillpatch_chemFunctCall( - int lev, const amrex::Real a_time, amrex::MultiFab& a_fctC, int nGhost) + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_fctC, + const int nGhost) { if (lev == 0) { amrex::PhysBCFunct> @@ -872,13 +899,16 @@ PeleLM::fillpatch_chemFunctCall( // Fill the state void PeleLM::fillcoarsepatch_state( - int lev, const amrex::Real a_time, amrex::MultiFab& a_state, int nGhost) + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_state, + const int nGhost) { AMREX_ASSERT(lev > 0); ProbParm const* lprobparm = prob_parm_d; auto const* lpmfdata = pmf_data.device_parm(); - int nCompState = (m_incompressible) != 0 ? AMREX_SPACEDIM : NVAR; + const int nCompState = (m_incompressible) != 0 ? AMREX_SPACEDIM : NVAR; fillTurbInflow(a_state, VELX, lev, a_time); @@ -909,7 +939,10 @@ PeleLM::fillcoarsepatch_state( // Fill the auxiliaries void PeleLM::fillcoarsepatch_aux( - int lev, const amrex::Real a_time, amrex::MultiFab& a_aux, int nGhost) + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_aux, + const int nGhost) { AMREX_ASSERT(lev > 0); ProbParm const* lprobparm = prob_parm_d; @@ -940,7 +973,10 @@ PeleLM::fillcoarsepatch_aux( // Fill the grad P void PeleLM::fillcoarsepatch_gradp( - int lev, const amrex::Real a_time, amrex::MultiFab& a_gp, int nGhost) + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_gp, + const int nGhost) { // Interpolator auto* mapper = getInterpolator(m_regrid_interp_method); @@ -960,7 +996,10 @@ PeleLM::fillcoarsepatch_gradp( // Fill the divu void PeleLM::fillcoarsepatch_divu( - int lev, const amrex::Real a_time, amrex::MultiFab& a_divu, int nGhost) + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_divu, + const int nGhost) { // Interpolator auto* mapper = getInterpolator(m_regrid_interp_method); @@ -979,7 +1018,10 @@ PeleLM::fillcoarsepatch_divu( // Fill coarse patch of reaction void PeleLM::fillcoarsepatch_reaction( - int lev, const amrex::Real a_time, amrex::MultiFab& a_I_R, int nGhost) + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_I_R, + const int nGhost) { // Interpolator auto* mapper = getInterpolator(m_regrid_interp_method); @@ -999,7 +1041,10 @@ PeleLM::fillcoarsepatch_reaction( // Fill coarse patch of chem function call void PeleLM::fillcoarsepatch_chemFunctCall( - int lev, const amrex::Real a_time, amrex::MultiFab& a_fctC, int nGhost) + const int lev, + const amrex::Real a_time, + amrex::MultiFab& a_fctC, + const int nGhost) { // Interpolator auto* mapper = getInterpolator(m_regrid_interp_method); @@ -1019,18 +1064,19 @@ PeleLM::fillcoarsepatch_chemFunctCall( // Fill the inflow boundary of a velocity MF // used for velocity projection void -PeleLM::setInflowBoundaryVel(amrex::MultiFab& a_vel, int lev, TimeStamp a_time) +PeleLM::setInflowBoundaryVel( + amrex::MultiFab& a_vel, const int lev, const TimeStamp a_time) { BL_PROFILE("PeleLMeX::setInflowBoundaryVel()"); - amrex::Real time = getTime(lev, a_time); + const amrex::Real time = getTime(lev, a_time); // Create a dummy BCRec from Velocity BCRec keeping only Inflow and set the // other to bogus auto realVelBCRec = fetchBCRecArray(VELX, AMREX_SPACEDIM); amrex::Vector dummyVelBCRec(AMREX_SPACEDIM); - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - for (int idim2 = 0; idim2 < AMREX_SPACEDIM; idim2++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + for (int idim2 = 0; idim2 < AMREX_SPACEDIM; ++idim2) { if (realVelBCRec[idim].lo(idim2) == amrex::BCType::ext_dir) { dummyVelBCRec[idim].setLo(idim2, amrex::BCType::ext_dir); } else { @@ -1063,7 +1109,10 @@ PeleLM::setInflowBoundaryVel(amrex::MultiFab& a_vel, int lev, TimeStamp a_time) void PeleLM::fillTurbInflow( - amrex::MultiFab& a_vel, int vel_comp, int lev, const amrex::Real a_time) + amrex::MultiFab& a_vel, + const int vel_comp, + const int lev, + const amrex::Real a_time) { if (turb_inflow.is_initialized()) { @@ -1093,8 +1142,8 @@ PeleLM::fillTurbInflow( velBCRec[0].lo()[dir] == amrex::BCType::ext_dir && bndryBoxLO.ok()) { // Create box with ghost cells and set them to zero amrex::IntVect growVect(amrex::IntVect::TheUnitVector()); - int Grow = 4; // Being conservative - for (int n = 0; n < AMREX_SPACEDIM; n++) { + constexpr int Grow = 4; // Being conservative + for (int n = 0; n < AMREX_SPACEDIM; ++n) { growVect[n] = Grow; } growVect[dir] = 0; @@ -1116,8 +1165,8 @@ PeleLM::fillTurbInflow( velBCRec[0].hi()[dir] == amrex::BCType::ext_dir && bndryBoxHI.ok()) { // Create box with ghost cells and set them to zero amrex::IntVect growVect(amrex::IntVect::TheUnitVector()); - int Grow = 4; - for (int n = 0; n < AMREX_SPACEDIM; n++) { + constexpr int Grow = 4; + for (int n = 0; n < AMREX_SPACEDIM; ++n) { growVect[n] = Grow; } growVect[dir] = 0; diff --git a/Source/PeleLMeX_BCfill.H b/Source/PeleLMeX_BCfill.H index 89b9b1a4d..fef1acae9 100644 --- a/Source/PeleLMeX_BCfill.H +++ b/Source/PeleLMeX_BCfill.H @@ -54,10 +54,10 @@ struct PeleLMCCFillExtDirState auto iv_3d = amrex::IntVectND<3>(0); AMREX_D_TERM(iv_3d[0] = iv[0];, iv_3d[1] = iv[1];, iv_3d[2] = iv[2];); - for (int idir = 0; idir < AMREX_SPACEDIM; idir++) { + for (int idir = 0; idir < AMREX_SPACEDIM; ++idir) { // Low - for (int n = 0; n < amrex::min(numcomp, NVAR); n++) { + for (int n = 0; n < amrex::min(numcomp, NVAR); ++n) { // bcnormal handles all the state components at once amrex::Real s_ext[NVAR] = {0.0}; @@ -91,7 +91,7 @@ struct PeleLMCCFillExtDirState } // High - for (int n = 0; n < amrex::min(numcomp, NVAR); n++) { + for (int n = 0; n < amrex::min(numcomp, NVAR); ++n) { // bcnormal handles all the state components at once amrex::Real s_ext[NVAR] = {0.0}; @@ -174,10 +174,10 @@ struct PeleLMCCFillExtDirAux auto aux_ext = aux.cellData(iv[0], iv[1], iv[2]); #endif - for (int idir = 0; idir < AMREX_SPACEDIM; idir++) { + for (int idir = 0; idir < AMREX_SPACEDIM; ++idir) { // Low - for (int n = 0; n < m_nAux; n++) { + for (int n = 0; n < m_nAux; ++n) { // Get current state component BC const int* bc = bcr[n].data(); @@ -190,7 +190,7 @@ struct PeleLMCCFillExtDirAux } } // High - for (int n = 0; n < m_nAux; n++) { + for (int n = 0; n < m_nAux; ++n) { // Get current state component BC const int* bc = bcr[n].data(); @@ -249,7 +249,7 @@ struct PeleLMCCFillExtDirSpec auto iv_3d = amrex::IntVectND<3>(0); AMREX_D_TERM(iv_3d[0] = iv[0];, iv_3d[1] = iv[1];, iv_3d[2] = iv[2];); - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { // Get the species component BC const int* bc = bcr[n].data(); @@ -257,7 +257,7 @@ struct PeleLMCCFillExtDirSpec // bcnormal handles all the state components at once amrex::Real s_ext[NVAR] = {0.0}; - for (int idir = 0; idir < AMREX_SPACEDIM; idir++) { + for (int idir = 0; idir < AMREX_SPACEDIM; ++idir) { // Low if ((bc[idir] == amrex::BCType::ext_dir) and (iv[idir] < domlo[idir])) { @@ -344,7 +344,7 @@ struct PeleLMCCFillExtDirDens // bcnormal handles all the state components at once amrex::Real s_ext[NVAR] = {0.0}; - for (int idir = 0; idir < AMREX_SPACEDIM; idir++) { + for (int idir = 0; idir < AMREX_SPACEDIM; ++idir) { // Low if ((bc[idir] == amrex::BCType::ext_dir) and (iv[idir] < domlo[idir])) { @@ -428,7 +428,7 @@ struct PeleLMCCFillExtDirRhoH // bcnormal handles all the state components at once amrex::Real s_ext[NVAR] = {0.0}; - for (int idir = 0; idir < AMREX_SPACEDIM; idir++) { + for (int idir = 0; idir < AMREX_SPACEDIM; ++idir) { // Low if ((bc[idir] == amrex::BCType::ext_dir) and (iv[idir] < domlo[idir])) { @@ -511,7 +511,7 @@ struct PeleLMCCFillExtDirTemp // bcnormal handles all the state components at once amrex::Real s_ext[NVAR] = {0.0}; - for (int idir = 0; idir < AMREX_SPACEDIM; idir++) { + for (int idir = 0; idir < AMREX_SPACEDIM; ++idir) { // Low if ((bc[idir] == amrex::BCType::ext_dir) and (iv[idir] < domlo[idir])) { @@ -596,7 +596,7 @@ struct PeleLMCCFillExtDirnE // bcnormal handles all the state components at once amrex::Real s_ext[NVAR] = {0.0}; - for (int idir = 0; idir < AMREX_SPACEDIM; idir++) { + for (int idir = 0; idir < AMREX_SPACEDIM; ++idir) { // Low if ((bc[idir] == amrex::BCType::ext_dir) and (iv[idir] < domlo[idir])) { @@ -682,7 +682,7 @@ struct PeleLMCCFillExtDirPhiV // bcnormal handles all the state components at once amrex::Real s_ext[NVAR] = {0.0}; - for (int idir = 0; idir < AMREX_SPACEDIM; idir++) { + for (int idir = 0; idir < AMREX_SPACEDIM; ++idir) { // Low if ((bc[idir] == amrex::BCType::ext_dir) and (iv[idir] < domlo[idir])) { diff --git a/Source/PeleLMeX_BCfillEB.H b/Source/PeleLMeX_BCfillEB.H index 1974f3e7e..b0e1890a8 100644 --- a/Source/PeleLMeX_BCfillEB.H +++ b/Source/PeleLMeX_BCfillEB.H @@ -140,7 +140,7 @@ struct PeleLMFillBCStateEB ebh.bcnormal_eb(xface, bcnorm, stateIn, stateExt, time, geom, *lprobparm); // Extract requested entries - for (int n = 0; n < numcomp; n++) { + for (int n = 0; n < numcomp; ++n) { ebState(i, j, k, n) = stateExt[firstcomp + n]; } } diff --git a/Source/PeleLMeX_BPatch.H b/Source/PeleLMeX_BPatch.H index 69a099df1..cae32fd4d 100644 --- a/Source/PeleLMeX_BPatch.H +++ b/Source/PeleLMeX_BPatch.H @@ -72,7 +72,7 @@ public: } #elif (AMREX_SPACEDIM == 3) amrex::Real zp; - const amrex::Real sqrt2 = sqrt(2.0); + constexpr amrex::Real sqrt2 = 1.4142135623730951; xp = point_coordinate[0]; yp = point_coordinate[1]; @@ -82,9 +82,9 @@ public: inside = true; } else if (m_patchtype_num == 1) { - amrex::Real patch_circle_radius_touse = + const amrex::Real patch_circle_radius_touse = m_patch_circle_radius + dx * sqrt2 / 2.0; - amrex::Real rad = sqrt( + const amrex::Real rad = sqrt( (xp - m_patch_circle_center[0]) * (xp - m_patch_circle_center[0]) + (yp - m_patch_circle_center[1]) * (yp - m_patch_circle_center[1]) + (zp - m_patch_circle_center[2]) * (zp - m_patch_circle_center[2])); @@ -116,12 +116,12 @@ public: inside = true; } } else if (m_patchtype_num == 3) { - amrex::Real patch_circ_ann_outer_radius_touse = + const amrex::Real patch_circ_ann_outer_radius_touse = m_patch_circ_ann_outer_radius + dx * sqrt2 / 2.0; - amrex::Real patch_circ_ann_inner_radius_touse = + const amrex::Real patch_circ_ann_inner_radius_touse = m_patch_circ_ann_inner_radius - dx * sqrt2 / 2.0; - amrex::Real rad = sqrt( + const amrex::Real rad = sqrt( (xp - m_patch_circ_ann_center[0]) * (xp - m_patch_circ_ann_center[0]) + (yp - m_patch_circ_ann_center[1]) * diff --git a/Source/PeleLMeX_BPatch.cpp b/Source/PeleLMeX_BPatch.cpp index 527e611b0..346e92d1a 100644 --- a/Source/PeleLMeX_BPatch.cpp +++ b/Source/PeleLMeX_BPatch.cpp @@ -156,19 +156,19 @@ BPatch::BPatch(const std::string& patch_name, const amrex::Geometry& geom) pele::physics::eos::speciesNames(names); names.resize(names.size()); - for (int n = 0; n < names.size(); n++) { + for (int n = 0; n < names.size(); ++n) { m_bpdata_h.speciesIndex[n] = -1; } - for (int m = 0; m < m_bpdata_h.num_species; m++) { - for (int n = 0; n < names.size(); n++) { + for (int m = 0; m < m_bpdata_h.num_species; ++m) { + for (int n = 0; n < names.size(); ++n) { if (speciesList[m] == names[n]) { m_bpdata_h.speciesIndex[m] = n; } } } - for (int n = 0; n < m_bpdata_h.num_species; n++) { + for (int n = 0; n < m_bpdata_h.num_species; ++n) { if (m_bpdata_h.speciesIndex[n] == -1) { std::string msg = "\nError! Unable to find species index " + std::to_string(n); diff --git a/Source/PeleLMeX_Data.cpp b/Source/PeleLMeX_Data.cpp index 5cd358993..8260d7e79 100644 --- a/Source/PeleLMeX_Data.cpp +++ b/Source/PeleLMeX_Data.cpp @@ -4,12 +4,12 @@ PeleLM::LevelData::LevelData( amrex::BoxArray const& ba, amrex::DistributionMapping const& dm, amrex::FabFactory const& factory, - int a_incompressible, - int a_has_divu, - int a_nAux, - int a_nGrowState, - int a_use_soret, - int a_do_les) + const int a_incompressible, + const int a_has_divu, + const int a_nAux, + const int a_nGrowState, + const int a_use_soret, + const int a_do_les) { if (a_incompressible != 0) { state.define( @@ -61,9 +61,10 @@ PeleLM::LevelDataReact::LevelDataReact( const amrex::DistributionMapping& dm, const amrex::FabFactory& factory) { - int IRsize = NUM_SPECIES; #ifdef PELE_USE_PLASMA - IRsize += 1; + constexpr int IRsize = NUM_SPECIES + 1; +#else + constexpr int IRsize = NUM_SPECIES; #endif I_R.define(ba, dm, IRsize, 0, amrex::MFInfo(), factory); functC.define(ba, dm, 1, 0, amrex::MFInfo(), factory); @@ -74,7 +75,7 @@ PeleLM::LevelDataNLSolve::LevelDataNLSolve( amrex::BoxArray const& ba, amrex::DistributionMapping const& dm, amrex::FabFactory const& factory, - int a_nGrow) + const int a_nGrow) { nlState.define(ba, dm, 2, a_nGrow, amrex::MFInfo(), factory); nlResid.define(ba, dm, 2, a_nGrow, amrex::MFInfo(), factory); @@ -90,67 +91,64 @@ PeleLM::LevelDataNLSolve::LevelDataNLSolve( #endif PeleLM::AdvanceDiffData::AdvanceDiffData( - int a_finestLevel, + const int a_finestLevel, const amrex::Vector& ba, const amrex::Vector& dm, const amrex::Vector>>& factory, - int nGrowAdv, - int a_use_wbar, - int a_use_soret, - int a_nAux, - int is_init) + const int nGrowAdv, + const int a_use_wbar, + const int a_use_soret, + const int a_nAux, + const int is_init) { if (is_init != 0) { // All I need is a container for a single diffusion term - // Resize Vectors - Dnp1.resize(a_finestLevel + 1); - + Dnp1.reserve(a_finestLevel + 1); // Define MFs - for (int lev = 0; lev <= a_finestLevel; lev++) { - Dnp1[lev].define( + for (int lev = 0; lev <= a_finestLevel; ++lev) { + Dnp1.emplace_back( ba[lev], dm[lev], NUM_SPECIES + 2, nGrowAdv, amrex::MFInfo(), *factory[lev]); } - if (a_nAux > 0) { - Dnp1_aux.resize(a_finestLevel + 1); - for (int lev = 0; lev <= a_finestLevel; lev++) { - Dnp1_aux[lev].define( + Dnp1_aux.reserve(a_finestLevel + 1); + for (int lev = 0; lev <= a_finestLevel; ++lev) { + Dnp1_aux.emplace_back( ba[lev], dm[lev], a_nAux, nGrowAdv, amrex::MFInfo(), *factory[lev]); } } } else { - // Resize Vectors - Dn.resize(a_finestLevel + 1); - Dnp1.resize(a_finestLevel + 1); - Dhat.resize(a_finestLevel + 1); + // Reserve/resize Vectors + Dn.reserve(a_finestLevel + 1); + Dnp1.reserve(a_finestLevel + 1); + Dhat.reserve(a_finestLevel + 1); if (a_nAux > 0) { - Dn_aux.resize(a_finestLevel + 1); - Dnp1_aux.resize(a_finestLevel + 1); - Dhat_aux.resize(a_finestLevel + 1); + Dn_aux.reserve(a_finestLevel + 1); + Dnp1_aux.reserve(a_finestLevel + 1); + Dhat_aux.reserve(a_finestLevel + 1); } if (a_use_wbar != 0) { - Dwbar.resize(a_finestLevel + 1); + Dwbar.reserve(a_finestLevel + 1); wbar_fluxes.resize(a_finestLevel + 1); } if (a_use_soret != 0) { - DT.resize(a_finestLevel + 1); + DT.reserve(a_finestLevel + 1); soret_fluxes.resize(a_finestLevel + 1); } // Define MFs - for (int lev = 0; lev <= a_finestLevel; lev++) { - Dn[lev].define( + for (int lev = 0; lev <= a_finestLevel; ++lev) { + Dn.emplace_back( ba[lev], dm[lev], NUM_SPECIES + 2, nGrowAdv, amrex::MFInfo(), *factory[lev]); - Dnp1[lev].define( + Dnp1.emplace_back( ba[lev], dm[lev], NUM_SPECIES + 2, nGrowAdv, amrex::MFInfo(), *factory[lev]); - Dhat[lev].define( + Dhat.emplace_back( ba[lev], dm[lev], NUM_SPECIES + 2, nGrowAdv, amrex::MFInfo(), *factory[lev]); if (a_use_wbar != 0) { - Dwbar[lev].define( + Dwbar.emplace_back( ba[lev], dm[lev], NUM_SPECIES, nGrowAdv, amrex::MFInfo(), *factory[lev]); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { @@ -161,7 +159,7 @@ PeleLM::AdvanceDiffData::AdvanceDiffData( } } if (a_use_soret != 0) { - DT[lev].define( + DT.emplace_back( ba[lev], dm[lev], NUM_SPECIES, nGrowAdv, amrex::MFInfo(), *factory[lev]); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { @@ -172,11 +170,11 @@ PeleLM::AdvanceDiffData::AdvanceDiffData( } } if (a_nAux > 0) { - Dn_aux[lev].define( + Dn_aux.emplace_back( ba[lev], dm[lev], a_nAux, nGrowAdv, amrex::MFInfo(), *factory[lev]); - Dnp1_aux[lev].define( + Dnp1_aux.emplace_back( ba[lev], dm[lev], a_nAux, nGrowAdv, amrex::MFInfo(), *factory[lev]); - Dhat_aux[lev].define( + Dhat_aux.emplace_back( ba[lev], dm[lev], a_nAux, nGrowAdv, amrex::MFInfo(), *factory[lev]); } } @@ -184,34 +182,34 @@ PeleLM::AdvanceDiffData::AdvanceDiffData( } PeleLM::AdvanceAdvData::AdvanceAdvData( - int a_finestLevel, + const int a_finestLevel, const amrex::Vector& ba, const amrex::Vector& dm, const amrex::Vector>>& factory, - int a_incompressible, - int a_nAux, - int nGrowAdv, - int nGrowMAC) + const int a_incompressible, + const int a_nAux, + const int nGrowAdv, + const int nGrowMAC) { // Resize Vectors umac.resize(a_finestLevel + 1); - AofS.resize(a_finestLevel + 1); + AofS.reserve(a_finestLevel + 1); if (a_incompressible == 0) { - chi.resize(a_finestLevel + 1); - Forcing.resize(a_finestLevel + 1); - mac_divu.resize(a_finestLevel + 1); + chi.reserve(a_finestLevel + 1); + Forcing.reserve(a_finestLevel + 1); + mac_divu.reserve(a_finestLevel + 1); } if (a_nAux > 0) { - AofS_aux.resize(a_finestLevel + 1); - Forcing_aux.resize(a_finestLevel + 1); + AofS_aux.reserve(a_finestLevel + 1); + Forcing_aux.reserve(a_finestLevel + 1); } #ifdef PELE_USE_PLASMA uDrift.resize(a_finestLevel + 1); #endif // Define MFs - for (int lev = 0; lev <= a_finestLevel; lev++) { + for (int lev = 0; lev <= a_finestLevel; ++lev) { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { const amrex::BoxArray& faceba = amrex::convert(ba[lev], amrex::IntVect::TheDimensionVector(idim)); @@ -223,28 +221,28 @@ PeleLM::AdvanceAdvData::AdvanceAdvData( #endif } if (a_incompressible != 0) { - AofS[lev].define( + AofS.emplace_back( ba[lev], dm[lev], AMREX_SPACEDIM, 0, amrex::MFInfo(), *factory[lev]); } else { - AofS[lev].define( + AofS.emplace_back( ba[lev], dm[lev], NVAR, 0, amrex::MFInfo(), *factory[lev]); - chi[lev].define(ba[lev], dm[lev], 1, 1, amrex::MFInfo(), *factory[lev]); + chi.emplace_back(ba[lev], dm[lev], 1, 1, amrex::MFInfo(), *factory[lev]); #ifdef PELE_USE_PLASMA - Forcing[lev].define( + Forcing.emplace_back( ba[lev], dm[lev], NUM_SPECIES + 2, nGrowAdv, amrex::MFInfo(), *factory[lev]); // Species + TEMP + nE #else - Forcing[lev].define( + Forcing.emplace_back( ba[lev], dm[lev], NUM_SPECIES + 1, nGrowAdv, amrex::MFInfo(), *factory[lev]); // Species + TEMP #endif - mac_divu[lev].define( + mac_divu.emplace_back( ba[lev], dm[lev], 1, nGrowAdv, amrex::MFInfo(), *factory[lev]); } if (a_nAux > 0) { - AofS_aux[lev].define( + AofS_aux.emplace_back( ba[lev], dm[lev], a_nAux, 0, amrex::MFInfo(), *factory[lev]); - Forcing_aux[lev].define( + Forcing_aux.emplace_back( ba[lev], dm[lev], a_nAux, nGrowAdv, amrex::MFInfo(), *factory[lev]); } } @@ -254,7 +252,7 @@ void PeleLM::copyStateNewToOld(int nGhost) { AMREX_ASSERT(nGhost <= m_nGrowState); - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { if (m_incompressible != 0) { amrex::MultiFab::Copy( m_leveldata_old[lev]->state, m_leveldata_new[lev]->state, 0, 0, @@ -280,7 +278,7 @@ PeleLM::copyStateNewToOld(int nGhost) void PeleLM::copyPressNewToOld() { - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { amrex::MultiFab::Copy( m_leveldata_old[lev]->press, m_leveldata_new[lev]->press, 0, 0, 1, 1); amrex::MultiFab::Copy( @@ -293,7 +291,7 @@ void PeleLM::copyStateOldToNew(int nGhost) { AMREX_ASSERT(nGhost <= m_nGrowState); - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { if (m_incompressible != 0) { amrex::MultiFab::Copy( m_leveldata_new[lev]->state, m_leveldata_old[lev]->state, 0, 0, @@ -319,7 +317,7 @@ PeleLM::copyStateOldToNew(int nGhost) void PeleLM::copyTransportOldToNew() { - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { amrex::MultiFab::Copy( m_leveldata_new[lev]->visc_cc, m_leveldata_old[lev]->visc_cc, 0, 0, 1, 1); if (m_incompressible == 0) { @@ -347,9 +345,9 @@ PeleLM::copyTransportOldToNew() } void -PeleLM::copyDiffusionOldToNew(std::unique_ptr& diffData) +PeleLM::copyDiffusionOldToNew(const std::unique_ptr& diffData) { - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { amrex::MultiFab::Copy( diffData->Dnp1[lev], diffData->Dn[lev], 0, 0, NUM_SPECIES + 2, m_nGrowAdv); @@ -365,7 +363,7 @@ bool PeleLM::checkForNaNs() { bool contains_nan = false; - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { if ( m_leveldata_new[lev]->state.contains_nan() || m_leveldata_new[lev]->state.contains_inf()) { diff --git a/Source/PeleLMeX_Derive.cpp b/Source/PeleLMeX_Derive.cpp index 69c8ab103..c31bf4d32 100644 --- a/Source/PeleLMeX_Derive.cpp +++ b/Source/PeleLMeX_Derive.cpp @@ -111,7 +111,7 @@ PeleLMDeriveRec::numDerive() const noexcept } const std::string& -PeleLMDeriveRec::variableName(int comp) const noexcept +PeleLMDeriveRec::variableName(const int comp) const noexcept { if (comp < variable_names.size()) { return variable_names[comp]; @@ -126,7 +126,7 @@ PeleLMDeriveRec::variableComp(const std::string& a_name) const noexcept if (n_derive == 1) { return 0; } - for (int comp = 0; comp < n_derive; comp++) { + for (int comp = 0; comp < n_derive; ++comp) { if (variable_names[comp] == a_name) { return comp; } @@ -196,7 +196,7 @@ PeleLMDeriveList::canDerive(const std::string& name) const { for (const auto& li : lst) { // Can be either a component name ... - for (int i = 0; i < li.numDerive(); i++) { + for (int i = 0; i < li.numDerive(); ++i) { if (li.variableName(i) == name) { return true; } @@ -214,7 +214,7 @@ PeleLMDeriveList::get(const std::string& name) const { for (const auto& li : lst) { // Can be either a component name ... - for (int i = 0; i < li.numDerive(); i++) { + for (int i = 0; i < li.numDerive(); ++i) { if (li.variableName(i) == name) { return &li; } diff --git a/Source/PeleLMeX_DeriveFunc.cpp b/Source/PeleLMeX_DeriveFunc.cpp index 58f9c5734..47d20a475 100644 --- a/Source/PeleLMeX_DeriveFunc.cpp +++ b/Source/PeleLMeX_DeriveFunc.cpp @@ -37,9 +37,10 @@ pelelmex_dertemp( AMREX_ASSERT(!a_pelelm->m_incompressible); auto const in_dat = statefab.array(); auto der = derfab.array(dcomp); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - der(i, j, k) = in_dat(i, j, k, TEMP); - }); + amrex::ParallelFor( + bx, [der, in_dat] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + der(i, j, k) = in_dat(i, j, k, TEMP); + }); } // @@ -75,13 +76,15 @@ pelelmex_derheatrelease( auto const& Hi = EnthFab.array(); auto HRR = derfab.array(dcomp); auto const* leosparm = a_pelelm->eos_parms.device_parm(); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - getHGivenT(i, j, k, temp, Hi, leosparm); - HRR(i, j, k) = 0.0; - for (int n = 0; n < NUM_SPECIES; n++) { - HRR(i, j, k) -= Hi(i, j, k, n) * react(i, j, k, n); - } - }); + amrex::ParallelFor( + bx, [temp, Hi, leosparm, HRR, + react] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + getHGivenT(i, j, k, temp, Hi, leosparm); + HRR(i, j, k) = 0.0; + for (int n = 0; n < NUM_SPECIES; ++n) { + HRR(i, j, k) -= Hi(i, j, k, n) * react(i, j, k, n); + } + }); } // @@ -113,8 +116,9 @@ pelelmex_dermassfrac( auto const in_dat = statefab.array(); auto der = derfab.array(dcomp); amrex::ParallelFor( - bx, NUM_SPECIES, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { - amrex::Real rhoinv = 1.0 / in_dat(i, j, k, DENSITY); + bx, NUM_SPECIES, + [in_dat, der] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + const amrex::Real rhoinv = 1.0 / in_dat(i, j, k, DENSITY); der(i, j, k, n) = in_dat(i, j, k, FIRSTSPEC + n) * rhoinv; }); } @@ -147,19 +151,20 @@ pelelmex_dermolefrac( auto const in_dat = statefab.array(); auto der = derfab.array(dcomp); auto const* leosparm = a_pelelm->eos_parms.device_parm(); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - amrex::Real Yt[NUM_SPECIES] = {0.0}; - amrex::Real Xt[NUM_SPECIES] = {0.0}; - amrex::Real rhoinv = 1.0 / in_dat(i, j, k, DENSITY); - for (int n = 0; n < NUM_SPECIES; n++) { - Yt[n] = in_dat(i, j, k, FIRSTSPEC + n) * rhoinv; - } - auto eos = pele::physics::PhysicsType::eos(leosparm); - eos.Y2X(Yt, Xt); - for (int n = 0; n < NUM_SPECIES; n++) { - der(i, j, k, n) = Xt[n]; - } - }); + amrex::ParallelFor( + bx, [in_dat, leosparm, der] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + amrex::Real Yt[NUM_SPECIES] = {0.0}; + amrex::Real Xt[NUM_SPECIES] = {0.0}; + amrex::Real rhoinv = 1.0 / in_dat(i, j, k, DENSITY); + for (int n = 0; n < NUM_SPECIES; ++n) { + Yt[n] = in_dat(i, j, k, FIRSTSPEC + n) * rhoinv; + } + auto eos = pele::physics::PhysicsType::eos(leosparm); + eos.Y2X(Yt, Xt); + for (int n = 0; n < NUM_SPECIES; ++n) { + der(i, j, k, n) = Xt[n]; + } + }); } // @@ -190,12 +195,13 @@ pelelmex_derrhomrhoy( AMREX_ASSERT(!a_pelelm->m_incompressible); auto const in_dat = statefab.array(); auto der = derfab.array(dcomp); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - der(i, j, k, 0) = in_dat(i, j, k, DENSITY); - for (int n = 0; n < NUM_SPECIES; n++) { - der(i, j, k, 0) -= in_dat(i, j, k, FIRSTSPEC + n); - } - }); + amrex::ParallelFor( + bx, [der, in_dat] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + der(i, j, k, 0) = in_dat(i, j, k, DENSITY); + for (int n = 0; n < NUM_SPECIES; ++n) { + der(i, j, k, 0) -= in_dat(i, j, k, FIRSTSPEC + n); + } + }); } // @@ -220,19 +226,20 @@ pelelmex_deravgpress( AMREX_ASSERT(derfab.box().contains(bx)); auto const in_dat = pressfab.array(); auto der = derfab.array(dcomp); - amrex::Real factor = 1.0 / (AMREX_D_TERM(2.0, *2.0, *2.0)); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - der(i, j, k) = - factor * (in_dat(i + 1, j, k) + in_dat(i, j, k) + amrex::ParallelFor( + bx, [der, in_dat] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + constexpr amrex::Real factor = 1.0 / (AMREX_D_TERM(2.0, *2.0, *2.0)); + der(i, j, k) = + factor * (in_dat(i + 1, j, k) + in_dat(i, j, k) #if (AMREX_SPACEDIM >= 2) - + in_dat(i + 1, j + 1, k) + in_dat(i, j + 1, k) + + in_dat(i + 1, j + 1, k) + in_dat(i, j + 1, k) #if (AMREX_SPACEDIM == 3) - + in_dat(i + 1, j, k + 1) + in_dat(i, j, k + 1) + - in_dat(i + 1, j + 1, k + 1) + in_dat(i, j + 1, k + 1) + + in_dat(i + 1, j, k + 1) + in_dat(i, j, k + 1) + + in_dat(i + 1, j + 1, k + 1) + in_dat(i, j + 1, k + 1) #endif #endif - ); - }); + ); + }); } // @@ -258,11 +265,12 @@ pelelmex_dermgvel( AMREX_ASSERT(statefab.box().contains(bx)); auto const vel = statefab.array(VELX); auto der = derfab.array(dcomp); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - der(i, j, k) = std::sqrt((AMREX_D_TERM( - vel(i, j, k, 0) * vel(i, j, k, 0), +vel(i, j, k, 1) * vel(i, j, k, 1), - +vel(i, j, k, 2) * vel(i, j, k, 2)))); - }); + amrex::ParallelFor( + bx, [der, vel] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + der(i, j, k) = std::sqrt((AMREX_D_TERM( + vel(i, j, k, 0) * vel(i, j, k, 0), +vel(i, j, k, 1) * vel(i, j, k, 1), + +vel(i, j, k, 2) * vel(i, j, k, 2)))); + }); } // @@ -299,58 +307,68 @@ pelelmex_dermgvort( auto typ = flags.getType(bx); if (typ == amrex::FabType::covered) { - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - vort_arr(i, j, k) = 0.0; - }); + amrex::ParallelFor( + bx, [vort_arr] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + vort_arr(i, j, k) = 0.0; + }); } else if (typ == amrex::FabType::singlevalued) { const auto& flag_fab = flags.const_array(); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - constexpr amrex::Real c0 = -1.5; - constexpr amrex::Real c1 = 2.0; - constexpr amrex::Real c2 = -0.5; - if (flag_fab(i, j, k).isCovered()) { - vort_arr(i, j, k) = 0.0; - } else { - // Define interpolation lambda - auto onesided = []( - const amrex::Real& v0, const amrex::Real& v1, - const amrex::Real& v2) -> amrex::Real { - return c0 * v0 + c1 * v1 + c2 * v2; - }; - - amrex::Real vx = 0.0; - amrex::Real uy = 0.0; -#if (AMREX_SPACEDIM == 2) - // Need to check if there are covered cells in neighbours -- - // -- if so, use one-sided difference computation (but still quadratic) - if (!flag_fab(i, j, k).isConnected(1, 0, 0)) { - vx = -onesided( - dat_arr(i, j, k, 1), dat_arr(i - 1, j, k, 1), - dat_arr(i - 2, j, k, 1)) * - idx; - } else if (!flag_fab(i, j, k).isConnected(-1, 0, 0)) { - vx = onesided( - dat_arr(i, j, k, 1), dat_arr(i + 1, j, k, 1), - dat_arr(i + 2, j, k, 1)) * - idx; - } else { - vx = 0.5 * (dat_arr(i + 1, j, k, 1) - dat_arr(i - 1, j, k, 1)) * idx; - } - // Do the same in y-direction - if (!flag_fab(i, j, k).isConnected(0, 1, 0)) { - uy = -onesided( - dat_arr(i, j, k, 0), dat_arr(i, j - 1, k, 0), - dat_arr(i, j - 2, k, 0)) * - idy; - } else if (!flag_fab(i, j, k).isConnected(0, -1, 0)) { - uy = onesided( - dat_arr(i, j, k, 0), dat_arr(i, j + 1, k, 0), - dat_arr(i, j + 2, k, 0)) * - idy; + amrex::ParallelFor( + bx, [vort_arr, dat_arr, flag_fab, idx, idy +#if AMREX_SPACEDIM == 3 + , + idz +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + constexpr amrex::Real c0 = -1.5; + constexpr amrex::Real c1 = 2.0; + constexpr amrex::Real c2 = -0.5; + if (flag_fab(i, j, k).isCovered()) { + vort_arr(i, j, k) = 0.0; } else { - uy = 0.5 * (dat_arr(i, j + 1, k, 0) - dat_arr(i, j - 1, k, 0)) * idy; - } - vort_arr(i, j, k) = std::abs(vx - uy); + // Define interpolation lambda + auto onesided = []( + const amrex::Real& v0, const amrex::Real& v1, + const amrex::Real& v2) -> amrex::Real { + return c0 * v0 + c1 * v1 + c2 * v2; + }; + + amrex::Real vx = 0.0; + amrex::Real uy = 0.0; +#if (AMREX_SPACEDIM == 2) + // Need to check if there are covered cells in neighbours -- + // -- if so, use one-sided difference computation (but still + // quadratic) + if (!flag_fab(i, j, k).isConnected(1, 0, 0)) { + vx = -onesided( + dat_arr(i, j, k, 1), dat_arr(i - 1, j, k, 1), + dat_arr(i - 2, j, k, 1)) * + idx; + } else if (!flag_fab(i, j, k).isConnected(-1, 0, 0)) { + vx = onesided( + dat_arr(i, j, k, 1), dat_arr(i + 1, j, k, 1), + dat_arr(i + 2, j, k, 1)) * + idx; + } else { + vx = + 0.5 * (dat_arr(i + 1, j, k, 1) - dat_arr(i - 1, j, k, 1)) * idx; + } + // Do the same in y-direction + if (!flag_fab(i, j, k).isConnected(0, 1, 0)) { + uy = -onesided( + dat_arr(i, j, k, 0), dat_arr(i, j - 1, k, 0), + dat_arr(i, j - 2, k, 0)) * + idy; + } else if (!flag_fab(i, j, k).isConnected(0, -1, 0)) { + uy = onesided( + dat_arr(i, j, k, 0), dat_arr(i, j + 1, k, 0), + dat_arr(i, j + 2, k, 0)) * + idy; + } else { + uy = + 0.5 * (dat_arr(i, j + 1, k, 0) - dat_arr(i, j - 1, k, 0)) * idy; + } + vort_arr(i, j, k) = std::abs(vx - uy); #elif (AMREX_SPACEDIM == 3) amrex::Real wx = 0.0; @@ -396,32 +414,38 @@ pelelmex_dermgvort( } vort_arr(i,j,k) = std::sqrt((wy-vz)*(wy-vz) + (uz-wx)*(uz-wx) + (vx-uy)*(vx-uy)); #endif - } - }); + } + }); } else #endif // Check on EB { - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + amrex::ParallelFor( + bx, [dat_arr, vort_arr, idx, idy +#if (AMREX_SPACEDIM == 3) + , + idz +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { #if (AMREX_SPACEDIM == 2) - amrex::Real vx = - 0.5 * (dat_arr(i + 1, j, k, 1) - dat_arr(i - 1, j, k, 1)) * idx; - amrex::Real uy = - 0.5 * (dat_arr(i, j + 1, k, 0) - dat_arr(i, j - 1, k, 0)) * idy; - vort_arr(i, j, k) = std::abs(vx - uy); + const amrex::Real vx = + 0.5 * (dat_arr(i + 1, j, k, 1) - dat_arr(i - 1, j, k, 1)) * idx; + const amrex::Real uy = + 0.5 * (dat_arr(i, j + 1, k, 0) - dat_arr(i, j - 1, k, 0)) * idy; + vort_arr(i, j, k) = std::abs(vx - uy); #elif (AMREX_SPACEDIM == 3) - amrex::Real vx = 0.5 * (dat_arr(i+1,j,k,1) - dat_arr(i-1,j,k,1)) * idx; - amrex::Real wx = 0.5 * (dat_arr(i+1,j,k,2) - dat_arr(i-1,j,k,2)) * idx; - - amrex::Real uy = 0.5 * (dat_arr(i,j+1,k,0) - dat_arr(i,j-1,k,0)) * idy; - amrex::Real wy = 0.5 * (dat_arr(i,j+1,k,2) - dat_arr(i,j-1,k,2)) * idy; - - amrex::Real uz = 0.5 * (dat_arr(i,j,k+1,0) - dat_arr(i,j,k-1,0)) * idz; - amrex::Real vz = 0.5 * (dat_arr(i,j,k+1,1) - dat_arr(i,j,k-1,1)) * idz; - - vort_arr(i,j,k) = std::sqrt((wy-vz)*(wy-vz) + (uz-wx)*(uz-wx) + (vx-uy)*(vx-uy)); + const amrex::Real vx = 0.5 * (dat_arr(i+1,j,k,1) - dat_arr(i-1,j,k,1)) * idx; + const amrex::Real wx = 0.5 * (dat_arr(i+1,j,k,2) - dat_arr(i-1,j,k,2)) * idx; + + const amrex::Real uy = 0.5 * (dat_arr(i,j+1,k,0) - dat_arr(i,j-1,k,0)) * idy; + const amrex::Real wy = 0.5 * (dat_arr(i,j+1,k,2) - dat_arr(i,j-1,k,2)) * idy; + + const amrex::Real uz = 0.5 * (dat_arr(i,j,k+1,0) - dat_arr(i,j,k-1,0)) * idz; + const amrex::Real vz = 0.5 * (dat_arr(i,j,k+1,1) - dat_arr(i,j,k-1,1)) * idz; + + vort_arr(i,j,k) = std::sqrt((wy-vz)*(wy-vz) + (uz-wx)*(uz-wx) + (vx-uy)*(vx-uy)); #endif - }); + }); } } @@ -442,9 +466,8 @@ pelelmex_dervort( amrex::Real /*time*/, const amrex::Vector& /*bcrec*/, int /*level*/) - { - amrex::ignore_unused(ncomp, bx); + amrex::ignore_unused(ncomp); AMREX_ASSERT(derfab.box().contains(bx)); AMREX_ASSERT(statefab.box().contains(bx)); AMREX_ASSERT(derfab.nComp() >= dcomp + ncomp); @@ -464,60 +487,70 @@ pelelmex_dervort( if (typ == amrex::FabType::covered) { amrex::ParallelFor( - bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + bx, ncomp, + [vort_arr] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { vort_arr(i, j, k, n) = 0.0; }); } else if (typ == amrex::FabType::singlevalued) { const auto& flag_fab = flags.const_array(); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - constexpr amrex::Real c0 = -1.5; - constexpr amrex::Real c1 = 2.0; - constexpr amrex::Real c2 = -0.5; - if (flag_fab(i, j, k).isCovered()) { - for (int n{0}; n < ncomp; ++n) { - vort_arr(i, j, k, n) = 0.0; - } - } else { - // Define interpolation lambda - auto onesided = []( - const amrex::Real& v0, const amrex::Real& v1, - const amrex::Real& v2) -> amrex::Real { - return c0 * v0 + c1 * v1 + c2 * v2; - }; - - amrex::Real vx = 0.0; - amrex::Real uy = 0.0; -#if (AMREX_SPACEDIM == 2) - // Need to check if there are covered cells in neighbours -- - // -- if so, use one-sided difference computation (but still quadratic) - if (!flag_fab(i, j, k).isConnected(1, 0, 0)) { - vx = -onesided( - dat_arr(i, j, k, 1), dat_arr(i - 1, j, k, 1), - dat_arr(i - 2, j, k, 1)) * - idx; - } else if (!flag_fab(i, j, k).isConnected(-1, 0, 0)) { - vx = onesided( - dat_arr(i, j, k, 1), dat_arr(i + 1, j, k, 1), - dat_arr(i + 2, j, k, 1)) * - idx; - } else { - vx = 0.5 * (dat_arr(i + 1, j, k, 1) - dat_arr(i - 1, j, k, 1)) * idx; - } - // Do the same in y-direction - if (!flag_fab(i, j, k).isConnected(0, 1, 0)) { - uy = -onesided( - dat_arr(i, j, k, 0), dat_arr(i, j - 1, k, 0), - dat_arr(i, j - 2, k, 0)) * - idy; - } else if (!flag_fab(i, j, k).isConnected(0, -1, 0)) { - uy = onesided( - dat_arr(i, j, k, 0), dat_arr(i, j + 1, k, 0), - dat_arr(i, j + 2, k, 0)) * - idy; + amrex::ParallelFor( + bx, [vort_arr, flag_fab, dat_arr, ncomp, idx, idy +#if (AMREX_SPACEDIM == 3) + , + idz +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + constexpr amrex::Real c0 = -1.5; + constexpr amrex::Real c1 = 2.0; + constexpr amrex::Real c2 = -0.5; + if (flag_fab(i, j, k).isCovered()) { + for (int n{0}; n < ncomp; ++n) { + vort_arr(i, j, k, n) = 0.0; + } } else { - uy = 0.5 * (dat_arr(i, j + 1, k, 0) - dat_arr(i, j - 1, k, 0)) * idy; - } - vort_arr(i, j, k) = vx - uy; + // Define interpolation lambda + auto onesided = []( + const amrex::Real& v0, const amrex::Real& v1, + const amrex::Real& v2) -> amrex::Real { + return c0 * v0 + c1 * v1 + c2 * v2; + }; + + amrex::Real vx = 0.0; + amrex::Real uy = 0.0; +#if (AMREX_SPACEDIM == 2) + // Need to check if there are covered cells in neighbours -- + // -- if so, use one-sided difference computation (but still + // quadratic) + if (!flag_fab(i, j, k).isConnected(1, 0, 0)) { + vx = -onesided( + dat_arr(i, j, k, 1), dat_arr(i - 1, j, k, 1), + dat_arr(i - 2, j, k, 1)) * + idx; + } else if (!flag_fab(i, j, k).isConnected(-1, 0, 0)) { + vx = onesided( + dat_arr(i, j, k, 1), dat_arr(i + 1, j, k, 1), + dat_arr(i + 2, j, k, 1)) * + idx; + } else { + vx = + 0.5 * (dat_arr(i + 1, j, k, 1) - dat_arr(i - 1, j, k, 1)) * idx; + } + // Do the same in y-direction + if (!flag_fab(i, j, k).isConnected(0, 1, 0)) { + uy = -onesided( + dat_arr(i, j, k, 0), dat_arr(i, j - 1, k, 0), + dat_arr(i, j - 2, k, 0)) * + idy; + } else if (!flag_fab(i, j, k).isConnected(0, -1, 0)) { + uy = onesided( + dat_arr(i, j, k, 0), dat_arr(i, j + 1, k, 0), + dat_arr(i, j + 2, k, 0)) * + idy; + } else { + uy = + 0.5 * (dat_arr(i, j + 1, k, 0) - dat_arr(i, j - 1, k, 0)) * idy; + } + vort_arr(i, j, k) = vx - uy; #elif (AMREX_SPACEDIM == 3) amrex::Real wx = 0.0; @@ -565,35 +598,41 @@ pelelmex_dervort( vort_arr(i,j,k,1) = (uz-wx)*(uz-wx); vort_arr(i,j,k,2) = (vx-uy)*(vx-uy); #endif - } - }); + } + }); } else #endif // Check on EB { - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + amrex::ParallelFor( + bx, [dat_arr, vort_arr, idx, idy +#if (AMREX_SPACEDIM == 3) + , + idz +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { #if (AMREX_SPACEDIM == 2) - amrex::Real vx = - 0.5 * (dat_arr(i + 1, j, k, 1) - dat_arr(i - 1, j, k, 1)) * idx; - amrex::Real uy = - 0.5 * (dat_arr(i, j + 1, k, 0) - dat_arr(i, j - 1, k, 0)) * idy; - vort_arr(i, j, k) = vx - uy; + const amrex::Real vx = + 0.5 * (dat_arr(i + 1, j, k, 1) - dat_arr(i - 1, j, k, 1)) * idx; + const amrex::Real uy = + 0.5 * (dat_arr(i, j + 1, k, 0) - dat_arr(i, j - 1, k, 0)) * idy; + vort_arr(i, j, k) = vx - uy; #elif (AMREX_SPACEDIM == 3) - amrex::Real vx = 0.5 * (dat_arr(i+1,j,k,1) - dat_arr(i-1,j,k,1)) * idx; - amrex::Real wx = 0.5 * (dat_arr(i+1,j,k,2) - dat_arr(i-1,j,k,2)) * idx; - - amrex::Real uy = 0.5 * (dat_arr(i,j+1,k,0) - dat_arr(i,j-1,k,0)) * idy; - amrex::Real wy = 0.5 * (dat_arr(i,j+1,k,2) - dat_arr(i,j-1,k,2)) * idy; - - amrex::Real uz = 0.5 * (dat_arr(i,j,k+1,0) - dat_arr(i,j,k-1,0)) * idz; - amrex::Real vz = 0.5 * (dat_arr(i,j,k+1,1) - dat_arr(i,j,k-1,1)) * idz; - - vort_arr(i,j,k,0) = (wy-vz)*(wy-vz); - vort_arr(i,j,k,1) = (uz-wx)*(uz-wx); - vort_arr(i,j,k,2) = (vx-uy)*(vx-uy); + const amrex::Real vx = 0.5 * (dat_arr(i+1,j,k,1) - dat_arr(i-1,j,k,1)) * idx; + const amrex::Real wx = 0.5 * (dat_arr(i+1,j,k,2) - dat_arr(i-1,j,k,2)) * idx; + + const amrex::Real uy = 0.5 * (dat_arr(i,j+1,k,0) - dat_arr(i,j-1,k,0)) * idy; + const amrex::Real wy = 0.5 * (dat_arr(i,j+1,k,2) - dat_arr(i,j-1,k,2)) * idy; + + const amrex::Real uz = 0.5 * (dat_arr(i,j,k+1,0) - dat_arr(i,j,k-1,0)) * idz; + const amrex::Real vz = 0.5 * (dat_arr(i,j,k+1,1) - dat_arr(i,j,k-1,1)) * idz; + + vort_arr(i,j,k,0) = (wy-vz)*(wy-vz); + vort_arr(i,j,k,1) = (uz-wx)*(uz-wx); + vort_arr(i,j,k,2) = (vx-uy)*(vx-uy); #endif - }); + }); } } @@ -644,30 +683,43 @@ pelelmex_dercoord( const auto& flag_arr = flags.const_array(); const auto& ccent_fab = ebfab.getCentroidData(); const auto& ccent_arr = ccent_fab->const_array(); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - const amrex::Real* prob_lo = geomdata.ProbLo(); - if (flag_arr(i, j, k).isCovered() || flag_arr(i, j, k).isRegular()) { - AMREX_D_TERM(coord_arr(i, j, k, 0) = prob_lo[0] + (i + 0.5) * dx; - , coord_arr(i, j, k, 1) = prob_lo[1] + (j + 0.5) * dy; - , coord_arr(i, j, k, 2) = prob_lo[2] + (k + 0.5) * dz;); - } else { - AMREX_D_TERM(coord_arr(i, j, k, 0) = - prob_lo[0] + (i + 0.5 + ccent_arr(i, j, k, 0)) * dx; - , coord_arr(i, j, k, 1) = - prob_lo[1] + (j + 0.5 + ccent_arr(i, j, k, 1)) * dy; - , coord_arr(i, j, k, 2) = + amrex::ParallelFor( + bx, [flag_arr, ccent_arr, coord_arr, geomdata, dx, dy +#if (AMREX_SPACEDIM == 3) + , + dz +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + const amrex::Real* prob_lo = geomdata.ProbLo(); + if (flag_arr(i, j, k).isCovered() || flag_arr(i, j, k).isRegular()) { + AMREX_D_TERM(coord_arr(i, j, k, 0) = prob_lo[0] + (i + 0.5) * dx; + , coord_arr(i, j, k, 1) = prob_lo[1] + (j + 0.5) * dy; + , coord_arr(i, j, k, 2) = prob_lo[2] + (k + 0.5) * dz;); + } else { + AMREX_D_TERM(coord_arr(i, j, k, 0) = + prob_lo[0] + (i + 0.5 + ccent_arr(i, j, k, 0)) * dx; + , coord_arr(i, j, k, 1) = + prob_lo[1] + (j + 0.5 + ccent_arr(i, j, k, 1)) * dy; + , + coord_arr(i, j, k, 2) = prob_lo[2] + (k + 0.5 + ccent_arr(i, j, k, 2)) * dz;); - } - }); + } + }); } else #endif { - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - const amrex::Real* prob_lo = geomdata.ProbLo(); - AMREX_D_TERM(coord_arr(i, j, k, 0) = prob_lo[0] + (i + 0.5) * dx; - , coord_arr(i, j, k, 1) = prob_lo[1] + (j + 0.5) * dy; - , coord_arr(i, j, k, 2) = prob_lo[2] + (k + 0.5) * dz;); - }); + const amrex::Real* prob_lo = geomdata.ProbLo(); + amrex::ParallelFor( + bx, [coord_arr, prob_lo, dx, dy +#if (AMREX_SPACEDIM == 3) + , + dz +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + AMREX_D_TERM(coord_arr(i, j, k, 0) = prob_lo[0] + (i + 0.5) * dx; + , coord_arr(i, j, k, 1) = prob_lo[1] + (j + 0.5) * dy; + , coord_arr(i, j, k, 2) = prob_lo[2] + (k + 0.5) * dz;); + }); } } @@ -726,132 +778,190 @@ pelelmex_derQcrit( auto typ = flags.getType(bx); if (typ == amrex::FabType::covered) { - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - qcrit_arr(i, j, k) = 0.0; - }); + amrex::ParallelFor( + bx, [qcrit_arr] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + qcrit_arr(i, j, k) = 0.0; + }); } else if (typ == amrex::FabType::singlevalued) { const auto& flag_fab = flags.const_array(); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - if (flag_fab(i, j, k).isCovered()) { - qcrit_arr(i, j, k) = 0.0; - } else { - // Define interpolation lambda - constexpr amrex::Real c0 = -1.5; - constexpr amrex::Real c1 = 2.0; - constexpr amrex::Real c2 = -0.5; - auto onesided = []( - const amrex::Real& v0, const amrex::Real& v1, - const amrex::Real& v2) -> amrex::Real { - return c0 * v0 + c1 * v1 + c2 * v2; - }; + amrex::ParallelFor( + bx, [qcrit_arr, dat_arr, flag_fab, idx, idy +#if (AMREX_SPACEDIM == 3) + , + idz +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + if (flag_fab(i, j, k).isCovered()) { + qcrit_arr(i, j, k) = 0.0; + } else { + // Define interpolation lambda + constexpr amrex::Real c0 = -1.5; + constexpr amrex::Real c1 = 2.0; + constexpr amrex::Real c2 = -0.5; + auto onesided = []( + const amrex::Real& v0, const amrex::Real& v1, + const amrex::Real& v2) -> amrex::Real { + return c0 * v0 + c1 * v1 + c2 * v2; + }; + + // Strain rate tensor + amrex::Array2D gradU; + if (!flag_fab(i, j, k).isConnected(1, 0, 0)) { + gradU(0, 0) = -onesided( + dat_arr(i, j, k, 0), dat_arr(i - 1, j, k, 0), + dat_arr(i - 2, j, k, 0)) * + idx; + gradU(1, 0) = -onesided( + dat_arr(i, j, k, 1), dat_arr(i - 1, j, k, 1), + dat_arr(i - 2, j, k, 1)) * + idx; + gradU(2, 0) = -onesided( + dat_arr(i, j, k, 2), dat_arr(i - 1, j, k, 2), + dat_arr(i - 2, j, k, 2)) * + idx; + } else if (!flag_fab(i, j, k).isConnected(-1, 0, 0)) { + gradU(0, 0) = onesided( + dat_arr(i, j, k, 0), dat_arr(i + 1, j, k, 0), + dat_arr(i + 2, j, k, 0)) * + idx; + gradU(1, 0) = onesided( + dat_arr(i, j, k, 1), dat_arr(i + 1, j, k, 1), + dat_arr(i + 2, j, k, 1)) * + idx; + gradU(2, 0) = onesided( + dat_arr(i, j, k, 2), dat_arr(i + 1, j, k, 2), + dat_arr(i + 2, j, k, 2)) * + idx; + } else { + gradU(0, 0) = + 0.5 * (dat_arr(i + 1, j, k, 0) - dat_arr(i - 1, j, k, 0)) * idx; + gradU(1, 0) = + 0.5 * (dat_arr(i + 1, j, k, 1) - dat_arr(i - 1, j, k, 1)) * idx; + gradU(2, 0) = + 0.5 * (dat_arr(i + 1, j, k, 2) - dat_arr(i - 1, j, k, 2)) * idx; + } + if (!flag_fab(i, j, k).isConnected(0, 1, 0)) { + gradU(0, 1) = -onesided( + dat_arr(i, j, k, 0), dat_arr(i, j - 1, k, 0), + dat_arr(i, j - 2, k, 0)) * + idy; + gradU(1, 1) = -onesided( + dat_arr(i, j, k, 1), dat_arr(i, j - 1, k, 1), + dat_arr(i, j - 2, k, 1)) * + idy; + gradU(2, 1) = -onesided( + dat_arr(i, j, k, 2), dat_arr(i, j - 1, k, 2), + dat_arr(i, j - 2, k, 2)) * + idy; + } else if (!flag_fab(i, j, k).isConnected(0, -1, 0)) { + gradU(0, 1) = onesided( + dat_arr(i, j, k, 0), dat_arr(i, j + 1, k, 0), + dat_arr(i, j + 2, k, 0)) * + idy; + gradU(1, 1) = onesided( + dat_arr(i, j, k, 1), dat_arr(i, j + 1, k, 1), + dat_arr(i, j + 2, k, 1)) * + idy; + gradU(2, 1) = onesided( + dat_arr(i, j, k, 2), dat_arr(i, j + 1, k, 2), + dat_arr(i, j + 2, k, 2)) * + idy; + } else { + gradU(0, 1) = + 0.5 * (dat_arr(i, j + 1, k, 0) - dat_arr(i, j - 1, k, 0)) * idy; + gradU(1, 1) = + 0.5 * (dat_arr(i, j + 1, k, 1) - dat_arr(i, j - 1, k, 1)) * idy; + gradU(2, 1) = + 0.5 * (dat_arr(i, j + 1, k, 2) - dat_arr(i, j - 1, k, 2)) * idy; + } + if (!flag_fab(i, j, k).isConnected(0, 0, 1)) { + gradU(0, 2) = -onesided( + dat_arr(i, j, k, 0), dat_arr(i, j, k - 1, 0), + dat_arr(i, j, k - 2, 0)) * + idz; + gradU(1, 2) = -onesided( + dat_arr(i, j, k, 1), dat_arr(i, j, k - 1, 1), + dat_arr(i, j, k - 2, 1)) * + idz; + gradU(2, 2) = -onesided( + dat_arr(i, j, k, 2), dat_arr(i, j, k - 1, 2), + dat_arr(i, j, k - 2, 2)) * + idz; + } else if (!flag_fab(i, j, k).isConnected(0, 0, -1)) { + gradU(0, 2) = onesided( + dat_arr(i, j, k, 0), dat_arr(i, j, k + 1, 0), + dat_arr(i, j, k + 2, 0)) * + idz; + gradU(1, 2) = onesided( + dat_arr(i, j, k, 1), dat_arr(i, j, k + 1, 1), + dat_arr(i, j, k + 2, 1)) * + idz; + gradU(2, 2) = onesided( + dat_arr(i, j, k, 2), dat_arr(i, j, k + 1, 2), + dat_arr(i, j, k + 2, 2)) * + idz; + } else { + gradU(0, 2) = + 0.5 * (dat_arr(i, j, k + 1, 0) - dat_arr(i, j, k - 1, 0)) * idz; + gradU(1, 2) = + 0.5 * (dat_arr(i, j, k + 1, 1) - dat_arr(i, j, k - 1, 1)) * idz; + gradU(2, 2) = + 0.5 * (dat_arr(i, j, k + 1, 2) - dat_arr(i, j, k - 1, 2)) * idz; + } + // Divu + const amrex::Real divU = gradU(0, 0) + gradU(1, 1) + gradU(2, 2); + + // Directly Assemble Sym. & AntiSym. into Qcrit. + // Remove divU (dilatation) from the Sym. tensor (due to + // mixing/reaction most often) + qcrit_arr(i, j, k) = 0.0; + for (int dim1 = 0; dim1 < AMREX_SPACEDIM; ++dim1) { + for (int dim2 = 0; dim2 < AMREX_SPACEDIM; ++dim2) { + const amrex::Real Ohm = + 0.5 * (gradU(dim1, dim2) - gradU(dim2, dim1)); + amrex::Real Sij = 0.5 * (gradU(dim1, dim2) + gradU(dim2, dim1)); + if (dim1 == dim2) { + Sij -= divU / static_cast(AMREX_SPACEDIM); + } + qcrit_arr(i, j, k) += Ohm * Ohm - Sij * Sij; + } + } + } + }); + } else +#endif + { + amrex::ParallelFor( + bx, [dat_arr, qcrit_arr, idx, idy +#if (AMREX_SPACEDIM == 3) + , + idz +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { // Strain rate tensor amrex::Array2D gradU; - if (!flag_fab(i, j, k).isConnected(1, 0, 0)) { - gradU(0, 0) = -onesided( - dat_arr(i, j, k, 0), dat_arr(i - 1, j, k, 0), - dat_arr(i - 2, j, k, 0)) * - idx; - gradU(1, 0) = -onesided( - dat_arr(i, j, k, 1), dat_arr(i - 1, j, k, 1), - dat_arr(i - 2, j, k, 1)) * - idx; - gradU(2, 0) = -onesided( - dat_arr(i, j, k, 2), dat_arr(i - 1, j, k, 2), - dat_arr(i - 2, j, k, 2)) * - idx; - } else if (!flag_fab(i, j, k).isConnected(-1, 0, 0)) { - gradU(0, 0) = onesided( - dat_arr(i, j, k, 0), dat_arr(i + 1, j, k, 0), - dat_arr(i + 2, j, k, 0)) * - idx; - gradU(1, 0) = onesided( - dat_arr(i, j, k, 1), dat_arr(i + 1, j, k, 1), - dat_arr(i + 2, j, k, 1)) * - idx; - gradU(2, 0) = onesided( - dat_arr(i, j, k, 2), dat_arr(i + 1, j, k, 2), - dat_arr(i + 2, j, k, 2)) * - idx; - } else { - gradU(0, 0) = - 0.5 * (dat_arr(i + 1, j, k, 0) - dat_arr(i - 1, j, k, 0)) * idx; - gradU(1, 0) = - 0.5 * (dat_arr(i + 1, j, k, 1) - dat_arr(i - 1, j, k, 1)) * idx; - gradU(2, 0) = - 0.5 * (dat_arr(i + 1, j, k, 2) - dat_arr(i - 1, j, k, 2)) * idx; - } - if (!flag_fab(i, j, k).isConnected(0, 1, 0)) { - gradU(0, 1) = -onesided( - dat_arr(i, j, k, 0), dat_arr(i, j - 1, k, 0), - dat_arr(i, j - 2, k, 0)) * - idy; - gradU(1, 1) = -onesided( - dat_arr(i, j, k, 1), dat_arr(i, j - 1, k, 1), - dat_arr(i, j - 2, k, 1)) * - idy; - gradU(2, 1) = -onesided( - dat_arr(i, j, k, 2), dat_arr(i, j - 1, k, 2), - dat_arr(i, j - 2, k, 2)) * - idy; - } else if (!flag_fab(i, j, k).isConnected(0, -1, 0)) { - gradU(0, 1) = onesided( - dat_arr(i, j, k, 0), dat_arr(i, j + 1, k, 0), - dat_arr(i, j + 2, k, 0)) * - idy; - gradU(1, 1) = onesided( - dat_arr(i, j, k, 1), dat_arr(i, j + 1, k, 1), - dat_arr(i, j + 2, k, 1)) * - idy; - gradU(2, 1) = onesided( - dat_arr(i, j, k, 2), dat_arr(i, j + 1, k, 2), - dat_arr(i, j + 2, k, 2)) * - idy; - } else { - gradU(0, 1) = - 0.5 * (dat_arr(i, j + 1, k, 0) - dat_arr(i, j - 1, k, 0)) * idy; - gradU(1, 1) = - 0.5 * (dat_arr(i, j + 1, k, 1) - dat_arr(i, j - 1, k, 1)) * idy; - gradU(2, 1) = - 0.5 * (dat_arr(i, j + 1, k, 2) - dat_arr(i, j - 1, k, 2)) * idy; - } - if (!flag_fab(i, j, k).isConnected(0, 0, 1)) { - gradU(0, 2) = -onesided( - dat_arr(i, j, k, 0), dat_arr(i, j, k - 1, 0), - dat_arr(i, j, k - 2, 0)) * - idz; - gradU(1, 2) = -onesided( - dat_arr(i, j, k, 1), dat_arr(i, j, k - 1, 1), - dat_arr(i, j, k - 2, 1)) * - idz; - gradU(2, 2) = -onesided( - dat_arr(i, j, k, 2), dat_arr(i, j, k - 1, 2), - dat_arr(i, j, k - 2, 2)) * - idz; - } else if (!flag_fab(i, j, k).isConnected(0, 0, -1)) { - gradU(0, 2) = onesided( - dat_arr(i, j, k, 0), dat_arr(i, j, k + 1, 0), - dat_arr(i, j, k + 2, 0)) * - idz; - gradU(1, 2) = onesided( - dat_arr(i, j, k, 1), dat_arr(i, j, k + 1, 1), - dat_arr(i, j, k + 2, 1)) * - idz; - gradU(2, 2) = onesided( - dat_arr(i, j, k, 2), dat_arr(i, j, k + 1, 2), - dat_arr(i, j, k + 2, 2)) * - idz; - } else { - gradU(0, 2) = - 0.5 * (dat_arr(i, j, k + 1, 0) - dat_arr(i, j, k - 1, 0)) * idz; - gradU(1, 2) = - 0.5 * (dat_arr(i, j, k + 1, 1) - dat_arr(i, j, k - 1, 1)) * idz; - gradU(2, 2) = - 0.5 * (dat_arr(i, j, k + 1, 2) - dat_arr(i, j, k - 1, 2)) * idz; - } + gradU(0, 0) = + 0.5 * (dat_arr(i + 1, j, k, 0) - dat_arr(i - 1, j, k, 0)) * idx; + gradU(0, 1) = + 0.5 * (dat_arr(i, j + 1, k, 0) - dat_arr(i, j - 1, k, 0)) * idy; + gradU(0, 2) = + 0.5 * (dat_arr(i, j, k + 1, 0) - dat_arr(i, j, k - 1, 0)) * idz; + gradU(1, 0) = + 0.5 * (dat_arr(i + 1, j, k, 1) - dat_arr(i - 1, j, k, 1)) * idx; + gradU(1, 1) = + 0.5 * (dat_arr(i, j + 1, k, 1) - dat_arr(i, j - 1, k, 1)) * idy; + gradU(1, 2) = + 0.5 * (dat_arr(i, j, k + 1, 1) - dat_arr(i, j, k - 1, 1)) * idz; + gradU(2, 0) = + 0.5 * (dat_arr(i + 1, j, k, 2) - dat_arr(i - 1, j, k, 2)) * idx; + gradU(2, 1) = + 0.5 * (dat_arr(i, j + 1, k, 2) - dat_arr(i, j - 1, k, 2)) * idy; + gradU(2, 2) = + 0.5 * (dat_arr(i, j, k + 1, 2) - dat_arr(i, j, k - 1, 2)) * idz; // Divu - amrex::Real divU = gradU(0, 0) + gradU(1, 1) + gradU(2, 2); + const amrex::Real divU = gradU(0, 0) + gradU(1, 1) + gradU(2, 2); // Directly Assemble Sym. & AntiSym. into Qcrit. // Remove divU (dilatation) from the Sym. tensor (due to mixing/reaction @@ -859,59 +969,16 @@ pelelmex_derQcrit( qcrit_arr(i, j, k) = 0.0; for (int dim1 = 0; dim1 < AMREX_SPACEDIM; ++dim1) { for (int dim2 = 0; dim2 < AMREX_SPACEDIM; ++dim2) { - amrex::Real Ohm = 0.5 * (gradU(dim1, dim2) - gradU(dim2, dim1)); + const amrex::Real Ohm = + 0.5 * (gradU(dim1, dim2) - gradU(dim2, dim1)); amrex::Real Sij = 0.5 * (gradU(dim1, dim2) + gradU(dim2, dim1)); if (dim1 == dim2) { - Sij -= divU / AMREX_SPACEDIM; + Sij -= divU / static_cast(AMREX_SPACEDIM); } qcrit_arr(i, j, k) += Ohm * Ohm - Sij * Sij; } } - } - }); - } else -#endif - { - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - // Strain rate tensor - amrex::Array2D gradU; - gradU(0, 0) = - 0.5 * (dat_arr(i + 1, j, k, 0) - dat_arr(i - 1, j, k, 0)) * idx; - gradU(0, 1) = - 0.5 * (dat_arr(i, j + 1, k, 0) - dat_arr(i, j - 1, k, 0)) * idy; - gradU(0, 2) = - 0.5 * (dat_arr(i, j, k + 1, 0) - dat_arr(i, j, k - 1, 0)) * idz; - gradU(1, 0) = - 0.5 * (dat_arr(i + 1, j, k, 1) - dat_arr(i - 1, j, k, 1)) * idx; - gradU(1, 1) = - 0.5 * (dat_arr(i, j + 1, k, 1) - dat_arr(i, j - 1, k, 1)) * idy; - gradU(1, 2) = - 0.5 * (dat_arr(i, j, k + 1, 1) - dat_arr(i, j, k - 1, 1)) * idz; - gradU(2, 0) = - 0.5 * (dat_arr(i + 1, j, k, 2) - dat_arr(i - 1, j, k, 2)) * idx; - gradU(2, 1) = - 0.5 * (dat_arr(i, j + 1, k, 2) - dat_arr(i, j - 1, k, 2)) * idy; - gradU(2, 2) = - 0.5 * (dat_arr(i, j, k + 1, 2) - dat_arr(i, j, k - 1, 2)) * idz; - - // Divu - amrex::Real divU = gradU(0, 0) + gradU(1, 1) + gradU(2, 2); - - // Directly Assemble Sym. & AntiSym. into Qcrit. - // Remove divU (dilatation) from the Sym. tensor (due to mixing/reaction - // most often) - qcrit_arr(i, j, k) = 0.0; - for (int dim1 = 0; dim1 < AMREX_SPACEDIM; ++dim1) { - for (int dim2 = 0; dim2 < AMREX_SPACEDIM; ++dim2) { - amrex::Real Ohm = 0.5 * (gradU(dim1, dim2) - gradU(dim2, dim1)); - amrex::Real Sij = 0.5 * (gradU(dim1, dim2) + gradU(dim2, dim1)); - if (dim1 == dim2) { - Sij -= divU / AMREX_SPACEDIM; - } - qcrit_arr(i, j, k) += Ohm * Ohm - Sij * Sij; - } - } - }); + }); } #endif } @@ -941,24 +1008,26 @@ pelelmex_derkineticenergy( auto const vel = statefab.array(VELX); auto der = derfab.array(dcomp); const auto rho = a_pelelm->m_rho; - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - der(i, j, k) = - 0.5 * rho * - (AMREX_D_TERM( - vel(i, j, k, 0) * vel(i, j, k, 0), +vel(i, j, k, 1) * vel(i, j, k, 1), - +vel(i, j, k, 2) * vel(i, j, k, 2))); - }); + amrex::ParallelFor( + bx, [der, vel, rho] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + der(i, j, k) = 0.5 * rho * + (AMREX_D_TERM( + vel(i, j, k, 0) * vel(i, j, k, 0), + +vel(i, j, k, 1) * vel(i, j, k, 1), + +vel(i, j, k, 2) * vel(i, j, k, 2))); + }); } else { auto const rho = statefab.array(DENSITY); auto const vel = statefab.array(VELX); auto der = derfab.array(dcomp); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - der(i, j, k) = - 0.5 * rho(i, j, k) * - (AMREX_D_TERM( - vel(i, j, k, 0) * vel(i, j, k, 0), +vel(i, j, k, 1) * vel(i, j, k, 1), - +vel(i, j, k, 2) * vel(i, j, k, 2))); - }); + amrex::ParallelFor( + bx, [der, vel, rho] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + der(i, j, k) = 0.5 * rho(i, j, k) * + (AMREX_D_TERM( + vel(i, j, k, 0) * vel(i, j, k, 0), + +vel(i, j, k, 1) * vel(i, j, k, 1), + +vel(i, j, k, 2) * vel(i, j, k, 2))); + }); } } @@ -999,65 +1068,74 @@ pelelmex_derenstrophy( auto typ = flags.getType(bx); if (typ == amrex::FabType::covered) { - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - ens_arr(i, j, k) = 0.0; - }); + amrex::ParallelFor( + bx, [ens_arr] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ens_arr(i, j, k) = 0.0; + }); } else if (typ == amrex::FabType::singlevalued) { const auto& flag_fab = flags.const_array(); const auto incomp = a_pelelm->m_incompressible; const auto rho = a_pelelm->m_rho; - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - constexpr amrex::Real c0 = -1.5; - constexpr amrex::Real c1 = 2.0; - constexpr amrex::Real c2 = -0.5; - if (flag_fab(i, j, k).isCovered()) { - ens_arr(i, j, k) = 0.0; - } else { - amrex::Real l_rho = rho; - if (incomp == 0) { - l_rho = rho_arr(i, j, k); - } - // Define interpolation lambda - auto onesided = []( - const amrex::Real& v0, const amrex::Real& v1, - const amrex::Real& v2) -> amrex::Real { - return c0 * v0 + c1 * v1 + c2 * v2; - }; - - amrex::Real vx = 0.0; - amrex::Real uy = 0.0; -#if (AMREX_SPACEDIM == 2) - // Need to check if there are covered cells in neighbours -- - // -- if so, use one-sided difference computation (but still - // quadratic) - if (!flag_fab(i, j, k).isConnected(1, 0, 0)) { - vx = -onesided( - dat_arr(i, j, k, 1), dat_arr(i - 1, j, k, 1), - dat_arr(i - 2, j, k, 1)) * - idx; - } else if (!flag_fab(i, j, k).isConnected(-1, 0, 0)) { - vx = onesided( - dat_arr(i, j, k, 1), dat_arr(i + 1, j, k, 1), - dat_arr(i + 2, j, k, 1)) * - idx; - } else { - vx = 0.5 * (dat_arr(i + 1, j, k, 1) - dat_arr(i - 1, j, k, 1)) * idx; - } - // Do the same in y-direction - if (!flag_fab(i, j, k).isConnected(0, 1, 0)) { - uy = -onesided( - dat_arr(i, j, k, 0), dat_arr(i, j - 1, k, 0), - dat_arr(i, j - 2, k, 0)) * - idy; - } else if (!flag_fab(i, j, k).isConnected(0, -1, 0)) { - uy = onesided( - dat_arr(i, j, k, 0), dat_arr(i, j + 1, k, 0), - dat_arr(i, j + 2, k, 0)) * - idy; + amrex::ParallelFor( + bx, [ens_arr, flag_fab, dat_arr, rho, incomp, rho_arr, idx, idy +#if (AMREX_SPACEDIM == 3) + , + idz +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + constexpr amrex::Real c0 = -1.5; + constexpr amrex::Real c1 = 2.0; + constexpr amrex::Real c2 = -0.5; + if (flag_fab(i, j, k).isCovered()) { + ens_arr(i, j, k) = 0.0; } else { - uy = 0.5 * (dat_arr(i, j + 1, k, 0) - dat_arr(i, j - 1, k, 0)) * idy; - } - ens_arr(i, j, k) = 0.5 * l_rho * (vx - uy) * (vx - uy); + amrex::Real l_rho = rho; + if (incomp == 0) { + l_rho = rho_arr(i, j, k); + } + // Define interpolation lambda + auto onesided = []( + const amrex::Real& v0, const amrex::Real& v1, + const amrex::Real& v2) -> amrex::Real { + return c0 * v0 + c1 * v1 + c2 * v2; + }; + + amrex::Real vx = 0.0; + amrex::Real uy = 0.0; +#if (AMREX_SPACEDIM == 2) + // Need to check if there are covered cells in neighbours -- + // -- if so, use one-sided difference computation (but still + // quadratic) + if (!flag_fab(i, j, k).isConnected(1, 0, 0)) { + vx = -onesided( + dat_arr(i, j, k, 1), dat_arr(i - 1, j, k, 1), + dat_arr(i - 2, j, k, 1)) * + idx; + } else if (!flag_fab(i, j, k).isConnected(-1, 0, 0)) { + vx = onesided( + dat_arr(i, j, k, 1), dat_arr(i + 1, j, k, 1), + dat_arr(i + 2, j, k, 1)) * + idx; + } else { + vx = + 0.5 * (dat_arr(i + 1, j, k, 1) - dat_arr(i - 1, j, k, 1)) * idx; + } + // Do the same in y-direction + if (!flag_fab(i, j, k).isConnected(0, 1, 0)) { + uy = -onesided( + dat_arr(i, j, k, 0), dat_arr(i, j - 1, k, 0), + dat_arr(i, j - 2, k, 0)) * + idy; + } else if (!flag_fab(i, j, k).isConnected(0, -1, 0)) { + uy = onesided( + dat_arr(i, j, k, 0), dat_arr(i, j + 1, k, 0), + dat_arr(i, j + 2, k, 0)) * + idy; + } else { + uy = + 0.5 * (dat_arr(i, j + 1, k, 0) - dat_arr(i, j - 1, k, 0)) * idy; + } + ens_arr(i, j, k) = 0.5 * l_rho * (vx - uy) * (vx - uy); #elif (AMREX_SPACEDIM == 3) amrex::Real wx = 0.0; @@ -1148,46 +1226,52 @@ pelelmex_derenstrophy( ((wy - vz) * (wy - vz) + (uz - wx) * (uz - wx) + (vx - uy) * (vx - uy)); #endif - } - }); + } + }); } else #endif { const auto incomp = a_pelelm->m_incompressible; const auto rho = a_pelelm->m_rho; - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - amrex::Real l_rho = rho; - if (incomp == 0) { - l_rho = rho_arr(i, j, k); - } + amrex::ParallelFor( + bx, [rho_arr, dat_arr, rho, incomp, ens_arr, idx, idy +#if (AMREX_SPACEDIM == 3) + , + idz +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + amrex::Real l_rho = rho; + if (incomp == 0) { + l_rho = rho_arr(i, j, k); + } #if (AMREX_SPACEDIM == 2) - amrex::Real vx = - 0.5 * (dat_arr(i + 1, j, k, 1) - dat_arr(i - 1, j, k, 1)) * idx; - amrex::Real uy = - 0.5 * (dat_arr(i, j + 1, k, 0) - dat_arr(i, j - 1, k, 0)) * idy; - ens_arr(i, j, k) = 0.5 * l_rho * (vx - uy) * (vx - uy); + const amrex::Real vx = + 0.5 * (dat_arr(i + 1, j, k, 1) - dat_arr(i - 1, j, k, 1)) * idx; + const amrex::Real uy = + 0.5 * (dat_arr(i, j + 1, k, 0) - dat_arr(i, j - 1, k, 0)) * idy; + ens_arr(i, j, k) = 0.5 * l_rho * (vx - uy) * (vx - uy); #elif (AMREX_SPACEDIM == 3) - amrex::Real vx = + const amrex::Real vx = 0.5 * (dat_arr(i + 1, j, k, 1) - dat_arr(i - 1, j, k, 1)) * idx; - amrex::Real wx = + const amrex::Real wx = 0.5 * (dat_arr(i + 1, j, k, 2) - dat_arr(i - 1, j, k, 2)) * idx; - amrex::Real uy = + const amrex::Real uy = 0.5 * (dat_arr(i, j + 1, k, 0) - dat_arr(i, j - 1, k, 0)) * idy; - amrex::Real wy = + const amrex::Real wy = 0.5 * (dat_arr(i, j + 1, k, 2) - dat_arr(i, j - 1, k, 2)) * idy; - amrex::Real uz = + const amrex::Real uz = 0.5 * (dat_arr(i, j, k + 1, 0) - dat_arr(i, j, k - 1, 0)) * idz; - amrex::Real vz = + const amrex::Real vz = 0.5 * (dat_arr(i, j, k + 1, 1) - dat_arr(i, j, k - 1, 1)) * idz; ens_arr(i, j, k) = 0.5 * l_rho * ((wy - vz) * (wy - vz) + (uz - wx) * (uz - wx) + (vx - uy) * (vx - uy)); #endif - }); + }); } } @@ -1230,14 +1314,16 @@ pelelmex_dermixfrac( fact_Bilger[n] = a_pelelm->spec_Bilger_fact[n]; } - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - amrex::Real rho_inv = 1.0 / density(i, j, k); - mixt_frac(i, j, k) = 0.0; - for (int n = 0; n < NUM_SPECIES; ++n) { - mixt_frac(i, j, k) += (rhoY(i, j, k, n) * fact_Bilger[n]) * rho_inv; - } - mixt_frac(i, j, k) = (mixt_frac(i, j, k) - Zox_lcl) * denom_inv; - }); + amrex::ParallelFor( + bx, [density, mixt_frac, rhoY, Zox_lcl, denom_inv, + fact_Bilger] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + const amrex::Real rho_inv = 1.0 / density(i, j, k); + mixt_frac(i, j, k) = 0.0; + for (int n = 0; n < NUM_SPECIES; ++n) { + mixt_frac(i, j, k) += (rhoY(i, j, k, n) * fact_Bilger[n]) * rho_inv; + } + mixt_frac(i, j, k) = (mixt_frac(i, j, k) - Zox_lcl) * denom_inv; + }); } // @@ -1273,28 +1359,30 @@ pelelmex_derprogvar( auto const temp = statefab.array(TEMP); auto prog_var = derfab.array(dcomp); - amrex::Real C0_lcl = a_pelelm->m_C0; - amrex::Real C1_lcl = a_pelelm->m_C1; - amrex::Real denom_inv = 1.0 / (C1_lcl - C0_lcl); + const amrex::Real C0_lcl = a_pelelm->m_C0; + const amrex::Real C1_lcl = a_pelelm->m_C1; + const amrex::Real denom_inv = 1.0 / (C1_lcl - C0_lcl); amrex::GpuArray Cweights; for (int n = 0; n < NUM_SPECIES + 1; ++n) { Cweights[n] = a_pelelm->m_Cweights[n]; } const auto revert = a_pelelm->m_Crevert; - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - amrex::Real rho_inv = 1.0 / density(i, j, k); - prog_var(i, j, k) = 0.0; - for (int n = 0; n < NUM_SPECIES; ++n) { - prog_var(i, j, k) += (rhoY(i, j, k, n) * Cweights[n]) * rho_inv; - } - prog_var(i, j, k) += temp(i, j, k) * Cweights[NUM_SPECIES]; - if (revert != 0) { - prog_var(i, j, k) = 1.0 - (prog_var(i, j, k) - C0_lcl) * denom_inv; - } else { - prog_var(i, j, k) = (prog_var(i, j, k) - C0_lcl) * denom_inv; - } - }); + amrex::ParallelFor( + bx, [density, prog_var, rhoY, Cweights, denom_inv, C0_lcl, temp, + revert] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + amrex::Real rho_inv = 1.0 / density(i, j, k); + prog_var(i, j, k) = 0.0; + for (int n = 0; n < NUM_SPECIES; ++n) { + prog_var(i, j, k) += (rhoY(i, j, k, n) * Cweights[n]) * rho_inv; + } + prog_var(i, j, k) += temp(i, j, k) * Cweights[NUM_SPECIES]; + if (revert != 0) { + prog_var(i, j, k) = 1.0 - (prog_var(i, j, k) - C0_lcl) * denom_inv; + } else { + prog_var(i, j, k) = (prog_var(i, j, k) - C0_lcl) * denom_inv; + } + }); } // @@ -1327,9 +1415,11 @@ pelelmex_dervisc( auto const& T = statefab.array(TEMP); auto der = derfab.array(dcomp); auto const* ltransparm = a_pelelm->trans_parms.device_parm(); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - getVelViscosity(i, j, k, rhoY, T, der, ltransparm); - }); + amrex::ParallelFor( + bx, [rhoY, T, der, + ltransparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + getVelViscosity(i, j, k, rhoY, T, der, ltransparm); + }); } } @@ -1361,9 +1451,9 @@ pelelmex_derdiffc( if (a_pelelm->m_use_soret == 0) { AMREX_ASSERT(ncomp == NUM_SPECIES); } - bool do_fixed_Le = (a_pelelm->m_fixed_Le != 0); - bool do_fixed_Pr = (a_pelelm->m_fixed_Pr != 0); - bool do_soret = (a_pelelm->m_use_soret != 0); + const bool do_fixed_Le = (a_pelelm->m_fixed_Le != 0); + const bool do_fixed_Pr = (a_pelelm->m_fixed_Pr != 0); + const bool do_soret = (a_pelelm->m_use_soret != 0); amrex::FArrayBox dummies(bx, NUM_SPECIES + 2, amrex::The_Async_Arena()); auto const& rhoY = statefab.const_array(FIRSTSPEC); auto const& T = statefab.array(TEMP); @@ -1376,11 +1466,14 @@ pelelmex_derdiffc( : dummies.array(2); // dummy for no soret amrex::Real LeInv = a_pelelm->m_Lewis_inv; amrex::Real PrInv = a_pelelm->m_Prandtl_inv; - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - getTransportCoeff( - i, j, k, do_fixed_Le, do_fixed_Pr, do_soret, LeInv, PrInv, rhoY, T, rhoD, - rhotheta, lambda, mu, ltransparm, leosparm); - }); + amrex::ParallelFor( + bx, [do_fixed_Le, do_fixed_Pr, do_soret, LeInv, PrInv, rhoY, T, rhoD, + rhotheta, lambda, mu, ltransparm, + leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + getTransportCoeff( + i, j, k, do_fixed_Le, do_fixed_Pr, do_soret, LeInv, PrInv, rhoY, T, + rhoD, rhotheta, lambda, mu, ltransparm, leosparm); + }); } // @@ -1419,11 +1512,14 @@ pelelmex_derlambda( auto const* leosparm = a_pelelm->eos_parms.device_parm(); amrex::Real LeInv = a_pelelm->m_Lewis_inv; amrex::Real PrInv = a_pelelm->m_Prandtl_inv; - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - getTransportCoeff( - i, j, k, do_fixed_Le, do_fixed_Pr, do_soret, LeInv, PrInv, rhoY, T, rhoD, - rhotheta, lambda, mu, ltransparm, leosparm); - }); + amrex::ParallelFor( + bx, [do_fixed_Le, do_fixed_Pr, do_soret, LeInv, PrInv, rhoY, T, rhoD, + rhotheta, lambda, mu, ltransparm, + leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + getTransportCoeff( + i, j, k, do_fixed_Le, do_fixed_Pr, do_soret, LeInv, PrInv, rhoY, T, + rhoD, rhotheta, lambda, mu, ltransparm, leosparm); + }); } // @@ -1447,9 +1543,10 @@ pelelmex_derdmap( AMREX_ASSERT(derfab.box().contains(bx)); auto der = derfab.array(dcomp); const int myrank = amrex::ParallelDescriptor::MyProc(); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - der(i, j, k) = myrank; - }); + amrex::ParallelFor( + bx, [der, myrank] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + der(i, j, k) = myrank; + }); } // @@ -1528,18 +1625,20 @@ pelelmex_dermaniout( auto const in_spec = statefab.array(FIRSTSPEC); auto der = derfab.array(dcomp); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - amrex::Real rho, rhoinv, maniparm[NUM_SPECIES]; - pele::physics::PhysicsType::eos_type::RY2RRinvY( - in_spec.cellData(i, j, k), rho, rhoinv, maniparm); - pele::physics::BlackBoxFunctionFactory< - pele::physics::eos::ManifoldFunctionType> - manfunc{d_manf_data}; - - // TODO: use get_all_values instead - for (int n = 0; n < nmanivar; ++n) { - manfunc.get_func()->get_value(n, maniparm, der(i, j, k, n)); - } - }); + amrex::ParallelFor( + bx, [in_spec, der, d_manf_data, + nmanivar] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + amrex::Real rho, rhoinv, maniparm[NUM_SPECIES]; + pele::physics::PhysicsType::eos_type::RY2RRinvY( + in_spec.cellData(i, j, k), rho, rhoinv, maniparm); + pele::physics::BlackBoxFunctionFactory< + pele::physics::eos::ManifoldFunctionType> + manfunc{d_manf_data}; + + // TODO: use get_all_values instead + for (int n = 0; n < nmanivar; ++n) { + manfunc.get_func()->get_value(n, maniparm, der(i, j, k, n)); + } + }); } #endif diff --git a/Source/PeleLMeX_Diffusion.cpp b/Source/PeleLMeX_Diffusion.cpp index c7739761e..d5595416c 100644 --- a/Source/PeleLMeX_Diffusion.cpp +++ b/Source/PeleLMeX_Diffusion.cpp @@ -20,7 +20,7 @@ PeleLM::getDiffusionOp() } DiffusionOp* -PeleLM::getMCDiffusionOp(int ncomp) +PeleLM::getMCDiffusionOp(const int ncomp) { if (!m_mcdiffusion_op || m_mcdiffusion_op->m_ncomp != ncomp) { m_mcdiffusion_op = std::make_unique(this, ncomp); @@ -39,9 +39,9 @@ PeleLM::getDiffusionTensorOp() void PeleLM::computeDifferentialDiffusionTerms( - const TimeStamp& a_time, - std::unique_ptr& diffData, - int is_init) + const TimeStamp a_time, + const std::unique_ptr& diffData, + const int is_init) { BL_PROFILE("PeleLMeX::computeDifferentialDiffusionTerms()"); @@ -78,7 +78,7 @@ PeleLM::computeDifferentialDiffusionTerms( for (int lev = 0; lev <= finest_level; ++lev) { const auto& ba = grids[lev]; const auto& factory = Factory(lev); - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { fluxes[lev][idim].define( amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)), dmap[lev], NUM_SPECIES + 2, nGrow, amrex::MFInfo(), factory); @@ -90,10 +90,11 @@ PeleLM::computeDifferentialDiffusionTerms( } } #ifdef AMREX_USE_EB - amrex::Vector EBfluxes(finest_level + 1); + amrex::Vector EBfluxes; if (m_isothermalEB != 0) { + EBfluxes.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - EBfluxes[lev].define( + EBfluxes.emplace_back( grids[lev], dmap[lev], 1, nGrow, amrex::MFInfo(), Factory(lev)); } } @@ -308,12 +309,13 @@ PeleLM::adjustSpeciesFluxes( flagfab.getType(amrex::grow(ebx, nGrow)) == amrex::FabType::regular) { amrex::ParallelFor( - ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - int idx[3] = {i, j, k}; - bool on_lo = + ebx, [idim, rhoY, flux_dir, edomain, bc_lo, + bc_hi] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + const int idx[3] = {i, j, k}; + const bool on_lo = ((bc_lo == amrex::BCType::ext_dir) && (idx[idim] <= edomain.smallEnd(idim))); - bool on_hi = + const bool on_hi = ((bc_hi == amrex::BCType::ext_dir) && (idx[idim] >= edomain.bigEnd(idim))); repair_flux(i, j, k, idim, on_lo, on_hi, rhoY, flux_dir); @@ -322,12 +324,14 @@ PeleLM::adjustSpeciesFluxes( auto const& rhoYed_ar = edgstate[idim].const_array(mfi); auto const& areafrac_ar = areafrac[idim]->const_array(mfi); amrex::ParallelFor( - ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - int idx[3] = {i, j, k}; - bool on_lo = + ebx, + [idim, rhoY, flux_dir, rhoYed_ar, areafrac_ar, edomain, bc_lo, + bc_hi] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + const int idx[3] = {i, j, k}; + const bool on_lo = ((bc_lo == amrex::BCType::ext_dir) && (idx[idim] <= edomain.smallEnd(idim))); - bool on_hi = + const bool on_hi = ((bc_hi == amrex::BCType::ext_dir) && (idx[idim] >= edomain.bigEnd(idim))); repair_flux_eb( @@ -338,12 +342,13 @@ PeleLM::adjustSpeciesFluxes( } #else amrex::ParallelFor( - ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - int idx[3] = {i, j, k}; - bool on_lo = + ebx, [idim, rhoY, flux_dir, edomain, bc_lo, + bc_hi] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + const int idx[3] = {i, j, k}; + const bool on_lo = ((bc_lo == amrex::BCType::ext_dir) && (idx[idim] <= edomain.smallEnd(idim))); - bool on_hi = + const bool on_hi = ((bc_hi == amrex::BCType::ext_dir) && (idx[idim] >= edomain.bigEnd(idim))); repair_flux(i, j, k, idim, on_lo, on_hi, rhoY, flux_dir); @@ -368,7 +373,7 @@ PeleLM::adjustSpeciesFluxes( void PeleLM::correctIsothermalBoundary( - const TimeStamp& a_time, + const TimeStamp a_time, const amrex::Vector& a_spec_boundary, const amrex::Vector>& a_wbarfluxes, @@ -382,8 +387,8 @@ PeleLM::correctIsothermalBoundary( amrex::Vector> soretfluxes( finest_level + 1); if (need_explicit_fluxes) { // need to fill the soret fluxes ourselves - for (int lev = 0; lev <= finest_level; lev++) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int lev = 0; lev <= finest_level; ++lev) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { soretfluxes[lev][idim] = new amrex::MultiFab( grids[lev], dmap[lev], NUM_SPECIES, 1, amrex::MFInfo(), Factory(lev)); soretfluxes[lev][idim]->setVal(0.0); @@ -393,8 +398,8 @@ PeleLM::correctIsothermalBoundary( soretfluxes, soretfluxes, GetVecOfConstPtrs(getTempVect(a_time)), GetVecOfConstPtrs(getDiffusivityVect(a_time))); } else { // have the lagged ones, alias to them - for (int lev = 0; lev <= finest_level; lev++) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int lev = 0; lev <= finest_level; ++lev) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { soretfluxes[lev][idim] = new amrex::MultiFab( *a_soretfluxes[lev][idim], amrex::make_alias, 0, NUM_SPECIES); } @@ -430,19 +435,23 @@ PeleLM::correctIsothermalBoundary( auto const& boundary_ar = a_spec_boundary[lev]->array(mfi); const auto use_wbar = m_use_wbar; amrex::ParallelFor( - ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [bc_lo, bc_hi, idim, need_explicit_fluxes, edomain, flux_soret, + rhoD_ec, flux_wbar, boundary_ar, + use_wbar] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { int idx[3] = {i, j, k}; - bool on_lo = (bc_lo == BoundaryCondition::BCNoSlipWallIsotherm || - bc_lo == BoundaryCondition::BCSlipWallIsotherm) && - (idx[idim] <= edomain.smallEnd(idim)); - bool on_hi = (bc_hi == BoundaryCondition::BCNoSlipWallIsotherm || - bc_hi == BoundaryCondition::BCSlipWallIsotherm) && - (idx[idim] >= edomain.bigEnd(idim)); + const bool on_lo = + (bc_lo == BoundaryCondition::BCNoSlipWallIsotherm || + bc_lo == BoundaryCondition::BCSlipWallIsotherm) && + (idx[idim] <= edomain.smallEnd(idim)); + const bool on_hi = + (bc_hi == BoundaryCondition::BCNoSlipWallIsotherm || + bc_hi == BoundaryCondition::BCSlipWallIsotherm) && + (idx[idim] >= edomain.bigEnd(idim)); if (on_lo || on_hi) { if (on_lo) { // need to move -1 for lo boundary idx[idim] -= 1; } - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { boundary_ar(idx[0], idx[1], idx[2], n) = flux_soret(i, j, k, n); // add lagged wbar flux if (use_wbar != 0 && !need_explicit_fluxes) { @@ -457,8 +466,8 @@ PeleLM::correctIsothermalBoundary( } } // TODO: wbar fluxes disabled for this case - boundary system becomes complex - for (int lev = 0; lev <= finest_level; lev++) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int lev = 0; lev <= finest_level; ++lev) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { delete soretfluxes[lev][idim]; } } @@ -466,7 +475,7 @@ PeleLM::correctIsothermalBoundary( void PeleLM::computeDifferentialDiffusionFluxes( - const TimeStamp& a_time, + const TimeStamp a_time, const amrex::Vector>& a_fluxes, const amrex::Vector& #ifdef AMREX_USE_EB @@ -488,20 +497,19 @@ PeleLM::computeDifferentialDiffusionFluxes( amrex::Vector spec_boundary; if (m_soret_boundary_override != 0) { - spec_boundary.resize(finest_level + 1); + spec_boundary.reserve(finest_level + 1); // this is the same regardless of lagged or not for (int lev = 0; lev <= finest_level; ++lev) { - auto* ldata_p = getLevelDataPtr(lev, a_time); - spec_boundary[lev].define( + spec_boundary.emplace_back( grids[lev], dmap[lev], NUM_SPECIES, 1, amrex::MFInfo(), Factory(lev)); - + auto* ldata_p = getLevelDataPtr(lev, a_time); // if we have a mix of Dirichlet and Isothermal walls, we need to give // Dirichlet boundaries and divide by density since diffuse_scalar doesn't // touch this boundary MF amrex::MultiFab::Copy( spec_boundary[lev], ldata_p->state, FIRSTSPEC, 0, NUM_SPECIES, 1); - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { amrex::MultiFab::Divide( spec_boundary[lev], ldata_p->state, DENSITY, n, 1, 1); } @@ -526,7 +534,7 @@ PeleLM::computeDifferentialDiffusionFluxes( GetVecOfConstPtrs(getDiffusivityVect(a_time)), 0, bcRecSpec, NUM_SPECIES - NUM_IONS, do_avgDown, {}); // Ions one by one - for (int n = 0; n < NUM_IONS; n++) { + for (int n = 0; n < NUM_IONS; ++n) { auto bcRecIons = fetchBCRecArray(FIRSTSPEC + NUM_SPECIES - NUM_IONS + n, 1); getDiffusionOp()->computeDiffFluxes( a_fluxes, NUM_SPECIES - NUM_IONS + n, @@ -600,10 +608,11 @@ PeleLM::computeDifferentialDiffusionFluxes( if (m_isothermalEB != 0) { AMREX_ASSERT(!a_EBfluxes.empty()); // Set up EB dirichlet value and diffusivity - amrex::Vector EBdiff(finest_level + 1); + amrex::Vector EBdiff; + EBdiff.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - EBdiff[lev].define( + EBdiff.emplace_back( grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), EBFactory(lev)); getEBDiff(lev, a_time, EBdiff[lev], NUM_SPECIES); } @@ -669,17 +678,18 @@ PeleLM::addWbarTerm( //------------------------------------------------------------------------ // Compute Wbar on all the levels int nGrow = 1; // Need one ghost cell to compute gradWbar - amrex::Vector Wbar(finest_level + 1); + amrex::Vector Wbar; + Wbar.reserve(finest_level + 1); amrex::Vector Wbar_boundary; if (have_boundary != 0) { - Wbar_boundary.resize(finest_level + 1); + Wbar_boundary.reserve(finest_level + 1); } auto const* leosparm = eos_parms.device_parm(); for (int lev = 0; lev <= finest_level; ++lev) { - Wbar[lev].define( + Wbar.emplace_back( grids[lev], dmap[lev], 1, nGrow, amrex::MFInfo(), Factory(lev)); if (have_boundary != 0) { - Wbar_boundary[lev].define( + Wbar_boundary.emplace_back( grids[lev], dmap[lev], 1, nGrow, amrex::MFInfo(), Factory(lev)); } const amrex::Box& domain = geom[lev].Domain(); @@ -727,7 +737,6 @@ PeleLM::addWbarTerm( }); } } - //------------------------------------------------------------------------ // Compute Wbar gradients and do average down to get gradients consistent // across levels Get the species BCRec @@ -740,7 +749,7 @@ PeleLM::addWbarTerm( for (int lev = 0; lev <= finest_level; ++lev) { const auto& ba = grids[lev]; const auto& factory = Factory(lev); - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { gradWbar[lev][idim].define( amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)), dmap[lev], NUM_SPECIES, nGrow, amrex::MFInfo(), factory); @@ -771,7 +780,7 @@ PeleLM::addWbarTerm( { for (amrex::MFIter mfi(*a_beta[lev], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { // Get edge centered rhoYs const amrex::Box ebx = mfi.nodaltilebox(idim); @@ -783,12 +792,13 @@ PeleLM::addWbarTerm( const auto bc_lo = bcRecSpec[0].lo(idim); const auto bc_hi = bcRecSpec[0].hi(idim); amrex::ParallelFor( - ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - int idx[3] = {i, j, k}; - bool on_lo = + ebx, [idim, bc_lo, bc_hi, use_harmonic_avg, rhoY_arr, rhoYed_arr, + edomain] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + const int idx[3] = {i, j, k}; + const bool on_lo = ((bc_lo == amrex::BCType::ext_dir) && (idx[idim] <= edomain.smallEnd(idim))); - bool on_hi = + const bool on_hi = ((bc_hi == amrex::BCType::ext_dir) && (idx[idim] >= edomain.bigEnd(idim))); cen2edg_cpp( @@ -809,28 +819,31 @@ PeleLM::addWbarTerm( // \overline{W} with beta_m = \rho * D_m * overline(W) / W_k below // need to divide by \overline(W) const auto* eosparm = leosparm; + auto eos = pele::physics::PhysicsType::eos(eosparm); + amrex::ParallelFor( - ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - auto eos = pele::physics::PhysicsType::eos(eosparm); + ebx, + [eos, rhoY, gradWbar_ar, beta_ar, spFlux_ar, spwbarFlux_ar, + need_wbar_fluxes] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { // Get Wbar from rhoYs amrex::Real rho = 0.0; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { rho += rhoY(i, j, k, n); } amrex::Real rho_inv = 1.0 / rho; amrex::Real y[NUM_SPECIES] = {0.0}; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { y[n] = rhoY(i, j, k, n) * rho_inv; } amrex::Real WBAR = 0.0; eos.Y2WBAR(y, WBAR); WBAR *= 0.001; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { spFlux_ar(i, j, k, n) -= y[n] / WBAR * beta_ar(i, j, k, n) * gradWbar_ar(i, j, k); } if (need_wbar_fluxes != 0) { - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { spwbarFlux_ar(i, j, k, n) = -y[n] / WBAR * beta_ar(i, j, k, n) * gradWbar_ar(i, j, k); } @@ -868,7 +881,7 @@ PeleLM::addSoretTerm( for (int lev = 0; lev <= finest_level; ++lev) { const auto& ba = grids[lev]; const auto& factory = Factory(lev); - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { gradT[lev][idim].define( amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)), dmap[lev], 1, nGrow, amrex::MFInfo(), factory); @@ -898,7 +911,7 @@ PeleLM::addSoretTerm( amrex::FArrayBox T_ed; for (amrex::MFIter mfi(*a_beta[lev], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { // Get edge centered rhoYs const amrex::Box ebx = mfi.nodaltilebox(idim); @@ -913,12 +926,14 @@ PeleLM::addSoretTerm( const auto bc_lo_temp = bcRecTemp[0].lo(idim); const auto bc_hi_temp = bcRecTemp[0].hi(idim); amrex::ParallelFor( - ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - int idx[3] = {i, j, k}; - bool on_lo = + ebx, + [idim, bc_lo_temp, bc_hi_temp, use_harmonic_avg, T_arr, Ted_arr, + edomain] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + const int idx[3] = {i, j, k}; + const bool on_lo = ((bc_lo_temp == amrex::BCType::ext_dir) && (idx[idim] <= edomain.smallEnd(idim))); - bool on_hi = + const bool on_hi = ((bc_hi_temp == amrex::BCType::ext_dir) && (idx[idim] >= edomain.bigEnd(idim))); cen2edg_cpp( @@ -938,19 +953,22 @@ PeleLM::addSoretTerm( // Soret flux is : - rho * D_m * chi_m * \nabla T / T // with beta_m = rho * D_m * chi_m below amrex::ParallelFor( - ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - for (int n = 0; n < NUM_SPECIES; n++) { - spFlux_ar(i, j, k, n) -= - beta_ar(i, j, k, n) * gradT_ar(i, j, k) / T(i, j, k); - } - - if (need_soret_fluxes != 0) { - for (int n = 0; n < NUM_SPECIES; n++) { - spsoretFlux_ar(i, j, k, n) = - -beta_ar(i, j, k, n) * gradT_ar(i, j, k) / T(i, j, k); - } - } + ebx, NUM_SPECIES, + [gradT_ar, beta_ar, T, + spFlux_ar] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + spFlux_ar(i, j, k, n) -= + beta_ar(i, j, k, n) * gradT_ar(i, j, k) / T(i, j, k); }); + + if (need_soret_fluxes != 0) { + amrex::ParallelFor( + ebx, NUM_SPECIES, + [gradT_ar, beta_ar, T, spsoretFlux_ar] AMREX_GPU_DEVICE( + int i, int j, int k, int n) noexcept { + spsoretFlux_ar(i, j, k, n) = + -beta_ar(i, j, k, n) * gradT_ar(i, j, k) / T(i, j, k); + }); + } } } } @@ -994,14 +1012,15 @@ PeleLM::computeSpeciesEnthalpyFlux( auto const& flag = flagfab.const_array(); if (flagfab.getType(gbx) == amrex::FabType::covered) { // Covered boxes amrex::ParallelFor( - gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + gbx, [Hi_arr] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { Hi_arr(i, j, k) = 0.0; }); } else if ( flagfab.getType(gbx) != amrex::FabType::regular) { // EB containing // boxes amrex::ParallelFor( - gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + gbx, [Hi_arr, flag, Temp_arr, + leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (flag(i, j, k).isCovered()) { Hi_arr(i, j, k) = 0.0; } else { @@ -1012,7 +1031,8 @@ PeleLM::computeSpeciesEnthalpyFlux( #endif { amrex::ParallelFor( - gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + gbx, [Temp_arr, Hi_arr, + leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { getHGivenT(i, j, k, Temp_arr, Hi_arr, leosparm); }); } @@ -1027,6 +1047,7 @@ PeleLM::computeSpeciesEnthalpyFlux( //------------------------------------------------------------------------ // Compute \sum_k { \Flux_k * h_k } + #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -1039,9 +1060,10 @@ PeleLM::computeSpeciesEnthalpyFlux( a_fluxes[lev][idim]->array(mfi, NUM_SPECIES + 1); auto const& enth_ar = Enth_ec[idim].const_array(mfi); amrex::ParallelFor( - ebox, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebox, [enthflux_ar, spflux_ar, + enth_ar] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { enthflux_ar(i, j, k) = 0.0; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { enthflux_ar(i, j, k) += spflux_ar(i, j, k, n) * enth_ar(i, j, k, n); } @@ -1053,8 +1075,8 @@ PeleLM::computeSpeciesEnthalpyFlux( void PeleLM::differentialDiffusionUpdate( - std::unique_ptr& advData, - std::unique_ptr& diffData) + const std::unique_ptr& advData, + const std::unique_ptr& diffData) { BL_PROFILE("PeleLMeX::differentialDiffusionUpdate()"); @@ -1071,7 +1093,7 @@ PeleLM::differentialDiffusionUpdate( for (int lev = 0; lev <= finest_level; ++lev) { const auto& ba = grids[lev]; const auto& factory = Factory(lev); - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { fluxes[lev][idim].define( amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)), dmap[lev], NUM_SPECIES + 2, nGrow, amrex::MFInfo(), factory); @@ -1083,10 +1105,11 @@ PeleLM::differentialDiffusionUpdate( } } #ifdef AMREX_USE_EB - amrex::Vector EBfluxes(finest_level + 1); + amrex::Vector EBfluxes; if (m_isothermalEB != 0) { + EBfluxes.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - EBfluxes[lev].define( + EBfluxes.emplace_back( grids[lev], dmap[lev], 1, nGrow, amrex::MFInfo(), Factory(lev)); } } @@ -1100,6 +1123,7 @@ PeleLM::differentialDiffusionUpdate( // Get t^{n} data pointer auto* ldata_p = getLevelDataPtr(lev, AmrOldTime); + #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -1137,11 +1161,11 @@ PeleLM::differentialDiffusionUpdate( auto bcRecAux = fetchBCRecAuxArray(0, m_nAux); amrex::Vector spec_boundary; if (m_soret_boundary_override != 0) { - spec_boundary.resize(finest_level + 1); + spec_boundary.reserve(finest_level + 1); // this is the same regardless of lagged or not for (int lev = 0; lev <= finest_level; ++lev) { auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); - spec_boundary[lev].define( + spec_boundary.emplace_back( grids[lev], dmap[lev], NUM_SPECIES, 1, amrex::MFInfo(), Factory(lev)); // if we have a mix of Dirichlet and Isothermal walls, we need to give @@ -1150,7 +1174,7 @@ PeleLM::differentialDiffusionUpdate( amrex::MultiFab::Copy( spec_boundary[lev], ldata_p->state, FIRSTSPEC, 0, NUM_SPECIES, 1); - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { amrex::MultiFab::Divide( spec_boundary[lev], ldata_p->state, DENSITY, n, 1, 1); } @@ -1178,7 +1202,7 @@ PeleLM::differentialDiffusionUpdate( GetVecOfConstPtrs(getDiffusivityVect(AmrNewTime)), 0, bcRecSpec, NUM_SPECIES - NUM_IONS, 0, m_dt, {}); // Ions one by one - for (int n = 0; n < NUM_IONS; n++) { + for (int n = 0; n < NUM_IONS; ++n) { auto bcRecIons = fetchBCRecArray(FIRSTSPEC + NUM_SPECIES - NUM_IONS + n, 1); getDiffusionOp()->diffuse_scalar( GetVecOfPtrs(getSpeciesVect(AmrNewTime)), NUM_SPECIES - NUM_IONS + n, @@ -1237,7 +1261,8 @@ PeleLM::differentialDiffusionUpdate( diffData->wbar_fluxes[lev][idim].const_array(mfi); amrex::ParallelFor( ebx, NUM_SPECIES, - [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + [flux_spec, + flux_wbar] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { flux_spec(i, j, k, n) += flux_wbar(i, j, k, n); }); } @@ -1260,7 +1285,8 @@ PeleLM::differentialDiffusionUpdate( diffData->soret_fluxes[lev][idim].const_array(mfi); amrex::ParallelFor( ebx, NUM_SPECIES, - [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + [flux_spec, + flux_soret] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { flux_spec(i, j, k, n) += flux_soret(i, j, k, n); }); } @@ -1298,6 +1324,7 @@ PeleLM::differentialDiffusionUpdate( for (int lev = 0; lev <= finest_level; ++lev) { auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); + auto dt = m_dt; #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) @@ -1323,7 +1350,6 @@ PeleLM::differentialDiffusionUpdate( ? advData->Forcing_aux[lev].const_array(mfi, 0) : DummyFab.const_array(); const auto nAux = m_nAux; - const auto dt = m_dt; const auto use_wbar = m_use_wbar; const auto use_soret = m_use_soret; amrex::ParallelFor( @@ -1364,9 +1390,10 @@ PeleLM::differentialDiffusionUpdate( #ifdef AMREX_USE_EB if (m_isothermalEB != 0) { // Set up EB dirichlet value and diffusivity - amrex::Vector EBdiff(finest_level + 1); + amrex::Vector EBdiff; + EBdiff.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - EBdiff[lev].define( + EBdiff.emplace_back( grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), EBFactory(lev)); getEBDiff(lev, AmrNewTime, EBdiff[lev], NUM_SPECIES); } @@ -1419,17 +1446,18 @@ PeleLM::differentialDiffusionUpdate( //------------------------------------------------------------------------ // Temporary data holders - amrex::Vector rhs( - finest_level + 1); // Linear deltaT solve RHS - amrex::Vector Tsave( - finest_level + 1); // Storage of T while working on deltaT - amrex::Vector RhoCp( - finest_level + 1); // Acoeff of the linear solve + amrex::Vector rhs; + rhs.reserve(finest_level + 1); // Linear deltaT solve RHS + amrex::Vector Tsave; + Tsave.reserve(finest_level + 1); // Storage of T while working on deltaT + amrex::Vector RhoCp; + RhoCp.reserve(finest_level + 1); // Acoeff of the linear solve for (int lev = 0; lev <= finest_level; ++lev) { - rhs[lev].define(grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), Factory(lev)); - Tsave[lev].define( + rhs.emplace_back( + grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), Factory(lev)); + Tsave.emplace_back( grids[lev], dmap[lev], 1, 1, amrex::MFInfo(), Factory(lev)); - RhoCp[lev].define( + RhoCp.emplace_back( grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), Factory(lev)); } @@ -1453,12 +1481,14 @@ PeleLM::differentialDiffusionUpdate( if (m_isothermalEB != 0) { // Set up EB dirichlet value and diffusivity // Dirichlet value is deltaT - amrex::Vector EBvalue(finest_level + 1); - amrex::Vector EBdiff(finest_level + 1); + amrex::Vector EBvalue; + EBvalue.reserve(finest_level + 1); + amrex::Vector EBdiff; + EBdiff.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - EBvalue[lev].define( + EBvalue.emplace_back( grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), EBFactory(lev)); - EBdiff[lev].define( + EBdiff.emplace_back( grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), EBFactory(lev)); getEBDiff(lev, AmrNewTime, EBdiff[lev], NUM_SPECIES); EBvalue[lev].setVal(0.0); @@ -1514,15 +1544,15 @@ PeleLM::deltaTIter_prepare( const amrex::Vector& a_rhs, const amrex::Vector& a_Tsave, const amrex::Vector& a_rhoCp, - std::unique_ptr& advData, - std::unique_ptr& diffData) + const std::unique_ptr& advData, + const std::unique_ptr& diffData) { auto const* leosparm = eos_parms.device_parm(); + const amrex::Real dtinv = 1.0 / m_dt; for (int lev = 0; lev <= finest_level; ++lev) { auto* ldataOld_p = getLevelDataPtr(lev, AmrOldTime); auto* ldataNew_p = getLevelDataPtr(lev, AmrNewTime); - #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -1537,7 +1567,6 @@ PeleLM::deltaTIter_prepare( auto const& diffDiff = diffData->Dhat[lev].const_array(mfi, NUM_SPECIES + 1); auto const& rhs = a_rhs[lev]->array(mfi); - const amrex::Real dtinv = 1.0 / m_dt; // Cpmix auto const& rho = ldataNew_p->state.const_array(mfi, DENSITY); @@ -1572,11 +1601,11 @@ PeleLM::deltaTIter_prepare( void PeleLM::deltaTIter_update( - int a_dtiter, + const int a_dtiter, const amrex::Vector>& a_fluxes, const amrex::Vector& a_ebfluxes, const amrex::Vector& a_Tsave, - std::unique_ptr& diffData, + const std::unique_ptr& diffData, amrex::Real& a_deltaT_norm) { #ifndef AMREX_USE_EB @@ -1611,9 +1640,10 @@ PeleLM::deltaTIter_update( #ifdef AMREX_USE_EB if (m_isothermalEB != 0) { // Set up EB dirichlet value and diffusivity - amrex::Vector EBdiff(finest_level + 1); + amrex::Vector EBdiff; + EBdiff.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - EBdiff[lev].define( + EBdiff.emplace_back( grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), EBFactory(lev)); getEBDiff(lev, AmrNewTime, EBdiff[lev], NUM_SPECIES); } @@ -1659,13 +1689,13 @@ PeleLM::deltaTIter_update( //------------------------------------------------------------------------ // Recompute RhoH + auto const* leosparm = eos_parms.device_parm(); for (int lev = 0; lev <= finest_level; ++lev) { - auto const* leosparm = eos_parms.device_parm(); auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); auto const& sma = ldata_p->state.arrays(); amrex::ParallelFor( - ldata_p->state, - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + ldata_p->state, [sma, leosparm] AMREX_GPU_DEVICE( + int box_no, int i, int j, int k) noexcept { getRHmixGivenTY( i, j, k, amrex::Array4(sma[box_no], DENSITY), amrex::Array4(sma[box_no], FIRSTSPEC), @@ -1678,12 +1708,12 @@ PeleLM::deltaTIter_update( void PeleLM::getScalarDiffForce( - std::unique_ptr& advData, - std::unique_ptr& diffData) + const std::unique_ptr& advData, + const std::unique_ptr& diffData) { - int* aux_advect_d = convertToDeviceVector(m_aux_advect).dataPtr(); - int* aux_diffuse_d = convertToDeviceVector(m_DiffTypeAux).dataPtr(); + const int* aux_advect_d = convertToDeviceVector(m_aux_advect).dataPtr(); + const int* aux_diffuse_d = convertToDeviceVector(m_DiffTypeAux).dataPtr(); for (int lev = 0; lev <= finest_level; ++lev) { @@ -1755,10 +1785,10 @@ PeleLM::getScalarDiffForce( void PeleLM::computeDivTau( - const TimeStamp& a_time, + const TimeStamp a_time, const amrex::Vector& a_divtau, - int use_density, - amrex::Real scale) + const int use_density, + const amrex::Real scale) { BL_PROFILE("PeleLMeX::computeDivTau()"); // Get the density component BCRec to get viscosity on faces @@ -1841,7 +1871,8 @@ PeleLM::getDiffusionLinOpBC( amrex::Vector> PeleLM::getDiffusionTensorOpBC( - amrex::Orientation::Side a_side, const amrex::Vector a_bc) + const amrex::Orientation::Side a_side, + const amrex::Vector& a_bc) { AMREX_ASSERT(a_bc.size() == AMREX_SPACEDIM); amrex::Vector> r( @@ -1853,7 +1884,7 @@ PeleLM::getDiffusionTensorOpBC( , r[1][idim] = amrex::LinOpBCType::Periodic; , r[2][idim] = amrex::LinOpBCType::Periodic;); } else { - for (int dir = 0; dir < AMREX_SPACEDIM; dir++) { + for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) { auto amrexbc = (a_side == amrex::Orientation::low) ? a_bc[dir].lo(idim) : a_bc[dir].hi(idim); if (amrexbc == amrex::BCType::ext_dir) { diff --git a/Source/PeleLMeX_DiffusionOp.H b/Source/PeleLMeX_DiffusionOp.H index ac4591c4a..49dd0e9f4 100644 --- a/Source/PeleLMeX_DiffusionOp.H +++ b/Source/PeleLMeX_DiffusionOp.H @@ -23,87 +23,87 @@ public: void diffuse_scalar( amrex::Vector const& a_phi, - int phi_comp, + const int phi_comp, amrex::Vector const& a_rhs, - int rhs_comp, + const int rhs_comp, amrex::Vector> const& a_fluxes, - int flux_comp, + const int flux_comp, amrex::Vector const& a_acoeff, amrex::Vector const& a_density, amrex::Vector const& a_bcoeff, - int bcoeff_comp, + const int bcoeff_comp, amrex::Vector a_bcrec, - int ncomp, - int isPoissonSolve, - amrex::Real dt, + const int ncomp, + const int isPoissonSolve, + const amrex::Real dt, amrex::Vector const& a_boundary); #ifdef AMREX_USE_EB void diffuse_scalar( amrex::Vector const& a_phi, - int phi_comp, + const int phi_comp, amrex::Vector const& a_phiEB, - int phiEB_comp, + const int phiEB_comp, amrex::Vector const& a_rhs, - int rhs_comp, + const int rhs_comp, amrex::Vector> const& a_fluxes, - int flux_comp, + const int flux_comp, amrex::Vector const& a_acoeff, amrex::Vector const& a_density, amrex::Vector const& a_bcoeff, - int bcoeff_comp, + const int bcoeff_comp, amrex::Vector const& a_bcoeffEB, - int bcoeffEB_comp, + const int bcoeffEB_comp, amrex::Vector a_bcrec, - int ncomp, - int isPoissonSolve, - amrex::Real dt, + const int ncomp, + const int isPoissonSolve, + const amrex::Real dt, amrex::Vector const& a_boundary); #endif void computeDiffLap( amrex::Vector const& laps, - int lap_comp, + const int lap_comp, amrex::Vector const& phi, - int phi_comp, + const int phi_comp, amrex::Vector const& bcoeff, - int bcoeff_comp, + const int bcoeff_comp, amrex::Vector a_bcrec, - int ncomp); + const int ncomp); void computeDiffFluxes( amrex::Vector> const& a_fluxes, - int flux_comp, + const int flux_comp, amrex::Vector const& a_phi, - int phi_comp, + const int phi_comp, amrex::Vector const& a_density, amrex::Vector const& a_bcoeff, - int bcoeff_comp, + const int bcoeff_comp, amrex::Vector a_bcrec, - int ncomp, - int do_avgDown, + const int ncomp, + const int do_avgDown, amrex::Vector const& a_boundary); #ifdef AMREX_USE_EB void computeDiffFluxes( amrex::Vector> const& a_fluxes, - int flux_comp, + const int flux_comp, amrex::Vector const& a_EBflux, - int ebflux_comp, + const int ebflux_comp, amrex::Vector const& a_phi, - int phi_comp, + const int phi_comp, amrex::Vector const& a_density, amrex::Vector const& a_bcoeff, - int bcoeff_comp, + const int bcoeff_comp, amrex::Vector const& a_EBvalue, amrex::Vector const& a_EBbcoeff, amrex::Vector a_bcrec, - int ncomp, - int do_avgDown, + const int ncomp, + const int do_avgDown, amrex::Vector const& a_boundary); #endif @@ -113,14 +113,14 @@ public: const amrex::Vector& a_phi, const amrex::Vector& a_boundary, const amrex::BCRec& a_bcrec, - int do_avgDown, + const int do_avgDown, int comp = 0) const; void avgDownFluxes( amrex::Vector> const& a_fluxes, - int flux_comp, - int ncomp) const; + const int flux_comp, + const int ncomp) const; void readParameters(); @@ -178,14 +178,14 @@ public: amrex::Vector const& a_density, amrex::Vector const& a_beta, const amrex::BCRec& a_bcrec, - amrex::Real scale); + const amrex::Real scale); void diffuse_velocity( amrex::Vector const& a_vel, amrex::Vector const& a_density, amrex::Vector const& a_beta, const amrex::BCRec& a_bcrec, - amrex::Real dt); + const amrex::Real dt); void computeGradientTensor( amrex::Vector> const& diff --git a/Source/PeleLMeX_DiffusionOp.cpp b/Source/PeleLMeX_DiffusionOp.cpp index 7f935523c..32bf8a9b0 100644 --- a/Source/PeleLMeX_DiffusionOp.cpp +++ b/Source/PeleLMeX_DiffusionOp.cpp @@ -11,7 +11,7 @@ //--------------------------------------------------------------------------------------- // Diffusion Operator -DiffusionOp::DiffusionOp(PeleLM* a_pelelm, int ncomp) +DiffusionOp::DiffusionOp(PeleLM* a_pelelm, const int ncomp) : m_pelelm(a_pelelm), m_ncomp(ncomp) { BL_PROFILE("DiffusionOp::DiffusionOp()"); @@ -93,30 +93,30 @@ DiffusionOp::DiffusionOp(PeleLM* a_pelelm, int ncomp) void DiffusionOp::diffuse_scalar( amrex::Vector const& a_phi, - int phi_comp, + const int phi_comp, amrex::Vector const& a_rhs, - int rhs_comp, + const int rhs_comp, amrex::Vector> const& a_flux, - int flux_comp, + const int flux_comp, amrex::Vector const& a_acoeff, amrex::Vector const& a_density, amrex::Vector const& a_bcoeff, - int bcoeff_comp, + const int bcoeff_comp, amrex::Vector a_bcrec, - int ncomp, - int isPoissonSolve, - amrex::Real a_dt, + const int ncomp, + const int isPoissonSolve, + const amrex::Real a_dt, amrex::Vector const& a_boundary) { BL_PROFILE("DiffusionOp::diffuse_scalar()"); //---------------------------------------------------------------- // What are we dealing with ? - int have_density = (a_density.empty()) ? 0 : 1; - int have_fluxes = (a_flux.empty()) ? 0 : 1; - int have_acoeff = (a_acoeff.empty()) ? 0 : 1; - int have_bcoeff = (a_bcoeff.empty()) ? 0 : 1; - int have_boundary = (a_boundary.empty()) ? 0 : 1; + const int have_density = (a_density.empty()) ? 0 : 1; + const int have_fluxes = (a_flux.empty()) ? 0 : 1; + const int have_acoeff = (a_acoeff.empty()) ? 0 : 1; + const int have_bcoeff = (a_bcoeff.empty()) ? 0 : 1; + const int have_boundary = (a_boundary.empty()) ? 0 : 1; //---------------------------------------------------------------- // Checks @@ -131,7 +131,7 @@ DiffusionOp::diffuse_scalar( AMREX_ASSERT(a_bcrec.size() >= ncomp); } - int finest_level = m_pelelm->finestLevel(); + const int finest_level = m_pelelm->finestLevel(); //---------------------------------------------------------------- // Duplicate phi_old to include rho scaling @@ -140,9 +140,10 @@ DiffusionOp::diffuse_scalar( // after adv., when we divide by \rho, it is inconsistent. But it only matters // if it screws up the ghost cell values 'cause interiors are just an initial // solution for the solve. - amrex::Vector phi(finest_level + 1); + amrex::Vector phi; + phi.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - phi[lev].define( + phi.emplace_back( a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), ncomp, 1, amrex::MFInfo(), a_phi[lev]->Factory()); #ifdef AMREX_USE_OMP @@ -167,15 +168,14 @@ DiffusionOp::diffuse_scalar( }); } } - //---------------------------------------------------------------- // Setup solve LinearOp coefficients // LinOp is \alpha A \phi - \beta \nabla \cdot B \nabla \phi = rhs // => \alpha = 1.0, A is a_acoeff if provided, 1.0 otherwise // => \beta = a_dt, B face centered diffusivity bcoeff^{np1,k} - amrex::Real alpha = (isPoissonSolve) != 0 ? 0.0 : 1.0; - amrex::Real beta = a_dt; + const amrex::Real alpha = (isPoissonSolve != 0) ? 0.0 : 1.0; + const amrex::Real beta = a_dt; m_scal_solve_op->setScalars(alpha, beta); for (int lev = 0; lev <= finest_level; ++lev) { if (have_acoeff != 0) { @@ -204,15 +204,15 @@ DiffusionOp::diffuse_scalar( // Set aliases and bcoeff comp for (int lev = 0; lev <= finest_level; ++lev) { if (have_fluxes != 0) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { fluxes[lev][idim] = new amrex::MultiFab( *a_flux[lev][idim], amrex::make_alias, flux_comp + comp, m_ncomp); } } if (have_bcoeff != 0) { - int doZeroVisc = 1; - int addTurbContrib = 1; + constexpr int doZeroVisc = 1; + constexpr int addTurbContrib = 1; amrex::Vector subBCRec = { a_bcrec.begin() + comp, a_bcrec.begin() + comp + m_ncomp}; amrex::Array bcoeff_ec = @@ -270,7 +270,7 @@ DiffusionOp::diffuse_scalar( #endif for (int lev = 0; lev <= finest_level; ++lev) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { delete fluxes[lev][idim]; } } @@ -310,34 +310,34 @@ DiffusionOp::diffuse_scalar( void DiffusionOp::diffuse_scalar( amrex::Vector const& a_phi, - int phi_comp, + const int phi_comp, amrex::Vector const& a_phiEB, - int /*phiEB_comp*/, + const int /*phiEB_comp*/, amrex::Vector const& a_rhs, - int rhs_comp, + const int rhs_comp, amrex::Vector> const& a_flux, - int flux_comp, + const int flux_comp, amrex::Vector const& a_acoeff, amrex::Vector const& a_density, amrex::Vector const& a_bcoeff, - int bcoeff_comp, + const int bcoeff_comp, amrex::Vector const& a_bcoeffEB, - int /*bcoeffEB_comp*/, + const int /*bcoeffEB_comp*/, amrex::Vector a_bcrec, - int ncomp, - int isPoissonSolve, - amrex::Real a_dt, + const int ncomp, + const int isPoissonSolve, + const amrex::Real a_dt, amrex::Vector const& a_boundary) { BL_PROFILE("DiffusionOp::diffuse_scalar()"); //---------------------------------------------------------------- // What are we dealing with ? - int have_density = (a_density.empty()) ? 0 : 1; - int have_fluxes = (a_flux.empty()) ? 0 : 1; - int have_acoeff = (a_acoeff.empty()) ? 0 : 1; - int have_bcoeff = (a_bcoeff.empty()) ? 0 : 1; - int have_boundary = (a_boundary.empty()) ? 0 : 1; + const int have_density = (a_density.empty()) ? 0 : 1; + const int have_fluxes = (a_flux.empty()) ? 0 : 1; + const int have_acoeff = (a_acoeff.empty()) ? 0 : 1; + const int have_bcoeff = (a_bcoeff.empty()) ? 0 : 1; + const int have_boundary = (a_boundary.empty()) ? 0 : 1; //---------------------------------------------------------------- // Checks @@ -352,7 +352,7 @@ DiffusionOp::diffuse_scalar( AMREX_ASSERT(a_bcrec.size() >= ncomp); } - int finest_level = m_pelelm->finestLevel(); + const int finest_level = m_pelelm->finestLevel(); //---------------------------------------------------------------- // Duplicate phi_old to include rho scaling @@ -361,9 +361,10 @@ DiffusionOp::diffuse_scalar( // after adv., when we divide by \rho, it is inconsistent. But it only matters // if it screws up the ghost cell values 'cause interiors are just an initial // solution for the solve. - amrex::Vector phi(finest_level + 1); + amrex::Vector phi; + phi.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - phi[lev].define( + phi.emplace_back( a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), ncomp, 1, amrex::MFInfo(), a_phi[lev]->Factory()); #ifdef AMREX_USE_OMP @@ -388,20 +389,21 @@ DiffusionOp::diffuse_scalar( }); } } - //---------------------------------------------------------------- // Setup solve LinearOp coefficients // LinOp is \alpha A \phi - \beta \nabla \cdot B \nabla \phi = rhs // => \alpha = 1.0, A is a_acoeff if provided, 1.0 otherwise // => \beta = a_dt, B face centered diffusivity bcoeff^{np1,k} - amrex::Real alpha = (isPoissonSolve) != 0 ? 0.0 : 1.0; - amrex::Real beta = a_dt; + const amrex::Real alpha = (isPoissonSolve != 0) ? 0.0 : 1.0; + const amrex::Real beta = a_dt; m_scal_solve_op->setScalars(alpha, beta); - for (int lev = 0; lev <= finest_level; ++lev) { - if (have_acoeff != 0) { + if (have_acoeff != 0) { + for (int lev = 0; lev <= finest_level; ++lev) { m_scal_solve_op->setACoeffs(lev, *a_acoeff[lev]); - } else { + } + } else { + for (int lev = 0; lev <= finest_level; ++lev) { m_scal_solve_op->setACoeffs(lev, 1.0); } } @@ -425,14 +427,14 @@ DiffusionOp::diffuse_scalar( // Set aliases and bcoeff comp for (int lev = 0; lev <= finest_level; ++lev) { if (have_fluxes != 0) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { fluxes[lev][idim] = new amrex::MultiFab( *a_flux[lev][idim], amrex::make_alias, flux_comp + comp, m_ncomp); } } if (have_bcoeff != 0) { - int doZeroVisc = 1; + constexpr int doZeroVisc = 1; amrex::Vector subBCRec = { a_bcrec.begin() + comp, a_bcrec.begin() + comp + m_ncomp}; amrex::Array bcoeff_ec = @@ -483,7 +485,7 @@ DiffusionOp::diffuse_scalar( mlmg.getFluxes(fluxes, amrex::MLMG::Location::FaceCentroid); for (int lev = 0; lev <= finest_level; ++lev) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { delete fluxes[lev][idim]; } } @@ -523,13 +525,13 @@ DiffusionOp::diffuse_scalar( void DiffusionOp::computeDiffLap( amrex::Vector const& a_laps, - int lap_comp, + const int lap_comp, amrex::Vector const& a_phi, - int phi_comp, + const int phi_comp, amrex::Vector const& a_bcoeff, - int bcoeff_comp, + const int bcoeff_comp, amrex::Vector a_bcrec, - int ncomp) + const int ncomp) { BL_PROFILE("DiffusionOp::computeDiffLap()"); @@ -541,12 +543,13 @@ DiffusionOp::computeDiffLap( AMREX_ASSERT(a_bcoeff[0]->nComp() >= bcoeff_comp + ncomp); AMREX_ASSERT(a_bcrec.size() >= ncomp); - int finest_level = m_pelelm->finestLevel(); + const int finest_level = m_pelelm->finestLevel(); // Copy phi with 1 ghost cell - amrex::Vector phi(finest_level + 1); + amrex::Vector phi; + phi.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - phi[lev].define( + phi.emplace_back( a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), ncomp, 1, amrex::MFInfo(), a_phi[lev]->Factory()); amrex::MultiFab::Copy(phi[lev], *a_phi[lev], phi_comp, 0, ncomp, 1); @@ -571,8 +574,8 @@ DiffusionOp::computeDiffLap( laps.emplace_back( *a_laps[lev], amrex::make_alias, lap_comp + comp, m_ncomp); component.emplace_back(phi[lev], amrex::make_alias, comp, m_ncomp); - int doZeroVisc = 0; - int addTurbContrib = 0; + constexpr int doZeroVisc = 0; + constexpr int addTurbContrib = 0; amrex::Vector subBCRec = { a_bcrec.begin() + comp, a_bcrec.begin() + comp + m_ncomp}; amrex::Array bcoeff_ec = @@ -597,15 +600,15 @@ DiffusionOp::computeDiffLap( void DiffusionOp::computeDiffFluxes( amrex::Vector> const& a_flux, - int flux_comp, + const int flux_comp, amrex::Vector const& a_phi, - int phi_comp, + const int phi_comp, amrex::Vector const& a_density, amrex::Vector const& a_bcoeff, - int bcoeff_comp, + const int bcoeff_comp, amrex::Vector a_bcrec, - int ncomp, - int do_avgDown, + const int ncomp, + const int do_avgDown, amrex::Vector const& a_boundary) { BL_PROFILE("DiffusionOp::computeDiffFluxes()"); @@ -618,16 +621,17 @@ DiffusionOp::computeDiffFluxes( AMREX_ASSERT(a_bcoeff[0]->nComp() >= bcoeff_comp + ncomp); AMREX_ASSERT(a_bcrec.size() >= ncomp); - int finest_level = m_pelelm->finestLevel(); + const int finest_level = m_pelelm->finestLevel(); - int have_density = (a_density.empty()) ? 0 : 1; - int have_boundary = (a_boundary.empty()) ? 0 : 1; + const int have_density = (a_density.empty()) ? 0 : 1; + const int have_boundary = (a_boundary.empty()) ? 0 : 1; // Duplicate phi since it is modified by the LinOp // and if have_density -> divide by density - amrex::Vector phi(finest_level + 1); + amrex::Vector phi; + phi.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - phi[lev].define( + phi.emplace_back( a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), ncomp, 1, amrex::MFInfo(), a_phi[lev]->Factory()); #ifdef AMREX_USE_OMP @@ -658,8 +662,8 @@ DiffusionOp::computeDiffFluxes( // => \beta = -1.0, B face centered diffusivity a_bcoeff // Set scalars \alpha & \beta - amrex::Real alpha = 0.0; - amrex::Real beta = -1.0; + constexpr amrex::Real alpha = 0.0; + constexpr amrex::Real beta = -1.0; m_scal_apply_op->setScalars(alpha, beta); // Get fluxes on a m_ncomp component(s) basis @@ -678,7 +682,7 @@ DiffusionOp::computeDiffFluxes( m_pelelm->getDiffusionLinOpBC(amrex::Orientation::high, a_bcrec[comp])); for (int lev = 0; lev <= finest_level; ++lev) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { fluxes[lev][idim] = new amrex::MultiFab( *a_flux[lev][idim], amrex::make_alias, flux_comp + comp, m_ncomp); } @@ -690,8 +694,8 @@ DiffusionOp::computeDiffFluxes( boundary.emplace_back(phi[lev], amrex::make_alias, comp, m_ncomp); } - int doZeroVisc = 1; - int addTurbContrib = 1; + constexpr int doZeroVisc = 1; + constexpr int addTurbContrib = 1; amrex::Vector subBCRec = { a_bcrec.begin() + comp, a_bcrec.begin() + comp + m_ncomp}; amrex::Array bcoeff_ec = @@ -720,7 +724,7 @@ DiffusionOp::computeDiffFluxes( fluxes, GetVecOfPtrs(component), amrex::MLMG::Location::FaceCenter); #endif for (int lev = 0; lev <= finest_level; ++lev) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { delete fluxes[lev][idim]; } } @@ -736,19 +740,19 @@ DiffusionOp::computeDiffFluxes( void DiffusionOp::computeDiffFluxes( amrex::Vector> const& a_flux, - int flux_comp, + const int flux_comp, amrex::Vector const& a_EBflux, - int ebflux_comp, + const int ebflux_comp, amrex::Vector const& a_phi, - int phi_comp, + const int phi_comp, amrex::Vector const& a_density, amrex::Vector const& a_bcoeff, - int bcoeff_comp, + const int bcoeff_comp, amrex::Vector const& a_EBvalue, amrex::Vector const& a_EBbcoeff, amrex::Vector a_bcrec, - int ncomp, - int do_avgDown, + const int ncomp, + const int do_avgDown, amrex::Vector const& a_boundary) { BL_PROFILE("DiffusionOp::computeDiffFluxes()"); @@ -761,16 +765,17 @@ DiffusionOp::computeDiffFluxes( AMREX_ASSERT(a_bcoeff[0]->nComp() >= bcoeff_comp + ncomp); AMREX_ASSERT(a_bcrec.size() >= ncomp); - int finest_level = m_pelelm->finestLevel(); + const int finest_level = m_pelelm->finestLevel(); - int have_density = (a_density.empty()) ? 0 : 1; - int have_boundary = (a_boundary.empty()) ? 0 : 1; + const int have_density = (a_density.empty()) ? 0 : 1; + const int have_boundary = (a_boundary.empty()) ? 0 : 1; // Duplicate phi since it is modified by the LinOp // and if have_density -> divide by density - amrex::Vector phi(finest_level + 1); + amrex::Vector phi; + phi.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - phi[lev].define( + phi.emplace_back( a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), ncomp, 1, amrex::MFInfo(), a_phi[lev]->Factory()); #ifdef AMREX_USE_OMP @@ -801,8 +806,8 @@ DiffusionOp::computeDiffFluxes( // => \beta = -1.0, B face centered diffusivity a_bcoeff // Set scalars \alpha & \beta - amrex::Real alpha = 0.0; - amrex::Real beta = -1.0; + constexpr amrex::Real alpha = 0.0; + constexpr amrex::Real beta = -1.0; m_scal_apply_op->setScalars(alpha, beta); // Get fluxes on a m_ncomp component(s) basis @@ -823,7 +828,7 @@ DiffusionOp::computeDiffFluxes( m_pelelm->getDiffusionLinOpBC(amrex::Orientation::high, a_bcrec[comp])); for (int lev = 0; lev <= finest_level; ++lev) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { fluxes[lev][idim] = std::make_unique( *a_flux[lev][idim], amrex::make_alias, flux_comp + comp, m_ncomp); } @@ -837,7 +842,7 @@ DiffusionOp::computeDiffFluxes( } else { boundary.emplace_back(phi[lev], amrex::make_alias, comp, m_ncomp); } - int doZeroVisc = 1; + constexpr int doZeroVisc = 1; amrex::Vector subBCRec = { a_bcrec.begin() + comp, a_bcrec.begin() + comp + m_ncomp}; amrex::Array bcoeff_ec = @@ -875,13 +880,13 @@ DiffusionOp::computeGradient( const amrex::Vector& a_phi, const amrex::Vector& a_boundary, const amrex::BCRec& a_bcrec, - int do_avgDown, - int comp) const + const int do_avgDown, + const int comp) const { BL_PROFILE("DiffusionOp::computeGradient()"); // Do I need the Laplacian out ? - int need_laplacian = (a_laps.empty()) ? 0 : 1; + const int need_laplacian = (a_laps.empty()) ? 0 : 1; // Force updating the operator for (int lev = 0; lev <= m_pelelm->finestLevel(); ++lev) { m_gradient_op->setBCoeffs(lev, -1.0); @@ -891,8 +896,8 @@ DiffusionOp::computeGradient( AMREX_ASSERT(a_phi[0]->nComp() > comp); AMREX_ASSERT(a_phi[0]->nGrow() >= 1); - int finest_level = m_pelelm->finestLevel(); - int have_boundary = (a_boundary.empty()) ? 0 : 1; + const int finest_level = m_pelelm->finestLevel(); + const int have_boundary = (a_boundary.empty()) ? 0 : 1; // Set domainBCs m_gradient_op->setDomainBC( @@ -901,14 +906,16 @@ DiffusionOp::computeGradient( // Duplicate phi since it is modified by the LinOp // and setup level BCs - amrex::Vector phi(finest_level + 1); - amrex::Vector boundary(finest_level + 1); + amrex::Vector phi; + phi.reserve(finest_level + 1); + amrex::Vector boundary; + boundary.reserve(finest_level + 1); amrex::Vector laps; for (int lev = 0; lev <= finest_level; ++lev) { - phi[lev].define( + phi.emplace_back( a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), 1, 1, amrex::MFInfo(), a_phi[lev]->Factory()); - boundary[lev].define( + boundary.emplace_back( a_phi[lev]->boxArray(), a_phi[lev]->DistributionMap(), 1, 1, amrex::MFInfo(), a_phi[lev]->Factory()); @@ -946,17 +953,17 @@ DiffusionOp::computeGradient( void DiffusionOp::avgDownFluxes( const amrex::Vector>& a_fluxes, - int flux_comp, - int ncomp) const + const int flux_comp, + const int ncomp) const { - int finest_level = m_pelelm->finestLevel(); + const int finest_level = m_pelelm->finestLevel(); for (int lev = finest_level; lev > 0; --lev) { // Get the requested components only amrex::Array flux_fine; amrex::Array flux_crse; - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { flux_fine[idim] = new amrex::MultiFab( *a_fluxes[lev][idim], amrex::make_alias, flux_comp, ncomp); flux_crse[idim] = new amrex::MultiFab( @@ -971,7 +978,7 @@ DiffusionOp::avgDownFluxes( GetArrOfConstPtrs(flux_fine), flux_crse, m_pelelm->refRatio(lev - 1), flux_crse[0]->nGrow()); #endif - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { delete flux_fine[idim]; delete flux_crse[idim]; } @@ -999,7 +1006,7 @@ DiffusionTensorOp::DiffusionTensorOp(PeleLM* a_pelelm) : m_pelelm(a_pelelm) readParameters(); - int finest_level = m_pelelm->finestLevel(); + const int finest_level = m_pelelm->finestLevel(); auto bcRecVel = m_pelelm->fetchBCRecArray(VELX, AMREX_SPACEDIM); @@ -1062,12 +1069,13 @@ DiffusionTensorOp::computeGradientTensor( // in 3D: dU/dx, dV/dx, dW/dx, dU/dy, dV/dy, dW/dy, dU/dz, dV/dz, // dW/dz - int finest_level = m_pelelm->finestLevel(); + const int finest_level = m_pelelm->finestLevel(); // Duplicate vel since it may be modified by the TensorOp - amrex::Vector vel(finest_level + 1); + amrex::Vector vel; + vel.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - vel[lev].define( + vel.emplace_back( a_vel[lev]->boxArray(), a_vel[lev]->DistributionMap(), AMREX_SPACEDIM, 1, amrex::MFInfo(), a_vel[lev]->Factory()); amrex::MultiFab::Copy(vel[lev], *a_vel[lev], 0, 0, AMREX_SPACEDIM, 1); @@ -1110,16 +1118,17 @@ DiffusionTensorOp::compute_divtau( amrex::Vector const& a_density, amrex::Vector const& a_beta, const amrex::BCRec& a_bcrec, - amrex::Real scale) + const amrex::Real scale) { - int finest_level = m_pelelm->finestLevel(); + const int finest_level = m_pelelm->finestLevel(); - int have_density = (a_density.empty()) ? 0 : 1; + const int have_density = (a_density.empty()) ? 0 : 1; // Duplicate vel since it is modified by the TensorOp - amrex::Vector vel(finest_level + 1); + amrex::Vector vel; + vel.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - vel[lev].define( + vel.emplace_back( a_vel[lev]->boxArray(), a_vel[lev]->DistributionMap(), AMREX_SPACEDIM, 2, amrex::MFInfo(), a_vel[lev]->Factory()); amrex::MultiFab::Copy(vel[lev], *a_vel[lev], 0, 0, AMREX_SPACEDIM, 2); @@ -1127,9 +1136,10 @@ DiffusionTensorOp::compute_divtau( #ifdef AMREX_USE_EB // Need a temporary divTau to apply redistribution - amrex::Vector divtau_tmp(finest_level + 1); + amrex::Vector divtau_tmp; + divtau_tmp.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - divtau_tmp[lev].define( + divtau_tmp.emplace_back( a_divtau[lev]->boxArray(), a_divtau[lev]->DistributionMap(), AMREX_SPACEDIM, 2, amrex::MFInfo(), a_divtau[lev]->Factory()); divtau_tmp[lev].setVal(0.0); @@ -1141,8 +1151,8 @@ DiffusionTensorOp::compute_divtau( // anything. m_apply_op->setACoeffs(lev, *a_density[lev]); } - int doZeroVisc = 0; - int addTurbContrib = 1; + constexpr int doZeroVisc = 0; + constexpr int addTurbContrib = 1; amrex::Array beta_ec = m_pelelm->getDiffusivity( lev, 0, 1, doZeroVisc, {a_bcrec}, *a_beta[lev], addTurbContrib); @@ -1175,8 +1185,8 @@ DiffusionTensorOp::compute_divtau( // anything. m_apply_op->setACoeffs(lev, *a_density[lev]); } - int doZeroVisc = 0; - int addTurbContrib = 1; + constexpr int doZeroVisc = 0; + constexpr int addTurbContrib = 1; amrex::Array beta_ec = m_pelelm->getDiffusivity( lev, 0, 1, doZeroVisc, {a_bcrec}, *a_beta[lev], addTurbContrib); @@ -1216,12 +1226,12 @@ DiffusionTensorOp::diffuse_velocity( amrex::Vector const& a_density, amrex::Vector const& a_beta, const amrex::BCRec& a_bcrec, - amrex::Real a_dt) + const amrex::Real a_dt) { const int finest_level = m_pelelm->finestLevel(); - int have_density = (a_density.empty()) ? 0 : 1; + const int have_density = (a_density.empty()) ? 0 : 1; AMREX_ASSERT( (!m_pelelm->m_incompressible && have_density) || @@ -1234,8 +1244,8 @@ DiffusionTensorOp::diffuse_velocity( } else { m_solve_op->setACoeffs(lev, m_pelelm->m_rho); } - int doZeroVisc = 0; - int addTurbContrib = 1; + constexpr int doZeroVisc = 0; + constexpr int addTurbContrib = 1; amrex::Array beta_ec = m_pelelm->getDiffusivity( lev, 0, 1, doZeroVisc, {a_bcrec}, *a_beta[lev], addTurbContrib); @@ -1256,9 +1266,10 @@ DiffusionTensorOp::diffuse_velocity( m_solve_op->setLevelBC(lev, a_vel[lev]); } - amrex::Vector rhs(finest_level + 1); + amrex::Vector rhs; + rhs.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - rhs[lev].define( + rhs.emplace_back( a_vel[lev]->boxArray(), a_vel[lev]->DistributionMap(), AMREX_SPACEDIM, 0); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) diff --git a/Source/PeleLMeX_EB.cpp b/Source/PeleLMeX_EB.cpp index 2d451c58a..796f4b8cf 100644 --- a/Source/PeleLMeX_EB.cpp +++ b/Source/PeleLMeX_EB.cpp @@ -56,15 +56,15 @@ PeleLM::makeEBGeometry() void PeleLM::redistributeAofS( - int a_lev, - amrex::Real& a_dt, + const int a_lev, + const amrex::Real a_dt, amrex::MultiFab& a_tmpDiv, - int div_comp, + const int div_comp, amrex::MultiFab& a_AofS, - int aofs_comp, + const int aofs_comp, amrex::MultiFab& a_state, - int state_comp, - int ncomp, + const int state_comp, + const int ncomp, const amrex::BCRec* d_bc, const amrex::Geometry& a_geom) const { @@ -120,7 +120,7 @@ PeleLM::redistributeAofS( if (m_adv_redist_type == "FluxRedist") { amrex::ParallelFor( amrex::Box(scratch), - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + [scratch] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { scratch(i, j, k) = 1.; }); } @@ -132,7 +132,9 @@ PeleLM::redistributeAofS( } else { // Move data to AofS for regular bx amrex::ParallelFor( - bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + bx, ncomp, + [aofs_ar, + divT_ar] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { aofs_ar(i, j, k, n) = divT_ar(i, j, k, n); }); } @@ -141,7 +143,7 @@ PeleLM::redistributeAofS( } void -PeleLM::getCoveredIMask(int a_lev, amrex::iMultiFab& a_imask) const +PeleLM::getCoveredIMask(const int a_lev, amrex::iMultiFab& a_imask) const { const auto& ebfact = EBFactory(a_lev); const auto& flags = ebfact.getMultiEBCellFlagFab(); @@ -192,15 +194,15 @@ PeleLM::getCoveredIMask(int a_lev, amrex::iMultiFab& a_imask) const void PeleLM::redistributeDiff( - int a_lev, - const amrex::Real& a_dt, + const int a_lev, + const amrex::Real a_dt, amrex::MultiFab& a_tmpDiv, - int div_comp, + const int div_comp, amrex::MultiFab& a_diff, - int diff_comp, + const int diff_comp, const amrex::MultiFab& a_state, - int state_comp, - int ncomp, + const int state_comp, + const int ncomp, const amrex::BCRec* d_bc, const amrex::Geometry& a_geom) const { @@ -256,7 +258,7 @@ PeleLM::redistributeDiff( if (m_diff_redist_type == "FluxRedist") { amrex::ParallelFor( amrex::Box(scratch), - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + [scratch] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { scratch(i, j, k) = 1.; }); } @@ -268,7 +270,9 @@ PeleLM::redistributeDiff( } else { // Move data to AofS for regular bx amrex::ParallelFor( - bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + bx, ncomp, + [diff_ar, + divT_ar] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { diff_ar(i, j, k, n) = divT_ar(i, j, k, n); }); } @@ -295,7 +299,7 @@ PeleLM::initCoveredState() coveredState_h[0] = 0.0;, coveredState_h[1] = 0.0; , coveredState_h[2] = 0.0;) coveredState_h[DENSITY] = typical_values[DENSITY]; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { coveredState_h[FIRSTSPEC + n] = typical_values[FIRSTSPEC + n]; } coveredState_h[RHOH] = typical_values[RHOH]; @@ -310,16 +314,16 @@ PeleLM::initCoveredState() } void -PeleLM::setCoveredState(const TimeStamp& a_time) +PeleLM::setCoveredState(const TimeStamp a_time) { BL_PROFILE("PeleLMeX::setCoveredState()"); - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { setCoveredState(lev, a_time); } } void -PeleLM::setCoveredState(int lev, const TimeStamp& a_time) +PeleLM::setCoveredState(const int lev, const TimeStamp a_time) { AMREX_ASSERT(a_time == AmrOldTime || a_time == AmrNewTime); @@ -420,7 +424,7 @@ PeleLM::initialRedistribution() } void -PeleLM::getEBDistance(int a_lev, amrex::MultiFab& a_signDistLev) +PeleLM::getEBDistance(const int a_lev, amrex::MultiFab& a_signDistLev) { BL_PROFILE("PeleLMeX::getEBDistance()"); @@ -472,7 +476,8 @@ PeleLM::getEBDistance(int a_lev, amrex::MultiFab& a_signDistLev) } amrex::Vector> -PeleLM::getEBState(int first_comp, int ncomp, const PeleLM::TimeStamp& a_time) +PeleLM::getEBState( + const int first_comp, const int ncomp, const TimeStamp a_time) { AMREX_ASSERT(first_comp >= VELX); AMREX_ASSERT(first_comp + ncomp <= NVAR); @@ -490,7 +495,10 @@ PeleLM::getEBState(int first_comp, int ncomp, const PeleLM::TimeStamp& a_time) std::unique_ptr PeleLM::getEBState( - int a_lev, int first_comp, int ncomp, const PeleLM::TimeStamp& a_time) + const int a_lev, + const int first_comp, + const int ncomp, + const TimeStamp a_time) { AMREX_ASSERT(first_comp >= VELX); AMREX_ASSERT(first_comp + ncomp <= NVAR); @@ -505,10 +513,10 @@ PeleLM::getEBState( amrex::FArrayBox PeleLM::getEBState( amrex::MFIter const& a_mfi, - int a_lev, - int first_comp, - int ncomp, - const PeleLM::TimeStamp& a_time) + const int a_lev, + const int first_comp, + const int ncomp, + const TimeStamp a_time) { AMREX_ASSERT(first_comp >= VELX); AMREX_ASSERT(first_comp + ncomp <= NVAR); @@ -536,21 +544,31 @@ PeleLM::getEBState( , const auto& ebfc_y = ebfact.getFaceCent()[1]->const_array(a_mfi); , const auto& ebfc_z = ebfact.getFaceCent()[2]->const_array(a_mfi);); const auto& ebnorm = ebfact.getBndryNormal().const_array(a_mfi); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - // Regular/covered cells -> 0.0 - if (flag(i, j, k).isCovered() || flag(i, j, k).isRegular()) { - for (int n = 0; n < ncomp; n++) { - ebscal_arr(i, j, k, n) = 0.0; + amrex::ParallelFor( + bx, [flag, ebscal_arr, state, ebnorm, ncomp, EBfiller, first_comp, + geomdata, time, ebfc_x, ebfc_y +#if (AMREX_SPACEDIM == 3) + , + ebfc_z +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + // Regular/covered cells -> 0.0 + if (flag(i, j, k).isCovered() || flag(i, j, k).isRegular()) { + for (int n = 0; n < ncomp; ++n) { + ebscal_arr(i, j, k, n) = 0.0; + } + } else { // cut-cells + EBfiller( + i, j, k, state, ebscal_arr, first_comp, ncomp, + AMREX_D_DECL(ebfc_x, ebfc_y, ebfc_z), ebnorm, geomdata, time); } - } else { // cut-cells - EBfiller( - i, j, k, state, ebscal_arr, first_comp, ncomp, - AMREX_D_DECL(ebfc_x, ebfc_y, ebfc_z), ebnorm, geomdata, time); - } - }); + }); } else { - AMREX_PARALLEL_FOR_4D( - bx, ncomp, i, j, k, n, { ebscal_arr(i, j, k, n) = 0.0; }); + amrex::ParallelFor( + bx, ncomp, + [ebscal_arr] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + ebscal_arr(i, j, k, n) = 0.0; + }); } return r; @@ -558,11 +576,11 @@ PeleLM::getEBState( void PeleLM::getEBState( - int a_lev, - const PeleLM::TimeStamp& a_time, + const int a_lev, + const TimeStamp a_time, amrex::MultiFab& a_EBstate, - int stateComp, - int nComp) + const int stateComp, + const int nComp) { AMREX_ASSERT(a_EBstate.nComp() >= nComp); @@ -607,10 +625,16 @@ PeleLM::getEBState( , const auto& ebfc_z = faceCentroid[2]->array(mfi);); const auto& ebnorm = ebfact.getBndryNormal().const_array(mfi); amrex::ParallelFor( - bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [flag, ebState, state, EBfiller, stateComp, nComp, ebnorm, geomdata, + time, ebfc_x, ebfc_y +#if (AMREX_SPACEDIM == 3) + , + ebfc_z +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { // Regular/covered cells -> 0.0 if (flag(i, j, k).isCovered() || flag(i, j, k).isRegular()) { - for (int n = 0; n < nComp; n++) { + for (int n = 0; n < nComp; ++n) { ebState(i, j, k, n) = 0.0; } } else { // cut-cells @@ -625,7 +649,10 @@ PeleLM::getEBState( void PeleLM::getEBDiff( - int a_lev, const TimeStamp& a_time, amrex::MultiFab& a_EBDiff, int diffComp) + const int a_lev, + const TimeStamp a_time, + amrex::MultiFab& a_EBDiff, + const int diffComp) { // Get Geom / EB data ProbParm const* lprobparm = prob_parm_d; @@ -666,7 +693,13 @@ PeleLM::getEBDiff( , const auto& ebfc_y = faceCentroid[1]->array(mfi); , const auto& ebfc_z = faceCentroid[2]->array(mfi);); amrex::ParallelFor( - bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, + [flag, ebdiff, diff_cc, EBTypfiller, geomdata, lprobparm, ebfc_x, ebfc_y +#if (AMREX_SPACEDIM == 3) + , + ebfc_z +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { // Regular/covered cells -> 0.0 if (flag(i, j, k).isCovered() || flag(i, j, k).isRegular()) { ebdiff(i, j, k) = 0.0; @@ -696,7 +729,7 @@ PeleLM::correct_vel_small_cells( { BL_PROFILE("PeleLMeX::correct_vel_small_cells"); - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -736,18 +769,23 @@ PeleLM::correct_vel_small_cells( // This FAB has cut cells -- we define the centroid value in terms of // the MAC velocities onfaces amrex::ParallelFor( - bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [vfrac_fab, ccvel_fab, umac_fab, apx_fab, vmac_fab, apy_fab +#if (AMREX_SPACEDIM == 3) + , + wmac_fab, apz_fab +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (vfrac_fab(i, j, k) > 0.0 && vfrac_fab(i, j, k) < 5.e-3) { AMREX_D_TERM( - amrex::Real u_avg = + const amrex::Real u_avg = (apx_fab(i, j, k) * umac_fab(i, j, k) + apx_fab(i + 1, j, k) * umac_fab(i + 1, j, k)) / (apx_fab(i, j, k) + apx_fab(i + 1, j, k)); - , amrex::Real v_avg = + , const amrex::Real v_avg = (apy_fab(i, j, k) * vmac_fab(i, j, k) + apy_fab(i, j + 1, k) * vmac_fab(i, j + 1, k)) / (apy_fab(i, j, k) + apy_fab(i, j + 1, k)); - , amrex::Real w_avg = + , const amrex::Real w_avg = (apz_fab(i, j, k) * wmac_fab(i, j, k) + apz_fab(i, j, k + 1) * wmac_fab(i, j, k + 1)) / (apz_fab(i, j, k) + apz_fab(i, j, k + 1));); @@ -817,8 +855,7 @@ PeleLM::checkEBInflowFunctions() amrex::Print() << "WARNING: EB-inflow capability is experimental. Scalar " "diffusion is not supported at these boundaries and future " - "interface changes are possible!" - << std::endl; + "interface changes are possible!\n"; } } #endif diff --git a/Source/PeleLMeX_Eos.cpp b/Source/PeleLMeX_Eos.cpp index 06d9314d9..525c8831f 100644 --- a/Source/PeleLMeX_Eos.cpp +++ b/Source/PeleLMeX_Eos.cpp @@ -3,7 +3,7 @@ #include void -PeleLM::setThermoPress(const TimeStamp& a_time) +PeleLM::setThermoPress(const TimeStamp a_time) { BL_PROFILE("PeleLMeX::setThermoPress()"); @@ -15,7 +15,7 @@ PeleLM::setThermoPress(const TimeStamp& a_time) } void -PeleLM::setThermoPress(int lev, const TimeStamp& a_time) +PeleLM::setThermoPress(const int lev, const TimeStamp a_time) { AMREX_ASSERT(a_time == AmrOldTime || a_time == AmrNewTime); @@ -26,7 +26,7 @@ PeleLM::setThermoPress(int lev, const TimeStamp& a_time) amrex::ParallelFor( ldata_p->state, - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + [sma, leosparm] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { getPGivenRTY( i, j, k, amrex::Array4(sma[box_no], DENSITY), amrex::Array4(sma[box_no], FIRSTSPEC), @@ -38,11 +38,11 @@ PeleLM::setThermoPress(int lev, const TimeStamp& a_time) void PeleLM::calcDivU( - int is_init, - int computeDiff, - int do_avgDown, - const TimeStamp& a_time, - std::unique_ptr& diffData) + const int is_init, + const int computeDiff, + const int do_avgDown, + const TimeStamp a_time, + const std::unique_ptr& diffData) { BL_PROFILE("PeleLMeX::calcDivU()"); @@ -57,7 +57,7 @@ PeleLM::calcDivU( } // Assemble divU on each level - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { auto* ldata_p = getLevelDataPtr(lev, a_time); @@ -126,14 +126,16 @@ PeleLM::calcDivU( #ifdef AMREX_USE_EB if (flagfab.getType(bx) == amrex::FabType::covered) { // Covered boxes amrex::ParallelFor( - bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [divu] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { divu(i, j, k) = 0.0; }); } else if (flagfab.getType(bx) != amrex::FabType::regular) { // EB // containing // boxes amrex::ParallelFor( - bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, + [flag, divu, rhoY, T, SpecD, Fourier, DiffDiff, r, extRhoY, extRhoH, + use_react, leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (flag(i, j, k).isCovered()) { divu(i, j, k) = 0.0; } else { @@ -146,7 +148,9 @@ PeleLM::calcDivU( #endif { amrex::ParallelFor( - bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, + [divu, rhoY, T, SpecD, Fourier, DiffDiff, r, extRhoY, extRhoH, + use_react, leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { compute_divu( i, j, k, rhoY, T, SpecD, Fourier, DiffDiff, r, extRhoY, extRhoH, divu, use_react, leosparm); @@ -172,14 +176,14 @@ PeleLM::calcDivU( // fillPatch a_time divu to get properly filled ghost cells for (int lev = 0; lev <= finest_level; ++lev) { - amrex::Real time = getTime(lev, a_time); + const amrex::Real time = getTime(lev, a_time); auto* ldata_p = getLevelDataPtr(lev, a_time); fillpatch_divu(lev, time, ldata_p->divu, m_nGrowdivu); } } void -PeleLM::setRhoToSumRhoY(int lev, const TimeStamp& a_time) +PeleLM::setRhoToSumRhoY(const int lev, const TimeStamp a_time) { AMREX_ASSERT(a_time == AmrOldTime || a_time == AmrNewTime); @@ -189,7 +193,7 @@ PeleLM::setRhoToSumRhoY(int lev, const TimeStamp& a_time) amrex::ParallelFor( ldata_p->state, - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + [sma] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { pele::physics::PhysicsType::eos_type::RY2R( sma[box_no].cellData(i, j, k), sma[box_no](i, j, k, DENSITY), FIRSTSPEC); @@ -198,7 +202,7 @@ PeleLM::setRhoToSumRhoY(int lev, const TimeStamp& a_time) } void -PeleLM::setTemperature(const TimeStamp& a_time) +PeleLM::setTemperature(const TimeStamp a_time) { BL_PROFILE_VAR("PeleLMeX::setTemperature()", setTemperature); @@ -211,7 +215,7 @@ PeleLM::setTemperature(const TimeStamp& a_time) } void -PeleLM::setTemperature(int lev, const TimeStamp& a_time) +PeleLM::setTemperature(const int lev, const TimeStamp a_time) { AMREX_ASSERT(a_time == AmrOldTime || a_time == AmrNewTime); @@ -222,7 +226,7 @@ PeleLM::setTemperature(int lev, const TimeStamp& a_time) amrex::ParallelFor( ldata_p->state, - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + [sma, leosparm] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { getTfromHY( i, j, k, amrex::Array4(sma[box_no], DENSITY), amrex::Array4(sma[box_no], FIRSTSPEC), @@ -234,7 +238,7 @@ PeleLM::setTemperature(int lev, const TimeStamp& a_time) void PeleLM::calc_dPdt( - const TimeStamp& a_time, const amrex::Vector& a_dPdt) + const TimeStamp a_time, const amrex::Vector& a_dPdt) { BL_PROFILE("PeleLMeX::calc_dPdt()"); @@ -254,7 +258,8 @@ PeleLM::calc_dPdt( } void -PeleLM::calc_dPdt(int lev, const TimeStamp& a_time, amrex::MultiFab* a_dPdt) +PeleLM::calc_dPdt( + const int lev, const TimeStamp a_time, amrex::MultiFab* a_dPdt) { auto const& sma = getLevelDataPtr(lev, a_time)->state.arrays(); auto const& dPdtma = a_dPdt->arrays(); @@ -264,7 +269,8 @@ PeleLM::calc_dPdt(int lev, const TimeStamp& a_time, amrex::MultiFab* a_dPdt) const auto dt = m_dt; const auto dpdt_fac = m_dpdtFactor; amrex::ParallelFor( - *a_dPdt, [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + *a_dPdt, [dPdtma, sma, p_amb, dt, dpdt_fac] AMREX_GPU_DEVICE( + int box_no, int i, int j, int k) noexcept { auto dPdta = dPdtma[box_no]; auto sa = sma[box_no]; dPdta(i, j, k) = @@ -274,7 +280,7 @@ PeleLM::calc_dPdt(int lev, const TimeStamp& a_time, amrex::MultiFab* a_dPdt) } amrex::Real -PeleLM::adjustPandDivU(std::unique_ptr& advData) +PeleLM::adjustPandDivU(const std::unique_ptr& advData) { BL_PROFILE("PeleLMeX::adjustPandDivU()"); @@ -294,24 +300,25 @@ PeleLM::adjustPandDivU(std::unique_ptr& advData) const auto pNew = m_pNew; amrex::ParallelFor( *ThetaHalft[lev], - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + [tma, sma_o, sma_n, pOld, pNew, + leosparm] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { auto theta = tma[box_no]; - amrex::Real gammaInv_o = getGammaInv( + const amrex::Real gammaInv_o = getGammaInv( i, j, k, amrex::Array4(sma_o[box_no], FIRSTSPEC), amrex::Array4(sma_o[box_no], TEMP), leosparm); - amrex::Real gammaInv_n = getGammaInv( + const amrex::Real gammaInv_n = getGammaInv( i, j, k, amrex::Array4(sma_n[box_no], FIRSTSPEC), amrex::Array4(sma_n[box_no], TEMP), leosparm); theta(i, j, k) = 0.5 * (gammaInv_o / pOld + gammaInv_n / pNew); }); + amrex::Gpu::streamSynchronize(); } - amrex::Gpu::streamSynchronize(); // Get the mean mac_divu (Sbar) and mean theta - amrex::Real Sbar = MFSum(GetVecOfConstPtrs(advData->mac_divu), 0); - Sbar /= m_uncoveredVol; - amrex::Real Thetabar = MFSum(GetVecOfConstPtrs(ThetaHalft), 0); - Thetabar /= m_uncoveredVol; + const amrex::Real Sbar = + MFSum(GetVecOfConstPtrs(advData->mac_divu), 0) / m_uncoveredVol; + const amrex::Real Thetabar = + MFSum(GetVecOfConstPtrs(ThetaHalft), 0) / m_uncoveredVol; // Adjust for (int lev = 0; lev <= finest_level; ++lev) { @@ -322,11 +329,11 @@ PeleLM::adjustPandDivU(std::unique_ptr& advData) } // Compute 1/Volume * int(U_inflow)dA across all boundary faces - amrex::Real umacFluxBalance = AMREX_D_TERM( + const amrex::Real umacFluxBalance = AMREX_D_TERM( m_domainUmacFlux[0] + m_domainUmacFlux[1], +m_domainUmacFlux[2] + m_domainUmacFlux[3], +m_domainUmacFlux[4] + m_domainUmacFlux[5]); - amrex::Real divu_vol = umacFluxBalance / m_uncoveredVol; + const amrex::Real divu_vol = umacFluxBalance / m_uncoveredVol; // Advance the ambient pressure m_pNew = m_pOld + m_dt * (Sbar - divu_vol) / Thetabar; @@ -337,8 +344,8 @@ PeleLM::adjustPandDivU(std::unique_ptr& advData) auto const& tma = ThetaHalft[lev]->arrays(); auto const& uma = advData->mac_divu[lev].arrays(); amrex::ParallelFor( - *ThetaHalft[lev], - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + *ThetaHalft[lev], [uma, Sbar, Thetabar, divu_vol, tma] AMREX_GPU_DEVICE( + int box_no, int i, int j, int k) noexcept { auto theta = tma[box_no]; uma[box_no](i, j, k) -= (theta(i, j, k) * Sbar / Thetabar - @@ -351,7 +358,7 @@ PeleLM::adjustPandDivU(std::unique_ptr& advData) amrex::Print() << " >> Closed chamber pOld: " << m_pOld << ", pNew: " << m_pNew << ", dp0dt: " << m_dp0dt << "\n"; amrex::Print() << " >> Total mass old: " << m_massOld - << ", mass new: " << m_massNew << std::endl; + << ", mass new: " << m_massNew << "\n"; } // Return Sbar so that we'll add it back to mac_divu after the MAC projection diff --git a/Source/PeleLMeX_Evaluate.cpp b/Source/PeleLMeX_Evaluate.cpp index 5939da65d..7e2916fbd 100644 --- a/Source/PeleLMeX_Evaluate.cpp +++ b/Source/PeleLMeX_Evaluate.cpp @@ -12,7 +12,7 @@ PeleLM::Evaluate() // of the container and entries names int ncomp = 0; amrex::Vector plt_VarsName; - for (int ivar = 0; ivar < m_evaluatePlotVarCount; ivar++) { + for (int ivar = 0; ivar < m_evaluatePlotVarCount; ++ivar) { bool itexists = derive_lst.canDerive(m_evaluatePlotVars[ivar]) || evaluate_lst.canDerive(m_evaluatePlotVars[ivar]) || isStateVariable(m_evaluatePlotVars[ivar]); @@ -23,13 +23,13 @@ PeleLM::Evaluate() if (derive_lst.canDerive(m_evaluatePlotVars[ivar])) { const PeleLMDeriveRec* rec = derive_lst.get(m_evaluatePlotVars[ivar]); ncomp += rec->numDerive(); - for (int dvar = 0; dvar < rec->numDerive(); dvar++) { + for (int dvar = 0; dvar < rec->numDerive(); ++dvar) { plt_VarsName.push_back(rec->variableName(dvar)); } } else if (evaluate_lst.canDerive(m_evaluatePlotVars[ivar])) { const PeleLMDeriveRec* rec = evaluate_lst.get(m_evaluatePlotVars[ivar]); ncomp += rec->numDerive(); - for (int dvar = 0; dvar < rec->numDerive(); dvar++) { + for (int dvar = 0; dvar < rec->numDerive(); ++dvar) { plt_VarsName.push_back(rec->variableName(dvar)); } } else if (isStateVariable(m_evaluatePlotVars[ivar])) { @@ -40,9 +40,10 @@ PeleLM::Evaluate() //---------------------------------------------------------------- // Define the outgoing container - amrex::Vector mf_plt(finest_level + 1); + amrex::Vector mf_plt; + mf_plt.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - mf_plt[lev].define( + mf_plt.emplace_back( grids[lev], dmap[lev], ncomp, 0, amrex::MFInfo(), Factory(lev)); } @@ -55,7 +56,7 @@ PeleLM::Evaluate() //---------------------------------------------------------------- // Fill the outgoing container int cnt = 0; - for (int ivar = 0; ivar < m_evaluatePlotVarCount; ivar++) { + for (int ivar = 0; ivar < m_evaluatePlotVarCount; ++ivar) { int cntIncr = 0; amrex::Print() << " --> Evaluating " << m_evaluatePlotVars[ivar] << "\n"; @@ -96,7 +97,7 @@ PeleLM::Evaluate() void PeleLM::MLevaluate( const amrex::Vector& a_MFVec, - int a_comp, + const int a_comp, int& nComp, const std::string& a_var) { @@ -105,9 +106,9 @@ PeleLM::MLevaluate( // used in PeleLM:::Evolve if (a_var == "divU") { - int is_initialization = 0; // No, use IRR - int computeDiffusionTerm = 1; // Needed here - int do_avgDown = 1; // Always + constexpr int is_initialization = 0; // No, use IRR + constexpr int computeDiffusionTerm = 1; // Needed here + constexpr int do_avgDown = 1; // Always // Light version of the diffusion data container std::unique_ptr diffData; @@ -124,9 +125,9 @@ PeleLM::MLevaluate( nComp = 1; } else if (a_var == "velProj") { // Will need DivU - int is_initialization = 0; // No, use IRR - int computeDiffusionTerm = 1; // Needed here - int do_avgDown = 1; // Always + constexpr int is_initialization = 0; // No, use IRR + constexpr int computeDiffusionTerm = 1; // Needed here + constexpr int do_avgDown = 1; // Always // Light version of the diffusion data container std::unique_ptr diffData; @@ -149,7 +150,7 @@ PeleLM::MLevaluate( nComp = AMREX_SPACEDIM; } else if (a_var == "divTau") { // Velocity tensor components - int use_density = 0; + constexpr int use_density = 0; amrex::Vector> aliasDivTau( finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { @@ -241,7 +242,7 @@ PeleLM::MLevaluate( } } else if (a_var == "velForce") { // Velocity forces used in computing the velocity advance - int add_gradP = 0; + constexpr int add_gradP = 0; amrex::Vector> aliasMFVec( finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { @@ -268,7 +269,7 @@ PeleLM::evaluateChemExtForces( m_dt = computeDt(0, AmrOldTime); // Update time vectors - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { m_t_old[lev] = m_cur_time; m_t_new[lev] = m_cur_time + m_dt; } @@ -374,7 +375,7 @@ PeleLM::evaluateAdvectionTerms( m_dt = computeDt(0, AmrOldTime); // Update time vectors - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { m_t_old[lev] = m_cur_time; m_t_new[lev] = m_cur_time + m_dt; } diff --git a/Source/PeleLMeX_Evolve.cpp b/Source/PeleLMeX_Evolve.cpp index 3ac3a71d7..86848e338 100644 --- a/Source/PeleLMeX_Evolve.cpp +++ b/Source/PeleLMeX_Evolve.cpp @@ -47,7 +47,7 @@ PeleLM::Evolve() SprayPostRegrid(); } #endif - int is_init = 0; + constexpr int is_init = 0; Advance(is_init); m_nstep++; m_cur_time += m_dt; @@ -59,7 +59,7 @@ PeleLM::Evolve() #endif // Active control - int is_restart = 0; + constexpr int is_restart = 0; activeControl(is_restart); // Temporals @@ -71,9 +71,9 @@ PeleLM::Evolve() doDiagnostics(); // Check message - bool dump_and_stop = checkMessage("dump_and_stop"); - bool plt_and_continue = checkMessage("plt_and_continue"); - bool chk_and_continue = checkMessage("chk_and_continue"); + const bool dump_and_stop = checkMessage("dump_and_stop"); + const bool plt_and_continue = checkMessage("plt_and_continue"); + const bool chk_and_continue = checkMessage("chk_and_continue"); // Check for plot file if (writePlotNow() || dump_and_stop || plt_and_continue) { @@ -95,9 +95,8 @@ PeleLM::Evolve() if (t_elapsed >= (m_max_wall_time * 3600)) { over_max_wall_time = true; if (m_verbose > 0) { - amrex::Print() << std::endl - << "Reached maximum allowed wall time, stopping ..." - << std::endl; + amrex::Print() + << "\n Reached maximum allowed wall time, stopping ...\n"; } } } @@ -218,13 +217,7 @@ PeleLM::writeCheckNow() const bool PeleLM::doTemporalsNow() const { - bool write_now = false; - - if ((m_do_temporals != 0) && (m_nstep % m_temp_int == 0)) { - write_now = true; - } - - return write_now; + return (m_do_temporals != 0) && (m_nstep % m_temp_int == 0); } bool diff --git a/Source/PeleLMeX_FlowController.cpp b/Source/PeleLMeX_FlowController.cpp index a476f19a5..6488f2391 100644 --- a/Source/PeleLMeX_FlowController.cpp +++ b/Source/PeleLMeX_FlowController.cpp @@ -81,7 +81,9 @@ PeleLM::initActiveControl() amrex::Box dumbx({AMREX_D_DECL(0, 0, 0)}, {AMREX_D_DECL(0, 0, 0)}); amrex::ParallelFor( - dumbx, [=] AMREX_GPU_DEVICE(int /*i*/, int /*j*/, int /*k*/) noexcept { + dumbx, + [x, fake_state, s_ext_d, ctrl_flameDir_l, time_l, geomdata, lprobparm, + lpmfdata] AMREX_GPU_DEVICE(int /*i*/, int /*j*/, int /*k*/) noexcept { const auto s_in = fake_state.cellData(0, 0, 0); ProblemSpecificFunctions::bcnormal( x, s_in, s_ext_d, ctrl_flameDir_l, 1, time_l, geomdata, *lprobparm, @@ -100,7 +102,7 @@ PeleLM::initActiveControl() // Compute some active control parameters amrex::Real area_tot = 1.0; - for (int idim{0}; idim < AMREX_SPACEDIM; idim++) { + for (int idim{0}; idim < AMREX_SPACEDIM; ++idim) { if (idim != m_ctrl_flameDir) { area_tot *= (Geom(0).ProbHi(idim) - Geom(0).ProbLo(idim)); } @@ -138,7 +140,7 @@ PeleLM::initActiveControl() } void -PeleLM::activeControl(int is_restart) +PeleLM::activeControl(const int is_restart) { if (m_ctrl_active == 0) { return; @@ -203,7 +205,7 @@ PeleLM::activeControl(int is_restart) if (m_ctrl_nfilled <= 0) { amrex::Real velIntegral = 0.0; for (int n = 1; n <= m_ctrl_NavgPts; - n++) { // Piecewise constant velocity over NavgPts last steps + ++n) { // Piecewise constant velocity over NavgPts last steps velIntegral += 0.5 * (m_ctrl_velo_pts[n - 1] + m_ctrl_velo_pts[n]) * (m_ctrl_time_pts[n - 1] - m_ctrl_time_pts[n]); } @@ -313,7 +315,7 @@ PeleLM::activeControl(int is_restart) void PeleLM::getActiveControlLowT(amrex::Real& a_coft) { - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { // Get t^{n+1} data pointer auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); @@ -328,7 +330,7 @@ PeleLM::getActiveControlLowT(amrex::Real& a_coft) if (lev != finest_level) { lowT = amrex::ReduceMin( ldata_p->state, *m_coveredMask[lev], 0, - [=] AMREX_GPU_HOST_DEVICE( + [geomdata, AC_Tcross, AC_FlameDir] AMREX_GPU_HOST_DEVICE( amrex::Box const& bx, amrex::Array4 const& T_arr, amrex::Array4 const& covered_arr) -> amrex::Real { const auto lo = amrex::lbound(bx); @@ -368,7 +370,7 @@ PeleLM::getActiveControlLowT(amrex::Real& a_coft) } else { lowT = amrex::ReduceMin( ldata_p->state, 0, - [=] AMREX_GPU_HOST_DEVICE( + [geomdata, AC_Tcross, AC_FlameDir] AMREX_GPU_HOST_DEVICE( amrex::Box const& bx, amrex::Array4 const& T_arr) -> amrex::Real { const auto lo = amrex::lbound(bx); @@ -449,7 +451,7 @@ PeleLM::loadActiveControlHistory() } if (m_ctrl_verbose != 0) { amrex::Print() << " AC history arrays: \n"; - for (long int n = 0; n < m_ctrl_time_pts.size(); n++) { + for (long int n = 0; n < m_ctrl_time_pts.size(); ++n) { amrex::Print() << " [" << n << "] time: " << m_ctrl_time_pts[n] << ", velo: " << m_ctrl_velo_pts[n] << ", coft: " << m_ctrl_cntl_pts[n] << "\n"; diff --git a/Source/PeleLMeX_Forces.cpp b/Source/PeleLMeX_Forces.cpp index d8c836ddd..9e4cf6910 100644 --- a/Source/PeleLMeX_Forces.cpp +++ b/Source/PeleLMeX_Forces.cpp @@ -6,14 +6,14 @@ // including divTau if input amrex::Vector not empty void PeleLM::getVelForces( - const TimeStamp& a_time, + const TimeStamp a_time, const amrex::Vector& a_divTau, const amrex::Vector& a_velForce, - int nGrowForce, - int add_gradP) + const int nGrowForce, + const int add_gradP) { BL_PROFILE("PeleLMeX::getVelForces()"); - int has_divTau = static_cast(!a_divTau.empty()); + const int has_divTau = static_cast(!a_divTau.empty()); for (int lev = 0; lev <= finest_level; ++lev) { if (has_divTau != 0) { @@ -31,11 +31,11 @@ PeleLM::getVelForces( void PeleLM::getVelForces( - const TimeStamp& a_time, - int lev, + const TimeStamp a_time, + const int lev, amrex::MultiFab* a_divTau, amrex::MultiFab* a_velForce, - int add_gradP) + const int add_gradP) { // Get level data @@ -47,7 +47,7 @@ PeleLM::getVelForces( auto* ldataGP_p = (m_t_old[lev] < 0.0) ? getLevelDataPtr(lev, AmrNewTime) : getLevelDataPtr(lev, AmrOldTime); - amrex::Real time = getTime(lev, a_time); + const amrex::Real time = getTime(lev, a_time); int has_divTau = static_cast(a_divTau != nullptr); @@ -175,58 +175,58 @@ PeleLM::getVelForces( } void -PeleLM::addSpark(const TimeStamp& a_timestamp) +PeleLM::addSpark(const TimeStamp a_timestamp) { - for (int lev = 0; lev <= finest_level; lev++) { - for (int n = 0; n < m_n_sparks; n++) { + for (int lev = 0; lev <= finest_level; ++lev) { + for (int n = 0; n < m_n_sparks; ++n) { // Do the checks first - amrex::Real time = getTime(lev, a_timestamp); - bool verb = m_spark_verbose > 1 && lev == 0; + const amrex::Real time = getTime(lev, a_timestamp); + const bool verb = m_spark_verbose > 1 && lev == 0; if ( time < m_spark_time[n] || time > m_spark_time[n] + m_spark_duration[n]) { if (verb) { - amrex::Print() << m_spark[n] << " not active" << std::endl; + amrex::Print() << m_spark[n] << " not active \n"; } continue; } const amrex::Real* probLo = geom[lev].ProbLo(); auto const dx = geom[lev].CellSizeArray(); amrex::IntVect spark_idx; - for (int d = 0; d < AMREX_SPACEDIM; d++) { - spark_idx[d] = (int)((m_spark_location[n][d] - probLo[d]) / dx[d]); + for (int d = 0; d < AMREX_SPACEDIM; ++d) { + spark_idx[d] = + static_cast((m_spark_location[n][d] - probLo[d]) / dx[d]); } - amrex::Box domainBox = geom[lev].Domain(); + const amrex::Box domainBox = geom[lev].Domain(); // just a check if (!domainBox.contains(spark_idx)) { amrex::Warning(m_spark[n] + " not in domain!"); continue; } if (verb) { - amrex::Print() << m_spark[n] << " active" << std::endl; + amrex::Print() << m_spark[n] << " active\n"; } + auto const* eosparm = eos_parms.device_parm(); + auto eos = pele::physics::PhysicsType::eos(eosparm); - auto statema = getLevelDataPtr(lev, a_timestamp)->state.const_arrays(); - auto extma = m_extSource[lev]->arrays(); - auto const* leosparm = eos_parms.device_parm(); - const auto spark_duration = m_spark_duration[n]; - const auto spark_temp = m_spark_temp[n]; - const auto* eosparm = leosparm; - const auto spark_radius = m_spark_radius[n]; - + auto const& statema = + getLevelDataPtr(lev, a_timestamp)->state.const_arrays(); + auto const& extma = m_extSource[lev]->arrays(); amrex::ParallelFor( *m_extSource[lev], - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { - auto eos = pele::physics::PhysicsType::eos(eosparm); - amrex::Real dist_to_center = std::sqrt(AMREX_D_TERM( + [statema, extma, eos, dx, spark_idx, + spark_duration = m_spark_duration[n], spark_temp = m_spark_temp[n], + spark_radius = m_spark_radius + [n]] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + const amrex::Real dist_to_center = std::sqrt(AMREX_D_TERM( (i - spark_idx[0]) * (i - spark_idx[0]) * dx[0] * dx[0], +(j - spark_idx[1]) * (j - spark_idx[1]) * dx[1] * dx[1], +(k - spark_idx[2]) * (k - spark_idx[2]) * dx[2] * dx[2])); if (dist_to_center < spark_radius) { amrex::Real rhoh_src_loc = 0; - amrex::Real rho = statema[box_no](i, j, k, DENSITY); + const amrex::Real rho = statema[box_no](i, j, k, DENSITY); amrex::Real Y[NUM_SPECIES]; - for (int ns = 0; ns < NUM_SPECIES; ns++) { + for (int ns = 0; ns < NUM_SPECIES; ++ns) { Y[ns] = statema[box_no](i, j, k, FIRSTSPEC + ns) / rho; } eos.TY2H(spark_temp, Y, rhoh_src_loc); @@ -234,6 +234,7 @@ PeleLM::addSpark(const TimeStamp& a_timestamp) extma[box_no](i, j, k, RHOH) = rhoh_src_loc; } }); + // Shift outside? amrex::Gpu::streamSynchronize(); } } @@ -241,7 +242,7 @@ PeleLM::addSpark(const TimeStamp& a_timestamp) // Manifold model - dissipation rate sources for variances void -PeleLM::addScalarVarianceSources(const TimeStamp& a_timestamp) +PeleLM::addScalarVarianceSources(const TimeStamp a_timestamp) { BL_PROFILE("PeleLM::addScalarVarianceSources"); // no scalar dissipation sources if not using a manifold model @@ -275,15 +276,15 @@ PeleLM::addScalarVarianceSources(const TimeStamp& a_timestamp) } else if (nvariances > 0) { // Compute scalar gradients (no need to average down here) - int do_avgDown = 0; + constexpr int do_avgDown = 0; auto bcRecScalar = fetchBCRecArray(var_of_scalar, 1); - int nGrow = 0; // No need for ghost face on fluxes + constexpr int nGrow = 0; // No need for ghost face on fluxes amrex::Vector> grad_fc( finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { const auto& ba = grids[lev]; const auto& factory = Factory(lev); - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { grad_fc[lev][idim].define( amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)), dmap[lev], 1, nGrow, amrex::MFInfo(), factory); @@ -295,8 +296,14 @@ PeleLM::addScalarVarianceSources(const TimeStamp& a_timestamp) GetVecOfConstPtrs(getStateVect(a_timestamp)), {}, bcRecScalar[0], do_avgDown, var_of_scalar); - // Add in Production and Dissipation source terms for subfilter variances - for (int lev = 0; lev <= finest_level; lev++) { + constexpr amrex::Real fact = + 0.5 / static_cast(AMREX_SPACEDIM); + const amrex::Real C_chi = m_les_c_chi; + const amrex::Real ScInv = m_Schmidt_inv; + + // Add in Production and Dissipation source terms for subfilter + // variances + for (int lev = 0; lev <= finest_level; ++lev) { auto* ldata_p = getLevelDataPtr(lev, a_timestamp); @@ -306,16 +313,13 @@ PeleLM::addScalarVarianceSources(const TimeStamp& a_timestamp) // The simple interpolation below probably isn't valid for EB #ifdef AMREX_USE_EB amrex::Abort( - "PeleLM::addScalarVarianceSources(): this is not supported with EB"); + "PeleLM::addScalarVarianceSources(): this is not supported with " + "EB"); #endif for (int n = 0; n < MANIFOLD_DIM; ++n) { if (leosparm.is_variance_of[n] >= 0) { - constexpr amrex::Real fact = 0.5 / AMREX_SPACEDIM; - const amrex::Real C_chi = m_les_c_chi; - const amrex::Real ScInv = m_Schmidt_inv; - AMREX_D_TERM( auto const& mut_arr_x = m_leveldata_old[lev]->visc_turb_fc[0].const_arrays(); @@ -327,23 +331,31 @@ PeleLM::addScalarVarianceSources(const TimeStamp& a_timestamp) auto const& gx = grad_fc[lev][0].const_arrays(); , auto const& gy = grad_fc[lev][1].const_arrays(); , auto const& gz = grad_fc[lev][2].const_arrays();) - auto extma = m_extSource[lev]->arrays(); - auto statema = ldata_p->state.const_arrays(); + auto const& extma = m_extSource[lev]->arrays(); + auto const& statema = ldata_p->state.const_arrays(); // l_scale will also need modification for EB const amrex::Real vol = AMREX_D_TERM( geom[lev].CellSize(0), *geom[lev].CellSize(1), *geom[lev].CellSize(2)); - const amrex::Real l_scale = - (AMREX_SPACEDIM == 2) ? std::sqrt(vol) : std::cbrt(vol); +#if AMREX_SPACEDIM == 2 + const amrex::Real l_scale = std::sqrt(vol); +#else + const amrex::Real l_scale = std::cbrt(vol); +#endif const amrex::Real inv_l_scale2 = 1.0 / (l_scale * l_scale); amrex::ParallelFor( - *m_extSource[lev], - [=] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { + *m_extSource[lev], [extma, statema, n, C_chi, ScInv, inv_l_scale2, + mut_arr_x, gx, mut_arr_y, gy +#if (AMREX_SPACEDIM == 3) + , + mut_arr_z, gz +#endif + ] AMREX_GPU_DEVICE(int bx, int i, int j, int k) noexcept { // Subfilter Scalar Dissipation: Linear Relaxation model // rho chi_sgs = C_chi * mu_t / Delta^2 * Variance - amrex::Real mu_t = + const amrex::Real mu_t = fact * (AMREX_D_TERM( mut_arr_x[bx](i, j, k) + mut_arr_x[bx](i + 1, j, k), @@ -357,7 +369,7 @@ PeleLM::addScalarVarianceSources(const TimeStamp& a_timestamp) // Production term (w/ Smagorinsky closure for turbulent flux) // -2 (rho - rho ) d/dx_j // = 2 *mu_t/Sc_t * d/dx_j * d/dx_j - amrex::Real mu_grad2 = + const amrex::Real mu_grad2 = fact * (AMREX_D_TERM( mut_arr_x[bx](i, j, k) * gx[bx](i, j, k) * gx[bx](i, j, k) + @@ -374,9 +386,9 @@ PeleLM::addScalarVarianceSources(const TimeStamp& a_timestamp) extma[bx](i, j, k, FIRSTSPEC + n) += 2.0 * ScInv * mu_grad2; }); + amrex::Gpu::streamSynchronize(); } } - amrex::Gpu::streamSynchronize(); } } } @@ -386,9 +398,9 @@ PeleLM::addScalarVarianceSources(const TimeStamp& a_timestamp) // Calculate additional external sources (soot, radiation, user defined, etc.) void PeleLM::getExternalSources( - int is_initIter, - const PeleLM::TimeStamp& a_timestamp_old, - const PeleLM::TimeStamp& a_timestamp_new) + const int is_initIter, + const PeleLM::TimeStamp a_timestamp_old, + const PeleLM::TimeStamp a_timestamp_new) { amrex::ignore_unused(is_initIter); @@ -418,7 +430,7 @@ PeleLM::getExternalSources( // User defined external sources if (m_user_defined_ext_sources) { - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { auto* ldata_p_old = getLevelDataPtr(lev, a_timestamp_old); auto* ldata_p_new = getLevelDataPtr(lev, a_timestamp_new); auto& ext_src = m_extSource[lev]; diff --git a/Source/PeleLMeX_Init.cpp b/Source/PeleLMeX_Init.cpp index fd3d2efc8..f9406fc4e 100644 --- a/Source/PeleLMeX_Init.cpp +++ b/Source/PeleLMeX_Init.cpp @@ -22,19 +22,18 @@ PeleLM::Init() void PeleLM::MakeNewLevelFromScratch( - int lev, - amrex::Real time, + const int lev, + const amrex::Real time, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm) { BL_PROFILE("PeleLMeX::MakeNewLevelFromScratch()"); if (m_verbose > 0) { - amrex::Print() << " Making new level " << lev << " from scratch" - << std::endl; + amrex::Print() << " Making new level " << lev << " from scratch \n"; if (m_verbose > 2 && lev > 0) { auto const dx = geom[lev].CellSizeArray(); - amrex::Real vol = AMREX_D_TERM(dx[0], *dx[1], *dx[2]); + const amrex::Real vol = AMREX_D_TERM(dx[0], *dx[1], *dx[2]); amrex::Print() << " with " << ba.numPts() << " cells," << ba.size() << " boxes," << " over " @@ -43,7 +42,7 @@ PeleLM::MakeNewLevelFromScratch( << "% of the domain \n"; } if (m_verbose > 3 && lev > 0) { - amrex::Print() << " with BoxArray " << ba << std::endl; + amrex::Print() << " with BoxArray " << ba << "\n"; } } @@ -141,8 +140,9 @@ PeleLM::MakeNewLevelFromScratch( static_cast(refRatio(ilev - 1)[0]), static_cast(ilev)); } - extentFactor *= - std::sqrt(2.0) * m_derefineEBBuffer; // Account for diagonals + // Account for diagonals + constexpr amrex::Real sqrt2 = 1.4142135623730951; + extentFactor *= sqrt2 * m_derefineEBBuffer; amrex::MultiFab signDist( convert(grids[0], amrex::IntVect::TheUnitVector()), dmap[0], 1, 1, @@ -168,6 +168,7 @@ PeleLM::MakeNewLevelFromScratch( sd_cc(i, j, k) *= fac; }); } + m_signedDist0->FillBoundary(geom[0].periodicity()); extendSignedDistance(m_signedDist0.get(), extentFactor); } @@ -202,7 +203,7 @@ PeleLM::initData() //---------------------------------------------------------------- // Set typical values - int is_init = 1; + constexpr int is_init = 1; setTypicalValues(AmrNewTime, is_init); // initialize temporals @@ -239,7 +240,7 @@ PeleLM::initData() projectInitSolution(); // Active control - int is_restart = 0; + constexpr int is_restart = 0; activeControl(is_restart); //---------------------------------------------------------------- @@ -302,8 +303,8 @@ PeleLM::initData() m_t_old[lev] = -1.0e200; } m_dt = -1.0; - int is_init = 1; - amrex::Real dtInit = computeDt(is_init, AmrNewTime); + constexpr int is_init = 1; + const amrex::Real dtInit = computeDt(is_init, AmrNewTime); amrex::Print() << " Initial dt: " << dtInit << "\n"; } @@ -335,13 +336,13 @@ PeleLM::initData() updateDiagnostics(); // Active control - int is_restart = 1; + constexpr int is_restart = 1; activeControl(is_restart); } } void -PeleLM::initLevelData(int lev) +PeleLM::initLevelData(const int lev) { BL_PROFILE("PeleLMeX::initLevelData()"); @@ -387,7 +388,7 @@ PeleLM::initLevelData(int lev) void PeleLM::projectInitSolution() { - const int is_init = 1; + constexpr int is_init = 1; #ifdef PELE_USE_PLASMA poissonSolveEF(AmrNewTime); @@ -395,7 +396,7 @@ PeleLM::projectInitSolution() #endif // Post data Init time step estimate - amrex::Real dtInit = computeDt(is_init, AmrNewTime); + const amrex::Real dtInit = computeDt(is_init, AmrNewTime); amrex::Print() << " Initial dt: " << dtInit << "\n"; if (m_do_init_proj != 0) { @@ -404,9 +405,9 @@ PeleLM::projectInitSolution() // Subcycling IAMR/PeleLM first does a projection with no reaction divU // which can make the dt for evaluating I_R better if (m_has_divu != 0) { - int is_initialization = 1; // Yes we are - int computeDiffusionTerm = 1; // Needed here - int do_avgDown = 1; // Always + constexpr int is_initialization = 1; // Yes we are + constexpr int computeDiffusionTerm = 1; // Needed here + constexpr int do_avgDown = 1; // Always // Light version of the diffusion data container std::unique_ptr diffData; @@ -431,7 +432,7 @@ PeleLM::projectInitSolution() //---------------------------------------------------------------- // Initial velocity projection iterations - for (int iter = 0; iter < m_numDivuIter; iter++) { + for (int iter = 0; iter < m_numDivuIter; ++iter) { if (m_do_react != 0) { // The new level data has been filled above // Copy new -> old since old used in advanceChemistry @@ -459,9 +460,9 @@ PeleLM::projectInitSolution() copyStateOldToNew(); } if (m_has_divu != 0) { - int is_initialization = 1; // Yes we are - int computeDiffusionTerm = 1; // Needed here - int do_avgDown = 1; // Always + constexpr int is_initialization = 1; // Yes we are + constexpr int computeDiffusionTerm = 1; // Needed here + constexpr int do_avgDown = 1; // Always // Light version of the diffusion data container std::unique_ptr diffData; @@ -511,13 +512,13 @@ PeleLM::initialIterations() //---------------------------------------------------------------- // Initial pressure iterations - for (int iter = 0; iter < m_init_iter; iter++) { + for (int iter = 0; iter < m_init_iter; ++iter) { if (m_verbose > 0) { amrex::Print() << "\n ================ INITIAL ITERATION [" << iter << "] ================ \n"; } - int is_init = 1; + constexpr int is_init = 1; Advance(is_init); // Pass new pressure and gp from New to Old @@ -529,15 +530,15 @@ PeleLM::initialIterations() } void -PeleLM::InitFromGridFile(amrex::Real time) +PeleLM::InitFromGridFile(const amrex::Real time) { { const amrex::BoxArray& ba = MakeBaseGrids(); - amrex::DistributionMapping dm(ba); + const amrex::DistributionMapping dm(ba); MakeNewLevelFromScratch(0, time, ba, dm); } finest_level = static_cast(m_initial_ba.size()); - for (int lev = 1; lev <= finest_level; lev++) { + for (int lev = 1; lev <= finest_level; ++lev) { const amrex::BoxArray ba = m_initial_ba[lev - 1]; amrex::DistributionMapping dm(ba); MakeNewLevelFromScratch(lev, time, ba, dm); diff --git a/Source/PeleLMeX_K.H b/Source/PeleLMeX_K.H index 81d8e93b3..6fac1bfe1 100644 --- a/Source/PeleLMeX_K.H +++ b/Source/PeleLMeX_K.H @@ -10,9 +10,9 @@ namespace c2m = pele::physics::utilities::cgs2mks; template AMREX_GPU_DEVICE AMREX_FORCE_INLINE void getTransportCoeff( - int i, - int j, - int k, + const int i, + const int j, + const int k, const bool do_fixed_Le, const bool do_fixed_Pr, const bool do_soret, @@ -36,19 +36,19 @@ getTransportCoeff( // Get rho & Y from rhoY amrex::Real rho = 0.0; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { rho += rhoY(i, j, k, n); } - amrex::Real rhoinv = 1.0 / rho; + const amrex::Real rhoinv = 1.0 / rho; amrex::Real y[NUM_SPECIES] = {0.0}; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { y[n] = rhoY(i, j, k, n) * rhoinv; } amrex::Real Wbar = 0.0; eos.Y2WBAR(y, Wbar); - amrex::Real rho_cgs = m2c::Rho(rho); // MKS -> CGS conversion + const amrex::Real rho_cgs = m2c::Rho(rho); // MKS -> CGS conversion amrex::Real rhoDi_cgs[NUM_SPECIES] = {0.0}; amrex::Real lambda_cgs = 0.0; amrex::Real mu_cgs = 0.0; @@ -58,9 +58,9 @@ getTransportCoeff( constexpr bool get_xi = false; // always ignore bulk viscosity constexpr bool get_mu = true; // always get dynamic viscosity - bool get_lam = !do_fixed_Pr; - bool get_Ddiag = !do_fixed_Le; - bool get_chi = do_soret; + const bool get_lam = !do_fixed_Pr; + const bool get_Ddiag = !do_fixed_Le; + const bool get_chi = do_soret; auto trans = pele::physics::PhysicsType::transport(); trans.transport( get_xi, get_mu, get_lam, get_Ddiag, get_chi, Tloc, rho_cgs, y, rhoDi_cgs, @@ -73,38 +73,41 @@ getTransportCoeff( amrex::Real cpmix_cgs = 0.0; eos.TY2Cp(T(i, j, k), y, cpmix_cgs); lambda(i, j, k) = PrInv * c2m::Cp(cpmix_cgs) * mu(i, j, k); - amrex::Real ScInv = PrInv * LeInv; - for (int n = 0; n < NUM_SPECIES; n++) { + const amrex::Real ScInv = PrInv * LeInv; + for (int n = 0; n < NUM_SPECIES; ++n) { rhoDi(i, j, k, n) = ScInv * mu(i, j, k); } } else if (do_fixed_Pr) { // fixed Pr, still use species diffs amrex::Real cpmix_cgs = 0.0; eos.TY2Cp(T(i, j, k), y, cpmix_cgs); lambda(i, j, k) = PrInv * c2m::Cp(cpmix_cgs) * mu(i, j, k); - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { rhoDi(i, j, k, n) = rhoDi_cgs[n] * Wbar * mwtinv[n] * 1.0e-1; } } else if (do_fixed_Le) { // fixed Le, still use thermal cond lambda(i, j, k) = c2m::Lambda(lambda_cgs); amrex::Real cpmix_cgs = 0.0; eos.TY2Cp(T(i, j, k), y, cpmix_cgs); - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { rhoDi(i, j, k, n) = lambda_cgs * 1.0e-1 * LeInv / cpmix_cgs; } } else { // full MA model, with Soret if needed lambda(i, j, k) = c2m::Lambda(lambda_cgs); - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { rhoDi(i, j, k, n) = rhoDi_cgs[n] * Wbar * mwtinv[n] * 1.0e-1; if (do_soret) { rhotheta(i, j, k, n) = -rhoDi_cgs[n] * chi_loc[n] * 1.0e-1; } } if (do_soret) { -#if defined(H2_ID) && defined(H_ID) // Soret coefficient scaling, see Howarth, Day, Pitsch & Aspden +#if defined(H2_ID) rhotheta(i, j, k, H2_ID) *= 0.664; +#endif +#if defined(H_ID) rhotheta(i, j, k, H_ID) *= 0.58; -#else +#endif +#if !defined(H2_ID) && !defined(H_ID) amrex::Abort( "Running with Soret without light species, waste of time " "and memory..."); @@ -116,9 +119,9 @@ getTransportCoeff( template <> AMREX_GPU_DEVICE AMREX_FORCE_INLINE void getTransportCoeff( - int i, - int j, - int k, + const int i, + const int j, + const int k, const bool /*do_fixed_Le*/, const bool /*do_fixed_Pr*/, const bool do_soret, @@ -143,13 +146,13 @@ getTransportCoeff( amrex::Real rho, rhoinv; amrex::Real y[NUM_SPECIES] = {0.0}; amrex::Real massdens[NUM_SPECIES]; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { massdens[n] = rhoY(i, j, k, n); } auto eos = pele::physics::PhysicsType::eos(eosparm); eos.RY2RRinvY(massdens, rho, rhoinv, y); - amrex::Real rho_cgs = m2c::Rho(rho); // MKS -> CGS conversion + const amrex::Real rho_cgs = m2c::Rho(rho); // MKS -> CGS conversion amrex::Real rhoDi_cgs[NUM_SPECIES] = {0.0}; amrex::Real lambda_cgs = 0.0; amrex::Real mu_cgs = 0.0; @@ -161,14 +164,14 @@ getTransportCoeff( constexpr bool get_mu = true; constexpr bool get_lam = false; constexpr bool get_Ddiag = true; - bool get_chi = do_soret; + const bool get_chi = do_soret; auto trans = pele::physics::PhysicsType::transport(); trans.transport( get_xi, get_mu, get_lam, get_Ddiag, get_chi, Tloc, rho_cgs, y, rhoDi_cgs, dummy_chi, mu_cgs, dummy_xi, lambda_cgs, trans_parm); // Do CGS -> MKS conversions - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { rhoDi(i, j, k, n) = rhoDi_cgs[n] * 1.0e-1; } lambda(i, j, k) = 0.0; // No need to carry lambda for manifold @@ -179,9 +182,9 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void getVelViscosity( - int i, - int j, - int k, + const int i, + const int j, + const int k, amrex::Array4 const& rhoY, amrex::Array4 const& T, amrex::Array4 const& mu, @@ -191,13 +194,13 @@ getVelViscosity( { // Get rho & Y from rhoY amrex::Real massdens[NUM_SPECIES]; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { massdens[n] = rhoY(i, j, k, n); } amrex::Real rho = 0.0, rhoinv = 0.0, y[NUM_SPECIES] = {0.0}; pele::physics::PhysicsType::eos_type::RY2RRinvY(massdens, rho, rhoinv, y); - amrex::Real rho_cgs = m2c::Rho(rho); // MKS -> CGS conversion + const amrex::Real rho_cgs = m2c::Rho(rho); // MKS -> CGS conversion amrex::Real temp = T(i, j, k); amrex::Real dummy_rhoDi[NUM_SPECIES] = {0.0}; amrex::Real dummy_chi[NUM_SPECIES] = {0.0}; @@ -223,9 +226,9 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void getPGivenRTY( - int i, - int j, - int k, + const int i, + const int j, + const int k, amrex::Array4 const& rho, amrex::Array4 const& rhoY, amrex::Array4 const& T, @@ -234,11 +237,11 @@ getPGivenRTY( eosparm) noexcept { auto eos = pele::physics::PhysicsType::eos(eosparm); - amrex::Real rhoinv = 1.0 / rho(i, j, k); - amrex::Real rho_cgs = m2c::Rho(rho(i, j, k)); + const amrex::Real rhoinv = 1.0 / rho(i, j, k); + const amrex::Real rho_cgs = m2c::Rho(rho(i, j, k)); amrex::Real y[NUM_SPECIES] = {0.0}; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { y[n] = rhoY(i, j, k, n) * rhoinv; } amrex::Real P_cgs = 0.0; @@ -249,9 +252,9 @@ getPGivenRTY( template AMREX_GPU_DEVICE AMREX_FORCE_INLINE void compute_divu( - int i, - int j, - int k, + const int i, + const int j, + const int k, amrex::Array4 const& rhoY, amrex::Array4 const& T, amrex::Array4 const& specDiff, @@ -261,7 +264,7 @@ compute_divu( amrex::Array4 const& extRhoY, amrex::Array4 const& extRhoH, amrex::Array4 const& divu, - int do_react, + const int do_react, pele::physics::eos::EosParm const* eosparm) noexcept { @@ -271,12 +274,12 @@ compute_divu( // Get rho & Y from rhoY amrex::Real rho = 0.0; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { rho += rhoY(i, j, k, n); } - amrex::Real rhoinv = 1.0 / rho; + const amrex::Real rhoinv = 1.0 / rho; amrex::Real y[NUM_SPECIES] = {0.0}; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { y[n] = rhoY(i, j, k, n) * rhoinv; } @@ -288,15 +291,15 @@ compute_divu( eos.T2Hi(T(i, j, k), hi_cgs); amrex::Real cpmix = c2m::Cp(cpmix_cgs); // CGS -> MKS conversion amrex::Real hi[NUM_SPECIES] = {0.0}; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { hi[n] = c2m::H(hi_cgs[n]); // CGS -> MKS conversion } // Note: divu does not depend on extRho. See docs and PR #428 for details. - amrex::Real denominv = 1.0 / (rho * cpmix * T(i, j, k)); + const amrex::Real denominv = 1.0 / (rho * cpmix * T(i, j, k)); divu(i, j, k) = (specEnthDiff(i, j, k) + tempDiff(i, j, k) + extRhoH(i, j, k)) * denominv; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { amrex::Real specTerm = specDiff(i, j, k, n) + extRhoY(i, j, k, n); if (do_react != 0) { specTerm += rhoYdot(i, j, k, n); @@ -308,9 +311,9 @@ compute_divu( template <> AMREX_GPU_DEVICE AMREX_FORCE_INLINE void compute_divu( - int i, - int j, - int k, + const int i, + const int j, + const int k, amrex::Array4 const& rhoY, amrex::Array4 const& /*T*/, amrex::Array4 const& specDiff, @@ -320,7 +323,7 @@ compute_divu( amrex::Array4 const& extRhoY, amrex::Array4 const& /*extRhoH*/, amrex::Array4 const& divu, - int do_react, + const int do_react, pele::physics::eos::EosParm const* eosparm) noexcept { @@ -328,7 +331,7 @@ compute_divu( amrex::Real rho, rhoinv; amrex::Real y[NUM_SPECIES] = {0.0}; amrex::Real massdens[NUM_SPECIES] = {0.0}; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { massdens[n] = rhoY(i, j, k, n); } @@ -341,7 +344,7 @@ compute_divu( divu(i, j, k) = 0.0; - for (int n = 0; n < MANIFOLD_DIM; n++) { + for (int n = 0; n < MANIFOLD_DIM; ++n) { derivs[n] *= 1.0e3; // Unit conversion CGS -> MKS divu(i, j, k) += derivs[n] * specDiff(i, j, k, n); if (do_react != 0) { @@ -359,21 +362,21 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void extFluxDivergence_K( - int i, - int j, - int k, - int ncomp, + const int i, + const int j, + const int k, + const int ncomp, AMREX_D_DECL( amrex::Array4 const& fluxX, amrex::Array4 const& fluxY, amrex::Array4 const& fluxZ), amrex::Array4 const& vol, - const amrex::Real& scaling, + const amrex::Real scaling, amrex::Array4 const& div) noexcept { - amrex::Real factor = scaling / vol(i, j, k); + const amrex::Real factor = scaling / vol(i, j, k); - for (int n = 0; n < ncomp; n++) { + for (int n = 0; n < ncomp; ++n) { div(i, j, k, n) = AMREX_D_TERM( (fluxX(i + 1, j, k, n) - fluxX(i, j, k, n)), +(fluxY(i, j + 1, k, n) - fluxY(i, j, k, n)), @@ -386,10 +389,10 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void EB_intFluxDivergence_K( - int i, - int j, - int k, - int ncomp, + const int i, + const int j, + const int k, + const int ncomp, AMREX_D_DECL( amrex::Array4 const& fluxX, amrex::Array4 const& fluxY, @@ -400,12 +403,12 @@ EB_intFluxDivergence_K( amrex::Array4 const& afracZ), AMREX_D_DECL(amrex::Real areax, amrex::Real areay, amrex::Real areaz), amrex::Array4 const& vol, - const amrex::Real& scaling, + const amrex::Real scaling, amrex::Array4 const& div) noexcept { - amrex::Real factor = scaling / vol(i, j, k); + const amrex::Real factor = scaling / vol(i, j, k); - for (int n = 0; n < ncomp; n++) { + for (int n = 0; n < ncomp; ++n) { div(i, j, k, n) = AMREX_D_TERM( (areax * afracX(i + 1, j, k) * fluxX(i + 1, j, k, n) - areax * afracX(i, j, k) * fluxX(i, j, k, n)), @@ -421,10 +424,10 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void EB_intFluxDivergence_K( - int i, - int j, - int k, - int ncomp, + const int i, + const int j, + const int k, + const int ncomp, AMREX_D_DECL( amrex::Array4 const& fluxX, amrex::Array4 const& fluxY, @@ -437,14 +440,15 @@ EB_intFluxDivergence_K( amrex::Array4 const& ebFlux, amrex::Array4 const& ebAreaFrac, amrex::Array4 const& vol, - const amrex::Real& a_dx, - const amrex::Real& scaling, + const amrex::Real a_dx, + const amrex::Real scaling, amrex::Array4 const& div) noexcept { - amrex::Real factor = scaling / vol(i, j, k); - amrex::Real eb_area = AMREX_D_TERM(1.0, *a_dx, *a_dx) * ebAreaFrac(i, j, k); + const amrex::Real factor = scaling / vol(i, j, k); + const amrex::Real eb_area = + AMREX_D_TERM(1.0, *a_dx, *a_dx) * ebAreaFrac(i, j, k); - for (int n = 0; n < ncomp; n++) { + for (int n = 0; n < ncomp; ++n) { div(i, j, k, n) = AMREX_D_TERM( (areax * afracX(i + 1, j, k) * fluxX(i + 1, j, k, n) - areax * afracX(i, j, k) * fluxX(i, j, k, n)), @@ -461,22 +465,22 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void intFluxDivergence_K( - int i, - int j, - int k, - int ncomp, + const int i, + const int j, + const int k, + const int ncomp, AMREX_D_DECL( amrex::Array4 const& fluxX, amrex::Array4 const& fluxY, amrex::Array4 const& fluxZ), AMREX_D_DECL(amrex::Real areax, amrex::Real areay, amrex::Real areaz), amrex::Array4 const& vol, - const amrex::Real& scaling, + const amrex::Real scaling, amrex::Array4 const& div) noexcept { - amrex::Real factor = scaling / vol(i, j, k); + const amrex::Real factor = scaling / vol(i, j, k); - for (int n = 0; n < ncomp; n++) { + for (int n = 0; n < ncomp; ++n) { div(i, j, k, n) = AMREX_D_TERM( (areax * fluxX(i + 1, j, k, n) - areax * fluxX(i, j, k, n)), +(areay * fluxY(i, j + 1, k, n) - areay * fluxY(i, j, k, n)), @@ -490,10 +494,10 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void intFluxDivergence_rz_K( - int i, - int j, - int k, - int ncomp, + const int i, + const int j, + const int k, + const int ncomp, AMREX_D_DECL( amrex::Array4 const& fluxX, amrex::Array4 const& fluxY, @@ -501,12 +505,12 @@ intFluxDivergence_rz_K( amrex::Array4 const& ax, amrex::Array4 const& ay, amrex::Array4 const& vol, - const amrex::Real& scaling, + const amrex::Real scaling, amrex::Array4 const& div) noexcept { - amrex::Real factor = scaling / vol(i, j, k); + const amrex::Real factor = scaling / vol(i, j, k); - for (int n = 0; n < ncomp; n++) { + for (int n = 0; n < ncomp; ++n) { div(i, j, k, n) = (ax(i + 1, j, k) * fluxX(i + 1, j, k, n) - ax(i, j, k) * fluxX(i, j, k, n)) + (ay(i, j + 1, k) * fluxY(i, j + 1, k, n) - @@ -520,9 +524,9 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void getMwmixGivenRY( - int i, - int j, - int k, + const int i, + const int j, + const int k, amrex::Array4 const& rho, amrex::Array4 const& rhoY, amrex::Array4 const& Mwmix, @@ -530,9 +534,9 @@ getMwmixGivenRY( eosparm) noexcept { auto eos = pele::physics::PhysicsType::eos(eosparm); - amrex::Real rhoinv = 1.0 / rho(i, j, k); + const amrex::Real rhoinv = 1.0 / rho(i, j, k); amrex::Real y[NUM_SPECIES] = {0.0}; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { y[n] = rhoY(i, j, k, n) * rhoinv; } @@ -544,9 +548,9 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void getGradMwmixGivengradYMwmix( - int i, - int j, - int k, + const int i, + const int j, + const int k, amrex::Array4 const& gradY, amrex::Array4 const& Mwmix, amrex::Array4 const& gradMwmix, @@ -557,7 +561,7 @@ getGradMwmixGivengradYMwmix( amrex::Real imw[NUM_SPECIES] = {0.0}; eos.inv_molecular_weight(imw); gradMwmix(i, j, k) = 0.0; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { imw[n] *= 1000.0; // CGS -> MKS gradMwmix(i, j, k) += gradY(i, j, k, n) * imw[n]; } @@ -569,11 +573,11 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void cen2edg_cpp( - int i, - int j, - int k, - int dir, - int ncomp, + const int i, + const int j, + const int k, + const int dir, + const int ncomp, const bool do_harmonic, const bool on_lo, const bool on_hi, @@ -595,7 +599,7 @@ cen2edg_cpp( id_h[dir] -= 1; } if (do_harmonic) { - for (int n = 0; n < ncomp; n++) { + for (int n = 0; n < ncomp; ++n) { if ( (cfab(id_l[0], id_l[1], id_l[2], n) * cfab(id_h[0], id_h[1], id_h[2], n)) > 0.0) { @@ -609,7 +613,7 @@ cen2edg_cpp( } } } else { - for (int n = 0; n < ncomp; n++) { + for (int n = 0; n < ncomp; ++n) { efab(i, j, k, n) = 0.5 * (cfab(id_l[0], id_l[1], id_l[2], n) + cfab(id_h[0], id_h[1], id_h[2], n)); } @@ -620,10 +624,10 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void repair_flux( - int i, - int j, - int k, - int dir, + const int i, + const int j, + const int k, + const int dir, const bool on_lo, const bool on_hi, amrex::Array4 const& rhoY, @@ -646,14 +650,14 @@ repair_flux( amrex::Real sumFlux = 0.0; amrex::Real sumRhoYe = 0.0; amrex::Real RhoYe[NUM_SPECIES] = {0.0}; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { sumFlux += flux(i, j, k, n); RhoYe[n] = 0.5 * (rhoY(id_l[0], id_l[1], id_l[2], n) + rhoY(id_h[0], id_h[1], id_h[2], n)); sumRhoYe += RhoYe[n]; } sumRhoYe = 1.0 / sumRhoYe; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { flux(i, j, k, n) -= sumFlux * RhoYe[n] * sumRhoYe; } } @@ -662,10 +666,10 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void repair_flux_eb( - int i, - int j, - int k, - int dir, + const int i, + const int j, + const int k, + const int dir, const bool on_lo, const bool on_hi, amrex::Array4 const& rhoY, @@ -680,7 +684,7 @@ repair_flux_eb( amrex::Real sumFlux = 0.0; amrex::Real sumRhoYe = 0.0; amrex::Real RhoYe[NUM_SPECIES] = {0.0}; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { sumFlux += flux(i, j, k, n); RhoYe[n] = rhoYed(i, j, k, n); if (on_lo) { @@ -692,11 +696,11 @@ repair_flux_eb( sumRhoYe += RhoYe[n]; } sumRhoYe = 1.0 / sumRhoYe; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { flux(i, j, k, n) -= sumFlux * RhoYe[n] * sumRhoYe; } } else { - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { flux(i, j, k, n) = 0.0; } } @@ -706,19 +710,19 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void getHGivenT( - int i, - int j, - int k, + const int i, + const int j, + const int k, amrex::Array4 const& T, amrex::Array4 const& Hi, pele::physics::eos::EosParm const* eosparm) noexcept { auto eos = pele::physics::PhysicsType::eos(eosparm); - amrex::Real hi_spec[NUM_SPECIES] = {0.0}; - eos.T2Hi(T(i, j, k), hi_spec); - for (int n = 0; n < NUM_SPECIES; n++) { - Hi(i, j, k, n) = hi_spec[n] * 0.0001; // CGS -> MKS conversion + amrex::Real hi_spec_cgs[NUM_SPECIES] = {0.0}; + eos.T2Hi(T(i, j, k), hi_spec_cgs); + for (int n = 0; n < NUM_SPECIES; ++n) { + Hi(i, j, k, n) = c2m::H(hi_spec_cgs[n]); // CGS -> MKS conversion } } @@ -776,9 +780,9 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void getRHmixGivenTY( - int i, - int j, - int k, + const int i, + const int j, + const int k, amrex::Array4 const& rho, amrex::Array4 const& rhoY, amrex::Array4 const& T, @@ -787,9 +791,9 @@ getRHmixGivenTY( eosparm) noexcept { auto eos = pele::physics::PhysicsType::eos(eosparm); - amrex::Real rhoinv = 1.0 / rho(i, j, k); + const amrex::Real rhoinv = 1.0 / rho(i, j, k); amrex::Real y[NUM_SPECIES] = {0.0}; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { y[n] = rhoY(i, j, k, n) * rhoinv; } amrex::Real hmix_cgs = 0.0; @@ -801,9 +805,9 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void getTfromHY( - int i, - int j, - int k, + const int i, + const int j, + const int k, amrex::Array4 const& rho, amrex::Array4 const& rhoY, amrex::Array4 const& rhoH, @@ -812,12 +816,12 @@ getTfromHY( eosparm) noexcept { auto eos = pele::physics::PhysicsType::eos(eosparm); - amrex::Real rhoinv = 1.0 / rho(i, j, k); + const amrex::Real rhoinv = 1.0 / rho(i, j, k); amrex::Real y[NUM_SPECIES] = {0.0}; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { y[n] = rhoY(i, j, k, n) * rhoinv; } - amrex::Real h_cgs = m2c::H(rhoH(i, j, k) * rhoinv); + const amrex::Real h_cgs = m2c::H(rhoH(i, j, k) * rhoinv); amrex::Real T_loc = T(i, j, k); eos.HY2T(h_cgs, y, T_loc); T(i, j, k) = T_loc; @@ -827,9 +831,9 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void getCpmixGivenRYT( - int i, - int j, - int k, + const int i, + const int j, + const int k, amrex::Array4 const& rho, amrex::Array4 const& rhoY, amrex::Array4 const& T, @@ -838,9 +842,9 @@ getCpmixGivenRYT( eosparm) noexcept { auto eos = pele::physics::PhysicsType::eos(eosparm); - amrex::Real rhoinv = 1.0 / rho(i, j, k); + const amrex::Real rhoinv = 1.0 / rho(i, j, k); amrex::Real y[NUM_SPECIES] = {0.0}; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { y[n] = rhoY(i, j, k, n) * rhoinv; } amrex::Real cpmix_cgs = 0.0; @@ -852,9 +856,9 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void buildAdvectionForcing( - int i, - int j, - int k, + const int i, + const int j, + const int k, amrex::Array4 const& rho, amrex::Array4 const& rhoY, amrex::Array4 const& T, @@ -863,15 +867,15 @@ buildAdvectionForcing( amrex::Array4 const& r, amrex::Array4 const& extRhoY, amrex::Array4 const& extRhoH, - amrex::Real const& dp0dt, - int const& closed_chamber, - int do_react, + const amrex::Real dp0dt, + const int closed_chamber, + const int do_react, amrex::Array4 const& forceY, amrex::Array4 const& forceT, amrex::Array4 const& forceAux, amrex::Array4 const& dn_aux, const int* aux_diffuse, - int nAux, + const int nAux, pele::physics::eos::EosParm const* eosparm) noexcept { @@ -881,11 +885,11 @@ buildAdvectionForcing( eos.T2Hi(T(i, j, k), hi_spec_cgs); // For species mass fractions - amrex::Real rhoinv = 1.0 / rho(i, j, k); + const amrex::Real rhoinv = 1.0 / rho(i, j, k); amrex::Real y[NUM_SPECIES] = {0.0}; forceT(i, j, k) = dn(i, j, k, NUM_SPECIES) + ddn(i, j, k); - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { y[n] = rhoY(i, j, k, n) * rhoinv; // get y forceY(i, j, k, n) = dn(i, j, k, n); // forceY = Dnk if (do_react != 0) { @@ -903,7 +907,7 @@ buildAdvectionForcing( forceT(i, j, k) += dp0dt; // Add dp0/dt } - for (int n = 0; n < nAux; n++) { + for (int n = 0; n < nAux; ++n) { if (aux_diffuse[n] != 0) { forceAux(i, j, k, n) = dn_aux(i, j, k, n); } else { @@ -914,7 +918,8 @@ buildAdvectionForcing( amrex::Real cpmix_cgs = 0.0; eos.TY2Cp(T(i, j, k), y, cpmix_cgs); - amrex::Real cpmixinv = 1.0 / c2m::Cp(cpmix_cgs); // CGS -> MKS conversion + const amrex::Real cpmixinv = + 1.0 / c2m::Cp(cpmix_cgs); // CGS -> MKS conversion forceT(i, j, k) *= rhoinv * cpmixinv; } @@ -922,35 +927,35 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void buildDiffusionForcing( - int i, - int j, - int k, + const int i, + const int j, + const int k, amrex::Array4 const& dn, amrex::Array4 const& ddn, amrex::Array4 const& dnp1k, amrex::Array4 const& ddnp1k, amrex::Array4 const& r, amrex::Array4 const& a, - amrex::Real const& dp0dt, - int const& closed_chamber, - int do_react, + const amrex::Real dp0dt, + const int closed_chamber, + const int do_react, amrex::Array4 const& forceY, amrex::Array4 const& forceT, amrex::Array4 const& dwbar, amrex::Array4 const& dT, amrex::Array4 const& extRhoY, amrex::Array4 const& extRhoH, - int use_wbar, - int use_soret, + const int use_wbar, + const int use_soret, amrex::Array4 const& forceAux, amrex::Array4 const& a_aux, amrex::Array4 const& dn_aux, amrex::Array4 const& dnp1k_aux, const int* aux_advect, const int* aux_diffuse, - int nAux) noexcept + const int nAux) noexcept { - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { forceY(i, j, k, n) = a(i, j, k, n) + 0.5 * (dn(i, j, k, n) - dnp1k(i, j, k, n)) + extRhoY(i, j, k, n); @@ -973,7 +978,7 @@ buildDiffusionForcing( forceT(i, j, k) += dp0dt; } - for (int n = 0; n < nAux; n++) { + for (int n = 0; n < nAux; ++n) { forceAux(i, j, k, n) = 0.0; if (aux_diffuse[n] != 0) { forceAux(i, j, k, n) += @@ -990,9 +995,9 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void reactionRateRhoY( - int i, - int j, - int k, + const int i, + const int j, + const int k, amrex::Array4 const& rhoY, amrex::Array4 const& rhoH, amrex::Array4 const& T, @@ -1008,15 +1013,15 @@ reactionRateRhoY( // Get T from Y/H. amrex::Real Tloc = T(i, j, k); - amrex::Real h_cgs = + const amrex::Real h_cgs = m2c::H(rhoH(i, j, k) * rhoinv); // Include MKS -> CGS conversion eos.HY2T(h_cgs, y, Tloc); // Get wdot. - amrex::Real rho_cgs = m2c::Rho(rho); // rho MKS -> CGS + const amrex::Real rho_cgs = m2c::Rho(rho); // rho MKS -> CGS amrex::Real wdot[NUM_SPECIES] = {0.0}; eos.RTY2WDOT(rho_cgs, Tloc, y, wdot); - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { rhoYdot(i, j, k, n) = wdot[n] * 1000.0; // CGS -> MKS conversion } } @@ -1025,15 +1030,16 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void fabMinMax( - int i, - int j, - int k, - int ncomp, - amrex::Real fmin, - amrex::Real fmax, + const int i, + const int j, + const int k, + const int ncomp, + const amrex::Real fmin, + const amrex::Real fmax, amrex::Array4 const& fab) noexcept { - for (int n = 0; n < ncomp; n++) { + + for (int n = 0; n < ncomp; ++n) { fab(i, j, k, n) = amrex::max(fmin, amrex::min(fmax, fab(i, j, k, n))); } } @@ -1044,11 +1050,11 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real est_divu_dt_1( - int i, - int j, - int k, - amrex::Real const& dtfactor, - amrex::Real const& rhomin, + const int i, + const int j, + const int k, + const amrex::Real dtfactor, + const amrex::Real rhomin, amrex::Array4 const& rho, amrex::Array4 const& divu) noexcept { @@ -1069,19 +1075,19 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real est_divu_dt_2( - int i, - int j, - int k, - amrex::Real const& dtfactor, - amrex::Real const& rhomin, - amrex::GpuArray const dxinv, + const int i, + const int j, + const int k, + const amrex::Real dtfactor, + const amrex::Real rhomin, + amrex::GpuArray const& dxinv, amrex::Array4 const& rho, amrex::Array4 const& vel, amrex::Array4 const& divu) noexcept { amrex::Real dtcell = 1.0e20; amrex::Real denom = 0.0; - bool rho_is_okay = (rho(i, j, k) > rhomin); + const bool rho_is_okay = (rho(i, j, k) > rhomin); denom = rho(i, j, k) * divu(i, j, k) + @@ -1101,13 +1107,13 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void check_divu_dt( - int i, - int j, - int k, - int check_type, - amrex::Real const& dtfactor, - amrex::Real const& rhomin, - amrex::GpuArray const dxinv, + const int i, + const int j, + const int k, + const int check_type, + const amrex::Real dtfactor, + const amrex::Real rhomin, + amrex::GpuArray const& dxinv, amrex::Array4 const& rho, amrex::Array4 const& vel, amrex::Array4 const& divu, @@ -1115,7 +1121,7 @@ check_divu_dt( { amrex::Real dtcell = 1.0e12; amrex::Real denom = 0.0; - bool rho_is_okay = (rho(i, j, k) > rhomin); + const bool rho_is_okay = (rho(i, j, k) > rhomin); if (check_type == 1) { // Check based on divU if (divu(i, j, k) > 0.0) { @@ -1145,9 +1151,9 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getGammaInv( - int i, - int j, - int k, + const int i, + const int j, + const int k, amrex::Array4 const& rhoY, amrex::Array4 const& T, pele::physics::eos::EosParm const* @@ -1156,12 +1162,12 @@ getGammaInv( auto eos = pele::physics::PhysicsType::eos(eosparm); // Get rho & Y from rhoY amrex::Real rho = 0.0; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { rho += rhoY(i, j, k, n); } - amrex::Real rhoinv = 1.0 / rho; + const amrex::Real rhoinv = 1.0 / rho; amrex::Real y[NUM_SPECIES] = {0.0}; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { y[n] = rhoY(i, j, k, n) * rhoinv; } @@ -1170,7 +1176,7 @@ getGammaInv( amrex::Real cvmix = 0.0; eos.TY2Cv(T(i, j, k), y, cvmix); - amrex::Real gammainv = cvmix / cpmix; + const amrex::Real gammainv = cvmix / cpmix; return gammainv; } @@ -1179,7 +1185,10 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getSijSij( - int i, int j, int k, amrex::Array4 const& velgrad) noexcept + const int i, + const int j, + const int k, + amrex::Array4 const& velgrad) noexcept { // This function returns SijSij, not strain rate magnitude |S| = sqrt(2 // SijSij) @@ -1199,7 +1208,10 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getVelGradTerm( - int i, int j, int k, amrex::Array4 const& velgrad) noexcept + const int i, + const int j, + const int k, + amrex::Array4 const& velgrad) noexcept { // Compute S^d_ij*S^d_ij // S^d_ij = 0.5(g2_ij + g2_ji) - (1/3)*delta_ij*g2_kk @@ -1216,7 +1228,7 @@ getVelGradTerm( } } } - amrex::Real g2_kk_third = + const amrex::Real g2_kk_third = (1.0 / AMREX_SPACEDIM) * (AMREX_D_TERM( g2_ij[0], +g2_ij[AMREX_SPACEDIM + 1], +g2_ij[2 * AMREX_SPACEDIM + 2])); @@ -1238,17 +1250,17 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void getTurbViscSmagorinsky( - int i, - int j, - int k, - amrex::Real prefactor, + const int i, + const int j, + const int k, + const amrex::Real prefactor, amrex::Array4 const& velgrad, amrex::Array4 const& rho, amrex::Array4 const& mu_t) noexcept { // mu_t = rho * Cs * Delta^2 * |S| where |S| = sqrt(2 SijSij) - amrex::Real Smag = std::sqrt(2.0 * getSijSij(i, j, k, velgrad)); + const amrex::Real Smag = std::sqrt(2.0 * getSijSij(i, j, k, velgrad)); mu_t(i, j, k) = prefactor * rho(i, j, k) * Smag; } @@ -1256,10 +1268,10 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void getTurbViscWALE( - int i, - int j, - int k, - amrex::Real prefactor, + const int i, + const int j, + const int k, + const amrex::Real prefactor, amrex::Array4 const& velgrad, amrex::Array4 const& rho, amrex::Array4 const& mu_t) noexcept @@ -1269,12 +1281,12 @@ getTurbViscWALE( // / (SijSij^(5/2) + (S^d_ijS^d_ij)^(5/4) + smallnum) // smallnum included to make sure denom is nonzero - amrex::Real SijSij = getSijSij(i, j, k, velgrad); - amrex::Real Sterm52 = std::pow(SijSij, 2.5); - amrex::Real Sdterm = getVelGradTerm(i, j, k, velgrad); // (S^d_ijS^d_ij) - amrex::Real Sdterm14 = std::pow(Sdterm, 0.25); - amrex::Real Sdterm54 = Sdterm * Sdterm14; - amrex::Real Sdterm32 = Sdterm54 * Sdterm14; + const amrex::Real SijSij = getSijSij(i, j, k, velgrad); + const amrex::Real Sterm52 = std::pow(SijSij, 2.5); + const amrex::Real Sdterm = getVelGradTerm(i, j, k, velgrad); // (S^d_ijS^d_ij) + const amrex::Real Sdterm14 = std::pow(Sdterm, 0.25); + const amrex::Real Sdterm54 = Sdterm * Sdterm14; + const amrex::Real Sdterm32 = Sdterm54 * Sdterm14; mu_t(i, j, k) = prefactor * rho(i, j, k) * Sdterm32 / (Sterm52 + Sdterm54 + 1.0e-12); } @@ -1283,10 +1295,10 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void getTurbViscSigma( - int i, - int j, - int k, - amrex::Real prefactor, + const int i, + const int j, + const int k, + const amrex::Real prefactor, amrex::Array4 const& velgrad, amrex::Array4 const& rho, amrex::Array4 const& mu_t) noexcept @@ -1336,7 +1348,7 @@ getTurbViscSigma( if (alpha1 != 0.0) { alpha2 = (I[0] / 3.0) * (I[0] / 3.0) * (I[0] / 3.0) - I[0] * I[1] / 6.0 + I[2] / 2.0; - amrex::Real arcCosArg = + const amrex::Real arcCosArg = amrex::min(1.0, amrex::max(-1.0, alpha2 / std::pow(alpha1, 1.5))); alpha3 = std::acos(arcCosArg) / 3.0; diff --git a/Source/PeleLMeX_ODEQty.cpp b/Source/PeleLMeX_ODEQty.cpp index aec597af0..061779ac5 100644 --- a/Source/PeleLMeX_ODEQty.cpp +++ b/Source/PeleLMeX_ODEQty.cpp @@ -7,14 +7,14 @@ PeleLM::predictODEQty() { // Uses forward Euler to predict values for ODE qty at tnp1 // If m_ext_sources_SDC = false, no SDC corrections used - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { auto const& state_arrs = getLevelDataPtr(lev, AmrNewTime)->state.arrays(); auto const& ext_src_arrs = m_extSource[lev]->arrays(); const auto dt = m_dt; - ParallelFor( - *m_extSource[lev], - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { - for (int n = 0; n < NUM_ODE; n++) { + amrex::ParallelFor( + *m_extSource[lev], [state_arrs, ext_src_arrs, dt] AMREX_GPU_DEVICE( + int box_no, int i, int j, int k) noexcept { + for (int n = 0; n < NUM_ODE; ++n) { amrex::Real const& B_n = state_arrs[box_no](i, j, k, FIRSTODE + n); amrex::Real const& S_ext_n = ext_src_arrs[box_no](i, j, k, FIRSTODE + n); diff --git a/Source/PeleLMeX_Plot.cpp b/Source/PeleLMeX_Plot.cpp index 80ea88b35..0c704d916 100644 --- a/Source/PeleLMeX_Plot.cpp +++ b/Source/PeleLMeX_Plot.cpp @@ -21,6 +21,9 @@ #include "PeleLMRad.H" #endif +namespace m2c = pele::physics::utilities::mks2cgs; +namespace c2m = pele::physics::utilities::cgs2mks; + namespace { const std::string level_prefix{"Level_"}; } @@ -36,9 +39,9 @@ void PeleLM::WriteDebugPlotFile( const amrex::Vector& a_MF, const std::string& pltname) { - int nComp = a_MF[0]->nComp(); + const int nComp = a_MF[0]->nComp(); amrex::Vector names(nComp); - for (int n = 0; n < nComp; n++) { + for (int n = 0; n < nComp; ++n) { names[n] = "comp" + std::to_string(n); } amrex::Vector istep(finest_level + 1, m_nstep); @@ -142,7 +145,7 @@ PeleLM::WritePlotFile() // Derive int deriveEntryCount = 0; - for (int ivar = 0; ivar < m_derivePlotVarCount; ivar++) { + for (int ivar = 0; ivar < m_derivePlotVarCount; ++ivar) { const PeleLMDeriveRec* rec = derive_lst.get(m_derivePlotVars[ivar]); deriveEntryCount += rec->numDerive(); } @@ -172,9 +175,10 @@ PeleLM::WritePlotFile() //---------------------------------------------------------------- // Plot MultiFabs - amrex::Vector mf_plt(finest_level + 1); + amrex::Vector mf_plt; + mf_plt.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - mf_plt[lev].define( + mf_plt.emplace_back( grids[lev], dmap[lev], ncomp, 0, amrex::MFInfo(), Factory(lev)); } @@ -191,7 +195,7 @@ PeleLM::WritePlotFile() if (m_incompressible == 0) { plt_VarsName.push_back("density"); if (m_plotStateSpec != 0) { - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { plt_VarsName.push_back("rho.Y(" + names[n] + ")"); } } @@ -203,8 +207,8 @@ PeleLM::WritePlotFile() plt_VarsName.push_back("phiV"); #endif #ifdef PELE_USE_SOOT - for (int mom = 0; mom < NUMSOOTVAR; mom++) { - std::string sootname = soot_model->sootVariableName(mom); + for (int mom = 0; mom < NUMSOOTVAR; ++mom) { + const std::string sootname = soot_model->sootVariableName(mom); plt_VarsName.push_back(sootname); } #endif @@ -226,12 +230,12 @@ PeleLM::WritePlotFile() , plt_VarsName.push_back("gradpz")); } - for (int n = 0; n < m_nAux; n++) { + for (int n = 0; n < m_nAux; ++n) { plt_VarsName.push_back(m_aux_names[n]); } if ((m_do_react != 0) && (m_skipInstantRR == 0) && (m_plot_react != 0)) { - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { plt_VarsName.push_back("I_R(" + names[n] + ")"); } #ifdef PELE_USE_PLASMA @@ -248,9 +252,9 @@ PeleLM::WritePlotFile() plt_VarsName.push_back("volFrac"); #endif - for (int ivar = 0; ivar < m_derivePlotVarCount; ivar++) { + for (int ivar = 0; ivar < m_derivePlotVarCount; ++ivar) { const PeleLMDeriveRec* rec = derive_lst.get(m_derivePlotVars[ivar]); - for (int dvar = 0; dvar < rec->numDerive(); dvar++) { + for (int dvar = 0; dvar < rec->numDerive(); ++dvar) { plt_VarsName.push_back(rec->variableName(dvar)); } } @@ -280,7 +284,7 @@ PeleLM::WritePlotFile() if (m_do_extraEFdiags) { for (int ivar = 0; ivar < NUM_IONS; ++ivar) { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - std::string dir = (idim == 0) ? "X" : ((idim == 1) ? "Y" : "Z"); + const std::string dir = (idim == 0) ? "X" : ((idim == 1) ? "Y" : "Z"); plt_VarsName.push_back( "DriftFlux_" + names[NUM_SPECIES - NUM_IONS + ivar] + "_" + dir); } @@ -293,7 +297,7 @@ PeleLM::WritePlotFile() } #if NUM_ODE > 0 - for (int n = 0; n < NUM_ODE; n++) { + for (int n = 0; n < NUM_ODE; ++n) { plt_VarsName.push_back(m_ode_names[n]); } #endif @@ -394,7 +398,7 @@ PeleLM::WritePlotFile() cnt += 1; #endif - for (int ivar = 0; ivar < m_derivePlotVarCount; ivar++) { + for (int ivar = 0; ivar < m_derivePlotVarCount; ++ivar) { std::unique_ptr mf; mf = derive(m_derivePlotVars[ivar], m_cur_time, lev, 0); amrex::MultiFab::Copy(mf_plt[lev], *mf, 0, cnt, mf->nComp(), 0); @@ -452,8 +456,12 @@ PeleLM::WritePlotFile() m_leveldata_old[lev]->visc_turb_fc[2].const_arrays();) // interpolate turbulent viscosity from faces to centers amrex::ParallelFor( - mf_plt[lev], - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + mf_plt[lev], [plot_arr, cnt, mut_arr_x, mut_arr_y +#if (AMREX_SPACEDIM == 3) + , + mut_arr_z +#endif + ] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { plot_arr[box_no](i, j, k, cnt) = fact * (AMREX_D_TERM( @@ -500,7 +508,7 @@ PeleLM::WritePlotFile() #ifdef PELE_USE_SPRAY if (do_spray_particles) { - bool is_spraycheck = false; + constexpr bool is_spraycheck = false; for (int lev = 0; lev <= finest_level; ++lev) { SprayPC->SprayParticleIO(lev, is_spraycheck, plotfilename); // Remove virtual particles that were made for derived variables @@ -511,7 +519,7 @@ PeleLM::WritePlotFile() } void -PeleLM::WriteHeader(const std::string& name, bool is_checkpoint) const +PeleLM::WriteHeader(const std::string& name, const bool is_checkpoint) const { if (amrex::ParallelDescriptor::IOProcessor()) { std::string HeaderFileName(name + "/Header"); @@ -599,7 +607,7 @@ PeleLM::WriteCheckPointFile() amrex::PreBuildDirectorHierarchy( checkpointname, level_prefix, finest_level + 1, true); - bool is_checkpoint = true; + constexpr bool is_checkpoint = true; WriteHeader(checkpointname, is_checkpoint); WriteJobInfo(checkpointname); @@ -642,7 +650,7 @@ PeleLM::WriteCheckPointFile() } #ifdef PELE_USE_SPRAY if (do_spray_particles) { - bool is_spraycheck = true; + constexpr bool is_spraycheck = true; for (int lev = 0; lev <= finest_level; ++lev) { SprayPC->SprayParticleIO(lev, is_spraycheck, checkpointname); } @@ -861,7 +869,7 @@ PeleLM::ReadCheckPointFile() } } if (m_verbose != 0) { - amrex::Print() << "Restart complete" << std::endl; + amrex::Print() << "Restart complete \n"; } } @@ -967,33 +975,31 @@ PeleLM::initLevelDataFromPlt(int a_lev, const std::string& a_dataPltFile) // in case the number of species differs. amrex::MultiFab speciesPlt(grids[a_lev], dmap[a_lev], nSpecPlt, 0); pltData.fillPatchFromPlt(a_lev, geom[a_lev], idY, 0, nSpecPlt, speciesPlt); - for (int i = 0; i < NUM_SPECIES; i++) { + for (int i = 0; i < NUM_SPECIES; ++i) { std::string specName = "Y(" + spec_names[i] + ")"; std::string specString = specName; if (!m_initDataPlt_specname_map.empty()) { specString = m_initDataPlt_specname_map[i]; } int foundSpec = 0; - for (int iplt = 0; iplt < nSpecPlt; iplt++) { + for (int iplt = 0; iplt < nSpecPlt; ++iplt) { if (specString == plt_vars[idY + iplt]) { amrex::MultiFab::Copy( ldata_p->state, speciesPlt, iplt, FIRSTSPEC + i, 1, 0); foundSpec = 1; if (m_verbose > 0) { amrex::Print() << "Loading species " << specName - << " from plotfile species " << specString - << std::endl; + << " from plotfile species " << specString << "\n"; } } } if (foundSpec == 0) { - for (int iplt = 0; iplt < plt_vars.size(); iplt++) { + for (int iplt = 0; iplt < plt_vars.size(); ++iplt) { if (specString == plt_vars[iplt]) { foundSpec = 1; if (m_verbose > 0) { amrex::Print() << "Loading species " << specName - << " from plotfile entry " << specString - << std::endl; + << " from plotfile entry " << specString << "\n"; } pltData.fillPatchFromPlt( a_lev, geom[a_lev], iplt, FIRSTSPEC + i, 1, ldata_p->state); @@ -1004,7 +1010,7 @@ PeleLM::initLevelDataFromPlt(int a_lev, const std::string& a_dataPltFile) ldata_p->state.setVal(0.0, FIRSTSPEC + i, 1); if (m_verbose > 0) { amrex::Print() << "For species " << specName << " entry " << specString - << " not found in plot file, setting to 0 " << std::endl; + << " not found in plot file, setting to 0\n"; } } } @@ -1186,22 +1192,7 @@ PeleLM::addLevelVelocityDataFromPlt(int a_lev, const std::string& a_velPltFile) a_lev, geom[a_lev], idXvel, sComp0, AMREX_SPACEDIM, tmpVel); // scale the velocity tmpVel.mult(m_velocity_plotfile_scale); - -#ifdef AMREX_USE_OMP -#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) -#endif - for (amrex::MFIter mfi(ldata_p->state, amrex::TilingIfNotGPU()); - mfi.isValid(); ++mfi) { - const amrex::Box& bx = mfi.tilebox(); - amrex::FArrayBox DummyFab(bx, 1); - auto const& state_arr = ldata_p->state.array(mfi); - auto const& tmpVel_arr = tmpVel.array(mfi); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - for (int n = 0; n < AMREX_SPACEDIM; n++) { - state_arr(i, j, k, XVEL + n) += tmpVel_arr(i, j, k, n); - } - }); - } + amrex::MultiFab::Add(ldata_p->state, tmpVel, 0, VELX, AMREX_SPACEDIM, 0); } void @@ -1274,11 +1265,11 @@ PeleLM::WriteJobInfo(const std::string& path) const jobInfoFile << " Grid Information\n"; jobInfoFile << PrettyLine; - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { jobInfoFile << " level: " << lev << "\n"; jobInfoFile << " number of boxes = " << grids[lev].size() << "\n"; jobInfoFile << " maximum zones = "; - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { jobInfoFile << geom[lev].Domain().length(idim) << " "; } jobInfoFile << "\n\n"; diff --git a/Source/PeleLMeX_ProblemSpecificFunctions.H b/Source/PeleLMeX_ProblemSpecificFunctions.H index 176dd1e9e..c40c1693f 100644 --- a/Source/PeleLMeX_ProblemSpecificFunctions.H +++ b/Source/PeleLMeX_ProblemSpecificFunctions.H @@ -181,7 +181,7 @@ struct DefaultProblemSpecificFunctions { #if NUM_ODE > 0 a_ode_names.resize(NUM_ODE); - for (int n = 0; n < NUM_ODE; n++) { + for (int n = 0; n < NUM_ODE; ++n) { a_ode_names[n] = "ODE_" + std::to_string(n); } #else @@ -219,11 +219,11 @@ struct DefaultProblemSpecificFunctions * auto const& state_old_arr = state_old.const_arrays(); * ParallelFor( * *ext_src, - * [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { - * for (int n = 0; n < NUM_ODE; n++) { - * amrex::Real B_n = state_old_arr[box_no](i, j, k, FIRSTODE + n); - * amrex::Real src = -1.0 * pow(10.0, n + 1) * B_n; - * ext_src_arr[box_no](i, j, k, FIRSTODE + n) += src; + * [ext_src_arr] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) + * noexcept { for (int n = 0; n < NUM_ODE; ++n) { amrex::Real B_n = + * state_old_arr[box_no](i, j, k, FIRSTODE + n); amrex::Real src = -1.0 * + * pow(10.0, n + 1) * B_n; ext_src_arr[box_no](i, j, k, FIRSTODE + n) += + * src; * } * }); * amrex::Gpu::streamSynchronize(); diff --git a/Source/PeleLMeX_Projection.cpp b/Source/PeleLMeX_Projection.cpp index 1f76627f3..fdff051c4 100644 --- a/Source/PeleLMeX_Projection.cpp +++ b/Source/PeleLMeX_Projection.cpp @@ -16,20 +16,18 @@ PeleLM::initialProjection() " W: " << velMax[2] <<) "\n"; } - amrex::Real dummy_dt = 1.0; - int incremental = 0; - int nGhost = 0; + constexpr int incremental = 0; + constexpr int nGhost = 0; + constexpr amrex::Real dummy_dt = 1.0; // Get sigma : density if not incompressible amrex::Vector> sigma(finest_level + 1); if (m_incompressible == 0) { for (int lev = 0; lev <= finest_level; ++lev) { - sigma[lev] = std::make_unique( grids[lev], dmap[lev], 1, nGhost, amrex::MFInfo(), *m_factory[lev]); auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); - #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -43,32 +41,40 @@ PeleLM::initialProjection() sig_arr(i, j, k) = dummy_dt / rho_arr(i, j, k); }); } - scaleProj_RZ(lev, *sigma[lev]); +#if AMREX_SPACEDIM == 2 + if (geom[lev].IsRZ()) { + scaleProj_RZ(lev, *sigma[lev]); + } +#endif } } // Get velocity - amrex::Vector> vel; + amrex::Vector> vel(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - vel.push_back( - std::make_unique( - m_leveldata_new[lev]->state, amrex::make_alias, VELX, AMREX_SPACEDIM)); + vel[lev] = std::make_unique( + m_leveldata_new[lev]->state, amrex::make_alias, VELX, AMREX_SPACEDIM); vel[lev]->setBndry(0.0); setInflowBoundaryVel(*vel[lev], lev, AmrNewTime); - scaleProj_RZ(lev, *vel[lev]); +#if AMREX_SPACEDIM == 2 + if (geom[lev].IsRZ()) { + scaleProj_RZ(lev, *vel[lev]); + }; +#endif } // Get RHS cc: - divU (- \int{divU}) amrex::Real Sbar = 0.0; - amrex::Vector rhs_cc(finest_level + 1); + amrex::Vector rhs_cc; if ((m_incompressible == 0) && (m_has_divu != 0)) { // Ensure integral of RHS is zero for closed chamber if (m_closed_chamber != 0) { Sbar = MFSum(GetVecOfConstPtrs(getDivUVect(AmrNewTime)), 0); Sbar /= m_uncoveredVol; // Transform in Mean. } + rhs_cc.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - rhs_cc[lev].define( + rhs_cc.emplace_back( grids[lev], dmap[lev], 1, m_leveldata_new[lev]->divu.nGrow()); amrex::MultiFab::Copy( rhs_cc[lev], m_leveldata_new[lev]->divu, 0, 0, 1, @@ -76,7 +82,11 @@ PeleLM::initialProjection() if (m_closed_chamber != 0) { rhs_cc[lev].plus(-Sbar, 0, 1); } - scaleProj_RZ(lev, rhs_cc[lev]); +#if AMREX_SPACEDIM == 2 + if (geom[lev].IsRZ()) { + scaleProj_RZ(lev, rhs_cc[lev]); + } +#endif rhs_cc[lev].mult(-1.0, 0, 1, rhs_cc[lev].nGrow()); } } @@ -87,7 +97,7 @@ PeleLM::initialProjection() // Set back press and gpress to zero and restore divu // and rescale velocity if 2D-RZ - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); ldata_p->press.setVal(0.0); ldata_p->gp.setVal(0.0); @@ -98,7 +108,11 @@ PeleLM::initialProjection() m_leveldata_new[lev]->divu.plus(Sbar, 0, 1); } } - unscaleProj_RZ(lev, *vel[lev]); +#if AMREX_SPACEDIM == 2 + if (geom[lev].IsRZ()) { + unscaleProj_RZ(lev, *vel[lev]); + } +#endif } // In R-Z, AMReX-Hydro do an average down of r*vel. @@ -109,8 +123,7 @@ PeleLM::initialProjection() } if (m_verbose != 0) { - amrex::Vector velMax(AMREX_SPACEDIM); - velMax = MLNorm0( + amrex::Vector velMax = MLNorm0( GetVecOfConstPtrs(getVelocityVect(AmrNewTime)), 0, AMREX_SPACEDIM); amrex::Print() << " >> After initial velocity projection: "; amrex::Print() << AMREX_D_TERM( @@ -128,20 +141,21 @@ PeleLM::initialPressProjection() amrex::Print() << " Initial pressure projection \n"; } - amrex::Real dummy_dt = 1.0; - int incremental = 0; - int nGhost = 1; + constexpr amrex::Real dummy_dt = 1.0; + constexpr int incremental = 0; + constexpr int nGhost = 1; // Get sigma : density if not incompressible - amrex::Vector> sigma(finest_level + 1); + amrex::Vector> sigma; if (m_incompressible == 0) { + sigma.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - sigma[lev] = std::make_unique( - grids[lev], dmap[lev], 1, nGhost, amrex::MFInfo(), *m_factory[lev]); + sigma.emplace_back( + std::make_unique( + grids[lev], dmap[lev], 1, nGhost, amrex::MFInfo(), *m_factory[lev])); auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); - #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -155,14 +169,19 @@ PeleLM::initialPressProjection() sig_arr(i, j, k) = dummy_dt / rho_arr(i, j, k); }); } - scaleProj_RZ(lev, *sigma[lev]); +#if AMREX_SPACEDIM == 2 + if (geom[lev].IsRZ()) { + scaleProj_RZ(lev, *sigma[lev]); + } +#endif } } // Set the velocity to the gravity field - amrex::Vector vel(finest_level + 1); + amrex::Vector vel; + vel.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - vel[lev].define( + vel.emplace_back( grids[lev], dmap[lev], AMREX_SPACEDIM, nGhost, amrex::MFInfo(), *m_factory[lev]); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { @@ -170,7 +189,11 @@ PeleLM::initialPressProjection() } vel[lev].setBndry(0.0); setInflowBoundaryVel(vel[lev], lev, AmrNewTime); - scaleProj_RZ(lev, vel[lev]); +#if AMREX_SPACEDIM == 2 + if (geom[lev].IsRZ()) { + scaleProj_RZ(lev, vel[lev]); + } +#endif } // Done without divU in IAMR @@ -180,18 +203,18 @@ PeleLM::initialPressProjection() void PeleLM::velocityProjection( - int is_initIter, const TimeStamp& a_rhoTime, const amrex::Real& a_dt) + const int is_initIter, const TimeStamp a_rhoTime, const amrex::Real a_dt) { BL_PROFILE("PeleLMeX::velocityProjection()"); - int nGhost = 0; - int incremental = (is_initIter) != 0 ? 1 : 0; + constexpr int nGhost = 0; + const int incremental = (is_initIter) != 0 ? 1 : 0; // Get sigma : scaled density inv. if not incompressible amrex::Vector> sigma(finest_level + 1); if (m_incompressible == 0) { - amrex::Vector> rhoHalf(finest_level + 1); - rhoHalf = getDensityVect(a_rhoTime); + amrex::Vector> rhoHalf = + getDensityVect(a_rhoTime); for (int lev = 0; lev <= finest_level; ++lev) { sigma[lev] = std::make_unique( @@ -213,7 +236,11 @@ PeleLM::velocityProjection( #ifdef AMREX_USE_EB EB_set_covered(*sigma[lev], 0.0); #endif - scaleProj_RZ(lev, *sigma[lev]); +#if AMREX_SPACEDIM == 2 + if (geom[lev].IsRZ()) { + scaleProj_RZ(lev, *sigma[lev]); + } +#endif } } @@ -265,11 +292,10 @@ PeleLM::velocityProjection( } // Get velocity - amrex::Vector> vel; + amrex::Vector> vel(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - vel.push_back( - std::make_unique( - m_leveldata_new[lev]->state, amrex::make_alias, VELX, AMREX_SPACEDIM)); + vel[lev] = std::make_unique( + m_leveldata_new[lev]->state, amrex::make_alias, VELX, AMREX_SPACEDIM); #ifdef AMREX_USE_EB EB_set_covered(*vel[lev], 0.0); #endif @@ -277,29 +303,33 @@ PeleLM::velocityProjection( if (incremental == 0) { setInflowBoundaryVel(*vel[lev], lev, AmrNewTime); } - scaleProj_RZ(lev, *vel[lev]); +#if AMREX_SPACEDIM == 2 + if (geom[lev].IsRZ()) { + scaleProj_RZ(lev, *vel[lev]); + } +#endif } // To ensure integral of RHS is zero for closed chamber, get mean divU amrex::Real SbarOld = 0.0; amrex::Real SbarNew = 0.0; if ((m_closed_chamber != 0) && (m_incompressible == 0)) { - SbarNew = MFSum(GetVecOfConstPtrs(getDivUVect(AmrNewTime)), 0); - SbarNew /= m_uncoveredVol; // Transform in Mean. + SbarNew = MFSum(GetVecOfConstPtrs(getDivUVect(AmrNewTime)), 0) / + m_uncoveredVol; // Transform in Mean. if (incremental != 0) { - SbarOld = MFSum(GetVecOfConstPtrs(getDivUVect(AmrOldTime)), 0); - SbarOld /= m_uncoveredVol; // Transform in Mean. + SbarOld = MFSum(GetVecOfConstPtrs(getDivUVect(AmrOldTime)), 0) / + m_uncoveredVol; // Transform in Mean. } } // Get RHS cc amrex::Vector rhs_cc; if ((m_incompressible == 0) && (m_has_divu != 0)) { - rhs_cc.resize(finest_level + 1); + rhs_cc.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { if (incremental == 0) { auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); - rhs_cc[lev].define( + rhs_cc.emplace_back( grids[lev], dmap[lev], 1, ldata_p->divu.nGrow(), amrex::MFInfo(), *m_factory[lev]); amrex::MultiFab::Copy( @@ -311,7 +341,7 @@ PeleLM::velocityProjection( } else { auto* ldataOld_p = getLevelDataPtr(lev, AmrOldTime); auto* ldataNew_p = getLevelDataPtr(lev, AmrNewTime); - rhs_cc[lev].define( + rhs_cc.emplace_back( grids[lev], dmap[lev], 1, ldataOld_p->divu.nGrow(), amrex::MFInfo(), *m_factory[lev]); #ifdef AMREX_USE_OMP @@ -337,7 +367,11 @@ PeleLM::velocityProjection( #ifdef AMREX_USE_EB EB_set_covered(rhs_cc[lev], 0.0); #endif - scaleProj_RZ(lev, rhs_cc[lev]); +#if AMREX_SPACEDIM == 2 + if (geom[lev].IsRZ()) { + scaleProj_RZ(lev, rhs_cc[lev]); + } +#endif } } @@ -345,30 +379,34 @@ PeleLM::velocityProjection( GetVecOfPtrs(vel), GetVecOfPtrs(sigma), GetVecOfPtrs(rhs_cc), {}, incremental, a_dt); +#if AMREX_SPACEDIM == 2 + // Unscaling New vel before adding back old one + for (int lev = 0; lev <= finest_level; ++lev) { + if (geom[lev].IsRZ()) { + unscaleProj_RZ(lev, *vel[lev]); + } + } +#endif + // If incremental // define back to be U^{np1} by adding U^{n} - // and handles scaling if 2D-RZ if (incremental != 0) { for (int lev = 0; lev <= finest_level; ++lev) { auto* ldataOld_p = getLevelDataPtr(lev, AmrOldTime); auto* ldataNew_p = getLevelDataPtr(lev, AmrNewTime); - unscaleProj_RZ( - lev, *vel[lev]); // Unscaling New vel before adding back old one amrex::MultiFab::Add( ldataNew_p->state, ldataOld_p->state, VELX, VELX, AMREX_SPACEDIM, 0); } - } else { - for (int lev = 0; lev <= finest_level; ++lev) { - unscaleProj_RZ(lev, *vel[lev]); - } } +#if AMREX_SPACEDIM == 2 // In R-Z, AMReX-Hydro do an average down of r*vel. // Now that we have unscaled vel, need to do average down again // to have consistent vel across levels if (Geom(0).IsRZ()) { averageDownVelocity(AmrNewTime); } +#endif } void @@ -377,8 +415,8 @@ PeleLM::doNodalProject( const amrex::Vector& a_sigma, const amrex::Vector& rhs_cc, const amrex::Vector& rhs_nd, - int incremental, - amrex::Real scaling_factor) + const int incremental, + const amrex::Real scaling_factor) { // Asserts AMREX_ASSERT(a_vel.size() == a_sigma.size()); @@ -392,7 +430,7 @@ PeleLM::doNodalProject( // BCs std::array lobc; std::array hibc; - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { if (Geom(0).isPeriodic(idim)) { lobc[idim] = hibc[idim] = amrex::LinOpBCType::Periodic; } else { @@ -417,7 +455,7 @@ PeleLM::doNodalProject( std::unique_ptr nodal_projector; if (m_incompressible != 0) { - amrex::Real constant_sigma = scaling_factor / m_rho; + const amrex::Real constant_sigma = scaling_factor / m_rho; nodal_projector = std::make_unique( a_vel, constant_sigma, Geom(0, finest_level), info); } else { @@ -452,7 +490,7 @@ PeleLM::doNodalProject( auto phi = nodal_projector->getPhi(); auto gphi = nodal_projector->getGradPhi(); - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); @@ -470,21 +508,25 @@ PeleLM::doNodalProject( if (incremental != 0) { amrex::ParallelFor( tbx, AMREX_SPACEDIM, - [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + [gp_lev_arr, + gp_proj_arr] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { gp_lev_arr(i, j, k, n) += gp_proj_arr(i, j, k, n); }); amrex::ParallelFor( - nbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + nbx, [p_lev_arr, + p_proj_arr] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { p_lev_arr(i, j, k) += p_proj_arr(i, j, k); }); } else { amrex::ParallelFor( tbx, AMREX_SPACEDIM, - [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + [gp_lev_arr, + gp_proj_arr] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { gp_lev_arr(i, j, k, n) = gp_proj_arr(i, j, k, n); }); amrex::ParallelFor( - nbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + nbx, [p_lev_arr, + p_proj_arr] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { p_lev_arr(i, j, k) = p_proj_arr(i, j, k); }); } @@ -505,77 +547,68 @@ PeleLM::doNodalProject( } } +#if AMREX_SPACEDIM == 2 void PeleLM::scaleProj_RZ( // NOLINT(readability-convert-member-functions-to-static) - int a_lev, - amrex::MultiFab& a_mf) + const int a_lev, + amrex::MultiFab& a_mf) { -#if AMREX_SPACEDIM == 2 // Scale nodal projection cell-centered mfs by radius - if (geom[a_lev].IsRZ()) { - amrex::Box domain = geom[a_lev].Domain(); - auto BCRecVel = fetchBCRecArray(VELX, 1); - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (BCRecVel[0].lo(idim) == amrex::BCType::ext_dir) { - domain.growLo(idim, 1); - } - if (BCRecVel[0].hi(idim) == amrex::BCType::ext_dir) { - domain.growHi(idim, 1); - } + amrex::Box domain = geom[a_lev].Domain(); + auto BCRecVel = fetchBCRecArray(VELX, 1); + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if (BCRecVel[0].lo(idim) == amrex::BCType::ext_dir) { + domain.growLo(idim, 1); + } + if (BCRecVel[0].hi(idim) == amrex::BCType::ext_dir) { + domain.growHi(idim, 1); } - const amrex::Real dr = geom[a_lev].CellSize()[0]; - auto const& mf_ma = a_mf.arrays(); - const auto ncomp = a_mf.nComp(); - amrex::ParallelFor( - a_mf, a_mf.nGrowVect(), - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { - auto mf = mf_ma[box_no]; - if (domain.contains(i, j, k)) { - for (int n = 0; n < ncomp; ++n) { - mf(i, j, k, n) *= (static_cast(i) + 0.5) * dr; - } - } else { - for (int n = 0; n < ncomp; ++n) { - mf(i, j, k, n) = 0.0; - } - } - }); - amrex::Gpu::streamSynchronize(); } -#else - amrex::ignore_unused(a_lev, a_mf); -#endif + const amrex::Real dr = geom[a_lev].CellSize()[0]; + auto const& mf_ma = a_mf.arrays(); + const auto ncomp = a_mf.nComp(); + amrex::ParallelFor( + a_mf, a_mf.nGrowVect(), + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + auto mf = mf_ma[box_no]; + if (domain.contains(i, j, k)) { + for (int n = 0; n < ncomp; ++n) { + mf(i, j, k, n) *= (static_cast(i) + 0.5) * dr; + } + } else { + for (int n = 0; n < ncomp; ++n) { + mf(i, j, k, n) = 0.0; + } + } + }); + amrex::Gpu::streamSynchronize(); } void PeleLM:: unscaleProj_RZ( // NOLINT(readability-convert-member-functions-to-static) - int a_lev, + const int a_lev, amrex::MultiFab& a_mf) { -#if AMREX_SPACEDIM == 2 // Unscale nodal projection cell-centered mfs by radius - if (geom[a_lev].IsRZ()) { - const amrex::Box& domain = geom[a_lev].Domain(); - const amrex::Real dr = geom[a_lev].CellSize()[0]; - auto const& mf_ma = a_mf.arrays(); - const auto ncomp = a_mf.nComp(); - amrex::ParallelFor( - a_mf, a_mf.nGrowVect(), - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { - auto mf = mf_ma[box_no]; - if (domain.contains(i, j, k)) { - for (int n = 0; n < ncomp; ++n) { - mf(i, j, k, n) /= (static_cast(i) + 0.5) * dr; - } - } else { - for (int n = 0; n < ncomp; ++n) { - mf(i, j, k, n) = 0.0; - } + const amrex::Box& domain = geom[a_lev].Domain(); + const amrex::Real dr = geom[a_lev].CellSize()[0]; + auto const& mf_ma = a_mf.arrays(); + const auto ncomp = a_mf.nComp(); + amrex::ParallelFor( + a_mf, a_mf.nGrowVect(), + [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + auto mf = mf_ma[box_no]; + if (domain.contains(i, j, k)) { + for (int n = 0; n < ncomp; ++n) { + mf(i, j, k, n) /= (static_cast(i) + 0.5) * dr; } - }); - } -#else - amrex::ignore_unused(a_lev, a_mf); -#endif + } else { + for (int n = 0; n < ncomp; ++n) { + mf(i, j, k, n) = 0.0; + } + } + }); + amrex::Gpu::streamSynchronize(); } +#endif diff --git a/Source/PeleLMeX_Radiation.cpp b/Source/PeleLMeX_Radiation.cpp index cef12e10d..3951a5f2d 100644 --- a/Source/PeleLMeX_Radiation.cpp +++ b/Source/PeleLMeX_Radiation.cpp @@ -35,14 +35,14 @@ PeleLM::RadInit() } void -PeleLM::computeRadSource(const PeleLM::TimeStamp& a_timestamp) +PeleLM::computeRadSource(const TimeStamp a_timestamp) { int const co2Indx = rad_model->readRadIndices().co2Indx; int const h2oIndx = rad_model->readRadIndices().h2oIndx; int const coIndx = rad_model->readRadIndices().coIndx; BL_PROFILE_VAR("PeleLM::advance::rad::spec", PLM_RAD_SPEC); - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { auto* ldata_p = PeleLM::getLevelDataPtr(lev, a_timestamp); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) @@ -72,7 +72,7 @@ PeleLM::computeRadSource(const PeleLM::TimeStamp& a_timestamp) BL_PROFILE_VAR("PeleLM::advance::rad::solve", PLM_RAD_SOLV); rad_model->evaluateRad(); - for (int lev = 0; lev <= finest_level; lev++) { + for (int lev = 0; lev <= finest_level; ++lev) { #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif diff --git a/Source/PeleLMeX_Reactions.cpp b/Source/PeleLMeX_Reactions.cpp index e1e93c852..3cd8668b3 100644 --- a/Source/PeleLMeX_Reactions.cpp +++ b/Source/PeleLMeX_Reactions.cpp @@ -5,7 +5,7 @@ #endif void -PeleLM::advanceChemistry(std::unique_ptr& advData) +PeleLM::advanceChemistry(const std::unique_ptr& advData) { BL_PROFILE("PeleLMeX::advanceChemistry()"); @@ -28,7 +28,7 @@ PeleLM::advanceChemistry(std::unique_ptr& advData) // It works with the AmrCore BoxArray and do not involve ParallelCopy void PeleLM::advanceChemistry( - int lev, const amrex::Real& a_dt, amrex::MultiFab& a_extForcing) + const int lev, const amrex::Real a_dt, amrex::MultiFab& a_extForcing) { BL_PROFILE("PeleLMeX::advanceChemistry_Lev" + std::to_string(lev) + "()"); @@ -65,15 +65,21 @@ PeleLM::advanceChemistry( auto const& mask_arr = mask.array(mfi); // Reset new to old and convert MKS -> CGS - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - for (int n = 0; n < NUM_SPECIES; n++) { - rhoY_n(i, j, k, n) = rhoY_o(i, j, k, n) * 1.0e-3; - extF_rhoY(i, j, k, n) *= 1.0e-3; - } - temp_n(i, j, k) = temp_o(i, j, k); - rhoH_n(i, j, k) = rhoH_o(i, j, k) * 10.0; - extF_rhoH(i, j, k) *= 10.0; - }); + amrex::ParallelFor( + bx, NUM_SPECIES, + [rhoY_o, rhoY_n, + extF_rhoY] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + rhoY_n(i, j, k, n) = m2c::Rho(rhoY_o(i, j, k, n)); + extF_rhoY(i, j, k, n) = m2c::Rho(extF_rhoY(i, j, k, n)); + ; + }); + amrex::ParallelFor( + bx, [temp_n, temp_o, rhoH_n, rhoH_o, + extF_rhoH] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + temp_n(i, j, k) = temp_o(i, j, k); + rhoH_n(i, j, k) = m2c::RhoH(rhoH_o(i, j, k)); + extF_rhoH(i, j, k) = m2c::RhoH(extF_rhoH(i, j, k)); + }); #ifdef PELE_USE_PLASMA // Pass nE -> rhoY_e & FnE -> FrhoY_e @@ -84,10 +90,12 @@ PeleLM::advanceChemistry( auto eos = pele::physics::PhysicsType::eos(&eos_parms.host_parm()); amrex::Real mwt[NUM_SPECIES] = {0.0}; eos.molecular_weight(mwt); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - rhoYe_n(i, j, k) = nE_o(i, j, k) / Na * mwt[E_ID] * 1.0e-6; - FrhoYe(i, j, k) = FnE(i, j, k) / Na * mwt[E_ID] * 1.0e-6; - }); + amrex::ParallelFor( + bx, [rhoYe_n, nE_o, mwt, FrhoYe, + FnE] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + rhoYe_n(i, j, k) = nE_o(i, j, k) / Na * mwt[E_ID] * 1.0e-6; + FrhoYe(i, j, k) = FnE(i, j, k) / Na * mwt[E_ID] * 1.0e-6; + }); #endif amrex::Real dt_incr = a_dt; @@ -103,25 +111,30 @@ PeleLM::advanceChemistry( ); // Convert CGS -> MKS - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - for (int n = 0; n < NUM_SPECIES; n++) { - rhoY_n(i, j, k, n) *= 1.0e3; - extF_rhoY(i, j, k, n) *= 1.0e3; - } - rhoH_n(i, j, k) *= 0.1; - extF_rhoH(i, j, k) *= 0.1; - }); - + amrex::ParallelFor( + bx, NUM_SPECIES, + [rhoY_n, + extF_rhoY] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + rhoY_n(i, j, k, n) = c2m::Rho(rhoY_n(i, j, k, n)); + extF_rhoY(i, j, k, n) = c2m::Rho(extF_rhoY(i, j, k, n)); + }); + amrex::ParallelFor( + bx, [rhoH_n, extF_rhoH] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + rhoH_n(i, j, k) = c2m::RhoH(rhoH_n(i, j, k)); + extF_rhoH(i, j, k) = c2m::RhoH(extF_rhoH(i, j, k)); + }); #ifdef PELE_USE_PLASMA // rhoY_e -> nE and set rhoY_e to zero auto const& nE_n = ldataNew_p->state.array(mfi, NE); amrex::Real invmwt[NUM_SPECIES] = {0.0}; eos.inv_molecular_weight(invmwt); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - nE_n(i, j, k) = rhoYe_n(i, j, k) * Na * invmwt[E_ID] * 1.0e3; - rhoYe_n(i, j, k) = 0.0; - extF_rhoY(i, j, k, E_ID) = 0.0; - }); + amrex::ParallelFor( + bx, [nE_n, rhoYe_n, invmwt, + extF_rhoY] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + nE_n(i, j, k) = rhoYe_n(i, j, k) * Na * invmwt[E_ID] * 1.0e3; + rhoYe_n(i, j, k) = 0.0; + extF_rhoY(i, j, k, E_ID) = 0.0; + }); #endif #ifdef AMREX_USE_GPU @@ -130,6 +143,7 @@ PeleLM::advanceChemistry( } // Set reaction term + #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -166,7 +180,7 @@ PeleLM::advanceChemistry( // on uncovered boxes. void PeleLM::advanceChemistryBAChem( - int lev, const amrex::Real& a_dt, amrex::MultiFab& a_extForcing) + const int lev, const amrex::Real a_dt, amrex::MultiFab& a_extForcing) { BL_PROFILE("PeleLMeX::advanceChemistry_Lev" + std::to_string(lev) + "()"); @@ -217,15 +231,19 @@ PeleLM::advanceChemistryBAChem( auto const& mask_arr = mask.array(mfi); // Convert MKS -> CGS - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - for (int n = 0; n < NUM_SPECIES; n++) { - rhoY_o(i, j, k, n) *= 1.0e-3; - extF_rhoY(i, j, k, n) *= 1.0e-3; - } - rhoH_o(i, j, k) *= 10.0; - extF_rhoH(i, j, k) *= 10.0; - }); - + amrex::ParallelFor( + bx, NUM_SPECIES, + [rhoY_o, + extF_rhoY] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + rhoY_o(i, j, k, n) = m2c::Rho(rhoY_o(i, j, k, n)); + extF_rhoY(i, j, k, n) = m2c::Rho(extF_rhoY(i, j, k, n)); + ; + }); + amrex::ParallelFor( + bx, [rhoH_o, extF_rhoH] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + rhoH_o(i, j, k) = m2c::RhoH(rhoH_o(i, j, k)); + extF_rhoH(i, j, k) = m2c::RhoH(extF_rhoH(i, j, k)); + }); #ifdef PELE_USE_PLASMA // Pass nE -> rhoY_e & FnE -> FrhoY_e auto const& nE_o = chemnE.array(mfi); @@ -235,14 +253,16 @@ PeleLM::advanceChemistryBAChem( auto eos = pele::physics::PhysicsType::eos(&eos_parms.host_parm()); amrex::Real mwt[NUM_SPECIES] = {0.0}; eos.molecular_weight(mwt); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - rhoYe_o(i, j, k) = nE_o(i, j, k) / Na * mwt[E_ID] * 1.0e-6; - FrhoYe(i, j, k) = FnE(i, j, k) / Na * mwt[E_ID] * 1.0e-6; - }); + amrex::ParallelFor( + bx, [rhoYe_o, nE_o, mwt, FrhoYe, + FnE] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + rhoYe_o(i, j, k) = nE_o(i, j, k) / Na * mwt[E_ID] * 1.0e-6; + FrhoYe(i, j, k) = FnE(i, j, k) / Na * mwt[E_ID] * 1.0e-6; + }); #endif // Do reaction only on uncovered box - int do_reactionBox = m_baChemFlag[lev][mfi.index()]; + const int do_reactionBox = m_baChemFlag[lev][mfi.index()]; if (do_reactionBox != 0) { // Do reaction as usual using PelePhysics chemistry integrator @@ -260,27 +280,31 @@ PeleLM::advanceChemistryBAChem( } else { // Just set the function call to 0.0 amrex::ParallelFor( - bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [fcl] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { fcl(i, j, k) = 0.0; }); } // Convert CGS -> MKS - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - for (int n = 0; n < NUM_SPECIES; n++) { - rhoY_o(i, j, k, n) *= 1.0e3; - } - rhoH_o(i, j, k) *= 0.1; - }); - + amrex::ParallelFor( + bx, NUM_SPECIES, + [rhoY_o] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + rhoY_o(i, j, k, n) = c2m::Rho(rhoY_o(i, j, k, n)); + }); + amrex::ParallelFor( + bx, [rhoH_o] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + rhoH_o(i, j, k) = c2m::RhoH(rhoH_o(i, j, k)); + }); #ifdef PELE_USE_PLASMA // rhoY_e -> nE and set rhoY_e to zero amrex::Real invmwt[NUM_SPECIES] = {0.0}; eos.inv_molecular_weight(invmwt); - ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - nE_o(i, j, k) = rhoYe_o(i, j, k) * Na * invmwt[E_ID] * 1.0e3; - rhoYe_o(i, j, k) = 0.0; - }); + ParallelFor( + bx, + [nE_o, rhoYe_o, invmwt] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + nE_o(i, j, k) = rhoYe_o(i, j, k) * Na * invmwt[E_ID] * 1.0e3; + rhoYe_o(i, j, k) = 0.0; + }); #endif #ifdef AMREX_USE_GPU @@ -296,6 +320,7 @@ PeleLM::advanceChemistryBAChem( #ifdef PELE_USE_PLASMA amrex::MultiFab nETemp(grids[lev], dmap[lev], 1, 0); nETemp.ParallelCopy(chemnE, 0, 0, 1); + auto const& nE_tmp_ma = nETemp.const_arrays(); #endif // Pass from temp state MF to leveldata and set reaction term @@ -346,7 +371,7 @@ PeleLM::advanceChemistryBAChem( void PeleLM::computeInstantaneousReactionRate( - const amrex::Vector& I_R, const TimeStamp& a_time) + const amrex::Vector& I_R, const TimeStamp a_time) { for (int lev = 0; lev <= finest_level; ++lev) { #ifdef PELE_USE_PLASMA @@ -359,7 +384,7 @@ PeleLM::computeInstantaneousReactionRate( void PeleLM::computeInstantaneousReactionRate( - int lev, const TimeStamp& a_time, amrex::MultiFab* a_I_R) + const int lev, const TimeStamp a_time, amrex::MultiFab* a_I_R) { BL_PROFILE("PeleLMeX::computeInstantaneousReactionRate()"); auto* ldata_p = getLevelDataPtr(lev, a_time); @@ -386,16 +411,17 @@ PeleLM::computeInstantaneousReactionRate( if (flagfab.getType(bx) == amrex::FabType::covered) { // Covered boxes amrex::ParallelFor( bx, NUM_SPECIES, - [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + [rhoYdot] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { rhoYdot(i, j, k, n) = 0.0; }); } else if (flagfab.getType(bx) != amrex::FabType::regular) { // EB // containing // boxes amrex::ParallelFor( - bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [rhoYdot, flag, rhoY, rhoH, T, + leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (flag(i, j, k).isCovered()) { - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { rhoYdot(i, j, k, n) = 0.0; } } else { @@ -406,7 +432,8 @@ PeleLM::computeInstantaneousReactionRate( #endif { amrex::ParallelFor( - bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [rhoY, rhoH, T, rhoYdot, + leosparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { reactionRateRhoY(i, j, k, rhoY, rhoH, T, rhoYdot, leosparm); }); } @@ -414,7 +441,7 @@ PeleLM::computeInstantaneousReactionRate( } void -PeleLM::getScalarReactForce(std::unique_ptr& advData) +PeleLM::getScalarReactForce(const std::unique_ptr& advData) { // The differentialDiffusionUpdate just provided the {np1,kp1} AD state // -> use it to build the external forcing for the chemistry @@ -453,7 +480,7 @@ PeleLM::getScalarReactForce(std::unique_ptr& advData) } void -PeleLM::getHeatRelease(int a_lev, amrex::MultiFab* a_HR) +PeleLM::getHeatRelease(const int a_lev, amrex::MultiFab* a_HR) { auto* ldataNew_p = getLevelDataPtr(a_lev, AmrNewTime); auto* ldataR_p = getLevelDataReactPtr(a_lev); diff --git a/Source/PeleLMeX_Regrid.cpp b/Source/PeleLMeX_Regrid.cpp index 24a13dcc0..16f6606f5 100644 --- a/Source/PeleLMeX_Regrid.cpp +++ b/Source/PeleLMeX_Regrid.cpp @@ -2,7 +2,7 @@ #include void -PeleLM::regrid(int lbase, amrex::Real time, bool initial) +PeleLM::regrid(const int lbase, const amrex::Real time, const bool initial) { BL_PROFILE("PeleLMeX::regrid()"); @@ -307,8 +307,8 @@ PeleLM::regrid(int lbase, amrex::Real time, bool initial) void PeleLM::MakeNewLevelFromCoarse( - int lev, - amrex::Real time, + const int lev, + const amrex::Real time, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm) { @@ -327,7 +327,7 @@ PeleLM::MakeNewLevelFromCoarse( << "% of the domain \n"; } if (m_verbose > 3) { - amrex::Print() << " with BoxArray " << ba << std::endl; + amrex::Print() << " with BoxArray " << ba << "\n"; } } @@ -423,8 +423,8 @@ PeleLM::MakeNewLevelFromCoarse( void PeleLM::RemakeLevel( - int lev, - amrex::Real time, + const int lev, + const amrex::Real time, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm) { @@ -443,7 +443,7 @@ PeleLM::RemakeLevel( << "% of the domain \n"; } if (m_verbose > 3) { - amrex::Print() << " with BoxArray " << ba << std::endl; + amrex::Print() << " with BoxArray " << ba << "\n"; } } @@ -538,7 +538,7 @@ PeleLM::RemakeLevel( } void -PeleLM::ClearLevel(int lev) +PeleLM::ClearLevel(const int lev) { BL_PROFILE("PeleLMeX::ClearLevel()"); @@ -571,32 +571,43 @@ PeleLM::ClearLevel(int lev) void PeleLM::computeCosts( - int a_lev, amrex::LayoutData& a_costs, int a_costMethod) + const int a_lev, + amrex::LayoutData& a_costs, + const int a_costMethod) { - if (a_costMethod == LoadBalanceCost::Ncell) { + switch (a_costMethod) { + case LoadBalanceCost::Ncell: { for (amrex::MFIter mfi(a_costs, false); mfi.isValid(); ++mfi) { a_costs[mfi] = static_cast(mfi.validbox().numPts()); } - } else if (a_costMethod == LoadBalanceCost::ChemFunctCallAvg) { + return; + } + case LoadBalanceCost::ChemFunctCallAvg: { amrex::MultiFab costMF(a_costs.boxArray(), a_costs.DistributionMap(), 1, 0); fillpatch_chemFunctCall(a_lev, m_cur_time, costMF, 0); for (amrex::MFIter mfi(costMF, false); mfi.isValid(); ++mfi) { a_costs[mfi] = costMF[mfi].sum(mfi.validbox(), 0) / static_cast(mfi.validbox().numPts()); } - } else if (a_costMethod == LoadBalanceCost::ChemFunctCallMax) { + return; + } + case LoadBalanceCost::ChemFunctCallMax: { amrex::MultiFab costMF(a_costs.boxArray(), a_costs.DistributionMap(), 1, 0); fillpatch_chemFunctCall(a_lev, m_cur_time, costMF, 0); for (amrex::MFIter mfi(costMF, false); mfi.isValid(); ++mfi) { a_costs[mfi] = costMF[mfi].max(mfi.validbox(), 0); } - } else if (a_costMethod == LoadBalanceCost::ChemFunctCallSum) { + return; + } + case LoadBalanceCost::ChemFunctCallSum: { amrex::MultiFab costMF(a_costs.boxArray(), a_costs.DistributionMap(), 1, 0); fillpatch_chemFunctCall(a_lev, m_cur_time, costMF, 0); for (amrex::MFIter mfi(costMF, false); mfi.isValid(); ++mfi) { a_costs[mfi] = costMF[mfi].sum(mfi.validbox(), 0); } - } else if (a_costMethod == LoadBalanceCost::UserDefinedDerivedAvg) { + return; + } + case LoadBalanceCost::UserDefinedDerivedAvg: { amrex::MultiFab costMF(a_costs.boxArray(), a_costs.DistributionMap(), 1, 0); costMF.setVal(0.0); std::unique_ptr mf; @@ -606,7 +617,9 @@ PeleLM::computeCosts( a_costs[mfi] = costMF[mfi].sum(mfi.validbox(), 0) / static_cast(mfi.validbox().numPts()); } - } else if (a_costMethod == LoadBalanceCost::UserDefinedDerivedSum) { + return; + } + case LoadBalanceCost::UserDefinedDerivedSum: { amrex::MultiFab costMF(a_costs.boxArray(), a_costs.DistributionMap(), 1, 0); costMF.setVal(0.0); std::unique_ptr mf; @@ -615,13 +628,15 @@ PeleLM::computeCosts( for (amrex::MFIter mfi(costMF, false); mfi.isValid(); ++mfi) { a_costs[mfi] = costMF[mfi].sum(mfi.validbox(), 0); } - } else { + return; + } + default: amrex::Abort(" Unknown cost estimate method !"); } } void -PeleLM::computeCosts(int a_lev) +PeleLM::computeCosts(const int a_lev) { computeCosts(a_lev, *m_costs[a_lev], m_loadBalanceCost); } @@ -651,7 +666,8 @@ PeleLM::resetMacProjector() } void -PeleLM::regridFromGridFile(int lbase, amrex::Real time, bool /*initial*/) +PeleLM::regridFromGridFile( + const int lbase, const amrex::Real time, const bool /*initial*/) { const int new_finest = static_cast(m_regrid_ba.size()); amrex::Vector new_grids(finest_level + 2); diff --git a/Source/PeleLMeX_Setup.cpp b/Source/PeleLMeX_Setup.cpp index 524659422..4b4c06912 100644 --- a/Source/PeleLMeX_Setup.cpp +++ b/Source/PeleLMeX_Setup.cpp @@ -101,34 +101,31 @@ PeleLM::Setup() amrex::Print() << " Using LES in transport with Sc = " << 1.0 / m_Schmidt_inv; if (pele::physics::PhysicsType::eos_type::identifier() == "Manifold") { - amrex::Print() << ", enthalpy not diffused for Manifold EOS " - << std::endl; + amrex::Print() << ", enthalpy not diffused for Manifold EOS \n"; } else { - amrex::Print() << " and Pr = " << 1.0 / m_Prandtl_inv << std::endl; + amrex::Print() << " and Pr = " << 1.0 / m_Prandtl_inv << "\n"; } } else if (m_verbose != 0) { if (m_fixed_Le == 0 && m_fixed_Pr == 0) { if (m_use_soret == 0) { - amrex::Print() << " Using mixture-averaged transport" << std::endl; + amrex::Print() << " Using mixture-averaged transport \n"; } else { amrex::Print() - << " Using mixture-averaged transport with Soret effects" - << std::endl; + << " Using mixture-averaged transport with Soret effects \n"; if (m_soret_boundary_override != 0) { - amrex::Print() - << " Imposing inhomogeneous Neumann conditions " - "for species on isothermal walls. WARNING: use_wbar disabled." - << std::endl; + amrex::Print() << " Imposing inhomogeneous Neumann conditions " + "for species on isothermal walls. WARNING: " + "use_wbar disabled. \n"; } } } else { if (m_fixed_Le != 0) { amrex::Print() << " Using fixed Le = " << 1.0 / m_Lewis_inv - << std::endl; + << "\n"; } if (m_fixed_Pr != 0) { amrex::Print() << " Using fixed Pr = " << 1.0 / m_Prandtl_inv - << std::endl; + << "\n"; } } } @@ -157,7 +154,7 @@ PeleLM::Setup() #ifdef PELE_USE_PLASMA pele::physics::eos::charge_mass(zk.arr); - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { zk[n] *= 1000.0; // CGS->MKS } #endif @@ -246,7 +243,7 @@ PeleLM::readParameters() // Boundary conditions // ----------------------------------------- int isOpenDomain = 0; - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { int lo_bc = BoundaryCondition::BCInterior; int hi_bc = BoundaryCondition::BCInterior; parseUserKey(pp, "lo_bc", boundarycondition, lo_bc, idim); @@ -277,7 +274,7 @@ PeleLM::readParameters() amrex::Vector hi_bc_char(AMREX_SPACEDIM); ppef.getarr("phiV_lo_bc", lo_bc_char, 0, AMREX_SPACEDIM); ppef.getarr("phiV_hi_bc", hi_bc_char, 0, AMREX_SPACEDIM); - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { if (lo_bc_char[idim] == "Interior") { m_phiV_bc.setLo(idim, 0); } else if (lo_bc_char[idim] == "Dirichlet") { @@ -301,7 +298,7 @@ PeleLM::readParameters() // Get the polarity of BCs ppef.getarr("phiV_polarity_lo", lo_bc_char, 0, AMREX_SPACEDIM); ppef.getarr("phiV_polarity_hi", hi_bc_char, 0, AMREX_SPACEDIM); - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { if (lo_bc_char[idim] == "Neutral") { m_phiV_bcpol.setLo(idim, 0); } else if (lo_bc_char[idim] == "Anode") { // Pos. elec = 1 @@ -351,7 +348,7 @@ PeleLM::readParameters() pp.queryarr("gravity", grav, 0, AMREX_SPACEDIM); amrex::Vector gp0(AMREX_SPACEDIM, 0); pp.queryarr("gradP0", gp0, 0, AMREX_SPACEDIM); - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_background_gp[idim] = gp0[idim]; m_gravity[idim] = grav[idim]; } @@ -373,7 +370,7 @@ PeleLM::readParameters() m_spark_temp.resize(m_n_sparks); m_spark_radius.resize(m_n_sparks); pp.query("spark_verbose", m_spark_verbose); - for (int n = 0; n < m_n_sparks; n++) { + for (int n = 0; n < m_n_sparks; ++n) { pp.get("sparks", m_spark[n], n); std::string spark_prefix = "peleLM." + m_spark[n]; amrex::ParmParse pps(spark_prefix); @@ -385,22 +382,21 @@ PeleLM::readParameters() pps.get("radius", m_spark_radius[n]); } if (m_spark_verbose > 0) { - amrex::Print() << "Spark list:" << std::endl; - for (int n = 0; n < m_n_sparks; n++) { - amrex::Print() << "Spark " << n << " name: " << m_spark[n] << std::endl; - amrex::Print() << "Spark " << n << " time: " << m_spark_time[n] - << std::endl; + amrex::Print() << "Spark list: \n"; + for (int n = 0; n < m_n_sparks; ++n) { + amrex::Print() << "Spark " << n << " name: " << m_spark[n] << "\n"; + amrex::Print() << "Spark " << n << " time: " << m_spark_time[n] << "\n"; amrex::Print() << "Spark " << n << " duration: " << m_spark_duration[n] - << std::endl; + << "\n"; amrex::Print() << "Spark " << n << " location: "; - for (int d = 0; d < AMREX_SPACEDIM; d++) { + for (int d = 0; d < AMREX_SPACEDIM; ++d) { amrex::Print() << m_spark_location[n][d] << " "; } - amrex::Print() << std::endl; + amrex::Print() << "\n"; amrex::Print() << "Spark " << n << " temperature: " << m_spark_temp[n] - << std::endl; + << "\n"; amrex::Print() << "Spark " << n << " radius: " << m_spark_radius[n] - << std::endl; + << "\n"; } } } @@ -412,7 +408,7 @@ PeleLM::readParameters() m_aux_advect.resize(m_nAux); m_DiffTypeAux.resize(m_nAux); m_aux_Schmidt.resize(m_nAux); - for (int n = 0; n < m_nAux; n++) { + for (int n = 0; n < m_nAux; ++n) { pp.get("aux_vars", m_aux_names[n], n); std::string aux_prefix = "peleLM." + m_aux_names[n]; amrex::ParmParse ppa(aux_prefix); @@ -468,7 +464,7 @@ PeleLM::readParameters() pp.query("use_wbar", m_use_wbar); if (m_use_soret != 0) { bool isothermal = false; - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { isothermal |= (m_phys_bc.lo(idim) == BoundaryCondition::BCSlipWallIsotherm || m_phys_bc.lo(idim) == BoundaryCondition::BCNoSlipWallIsotherm || @@ -489,8 +485,7 @@ PeleLM::readParameters() if (m_unity_Le != 0) { m_fixed_Le = 1; amrex::Print() << "WARNING: unity_Le is deprecated and will be removed in " - "future version, use fixed_Le instead" - << std::endl; + "future version, use fixed_Le instead \n"; } if (m_do_les) { // For LES, Prandtl and Schmidt numbers are fixed m_fixed_Le = 1; @@ -526,8 +521,7 @@ PeleLM::readParameters() m_use_wbar = 0; m_use_soret = 0; amrex::Print() << "WARNING: use_wbar and use_soret set to false because " - "fixed_Pr or fixed_Le is true" - << std::endl; + "fixed_Pr or fixed_Le is true \n"; } // Manifold EOS: invPrandtl needs to be 0 because H not used @@ -759,7 +753,7 @@ PeleLM::readParameters() m_evaluatePlotVarCount = (pp.countval("evaluate_vars")); if (m_evaluatePlotVarCount != 0) { m_evaluatePlotVars.resize(m_evaluatePlotVarCount); - for (int ivar = 0; ivar < m_evaluatePlotVarCount; ivar++) { + for (int ivar = 0; ivar < m_evaluatePlotVarCount; ++ivar) { pp.get("evaluate_vars", m_evaluatePlotVars[ivar], ivar); } } @@ -904,7 +898,7 @@ PeleLM::readIOParameters() m_derivePlotVarCount = (pp.countval("derive_plot_vars")); if (m_derivePlotVarCount != 0) { m_derivePlotVars.resize(m_derivePlotVarCount); - for (int ivar = 0; ivar < m_derivePlotVarCount; ivar++) { + for (int ivar = 0; ivar < m_derivePlotVarCount; ++ivar) { pp.get("derive_plot_vars", m_derivePlotVars[ivar], ivar); } } @@ -915,8 +909,10 @@ PeleLM::readIOParameters() pp.query("initial_grid_file", m_initial_grid_file); pp.query("regrid_file", m_regrid_file); pp.query("file_stepDigits", m_ioDigits); - pp.query("use_hdf5_plt", m_write_hdf5_pltfile); pp.query("regrid_interp_method", m_regrid_interp_method); +#ifdef AMREX_USE_HDF5 + pp.query("use_hdf5_plt", m_write_hdf5_pltfile); +#endif AMREX_ASSERT(m_regrid_interp_method == 0 || m_regrid_interp_method == 1); } @@ -953,7 +949,7 @@ PeleLM::variablesSetup() amrex::Vector names; pele::physics::eos::speciesNames( names, &(eos_parms.host_parm())); - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { stateComponents.emplace_back(FIRSTSPEC + n, "rho.Y(" + names[n] + ")"); reactComponents.emplace_back(n, "I_R(" + names[n] + ")"); } @@ -970,7 +966,7 @@ PeleLM::variablesSetup() stateComponents.emplace_back(PHIV, "PhiV"); #endif #ifdef PELE_USE_SOOT - for (int mom = 0; mom < NUMSOOTVAR; mom++) { + for (int mom = 0; mom < NUMSOOTVAR; ++mom) { std::string sootname = soot_model->sootVariableName(mom); amrex::Print() << " " << sootname << ": " << FIRSTSOOT + mom << "\n"; stateComponents.emplace_back(FIRSTSOOT + mom, sootname); @@ -1003,7 +999,7 @@ PeleLM::variablesSetup() amrex::Print() << " => Total number of state variables: " << NVAR << "\n"; } if (m_nAux > 0) { - for (int n = 0; n < m_nAux; n++) { + for (int n = 0; n < m_nAux; ++n) { amrex::Print() << " Auxiliary " + std::to_string(n + 1) + ": " << m_aux_names[n] << "\n"; amrex::Print() << " Advective: " << m_aux_advect[n] << "\n"; @@ -1056,7 +1052,7 @@ PeleLM::variablesSetup() m_DiffTypeState[PHIV] = 0; #endif #ifdef PELE_USE_SOOT - for (int mom = 0; mom < NUMSOOTVAR; mom++) { + for (int mom = 0; mom < NUMSOOTVAR; ++mom) { m_AdvTypeState[FIRSTSOOT + mom] = 0; m_DiffTypeState[FIRSTSOOT + mom] = 0; } @@ -1103,7 +1099,7 @@ PeleLM::variablesSetup() void PeleLM::readGridFile( - std::string grid_file, amrex::Vector& input_ba) + const std::string& grid_file, amrex::Vector& input_ba) { #define STRIP \ while (is.get() != '\n') { \ @@ -1123,11 +1119,11 @@ PeleLM::readGridFile( "You have fewer levels in your inputs file then in your grids file!"); } - for (int lev = 1; lev <= in_finest; lev++) { + for (int lev = 1; lev <= in_finest; ++lev) { amrex::BoxList bl; is >> ngrid; STRIP; - for (int i = 0; i < ngrid; i++) { + for (int i = 0; i < ngrid; ++i) { amrex::Box bx; is >> bx; STRIP; @@ -1155,14 +1151,14 @@ PeleLM::derivedSetup() // Set species mass fractions amrex::Vector var_names_massfrac(NUM_SPECIES); - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { var_names_massfrac[n] = "Y(" + spec_names[n] + ")"; } derive_lst.add( "mass_fractions", amrex::IndexType::TheCellType(), NUM_SPECIES, var_names_massfrac, pelelmex_dermassfrac, the_same_box); - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { var_names_massfrac[n] = "X(" + spec_names[n] + ")"; } derive_lst.add( @@ -1170,12 +1166,12 @@ PeleLM::derivedSetup() var_names_massfrac, pelelmex_dermolefrac, the_same_box); // Species diffusion coefficients - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { var_names_massfrac[n] = "D_" + spec_names[n]; } if (m_use_soret != 0) { var_names_massfrac.resize(2 * NUM_SPECIES); - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { var_names_massfrac[n + NUM_SPECIES] = "theta_" + spec_names[n]; } derive_lst.add( @@ -1296,7 +1292,7 @@ PeleLM::derivedSetup() auto& mani_data = eos_parms.host_only_parm().manfunc_par->host_parm(); const int nmanivar = mani_data.Nvar; amrex::Vector var_names_maniout(nmanivar); - for (int n = 0; n < nmanivar; n++) { + for (int n = 0; n < nmanivar; ++n) { std::string nametmp = std::string( &(mani_data.varnames)[n * mani_data.len_str], mani_data.len_str); var_names_maniout[n] = "MANI_" + amrex::trim(nametmp); @@ -1395,7 +1391,7 @@ PeleLM::evaluateSetup() // scalar diffusion term { amrex::Vector var_names(NUM_SPECIES + 2); - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { var_names[n] = "D(" + spec_names[n] + ")"; } var_names[NUM_SPECIES] = "D(RhoH)"; @@ -1413,7 +1409,7 @@ PeleLM::evaluateSetup() var_names[VELX] = "A(VELX)";, var_names[VELY] = "A(VELY)"; , var_names[VELZ] = "A(VELZ)"); var_names[DENSITY] = "A(Rho)"; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { var_names[FIRSTSPEC + n] = "A(" + spec_names[n] + ")"; } var_names[RHOH] = "A(RhoH)"; @@ -1425,12 +1421,12 @@ PeleLM::evaluateSetup() // Chemical state and external chem. forcing (used in ReactEval) { amrex::Vector var_names(2 * (NUM_SPECIES + 1) + 1); - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { var_names[n] = "rhoY(" + spec_names[n] + ")"; } var_names[NUM_SPECIES] = "rhoH"; var_names[NUM_SPECIES + 1] = "Temp"; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { var_names[NUM_SPECIES + 2 + n] = "F_rhoY(" + spec_names[n] + ")"; } var_names[2 * NUM_SPECIES + 2] = "F_rhoH"; @@ -1442,7 +1438,7 @@ PeleLM::evaluateSetup() // instantaneous reaction rate { amrex::Vector var_names(NUM_SPECIES); - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { var_names[n] = "I_R(" + spec_names[n] + ")"; } evaluate_lst.add( @@ -1453,7 +1449,7 @@ PeleLM::evaluateSetup() // cell-centered transport coefficients { amrex::Vector var_names(NUM_SPECIES + 2); - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { var_names[n] = "rhoD(" + spec_names[n] + ")"; } var_names[NUM_SPECIES] = "Lamdba"; @@ -1477,7 +1473,9 @@ PeleLM::taggingSetup() "refinement_indicators", refinement_indicators, 0, ppamr.countval("refinement_indicators")); for (const auto& refinement_indicator : refinement_indicators) { - std::string ref_prefix = amr_prefix + "." + refinement_indicator; + std::string ref_prefix = amr_prefix; + ref_prefix += "."; + ref_prefix += refinement_indicator; amrex::ParmParse ppr(ref_prefix); // Tag a given box diff --git a/Source/PeleLMeX_Soot.cpp b/Source/PeleLMeX_Soot.cpp index f32a601e2..52832d130 100644 --- a/Source/PeleLMeX_Soot.cpp +++ b/Source/PeleLMeX_Soot.cpp @@ -29,13 +29,13 @@ PeleLM::cleanupSootModel() } void -PeleLM::computeSootSource( - const PeleLM::TimeStamp& a_timestamp, const amrex::Real a_dt) +PeleLM::computeSootSource(const TimeStamp a_timestamp, const amrex::Real a_dt) { - bool pres_term = false; // Do not include change in pressure in energy - for (int lev = 0; lev <= finest_level; lev++) { + constexpr bool pres_term = + false; // Do not include change in pressure in energy + for (int lev = 0; lev <= finest_level; ++lev) { auto* ldata_p = getLevelDataPtr(lev, a_timestamp); - amrex::Real time = getTime(lev, a_timestamp); + const amrex::Real time = getTime(lev, a_timestamp); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -54,7 +54,8 @@ PeleLM::computeSootSource( void PeleLM::clipSootMoments() { - for (int lev = 0; lev <= finest_level; lev++) { + SootData* sd = soot_model->getSootData_d(); + for (int lev = 0; lev <= finest_level; ++lev) { auto* ldata_p = getLevelDataPtr(lev, AmrNewTime); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) @@ -63,7 +64,6 @@ PeleLM::clipSootMoments() mfi.isValid(); ++mfi) { amrex::Box const& gbx = mfi.tilebox(); auto const& state_arr = ldata_p->state.array(mfi, FIRSTSOOT); - SootData* sd = soot_model->getSootData_d(); amrex::ParallelFor( gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { amrex::GpuArray moments; diff --git a/Source/PeleLMeX_SprayParticles.cpp b/Source/PeleLMeX_SprayParticles.cpp index 66b2a2e96..5709aa273 100644 --- a/Source/PeleLMeX_SprayParticles.cpp +++ b/Source/PeleLMeX_SprayParticles.cpp @@ -172,8 +172,7 @@ PeleLM::SprayInit() SprayInjectRedist(); if (spray_verbose >= 1) { amrex::Print() << "Total number of initial particles " - << SprayPC->TotalNumberOfParticles(false, false) - << std::endl; + << SprayPC->TotalNumberOfParticles(false, false) << "\n"; } } @@ -280,7 +279,7 @@ PeleLM::SprayMKD(const amrex::Real time, const amrex::Real dt) setupVirtualParticles(0); for (int lev = 0; lev <= finest_level; ++lev) { if (spray_verbose > 1) { - amrex::Print() << "SprayMKDLevel " << lev << std::endl; + amrex::Print() << "SprayMKDLevel " << lev << "\n"; } m_spraysource[lev]->setVal(0.); SprayMKDLevel(lev, time, dt); @@ -346,7 +345,7 @@ PeleLM::SprayPostRegrid() prev_source.resize(finest_level + 1); changed = true; } else { - for (int lev = 0; lev <= finest_level && !changed; lev++) { + for (int lev = 0; lev <= finest_level && !changed; ++lev) { if (ba_spray[lev] != grids[lev]) { changed = true; } @@ -394,7 +393,7 @@ PeleLM::SprayInjectRedist() amrex::Long new_count = SprayPC->TotalNumberOfParticles(true, false); amrex::Long num_inj = new_count - prev_count; amrex::Print() << "Injected " << num_inj << " particles at time " - << m_t_new[0] << std::endl; + << m_t_new[0] << "\n"; } } diff --git a/Source/PeleLMeX_Tagging.cpp b/Source/PeleLMeX_Tagging.cpp index e1ace5a1c..46861a921 100644 --- a/Source/PeleLMeX_Tagging.cpp +++ b/Source/PeleLMeX_Tagging.cpp @@ -40,11 +40,12 @@ PeleLM::ErrorEst( const auto& bx = mfi.tilebox(); auto tag = tags.array(mfi); auto vfrac = EBFactory(lev).getVolFrac().const_array(mfi); - amrex::ParallelFor(bx, [=] AMREX_GPU_HOST_DEVICE(int i, int j, int k) { - if (vfrac(i, j, k) <= 0.0) { - tag(i, j, k) = amrex::TagBox::CLEAR; - } - }); + amrex::ParallelFor( + bx, [tag, vfrac] AMREX_GPU_HOST_DEVICE(int i, int j, int k) { + if (vfrac(i, j, k) <= 0.0) { + tag(i, j, k) = amrex::TagBox::CLEAR; + } + }); } // Untag cell close to EB @@ -55,7 +56,8 @@ PeleLM::ErrorEst( getEBDistance(lev, signDist); // Estimate how far I need to derefine - amrex::Real diagFac = std::sqrt(2.0) * m_derefineEBBuffer; + constexpr amrex::Real sqrt2 = 1.4142135623730951; + const amrex::Real diagFac = sqrt2 * m_derefineEBBuffer; amrex::Real clearTagDist = Geom(m_EB_refine_LevMax).CellSize(0) * static_cast(nErrorBuf(m_EB_refine_LevMax)) * diagFac; @@ -73,11 +75,13 @@ PeleLM::ErrorEst( const auto& bx = mfi.tilebox(); const auto& dist = signDist.const_array(mfi); auto tag = tags.array(mfi); - amrex::ParallelFor(bx, [=] AMREX_GPU_HOST_DEVICE(int i, int j, int k) { - if (dist(i, j, k) < clearTagDist) { - tag(i, j, k) = amrex::TagBox::CLEAR; - } - }); + amrex::ParallelFor( + bx, + [dist, clearTagDist, tag] AMREX_GPU_HOST_DEVICE(int i, int j, int k) { + if (dist(i, j, k) < clearTagDist) { + tag(i, j, k) = amrex::TagBox::CLEAR; + } + }); } } #endif diff --git a/Source/PeleLMeX_Temporals.cpp b/Source/PeleLMeX_Temporals.cpp index 7af899344..fc6ab93a6 100644 --- a/Source/PeleLMeX_Temporals.cpp +++ b/Source/PeleLMeX_Temporals.cpp @@ -2,7 +2,7 @@ #include void -PeleLM::initTemporals(const PeleLM::TimeStamp& a_time) +PeleLM::initTemporals(const PeleLM::TimeStamp a_time) { if ((m_do_temporals == 0) && !(m_nstep % m_temp_int == 0)) { return; @@ -11,23 +11,23 @@ PeleLM::initTemporals(const PeleLM::TimeStamp& a_time) // Reset mass fluxes integrals on domain boundaries if ((m_do_massBalance != 0) && (m_incompressible == 0)) { m_massOld = MFSum(GetVecOfConstPtrs(getDensityVect(a_time)), 0); - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_domainMassFlux[2 * idim] = 0.0; m_domainMassFlux[2 * idim + 1] = 0.0; } } if ((m_do_energyBalance != 0) && (m_incompressible == 0)) { m_RhoHOld = MFSum(GetVecOfConstPtrs(getRhoHVect(a_time)), 0); - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_domainRhoHFlux[2 * idim] = 0.0; m_domainRhoHFlux[2 * idim + 1] = 0.0; } } if ((m_do_speciesBalance != 0) && (m_incompressible == 0)) { - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { m_RhoYOld[n] = MFSum(GetVecOfConstPtrs(getSpeciesVect(a_time)), n); - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_domainRhoYFlux[2 * n * AMREX_SPACEDIM + 2 * idim] = 0.0; m_domainRhoYFlux[1 + 2 * n * AMREX_SPACEDIM + 2 * idim] = 0.0; } @@ -60,7 +60,7 @@ PeleLM::speciesBalancePatch() tmppatchmfrFile << m_nstep << "," << m_cur_time; // Time info for (const auto& m_bPatche : m_bPatches) { BPatch::BpatchDataContainer* bphost = m_bPatche->getHostDataPtr(); - for (int i = 0; i < bphost->num_species; i++) { + for (int i = 0; i < bphost->num_species; ++i) { tmppatchmfrFile << "," << bphost->speciesFlux[i]; } } @@ -75,7 +75,7 @@ PeleLM::speciesBalance() amrex::Array dmYdt; amrex::Array massYFluxBalance; amrex::Array rhoYdots; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { m_RhoYNew[n] = MFSum(GetVecOfConstPtrs(getSpeciesVect(AmrNewTime)), n); rhoYdots[n] = MFSum(GetVecOfConstPtrs(getIRVect()), n); dmYdt[n] = (m_RhoYNew[n] - m_RhoYOld[n]) / m_dt; @@ -89,7 +89,7 @@ PeleLM::speciesBalance() } tmpSpecFile << m_nstep << "," << m_cur_time; // Time info - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { tmpSpecFile << "," << m_RhoYNew[n] // mass of Y << "," << dmYdt[n] // mass temporal derivative << "," << massYFluxBalance[n] // domain boundaries mass fluxes @@ -128,7 +128,7 @@ PeleLM::addMassFluxes( area[2] = dx[0] * dx[1]; #endif - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { auto faceDomain = amrex::convert(a_geom.Domain(), amrex::IntVect::TheDimensionVector(idim)); @@ -146,8 +146,9 @@ PeleLM::addMassFluxes( amrex::TypeList{}, amrex::TypeList{}, *a_fluxes[idim], amrex::IntVect(0), - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept - -> amrex::GpuTuple { + [fma, ama, idim, faceDomain] AMREX_GPU_DEVICE( + int box_no, int i, int j, + int k) noexcept -> amrex::GpuTuple { amrex::Array4 const& flux = fma[box_no]; amrex::Array4 const& area_ar = ama[box_no]; @@ -155,14 +156,14 @@ PeleLM::addMassFluxes( // low amrex::Real low = 0.0; if (idx == faceDomain.smallEnd(idim)) { - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { low += flux(i, j, k, n) * area_ar(i, j, k); } } // high amrex::Real high = 0.0; if (idx == faceDomain.bigEnd(idim)) { - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { high += flux(i, j, k, n) * area_ar(i, j, k); } } @@ -177,22 +178,23 @@ PeleLM::addMassFluxes( amrex::TypeList{}, amrex::TypeList{}, *a_fluxes[idim], amrex::IntVect(0), - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept - -> amrex::GpuTuple { + [fma, idim, faceDomain, area] AMREX_GPU_DEVICE( + int box_no, int i, int j, + int k) noexcept -> amrex::GpuTuple { amrex::Array4 const& flux = fma[box_no]; int idx = (idim == 0) ? i : ((idim == 1) ? j : k); // low amrex::Real low = 0.0; if (idx == faceDomain.smallEnd(idim)) { - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { low += flux(i, j, k, n) * area[idim]; } } // high amrex::Real high = 0.0; if (idx == faceDomain.bigEnd(idim)) { - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { high += flux(i, j, k, n) * area[idim]; } } @@ -210,7 +212,7 @@ PeleLM::addMassFluxes( void PeleLM::addUmacFluxes( - std::unique_ptr& advData, const amrex::Geometry& a_geom) + const std::unique_ptr& advData, const amrex::Geometry& a_geom) { // Get the face areas const amrex::Real* dx = a_geom.CellSize(); @@ -227,9 +229,9 @@ PeleLM::addUmacFluxes( #endif // Just use level 0 since we are calling after averaging down - int lev = 0; + constexpr int lev = 0; - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { auto faceDomain = amrex::convert(a_geom.Domain(), amrex::IntVect::TheDimensionVector(idim)); @@ -247,8 +249,9 @@ PeleLM::addUmacFluxes( amrex::TypeList{}, amrex::TypeList{}, advData->umac[lev][idim], amrex::IntVect(0), - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept - -> amrex::GpuTuple { + [fma, ama, idim, faceDomain] AMREX_GPU_DEVICE( + int box_no, int i, int j, + int k) noexcept -> amrex::GpuTuple { amrex::Array4 const& flux = fma[box_no]; amrex::Array4 const& area_ar = ama[box_no]; @@ -274,8 +277,9 @@ PeleLM::addUmacFluxes( amrex::TypeList{}, amrex::TypeList{}, advData->umac[lev][idim], amrex::IntVect(0), - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept - -> amrex::GpuTuple { + [fma, idim, faceDomain, area] AMREX_GPU_DEVICE( + int box_no, int i, int j, + int k) noexcept -> amrex::GpuTuple { amrex::Array4 const& flux = fma[box_no]; int idx = (idim == 0) ? i : ((idim == 1) ? j : k); @@ -346,7 +350,7 @@ PeleLM::addRhoHFluxes( area[2] = dx[0] * dx[1]; #endif - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { auto faceDomain = amrex::convert(a_geom.Domain(), amrex::IntVect::TheDimensionVector(idim)); @@ -364,8 +368,9 @@ PeleLM::addRhoHFluxes( amrex::TypeList{}, amrex::TypeList{}, *a_fluxes[idim], amrex::IntVect(0), - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept - -> amrex::GpuTuple { + [fma, ama, idim, faceDomain] AMREX_GPU_DEVICE( + int box_no, int i, int j, + int k) noexcept -> amrex::GpuTuple { amrex::Array4 const& flux = fma[box_no]; amrex::Array4 const& area_ar = ama[box_no]; @@ -391,8 +396,9 @@ PeleLM::addRhoHFluxes( amrex::TypeList{}, amrex::TypeList{}, *a_fluxes[idim], amrex::IntVect(0), - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept - -> amrex::GpuTuple { + [fma, idim, faceDomain, area] AMREX_GPU_DEVICE( + int box_no, int i, int j, + int k) noexcept -> amrex::GpuTuple { amrex::Array4 const& flux = fma[box_no]; int idx = (idim == 0) ? i : ((idim == 1) ? j : k); @@ -446,9 +452,9 @@ PeleLM::addRhoYFluxes( #endif // Outer loop over species - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { // Inner loop over dimensions - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { auto faceDomain = amrex::convert( a_geom.Domain(), amrex::IntVect::TheDimensionVector(idim)); @@ -466,8 +472,9 @@ PeleLM::addRhoYFluxes( amrex::TypeList{}, amrex::TypeList{}, *a_fluxes[idim], amrex::IntVect(0), - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept - -> amrex::GpuTuple { + [fma, ama, idim, faceDomain, n] AMREX_GPU_DEVICE( + int box_no, int i, int j, + int k) noexcept -> amrex::GpuTuple { amrex::Array4 const& flux = fma[box_no]; amrex::Array4 const& area_ar = ama[box_no]; @@ -493,8 +500,9 @@ PeleLM::addRhoYFluxes( amrex::TypeList{}, amrex::TypeList{}, *a_fluxes[idim], amrex::IntVect(0), - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept - -> amrex::GpuTuple { + [fma, idim, faceDomain, area, n] AMREX_GPU_DEVICE( + int box_no, int i, int j, + int k) noexcept -> amrex::GpuTuple { amrex::Array4 const& flux = fma[box_no]; int idx = (idim == 0) ? i : ((idim == 1) ? j : k); @@ -523,7 +531,7 @@ PeleLM::addRhoYFluxes( } void -PeleLM::initBPatches(amrex::Geometry& a_geom) +PeleLM::initBPatches(const amrex::Geometry& a_geom) { std::string pele_prefix = "peleLM.bpatch"; amrex::ParmParse pp(pele_prefix); @@ -541,7 +549,7 @@ PeleLM::initBPatches(amrex::Geometry& a_geom) m_bPatches[n] = std::make_unique(bpatch_name[n], a_geom); if (m_verbose > 0) { amrex::Print() << " Initializing boundary patch: " << bpatch_name[n] - << std::endl; + << "\n"; } } } @@ -588,7 +596,7 @@ PeleLM::addRhoYFluxesPatch( auto const& fma = a_fluxes[idim]->const_arrays(); // Loop through species specified by user - for (int m = 0; m < bphost->num_species; m++) { + for (int m = 0; m < bphost->num_species; ++m) { amrex::Real sum_species_flux_global = 0.0; @@ -597,7 +605,8 @@ PeleLM::addRhoYFluxesPatch( amrex::TypeList{}, amrex::TypeList{}, *a_fluxes[idim], amrex::IntVect(0), - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept + [fma, bpdevice, faceDomain, idim, prob_lo, dx, m, + area] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept -> amrex::GpuTuple { amrex::Array4 const& flux = fma[box_no]; int idx = @@ -744,7 +753,7 @@ PeleLM::openTempFile() std::ios::out | std::ios::app | std::ios_base::binary); tmpSpecFile.precision(12); tmpSpecFile << "iter,time"; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { tmpSpecFile << ",rhoYnew_" << PeleLM::stateVariableName(FIRSTSPEC + n); tmpSpecFile << ",drhoYdt_" << PeleLM::stateVariableName(FIRSTSPEC + n); tmpSpecFile << ",netFlux_" << PeleLM::stateVariableName(FIRSTSPEC + n); @@ -776,7 +785,7 @@ PeleLM::openTempFile() for (const auto& m_bPatche : m_bPatches) { BPatch* patch = m_bPatche.get(); BPatch::BpatchDataContainer bphost = patch->getHostData(); - for (int i = 0; i < bphost.num_species; i++) { + for (int i = 0; i < bphost.num_species; ++i) { tmppatchmfrFile << "," << patch->m_patchname + "_" + patch->speciesList[i]; } @@ -791,7 +800,7 @@ PeleLM::openTempFile() std::ios::out | std::ios::app | std::ios_base::binary); tmpIonsFile.precision(12); tmpIonsFile << "iter,time"; - for (int i = 0; i < AMREX_SPACEDIM; i++) { + for (int i = 0; i < AMREX_SPACEDIM; ++i) { tmpIonsFile << ",curr_" << i << "_low,curr_" << i << "_hi"; } tmpIonsFile << "\n"; diff --git a/Source/PeleLMeX_Timestep.cpp b/Source/PeleLMeX_Timestep.cpp index de67079cb..ecafa68c3 100644 --- a/Source/PeleLMeX_Timestep.cpp +++ b/Source/PeleLMeX_Timestep.cpp @@ -2,7 +2,7 @@ #include amrex::Real -PeleLM::computeDt(int is_init, const TimeStamp& a_time) +PeleLM::computeDt(const int is_init, const TimeStamp a_time) { BL_PROFILE("PeleLMeX::computeDt()"); @@ -96,7 +96,7 @@ PeleLM::computeDt(int is_init, const TimeStamp& a_time) } amrex::Real -PeleLM::estConvectiveDt(const TimeStamp& a_time) +PeleLM::estConvectiveDt(const TimeStamp a_time) { amrex::Real estdt = 1.0e200; @@ -114,12 +114,12 @@ PeleLM::estConvectiveDt(const TimeStamp& a_time) //---------------------------------------------------------------- // Get velocity forces - int nGrow_force = 0; + constexpr int nGrow_force = 0; amrex::MultiFab velForces( grids[lev], dmap[lev], AMREX_SPACEDIM, nGrow_force, amrex::MFInfo(), Factory(lev)); - int add_gradP = 1; + constexpr int add_gradP = 1; getVelForces(a_time, lev, nullptr, &velForces, add_gradP); //---------------------------------------------------------------- @@ -155,7 +155,7 @@ PeleLM::estConvectiveDt(const TimeStamp& a_time) } amrex::Real -PeleLM::estDivUDt(const TimeStamp& a_time) +PeleLM::estDivUDt(const TimeStamp a_time) { amrex::Real estdt = 1.0e200; @@ -227,7 +227,7 @@ PeleLM::estDivUDt(const TimeStamp& a_time) } void -PeleLM::checkDt(const TimeStamp& a_time, const amrex::Real& a_dt) +PeleLM::checkDt(const TimeStamp a_time, const amrex::Real a_dt) { BL_PROFILE("PeleLMeX::checkDt()"); @@ -236,10 +236,9 @@ PeleLM::checkDt(const TimeStamp& a_time, const amrex::Real& a_dt) } for (int lev = 0; lev <= finest_level; ++lev) { - auto* ldata_p = getLevelDataPtr(lev, a_time); - - const auto dxinv = geom[lev].InvCellSizeArray(); + const amrex::GpuArray dxinv = + geom[lev].InvCellSizeArray(); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) diff --git a/Source/PeleLMeX_TransportProp.cpp b/Source/PeleLMeX_TransportProp.cpp index cbb2fb09c..eae3b1bbb 100644 --- a/Source/PeleLMeX_TransportProp.cpp +++ b/Source/PeleLMeX_TransportProp.cpp @@ -7,7 +7,7 @@ #endif void -PeleLM::calcTurbViscosity(const TimeStamp& a_time) +PeleLM::calcTurbViscosity(const TimeStamp a_time) { BL_PROFILE("PeleLMeX::calcTurbViscosity()"); @@ -16,8 +16,7 @@ PeleLM::calcTurbViscosity(const TimeStamp& a_time) if (m_les_verbose > 0) { amrex::Print() << " Computing Turbulent Viscosity with LES model: " - << m_les_model << " for time " << getTime(0, a_time) - << std::endl; + << m_les_model << " for time " << getTime(0, a_time) << "\n"; } // Create temporary multifab to store velocity gradient tensor @@ -62,14 +61,15 @@ PeleLM::calcTurbViscosity(const TimeStamp& a_time) } } else { // get cp_cc (valid in 1 grow cell for interpolation to FCs) - int ngrow = 1; + constexpr int ngrow = 1; auto const* leosparm = eos_parms.device_parm(); cp_cc.define(ba, dm, 1, ngrow, amrex::MFInfo(), factory); auto const& state_arr = ldata_p->state.const_arrays(); auto const& cp_arr = cp_cc.arrays(); amrex::ParallelFor( cp_cc, cp_cc.nGrowVect(), - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + [state_arr, cp_arr, + leosparm] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { getCpmixGivenRYT( i, j, k, amrex::Array4(state_arr[box_no], DENSITY), @@ -80,7 +80,7 @@ PeleLM::calcTurbViscosity(const TimeStamp& a_time) amrex::Gpu::streamSynchronize(); // this function really just interpolates CCs to FCs in this case - int doZeroVisc = 0; + constexpr int doZeroVisc = 0; auto bcRec = fetchBCRecArray(DENSITY, 1); dens_fc = getDiffusivity(lev, DENSITY, 1, doZeroVisc, {bcRec}, ldata_p->state); @@ -94,56 +94,42 @@ PeleLM::calcTurbViscosity(const TimeStamp& a_time) auto const& mut_arr = ldata_p->visc_turb_fc[idim].arrays(); const amrex::Real vol = AMREX_D_TERM( geom[lev].CellSize(0), *geom[lev].CellSize(1), *geom[lev].CellSize(2)); + const amrex::Real l_scale = - (AMREX_SPACEDIM == 2) ? std::sqrt(vol) : std::cbrt(vol); + AMREX_D_PICK(vol, std::sqrt(vol), std::cbrt(vol)); #ifdef AMREX_USE_EB auto const& ebfact = EBFactory(lev); - auto const vfrac = ebfact.getVolFrac().const_arrays(); + auto const& vfrac = ebfact.getVolFrac().const_arrays(); #endif if (m_les_model == "Smagorinsky") { const amrex::Real prefact = m_les_cs_smag * m_les_cs_smag * l_scale * l_scale; amrex::ParallelFor( ldata_p->visc_turb_fc[idim], ldata_p->visc_turb_fc[idim].nGrowVect(), - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + [prefact, velgrad_arr, dens_arr, mut_arr +#ifdef AMREX_USE_EB + , + vfrac, idim +#endif + ] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { getTurbViscSmagorinsky( i, j, k, prefact, amrex::Array4(velgrad_arr[box_no]), amrex::Array4(dens_arr[box_no]), amrex::Array4(mut_arr[box_no])); #ifdef AMREX_USE_EB - if (idim == 0) { - const amrex::Real vfr_m = - (AMREX_SPACEDIM == 2) ? vfrac[box_no](i - 1, j, k) - : std::cbrt(vfrac[box_no](i - 1, j, k)) * - std::cbrt(vfrac[box_no](i - 1, j, k)); - const amrex::Real vfr_p = (AMREX_SPACEDIM == 2) - ? vfrac[box_no](i, j, k) - : std::cbrt(vfrac[box_no](i, j, k)) * - std::cbrt(vfrac[box_no](i, j, k)); - mut_arr[box_no](i, j, k) *= amrex::min(vfr_m, vfr_p); - } else if (idim == 1) { - const amrex::Real vfr_m = - (AMREX_SPACEDIM == 2) ? vfrac[box_no](i, j - 1, k) - : std::cbrt(vfrac[box_no](i, j - 1, k)) * - std::cbrt(vfrac[box_no](i, j - 1, k)); - const amrex::Real vfr_p = (AMREX_SPACEDIM == 2) - ? vfrac[box_no](i, j, k) - : std::cbrt(vfrac[box_no](i, j, k)) * - std::cbrt(vfrac[box_no](i, j, k)); - mut_arr[box_no](i, j, k) *= amrex::min(vfr_m, vfr_p); - } else if (idim == 2) { - const amrex::Real vfr_m = - (AMREX_SPACEDIM == 2) ? vfrac[box_no](i, j, k - 1) - : std::cbrt(vfrac[box_no](i, j, k - 1)) * - std::cbrt(vfrac[box_no](i, j, k - 1)); - const amrex::Real vfr_p = (AMREX_SPACEDIM == 2) - ? vfrac[box_no](i, j, k) - : std::cbrt(vfrac[box_no](i, j, k)) * - std::cbrt(vfrac[box_no](i, j, k)); - mut_arr[box_no](i, j, k) *= amrex::min(vfr_m, vfr_p); - } + int idx[3] = {i, j, k}; + idx[idim] -= 1; + const amrex::Real vfr_m = AMREX_D_PICK( + 1.0, vfrac[box_no](idx[0], idx[1], idx[2]), + std::cbrt( + vfrac[box_no](idx[0], idx[1], idx[2]) * + vfrac[box_no](idx[0], idx[1], idx[2]))); + const amrex::Real vfr_p = AMREX_D_PICK( + 1.0, vfrac[box_no](i, j, k), + std::cbrt(vfrac[box_no](i, j, k) * vfrac[box_no](i, j, k))); + mut_arr[box_no](i, j, k) *= amrex::min(vfr_m, vfr_p); #endif }); } else if (m_les_model == "WALE") { @@ -151,44 +137,29 @@ PeleLM::calcTurbViscosity(const TimeStamp& a_time) m_les_cm_wale * m_les_cm_wale * l_scale * l_scale; amrex::ParallelFor( ldata_p->visc_turb_fc[idim], ldata_p->visc_turb_fc[idim].nGrowVect(), - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + [prefact, velgrad_arr, dens_arr, mut_arr +#ifdef AMREX_USE_EB + , + vfrac, idim +#endif + ] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { getTurbViscWALE( i, j, k, prefact, amrex::Array4(velgrad_arr[box_no]), amrex::Array4(dens_arr[box_no]), amrex::Array4(mut_arr[box_no])); #ifdef AMREX_USE_EB - if (idim == 0) { - const amrex::Real vfr_m = - (AMREX_SPACEDIM == 2) ? vfrac[box_no](i - 1, j, k) - : std::cbrt(vfrac[box_no](i - 1, j, k)) * - std::cbrt(vfrac[box_no](i - 1, j, k)); - const amrex::Real vfr_p = (AMREX_SPACEDIM == 2) - ? vfrac[box_no](i, j, k) - : std::cbrt(vfrac[box_no](i, j, k)) * - std::cbrt(vfrac[box_no](i, j, k)); - mut_arr[box_no](i, j, k) *= amrex::min(vfr_m, vfr_p); - } else if (idim == 1) { - const amrex::Real vfr_m = - (AMREX_SPACEDIM == 2) ? vfrac[box_no](i, j - 1, k) - : std::cbrt(vfrac[box_no](i, j - 1, k)) * - std::cbrt(vfrac[box_no](i, j - 1, k)); - const amrex::Real vfr_p = (AMREX_SPACEDIM == 2) - ? vfrac[box_no](i, j, k) - : std::cbrt(vfrac[box_no](i, j, k)) * - std::cbrt(vfrac[box_no](i, j, k)); - mut_arr[box_no](i, j, k) *= amrex::min(vfr_m, vfr_p); - } else if (idim == 2) { - const amrex::Real vfr_m = - (AMREX_SPACEDIM == 2) ? vfrac[box_no](i, j, k - 1) - : std::cbrt(vfrac[box_no](i, j, k - 1)) * - std::cbrt(vfrac[box_no](i, j, k - 1)); - const amrex::Real vfr_p = (AMREX_SPACEDIM == 2) - ? vfrac[box_no](i, j, k) - : std::cbrt(vfrac[box_no](i, j, k)) * - std::cbrt(vfrac[box_no](i, j, k)); - mut_arr[box_no](i, j, k) *= amrex::min(vfr_m, vfr_p); - } + int idx[3] = {i, j, k}; + idx[idim] -= 1; + const amrex::Real vfr_m = AMREX_D_PICK( + 1.0, vfrac[box_no](idx[0], idx[1], idx[2]), + std::cbrt( + vfrac[box_no](idx[0], idx[1], idx[2]) * + vfrac[box_no](idx[0], idx[1], idx[2]))); + const amrex::Real vfr_p = AMREX_D_PICK( + 1.0, vfrac[box_no](i, j, k), + std::cbrt(vfrac[box_no](i, j, k) * vfrac[box_no](i, j, k))); + mut_arr[box_no](i, j, k) *= amrex::min(vfr_m, vfr_p); #endif }); } else if (m_les_model == "Sigma") { @@ -196,44 +167,29 @@ PeleLM::calcTurbViscosity(const TimeStamp& a_time) m_les_cs_sigma * m_les_cs_sigma * l_scale * l_scale; amrex::ParallelFor( ldata_p->visc_turb_fc[idim], ldata_p->visc_turb_fc[idim].nGrowVect(), - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + [prefact, velgrad_arr, dens_arr, mut_arr +#ifdef AMREX_USE_EB + , + vfrac, idim +#endif + ] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { getTurbViscSigma( i, j, k, prefact, amrex::Array4(velgrad_arr[box_no]), amrex::Array4(dens_arr[box_no]), amrex::Array4(mut_arr[box_no])); #ifdef AMREX_USE_EB - if (idim == 0) { - const amrex::Real vfr_m = - (AMREX_SPACEDIM == 2) ? vfrac[box_no](i - 1, j, k) - : std::cbrt(vfrac[box_no](i - 1, j, k)) * - std::cbrt(vfrac[box_no](i - 1, j, k)); - const amrex::Real vfr_p = (AMREX_SPACEDIM == 2) - ? vfrac[box_no](i, j, k) - : std::cbrt(vfrac[box_no](i, j, k)) * - std::cbrt(vfrac[box_no](i, j, k)); - mut_arr[box_no](i, j, k) *= amrex::min(vfr_m, vfr_p); - } else if (idim == 1) { - const amrex::Real vfr_m = - (AMREX_SPACEDIM == 2) ? vfrac[box_no](i, j - 1, k) - : std::cbrt(vfrac[box_no](i, j - 1, k)) * - std::cbrt(vfrac[box_no](i, j - 1, k)); - const amrex::Real vfr_p = (AMREX_SPACEDIM == 2) - ? vfrac[box_no](i, j, k) - : std::cbrt(vfrac[box_no](i, j, k)) * - std::cbrt(vfrac[box_no](i, j, k)); - mut_arr[box_no](i, j, k) *= amrex::min(vfr_m, vfr_p); - } else if (idim == 2) { - const amrex::Real vfr_m = - (AMREX_SPACEDIM == 2) ? vfrac[box_no](i, j, k - 1) - : std::cbrt(vfrac[box_no](i, j, k - 1)) * - std::cbrt(vfrac[box_no](i, j, k - 1)); - const amrex::Real vfr_p = (AMREX_SPACEDIM == 2) - ? vfrac[box_no](i, j, k) - : std::cbrt(vfrac[box_no](i, j, k)) * - std::cbrt(vfrac[box_no](i, j, k)); - mut_arr[box_no](i, j, k) *= amrex::min(vfr_m, vfr_p); - } + int idx[3] = {i, j, k}; + idx[idim] -= 1; + const amrex::Real vfr_m = AMREX_D_PICK( + 1.0, vfrac[box_no](idx[0], idx[1], idx[2]), + std::cbrt( + vfrac[box_no](idx[0], idx[1], idx[2]) * + vfrac[box_no](idx[0], idx[1], idx[2]))); + const amrex::Real vfr_p = AMREX_D_PICK( + 1.0, vfrac[box_no](i, j, k), + std::cbrt(vfrac[box_no](i, j, k) * vfrac[box_no](i, j, k))); + mut_arr[box_no](i, j, k) *= amrex::min(vfr_m, vfr_p); #endif }); } @@ -253,7 +209,7 @@ PeleLM::calcTurbViscosity(const TimeStamp& a_time) } void -PeleLM::calcViscosity(const TimeStamp& a_time) +PeleLM::calcViscosity(const TimeStamp a_time) { BL_PROFILE("PeleLMeX::calcViscosity()"); @@ -274,7 +230,8 @@ PeleLM::calcViscosity(const TimeStamp& a_time) amrex::ParallelFor( ldata_p->visc_cc, ldata_p->visc_cc.nGrowVect(), - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + [sma, vma, ltransparm] AMREX_GPU_DEVICE( + int box_no, int i, int j, int k) noexcept { getVelViscosity( i, j, k, amrex::Array4(sma[box_no], FIRSTSPEC), amrex::Array4(sma[box_no], TEMP), @@ -286,7 +243,7 @@ PeleLM::calcViscosity(const TimeStamp& a_time) } void -PeleLM::calcDiffusivity(const TimeStamp& a_time) +PeleLM::calcDiffusivity(const TimeStamp a_time) { BL_PROFILE("PeleLMeX::calcDiffusivity()"); @@ -322,7 +279,13 @@ PeleLM::calcDiffusivity(const TimeStamp& a_time) amrex::ParallelFor( ldata_p->diff_cc, ldata_p->diff_cc.nGrowVect(), - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + [sma, dma, soret_idx, ltransparm, do_fixed_Le, do_fixed_Pr, do_soret, + Le_inv, Pr_inv, leosparm +#if PELE_USE_PLASMA + , + kma, mwt, zk = zk +#endif + ] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { getTransportCoeff( i, j, k, do_fixed_Le, do_fixed_Pr, do_soret, Le_inv, Pr_inv, amrex::Array4(sma[box_no], FIRSTSPEC), @@ -341,9 +304,11 @@ PeleLM::calcDiffusivity(const TimeStamp& a_time) amrex::Array4(kma[box_no], 0)); #endif }); + // Shift outside/combine with below? + amrex::Gpu::streamSynchronize(); // Fill the diff_aux MF with specified Schmidt number - for (int n = 0; n < m_nAux; n++) { + for (int n = 0; n < m_nAux; ++n) { if (m_aux_Schmidt[n] > 0) { amrex::MultiFab::Copy( ldata_p->diff_aux_cc, ldata_p->diff_cc, NUM_SPECIES + 1, n, 1, @@ -360,13 +325,14 @@ PeleLM::calcDiffusivity(const TimeStamp& a_time) const auto& factory = ldata_p->diff_cc.Factory(); amrex::MultiFab cp_cc; - int ngrow = ldata_p->diff_cc.nGrow(); + const int ngrow = ldata_p->diff_cc.nGrow(); cp_cc.define(ba, dm, 1, ngrow, amrex::MFInfo(), factory); auto const& state_arr = ldata_p->state.const_arrays(); auto const& cp_arr = cp_cc.arrays(); amrex::ParallelFor( cp_cc, cp_cc.nGrowVect(), - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + [state_arr, cp_arr, leosparm] AMREX_GPU_DEVICE( + int box_no, int i, int j, int k) noexcept { getCpmixGivenRYT( i, j, k, amrex::Array4(state_arr[box_no], DENSITY), @@ -374,23 +340,23 @@ PeleLM::calcDiffusivity(const TimeStamp& a_time) amrex::Array4(state_arr[box_no], TEMP), amrex::Array4(cp_arr[box_no]), leosparm); }); - + // Combine with the one above? + amrex::Gpu::streamSynchronize(); ldata_p->diff_aux_cc.divide(cp_cc, n, 1, ldata_p->diff_cc.nGrow()); } } } - amrex::Gpu::streamSynchronize(); } amrex::Array PeleLM::getDiffusivity( - int lev, - int beta_comp, - int ncomp, - int doZeroVisc, - amrex::Vector bcrec, + const int lev, + const int beta_comp, + const int ncomp, + const int doZeroVisc, + amrex::Vector const& bcrec, amrex::MultiFab const& beta_cc, - int addTurbContrib) + const int addTurbContrib) { BL_PROFILE("PeleLMeX::getDiffusivity()"); @@ -420,14 +386,14 @@ PeleLM::getDiffusivity( EB_set_covered_faces(GetArrOfPtrs(beta_ec), 1.234e40); #else // NON-EB : use cen2edg_cpp - bool use_harmonic_avg = m_harm_avg_cen2edge != 0; + const bool use_harmonic_avg = m_harm_avg_cen2edge != 0; #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif for (amrex::MFIter mfi(beta_cc, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { const amrex::Box ebx = mfi.nodaltilebox(idim); const amrex::Box& edomain = amrex::surroundingNodes(domain, idim); const auto& diff_c = beta_cc.const_array(mfi, beta_comp); @@ -435,12 +401,13 @@ PeleLM::getDiffusivity( const auto bc_lo = bcrec[0].lo(idim); const auto bc_hi = bcrec[0].hi(idim); amrex::ParallelFor( - ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - int idx[3] = {i, j, k}; - bool on_lo = + ebx, [bc_lo, bc_hi, diff_ec, use_harmonic_avg, ncomp, idim, edomain, + diff_c] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + const int idx[3] = {i, j, k}; + const bool on_lo = ((bc_lo == amrex::BCType::ext_dir) && (idx[idim] <= edomain.smallEnd(idim))); - bool on_hi = + const bool on_hi = ((bc_hi == amrex::BCType::ext_dir) && (idx[idim] >= edomain.bigEnd(idim))); cen2edg_cpp( @@ -474,13 +441,13 @@ PeleLM::getDiffusivity( auto* ldata_p = getLevelDataPtr(lev, tstamp); // Identify and add the correct turbulent contribution - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { if ((ncomp == 1) and (beta_comp == 0)) { // Viscosity amrex::MultiFab::Add( beta_ec[idim], ldata_p->visc_turb_fc[idim], 0, 0, 1, 0); } else if ((ncomp == NUM_SPECIES) and (beta_comp == 0)) { // Species // diffusivity - for (int ispec = 0; ispec < NUM_SPECIES; ispec++) { + for (int ispec = 0; ispec < NUM_SPECIES; ++ispec) { amrex::MultiFab::Saxpy( beta_ec[idim], m_Schmidt_inv, ldata_p->visc_turb_fc[idim], 0, ispec, 1, 0); @@ -501,7 +468,7 @@ PeleLM::getDiffusivity( if (doZeroVisc != 0) { ProbParm const* lprobparm = prob_parm_d; const auto geomdata = geom[lev].data(); - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { const amrex::Box& edomain = amrex::surroundingNodes(domain, idim); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) @@ -511,7 +478,8 @@ PeleLM::getDiffusivity( const amrex::Box ebx = mfi.tilebox(); const auto& diff_ec = beta_ec[idim].array(mfi); amrex::ParallelFor( - ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ebx, [diff_ec, geomdata, edomain, idim, beta_comp, ncomp, + lprobparm] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { ProblemSpecificFunctions::zero_visc( i, j, k, diff_ec, geomdata, edomain, idim, beta_comp, ncomp, *lprobparm); diff --git a/Source/PeleLMeX_UMac.cpp b/Source/PeleLMeX_UMac.cpp index d85a8a450..ee16ba1b9 100644 --- a/Source/PeleLMeX_UMac.cpp +++ b/Source/PeleLMeX_UMac.cpp @@ -5,7 +5,7 @@ #include void -PeleLM::predictVelocity(std::unique_ptr& advData) +PeleLM::predictVelocity(const std::unique_ptr& advData) { BL_PROFILE("PeleLMeX::predictVelocity()"); @@ -20,23 +20,25 @@ PeleLM::predictVelocity(std::unique_ptr& advData) //---------------------------------------------------------------- // Get viscous forces - int nGrow_force = 1; - amrex::Vector divtau(finest_level + 1); - amrex::Vector velForces(finest_level + 1); + constexpr int nGrow_force = 1; + amrex::Vector divtau; + divtau.reserve(finest_level + 1); + amrex::Vector velForces; + velForces.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - divtau[lev].define( + divtau.emplace_back( grids[lev], dmap[lev], AMREX_SPACEDIM, 0, amrex::MFInfo(), Factory(lev)); - velForces[lev].define( + velForces.emplace_back( grids[lev], dmap[lev], AMREX_SPACEDIM, nGrow_force, amrex::MFInfo(), Factory(lev)); } - int use_density = 0; + constexpr int use_density = 0; computeDivTau(AmrOldTime, GetVecOfPtrs(divtau), use_density); //---------------------------------------------------------------- // Gather all the velocity forces // F = [ (gravity+...) - gradP + divTau ] / rho - int add_gradP = 1; + constexpr int add_gradP = 1; getVelForces( AmrOldTime, GetVecOfPtrs(divtau), GetVecOfPtrs(velForces), nGrow_force, add_gradP); @@ -71,7 +73,7 @@ PeleLM::predictVelocity(std::unique_ptr& advData) } void -PeleLM::createMACRHS(std::unique_ptr& advData) +PeleLM::createMACRHS(const std::unique_ptr& advData) { BL_PROFILE("PeleLMeX::createMACRHS()"); @@ -83,16 +85,17 @@ PeleLM::createMACRHS(std::unique_ptr& advData) void PeleLM::addChiIncrement( - int a_sdcIter, - const TimeStamp& a_time, - std::unique_ptr& advData) + const int a_sdcIter, + const TimeStamp a_time, + const std::unique_ptr& advData) { BL_PROFILE("PeleLMeX::addChiIncrement()"); - int nGrow = m_nGrowAdv; - amrex::Vector chiIncr(finest_level + 1); + const int nGrow = m_nGrowAdv; + amrex::Vector chiIncr; + chiIncr.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - chiIncr[lev].define( + chiIncr.emplace_back( grids[lev], dmap[lev], 1, nGrow, amrex::MFInfo(), Factory(lev)); } @@ -148,23 +151,24 @@ PeleLM::addChiIncrement( } } } + if (m_print_chi_convergence) { - amrex::Real max_corr = + const amrex::Real max_corr = MLNorm0(GetVecOfConstPtrs(chiIncr)) * m_dt / m_dpdtFactor; amrex::Print() << " Before SDC " << a_sdcIter - << ": max relative P mismatch is " << max_corr << std::endl; + << ": max relative P mismatch is " << max_corr << "\n"; } } void PeleLM::macProject( - const TimeStamp& a_time, - std::unique_ptr& advData, + const TimeStamp a_time, + const std::unique_ptr& advData, const amrex::Vector& a_divu) { BL_PROFILE("PeleLMeX::macProject()"); - int has_divu = static_cast(!a_divu.empty()); + const int has_divu = static_cast(!a_divu.empty()); // Get face rho inv auto bcRec = fetchBCRecArray(DENSITY, 1); @@ -181,7 +185,7 @@ PeleLM::macProject( } } else { auto* ldata_p = getLevelDataPtr(lev, a_time); - int doZeroVisc = 0; + constexpr int doZeroVisc = 0; rho_inv[lev] = getDiffusivity(lev, DENSITY, 1, doZeroVisc, {bcRec}, ldata_p->state); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { @@ -238,32 +242,29 @@ PeleLM::macProject( } // FillBoundary umac - for (int lev = 0; lev <= finest_level; ++lev) { - if (lev > 0) { - // We need to fill the MAC velocities outside the fine region so we can - // use them in the Godunov method - amrex::IntVect rr = - geom[lev].Domain().size() / geom[lev - 1].Domain().size(); - create_constrained_umac_grown( - m_nGrowMAC, &geom[lev - 1], &geom[lev], - GetArrOfPtrs(advData->umac[lev - 1]), GetArrOfPtrs(advData->umac[lev]), - rr); - } else { - AMREX_D_TERM( - advData->umac[lev][0].FillBoundary(geom[lev].periodicity()); - , advData->umac[lev][1].FillBoundary(geom[lev].periodicity()); - , advData->umac[lev][2].FillBoundary(geom[lev].periodicity())); - } + // Do coarse first + AMREX_D_TERM(advData->umac[0][0].FillBoundary(geom[0].periodicity()); + , advData->umac[0][1].FillBoundary(geom[0].periodicity()); + , advData->umac[0][2].FillBoundary(geom[0].periodicity())); + for (int lev = 1; lev <= finest_level; ++lev) { + // We need to fill the MAC velocities outside the fine region so we can + // use them in the Godunov method + const amrex::IntVect rr = + geom[lev].Domain().size() / geom[lev - 1].Domain().size(); + create_constrained_umac_grown( + m_nGrowMAC, &geom[lev - 1], &geom[lev], + GetArrOfPtrs(advData->umac[lev - 1]), GetArrOfPtrs(advData->umac[lev]), + rr); } } void PeleLM::create_constrained_umac_grown( - int a_nGrow, + const int a_nGrow, const amrex::Geometry* crse_geom, const amrex::Geometry* fine_geom, - amrex::Array u_mac_crse, - amrex::Array u_mac_fine, + const amrex::Array& u_mac_crse, + const amrex::Array& u_mac_fine, const amrex::IntVect& crse_ratio) { // Divergence preserving interp @@ -271,7 +272,7 @@ PeleLM::create_constrained_umac_grown( // Set BCRec for Umac amrex::Vector bcrec(1); - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { if (crse_geom->isPeriodic(idim)) { bcrec[0].setLo(idim, amrex::BCType::int_dir); bcrec[0].setHi(idim, amrex::BCType::int_dir); @@ -299,7 +300,7 @@ PeleLM::create_constrained_umac_grown( // Use piecewise constant interpolation in time, so create dummy variable for // time - amrex::Real dummy = 0.; + constexpr amrex::Real dummy = 0.; FillPatchTwoLevels( u_mac_fine, amrex::IntVect(a_nGrow), dummy, {u_mac_crse}, {dummy}, {u_mac_fine}, {dummy}, 0, 0, 1, *crse_geom, *fine_geom, cbndyFuncArr, 0, diff --git a/Source/PeleLMeX_UserKeys.H b/Source/PeleLMeX_UserKeys.H index 097908005..69e23e5ba 100644 --- a/Source/PeleLMeX_UserKeys.H +++ b/Source/PeleLMeX_UserKeys.H @@ -11,7 +11,7 @@ struct BoundaryCondition { BoundaryCondition() = default; - enum { + enum : std::uint8_t { BCInterior = 0, BCInflow, BCOutflow, @@ -41,7 +41,7 @@ const BoundaryCondition boundarycondition; struct NSSolver { NSSolver() = default; - enum { LowMachNumber = 0, Incompressible }; + enum : std::uint8_t { LowMachNumber = 0, Incompressible }; const std::map str2int = { {"lowmachnumber", LowMachNumber}, {"incompressible", Incompressible}, @@ -57,7 +57,7 @@ const NSSolver nssolver; struct Interpolator { Interpolator() = default; - enum { + enum : std::uint8_t { PiecewiseLinearConserv = 0, PiecewiseLinearConservMinMax, PiecewiseConstant @@ -78,7 +78,7 @@ const Interpolator interpolator; struct LESModel { LESModel() = default; - enum { None = 0, Smagorinsky, WALE, Sigma }; + enum : std::uint8_t { None = 0, Smagorinsky, WALE, Sigma }; const std::map str2int = { {"none", None}, {"smagorinsky", Smagorinsky}, @@ -96,7 +96,7 @@ const LESModel lesmodel; struct AdvectionScheme { AdvectionScheme() = default; - enum { + enum : std::uint8_t { Godunov_PLM = 0, Godunov_PPM, Godunov_PPM_WENOZ, @@ -121,7 +121,7 @@ const AdvectionScheme advscheme; struct LoadBalanceCost { LoadBalanceCost() = default; - enum { + enum : std::uint8_t { Ncell = 0, ChemFunctCallAvg, ChemFunctCallMax, @@ -150,7 +150,7 @@ const LoadBalanceCost lbcost; struct LoadBalanceMethod { LoadBalanceMethod() = default; - enum { SFC = 0, Knapsack, RoundRobin }; + enum : std::uint8_t { SFC = 0, Knapsack, RoundRobin }; const std::map str2int = { {"sfc", SFC}, {"knapsack", Knapsack}, @@ -168,7 +168,7 @@ const LoadBalanceMethod lbmethod; struct ChiCorrectionType { ChiCorrectionType() = default; - enum { DivuEveryIter = 0, DivuFirstIter, NoDivu }; + enum : std::uint8_t { DivuEveryIter = 0, DivuFirstIter, NoDivu }; const std::map str2int = { {"divueveryiter", DivuEveryIter}, {"divufirstiter", DivuFirstIter}, diff --git a/Source/PeleLMeX_Utils.cpp b/Source/PeleLMeX_Utils.cpp index 0b5c26afc..7a9d0a79e 100644 --- a/Source/PeleLMeX_Utils.cpp +++ b/Source/PeleLMeX_Utils.cpp @@ -37,7 +37,7 @@ writeBuildInfo() std::cout << "\n"; - for (int n = 1; n <= amrex::buildInfoGetNumModules(); n++) { + for (int n = 1; n <= amrex::buildInfoGetNumModules(); ++n) { std::cout << amrex::buildInfoGetModuleName(n) << ": " << amrex::buildInfoGetModuleVal(n) << "\n"; } @@ -70,11 +70,11 @@ writeBuildInfo() void PeleLM::fluxDivergence( const amrex::Vector& a_divergence, - int div_comp, + const int div_comp, const amrex::Vector>& a_fluxes, - int flux_comp, - int ncomp, - int intensiveFluxes, + const int flux_comp, + const int ncomp, + const int intensiveFluxes, amrex::Real scale) { BL_PROFILE("PeleLMeX::fluxDivergence()"); @@ -97,14 +97,14 @@ PeleLM::fluxDivergence( void PeleLM::fluxDivergence( const amrex::Vector& a_divergence, - int div_comp, + const int div_comp, const amrex::Vector>& a_fluxes, - int flux_comp, + const int flux_comp, const amrex::Vector& a_EBfluxes, - int ebflux_comp, - int ncomp, - int intensiveFluxes, - amrex::Real scale) + const int ebflux_comp, + const int ncomp, + const int intensiveFluxes, + const amrex::Real scale) { BL_PROFILE("PeleLMeX::fluxDivergence()"); @@ -127,26 +127,26 @@ PeleLM::fluxDivergence( void PeleLM::fluxDivergenceRD( const amrex::Vector& a_state, - int state_comp, + const int state_comp, const amrex::Vector& a_divergence, - int div_comp, + const int div_comp, const amrex::Vector>& a_fluxes, - int flux_comp, + const int flux_comp, const amrex::Vector& a_EBfluxes, - int ebflux_comp, - int ncomp, - int intensiveFluxes, + const int ebflux_comp, + const int ncomp, + const int intensiveFluxes, const amrex::BCRec* state_bc_d, const amrex::Real& scale, const amrex::Real& a_dt) { BL_PROFILE("PeleLMeX::fluxDivergenceRD()"); #ifdef AMREX_USE_EB - int have_ebfluxes = (a_EBfluxes.empty()) ? 0 : 1; + const int have_ebfluxes = (a_EBfluxes.empty()) ? 0 : 1; for (int lev = 0; lev <= finest_level; ++lev) { //---------------------------------------------------------------- // Use a temporary MF to hold divergence before redistribution - int nGrow_divTmp = 3; + constexpr int nGrow_divTmp = 3; amrex::MultiFab divTmp( grids[lev], dmap[lev], ncomp, nGrow_divTmp, amrex::MFInfo(), EBFactory(lev)); @@ -188,13 +188,13 @@ PeleLM::fluxDivergenceRD( void PeleLM::extFluxDivergenceLevel( - int lev, + const int lev, amrex::MultiFab& a_divergence, - int div_comp, + const int div_comp, const amrex::Array& a_fluxes, - int flux_comp, - int ncomp, - amrex::Real scale) + const int flux_comp, + const int ncomp, + const amrex::Real scale) { AMREX_ASSERT(a_divergence.nComp() >= div_comp + ncomp); @@ -219,16 +219,14 @@ PeleLM::extFluxDivergenceLevel( , auto const& fluxZ = a_fluxes[2]->const_array(mfi, flux_comp);); auto const& divergence = a_divergence.array(mfi, div_comp); auto const& vol = volume.const_array(mfi); - #ifdef AMREX_USE_EB auto const& flagfab = ebfact.getMultiEBCellFlagFab()[mfi]; auto const& flag = flagfab.const_array(); -#endif -#ifdef AMREX_USE_EB if (flagfab.getType(bx) == amrex::FabType::covered) { // Covered boxes amrex::ParallelFor( - bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + bx, ncomp, + [divergence] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { divergence(i, j, k, n) = 0.0; }); } else if (flagfab.getType(bx) != amrex::FabType::regular) { // EB @@ -236,22 +234,25 @@ PeleLM::extFluxDivergenceLevel( // boxes auto vfrac = ebfact.getVolFrac().const_array(mfi); amrex::ParallelFor( - bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - if (flag(i, j, k).isCovered()) { - for (int n = 0; n < ncomp; n++) { + bx, [flag, divergence, ncomp, vol, scale, vfrac, fluxX, fluxY +#if AMREX_SPACEDIM == 3 + , + fluxZ +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + if (flag(i, j, k).isCovered()) { // Covered cell + for (int n = 0; n < ncomp; ++n) { divergence(i, j, k, n) = 0.0; } - } else if (flag(i, j, k).isRegular()) { + } else { // Regular or cut cell extFluxDivergence_K( i, j, k, ncomp, AMREX_D_DECL(fluxX, fluxY, fluxZ), vol, scale, divergence); - } else { - amrex::Real vfracinv = 1.0 / vfrac(i, j, k); - extFluxDivergence_K( - i, j, k, ncomp, AMREX_D_DECL(fluxX, fluxY, fluxZ), vol, scale, - divergence); - for (int n = 0; n < ncomp; n++) { - divergence(i, j, k, n) *= vfracinv; + if (!flag(i, j, k).isRegular()) { // Cut cell + const amrex::Real vfracinv = 1.0 / vfrac(i, j, k); + for (int n = 0; n < ncomp; ++n) { + divergence(i, j, k, n) *= vfracinv; + } } } }); @@ -259,7 +260,12 @@ PeleLM::extFluxDivergenceLevel( #endif { amrex::ParallelFor( - bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [ncomp, vol, scale, divergence, fluxX, fluxY +#if AMREX_SPACEDIM == 3 + , + fluxZ +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { extFluxDivergence_K( i, j, k, ncomp, AMREX_D_DECL(fluxX, fluxY, fluxZ), vol, scale, divergence); @@ -270,13 +276,13 @@ PeleLM::extFluxDivergenceLevel( void PeleLM::intFluxDivergenceLevel( - int lev, + const int lev, amrex::MultiFab& a_divergence, - int div_comp, + const int div_comp, const amrex::Array& a_fluxes, - int flux_comp, - int ncomp, - amrex::Real scale) + const int flux_comp, + const int ncomp, + const amrex::Real scale) { AMREX_ASSERT(a_divergence.nComp() >= div_comp + ncomp); @@ -327,7 +333,8 @@ PeleLM::intFluxDivergenceLevel( if (flagfab.getType(bx) == amrex::FabType::covered) { // Covered boxes amrex::ParallelFor( - bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + bx, ncomp, + [divergence] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { divergence(i, j, k, n) = 0.0; }); } else if (flagfab.getType(bx) != amrex::FabType::regular) { // EB @@ -339,9 +346,15 @@ PeleLM::intFluxDivergenceLevel( , const auto& afrac_y = areafrac[1]->array(mfi); , const auto& afrac_z = areafrac[2]->array(mfi);); amrex::ParallelFor( - bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - if (flag(i, j, k).isCovered()) { - for (int n = 0; n < ncomp; n++) { + bx, [flag, divergence, ncomp, vol, scale, vfrac, fluxX, areax, afrac_x, + fluxY, areay, afrac_y +#if (AMREX_SPACEDIM == 3) + , + fluxZ, areaz, afrac_z +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + if (flag(i, j, k).isCovered()) { // Covered cells + for (int n = 0; n < ncomp; ++n) { divergence(i, j, k, n) = 0.0; } } else if (flag(i, j, k).isRegular()) { @@ -349,12 +362,12 @@ PeleLM::intFluxDivergenceLevel( i, j, k, ncomp, AMREX_D_DECL(fluxX, fluxY, fluxZ), AMREX_D_DECL(areax, areay, areaz), vol, scale, divergence); } else { - amrex::Real vfracinv = 1.0 / vfrac(i, j, k); + const amrex::Real vfracinv = 1.0 / vfrac(i, j, k); EB_intFluxDivergence_K( i, j, k, ncomp, AMREX_D_DECL(fluxX, fluxY, fluxZ), AMREX_D_DECL(afrac_x, afrac_y, afrac_z), AMREX_D_DECL(areax, areay, areaz), vol, scale, divergence); - for (int n = 0; n < ncomp; n++) { + for (int n = 0; n < ncomp; ++n) { divergence(i, j, k, n) *= vfracinv; } } @@ -367,16 +380,21 @@ PeleLM::intFluxDivergenceLevel( amrex::Array4 const& ax = mf_ax.const_array(mfi); amrex::Array4 const& ay = mf_ay.const_array(mfi); amrex::ParallelFor( - bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [ncomp, ax, ay, fluxX, fluxY, vol, scale, + divergence] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { intFluxDivergence_rz_K( - i, j, k, ncomp, AMREX_D_DECL(fluxX, fluxY, fluxZ), ax, ay, vol, - scale, divergence); + i, j, k, ncomp, fluxX, fluxY, ax, ay, vol, scale, divergence); }); } else #endif { amrex::ParallelFor( - bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [ncomp, vol, scale, divergence, fluxX, areax, fluxY, areay +#if (AMREX_SPACEDIM == 3) + , + fluxZ, areaz +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { intFluxDivergence_K( i, j, k, ncomp, AMREX_D_DECL(fluxX, fluxY, fluxZ), AMREX_D_DECL(areax, areay, areaz), vol, scale, divergence); @@ -388,15 +406,15 @@ PeleLM::intFluxDivergenceLevel( void PeleLM::intFluxDivergenceLevelEB( - int lev, + const int lev, amrex::MultiFab& a_divergence, - int div_comp, + const int div_comp, const amrex::Array& a_fluxes, - int flux_comp, + const int flux_comp, const amrex::MultiFab* a_EBfluxes, - int ebflux_comp, - int ncomp, - amrex::Real scale) + const int ebflux_comp, + const int ncomp, + const amrex::Real scale) { AMREX_ASSERT(a_divergence.nComp() >= div_comp + ncomp); @@ -447,7 +465,8 @@ PeleLM::intFluxDivergenceLevelEB( if (flagfab.getType(bx) == amrex::FabType::covered) { // Covered boxes amrex::ParallelFor( - bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + bx, ncomp, + [divergence] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { divergence(i, j, k, n) = 0.0; }); } else if (flagfab.getType(bx) != amrex::FabType::regular) { // EB @@ -460,9 +479,15 @@ PeleLM::intFluxDivergenceLevelEB( , const auto& afrac_z = areafrac[2]->array(mfi);); const auto& ebarea = eb_area->array(mfi); amrex::ParallelFor( - bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [flag, divergence, vol, scale, ncomp, ebflux, ebarea, dx, vfrac, + fluxX, areax, afrac_x, fluxY, areay, afrac_y +#if (AMREX_SPACEDIM == 3) + , + fluxZ, areaz, afrac_z +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (flag(i, j, k).isCovered()) { - for (int n = 0; n < ncomp; n++) { + for (int n = 0; n < ncomp; ++n) { divergence(i, j, k, n) = 0.0; } } else if (flag(i, j, k).isRegular()) { @@ -470,13 +495,13 @@ PeleLM::intFluxDivergenceLevelEB( i, j, k, ncomp, AMREX_D_DECL(fluxX, fluxY, fluxZ), AMREX_D_DECL(areax, areay, areaz), vol, scale, divergence); } else { - amrex::Real vfracinv = 1.0 / vfrac(i, j, k); + const amrex::Real vfracinv = 1.0 / vfrac(i, j, k); EB_intFluxDivergence_K( i, j, k, ncomp, AMREX_D_DECL(fluxX, fluxY, fluxZ), AMREX_D_DECL(afrac_x, afrac_y, afrac_z), AMREX_D_DECL(areax, areay, areaz), ebflux, ebarea, vol, dx[0], scale, divergence); - for (int n = 0; n < ncomp; n++) { + for (int n = 0; n < ncomp; ++n) { divergence(i, j, k, n) *= vfracinv; } } @@ -485,7 +510,12 @@ PeleLM::intFluxDivergenceLevelEB( #endif { amrex::ParallelFor( - bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [ncomp, vol, scale, divergence, fluxX, areax, fluxY, areay +#if (AMREX_SPACEDIM == 3) + , + fluxZ, areaz +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { intFluxDivergence_K( i, j, k, ncomp, AMREX_D_DECL(fluxX, fluxY, fluxZ), AMREX_D_DECL(areax, areay, areaz), vol, scale, divergence); @@ -495,21 +525,20 @@ PeleLM::intFluxDivergenceLevelEB( } void -PeleLM:: - advFluxDivergence( // NOLINT(readability-convert-member-functions-to-static) - int a_lev, - amrex::MultiFab& a_divergence, - int div_comp, - amrex::MultiFab& a_divu, - const amrex::Array& a_fluxes, - int flux_comp, - const amrex::Array& a_faceState, - int face_comp, - int ncomp, - int const* l_conserv_d, - const amrex::Geometry& a_geom, - amrex::Real scale, - bool fluxes_are_area_weighted) const +PeleLM::advFluxDivergence( + const int a_lev, + amrex::MultiFab& a_divergence, + const int div_comp, + amrex::MultiFab& a_divu, + const amrex::Array& a_fluxes, + const int flux_comp, + const amrex::Array& a_faceState, + const int face_comp, + const int ncomp, + int const* l_conserv_d, + const amrex::Geometry& a_geom, + const amrex::Real scale, + const bool fluxes_are_area_weighted) const { BL_PROFILE("PeleLMeX::advFluxDivergence()"); @@ -579,16 +608,23 @@ PeleLM:: , auto const& apy_arr = ebfact.getAreaFrac()[1]->const_array(mfi); , auto const& apz_arr = ebfact.getAreaFrac()[2]->const_array(mfi);); amrex::ParallelFor( - bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + bx, ncomp, + [l_conserv_d, vfrac_arr, div_arr, divu_arr, apx_arr, facex, apy_arr, + facey +#if (AMREX_SPACEDIM == 3) + , + apz_arr, facez +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { if ((l_conserv_d[n] == 0) && vfrac_arr(i, j, k) > 0.) { - amrex::Real qwsum = AMREX_D_TERM( + const amrex::Real qwsum = AMREX_D_TERM( apx_arr(i, j, k) * facex(i, j, k, n) + apx_arr(i + 1, j, k) * facex(i + 1, j, k, n), +apy_arr(i, j, k) * facey(i, j, k, n) + apy_arr(i, j + 1, k) * facey(i, j + 1, k, n), +apz_arr(i, j, k) * facez(i, j, k, n) + apz_arr(i, j, k + 1) * facez(i, j, k + 1, n)); - amrex::Real areasum = AMREX_D_TERM( + const amrex::Real areasum = AMREX_D_TERM( apx_arr(i, j, k) + apx_arr(i + 1, j, k), +apy_arr(i, j, k) + apy_arr(i, j + 1, k), +apz_arr(i, j, k) + apz_arr(i, j, k + 1)); @@ -605,15 +641,16 @@ PeleLM:: amrex::Array4 const& ax = mf_ax.const_array(mfi); amrex::Array4 const& ay = mf_ay.const_array(mfi); amrex::ParallelFor( - bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + bx, ncomp, + [l_conserv_d, ax, ay, facex, facey, divu_arr, + div_arr] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { if (l_conserv_d[n] == 0) { amrex::Real qavg = AMREX_D_TERM( ax(i, j, k) * facex(i, j, k, n) + ax(i + 1, j, k) * facex(i + 1, j, k, n), +ay(i, j, k) * facey(i, j, k, n) + - ay(i, j + 1, k) * facey(i, j + 1, k, n), - +0.0); - amrex::Real areasum = + ay(i, j + 1, k) * facey(i, j + 1, k, n), ); + const amrex::Real areasum = ax(i, j, k) + ax(i + 1, j, k) + ay(i, j, k) + ay(i, j + 1, k); qavg /= areasum; // Note that because we define adv update as MINUS div(u q), here @@ -625,7 +662,13 @@ PeleLM:: #endif { amrex::ParallelFor( - bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + bx, ncomp, + [l_conserv_d, div_arr, divu_arr, facex, facey +#if (AMREX_SPACEDIM == 3) + , + facez +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { if (l_conserv_d[n] == 0) { amrex::Real qavg = AMREX_D_TERM( facex(i, j, k, n) + facex(i + 1, j, k, n), @@ -645,21 +688,21 @@ PeleLM:: #ifdef AMREX_USE_EB void PeleLM::advFluxDivergence( - int a_lev, + const int a_lev, amrex::MultiFab& a_divergence, - int div_comp, + const int div_comp, amrex::MultiFab& a_divu, const amrex::Array& a_fluxes, - int flux_comp, + const int flux_comp, const amrex::Array& a_faceState, - int face_comp, + const int face_comp, const amrex::MultiFab* a_EBvelocity, const amrex::MultiFab* a_EBvalue, - int ncomp, + const int ncomp, int const* l_conserv_d, const amrex::Geometry& a_geom, - amrex::Real scale, - bool fluxes_are_area_weighted) const + const amrex::Real scale, + const bool fluxes_are_area_weighted) const { BL_PROFILE("PeleLM::advFluxDivergence()"); @@ -718,16 +761,23 @@ PeleLM::advFluxDivergence( , auto const& apy_arr = ebfact.getAreaFrac()[1]->const_array(mfi); , auto const& apz_arr = ebfact.getAreaFrac()[2]->const_array(mfi);); amrex::ParallelFor( - bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + bx, ncomp, + [l_conserv_d, vfrac_arr, div_arr, divu_arr, facex, apx_arr, facey, + apy_arr +#if (AMREX_SPACEDIM == 3) + , + facez, apz_arr +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { if (l_conserv_d[n] == 0 && vfrac_arr(i, j, k) > 0.) { - amrex::Real qwsum = AMREX_D_TERM( + const amrex::Real qwsum = AMREX_D_TERM( apx_arr(i, j, k) * facex(i, j, k, n) + apx_arr(i + 1, j, k) * facex(i + 1, j, k, n), +apy_arr(i, j, k) * facey(i, j, k, n) + apy_arr(i, j + 1, k) * facey(i, j + 1, k, n), +apz_arr(i, j, k) * facez(i, j, k, n) + apz_arr(i, j, k + 1) * facez(i, j, k + 1, n)); - amrex::Real areasum = AMREX_D_TERM( + const amrex::Real areasum = AMREX_D_TERM( apx_arr(i, j, k) + apx_arr(i + 1, j, k), +apy_arr(i, j, k) + apy_arr(i, j + 1, k), +apz_arr(i, j, k) + apz_arr(i, j, k + 1)); @@ -738,7 +788,13 @@ PeleLM::advFluxDivergence( }); } else { amrex::ParallelFor( - bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + bx, ncomp, + [l_conserv_d, div_arr, divu_arr, facex, facey +#if (AMREX_SPACEDIM == 3) + , + facez +#endif + ] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { if (l_conserv_d[n] == 0) { amrex::Real qavg = AMREX_D_TERM( facex(i, j, k, n) + facex(i + 1, j, k, n), @@ -756,7 +812,7 @@ PeleLM::advFluxDivergence( #endif void -PeleLM::floorSpecies(const TimeStamp& a_time) +PeleLM::floorSpecies(const TimeStamp a_time) { BL_PROFILE("PeleLMeX::floorSpecies()"); AMREX_ASSERT(a_time == AmrOldTime || a_time == AmrNewTime); @@ -771,8 +827,8 @@ PeleLM::floorSpecies(const TimeStamp& a_time) auto const* leosparm = eos_parms.device_parm(); amrex::ParallelFor( - ldata_p->state, - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { + ldata_p->state, [sma, leosparm] AMREX_GPU_DEVICE( + int box_no, int i, int j, int k) noexcept { fabMinMax( i, j, k, NUM_SPECIES, 0.0, AMREX_REAL_MAX, amrex::Array4(sma[box_no], FIRSTSPEC)); @@ -785,7 +841,7 @@ PeleLM::floorSpecies(const TimeStamp& a_time) amrex::Real massfrac[NUM_SPECIES] = {0.0}; amrex::Real massdens[NUM_SPECIES] = {0.0}; amrex::Real rhoinv, h_cgs = 0.0; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { massdens[n] = sma[box_no](i, j, k, FIRSTSPEC + n); } auto eos = pele::physics::PhysicsType::eos(leosparm); @@ -795,6 +851,7 @@ PeleLM::floorSpecies(const TimeStamp& a_time) sma[box_no](i, j, k, RHOH) = h_cgs * 1.0e-4 * sma[box_no](i, j, k, DENSITY); }); + // Shift outside? amrex::Gpu::streamSynchronize(); } } @@ -825,7 +882,7 @@ PeleLM::resetCoveredMask() baf.intersections(grids[lev][mfi.index()], isects); for (const auto& is : isects) { amrex::ParallelFor( - is.second, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + is.second, [mask] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { mask(i, j, k) = 0; }); } @@ -899,9 +956,10 @@ PeleLM::resetCoveredMask() //---------------------------------------------------------------------------- // Need to compute the uncovered volume if (m_uncoveredVol < 0.0) { - amrex::Vector dummy(finest_level + 1); + amrex::Vector dummy; + dummy.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - dummy[lev].define( + dummy.emplace_back( grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), *m_factory[lev]); dummy[lev].setVal(1.0); } @@ -924,7 +982,7 @@ PeleLM::loadBalanceChem() } void -PeleLM::loadBalanceChemLev(int a_lev) +PeleLM::loadBalanceChemLev(const int a_lev) { amrex::LayoutData new_cost(*m_baChem[a_lev], *m_dmapChem[a_lev]); computeCosts(a_lev, new_cost, m_loadBalanceCostChem); @@ -936,7 +994,6 @@ PeleLM::loadBalanceChemLev(int a_lev) amrex::DistributionMapping test_dmap; // Build the test dmap, w/o braodcasting if (m_loadBalanceMethodChem == LoadBalanceMethod::SFC) { - test_dmap = amrex::DistributionMapping::makeSFC( new_cost, currentEfficiency, testEfficiency, false, amrex::ParallelDescriptor::IOProcessorNumber()); @@ -998,7 +1055,10 @@ PeleLM::loadBalanceChemLev(int a_lev) // Return a unique_ptr with the entire derive std::unique_ptr PeleLM::derive( - const std::string& a_name, amrex::Real a_time, int lev, int nGrow) + const std::string& a_name, + const amrex::Real a_time, + const int lev, + const int nGrow) { BL_PROFILE("PeleLMeX::derive()"); AMREX_ASSERT(nGrow >= 0); @@ -1065,7 +1125,10 @@ PeleLM::derive( // Return a unique_ptr with only the required component of a derive std::unique_ptr PeleLM::deriveComp( - const std::string& a_name, amrex::Real a_time, int lev, int nGrow) + const std::string& a_name, + const amrex::Real a_time, + const int lev, + const int nGrow) { BL_PROFILE("PeleLMeX::derive()"); AMREX_ASSERT(nGrow >= 0); @@ -1225,7 +1288,7 @@ PeleLM::parseVars( // For each entry in the user-provided composition, parse name and value std::string delimiter = ":"; - for (int i = 0; i < varCountIn; i++) { + for (int i = 0; i < varCountIn; ++i) { long unsigned sep = a_stringIn[i].find(delimiter); if (sep == std::string::npos) { amrex::Abort( @@ -1235,7 +1298,7 @@ PeleLM::parseVars( amrex::Real value = std::stod(a_stringIn[i].substr(sep + 1, a_stringIn[i].length())); int foundIt = 0; - for (int k = 0; k < a_varsNames.size(); k++) { + for (int k = 0; k < a_varsNames.size(); ++k) { if (varNameIn == a_varsNames[k]) { a_rVars[k] = value; foundIt = 1; @@ -1267,22 +1330,24 @@ PeleLM::MLNorm0(const amrex::Vector& a_MF) amrex::Vector PeleLM::MLNorm0( - const amrex::Vector& a_MF, int startcomp, int ncomp) + const amrex::Vector& a_MF, + const int startcomp, + const int ncomp) { BL_PROFILE("PeleLMeX::MLNorm0()"); AMREX_ASSERT(a_MF[0]->nComp() >= startcomp + ncomp); amrex::Vector r(ncomp); - for (int n = 0; n < ncomp; n++) { + for (int n = 0; n < ncomp; ++n) { r[n] = 0.0; } for (int lev = 0; lev < a_MF.size(); ++lev) { if (lev != finest_level) { - for (int n = 0; n < ncomp; n++) { + for (int n = 0; n < ncomp; ++n) { r[n] = amrex::max( r[n], a_MF[lev]->norm0(*m_coveredMask[lev], startcomp + n, 0, true)); } } else { - for (int n = 0; n < ncomp; n++) { + for (int n = 0; n < ncomp; ++n) { r[n] = amrex::max(r[n], a_MF[lev]->norm0(startcomp + n, 0, true, true)); } } @@ -1297,7 +1362,7 @@ PeleLM::isStateVariable(std::string_view a_name) // Check state return std::any_of( stateComponents.begin(), stateComponents.end(), - [=](const auto& stateComponent) { + [a_name](const auto& stateComponent) { return std::get<1>(stateComponent) == a_name; }); } @@ -1308,7 +1373,7 @@ PeleLM::isReactVariable(std::string_view a_name) // Check reaction state return std::any_of( reactComponents.begin(), reactComponents.end(), - [=](const auto& reactComponent) { + [a_name](const auto& reactComponent) { return std::get<1>(reactComponent) == a_name; }); } @@ -1369,7 +1434,7 @@ amrex::Vector PeleLM::fetchAdvTypeArray(int scomp, int ncomp) { amrex::Vector types(ncomp); - for (int comp = 0; comp < ncomp; comp++) { + for (int comp = 0; comp < ncomp; ++comp) { types[comp] = m_AdvTypeState[scomp + comp]; } return types; @@ -1379,7 +1444,7 @@ amrex::Vector PeleLM::fetchDiffTypeArray(int scomp, int ncomp) { amrex::Vector types(ncomp); - for (int comp = 0; comp < ncomp; comp++) { + for (int comp = 0; comp < ncomp; ++comp) { types[comp] = m_DiffTypeState[scomp + comp]; } return types; @@ -1389,7 +1454,7 @@ amrex::Vector PeleLM::fetchAdvTypeAuxArray(int scomp, int ncomp) { amrex::Vector types(ncomp); - for (int comp = 0; comp < ncomp; comp++) { + for (int comp = 0; comp < ncomp; ++comp) { types[comp] = m_AdvTypeAux[scomp + comp]; } return types; @@ -1399,14 +1464,14 @@ amrex::Vector PeleLM::fetchDiffTypeAuxArray(int scomp, int ncomp) { amrex::Vector types(ncomp); - for (int comp = 0; comp < ncomp; comp++) { + for (int comp = 0; comp < ncomp; ++comp) { types[comp] = m_DiffTypeAux[scomp + comp]; } return types; } amrex::Real -PeleLM::MFSum(const amrex::Vector& a_mf, int comp) +PeleLM::MFSum(const amrex::Vector& a_mf, const int comp) { BL_PROFILE("PeleLMeX::MFSum()"); // Get the integral of the MF, not including the fine-covered and @@ -1505,7 +1570,7 @@ cells */ void -PeleLM::setTypicalValues(const TimeStamp& a_time, int is_init) +PeleLM::setTypicalValues(const TimeStamp a_time, const int is_init) { // Get state Max/Min auto stateMax = @@ -1518,7 +1583,7 @@ PeleLM::setTypicalValues(const TimeStamp& a_time, int is_init) : MLmin(GetVecOfConstPtrs(getStateVect(a_time)), 0, NVAR); // Fill typical values vector - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { typical_values[idim] = amrex::max(stateMax[VELX + idim], std::abs(stateMin[VELX + idim])); } @@ -1526,7 +1591,7 @@ PeleLM::setTypicalValues(const TimeStamp& a_time, int is_init) if (m_incompressible == 0) { // Average between max/min typical_values[DENSITY] = 0.5 * (stateMax[DENSITY] + stateMin[DENSITY]); - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { typical_values[FIRSTSPEC + n] = 0.5 * (stateMax[FIRSTSPEC + n] + stateMin[FIRSTSPEC + n]) / typical_values[DENSITY]; @@ -1539,7 +1604,7 @@ PeleLM::setTypicalValues(const TimeStamp& a_time, int is_init) typical_values[NE] = 0.5 * (stateMax[NE] + stateMin[NE]); #endif #if NUM_ODE > 0 - for (int n = 0; n < NUM_ODE; n++) { + for (int n = 0; n < NUM_ODE; ++n) { typical_values[FIRSTODE + n] = 0.5 * (stateMax[FIRSTODE + n] + stateMin[FIRSTODE + n]); } @@ -1564,7 +1629,7 @@ PeleLM::setTypicalValues(const TimeStamp& a_time, int is_init) amrex::Vector spec_names; pele::physics::eos::speciesNames( spec_names, &(eos_parms.host_parm())); - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { amrex::Print() << "\tY_" << spec_names[n] << std::setw( amrex::max( @@ -1576,7 +1641,7 @@ PeleLM::setTypicalValues(const TimeStamp& a_time, int is_init) amrex::Print() << "\tnE: " << typical_values[NE] << '\n'; #endif #if NUM_ODE > 0 - for (int n = 0; n < NUM_ODE; n++) { + for (int n = 0; n < NUM_ODE; ++n) { amrex::Print() << "\t" << m_ode_names[n] << std::setw( @@ -1625,7 +1690,7 @@ PeleLM::updateTypicalValuesChem() // amrex::MultiFab max, excluding EB-covered/fine-covered cells, local amrex::Real PeleLM::MFmax( - const amrex::MultiFab* a_MF, const amrex::iMultiFab& a_mask, int comp) + const amrex::MultiFab* a_MF, const amrex::iMultiFab& a_mask, const int comp) { BL_PROFILE("PeleLMeX::MFmax()"); amrex::Real mx = std::numeric_limits::lowest(); @@ -1643,8 +1708,9 @@ PeleLM::MFmax( mx = ParReduce( amrex::TypeList{}, amrex::TypeList{}, *a_MF, amrex::IntVect(0), - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept - -> amrex::GpuTuple { + [flagsma, mask, ma, comp] AMREX_GPU_DEVICE( + int box_no, int i, int j, + int k) noexcept -> amrex::GpuTuple { if (flagsma[box_no](i, j, k).isCovered() || !mask[box_no](i, j, k)) { return AMREX_REAL_LOWEST; } else { @@ -1681,8 +1747,9 @@ PeleLM::MFmax( mx = ParReduce( amrex::TypeList{}, amrex::TypeList{}, *a_MF, amrex::IntVect(0), - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept - -> amrex::GpuTuple { + [mask, ma, comp] AMREX_GPU_DEVICE( + int box_no, int i, int j, + int k) noexcept -> amrex::GpuTuple { if (!mask[box_no](i, j, k)) { return AMREX_REAL_LOWEST; } else { @@ -1714,7 +1781,7 @@ PeleLM::MFmax( // amrex::MultiFab min, excluding EB-covered/fine-covered cells, local amrex::Real PeleLM::MFmin( - const amrex::MultiFab* a_MF, const amrex::iMultiFab& a_mask, int comp) + const amrex::MultiFab* a_MF, const amrex::iMultiFab& a_mask, const int comp) { BL_PROFILE("PeleLMeX::MFmin()"); amrex::Real mn = std::numeric_limits::max(); @@ -1732,8 +1799,9 @@ PeleLM::MFmin( mn = ParReduce( amrex::TypeList{}, amrex::TypeList{}, *a_MF, amrex::IntVect(0), - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept - -> amrex::GpuTuple { + [flagsma, mask, ma, comp] AMREX_GPU_DEVICE( + int box_no, int i, int j, + int k) noexcept -> amrex::GpuTuple { if (flagsma[box_no](i, j, k).isCovered() || !mask[box_no](i, j, k)) { return AMREX_REAL_MAX; } else { @@ -1770,8 +1838,9 @@ PeleLM::MFmin( mn = ParReduce( amrex::TypeList{}, amrex::TypeList{}, *a_MF, amrex::IntVect(0), - [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept - -> amrex::GpuTuple { + [mask, ma, comp] AMREX_GPU_DEVICE( + int box_no, int i, int j, + int k) noexcept -> amrex::GpuTuple { if (!mask[box_no](i, j, k)) { return AMREX_REAL_MAX; } else { @@ -1803,7 +1872,9 @@ PeleLM::MFmin( // MultiLevel max, exlucing EB-covered/fine-covered cells amrex::Vector PeleLM::MLmax( - const amrex::Vector& a_MF, int scomp, int ncomp) + const amrex::Vector& a_MF, + const int scomp, + const int ncomp) { BL_PROFILE("PeleLMeX::MLmax()"); AMREX_ASSERT(a_MF[0]->nComp() >= scomp + ncomp); @@ -1812,12 +1883,12 @@ PeleLM::MLmax( for (int lev = 0; lev < a_MF.size(); ++lev) { if (lev != finest_level) { - for (int n = 0; n < ncomp; n++) { + for (int n = 0; n < ncomp; ++n) { nmax[n] = amrex::max(nmax[n], MFmax(a_MF[lev], *m_coveredMask[lev], scomp + n)); } } else { - for (int n = 0; n < ncomp; n++) { + for (int n = 0; n < ncomp; ++n) { nmax[n] = amrex::max(nmax[n], a_MF[lev]->max(scomp + n, 0, true)); } } @@ -1830,7 +1901,9 @@ PeleLM::MLmax( // MultiLevel min, exlucing EB-covered/fine-covered cells amrex::Vector PeleLM::MLmin( - const amrex::Vector& a_MF, int scomp, int ncomp) + const amrex::Vector& a_MF, + const int scomp, + const int ncomp) { BL_PROFILE("PeleLMeX::MLmin()"); AMREX_ASSERT(a_MF[0]->nComp() >= scomp + ncomp); @@ -1839,12 +1912,12 @@ PeleLM::MLmin( for (int lev = 0; lev < a_MF.size(); ++lev) { if (lev != finest_level) { - for (int n = 0; n < ncomp; n++) { + for (int n = 0; n < ncomp; ++n) { nmin[n] = amrex::min(nmin[n], MFmin(a_MF[lev], *m_coveredMask[lev], scomp + n)); } } else { - for (int n = 0; n < ncomp; n++) { + for (int n = 0; n < ncomp; ++n) { nmin[n] = amrex::min(nmin[n], a_MF[lev]->min(scomp + n, 0, true)); } } @@ -2037,7 +2110,7 @@ PeleLM::initMixtureFraction() Zox = 0.0; for (int i = 0; i < NUM_SPECIES; ++i) { spec_Bilger_fact[i] = 0.0; - for (int k = 0; k < 4; k++) { + for (int k = 0; k < 4; ++k) { spec_Bilger_fact[i] += Beta_mix[k] * (ecompCHON[i * 4 + k] * atwCHON[k] / mwt[i]); } @@ -2048,8 +2121,8 @@ PeleLM::initMixtureFraction() void PeleLM::parseComposition( - amrex::Vector compositionIn, - std::string compositionType, + const amrex::Vector& compositionIn, + const std::string& compositionType, amrex::Real* massFrac) { amrex::Real compoIn[NUM_SPECIES] = {0.0}; @@ -2062,7 +2135,7 @@ PeleLM::parseComposition( // For each entry in the user-provided composition, parse name and value std::string delimiter = ":"; const int specCountIn = static_cast(compositionIn.size()); - for (int i = 0; i < specCountIn; i++) { + for (int i = 0; i < specCountIn; ++i) { long unsigned sep = compositionIn[i].find(delimiter); if (sep == std::string::npos) { amrex::Abort( @@ -2073,7 +2146,7 @@ PeleLM::parseComposition( amrex::Real value = std::stod(compositionIn[i].substr(sep + 1, compositionIn[i].length())); int foundIt = 0; - for (int k = 0; k < NUM_SPECIES; k++) { + for (int k = 0; k < NUM_SPECIES; ++k) { if (specNameIn == specNames[k]) { compoIn[k] = value; foundIt = 1; @@ -2097,7 +2170,7 @@ PeleLM::parseComposition( // Fill the massFrac array, convert from mole fraction if necessary if (compositionType == "mass") { // mass - for (int i = 0; i < NUM_SPECIES; i++) { + for (int i = 0; i < NUM_SPECIES; ++i) { massFrac[i] = compoIn[i]; } } else if (compositionType == "mole") { // mole @@ -2113,20 +2186,22 @@ PeleLM::parseComposition( // Extend the cell-centered based signed distance function void PeleLM::extendSignedDistance( - amrex::MultiFab* a_signDist, amrex::Real a_extendFactor) + amrex::MultiFab* a_signDist, const amrex::Real a_extendFactor) { BL_PROFILE("PeleLMeX::extendSignedDistance()"); // This is a not-so-pretty piece of code that'll take AMReX cell-averaged // signed distance and propagates it manually up to the point where we need to // have it for derefining. const auto geomdata = geom[0].data(); - amrex::Real maxSignedDist = a_signDist->max(0); + const amrex::Real maxSignedDist = a_signDist->max(0); const auto& ebfactory = dynamic_cast(a_signDist->Factory()); const auto& flags = ebfactory.getMultiEBCellFlagFab(); - int nGrowFac = flags.nGrow() + 1; + const int nGrowFac = flags.nGrow() + 1; // First set the region far away at the max value we need + const amrex::Real* dx = geomdata.CellSize(); + #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -2136,7 +2211,6 @@ PeleLM::extendSignedDistance( auto const& sd_cc = a_signDist->array(mfi); amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (sd_cc(i, j, k) >= maxSignedDist - 1e-12) { - const amrex::Real* dx = geomdata.CellSize(); sd_cc(i, j, k) = nGrowFac * dx[0] * a_extendFactor; } }); @@ -2145,8 +2219,8 @@ PeleLM::extendSignedDistance( // Iteratively compute the distance function in boxes, propagating across // boxes using ghost cells If needed, increase the number of loop to extend // the reach of the distance function - int nMaxLoop = 4; - for (int dloop = 1; dloop <= nMaxLoop; dloop++) { + constexpr int nMaxLoop = 4; + for (int dloop = 1; dloop <= nMaxLoop; ++dloop) { #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -2159,10 +2233,10 @@ PeleLM::extendSignedDistance( } auto const& sd_cc = a_signDist->array(mfi); amrex::ParallelFor( - bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bx, [gbx, dx, a_extendFactor, sd_cc, + maxSignedDist] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { const auto glo = amrex::lbound(gbx); const auto ghi = amrex::ubound(gbx); - const amrex::Real* dx = geomdata.CellSize(); amrex::Real extendedDist = dx[0] * a_extendFactor; if (sd_cc(i, j, k) >= maxSignedDist - 1e-12) { amrex::Real closestEBDist = 1e12; diff --git a/Source/Plasma/GMRES/MLGMRES.cpp b/Source/Plasma/GMRES/MLGMRES.cpp index 0b6efb3a8..c9c77b759 100644 --- a/Source/Plasma/GMRES/MLGMRES.cpp +++ b/Source/Plasma/GMRES/MLGMRES.cpp @@ -18,13 +18,13 @@ MLGMRESSolver::define(PeleLM* a_pelelm, const int a_nComp, const int a_nGrow) m_pelelm = a_pelelm; - int finest_level = m_pelelm->finestLevel(); + const int finest_level = m_pelelm->finestLevel(); // Resize level vector m_grids.resize(finest_level + 1); m_dmap.resize(finest_level + 1); - Ax.resize(finest_level + 1); - res.resize(finest_level + 1); + Ax.reserve(finest_level + 1); + res.reserve(finest_level + 1); // Grab stuff from AmrCore m_geom = m_pelelm->Geom(0, finest_level); @@ -38,14 +38,14 @@ MLGMRESSolver::define(PeleLM* a_pelelm, const int a_nComp, const int a_nGrow) // Build krylov base memory and work MFs KspBase.resize(m_krylovSize + 1); for (int n = 0; n <= m_krylovSize; ++n) { - KspBase[n].resize(finest_level + 1); - for (int lev = 0; lev <= finest_level; lev++) { - KspBase[n][lev].define(m_grids[lev], m_dmap[lev], m_nComp, m_nGrow); + KspBase[n].reserve(finest_level + 1); + for (int lev = 0; lev <= finest_level; ++lev) { + KspBase[n].emplace_back(m_grids[lev], m_dmap[lev], m_nComp, m_nGrow); } } - for (int lev = 0; lev <= finest_level; lev++) { - Ax[lev].define(m_grids[lev], m_dmap[lev], m_nComp, m_nGrow); - res[lev].define(m_grids[lev], m_dmap[lev], m_nComp, m_nGrow); + for (int lev = 0; lev <= finest_level; ++lev) { + Ax.emplace_back(m_grids[lev], m_dmap[lev], m_nComp, m_nGrow); + res.emplace_back(m_grids[lev], m_dmap[lev], m_nComp, m_nGrow); } // Work Reals @@ -340,7 +340,7 @@ MLGMRESSolver::computeMLNorm(const amrex::Vector& a_vec) if (m_norm != nullptr) { MEMBER_FUNC_PTR(*m_pelelm, m_norm)(a_vec, r); } else { - for (int comp = 0; comp < m_nComp; comp++) { + for (int comp = 0; comp < m_nComp; ++comp) { amrex::Real norm = 0.0; for (int lev = 0; lev <= a_vec.size(); ++lev) { norm += diff --git a/Source/Plasma/LinOps/AMReX_MLABecCecLap_3D_K.H b/Source/Plasma/LinOps/AMReX_MLABecCecLap_3D_K.H index 9677eda5f..f0c4e2f29 100644 --- a/Source/Plasma/LinOps/AMReX_MLABecCecLap_3D_K.H +++ b/Source/Plasma/LinOps/AMReX_MLABecCecLap_3D_K.H @@ -752,14 +752,14 @@ tridiagonal_solve( Real bet = b_ls(0); u_ls(0) = r_ls(0) / bet; - for (int i = 1; i <= ilen - 1; i++) { + for (int i = 1; i <= ilen - 1; ++i) { gam(i) = c_ls(i - 1) / bet; bet = b_ls(i) - a_ls(i) * gam(i); if (bet == 0) amrex::Abort(">>>TRIDIAG FAILED"); u_ls(i) = (r_ls(i) - a_ls(i) * u_ls(i - 1)) / bet; } - for (int i = ilen - 2; i >= 0; i--) { + for (int i = ilen - 2; i >= 0; --i) { u_ls(i) = u_ls(i) - gam(i + 1) * u_ls(i + 1); } } diff --git a/Source/Plasma/LinOps/AMReX_MLCellABecCecLap.cpp b/Source/Plasma/LinOps/AMReX_MLCellABecCecLap.cpp index 69af3c5e9..0e713c61f 100644 --- a/Source/Plasma/LinOps/AMReX_MLCellABecCecLap.cpp +++ b/Source/Plasma/LinOps/AMReX_MLCellABecCecLap.cpp @@ -80,7 +80,7 @@ MLCellABecCecLap::define( Array4 const& cmsk = crse->array(mfi); reduce_op.eval( bx, reduce_data, - [=] AMREX_GPU_HOST_DEVICE(Box const& b) -> ReduceTuple { + [cmsk, fmsk] AMREX_GPU_HOST_DEVICE(Box const& b) -> ReduceTuple { return {coarsen_overset_mask(b, cmsk, fmsk)}; }); } diff --git a/Source/Plasma/PeleLMeX_EFDeriveFunc.cpp b/Source/Plasma/PeleLMeX_EFDeriveFunc.cpp index ddca235a0..bbe4be7e3 100644 --- a/Source/Plasma/PeleLMeX_EFDeriveFunc.cpp +++ b/Source/Plasma/PeleLMeX_EFDeriveFunc.cpp @@ -29,13 +29,14 @@ pelelmex_derchargedist( amrex::GpuArray zk; pele::physics::eos::charge_mass(zk.arr); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - der(i, j, k) = -nE(i, j, k) * elemCharge; - for (int n = 0; n < NUM_SPECIES; ++n) { - der(i, j, k) += - zk[n] * 1000.0 * rhoY(i, j, k, n); // CGS->MKS conversion of zk - } - }); + amrex::ParallelFor( + bx, [der, nE, zk, rhoY] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + der(i, j, k) = -nE(i, j, k) * elemCharge; + for (int n = 0; n < NUM_SPECIES; ++n) { + der(i, j, k) += + zk[n] * 1000.0 * rhoY(i, j, k, n); // CGS->MKS conversion of zk + } + }); } void @@ -60,22 +61,25 @@ pelelmex_derefx( const auto dxinv = geomdata.InvCellSizeArray(); const auto domain = geomdata.Domain(); - amrex::Real factor = -0.5 * dxinv[0]; + const amrex::Real factor = -0.5 * dxinv[0]; const auto bc_lo = bcrec[PHIV].lo(0); const auto bc_hi = bcrec[PHIV].hi(0); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - bool on_lo = ((bc_lo == amrex::BCType::ext_dir) && i <= domain.smallEnd(0)); - bool on_hi = ((bc_hi == amrex::BCType::ext_dir) && i >= domain.bigEnd(0)); - der(i, j, k) = factor * (phiV(i + 1, j, k) - phiV(i - 1, j, k)); - if (on_lo) - der(i, j, k) = - factor * (phiV(i + 1, j, k) + phiV(i, j, k) - 2.0 * phiV(i - 1, j, k)); - if (on_hi) - der(i, j, k) = - factor * (2.0 * phiV(i + 1, j, k) - phiV(i, j, k) - phiV(i - 1, j, k)); - }); + amrex::ParallelFor( + bx, [bc_lo, bc_hi, domain, factor, phiV, + der] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bool on_lo = + ((bc_lo == amrex::BCType::ext_dir) && i <= domain.smallEnd(0)); + bool on_hi = ((bc_hi == amrex::BCType::ext_dir) && i >= domain.bigEnd(0)); + der(i, j, k) = factor * (phiV(i + 1, j, k) - phiV(i - 1, j, k)); + if (on_lo) + der(i, j, k) = factor * (phiV(i + 1, j, k) + phiV(i, j, k) - + 2.0 * phiV(i - 1, j, k)); + if (on_hi) + der(i, j, k) = factor * (2.0 * phiV(i + 1, j, k) - phiV(i, j, k) - + phiV(i - 1, j, k)); + }); } void @@ -105,30 +109,33 @@ pelelmex_derLorentzx( const auto dxinv = geomdata.InvCellSizeArray(); const auto domain = geomdata.Domain(); - amrex::Real factor = -0.5 * dxinv[0]; + const amrex::Real factor = -0.5 * dxinv[0]; const auto bc_lo = bcrec[PHIV].lo(0); const auto bc_hi = bcrec[PHIV].hi(0); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - // Get gradient of PhiV - bool on_lo = ((bc_lo == amrex::BCType::ext_dir) && i <= domain.smallEnd(0)); - bool on_hi = ((bc_hi == amrex::BCType::ext_dir) && i >= domain.bigEnd(0)); - amrex::Real EFx = factor * (phiV(i + 1, j, k) - phiV(i - 1, j, k)); - if (on_lo) - EFx = - factor * (phiV(i + 1, j, k) + phiV(i, j, k) - 2.0 * phiV(i - 1, j, k)); - if (on_hi) - EFx = - factor * (2.0 * phiV(i + 1, j, k) - phiV(i, j, k) - phiV(i - 1, j, k)); - - // Assemble Lorentz force in X - der(i, j, k) = -nE(i, j, k) * elemCharge * EFx; - for (int n = 0; n < NUM_SPECIES; ++n) { - der(i, j, k) += - zk[n] * 1000.0 * rhoY(i, j, k, n) * EFx; // CGS->MKS conversion of zk - } - }); + amrex::ParallelFor( + bx, [bc_lo, bc_hi, domain, factor, phiV, der, nE, zk, + rhoY] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + // Get gradient of PhiV + bool on_lo = + ((bc_lo == amrex::BCType::ext_dir) && i <= domain.smallEnd(0)); + bool on_hi = ((bc_hi == amrex::BCType::ext_dir) && i >= domain.bigEnd(0)); + amrex::Real EFx = factor * (phiV(i + 1, j, k) - phiV(i - 1, j, k)); + if (on_lo) + EFx = factor * + (phiV(i + 1, j, k) + phiV(i, j, k) - 2.0 * phiV(i - 1, j, k)); + if (on_hi) + EFx = factor * + (2.0 * phiV(i + 1, j, k) - phiV(i, j, k) - phiV(i - 1, j, k)); + + // Assemble Lorentz force in X + der(i, j, k) = -nE(i, j, k) * elemCharge * EFx; + for (int n = 0; n < NUM_SPECIES; ++n) { + der(i, j, k) += + zk[n] * 1000.0 * rhoY(i, j, k, n) * EFx; // CGS->MKS conversion of zk + } + }); } #if (AMREX_SPACEDIM > 1) @@ -154,22 +161,25 @@ pelelmex_derefy( const auto dxinv = geomdata.InvCellSizeArray(); const auto domain = geomdata.Domain(); - amrex::Real factor = -0.5 * dxinv[1]; + const amrex::Real factor = -0.5 * dxinv[1]; const auto bc_lo = bcrec[PHIV].lo(1); const auto bc_hi = bcrec[PHIV].hi(1); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - bool on_lo = ((bc_lo == amrex::BCType::ext_dir) && j <= domain.smallEnd(1)); - bool on_hi = ((bc_hi == amrex::BCType::ext_dir) && j >= domain.bigEnd(1)); - der(i, j, k) = factor * (phiV(i, j + 1, k) - phiV(i, j - 1, k)); - if (on_lo) - der(i, j, k) = - factor * (phiV(i, j + 1, k) + phiV(i, j, k) - 2.0 * phiV(i, j - 1, k)); - if (on_hi) - der(i, j, k) = - factor * (2.0 * phiV(i, j + 1, k) - phiV(i, j, k) - phiV(i, j - 1, k)); - }); + amrex::ParallelFor( + bx, [bc_lo, bc_hi, domain, der, phiV, + factor] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bool on_lo = + ((bc_lo == amrex::BCType::ext_dir) && j <= domain.smallEnd(1)); + bool on_hi = ((bc_hi == amrex::BCType::ext_dir) && j >= domain.bigEnd(1)); + der(i, j, k) = factor * (phiV(i, j + 1, k) - phiV(i, j - 1, k)); + if (on_lo) + der(i, j, k) = factor * (phiV(i, j + 1, k) + phiV(i, j, k) - + 2.0 * phiV(i, j - 1, k)); + if (on_hi) + der(i, j, k) = factor * (2.0 * phiV(i, j + 1, k) - phiV(i, j, k) - + phiV(i, j - 1, k)); + }); } void @@ -199,30 +209,33 @@ pelelmex_derLorentzy( const auto dxinv = geomdata.InvCellSizeArray(); const auto domain = geomdata.Domain(); - amrex::Real factor = -0.5 * dxinv[1]; + const amrex::Real factor = -0.5 * dxinv[1]; const auto bc_lo = bcrec[PHIV].lo(1); const auto bc_hi = bcrec[PHIV].hi(1); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - // Get gradient of PhiV - bool on_lo = ((bc_lo == amrex::BCType::ext_dir) && j <= domain.smallEnd(1)); - bool on_hi = ((bc_hi == amrex::BCType::ext_dir) && j >= domain.bigEnd(1)); - amrex::Real EFy = factor * (phiV(i, j + 1, k) - phiV(i, j - 1, k)); - if (on_lo) - EFy = - factor * (phiV(i, j + 1, k) + phiV(i, j, k) - 2.0 * phiV(i, j - 1, k)); - if (on_hi) - EFy = - factor * (2.0 * phiV(i, j - 1, k) - phiV(i, j, k) - phiV(i, j - 1, k)); - - // Assemble Lorentz force in Y - der(i, j, k) = -nE(i, j, k) * elemCharge * EFy; - for (int n = 0; n < NUM_SPECIES; ++n) { - der(i, j, k) += - zk[n] * 1000.0 * rhoY(i, j, k, n) * EFy; // CGS->MKS conversion of zk - } - }); + amrex::ParallelFor( + bx, [bc_lo, bc_hi, domain, factor, der, nE, phiV, rhoY, + zk] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + // Get gradient of PhiV + bool on_lo = + ((bc_lo == amrex::BCType::ext_dir) && j <= domain.smallEnd(1)); + bool on_hi = ((bc_hi == amrex::BCType::ext_dir) && j >= domain.bigEnd(1)); + amrex::Real EFy = factor * (phiV(i, j + 1, k) - phiV(i, j - 1, k)); + if (on_lo) + EFy = factor * + (phiV(i, j + 1, k) + phiV(i, j, k) - 2.0 * phiV(i, j - 1, k)); + if (on_hi) + EFy = factor * + (2.0 * phiV(i, j - 1, k) - phiV(i, j, k) - phiV(i, j - 1, k)); + + // Assemble Lorentz force in Y + der(i, j, k) = -nE(i, j, k) * elemCharge * EFy; + for (int n = 0; n < NUM_SPECIES; ++n) { + der(i, j, k) += + zk[n] * 1000.0 * rhoY(i, j, k, n) * EFy; // CGS->MKS conversion of zk + } + }); } #if (AMREX_SPACEDIM > 2) @@ -253,17 +266,20 @@ pelelmex_derefz( const auto bc_lo = bcrec[PHIV].lo(2); const auto bc_hi = bcrec[PHIV].hi(2); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - bool on_lo = ((bc_lo == amrex::BCType::ext_dir) && k <= domain.smallEnd(2)); - bool on_hi = ((bc_hi == amrex::BCType::ext_dir) && k >= domain.bigEnd(2)); - der(i, j, k) = factor * (phiV(i, j, k + 1) - phiV(i, j, k - 1)); - if (on_lo) - der(i, j, k) = - factor * (phiV(i, j, k + 1) + phiV(i, j, k) - 2.0 * phiV(i, j, k - 1)); - if (on_hi) - der(i, j, k) = - factor * (2.0 * phiV(i, j, k + 1) - phiV(i, j, k) - phiV(i, j, k - 1)); - }); + amrex::ParallelFor( + bx, [bc_lo, bc_hi, domain, der, phiV, + factor] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + bool on_lo = + ((bc_lo == amrex::BCType::ext_dir) && k <= domain.smallEnd(2)); + bool on_hi = ((bc_hi == amrex::BCType::ext_dir) && k >= domain.bigEnd(2)); + der(i, j, k) = factor * (phiV(i, j, k + 1) - phiV(i, j, k - 1)); + if (on_lo) + der(i, j, k) = factor * (phiV(i, j, k + 1) + phiV(i, j, k) - + 2.0 * phiV(i, j, k - 1)); + if (on_hi) + der(i, j, k) = factor * (2.0 * phiV(i, j, k + 1) - phiV(i, j, k) - + phiV(i, j, k - 1)); + }); } void @@ -298,25 +314,28 @@ pelelmex_derLorentzz( const auto bc_lo = bcrec[PHIV].lo(2); const auto bc_hi = bcrec[PHIV].hi(2); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - // Get gradient of PhiV - bool on_lo = ((bc_lo == amrex::BCType::ext_dir) && k <= domain.smallEnd(2)); - bool on_hi = ((bc_hi == amrex::BCType::ext_dir) && k >= domain.bigEnd(2)); - amrex::Real EFz = factor * (phiV(i, j, k + 1) - phiV(i, j, k - 1)); - if (on_lo) - EFz = - factor * (phiV(i, j, k + 1) + phiV(i, j, k) - 2.0 * phiV(i, j, k - 1)); - if (on_hi) - EFz = - factor * (2.0 * phiV(i, j, k + 1) - phiV(i, j, k) - phiV(i, j, k - 1)); - - // Assemble Lorentz force in Z - der(i, j, k) = -nE(i, j, k) * elemCharge * EFz; - for (int n = 0; n < NUM_SPECIES; ++n) { - der(i, j, k) += - zk[n] * 1000.0 * rhoY(i, j, k, n) * EFz; // CGS->MKS conversion of zk - } - }); + amrex::ParallelFor( + bx, [bc_lo, bc_hi, domain, factor, phiV, der, nE, zk, + rhoY] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + // Get gradient of PhiV + bool on_lo = + ((bc_lo == amrex::BCType::ext_dir) && k <= domain.smallEnd(2)); + bool on_hi = ((bc_hi == amrex::BCType::ext_dir) && k >= domain.bigEnd(2)); + amrex::Real EFz = factor * (phiV(i, j, k + 1) - phiV(i, j, k - 1)); + if (on_lo) + EFz = factor * + (phiV(i, j, k + 1) + phiV(i, j, k) - 2.0 * phiV(i, j, k - 1)); + if (on_hi) + EFz = factor * + (2.0 * phiV(i, j, k + 1) - phiV(i, j, k) - phiV(i, j, k - 1)); + + // Assemble Lorentz force in Z + der(i, j, k) = -nE(i, j, k) * elemCharge * EFz; + for (int n = 0; n < NUM_SPECIES; ++n) { + der(i, j, k) += + zk[n] * 1000.0 * rhoY(i, j, k, n) * EFz; // CGS->MKS conversion of zk + } + }); } #endif #endif diff --git a/Source/Plasma/PeleLMeX_EFIonDrift.cpp b/Source/Plasma/PeleLMeX_EFIonDrift.cpp index 01d4fadb3..9385c296a 100644 --- a/Source/Plasma/PeleLMeX_EFIonDrift.cpp +++ b/Source/Plasma/PeleLMeX_EFIonDrift.cpp @@ -5,7 +5,7 @@ #include void -PeleLM::ionDriftVelocity(std::unique_ptr& advData) +PeleLM::ionDriftVelocity(const std::unique_ptr& advData) { //---------------------------------------------------------------- // set udrift boundaries to zero @@ -27,7 +27,7 @@ PeleLM::ionDriftVelocity(std::unique_ptr& advData) for (int lev = 0; lev <= finest_level; ++lev) { const auto& ba = grids[lev]; const auto& factory = Factory(lev); - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { gphiVOld[lev][idim].define( amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)), dmap[lev], 1, nGrow, amrex::MFInfo(), factory); @@ -37,7 +37,7 @@ PeleLM::ionDriftVelocity(std::unique_ptr& advData) } } - int do_avgDown = 0; // TODO or should I ? + constexpr int do_avgDown = 0; // TODO or should I ? auto bcRecPhiV = fetchBCRecArray(PHIV, 1); getDiffusionOp()->computeGradient( GetVecOfArrOfPtrs(gphiVOld), {}, // don't need the laplacian out @@ -58,6 +58,7 @@ PeleLM::ionDriftVelocity(std::unique_ptr& advData) auto ldataNew_p = getLevelDataPtr(lev, AmrNewTime); amrex::MultiFab mobH_cc(grids[lev], dmap[lev], NUM_IONS, 1); + #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -73,14 +74,13 @@ PeleLM::ionDriftVelocity(std::unique_ptr& advData) mob_h(i, j, k, n) = 0.5 * (mob_o(i, j, k, n) + mob_n(i, j, k, n)); }); } - // Get the face centered ions mobility - int doZeroVisc = 0; + constexpr int doZeroVisc = 0; amrex::Array mobH_ec = getDiffusivity(lev, 0, NUM_IONS, doZeroVisc, bcRecIons, mobH_cc); // Assemble the ions drift velocity - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -124,8 +124,8 @@ PeleLM::ionDriftVelocity(std::unique_ptr& advData) // Set BCRec for Umac amrex::Vector bcrec(NUM_IONS); - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - for (int ion = 0; ion < NUM_IONS; ion++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + for (int ion = 0; ion < NUM_IONS; ++ion) { if (geom[lev - 1].isPeriodic(idim)) { bcrec[ion].setLo(idim, amrex::BCType::int_dir); bcrec[ion].setHi(idim, amrex::BCType::int_dir); @@ -169,10 +169,11 @@ PeleLM::ionDriftVelocity(std::unique_ptr& advData) } void -PeleLM::ionDriftAddUmac(int lev, std::unique_ptr& advData) +PeleLM::ionDriftAddUmac( + const int lev, const std::unique_ptr& advData) { // Add umac to the ions drift velocity to get the effective velocity - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif diff --git a/Source/Plasma/PeleLMeX_EFNLSolve.cpp b/Source/Plasma/PeleLMeX_EFNLSolve.cpp index ec5103d55..851b5775c 100644 --- a/Source/Plasma/PeleLMeX_EFNLSolve.cpp +++ b/Source/Plasma/PeleLMeX_EFNLSolve.cpp @@ -16,10 +16,10 @@ PeleLM::getPrecondOp() void PeleLM::implicitNonLinearSolve( - int sdcIter, - const amrex::Real& a_dt, - std::unique_ptr& diffData, - std::unique_ptr& advData) + const int sdcIter, + const amrex::Real a_dt, + const std::unique_ptr& diffData, + const std::unique_ptr& advData) { BL_PROFILE_VAR("PeleLMeX::implicitNonLinearSolve()", implicitNonLinearSolve); @@ -48,7 +48,7 @@ PeleLM::implicitNonLinearSolve( } // Gradient of PhiV at t^{n} - int do_avgDown = 0; // TODO or should I ? + constexpr int do_avgDown = 0; // TODO or should I ? auto bcRecPhiV = fetchBCRecArray(PHIV, 1); getDiffusionOp()->computeGradient( getNLgradPhiVVect(), {}, // don't need the laplacian out @@ -58,7 +58,7 @@ PeleLM::implicitNonLinearSolve( for (int lev = 0; lev <= finest_level; ++lev) { // Get nl solve data pointer auto ldataNLs_p = getLevelDataNLSolvePtr(lev); - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { amrex::MultiFab::Copy( ldataNLs_p->umac[idim], advData->umac[lev][idim], 0, 0, 1, 0); } @@ -82,7 +82,7 @@ PeleLM::implicitNonLinearSolve( if (ef_substep > 1) amrex::Abort("Non-linear solve sub-stepping not re-implemented yet"); int NK_tot_count = 0; - for (int sstep = 0; sstep < ef_substep; sstep++) { + for (int sstep = 0; sstep < ef_substep; ++sstep) { curtime = getTime(0, AmrOldTime) + (sstep + 1) * dtsub; // ----------------- @@ -141,9 +141,10 @@ PeleLM::implicitNonLinearSolve( } // Solve for Newton direction - amrex::Vector newtonDir(finest_level + 1); + amrex::Vector newtonDir; + newtonDir.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - newtonDir[lev].define( + newtonDir.emplace_back( grids[lev], dmap[lev], 2, 1, amrex::MFInfo(), Factory(lev)); newtonDir[lev].setVal(0.0, 0, 2, 1); } @@ -219,7 +220,9 @@ PeleLM::implicitNonLinearSolve( int PeleLM::testExitNewton( - int newtonIter, const amrex::Real& max_res, const amrex::Real& norm_NewtonDir) + const int newtonIter, + const amrex::Real max_res, + const amrex::Real norm_NewtonDir) { int exit = 0; if (max_res <= m_ef_newtonTol || norm_NewtonDir <= 1e-11) { @@ -313,10 +316,10 @@ PeleLM::updateNLState(const amrex::Vector& a_update) void PeleLM::incrementElectronForcing( - int a_sstep, std::unique_ptr& advData) + const int a_sstep, const std::unique_ptr& advData) { - for (int lev = 0; lev <= finest_level; ++lev) { + for (int lev = 0; lev <= finest_level; ++lev) { auto ldata_p = getLevelDataPtr(lev, AmrOldTime); // Old time electron auto ldataR_p = getLevelDataReactPtr(lev); // Reaction auto ldataNLs_p = getLevelDataNLSolvePtr(lev); // NL data @@ -349,12 +352,12 @@ PeleLM::incrementElectronForcing( void PeleLM::computeBGcharge( - const amrex::Real& a_time, - std::unique_ptr& diffData, - std::unique_ptr& advData) + const amrex::Real a_time, + const std::unique_ptr& diffData, + const std::unique_ptr& advData) { // Get integration dt - amrex::Real dt_int = a_time - getTime(0, AmrOldTime); + const amrex::Real dt_int = a_time - getTime(0, AmrOldTime); for (int lev = 0; lev <= finest_level; ++lev) { // Get data pointers @@ -397,19 +400,21 @@ PeleLM::computeBGcharge( void PeleLM::nonLinearResidual( - const amrex::Real& a_dt, + const amrex::Real a_dt, const amrex::Vector& a_nlstate, const amrex::Vector& a_nlresid, - int updateScaling, - int updatePrecond) + const int updateScaling, + const int updatePrecond) { // Get unscaled copy of the NL state - amrex::Vector nE(finest_level + 1); - amrex::Vector phiV(finest_level + 1); + amrex::Vector nE; + nE.reserve(finest_level + 1); + amrex::Vector phiV; + phiV.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - nE[lev].define( + nE.emplace_back( grids[lev], dmap[lev], 1, m_nGrowState, amrex::MFInfo(), Factory(lev)); - phiV[lev].define( + phiV.emplace_back( grids[lev], dmap[lev], 1, m_nGrowState, amrex::MFInfo(), Factory(lev)); amrex::MultiFab::Copy(nE[lev], *a_nlstate[lev], 0, 0, 1, m_nGrowState); nE[lev].mult(nE_scale, 0, 1, m_nGrowState); @@ -421,29 +426,31 @@ PeleLM::nonLinearResidual( fillPatchNLnE(m_cur_time, GetVecOfPtrs(nE), m_nGrowState); // Get L(phiV) and Grad(phiV) - amrex::Vector laplacian(finest_level + 1); + amrex::Vector laplacian; + laplacian.reserve(finest_level + 1); amrex::Vector> gradPhiVCur( finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - laplacian[lev].define( + laplacian.emplace_back( grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), Factory(lev)); - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { const auto& fba = amrex::convert(grids[lev], amrex::IntVect::TheDimensionVector(idim)); gradPhiVCur[lev][idim].define( fba, dmap[lev], 1, 0, amrex::MFInfo(), Factory(lev)); } } - int do_avgDown = 0; // TODO or should I ? + constexpr int do_avgDown = 0; // TODO or should I ? auto bcRecPhiV = fetchBCRecArray(PHIV, 1); getDiffusionOp()->computeGradient( GetVecOfArrOfPtrs(gradPhiVCur), GetVecOfPtrs(laplacian), GetVecOfConstPtrs(phiV), {}, bcRecPhiV[0], do_avgDown); // Get nE diffusion term - amrex::Vector diffnE(finest_level + 1); + amrex::Vector diffnE; + diffnE.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - diffnE[lev].define( + diffnE.emplace_back( grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), Factory(lev)); } auto bcRecnE = fetchBCRecArray(NE, 1); @@ -455,9 +462,10 @@ PeleLM::nonLinearResidual( // amrex::VisMF::Write(diffnE[0],"diffnEnlResid"); // Get nE advection term - amrex::Vector advnE(finest_level + 1); + amrex::Vector advnE; + advnE.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - advnE[lev].define( + advnE.emplace_back( grids[lev], dmap[lev], 1, 0, amrex::MFInfo(), Factory(lev)); } getAdvectionTerm( @@ -477,7 +485,6 @@ PeleLM::nonLinearResidual( // Init the ghostcells too a_nlresid[lev]->setVal(0.0); - #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -554,7 +561,7 @@ PeleLM::getAdvectionTerm( for (int lev = 0; lev <= finest_level; ++lev) { const auto& ba = grids[lev]; const auto& factory = Factory(lev); - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { fluxes[lev][idim].define( amrex::convert(ba, amrex::IntVect::TheDimensionVector(idim)), dmap[lev], 1, nGrow, amrex::MFInfo(), factory); @@ -573,10 +580,12 @@ PeleLM::getAdvectionTerm( auto ldataNLs_p = getLevelDataNLSolvePtr(lev); // Get the face centered electron mobility - int doZeroVisc = 0; + constexpr int doZeroVisc = 0; amrex::Array mobE_ec = getDiffusivity(lev, 0, 1, doZeroVisc, bcRecnE, ldata_p->mobE_cc); + // Get the electron effective velocity + // Get the electron effective velocity for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { #ifdef AMREX_USE_OMP @@ -643,14 +652,14 @@ PeleLM::getAdvectionTerm( } // Compute divergence - int intensiveFluxes = 1; + constexpr int intensiveFluxes = 1; fluxDivergence( a_advTerm, 0, GetVecOfArrOfPtrs(fluxes), 0, 1, intensiveFluxes, -1.0); } void PeleLM::getAdvectionFluxesMOL( - int lev, + const int lev, const amrex::Array& a_fluxes, const amrex::MultiFab& a_nE, const amrex::Array& a_ueff, @@ -698,9 +707,9 @@ PeleLM::getAdvectionFluxesMOL( auto const& divu_arr = a_nE.const_array(mfi); auto const& force_arr = a_nE.const_array(mfi); - bool is_velocity = false; - bool fluxes_are_area_weighted = false; - bool knownEdgeState = false; + constexpr bool is_velocity = false; + constexpr bool fluxes_are_area_weighted = false; + constexpr bool knownEdgeState = false; std::string mol = "MOL"; HydroUtils::ComputeFluxesOnBoxFromState( bx, 1, mfi, nE_arr, AMREX_D_DECL(fx, fy, fz), @@ -717,7 +726,7 @@ PeleLM::getAdvectionFluxesMOL( void PeleLM::getAdvectionFluxes( - int lev, + const int lev, const amrex::Array& a_fluxes, const amrex::MultiFab& a_nE, const amrex::Array& a_ueff, @@ -725,7 +734,7 @@ PeleLM::getAdvectionFluxes( { const amrex::Box& domain = geom[lev].Domain(); - int order = m_nEAdvOrder; + const int order = m_nEAdvOrder; #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) @@ -771,22 +780,23 @@ PeleLM::getAdvectionFluxes( const auto bc_hi = bcrec.hi(0); amrex::ParallelFor( - xbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - int idx[3] = {i, j, k}; - bool on_lo = + xbx, [bc_lo, bc_hi, edomain, ne_arr, xstate, u, order, domain, + xbx] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + const int idx[3] = {i, j, k}; + const bool on_lo = ((bc_lo == amrex::BCType::ext_dir) && (idx[0] <= edomain.smallEnd(0))); - bool on_hi = + const bool on_hi = ((bc_hi == amrex::BCType::ext_dir) && (idx[0] >= edomain.bigEnd(0))); if (order == 1) { xstate(i, j, k) = ef_edge_state_extdir(i, j, k, 0, on_lo, on_hi, ne_arr, u); } else if (order == 2) { - bool extdir_or_ho_lo = (bc_lo == amrex::BCType::ext_dir) || - (bc_lo == amrex::BCType::hoextrap); - bool extdir_or_ho_hi = (bc_hi == amrex::BCType::ext_dir) || - (bc_hi == amrex::BCType::hoextrap); + const bool extdir_or_ho_lo = (bc_lo == amrex::BCType::ext_dir) || + (bc_lo == amrex::BCType::hoextrap); + const bool extdir_or_ho_hi = (bc_hi == amrex::BCType::ext_dir) || + (bc_hi == amrex::BCType::hoextrap); xstate(i, j, k) = ef_edge_state_2ndO_extdir( i, j, k, 0, on_lo, on_hi, extdir_or_ho_lo, extdir_or_ho_hi, domain.smallEnd(0), domain.bigEnd(0), ne_arr, u, xbx); @@ -802,22 +812,23 @@ PeleLM::getAdvectionFluxes( const auto bc_hi = bcrec.hi(1); amrex::ParallelFor( - ybx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - int idx[3] = {i, j, k}; - bool on_lo = + ybx, [bc_lo, bc_hi, edomain, domain, order, ystate, ne_arr, v, + ybx] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + const int idx[3] = {i, j, k}; + const bool on_lo = ((bc_lo == amrex::BCType::ext_dir) && (idx[1] <= edomain.smallEnd(1))); - bool on_hi = + const bool on_hi = ((bc_hi == amrex::BCType::ext_dir) && (idx[1] >= edomain.bigEnd(1))); if (order == 1) { ystate(i, j, k) = ef_edge_state_extdir(i, j, k, 1, on_lo, on_hi, ne_arr, v); } else if (order == 2) { - bool extdir_or_ho_lo = (bc_lo == amrex::BCType::ext_dir) || - (bc_lo == amrex::BCType::hoextrap); - bool extdir_or_ho_hi = (bc_hi == amrex::BCType::ext_dir) || - (bc_hi == amrex::BCType::hoextrap); + const bool extdir_or_ho_lo = (bc_lo == amrex::BCType::ext_dir) || + (bc_lo == amrex::BCType::hoextrap); + const bool extdir_or_ho_hi = (bc_hi == amrex::BCType::ext_dir) || + (bc_hi == amrex::BCType::hoextrap); ystate(i, j, k) = ef_edge_state_2ndO_extdir( i, j, k, 1, on_lo, on_hi, extdir_or_ho_lo, extdir_or_ho_hi, domain.smallEnd(1), domain.bigEnd(1), ne_arr, v, ybx); @@ -833,22 +844,23 @@ PeleLM::getAdvectionFluxes( const auto bc_hi = bcrec.hi(2); amrex::ParallelFor( - zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - int idx[3] = {i, j, k}; - bool on_lo = + zbx, [bc_lo, bc_hi, edomain, zstate, order, ne_arr, w, zbx, + domain] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + const int idx[3] = {i, j, k}; + const bool on_lo = ((bc_lo == amrex::BCType::ext_dir) && (idx[2] <= edomain.smallEnd(2))); - bool on_hi = + const bool on_hi = ((bc_hi == amrex::BCType::ext_dir) && (idx[2] >= edomain.bigEnd(2))); if (order == 1) { zstate(i, j, k) = ef_edge_state_extdir(i, j, k, 2, on_lo, on_hi, ne_arr, w); } else if (order == 2) { - bool extdir_or_ho_lo = (bc_lo == amrex::BCType::ext_dir) || - (bc_lo == amrex::BCType::hoextrap); - bool extdir_or_ho_hi = (bc_hi == amrex::BCType::ext_dir) || - (bc_hi == amrex::BCType::hoextrap); + const bool extdir_or_ho_lo = (bc_lo == amrex::BCType::ext_dir) || + (bc_lo == amrex::BCType::hoextrap); + const bool extdir_or_ho_hi = (bc_hi == amrex::BCType::ext_dir) || + (bc_hi == amrex::BCType::hoextrap); zstate(i, j, k) = ef_edge_state_2ndO_extdir( i, j, k, 2, on_lo, on_hi, extdir_or_ho_lo, extdir_or_ho_hi, domain.smallEnd(2), domain.bigEnd(2), ne_arr, w, zbx); @@ -860,17 +872,17 @@ PeleLM::getAdvectionFluxes( // Computing fluxes amrex::ParallelFor( - xbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + xbx, [xflux, u, xstate] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { xflux(i, j, k) = u(i, j, k) * xstate(i, j, k); }); #if (AMREX_SPACEDIM > 1) amrex::ParallelFor( - ybx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + ybx, [yflux, v, ystate] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { yflux(i, j, k) = v(i, j, k) * ystate(i, j, k); }); #if (AMREX_SPACEDIM == 3) amrex::ParallelFor( - zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + zbx, [zflux, w, zstate] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { zflux(i, j, k) = w(i, j, k) * zstate(i, j, k); }); #endif @@ -881,7 +893,7 @@ PeleLM::getAdvectionFluxes( void PeleLM::setUpPrecond( - const amrex::Real& a_dt, const amrex::Vector& a_nE) + const amrex::Real a_dt, const amrex::Vector& a_nE) { BL_PROFILE("PeleLMeX::setUpPrecond()"); @@ -910,7 +922,7 @@ PeleLM::setUpPrecond( // Get nl solve data pointer auto ldataNLs_p = getLevelDataNLSolvePtr(lev); - int doZeroVisc = 0; + constexpr int doZeroVisc = 0; amrex::Array diffE_ec = getDiffusivity(lev, 0, 1, doZeroVisc, bcRecnE, ldata_p->diffE_cc); @@ -920,10 +932,11 @@ PeleLM::setUpPrecond( } // Stilda approx first-order - amrex::Vector diagDiffOp(finest_level + 1); + amrex::Vector diagDiffOp; + diagDiffOp.reserve(finest_level + 1); if (m_ef_PC_approx == 2) { for (int lev = 0; lev <= finest_level; ++lev) { - diagDiffOp[lev].define(grids[lev], dmap[lev], 1, 1); + diagDiffOp.emplace_back(grids[lev], dmap[lev], 1, 1); getPrecondOp()->getDiffOpDiagonal(lev, diagDiffOp[lev]); diagDiffOp[lev].mult(FnE_scale / nE_scale); } @@ -995,7 +1008,7 @@ PeleLM::setUpPrecond( if (m_ef_PC_approx == 1) { // Assuming identity of the inverse of DiffOp // Add Stilda pieces amrex::Real scalLap = eps0 * epsr / elemCharge; - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { neKe_ec[idim].mult(0.5 * a_dt, 0, 1); neKe_ec[idim].plus(scalLap, 0, 1); } @@ -1007,7 +1020,7 @@ PeleLM::setUpPrecond( lev, 0, 1, bcRecnE, Schur_nEKe, GetArrOfConstPtrs(ldataNLs_p->uEffnE)); amrex::Real scalLap = eps0 * epsr / elemCharge; - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { Schur_neKe_ec[idim].plus(scalLap, 0, 1); } getPrecondOp()->setStildaOpBCoeff(lev, GetArrOfConstPtrs(Schur_neKe_ec)); @@ -1019,10 +1032,10 @@ PeleLM::setUpPrecond( amrex::Array PeleLM::getUpwindedEdge( - int lev, - int edge_comp, - int ncomp, - amrex::Vector bcrec, + const int lev, + const int edge_comp, + const int ncomp, + const amrex::Vector& bcrec, const amrex::MultiFab& ccMF, const amrex::Array& ecVel) { @@ -1050,7 +1063,7 @@ PeleLM::getUpwindedEdge( #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif for (amrex::MFIter mfi(ccMF, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { const amrex::Box ebx = mfi.nodaltilebox(idim); const amrex::Box& edomain = amrex::surroundingNodes(domain, idim); const auto& ccVal = ccMF.const_array(mfi, edge_comp); @@ -1059,12 +1072,13 @@ PeleLM::getUpwindedEdge( const auto bc_lo = bcrec[0].lo(idim); const auto bc_hi = bcrec[0].hi(idim); amrex::ParallelFor( - ebx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - int idx[3] = {i, j, k}; - bool on_lo = + ebx, [bc_lo, bc_hi, idim, edomain, ecUeff, ccVal, + ecVal] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + const int idx[3] = {i, j, k}; + const bool on_lo = ((bc_lo == amrex::BCType::ext_dir) && (idx[idim] <= edomain.smallEnd(idim))); - bool on_hi = + const bool on_hi = ((bc_hi == amrex::BCType::ext_dir) && (idx[idim] >= edomain.bigEnd(idim))); cen2edg_upwind(i, j, k, idim, 1, on_lo, on_hi, ecUeff, ccVal, ecVal); @@ -1097,21 +1111,23 @@ PeleLM::jTimesV( m_ef_lambda_jfnk * (m_ef_lambda_jfnk + nl_stateNorm / vNorm); if (m_ef_diffT_jfnk == 1) { - amrex::Vector statePert(finest_level + 1); - amrex::Vector residPert(finest_level + 1); + amrex::Vector statePert; + statePert.reserve(finest_level + 1); + amrex::Vector residPert; + residPert.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { auto ldataNLs_p = getLevelDataNLSolvePtr(lev); - statePert[lev].define( + statePert.emplace_back( grids[lev], dmap[lev], 2, m_nGrowState, amrex::MFInfo(), Factory(lev)); - residPert[lev].define( + residPert.emplace_back( grids[lev], dmap[lev], 2, m_nGrowState, amrex::MFInfo(), Factory(lev)); amrex::MultiFab::Copy( statePert[lev], ldataNLs_p->nlState, 0, 0, 2, m_nGrowState); amrex::MultiFab::Saxpy(statePert[lev], delta_pert, *a_v[lev], 0, 0, 2, 0); } - int update_scaling = 0; - int update_precond = 0; + constexpr int update_scaling = 0; + constexpr int update_precond = 0; nonLinearResidual( dtsub, GetVecOfPtrs(statePert), GetVecOfPtrs(residPert), update_scaling, update_precond); @@ -1124,19 +1140,23 @@ PeleLM::jTimesV( a_Jv[lev]->mult(-1.0 / delta_pert); } } else if (m_ef_diffT_jfnk == 2) { - amrex::Vector statePertPls(finest_level + 1); - amrex::Vector residPertPls(finest_level + 1); - amrex::Vector statePertMns(finest_level + 1); - amrex::Vector residPertMns(finest_level + 1); + amrex::Vector statePertPls; + statePertPls.reserve(finest_level + 1); + amrex::Vector residPertPls; + residPertPls.reserve(finest_level + 1); + amrex::Vector statePertMns; + statePertMns.reserve(finest_level + 1); + amrex::Vector residPertMns; + residPertMns.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { auto ldataNLs_p = getLevelDataNLSolvePtr(lev); - statePertPls[lev].define( + statePertPls.emplace_back( grids[lev], dmap[lev], 2, m_nGrowState, amrex::MFInfo(), Factory(lev)); - residPertPls[lev].define( + residPertPls.emplace_back( grids[lev], dmap[lev], 2, m_nGrowState, amrex::MFInfo(), Factory(lev)); - statePertMns[lev].define( + statePertMns.emplace_back( grids[lev], dmap[lev], 2, m_nGrowState, amrex::MFInfo(), Factory(lev)); - residPertMns[lev].define( + residPertMns.emplace_back( grids[lev], dmap[lev], 2, m_nGrowState, amrex::MFInfo(), Factory(lev)); amrex::MultiFab::Copy( statePertPls[lev], ldataNLs_p->nlState, 0, 0, 2, m_nGrowState); @@ -1148,8 +1168,8 @@ PeleLM::jTimesV( statePertMns[lev], -delta_pert, *a_v[lev], 0, 0, 2, 0); } - int update_scaling = 0; - int update_precond = 0; + constexpr int update_scaling = 0; + constexpr int update_precond = 0; nonLinearResidual( dtsub, GetVecOfPtrs(statePertPls), GetVecOfPtrs(residPertPls), update_scaling, update_precond); @@ -1176,9 +1196,13 @@ PeleLM::applyPrecond( // Setup aliases and temps amrex::Vector nE_al; + nE_al.reserve(finest_level + 1); amrex::Vector phiV_al; + phiV_al.reserve(finest_level + 1); amrex::Vector PnE_al; + PnE_al.reserve(finest_level + 1); amrex::Vector PphiV_al; + PphiV_al.reserve(finest_level + 1); amrex::Vector> temp(finest_level + 1); amrex::Vector> temp2(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { @@ -1249,8 +1273,8 @@ void PeleLM::nlSolveNorm(const amrex::Vector& a_MF, amrex::Real& r) { r = 0.0; - int nComp = a_MF[0]->nComp(); - for (int comp = 0; comp < nComp; comp++) { + const int nComp = a_MF[0]->nComp(); + for (int comp = 0; comp < nComp; ++comp) { amrex::Real norm = 0.0; for (int lev = 0; lev < a_MF.size(); ++lev) { // TODO : norm not weighted by cell size, should it ? diff --git a/Source/Plasma/PeleLMeX_EFPoisson.cpp b/Source/Plasma/PeleLMeX_EFPoisson.cpp index 5f41ba613..676fec23e 100644 --- a/Source/Plasma/PeleLMeX_EFPoisson.cpp +++ b/Source/Plasma/PeleLMeX_EFPoisson.cpp @@ -1,10 +1,9 @@ #include -#include #include #include void -PeleLM::poissonSolveEF(const TimeStamp& a_time) +PeleLM::poissonSolveEF(const TimeStamp a_time) { BL_PROFILE("PeleLMeX::poissonSolveEF()"); if (ef_verbose) { @@ -15,13 +14,17 @@ PeleLM::poissonSolveEF(const TimeStamp& a_time) auto bcRecPhiV = fetchBCRecArray(PHIV, 1); // Build Poisson RHS: charge distribution - int nGhost = 0; + constexpr int nGhost = 0; amrex::Vector> rhsPoisson(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { rhsPoisson[lev].reset(new amrex::MultiFab( grids[lev], dmap[lev], 1, nGhost, amrex::MFInfo(), *m_factory[lev])); auto ldata_p = getLevelDataPtr(lev, a_time); + + auto const& state_ma = ldata_p->state.const_arrays(); + auto const& rhs_ma = rhsPoisson[lev]->arrays(); + #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -41,7 +44,6 @@ PeleLM::poissonSolveEF(const TimeStamp& a_time) }); } } - // Solve for PhiV getDiffusionOp()->diffuse_scalar( GetVecOfPtrs(getPhiVVect(a_time)), 0, GetVecOfConstPtrs(rhsPoisson), 0, {}, diff --git a/Source/Plasma/PeleLMeX_EFReactions.cpp b/Source/Plasma/PeleLMeX_EFReactions.cpp index 193e665fb..033e9954b 100644 --- a/Source/Plasma/PeleLMeX_EFReactions.cpp +++ b/Source/Plasma/PeleLMeX_EFReactions.cpp @@ -3,7 +3,7 @@ void PeleLM::computeInstantaneousReactionRateEF( - int lev, const TimeStamp& a_time, amrex::MultiFab* a_I_R) + const int lev, const TimeStamp a_time, amrex::MultiFab* a_I_R) { auto ldata_p = getLevelDataPtr(lev, a_time); diff --git a/Source/Plasma/PeleLMeX_EFTimeStep.cpp b/Source/Plasma/PeleLMeX_EFTimeStep.cpp index 159cf2761..d399cc7d0 100644 --- a/Source/Plasma/PeleLMeX_EFTimeStep.cpp +++ b/Source/Plasma/PeleLMeX_EFTimeStep.cpp @@ -1,7 +1,7 @@ #include amrex::Real -PeleLM::estEFIonsDt(const TimeStamp& a_time) +PeleLM::estEFIonsDt(const TimeStamp a_time) { amrex::Real estdt = 1.0e200; constexpr amrex::Real small = 1.0e-8; @@ -93,7 +93,6 @@ PeleLM::estEFIonsDt(const TimeStamp& a_time) #endif #endif } - // Get cell centered max effective velocities across // all dimension/ions #ifdef AMREX_USE_OMP @@ -121,7 +120,7 @@ PeleLM::estEFIonsDt(const TimeStamp& a_time) } const auto dx = Geom(lev).CellSizeArray(); - amrex::Real cfl_lcl = m_cfl; + const amrex::Real cfl_lcl = m_cfl; estdt_lev = amrex::ReduceMin( driftVelMax_cc, 0, [dx, cfl_lcl] AMREX_GPU_HOST_DEVICE( diff --git a/Source/Plasma/PeleLMeX_EFTransport.cpp b/Source/Plasma/PeleLMeX_EFTransport.cpp index 9aa18fa5f..7649cd620 100644 --- a/Source/Plasma/PeleLMeX_EFTransport.cpp +++ b/Source/Plasma/PeleLMeX_EFTransport.cpp @@ -2,7 +2,7 @@ #include void -PeleLM::calcEFTransport(const TimeStamp& a_time) +PeleLM::calcEFTransport(const TimeStamp a_time) { BL_PROFILE("PeleLMeX::calcEFTransport()"); diff --git a/Source/Plasma/PeleLMeX_EFUtils.cpp b/Source/Plasma/PeleLMeX_EFUtils.cpp index 4ccf6fdf2..20a5a0a60 100644 --- a/Source/Plasma/PeleLMeX_EFUtils.cpp +++ b/Source/Plasma/PeleLMeX_EFUtils.cpp @@ -63,7 +63,7 @@ void PeleLM::getNLStateScaling(amrex::Real& nEScale, amrex::Real& phiVScale) { amrex::Array r = {0.0, 0.0}; - for (int comp = 0; comp < 2; comp++) { + for (int comp = 0; comp < 2; ++comp) { for (int lev = 0; lev <= finest_level; ++lev) { if (lev != finest_level) { r[comp] = amrex::max( @@ -84,7 +84,7 @@ void PeleLM::getNLResidScaling(amrex::Real& nEScale, amrex::Real& phiVScale) { amrex::Array r = {0.0, 0.0}; - for (int comp = 0; comp < 2; comp++) { + for (int comp = 0; comp < 2; ++comp) { for (int lev = 0; lev <= finest_level; ++lev) { if (lev != finest_level) { r[comp] = amrex::max( @@ -102,7 +102,7 @@ PeleLM::getNLResidScaling(amrex::Real& nEScale, amrex::Real& phiVScale) } void -PeleLM::scaleNLState(const amrex::Real& nEScale, const amrex::Real& phiVScale) +PeleLM::scaleNLState(const amrex::Real nEScale, const amrex::Real phiVScale) { for (int lev = 0; lev <= finest_level; ++lev) { m_leveldatanlsolve[lev]->nlState.mult(1.0 / nE_scale, 0, 1, m_nGrowState); @@ -113,8 +113,8 @@ PeleLM::scaleNLState(const amrex::Real& nEScale, const amrex::Real& phiVScale) void PeleLM::scaleNLResid( const amrex::Vector& a_resid, - const amrex::Real& nEScale, - const amrex::Real& phiVScale) + const amrex::Real nEScale, + const amrex::Real phiVScale) { for (int lev = 0; lev <= finest_level; ++lev) { a_resid[lev]->mult(1.0 / FnE_scale, 0, 1, 1); @@ -124,14 +124,14 @@ PeleLM::scaleNLResid( amrex::BCRec PeleLM::hackBCChargedParticle( - const amrex::Real& charge, const amrex::BCRec& bc_in) + const amrex::Real charge, const amrex::BCRec& bc_in) { amrex::BCRec bc_hacked; const int* lo_bc = bc_in.lo(); const int* hi_bc = bc_in.hi(); - for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { int lo = lo_bc[idim]; int hi = hi_bc[idim]; // Spec is In/Out and it's cathode (neg electrode) @@ -181,9 +181,9 @@ PeleLM::hackBCChargedParticle( void PeleLM::addLorentzVelForces( - int lev, + const int lev, const amrex::Box& bx, - const amrex::Real& a_time, + const amrex::Real a_time, amrex::Array4 const& force, amrex::Array4 const& rhoY, amrex::Array4 const& phiV, @@ -193,9 +193,11 @@ PeleLM::addLorentzVelForces( amrex::GpuArray blo = bx.loVect3d(); amrex::GpuArray bhi = bx.hiVect3d(); - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - addLorentzForce(i, j, k, blo, bhi, a_time, dx, zk, rhoY, nE, phiV, force); - }); + amrex::ParallelFor( + bx, [blo, bhi, a_time, dx, zk = zk, rhoY, nE, phiV, + force] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + addLorentzForce(i, j, k, blo, bhi, a_time, dx, zk, rhoY, nE, phiV, force); + }); } void @@ -254,7 +256,9 @@ PeleLM::initializeElectronFromMassFraction() void PeleLM::fillPatchExtrap( - amrex::Real a_time, amrex::Vector const& a_MF, int a_nGrow) + const amrex::Real a_time, + amrex::Vector const& a_MF, + const int a_nGrow) { AMREX_ASSERT(a_MF[0]->nComp() <= m_bcrec_force.size()); const int nComp = a_MF[0]->nComp(); @@ -375,14 +379,14 @@ PeleLM::ionsBalance() { // Compute the sum of ions on the domain boundaries amrex::Array ionsCurrent{0.0}; - for (int n = NUM_SPECIES - NUM_IONS; n < NUM_SPECIES; n++) { - for (int i = 0; i < 2 * AMREX_SPACEDIM; i++) { + for (int n = NUM_SPECIES - NUM_IONS; n < NUM_SPECIES; ++n) { + for (int i = 0; i < 2 * AMREX_SPACEDIM; ++i) { ionsCurrent[i] += m_domainRhoYFlux[2 * n * AMREX_SPACEDIM + i] * zk[n]; } } tmpIonsFile << m_nstep << "," << m_cur_time; // Time info - for (int i = 0; i < 2 * AMREX_SPACEDIM; i++) { + for (int i = 0; i < 2 * AMREX_SPACEDIM; ++i) { tmpIonsFile << "," << ionsCurrent[i]; // ions current as xlo, xhi, ylo, ... } tmpIonsFile << "\n"; diff --git a/Source/Plasma/PeleLMeX_EF_K.H b/Source/Plasma/PeleLMeX_EF_K.H index 1669a70fa..cf5f6fbdb 100644 --- a/Source/Plasma/PeleLMeX_EF_K.H +++ b/Source/Plasma/PeleLMeX_EF_K.H @@ -29,12 +29,12 @@ getKappaE( // Get rho & Y from rhoY amrex::Real rho = 0.0; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { rho += rhoY(i, j, k, n); } amrex::Real rhoinv = 1.0 / rho; amrex::Real y[NUM_SPECIES] = {0.0}; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { y[n] = rhoY(i, j, k, n) * rhoinv; } @@ -77,11 +77,11 @@ getKappaSp( amrex::Array4 const& Ksp) noexcept { amrex::Real rho = 0.0; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { rho += rhoY(i, j, k, n); } int ionCnt = 0; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { if (a_zk[n] != 0.0) { Ksp(i, j, k, ionCnt) = rhoD(i, j, k, n) * mwt[n] * 0.001 * a_zk[n] / (rho * PP_RU_MKS * T(i, j, k)); @@ -262,7 +262,7 @@ cen2edg_upwind( id_h[dir] -= 1; } - for (int n = 0; n < ncomp; n++) { + for (int n = 0; n < ncomp; ++n) { if (ueff(i, j, k) > smallVel) { efab(i, j, k, n) = cfab(id_l[0], id_l[1], id_l[2], n); } else if (ueff(i, j, k) < -smallVel) { @@ -290,12 +290,12 @@ reactionRateRhoY_EF( { // Get rho from rhoY. Is there a better way ? amrex::Real rho = 0.0; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { rho += rhoY(i, j, k, n); } amrex::Real rhoinv = 1.0 / rho; amrex::Real y[NUM_SPECIES] = {0.0}; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { y[n] = rhoY(i, j, k, n) * rhoinv; } @@ -315,7 +315,7 @@ reactionRateRhoY_EF( rho = rho * 0.001; // rho MKS -> CGS amrex::Real wdot[NUM_SPECIES] = {0.0}; eos.RTY2WDOT(rho, Tloc, y, wdot); - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { rhoYdot(i, j, k, n) = wdot[n] * 1000.0; // CGS -> MKS conversion } nEdot(i, j, k) = wdot[E_ID] * Na * 1.0e6; @@ -366,7 +366,7 @@ addLorentzForce( EFz = -0.5 / dx[2] * (phiV(i, j, k + 1) - phiV(i, j, k - 1)); } #endif - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { force(i, j, k, 0) += a_zk[n] * rhoY(i, j, k, n) * EFx; force(i, j, k, 1) += a_zk[n] * rhoY(i, j, k, n) * EFy; #if (AMREX_SPACEDIM == 3) @@ -399,14 +399,14 @@ initElecNeutral( // Get species charge/mass amrex::Real zk[NUM_SPECIES] = {0.0}; pele::physics::eos::charge_mass(zk); - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { zk[n] *= 1000.0; // CGS->MKS } // Get mass fractions amrex::Real massfrac[NUM_SPECIES] = {0.0}; amrex::Real rho_inv = 1.0 / rho(i, j, k); - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { massfrac[n] = rhoY(i, j, k, n) * rho_inv; } @@ -415,7 +415,7 @@ initElecNeutral( // First remove Y_nE massfrac[E_ID] = 0.0; amrex::Real sum = 0.0; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { massfrac[n] = std::max(0.0, massfrac[n]); if (n != N2_ID) sum += massfrac[n]; @@ -424,20 +424,20 @@ initElecNeutral( // Change ions massfrac where charge already < 0.0 amrex::Real chargeDist = 0.0; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { chargeDist += massfrac[n] * zk[n]; } if (chargeDist < 0.0) { amrex::Real sum_pos_ion = 0.0; amrex::Real mean_zk_pos_ion = 0.0; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { if (zk[n] > 0.0) { sum_pos_ion += massfrac[n]; mean_zk_pos_ion += massfrac[n] * zk[n]; } } if (sum_pos_ion > 0.0) { - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { if (zk[n] > 0.0) { massfrac[N2_ID] += massfrac[n] / sum_pos_ion * chargeDist / mean_zk_pos_ion * sum_pos_ion; @@ -449,7 +449,7 @@ initElecNeutral( } // Add Electrons chargeDist = 0.0; - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { chargeDist += massfrac[n] * zk[n]; } nE(i, j, k) = std::max(1.0e-24, chargeDist / elemCharge); @@ -467,7 +467,7 @@ initElecNeutral( rhoH(i, j, k) = h_cgs * 1.0e-4 * rho(i, j, k); // Species mass - for (int n = 0; n < NUM_SPECIES; n++) { + for (int n = 0; n < NUM_SPECIES; ++n) { rhoY(i, j, k, n) = massfrac[n] * rho(i, j, k); } diff --git a/Source/main.cpp b/Source/main.cpp index b06bdeb93..1a06d3957 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -10,7 +10,7 @@ int main(int argc, char* argv[]) { if (argc >= 2) { - for (auto i = 1; i < argc; i++) { + for (auto i = 1; i < argc; ++i) { if (std::string(argv[i]) == "--describe") { writeBuildInfo(); return 0;