From 99b24aed8d1b0934f0e2ffc3b903e09382ba120a Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Tue, 11 Mar 2025 11:00:45 -0600 Subject: [PATCH 1/9] MueLu: Initial version of the KokkosTuning interface that compiles, but isn't hooked up to anything Signed-off-by: Chris Siefert --- packages/muelu/doc/UsersGuide/masterList.xml | 12 + .../muelu/doc/UsersGuide/paramlist_hidden.tex | 2 + .../muelu/src/MueCentral/MueLu_MasterList.cpp | 3 + .../src/Utils/MueLu_KokkosTuningInterface.cpp | 221 ++++++++++++++++++ .../src/Utils/MueLu_KokkosTuningInterface.hpp | 77 ++++++ 5 files changed, 315 insertions(+) create mode 100644 packages/muelu/src/Utils/MueLu_KokkosTuningInterface.cpp create mode 100644 packages/muelu/src/Utils/MueLu_KokkosTuningInterface.hpp diff --git a/packages/muelu/doc/UsersGuide/masterList.xml b/packages/muelu/doc/UsersGuide/masterList.xml index 0ccab26c6b24..bf2eb3c0e629 100644 --- a/packages/muelu/doc/UsersGuide/masterList.xml +++ b/packages/muelu/doc/UsersGuide/masterList.xml @@ -1987,6 +1987,18 @@ optimized neighbor discovery for Importer construction. + + + kokkos tuning: muelu parameter mapping + \parameterlist + Sublist for Kokkos Tuning of MueLu Parameters + false + not supported by ML + + + + + diff --git a/packages/muelu/doc/UsersGuide/paramlist_hidden.tex b/packages/muelu/doc/UsersGuide/paramlist_hidden.tex index d46fe00445b3..3577ed7a43ad 100644 --- a/packages/muelu/doc/UsersGuide/paramlist_hidden.tex +++ b/packages/muelu/doc/UsersGuide/paramlist_hidden.tex @@ -447,6 +447,8 @@ \cba{amgx:params}{\parameterlist}{Sublist for listing AMGX configuration parameters} +\cba{kokkos tuning: muelu parameter mapping}{\parameterlist}{Sublist for Kokkos Tuning of MueLu Parameters} + \cbb{debug: graph level}{int}{-2}{Output dependency graph on level X (use -1 for all levels).} \cbb{maxwell1: mode}{string}{"standard"}{Specifying the order of solve of the block system. Allowed values are: "standard" (default), "refmaxwell"} diff --git a/packages/muelu/src/MueCentral/MueLu_MasterList.cpp b/packages/muelu/src/MueCentral/MueLu_MasterList.cpp index f2e040c8cb9a..38406af47ca3 100644 --- a/packages/muelu/src/MueCentral/MueLu_MasterList.cpp +++ b/packages/muelu/src/MueCentral/MueLu_MasterList.cpp @@ -338,6 +338,7 @@ namespace MueLu { "" "" "" + "" "" "" "" @@ -946,6 +947,8 @@ namespace MueLu { ("amgx:params","amgx:params") + ("kokkos tuning: muelu parameter mapping","kokkos tuning: muelu parameter mapping") + ("debug: graph level","debug: graph level") ("maxwell1: mode","maxwell1: mode") diff --git a/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.cpp b/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.cpp new file mode 100644 index 000000000000..45cc6dbe4912 --- /dev/null +++ b/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.cpp @@ -0,0 +1,221 @@ +// @HEADER +// ***************************************************************************** +// MueLu: A package for multigrid based preconditioning +// +// Copyright 2012 NTESS and the MueLu contributors. +// SPDX-License-Identifier: BSD-3-Clause +// ***************************************************************************** +// @HEADER + +#include "MueLu_KokkosTuningInterface.hpp" + +#include +#include +#include "Teuchos_Array.hpp" +#include "Teuchos_CommHelpers.hpp" +#include "Teuchos_RawParameterListHelpers.hpp" +#include "MueLu_BaseClass.hpp" +#include "MueLu_Exceptions.hpp" + + + +// *********************************************************************** +/* Notional Parameterlist Structure + "kokkos tuning: muelu parameter mapping" + - "input variables" "{"Chebyshev","parallel_for"} + - "param0" + - "muelu parameter" "smoother: sweeps" + - "discrete range" "{1,6,1}" # (low, high, step') + - "initial guess" "2" + - "param1" + - "muelu parameter" "chebyshev: eigenvalue ratio" + - "continuous range" "{5.0,50.0,5.0}" # (low, high, step') + - "initial guess" "10.0" + + + The input variables should be handled by the tuning tool. + The output variable should be calculated automatically based on the types above + */ + + +namespace MueLu { + + // FIXME: Will probably need to bump this + namespace KokkosTuningParams { + const int MAX_VALID_PARAMS=10; + }; + +// *********************************************************************** +RCP KokkosTuningInterface::GetValidParameterList() const { + RCP topValidParamList = rcp(new ParameterList()); + ParameterList validParamList; + + ParameterList pl_dummy; + Teuchos::Array ar_dummy; + std::string s_dummy; + + + // Input variables for Kokkos tuning + validParamList.set>("input variables", ar_dummy, "Names of the input variables for Kokkos tuning"); + + for(int i=0;i(name,pl_dummy,"Parameter-specific sublist"); + } + + topValidParamList->set("kokkos tuning: muelu parameter mapping",validParamList,"Sublist for Kokkos tuning of MueLu"); + + return topValidParamList; +} + + + +// *********************************************************************** +void KokkosTuningInterface::Setup() { + // Sanity check + if (comm_.is_null()) throw std::runtime_error("MueLu::KokkosTuningInterface::Setup(): Communicator cannot be null"); + + // Unpack the MueLu Mapping into something actionable + UnpackMueLuMapping(); +} + + +// *********************************************************************** +void KokkosTuningInterface::UnpackMueLuMapping() { + const Teuchos::ParameterList& pL = params_.get("kokkos tuning: muelu parameter mapping"); + namespace KTE = Kokkos::Tools::Experimental; + + + /********************************/ + /* Process the output variables */ + /********************************/ + out_variables.clear(); + out_names.clear(); + + for(int i=0;i("muelu parameter"); + + // Infer types from initial guess + if(sublist.isType("initial guess")) { + // Discrete range + int guess = sublist.get("initial guess"); + const Teuchos::Array & range = sublist.get >("discrete range"); + TEUCHOS_TEST_FOR_EXCEPTION(range.size() !=3, Exceptions::RuntimeError, "MueLu::KokkosTuningInterface: 'discrete range' needs to be (low, high, step)"); + + // Copy the range to int64_t because Kokkos + std::vector range64(range.size()); + for(int j=0;j<(int)range.size(); j++) + range64[j] = range[j]; + int64_ranges.push_back(range64); + + + // Set the VariableInfo + KTE::VariableInfo out_info; + out_info.type = Kokkos::Tools::Experimental::ValueType::kokkos_value_int64; + out_info.category = Kokkos::Tools::Experimental::StatisticalCategory::kokkos_value_ordinal; + out_info.valueQuantity = Kokkos::Tools::Experimental::CandidateValueType::kokkos_value_set; + out_info.candidates = Kokkos::Tools::Experimental::make_candidate_set(3,int64_ranges[int64_ranges.size()-1].data()); + size_t var_id = Kokkos::Tools::Experimental::declare_output_type(muelu_param,out_info); + out_variables.push_back(KTE::make_variable_value(var_id,int64_t(guess))); + + out_typenames.push_back("int"); + } + else if(sublist.isType("initial guess")) { + // Continuous range + double guess = sublist.get("initial guess"); + const Teuchos::Array & range = sublist.get >("continuous range"); + TEUCHOS_TEST_FOR_EXCEPTION(range.size() !=3, Exceptions::RuntimeError, "MueLu::KokkosTuningInterface: 'continuous range' needs to be (low, high, step)"); + + + // Copy the range because Kokkos + std::vector rangeD(range.size()); + for(int j=0;j<(int)range.size(); j++) + rangeD[j] = range[j]; + double_ranges.push_back(rangeD); + + + // Set the VariableInfo + KTE::VariableInfo out_info; + out_info.type = Kokkos::Tools::Experimental::ValueType::kokkos_value_double; + out_info.category = Kokkos::Tools::Experimental::StatisticalCategory::kokkos_value_ordinal; + out_info.valueQuantity = Kokkos::Tools::Experimental::CandidateValueType::kokkos_value_set; + // FIXME: This is a memory error waiting to happen + out_info.candidates = Kokkos::Tools::Experimental::make_candidate_set(3,double_ranges[double_ranges.size()-1].data()); + size_t var_id = Kokkos::Tools::Experimental::declare_output_type(muelu_param,out_info); + out_variables.push_back(KTE::make_variable_value(var_id,guess)); + + out_typenames.push_back("double"); + } + /*else if(sublist.isType("initial guess")) { + // Categorical + + }*/ + else { + TEUCHOS_TEST_FOR_EXCEPTION(true, Exceptions::RuntimeError, "MueLu::KokkosTuningInterface: Only handles int and double parameters"); + } + + // Stash the parameter name + out_names.push_back(muelu_param); + + }// end if pL.isSublist + + }// end for + + // Sanity check + TEUCHOS_TEST_FOR_EXCEPTION(out_names.size() != out_variables.size(), Exceptions::RuntimeError, "MueLu::KokkosTuningInterface: Error in option processing"); + TEUCHOS_TEST_FOR_EXCEPTION(out_names.size() != out_typenames.size(), Exceptions::RuntimeError, "MueLu::KokkosTuningInterface: Error in option processing"); + + /********************************/ + /* Process the input variables */ + /********************************/ + in_variables.clear(); + + const Teuchos::Array & inputs = pL.get >("input_variables"); + for (int i=0; i< (int)inputs.size(); i++) { + in_variables.push_back(KTE::make_variable_value(i,inputs[i].c_str())); + } + + +} + + +// *********************************************************************** + void KokkosTuningInterface::SetMueLuParameters(size_t kokkos_context_id, Teuchos::ParameterList& mueluParams, bool overwrite) const { + namespace KTE = Kokkos::Tools::Experimental; + Teuchos::ParameterList tunedParams; + + if (comm_->getRank() == 0) { + // Only Rank 0 calls KokkosTuning + GetOStream(Runtime0) << "MueLu::KokkosTuningInterface: Tuning " << out_variables.size() << " parameters" < +#include +#include "Teuchos_Comm.hpp" +#include "Teuchos_RCP.hpp" +#include "Teuchos_ArrayRCP.hpp" +#include "Teuchos_ParameterList.hpp" +#include "MueLu_BaseClass.hpp" + +namespace MueLu { + +/*! @class + Manages the interface to the KokkosTuning interface + + Note only proc 0 (as defined by comm) will actually have kokkos-tuning instantiated. The options determined + by kokkostune will be broadcast to all processors +*/ +class KokkosTuningInterface : public BaseClass { + public: + KokkosTuningInterface(Teuchos::RCP >& comm) + : comm_(comm) {} + + // KokkosTuningInterface(Teuchos::RCP >& comm, Teuchos::ParameterList& inParams) + // : comm_(comm) + // , params_(inParams){}; + + virtual ~KokkosTuningInterface() { } + + Teuchos::RCP GetValidParameterList() const; + + // Sets the input parameters for the KokkosTuneInterface + // NOTE: These are *not* the parameters which KokkosTuning is going to overwrite, rather the + // parameters to tell KokkosTuning what to do. + void SetParameterList(Teuchos::ParameterList& inParams) { params_ = inParams; } + + // Sets up Kokkos Tuning + void Setup(); + + // Calls Kokkos Tuning to set MueLu Parameters + void SetMueLuParameters(size_t kokkos_context_id, Teuchos::ParameterList& mueluParams, bool overwrite = true) const; + + private: + // Utility functions + void UnpackMueLuMapping(); + + // Cached data + Teuchos::RCP > comm_; + mutable Teuchos::ParameterList params_; // The mutable is a hack to deal with issues in Teuchos + + + // Tuning information + std::vector in_variables; + mutable std::vector out_variables; + std::vector out_names; + std::vector out_typenames; + + // FIXME: Dealing with Kokkos grabbing the pointer + std::vector > int64_ranges; + std::vector > double_ranges; + + +}; + +} // namespace MueLu + +#endif // MUELU_KOKKOSTUNEINTERFACE_HPP From e2308d7a25db0128619ffba534bb5d26202d5b5f Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Wed, 12 Mar 2025 16:21:27 -0600 Subject: [PATCH 2/9] MueLu: Adding kokkos tune support to driver Signed-off-by: Chris Siefert --- .../src/Utils/MueLu_KokkosTuningInterface.hpp | 6 +- packages/muelu/test/scaling/Driver.cpp | 161 ++++++++++++------ packages/muelu/test/scaling/DriverCore.hpp | 1 + 3 files changed, 111 insertions(+), 57 deletions(-) diff --git a/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.hpp b/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.hpp index 46bd465eddc2..b4f070ce88b4 100644 --- a/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.hpp +++ b/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.hpp @@ -44,9 +44,6 @@ class KokkosTuningInterface : public BaseClass { // parameters to tell KokkosTuning what to do. void SetParameterList(Teuchos::ParameterList& inParams) { params_ = inParams; } - // Sets up Kokkos Tuning - void Setup(); - // Calls Kokkos Tuning to set MueLu Parameters void SetMueLuParameters(size_t kokkos_context_id, Teuchos::ParameterList& mueluParams, bool overwrite = true) const; @@ -54,6 +51,9 @@ class KokkosTuningInterface : public BaseClass { // Utility functions void UnpackMueLuMapping(); + // Sets up Kokkos Tuning - This gets called from SetParameterList + void Setup(); + // Cached data Teuchos::RCP > comm_; mutable Teuchos::ParameterList params_; // The mutable is a hack to deal with issues in Teuchos diff --git a/packages/muelu/test/scaling/Driver.cpp b/packages/muelu/test/scaling/Driver.cpp index c29ad36f18cf..85b38a5fd3b4 100644 --- a/packages/muelu/test/scaling/Driver.cpp +++ b/packages/muelu/test/scaling/Driver.cpp @@ -19,6 +19,9 @@ #include #include +// Kokkos +#include + // Xpetra #include #include @@ -83,6 +86,8 @@ #include "Xpetra_EpetraMultiVector.hpp" #endif +#include "MueLu_KokkosTuningInterface.hpp" + /*********************************************************************/ #include "KokkosBlas1_abs_impl.hpp" @@ -243,6 +248,8 @@ int main_(Teuchos::CommandLineProcessor& clp, Xpetra::UnderlyingLib& lib, int ar clp.setOption("rebuild", &numRebuilds, "#times to rebuild hierarchy"); int numResolves = 0; clp.setOption("resolve", &numResolves, "#times to redo solve"); + int numLoops = 1; + clp.setOption("loops", &numLoops, "#times to do setup/solve pairs. This is multiplicative with numRebuilds and numSolves (defaults to 1)."); int maxIts = 200; clp.setOption("its", &maxIts, "maximum number of solver iterations"); int numVectors = 1; @@ -258,15 +265,14 @@ int main_(Teuchos::CommandLineProcessor& clp, Xpetra::UnderlyingLib& lib, int ar std::string equilibrate = "no"; clp.setOption("equilibrate", &equilibrate, "equilibrate the system (no | diag | 1-norm)"); -#ifdef HAVE_MUELU_CUDA + bool profileSetup = false; - clp.setOption("cuda-profile-setup", "no-cuda-profile-setup", &profileSetup, "enable CUDA profiling for setup"); bool profileSolve = false; +#ifdef HAVE_MUELU_CUDA + clp.setOption("cuda-profile-setup", "no-cuda-profile-setup", &profileSetup, "enable CUDA profiling for setup"); clp.setOption("cuda-profile-solve", "no-cuda-profile-solve", &profileSolve, "enable CUDA profiling for solve"); -#else - bool profileSetup = false; - bool profileSolve = false; #endif + int cacheSize = 0; clp.setOption("cachesize", &cacheSize, "cache size (in KB)"); #ifdef HAVE_MPI @@ -283,6 +289,12 @@ int main_(Teuchos::CommandLineProcessor& clp, Xpetra::UnderlyingLib& lib, int ar clp.setOption("filesuffix", &rerunFileSuffix, "if doing reruns, optional suffix to append to output files"); std::string levelPerformanceModel = "no"; clp.setOption("performance-model", &levelPerformanceModel, "runs the level-by-level performance mode options- 'no', 'yes' or 'verbose'"); + + bool kokkosTuning = false; +#ifdef KOKKOS_ENABLE_TUNING + clp.setOption("kokkos-tuning", "no-kokkos-tuning", &kokkosTuning, "enable Kokkos tuning inferface"); +#endif + clp.recogniseAllOptions(true); switch (clp.parse(argc, argv)) { @@ -390,6 +402,9 @@ int main_(Teuchos::CommandLineProcessor& clp, Xpetra::UnderlyingLib& lib, int ar RCP X; RCP B; + // Create the Kokkos Tuning Interface + MueLu::KokkosTuningInterface KokkosTuner(comm); + // Load the matrix off disk (or generate it via Galeri) MatrixLoad(comm, lib, binaryFormat, matrixFile, rhsFile, rowMapFile, colMapFile, domainMapFile, rangeMapFile, coordFile, coordMapFile, nullFile, materialFile, map, A, coordinates, nullspace, material, X, B, numVectors, galeriParameters, xpetraParameters, galeriStream); comm->barrier(); @@ -472,6 +487,8 @@ int main_(Teuchos::CommandLineProcessor& clp, Xpetra::UnderlyingLib& lib, int ar int runCount = 1; int savedOut = -1; FILE* openedOut = NULL; + + // NOTE: This do loop is for handling multiple input decks in a single run do { solveType = dsolveType; tol = dtol; @@ -508,9 +525,8 @@ int main_(Teuchos::CommandLineProcessor& clp, Xpetra::UnderlyingLib& lib, int ar out2.setOutputToRootOnly(0); out2 << galeriStream.str(); - // ========================================================================= - // Preconditioner construction - // ========================================================================= + + // Preconditioner control options bool useAMGX = mueluList.isParameter("use external multigrid package") && (mueluList.get("use external multigrid package") == "amgx"); bool useML = mueluList.isParameter("use external multigrid package") && (mueluList.get("use external multigrid package") == "ml"); #ifdef HAVE_MPI @@ -519,64 +535,101 @@ int main_(Teuchos::CommandLineProcessor& clp, Xpetra::UnderlyingLib& lib, int ar userParamList.set("Node Comm", nodeComm); } #endif - RCP H; - RCP Prec; - // Build the preconditioner numRebuilds+1 times - if (solvePreconditioned) { - out2 << "*********** MueLu ParameterList ***********" << std::endl; - out2 << mueluList; - out2 << "*******************************************" << std::endl; - - MUELU_SWITCH_TIME_MONITOR(tm, "Driver: 2 - MueLu Setup"); - PreconditionerSetup(A, coordinates, nullspace, material, mueluList, profileSetup, useAMGX, useML, setNullSpace, numRebuilds, H, Prec); + // Get a Kokkos context for tuning and setup the tuner + size_t kokkos_context_id = 0; + if(kokkosTuning) { + kokkos_context_id = Kokkos::Tools::Experimental::get_new_context_id(); + KokkosTuner.SetParameterList(mueluList); } - comm->barrier(); - tm = Teuchos::null; - - size_t mem = get_current_memory_usage(); - out2 << "Memory use after preconditioner setup (GB): " << (mem / 1024.0 / 1024.0) << std::endl; // ========================================================================= - // System solution (Ax = b) + // Loop over the setup/solve pairs // ========================================================================= - try { + for(int l=0; l H; + RCP Prec; + // Build the preconditioner numRebuilds+1 times + if (solvePreconditioned) { + out2 << "*********** MueLu ParameterList ***********" << std::endl; + out2 << mueluList; + out2 << "*******************************************" << std::endl; + + MUELU_SWITCH_TIME_MONITOR(tm, "Driver: 2 - MueLu Setup"); + + PreconditionerSetup(A, coordinates, nullspace, material, mueluList, profileSetup, useAMGX, useML, setNullSpace, numRebuilds, H, Prec); + + } comm->barrier(); - if (writeMatricesOPT > -2) { - tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("Driver: 3.5 - Matrix output"))); - H->Write(writeMatricesOPT, writeMatricesOPT); - if (writeMatricesOPT == 0 || writeMatricesOPT == -1) { - Xpetra::IO::Write("b_0.m", *B); + tm = Teuchos::null; + + size_t mem = get_current_memory_usage(); + out2 << "Memory use after preconditioner setup (GB): " << (mem / 1024.0 / 1024.0) << std::endl; + + + // Matrix output: first time only + comm->barrier(); + if(l==0) { + if (writeMatricesOPT > -2) { + tm = rcp(new TimeMonitor(*TimeMonitor::getNewTimer("Driver: 3.5 - Matrix output"))); + H->Write(writeMatricesOPT, writeMatricesOPT); + if (writeMatricesOPT == 0 || writeMatricesOPT == -1) { + Xpetra::IO::Write("b_0.m", *B); + } + tm = Teuchos::null; } - tm = Teuchos::null; } - + // ========================================================================= + // System solution (Ax = b) + // ========================================================================= // Solve the system numResolves+1 times - SystemSolve(A, X, B, H, Prec, out2, solveType, belosType, profileSolve, useAMGX, useML, cacheSize, numResolves, scaleResidualHist, solvePreconditioned, maxIts, tol, computeCondEst, enforceBoundaryConditionsOnInitialGuess); + try { + SystemSolve(A, X, B, H, Prec, out2, solveType, belosType, profileSolve, useAMGX, useML, cacheSize, numResolves, scaleResidualHist, solvePreconditioned, maxIts, tol, computeCondEst, enforceBoundaryConditionsOnInitialGuess); + + comm->barrier(); + } catch (const std::exception& e) { + if (isDriver) + out2 << "MueLu_Driver: solver crashed w/ message:" << e.what() << std::endl; + else + throw; + } comm->barrier(); - } catch (const std::exception& e) { - if (isDriver) - out2 << "MueLu_Driver: solver crashed w/ message:" << e.what() << std::endl; - else - throw; - } + tm = Teuchos::null; - tm = Teuchos::null; - - // If we want Level-specific performance model diagnostics, now is the time! - if ((levelPerformanceModel == "yes" || levelPerformanceModel == "verbose") && !H.is_null()) { - for (int i = 0; i < H->GetNumLevels(); i++) { - RCP level = H->GetLevel(i); - try { - RCP A_level = level->Get >("A"); - std::string level_name = std::string("Level-") + std::to_string(i) + std::string(": "); - std::vector timers; // MueLu: Laplace2D: Hierarchy: Solve (level=0) - MueLu::report_spmv_performance_models(A_level, 100, timers, globalTimeMonitor, level_name, levelPerformanceModel == "verbose"); - } catch (...) { - ; - } + if(kokkosTuning) { + Kokkos::Tools::Experimental::end_context(kokkos_context_id); } - } + + + // If we want Level-specific performance model diagnostics, now is the time! + if ((levelPerformanceModel == "yes" || levelPerformanceModel == "verbose") && !H.is_null()) { + for (int i = 0; i < H->GetNumLevels(); i++) { + RCP level = H->GetLevel(i); + try { + RCP A_level = level->Get >("A"); + std::string level_name = std::string("Level-") + std::to_string(i) + std::string(": "); + std::vector timers; // MueLu: Laplace2D: Hierarchy: Solve (level=0) + MueLu::report_spmv_performance_models(A_level, 100, timers, globalTimeMonitor, level_name, levelPerformanceModel == "verbose"); + } catch (...) { + ; + } + } + }// end performance diagnostics + + }//end loop over setup/solve pairs + globalTimeMonitor = Teuchos::null; if (useStackedTimer) diff --git a/packages/muelu/test/scaling/DriverCore.hpp b/packages/muelu/test/scaling/DriverCore.hpp index a439d501a3ef..3dbbe33afc60 100644 --- a/packages/muelu/test/scaling/DriverCore.hpp +++ b/packages/muelu/test/scaling/DriverCore.hpp @@ -134,6 +134,7 @@ void PreconditionerSetup(Teuchos::RCP Date: Thu, 13 Mar 2025 11:09:50 -0600 Subject: [PATCH 3/9] Minor fixes to tuning interface (and debugging statements) Signed-off-by: Chris Siefert --- packages/muelu/src/Utils/MueLu_KokkosTuningInterface.cpp | 4 +++- packages/muelu/src/Utils/MueLu_KokkosTuningInterface.hpp | 2 +- packages/muelu/test/scaling/Driver.cpp | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.cpp b/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.cpp index 45cc6dbe4912..7c8e8e55a2ea 100644 --- a/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.cpp +++ b/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.cpp @@ -89,6 +89,8 @@ void KokkosTuningInterface::UnpackMueLuMapping() { const Teuchos::ParameterList& pL = params_.get("kokkos tuning: muelu parameter mapping"); namespace KTE = Kokkos::Tools::Experimental; + // FIXME: Remove + std::cout<<"*** Params for Kokkos Tuning ***"< & inputs = pL.get >("input_variables"); + const Teuchos::Array & inputs = pL.get >("input variables"); for (int i=0; i< (int)inputs.size(); i++) { in_variables.push_back(KTE::make_variable_value(i,inputs[i].c_str())); } diff --git a/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.hpp b/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.hpp index b4f070ce88b4..417a12141853 100644 --- a/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.hpp +++ b/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.hpp @@ -42,7 +42,7 @@ class KokkosTuningInterface : public BaseClass { // Sets the input parameters for the KokkosTuneInterface // NOTE: These are *not* the parameters which KokkosTuning is going to overwrite, rather the // parameters to tell KokkosTuning what to do. - void SetParameterList(Teuchos::ParameterList& inParams) { params_ = inParams; } + void SetParameterList(Teuchos::ParameterList& inParams) { params_ = inParams; Setup();} // Calls Kokkos Tuning to set MueLu Parameters void SetMueLuParameters(size_t kokkos_context_id, Teuchos::ParameterList& mueluParams, bool overwrite = true) const; diff --git a/packages/muelu/test/scaling/Driver.cpp b/packages/muelu/test/scaling/Driver.cpp index 85b38a5fd3b4..8c5c4f6fb346 100644 --- a/packages/muelu/test/scaling/Driver.cpp +++ b/packages/muelu/test/scaling/Driver.cpp @@ -551,6 +551,7 @@ int main_(Teuchos::CommandLineProcessor& clp, Xpetra::UnderlyingLib& lib, int ar if(kokkosTuning) { // FIXME: Ideally we'd have the context bracket the solve only, not the setup, // but we need to know if that will work with Kokkos or not first. + out2<< "Enabling MueLu::KokkosTuning"< Date: Thu, 13 Mar 2025 14:35:17 -0600 Subject: [PATCH 4/9] MueLu: Adding nested list support to KTI Signed-off-by: Chris Siefert --- .../src/Utils/MueLu_KokkosTuningInterface.cpp | 48 +++++++++++++++++-- .../src/Utils/MueLu_KokkosTuningInterface.hpp | 5 +- packages/muelu/test/scaling/Driver.cpp | 3 ++ 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.cpp b/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.cpp index 7c8e8e55a2ea..cf7d8c692b32 100644 --- a/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.cpp +++ b/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.cpp @@ -190,9 +190,28 @@ void KokkosTuningInterface::UnpackMueLuMapping() { } +// *********************************************************************** +std::vector KokkosTuningInterface::SplitString(const std::string& base_string, const std::string& delimiter) const { + // Props to Sandia AI Chat for this routine + std::vector tokens; + size_t start = 0; + + size_t end = base_string.find(delimiter); + + while (end != std::string::npos) { + tokens.push_back(base_string.substr(start, end - start)); + start = end + delimiter.length(); + end = base_string.find(delimiter, start); + } + + // And the final token... + tokens.push_back(base_string.substr(start, end)); + + return tokens; +} // *********************************************************************** - void KokkosTuningInterface::SetMueLuParameters(size_t kokkos_context_id, Teuchos::ParameterList& mueluParams, bool overwrite) const { +void KokkosTuningInterface::SetMueLuParameters(size_t kokkos_context_id, Teuchos::ParameterList& mueluParams, bool overwrite) const { namespace KTE = Kokkos::Tools::Experimental; Teuchos::ParameterList tunedParams; @@ -205,18 +224,39 @@ void KokkosTuningInterface::UnpackMueLuMapping() { // Unpack the tuned values for(int i=0; i<(int)out_names.size(); i++) { - // FIXME: Allow for '/' separated sublisted params + + // Because we'll want to set parameters inside sublists we'll allow the "muelu parameter" option to specify sublists with '||' as a nesting delimiter + // That's really, really unlikely to be part of a parameter list item name, so we'll go with it. + Teuchos::ParameterList *activeList = &tunedParams; + std::vector treeWalk = SplitString(out_names[i],"||"); + + for(int j=0; j<(int) treeWalk.size(); j++) + std::cout<<"["<sublist(treeWalk[j])); + } + + std::string activeName=treeWalk[treeWalk.size()-1]; + if (out_typenames[i] == "int") - tunedParams.set(out_names[i], (int) out_variables[i].value.int_value); + activeList->set(activeName, (int) out_variables[i].value.int_value); else if (out_typenames[i] == "double") - tunedParams.set(out_names[i], out_variables[i].value.double_value); + activeList->set(activeName, out_variables[i].value.double_value); else { TEUCHOS_TEST_FOR_EXCEPTION(true, Exceptions::RuntimeError, "MueLu::KokkosTuningInterface: Unknown variable output type"); } } } +#if 0 + std::cout<<"*** FINAL TUNED PARAMS ***"< SplitString(const std::string& base_string, const std::string& delimiter) const; + // Sets up Kokkos Tuning - This gets called from SetParameterList void Setup(); diff --git a/packages/muelu/test/scaling/Driver.cpp b/packages/muelu/test/scaling/Driver.cpp index 8c5c4f6fb346..82c63aca4f32 100644 --- a/packages/muelu/test/scaling/Driver.cpp +++ b/packages/muelu/test/scaling/Driver.cpp @@ -293,8 +293,11 @@ int main_(Teuchos::CommandLineProcessor& clp, Xpetra::UnderlyingLib& lib, int ar bool kokkosTuning = false; #ifdef KOKKOS_ENABLE_TUNING clp.setOption("kokkos-tuning", "no-kokkos-tuning", &kokkosTuning, "enable Kokkos tuning inferface"); +#else + #error "KokkosTuning not enabled" #endif + clp.recogniseAllOptions(true); switch (clp.parse(argc, argv)) { From 901344cbd9c89cc8dc49b592b2797f5625b6a990 Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Thu, 13 Mar 2025 14:48:15 -0600 Subject: [PATCH 5/9] Minor fixes to tuning interface (and debugging statements) Signed-off-by: Chris Siefert --- .../muelu/src/Utils/MueLu_KokkosTuningInterface.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.cpp b/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.cpp index cf7d8c692b32..42cde14af1ed 100644 --- a/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.cpp +++ b/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.cpp @@ -222,6 +222,17 @@ void KokkosTuningInterface::SetMueLuParameters(size_t kokkos_context_id, Teuchos // Start the tuning KTE::request_output_values(kokkos_context_id, out_variables.size(), out_variables.data()); + // Diagnostic output + if (IsPrint(Runtime1)) { + GetOStream(Runtime1) << "Tuned Parameters: "< treeWalk = SplitString(out_names[i],"||"); for(int j=0; j<(int) treeWalk.size(); j++) - std::cout<<"["< Date: Fri, 14 Mar 2025 11:18:48 -0600 Subject: [PATCH 6/9] MueLu: Getting the finally working version of the KokkosTuning MueLu Driver Signed-off-by: Chris Siefert --- .../src/Utils/MueLu_KokkosTuningInterface.cpp | 73 ++++++++----------- .../src/Utils/MueLu_KokkosTuningInterface.hpp | 6 +- packages/muelu/test/scaling/Driver.cpp | 9 ++- 3 files changed, 38 insertions(+), 50 deletions(-) diff --git a/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.cpp b/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.cpp index 42cde14af1ed..bf5174f68aa6 100644 --- a/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.cpp +++ b/packages/muelu/src/Utils/MueLu_KokkosTuningInterface.cpp @@ -89,9 +89,6 @@ void KokkosTuningInterface::UnpackMueLuMapping() { const Teuchos::ParameterList& pL = params_.get("kokkos tuning: muelu parameter mapping"); namespace KTE = Kokkos::Tools::Experimental; - // FIXME: Remove - std::cout<<"*** Params for Kokkos Tuning ***"< & range = sublist.get >("discrete range"); TEUCHOS_TEST_FOR_EXCEPTION(range.size() !=3, Exceptions::RuntimeError, "MueLu::KokkosTuningInterface: 'discrete range' needs to be (low, high, step)"); - // Copy the range to int64_t because Kokkos - std::vector range64(range.size()); - for(int j=0;j<(int)range.size(); j++) - range64[j] = range[j]; - int64_ranges.push_back(range64); - - // Set the VariableInfo KTE::VariableInfo out_info; - out_info.type = Kokkos::Tools::Experimental::ValueType::kokkos_value_int64; - out_info.category = Kokkos::Tools::Experimental::StatisticalCategory::kokkos_value_ordinal; - out_info.valueQuantity = Kokkos::Tools::Experimental::CandidateValueType::kokkos_value_set; - out_info.candidates = Kokkos::Tools::Experimental::make_candidate_set(3,int64_ranges[int64_ranges.size()-1].data()); - size_t var_id = Kokkos::Tools::Experimental::declare_output_type(muelu_param,out_info); + out_info.type = KTE::ValueType::kokkos_value_int64; + out_info.category = KTE::StatisticalCategory::kokkos_value_interval; + out_info.valueQuantity = KTE::CandidateValueType::kokkos_value_range; + + // Unlike the ordinal lists, the ranges get copied into Kokkos + // TODO: Add support for open/closed ranges + out_info.candidates = KTE::make_candidate_range((int64_t)range[0],(int64_t)range[1],(int64_t)range[2],false,false); + size_t var_id = KTE::declare_output_type(muelu_param,out_info); out_variables.push_back(KTE::make_variable_value(var_id,int64_t(guess))); out_typenames.push_back("int"); @@ -138,32 +131,23 @@ void KokkosTuningInterface::UnpackMueLuMapping() { const Teuchos::Array & range = sublist.get >("continuous range"); TEUCHOS_TEST_FOR_EXCEPTION(range.size() !=3, Exceptions::RuntimeError, "MueLu::KokkosTuningInterface: 'continuous range' needs to be (low, high, step)"); - - // Copy the range because Kokkos - std::vector rangeD(range.size()); - for(int j=0;j<(int)range.size(); j++) - rangeD[j] = range[j]; - double_ranges.push_back(rangeD); - - // Set the VariableInfo KTE::VariableInfo out_info; - out_info.type = Kokkos::Tools::Experimental::ValueType::kokkos_value_double; - out_info.category = Kokkos::Tools::Experimental::StatisticalCategory::kokkos_value_ordinal; - out_info.valueQuantity = Kokkos::Tools::Experimental::CandidateValueType::kokkos_value_set; - // FIXME: This is a memory error waiting to happen - out_info.candidates = Kokkos::Tools::Experimental::make_candidate_set(3,double_ranges[double_ranges.size()-1].data()); - size_t var_id = Kokkos::Tools::Experimental::declare_output_type(muelu_param,out_info); + out_info.type = KTE::ValueType::kokkos_value_double; + out_info.category = KTE::StatisticalCategory::kokkos_value_interval; + out_info.valueQuantity = KTE::CandidateValueType::kokkos_value_range; + + // Unlike the ordinal lists, the ranges get copied into Kokkos + // TODO: Add support for open/closed ranges + out_info.candidates = KTE::make_candidate_range(range[0],range[1],range[2],false,false); + size_t var_id = KTE::declare_output_type(muelu_param,out_info); out_variables.push_back(KTE::make_variable_value(var_id,guess)); out_typenames.push_back("double"); } - /*else if(sublist.isType("initial guess")) { - // Categorical - - }*/ + // TODO: Add support for categorical and set parameters else { - TEUCHOS_TEST_FOR_EXCEPTION(true, Exceptions::RuntimeError, "MueLu::KokkosTuningInterface: Only handles int and double parameters"); + TEUCHOS_TEST_FOR_EXCEPTION(true, Exceptions::RuntimeError, "MueLu::KokkosTuningInterface: We currently only handle int and double ranges."); } // Stash the parameter name @@ -183,8 +167,13 @@ void KokkosTuningInterface::UnpackMueLuMapping() { in_variables.clear(); const Teuchos::Array & inputs = pL.get >("input variables"); + for (int i=0; i< (int)inputs.size(); i++) { - in_variables.push_back(KTE::make_variable_value(i,inputs[i].c_str())); + // NOTE: The string name is copied in here (unlike double/int) so we don't need to cache. + KTE::VariableInfo in_info; + in_info.type = KTE::ValueType::kokkos_value_string; + size_t var_id = KTE::declare_input_type(inputs[i].c_str(),in_info); + in_variables.push_back(KTE::make_variable_value(var_id,inputs[i].c_str())); } @@ -219,6 +208,12 @@ void KokkosTuningInterface::SetMueLuParameters(size_t kokkos_context_id, Teuchos // Only Rank 0 calls KokkosTuning GetOStream(Runtime0) << "MueLu::KokkosTuningInterface: Tuning " << out_variables.size() << " parameters" < treeWalk = SplitString(out_names[i],"||"); - for(int j=0; j<(int) treeWalk.size(); j++) - // Walk down all but the last guy for(int j=0;j<(int) treeWalk.size()-1; j++) { activeList = &(activeList->sublist(treeWalk[j])); @@ -259,14 +252,8 @@ void KokkosTuningInterface::SetMueLuParameters(size_t kokkos_context_id, Teuchos } } } -#if 0 - std::cout<<"*** FINAL TUNED PARAMS ***"< in_variables; + mutable std::vector in_variables; mutable std::vector out_variables; std::vector out_names; std::vector out_typenames; - // FIXME: Dealing with Kokkos grabbing the pointer - std::vector > int64_ranges; - std::vector > double_ranges; - }; diff --git a/packages/muelu/test/scaling/Driver.cpp b/packages/muelu/test/scaling/Driver.cpp index 82c63aca4f32..6694cf5171dc 100644 --- a/packages/muelu/test/scaling/Driver.cpp +++ b/packages/muelu/test/scaling/Driver.cpp @@ -30,6 +30,9 @@ #include #include +// Tpetra +#include + // Galeri #include #include @@ -292,7 +295,7 @@ int main_(Teuchos::CommandLineProcessor& clp, Xpetra::UnderlyingLib& lib, int ar bool kokkosTuning = false; #ifdef KOKKOS_ENABLE_TUNING - clp.setOption("kokkos-tuning", "no-kokkos-tuning", &kokkosTuning, "enable Kokkos tuning inferface"); + clp.setOption("tuning-with-kokkos", "no-tuning-with-kokkos", &kokkosTuning, "enable Kokkos tuning inferface"); #else #error "KokkosTuning not enabled" #endif @@ -541,7 +544,6 @@ int main_(Teuchos::CommandLineProcessor& clp, Xpetra::UnderlyingLib& lib, int ar // Get a Kokkos context for tuning and setup the tuner size_t kokkos_context_id = 0; if(kokkosTuning) { - kokkos_context_id = Kokkos::Tools::Experimental::get_new_context_id(); KokkosTuner.SetParameterList(mueluList); } @@ -549,6 +551,9 @@ int main_(Teuchos::CommandLineProcessor& clp, Xpetra::UnderlyingLib& lib, int ar // Loop over the setup/solve pairs // ========================================================================= for(int l=0; l Date: Fri, 14 Mar 2025 13:31:20 -0600 Subject: [PATCH 7/9] MueLu: Adding KokkosTuning unit test Signed-off-by: Chris Siefert --- packages/muelu/test/unit_tests/CMakeLists.txt | 1 + .../muelu/test/unit_tests/KokkosTuning.cpp | 88 +++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 packages/muelu/test/unit_tests/KokkosTuning.cpp diff --git a/packages/muelu/test/unit_tests/CMakeLists.txt b/packages/muelu/test/unit_tests/CMakeLists.txt index cbc8ed6d459d..bea05262f432 100644 --- a/packages/muelu/test/unit_tests/CMakeLists.txt +++ b/packages/muelu/test/unit_tests/CMakeLists.txt @@ -39,6 +39,7 @@ APPEND_SET(SOURCES Level.cpp LowPrecision.cpp LWGraph.cpp + KokkosTuning.cpp MapTransferFactory.cpp Memory.cpp MueLu_UnitTests.cpp diff --git a/packages/muelu/test/unit_tests/KokkosTuning.cpp b/packages/muelu/test/unit_tests/KokkosTuning.cpp new file mode 100644 index 000000000000..ef33c4f7bf19 --- /dev/null +++ b/packages/muelu/test/unit_tests/KokkosTuning.cpp @@ -0,0 +1,88 @@ +// @HEADER +// ***************************************************************************** +// MueLu: A package for multigrid based preconditioning +// +// Copyright 2012 NTESS and the MueLu contributors. +// SPDX-License-Identifier: BSD-3-Clause +// ***************************************************************************** +// @HEADER + +#include +#include + +#include + +#include +#include + +#include + + +namespace MueLuTests { + +TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(KokkosTuningInterface, Basic, Scalar, LocalOrdinal, GlobalOrdinal, Node) { +#include + MUELU_TESTING_SET_OSTREAM; + MUELU_TESTING_LIMIT_SCOPE(Scalar, GlobalOrdinal, Node); + out << "version: " << MueLu::Version() << std::endl; + RCP > comm = TestHelpers::Parameters::getDefaultComm(); + + + + // Driver parameters (like you'd actually use) + Teuchos::ParameterList baseList; + Teuchos::ParameterList & pL = baseList.sublist("kokkos tuning: muelu parameter mapping"); + Teuchos::Array input_vars(1); input_vars[0] = "Parameter List Item"; + Teuchos::Array i_range{1,6,1}; + Teuchos::Array d_range{5.0,50.0,5.0}; + pL.set("input variables",input_vars); + pL.sublist("param0").set("muelu parameter","smoother: params||chebyshev: degree"); + pL.sublist("param0").set("discrete range",i_range); + pL.sublist("param0").set("initial guess",(int) 1); + pL.sublist("param1").set("muelu parameter","smoother: params||chebyshev: ratio eigenvalue"); + pL.sublist("param1").set("continuous range",d_range); + pL.sublist("param1").set("initial guess",(double) 10.0); + + + // Actually make the interface + MueLu::KokkosTuningInterface interface(comm); + interface.SetParameterList(baseList); + + // Call the tuner + size_t kokkos_context_id; + Kokkos::Tools::Experimental::begin_context(kokkos_context_id); + Teuchos::ParameterList outputList; + interface.SetMueLuParameters(kokkos_context_id,outputList); + Kokkos::Tools::Experimental::end_context(kokkos_context_id); + + // Check that the output has the varables set to something + TEST_EQUALITY(outputList.isSublist("smoother: params"),true); + TEST_EQUALITY(outputList.sublist("smoother: params").isParameter("chebyshev: degree"),true); + TEST_EQUALITY(outputList.sublist("smoother: params").isParameter("chebyshev: ratio eigenvalue"),true); + + +} + + +TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(KokkosTuningInterface, Advanced, Scalar, LocalOrdinal, GlobalOrdinal, Node) { +#include + MUELU_TESTING_SET_OSTREAM; + MUELU_TESTING_LIMIT_SCOPE(Scalar, GlobalOrdinal, Node); + out << "version: " << MueLu::Version() << std::endl; + + RCP > comm = TestHelpers::Parameters::getDefaultComm(); + + RCP interface = rcp(new MueLu::KokkosTuningInterface(comm)); + + TEST_INEQUALITY(interface, Teuchos::null); +} + + +#define MUELU_ETI_GROUP(Scalar, LO, GO, Node) \ + TEUCHOS_UNIT_TEST_TEMPLATE_4_INSTANT(KokkosTuningInterface, Basic, Scalar, LO, GO, Node) \ + TEUCHOS_UNIT_TEST_TEMPLATE_4_INSTANT(KokkosTuningInterface, Advanced, Scalar, LO, GO, Node) + + +#include + +} // namespace MueLuTests From 4f76e967ae613a366c2530f02f8000a6f1ad5984 Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Fri, 14 Mar 2025 15:49:37 -0600 Subject: [PATCH 8/9] MueLu: That wasn't supposed to go in Signed-off-by: Chris Siefert --- packages/muelu/test/scaling/Driver.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/muelu/test/scaling/Driver.cpp b/packages/muelu/test/scaling/Driver.cpp index 6694cf5171dc..88b075691bdf 100644 --- a/packages/muelu/test/scaling/Driver.cpp +++ b/packages/muelu/test/scaling/Driver.cpp @@ -296,8 +296,6 @@ int main_(Teuchos::CommandLineProcessor& clp, Xpetra::UnderlyingLib& lib, int ar bool kokkosTuning = false; #ifdef KOKKOS_ENABLE_TUNING clp.setOption("tuning-with-kokkos", "no-tuning-with-kokkos", &kokkosTuning, "enable Kokkos tuning inferface"); -#else - #error "KokkosTuning not enabled" #endif From a07c969faec41732aef0b4cd0d6930ae6683723b Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Fri, 14 Mar 2025 16:36:36 -0600 Subject: [PATCH 9/9] MueLu: Advanced test not working yet Signed-off-by: Chris Siefert --- .../muelu/test/unit_tests/KokkosTuning.cpp | 149 +++++++++++++++++- 1 file changed, 144 insertions(+), 5 deletions(-) diff --git a/packages/muelu/test/unit_tests/KokkosTuning.cpp b/packages/muelu/test/unit_tests/KokkosTuning.cpp index ef33c4f7bf19..7dbd448c4d79 100644 --- a/packages/muelu/test/unit_tests/KokkosTuning.cpp +++ b/packages/muelu/test/unit_tests/KokkosTuning.cpp @@ -9,6 +9,7 @@ #include #include +#include #include @@ -64,23 +65,161 @@ TEUCHOS_UNIT_TEST_TEMPLATE_4_DECL(KokkosTuningInterface, Basic, Scalar, LocalOrd } +namespace TestUtilities { + std::vector output_info; + std::vector input_info; + + const char* str_type(Kokkos_Tools_VariableInfo_ValueType x){ + if(x == Kokkos_Tools_VariableInfo_ValueType::kokkos_value_double) + return "double"; + else if(x == Kokkos_Tools_VariableInfo_ValueType::kokkos_value_int64) + return "int64"; + else if(x == Kokkos_Tools_VariableInfo_ValueType::kokkos_value_string) + return "string"; + else + return "unknown"; + } + + const char* str_category(const enum Kokkos_Tools_VariableInfo_StatisticalCategory x){ + if(x == Kokkos_Tools_VariableInfo_StatisticalCategory::kokkos_value_categorical) + return "categorical"; + else if(x == Kokkos_Tools_VariableInfo_StatisticalCategory::kokkos_value_ordinal) + return "ordinal"; + else if(x == Kokkos_Tools_VariableInfo_StatisticalCategory::kokkos_value_interval) + return "interval"; + else + return "unknown"; + } + + const char* str_candidatevalue(const enum Kokkos_Tools_VariableInfo_CandidateValueType x){ + if(x == Kokkos_Tools_VariableInfo_CandidateValueType::kokkos_value_set) + return "set"; + else if(x == Kokkos_Tools_VariableInfo_CandidateValueType::kokkos_value_range) + return "range"; + else if (x == Kokkos_Tools_VariableInfo_CandidateValueType::kokkos_value_unbounded) + return "unbounded"; + else + return "unknown"; + } + + void declare_input_type(const char* name, const size_t id, + Kokkos::Tools::Experimental::VariableInfo* info) { + // Yeah, we're aliasing the pointer. This is a test. Deal. + input_info.push_back(info); + } + + void declare_output_type(const char* name, const size_t id, + Kokkos::Tools::Experimental::VariableInfo* info) { + // Yeah, we're aliasing the pointer. This is a test. Deal. + output_info.push_back(info); + } + + void request_output_values(const size_t context, const size_t num_inputs, + const Kokkos::Tools::Experimental::VariableValue* inputs_in, + const size_t num_outputs, + Kokkos::Tools::Experimental::VariableValue* outputs_in) { + // This dummy callback will set the output value to one step more than the bottom guy in the rang + std::cout<<"\nDEBUG: request_ouput_values called with "<category == Kokkos_Tools_VariableInfo_StatisticalCategory::kokkos_value_interval && + info->valueQuantity == Kokkos_Tools_VariableInfo_CandidateValueType::kokkos_value_range) { + auto range = info->candidates.range; + + if(info->type == Kokkos_Tools_VariableInfo_ValueType::kokkos_value_int64) { + outputs_in[i].value.int_value = (int) (range.lower.int_value + range.step.int_value); + std::cout<<"Setting parameter "<type = "<type)<category)<<","<valueQuantity)< MUELU_TESTING_SET_OSTREAM; MUELU_TESTING_LIMIT_SCOPE(Scalar, GlobalOrdinal, Node); out << "version: " << MueLu::Version() << std::endl; - RCP > comm = TestHelpers::Parameters::getDefaultComm(); + MUELU_TESTING_SET_OSTREAM; + MUELU_TESTING_LIMIT_SCOPE(Scalar, GlobalOrdinal, Node); + out << "version: " << MueLu::Version() << std::endl; + RCP > comm = TestHelpers::Parameters::getDefaultComm(); + + + + // Driver parameters (like you'd actually use) + Teuchos::ParameterList baseList; + Teuchos::ParameterList & pL = baseList.sublist("kokkos tuning: muelu parameter mapping"); + Teuchos::Array input_vars(1); input_vars[0] = "Parameter List Item"; + Teuchos::Array i_range{1,6,1}; + Teuchos::Array d_range{5.0,50.0,5.0}; + pL.set("input variables",input_vars); + pL.sublist("param0").set("muelu parameter","smoother: params||chebyshev: degree"); + pL.sublist("param0").set("discrete range",i_range); + pL.sublist("param0").set("initial guess",(int) 1); + pL.sublist("param1").set("muelu parameter","smoother: params||chebyshev: ratio eigenvalue"); + pL.sublist("param1").set("continuous range",d_range); + pL.sublist("param1").set("initial guess",(double) 5.0); + + + // Set the callbacks + Kokkos::Tools::Experimental::set_declare_input_type_callback(TestUtilities::declare_input_type); + Kokkos::Tools::Experimental::set_declare_output_type_callback(TestUtilities::declare_output_type); + Kokkos::Tools::Experimental::set_request_output_values_callback(TestUtilities::request_output_values); + + // Actually make the interface + MueLu::KokkosTuningInterface interface(comm); + interface.SetParameterList(baseList); + + // Call the tuner + size_t kokkos_context_id; + Kokkos::Tools::Experimental::begin_context(kokkos_context_id); + Teuchos::ParameterList outputList; + interface.SetMueLuParameters(kokkos_context_id,outputList); + Kokkos::Tools::Experimental::end_context(kokkos_context_id); + + // Check that the output has the varables set to something + TEST_EQUALITY(outputList.isSublist("smoother: params"),true); + TEST_EQUALITY(outputList.sublist("smoother: params").isParameter("chebyshev: degree"),true); + TEST_EQUALITY(outputList.sublist("smoother: params").isParameter("chebyshev: ratio eigenvalue"),true); - RCP interface = rcp(new MueLu::KokkosTuningInterface(comm)); - TEST_INEQUALITY(interface, Teuchos::null); +#ifdef KOKKOS_ENABLE_TUNING + // Check that the variables are set to one step off the bottom + int degree = 2; + double ratio = 10.0; +#else + // Expect the default values + int degree = 1; + double ratio = 5.0; +#endif + TEST_EQUALITY(outputList.sublist("smoother: params").get("chebyshev: degree"),degree); + TEST_FLOATING_EQUALITY(outputList.sublist("smoother: params").get("chebyshev: ratio eigenvalue"),ratio,1e-10); } #define MUELU_ETI_GROUP(Scalar, LO, GO, Node) \ - TEUCHOS_UNIT_TEST_TEMPLATE_4_INSTANT(KokkosTuningInterface, Basic, Scalar, LO, GO, Node) \ - TEUCHOS_UNIT_TEST_TEMPLATE_4_INSTANT(KokkosTuningInterface, Advanced, Scalar, LO, GO, Node) + TEUCHOS_UNIT_TEST_TEMPLATE_4_INSTANT(KokkosTuningInterface, Basic, Scalar, LO, GO, Node) + + //TEUCHOS_UNIT_TEST_TEMPLATE_4_INSTANT(KokkosTuningInterface, Advanced, Scalar, LO, GO, Node) #include