Finish OneAPI MKL support, removing KK override

trilinos · Apr 19, 2024 · 1663147 · 1663147
1 parent 64e4632
commit 1663147
Show file tree

Hide file tree

Showing 12 changed files with 29 additions and 29 deletions.
diff --git a/cmake/TPLs/FindTPLMKL.cmake b/cmake/TPLs/FindTPLMKL.cmake
@@ -70,14 +70,28 @@
 # pseudorandom number generators.  That's why we require a header
 # file, to access the function declarations.
 
-IF(KOKKOS_ENABLE_SYCL)
+IF(Kokkos_ENABLE_SYCL)
   # For OneAPI MKL on GPU, use the CMake target
+  # Temporarily change CMAKE_CXX_COMPILER to icpx to convince MKL to add the DPCPP target
+  # If it sees that CMAKE_CXX_COMPILER is just ".../mpicxx", it won't do this.
+  SET(CMAKE_CXX_COMPILER_PREVIOUS "${CMAKE_CXX_COMPILER}")
+  SET(CMAKE_CXX_COMPILER "icpx")
   find_package(MKL)
-  IF(MKL_FOUND)
-    tribits_extpkg_create_imported_all_libs_target_and_config_file( MKL
-      INNER_FIND_PACKAGE_NAME  MKL
-      IMPORTED_TARGETS_FOR_ALL_LIBS  MKL::MKL)
-    ENDIF()
+  SET(CMAKE_CXX_COMPILER "${CMAKE_CXX_COMPILER_PREVIOUS}")
+  IF (NOT MKL_FOUND)
+    MESSAGE(FATAL_ERROR "MKL (as CMake package) was not found! This is required for SYCL+MKL")
+  ENDIF()
+  IF (NOT TARGET MKL::MKL_DPCPP)
+    MESSAGE(FATAL_ERROR "SYCL and MKL are enabled, but the target MKL_DPCPP wasn't found")
+  ENDIF()
+  tribits_extpkg_create_imported_all_libs_target_and_config_file( MKL
+    INNER_FIND_PACKAGE_NAME  MKL
+    IMPORTED_TARGETS_FOR_ALL_LIBS  MKL::MKL MKL::MKL_DPCPP)
+
+  #  TARGET_LINK_LIBRARIES(MKL INTERFACE MKL::MKL MKL::MKL_DPCPP)
+  IF (NOT TARGET MKL::all_libs)
+    MESSAGE(FATAL_ERROR "MKL::all_libs was not created as a target")
+  ENDIF()
 ELSEIF()
   # For host MKL, the single library libmkl_rt is sufficient
   TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( MKL

diff --git a/packages/kokkos-kernels/blas/tpls/KokkosBlas1_nrm1_tpl_spec_avail.hpp b/packages/kokkos-kernels/blas/tpls/KokkosBlas1_nrm1_tpl_spec_avail.hpp
@@ -116,8 +116,7 @@ KOKKOSBLAS1_NRM1_TPL_SPEC_AVAIL_ROCBLAS(Kokkos::complex<float>,
 // oneMKL
 #ifdef KOKKOSKERNELS_ENABLE_TPL_MKL
 
-#if defined(KOKKOS_ENABLE_SYCL) && \
-    !defined(KOKKOSKERNELS_ENABLE_TPL_MKL_SYCL_OVERRIDE)
+#if defined(KOKKOS_ENABLE_SYCL)
 
 #define KOKKOSBLAS1_NRM1_TPL_SPEC_AVAIL_MKL_SYCL(SCALAR, LAYOUT, MEMSPACE)     \
   template <class ExecSpace>                                                   \

diff --git a/packages/kokkos-kernels/blas/tpls/KokkosBlas1_nrm1_tpl_spec_decl.hpp b/packages/kokkos-kernels/blas/tpls/KokkosBlas1_nrm1_tpl_spec_decl.hpp
@@ -334,8 +334,7 @@ KOKKOSBLAS1_NRM1_TPL_SPEC_DECL_ROCBLAS(Kokkos::complex<double>,
 // oneMKL
 #ifdef KOKKOSKERNELS_ENABLE_TPL_MKL
 
-#if defined(KOKKOS_ENABLE_SYCL) && \
-    !defined(KOKKOSKERNELS_ENABLE_TPL_MKL_SYCL_OVERRIDE)
+#if defined(KOKKOS_ENABLE_SYCL)
 
 #include <KokkosBlas_tpl_spec.hpp>
 #include <oneapi/mkl/blas.hpp>

diff --git a/packages/kokkos-kernels/blas/tpls/KokkosBlas1_nrm2_tpl_spec_avail.hpp b/packages/kokkos-kernels/blas/tpls/KokkosBlas1_nrm2_tpl_spec_avail.hpp
@@ -88,8 +88,7 @@ KOKKOSBLAS1_NRM2_TPL_SPEC_AVAIL(Kokkos::LayoutLeft, Kokkos::HIP,
                                 Kokkos::HIPSpace)
 #endif
 
-#if defined(KOKKOSKERNELS_ENABLE_TPL_MKL) &&                \
-    !defined(KOKKOSKERNELS_ENABLE_TPL_MKL_SYCL_OVERRIDE) && \
+#if defined(KOKKOSKERNELS_ENABLE_TPL_MKL) && \
     defined(KOKKOS_ENABLE_SYCL)
 KOKKOSBLAS1_NRM2_TPL_SPEC_AVAIL(Kokkos::LayoutLeft, Kokkos::Experimental::SYCL,
                                 Kokkos::Experimental::SYCLDeviceUSMSpace)

diff --git a/packages/kokkos-kernels/blas/tpls/KokkosBlas1_nrm2_tpl_spec_decl.hpp b/packages/kokkos-kernels/blas/tpls/KokkosBlas1_nrm2_tpl_spec_decl.hpp
@@ -364,8 +364,7 @@ KOKKOSBLAS1_NRM2_TPL_SPEC_DECL_ROCBLAS_EXT(false)
 
 #endif
 
-#if defined(KOKKOSKERNELS_ENABLE_TPL_MKL) &&                \
-    !defined(KOKKOSKERNELS_ENABLE_TPL_MKL_SYCL_OVERRIDE) && \
+#if defined(KOKKOSKERNELS_ENABLE_TPL_MKL) && \
     defined(KOKKOS_ENABLE_SYCL)
 #include <mkl.h>
 #include <oneapi/mkl/blas.hpp>

diff --git a/packages/kokkos-kernels/blas/tpls/KokkosBlas2_gemv_tpl_spec_avail.hpp b/packages/kokkos-kernels/blas/tpls/KokkosBlas2_gemv_tpl_spec_avail.hpp
@@ -161,8 +161,7 @@ KOKKOSBLAS2_GEMV_TPL_SPEC_AVAIL_ROCBLAS(Kokkos::complex<float>,
 
 #ifdef KOKKOSKERNELS_ENABLE_TPL_MKL
 
-#if defined(KOKKOS_ENABLE_SYCL) && \
-    !defined(KOKKOSKERNELS_ENABLE_TPL_MKL_SYCL_OVERRIDE)
+#if defined(KOKKOS_ENABLE_SYCL)
 
 #define KOKKOSBLAS2_GEMV_TPL_SPEC_AVAIL_ONEMKL(SCALAR, LAYOUT)               \
   template <class ExecSpace>                                                 \

diff --git a/packages/kokkos-kernels/blas/tpls/KokkosBlas2_gemv_tpl_spec_decl.hpp b/packages/kokkos-kernels/blas/tpls/KokkosBlas2_gemv_tpl_spec_decl.hpp
@@ -767,8 +767,7 @@ KOKKOSBLAS2_CGEMV_ROCBLAS(Kokkos::LayoutRight, Kokkos::HIPSpace, false)
 #endif  // KOKKOSKERNELS_ENABLE_TPL_ROCBLAS
 
 // ONEMKL
-#if defined(KOKKOSKERNELS_ENABLE_TPL_MKL) &&                \
-    !defined(KOKKOSKERNELS_ENABLE_TPL_MKL_SYCL_OVERRIDE) && \
+#if defined(KOKKOSKERNELS_ENABLE_TPL_MKL) && \
     defined(KOKKOS_ENABLE_SYCL)
 #include <mkl.h>
 #include <oneapi/mkl/blas.hpp>

diff --git a/packages/kokkos-kernels/cmake/KokkosKernels_config.h.in b/packages/kokkos-kernels/cmake/KokkosKernels_config.h.in
@@ -29,7 +29,6 @@
    requires (a) header file(s) as well, and may use functions other
    than just BLAS and LAPACK functions.  */
 #cmakedefine HAVE_KOKKOSKERNELS_MKL
-#cmakedefine KOKKOSKERNELS_ENABLE_TPL_MKL_SYCL_OVERRIDE
 
 #cmakedefine KOKKOSKERNELS_ENABLE_TESTS_AND_PERFSUITE
 #cmakedefine KOKKOSKERNELS_ENABLE_BENCHMARK

diff --git a/packages/kokkos-kernels/cmake/kokkoskernels_tpls.cmake b/packages/kokkos-kernels/cmake/kokkoskernels_tpls.cmake
@@ -31,10 +31,6 @@ MACRO(KOKKOSKERNELS_ADD_TPL_OPTION NAME DEFAULT_VALUE DOCSTRING)
   SET(ROOT_DEFAULT $ENV{${_NAME_ORIG}_ROOT})
   KOKKOSKERNELS_ADD_OPTION(${_NAME_ORIG}_ROOT "${ROOT_DEFAULT}" PATH "Location of ${_NAME} install root. Default: None or the value of the environment variable ${_NAME}_ROOT if set")
   IF (DEFINED TPL_ENABLE_${_NAME})
-    IF (${_NAME} STREQUAL MKL AND KOKKOSKERNELS_HAS_TRILINOS)
-      MESSAGE("Trilinos has enabled MKL and SYCL but it does not detect oneMKL correctly so we disable it!")
-      SET(KOKKOSKERNELS_ENABLE_TPL_MKL_SYCL_OVERRIDE ON)
-    ENDIF ()
     IF (TPL_ENABLE_${_NAME} AND NOT KOKKOSKERNELS_ENABLE_TPL_${_NAME})
       MESSAGE("Overriding KOKKOSKERNELS_ENABLE_TPL_${_NAME_ORIG}=OFF with TPL_ENABLE_${_NAME}=ON")
       SET(KOKKOSKERNELS_ENABLE_TPL_${_NAME_ORIG} ON)

diff --git a/packages/kokkos-kernels/sparse/src/KokkosSparse_spmv_handle.hpp b/packages/kokkos-kernels/sparse/src/KokkosSparse_spmv_handle.hpp
@@ -189,8 +189,7 @@ struct MKL_SpMV_Data : public TPL_SpMV_Data<ExecutionSpace> {
 };
 #endif
 
-#if defined(KOKKOS_ENABLE_SYCL) && \
-    !defined(KOKKOSKERNELS_ENABLE_TPL_MKL_SYCL_OVERRIDE)
+#if defined(KOKKOS_ENABLE_SYCL)
 struct OneMKL_SpMV_Data : public TPL_SpMV_Data<Kokkos::Experimental::SYCL> {
   OneMKL_SpMV_Data(const Kokkos::Experimental::SYCL& exec_)
       : TPL_SpMV_Data(exec_) {}

diff --git a/packages/kokkos-kernels/sparse/tpls/KokkosSparse_spmv_tpl_spec_avail.hpp b/packages/kokkos-kernels/sparse/tpls/KokkosSparse_spmv_tpl_spec_avail.hpp
@@ -257,8 +257,7 @@ KOKKOSSPARSE_SPMV_TPL_SPEC_AVAIL_MKL(Kokkos::complex<float>, Kokkos::OpenMP)
 KOKKOSSPARSE_SPMV_TPL_SPEC_AVAIL_MKL(Kokkos::complex<double>, Kokkos::OpenMP)
 #endif
 
-#if defined(KOKKOS_ENABLE_SYCL) && \
-    !defined(KOKKOSKERNELS_ENABLE_TPL_MKL_SYCL_OVERRIDE)
+#if defined(KOKKOS_ENABLE_SYCL)
 #define KOKKOSSPARSE_SPMV_TPL_SPEC_AVAIL_ONEMKL(SCALAR, ORDINAL, MEMSPACE)     \
   template <>                                                                  \
   struct spmv_tpl_spec_avail<                                                  \

diff --git a/packages/kokkos-kernels/sparse/tpls/KokkosSparse_spmv_tpl_spec_decl.hpp b/packages/kokkos-kernels/sparse/tpls/KokkosSparse_spmv_tpl_spec_decl.hpp
@@ -720,8 +720,7 @@ KOKKOSSPARSE_SPMV_MKL(Kokkos::complex<double>, Kokkos::OpenMP,
 #undef KOKKOSSPARSE_SPMV_MKL
 #endif
 
-#if defined(KOKKOS_ENABLE_SYCL) && \
-    !defined(KOKKOSKERNELS_ENABLE_TPL_MKL_SYCL_OVERRIDE)
+#if defined(KOKKOS_ENABLE_SYCL)
 inline oneapi::mkl::transpose mode_kk_to_onemkl(char mode_kk) {
   switch (toupper(mode_kk)) {
     case 'N': return oneapi::mkl::transpose::nontrans;