LLNL · rchen20 · Oct 28, 2023 · Oct 28, 2023 · Oct 28, 2023 · Nov 3, 2023
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,6 +1,6 @@
 ###############################################################################
 #
-# Copyright (c) 2017-22, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2017-23, Lawrence Livermore National Security, LLC.
 #
 # Produced at the Lawrence Livermore National Laboratory
 #
@@ -30,9 +30,7 @@ endif()
 ################################
 # BLT
 ################################
-if (ENABLE_KRIPKE)
-  set(BLT_CXX_STD "c++14" CACHE STRING "Version of C++ Standard")
-endif()
+set(BLT_CXX_STD "c++14" CACHE STRING "Version of C++ Standard")
 if (NOT EXISTS ${PROJECT_SOURCE_DIR}/blt/SetupBLT.cmake)
   message(FATAL_ERROR "\
   The BLT submodule is not present. \
@@ -60,9 +58,13 @@ endif ()
 
 if (ENABLE_KRIPKE)
   if (ENABLE_CUDA)
+    set(CMAKE_CUDA_ARCHITECTURES "70")
     set(KRIPKE_ARCH "Cuda")
-    set(kripke_compile_options --expt-extended-lambda -arch=${CUDA_ARCH})
+    set(kripke_compile_options --expt-extended-lambda)
   elseif (ENABLE_HIP)
+    set(CMAKE_HIP_ARCHITECTURES "--offload-arch=gfx90a")
+    set(GPU_TARGETS "gfx90a")
+    set(AMDGPU_TARGETS "gfx90a")
     set(ENABLE_CHAI On CACHE BOOL "")
     set(KRIPKE_ARCH "HIP")
   elseif (ENABLE_OPENMP)

diff --git a/CoMD/CMakeLists.txt b/CoMD/CMakeLists.txt
@@ -47,6 +47,7 @@ if(ENABLE_MPI)
    set(MPI_CXX_SKIP_FLAGS
      -DMPI_NO_CPPBIND -DOMPI_SKIP_MPICXX -D_MPICC_H -DMPICH_SKIP_MPICXX -DMPIBULL_SKIP_MPICXX
      CACHE STRING "Default skip-flags used (for SGI, OpenMPI, HP-MPI, MPICH, BULL-MPI)")
+   cmake_policy(SET CMP0074 NEW)
    find_package(MPI REQUIRED)
    if (MPI_FOUND)
       set(COMD2_HAVE_MPI TRUE)

diff --git a/CoMD/src-threaded/CMakeLists.txt b/CoMD/src-threaded/CMakeLists.txt
@@ -4,7 +4,7 @@ set (CoMD_depends RAJA)
 
 if (ENABLE_MPI)
   add_definitions(-DENABLE_MPI)
-  set (CoMD_depends ${CoMD_depends} MPI)
+  set (CoMD_depends ${CoMD_depends} mpi)
 endif()
 
 if (ENABLE_OPENMP)
@@ -14,9 +14,13 @@ endif ()
 
 if (ENABLE_CUDA)
   add_definitions(-DENABLE_CUDA)
+  set (CMAKE_CUDA_ARCHITECTURES "70")
   set (CoMD_depends ${CoMD_depends} cuda)
 elseif (ENABLE_HIP)
   add_definitions(-DENABLE_HIP)
+  set (CMAKE_HIP_ARCHITECTURES "--offload-arch=gfx90a")
+  set (GPU_TARGETS "gfx90a")
+  set (AMDGPU_TARGETS "gfx90a")
   set (CoMD_depends ${CoMD_depends} blt::hip)
   set (CoMD_depends ${CoMD_depends} blt::hip_runtime)
 endif ()
@@ -64,7 +68,7 @@ blt_add_executable(
 
 if(ENABLE_CUDA)
   set (comd_compile_options
-    --expt-extended-lambda -arch=${CUDA_ARCH})
+    --expt-extended-lambda)
 target_compile_options(
   coMD_threaded
   PUBLIC

diff --git a/CoMD/src-threaded/CoMDTypes.h b/CoMD/src-threaded/CoMDTypes.h
@@ -140,9 +140,8 @@ typedef RAJA::KernelPolicy<
 #else
   RAJA::statement::CudaKernel<
 #endif
-    RAJA::statement::Tile<0, RAJA::tile_fixed<128>, RAJA::cuda_block_x_loop,
-    RAJA::statement::For<0, RAJA::cuda_thread_x_loop,
-    RAJA::statement::Lambda<0> > > > > redistributeKernel;
+    RAJA::statement::For<0, RAJA::cuda_global_size_x_direct<128>,
+    RAJA::statement::Lambda<0> > > > redistributeKernel;
 
 typedef RAJA::KernelPolicy<
 #ifdef CUDA_ASYNC
@@ -196,9 +195,8 @@ typedef RAJA::KernelPolicy<
 #else
   RAJA::statement::HipKernel<
 #endif
-    RAJA::statement::Tile<0, RAJA::tile_fixed<128>, RAJA::hip_block_x_loop,
-    RAJA::statement::For<0, RAJA::hip_thread_x_loop,
-    RAJA::statement::Lambda<0> > > > > redistributeKernel;
+    RAJA::statement::For<0, RAJA::hip_global_size_x_direct<128>,
+    RAJA::statement::Lambda<0> > > > redistributeKernel;
 
 typedef RAJA::KernelPolicy<
 #ifdef HIP_ASYNC

diff --git a/RELEASE.md b/RELEASE.md
@@ -1,8 +1,9 @@
 # RAJAProxies 2.0.0
 
-Copyright (c) 2017-22, Lawrence Livermore National Security, LLC. 
+Copyright (c) 2017-23, Lawrence Livermore National Security, LLC. 
 Produced at the Lawrence Livermore National Laboratory.
 All rights reserved. 
+This file is part of the RAJA Proxy App Suite.
 
 The proxy applications available in this repository have been relased
 individually. The release numbers and license are for each application are

diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
@@ -1,7 +1,8 @@
 [comment]: # (#################################################################)
-[comment]: # (Copyright 2017-22, Lawrence Livermore National Security, LLC)
+[comment]: # (Copyright (c) 2017-23, Lawrence Livermore National Security, LLC)
 [comment]: # (Produced at the Lawrence Livermore National Laboratory.)
 [comment]: # (All rights reserved.)
+[comment]: # (This file is part of the RAJA Proxy App Suite.)
 [comment]: # 
 [comment]: # (#################################################################)
 

diff --git a/blt b/blt
diff --git a/host-configs/blueos/nvcc_xl_X.cmake b/host-configs/blueos/nvcc_xl_X.cmake
@@ -7,16 +7,19 @@
 
 set(RAJA_COMPILER "RAJA_COMPILER_XLC" CACHE STRING "")
 
-set(CMAKE_CXX_FLAGS_RELEASE "-O3" CACHE STRING "")
-set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g9" CACHE STRING "")
-set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g -qsmp=omp:noopt" CACHE STRING "")
+set(CMAKE_C_COMPILER "xlc_r" CACHE PATH "")
+set(CMAKE_CXX_COMPILER "xlc++_r" CACHE PATH "")
+
+set(CMAKE_CXX_FLAGS_RELEASE "-O3 -std=c++1y --gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1/" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g9 -std=c++1y --gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1/" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g -qsmp=omp:noopt -std=c++1y --gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1/" CACHE STRING "")
 #set(CMAKE_EXE_LINKER_FLAGS "-qpic,-Wl,-z,muldefs" CACHE STRING "")
 
-set(HOST_OPT_FLAGS "-Xcompiler -O3 -Xcompiler -qxlcompatmacros -Xcompiler -qalias=noansi -Xcompiler -qsmp=omp -Xcompiler -qhot -Xcompiler -qnoeh -Xcompiler -qsuppress=1500-029 -Xcompiler -qsuppress=1500-036")
+set(HOST_OPT_FLAGS "-Xcompiler -O3 -Xcompiler -qxlcompatmacros -Xcompiler -qalias=noansi -Xcompiler -qsmp=omp -Xcompiler -qhot -Xcompiler -qnoeh -Xcompiler -qsuppress=1500-029 -Xcompiler -std=c++1y -Xcompiler --gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1/ -Xcompiler -qsuppress=1500-036")
 
-set(CMAKE_CUDA_FLAGS_RELEASE "-O3 ${HOST_OPT_FLAGS}" CACHE STRING "")
-set(CMAKE_CUDA_FLAGS_DEBUG "-g -G -O0" CACHE STRING "")
-set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-g -lineinfo -O3 ${HOST_OPT_FLAGS}" CACHE STRING "")
+set(CMAKE_CUDA_FLAGS_RELEASE "-O3 -std=c++14 --gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1/ ${HOST_OPT_FLAGS}" CACHE STRING "")
+set(CMAKE_CUDA_FLAGS_DEBUG "-g -G -O0 -std=c++14 --gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1/" CACHE STRING "")
+set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-g -lineinfo -O3 -std=c++14 --gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1/ ${HOST_OPT_FLAGS}" CACHE STRING "")
 
 # Suppressed XLC warnings:
 # - 1500-029 cannot inline

diff --git a/host-configs/toss4/hip_3_X.cmake b/host-configs/toss4/hip_3_X.cmake
diff --git a/host-configs/toss4/hip_4_link_X.cmake → host-configs/toss4/hip_link_X.cmake b/host-configs/toss4/hip_4_link_X.cmake → host-configs/toss4/hip_link_X.cmake
diff --git a/kripke b/kripke
diff --git a/lulesh-v1.0/CMakeLists.txt b/lulesh-v1.0/CMakeLists.txt
@@ -1,6 +1,6 @@
 ###############################################################################
 #
-# Copyright (c) 2017-22, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2017-23, Lawrence Livermore National Security, LLC.
 #
 # Produced at the Lawrence Livermore National Laboratory
 #

diff --git a/lulesh-v1.0/RAJA/CMakeLists.txt b/lulesh-v1.0/RAJA/CMakeLists.txt
@@ -1,6 +1,6 @@
 ###############################################################################
 #
-# Copyright (c) 2017-22, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2017-23, Lawrence Livermore National Security, LLC.
 #
 # Produced at the Lawrence Livermore National Laboratory
 #
@@ -23,9 +23,13 @@ if (ENABLE_CUDA)
   set (lulesh_depends
     ${lulesh_depends}
     cuda)
+  set (CMAKE_CUDA_ARCHITECTURES "70")
   set (lulesh_compile_options
-    --expt-extended-lambda -arch=${CUDA_ARCH})
+    --expt-extended-lambda)
 elseif (ENABLE_HIP)
+  set (CMAKE_HIP_ARCHITECTURES "--offload-arch=gfx90a")
+  set (GPU_TARGETS "gfx90a")
+  set (AMDGPU_TARGETS "gfx90a")
   set (lulesh_depends
     ${lulesh_depends}
     blt::hip)

diff --git a/lulesh-v1.0/baseline/CMakeLists.txt b/lulesh-v1.0/baseline/CMakeLists.txt
@@ -1,6 +1,6 @@
 ###############################################################################
 #
-# Copyright (c) 2017-22, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2017-23, Lawrence Livermore National Security, LLC.
 #
 # Produced at the Lawrence Livermore National Laboratory
 #

diff --git a/lulesh-v2.0/CMakeLists.txt b/lulesh-v2.0/CMakeLists.txt
@@ -1,6 +1,6 @@
 ###############################################################################
 #
-# Copyright (c) 2017-22, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2017-23, Lawrence Livermore National Security, LLC.
 #
 # Produced at the Lawrence Livermore National Laboratory
 #

diff --git a/lulesh-v2.0/RAJA/CMakeLists.txt b/lulesh-v2.0/RAJA/CMakeLists.txt
@@ -1,6 +1,6 @@
 ###############################################################################
 #
-# Copyright (c) 2017-22, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2017-23, Lawrence Livermore National Security, LLC.
 #
 # Produced at the Lawrence Livermore National Laboratory
 #
@@ -30,9 +30,13 @@ if (ENABLE_CUDA)
   set (lulesh_depends
     ${lulesh_depends}
     cuda)
+  set (CMAKE_CUDA_ARCHITECTURES "70")
   set (lulesh_compile_options
-    --expt-extended-lambda -arch=${CUDA_ARCH})
+    --expt-extended-lambda)
 elseif (ENABLE_HIP)
+  set (CMAKE_HIP_ARCHITECTURES "--offload-arch=gfx90a")
+  set (GPU_TARGETS "gfx90a")
+  set (AMDGPU_TARGETS "gfx90a")
   set (lulesh_depends
     ${lulesh_depends}
     blt::hip)

diff --git a/lulesh-v2.0/baseline/CMakeLists.txt b/lulesh-v2.0/baseline/CMakeLists.txt
@@ -1,6 +1,6 @@
 ###############################################################################
 #
-# Copyright (c) 2017-22, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2017-23, Lawrence Livermore National Security, LLC.
 #
 # Produced at the Lawrence Livermore National Laboratory
 #

diff --git a/scripts/update_copyright.sh b/scripts/update_copyright.sh
@@ -50,7 +50,7 @@ for i in `cat files2change`
 do
     echo $i
     cp $i $i.sed.bak
-    sed "s/Copyright (c) 2017/Copyright (c) 2017-20/" $i.sed.bak > $i
+    sed "s/Copyright (c) 2017-22/Copyright (c) 2017-23/" $i.sed.bak > $i
 done
 
 

diff --git a/tpl/RAJA b/tpl/RAJA
+2 −0		CHANGELOG
+47 −7		CMakeLists.txt
+1 −1		LICENSE
+21 −16		README.md
+2 −2		RELEASE
+1 −1		blt
+6 −8		host-configs/llnl-bgqos-clang.cmake
+6 −8		host-configs/llnl-blueos-P100-nvcc-clang.cmake
+16 −17		host-configs/llnl-blueos-V100-nvcc-clang.cmake
+6 −8		host-configs/llnl-toss3-clang8.cmake
+6 −8		host-configs/llnl-toss3-gcc7.1.cmake
+7 −9		host-configs/llnl-toss3-gcc8.1.cmake
+6 −8		host-configs/llnl-toss3-intel18.cmake
+6 −8		host-configs/llnl-toss4-MI100-rocm4-nevada.cmake
+6 −8		host-configs/llnl-toss4-MI100-rocm4-whamo.cmake
+36 −0		host-configs/llnl-toss4-MI250X-rocm5-vernal.cmake
+23 −0		host-configs/llnl-toss4-intel19.cmake
+23 −0		host-configs/llnl-toss4-intel22.cmake
+2 −2		scripts/gitlab/build_and_test.sh
+26 −0		scripts/make_release_tarball.sh
+2 −2		scripts/uberenv/packages/chai/package.py
+2 −2		scripts/uberenv/packages/kripke/package.py
+2 −2		scripts/uberenv/packages/raja/package.py
+2 −2		scripts/uberenv/packages/umpire/package.py
+66 −0		scripts/update_copyright.sh
+14 −8		src/Kripke.h
+2 −2		src/Kripke/Arch/LPlusTimes.h
+2 −2		src/Kripke/Arch/LTimes.h
+52 −75		src/Kripke/Arch/Population.h
+2 −2		src/Kripke/Arch/Scattering.h
+14 −22		src/Kripke/Arch/Source.h
+26 −26		src/Kripke/Arch/SweepSubdomains.h
+2 −2		src/Kripke/ArchLayout.h
+2 −2		src/Kripke/Core/BaseVar.cpp
+2 −2		src/Kripke/Core/BaseVar.h
+2 −2		src/Kripke/Core/Comm.h
+2 −2		src/Kripke/Core/DataStore.cpp
+2 −2		src/Kripke/Core/DataStore.h
+2 −2		src/Kripke/Core/DomainVar.cpp
+2 −2		src/Kripke/Core/DomainVar.h
+2 −2		src/Kripke/Core/Field.h
+2 −2		src/Kripke/Core/PartitionSpace.cpp
+2 −2		src/Kripke/Core/PartitionSpace.h
+2 −2		src/Kripke/Core/Set.cpp
+2 −2		src/Kripke/Core/Set.h
+2 −2		src/Kripke/Core/VarLayout.h
+2 −2		src/Kripke/Generate.cpp
+2 −2		src/Kripke/Generate.h
+2 −2		src/Kripke/Generate/Data.cpp
+2 −2		src/Kripke/Generate/Decomp.cpp
+2 −2		src/Kripke/Generate/Energy.cpp
+2 −2		src/Kripke/Generate/Quadrature.cpp
+2 −2		src/Kripke/Generate/Space.cpp
+2 −2		src/Kripke/InputVariables.cpp
+2 −2		src/Kripke/InputVariables.h
+2 −2		src/Kripke/Kernel.h
+2 −2		src/Kripke/Kernel/LPlusTimes.cpp
+2 −2		src/Kripke/Kernel/LTimes.cpp
+2 −2		src/Kripke/Kernel/Population.cpp
+2 −2		src/Kripke/Kernel/Scattering.cpp
+2 −2		src/Kripke/Kernel/Source.cpp
+2 −2		src/Kripke/Kernel/SweepSubdomain.cpp
+2 −2		src/Kripke/ParallelComm.cpp
+2 −2		src/Kripke/ParallelComm.h
+2 −2		src/Kripke/ParallelComm/BlockJacobiComm.cpp
+2 −2		src/Kripke/ParallelComm/SweepComm.cpp
+2 −2		src/Kripke/SteadyStateSolver.cpp
+2 −2		src/Kripke/SteadyStateSolver.h
+2 −2		src/Kripke/SweepSolver.cpp
+2 −2		src/Kripke/SweepSolver.h
+2 −2		src/Kripke/Timing.cpp
+2 −2		src/Kripke/Timing.h
+2 −2		src/Kripke/VarTypes.h
+3 −3		src/kripke.cpp
+1 −1		tpl/chai
+1 −1		tpl/raja
+1 −1		tpl/umpire
-Original file line number
+Diff line change
@@ Expand Up / @@ -50,7 +50,7 @@ for i in `cat files2change` @@
     do
         echo $i
         cp $i $i.sed.bak
-        sed "s/Copyright (c) 2017/Copyright (c) 2017-20/" $i.sed.bak > $i
+        sed "s/Copyright (c) 2017-22/Copyright (c) 2017-23/" $i.sed.bak > $i
     done
@@ Expand Down @@