Add machine matrix

AUTOTEST/cmake.sh

@@ -82,7 +82,8 @@ rm -fr src/hypre
 # Configure
 cd $root_dir/build
 eval cmake $copts ../src
-make $mopts install
+eval cmake --build . $mopts
+eval cmake --install .
 
 cd $test_dir
 

AUTOTEST/machine-matrix.sh

@@ -0,0 +1,150 @@
+#!/bin/bash
+# Copyright (c) 1998 Lawrence Livermore National Security, LLC and other
+# HYPRE Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+testname=`basename $0 .sh`
+
+# Echo usage information
+case $1 in
+   -h|-help)
+      cat <<EOF
+
+   **** Only run this script on LC's matrix cluster ****
+
+   $0 [-h|-help] {src_dir}
+
+   where: -h|-help   prints this usage information and exits
+          {src_dir}  is the hypre source directory
+
+   This script runs a number of tests suitable for the matrix cluster.
+
+   Example usage: $0 ../src
+
+EOF
+      exit
+      ;;
+esac
+
+# Setup
+cmake_version=3.30
+test_dir=`pwd`
+output_dir=`pwd`/$testname.dir
+rm -fr $output_dir
+mkdir -p $output_dir
+src_dir=`cd $1; pwd`
+root_dir=`cd $src_dir/..; pwd`
+shift
+
+# Basic build and run tests
+aco="LIBS=-ldl"
+cco="-DMPIEXEC_EXECUTABLE=\"srun\" -DMPIEXEC_NUMPROC_FLAG=\"n\""
+cmo="--parallel"
+mo="-j test"
+eo=""
+rtol="0.0"
+atol="3e-15"
+save="matrix"
+
+# 1C) GCC 13.3.1 + CUDA 12.9.1 with UM and memory tracker OFF in debug mode [error, ij, ams, struct, sstruct]
+module -q load cmake/${cmake_version} cuda/12.9.1 gcc/13.3.1
+co="${cco} -DHYPRE_ENABLE_CUDA=ON -DHYPRE_ENABLE_UMPIRE=OFF -DHYPRE_ENABLE_UNIFIED_MEMORY=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CUDA_ARCHITECTURES=90 -DHYPRE_ENABLE_MEMORY_TRACKER=OFF -DHYPRE_ENABLE_PERSISTENT_COMM=ON -DHYPRE_ENABLE_PRINT_ERRORS=ON"
+ro="-error -ij -ams -struct -sstruct -rt -save ${save} -rtol ${rtol} -atol ${atol}"
+./test.sh cmake.sh $root_dir -co: $co -mo: $cmo -ro: $ro
+./renametest.sh cmake $output_dir/cmake-cuda-um-dbg
+
+# 2C) GCC 13.3.1 + CUDA 12.9.1 with mixed integers and UM in debug mode [ij-mixed, ams, struct, sstruct-mixed]
+module reset && module -q load cmake/${cmake_version} cuda/12.9.1 gcc/13.3.1
+co="${cco} -DHYPRE_ENABLE_UMPIRE=OFF -DHYPRE_ENABLE_UNIFIED_MEMORY=ON -DHYPRE_ENABLE_CUDA=ON -DHYPRE_ENABLE_MIXEDINT=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CUDA_ARCHITECTURES=90 -DHYPRE_ENABLE_PRINT_ERRORS=ON"
+ro="-error -ij-mixed -ams -struct -sstruct-mixed -rt -save ${save} -rtol ${rtol} -atol ${atol}"
+./test.sh cmake.sh $root_dir -co: $co -mo: $cmo -ro: $ro
+./renametest.sh cmake $output_dir/cmake-cuda-um-mixedint
+
+# 3C) GCC 13.3.1 + CUDA 12.9.1 with OMP and shared library in release mode
+module reset && module -q load cmake/${cmake_version} cuda/12.9.1 gcc/13.3.1
+co="${cco} -DHYPRE_ENABLE_UMPIRE=OFF -DHYPRE_ENABLE_UNIFIED_MEMORY=ON -DHYPRE_ENABLE_CUDA=ON -DHYPRE_ENABLE_OPENMP=ON -DHYPRE_ENABLE_HOPSCOTCH=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_CUDA_ARCHITECTURES=90 -DHYPRE_ENABLE_PRINT_ERRORS=ON"
+ro="-gpumemcheck -rt -cudasan -save ${save} -rtol ${rtol} -atol ${atol}"
+./test.sh cmake.sh $root_dir -co: $co -mo: $cmo -ro: $ro
+./renametest.sh cmake $output_dir/cmake-cuda-um-shared
+
+# 4C) GCC 13.3.1 + CUDA 12.9.1 with UM and single precision in debug mode
+module reset && module -q load cmake/${cmake_version} cuda/12.9.1 gcc/13.3.1
+co="${cco} -DHYPRE_ENABLE_SINGLE=ON -DHYPRE_ENABLE_UMPIRE=OFF -DHYPRE_ENABLE_UNIFIED_MEMORY=ON -DHYPRE_ENABLE_CUDA=ON -DHYPRE_ENABLE_CUSOLVER=ON -DHYPRE_ENABLE_OPENMP=ON -DHYPRE_ENABLE_HOPSCOTCH=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_CUDA_ARCHITECTURES=90 -DHYPRE_ENABLE_PRINT_ERRORS=ON"
+ro="-single -rt -save ${save} -rtol ${rtol} -atol ${atol}"
+./test.sh cmake.sh $root_dir -co: $co -mo: $cmo -ro: $ro
+./renametest.sh cmake $output_dir/cmake-cuda-um-single
+
+# 5C) GCC 13.3.1 + CUDA 12.9.1 without MPI [no run]
+module reset && module -q load cmake/${cmake_version} cuda/12.9.1 gcc/13.3.1
+co="${cco} -DHYPRE_ENABLE_MPI=OFF -DHYPRE_ENABLE_UMPIRE=OFF -DHYPRE_ENABLE_UNIFIED_MEMORY=ON -DHYPRE_ENABLE_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=90 -DHYPRE_ENABLE_PRINT_ERRORS=ON"
+./test.sh cmake.sh $root_dir -co: $co -mo: $cmo
+./renametest.sh cmake $output_dir/cmake-cuda-um-without-MPI
+
+# 6C) GCC 13.3.1 + CUDA 12.9.1 with Umpire [benchmark]
+module reset && module -q load cmake/${cmake_version} cuda/12.9.1 gcc/13.3.1
+UMPIRE_DIR=/usr/workspace/hypre/ext-libs/Umpire/install-umpire_2025.09.0-cuda_12.9_sm90-gcc_13.3
+co="${cco} -DHYPRE_ENABLE_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=90 -DHYPRE_ENABLE_PRINT_ERRORS=ON -Dumpire_DIR=${UMPIRE_DIR}/lib/cmake"
+ro="-bench -rt -mpibind -save ${save}"
+./test.sh cmake.sh $root_dir -co: $co -mo: $cmo -ro: $ro
+./renametest.sh cmake $output_dir/cmake-cuda-bench
+
+# 7C) GCC 13.3.1 + CUDA 12.9.1 with host execution
+module reset && module -q load cmake/${cmake_version} cuda/12.9.1 gcc/13.3.1
+co="${cco} -DHYPRE_ENABLE_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=90 -DHYPRE_ENABLE_UMPIRE=OFF -DHYPRE_ENABLE_TEST_USING_HOST=ON -DHYPRE_ENABLE_MEMORY_TRACKER=OFF -DHYPRE_ENABLE_PRINT_ERRORS=ON -DCMAKE_BUILD_TYPE=Debug"
+ro="-ij-noilu -ams -struct -sstruct"
+./test.sh cmake.sh $root_dir -co: $co -mo: $cmo -ro: $ro
+./renametest.sh cmake $output_dir/cmake-cuda-cpu
+
+##################################
+## Autotools (build only) tests ##
+##################################
+
+# 1A) GCC 13.3.1 + CUDA 12.9.1 with UM and memory tracker in debug mode
+module -q load cuda/12.9.1 gcc/13.3.1
+co="${aco} --with-cuda --without-umpire --enable-unified-memory --enable-debug --with-gpu-arch=90 --with-memory-tracker --enable-persistent --with-print-errors"
+./test.sh basic.sh $src_dir -co: $co -mo: $mo
+./renametest.sh basic $output_dir/basic-cuda-um-dbg
+
+# 2A) GCC 13.3.1 + CUDA 12.9.1 with mixed integers and UM in debug mode
+module reset && module -q load cuda/12.9.1 gcc/13.3.1
+co="${aco} --with-cuda --without-umpire --enable-unified-memory --enable-debug --with-gpu-arch=90 --enable-mixedint --with-print-errors"
+./test.sh basic.sh $src_dir -co: $co -mo: $mo
+./renametest.sh basic $output_dir/basic-cuda-um-mixedint
+
+# 3A) GCC 13.3.1 + CUDA 12.9.1 with OMP and shared library in release mode
+module reset && module -q load cuda/12.9.1 gcc/13.3.1
+co="${aco} --with-cuda --without-umpire --enable-unified-memory --with-openmp --enable-hopscotch --enable-shared --with-gpu-arch=90 --with-print-errors"
+./test.sh basic.sh $src_dir -co: $co -mo: $mo
+./renametest.sh basic $output_dir/basic-cuda-um-shared
+
+# 4A) GCC 13.3.1 + CUDA 12.9.1 with UM and single precision in debug mode
+module reset && module -q load cuda/12.9.1 gcc/13.3.1
+co="${aco} --enable-single --without-umpire --enable-unified-memory --with-cuda --enable-cusolver --with-openmp --enable-hopscotch --enable-shared --with-gpu-arch=90 --with-print-errors"
+./test.sh basic.sh $src_dir -co: $co -mo: $mo
+./renametest.sh basic $output_dir/basic-cuda-um-single
+
+# 5A) GCC 13.3.1 + CUDA 12.9.1 without MPI
+module reset && module -q load cuda/12.9.1 gcc/13.3.1
+co="${aco} --without-MPI --without-umpire --enable-unified-memory --with-cuda --with-gpu-arch=90 --with-print-errors"
+./test.sh basic.sh $src_dir -co: $co -mo: $mo
+./renametest.sh basic $output_dir/basic-cuda-um-without-MPI
+
+# 6A) GCC 13.3.1 + CUDA 12.9.1 with Umpire
+module reset && module -q load cuda/12.9.1 gcc/13.3.1
+UMPIRE_DIR=/usr/workspace/hypre/ext-libs/Umpire/install-umpire_2025.09.0-cuda_12.9_sm90-gcc_13.3
+co="${aco} --with-cuda --with-gpu-arch=90 --with-umpire --with-umpire-include=${UMPIRE_DIR}/include --with-umpire-lib-dirs=${UMPIRE_DIR}/lib --with-umpire-libs=\"umpire camp\" --with-print-errors"
+./test.sh basic.sh $src_dir -co: $co -mo: $mo
+./renametest.sh basic $output_dir/basic-cuda-bench
+
+# 7A) GCC 13.3.1 + CUDA 12.9.1 with host execution
+module reset && module -q load cuda/12.9.1 gcc/13.3.1
+co="${aco} --with-cuda --with-gpu-arch=90 --without-umpire --with-test-using-host --with-memory-tracker --enable-debug --with-print-errors"
+./test.sh basic.sh $root_dir -co: $co -mo: $mo
+./renametest.sh basic $output_dir/basic-cuda-cpu
+
+# Echo to stderr all nonempty error files in $output_dir
+for errfile in $( find $output_dir ! -size 0 -name "*.err" )
+do
+   echo $errfile >&2
+done

src/test/TEST_ams/solvers.saved.matrix

@@ -0,0 +1,117 @@
+# Output file: solvers.out.0
+    Cycle 55   2.175368e-05    0.792497     1.620091e-06 
+    Cycle 56   1.724221e-05    0.792611     1.284102e-06 
+    Cycle 57   1.366597e-05    0.792588     1.017764e-06 
+    Cycle 58   1.083058e-05    0.792522     8.066004e-07 
+
+
+ Average Convergence Factor = 0.785131
+
+# Output file: solvers.out.1
+    Cycle 55   2.175368e-05    0.792497     1.620091e-06 
+    Cycle 56   1.724221e-05    0.792611     1.284102e-06 
+    Cycle 57   1.366597e-05    0.792588     1.017764e-06 
+    Cycle 58   1.083058e-05    0.792522     8.066004e-07 
+
+
+ Average Convergence Factor = 0.785131
+
+# Output file: solvers.out.2
+    Cycle 102   1.794451e-05    0.878441     1.336405e-06 
+    Cycle 103   1.576536e-05    0.878562     1.174114e-06 
+    Cycle 104   1.385191e-05    0.878629     1.031611e-06 
+    Cycle 105   1.216904e-05    0.878510     9.062812e-07 
+
+
+ Average Convergence Factor = 0.875891
+
+# Output file: solvers.out.3
+    Cycle 102   1.794451e-05    0.878441     1.336405e-06 
+    Cycle 103   1.576536e-05    0.878562     1.174114e-06 
+    Cycle 104   1.385191e-05    0.878629     1.031611e-06 
+    Cycle 105   1.216904e-05    0.878510     9.062812e-07 
+
+
+ Average Convergence Factor = 0.875891
+
+# Output file: solvers.out.4
+
+Iterations = 13
+Final Relative Residual Norm = 3.213286e-07
+
+# Output file: solvers.out.5
+
+Iterations = 13
+Final Relative Residual Norm = 3.213286e-07
+
+# Output file: solvers.out.6
+
+Iterations = 16
+Final Relative Residual Norm = 7.960715e-07
+
+# Output file: solvers.out.7
+
+Iterations = 16
+Final Relative Residual Norm = 7.960715e-07
+
+# Output file: solvers.out.12
+
+Iterations = 18
+Final Relative Residual Norm = 4.021817e+00
+
+# Output file: solvers.out.8
+
+Eigenvalue lambda   3.02357653920962e+01
+Eigenvalue lambda   3.03135374702830e+01
+Eigenvalue lambda   3.85013899429209e+01
+Eigenvalue lambda   5.14395940113317e+01
+Eigenvalue lambda   5.15742481827998e+01
+Residual   1.28206884875916e-04
+Residual   1.15405099456959e-04
+Residual   1.04689465214239e-04
+Residual   1.38172405812504e-04
+Residual   1.23675952898175e-04
+
+25 iterations
+# Output file: solvers.out.9
+
+Eigenvalue lambda   3.02357653920962e+01
+Eigenvalue lambda   3.03135374702830e+01
+Eigenvalue lambda   3.85013899429209e+01
+Eigenvalue lambda   5.14395940113317e+01
+Eigenvalue lambda   5.15742481827998e+01
+Residual   1.28206884875916e-04
+Residual   1.15405099456959e-04
+Residual   1.04689465214239e-04
+Residual   1.38172405812504e-04
+Residual   1.23675952898175e-04
+
+25 iterations
+# Output file: solvers.out.10
+
+Eigenvalue lambda   3.02357653918892e+01
+Eigenvalue lambda   3.03135374702453e+01
+Eigenvalue lambda   3.85013899430456e+01
+Eigenvalue lambda   5.14395940111879e+01
+Eigenvalue lambda   5.15742481838279e+01
+Residual   8.52199408895113e-05
+Residual   9.28400783649100e-05
+Residual   1.24500310590999e-04
+Residual   1.48772911719422e-04
+Residual   1.31345900292041e-04
+
+37 iterations
+# Output file: solvers.out.11
+
+Eigenvalue lambda   3.02357653918892e+01
+Eigenvalue lambda   3.03135374702453e+01
+Eigenvalue lambda   3.85013899430456e+01
+Eigenvalue lambda   5.14395940111879e+01
+Eigenvalue lambda   5.15742481838279e+01
+Residual   8.52199408895113e-05
+Residual   9.28400783649100e-05
+Residual   1.24500310590999e-04
+Residual   1.48772911719422e-04
+Residual   1.31345900292041e-04
+
+37 iterations