GPU mixed krylov -- WIP

src/IJ_mv/mup.fixed.gpu

@@ -0,0 +1,18 @@
+hypreCUDAKernel_IJVectorUpdateValues
+hypreGPUKernel_IJMatrixValues_dev1
+hypreGPUKernel_IJVectorAssemblePar
+hypre_AuxParCSRMatrixStackReallocate
+hypre_IJMatrixAssembleCommunicate
+hypre_IJMatrixAssembleCompressDevice
+hypre_IJMatrixAssembleParCSRDevice
+hypre_IJMatrixAssembleSortAndReduce1
+hypre_IJMatrixAssembleSortAndReduce2
+hypre_IJMatrixAssembleSortAndReduce3
+hypre_IJMatrixGetValuesParCSRDevice
+hypre_IJMatrixSetAddValuesParCSRDevice
+hypre_IJMatrixSetConstantValuesParCSRDevice
+hypre_IJVectorAssembleParDevice
+hypre_IJVectorAssembleSortAndReduce1
+hypre_IJVectorAssembleSortAndReduce3
+hypre_IJVectorSetAddValuesParDevice
+hypre_IJVectorUpdateValuesDevice

src/config/configure.in

@@ -347,10 +347,6 @@ AS_HELP_STRING([--enable-longdouble],
  esac],
 [hypre_using_longdouble=no]
 )
-if test "$hypre_using_longdouble" = "yes"
-then
-   AC_DEFINE(HYPRE_LONG_DOUBLE, 1, [Define to 1 if using quad precision values for HYPRE_Real])
-fi
 
 AC_ARG_ENABLE(complex,
 AS_HELP_STRING([--enable-complex],
@@ -2424,6 +2420,23 @@ AS_IF([ test x"$hypre_using_hip" == x"yes" ],
                          [AC_MSG_ERROR([unable to find ${HYPRE_ROCM_PREFIX}/include/hip/hip_common.h ... Ensure ROCm is installed and set ROCM_PATH environment variable to ROCm installation path.])] )],
                          [])
 
+dnl *********************************************************************
+dnl * Check for longdouble support
+dnl *********************************************************************
+
+if test "$hypre_using_longdouble" = "yes"
+then
+   if [test "x$hypre_using_cuda" = "xyes" || test "x$hypre_using_device_openmp" = "xyes" || test "x$hypre_using_hip" = "xyes" || test "x$hypre_using_sycl" = "xyes"]
+   then
+      AC_MSG_ERROR([******************** Incompatible precision on device **********************
+                  Long double data format is not supported on device.
+                  For GPU builds, please use the default double precision or --enable-single.
+                  ****************************************************************************])
+   else   
+      AC_DEFINE(HYPRE_LONG_DOUBLE, 1, [Define to 1 if using quad precision values for HYPRE_Real])
+   fi
+fi
+
 dnl *********************************************************************
 dnl * Set GPU warp size
 dnl *********************************************************************

src/config/gen_code.sh

@@ -30,14 +30,38 @@ OUTP=$3
 #### then need to do a manual merge.  The header files can be generated fully
 #### automatically as below.
 
+############################################################################
+# Define gpu tag - set to an empty string if there are no gpu list files
+#
+# Note that the following line will always generate an empty-string value for c,
+# but it will only generate another value for c when $gpu is non-empty
+#
+#     for c in "" $gpu    --> c=""               when gpu=""
+#                         --> c="", c="$gpu"     when gpu="non-empty-string"
+#
+# This is useful below for executing the same lines of code for the normal case
+# and the gpu case (and potential future cases).
+#
+############################################################################
+
+gpu=""
+if [ -f mup.fixed.gpu ]
+then
+   gpu=".gpu"
+fi
+
 ############################################################################
 # Make sure the function list files are sorted (list capital letters first)
 ############################################################################
 
-for i in fixed functions methods
+for c in "" $gpu
 do
-    (export LC_COLLATE=C; sort mup.${i} | uniq) > mup.${i}.tmp
-    mv mup.${i}.tmp mup.${i}
+   for i in fixed functions methods
+   do
+      tag=${i}${c}
+      (export LC_COLLATE=C; sort mup.${tag} | uniq) > mup.${tag}.tmp
+      mv mup.${tag}.tmp mup.${tag}
+   done
 done
 
 ############################################################################
@@ -78,14 +102,25 @@ cat >> $MUP_HEADER <<@
 
 cat>> $MUP_HEADER <<@
 $(
-cat mup.functions mup.methods | while read -r func_name
-do
-   echo "#define $func_name HYPRE_MULTIPRECISION_FUNC ( $func_name )"
-done
-cat mup.fixed | while read -r func_name
-do
-   echo "#define $func_name HYPRE_FIXEDPRECISION_FUNC ( $func_name )"
-done
+   for c in "" $gpu
+   do
+      for i in functions methods
+      do
+         tag=${i}${c}
+         cat mup.${tag} | while read -r func_name
+         do
+            echo "#define $func_name HYPRE_MULTIPRECISION_FUNC ( $func_name )"
+         done
+      done
+      for i in fixed
+      do
+         tag=${i}${c}
+         cat mup.${tag} | while read -r func_name
+         do
+            echo "#define $func_name HYPRE_FIXEDPRECISION_FUNC ( $func_name )"
+         done
+      done
+   done
 )
 
 #endif
@@ -97,7 +132,7 @@ done
 
 if [ "${OUTP}" = "onlydef" ]
 then
-    exit
+   exit
 fi
 
 #===========================================================================
@@ -124,34 +159,46 @@ cat >> $MUP_HEADER <<@
 
 cat>> $MUP_HEADER <<@
 $(
-cat mup.functions mup.methods mup.fixed | while read -r func_name
-do
-   echo "#undef $func_name"
-done
+   for c in "" $gpu
+   do
+      for i in functions methods fixed
+      do
+         tag=${i}${c}
+         cat mup.${tag} | while read -r func_name
+         do
+            echo "#undef $func_name"
+         done
+      done
+   done
 )
 @
 
 ############################################################################
 # Create temporary files and initial code
 ############################################################################
 
-# Create file with list of MUP functions and methods
-cat mup.functions mup.methods > mup.pre
-
 # Create prototype information files
-for i in fixed functions
+for c in "" $gpu
 do
-    $scriptdir/gen_proto_info.sh mup.${i} ${EXTH} > ${OUTP}.${i}.ext.proto
-    $scriptdir/gen_proto_info.sh mup.${i} ${INTH} > ${OUTP}.${i}.int.proto
+   for i in fixed functions methods
+   do
+      tag=${i}${c}
+      $scriptdir/gen_proto_info.sh mup.${tag} ${EXTH} > ${OUTP}.${tag}.ext.proto
+      $scriptdir/gen_proto_info.sh mup.${tag} ${INTH} > ${OUTP}.${tag}.int.proto
+   done
+   cat ${OUTP}.functions${c}.ext.proto ${OUTP}.methods${c}.ext.proto > ${OUTP}.pre${c}.ext.proto
+   cat ${OUTP}.functions${c}.int.proto ${OUTP}.methods${c}.int.proto > ${OUTP}.pre${c}.int.proto
 done
-cat ${OUTP}.functions.ext.proto > ${OUTP}.pre.ext.proto
-cat ${OUTP}.functions.int.proto > ${OUTP}.pre.int.proto
 
 # Create C implementation files and header files
-for i in fixed functions pre
+for c in "" $gpu
 do
-    $scriptdir/gen_code_awk.sh ${OUTP}.${i}.ext.proto ${OUTP}_${i}_ext ${i}
-    $scriptdir/gen_code_awk.sh ${OUTP}.${i}.int.proto ${OUTP}_${i}_int ${i}
+   for i in fixed functions pre
+   do
+      tag=${i}${c}
+      $scriptdir/gen_code_awk.sh ${OUTP}.${tag}.ext.proto ${OUTP}_${tag}_ext ${i}
+      $scriptdir/gen_code_awk.sh ${OUTP}.${tag}.int.proto ${OUTP}_${tag}_int ${i}
+   done
 done
 
 ############################################################################
@@ -179,6 +226,12 @@ cat > $FOUT <<@
 
 $scriptdir/write_header.sh >> $FOUT
 cat ${OUTP}_${i}_ext.c ${OUTP}_${i}_int.c >> $FOUT
+if [ -n "$gpu" ]
+then
+   echo "#if defined(HYPRE_USING_GPU)"                   >> $FOUT
+   cat ${OUTP}_${i}${gpu}_ext.c ${OUTP}_${i}${gpu}_int.c >> $FOUT
+   echo "#endif"                                         >> $FOUT
+fi
 
 cat >> $FOUT <<@
 
@@ -190,7 +243,15 @@ cat >> $FOUT <<@
 # add header info to the prototype files
 #========================================
 
+# external file
+
 FOUT=${OUTP}_${i}_ext.h
+if [ -n "$gpu" ]
+then
+   echo "#if defined(HYPRE_USING_GPU)"   >> $FOUT
+   cat ${OUTP}_${i}${gpu}_ext.h          >> $FOUT
+   echo "#endif"                         >> $FOUT
+fi
 cat > $FOUT.tmp <<@
 
 /*** DO NOT EDIT THIS FILE DIRECTLY (use $0 to generate) ***/
@@ -200,7 +261,15 @@ $scriptdir/write_header.sh >> $FOUT.tmp
 cat $FOUT >> $FOUT.tmp
 mv $FOUT.tmp $FOUT
 
+# internal file
+
 FOUT=${OUTP}_${i}_int.h
+if [ -n "$gpu" ]
+then
+   echo "#if defined(HYPRE_USING_GPU)"   >> $FOUT
+   cat ${OUTP}_${i}${gpu}_int.h          >> $FOUT
+   echo "#endif"                         >> $FOUT
+fi
 cat > $FOUT.tmp <<@
 
 /*** DO NOT EDIT THIS FILE DIRECTLY (use $0 to generate) ***/
@@ -212,15 +281,6 @@ mv $FOUT.tmp $FOUT
 
 done
 
-############################################################################
-# Remove temporary files
-############################################################################
-
-rm -f mup.pre
-rm -f ${OUTP}.*.proto
-rm -f ${OUTP}_*_ext.c
-rm -f ${OUTP}_*_int.c
-
 ############################################################################
 # Generate header files
 ############################################################################
@@ -245,9 +305,10 @@ extern "C" {
 #if defined (HYPRE_MIXED_PRECISION)
 @
 
-cat ${OUTP}_fixed_ext.h      >> $MUP_HEADER
-cat ${OUTP}_functions_ext.h  >> $MUP_HEADER
-cat ${OUTP}_pre_ext.h        >> $MUP_HEADER
+for i in fixed functions pre
+do
+   cat ${OUTP}_${i}_ext.h      >> $MUP_HEADER
+done
 
 cat >> $MUP_HEADER <<@
 
@@ -261,8 +322,6 @@ cat >> $MUP_HEADER <<@
 
 @
 
-rm -f ${OUTP}_fixed_ext.h ${OUTP}_functions_ext.h ${OUTP}_pre_ext.h
-
 #===========================================================================
 # Create internal header file
 #===========================================================================
@@ -283,9 +342,10 @@ extern "C" {
 #if defined (HYPRE_MIXED_PRECISION)
 @
 
-cat ${OUTP}_fixed_int.h      >> $MUP_HEADER
-cat ${OUTP}_functions_int.h  >> $MUP_HEADER
-cat ${OUTP}_pre_int.h        >> $MUP_HEADER
+for i in fixed functions pre
+do
+   cat ${OUTP}_${i}_int.h      >> $MUP_HEADER
+done
 
 cat >> $MUP_HEADER <<@
 
@@ -299,4 +359,19 @@ cat >> $MUP_HEADER <<@
 
 @
 
-rm -f ${OUTP}_fixed_int.h ${OUTP}_functions_int.h ${OUTP}_pre_int.h
+############################################################################
+# Remove temporary files
+############################################################################
+
+for c in "" $gpu
+do
+   for i in fixed functions methods pre
+   do
+      tag=${i}${c}
+      rm -f ${OUTP}.${tag}.ext.proto
+      rm -f ${OUTP}.${tag}.int.proto
+      rm -f ${OUTP}_${tag}_ext.[ch]
+      rm -f ${OUTP}_${tag}_int.[ch]
+   done
+done
+

src/config/generate_function_list.sh

@@ -13,4 +13,6 @@
 shopt -s nullglob
 
 # Use awk to avoid issues with spacing
-nm -P *.o *.obj | awk '$2 == "T" {print $1}' | sed -e 's/^_//' -e 's/_$//'
+# Demangle any c++ name mangling and filter _device_stub_ prefixes.
+nm -P *.o *.obj | awk '$2 == "T" {print $1}' | c++filt | sed -e 's/(.*$//' -e 's/^__device_stub__//' -e 's/^_//' -e 's/_$//'
+

src/config/mup_check_dir.sh

@@ -15,6 +15,8 @@
 
 scriptdir=`dirname $0`
 
+BUILD_TYPE=$(echo "$1" | tr '[:lower:]' '[:upper:]')
+
 # Check if terminal supports colors
 if [ -t 1 ]; then
     # Use colors
@@ -30,7 +32,16 @@ fi
 
 export LC_COLLATE=C  # sort by listing capital letters first
 
-cat mup.fixed mup.functions mup.methods | sort | uniq  > mup_check.old
+
+if [ "$BUILD_TYPE" = "GPU" ]; then
+    cat mup.fixed mup.fixed.gpu \
+        mup.functions mup.functions.gpu \
+        mup.methods mup.methods.gpu \
+        | sort | uniq  > mup_check.old
+else
+    cat mup.fixed mup.functions mup.methods | sort | uniq  > mup_check.old
+fi
+
 $scriptdir/generate_function_list.sh    | sort | uniq  > mup_check.new
 
 # Remove functions listed in mup.exclude (if it exists)

src/configure

@@ -3608,12 +3608,6 @@ else $as_nop
 
 fi
 
-if test "$hypre_using_longdouble" = "yes"
-then
-
-printf "%s\n" "#define HYPRE_LONG_DOUBLE 1" >>confdefs.h
-
-fi
 
 # Check whether --enable-complex was given.
 if test ${enable_complex+y}
@@ -10651,6 +10645,22 @@ fi
 fi
 
 
+if test "$hypre_using_longdouble" = "yes"
+then
+   if test "x$hypre_using_cuda" = "xyes" || test "x$hypre_using_device_openmp" = "xyes" || test "x$hypre_using_hip" = "xyes" || test "x$hypre_using_sycl" = "xyes"
+   then
+      as_fn_error $? "******************** Incompatible precision on device **********************
+                  Long double data format is not supported on device.
+                  For GPU builds, please use the default double precision or --enable-single.
+                  ****************************************************************************" "$LINENO" 5
+   else
+
+printf "%s\n" "#define HYPRE_LONG_DOUBLE 1" >>confdefs.h
+
+   fi
+fi
+
+
 if test "x$hypre_warp_size" = "xauto"
 then
   if test "x$hypre_using_cuda" = "xyes" || test "x$hypre_using_sycl" = "xyes"

src/parcsr_block_mv/_hypre_parcsr_block_mv.h

@@ -243,7 +243,7 @@ typedef struct
    HYPRE_Int      owns_data;
 
    HYPRE_BigInt   num_nonzeros;
-   HYPRE_Real     d_num_nonzeros;
+   hypre_double   d_num_nonzeros;
 
    /* Buffers used by GetRow to hold row currently being accessed. AJC, 4/99 */
    HYPRE_Int     *rowindices;