From bcec5bb6035c73bfdbad812f2fcc84681aaa856d Mon Sep 17 00:00:00 2001 From: Jorge Galvez Vallejo Date: Wed, 23 Jul 2025 15:37:33 +1000 Subject: [PATCH 1/9] add nvhpc options to compile FMS --- CMakeLists.txt | 4 ++-- cmake/compiler_flags_NVHPC_C.cmake | 6 ++++++ cmake/compiler_flags_NVHPC_Fortran.cmake | 11 +++++++++++ cmake/fms_compiler_flags.cmake | 4 ++++ 4 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 cmake/compiler_flags_NVHPC_C.cmake create mode 100644 cmake/compiler_flags_NVHPC_Fortran.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 0451c3dc06..34f1d71475 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,12 +38,12 @@ if(NOT CMAKE_BUILD_TYPE MATCHES "^(Debug|Release|RelWithDebInfo|MinSizeRel)$") set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") endif() -if(NOT CMAKE_C_COMPILER_ID MATCHES "^(Intel|GNU|Clang|IntelLLVM)$") +if(NOT CMAKE_C_COMPILER_ID MATCHES "^(Intel|GNU|Clang|IntelLLVM|NVHPC)$") message( WARNING "Compiler not officially supported: ${CMAKE_C_COMPILER_ID}") endif() -if(NOT CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel|GNU|IntelLLVM)$") +if(NOT CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel|GNU|IntelLLVM|NVHPC)$") message( WARNING "Compiler not officially supported: ${CMAKE_Fortran_COMPILER_ID}") endif() diff --git a/cmake/compiler_flags_NVHPC_C.cmake b/cmake/compiler_flags_NVHPC_C.cmake new file mode 100644 index 0000000000..1433f3efc3 --- /dev/null +++ b/cmake/compiler_flags_NVHPC_C.cmake @@ -0,0 +1,6 @@ +# GNU C +set(CMAKE_C_FLAGS_RELEASE "-O3") + +set(CMAKE_C_FLAGS_DEBUG "-O0 -g") + +set(CMAKE_C_LINK_FLAGS "") diff --git a/cmake/compiler_flags_NVHPC_Fortran.cmake b/cmake/compiler_flags_NVHPC_Fortran.cmake new file mode 100644 index 0000000000..c7c75d7bdc --- /dev/null +++ b/cmake/compiler_flags_NVHPC_Fortran.cmake @@ -0,0 +1,11 @@ +# Precision-based Fortran compiler flags +set(r8_flags "-r8") # Fortran flags for 64BIT precision + +# GNU Fortan +set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ") + +set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -fast") + +set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -g -fcheck=bounds -ffpe-trap=invalid,zero,overflow,underflow" ) + +set(CMAKE_Fortran_LINK_FLAGS "" ) diff --git a/cmake/fms_compiler_flags.cmake b/cmake/fms_compiler_flags.cmake index cc7a2fee0f..be84877342 100644 --- a/cmake/fms_compiler_flags.cmake +++ b/cmake/fms_compiler_flags.cmake @@ -8,6 +8,8 @@ elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "Intel") include(compiler_flags_Intel_Fortran) elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "IntelLLVM") include(compiler_flags_IntelLLVM_Fortran) +elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") + include(compiler_flags_NVHPC_Fortran) else() message(WARNING "Fortran compiler with ID ${CMAKE_Fortran_COMPILER_ID} will be used with CMake default options") endif() @@ -24,6 +26,8 @@ elseif(CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM") include(compiler_flags_IntelLLVM_C) elseif(CMAKE_C_COMPILER_ID MATCHES "Clang") include(compiler_flags_Clang_C) +elseif(CMAKE_C_COMPILER_ID MATCHES "NVHPC") + include(compiler_flags_NVHPC_C) else() message(WARNING "C compiler with ID ${CMAKE_C_COMPILER_ID} will be used with CMake default options") endif() From 35ca0da4b13cfb6656f87f6a9b5a6d7b2c2e5ca6 Mon Sep 17 00:00:00 2001 From: Jorge Galvez Vallejo Date: Wed, 23 Jul 2025 15:43:40 +1000 Subject: [PATCH 2/9] turn off quad precision with nvhpc --- CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 34f1d71475..c6540da035 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,6 +67,10 @@ option(LARGEFILE "Enable compiler definition -Duse_LARGEFILE" option(WITH_YAML "Enable compiler definition -Duse_yaml" OFF) option(USE_DEPRECATED_IO "THIS OPTION HAS NO EFFECT AND WILL BE REMOVED IN A FUTURE RELEASE" OFF) +if(CMAKE_Fortran_COMPILER_ID MATCHES "NVHPC") + set(ENABLE_QUAD_PRECISION OFF) +endif() + if(32BIT) list(APPEND kinds "r4") endif() From ca4d10f73db671dfaf94c5121074f669ce2773ad Mon Sep 17 00:00:00 2001 From: Jorge Date: Thu, 7 Aug 2025 23:49:28 -0500 Subject: [PATCH 3/9] add the include dir for nvhpc to find --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1e4a934be5..4296731285 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ project(FMS DESCRIPTION "GFDL FMS Library" HOMEPAGE_URL "https://www.gfdl.noaa.gov/fms" LANGUAGES C Fortran) - +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) include(GNUInstallDirs) if(NOT CMAKE_BUILD_TYPE MATCHES "^(Debug|Release|RelWithDebInfo|MinSizeRel)$") @@ -336,6 +336,7 @@ foreach(kind ${kinds}) mosaic2/include constants astronomy/include + field_manager/ field_manager/include time_interp/include tracer_manager/include From d2883c1f95b9b613206a584910e4b7c6530c7955 Mon Sep 17 00:00:00 2001 From: Jorge Date: Fri, 8 Aug 2025 00:25:10 -0500 Subject: [PATCH 4/9] comment on nvhpc --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4296731285..be3ee75eac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,6 +68,7 @@ option(WITH_YAML "Enable compiler definition -Duse_yaml" option(USE_DEPRECATED_IO "THIS OPTION HAS NO EFFECT AND WILL BE REMOVED IN A FUTURE RELEASE" OFF) if(CMAKE_Fortran_COMPILER_ID MATCHES "NVHPC") +# it seems nvidia does not support quad prec set(ENABLE_QUAD_PRECISION OFF) endif() From 2f07ef34cb13e2ced606e86a5798386c92d58433 Mon Sep 17 00:00:00 2001 From: Jorge Date: Fri, 8 Aug 2025 00:29:52 -0500 Subject: [PATCH 5/9] add comments and fix the debug flags --- cmake/compiler_flags_NVHPC_C.cmake | 2 +- cmake/compiler_flags_NVHPC_Fortran.cmake | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cmake/compiler_flags_NVHPC_C.cmake b/cmake/compiler_flags_NVHPC_C.cmake index 1433f3efc3..fe6ac05f9f 100644 --- a/cmake/compiler_flags_NVHPC_C.cmake +++ b/cmake/compiler_flags_NVHPC_C.cmake @@ -1,4 +1,4 @@ -# GNU C +# NVHPC C set(CMAKE_C_FLAGS_RELEASE "-O3") set(CMAKE_C_FLAGS_DEBUG "-O0 -g") diff --git a/cmake/compiler_flags_NVHPC_Fortran.cmake b/cmake/compiler_flags_NVHPC_Fortran.cmake index c7c75d7bdc..7bdc9e5a15 100644 --- a/cmake/compiler_flags_NVHPC_Fortran.cmake +++ b/cmake/compiler_flags_NVHPC_Fortran.cmake @@ -1,11 +1,13 @@ # Precision-based Fortran compiler flags set(r8_flags "-r8") # Fortran flags for 64BIT precision -# GNU Fortan +# NVHPC Fortan set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ") set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -fast") -set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -g -fcheck=bounds -ffpe-trap=invalid,zero,overflow,underflow" ) +set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -Ktrap=fp" ) +# -g can cause bugs, see: https://forums.developer.nvidia.com/t/bug-compiling-with-g-o0-produces-a-compute-sanitizer-error-removing-g-removes-the-error/341478 +# not sure if this is only on GPUs set(CMAKE_Fortran_LINK_FLAGS "" ) From 58d01eeea15b6a070c63767fc39369562e61e395 Mon Sep 17 00:00:00 2001 From: Jorge Date: Fri, 8 Aug 2025 00:32:30 -0500 Subject: [PATCH 6/9] make sure that nvhpc gets warning --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index be3ee75eac..0cf51a8682 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,12 +38,12 @@ if(NOT CMAKE_BUILD_TYPE MATCHES "^(Debug|Release|RelWithDebInfo|MinSizeRel)$") set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") endif() -if(NOT CMAKE_C_COMPILER_ID MATCHES "^(Intel|GNU|Clang|IntelLLVM|NVHPC)$") +if(NOT CMAKE_C_COMPILER_ID MATCHES "^(Intel|GNU|Clang|IntelLLVM)$") message( WARNING "Compiler not officially supported: ${CMAKE_C_COMPILER_ID}") endif() -if(NOT CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel|GNU|IntelLLVM|NVHPC)$") +if(NOT CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel|GNU|IntelLLVM)$") message( WARNING "Compiler not officially supported: ${CMAKE_Fortran_COMPILER_ID}") endif() From ba9b3c1a19700dcc6e808f510cb03aad9498dcb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Luis=20G=C3=A1lvez=20Vallejo?= Date: Tue, 26 Aug 2025 10:00:39 +1000 Subject: [PATCH 7/9] Update cmake/compiler_flags_NVHPC_Fortran.cmake Co-authored-by: Ryan Mulhall <35538242+rem1776@users.noreply.github.com> --- cmake/compiler_flags_NVHPC_Fortran.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/compiler_flags_NVHPC_Fortran.cmake b/cmake/compiler_flags_NVHPC_Fortran.cmake index 7bdc9e5a15..daca1015bb 100644 --- a/cmake/compiler_flags_NVHPC_Fortran.cmake +++ b/cmake/compiler_flags_NVHPC_Fortran.cmake @@ -4,7 +4,7 @@ set(r8_flags "-r8") # Fortran flags for 64BIT precision # NVHPC Fortan set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ") -set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -fast") +set(CMAKE_Fortran_FLAGS_RELEASE "-O3") set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -Ktrap=fp" ) # -g can cause bugs, see: https://forums.developer.nvidia.com/t/bug-compiling-with-g-o0-produces-a-compute-sanitizer-error-removing-g-removes-the-error/341478 From 948c30bb865387faf2d802fcf521cc87d8309884 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Luis=20G=C3=A1lvez=20Vallejo?= Date: Tue, 26 Aug 2025 10:00:56 +1000 Subject: [PATCH 8/9] Update cmake/compiler_flags_NVHPC_Fortran.cmake Co-authored-by: Ryan Mulhall <35538242+rem1776@users.noreply.github.com> --- cmake/compiler_flags_NVHPC_Fortran.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/compiler_flags_NVHPC_Fortran.cmake b/cmake/compiler_flags_NVHPC_Fortran.cmake index daca1015bb..4095ba0b63 100644 --- a/cmake/compiler_flags_NVHPC_Fortran.cmake +++ b/cmake/compiler_flags_NVHPC_Fortran.cmake @@ -6,7 +6,7 @@ set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ") set(CMAKE_Fortran_FLAGS_RELEASE "-O3") -set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -Ktrap=fp" ) +set(CMAKE_Fortran_FLAGS_DEBUG "-O0" ) # -g can cause bugs, see: https://forums.developer.nvidia.com/t/bug-compiling-with-g-o0-produces-a-compute-sanitizer-error-removing-g-removes-the-error/341478 # not sure if this is only on GPUs From 92251e852e855b80458d3cae95778ab4550e7324 Mon Sep 17 00:00:00 2001 From: Jorge Galvez Vallejo Date: Tue, 26 Aug 2025 10:03:26 +1000 Subject: [PATCH 9/9] add warning for nvhpc and quad precision --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0cf51a8682..a153ccdec5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,8 +68,8 @@ option(WITH_YAML "Enable compiler definition -Duse_yaml" option(USE_DEPRECATED_IO "THIS OPTION HAS NO EFFECT AND WILL BE REMOVED IN A FUTURE RELEASE" OFF) if(CMAKE_Fortran_COMPILER_ID MATCHES "NVHPC") -# it seems nvidia does not support quad prec set(ENABLE_QUAD_PRECISION OFF) + message(WARNING "NVHPC does not support building FMS with QUAD_PRECISION!") endif() if(32BIT)