diff --git a/.github/workflows/windows-mpi.yml b/.github/workflows/windows-mpi.yml new file mode 100644 index 0000000000..cdb98b42b5 --- /dev/null +++ b/.github/workflows/windows-mpi.yml @@ -0,0 +1,163 @@ +name: Windows + +on: [push, pull_request] + +concurrency: + group: ${{ github.ref }}-${{ github.head_ref }}-windows + cancel-in-progress: true + +jobs: + WIN64-MSVC: + name: WIN64-MSVC - MPI ${{ matrix.mpi }} - Particles ${{ matrix.particles }} + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + particles: [OFF, ON] + mpi: [ON] + + steps: + - name: Checkout (with submodules) + uses: actions/checkout@v4 + with: + submodules: true + + - name: Install MS-MPI SDK only + shell: pwsh + run: | + curl -L -o msmpisdk.msi https://github.com/microsoft/Microsoft-MPI/releases/download/v10.1.1/msmpisdk.msi + msiexec /i msmpisdk.msi /qn /norestart + + $msmpiInc = 'C:\Program Files (x86)\Microsoft SDKs\MPI\Include' + $msmpiLib = 'C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64' + "MSMPI_INC=$msmpiInc" | Out-File -FilePath $env:GITHUB_ENV -Append + "MSMPI_LIB=$msmpiLib" | Out-File -FilePath $env:GITHUB_ENV -Append + + - name: Configure (CMake) + shell: pwsh + run: | + cmake -S . -B build ` + -DCMAKE_BUILD_TYPE=Release ` + -DERF_ENABLE_MPI=ON ` + -DMPI_CXX_LIB_NAMES=msmpi ` + -DMPI_C_LIB_NAMES=msmpi ` + -DMPI_msmpi_LIBRARY="$env:MSMPI_LIB\msmpi.lib" ` + ${{ github.workspace }} + + - name: Build + shell: pwsh + run: cmake --build build --parallel 2 --verbose + + - name: Create Installation README + shell: pwsh + run: | + @" + # ERF Windows Build - Installation Instructions + + ## Build Information + - Build Type: Release + - MPI Enabled: ${{ matrix.mpi }} + - Particles Enabled: ${{ matrix.particles }} + - MS-MPI Version: 10.1.1 + - Built on: $(Get-Date -Format "yyyy-MM-dd HH:mm:ss UTC") + - Commit: ${{ github.sha }} + + ## Required Runtime Installation + + This executable requires MS-MPI Runtime v10.1.1 to run. + + ### Quick Install (PowerShell): + ``````powershell + # Download and install MS-MPI Runtime v10.1.1 + Invoke-WebRequest -Uri "https://github.com/microsoft/Microsoft-MPI/releases/download/v10.1.1/msmpisetup.exe" -OutFile "msmpisetup.exe" + .\msmpisetup.exe + + # After installation, open a NEW PowerShell window to refresh PATH + `````` + + ### Alternative Install (using vcpkg): + ``````powershell + # vcpkg will download the installer and prompt you to run it + vcpkg install msmpi:x64-windows + # Follow the instructions vcpkg provides to run the downloaded installer + `````` + + ## Running the Executable + + Navigate to the executable directory (e.g., `Exec\ABL\Debug\` or `Exec\ABL\Release\`) and run: + + ``````powershell + # Single process + mpiexec -n 1 .\erf_abl.exe path\to\inputs_file + + # Multiple processes (e.g., 4) + mpiexec -n 4 .\erf_abl.exe path\to\inputs_file + `````` + + **Important:** Always use `mpiexec` or `mpirun` to launch. Running `.\erf_abl.exe` directly will trigger Windows Defender firewall warnings. + + ## Troubleshooting + + ### Executable runs but produces no output: + - MS-MPI runtime is not installed, or the wrong version is installed. + - Install MS-MPI v10.1.1 runtime using the script above + - Open a NEW terminal after installation + + ### "mpiexec not recognized": + - Open a new PowerShell window after installing MS-MPI + - Or manually refresh PATH: `$env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine")` + + ## Build Details + + This build was compiled with: + - Compiler: MSVC 19.29+ + - CMake Configuration: + - CMAKE_BUILD_TYPE=Release + - ERF_ENABLE_MPI=ON + - ERF_ENABLE_PARTICLES=${{ matrix.particles }} + "@ | Out-File -FilePath build\INSTALL.txt -Encoding utf8 + + - name: Add Installation Instructions to Job Summary + shell: pwsh + run: | + @" + ## ✅ Build Complete: ERF Windows (MPI: ${{ matrix.mpi }}, Particles: ${{ matrix.particles }}) + + ### 📦 Artifact Information + - **Build Type:** Release + - **MS-MPI Version:** 10.1.1 + - **Commit:** ${{ github.sha }} + + ### ⚠️ Required for Running + + This executable requires **MS-MPI Runtime v10.1.1** + + #### Quick Install: + ``````powershell + Invoke-WebRequest -Uri "https://github.com/microsoft/Microsoft-MPI/releases/download/v10.1.1/msmpisetup.exe" -OutFile "msmpisetup.exe" + .\msmpisetup.exe + `````` + + #### Or use vcpkg: + ``````powershell + vcpkg install msmpi:x64-windows + `````` + + ### 🚀 Running the Executable + + ``````powershell + mpiexec -n 4 .\erf_abl.exe path\to\inputs_file + `````` + + **Note:** Always use `mpiexec` to launch (not `.\erf_abl.exe` directly) to avoid Windows Defender warnings. + + See `INSTALL.txt` in the artifact for complete instructions. + "@ | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append -Encoding utf8 + + - name: Upload artifact (from build tree) + uses: actions/upload-artifact@v4 + with: + name: ERF-win64-Release-mpi-${{ matrix.mpi }}-particles-${{ matrix.particles }} + path: | + build + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index 880d4460cd..39ec67b33e 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,41 @@ cmake*ser* cmake*par* .idea *.ipynb + +# CMake build artifacts +CMakeCache.txt +CMakeFiles/ +cmake_install.cmake +CTestTestfile.cmake +DartConfiguration.tcl +Makefile +compile_commands.json +Testing/ +_deps/ + +# Project-specific generated files +ERFConfig.cmake +*.pc +git-state.txt + +# Build outputs +lib*.a +lib*.so +bin/ +Exec/ +Submodules/ +Tests/ +cmake_packages/ +externals/ + +# Build and install directories +build_*/ +install_*/ + +# Log files +build_*.log + +# Editor backups +*~ +\#*\# +.#* \ No newline at end of file diff --git a/Build/Perlmutter/build_erf_with_shoc_cuda_Perlmutter.sh b/Build/Perlmutter/build_erf_with_shoc_cuda_Perlmutter.sh new file mode 100755 index 0000000000..1d577bd56e --- /dev/null +++ b/Build/Perlmutter/build_erf_with_shoc_cuda_Perlmutter.sh @@ -0,0 +1,104 @@ +#!/bin/bash +set -e +set -o pipefail + +# Function to verify if a directory is the ERF repo root +verify_erf_dir() { + local dir=$1 + + # Check for basic structure + if [ ! -f "$dir/CMakeLists.txt" ] || [ ! -d "$dir/Source" ]; then + return 1 + fi + + # Check for "Energy Research and Forecasting" in key files + local found=0 + + if [ -f "$dir/README.rst" ]; then + if grep -q "Energy Research and Forecasting" "$dir/README.rst" 2>/dev/null; then + found=1 + fi + fi + + if [ $found -eq 0 ] && [ -f "$dir/LICENSE.md" ]; then + if grep -q "Energy Research and Forecasting" "$dir/LICENSE.md" 2>/dev/null; then + found=1 + fi + fi + + if [ $found -eq 0 ] && [ -f "$dir/CITATION.cff" ]; then + if grep -q "Energy Research and Forecasting" "$dir/CITATION.cff" 2>/dev/null; then + found=1 + fi + fi + + return $((1 - found)) +} + +# Function to find ERF repo root with multiple fallbacks +find_erf_dir() { + # Method 1: Use git to find repo root + if command -v git &> /dev/null; then + if git rev-parse --is-inside-work-tree &> /dev/null 2>&1; then + local git_root="$(git rev-parse --show-toplevel)" + if verify_erf_dir "$git_root"; then + ERF_DIR="$git_root" + echo "Detected ERF_DIR from git: $ERF_DIR" + return 0 + fi + fi + fi + + # Method 2: Try going up from script location + local script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + # Script is in Build/Perlmutter/, so go up 2 levels + local candidate="$(cd "$script_dir/../.." && pwd)" + if verify_erf_dir "$candidate"; then + ERF_DIR="$candidate" + echo "Detected ERF_DIR from script location: $ERF_DIR" + return 0 + fi + + # Method 3: Check current directory + if verify_erf_dir "$PWD"; then + ERF_DIR="$PWD" + echo "Detected ERF_DIR from current directory: $ERF_DIR" + return 0 + fi + + echo "Error: Could not auto-detect ERF_DIR" + echo "Verification requires:" + echo " - CMakeLists.txt and Source/ directory" + echo " - 'Energy Research and Forecasting' in README.rst, LICENSE.md, or CITATION.cff" + return 1 +} + +################################################################################### + +# 1. Resolve ERF_DIR +# Detect ERF_DIR +if ! find_erf_dir; then + exit 1 +fi + +export ERF_DIR + +E3SM_DIR="$ERF_DIR/external/E3SM" +if [ ! -d "$E3SM_DIR" ]; then + echo "external/E3SM folder not found, running eamxx_clone.sh..." + source "$ERF_DIR/Build/GNU_Ekat/eamxx_clone.sh" +else + echo "external/E3SM folder already exists, skipping clone." +fi + +# 3. Prepare build directory +echo "Preparing build directory..." +mkdir -p "$ERF_DIR/build" +cp "$ERF_DIR/Build/Perlmutter/cmake_with_cuda_shoc_Perlmutter.sh" "$ERF_DIR/build/" + +# 4. Move into build directory +cd "$ERF_DIR/build" + +# Run cmake setup +echo "Running cmake_with_cuda_shoc_Perlmutter.sh..." +source cmake_with_cuda_shoc_Perlmutter.sh diff --git a/Build/Perlmutter/cmake_with_cuda_perlmutter.sh b/Build/Perlmutter/cmake_with_cuda_perlmutter.sh new file mode 100755 index 0000000000..316dbc2294 --- /dev/null +++ b/Build/Perlmutter/cmake_with_cuda_perlmutter.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Load the needed modules +module load gcc-native cmake cray-mpich cray-libsci cray-hdf5-parallel cray-netcdf-hdf5parallel + +# GPU-aware mpi is on by default (set it anyways) +export MPICH_GPU_SUPPORT_ENABLED=1 + +# Deduce the lib paths and files with $(CC/cc/ftn --cray-print-opts=libs) +CRAY_LIBS_CLEAN=$(CC --cray-print-opts=libs | sed 's/-Wl,--as-needed,//g; s/,--no-as-needed//g; s/,-l/ -l/g') +CRAY_LIBS_CLEAN="$CRAY_LIBS_CLEAN $(cc --cray-print-opts=libs | sed 's/-Wl,--as-needed,//g; s/,--no-as-needed//g; s/,-l/ -l/g')" +CRAY_LIBS_CLEAN="$CRAY_LIBS_CLEAN $(ftn --cray-print-opts=libs | sed 's/-Wl,--as-needed,//g; s/,--no-as-needed//g; s/,-l/ -l/g')" + +# Configure and build +cmake -DCMAKE_INSTALL_PREFIX:PATH=./install \ + -DCMAKE_C_COMPILER=cc \ + -DCMAKE_CXX_COMPILER=CC \ + -DCMAKE_C_FLAGS="$(cc --cray-print-opts=cflags)" \ + -DCMAKE_CXX_FLAGS="$(CC --cray-print-opts=cflags)" \ + -DCMAKE_CUDA_FLAGS="$(CC --cray-print-opts=cflags)" \ + -DCMAKE_CXX_STANDARD_LIBRARIES="-lmpi_gnu_123 -lmpi_gtl_cuda" \ + -DCMAKE_CUDA_STANDARD_LIBRARIES="-lmpi_gnu_123 -lmpi_gtl_cuda" \ + -DCMAKE_EXE_LINKER_FLAGS="-Wl,--no-as-needed $CRAY_LIBS_CLEAN" \ + -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DERF_DIM:STRING=3 \ + -DERF_ENABLE_MPI:BOOL=ON \ + -DERF_ENABLE_TESTS:BOOL=ON \ + -DAMReX_CUDA_ARCH=8.0 \ + -DERF_ENABLE_CUDA:BOOL=ON \ + -DERF_ENABLE_FCOMPARE:BOOL=ON \ + -DERF_ENABLE_DOCUMENTATION:BOOL=OFF \ + -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \ + .. && make -j8 diff --git a/Build/Perlmutter/cmake_with_cuda_perlmutter.sh~ b/Build/Perlmutter/cmake_with_cuda_perlmutter.sh~ new file mode 100644 index 0000000000..747a399124 --- /dev/null +++ b/Build/Perlmutter/cmake_with_cuda_perlmutter.sh~ @@ -0,0 +1,49 @@ +#!/bin/bash + +# Load the needed modules +module load gcc-native cmake cray-mpich cray-libsci cray-hdf5-parallel cray-netcdf-hdf5parallel + +# GPU-aware mpi is on by default (set it anyways) +export MPICH_GPU_SUPPORT_ENABLED=1 + +# Deduce the lib paths and files with $(CC/cc/ftn --cray-print-opts=libs) +CRAY_LIBS_CLEAN=$(CC --cray-print-opts=libs | sed 's/-Wl,--as-needed,//g; s/,--no-as-needed//g; s/,-l/ -l/g') +CRAY_LIBS_CLEAN="$CRAY_LIBS_CLEAN $(cc --cray-print-opts=libs | sed 's/-Wl,--as-needed,//g; s/,--no-as-needed//g; s/,-l/ -l/g')" +CRAY_LIBS_CLEAN="$CRAY_LIBS_CLEAN $(ftn --cray-print-opts=libs | sed 's/-Wl,--as-needed,//g; s/,--no-as-needed//g; s/,-l/ -l/g')" + +# Configure and build +cmake -DCMAKE_INSTALL_PREFIX:PATH=./install \ + -DCMAKE_C_COMPILER=cc \ + -DCMAKE_CXX_COMPILER=CC \ + -DCMAKE_C_FLAGS="$(cc --cray-print-opts=cflags)" \ + -DCMAKE_CXX_FLAGS="$(CC --cray-print-opts=cflags)" \ + -DCMAKE_CUDA_FLAGS="$(CC --cray-print-opts=cflags)" \ + -DCMAKE_CXX_STANDARD_LIBRARIES="-lmpi_gnu_123" \ + -DCMAKE_CUDA_STANDARD_LIBRARIES="-lmpi_gnu_123" \ + -DCMAKE_EXE_LINKER_FLAGS="-Wl,--no-as-needed $CRAY_LIBS_CLEAN" \ + -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DERF_DIM:STRING=3 \ + -DERF_ENABLE_MPI:BOOL=ON \ + -DERF_ENABLE_TESTS:BOOL=ON \ + -DERF_ENABLE_CUDA:BOOL=ON \ + -DERF_ENABLE_FCOMPARE:BOOL=ON \ + -DERF_ENABLE_DOCUMENTATION:BOOL=OFF \ + -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \ + .. && make -j8 + + + +cmake -DCMAKE_INSTALL_PREFIX:PATH=./install \ + -DCMAKE_PREFIX_PATH:PATH=${CUDA_HOME}/../../ \ + -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DERF_DIM:STRING=3 \ + -DERF_ENABLE_MPI:BOOL=ON \ + -DERF_ENABLE_CUDA:BOOL=ON \ + -DERF_ENABLE_NVHPC:BOOL=ON \ + -DERF_ENABLE_TESTS:BOOL=ON \ + -DERF_ENABLE_FCOMPARE:BOOL=ON \ + -DERF_ENABLE_DOCUMENTATION:BOOL=OFF \ + -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \ + .. && make -j8 diff --git a/Build/Perlmutter/cmake_with_cuda_shoc_Perlmutter.sh b/Build/Perlmutter/cmake_with_cuda_shoc_Perlmutter.sh new file mode 100644 index 0000000000..c681d79617 --- /dev/null +++ b/Build/Perlmutter/cmake_with_cuda_shoc_Perlmutter.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +module load gcc-native cmake cray-mpich cray-libsci cray-hdf5-parallel cray-netcdf-hdf5parallel + +# NOTE: $(CC --cray-print-opts=libs) can be used to deduce libmpi_gnu_123.so +# Depending on your module version, you may want to add all flags to EXE_LINKER_FLAGS without the as-needed flag if you're building with the fcompare tools + +CRAY_LIBS_CLEAN=$(CC --cray-print-opts=libs | sed 's/-Wl,--as-needed,//g; s/,--no-as-needed//g; s/,-l/ -l/g') +CRAY_LIBS_CLEAN="$CRAY_LIBS_CLEAN $(cc --cray-print-opts=libs | sed 's/-Wl,--as-needed,//g; s/,--no-as-needed//g; s/,-l/ -l/g')" +CRAY_LIBS_CLEAN="$CRAY_LIBS_CLEAN $(ftn --cray-print-opts=libs | sed 's/-Wl,--as-needed,//g; s/,--no-as-needed//g; s/,-l/ -l/g')" + +cmake -DCMAKE_INSTALL_PREFIX:PATH=./install \ + -DCMAKE_PREFIX_PATH:PATH=${CUDATOOLKIT_HOME}/../../ \ + -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DCMAKE_C_COMPILER=cc \ + -DCMAKE_CXX_COMPILER=CC \ + -DCMAKE_C_FLAGS="$(cc --cray-print-opts=cflags)" \ + -DCMAKE_CXX_FLAGS="$(CC --cray-print-opts=cflags)" \ + -DCMAKE_CUDA_FLAGS="$(CC --cray-print-opts=cflags)" \ + -DCMAKE_EXE_LINKER_FLAGS="-Wl,--no-as-needed $CRAY_LIBS_CLEAN" \ + -DCMAKE_CXX_STANDARD_LIBRARIES="-lmpi_gnu_123" \ + -DCMAKE_CUDA_STANDARD_LIBRARIES="-lmpi_gnu_123" \ + -DERF_DIM:STRING=3 \ + -DERF_ENABLE_MPI:BOOL=ON \ + -DERF_ENABLE_TESTS:BOOL=ON \ + -DERF_ENABLE_CUDA:BOOL=ON \ + -DERF_ENABLE_NVHPC:BOOL=ON \ + -DERF_ENABLE_EKAT:BOOL=ON \ + -DERF_ENABLE_SHOC:BOOL=ON \ + -DERF_ENABLE_FCOMPARE:BOOL=ON \ + -DERF_ENABLE_DOCUMENTATION:BOOL=OFF \ + -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \ + .. && make -j8 diff --git a/Build/Perlmutter/cmake_with_cuda_shoc_netcdf_perlmutter.sh b/Build/Perlmutter/cmake_with_cuda_shoc_netcdf_perlmutter.sh new file mode 100644 index 0000000000..8f16e72384 --- /dev/null +++ b/Build/Perlmutter/cmake_with_cuda_shoc_netcdf_perlmutter.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Load the needed modules +module load gcc-native cmake cray-mpich cray-libsci cray-hdf5-parallel cray-netcdf-hdf5parallel + +# GPU-aware mpi is on by default (set it anyways) +export MPICH_GPU_SUPPORT_ENABLED=1 + +# Deduce the lib paths and files with $(CC/cc/ftn --cray-print-opts=libs) +CRAY_LIBS_CLEAN=$(CC --cray-print-opts=libs | sed 's/-Wl,--as-needed,//g; s/,--no-as-needed//g; s/,-l/ -l/g') +CRAY_LIBS_CLEAN="$CRAY_LIBS_CLEAN $(cc --cray-print-opts=libs | sed 's/-Wl,--as-needed,//g; s/,--no-as-needed//g; s/,-l/ -l/g')" +CRAY_LIBS_CLEAN="$CRAY_LIBS_CLEAN $(ftn --cray-print-opts=libs | sed 's/-Wl,--as-needed,//g; s/,--no-as-needed//g; s/,-l/ -l/g')" + +# Configure and build +cmake -DCMAKE_INSTALL_PREFIX:PATH=./install \ + -DCMAKE_C_COMPILER=cc \ + -DCMAKE_CXX_COMPILER=CC \ + -DCMAKE_C_FLAGS="$(cc --cray-print-opts=cflags)" \ + -DCMAKE_CXX_FLAGS="$(CC --cray-print-opts=cflags)" \ + -DCMAKE_CUDA_FLAGS="$(CC --cray-print-opts=cflags)" \ + -DCMAKE_CXX_STANDARD_LIBRARIES="-lmpi_gnu_123 -lmpi_gtl_cuda" \ + -DCMAKE_CUDA_STANDARD_LIBRARIES="-lmpi_gnu_123 -lmpi_gtl_cuda" \ + -DCMAKE_EXE_LINKER_FLAGS="-Wl,--no-as-needed $CRAY_LIBS_CLEAN" \ + -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DERF_DIM:STRING=3 \ + -DERF_ENABLE_MPI:BOOL=ON \ + -DERF_ENABLE_TESTS:BOOL=ON \ + -DERF_ENABLE_CUDA:BOOL=ON \ + -DERF_ENABLE_SHOC:BOOL=ON \ + -DERF_ENABLE_HDF5:BOOL=ON \ + -DERF_ENABLE_NETCDF:BOOL=ON \ + -DERF_ENABLE_FCOMPARE:BOOL=ON \ + -DERF_ENABLE_DOCUMENTATION:BOOL=OFF \ + -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \ + .. && make -j8 diff --git a/Build/cmake_with_kokkos_ekat_hdf5_netcdf_fftw3_perlmutter.sh b/Build/Perlmutter/cmake_with_kokkos_ekat_hdf5_netcdf_fftw3_perlmutter.sh similarity index 100% rename from Build/cmake_with_kokkos_ekat_hdf5_netcdf_fftw3_perlmutter.sh rename to Build/Perlmutter/cmake_with_kokkos_ekat_hdf5_netcdf_fftw3_perlmutter.sh diff --git a/Build/Perlmutter/cmake_with_shoc_netcdf_perlmutter.sh b/Build/Perlmutter/cmake_with_shoc_netcdf_perlmutter.sh new file mode 100644 index 0000000000..7f9fb88262 --- /dev/null +++ b/Build/Perlmutter/cmake_with_shoc_netcdf_perlmutter.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Load the needed modules +module load gcc-native cmake cray-mpich cray-libsci cray-hdf5-parallel cray-netcdf-hdf5parallel + +# Deactivate GPU aware MPI for CPU build +export MPICH_GPU_SUPPORT_ENABLED=0 +export CRAY_ACCEL_TARGET=none + +# Deduce the lib paths and files with $(CC/cc/ftn --cray-print-opts=libs) +CRAY_LIBS_CLEAN=$(CC --cray-print-opts=libs | sed 's/-Wl,--as-needed,//g; s/,--no-as-needed//g; s/,-l/ -l/g') +CRAY_LIBS_CLEAN="$CRAY_LIBS_CLEAN $(cc --cray-print-opts=libs | sed 's/-Wl,--as-needed,//g; s/,--no-as-needed//g; s/,-l/ -l/g')" +CRAY_LIBS_CLEAN="$CRAY_LIBS_CLEAN $(ftn --cray-print-opts=libs | sed 's/-Wl,--as-needed,//g; s/,--no-as-needed//g; s/,-l/ -l/g')" + +# Configure and build +cmake -DCMAKE_INSTALL_PREFIX:PATH=./install \ + -DCMAKE_C_COMPILER=cc \ + -DCMAKE_CXX_COMPILER=CC \ + -DCMAKE_C_FLAGS="$(cc --cray-print-opts=cflags)" \ + -DCMAKE_CXX_FLAGS="$(CC --cray-print-opts=cflags)" \ + -DCMAKE_CUDA_FLAGS="$(CC --cray-print-opts=cflags)" \ + -DCMAKE_CXX_STANDARD_LIBRARIES="-lmpi_gnu_123" \ + -DCMAKE_CUDA_STANDARD_LIBRARIES="-lmpi_gnu_123" \ + -DCMAKE_EXE_LINKER_FLAGS="-Wl,--no-as-needed $CRAY_LIBS_CLEAN" \ + -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DERF_DIM:STRING=3 \ + -DERF_ENABLE_MPI:BOOL=ON \ + -DERF_ENABLE_TESTS:BOOL=ON \ + -DERF_ENABLE_SHOC:BOOL=ON \ + -DERF_ENABLE_HDF5:BOOL=ON \ + -DERF_ENABLE_NETCDF:BOOL=ON \ + -DERF_ENABLE_FCOMPARE:BOOL=ON \ + -DERF_ENABLE_DOCUMENTATION:BOOL=OFF \ + -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \ + .. && make -j8 diff --git a/Build/build_erf_with_shoc.sh b/Build/build_erf_with_shoc.sh new file mode 100644 index 0000000000..d212f40033 --- /dev/null +++ b/Build/build_erf_with_shoc.sh @@ -0,0 +1,104 @@ +#!/bin/bash +set -e +set -o pipefail + +# Function to verify if a directory is the ERF repo root +verify_erf_dir() { + local dir=$1 + + # Check for basic structure + if [ ! -f "$dir/CMakeLists.txt" ] || [ ! -d "$dir/Source" ]; then + return 1 + fi + + # Check for "Energy Research and Forecasting" in key files + local found=0 + + if [ -f "$dir/README.rst" ]; then + if grep -q "Energy Research and Forecasting" "$dir/README.rst" 2>/dev/null; then + found=1 + fi + fi + + if [ $found -eq 0 ] && [ -f "$dir/LICENSE.md" ]; then + if grep -q "Energy Research and Forecasting" "$dir/LICENSE.md" 2>/dev/null; then + found=1 + fi + fi + + if [ $found -eq 0 ] && [ -f "$dir/CITATION.cff" ]; then + if grep -q "Energy Research and Forecasting" "$dir/CITATION.cff" 2>/dev/null; then + found=1 + fi + fi + + return $((1 - found)) +} + +# Function to find ERF repo root with multiple fallbacks +find_erf_dir() { + # Method 1: Use git to find repo root + if command -v git &> /dev/null; then + if git rev-parse --is-inside-work-tree &> /dev/null 2>&1; then + local git_root="$(git rev-parse --show-toplevel)" + if verify_erf_dir "$git_root"; then + ERF_DIR="$git_root" + echo "Detected ERF_DIR from git: $ERF_DIR" + return 0 + fi + fi + fi + + # Method 2: Try going up from script location + local script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + # Script is in Build/Perlmutter/, so go up 2 levels + local candidate="$(cd "$script_dir/../.." && pwd)" + if verify_erf_dir "$candidate"; then + ERF_DIR="$candidate" + echo "Detected ERF_DIR from script location: $ERF_DIR" + return 0 + fi + + # Method 3: Check current directory + if verify_erf_dir "$PWD"; then + ERF_DIR="$PWD" + echo "Detected ERF_DIR from current directory: $ERF_DIR" + return 0 + fi + + echo "Error: Could not auto-detect ERF_DIR" + echo "Verification requires:" + echo " - CMakeLists.txt and Source/ directory" + echo " - 'Energy Research and Forecasting' in README.rst, LICENSE.md, or CITATION.cff" + return 1 +} + +################################################################################### + +# 1. Resolve ERF_DIR +# Detect ERF_DIR +if ! find_erf_dir; then + exit 1 +fi + +export ERF_DIR + +E3SM_DIR="$ERF_DIR/external/E3SM" +if [ ! -d "$E3SM_DIR" ]; then + echo "external/E3SM folder not found, running eamxx_clone.sh..." + source "$ERF_DIR/Build/GNU_Ekat/eamxx_clone.sh" +else + echo "external/E3SM folder already exists, skipping clone." +fi + +# 3. Prepare build directory +echo "Preparing build directory..." +mkdir -p "$ERF_DIR/build" +cp "$ERF_DIR/Build/cmake_with_shoc.sh" "$ERF_DIR/build/" + +# 4. Move into build directory +cd "$ERF_DIR/build" + +# Run cmake setup +echo "Running cmake_with_shoc.sh..." +source cmake_with_shoc.sh diff --git a/Build/build_erf_with_shoc_cuda_Perlmutter.sh b/Build/build_erf_with_shoc_cuda_Perlmutter.sh deleted file mode 100644 index add85b002e..0000000000 --- a/Build/build_erf_with_shoc_cuda_Perlmutter.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -set -e -set -o pipefail - -# 1. Resolve ERF_DIR to the repo root (one level up from this script) -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -export ERF_DIR="$(dirname "$SCRIPT_DIR")" - -echo "ERF_DIR set to: $ERF_DIR" - -E3SM_DIR="$ERF_DIR/external/E3SM" -if [ ! -d "$E3SM_DIR" ]; then -echo "external/E3SM folder not found, running eamxx_clone.sh..." -source "$ERF_DIR/Build/GNU_Ekat/eamxx_clone.sh" -else -echo "external/E3SM folder already exists, skipping clone." -fi - -# 3. Prepare build directory -echo "Preparing build directory..." -mkdir -p "$ERF_DIR/build" -cp "$ERF_DIR/Build/cmake_with_shoc_cuda_Perlmutter.sh" "$ERF_DIR/build/" - -# 4. Move into build directory -cd "$ERF_DIR/build" - -# 5. Run cmake setup -echo "Running cmake_with_shoc_cuda_Perlmutter.sh..." -source cmake_with_shoc_cuda_Perlmutter.sh diff --git a/Build/cmake_cuda_perlmutter.sh b/Build/cmake_cuda_perlmutter.sh deleted file mode 100644 index adfe9a1257..0000000000 --- a/Build/cmake_cuda_perlmutter.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -# Example CMake config script for an OSX laptop with OpenMPI - -cmake -DCMAKE_INSTALL_PREFIX:PATH=./install \ - -DCMAKE_PREFIX_PATH:PATH=${CUDA_HOME}/../../ \ - -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \ - -DCMAKE_BUILD_TYPE:STRING=Release \ - -DERF_DIM:STRING=3 \ - -DERF_ENABLE_MPI:BOOL=ON \ - -DERF_ENABLE_CUDA:BOOL=ON \ - -DERF_ENABLE_NVHPC:BOOL=ON \ - -DERF_ENABLE_TESTS:BOOL=ON \ - -DERF_ENABLE_FCOMPARE:BOOL=ON \ - -DERF_ENABLE_DOCUMENTATION:BOOL=OFF \ - -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \ - .. && make -j8 diff --git a/Build/cmake_with_kokkos_many.sh b/Build/cmake_with_kokkos_many.sh new file mode 100755 index 0000000000..43218b0dbe --- /dev/null +++ b/Build/cmake_with_kokkos_many.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +#Example cmake configuration script that assumes cray detection + +cmake -DCMAKE_INSTALL_PREFIX:PATH=./install_erf \ + -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DERF_DIM:STRING=3 \ + -DERF_ENABLE_FFT:BOOL=ON \ + -DERF_ENABLE_NETCDF:BOOL=ON \ + -DERF_ENABLE_HDF5:BOOL=ON \ + -DERF_ENABLE_RRTMGP:BOOL=ON \ + -DERF_ENABLE_SHOC:BOOL=OFF \ + -DERF_ENABLE_MPI:BOOL=ON \ + -DERF_ENABLE_CUDA:BOOL=OFF \ + -DERF_ENABLE_HIP:BOOL=OFF \ + -DERF_ENABLE_SYCL:BOOL=OFF \ + -DERF_ENABLE_TESTS:BOOL=ON \ + -DERF_ENABLE_FCOMPARE:BOOL=ON \ + -DERF_ENABLE_DOCUMENTATION:BOOL=OFF \ + -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \ + -B build_erf .. + +cmake --build build_erf -j10 -v +cmake --install build_erf --prefix=install_erf diff --git a/Build/cmake_with_kokkos_many_cuda.sh b/Build/cmake_with_kokkos_many_cuda.sh new file mode 100755 index 0000000000..f13cf79ee9 --- /dev/null +++ b/Build/cmake_with_kokkos_many_cuda.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +#Example cmake configuration script that assumes cray detection + +cmake -DCMAKE_INSTALL_PREFIX:PATH=./install_erf \ + -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DERF_DIM:STRING=3 \ + -DERF_ENABLE_FFT:BOOL=ON \ + -DERF_ENABLE_NETCDF:BOOL=ON \ + -DERF_ENABLE_HDF5:BOOL=ON \ + -DERF_ENABLE_RRTMGP:BOOL=ON \ + -DERF_ENABLE_SHOC:BOOL=OFF \ + -DERF_ENABLE_MPI:BOOL=ON \ + -DERF_ENABLE_CUDA:BOOL=ON \ + -DERF_ENABLE_HIP:BOOL=OFF \ + -DERF_ENABLE_SYCL:BOOL=OFF \ + -DERF_ENABLE_TESTS:BOOL=ON \ + -DERF_ENABLE_FCOMPARE:BOOL=ON \ + -DERF_ENABLE_DOCUMENTATION:BOOL=OFF \ + -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \ + --log-context \ + -B build_erf .. + +cmake --build build_erf -j10 -v +cmake --install build_erf --prefix=install_erf diff --git a/Build/cmake_with_kokkos_many_noradiation_hip.sh b/Build/cmake_with_kokkos_many_noradiation_hip.sh new file mode 100755 index 0000000000..839713de50 --- /dev/null +++ b/Build/cmake_with_kokkos_many_noradiation_hip.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +#Example cmake configuration script that assumes cray detection + +cmake -DCMAKE_INSTALL_PREFIX:PATH=./install_erf \ + -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DERF_DIM:STRING=3 \ + -DERF_ENABLE_FFT:BOOL=ON \ + -DERF_ENABLE_NETCDF:BOOL=ON \ + -DERF_ENABLE_HDF5:BOOL=ON \ + -DERF_ENABLE_RRTMGP:BOOL=OFF \ + -DERF_ENABLE_SHOC:BOOL=OFF \ + -DERF_ENABLE_MPI:BOOL=ON \ + -DERF_ENABLE_CUDA:BOOL=OFF \ + -DERF_ENABLE_HIP:BOOL=ON \ + -DERF_ENABLE_SYCL:BOOL=OFF \ + -DERF_ENABLE_TESTS:BOOL=ON \ + -DERF_ENABLE_FCOMPARE:BOOL=ON \ + -DERF_ENABLE_DOCUMENTATION:BOOL=OFF \ + -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \ + --log-context \ + -B build_erf .. + +cmake --build build_erf -j10 -v +cmake --install build_erf --prefix=install_erf diff --git a/Build/cmake_with_kokkos_many_sycl.sh b/Build/cmake_with_kokkos_many_sycl.sh new file mode 100755 index 0000000000..b01c6dad09 --- /dev/null +++ b/Build/cmake_with_kokkos_many_sycl.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +#Example cmake configuration script that assumes cray detection + +cmake -DCMAKE_INSTALL_PREFIX:PATH=./install_erf \ + -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DERF_DIM:STRING=3 \ + -DERF_ENABLE_FFT:BOOL=ON \ + -DERF_ENABLE_NETCDF:BOOL=ON \ + -DERF_ENABLE_HDF5:BOOL=ON \ + -DERF_ENABLE_RRTMGP:BOOL=ON \ + -DERF_ENABLE_SHOC:BOOL=OFF \ + -DERF_ENABLE_MPI:BOOL=ON \ + -DERF_ENABLE_CUDA:BOOL=OFF \ + -DERF_ENABLE_HIP:BOOL=OFF \ + -DERF_ENABLE_SYCL:BOOL=ON \ + -DERF_ENABLE_TESTS:BOOL=ON \ + -DERF_ENABLE_FCOMPARE:BOOL=ON \ + -DERF_ENABLE_DOCUMENTATION:BOOL=OFF \ + -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \ + -B build_erf .. + +cmake --build build_erf -j10 -v +cmake --install build_erf --prefix=install_erf diff --git a/Build/cmake_with_shoc_cuda_Perlmutter.sh b/Build/cmake_with_shoc_cuda_Perlmutter.sh deleted file mode 100644 index 82d06ffb9f..0000000000 --- a/Build/cmake_with_shoc_cuda_Perlmutter.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -cmake -DCMAKE_INSTALL_PREFIX:PATH=./install \ - -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \ - -DCMAKE_BUILD_TYPE:STRING=Release \ - -DCMAKE_PREFIX_PATH:PATH=${CUDATOOLKIT_HOME}/../../ \ - -DERF_DIM:STRING=3 \ - -DERF_ENABLE_MPI:BOOL=ON \ - -DERF_ENABLE_TESTS:BOOL=ON \ - -DERF_ENABLE_CUDA:BOOL=ON \ - -DERF_ENABLE_NVHPC:BOOL=ON \ - -DERF_ENABLE_EKAT:BOOL=ON \ - -DERF_ENABLE_SHOC:BOOL=ON \ - -DERF_ENABLE_FCOMPARE:BOOL=ON \ - -DERF_ENABLE_DOCUMENTATION:BOOL=OFF \ - -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \ - .. && make -j8 diff --git a/Build/machines/aurora_erf.profile b/Build/machines/aurora_erf.profile new file mode 100644 index 0000000000..8a4ada731f --- /dev/null +++ b/Build/machines/aurora_erf.profile @@ -0,0 +1,8 @@ +# required dependencies +module load cmake + +module load hdf5/1.14.6 +module load netcdf-cxx4 + +# necessary to use build or run with GPU-aware MPICH +# export MPIR_CVAR_ENABLE_GPU=1 \ No newline at end of file diff --git a/Build/machines/frontier_erf.profile b/Build/machines/frontier_erf.profile new file mode 100644 index 0000000000..01645623b8 --- /dev/null +++ b/Build/machines/frontier_erf.profile @@ -0,0 +1,10 @@ +module load cmake/3.30.5 +module load craype-accel-amd-gfx90a +module load rocm/6.2.4 +module load cray-mpich/8.1.31 +module load cce/18.0.1 + +module load cray-hdf5-parallel +module load cray-netcdf-hdf5parallel + +# export MPICH_GPU_SUPPORT_ENABLED=1 diff --git a/Build/machines/perlmutter_erf.profile b/Build/machines/perlmutter_erf.profile new file mode 100644 index 0000000000..c60418c6b8 --- /dev/null +++ b/Build/machines/perlmutter_erf.profile @@ -0,0 +1,14 @@ +#!/bin/bash + +module load gcc-native/13.2 cmake cudatoolkit cray-hdf5-parallel cray-netcdf-hdf5parallel cray-libsci + +#module load gcc-native/13.2 +#module load cray-mpich/8.1.30 +#module load cray-hdf5-parallel/1.14.3.1 +#module load cray-netcdf-hdf5parallel/4.9.0.13 +#module load cmake/3.30.2 +#module load cray-libsci/24.07.0 +#module load cray-parallel-netcdf/1.12.3.13 + +# Automatically included with module load gpu +# export MPICH_GPU_SUPPORT_ENABLED=1 \ No newline at end of file diff --git a/Build/machines/polaris_erf.profile b/Build/machines/polaris_erf.profile new file mode 100644 index 0000000000..6502c320e8 --- /dev/null +++ b/Build/machines/polaris_erf.profile @@ -0,0 +1,25 @@ +# swap to the Milan cray package +module load craype-x86-milan + +# extra modules +module use /soft/modulefiles +module load spack-pe-gnu + +# add cuda +module load cuda/12.6 +module load cudatoolkit-standalone/12.6 +module load craype-accel-nvidia80 + +# required dependencies +module load cmake + +# default gcc-native too new for cuda/12.6 +module load gcc-native/13.2 + +module load cray-hdf5-parallel +module load cray-libsci/25.03.0 + + +module load cray-netcdf-hdf5parallel + +# export MPICH_GPU_SUPPORT_ENABLED=1 \ No newline at end of file diff --git a/Build/setup_cmake_validation.sh b/Build/setup_cmake_validation.sh new file mode 100755 index 0000000000..db40ed3d25 --- /dev/null +++ b/Build/setup_cmake_validation.sh @@ -0,0 +1,316 @@ +#!/bin/bash + +set -e +set -o pipefail + +# Function to verify if a directory is the ERF repo root +verify_erf_dir() { + local dir=$1 + + # Check for basic structure + if [ ! -f "$dir/CMakeLists.txt" ] || [ ! -d "$dir/Source" ]; then + return 1 + fi + + # Check for "Energy Research and Forecasting" in key files + local found=0 + + if [ -f "$dir/README.rst" ]; then + if grep -q "Energy Research and Forecasting" "$dir/README.rst" 2>/dev/null || true; then + found=1 + fi + fi + + if [ $found -eq 0 ] && [ -f "$dir/LICENSE.md" ]; then + if grep -q "Energy Research and Forecasting" "$dir/LICENSE.md" 2>/dev/null || true; then + found=1 + fi + fi + + if [ $found -eq 0 ] && [ -f "$dir/CITATION.cff" ]; then + if grep -q "Energy Research and Forecasting" "$dir/CITATION.cff" 2>/dev/null || true; then + found=1 + fi + fi + + return $((1 - found)) +} + +# Function to find ERF repo root +find_erf_dir() { + # Method 1: Check if we're already in Build/ + if [ -f "../CMakeLists.txt" ] && [ -d "../Source" ]; then + local candidate="$(cd .. && pwd)" + if verify_erf_dir "$candidate"; then + ERF_DIR="$candidate" + echo "Detected ERF_DIR from Build location: $ERF_DIR" + return 0 + fi + fi + + # Method 2: Use git to find repo root + if command -v git &> /dev/null; then + if git rev-parse --is-inside-work-tree &> /dev/null 2>&1; then + local git_root="$(git rev-parse --show-toplevel)" + if verify_erf_dir "$git_root"; then + ERF_DIR="$git_root" + echo "Detected ERF_DIR from git: $ERF_DIR" + return 0 + fi + fi + fi + + # Method 3: Try going up from script location + local script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + # Check if script is in Build/ directory + if [[ "$script_dir" =~ /Build$ ]]; then + local candidate="$(dirname "$script_dir")" + if verify_erf_dir "$candidate"; then + ERF_DIR="$candidate" + echo "Detected ERF_DIR from script location: $ERF_DIR" + return 0 + fi + fi + + # Method 4: Check current directory + if verify_erf_dir "$PWD"; then + ERF_DIR="$PWD" + echo "Detected ERF_DIR from current directory: $ERF_DIR" + return 0 + fi + + return 1 +} + +# Parse arguments +if [ $# -lt 1 ] || [ $# -gt 3 ]; then + echo "Usage: $0 [script_pattern] [erf_dir]" + echo "" + echo "Sets:" + echo " default - Scripts from Build/" + echo " perlmutter - Scripts from Build/Perlmutter/" + echo " gnu_ekat - Scripts from Build/GNU_Ekat/" + echo "" + echo "If script_pattern is provided, creates build_/" + echo "Otherwise creates build_/" + echo "" + echo "If erf_dir is provided, uses that as ERF_DIR" + echo "Otherwise auto-detects ERF repo root" + exit 1 +fi + +SET=$1 +PATTERN=${2:-} +ERF_DIR_ARG=${3:-} + +# Set ERF_DIR +if [ -n "$ERF_DIR_ARG" ]; then + ERF_DIR="$ERF_DIR_ARG" + echo "Using provided ERF_DIR: $ERF_DIR" + if ! verify_erf_dir "$ERF_DIR"; then + echo "Error: Provided directory is not a valid ERF repository" + echo "Must contain 'Energy Research and Forecasting' in README.rst, LICENSE.md, or CITATION.cff" + exit 1 + fi +else + if ! find_erf_dir; then + echo "Error: Could not auto-detect ERF_DIR" + echo "Please provide it as the third argument or run from ERF Build/ directory" + echo "" + echo "Verification checks for:" + echo " - CMakeLists.txt and Source/ directory" + echo " - 'Energy Research and Forecasting' in README.rst, LICENSE.md, or CITATION.cff" + exit 1 + fi +fi + +echo "ERF_DIR set to: $ERF_DIR" + +# Define source directories relative to ERF_DIR +DEFAULT_DIR="$ERF_DIR/Build" +PERLMUTTER_DIR="$ERF_DIR/Build/Perlmutter" +GNU_EKAT_DIR="$ERF_DIR/Build/GNU_Ekat" + +case $SET in + default) + SRC_DIR="$DEFAULT_DIR" + ;; + perlmutter) + SRC_DIR="$PERLMUTTER_DIR" + ;; + gnu_ekat) + SRC_DIR="$GNU_EKAT_DIR" + ;; + *) + echo "Error: Invalid set '$SET'" + echo "Choose: default, perlmutter, or gnu_ekat" + exit 1 + ;; +esac + +if [ ! -d "$SRC_DIR" ]; then + echo "Error: Source directory does not exist: $SRC_DIR" + exit 1 +fi + +# Determine build directory name +if [ -n "$PATTERN" ]; then + BUILD_DIR="$ERF_DIR/build_${PATTERN}" +else + BUILD_DIR="$ERF_DIR/build_${SET}" +fi + +# Create build directory +mkdir -p "$BUILD_DIR" +echo "Created directory: $BUILD_DIR" + +# Find and copy ERF cmake build scripts +echo "Scanning for ERF cmake scripts in $SRC_DIR:" +COPIED=0 +SKIPPED=0 + +# Temporarily disable exit on error for the loop +set +e + +for script in "$SRC_DIR"/*.sh; do + # Check if file exists (glob might not match anything) + if [ ! -f "$script" ]; then + continue + fi + + basename_script=$(basename "$script") + + # Skip backup files + if [[ "$basename_script" =~ ~$ ]]; then + SKIPPED=$((SKIPPED + 1)) + continue + fi + + # Check if it's an ERF cmake script (contains DERF or cmake) + has_derf=0 + has_cmake=0 + + grep -q "DERF" "$script" 2>/dev/null && has_derf=1 + grep -q "cmake" "$script" 2>/dev/null && has_cmake=1 + + if [ $has_derf -eq 1 ] || [ $has_cmake -eq 1 ]; then + cp "$script" "$BUILD_DIR/" + chmod +x "$BUILD_DIR/$basename_script" + echo " DONE: $basename_script" + COPIED=$((COPIED + 1)) + else + echo " ERROR: $basename_script (no DERF or cmake found)" + SKIPPED=$((SKIPPED + 1)) + fi +done + +# Re-enable exit on error +set -e + +echo "" +echo "Summary: Copied $COPIED script(s), skipped $SKIPPED" + +if [ $COPIED -eq 0 ]; then + echo "Warning: No ERF cmake scripts found" + echo "Scripts should contain 'DERF' or 'cmake'" +fi + +# Create a run script in the build directory +cat > "$BUILD_DIR/run.sh" << 'EOF' +#!/bin/bash + +set -e +set -o pipefail + +# Resolve ERF_DIR (go up from build directory) +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +export ERF_DIR="$(dirname "$SCRIPT_DIR")" + +echo "ERF_DIR set to: $ERF_DIR" + +# Find all .sh scripts (excluding run.sh and backups) +SCRIPTS=() +for script in *.sh; do + if [ "$script" = "run.sh" ]; then + continue + fi + if [[ "$script" =~ ~$ ]]; then + continue + fi + if [ -f "$script" ]; then + SCRIPTS+=("$script") + fi +done + +# Sort scripts alphabetically +IFS=$'\n' SCRIPTS=($(sort <<<"${SCRIPTS[*]}")) +unset IFS + +if [ ${#SCRIPTS[@]} -eq 0 ]; then + echo "Error: No build scripts found in this directory" + exit 1 +fi + +if [ $# -ne 1 ]; then + echo "Usage: $0 " + echo "" + echo "Available ERF cmake scripts:" + for i in "${!SCRIPTS[@]}"; do + script_base="${SCRIPTS[$i]%.sh}" + printf "%3d: %s\n" $((i+1)) "${SCRIPTS[$i]}" + printf " -> subdirectory: %s/script_%s/\n" "$ERF_DIR" "$script_base" + done + echo "" + echo "Each script will run in its own clean subdirectory at ERF root." + exit 1 +fi + +NUM=$1 +if [ $NUM -lt 1 ] || [ $NUM -gt ${#SCRIPTS[@]} ]; then + echo "Error: Number must be between 1 and ${#SCRIPTS[@]}" + exit 1 +fi + +SCRIPT="${SCRIPTS[$((NUM-1))]}" +SCRIPT_BASE="${SCRIPT%.sh}" +SUBDIR="$ERF_DIR/script_${SCRIPT_BASE}" + +# Create a clean subdirectory for this script +if [ -d "$SUBDIR" ]; then + echo "Warning: $SUBDIR already exists" + read -p "Delete and recreate? (y/N) " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + rm -rf "$SUBDIR" + else + echo "Aborting. Please remove $SUBDIR manually or choose a different script." + exit 1 + fi +fi + +mkdir -p "$SUBDIR" +echo "========================================" +echo "Running: $SCRIPT" +echo "Build directory: $SUBDIR" +echo "Working directory: $SUBDIR" +echo "========================================" +echo "" + +# Copy the script into the subdirectory and run it there +cp "$SCRIPT" "$SUBDIR/" +cd "$SUBDIR" +bash "./$SCRIPT" +EOF + +chmod +x "$BUILD_DIR/run.sh" + +echo "" +echo "Setup complete!" +echo "Build directory: $BUILD_DIR" +echo "" +echo "To use:" +echo " cd $BUILD_DIR" +echo " ./run.sh # List available scripts" +echo " ./run.sh # Run a specific script" +echo "" +echo "Copied $COPIED script(s)" diff --git a/Build/wrapper_clean_build.sh b/Build/wrapper_clean_build.sh new file mode 100755 index 0000000000..8dec278538 --- /dev/null +++ b/Build/wrapper_clean_build.sh @@ -0,0 +1,271 @@ +#!/bin/bash +set -e + +# ============================================================================ +# CMake Build Wrapper with Cleanup (distclean equivalent) +# ============================================================================ +# +# MODERN CMAKE PRACTICE: +# ---------------------- +# The current best practice for CMake is to use out-of-source builds with: +# cmake -S -B +# +# For example: +# cmake -S .. -B build_release +# cmake --build build_release +# cmake --install build_release --prefix install_release +# +# This keeps your source tree clean and allows multiple build configurations +# (debug, release, different compilers, etc.) in separate directories. +# +# MODERN CMAKE INSTALL: +# --------------------- +# The newer cmake --install command (CMake 3.15+) provides a cleaner interface +# than the older "make install" or "cmake --build . --target install": +# +# cmake --install --prefix +# +# Examples: +# # Install to default CMAKE_INSTALL_PREFIX (set during configure) +# cmake --install build_release +# +# # Install to custom location +# cmake --install build_release --prefix /opt/erf +# +# # Install to local directory +# cmake --install build_release --prefix ./install +# +# This is preferred because: +# - Works regardless of build system (make, ninja, etc.) +# - Doesn't require entering the build directory +# - More explicit and consistent syntax +# - Allows overriding install location without reconfiguring +# +# Note: You can still set CMAKE_INSTALL_PREFIX during configuration: +# cmake -S .. -B build_release -DCMAKE_INSTALL_PREFIX=/usr/local +# +# ABOUT ERF/Build DIRECTORY: +# -------------------------- +# The ERF/Build directory is primarily intended as a single build directory +# for users doing one configuration. If you're testing multiple configurations +# (CPU, GPU, different flags), you should use separate build directories: +# ERF/build_cpu/ +# ERF/build_gpu/ +# ERF/build_debug/ +# etc. +# +# With corresponding install directories if needed: +# ERF/install_cpu/ +# ERF/install_gpu/ +# ERF/install_debug/ +# +# CLEANUP BEHAVIOR (GNU Make Standard): +# ------------------------------------- +# This script performs a 'distclean' equivalent operation, which per GNU +# standards means: "Delete all files in the current directory (or created +# by this makefile) that are created by configuring or building the program." +# +# For CMake, this includes: +# - CMakeCache.txt (configuration file) +# - CMakeFiles/ (generated build system files) +# - *.cmake (generated configuration scripts) +# - Makefile (if generated) +# - Any other CMake-generated artifacts +# +# This ensures a completely fresh configuration and build, as if you had +# just unpacked the source distribution. +# +# Note: This does NOT delete install directories - those should be managed +# separately (equivalent to 'uninstall' target in GNU make). +# +# ============================================================================ + +SCRIPT=$1 + +if [ -z "$SCRIPT" ]; then + echo "ERROR: No build script provided" + echo "Usage: $0 " + exit 1 +fi + +if [ ! -f "$SCRIPT" ]; then + echo "ERROR: Build script not found: $SCRIPT" + exit 1 +fi + +# Check what would be deleted (distclean items) +FILES_TO_DELETE="" +[ -f "CMakeCache.txt" ] && FILES_TO_DELETE="$FILES_TO_DELETE CMakeCache.txt" +[ -d "CMakeFiles" ] && FILES_TO_DELETE="$FILES_TO_DELETE CMakeFiles/" +[ -f "Makefile" ] && FILES_TO_DELETE="$FILES_TO_DELETE Makefile" +[ -f "cmake_install.cmake" ] && FILES_TO_DELETE="$FILES_TO_DELETE cmake_install.cmake" +[ -f "CTestTestfile.cmake" ] && FILES_TO_DELETE="$FILES_TO_DELETE CTestTestfile.cmake" + +# Find any other .cmake files (excluding those we might want to keep) +OTHER_CMAKE=$(find . -maxdepth 1 -name "*.cmake" -type f 2>/dev/null | \ + grep -v "cmake_install.cmake\|CTestTestfile.cmake" || true) +if [ -n "$OTHER_CMAKE" ]; then + FILES_TO_DELETE="$FILES_TO_DELETE $OTHER_CMAKE" +fi + +# Additional common CMake artifacts +[ -d "Testing" ] && FILES_TO_DELETE="$FILES_TO_DELETE Testing/" +[ -d "_deps" ] && FILES_TO_DELETE="$FILES_TO_DELETE _deps/" +[ -f "compile_commands.json" ] && FILES_TO_DELETE="$FILES_TO_DELETE compile_commands.json" + +# === Add after initial FILES_TO_DELETE setup === + +# Built artifact directories (these are built, not source) +for d in Exec Submodules Tests bin cmake_packages externals; do + [ -d "$d" ] && FILES_TO_DELETE="$FILES_TO_DELETE $d/" +done + +# CTest artifacts +[ -f "DartConfiguration.tcl" ] && FILES_TO_DELETE="$FILES_TO_DELETE DartConfiguration.tcl" + +# Generated project config +[ -f "ERFConfig.cmake" ] && FILES_TO_DELETE="$FILES_TO_DELETE ERFConfig.cmake" + +# pkg-config files +find . -maxdepth 1 -name "*.pc" -type f 2>/dev/null | while read -r f; do + FILES_TO_DELETE="$FILES_TO_DELETE $f" +done + +# Build artifacts (libraries) +find . -maxdepth 1 \( -name "lib*.a" -o -name "lib*.so" \) -type f 2>/dev/null | while read -r f; do + FILES_TO_DELETE="$FILES_TO_DELETE $f" +done + +# Build logs +for f in build_*.log git-state.txt; do + [ -f "$f" ] && FILES_TO_DELETE="$FILES_TO_DELETE $f" +done + +# === Check for install directory from CMakeCache.txt === +INSTALL_DIR="" +if [ -f "CMakeCache.txt" ]; then + # Extract CMAKE_INSTALL_PREFIX from cache + INSTALL_PREFIX=$(grep "^CMAKE_INSTALL_PREFIX:" CMakeCache.txt | cut -d'=' -f2) + + if [ -n "$INSTALL_PREFIX" ] && [ -d "$INSTALL_PREFIX" ]; then + # Convert to absolute path for comparison + INSTALL_DIR=$(cd "$INSTALL_PREFIX" 2>/dev/null && pwd || echo "$INSTALL_PREFIX") + + # Check if it's a subdirectory of current directory (local install) + CURRENT_DIR=$(pwd) + if [[ "$INSTALL_DIR" == "$CURRENT_DIR"/* ]]; then + # It's a local install directory + INSTALL_DIR_RELATIVE=$(realpath --relative-to="$CURRENT_DIR" "$INSTALL_DIR" 2>/dev/null || \ + python3 -c "import os.path; print(os.path.relpath('$INSTALL_DIR', '$CURRENT_DIR'))" 2>/dev/null || \ + echo "$INSTALL_DIR") + + echo "" + echo "==========================================" + echo "Install Directory Detected" + echo "==========================================" + echo "This build is configured to install to:" + echo " $INSTALL_DIR_RELATIVE" + echo "" + echo "This directory contains installed artifacts and is separate" + echo "from the build configuration (distclean does NOT remove it)." + echo "" + + if [ -d "$INSTALL_DIR" ]; then + read -p "Also remove install directory? [y/N] " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + CLEAN_INSTALL_DIR="$INSTALL_DIR" + fi + fi + fi + fi +fi + +# === Then in the deletion section, after FILES_TO_DELETE cleanup === + +# If there's nothing to clean, just run the script +if [ -z "$FILES_TO_DELETE" ]; then + echo "Directory is already clean, proceeding with build..." + echo "" +else + # Show what will be deleted + echo "==========================================" + echo "WARNING: About to perform 'distclean'" + echo "==========================================" + echo "This will delete all CMake configuration and build artifacts:" + echo "" + for f in $FILES_TO_DELETE; do + if [ -d "$f" ]; then + echo " - $f (directory)" + else + echo " - $f" + fi + done + echo "" + echo "Current directory: $(pwd)" + echo "" + echo "This operation matches the GNU make 'distclean' target:" + echo " \"Delete all files created by configuring or building\"" + echo "" + echo "Note: Install directories (if any) will NOT be deleted." + echo " Use 'cmake --install --prefix ...' to manage installations." + echo "" + + # Prompt user + read -p "Delete these files/directories? [y/N] " -n 1 -r + echo + + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo "Aborted by user. Not deleting anything." + echo "" + echo "To proceed without cleaning, run the build script directly:" + echo " bash $SCRIPT" + echo "" + echo "Modern CMake workflow reminder:" + echo " 1. Configure: cmake -S -B " + echo " 2. Build: cmake --build " + echo " 3. Install: cmake --install --prefix " + exit 1 + fi + + # Actually delete + echo "" + echo "Performing distclean..." + for f in $FILES_TO_DELETE; do + if [ -d "$f" ]; then + rm -rf "$f" && echo " DONE: Deleted directory: $f" + elif [ -f "$f" ]; then + rm -f "$f" && echo " DONE: Deleted file: $f" + fi + done + + # Clean install directory if requested + if [ -n "$CLEAN_INSTALL_DIR" ]; then + echo "" + echo "Removing install directory..." + rm -rf "$CLEAN_INSTALL_DIR" && echo " DONE: Deleted: $CLEAN_INSTALL_DIR" + fi + + echo "" + echo "Distclean complete. Ready for fresh configuration." + echo "" +fi + +# Set ERF_DIR to the source tree regtest checked out for us +if [ -z "$ERF_DIR" ]; then + if [ -d "../source" ]; then + export ERF_DIR=$(cd ../source && pwd) + echo "Auto-detected ERF_DIR: $ERF_DIR" + else + echo "WARNING: Could not auto-detect ERF_DIR" + echo "Build script may fail if it requires ERF_DIR" + fi +fi + +# Run the actual build script +echo "==========================================" +echo "Running build script: $SCRIPT" +echo "==========================================" +echo "" + +bash "$SCRIPT" diff --git a/Build/wrapper_clean_build_auto.sh b/Build/wrapper_clean_build_auto.sh new file mode 100755 index 0000000000..df0b483a7e --- /dev/null +++ b/Build/wrapper_clean_build_auto.sh @@ -0,0 +1,103 @@ +#!/bin/bash +set -e + +# ============================================================================ +# Automated CMake Build Wrapper with Cleanup (distclean equivalent) +# ============================================================================ +# Non-interactive version for CI/regression testing +# See wrapper_clean_build.sh for detailed documentation on: +# - Modern cmake -S -B workflow +# - Modern cmake --install --prefix usage +# - GNU distclean behavior +# ============================================================================ + +SCRIPT=$1 +CLEAN_INSTALL=${2:-no} # Optional: "yes" to also clean install dir + +if [ -z "$SCRIPT" ]; then + echo "ERROR: No build script provided" + echo "Usage: $0 [clean_install]" + echo " clean_install: 'yes' to also remove CMAKE_INSTALL_PREFIX (default: no)" + exit 1 +fi + +if [ ! -f "$SCRIPT" ]; then + echo "ERROR: Build script not found: $SCRIPT" + exit 1 +fi + +echo "==========================================" +echo "AUTO MODE: Performing distclean" +echo "==========================================" +echo "Deleting CMake configuration and build artifacts..." +echo "" + +# Check for install directory before deleting CMakeCache.txt +INSTALL_DIR="" +if [ -f "CMakeCache.txt" ] && [ "$CLEAN_INSTALL" = "yes" ]; then + INSTALL_PREFIX=$(grep "^CMAKE_INSTALL_PREFIX:" CMakeCache.txt 2>/dev/null | cut -d'=' -f2 || true) + if [ -n "$INSTALL_PREFIX" ] && [ -d "$INSTALL_PREFIX" ]; then + INSTALL_DIR="$INSTALL_PREFIX" + echo "Install directory found: $INSTALL_DIR" + fi +fi + +# Delete CMake configuration files +rm -rf CMakeCache.txt CMakeFiles/ Makefile cmake_install.cmake \ + CTestTestfile.cmake Testing/ _deps/ compile_commands.json \ + 2>/dev/null || true + +# Delete CTest/CDash artifacts +rm -f DartConfiguration.tcl 2>/dev/null || true + +# Delete generated project config +rm -f ERFConfig.cmake 2>/dev/null || true + +# Delete any remaining .cmake files (excluding source CMakeLists.txt) +find . -maxdepth 1 -name "*.cmake" -type f -exec rm -f {} \; 2>/dev/null || true + +# Delete pkg-config files +rm -f *.pc 2>/dev/null || true + +# Delete built artifact directories +rm -rf Exec/ Submodules/ Tests/ bin/ cmake_packages/ externals/ 2>/dev/null || true + +# Delete built libraries +rm -f lib*.a lib*.so 2>/dev/null || true + +# Delete build logs +rm -f build_*.log git-state.txt 2>/dev/null || true + +echo " DONE: Cleaned: CMake configuration files" +echo " DONE: Cleaned: Build artifacts (Exec/, Submodules/, Tests/, bin/, etc.)" +echo " DONE: Cleaned: Libraries (lib*.a, lib*.so)" +echo " DONE: Cleaned: Build logs and editor backups" + +# Clean install directory if requested +if [ -n "$INSTALL_DIR" ]; then + echo "" + echo "Cleaning install directory (CLEAN_INSTALL=yes): $INSTALL_DIR" + rm -rf "$INSTALL_DIR" && echo " DONE: Deleted: $INSTALL_DIR" +elif [ -f "CMakeCache.txt.deleted" ]; then + echo "" + echo "Install directory NOT cleaned (use CLEAN_INSTALL=yes to clean)" +fi + +echo "" +echo " DONE: Directory ready for fresh configuration" +echo "" + +# Set ERF_DIR +if [ -z "$ERF_DIR" ]; then + if [ -d "../source" ]; then + export ERF_DIR=$(cd ../source && pwd) + echo "Auto-detected ERF_DIR: $ERF_DIR" + fi +fi + +echo "==========================================" +echo "Running build script: $SCRIPT" +echo "==========================================" +echo "" + +bash "$SCRIPT" diff --git a/CMake/CrayCompilerDetection.cmake b/CMake/CrayCompilerDetection.cmake new file mode 100644 index 0000000000..39b3a9a382 --- /dev/null +++ b/CMake/CrayCompilerDetection.cmake @@ -0,0 +1,354 @@ +# ============================================================================== +# Cray Compiler Detection (Pre-Project Stage) +# ============================================================================== +# This file runs BEFORE project() to detect Cray systems and set compilers +# The main CrayDetection.cmake runs AFTER project() to apply build fixes +# ============================================================================== + +# ----------------------------------------------------------------------------- +# Helper function: Suggest machine profile +# ----------------------------------------------------------------------------- +function(erf_suggest_machine_profile) + execute_process( + COMMAND hostname -s + OUTPUT_VARIABLE hostname + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET + ) + + file(GLOB profiles "${CMAKE_SOURCE_DIR}/Build/machines/*_erf.profile") + + message(STATUS " Load modules from your machine profile:") + message(STATUS "") + + if(profiles) + foreach(p ${profiles}) + get_filename_component(name ${p} NAME_WE) + # Check if hostname contains profile name + if(hostname MATCHES "${name}") + message(STATUS " source ${p} <-- matches hostname '${hostname}'") + else() + message(STATUS " source ${p}") + endif() + endforeach() + else() + message(STATUS " No profiles found in ${CMAKE_SOURCE_DIR}/Build/machines/") + endif() +endfunction() + +# Skip if user already set compilers explicitly +if(CMAKE_C_COMPILER OR CMAKE_CXX_COMPILER OR CMAKE_Fortran_COMPILER) + message(STATUS "ERF: Compilers already specified by user, skipping Cray auto-detection") + return() +endif() + +# ----------------------------------------------------------------------------- +# Detect Cray Environment (using only environment variables) +# ----------------------------------------------------------------------------- + +set(ERF_ON_CRAY_PREPROJECT FALSE) + +# Check for Cray Programming Environment +if(DEFINED ENV{CRAYPE_VERSION}) + set(ERF_ON_CRAY_PREPROJECT TRUE) + message(STATUS "ERF: Detected Cray Programming Environment (CRAYPE_VERSION=$ENV{CRAYPE_VERSION})") +endif() + +# Additional check for Cray MPI +if(DEFINED ENV{CRAY_MPICH_DIR}) + set(ERF_ON_CRAY_PREPROJECT TRUE) + message(STATUS "ERF: Detected Cray MPI (CRAY_MPICH_DIR=$ENV{CRAY_MPICH_DIR})") +endif() + +# Check for Cray compiler module +if(DEFINED ENV{PE_ENV}) + set(ERF_ON_CRAY_PREPROJECT TRUE) + message(STATUS "ERF: Detected Cray Programming Environment: $ENV{PE_ENV}") +endif() + +if(NOT ERF_ON_CRAY_PREPROJECT) + # Not on Cray, skip compiler setup + return() +endif() + +# ----------------------------------------------------------------------------- +# Set Cray Compiler Wrappers as Defaults +# ----------------------------------------------------------------------------- + +message(STATUS "ERF: Setting Cray compiler wrappers...") + +# Find Cray C compiler wrapper +find_program(ERF_CRAY_CC cc) +if(ERF_CRAY_CC) + set(CMAKE_C_COMPILER "${ERF_CRAY_CC}" CACHE FILEPATH "C compiler") + message(STATUS " Set CMAKE_C_COMPILER = ${ERF_CRAY_CC}") +else() + message(WARNING "ERF: On Cray system but 'cc' wrapper not found in PATH") +endif() + +# Find Cray C++ compiler wrapper +find_program(ERF_CRAY_CXX CC) +if(ERF_CRAY_CXX) + set(CMAKE_CXX_COMPILER "${ERF_CRAY_CXX}" CACHE FILEPATH "C++ compiler") + message(STATUS " Set CMAKE_CXX_COMPILER = ${ERF_CRAY_CXX}") +else() + message(WARNING "ERF: On Cray system but 'CC' wrapper not found in PATH") +endif() + +# Find Cray Fortran compiler wrapper (if needed) +if(ERF_ENABLE_MORR_FORT OR ERF_ENABLE_NOAHMP) + find_program(ERF_CRAY_FC ftn) + if(ERF_CRAY_FC) + set(CMAKE_Fortran_COMPILER "${ERF_CRAY_FC}" CACHE FILEPATH "Fortran compiler") + message(STATUS " Set CMAKE_Fortran_COMPILER = ${ERF_CRAY_FC}") + else() + message(WARNING "ERF: On Cray system but 'ftn' wrapper not found in PATH") + endif() +endif() + +message(STATUS "") + +# ----------------------------------------------------------------------------- +# GPU Host Compilers (for CUDA, HIP, SYCL) +# ----------------------------------------------------------------------------- + +# CUDA - Check if craype-accel module is loaded on Cray systems +if(DEFINED ENV{CUDA_HOME} OR DEFINED ENV{CUDATOOLKIT_HOME} OR DEFINED ENV{CRAY_ACCEL_TARGET}) + message(STATUS " Detected CUDA environment") + + # On Cray systems, need craype-accel-* module loaded + if(DEFINED ENV{CRAYPE_VERSION}) + if(NOT DEFINED ENV{CRAY_ACCEL_TARGET}) + message(STATUS "") + message(STATUS "====================================================================") + message(STATUS "CUDA on Cray: Missing craype-accel Module") + message(STATUS "====================================================================") + message(STATUS "") + message(STATUS "The Cray compiler wrappers need a craype-accel-* module loaded") + message(STATUS "to configure GPU support (sets CRAY_ACCEL_TARGET).") + message(STATUS "") + message(STATUS "To fix, load the appropriate module from your machine profile:") + message(STATUS "") + erf_suggest_machine_profile() + message(STATUS "") + message(STATUS " Examples of craype-accel modules:") + message(STATUS " craype-accel-nvidia80 (A100)") + message(STATUS " craype-accel-nvidia90 (H100)") + message(STATUS " craype-accel-amd-gfx90a (MI250X)") + message(STATUS "") + message(STATUS "====================================================================") + message(FATAL_ERROR "CUDA requires craype-accel module on Cray systems") + else() + message(STATUS " craype-accel module loaded: CRAY_ACCEL_TARGET=$ENV{CRAY_ACCEL_TARGET}") + message(STATUS " Cray wrappers will handle CUDA compilation") + endif() + endif() + + # Set CUDA compiler (default to Cray wrapper, can be overridden) + # Respect: CMAKE_CUDA_COMPILER (cache), CUDACXX (env) +# if(NOT CMAKE_CUDA_COMPILER AND NOT DEFINED ENV{CUDACXX}) +# if(ERF_CRAY_CXX) +# set(CMAKE_CUDA_COMPILER "${ERF_CRAY_CXX}" CACHE FILEPATH "CUDA compiler (Cray wrapper)") +# message(STATUS " Set CMAKE_CUDA_COMPILER = ${ERF_CRAY_CXX}") +# message(STATUS " -> Inherits MPI paths automatically (no Fix 1 needed)") +# endif() +# elseif(CMAKE_CUDA_COMPILER) +# message(STATUS " CMAKE_CUDA_COMPILER already set: ${CMAKE_CUDA_COMPILER}") +# elseif(DEFINED ENV{CUDACXX}) +# message(STATUS " CUDACXX environment variable set: $ENV{CUDACXX}") +# endif() + + # Set CUDA host compiler (used when nvcc or nvcc_wrapper is the CUDA compiler) + # Respect: CMAKE_CUDA_HOST_COMPILER (cache), CUDAHOSTCXX (env) + if(NOT CMAKE_CUDA_HOST_COMPILER AND NOT DEFINED ENV{CUDAHOSTCXX}) + if(ERF_CRAY_CXX) + set(CMAKE_CUDA_HOST_COMPILER "${ERF_CRAY_CXX}" CACHE FILEPATH "CUDA host compiler") + message(STATUS " Set CMAKE_CUDA_HOST_COMPILER = ${ERF_CRAY_CXX}") + endif() + elseif(CMAKE_CUDA_HOST_COMPILER) + message(STATUS " CMAKE_CUDA_HOST_COMPILER already set: ${CMAKE_CUDA_HOST_COMPILER}") + elseif(DEFINED ENV{CUDAHOSTCXX}) + message(STATUS " CUDAHOSTCXX environment variable set: $ENV{CUDAHOSTCXX}") + endif() +endif() + +# HIP - Check if craype-accel module is loaded on Cray systems +if(DEFINED ENV{ROCM_PATH} OR DEFINED ENV{HIP_PATH}) + message(STATUS " Detected ROCm/HIP environment") + + if(DEFINED ENV{CRAYPE_VERSION}) + if(NOT DEFINED ENV{CRAY_ACCEL_TARGET}) + message(STATUS "") + message(STATUS "====================================================================") + message(STATUS "HIP on Cray: Missing craype-accel Module") + message(STATUS "====================================================================") + message(STATUS "") + message(STATUS "The Cray compiler wrappers need a craype-accel-* module loaded") + message(STATUS "to configure GPU support (sets CRAY_ACCEL_TARGET).") + message(STATUS "") + message(STATUS "To fix, load the appropriate module from your machine profile:") + message(STATUS "") + erf_suggest_machine_profile() + message(STATUS "") + message(STATUS " Examples of craype-accel modules:") + message(STATUS " craype-accel-amd-gfx90a (MI250X)") + message(STATUS " craype-accel-amd-gfx942 (MI300)") + message(STATUS "") + message(STATUS "====================================================================") + message(FATAL_ERROR "HIP requires craype-accel module on Cray systems") + else() + message(STATUS " craype-accel module loaded: CRAY_ACCEL_TARGET=$ENV{CRAY_ACCEL_TARGET}") + endif() + endif() + + # Set HIP compiler (Cray wrapper handles HIP via hipcc) + # Respect: CMAKE_HIP_COMPILER (cache), HIPCXX (env) +# if(NOT CMAKE_HIP_COMPILER AND NOT DEFINED ENV{HIPCXX}) +# if(ERF_CRAY_CXX) +# set(CMAKE_HIP_COMPILER "${ERF_CRAY_CXX}" CACHE FILEPATH "HIP compiler (Cray wrapper)") +# message(STATUS " Set CMAKE_HIP_COMPILER = ${ERF_CRAY_CXX}") +# endif() +# elseif(CMAKE_HIP_COMPILER) +# message(STATUS " CMAKE_HIP_COMPILER already set: ${CMAKE_HIP_COMPILER}") +# elseif(DEFINED ENV{HIPCXX}) +# message(STATUS " HIPCXX environment variable set: $ENV{HIPCXX}") +# endif() + + # Set HIP host compiler + # Respect: CMAKE_HIP_HOST_COMPILER (cache), HIPHOSTCXX (env) + if(NOT CMAKE_HIP_HOST_COMPILER AND NOT DEFINED ENV{HIPHOSTCXX}) + if(ERF_CRAY_CXX) + set(CMAKE_HIP_HOST_COMPILER "${ERF_CRAY_CXX}" CACHE FILEPATH "HIP host compiler") + message(STATUS " Set CMAKE_HIP_HOST_COMPILER = ${ERF_CRAY_CXX}") + endif() + elseif(CMAKE_HIP_HOST_COMPILER) + message(STATUS " CMAKE_HIP_HOST_COMPILER already set: ${CMAKE_HIP_HOST_COMPILER}") + elseif(DEFINED ENV{HIPHOSTCXX}) + message(STATUS " HIPHOSTCXX environment variable set: $ENV{HIPHOSTCXX}") + endif() +endif() + +# SYCL - detect via Intel oneAPI +if(DEFINED ENV{ONEAPI_ROOT} OR DEFINED ENV{I_MPI_ROOT}) + message(STATUS " Detected Intel oneAPI environment") + message(STATUS " SYCL will use CMAKE_CXX_COMPILER = ${CMAKE_CXX_COMPILER}") +endif() + +# ----------------------------------------------------------------------------- +# Detect Cray MPI and GTL Library Names (with smart fallbacks) +# ----------------------------------------------------------------------------- + +set(CRAY_MPI_LIB "") +set(CRAY_GTL_LIB "") + +# Method 1: Parse from CC --cray-print-opts=libs (BEST) +if(ERF_CRAY_CXX) + execute_process( + COMMAND ${ERF_CRAY_CXX} --cray-print-opts=libs + OUTPUT_VARIABLE CRAY_LIBS_OUTPUT + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET + RESULT_VARIABLE CRAY_LIBS_RESULT + ) + + if(CRAY_LIBS_RESULT EQUAL 0 AND CRAY_LIBS_OUTPUT) + string(REGEX MATCH "-lmpi_gnu_[0-9]+" CRAY_MPI_LIB "${CRAY_LIBS_OUTPUT}") + string(REGEX MATCH "-lmpi_gtl_[a-z]+" CRAY_GTL_LIB "${CRAY_LIBS_OUTPUT}") + endif() +endif() + +# Method 2: Fallback from environment variables +if(NOT CRAY_MPI_LIB AND DEFINED ENV{CRAY_MPICH_DIR}) + string(REGEX MATCH "/gnu/([0-9]+)\\.([0-9]+)" MATCH_RESULT "$ENV{CRAY_MPICH_DIR}") + if(CMAKE_MATCH_1 AND CMAKE_MATCH_2) + set(CRAY_MPI_LIB "-lmpi_gnu_${CMAKE_MATCH_1}${CMAKE_MATCH_2}") + endif() +endif() + +if(NOT CRAY_GTL_LIB AND DEFINED ENV{CRAY_ACCEL_TARGET}) + set(GTL_VAR "PE_MPICH_GTL_LIBS_$ENV{CRAY_ACCEL_TARGET}") + if(DEFINED ENV{${GTL_VAR}}) + set(CRAY_GTL_LIB "$ENV{${GTL_VAR}}") + elseif("$ENV{CRAY_ACCEL_TARGET}" MATCHES "nvidia") + set(CRAY_GTL_LIB "-lmpi_gtl_cuda") + elseif("$ENV{CRAY_ACCEL_TARGET}" MATCHES "amd") + set(CRAY_GTL_LIB "-lmpi_gtl_hsa") + endif() +endif() + +# Method 3: Ultimate fallback +if(NOT CRAY_MPI_LIB) + set(CRAY_MPI_LIB "-lmpi") +endif() + +# Combine +if(CRAY_MPI_LIB AND CRAY_GTL_LIB) + set(GTL_LIBS "${CRAY_MPI_LIB} ${CRAY_GTL_LIB}") +elseif(CRAY_MPI_LIB) + set(GTL_LIBS "${CRAY_MPI_LIB}") +endif() + +# ----------------------------------------------------------------------------- +# Set Minimal Flags for Compiler Tests (using detected libraries) +# ----------------------------------------------------------------------------- +message(STATUS "ERF: Setting minimal flags for compiler tests...") + +if(DEFINED ENV{MPICH_GPU_SUPPORT_ENABLED} AND "$ENV{MPICH_GPU_SUPPORT_ENABLED}" STREQUAL "1") + message(STATUS " GPU-aware MPI detected") + message(STATUS " Detected libraries: ${GTL_LIBS}") + + # Only add CUDA runtime if CUDA is actually available + set(NEED_CUDA_RUNTIME FALSE) + if(DEFINED ENV{CRAY_ACCEL_TARGET}) + if("$ENV{CRAY_ACCEL_TARGET}" MATCHES "nvidia") + set(NEED_CUDA_RUNTIME TRUE) + endif() + endif() + + if(NEED_CUDA_RUNTIME) + # Check if CUDA toolkit is available + if(DEFINED ENV{CUDA_HOME} OR DEFINED ENV{CUDATOOLKIT_HOME}) + message(STATUS " Adding CUDA runtime libraries for GPU-aware MPI tests") + + # APPEND to linker flags + if(CMAKE_EXE_LINKER_FLAGS) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lcudart -lcuda" CACHE STRING "" FORCE) + else() + set(CMAKE_EXE_LINKER_FLAGS "-lcudart -lcuda" CACHE STRING "" FORCE) + endif() + else() + message(STATUS "") + message(STATUS "====================================================================") + message(STATUS "GPU-Aware MPI: CUDA Runtime Not Found") + message(STATUS "====================================================================") + message(STATUS "") + message(STATUS "GPU-aware MPI is enabled (MPICH_GPU_SUPPORT_ENABLED=1) but") + message(STATUS "CUDA toolkit is not loaded.") + message(STATUS "") + message(STATUS "To fix, load the appropriate modules from your machine profile:") + message(STATUS "") + erf_suggest_machine_profile() + message(STATUS "") + message(STATUS "====================================================================") + message(FATAL_ERROR "GPU-aware MPI requires CUDA toolkit") + endif() + endif() + + # APPEND to standard libraries (use DETECTED GTL_LIBS, not hardcoded!) + if(CMAKE_CXX_STANDARD_LIBRARIES) + set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} ${GTL_LIBS}" CACHE STRING "" FORCE) + else() + set(CMAKE_CXX_STANDARD_LIBRARIES "${GTL_LIBS}" CACHE STRING "" FORCE) + endif() + + if(CMAKE_CUDA_STANDARD_LIBRARIES) + set(CMAKE_CUDA_STANDARD_LIBRARIES "${CMAKE_CUDA_STANDARD_LIBRARIES} ${GTL_LIBS}" CACHE STRING "" FORCE) + else() + set(CMAKE_CUDA_STANDARD_LIBRARIES "${GTL_LIBS}" CACHE STRING "" FORCE) + endif() + + message(STATUS " CMAKE_EXE_LINKER_FLAGS: ${CMAKE_EXE_LINKER_FLAGS}") + message(STATUS " CMAKE_CXX_STANDARD_LIBRARIES: ${CMAKE_CXX_STANDARD_LIBRARIES}") +else() + message(STATUS " GPU-aware MPI not enabled") +endif() \ No newline at end of file diff --git a/CMake/CrayDetection.cmake b/CMake/CrayDetection.cmake new file mode 100644 index 0000000000..e374b6765a --- /dev/null +++ b/CMake/CrayDetection.cmake @@ -0,0 +1,1184 @@ +# ============================================================================== +# Cray System Auto-Detection and Workarounds +# ============================================================================== +# This module detects Cray systems and automatically applies workarounds for +# common build issues. Each fix corresponds to a documented checklist item. +# +# CMake 3.25+ version using standard message log levels: +# cmake .. # Quiet (STATUS messages only) +# cmake --log-level=VERBOSE .. # Show detection details +# cmake --log-level=DEBUG .. # Show all diagnostics +# cmake --log-context .. # Show message hierarchy +# +# Options: +# -DERF_DISABLE_CRAY_AUTO_FIXES=ON : Disable automatic Cray system fixes +# ============================================================================== + +option(ERF_DISABLE_CRAY_AUTO_FIXES "Disable automatic Cray system fixes" OFF) + +# Set Cray context for hierarchical logging +list(APPEND CMAKE_MESSAGE_CONTEXT "Cray") + +if(ERF_DISABLE_CRAY_AUTO_FIXES) + message(STATUS "Auto-fixes disabled by user") + list(POP_BACK CMAKE_MESSAGE_CONTEXT) + return() +endif() + +message(DEBUG "Starting Cray detection and workaround application") + +# ============================================================================== +# Detect Cray Environment +# ============================================================================== + +set(ERF_ON_CRAY FALSE) + +message(DEBUG "Checking for Cray environment") +message(TRACE " CMAKE_C_COMPILER: ${CMAKE_C_COMPILER}") +message(TRACE " CMAKE_CXX_COMPILER: ${CMAKE_CXX_COMPILER}") +message(TRACE " CRAY_MPICH_DIR: $ENV{CRAY_MPICH_DIR}") + +# Check for Cray compiler wrappers +if(CMAKE_C_COMPILER MATCHES ".*cc$" AND + CMAKE_CXX_COMPILER MATCHES ".*CC$" AND + DEFINED ENV{CRAY_MPICH_DIR}) + set(ERF_ON_CRAY TRUE) + message(STATUS "Detected Cray system via compiler wrappers") + message(VERBOSE " C compiler: ${CMAKE_C_COMPILER}") + message(VERBOSE " C++ compiler: ${CMAKE_CXX_COMPILER}") + message(VERBOSE " CRAY_MPICH_DIR: $ENV{CRAY_MPICH_DIR}") +endif() + +# Additional check for Cray environment variables +if(DEFINED ENV{CRAYPE_VERSION}) + set(ERF_ON_CRAY TRUE) + message(STATUS "Detected Cray Programming Environment") + message(VERBOSE " CRAYPE_VERSION: $ENV{CRAYPE_VERSION}") +endif() + +if(NOT ERF_ON_CRAY) + message(STATUS "Not on a Cray system, skipping Cray-specific fixes") + message(DEBUG "Detection criteria not met:") + message(DEBUG " Compiler wrappers cc/CC: NO") + message(DEBUG " CRAY_MPICH_DIR set: NO") + message(DEBUG " CRAYPE_VERSION set: NO") + list(POP_BACK CMAKE_MESSAGE_CONTEXT) + return() +endif() + +# ============================================================================== +# Optional: Check for Stale Configuration +# ============================================================================== + +option(ERF_CHECK_MODULES "Check for stale configuration from module changes" ON) + +if(ERF_CHECK_MODULES) + list(APPEND CMAKE_MESSAGE_CONTEXT "CrayConfigCheck") + + message(DEBUG "Starting configuration verification") + + # Detection log for issues + set(STALE_CONFIG_LOG "") + + # Determine if this is first configure (before we cache anything) + set(IS_FIRST_CONFIGURE FALSE) + if(NOT DEFINED CACHED_LOADED_MODULES) + set(IS_FIRST_CONFIGURE TRUE) + message(DEBUG "First configure detected") + endif() + + # Check 1: Module environment changed + if(DEFINED ENV{LOADEDMODULES}) + set(CURRENT_MODULES "$ENV{LOADEDMODULES}") + if(DEFINED CACHED_LOADED_MODULES) + if(NOT "${CURRENT_MODULES}" STREQUAL "${CACHED_LOADED_MODULES}") + message(VERBOSE "Module environment changed since last configure") + list(APPEND STALE_CONFIG_LOG "LOADEDMODULES changed") + list(APPEND STALE_CONFIG_LOG " Previous: ${CACHED_LOADED_MODULES}") + list(APPEND STALE_CONFIG_LOG " Current: ${CURRENT_MODULES}") + else() + message(DEBUG "Module environment unchanged") + endif() + endif() + set(CACHED_LOADED_MODULES "${CURRENT_MODULES}" CACHE INTERNAL "Modules at configure time") + endif() + + # Check 2: PE_ENV changed + if(DEFINED ENV{PE_ENV}) + set(CURRENT_PE_ENV "$ENV{PE_ENV}") + if(DEFINED CACHED_PE_ENV AND NOT "${CURRENT_PE_ENV}" STREQUAL "${CACHED_PE_ENV}") + message(VERBOSE "PE_ENV changed: ${CACHED_PE_ENV} -> ${CURRENT_PE_ENV}") + list(APPEND STALE_CONFIG_LOG "PE_ENV changed from ${CACHED_PE_ENV} to ${CURRENT_PE_ENV}") + endif() + set(CACHED_PE_ENV "${CURRENT_PE_ENV}" CACHE INTERNAL "") + endif() + + # Check 3: Compiler version changed + if(DEFINED CMAKE_CXX_COMPILER_VERSION) + if(DEFINED CACHED_CXX_COMPILER_VERSION AND NOT "${CMAKE_CXX_COMPILER_VERSION}" STREQUAL "${CACHED_CXX_COMPILER_VERSION}") + message(VERBOSE "Compiler version changed") + list(APPEND STALE_CONFIG_LOG "Compiler version changed from ${CACHED_CXX_COMPILER_VERSION} to ${CMAKE_CXX_COMPILER_VERSION}") + endif() + set(CACHED_CXX_COMPILER_VERSION "${CMAKE_CXX_COMPILER_VERSION}" CACHE INTERNAL "") + endif() + + # Check 4: CMAKE_*_STANDARD_LIBRARIES already contains MPI (from previous run) + if(NOT IS_FIRST_CONFIGURE) # Only check on reconfigure + if(DEFINED CMAKE_CXX_STANDARD_LIBRARIES AND CMAKE_CXX_STANDARD_LIBRARIES) + if(CMAKE_CXX_STANDARD_LIBRARIES MATCHES "mpi_") + message(VERBOSE "CMAKE_CXX_STANDARD_LIBRARIES already contains MPI libraries") + list(APPEND STALE_CONFIG_LOG "CMAKE_CXX_STANDARD_LIBRARIES pre-populated with MPI libs") + list(APPEND STALE_CONFIG_LOG " Found: ${CMAKE_CXX_STANDARD_LIBRARIES}") + endif() + endif() + + if(DEFINED CMAKE_CUDA_STANDARD_LIBRARIES AND CMAKE_CUDA_STANDARD_LIBRARIES) + if(CMAKE_CUDA_STANDARD_LIBRARIES MATCHES "mpi_") + message(VERBOSE "CMAKE_CUDA_STANDARD_LIBRARIES already contains MPI libraries") + list(APPEND STALE_CONFIG_LOG "CMAKE_CUDA_STANDARD_LIBRARIES pre-populated with MPI libs") + list(APPEND STALE_CONFIG_LOG " Found: ${CMAKE_CUDA_STANDARD_LIBRARIES}") + endif() + endif() + else() + message(DEBUG "First configure - skipping pre-populated library check") + endif() + + list(POP_BACK CMAKE_MESSAGE_CONTEXT) +endif() + +# ============================================================================== +# Compiler Version Checks +# ============================================================================== + +message(VERBOSE "Checking compiler versions") + +# GCC Version Check (for std::filesystem support) +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") + message(VERBOSE "Detected GNU compiler: ${CMAKE_CXX_COMPILER_VERSION}") + + if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "8.0") + message(FATAL_ERROR + "\n" + "════════════════════════════════════════════════════════════════\n" + "ERF requires GCC 8.0+ for C++17 support\n" + "Found: GCC ${CMAKE_CXX_COMPILER_VERSION}\n" + "════════════════════════════════════════════════════════════════\n" + "\n" + "On Cray systems:\n" + " 1. Load newer compiler: module load PrgEnv-gnu gcc\n" + " 2. Verify version: CC --version\n" + "") + endif() + + message(DEBUG "GCC ${CMAKE_CXX_COMPILER_VERSION} >= 8.0 (C++17 filesystem supported)") + +elseif(CMAKE_CXX_COMPILER_ID MATCHES "Cray") + message(VERBOSE "Detected Cray compiler: ${CMAKE_CXX_COMPILER_VERSION}") + if(DEFINED ENV{PE_ENV}) + message(DEBUG "Programming Environment: $ENV{PE_ENV}") + endif() +else() + message(VERBOSE "Detected ${CMAKE_CXX_COMPILER_ID} compiler: ${CMAKE_CXX_COMPILER_VERSION}") +endif() + +# GPU Compiler Checks +if(ERF_ENABLE_CUDA) + message(VERBOSE "Checking CUDA compiler configuration") + + if(CMAKE_CUDA_COMPILER) + message(DEBUG "CMAKE_CUDA_COMPILER: ${CMAKE_CUDA_COMPILER}") + else() + message(DEBUG "CMAKE_CUDA_COMPILER not set (will auto-detect)") + endif() + + if(CMAKE_CUDA_FLAGS) + message(DEBUG "CMAKE_CUDA_FLAGS: ${CMAKE_CUDA_FLAGS}") + endif() + + # Detect AMReX CUDA architecture + message(DEBUG "Detecting CUDA architecture") + + if(AMReX_CUDA_ARCH) + message(VERBOSE "AMReX_CUDA_ARCH: ${AMReX_CUDA_ARCH} (user specified)") + + elseif(DEFINED ENV{AMREX_CUDA_ARCH}) + set(AMReX_CUDA_ARCH "$ENV{AMREX_CUDA_ARCH}" CACHE STRING "CUDA arch from AMREX_CUDA_ARCH") + message(VERBOSE "AMReX_CUDA_ARCH: $ENV{AMREX_CUDA_ARCH} (from environment)") + + elseif(DEFINED ENV{CMAKE_CUDA_ARCH}) + set(ENV_CUDA_ARCH "$ENV{CMAKE_CUDA_ARCH}") + message(DEBUG "Found CMAKE_CUDA_ARCH: ${ENV_CUDA_ARCH}") + + # Convert to AMReX format + if(ENV_CUDA_ARCH MATCHES "^[0-9][0-9]$") + string(SUBSTRING "${ENV_CUDA_ARCH}" 0 1 MAJOR) + string(SUBSTRING "${ENV_CUDA_ARCH}" 1 1 MINOR) + set(DETECTED_CUDA_ARCH "${MAJOR}.${MINOR}") + message(TRACE "Converted ${ENV_CUDA_ARCH} -> ${DETECTED_CUDA_ARCH}") + else() + set(DETECTED_CUDA_ARCH "${ENV_CUDA_ARCH}") + endif() + + set(AMReX_CUDA_ARCH "${DETECTED_CUDA_ARCH}" CACHE STRING "CUDA arch from CMAKE_CUDA_ARCH") + message(VERBOSE "AMReX_CUDA_ARCH: ${DETECTED_CUDA_ARCH} (from CMAKE_CUDA_ARCH)") + + elseif(DEFINED ENV{CRAY_ACCEL_TARGET}) + set(CRAY_ACCEL_TARGET "$ENV{CRAY_ACCEL_TARGET}") + message(VERBOSE "CRAY_ACCEL_TARGET: ${CRAY_ACCEL_TARGET}") + + if(CRAY_ACCEL_TARGET STREQUAL "nvidia70") + set(AMReX_CUDA_ARCH "7.0" CACHE STRING "CUDA arch from CRAY_ACCEL_TARGET") + message(VERBOSE "AMReX_CUDA_ARCH: 7.0 (Tesla V100)") + elseif(CRAY_ACCEL_TARGET STREQUAL "nvidia80") + set(AMReX_CUDA_ARCH "8.0" CACHE STRING "CUDA arch from CRAY_ACCEL_TARGET") + message(VERBOSE "AMReX_CUDA_ARCH: 8.0 (A100)") + elseif(CRAY_ACCEL_TARGET STREQUAL "nvidia90") + set(AMReX_CUDA_ARCH "9.0" CACHE STRING "CUDA arch from CRAY_ACCEL_TARGET") + message(VERBOSE "AMReX_CUDA_ARCH: 9.0 (H100)") + else() + message(WARNING "Unknown CRAY_ACCEL_TARGET: ${CRAY_ACCEL_TARGET}") + endif() + else() + message(WARNING "AMReX_CUDA_ARCH not detected") + message(STATUS " For Perlmutter: module load gpu") + message(STATUS " Or: export CMAKE_CUDA_ARCH=80") + message(STATUS " Or: -DAMReX_CUDA_ARCH=8.0") + endif() +endif() + +# AMD GPU Architecture Detection (HIP) +if(AMReX_GPU_BACKEND MATCHES "HIP" OR ERF_ENABLE_HIP) + message(VERBOSE "Checking HIP/ROCm configuration") + + if(AMReX_AMD_ARCH) + message(VERBOSE "AMReX_AMD_ARCH: ${AMReX_AMD_ARCH} (user specified)") + elseif(DEFINED ENV{AMREX_AMD_ARCH}) + set(AMReX_AMD_ARCH "$ENV{AMREX_AMD_ARCH}" CACHE STRING "AMD arch from AMREX_AMD_ARCH") + message(VERBOSE "AMReX_AMD_ARCH: $ENV{AMREX_AMD_ARCH} (from environment)") + elseif(DEFINED ENV{CMAKE_AMD_ARCH}) + set(AMReX_AMD_ARCH "$ENV{CMAKE_AMD_ARCH}" CACHE STRING "AMD arch from CMAKE_AMD_ARCH") + message(VERBOSE "AMReX_AMD_ARCH: $ENV{CMAKE_AMD_ARCH} (from CMAKE_AMD_ARCH)") + elseif(DEFINED ENV{CRAY_ACCEL_TARGET}) + set(CRAY_ACCEL_TARGET "$ENV{CRAY_ACCEL_TARGET}") + message(VERBOSE "CRAY_ACCEL_TARGET: ${CRAY_ACCEL_TARGET}") + + if(CRAY_ACCEL_TARGET STREQUAL "amd_gfx90a") + set(AMReX_AMD_ARCH "gfx90a" CACHE STRING "AMD arch from CRAY_ACCEL_TARGET") + message(VERBOSE "AMReX_AMD_ARCH: gfx90a (MI200)") + elseif(CRAY_ACCEL_TARGET STREQUAL "amd_gfx908") + set(AMReX_AMD_ARCH "gfx908" CACHE STRING "AMD arch from CRAY_ACCEL_TARGET") + message(VERBOSE "AMReX_AMD_ARCH: gfx908 (MI100)") + elseif(CRAY_ACCEL_TARGET STREQUAL "amd_gfx942") + set(AMReX_AMD_ARCH "gfx942" CACHE STRING "AMD arch from CRAY_ACCEL_TARGET") + message(VERBOSE "AMReX_AMD_ARCH: gfx942 (MI300)") + else() + message(WARNING "Unknown CRAY_ACCEL_TARGET: ${CRAY_ACCEL_TARGET}") + endif() + else() + message(WARNING "AMReX_AMD_ARCH not detected") + message(STATUS " For Frontier: module load craype-accel-amd-gfx90a") + message(STATUS " Or: export CMAKE_AMD_ARCH=gfx90a") + endif() +endif() + +# Kokkos Architecture Detection (for EKAT physics) +if(ERF_ENABLE_RRTMGP OR ERF_ENABLE_SHOC OR ERF_ENABLE_P3) + message(VERBOSE "EKAT-based physics enabled, checking Kokkos architecture") + + set(KOKKOS_ARCH_SET FALSE) + + # Check if user already set via CMake + if(Kokkos_ARCH_VOLTA70 OR Kokkos_ARCH_AMPERE80 OR Kokkos_ARCH_HOPPER90 OR + Kokkos_ARCH_VEGA90A OR Kokkos_ARCH_VEGA908 OR Kokkos_ARCH_MI300A) + set(KOKKOS_ARCH_SET TRUE) + message(VERBOSE "Kokkos architecture already set by user") + + elseif(DEFINED ENV{KOKKOS_GPU_ARCH}) + set(KOKKOS_GPU_ARCH_ENV "$ENV{KOKKOS_GPU_ARCH}") + message(VERBOSE "KOKKOS_GPU_ARCH: ${KOKKOS_GPU_ARCH_ENV}") + + # Map to Kokkos arch variables + if(KOKKOS_GPU_ARCH_ENV STREQUAL "VOLTA70") + set(Kokkos_ARCH_VOLTA70 ON CACHE BOOL "Kokkos arch from KOKKOS_GPU_ARCH") + set(KOKKOS_ARCH_SET TRUE) + message(DEBUG "Mapped VOLTA70 -> Kokkos_ARCH_VOLTA70") + elseif(KOKKOS_GPU_ARCH_ENV STREQUAL "AMPERE80") + set(Kokkos_ARCH_AMPERE80 ON CACHE BOOL "Kokkos arch from KOKKOS_GPU_ARCH") + set(KOKKOS_ARCH_SET TRUE) + message(DEBUG "Mapped AMPERE80 -> Kokkos_ARCH_AMPERE80") + elseif(KOKKOS_GPU_ARCH_ENV STREQUAL "HOPPER90") + set(Kokkos_ARCH_HOPPER90 ON CACHE BOOL "Kokkos arch from KOKKOS_GPU_ARCH") + set(KOKKOS_ARCH_SET TRUE) + message(DEBUG "Mapped HOPPER90 -> Kokkos_ARCH_HOPPER90") + elseif(KOKKOS_GPU_ARCH_ENV STREQUAL "VEGA90A") + set(Kokkos_ARCH_VEGA90A ON CACHE BOOL "Kokkos arch from KOKKOS_GPU_ARCH") + set(KOKKOS_ARCH_SET TRUE) + message(DEBUG "Mapped VEGA90A -> Kokkos_ARCH_VEGA90A") + elseif(KOKKOS_GPU_ARCH_ENV STREQUAL "VEGA908") + set(Kokkos_ARCH_VEGA908 ON CACHE BOOL "Kokkos arch from KOKKOS_GPU_ARCH") + set(KOKKOS_ARCH_SET TRUE) + message(DEBUG "Mapped VEGA908 -> Kokkos_ARCH_VEGA908") + else() + message(WARNING "Unknown KOKKOS_GPU_ARCH: ${KOKKOS_GPU_ARCH_ENV}") + endif() + + elseif(DEFINED ENV{CRAY_ACCEL_TARGET}) + set(CRAY_ACCEL_TARGET "$ENV{CRAY_ACCEL_TARGET}") + message(DEBUG "Using CRAY_ACCEL_TARGET for Kokkos: ${CRAY_ACCEL_TARGET}") + + # Map NVIDIA targets + if(CRAY_ACCEL_TARGET STREQUAL "nvidia70") + set(Kokkos_ARCH_VOLTA70 ON CACHE BOOL "Kokkos arch from CRAY_ACCEL_TARGET") + set(KOKKOS_ARCH_SET TRUE) + message(VERBOSE "Set Kokkos_ARCH_VOLTA70 from CRAY_ACCEL_TARGET") + elseif(CRAY_ACCEL_TARGET STREQUAL "nvidia80") + set(Kokkos_ARCH_AMPERE80 ON CACHE BOOL "Kokkos arch from CRAY_ACCEL_TARGET") + set(KOKKOS_ARCH_SET TRUE) + message(VERBOSE "Set Kokkos_ARCH_AMPERE80 from CRAY_ACCEL_TARGET") + elseif(CRAY_ACCEL_TARGET STREQUAL "nvidia90") + set(Kokkos_ARCH_HOPPER90 ON CACHE BOOL "Kokkos arch from CRAY_ACCEL_TARGET") + set(KOKKOS_ARCH_SET TRUE) + message(VERBOSE "Set Kokkos_ARCH_HOPPER90 from CRAY_ACCEL_TARGET") + # Map AMD targets + elseif(CRAY_ACCEL_TARGET STREQUAL "amd_gfx90a") + set(Kokkos_ARCH_VEGA90A ON CACHE BOOL "Kokkos arch from CRAY_ACCEL_TARGET") + set(KOKKOS_ARCH_SET TRUE) + message(VERBOSE "Set Kokkos_ARCH_VEGA90A from CRAY_ACCEL_TARGET") + elseif(CRAY_ACCEL_TARGET STREQUAL "amd_gfx908") + set(Kokkos_ARCH_VEGA908 ON CACHE BOOL "Kokkos arch from CRAY_ACCEL_TARGET") + set(KOKKOS_ARCH_SET TRUE) + message(VERBOSE "Set Kokkos_ARCH_VEGA908 from CRAY_ACCEL_TARGET") + elseif(CRAY_ACCEL_TARGET STREQUAL "amd_gfx942") + set(Kokkos_ARCH_MI300A ON CACHE BOOL "Kokkos arch from CRAY_ACCEL_TARGET") + set(KOKKOS_ARCH_SET TRUE) + message(VERBOSE "Set Kokkos_ARCH_MI300A from CRAY_ACCEL_TARGET") + endif() + endif() + + if(NOT KOKKOS_ARCH_SET) + message(WARNING "Kokkos architecture not detected") + message(STATUS " For Perlmutter: module load gpu") + message(STATUS " For Frontier: module load craype-accel-amd-gfx90a") + message(STATUS " Or: export KOKKOS_GPU_ARCH=AMPERE80") + else() + message(DEBUG "Note: Kokkos will set CMAKE_CUDA_ARCHITECTURES when CUDA language is enabled") + endif() +endif() + +# ============================================================================== +# Prerequisite Checks +# ============================================================================== + +message(VERBOSE "Checking prerequisites") + +# CMake Version Check +set(ERF_RECOMMENDED_CMAKE_VERSION "3.24.0") +if(CMAKE_VERSION VERSION_LESS ${ERF_RECOMMENDED_CMAKE_VERSION}) + message(WARNING "CMake ${CMAKE_VERSION} < recommended ${ERF_RECOMMENDED_CMAKE_VERSION}") + message(STATUS " Fix: module load cmake") + message(DEBUG "Older CMake may have issues with Cray wrappers and CUDA") +else() + message(DEBUG "CMake ${CMAKE_VERSION} >= ${ERF_RECOMMENDED_CMAKE_VERSION}") +endif() + +# CUDA Toolkit Check +if(ERF_ENABLE_CUDA) + message(VERBOSE "Checking CUDA toolkit") + + set(CUDA_TOOLKIT_LOADED FALSE) + + if(DEFINED ENV{CUDA_HOME}) + set(CUDA_TOOLKIT_LOADED TRUE) + message(VERBOSE " CUDA_HOME: $ENV{CUDA_HOME}") + endif() + + if(NOT CUDA_TOOLKIT_LOADED AND DEFINED ENV{CUDATOOLKIT_HOME}) + set(CUDA_TOOLKIT_LOADED TRUE) + message(VERBOSE " CUDATOOLKIT_HOME: $ENV{CUDATOOLKIT_HOME}") + endif() + + find_program(NVCC_EXECUTABLE nvcc) + if(NVCC_EXECUTABLE) + set(CUDA_TOOLKIT_LOADED TRUE) + message(VERBOSE " nvcc: ${NVCC_EXECUTABLE}") + endif() + + if(NOT CUDA_TOOLKIT_LOADED) + message(WARNING "CUDA toolkit not detected") + message(STATUS " Fix: module load cuda") + endif() +else() + message(DEBUG "CUDA not enabled, skipping toolkit check") +endif() + +# NetCDF Module Check +if(ERF_ENABLE_NETCDF) + message(VERBOSE "Checking NetCDF") + + if(DEFINED ENV{NETCDF_DIR}) + message(VERBOSE " NETCDF_DIR: $ENV{NETCDF_DIR}") + else() + message(STATUS " NetCDF module not detected") + message(STATUS " Recommended: module load cray-netcdf-hdf5parallel") + endif() +else() + message(DEBUG "NetCDF not enabled") +endif() + +# HDF5 Module Check +if(AMReX_HDF5) + message(VERBOSE "Checking HDF5") + + if(DEFINED ENV{HDF5_DIR}) + message(VERBOSE " HDF5_DIR: $ENV{HDF5_DIR}") + elseif(DEFINED ENV{HDF5_ROOT}) + message(VERBOSE " HDF5_ROOT: $ENV{HDF5_ROOT}") + else() + message(STATUS " HDF5 module not detected") + message(STATUS " Recommended: module load cray-hdf5-parallel") + endif() +else() + message(DEBUG "HDF5 not enabled") +endif() + +# FFTW Module Check +if(ERF_ENABLE_FFT) + message(VERBOSE "Checking FFTW") + + if(DEFINED ENV{FFTW_DIR}) + message(VERBOSE " FFTW_DIR: $ENV{FFTW_DIR}") + elseif(DEFINED ENV{CRAY_FFTW_DIR}) + message(VERBOSE " CRAY_FFTW_DIR: $ENV{CRAY_FFTW_DIR}") + else() + message(STATUS " FFTW module not detected") + message(STATUS " Recommended: module load cray-fftw") + endif() +else() + message(DEBUG "FFTW not enabled") +endif() + +# E3SM Submodule Check +if(ERF_ENABLE_RRTMGP OR ERF_ENABLE_SHOC OR ERF_ENABLE_P3) + message(VERBOSE "Checking E3SM submodule") + + set(E3SM_EXPECTED_PATH "${CMAKE_SOURCE_DIR}/external/E3SM") + + if(EXISTS "${E3SM_EXPECTED_PATH}") + file(GLOB E3SM_CONTENTS "${E3SM_EXPECTED_PATH}/*") + if(E3SM_CONTENTS) + message(VERBOSE " E3SM submodule found") + else() + message(WARNING "E3SM directory exists but is empty") + message(STATUS " Fix: git submodule update --init --recursive external/E3SM") + endif() + else() + message(WARNING "E3SM submodule not found") + message(STATUS " Fix: git submodule update --init --recursive external/E3SM") + endif() +else() + message(DEBUG "EKAT physics not enabled, skipping E3SM check") +endif() + +# Environment summary (DEBUG level) +message(DEBUG "Key environment variables:") +message(DEBUG " CRAYPE_VERSION: $ENV{CRAYPE_VERSION}") +message(DEBUG " CRAY_MPICH_DIR: $ENV{CRAY_MPICH_DIR}") +message(DEBUG " CUDA_HOME: $ENV{CUDA_HOME}") +message(DEBUG " NETCDF_DIR: $ENV{NETCDF_DIR}") +message(DEBUG " HDF5_DIR: $ENV{HDF5_DIR}") +message(DEBUG " FFTW_DIR: $ENV{FFTW_DIR}") +message(DEBUG " MPICH_GPU_SUPPORT_ENABLED: $ENV{MPICH_GPU_SUPPORT_ENABLED}") + +# ============================================================================== +# Fix 1: CUDA + EKAT -> nvcc_wrapper complications +# ============================================================================== + +if(ERF_ENABLE_CUDA AND (ERF_ENABLE_RRTMGP OR ERF_ENABLE_SHOC OR ERF_ENABLE_P3)) + message(STATUS "Applying Fix 1: CUDA+EKAT nvcc_wrapper") + + message(DEBUG "Problem: nvcc_wrapper doesn't inherit Cray include paths") + message(DEBUG "Solution: Add flags from CC --cray-print-opts=cflags") + + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} --cray-print-opts=cflags + OUTPUT_VARIABLE CRAY_CUDA_FLAGS + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET + RESULT_VARIABLE CRAY_CUDA_FLAGS_RESULT + ) + + if(CRAY_CUDA_FLAGS_RESULT EQUAL 0 AND CRAY_CUDA_FLAGS) + message(VERBOSE "Adding Cray flags to CMAKE_CUDA_FLAGS") + message(DEBUG "Flags: ${CRAY_CUDA_FLAGS}") + + if(CMAKE_CUDA_FLAGS) + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${CRAY_CUDA_FLAGS}" CACHE STRING "" FORCE) + else() + set(CMAKE_CUDA_FLAGS "${CRAY_CUDA_FLAGS}" CACHE STRING "" FORCE) + endif() + else() + message(WARNING "Could not retrieve Cray CUDA flags") + message(STATUS " Try: -DCMAKE_CUDA_FLAGS=\"\$(CC --cray-print-opts=cflags)\"") + endif() +else() + message(DEBUG "Fix 1 not needed (CUDA+EKAT not both enabled)") +endif() + +# ============================================================================== +# Fix 2: FCOMPARE + Cray -> mpi_gnu_123 not found +# ============================================================================== + +if(ERF_ENABLE_FCOMPARE) + message(STATUS "Applying Fix 2: fcompare linker") + + message(DEBUG "Problem: --as-needed drops required MPI libs") + message(DEBUG "Solution: Clean Cray libs and add --no-as-needed") + + set(CRAY_LIBS_CLEAN "") + + foreach(COMPILER IN ITEMS ${CMAKE_CXX_COMPILER} ${CMAKE_C_COMPILER} ${CMAKE_Fortran_COMPILER}) + if(EXISTS ${COMPILER}) + execute_process( + COMMAND ${COMPILER} --cray-print-opts=libs + OUTPUT_VARIABLE COMPILER_LIBS + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET + RESULT_VARIABLE COMPILER_LIBS_RESULT + ) + + if(COMPILER_LIBS_RESULT EQUAL 0) + message(TRACE "Libs from ${COMPILER}: ${COMPILER_LIBS}") + + # Remove problematic flags + string(REGEX REPLACE "-Wl,--as-needed," "" COMPILER_LIBS "${COMPILER_LIBS}") + string(REGEX REPLACE ",--no-as-needed" "" COMPILER_LIBS "${COMPILER_LIBS}") + string(REGEX REPLACE ",-l" " -l" COMPILER_LIBS "${COMPILER_LIBS}") + + set(CRAY_LIBS_CLEAN "${CRAY_LIBS_CLEAN} ${COMPILER_LIBS}") + endif() + endif() + endforeach() + + if(CRAY_LIBS_CLEAN) + message(VERBOSE "Adding: -Wl,--no-as-needed + cleaned libs") + message(DEBUG "Cleaned libs: ${CRAY_LIBS_CLEAN}") + + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-as-needed ${CRAY_LIBS_CLEAN}" + CACHE STRING "" FORCE) + else() + message(WARNING "Could not retrieve Cray library paths") + message(STATUS " Fcompare may fail to link") + endif() +else() + message(DEBUG "Fix 2 not needed (fcompare disabled)") +endif() + +# ============================================================================== +# Fix 3: CUDA math libs not found +# ============================================================================== + +if(ERF_ENABLE_CUDA AND DEFINED ENV{CUDA_HOME}) + set(CUDA_MATH_PATH "$ENV{CUDA_HOME}/../../math_libs/lib64") + + message(DEBUG "Checking CUDA math libs: ${CUDA_MATH_PATH}") + + if(EXISTS ${CUDA_MATH_PATH}) + message(STATUS "Applying Fix 3: CUDA math libraries") + message(VERBOSE "Adding: ${CUDA_MATH_PATH}") + + list(APPEND CMAKE_PREFIX_PATH ${CUDA_MATH_PATH}) + set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} CACHE STRING "" FORCE) + else() + message(WARNING "CUDA math libs not found at ${CUDA_MATH_PATH}") + message(STATUS " Fix: module load cuda") + endif() +else() + message(DEBUG "Fix 3 not needed (CUDA disabled or CUDA_HOME not set)") +endif() + +# ============================================================================== +# Fix 4: GPU-aware MPI with Cray GTL +# ============================================================================== + +if(ERF_ENABLE_MPI AND "$ENV{MPICH_GPU_SUPPORT_ENABLED}" STREQUAL "1") + set(APPLY_FIX4 FALSE) + set(GPU_TYPE "") + set(GTL_LIB "") + set(MPI_BASE_LIB "") + + message(VERBOSE "Detecting MPI library for GPU-aware support") + + # Try pkg-config first + find_package(PkgConfig QUIET) + if(PkgConfig_FOUND) + execute_process( + COMMAND CC --cray-print-opts=pkg_config_path + OUTPUT_VARIABLE CRAY_PKG_CONFIG_PATH + ERROR_QUIET + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE CC_RESULT + ) + + if(CC_RESULT EQUAL 0 AND CRAY_PKG_CONFIG_PATH) + set(ENV{PKG_CONFIG_PATH} "${CRAY_PKG_CONFIG_PATH}:$ENV{PKG_CONFIG_PATH}") + message(TRACE "PKG_CONFIG_PATH: ${CRAY_PKG_CONFIG_PATH}") + endif() + + pkg_check_modules(CRAY_MPI QUIET mpich) + if(CRAY_MPI_FOUND) + message(DEBUG "Found mpich via pkg-config") + foreach(lib IN LISTS CRAY_MPI_LIBRARIES CRAY_MPI_LINK_LIBRARIES) + if(lib MATCHES "^mpi_" AND NOT lib MATCHES "mpi_gtl") + set(MPI_BASE_LIB "${lib}") + message(DEBUG "Detected MPI base: ${MPI_BASE_LIB}") + break() + endif() + endforeach() + endif() + endif() + + # Fallback: Search filesystem + if(NOT MPI_BASE_LIB) + message(DEBUG "Falling back to filesystem search") + set(MPI_LIB_SEARCH_PATHS "") + if(DEFINED ENV{MPICH_DIR}) + list(APPEND MPI_LIB_SEARCH_PATHS "$ENV{MPICH_DIR}/lib") + endif() + if(DEFINED ENV{CRAY_MPICH_DIR}) + list(APPEND MPI_LIB_SEARCH_PATHS "$ENV{CRAY_MPICH_DIR}/lib") + endif() + + foreach(path IN LISTS MPI_LIB_SEARCH_PATHS) + file(GLOB mpi_libs "${path}/libmpi_*.so" "${path}/libmpi_*.a") + foreach(lib IN LISTS mpi_libs) + get_filename_component(libname "${lib}" NAME_WE) + string(REGEX REPLACE "^lib" "" libname "${libname}") + if(libname MATCHES "^mpi_(gnu|cray|intel)" AND NOT MPI_BASE_LIB) + set(MPI_BASE_LIB "${libname}") + message(DEBUG "Found MPI lib: ${MPI_BASE_LIB}") + break() + endif() + endforeach() + if(MPI_BASE_LIB) + break() + endif() + endforeach() + endif() + + # Last resort: Heuristic + if(NOT MPI_BASE_LIB) + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") + set(MPI_BASE_LIB "mpi_gnu_123") + message(WARNING "Using heuristic MPI library: ${MPI_BASE_LIB}") + elseif(CMAKE_CXX_COMPILER_ID MATCHES "Cray") + set(MPI_BASE_LIB "mpi_cray") + message(WARNING "Using heuristic MPI library: ${MPI_BASE_LIB}") + else() + set(MPI_BASE_LIB "mpi") + endif() + endif() + + # Verify MPI library exists (if checking enabled) + if(ERF_CHECK_MODULES AND MPI_BASE_LIB) + message(DEBUG "Verifying MPI library: ${MPI_BASE_LIB}") + + find_library(MPI_BASE_VERIFY + NAMES ${MPI_BASE_LIB} + PATHS + $ENV{MPICH_DIR}/lib + $ENV{CRAY_MPICH_DIR}/lib + NO_DEFAULT_PATH + ) + + if(MPI_BASE_VERIFY) + message(DEBUG "Verified MPI library exists: ${MPI_BASE_VERIFY}") + else() + message(VERBOSE "MPI library ${MPI_BASE_LIB} not found") + list(APPEND STALE_CONFIG_LOG "MPI library lib${MPI_BASE_LIB}.so not found") + list(APPEND STALE_CONFIG_LOG " Searched in: \$MPICH_DIR/lib, \$CRAY_MPICH_DIR/lib") + + # Try to suggest correct version + if(CMAKE_CXX_COMPILER_VERSION MATCHES "^([0-9]+)\\.([0-9]+)") + set(EXPECTED_VER "${CMAKE_MATCH_1}${CMAKE_MATCH_2}") + list(APPEND STALE_CONFIG_LOG " Expected based on GCC ${CMAKE_CXX_COMPILER_VERSION}: mpi_gnu_${EXPECTED_VER}") + endif() + endif() + unset(MPI_BASE_VERIFY CACHE) + endif() + + # Determine GPU type and GTL library + if(ERF_ENABLE_CUDA) + set(APPLY_FIX4 TRUE) + set(GPU_TYPE "CUDA") + set(GTL_LIB "mpi_gtl_cuda") + elseif(AMReX_GPU_BACKEND MATCHES "HIP") + set(APPLY_FIX4 TRUE) + set(GPU_TYPE "HIP") + set(GTL_LIB "mpi_gtl_hsa") + endif() + +if(APPLY_FIX4) + message(STATUS "Applying Fix 4: GPU-aware MPI (${GPU_TYPE})") + message(VERBOSE "MPI base library: ${MPI_BASE_LIB}") + message(VERBOSE "GTL library: ${GTL_LIB}") + + set(CRAY_MPI_LIBS "-l${MPI_BASE_LIB} -l${GTL_LIB}") + + # Only append if not already present + if(ERF_ENABLE_CUDA) + string(FIND "${CMAKE_CUDA_STANDARD_LIBRARIES}" "${CRAY_MPI_LIBS}" already_present) + if(already_present EQUAL -1) + message(DEBUG "Adding to CMAKE_CUDA_STANDARD_LIBRARIES: ${CRAY_MPI_LIBS}") + set(CMAKE_CUDA_STANDARD_LIBRARIES "${CMAKE_CUDA_STANDARD_LIBRARIES} ${CRAY_MPI_LIBS}" + CACHE STRING "" FORCE) + else() + message(DEBUG "CUDA libraries already contain MPI libs, skipping") + endif() + else() + string(FIND "${CMAKE_HIP_STANDARD_LIBRARIES}" "${CRAY_MPI_LIBS}" already_present) + if(already_present EQUAL -1) + message(DEBUG "Adding to CMAKE_HIP_STANDARD_LIBRARIES: ${CRAY_MPI_LIBS}") + set(CMAKE_HIP_STANDARD_LIBRARIES "${CMAKE_HIP_STANDARD_LIBRARIES} ${CRAY_MPI_LIBS}" + CACHE STRING "" FORCE) + else() + message(DEBUG "HIP libraries already contain MPI libs, skipping") + endif() + endif() + + string(FIND "${CMAKE_CXX_STANDARD_LIBRARIES}" "${CRAY_MPI_LIBS}" already_present) + if(already_present EQUAL -1) + message(DEBUG "Adding to CMAKE_CXX_STANDARD_LIBRARIES: ${CRAY_MPI_LIBS}") + set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} ${CRAY_MPI_LIBS}" + CACHE STRING "" FORCE) + else() + message(DEBUG "CXX libraries already contain MPI libs, skipping") + endif() + endif() +else() + message(DEBUG "Fix 4 not needed (GPU+MPI not enabled or GPU support not enabled)") +endif() + +# ============================================================================== +# Fix 5-6: NetCDF with cray-netcdf-hdf5parallel +# ============================================================================== + +if(ERF_ENABLE_NETCDF) + message(STATUS "Applying Fix 5-6: NetCDF configuration") + + message(DEBUG "Setting up pkg-config path for NetCDF/MPI") + + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} --cray-print-opts=PKG_CONFIG_PATH + OUTPUT_VARIABLE CRAY_PKG_CONFIG_PATH + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET + RESULT_VARIABLE PKG_RESULT + ) + + if(PKG_RESULT EQUAL 0 AND CRAY_PKG_CONFIG_PATH) + message(VERBOSE "PKG_CONFIG_PATH from Cray wrapper") + message(DEBUG " ${CRAY_PKG_CONFIG_PATH}") + + if(DEFINED ENV{PKG_CONFIG_PATH}) + set(ENV{PKG_CONFIG_PATH} "${CRAY_PKG_CONFIG_PATH}:$ENV{PKG_CONFIG_PATH}") + else() + set(ENV{PKG_CONFIG_PATH} "${CRAY_PKG_CONFIG_PATH}") + endif() + endif() + + # Add NetCDF/HDF5 to search paths + if(DEFINED ENV{NETCDF_DIR}) + list(APPEND CMAKE_PREFIX_PATH $ENV{NETCDF_DIR}) + message(VERBOSE "Added NETCDF_DIR to search: $ENV{NETCDF_DIR}") + endif() + + if(DEFINED ENV{HDF5_DIR}) + list(APPEND CMAKE_PREFIX_PATH $ENV{HDF5_DIR}) + message(VERBOSE "Added HDF5_DIR to search: $ENV{HDF5_DIR}") + endif() +else() + message(DEBUG "Fix 5-6 not needed (NetCDF disabled)") +endif() + +# ============================================================================== +# Fix 7: HDF5 parallel detection for HIP builds +# ============================================================================== + +if(AMReX_GPU_BACKEND MATCHES "HIP" AND AMReX_HDF5) + message(STATUS "Applying Fix 7: HDF5 for HIP") + + message(DEBUG "Configuring HDF5 hints for HIP build") + + find_package(PkgConfig QUIET) + if(PkgConfig_FOUND) + execute_process( + COMMAND CC --cray-print-opts=pkg_config_path + OUTPUT_VARIABLE CRAY_PKG_CONFIG_PATH + ERROR_QUIET + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE CC_RESULT + ) + + if(CC_RESULT EQUAL 0 AND CRAY_PKG_CONFIG_PATH) + set(ENV{PKG_CONFIG_PATH} "${CRAY_PKG_CONFIG_PATH}:$ENV{PKG_CONFIG_PATH}") + endif() + + pkg_check_modules(PC_HDF5 QUIET hdf5) + if(PC_HDF5_FOUND) + message(VERBOSE "Found HDF5 via pkg-config: ${PC_HDF5_PREFIX}") + + set(HDF5_ROOT "${PC_HDF5_PREFIX}" CACHE PATH "HDF5 root from pkg-config") + set(HDF5_PREFER_PARALLEL ON CACHE BOOL "Prefer parallel HDF5") + set(HDF5_IS_PARALLEL TRUE CACHE BOOL "HDF5 is parallel") + + list(APPEND CMAKE_PREFIX_PATH "${PC_HDF5_PREFIX}") + + message(DEBUG "Set HDF5_ROOT: ${PC_HDF5_PREFIX}") + message(DEBUG "Set HDF5_PREFER_PARALLEL: ON") + else() + message(WARNING "pkg-config could not find HDF5") + endif() + else() + message(WARNING "PkgConfig not found, cannot auto-configure HDF5") + endif() +else() + message(DEBUG "Fix 7 not needed (not HIP+HDF5)") +endif() + +# ============================================================================== +# Summary +# ============================================================================== + +message(STATUS "Cray configuration complete") + +# Track which fixes were applied +set(FIX1_ACTIVE OFF) +set(FIX2_ACTIVE OFF) +set(FIX3_ACTIVE OFF) +set(FIX4_ACTIVE OFF) +set(FIX56_ACTIVE OFF) +set(FIX7_ACTIVE OFF) + +# Fix 1: CUDA + EKAT +if(ERF_ENABLE_CUDA AND (ERF_ENABLE_RRTMGP OR ERF_ENABLE_SHOC OR ERF_ENABLE_P3) AND CRAY_CUDA_FLAGS) + set(FIX1_ACTIVE ON) +endif() + +# Fix 2: fcompare +if(ERF_ENABLE_FCOMPARE AND CRAY_LIBS_CLEAN) + set(FIX2_ACTIVE ON) +endif() + +# Fix 3: CUDA math libs +if(ERF_ENABLE_CUDA AND DEFINED ENV{CUDA_HOME}) + set(CUDA_MATH_CHECK "$ENV{CUDA_HOME}/../../math_libs/lib64") + if(EXISTS ${CUDA_MATH_CHECK}) + set(FIX3_ACTIVE ON) + endif() +endif() + +# Fix 4: GPU-aware MPI +if(APPLY_FIX4) + set(FIX4_ACTIVE ON) +endif() + +# Fix 5-6: NetCDF +if(ERF_ENABLE_NETCDF) + set(FIX56_ACTIVE ON) +endif() + +# Fix 7: HDF5 for HIP +if(AMReX_GPU_BACKEND MATCHES "HIP" AND AMReX_HDF5) + set(FIX7_ACTIVE ON) +endif() + +# Show summary at VERBOSE level +message(VERBOSE "Applied fixes:") +if(FIX1_ACTIVE) + message(VERBOSE " Fix 1 (CUDA+EKAT): ACTIVE") +endif() +if(FIX2_ACTIVE) + message(VERBOSE " Fix 2 (fcompare): ACTIVE") +endif() +if(FIX3_ACTIVE) + message(VERBOSE " Fix 3 (CUDA math): ACTIVE") +endif() +if(FIX4_ACTIVE) + message(VERBOSE " Fix 4 (GPU-aware MPI): ACTIVE") +endif() +if(FIX56_ACTIVE) + message(VERBOSE " Fix 5-6 (NetCDF): ACTIVE") +endif() +if(FIX7_ACTIVE) + message(VERBOSE " Fix 7 (HDF5+HIP): ACTIVE") +endif() + +# Command-line equivalents (DEBUG level) +message(DEBUG "Command-line equivalents for active fixes:") +message(DEBUG "=====================================================================") + +if(FIX1_ACTIVE) + message(DEBUG "Fix 1 (CUDA+EKAT):") + message(DEBUG " -DCMAKE_CUDA_FLAGS=\"\$(CC --cray-print-opts=cflags)\"") + message(DEBUG "") +endif() + +if(FIX2_ACTIVE) + message(DEBUG "Fix 2 (fcompare):") + message(DEBUG " CRAY_LIBS=\"\$(CC --cray-print-opts=libs | sed 's/-Wl,--as-needed,//g; s/,--no-as-needed//g; s/,-l/ -l/g')\"") + message(DEBUG " CRAY_LIBS=\"\$CRAY_LIBS \$(cc --cray-print-opts=libs | sed ...)\"") + message(DEBUG " CRAY_LIBS=\"\$CRAY_LIBS \$(ftn --cray-print-opts=libs | sed ...)\"") + message(DEBUG " -DCMAKE_EXE_LINKER_FLAGS=\"-Wl,--no-as-needed \$CRAY_LIBS\"") + message(DEBUG "") +endif() + +if(FIX3_ACTIVE) + message(DEBUG "Fix 3 (CUDA math):") + message(DEBUG " -DCMAKE_PREFIX_PATH=\"\$CUDA_HOME/../../math_libs/lib64\"") + message(DEBUG "") +endif() + +if(FIX4_ACTIVE) + message(DEBUG "Fix 4 (GPU-aware MPI):") + message(DEBUG " export MPICH_GPU_SUPPORT_ENABLED=1") + if(ERF_ENABLE_CUDA) + message(DEBUG " -DCMAKE_CUDA_STANDARD_LIBRARIES=\"-l${MPI_BASE_LIB} -l${GTL_LIB}\"") + else() + message(DEBUG " -DCMAKE_HIP_STANDARD_LIBRARIES=\"-l${MPI_BASE_LIB} -l${GTL_LIB}\"") + endif() + message(DEBUG " -DCMAKE_CXX_STANDARD_LIBRARIES=\"-l${MPI_BASE_LIB} -l${GTL_LIB}\"") + message(DEBUG "") +endif() + +if(FIX56_ACTIVE) + message(DEBUG "Fix 5-6 (NetCDF):") + message(DEBUG " export PKG_CONFIG_PATH=\"\$(CC --cray-print-opts=PKG_CONFIG_PATH):\$PKG_CONFIG_PATH\"") + if(DEFINED ENV{NETCDF_DIR}) + message(DEBUG " -DCMAKE_PREFIX_PATH=\"\$NETCDF_DIR\"") + endif() + if(DEFINED ENV{HDF5_DIR}) + message(DEBUG " -DCMAKE_PREFIX_PATH=\"\$CMAKE_PREFIX_PATH:\$HDF5_DIR\"") + endif() + message(DEBUG "") +endif() + +if(FIX7_ACTIVE) + message(DEBUG "Fix 7 (HDF5+HIP):") + message(DEBUG " -DHDF5_ROOT=\$(pkg-config --variable=prefix hdf5)") + message(DEBUG " -DHDF5_PREFER_PARALLEL=ON") + message(DEBUG " -DHDF5_IS_PARALLEL=TRUE") + message(DEBUG "") +endif() + +if(FIX1_ACTIVE OR FIX2_ACTIVE OR FIX3_ACTIVE OR FIX4_ACTIVE OR FIX56_ACTIVE OR FIX7_ACTIVE) + message(DEBUG "Complete manual equivalent (all active fixes):") + message(DEBUG "=====================================================================") + if(FIX1_ACTIVE) + message(DEBUG " -DCMAKE_CUDA_FLAGS=\"\$(CC --cray-print-opts=cflags)\" \\") + endif() + if(FIX2_ACTIVE) + message(DEBUG " -DCMAKE_EXE_LINKER_FLAGS=\"-Wl,--no-as-needed ${CRAY_LIBS_CLEAN}\" \\") + endif() + if(FIX3_ACTIVE) + message(DEBUG " -DCMAKE_PREFIX_PATH=\"\$CUDA_HOME/../../math_libs/lib64\" \\") + endif() + if(FIX4_ACTIVE) + if(ERF_ENABLE_CUDA) + message(DEBUG " -DCMAKE_CUDA_STANDARD_LIBRARIES=\"-l${MPI_BASE_LIB} -l${GTL_LIB}\" \\") + else() + message(DEBUG " -DCMAKE_HIP_STANDARD_LIBRARIES=\"-l${MPI_BASE_LIB} -l${GTL_LIB}\" \\") + endif() + message(DEBUG " -DCMAKE_CXX_STANDARD_LIBRARIES=\"-l${MPI_BASE_LIB} -l${GTL_LIB}\" \\") + endif() + message(DEBUG "") +endif() + +# ============================================================================== +# Generate Concise Config File +# ============================================================================== + +set(CRAY_CONFIG_FILE "${CMAKE_BINARY_DIR}/cray_detected_config.cmake") + +file(WRITE ${CRAY_CONFIG_FILE} +"# ============================================================================== +# Auto-detected Cray Configuration +# Generated: ${CMAKE_CURRENT_LIST_FILE} +# Date: ${CMAKE_TIMESTAMP} +# ============================================================================== +# This file shows the settings auto-detected by CrayDetection.cmake +# You can use this as a starting point for a manual config file. +# +# To use this config manually: +# +# From build directory: +# cmake -C ${CRAY_CONFIG_FILE} ${CMAKE_SOURCE_DIR} +# +# From source directory: +# cmake -C ${CRAY_CONFIG_FILE} -B ${CMAKE_BINARY_DIR} +# +# ============================================================================== + +") + +# System info +file(APPEND ${CRAY_CONFIG_FILE} " +# System Detection +set(ERF_ON_CRAY TRUE CACHE BOOL \"Detected Cray system\") +set(CRAYPE_VERSION \"$ENV{CRAYPE_VERSION}\" CACHE STRING \"Cray PE version\") +") + +# Compiler info +file(APPEND ${CRAY_CONFIG_FILE} " +# Compiler Configuration +set(CMAKE_C_COMPILER \"${CMAKE_C_COMPILER}\" CACHE FILEPATH \"\") +set(CMAKE_CXX_COMPILER \"${CMAKE_CXX_COMPILER}\" CACHE FILEPATH \"\") +set(CMAKE_CXX_COMPILER_ID \"${CMAKE_CXX_COMPILER_ID}\" CACHE STRING \"\") +set(CMAKE_CXX_COMPILER_VERSION \"${CMAKE_CXX_COMPILER_VERSION}\" CACHE STRING \"\") +") + +# GPU architectures +if(ERF_ENABLE_CUDA AND AMReX_CUDA_ARCH) + file(APPEND ${CRAY_CONFIG_FILE} " +# CUDA Configuration +set(AMReX_CUDA_ARCH \"${AMReX_CUDA_ARCH}\" CACHE STRING \"Auto-detected\") +") +endif() + +if(AMReX_AMD_ARCH) + file(APPEND ${CRAY_CONFIG_FILE} " +# HIP Configuration +set(AMReX_AMD_ARCH \"${AMReX_AMD_ARCH}\" CACHE STRING \"Auto-detected\") +") +endif() + +if(KOKKOS_ARCH_SET) + file(APPEND ${CRAY_CONFIG_FILE} " +# Kokkos Architecture +") + foreach(arch IN ITEMS VOLTA70 AMPERE80 HOPPER90 VEGA90A VEGA908 MI300A) + if(Kokkos_ARCH_${arch}) + file(APPEND ${CRAY_CONFIG_FILE} "set(Kokkos_ARCH_${arch} ON CACHE BOOL \"Auto-detected\")\n") + endif() + endforeach() +endif() + +# Applied fixes +file(APPEND ${CRAY_CONFIG_FILE} " +# Applied Fixes +") + +if(FIX1_ACTIVE) + file(APPEND ${CRAY_CONFIG_FILE} " +# Fix 1: CUDA+EKAT nvcc_wrapper flags +set(CMAKE_CUDA_FLAGS \"${CMAKE_CUDA_FLAGS}\" CACHE STRING \"\") +") +endif() + +if(FIX2_ACTIVE) + file(APPEND ${CRAY_CONFIG_FILE} " +# Fix 2: fcompare linker flags +set(CMAKE_EXE_LINKER_FLAGS \"${CMAKE_EXE_LINKER_FLAGS}\" CACHE STRING \"\") +") +endif() + +if(FIX3_ACTIVE) + file(APPEND ${CRAY_CONFIG_FILE} " +# Fix 3: CUDA math libraries path +list(APPEND CMAKE_PREFIX_PATH \"${CUDA_MATH_PATH}\") +") +endif() + +if(FIX4_ACTIVE) + file(APPEND ${CRAY_CONFIG_FILE} " +# Fix 4: GPU-aware MPI (${GPU_TYPE}) +set(CMAKE_CXX_STANDARD_LIBRARIES \"${CMAKE_CXX_STANDARD_LIBRARIES}\" CACHE STRING \"\") +") + if(ERF_ENABLE_CUDA) + file(APPEND ${CRAY_CONFIG_FILE} "set(CMAKE_CUDA_STANDARD_LIBRARIES \"${CMAKE_CUDA_STANDARD_LIBRARIES}\" CACHE STRING \"\")\n") + endif() +endif() + +if(FIX56_ACTIVE) + file(APPEND ${CRAY_CONFIG_FILE} " +# Fix 5-6: NetCDF/HDF5 paths +set(ENV{PKG_CONFIG_PATH} \"$ENV{PKG_CONFIG_PATH}\") +") + if(DEFINED ENV{NETCDF_DIR}) + file(APPEND ${CRAY_CONFIG_FILE} "list(APPEND CMAKE_PREFIX_PATH \"$ENV{NETCDF_DIR}\")\n") + endif() +endif() + +if(FIX7_ACTIVE) + file(APPEND ${CRAY_CONFIG_FILE} " +# Fix 7: HDF5 parallel for HIP +set(HDF5_ROOT \"${HDF5_ROOT}\" CACHE PATH \"\") +set(HDF5_PREFER_PARALLEL ON CACHE BOOL \"\") +set(HDF5_IS_PARALLEL TRUE CACHE BOOL \"\") +") +endif() + +message(STATUS "Generated config: ${CRAY_CONFIG_FILE}") + +# Add a target to display it +add_custom_target(show-cray-config + COMMAND ${CMAKE_COMMAND} -E echo "Displaying auto-detected Cray configuration" + COMMAND ${CMAKE_COMMAND} -E echo "===================================================================" + COMMAND ${CMAKE_COMMAND} -E echo "Auto-detected Cray Configuration:" + COMMAND ${CMAKE_COMMAND} -E echo "===================================================================" + COMMAND ${CMAKE_COMMAND} -E cat ${CRAY_CONFIG_FILE} + COMMAND ${CMAKE_COMMAND} -E echo "" + COMMAND ${CMAKE_COMMAND} -E echo "To use this config manually:" + COMMAND ${CMAKE_COMMAND} -E echo " cmake -C ${CRAY_CONFIG_FILE} ${CMAKE_SOURCE_DIR}" + COMMAND ${CMAKE_COMMAND} -E echo "===================================================================" + DEPENDS ${CRAY_CONFIG_FILE} + COMMENT "Showing Cray configuration from ${CRAY_CONFIG_FILE}" +) + +# ============================================================================== +# Display Configuration Verification Results +# ============================================================================== + +if(ERF_CHECK_MODULES AND STALE_CONFIG_LOG) + message(STATUS "") + message(STATUS "====================================================================") + message(STATUS "STALE CONFIGURATION DETECTED") + message(STATUS "====================================================================") + message(STATUS "") + message(STATUS "Configuration issues found:") + foreach(issue ${STALE_CONFIG_LOG}) + message(STATUS " ${issue}") + endforeach() + message(STATUS "") + message(STATUS "This usually happens when:") + message(STATUS " - You changed which modules are loaded") + message(STATUS " - You switched compiler versions") + message(STATUS " - CMake cache contains old settings") + message(STATUS "") + message(STATUS "To resolve, clean your build:") + message(STATUS "") + message(STATUS " Recommended:") + message(STATUS " cmake --build . --target distclean") + message(STATUS "") + message(STATUS " Or manually:") + message(STATUS " rm -rf CMakeCache.txt CMakeFiles/ cray_detected_config.cmake") + message(STATUS "") + message(STATUS " Then reconfigure:") + message(STATUS " cmake ..") + message(STATUS "") + message(STATUS "To disable this check:") + message(STATUS " cmake -DERF_CHECK_MODULES=OFF ..") + message(STATUS "") + message(STATUS "====================================================================") + message(STATUS "") + + # Make it a hard error instead of warning + message(FATAL_ERROR "Stale configuration detected - clean build required") +endif() + +message(DEBUG "=====================================================================") +message(DEBUG "To disable auto-fixes: -DERF_DISABLE_CRAY_AUTO_FIXES=ON") +message(DEBUG "For verbose output: cmake --log-level=VERBOSE ..") +message(DEBUG "For debug output: cmake --log-level=DEBUG ..") + +# Pop Cray context +list(POP_BACK CMAKE_MESSAGE_CONTEXT) diff --git a/CMake/FindNetCDF.cmake b/CMake/FindNetCDF.cmake index a2091a906f..be76f7ef5d 100644 --- a/CMake/FindNetCDF.cmake +++ b/CMake/FindNetCDF.cmake @@ -15,18 +15,45 @@ # find_package (NetCDF REQUIRED) # target_link_libraries (target_name PUBLIC ${NETCDF_LINK_LIBRARIES}) -if (NETCDF_INCLUDES AND NETCDF_LIBRARIES) - # Already in cache, be silent - set (NETCDF_FIND_QUIETLY TRUE) -endif (NETCDF_INCLUDES AND NETCDF_LIBRARIES) +# Set FindNetCDF context +list(APPEND CMAKE_MESSAGE_CONTEXT "FindNetCDF") -# Build hints from user variables first, then pkg-config +message(DEBUG "Starting NetCDF detection") + +# Detection log for failures +set(NETCDF_DETECTION_LOG "") + +# Check cache +if(NETCDF_INCLUDES AND NETCDF_LIBRARIES) + set(NETCDF_FIND_QUIETLY TRUE) + message(VERBOSE "NetCDF already in cache") + message(DEBUG " NETCDF_INCLUDES: ${NETCDF_INCLUDES}") + message(DEBUG " NETCDF_LIBRARIES: ${NETCDF_LIBRARIES}") +endif() + +# Build hints set(NETCDF_INCLUDE_HINTS) set(NETCDF_LIBRARY_HINTS) +message(DEBUG "Building search hints") + if(NETCDF_DIR) list(APPEND NETCDF_INCLUDE_HINTS ${NETCDF_DIR}/include) list(APPEND NETCDF_LIBRARY_HINTS ${NETCDF_DIR}/lib) + message(VERBOSE "Using NETCDF_DIR: ${NETCDF_DIR}") + list(APPEND NETCDF_DETECTION_LOG "NETCDF_DIR=${NETCDF_DIR}") +else() + message(DEBUG "NETCDF_DIR not set") + list(APPEND NETCDF_DETECTION_LOG "NETCDF_DIR not set") +endif() + +if(DEFINED ENV{NETCDF_DIR}) + list(APPEND NETCDF_INCLUDE_HINTS $ENV{NETCDF_DIR}/include) + list(APPEND NETCDF_LIBRARY_HINTS $ENV{NETCDF_DIR}/lib) + message(VERBOSE "Using ENV NETCDF_DIR: $ENV{NETCDF_DIR}") + list(APPEND NETCDF_DETECTION_LOG "ENV NETCDF_DIR=$ENV{NETCDF_DIR}") +else() + list(APPEND NETCDF_DETECTION_LOG "ENV NETCDF_DIR not set") endif() if(NETCDF_INCLUDE_DIR) @@ -37,29 +64,112 @@ if(NETCDF_LIBRARY_DIR) list(APPEND NETCDF_LIBRARY_HINTS ${NETCDF_LIBRARY_DIR}) endif() +# Pkg-config +message(VERBOSE "Attempting pkg-config detection") set(ENV{PKG_CONFIG_PATH} "$ENV{MPICH_DIR}/lib/pkgconfig:$ENV{PKG_CONFIG_PATH}") -message(STATUS "PKG_CONFIG_PATH = $ENV{PKG_CONFIG_PATH}") +message(DEBUG "PKG_CONFIG_PATH: $ENV{PKG_CONFIG_PATH}") -find_package(PkgConfig REQUIRED QUIET) -pkg_check_modules(NETCDF QUIET IMPORTED_TARGET netcdf) -if(NOT NETCDF_FOUND) - pkg_check_modules(NETCDF REQUIRED IMPORTED_TARGET netcdf-cxx4_parallel) +find_package(PkgConfig QUIET) +if(PKG_CONFIG_FOUND) + message(DEBUG "pkg-config available") + + set(PKG_VARIANTS netcdf netcdf-mpi netcdf_parallel netcdf-cxx4_parallel) + foreach(variant ${PKG_VARIANTS}) + if(NOT NETCDF_FOUND) + message(DEBUG " Trying: ${variant}") + pkg_check_modules(NETCDF QUIET IMPORTED_TARGET ${variant}) + + if(NETCDF_FOUND) + message(VERBOSE "Found via pkg-config: ${variant}") + message(DEBUG " Version: ${NETCDF_VERSION}") + list(APPEND NETCDF_DETECTION_LOG "pkg-config ${variant}: found") + list(APPEND NETCDF_INCLUDE_HINTS ${NETCDF_INCLUDE_DIRS}) + list(APPEND NETCDF_LIBRARY_HINTS ${NETCDF_LIBRARY_DIRS}) + break() + else() + list(APPEND NETCDF_DETECTION_LOG "pkg-config ${variant}: not found") + endif() + endif() + endforeach() +else() + message(DEBUG "pkg-config not available") + list(APPEND NETCDF_DETECTION_LOG "pkg-config: not available") endif() -# Add pkg-config results to hints -list(APPEND NETCDF_INCLUDE_HINTS ${NETCDF_INCLUDE_DIRS}) -list(APPEND NETCDF_LIBRARY_HINTS ${NETCDF_LIBRARY_DIRS}) +# Manual search +message(VERBOSE "Searching for netcdf.h and libnetcdf") +message(DEBUG " Include hints: ${NETCDF_INCLUDE_HINTS}") +message(DEBUG " Library hints: ${NETCDF_LIBRARY_HINTS}") find_path(NETCDF_INCLUDES netcdf.h HINTS ${NETCDF_INCLUDE_HINTS} $ENV{NETCDF_DIR}/include) +if(NETCDF_INCLUDES) + message(VERBOSE "Found netcdf.h: ${NETCDF_INCLUDES}") + list(APPEND NETCDF_DETECTION_LOG "find_path: ${NETCDF_INCLUDES}") +else() + message(DEBUG "netcdf.h not found") + list(APPEND NETCDF_DETECTION_LOG "find_path: failed") +endif() + find_library(NETCDF_LIBRARIES_C NAMES netcdf HINTS ${NETCDF_LIBRARY_HINTS} $ENV{NETCDF_DIR}/lib) mark_as_advanced(NETCDF_LIBRARIES_C) -set(NetCDF_has_interfaces "YES") # will be set to NO if we're missing any interfaces +if(NETCDF_LIBRARIES_C) + message(VERBOSE "Found libnetcdf: ${NETCDF_LIBRARIES_C}") + list(APPEND NETCDF_DETECTION_LOG "find_library: ${NETCDF_LIBRARIES_C}") + + # Only add HDF5 if pkg-config told us NetCDF needs it + if(NETCDF_LINK_LIBRARIES) + # Check if pkg-config's library list includes hdf5 + string(FIND "${NETCDF_LINK_LIBRARIES}" "hdf5" HDF5_IN_NETCDF) + if(HDF5_IN_NETCDF GREATER -1) + message(STATUS "NetCDF was built with HDF5 support") + # Check if HDF5 was already found (e.g., by AMReX) + if(TARGET hdf5::hdf5 OR HDF5_FOUND) + list(APPEND NETCDF_LIBRARIES_C ${HDF5_LIBRARIES}) + message(STATUS " Using HDF5 libraries (already found): ${HDF5_LIBRARIES}") + else() + # Fallback: use pkg-config's complete library list which includes HDF5 + set(NETCDF_LIBRARIES_C ${NETCDF_LINK_LIBRARIES}) + message(STATUS " HDF5 not already a target, using pkg-config's complete library list:") + message(STATUS " NETCDF_LIBRARIES_C = ${NETCDF_LINK_LIBRARIES}") + endif() + else() + message(STATUS "NetCDF has no HDF5 dependency in pkg-config") + endif() + endif() # <-- THIS WAS MISSING! + +# FALLBACK: If find_library failed but pkg-config succeeded, use pkg-config's library list +elseif(NETCDF_FOUND AND NETCDF_LINK_LIBRARIES) + set(NETCDF_LIBRARIES_C ${NETCDF_LINK_LIBRARIES}) + message(STATUS "Using NetCDF libraries from pkg-config: ${NETCDF_LINK_LIBRARIES}") + list(APPEND NETCDF_DETECTION_LOG "pkg-config fallback: ${NETCDF_LINK_LIBRARIES}") +else() + message(DEBUG "libnetcdf not found") + list(APPEND NETCDF_DETECTION_LOG "find_library: failed") +endif() + +# HDF5 dependency +message(DEBUG "Checking HDF5 dependency") +if(NETCDF_LIBRARIES_C AND NETCDF_LINK_LIBRARIES) + string(FIND "${NETCDF_LINK_LIBRARIES}" "hdf5" HDF5_IN_NETCDF) + if(HDF5_IN_NETCDF GREATER -1) + message(VERBOSE "NetCDF requires HDF5") + if(TARGET hdf5::hdf5 OR HDF5_FOUND) + list(APPEND NETCDF_LIBRARIES_C ${HDF5_LIBRARIES}) + message(DEBUG "Using HDF5: ${HDF5_LIBRARIES}") + else() + set(NETCDF_LIBRARIES_C ${NETCDF_LINK_LIBRARIES}) + message(DEBUG "Using pkg-config libraries with HDF5") + endif() + endif() +endif() + +set(NetCDF_has_interfaces "YES") set(NetCDF_libs "${NETCDF_LIBRARIES_C}") get_filename_component(NetCDF_lib_dirs "${NETCDF_LIBRARIES_C}" PATH) @@ -91,14 +201,59 @@ set(NETCDF_LIBRARIES "${NetCDF_libs}" CACHE STRING "All NetCDF libraries require set(NETCDF_LINK_LIBRARIES ${NetCDF_libs}) set(NETCDF_INCLUDE_DIRS ${NETCDF_INCLUDES}) -# handle the QUIETLY and REQUIRED arguments and set NETCDF_FOUND to TRUE if -# all listed variables are TRUE -include (FindPackageHandleStandardArgs) +# Check if detection failed - show helpful error BEFORE standard handler +if(NOT NETCDF_LIBRARIES_C OR NOT NETCDF_INCLUDES) + message(STATUS "") + message(STATUS "====================================================================") + message(STATUS "NetCDF Detection Failed") + message(STATUS "====================================================================") + message(STATUS "") + message(STATUS "Detection attempts:") + foreach(attempt ${NETCDF_DETECTION_LOG}) + message(STATUS " ${attempt}") + endforeach() + message(STATUS "") + message(STATUS "Missing components:") + message(STATUS " netcdf.h: ${NETCDF_INCLUDES}") + message(STATUS " libnetcdf: ${NETCDF_LIBRARIES_C}") + message(STATUS "") + message(STATUS "To resolve:") + message(STATUS "") + message(STATUS " On Perlmutter/NERSC:") + message(STATUS " module load cray-netcdf-hdf5parallel") + message(STATUS "") + message(STATUS " On other Cray systems:") + message(STATUS " module load cray-netcdf") + message(STATUS "") + message(STATUS " Or specify manually:") + message(STATUS " cmake -DNETCDF_DIR=/path/to/netcdf ..") + message(STATUS "") + message(STATUS " Or via environment:") + message(STATUS " export NETCDF_DIR=/path/to/netcdf") + message(STATUS "") + message(STATUS "====================================================================") + message(STATUS "") + message(FATAL_ERROR "NetCDF not found") +endif() + +# Standard find package handling +include(FindPackageHandleStandardArgs) find_package_handle_standard_args (NetCDF DEFAULT_MSG NETCDF_LIBRARIES NETCDF_LINK_LIBRARIES NETCDF_INCLUDE_DIRS NETCDF_INCLUDES NetCDF_has_interfaces) -message(STATUS " NETCDF_LIBRARIES = ${NETCDF_LIBRARIES}") -#message(STATUS " NETCDF_LINK_LIBRARIES = ${NETCDF_LINK_LIBRARIES}") -#message(STATUS " NETCDF_INCLUDE_DIRS = ${NETCDF_INCLUDE_DIRS}") -message(STATUS " NETCDF_INCLUDES = ${NETCDF_INCLUDES}") -mark_as_advanced (NETCDF_LIBRARIES NETCDF_INCLUDES) +# Show diagnostics on failure +if(NOT NETCDF_FOUND) + message(STATUS "Detection attempts:") + foreach(attempt ${NETCDF_DETECTION_LOG}) + message(STATUS " ${attempt}") + endforeach() + message(STATUS "") + message(STATUS "To resolve:") + message(STATUS " Cray: module load cray-netcdf-hdf5parallel") + message(STATUS " Manual: -DNETCDF_DIR=/path/to/netcdf") + message(STATUS " Env var: export NETCDF_DIR=/path/to/netcdf") +endif() + +mark_as_advanced(NETCDF_LIBRARIES NETCDF_INCLUDES) +# Pop FindNetCDF context +list(POP_BACK CMAKE_MESSAGE_CONTEXT) diff --git a/CMake/UtilityTargets.cmake b/CMake/UtilityTargets.cmake new file mode 100644 index 0000000000..2876393b2e --- /dev/null +++ b/CMake/UtilityTargets.cmake @@ -0,0 +1,77 @@ +# Add uninstall target +if(NOT TARGET uninstall) + configure_file( + "${CMAKE_SOURCE_DIR}/CMake/cmake_uninstall.cmake.in" + "${CMAKE_BINARY_DIR}/cmake_uninstall.cmake" + IMMEDIATE @ONLY) + + add_custom_target(uninstall + COMMAND ${CMAKE_COMMAND} -P ${CMAKE_BINARY_DIR}/cmake_uninstall.cmake + COMMENT "Uninstalling files listed in install_manifest.txt" + ) +endif() + +# Add distclean target +add_custom_target(distclean + # Header + COMMAND ${CMAKE_COMMAND} -E echo "==================================================================================" + COMMAND ${CMAKE_COMMAND} -E echo "Distclean: ${CMAKE_BINARY_DIR}" + COMMAND ${CMAKE_COMMAND} -E echo "==================================================================================" + + # CMake configuration files + COMMAND ${CMAKE_COMMAND} -E remove -f + ${CMAKE_BINARY_DIR}/CMakeCache.txt + ${CMAKE_BINARY_DIR}/cmake_install.cmake + ${CMAKE_BINARY_DIR}/cmake_uninstall.cmake + ${CMAKE_BINARY_DIR}/Makefile + ${CMAKE_BINARY_DIR}/install_manifest.txt + ${CMAKE_BINARY_DIR}/cray_detected_config.cmake + + # CPack files + COMMAND ${CMAKE_COMMAND} -E remove -f + ${CMAKE_BINARY_DIR}/CPackConfig.cmake + ${CMAKE_BINARY_DIR}/CPackSourceConfig.cmake + + # CTest files + COMMAND ${CMAKE_COMMAND} -E remove -f + ${CMAKE_BINARY_DIR}/CTestTestfile.cmake + ${CMAKE_BINARY_DIR}/DartConfiguration.tcl + + # Project-specific generated files + COMMAND ${CMAKE_COMMAND} -E remove -f + ${CMAKE_BINARY_DIR}/ERFConfig.cmake + ${CMAKE_BINARY_DIR}/compile_commands.json + ${CMAKE_BINARY_DIR}/git-state.txt + + # CMake-generated directories + COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_BINARY_DIR}/CMakeFiles + COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_BINARY_DIR}/Testing + COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_BINARY_DIR}/_deps + + # Build output directories + COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_BINARY_DIR}/Exec + COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_BINARY_DIR}/Submodules + COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_BINARY_DIR}/Tests + COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_BINARY_DIR}/bin + COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_BINARY_DIR}/erf_srclib + COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_BINARY_DIR}/cmake_packages + COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_BINARY_DIR}/externals + + # Use shell commands with explicit directory (if needed for glob patterns) + COMMAND ${CMAKE_COMMAND} -E echo "Removing generated files..." + COMMAND sh -c "cd ${CMAKE_BINARY_DIR} && rm -f *.pc lib*.a lib*.so build_*.log 2>/dev/null || true" + + # Summary + COMMAND ${CMAKE_COMMAND} -E echo "" + COMMAND ${CMAKE_COMMAND} -E echo " DONE: Distclean complete" + COMMAND ${CMAKE_COMMAND} -E echo "" + COMMAND ${CMAKE_COMMAND} -E echo "Next steps to reconfigure:" + COMMAND ${CMAKE_COMMAND} -E echo " From build directory: cmake ${CMAKE_SOURCE_DIR}" + COMMAND ${CMAKE_COMMAND} -E echo " Or more simply: cmake .." + COMMAND ${CMAKE_COMMAND} -E echo " From source directory: cmake -B ${CMAKE_BINARY_DIR}" + COMMAND ${CMAKE_COMMAND} -E echo "" + COMMAND ${CMAKE_COMMAND} -E echo "Note: Install directories preserved" + + COMMENT "Removing all CMake configuration and build artifacts" + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} +) \ No newline at end of file diff --git a/CMake/cmake_uninstall.cmake.in b/CMake/cmake_uninstall.cmake.in new file mode 100644 index 0000000000..830a6c6230 --- /dev/null +++ b/CMake/cmake_uninstall.cmake.in @@ -0,0 +1,25 @@ +if(NOT EXISTS "@CMAKE_BINARY_DIR@/install_manifest.txt") + message(FATAL_ERROR "Cannot find install manifest: @CMAKE_BINARY_DIR@/install_manifest.txt") +endif() + +file(READ "@CMAKE_BINARY_DIR@/install_manifest.txt" files) +string(REGEX REPLACE "\n" ";" files "${files}") + +foreach(file ${files}) + message(STATUS "Uninstalling $ENV{DESTDIR}${file}") + if(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}") + execute_process( + COMMAND "@CMAKE_COMMAND@" -E remove "$ENV{DESTDIR}${file}" + RESULT_VARIABLE rm_retval + OUTPUT_VARIABLE rm_out + ERROR_VARIABLE rm_err + ) + if(NOT "${rm_retval}" STREQUAL "0") + message(FATAL_ERROR "Problem when removing $ENV{DESTDIR}${file}: ${rm_err}") + endif() + else() + message(STATUS "File $ENV{DESTDIR}${file} does not exist.") + endif() +endforeach() + +message(STATUS "Uninstall complete") \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 3331215cfc..a95120bceb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,37 +1,82 @@ ############################ BASE ###################################### cmake_minimum_required (VERSION 3.14 FATAL_ERROR) + +# Set main ERF context +list(APPEND CMAKE_MESSAGE_CONTEXT "ERF") + +# Include Cray compiler detection BEFORE project() to set compilers +list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMake") +include(CrayCompilerDetection) + if(ERF_ENABLE_MORR_FORT OR ERF_ENABLE_NOAHMP) project(ERF CXX C Fortran) else() project(ERF CXX C) endif() +message(STATUS "Configuring ERF") +message(VERBOSE "Source directory: ${CMAKE_SOURCE_DIR}") +message(VERBOSE "Build directory: ${CMAKE_BINARY_DIR}") + +# Find NVHPC package and create aliases if needed # Find NVHPC package and create aliases if needed if(ERF_ENABLE_CUDA AND ERF_ENABLE_NVHPC) - find_package(NVHPC REQUIRED COMPONENTS MATH CUDA) + list(APPEND CMAKE_MESSAGE_CONTEXT "NVHPC") + message(STATUS "Configuring NVHPC CUDA support") + + find_package(NVHPC QUIET COMPONENTS MATH CUDA) + + if(NOT NVHPC_FOUND) + message(STATUS "") + message(STATUS "====================================================================") + message(STATUS "NVHPC Detection Failed") + message(STATUS "====================================================================") + message(STATUS "") + message(STATUS "To resolve:") + message(STATUS " Load NVHPC module or set NVHPC_ROOT") + message(STATUS "") + message(STATUS " Example:") + message(STATUS " module load nvhpc") + message(STATUS " cmake -DNVHPC_ROOT=/path/to/nvhpc ..") + message(STATUS "") + message(STATUS "====================================================================") + message(STATUS "") + message(FATAL_ERROR "NVHPC required but not found") + endif() + + message(STATUS "Found NVHPC") + function(create_cuda_alias nvhpc_target cuda_name) if(TARGET NVHPC::${nvhpc_target} AND NOT TARGET CUDA::${cuda_name}) add_library(CUDA::${cuda_name} ALIAS NVHPC::${nvhpc_target}) - message(STATUS " Created alias: CUDA::${cuda_name} -> NVHPC::${nvhpc_target}") + message(STATUS " Created alias: CUDA::${cuda_name} -> NVHPC::${nvhpc_target}") elseif(NOT TARGET NVHPC::${nvhpc_target}) - message(WARNING "X Cannot create alias CUDA::${cuda_name}: NVHPC::${nvhpc_target} not found") - endif() + message(WARNING " Cannot create alias CUDA::${cuda_name}: NVHPC::${nvhpc_target} not found") + endif() endfunction() + create_cuda_alias(CUBLAS cublas) create_cuda_alias(CUBLAS_STATIC cublas_static) create_cuda_alias(CURAND curand) create_cuda_alias(CURAND_STATIC curand_static) create_cuda_alias(CUSPARSE cusparse) create_cuda_alias(CUSPARSE_STATIC cusparse_static) -endif() + list(POP_BACK CMAKE_MESSAGE_CONTEXT) +endif() list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMake") include(CMakePackageConfigHelpers) +# Include Cray/Perlmutter auto-detection and fixes +include(CrayDetection) +include(UtilityTargets) + ########################## OPTIONS ##################################### +message(VERBOSE "Configuring build options") + #General options for all executables in the project set(ERF_DIM "3" CACHE STRING "Number of physical dimensions") option(ERF_ENABLE_DOCUMENTATION "Build documentation" OFF) @@ -82,6 +127,7 @@ endif() # Configure measuring code coverage in tests option(CODECOVERAGE "Enable code coverage profiling" OFF) if(CODECOVERAGE) + message(VERBOSE "Enabling code coverage profiling") # Only supports GNU if(NOT CMAKE_CXX_COMPILER_ID MATCHES GNU) message(WARNING "CODECOVERAGE is only support with GNU Compilers. The current C++ compiler is ${CMAKE_CXX_COMPILER_ID}") @@ -96,7 +142,10 @@ endif() ########################### AMReX ##################################### +list(APPEND CMAKE_MESSAGE_CONTEXT "AMReX") + if (${ERF_USE_INTERNAL_AMREX}) + message(STATUS "Using internal AMReX submodule") set(AMREX_SUBMOD_LOCATION "${CMAKE_SOURCE_DIR}/Submodules/AMReX") include(${CMAKE_SOURCE_DIR}/CMake/SetAmrexOptions.cmake) list(APPEND CMAKE_MODULE_PATH "${AMREX_SUBMOD_LOCATION}/Tools/CMake") @@ -114,7 +163,10 @@ if (${ERF_USE_INTERNAL_AMREX}) set(FCOMPARE_EXE ${CMAKE_BINARY_DIR}/Submodules/AMReX/Tools/Plotfile/amrex_fcompare CACHE STRING "Path to fcompare executable for regression tests") endif() + message(VERBOSE "fcompare executable: ${FCOMPARE_EXE}") else() + message(STATUS "Using external AMReX") + set(CMAKE_PREFIX_PATH ${AMREX_DIR} ${CMAKE_PREFIX_PATH}) list(APPEND AMREX_COMPONENTS "3D" "PIC" "PARTICLES" "PDOUBLE" "DOUBLE" "LSOLVERS") @@ -139,10 +191,36 @@ else() if (ERF_ENABLE_TINY_PROFILE) list(APPEND AMREX_COMPONENTS "TINY_PROFILE") endif() + separate_arguments(AMREX_COMPONENTS) - find_package(AMReX CONFIG REQUIRED - COMPONENTS ${AMREX_COMPONENTS}) - message(STATUS "Found AMReX = ${AMReX_DIR}") + message(VERBOSE "Required AMReX components: ${AMREX_COMPONENTS}") + + find_package(AMReX CONFIG QUIET COMPONENTS ${AMREX_COMPONENTS}) + + if(NOT AMReX_FOUND) + message(STATUS "") + message(STATUS "====================================================================") + message(STATUS "AMReX Detection Failed") + message(STATUS "====================================================================") + message(STATUS "") + message(STATUS "Required components: ${AMREX_COMPONENTS}") + message(STATUS "") + message(STATUS "To resolve:") + message(STATUS "") + message(STATUS " Option 1: Use internal AMReX (recommended):") + message(STATUS " cmake -DERF_USE_INTERNAL_AMREX=ON ..") + message(STATUS "") + message(STATUS " Option 2: Build and install AMReX separately, then:") + message(STATUS " cmake -DAMReX_DIR=/path/to/amrex/lib/cmake/AMReX ..") + message(STATUS "") + message(STATUS "====================================================================") + message(STATUS "") + message(FATAL_ERROR "AMReX required but not found") + endif() + + message(STATUS "Found AMReX: ${AMReX_DIR}") + message(VERBOSE " AMReX_VERSION: ${AMReX_VERSION}") + if(WIN32) set(FCOMPARE_EXE ${AMReX_DIR}/../../../*/amrex_fcompare.exe CACHE STRING "Path to fcompare executable for regression tests") @@ -150,8 +228,12 @@ else() set(FCOMPARE_EXE ${AMReX_DIR}/../../../bin/amrex_fcompare CACHE STRING "Path to fcompare executable for regression tests") endif() + message(VERBOSE "fcompare executable: ${FCOMPARE_EXE}") + endif() +list(POP_BACK CMAKE_MESSAGE_CONTEXT) + ########################## EKAT ################################## if(ERF_ENABLE_RRTMGP OR ERF_ENABLE_SHOC OR ERF_ENABLE_P3) @@ -161,6 +243,9 @@ else() endif() if(ERF_ENABLE_EKAT) + list(APPEND CMAKE_MESSAGE_CONTEXT "EKAT") + + message(STATUS "Configuring EKAT") if(NOT ERF_ENABLE_MPI) message(FATAL_ERROR "CMake Error: MPI must be enabled if EKAT is enabled.") @@ -168,19 +253,22 @@ if(ERF_ENABLE_EKAT) # NOTE: EKAT provides KOKKOS, so set relevant flags for KOKKOS if(ERF_ENABLE_CUDA) + message(VERBOSE "Enabling Kokkos CUDA support") set(Kokkos_ENABLE_CUDA ON CACHE BOOL "kokkos enable cuda") set(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "kokkos enable cuda lambda") set(Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE ON CACHE BOOL "kokkos enable cuda RDC") endif() if(ERF_ENABLE_HIP) + message(VERBOSE "Enabling Kokkos HIP support") set(Kokkos_ENABLE_HIP ON CACHE BOOL "kokkos enable hip") set(Kokkos_ENABLE_HIP_LAMBDA ON CACHE BOOL "kokkos enable hip lambda") set(Kokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE ON CACHE BOOL "kokkos enable hip RDC") endif() if(ERF_ENABLE_SYCL) - set(Kokkos_ENABLE_SYCL ON CACHE BOOL "kokkos enable hip") + message(VERBOSE "Enabling Kokkos SYCL support") + set(Kokkos_ENABLE_SYCL ON CACHE BOOL "kokkos enable sycl") set(Kokkos_ENABLE_SYCL_LAMBDA ON CACHE BOOL "kokkos enable sycl lambda") endif() @@ -190,96 +278,207 @@ if(ERF_ENABLE_EKAT) set(EKAT_ENABLE_KOKKOS ON CACHE BOOL "ekat enable kokkos") set(EKAT_ENABLE_LOGGING ON CACHE BOOL "ekat enable spdlog") set(EKAT_BIN ${CMAKE_BINARY_DIR}/Submodules/ekat) + + message(VERBOSE "EKAT binary directory: ${EKAT_BIN}") add_subdirectory(${CMAKE_SOURCE_DIR}/Submodules/ekat ${EKAT_BIN}) + list(POP_BACK CMAKE_MESSAGE_CONTEXT) endif() + ########################### MPI ##################################### -##if(ERF_ENABLE_MPI) -## set(_mpi_comps C CXX) # Do we need MPI_C ? -## if(ERF_ENABLE_MORR_FORT OR ERF_ENABLE_NOAHMP) -## list(APPEND _mpi_comps Fortran) -## endif() -## find_package(MPI REQUIRED ${_mpi_comps}) -### list(TRANSFORM _mpi_comps PREPEND "MPI::MPI_") -### foreach(D IN LISTS AMReX_SPACEDIM) -### target_link_libraries(amrex_${D}d PUBLIC ${_mpi_comps}) -### endforeach() -### unset(_mpi_comps) -##endif() - message(STATUS "mpi section done") +########################### MPI ##################################### + +if(ERF_ENABLE_MPI) + list(APPEND CMAKE_MESSAGE_CONTEXT "MPI") + message(STATUS "Configuring MPI") + + # Check if we're on Cray with bare MPI wrappers (which will hang) + set(SKIP_MPI_DETECTION FALSE) + + if(DEFINED ENV{CRAYPE_VERSION} OR DEFINED ENV{CRAY_MPICH_DIR}) + # On Cray system - check if using problematic bare MPI wrappers + if(CMAKE_CXX_COMPILER MATCHES "mpicxx" OR + CMAKE_C_COMPILER MATCHES "mpicc" OR + CMAKE_Fortran_COMPILER MATCHES "mpifort") + message(STATUS "Detected bare MPI wrappers on Cray - skipping detection (would hang)") + set(SKIP_MPI_DETECTION TRUE) + endif() + endif() + + if(SKIP_MPI_DETECTION) + # Workaround: Manual MPI setup (avoids hang) + message(VERBOSE "Manually configuring MPI (bypassing find_package)") + + # Get Cray MPICH version for informational purposes + set(MPICH_VERSION "UNKNOWN") + if(DEFINED ENV{CRAY_MPICH_VERSION}) + set(MPICH_VERSION "$ENV{CRAY_MPICH_VERSION}") + elseif(DEFINED ENV{CRAY_MPICH_VER}) + set(MPICH_VERSION "$ENV{CRAY_MPICH_VER}") + endif() + + # Cray MPICH 8.x supports MPI 3.1 standard + set(MPI_VERSION "3.1") + + # Create MPI targets + if(NOT TARGET MPI::MPI_CXX) + add_library(MPI::MPI_CXX INTERFACE IMPORTED) + message(DEBUG "Created MPI::MPI_CXX target") + endif() + + if(NOT TARGET MPI::MPI_C) + add_library(MPI::MPI_C INTERFACE IMPORTED) + message(DEBUG "Created MPI::MPI_C target") + endif() + + # Set MPI variables + set(MPI_FOUND TRUE) + set(MPI_CXX_FOUND TRUE) + set(MPI_C_FOUND TRUE) + set(MPI_C_VERSION "${MPI_VERSION}") + set(MPI_CXX_VERSION "${MPI_VERSION}") + + message(STATUS "Cray MPICH: ${MPICH_VERSION}") + message(STATUS "MPI standard: ${MPI_VERSION}") + message(VERBOSE "Created MPI::MPI_CXX and MPI::MPI_C targets") + else() + # Normal path: Use find_package with QUIET + message(VERBOSE "Using find_package(MPI) for detection") + set(_mpi_comps C CXX) + if(ERF_ENABLE_MORR_FORT OR ERF_ENABLE_NOAHMP) + list(APPEND _mpi_comps Fortran) + endif() + message(DEBUG "MPI components: ${_mpi_comps}") + + find_package(MPI QUIET COMPONENTS ${_mpi_comps}) + + if(NOT MPI_FOUND) + message(STATUS "") + message(STATUS "====================================================================") + message(STATUS "MPI Detection Failed") + message(STATUS "====================================================================") + message(STATUS "") + message(STATUS "To resolve, load modules from your machine profile:") + message(STATUS "") + erf_suggest_machine_profile() + message(STATUS "") + message(STATUS " Or on non-Cray systems, install MPI:") + message(STATUS " OpenMPI, MPICH, Intel MPI, etc.") + message(STATUS "") + message(STATUS " Then configure with:") + message(STATUS " cmake -DMPI_C_COMPILER=mpicc -DMPI_CXX_COMPILER=mpicxx ..") + message(STATUS "") + message(STATUS "====================================================================") + message(STATUS "") + message(FATAL_ERROR "MPI required but not found") + endif() + + message(STATUS "Found MPI") + message(VERBOSE " MPI_C_VERSION: ${MPI_C_VERSION}") + message(VERBOSE " MPI_CXX_VERSION: ${MPI_CXX_VERSION}") + endif() + + message(STATUS "MPI configuration complete") + list(POP_BACK CMAKE_MESSAGE_CONTEXT) +else() + message(DEBUG "MPI not enabled") +endif() + ########################## NETCDF ################################## if(ERF_ENABLE_NETCDF OR ERF_ENABLE_TOOLS) + list(APPEND CMAKE_MESSAGE_CONTEXT "NetCDF") + + message(STATUS "Configuring NetCDF") + message(DEBUG "ERF_ENABLE_NETCDF: ${ERF_ENABLE_NETCDF}") + message(DEBUG "ERF_ENABLE_TOOLS: ${ERF_ENABLE_TOOLS}") + set(CMAKE_PREFIX_PATH ${NETCDF_DIR} ${CMAKE_PREFIX_PATH}) -## set(NETCDF_CXX "YES") find_package (NetCDF REQUIRED) if(NETCDF_FOUND) - message(STATUS "Found NetCDF, NETCDF_DIR = ${NETCDF_DIR}") + message(STATUS "Found NetCDF: ${NETCDF_DIR}") + message(VERBOSE "NetCDF includes: ${NETCDF_INCLUDES}") + message(VERBOSE "NetCDF libraries: ${NETCDF_LIBRARIES}") endif() + + message(STATUS "NetCDF configuration complete") + list(POP_BACK CMAKE_MESSAGE_CONTEXT) endif() ########################## NOAH-MP ################################## if(ERF_ENABLE_NOAHMP) + list(APPEND CMAKE_MESSAGE_CONTEXT "Noah-MP") + + message(STATUS "Configuring Noah-MP") + if(ERF_ENABLE_NETCDF) set(NOAHMP_HOME ${CMAKE_SOURCE_DIR}/Submodules/Noah-MP/drivers/erf) set(NOAHMP_BIN ${CMAKE_BINARY_DIR}/Submodules/Noah-MP/drivers/erf) + message(VERBOSE "Noah-MP source: ${NOAHMP_HOME}") + message(VERBOSE "Noah-MP binary: ${NOAHMP_BIN}") add_subdirectory(${NOAHMP_HOME} ${NOAHMP_BIN}) else() message(FATAL_ERROR "Noah-MP requires NetCDF be enabled") endif() + + message(STATUS "Noah-MP configuration complete") + list(POP_BACK CMAKE_MESSAGE_CONTEXT) endif() ########################### RRTMGP ################################# if(ERF_ENABLE_RRTMGP) - if(NOT ERF_ENABLE_EKAT) - message(FATAL_ERROR "CMake Error: EKAT must be enabled if RRTMGP is enabled.") - endif() + list(APPEND CMAKE_MESSAGE_CONTEXT "RRTMGP") - if(NOT ERF_ENABLE_NETCDF) - message(FATAL_ERROR "CMake Error: NetCDF must be enabled if RRTMGP is enabled.") - endif() + message(STATUS "Configuring RRTMGP") - message(STATUS "Building RRTMGP + KOKKOS...") + if(NOT ERF_ENABLE_EKAT) + message(FATAL_ERROR "EKAT must be enabled if RRTMGP is enabled") + endif() + + if(NOT ERF_ENABLE_NETCDF) + message(FATAL_ERROR "NetCDF must be enabled if RRTMGP is enabled") + endif() + + message(VERBOSE "Building RRTMGP with Kokkos support") + + # Build the static rrtmgp library + set(RRTMGP_BIN ${CMAKE_BINARY_DIR}/Submodules/rrtmgp) + message(VERBOSE "RRTMGP binary directory: ${RRTMGP_BIN}") + add_subdirectory(${CMAKE_SOURCE_DIR}/Submodules/RRTMGP/cpp ${RRTMGP_BIN}) - # Build the static rrtmgp library - set(RRTMGP_BIN ${CMAKE_BINARY_DIR}/Submodules/rrtmgp) - add_subdirectory(${CMAKE_SOURCE_DIR}/Submodules/RRTMGP/cpp ${RRTMGP_BIN}) + # Set up kokkos library and definitions + set(RRTMGP_ENABLE_KOKKOS TRUE) + target_compile_definitions(rrtmgp PUBLIC RRTMGP_ENABLE_KOKKOS) + target_link_libraries(rrtmgp kokkos) + message(DEBUG "Added Kokkos to RRTMGP target") - # Set up kokkos library and definitions - set(RRTMGP_ENABLE_KOKKOS TRUE) - target_compile_definitions(rrtmgp PUBLIC RRTMGP_ENABLE_KOKKOS) - target_link_libraries(rrtmgp kokkos) + message(STATUS "RRTMGP configuration complete") + list(POP_BACK CMAKE_MESSAGE_CONTEXT) endif() ########################### SHOC ################################# if(ERF_ENABLE_SHOC) - if(NOT ERF_ENABLE_EKAT) - message(FATAL_ERROR "CMake Error: EKAT must be enabled if SHOC is enabled.") - endif() + list(APPEND CMAKE_MESSAGE_CONTEXT "SHOC") - # NOTE: We compile shoc src files directly + message(STATUS "Configuring SHOC") -endif() + if(NOT ERF_ENABLE_EKAT) + message(FATAL_ERROR "EKAT must be enabled if SHOC is enabled") + endif() -########################### ERF ##################################### + # NOTE: We compile shoc src files directly + message(VERBOSE "SHOC source files will be compiled directly") -if(ERF_ENABLE_MPI) - set(_mpi_comps C CXX) # Do we need MPI_C ? - if(ERF_ENABLE_MORR_FORT OR ERF_ENABLE_NOAHMP) - list(APPEND _mpi_comps Fortran) - endif() - find_package(MPI REQUIRED ${_mpi_comps}) -# list(TRANSFORM _mpi_comps PREPEND "MPI::MPI_") -# foreach(D IN LISTS AMReX_SPACEDIM) -# target_link_libraries(amrex_${D}d PUBLIC ${_mpi_comps}) -# endforeach() -# unset(_mpi_comps) + message(STATUS "SHOC configuration complete") + list(POP_BACK CMAKE_MESSAGE_CONTEXT) endif() +########################### ERF ##################################### + # General information about machine, compiler, and build type message(STATUS "ERF Information:") message(STATUS "CMAKE_SYSTEM_NAME = ${CMAKE_SYSTEM_NAME}") @@ -294,15 +493,21 @@ include(${CMAKE_SOURCE_DIR}/CMake/SetRpath.cmake) add_subdirectory(Exec) if(ERF_ENABLE_TESTS) + message(STATUS "Configuring tests") include(CTest) add_subdirectory(Tests) endif() if(ERF_ENABLE_DOCUMENTATION) - add_subdirectory(Docs) + message(STATUS "Configuring documentation") + add_subdirectory(Docs) endif() # Installation rules +list(APPEND CMAKE_MESSAGE_CONTEXT "Install") + +message(STATUS "Configuring installation") + include(CMakePackageConfigHelpers) include(GNUInstallDirs) @@ -355,3 +560,13 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME} ) + +message(VERBOSE "Install prefix: ${CMAKE_INSTALL_PREFIX}") +message(VERBOSE "Install libdir: ${CMAKE_INSTALL_LIBDIR}") + +list(POP_BACK CMAKE_MESSAGE_CONTEXT) + +# Pop main ERF context +list(POP_BACK CMAKE_MESSAGE_CONTEXT) + +message(STATUS "ERF configuration complete") diff --git a/Docs/sphinx_doc/CouplingToNoahMP.rst b/Docs/sphinx_doc/CouplingToNoahMP.rst index 45067dac17..30657837e6 100644 --- a/Docs/sphinx_doc/CouplingToNoahMP.rst +++ b/Docs/sphinx_doc/CouplingToNoahMP.rst @@ -126,8 +126,8 @@ model (e.g., OpenAI, Argo, etc.). Tutorials are available at .. code-block:: bash code-scribe update NoahmpIO.H NoahmpIO.cpp NoahmpIO_fi.F90 \ - -r prompts/noahmpio_update.toml \ - -p "Write a natural language prompt with variable names, dimensions, etc." \ + -p prompts/noahmpio_update.toml \ + -q "Write a natural language prompt with variable names, dimensions, etc." \ -m 3. Run the following to generate or update bindings in **Source/LandSurfaceModel/Noah-MP** directory: @@ -135,8 +135,8 @@ model (e.g., OpenAI, Argo, etc.). Tutorials are available at .. code-block:: bash code-scribe update ERF_NOAHMP.cpp \ - -r prompts/noahmpio_update.toml \ - -p "Write a natural language prompt with variable names, dimensions, etc." \ + -p prompts/noahmpio_update.toml \ + -q "Write a natural language prompt with variable names, dimensions, etc." \ -m You may need to manually edit **Submodules/Noah-MP/drivers/erf/NoahmpIOVarType.F90** to replace: @@ -157,8 +157,8 @@ model’s context length) using: .. code-block:: bash code-scribe update NoahmpIOVarType.F90 \ - -r prompts/noahmpio_update.toml \ - -p "Write a natural language prompt with variable names, dimensions, etc." \ + -p prompts/noahmpio_update.toml \ + -q "Write a natural language prompt with variable names, dimensions, etc." \ -m If you want to control Noah-MP plot variables, you can update **Submodules/Noah-MP/drivers/erf/NoahmpWriteLandMod.F90** file: @@ -166,6 +166,6 @@ If you want to control Noah-MP plot variables, you can update **Submodules/Noah- .. code-block:: bash code-scribe update NoahmpWriteLandMod.F90 \ - -r prompts/noahmpwriteland_update.toml \ - -p "Write a natural language prompt with variable names, dimensions, etc." \ + -p prompts/noahmpwriteland_update.toml \ + -q "Write a natural language prompt with variable names, dimensions, etc." \ -m diff --git a/Exec/DryRegTests/WitchOfAgnesi/inputs_FittedMesh b/Exec/DryRegTests/WitchOfAgnesi/inputs_FittedMesh index 7010714e1b..cea1375759 100644 --- a/Exec/DryRegTests/WitchOfAgnesi/inputs_FittedMesh +++ b/Exec/DryRegTests/WitchOfAgnesi/inputs_FittedMesh @@ -64,8 +64,8 @@ erf.abl_driver_type = "PressureGradient" erf.abl_pressure_grad = -0.02 0. 0. erf.rayleigh_damp_W = true -erf.rayleigh_zdamp = 5000.0 -erf.rayleigh_dampcoef = 0.2 +erf.rayleigh_zdamp = 50.0 +erf.rayleigh_dampcoef = 0.25 #erf.init_type = "input_sounding" #erf.init_sounding_ideal = true diff --git a/Exec/Make.ERF b/Exec/Make.ERF index 79667a88ca..1aaca5c755 100644 --- a/Exec/Make.ERF +++ b/Exec/Make.ERF @@ -208,8 +208,21 @@ ifeq ($(USE_NOAHMP), TRUE) $(error USE_NETCDF must be true for using NOAH-MP interface) else DEFINES += -DERF_USE_NOAHMP - includes += $(shell pkg-config --cflags netcdf-fortran) - LIBRARIES += $(shell pkg-config --libs netcdf-fortran) + + # Try netcdf-fortran => netcdf-fortran_parallel + has_netcdf_fortran := $(shell if [ -n "$$MPICH_DIR" ]; then export PKG_CONFIG_PATH="$$MPICH_DIR/lib/pkgconfig:$$PKG_CONFIG_PATH"; fi; pkg-config --exists netcdf-fortran 2>/dev/null; echo $$?) + ifeq ($(has_netcdf_fortran),0) + includes += $(shell if [ -n "$$MPICH_DIR" ]; then export PKG_CONFIG_PATH="$$MPICH_DIR/lib/pkgconfig:$$PKG_CONFIG_PATH"; fi; pkg-config --cflags netcdf-fortran) + LIBRARIES += $(shell if [ -n "$$MPICH_DIR" ]; then export PKG_CONFIG_PATH="$$MPICH_DIR/lib/pkgconfig:$$PKG_CONFIG_PATH"; fi; pkg-config --libs netcdf-fortran) + else + has_netcdf_fortran_parallel := $(shell if [ -n "$$MPICH_DIR" ]; then export PKG_CONFIG_PATH="$$MPICH_DIR/lib/pkgconfig:$$PKG_CONFIG_PATH"; fi; pkg-config --exists netcdf-fortran_parallel 2>/dev/null; echo $$?) + ifeq ($(has_netcdf_fortran_parallel),0) + includes += $(shell if [ -n "$$MPICH_DIR" ]; then export PKG_CONFIG_PATH="$$MPICH_DIR/lib/pkgconfig:$$PKG_CONFIG_PATH"; fi; pkg-config --cflags netcdf-fortran_parallel) + LIBRARIES += $(shell if [ -n "$$MPICH_DIR" ]; then export PKG_CONFIG_PATH="$$MPICH_DIR/lib/pkgconfig:$$PKG_CONFIG_PATH"; fi; pkg-config --libs netcdf-fortran_parallel) + else + $(error NetCDF Fortran not found. Tried netcdf-fortran and netcdf-fortran_parallel) + endif + endif NOAHMP_HOME ?= $(ERF_HOME)/Submodules/Noah-MP @@ -396,13 +409,26 @@ endif # Turn on NetCDF macro define ifeq ($(USE_NETCDF), TRUE) DEFINES += -DERF_USE_NETCDF - has_netcdf_mpi := $(shell pkg-config --cflags netcdf-mpi > /dev/null 2>&1; echo $$?) - ifeq ($(has_netcdf_mpi),0) - includes += $(shell pkg-config --cflags netcdf-mpi) - LIBRARIES += $(shell pkg-config --libs netcdf-mpi) + + # Try netcdf => netcdf-cxx4_parallel => netcdf_parallel + has_netcdf := $(shell if [ -n "$$MPICH_DIR" ]; then export PKG_CONFIG_PATH="$$MPICH_DIR/lib/pkgconfig:$$PKG_CONFIG_PATH"; fi; pkg-config --exists netcdf 2>/dev/null; echo $$?) + ifeq ($(has_netcdf),0) + includes += $(shell if [ -n "$$MPICH_DIR" ]; then export PKG_CONFIG_PATH="$$MPICH_DIR/lib/pkgconfig:$$PKG_CONFIG_PATH"; fi; pkg-config --cflags netcdf) + LIBRARIES += $(shell if [ -n "$$MPICH_DIR" ]; then export PKG_CONFIG_PATH="$$MPICH_DIR/lib/pkgconfig:$$PKG_CONFIG_PATH"; fi; pkg-config --libs netcdf) else - includes += $(shell pkg-config --cflags netcdf) - LIBRARIES += $(shell pkg-config --libs netcdf) + has_netcdf_cxx4_parallel := $(shell if [ -n "$$MPICH_DIR" ]; then export PKG_CONFIG_PATH="$$MPICH_DIR/lib/pkgconfig:$$PKG_CONFIG_PATH"; fi; pkg-config --exists netcdf-cxx4_parallel 2>/dev/null; echo $$?) + ifeq ($(has_netcdf_cxx4_parallel),0) + includes += $(shell if [ -n "$$MPICH_DIR" ]; then export PKG_CONFIG_PATH="$$MPICH_DIR/lib/pkgconfig:$$PKG_CONFIG_PATH"; fi; pkg-config --cflags netcdf-cxx4_parallel) + LIBRARIES += $(shell if [ -n "$$MPICH_DIR" ]; then export PKG_CONFIG_PATH="$$MPICH_DIR/lib/pkgconfig:$$PKG_CONFIG_PATH"; fi; pkg-config --libs netcdf-cxx4_parallel) + else + has_netcdf_parallel := $(shell if [ -n "$$MPICH_DIR" ]; then export PKG_CONFIG_PATH="$$MPICH_DIR/lib/pkgconfig:$$PKG_CONFIG_PATH"; fi; pkg-config --exists netcdf_parallel 2>/dev/null; echo $$?) + ifeq ($(has_netcdf_parallel),0) + includes += $(shell if [ -n "$$MPICH_DIR" ]; then export PKG_CONFIG_PATH="$$MPICH_DIR/lib/pkgconfig:$$PKG_CONFIG_PATH"; fi; pkg-config --cflags netcdf_parallel) + LIBRARIES += $(shell if [ -n "$$MPICH_DIR" ]; then export PKG_CONFIG_PATH="$$MPICH_DIR/lib/pkgconfig:$$PKG_CONFIG_PATH"; fi; pkg-config --libs netcdf_parallel) + else + $(error NetCDF not found. Tried netcdf, netcdf-cxx4_parallel, and netcdf_parallel) + endif + endif endif endif diff --git a/Source/BoundaryConditions/ERF_BoundaryConditionsRealbdy.cpp b/Source/BoundaryConditions/ERF_BoundaryConditionsRealbdy.cpp index bb7d5a83ba..c81ad955a5 100644 --- a/Source/BoundaryConditions/ERF_BoundaryConditionsRealbdy.cpp +++ b/Source/BoundaryConditions/ERF_BoundaryConditionsRealbdy.cpp @@ -380,16 +380,27 @@ ERF::fill_from_realbdy_upwind (const Vector& mfs, const Array4& u_arr = mf_u.array(mfi); const Array4& v_arr = mf_v.array(mfi); + auto lb_u = lbound(u_arr); lb_u.x += ngvect_vels[0]; lb_u.y += ngvect_vels[1]; + auto ub_u = ubound(u_arr); ub_u.x -= ngvect_vels[0]; ub_u.y -= ngvect_vels[1]; + auto lb_v = lbound(v_arr); lb_v.x += ngvect_vels[0]; lb_v.y += ngvect_vels[1]; + auto ub_v = ubound(v_arr); ub_v.x -= ngvect_vels[0]; ub_v.y -= ngvect_vels[1]; + // NOTE: Xlo/hi boxes own corner cells (Ylo/hi) ParallelFor(bx_xlo, bx_xhi, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + // Limit for BDY FAB data int ii = std::max(i , dom_lo.x); int jj = std::max(j , dom_lo.y); jj = std::min(jj, dom_hi.y); - if ( (u_arr(dom_lo.x,jj,k) >= 0.0) || - ((jj == dom_lo.y) && (v_arr(ii,dom_lo.y ,k) >= 0.0)) || - ((jj == dom_hi.y) && (v_arr(ii,dom_hi.y+1,k) <= 0.0)) ) { + + // Limit for u_arr and v_arr + int ju = std::min(std::max(j, lb_u.y), ub_u.y); + int iv = std::min(std::max(i, lb_v.x), ub_v.x); + + if ( (u_arr(dom_lo.x,ju,k) >= 0.0) || + ((jj == dom_lo.y) && (v_arr(iv,dom_lo.y ,k) >= 0.0)) || + ((jj == dom_hi.y) && (v_arr(iv,dom_hi.y+1,k) <= 0.0)) ) { dest_arr(i,j,k,comp_idx) = oma * bdatxlo_n (ii,jj,k,0) + alpha * bdatxlo_np1(ii,jj,k,0); if (var_idx == Vars::cons) { @@ -401,12 +412,18 @@ ERF::fill_from_realbdy_upwind (const Vector& mfs, }, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + // Limit for BDY FAB data int ii = std::min(i , dom_hi.x); int jj = std::max(j , dom_lo.y); jj = std::min(jj, dom_hi.y); - if ( (u_arr(dom_hi.x+1,jj,k) <= 0.0) || - ((jj == dom_lo.y) && (v_arr(ii,dom_lo.y ,k) >= 0.0)) || - ((jj == dom_hi.y) && (v_arr(ii,dom_hi.y+1,k) <= 0.0)) ) { + + // Limit for u_arr and v_arr + int ju = std::min(std::max(j, lb_u.y), ub_u.y); + int iv = std::min(std::max(i, lb_v.x), ub_v.x); + + if ( (u_arr(dom_hi.x+1,ju,k) <= 0.0) || + ((jj == dom_lo.y) && (v_arr(iv,dom_lo.y ,k) >= 0.0)) || + ((jj == dom_hi.y) && (v_arr(iv,dom_hi.y+1,k) <= 0.0)) ) { dest_arr(i,j,k,comp_idx) = oma * bdatxhi_n (ii,jj,k,0) + alpha * bdatxhi_np1(ii,jj,k,0); if (var_idx == Vars::cons) { @@ -423,8 +440,13 @@ ERF::fill_from_realbdy_upwind (const Vector& mfs, ParallelFor(bx_ylo, bx_yhi, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + // Limit for BDY FAB data int jj = std::max(j, dom_lo.y); - if (v_arr(i,dom_lo.y,k) >= 0.0) { + + // Limit for v_arr + int iv = std::min(std::max(i, lb_v.x), ub_v.x); + + if (v_arr(iv,dom_lo.y,k) >= 0.0) { dest_arr(i,j,k,comp_idx) = oma * bdatylo_n (i,jj,k,0) + alpha * bdatylo_np1(i,jj,k,0); if (var_idx == Vars::cons) { @@ -436,8 +458,13 @@ ERF::fill_from_realbdy_upwind (const Vector& mfs, }, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + // Limit for BDY FAB data int jj = std::min(j, dom_hi.y); - if (v_arr(i,dom_hi.y+1,k) <= 0.0) { + + // Limit for v_arr + int iv = std::min(std::max(i, lb_v.x), ub_v.x); + + if (v_arr(iv,dom_hi.y+1,k) <= 0.0) { dest_arr(i,j,k,comp_idx) = oma * bdatyhi_n (i,jj,k,0) + alpha * bdatyhi_np1(i,jj,k,0); if (var_idx == Vars::cons) { diff --git a/Source/EB/ERF_EB.H b/Source/EB/ERF_EB.H index fd6d240756..28aab3491c 100644 --- a/Source/EB/ERF_EB.H +++ b/Source/EB/ERF_EB.H @@ -99,8 +99,44 @@ class eb_ { inline amrex::FabArray& getNonConstEBCellFlags(const amrex::EBFArrayBoxFactory& ebfact) { - const amrex::FabArray& flags_const = ebfact.getMultiEBCellFlagFab(); - return const_cast&>(flags_const); + const amrex::FabArray& flags_const = ebfact.getMultiEBCellFlagFab(); + return const_cast&>(flags_const); + } + + inline amrex::MultiFab& + getNonConstVolFrac(const amrex::EBFArrayBoxFactory& ebfact) + { + const amrex::MultiFab& vfrac_const = ebfact.getVolFrac(); + return const_cast(vfrac_const); + } + + inline amrex::MultiCutFab& + getNonConstCentroid(const amrex::EBFArrayBoxFactory& ebfact) + { + const amrex::MultiCutFab& vcent_const = ebfact.getCentroid(); + return const_cast(vcent_const); + } + + inline amrex::Array + getNonConstAreaFrac(const amrex::EBFArrayBoxFactory& ebfact) + { + auto afrac_const = ebfact.getAreaFrac(); + amrex::Array afrac; + for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) { + afrac[dir] = const_cast(afrac_const[dir]); + } + return afrac; + } + + inline amrex::Array + getNonConstFaceCent(const amrex::EBFArrayBoxFactory& ebfact) + { + auto fcent_const = ebfact.getFaceCent(); + amrex::Array fcent; + for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) { + fcent[dir] = const_cast(fcent_const[dir]); + } + return fcent; } }; diff --git a/Source/EB/ERF_EBAux.cpp b/Source/EB/ERF_EBAux.cpp index 584a547c05..5922d3e516 100644 --- a/Source/EB/ERF_EBAux.cpp +++ b/Source/EB/ERF_EBAux.cpp @@ -53,14 +53,8 @@ define( [[maybe_unused]] int const& a_level, m_volcent = new MultiFab(my_grids, a_dmap, AMREX_SPACEDIM, a_ngrow[2], MFInfo(), FArrayBoxFactory()); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - const BoxArray& faceba = amrex::convert(a_grids, IntVect::TheDimensionVector(idim)); - if (idim == a_idim) { - m_areafrac[idim] = new MultiFab(a_grids, a_dmap, 1, a_ngrow[1]+1, MFInfo(), FArrayBoxFactory()); - m_facecent[idim] = new MultiFab(a_grids, a_dmap, AMREX_SPACEDIM-1, a_ngrow[2], MFInfo(), FArrayBoxFactory()); - } else { - m_areafrac[idim] = new MultiFab(faceba, a_dmap, 1, a_ngrow[1], MFInfo(), FArrayBoxFactory()); - m_facecent[idim] = new MultiFab(faceba, a_dmap, AMREX_SPACEDIM-1, a_ngrow[2], MFInfo(), FArrayBoxFactory()); - } + m_areafrac[idim] = new MultiFab(a_grids, a_dmap, 1, a_ngrow[1]+1, MFInfo(), FArrayBoxFactory()); + m_facecent[idim] = new MultiFab(a_grids, a_dmap, AMREX_SPACEDIM-1, a_ngrow[2], MFInfo(), FArrayBoxFactory()); } m_bndryarea = new MultiFab(my_grids, a_dmap, 1, a_ngrow[2], MFInfo(), FArrayBoxFactory()); @@ -848,23 +842,22 @@ define( [[maybe_unused]] int const& a_level, for (MFIter mfi(*m_cellflags, false); mfi.isValid(); ++mfi) { - const Box& bx = mfi.validbox(); - const Box& bx_grown = mfi.growntilebox(); - - Array4 const& aux_flag = m_cellflags->array(mfi); - Array4 const& aux_vfrac = m_volfrac->array(mfi); - Array4 const& aux_afrac_x = m_areafrac[0]->array(mfi); - Array4 const& aux_afrac_y = m_areafrac[1]->array(mfi); - Array4 const& aux_afrac_z = m_areafrac[2]->array(mfi); + const Box& bx = mfi.validbox(); + const Box& bx_grown = mfi.growntilebox(); - Array4 const& aux_vcent = m_volcent->array(mfi); - Array4 const& aux_fcent_x = m_facecent[0]->array(mfi); - Array4 const& aux_fcent_y = m_facecent[1]->array(mfi); - Array4 const& aux_fcent_z = m_facecent[2]->array(mfi); - Array4 const& aux_barea = m_bndryarea->array(mfi); - Array4 const& aux_bcent = m_bndrycent->array(mfi); - Array4 const& aux_bnorm = m_bndrynorm->array(mfi); + Array4 const& aux_flag = m_cellflags->array(mfi); + Array4 const& aux_vfrac = m_volfrac->array(mfi); + Array4 const& aux_afrac_x = m_areafrac[0]->array(mfi); + Array4 const& aux_afrac_y = m_areafrac[1]->array(mfi); + Array4 const& aux_afrac_z = m_areafrac[2]->array(mfi); + Array4 const& aux_vcent = m_volcent->array(mfi); + Array4 const& aux_fcent_x = m_facecent[0]->array(mfi); + Array4 const& aux_fcent_y = m_facecent[1]->array(mfi); + Array4 const& aux_fcent_z = m_facecent[2]->array(mfi); + Array4 const& aux_barea = m_bndryarea->array(mfi); + Array4 const& aux_bcent = m_bndrycent->array(mfi); + Array4 const& aux_bnorm = m_bndrynorm->array(mfi); if (FlagFab[mfi].getType(bx) == FabType::singlevalued ) { diff --git a/Source/IO/ERF_ReadFromMetgrid.cpp b/Source/IO/ERF_ReadFromMetgrid.cpp index d5e1658a11..a78a5f1297 100644 --- a/Source/IO/ERF_ReadFromMetgrid.cpp +++ b/Source/IO/ERF_ReadFromMetgrid.cpp @@ -117,7 +117,7 @@ read_from_metgrid (int lev, const Box& domain, const std::string& fname, Vector success_i; success_i.resize(NC_iabs.size()); BuildFABsFromNetCDFFile(domain, fname, NC_inames, NC_idim_types, NC_iabs, success_i); for (int i = 0; i < success_i.size(); i++) { - if (NC_inames[i] == "LANDMASK" && success[i] == 1) {flag_lmask = 1;} + if (NC_inames[i] == "LANDMASK" && success_i[i] == 1) {flag_lmask = 1;} } // TODO: FIND OUT IF WE NEED TO DIVIDE VELS BY MAPFAC diff --git a/Source/LandSurfaceModel/Noah-MP/prompts/noahmpio_update.toml b/Source/LandSurfaceModel/Noah-MP/prompts/noahmpio_update.toml index c8c0501973..cdc6ef230c 100644 --- a/Source/LandSurfaceModel/Noah-MP/prompts/noahmpio_update.toml +++ b/Source/LandSurfaceModel/Noah-MP/prompts/noahmpio_update.toml @@ -1,6 +1,6 @@ # Usage: code-scribe update ERF_NOAHMP.cpp \ # -p prompts/noahmpio_update.toml \ -# -p "Write a natural language prompt with variable name dimension etc." \ +# -q "Write a natural language prompt with variable name dimension etc." \ # -m [[chat.user]] diff --git a/Source/Utils/ERF_HurricaneDiagnostics.H b/Source/Utils/ERF_HurricaneDiagnostics.H index 61ed21b47a..4337cdd750 100644 --- a/Source/Utils/ERF_HurricaneDiagnostics.H +++ b/Source/Utils/ERF_HurricaneDiagnostics.H @@ -326,7 +326,6 @@ HurricaneMaxVelTracker(const amrex::Geometry& geom, amrex::Real h_val_max_global = -1.0e30; #ifdef AMREX_USE_MPI MPI_Allreduce(&h_val_max_local, &h_val_max_global, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); - h_val_max_global = h_val_max_local; #else h_val_max_global = h_val_max_local; #endif diff --git a/Submodules/Noah-MP b/Submodules/Noah-MP index 128b02d0d4..6dc16027de 160000 --- a/Submodules/Noah-MP +++ b/Submodules/Noah-MP @@ -1 +1 @@ -Subproject commit 128b02d0d4ffbe6d69b799f65bf670d1e9966d7b +Subproject commit 6dc16027de56f2abc9fe430016acd086625e9ae2