Cray specific detection and enhanced netcdf pkg-config support

.github/workflows/windows-mpi.yml

@@ -0,0 +1,163 @@
+name: Windows
+
+on: [push, pull_request]
+
+concurrency:
+  group: ${{ github.ref }}-${{ github.head_ref }}-windows
+  cancel-in-progress: true
+
+jobs:
+  WIN64-MSVC:
+    name: WIN64-MSVC - MPI ${{ matrix.mpi }} - Particles ${{ matrix.particles }}
+    runs-on: windows-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        particles: [OFF, ON]
+        mpi: [ON]
+
+    steps:
+      - name: Checkout (with submodules)
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Install MS-MPI SDK only
+        shell: pwsh
+        run: |
+          curl -L -o msmpisdk.msi  https://github.com/microsoft/Microsoft-MPI/releases/download/v10.1.1/msmpisdk.msi
+          msiexec /i msmpisdk.msi /qn /norestart
+
+          $msmpiInc = 'C:\Program Files (x86)\Microsoft SDKs\MPI\Include'
+          $msmpiLib = 'C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64'
+          "MSMPI_INC=$msmpiInc" | Out-File -FilePath $env:GITHUB_ENV -Append
+          "MSMPI_LIB=$msmpiLib" | Out-File -FilePath $env:GITHUB_ENV -Append
+
+      - name: Configure (CMake)
+        shell: pwsh
+        run: |
+          cmake -S . -B build `
+            -DCMAKE_BUILD_TYPE=Release `
+            -DERF_ENABLE_MPI=ON `
+            -DMPI_CXX_LIB_NAMES=msmpi `
+            -DMPI_C_LIB_NAMES=msmpi `
+            -DMPI_msmpi_LIBRARY="$env:MSMPI_LIB\msmpi.lib" `
+              ${{ github.workspace }}
+
+      - name: Build
+        shell: pwsh
+        run: cmake --build build --parallel 2 --verbose
+
+      - name: Create Installation README
+        shell: pwsh
+        run: |
+          @"
+          # ERF Windows Build - Installation Instructions
+
+          ## Build Information
+          - Build Type: Release
+          - MPI Enabled: ${{ matrix.mpi }}
+          - Particles Enabled: ${{ matrix.particles }}
+          - MS-MPI Version: 10.1.1
+          - Built on: $(Get-Date -Format "yyyy-MM-dd HH:mm:ss UTC")
+          - Commit: ${{ github.sha }}
+
+          ## Required Runtime Installation
+
+          This executable requires MS-MPI Runtime v10.1.1 to run.
+
+          ### Quick Install (PowerShell):
+          ``````powershell
+          # Download and install MS-MPI Runtime v10.1.1
+          Invoke-WebRequest -Uri "https://github.com/microsoft/Microsoft-MPI/releases/download/v10.1.1/msmpisetup.exe" -OutFile "msmpisetup.exe"
+          .\msmpisetup.exe
+
+          # After installation, open a NEW PowerShell window to refresh PATH
+          ``````
+
+          ### Alternative Install (using vcpkg):
+          ``````powershell
+          # vcpkg will download the installer and prompt you to run it
+          vcpkg install msmpi:x64-windows
+          # Follow the instructions vcpkg provides to run the downloaded installer
+          ``````
+
+          ## Running the Executable
+
+          Navigate to the executable directory (e.g., `Exec\ABL\Debug\` or `Exec\ABL\Release\`) and run:
+
+          ``````powershell
+          # Single process
+          mpiexec -n 1 .\erf_abl.exe path\to\inputs_file
+
+          # Multiple processes (e.g., 4)
+          mpiexec -n 4 .\erf_abl.exe path\to\inputs_file
+          ``````
+
+          **Important:** Always use `mpiexec` or `mpirun` to launch. Running `.\erf_abl.exe` directly will trigger Windows Defender firewall warnings.
+
+          ## Troubleshooting
+
+          ### Executable runs but produces no output:
+          - MS-MPI runtime is not installed, or the wrong version is installed.
+          - Install MS-MPI v10.1.1 runtime using the script above
+          - Open a NEW terminal after installation
+
+          ### "mpiexec not recognized":
+          - Open a new PowerShell window after installing MS-MPI
+          - Or manually refresh PATH: `$env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine")`
+
+          ## Build Details
+
+          This build was compiled with:
+          - Compiler: MSVC 19.29+
+          - CMake Configuration:
+            - CMAKE_BUILD_TYPE=Release
+            - ERF_ENABLE_MPI=ON
+            - ERF_ENABLE_PARTICLES=${{ matrix.particles }}
+          "@ | Out-File -FilePath build\INSTALL.txt -Encoding utf8
+
+      - name: Add Installation Instructions to Job Summary
+        shell: pwsh
+        run: |
+          @"
+          ## ✅ Build Complete: ERF Windows (MPI: ${{ matrix.mpi }}, Particles: ${{ matrix.particles }})
+
+          ### 📦 Artifact Information
+          - **Build Type:** Release
+          - **MS-MPI Version:** 10.1.1
+          - **Commit:** ${{ github.sha }}
+
+          ### ⚠️ Required for Running
+
+          This executable requires **MS-MPI Runtime v10.1.1**
+
+          #### Quick Install:
+          ``````powershell
+          Invoke-WebRequest -Uri "https://github.com/microsoft/Microsoft-MPI/releases/download/v10.1.1/msmpisetup.exe" -OutFile "msmpisetup.exe"
+          .\msmpisetup.exe
+          ``````
+
+          #### Or use vcpkg:
+          ``````powershell
+          vcpkg install msmpi:x64-windows
+          ``````
+
+          ### 🚀 Running the Executable
+
+          ``````powershell
+          mpiexec -n 4 .\erf_abl.exe path\to\inputs_file
+          ``````
+
+          **Note:** Always use `mpiexec` to launch (not `.\erf_abl.exe` directly) to avoid Windows Defender warnings.
+
+          See `INSTALL.txt` in the artifact for complete instructions.
+          "@ | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append -Encoding utf8
+
+      - name: Upload artifact (from build tree)
+        uses: actions/upload-artifact@v4
+        with:
+          name: ERF-win64-Release-mpi-${{ matrix.mpi }}-particles-${{ matrix.particles }}
+          path: |
+            build
+          if-no-files-found: warn

.gitignore

@@ -31,3 +31,41 @@ cmake*ser*
 cmake*par*
 .idea
 *.ipynb
+
+# CMake build artifacts
+CMakeCache.txt
+CMakeFiles/
+cmake_install.cmake
+CTestTestfile.cmake
+DartConfiguration.tcl
+Makefile
+compile_commands.json
+Testing/
+_deps/
+
+# Project-specific generated files
+ERFConfig.cmake
+*.pc
+git-state.txt
+
+# Build outputs
+lib*.a
+lib*.so
+bin/
+Exec/
+Submodules/
+Tests/
+cmake_packages/
+externals/
+
+# Build and install directories
+build_*/
+install_*/
+
+# Log files
+build_*.log
+
+# Editor backups
+*~
+\#*\#
+.#*

Build/Perlmutter/build_erf_with_shoc_cuda_Perlmutter.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+set -e
+set -o pipefail
+
+# Function to verify if a directory is the ERF repo root
+verify_erf_dir() {
+    local dir=$1
+
+    # Check for basic structure
+    if [ ! -f "$dir/CMakeLists.txt" ] || [ ! -d "$dir/Source" ]; then
+        return 1
+    fi
+
+    # Check for "Energy Research and Forecasting" in key files
+    local found=0
+
+    if [ -f "$dir/README.rst" ]; then
+        if grep -q "Energy Research and Forecasting" "$dir/README.rst" 2>/dev/null; then
+            found=1
+        fi
+    fi
+
+    if [ $found -eq 0 ] && [ -f "$dir/LICENSE.md" ]; then
+        if grep -q "Energy Research and Forecasting" "$dir/LICENSE.md" 2>/dev/null; then
+            found=1
+        fi
+    fi
+
+    if [ $found -eq 0 ] && [ -f "$dir/CITATION.cff" ]; then
+        if grep -q "Energy Research and Forecasting" "$dir/CITATION.cff" 2>/dev/null; then
+            found=1
+        fi
+    fi
+
+    return $((1 - found))
+}
+
+# Function to find ERF repo root with multiple fallbacks
+find_erf_dir() {
+    # Method 1: Use git to find repo root
+    if command -v git &> /dev/null; then
+        if git rev-parse --is-inside-work-tree &> /dev/null 2>&1; then
+            local git_root="$(git rev-parse --show-toplevel)"
+            if verify_erf_dir "$git_root"; then
+                ERF_DIR="$git_root"
+                echo "Detected ERF_DIR from git: $ERF_DIR"
+                return 0
+            fi
+        fi
+    fi
+
+    # Method 2: Try going up from script location
+    local script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+    # Script is in Build/Perlmutter/, so go up 2 levels
+    local candidate="$(cd "$script_dir/../.." && pwd)"
+    if verify_erf_dir "$candidate"; then
+        ERF_DIR="$candidate"
+        echo "Detected ERF_DIR from script location: $ERF_DIR"
+        return 0
+    fi
+
+    # Method 3: Check current directory
+    if verify_erf_dir "$PWD"; then
+        ERF_DIR="$PWD"
+        echo "Detected ERF_DIR from current directory: $ERF_DIR"
+        return 0
+    fi
+
+    echo "Error: Could not auto-detect ERF_DIR"
+    echo "Verification requires:"
+    echo "  - CMakeLists.txt and Source/ directory"
+    echo "  - 'Energy Research and Forecasting' in README.rst, LICENSE.md, or CITATION.cff"
+    return 1
+}
+
+###################################################################################
+
+# 1. Resolve ERF_DIR
+# Detect ERF_DIR
+if ! find_erf_dir; then
+    exit 1
+fi
+
+export ERF_DIR
+
+E3SM_DIR="$ERF_DIR/external/E3SM"
+if [ ! -d "$E3SM_DIR" ]; then
+    echo "external/E3SM folder not found, running eamxx_clone.sh..."
+    source "$ERF_DIR/Build/GNU_Ekat/eamxx_clone.sh"
+else
+    echo "external/E3SM folder already exists, skipping clone."
+fi
+
+# 3. Prepare build directory
+echo "Preparing build directory..."
+mkdir -p "$ERF_DIR/build"
+cp "$ERF_DIR/Build/Perlmutter/cmake_with_cuda_shoc_Perlmutter.sh" "$ERF_DIR/build/"
+
+# 4. Move into build directory
+cd "$ERF_DIR/build"
+
+# Run cmake setup
+echo "Running cmake_with_cuda_shoc_Perlmutter.sh..."
+source cmake_with_cuda_shoc_Perlmutter.sh

Build/Perlmutter/cmake_with_cuda_perlmutter.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Load the needed modules
+module load gcc-native cmake cray-mpich cray-libsci cray-hdf5-parallel cray-netcdf-hdf5parallel
+
+# GPU-aware mpi is on by default (set it anyways)
+export MPICH_GPU_SUPPORT_ENABLED=1
+
+# Deduce the lib paths and files with $(CC/cc/ftn --cray-print-opts=libs)
+CRAY_LIBS_CLEAN=$(CC --cray-print-opts=libs | sed 's/-Wl,--as-needed,//g; s/,--no-as-needed//g; s/,-l/ -l/g')
+CRAY_LIBS_CLEAN="$CRAY_LIBS_CLEAN $(cc --cray-print-opts=libs | sed 's/-Wl,--as-needed,//g; s/,--no-as-needed//g; s/,-l/ -l/g')"
+CRAY_LIBS_CLEAN="$CRAY_LIBS_CLEAN $(ftn --cray-print-opts=libs | sed 's/-Wl,--as-needed,//g; s/,--no-as-needed//g; s/,-l/ -l/g')"
+
+# Configure and build
+cmake -DCMAKE_INSTALL_PREFIX:PATH=./install \
+      -DCMAKE_C_COMPILER=cc \
+      -DCMAKE_CXX_COMPILER=CC \
+      -DCMAKE_C_FLAGS="$(cc --cray-print-opts=cflags)" \
+      -DCMAKE_CXX_FLAGS="$(CC --cray-print-opts=cflags)" \
+      -DCMAKE_CUDA_FLAGS="$(CC --cray-print-opts=cflags)" \
+      -DCMAKE_CXX_STANDARD_LIBRARIES="-lmpi_gnu_123 -lmpi_gtl_cuda" \
+      -DCMAKE_CUDA_STANDARD_LIBRARIES="-lmpi_gnu_123 -lmpi_gtl_cuda" \
+      -DCMAKE_EXE_LINKER_FLAGS="-Wl,--no-as-needed $CRAY_LIBS_CLEAN" \
+      -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \
+      -DCMAKE_BUILD_TYPE:STRING=Release \
+      -DERF_DIM:STRING=3 \
+      -DERF_ENABLE_MPI:BOOL=ON \
+      -DERF_ENABLE_TESTS:BOOL=ON \
+      -DAMReX_CUDA_ARCH=8.0 \
+      -DERF_ENABLE_CUDA:BOOL=ON \
+      -DERF_ENABLE_FCOMPARE:BOOL=ON \
+      -DERF_ENABLE_DOCUMENTATION:BOOL=OFF \
+      -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \
+      .. && make -j8

Build/Perlmutter/cmake_with_cuda_perlmutter.sh~

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Load the needed modules
+module load gcc-native cmake cray-mpich cray-libsci cray-hdf5-parallel cray-netcdf-hdf5parallel
+
+# GPU-aware mpi is on by default (set it anyways)
+export MPICH_GPU_SUPPORT_ENABLED=1
+
+# Deduce the lib paths and files with $(CC/cc/ftn --cray-print-opts=libs)
+CRAY_LIBS_CLEAN=$(CC --cray-print-opts=libs | sed 's/-Wl,--as-needed,//g; s/,--no-as-needed//g; s/,-l/ -l/g')
+CRAY_LIBS_CLEAN="$CRAY_LIBS_CLEAN $(cc --cray-print-opts=libs | sed 's/-Wl,--as-needed,//g; s/,--no-as-needed//g; s/,-l/ -l/g')"
+CRAY_LIBS_CLEAN="$CRAY_LIBS_CLEAN $(ftn --cray-print-opts=libs | sed 's/-Wl,--as-needed,//g; s/,--no-as-needed//g; s/,-l/ -l/g')"
+
+# Configure and build
+cmake -DCMAKE_INSTALL_PREFIX:PATH=./install \
+      -DCMAKE_C_COMPILER=cc \
+      -DCMAKE_CXX_COMPILER=CC \
+      -DCMAKE_C_FLAGS="$(cc --cray-print-opts=cflags)" \
+      -DCMAKE_CXX_FLAGS="$(CC --cray-print-opts=cflags)" \
+      -DCMAKE_CUDA_FLAGS="$(CC --cray-print-opts=cflags)" \
+      -DCMAKE_CXX_STANDARD_LIBRARIES="-lmpi_gnu_123" \
+      -DCMAKE_CUDA_STANDARD_LIBRARIES="-lmpi_gnu_123" \
+      -DCMAKE_EXE_LINKER_FLAGS="-Wl,--no-as-needed $CRAY_LIBS_CLEAN" \
+      -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \
+      -DCMAKE_BUILD_TYPE:STRING=Release \
+      -DERF_DIM:STRING=3 \
+      -DERF_ENABLE_MPI:BOOL=ON \
+      -DERF_ENABLE_TESTS:BOOL=ON \
+      -DERF_ENABLE_CUDA:BOOL=ON \
+      -DERF_ENABLE_FCOMPARE:BOOL=ON \
+      -DERF_ENABLE_DOCUMENTATION:BOOL=OFF \
+      -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \
+      .. && make -j8
+
+
+
+cmake -DCMAKE_INSTALL_PREFIX:PATH=./install \
+      -DCMAKE_PREFIX_PATH:PATH=${CUDA_HOME}/../../ \
+      -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \
+      -DCMAKE_BUILD_TYPE:STRING=Release \
+      -DERF_DIM:STRING=3 \
+      -DERF_ENABLE_MPI:BOOL=ON \
+      -DERF_ENABLE_CUDA:BOOL=ON \
+      -DERF_ENABLE_NVHPC:BOOL=ON \
+      -DERF_ENABLE_TESTS:BOOL=ON \
+      -DERF_ENABLE_FCOMPARE:BOOL=ON \
+      -DERF_ENABLE_DOCUMENTATION:BOOL=OFF \
+      -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \
+      .. && make -j8