pnnl · nkoukpaizan · Jan 7, 2025 · Jan 7, 2025 · Jan 8, 2025 · Jan 9, 2025
diff --git a/.github/workflows/pnnl_mirror.yaml b/.github/workflows/pnnl_mirror.yaml
@@ -2,11 +2,11 @@ name: PNNL Mirror
 
 # triggers a github action everytime there is a push or mr
 on:
-  pull_request:
-  push:
-    branches:
-      - develop
-      - main
+  #  pull_request:
+  #  push:
+  #    branches:
+  #      - develop
+  #      - main
 
 jobs:
   # To test on HPC resources we must first mirror the repo and then trigger a pipeline

diff --git a/.github/workflows/spack_cpu_build.yaml b/.github/workflows/spack_cpu_build.yaml
@@ -7,31 +7,19 @@ env:
   # Our repo name contains upper case characters, so we can't use ${{ github.repository }}
   IMAGE_NAME: exago
   USERNAME: exago-bot
-  BASE_VERSION: ubuntu-24.04-fortran
-  SPACK_CACHE: /opt/spack-cache
-  tempdir: /opt/spack-cache
-  TMP: /opt/spack-cache
-  TMPDIR: /opt/spack-cache
+  BASE_VERSION: ubuntu-24.04-fortran-v0.0.1
 
 # Until we remove the need to clone submodules to build, this should on be in PRs
 on: [pull_request]
 
 jobs:
   base_image_build:
+    name: Build Custom Base Image
     runs-on: ubuntu-24.04
     permissions:
-      packages: write
       contents: read
-
-    name: Build Custom Base Image
+      packages: write
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          # Once we move submodule deps into spack, we can do some more builds
-          # Also need to change build script to use spack from base image
-          submodules: true
-
       # No GHCR base image with skopeo, so this will do...
       - name: "Set up skopeo"
         uses: warjiang/[email protected]
@@ -52,6 +40,14 @@ jobs:
             > /dev/null && echo "Image already exists. Please bump version." && exit 0
           echo "IMAGE_EXISTS=false" >> $GITHUB_ENV
 
+      # https://docs.github.com/en/actions/publishing-packages/publishing-docker-images
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ env.USERNAME }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       # Need to build custom base image with gfortran
       - name: Create Dockerfile heredoc
         if: ${{ env.IMAGE_EXISTS == 'false' }}
@@ -73,14 +69,6 @@ jobs:
               && rm -rf /var/lib/apt/lists/*
           EOF
 
-      # https://docs.github.com/en/actions/publishing-packages/publishing-docker-images
-      - name: Log in to the Container registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ env.USERNAME }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Extract metadata (tags, labels) for Docker
         if: ${{ env.IMAGE_EXISTS == 'false' }}
         id: meta
@@ -104,6 +92,7 @@ jobs:
     permissions:
       packages: write
       contents: read
+    timeout-minutes: 120
 
     strategy:
       matrix:
@@ -118,7 +107,8 @@ jobs:
           # - exago@develop~mpi~ipopt+hiop~python+raja
           # See #16 - +python~mpi causes issues
           # - exago@develop~mpi~ipopt+hiop+python~raja
-          - exago@develop+mpi~ipopt+hiop+python~raja ^openmpi
+          # Python build in GH actions is failing. Using system python.
+          # - exago@develop+mpi~ipopt+hiop+python~raja ^openmpi
           # See #40 - +hiop+raja~ipopt ^hiop~sparse is useful for edge cases
           - exago@develop+mpi~ipopt+hiop~python+raja ^openmpi ^hiop+raja~sparse
 
@@ -143,15 +133,12 @@ jobs:
             concretizer:
               reuse: dependencies
             config:
-              source_cache: $SPACK_CACHE/source_cache
-              misc_cache: $SPACK_CACHE/misc_cache
-              build_stage: $SPACK_CACHE/build_stage
               install_tree:
                 root: /opt/spack
                 padded_length: False
             mirrors:
               local-buildcache: oci://${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-            # spack: https://binaries.spack.io/develop
+              spack: https://binaries.spack.io/develop
             packages:
               all:
                 require: "%gcc"
@@ -164,7 +151,7 @@ jobs:
         run: spack -e . buildcache keys --install --trust
 
       - name: Find external packages
-        run: spack -e . external find --all --exclude python
+        run: spack -e . external find --all
 
       - name: Spack develop exago
         run: spack -e . develop --path=$(pwd) exago@develop
@@ -175,8 +162,8 @@ jobs:
       - name: Install
         run: spack -e . install --no-check-signature --fail-fast --keep-stage
 
-      - name: Test Build
-        run: cd $(spack -e . location --build-dir exago@develop) && ctest -VV
+        #- name: Test Build
+        #  run: cd $(spack -e . location --build-dir exago@develop) && ctest -VV
 
       # Push with force to override existing binaries...
       - name: Push to binaries to buildcache

diff --git a/buildsystem/README.md b/buildsystem/README.md
@@ -17,11 +17,11 @@ Each folder which builds a configuration of ExaGO should have a following:
 
 Platforms:
 
-- Crusher
+- frontier
 
 Description:
 
-Crusher clang build of exago@crusher-dev + hiop@develop
+frontier clang build of exago@frontier-dev + hiop@develop
 
 ### clang-omp
 
@@ -52,11 +52,9 @@ https://gitlab.pnnl.gov/exasgd/frameworks/exago/-/commit/47ea09e648dfa81ca8a70cc
 ### gcc-cuda
 
 Platforms:
-- Ascent
 - Deception
 - Marianas
 - Newell
-- Summit
 
 Description:
 

diff --git a/buildsystem/build.sh b/buildsystem/build.sh
@@ -221,7 +221,16 @@ module purge
 varfile="$SRCDIR/buildsystem/$JOB/$(echo $MY_CLUSTER)Variables.sh"
 
 if [[ -f "$varfile" ]]; then
-  source $varfile || { echo "Could not source $varfile"; exit 1; }
+  source $varfile
+  if [ $? ]; then
+    if [[ $MY_CLUSTER==frontier ]]; then
+      echo "Allowing non-zero exit code for $varfile."
+      echo "Frontier modules are currently generating warnings that will go away in future updates."
+    else
+      echo "Could not source $varfile"; 
+      exit 1;
+    fi
+  fi
 fi
 
 # module list

diff --git a/buildsystem/clang-hip/crusher/crusherExago.sh b/buildsystem/clang-hip/crusher/crusherExago.sh
diff --git a/buildsystem/clang-hip/crusher/crusherOptimizedExago.sh b/buildsystem/clang-hip/crusher/crusherOptimizedExago.sh
diff --git a/buildsystem/clang-hip/crusher/crusherOptimizedVariables.sh b/buildsystem/clang-hip/crusher/crusherOptimizedVariables.sh
diff --git a/buildsystem/clang-hip/crusherVariables.sh b/buildsystem/clang-hip/crusherVariables.sh
diff --git a/buildsystem/clang-hip/crusher/base.sh → buildsystem/clang-hip/frontier/base.sh b/buildsystem/clang-hip/crusher/base.sh → buildsystem/clang-hip/frontier/base.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-export MY_CLUSTER=crusher
+export MY_CLUSTER=frontier
 export PROJ_DIR=/autofs/nccs-svm1_proj/eng145
 
 module reset
@@ -10,16 +10,16 @@ module load PrgEnv-gnu-amd
 module load cpe/23.12
 module load craype-x86-trento
 module load craype-accel-amd-gfx90a
-module load amd-mixed/5.7.1
-module load rocm/5.7.1
+module load amd-mixed/6.3.1
+module load rocm/6.3.1
 module load gcc-native/12.3
 module load cray-mpich/8.1.28
 module load libfabric
 
 # Consider changing to $(which clang) as for deception
-export CC=/opt/rocm-5.7.1/llvm/bin/amdclang
-export CXX=/opt/rocm-5.7.1/llvm/bin/amdclang++
-export FC=/opt/rocm-5.7.1/llvm/bin/amdflang
+export CC=/opt/rocm-6.3.1/llvm/bin/amdclang
+export CXX=/opt/rocm-6.3.1/llvm/bin/amdclang++
+export FC=/opt/rocm-6.3.1/llvm/bin/amdflang
 
 export EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -DEXAGO_CTEST_LAUNCH_COMMAND='srun'"
 export EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -DAMDGPU_TARGETS='gfx90a'"
diff --git a/buildsystem/clang-hip/frontier/frontierExago.sh b/buildsystem/clang-hip/frontier/frontierExago.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+export SRCDIR=${SRCDIR:-$PWD}
+
+# Platform specific configuration
+source $SRCDIR/buildsystem/clang-hip/frontier/base.sh
+
+# Spack modules
+source $SRCDIR/buildsystem/spack/frontier/modules/exago.sh
diff --git a/buildsystem/clang-hip/frontier/frontierOptimizedExago.sh b/buildsystem/clang-hip/frontier/frontierOptimizedExago.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+export SRCDIR=${SRCDIR:-$PWD}
+
+# Platform specific configuration
+source $SRCDIR/buildsystem/clang-hip/frontier/base.sh
+
+# Spack modules
+source $SRCDIR/buildsystem/spack/frontier/modules/exago-optimized.sh
diff --git a/buildsystem/clang-hip/frontier/frontierOptimizedVariables.sh b/buildsystem/clang-hip/frontier/frontierOptimizedVariables.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+export SRCDIR=${SRCDIR:-$PWD}
+
+# Platform specific configuration
+source $SRCDIR/buildsystem/clang-hip/frontier/base.sh
+
+# Spack modules
+source $SRCDIR/buildsystem/spack/frontier/modules/optimized-dependencies.sh 
diff --git a/buildsystem/clang-hip/frontierVariables.sh b/buildsystem/clang-hip/frontierVariables.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+SRCDIR=${SRCDIR:-$PWD}
+
+# Platform specific configuration
+source $SRCDIR/buildsystem/clang-hip/frontier/base.sh
+
+# Spack modules
+source $SRCDIR/buildsystem/spack/frontier/modules/dependencies.sh 
diff --git a/buildsystem/gcc-cuda/ascent/base.sh b/buildsystem/gcc-cuda/ascent/base.sh
diff --git a/buildsystem/gcc-cuda/ascentVariables.sh b/buildsystem/gcc-cuda/ascentVariables.sh
diff --git a/buildsystem/gcc-cuda/summit/base.sh b/buildsystem/gcc-cuda/summit/base.sh
diff --git a/buildsystem/gcc-cuda/summitVariables.sh b/buildsystem/gcc-cuda/summitVariables.sh
diff --git a/buildsystem/spack/README.md b/buildsystem/spack/README.md
@@ -124,7 +124,7 @@ in `configure_modules.sh` that aligns with that is specified in the environment.
 These modules are used in `buildsystem/<build_type>/` in various scripts for each
 platform, and you should ensure these are compatible with the module configuration
 used for each platform. Make sure to create variable files that are consistent
-with examples for `gcc-cuda/newell` and `clang-hip/crusher`.
+with examples for `gcc-cuda/newell` and `clang-hip/frontier`.
 
 If you have an update in the ExaGO/HiOp spack package, you might need to update
 the relevant `spack.yaml` and configure module scripts for each platform.