ubuntu-ci-x86_64-gnu.yaml

name: ubuntu-ci-c6a-x86_64-gnu-build
on:
  pull_request:
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash

jobs:
  ubuntu-ci-c6a-x86_64-gnu-build:
    runs-on: [ubuntu-ci-c6a-x86_64]

    steps:
      - name: cleanup
        run: |
          pwd
          ls -lart
          set +e
          find ./* -type d -exec chmod u+xw {} \;
          set -e
          rm -fr *

      - name: checkout
        uses: actions/checkout@v4
        with:
          submodules: true

      - name: prepare-directories
        run: |
          mkdir -p /home/ubuntu/spack-stack/build-cache/
          mkdir -p /home/ubuntu/spack-stack/source-cache/

      - name: create-buildcache
        run: |
          # Get day of week to decide whether to use build caches or not
          DOW=$(date +%u)
          # Monday is 1 ... Sunday is 7
          if [[ $DOW == 7 ]]; then
            export USE_BINARY_CACHE=false
            echo "Ignore existing binary cache for creating buildcache environment"
          else
            export USE_BINARY_CACHE=true
            echo "Use existing binary cache for creating buildcache environment"
          fi

          # Set up spack-stack
          source ./setup.sh

          declare -a TEMPLATES=("unified-dev" "skylab-dev" "cylc-dev")
          for TEMPLATE in "${TEMPLATES[@]}"; do
            if [[ "${TEMPLATE}" == *"unified-dev"* ]]; then
              export ENVNAME=ue-gcc-11.4.0-buildcache
            elif [[ "${TEMPLATE}" == *"skylab-dev"* ]]; then
              export ENVNAME=se-gcc-11.4.0-buildcache
            elif [[ "${TEMPLATE}" == *"cylc-dev"* ]]; then
              export ENVNAME=ce-gcc-11.4.0-buildcache
            fi
            echo "Creating environment ${ENVNAME} from template ${TEMPLATE}"

            export ENVDIR=$PWD/envs/${ENVNAME}
            spack stack create env --site linux.default --template ${TEMPLATE} --name ${ENVNAME} --compiler gcc
            spack env activate ${ENVDIR}
            export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site"

            # Find external packages
            spack external find --scope system \
                --exclude bison --exclude openssl \
                --exclude curl --exclude python \
                --exclude meson
            spack external find --scope system grep
            spack external find --scope system sed
            spack external find --scope system perl
            spack external find --scope system wget
            spack external find --scope system texlive
            spack external find --scope system mysql

            # Find compilers
            spack compiler find --scope system

            export -n SPACK_SYSTEM_CONFIG_PATH

            # For buildcaches
            spack config add config:install_tree:padded_length:200

            # Set compiler and MPI specs
            spack config add "packages:all:providers:mpi:[openmpi@5.0.3]"
            spack config add "packages:all:compiler:[gcc@11.4.0]"

            # Add additional variants for MET packages, different from config/common/packages.yaml
            spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"

            # Concretize and check for duplicates
            spack concretize 2>&1 | tee log.concretize.gnu-11.4.0-buildcache
            ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.gnu-11.4.0-buildcache -i fms -i crtm -i crtm-fix -i esmf -i mapl

            # Add and update source cache
            spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/
            spack mirror create -a -d /home/ubuntu/spack-stack/source-cache/

            # Add binary cache if requested
            if [ "$USE_BINARY_CACHE" = true ] ; then
              set +e
              spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
              spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary)
              set -e
              echo "Packages in spack binary cache:"
              spack buildcache list
            fi

            # Break installation up in pieces and create build caches in between
            # This allows us to "spin up" builds that altogether take longer than
            # six hours, and/or fail later in the build process.

            if [[ "${TEMPLATE}" == *"unified-dev"* || "${TEMPLATE}" == *"skylab-dev"* ]]; then
              # base-env
              echo "base-env ..."
              spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.base-env
              spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env

              # jedi-base-env
              echo "jedi-base-env ..."
              spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-base-env
              spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env

              # jedi-ufs-env
              echo "jedi-ufs-env ..."
              spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-ufs-env
              spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env

            elif [[ "${TEMPLATE}" == *"cylc-dev"* ]]; then
              # Workaround for not being able to install rust from build-cache, see 
              # https://github.com/spack/spack/issues/48971
              echo "rust dependencies ..."
              spack install --verbose --source --no-check-signature --only=dependencies rust 2>&1 | tee log.install.gnu-11.4.0-buildcache.${TEMPLATE}.001.rust-dependencies

              # rust from source
              echo "rust (from source) ..."
              spack install --verbose --source --no-cache rust 2>&1 | tee log.install.gnu-11.4.0-buildcache.${TEMPLATE}.002.rust
            fi

            # the rest
            echo "${TEMPLATE} ..."
            spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.gnu-11.4.0-buildcache.${TEMPLATE}
            spack buildcache create -u /home/ubuntu/spack-stack/build-cache/

            # Remove binary cache for next round of concretization
            if [ "$USE_BINARY_CACHE" = true ] ; then
              spack mirror rm local-binary
            fi

            # Remove buildcache config settings
            spack config remove config:install_tree:padded_length

            # Next steps: synchronize source and build cache to a central/combined mirror?

            # Cleanup
            spack clean -a
            spack env deactivate

          done

      - name: create-env
        run: |
          # Set up spack-stack
          source ./setup.sh
          export BUILDCACHE_ENVNAME=ue-gcc-11.4.0-buildcache
          export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME}
          export ENVNAME=ue-gcc-11.4.0
          export ENVDIR=$PWD/envs/${ENVNAME}
          rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/
          spack env activate ${ENVDIR}

          # Concretize and check for duplicates
          spack concretize --force 2>&1 | tee log.concretize.gnu-11.4.0
          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.gnu-11.4.0 -i fms -i crtm -i crtm-fix -i esmf -i mapl

          # Add binary cache back in
          spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
          echo "Packages in combined spack build caches:"
          spack buildcache list

          # Install from cache
          spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.gnu-11.4.0.unified-env

          # Check shared libraries
          ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check

          # Create modules
          spack clean -a
          spack module tcl refresh -y
          spack stack setup-meta-modules
          spack env deactivate

          # Test environment chaining
          echo "Test environment chaining"
          spack stack create env --name chaintest --template empty --site linux.default --compiler gcc --upstream ${ENVDIR}/install
          # Retain config from upstream so we don't have to rebuild:
          cp -r ${ENVDIR}/{site,common} $PWD/envs/chaintest/.
          spack env activate ${PWD}/envs/chaintest
          spack add nccmp@1.9.0.1%gcc
          spack concretize | tee envs/chaintest/log.concretize
          unwanted_duplicates=$(( cat envs/chaintest/log.concretize | grep -E '^ - ' | grep -Fv 'nccmp@1.9.0.1' || true ) | wc -l)
          if [ ${unwanted_duplicates} -gt 0 ]; then echo "Environment chaining test failed"; exit 1; fi
          spack env deactivate
          echo "Verify 'create env' warnings"
          echo "# nothing" >> ${SPACK_STACK_DIR}/envs/chaintest/site/packages.yaml
          echo "# nothing" >> ${SPACK_STACK_DIR}/envs/chaintest/common/packages.yaml
          spack stack create env --name chaintest3 --site linux.default --compiler gcc --upstream ${SPACK_STACK_DIR}/envs/chaintest/install 2>&1 | tee stderr.txt
          cnt=$(grep -c "WARNING.*do not match" stderr.txt || true)
          if [ $cnt -lt 3 ]; then echo "Missing 'create env' warnings"; exit 1; fi

      - name: test-env
        run: |
          source /etc/profile.d/modules.sh
          module use /home/ubuntu/spack-stack/modulefiles

          export ENVNAME=ue-gcc-11.4.0
          export ENVDIR=$PWD/envs/${ENVNAME}
          ls -l ${ENVDIR}/install/modulefiles/Core

          module use ${ENVDIR}/install/modulefiles/Core
          module load stack-gcc/11.4.0
          module load stack-openmpi/5.0.5
          module load stack-python/3.11.7
          module available

          module load jedi-ufs-env
          module load ewok-env
          module load soca-env
          module list