self-hosted.yml

name: self-hosted.yml

on:
  pull_request:
    branches:
      - develop
  # Manual trigger
  workflow_dispatch:

jobs:
  build_and_test:
    # The asics environment is only for launching on self-hosted runners.
    # Do not use or edit this environment for any other purpose.
    environment: asics
    permissions:
      contents: read
    # The type of runner that the job will run on
    runs-on: [self-hosted, linux, x64, gpu]

    steps:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@v4
        with:
          # ref: refs/pull/${{ github.event.issue.number }}/head
          submodules: recursive
          path: 'pcms_${{ github.event.id }}' #under $GITHUB_WORKSPACE

      # note: modules match to https://github.com/LACES-LAB/software-dependencies/blob/main/loads-rhel9.sh
      # However, due to risk of running user modifiable code, we repeat modules here.
      - name: setup
        id: setup
        shell: bash
        run: |
          echo "github.workspace ${{github.workspace}}"
          date=`date +%F-%H-%M`
          workDir=${{github.workspace}}/pcmsCI_${date}
          mkdir -p $workDir
          echo "PCMS_WORK_DIR=$workDir" >> $GITHUB_ENV
          echo "workDir $workDir"
          cat << 'EOF' > $workDir/envGcc13.sh #quotes prevent variable expansion in doc contents
          set +e #avoid exiting when lua modules return non-zero on 'warning' messages
          source /etc/profile #provides module command
          module purge
          module use /opt/scorec/spack/rhel9/v0201_4/lmod/linux-rhel9-x86_64/Core/
          module load gcc/12.3.0-iil3lno mpich/4.1.1-xpoyz4t cuda/12.1.1-zxa4msk
          module load fftw/3.3.10-qqvl57a
          module load cmake/3.26.3-xxgzxlv
          #module load openblas/0.3.23-wqm7iud
          module load netlib-lapack/3.11.0-b22mgwg #netlib-lapack includes blas
          module load netlib-scalapack/2.2.0-fzd4jvl
          module load python py-pytest py-numpy
          set -e
          EOF

      - name: build pcms
        id: build_pcms_kokkos_cuda
        shell: bash
        run: |
          workDir=$PCMS_WORK_DIR
          source $workDir/envGcc13.sh
          bdir=${workDir}/build-pcms

          export NVCC_WRAPPER_DEFAULT_COMPILER=`which mpicxx`
          export PETSC_OPTIONS='-use_gpu_aware_mpi 0'

          git clone https://github.com/jacobmerson/pcms_testcases.git ${workDir}/pcms_testcases
          # PCMS dependencies are built with https://github.com/LACES-LAB/software-dependencies/blob/main/build-deps-rhel9.sh
          # Built artifacts are located in the $DEPENDENCY_DIR
          DEPENDENCY_DIR=/users/mersoj2/laces-software/build/
          DEVICE_ARCH=AMPERE80
          # debug output, check if this directory is readable
          ls $DEPENDENCY_DIR/${DEVICE_ARCH}
          ls $DEPENDENCY_DIR/${DEVICE_ARCH}/redev/install/lib64/cmake/redev/
          echo $DEPENDENCY_DIR/${DEVICE_ARCH}/redev/install/lib64/cmake/redev/

          # Build PCMS without PETSc
          cmake -S ${{github.workspace}}/pcms_${{ github.event.id }} -B $bdir \
            -DCMAKE_CXX_COMPILER=`which mpicxx` \
            -DCMAKE_C_COMPILER=`which mpicc` \
            -DCMAKE_Fortran_COMPILER=`which mpifort`\
            -DPCMS_TIMEOUT=20 \
            -DCMAKE_BUILD_TYPE=Release \
            -DCatch2_DIR=$DEPENDENCY_DIR/Catch2/install/lib64/cmake/Catch2/ \
            -Dmeshfields_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/meshFields/install/lib64/cmake/meshfields \
            -DOmega_h_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/omega_h/install/lib64/cmake/Omega_h/ \
            -Dredev_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/redev/install/lib64/cmake/redev/ \
            -DPCMS_TEST_DATA_DIR=${workDir}/pcms_testcases/ \
            -DMPIEXEC_EXECUTABLE=`which mpirun` \
            -DADIOS2_DIR=$DEPENDENCY_DIR/adios2/install/lib64/cmake/adios2/ \
            -Dperfstubs_DIR=$DEPENDENCY_DIR/perfstubs/install/lib/cmake/ \
            -DKokkos_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/kokkos/install/lib64/cmake/Kokkos/ \
            -DKokkosKernels_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/kokkos-kernels/install/lib64/cmake/KokkosKernels/ \
            -DBUILD_TESTING=ON \
            -DPCMS_ENABLE_PETSC=OFF \
            -DPETSC_ARCH="" \
            -DPETSC_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/petsc/install \
            -DPCMS_ENABLE_SPDLOG=OFF

          cmake --build $bdir -j 4
          ctest --test-dir $bdir --output-on-failure

          # Build pcms with PETSc and python enabled
          cmake -S ${{github.workspace}}/pcms_${{ github.event.id }} -B $bdir \
            -DCMAKE_CXX_COMPILER=`which mpicxx` \
            -DCMAKE_C_COMPILER=`which mpicc` \
            -DCMAKE_Fortran_COMPILER=`which mpifort`\
            -DPCMS_TIMEOUT=20 \
            -DCMAKE_BUILD_TYPE=DEBUG \
            -DCatch2_DIR=$DEPENDENCY_DIR/Catch2/install/lib64/cmake/Catch2/ \
            -Dmeshfields_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/meshFields/install/lib64/cmake/meshfields \
            -DOmega_h_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/omega_h/install/lib64/cmake/Omega_h/ \
            -Dredev_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/redev/install/lib64/cmake/redev/ \
            -DPCMS_TEST_DATA_DIR=${workDir}/pcms_testcases/ \
            -DMPIEXEC_EXECUTABLE=`which mpirun` \
            -DADIOS2_DIR=$DEPENDENCY_DIR/adios2/install/lib64/cmake/adios2/ \
            -Dperfstubs_DIR=$DEPENDENCY_DIR/perfstubs/install/lib/cmake/ \
            -DKokkos_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/kokkos/install/lib64/cmake/Kokkos/ \
            -DKokkosKernels_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/kokkos-kernels/install/lib64/cmake/KokkosKernels/ \
            -DBUILD_TESTING=ON \
            -DPCMS_ENABLE_Python=ON \
            -DPCMS_ENABLE_PETSC=ON \
            -DPETSC_ARCH="" \
            -DPETSC_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/petsc/install \
            -Dpybind11_ROOT=$DEPENDENCY_DIR/pybind11/install \
            -DPCMS_ENABLE_SPDLOG=OFF

          cmake --build $bdir -j 4
          ctest --test-dir $bdir --output-on-failure

          export PYTHONPATH=$DEPENDENCY_DIR/${DEVICE_ARCH}/omega_h/install/lib64/python./site-packages:$bdir/src/pcms/pythonapi:$PYTHONPATH
          export LD_LIBRARY_PATH=$DEPENDENCY_DIR/${DEVICE_ARCH}/omega_h/install/lib64:$LD_LIBRARY_PATH
          # The examples need Exodus data, which is stored with Git LFS
          # and is not fetched by the clone above, so use the copy shared with
          # the dependencies. We can resolve this when git-lfs installed on SCOREC asics
          export PCMS_TEST_DATA_DIR=/users/mersoj2/laces-software/sources/pcms_testcases
          export PCMS_REQUIRE_EXAMPLES=1
          pytest -v -p no:cacheprovider ${{github.workspace}}/pcms_${{ github.event.id }}/test/python

      - name: build pcms (Kokkos OpenMP)
        id: build_pcms_kokkos_openmp
        shell: bash
        run: |
          workDir=$PCMS_WORK_DIR
          source $workDir/envGcc13.sh
          bdir=${workDir}/build-pcms-openmp

          # The NATIVE dependencies use the Kokkos OpenMP (host) backend.
          DEPENDENCY_DIR=/users/mersoj2/laces-software/build/
          DEVICE_ARCH=NATIVE
          # Use more than one thread so threading bugs can show up.
          export OMP_NUM_THREADS=2
          export OMP_PROC_BIND=false

          cmake -S ${{github.workspace}}/pcms_${{ github.event.id }} -B $bdir \
            -DCMAKE_CXX_COMPILER=`which mpicxx` \
            -DCMAKE_C_COMPILER=`which mpicc` \
            -DCMAKE_Fortran_COMPILER=`which mpifort`\
            -DPCMS_TIMEOUT=20 \
            -DCMAKE_BUILD_TYPE=Release \
            -DCatch2_DIR=$DEPENDENCY_DIR/Catch2/install/lib64/cmake/Catch2/ \
            -Dmeshfields_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/meshFields/install/lib64/cmake/meshfields \
            -DOmega_h_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/omega_h/install/lib64/cmake/Omega_h/ \
            -Dredev_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/redev/install/lib64/cmake/redev/ \
            -DPCMS_TEST_DATA_DIR=${workDir}/pcms_testcases/ \
            -DMPIEXEC_EXECUTABLE=`which mpirun` \
            -DADIOS2_DIR=$DEPENDENCY_DIR/adios2/install/lib64/cmake/adios2/ \
            -Dperfstubs_DIR=$DEPENDENCY_DIR/perfstubs/install/lib/cmake/ \
            -DKokkos_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/kokkos/install/lib64/cmake/Kokkos/ \
            -DKokkosKernels_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/kokkos-kernels/install/lib64/cmake/KokkosKernels/ \
            -DBUILD_TESTING=ON \
            -DPCMS_ENABLE_Python=ON \
            -DPCMS_ENABLE_PETSC=ON \
            -DPETSC_ARCH="" \
            -DPETSC_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/petsc/install \
            -Dpybind11_ROOT=$DEPENDENCY_DIR/pybind11/install \
            -DPCMS_ENABLE_SPDLOG=OFF

          cmake --build $bdir -j 4
          ctest --test-dir $bdir --output-on-failure

          export PYTHONPATH=$DEPENDENCY_DIR/${DEVICE_ARCH}/omega_h/install/lib64/python./site-packages:$bdir/src/pcms/pythonapi:$PYTHONPATH
          export LD_LIBRARY_PATH=$DEPENDENCY_DIR/${DEVICE_ARCH}/omega_h/install/lib64:$LD_LIBRARY_PATH
          pytest -v -p no:cacheprovider ${{github.workspace}}/pcms_${{ github.event.id }}/test/python


      - name: Save Result Link
        if: ${{ !cancelled() }} #prepare report unless the job was cancelled
        run: |
          mkdir -p ./pr
          echo "${{ github.event.id }}"  > ./pr/issueNumber
          echo "Test Results:" > ./pr/message
          echo "" >> ./pr/message
          echo "[(details)](https://github.com/${{github.repository}}/actions/runs/${{ github.run_id }})" >> ./pr/message

      - name: Cleanup
        if: ${{ !cancelled() }}
        run: |
          echo "PCMS_WORK_DIR $PCMS_WORK_DIR"
          rm -rf $PCMS_WORK_DIR