Repository navigation
Dof access #373
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: self-hosted.yml | |
| on: | |
| pull_request: | |
| branches: | |
| - develop | |
| # Manual trigger | |
| workflow_dispatch: | |
| jobs: | |
| build_and_test: | |
| # The asics environment is only for launching on self-hosted runners. | |
| # Do not use or edit this environment for any other purpose. | |
| environment: asics | |
| permissions: | |
| contents: read | |
| # The type of runner that the job will run on | |
| runs-on: [self-hosted, linux, x64, gpu] | |
| steps: | |
| # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it | |
| - uses: actions/checkout@v4 | |
| with: | |
| # ref: refs/pull/${{ github.event.issue.number }}/head | |
| submodules: recursive | |
| path: 'pcms_${{ github.event.id }}' #under $GITHUB_WORKSPACE | |
| # note: modules match to https://github.com/LACES-LAB/software-dependencies/blob/main/loads-rhel9.sh | |
| # However, due to risk of running user modifiable code, we repeat modules here. | |
| - name: setup | |
| id: setup | |
| shell: bash | |
| run: | | |
| echo "github.workspace ${{github.workspace}}" | |
| date=`date +%F-%H-%M` | |
| workDir=${{github.workspace}}/pcmsCI_${date} | |
| mkdir -p $workDir | |
| echo "PCMS_WORK_DIR=$workDir" >> $GITHUB_ENV | |
| echo "workDir $workDir" | |
| cat << 'EOF' > $workDir/envGcc13.sh #quotes prevent variable expansion in doc contents | |
| set +e #avoid exiting when lua modules return non-zero on 'warning' messages | |
| source /etc/profile #provides module command | |
| module purge | |
| module use /opt/scorec/spack/rhel9/v0201_4/lmod/linux-rhel9-x86_64/Core/ | |
| module load gcc/12.3.0-iil3lno mpich/4.1.1-xpoyz4t cuda/12.1.1-zxa4msk | |
| module load fftw/3.3.10-qqvl57a | |
| module load cmake/3.26.3-xxgzxlv | |
| #module load openblas/0.3.23-wqm7iud | |
| module load netlib-lapack/3.11.0-b22mgwg #netlib-lapack includes blas | |
| module load netlib-scalapack/2.2.0-fzd4jvl | |
| module load python py-pytest py-numpy | |
| set -e | |
| EOF | |
| - name: build pcms | |
| id: build_pcms_kokkos_cuda | |
| shell: bash | |
| run: | | |
| workDir=$PCMS_WORK_DIR | |
| source $workDir/envGcc13.sh | |
| bdir=${workDir}/build-pcms | |
| export NVCC_WRAPPER_DEFAULT_COMPILER=`which mpicxx` | |
| export PETSC_OPTIONS='-use_gpu_aware_mpi 0' | |
| git clone https://github.com/jacobmerson/pcms_testcases.git ${workDir}/pcms_testcases | |
| # PCMS dependencies are built with https://github.com/LACES-LAB/software-dependencies/blob/main/build-deps-rhel9.sh | |
| # Built artifacts are located in the $DEPENDENCY_DIR | |
| DEPENDENCY_DIR=/users/mersoj2/laces-software/build/ | |
| DEVICE_ARCH=AMPERE80 | |
| # debug output, check if this directory is readable | |
| ls $DEPENDENCY_DIR/${DEVICE_ARCH} | |
| ls $DEPENDENCY_DIR/${DEVICE_ARCH}/redev/install/lib64/cmake/redev/ | |
| echo $DEPENDENCY_DIR/${DEVICE_ARCH}/redev/install/lib64/cmake/redev/ | |
| # Build PCMS without PETSc | |
| cmake -S ${{github.workspace}}/pcms_${{ github.event.id }} -B $bdir \ | |
| -DCMAKE_CXX_COMPILER=`which mpicxx` \ | |
| -DCMAKE_C_COMPILER=`which mpicc` \ | |
| -DCMAKE_Fortran_COMPILER=`which mpifort`\ | |
| -DPCMS_TIMEOUT=20 \ | |
| -DCMAKE_BUILD_TYPE=Release \ | |
| -DCatch2_DIR=$DEPENDENCY_DIR/Catch2/install/lib64/cmake/Catch2/ \ | |
| -Dmeshfields_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/meshFields/install/lib64/cmake/meshfields \ | |
| -DOmega_h_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/omega_h/install/lib64/cmake/Omega_h/ \ | |
| -Dredev_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/redev/install/lib64/cmake/redev/ \ | |
| -DPCMS_TEST_DATA_DIR=${workDir}/pcms_testcases/ \ | |
| -DMPIEXEC_EXECUTABLE=`which mpirun` \ | |
| -DADIOS2_DIR=$DEPENDENCY_DIR/adios2/install/lib64/cmake/adios2/ \ | |
| -Dperfstubs_DIR=$DEPENDENCY_DIR/perfstubs/install/lib/cmake/ \ | |
| -DKokkos_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/kokkos/install/lib64/cmake/Kokkos/ \ | |
| -DKokkosKernels_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/kokkos-kernels/install/lib64/cmake/KokkosKernels/ \ | |
| -DBUILD_TESTING=ON \ | |
| -DPCMS_ENABLE_PETSC=OFF \ | |
| -DPETSC_ARCH="" \ | |
| -DPETSC_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/petsc/install \ | |
| -DPCMS_ENABLE_SPDLOG=OFF | |
| cmake --build $bdir -j 4 | |
| ctest --test-dir $bdir --output-on-failure | |
| # Build pcms with PETSc and python enabled | |
| cmake -S ${{github.workspace}}/pcms_${{ github.event.id }} -B $bdir \ | |
| -DCMAKE_CXX_COMPILER=`which mpicxx` \ | |
| -DCMAKE_C_COMPILER=`which mpicc` \ | |
| -DCMAKE_Fortran_COMPILER=`which mpifort`\ | |
| -DPCMS_TIMEOUT=20 \ | |
| -DCMAKE_BUILD_TYPE=DEBUG \ | |
| -DCatch2_DIR=$DEPENDENCY_DIR/Catch2/install/lib64/cmake/Catch2/ \ | |
| -Dmeshfields_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/meshFields/install/lib64/cmake/meshfields \ | |
| -DOmega_h_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/omega_h/install/lib64/cmake/Omega_h/ \ | |
| -Dredev_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/redev/install/lib64/cmake/redev/ \ | |
| -DPCMS_TEST_DATA_DIR=${workDir}/pcms_testcases/ \ | |
| -DMPIEXEC_EXECUTABLE=`which mpirun` \ | |
| -DADIOS2_DIR=$DEPENDENCY_DIR/adios2/install/lib64/cmake/adios2/ \ | |
| -Dperfstubs_DIR=$DEPENDENCY_DIR/perfstubs/install/lib/cmake/ \ | |
| -DKokkos_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/kokkos/install/lib64/cmake/Kokkos/ \ | |
| -DKokkosKernels_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/kokkos-kernels/install/lib64/cmake/KokkosKernels/ \ | |
| -DBUILD_TESTING=ON \ | |
| -DPCMS_ENABLE_Python=ON \ | |
| -DPCMS_ENABLE_PETSC=ON \ | |
| -DPETSC_ARCH="" \ | |
| -DPETSC_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/petsc/install \ | |
| -Dpybind11_ROOT=$DEPENDENCY_DIR/pybind11/install \ | |
| -DPCMS_ENABLE_SPDLOG=OFF | |
| cmake --build $bdir -j 4 | |
| ctest --test-dir $bdir --output-on-failure | |
| export PYTHONPATH=$DEPENDENCY_DIR/${DEVICE_ARCH}/omega_h/install/lib64/python./site-packages:$bdir/src/pcms/pythonapi:$PYTHONPATH | |
| export LD_LIBRARY_PATH=$DEPENDENCY_DIR/${DEVICE_ARCH}/omega_h/install/lib64:$LD_LIBRARY_PATH | |
| # The examples need Exodus data, which is stored with Git LFS | |
| # and is not fetched by the clone above, so use the copy shared with | |
| # the dependencies. We can resolve this when git-lfs installed on SCOREC asics | |
| export PCMS_TEST_DATA_DIR=/users/mersoj2/laces-software/sources/pcms_testcases | |
| export PCMS_REQUIRE_EXAMPLES=1 | |
| pytest -v -p no:cacheprovider ${{github.workspace}}/pcms_${{ github.event.id }}/test/python | |
| - name: build pcms (Kokkos OpenMP) | |
| id: build_pcms_kokkos_openmp | |
| shell: bash | |
| run: | | |
| workDir=$PCMS_WORK_DIR | |
| source $workDir/envGcc13.sh | |
| bdir=${workDir}/build-pcms-openmp | |
| # The NATIVE dependencies use the Kokkos OpenMP (host) backend. | |
| DEPENDENCY_DIR=/users/mersoj2/laces-software/build/ | |
| DEVICE_ARCH=NATIVE | |
| # Use more than one thread so threading bugs can show up. | |
| export OMP_NUM_THREADS=2 | |
| export OMP_PROC_BIND=false | |
| cmake -S ${{github.workspace}}/pcms_${{ github.event.id }} -B $bdir \ | |
| -DCMAKE_CXX_COMPILER=`which mpicxx` \ | |
| -DCMAKE_C_COMPILER=`which mpicc` \ | |
| -DCMAKE_Fortran_COMPILER=`which mpifort`\ | |
| -DPCMS_TIMEOUT=20 \ | |
| -DCMAKE_BUILD_TYPE=Release \ | |
| -DCatch2_DIR=$DEPENDENCY_DIR/Catch2/install/lib64/cmake/Catch2/ \ | |
| -Dmeshfields_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/meshFields/install/lib64/cmake/meshfields \ | |
| -DOmega_h_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/omega_h/install/lib64/cmake/Omega_h/ \ | |
| -Dredev_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/redev/install/lib64/cmake/redev/ \ | |
| -DPCMS_TEST_DATA_DIR=${workDir}/pcms_testcases/ \ | |
| -DMPIEXEC_EXECUTABLE=`which mpirun` \ | |
| -DADIOS2_DIR=$DEPENDENCY_DIR/adios2/install/lib64/cmake/adios2/ \ | |
| -Dperfstubs_DIR=$DEPENDENCY_DIR/perfstubs/install/lib/cmake/ \ | |
| -DKokkos_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/kokkos/install/lib64/cmake/Kokkos/ \ | |
| -DKokkosKernels_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/kokkos-kernels/install/lib64/cmake/KokkosKernels/ \ | |
| -DBUILD_TESTING=ON \ | |
| -DPCMS_ENABLE_Python=ON \ | |
| -DPCMS_ENABLE_PETSC=ON \ | |
| -DPETSC_ARCH="" \ | |
| -DPETSC_DIR=$DEPENDENCY_DIR/${DEVICE_ARCH}/petsc/install \ | |
| -Dpybind11_ROOT=$DEPENDENCY_DIR/pybind11/install \ | |
| -DPCMS_ENABLE_SPDLOG=OFF | |
| cmake --build $bdir -j 4 | |
| ctest --test-dir $bdir --output-on-failure | |
| export PYTHONPATH=$DEPENDENCY_DIR/${DEVICE_ARCH}/omega_h/install/lib64/python./site-packages:$bdir/src/pcms/pythonapi:$PYTHONPATH | |
| export LD_LIBRARY_PATH=$DEPENDENCY_DIR/${DEVICE_ARCH}/omega_h/install/lib64:$LD_LIBRARY_PATH | |
| pytest -v -p no:cacheprovider ${{github.workspace}}/pcms_${{ github.event.id }}/test/python | |
| - name: Save Result Link | |
| if: ${{ !cancelled() }} #prepare report unless the job was cancelled | |
| run: | | |
| mkdir -p ./pr | |
| echo "${{ github.event.id }}" > ./pr/issueNumber | |
| echo "Test Results:" > ./pr/message | |
| echo "" >> ./pr/message | |
| echo "[(details)](https://github.com/${{github.repository}}/actions/runs/${{ github.run_id }})" >> ./pr/message | |
| - name: Cleanup | |
| if: ${{ !cancelled() }} | |
| run: | | |
| echo "PCMS_WORK_DIR $PCMS_WORK_DIR" | |
| rm -rf $PCMS_WORK_DIR |