diff --git a/.github/workflows/create_cache.yml b/.github/workflows/create_cache.yml new file mode 100644 index 00000000..530e2af4 --- /dev/null +++ b/.github/workflows/create_cache.yml @@ -0,0 +1,73 @@ +name: "Create GHA cache" + +# GitHub puts the following restrictions on cache sharing. PRs can access +# +# - caches that were created by the PR / earlier runs of the PR +# - caches that were created on the target branch +# +# To get effective cache sharing between PRs, we create caches on the `develop` +# branch (which is where almost all PRs merge into). + +on: + push: + branches: [develop] + +# Cancel running jobs if there's a newer push +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # GitHub Actions cache of the pre-commit environment + pre-commit: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Setup Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - uses: actions/cache@v4 + id: cache + with: + path: ~/.cache/pre-commit + key: pre-commit_${{ env.pythonLocation }}_${{ hashFiles('.pre-commit-config.yaml') }} + lookup-only: true # don't actually download the cache + + - name: Populate pre-commit environment (if not cached) + if: steps.cache.outputs.cache-hit != 'true' + run: | + pip install pre-commit + pre-commit install --install-hooks + + # GitHub Actions cache of pySHiELD test data + test-data: + runs-on: ubuntu-latest + env: + TEST_DATA_PATH: "./test_data/8.1.3/c12_6ranks_baroclinic/physics" + TEST_DATA_URL: "https://portal.nccs.nasa.gov/datashare/astg/smt/pace-regression-data/8.1.3_c12_6ranks_baroclinic.physics.tar.gz" + + steps: + - uses: actions/checkout@v5 + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - uses: actions/cache@v4 + id: cache + with: + path: pySHiELD/test_data + key: ${{ env.TEST_DATA_PATH }} + lookup-only: true # don't actually download the cache + + - name: Download test data (if not already cached) + if: steps.cache-restore.outputs.cache-hit != 'true' + run: | + mkdir -p pySHiELD/test_data && cd pySHiELD/test_data + wget ${{ env.TEST_DATA_URL }} + tar -xzvf 8.1.3_c12_6ranks_baroclinic.physics.tar.gz + rm 8.1.3_c12_6ranks_baroclinic.physics.tar.gz diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 2a60c1d9..eb744c5e 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -22,6 +22,13 @@ jobs: with: python-version: '3.11' + # Only restore (don't save) caches on PRs. New caches created from PRs won't be + # accessible from other PRs, see workflows/create_cache.yaml. + - uses: actions/cache/restore@v4 + with: + path: ~/.cache/pre-commit + key: pre-commit_${{ env.pythonLocation }}_${{ hashFiles('.pre-commit-config.yaml') }} + - name: Install pre-commit run: pip install pre-commit diff --git a/.github/workflows/translate.yaml b/.github/workflows/translate.yaml index 95a02866..841c7569 100644 --- a/.github/workflows/translate.yaml +++ b/.github/workflows/translate.yaml @@ -26,6 +26,10 @@ concurrency: jobs: pyshield_translate_tests: runs-on: ubuntu-latest + env: + TEST_DATA_PATH: "./test_data/8.1.3/c12_6ranks_baroclinic/physics" + TEST_DATA_URL: "https://portal.nccs.nasa.gov/datashare/astg/smt/pace-regression-data/8.1.3_c12_6ranks_baroclinic.physics.tar.gz" + steps: - name: External trigger Checkout pySHiELD if: ${{inputs.component_trigger}} @@ -91,13 +95,21 @@ jobs: pip install numpy==1.26.4 conda remove sqlite - - name: Download data + # Only restore (don't save) caches on PRs. New caches created from PRs won't be + # accessible from other PRs, see workflows/create_cache.yaml. + - name: Restore test_data (if cached) + id: cache-restore + uses: actions/cache/restore@v4 + with: + key: ${{ env.TEST_DATA_PATH }} + path: pySHiELD/test_data + + - name: Download data (if not cached) + if: steps.cache-restore.outputs.cache-hit != 'true' shell: bash -l {0} run: | - cd ${GITHUB_WORKSPACE}/pySHiELD - mkdir -p test_data - cd test_data - wget https://portal.nccs.nasa.gov/datashare/astg/smt/pace-regression-data/8.1.3_c12_6ranks_baroclinic.physics.tar.gz + mkdir -p pySHiELD/test_data && cd pySHiELD/test_data + wget ${{ env.TEST_DATA_URL }} tar -xzvf 8.1.3_c12_6ranks_baroclinic.physics.tar.gz - name: Numpy Translate Test @@ -105,7 +117,7 @@ jobs: run: | cd ${GITHUB_WORKSPACE}/pySHiELD pytest \ - -v -s --data_path=./test_data/8.1.3/c12_6ranks_baroclinic/physics \ + -v -s --data_path=${{ env.TEST_DATA_PATH }} \ --backend=numpy \ --threshold_overrides_file=./tests/savepoint/translate/overrides/standard.yaml \ ./tests/savepoint @@ -119,7 +131,7 @@ jobs: export OMP_NUM_THREADS=10 export PACE_LOGLEVEL=Debug pytest \ - -vvv -x -s --data_path=./test_data/8.1.3/c12_6ranks_baroclinic/physics \ + -vvv -x -s --data_path=${{ env.TEST_DATA_PATH }} \ --backend=dace:cpu \ --threshold_overrides_file=./tests/savepoint/translate/overrides/standard.yaml \ ./tests/savepoint