diff --git a/.github/workflows/upstream-dev-ci.yaml b/.github/workflows/upstream-dev-ci.yaml new file mode 100644 index 0000000..e623697 --- /dev/null +++ b/.github/workflows/upstream-dev-ci.yaml @@ -0,0 +1,174 @@ +name: CI Upstream +on: + push: + branches: + - master + pull_request: + branches: + - master + schedule: + - cron: "0 0 * * *" # Daily “At 00:00” UTC + workflow_dispatch: # allows you to trigger the workflow run manually + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + detect-ci-trigger: + name: detect upstream-dev ci trigger + runs-on: ubuntu-latest + if: | + github.repository == 'intake/intake-xarray' + && (github.event_name == 'push' || github.event_name == 'pull_request') + outputs: + triggered: ${{ steps.detect-trigger.outputs.trigger-found }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 2 + - uses: xarray-contrib/ci-trigger@v1.1 + id: detect-trigger + with: + keyword: "[test-upstream]" + + upstream-dev: + name: upstream-dev + runs-on: ubuntu-latest + needs: detect-ci-trigger + if: | + always() + && ( + (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') + || needs.detect-ci-trigger.outputs.triggered == 'true' + ) + defaults: + run: + shell: bash -l {0} + strategy: + fail-fast: false + matrix: + python-version: ["3.10"] + outputs: + artifacts_availability: ${{ steps.status.outputs.ARTIFACTS_AVAILABLE }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # Fetch all history for all branches and tags. + - uses: conda-incubator/setup-miniconda@v2 + with: + channels: conda-forge + channel-priority: strict + mamba-version: "*" + activate-environment: xarray-tests + auto-update-conda: false + python-version: ${{ matrix.python-version }} + - name: Set up conda environment + run: | + mamba env update -f ci/environment-upstream.yml + bash ci/install-upstream-wheels.sh + - name: Install intake-xarray + run: | + python -m pip install --no-deps -e . + - name: import intake-xarray + run: | + python -c 'import intake_xarray' + - name: Run Tests + if: success() + id: status + run: | + set -euo pipefail + python -m pytest --timeout=60 -rf | tee output-${{ matrix.python-version }}-log || ( + echo '::set-output name=ARTIFACTS_AVAILABLE::true' && false + ) + - name: Upload artifacts + if: | + failure() + && steps.status.outcome == 'failure' + && github.event_name == 'schedule' + && github.repository == 'intake/intake-xarray' + uses: actions/upload-artifact@v2 + with: + name: output-${{ matrix.python-version }}-log + path: output-${{ matrix.python-version }}-log + retention-days: 5 + + report: + name: report + needs: upstream-dev + if: | + failure() + && github.event_name == 'schedule' + && needs.upstream-dev.outputs.artifacts_availability == 'true' + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: "3.x" + - uses: actions/download-artifact@v2 + with: + path: /tmp/workspace/logs + - name: Move all log files into a single directory + run: | + rsync -a /tmp/workspace/logs/output-*/ ./logs + ls -R ./logs + - name: Parse logs + run: | + shopt -s globstar + python .github/workflows/parse_logs.py logs/**/*-log + - name: Report failures + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const pytest_logs = fs.readFileSync('pytest-logs.txt', 'utf8'); + const title = "⚠️ Nightly upstream-dev CI failed ⚠️" + const workflow_url = `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}` + const issue_body = `[Workflow Run URL](${workflow_url})\n${pytest_logs}` + + // Run GraphQL query against GitHub API to find the most recent open issue used for reporting failures + const query = `query($owner:String!, $name:String!, $creator:String!, $label:String!){ + repository(owner: $owner, name: $name) { + issues(first: 1, states: OPEN, filterBy: {createdBy: $creator, labels: [$label]}, orderBy: {field: CREATED_AT, direction: DESC}) { + edges { + node { + body + id + number + } + } + } + } + }`; + + const variables = { + owner: context.repo.owner, + name: context.repo.repo, + label: 'CI', + creator: "github-actions[bot]" + } + const result = await github.graphql(query, variables) + + // If no issue is open, create a new issue, + // else update the body of the existing issue. + if (result.repository.issues.edges.length === 0) { + github.rest.issues.create({ + owner: variables.owner, + repo: variables.name, + body: issue_body, + title: title, + labels: [variables.label] + }) + } else { + github.rest.issues.update({ + owner: variables.owner, + repo: variables.name, + issue_number: result.repository.issues.edges[0].node.number, + body: issue_body + }) + } \ No newline at end of file diff --git a/ci/install-upstream-wheels.sh b/ci/install-upstream-wheels.sh new file mode 100644 index 0000000..a427fc6 --- /dev/null +++ b/ci/install-upstream-wheels.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +conda uninstall -y --force \ + aiohttp \ + flask \ + h5netcdf \ + netcdf4 \ + pydap \ + rasterio \ + tornado \ + scikit-image \ + zarr \ + s3fs \ + moto +# to limit the runtime of Upstream CI +python -m pip install pytest-timeout +python -m pip install \ + --no-deps \ + --upgrade \ + git+https://github.com/aio-libs/aiohttp \ + git+https://github.com/boto/boto3 \ + git+https://github.com/pallets/flask \ + git+https://github.com/h5netcdf/h5netcdf \ + git+https://github.com/Unidata/netcdf4-python \ + git+https://github.com/pydap/pydap \ + git+https://github.com/rasterio/rasterio \ + git+https://github.com/scikit-image/scikit-image \ + git+https://github.com/zarr-developers/zarr-python \ + git+https://github.com/fsspec/s3fs \ + git+https://github.com/spulec/moto \ + git+https://github.com/fsspec/filesystem_spec.git \ + git+https://github.com/pydata/xarray.git \ + git+https://github.com/danvk/RangeHTTPServer.git