whirl-ci.yml

name: execute whirl ci for examples
on:
  push:
    branches:
      - master
  pull_request:

jobs:
  shellcheck: # Job that list subdirectories of ./examples
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: run shellcheck
        uses: sudo-bot/action-shellcheck@latest
        with:
          # https://github.com/koalaman/shellcheck#how-to-use
          cli-args: "-x whirl"

  directories: # Job that list subdirectories of ./examples
    runs-on: ubuntu-latest
    outputs:
      # generate output name dir by using inner step output
      dir: ${{ steps.setdirs.outputs.dir }}
    steps:
      - uses: actions/checkout@v2
      - id: setdirs # Give it an id to handle to get step outputs in the outputs key above
        # run: echo "::set-output name=dir::$(ls -d ./examples/* | jq -R -s -c 'split("\n")[:-1]')"
        run: echo "dir=$(ls -d ./examples/* | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
        # Define step output named dir based on ls command transformed to JSON thanks to jq

  examples: # Job that list subdirectories of ./examples
    runs-on: ubuntu-latest
    outputs:
      # generate output name example by using inner step output
      example: ${{ steps.setexamples.outputs.example }}
    steps:
      - uses: actions/checkout@v2
      - id: setexamples # Give it an id to handle to get step outputs in the outputs key above
        run: echo "example=$(ls -d ./examples/* | sed -r 's/\.\/examples\/(.*)/\1/g' | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
        # Define step output named example based on ls command transformed to JSON thanks to jq

  whirl-ci-default-envs:
    needs: [directories]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      max-parallel: 4
      matrix:
        example_dir: ${{ fromJson(needs.directories.outputs.dir) }}
        python_version: ["3.9", "3.12"]
        airflow_version: ["2.2.5", "2.9.1"]
        exclude:
          - airflow_version: "2.2.5"
            python_version: "3.12"
          # Needs more memory than available on the runner
          - example_dir: ./examples/dbt-spark-example
          - example_dir: ./examples/spark-delta-sharing
          - example_dir: ./examples/spark-s3-to-hive
          # No default dag available for CI
          - example_dir: ./examples/airflow-datasets
          # Run without parallelism separately
          - example_dir: ./examples/dbt-example
          # Requires artifact downloaded from github.com/godatadriven/spot, which doesn't publish artifacts yet (2024-12-11)
          - example_dir: ./examples/spark-opentelemetry
    env:
      PYTHON_VERSION: ${{ matrix.python_version }}
      AIRFLOW_VERSION: ${{ matrix.airflow_version }}
    steps:
      - uses: actions/checkout@v2
      - name: Run whirl CI ${{ matrix.example_dir }}
        working-directory: ${{ matrix.example_dir }}
        run: |
          echo Run Ci from example directory ${{ matrix.example_dir }}
          ../../whirl ci

  whirl-ci-default-envs-from-root-dir:
    needs: [examples]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      max-parallel: 4
      matrix:
        example: ${{ fromJson(needs.examples.outputs.example) }}
        python_version: ["3.9", "3.12"]
        airflow_version: ["2.2.5", "2.9.1"]
        exclude:
          - airflow_version: "2.2.5"
            python_version: "3.12"
          # Needs more memory than available on the runner
          - example: dbt-spark-example
          - example: spark-delta-sharing
          - example: spark-s3-to-hive
          # No default dag available for CI
          - example: airflow-datasets
          # Run without parallelism separately
          - example: dbt-example
          # Requires artifact downloaded from github.com/godatadriven/spot, which doesn't publish artifacts yet (2024-12-11)
          - example: spark-opentelemetry
    env:
      PYTHON_VERSION: ${{ matrix.python_version }}
      AIRFLOW_VERSION: ${{ matrix.airflow_version }}
    steps:
      - uses: actions/checkout@v2
      - name: Run whirl CI example ${{ matrix.example }}
        run: |
          echo Run Ci for example ${{ matrix.example }}
          ./whirl -x ${{ matrix.example }} ci

  whirl-ci-dbt-example:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        python_version: ["3.9", "3.12"]
        airflow_version: ["2.2.5", "2.9.1"]
        exclude:
          - airflow_version: "2.2.5"
            python_version: "3.12"
    env:
      PYTHON_VERSION: ${{ matrix.python_version }}
      AIRFLOW_VERSION: ${{ matrix.airflow_version }}
    steps:
      - uses: actions/checkout@v2
      - name: Run whirl CI dbt example
        working-directory: ./examples/dbt-example
        run: |
          echo Run Ci from dbt-example directory
          ../../whirl ci

  whirl-ci-opentelemetry-example:
    if: false  # Disabled until github.com/godatadriven/spot has public Releases (need a stable download URL)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: Run whirl CI opentelemetry example
        working-directory: ./examples/spark-opentelemetry/
        run: |
          curl -fLSs \
               -o ./whirl.setup.d/spot-complete.jar \
               --url "https://github.com/godatadriven/spot/releases/..." # TODO spot-complete-3.5_2.21.jar
          echo Run Ci from spark-opentelemetry directory
          ../../whirl ci

  whirl-ci-extra-env-spark-s3-to-postgres:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: Run whirl CI non-default env
        working-directory: ./examples/spark-s3-to-postgres/
        run: |
          echo Run Ci for example spark-s3-to-postgres with non-default env
          ../../whirl ci -e postgres-s3-spark

  whirl-ci-extra-env-api-to-s3:
    strategy:
      fail-fast: false
      max-parallel: 4
      matrix:
        python_version: ["3.9", "3.12"]
        airflow_version: ["2.2.5", "2.9.1"]
        exclude:
          - airflow_version: "2.2.5"
            python_version: "3.12"
    runs-on: ubuntu-latest
    env:
      PYTHON_VERSION: ${{ matrix.python_version }}
      AIRFLOW_VERSION: ${{ matrix.airflow_version }}
    steps:
      - uses: actions/checkout@v2
      - name: Run whirl CI api-to-s3 on k8s executor
        working-directory: ./examples/api-to-s3/
        run: |
          echo Run Ci for example api-to-s3 with k8s executor env
          ../../whirl ci -e api-python-s3-k8s

  whirl-ci-extra-env-ha-scheduler:
    strategy:
      fail-fast: false
      max-parallel: 4
      matrix:
        python_version: ["3.9", "3.12"]
        airflow_version: ["2.2.5", "2.9.1"]
        exclude:
          - airflow_version: "2.2.5"
            python_version: "3.12"
    runs-on: ubuntu-latest
    env:
      PYTHON_VERSION: ${{ matrix.python_version }}
      AIRFLOW_VERSION: ${{ matrix.airflow_version }}
    steps:
      - uses: actions/checkout@v2
      - name: Run whirl CI ha-scheduler env
        working-directory: ./examples/external-airflow-db/
        run: |
          echo Run Ci for example external-airflow-db with ha-scheduler env
          ../../whirl ci -e ha-scheduler