publish-to-pypi.yml

name: Publish to PyPI

on:
  push:
    # Pattern matched against refs/tags
    tags:
    - "**"  # Push events to every tag including tags with slashes

# How to publish a new version:
# 1. Update all hardcoded versions in this file (automation TBD): plugin's, Airflow's, and Python's ones.
#    Pick the latest Airflow's and Python's versions supported by the plugin.
# 2. Publish a tag with a name equal to the plugin's current version and "v" prefix: e.g. v1.0.0.

jobs:
  run-local-tests:
    name: Test locally on ClickHouse
    runs-on: ubuntu-latest
    services:
      clickhouse:
        image: clickhouse/clickhouse-server
        ports:
          - 9000/tcp
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: "3.12"
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install \
          -r requirements.txt \
          --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.0/constraints-3.12.txt"
    - name: Run tests on ClickHouse server
      env:
        AIRFLOW_CONN_CLICKHOUSE_DEFAULT: "clickhouse://localhost:${{ job.services.clickhouse.ports['9000'] }}"
        PYTHONPATH: "${{ github.workspace }}/src"  # for tests to import src/airflow_clickhouse_plugin
      run: |
        cd tests
        python -m unittest

  build:
    name: Build distribution 📦
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: "3.x"
    - name: Install pypa/build
      run: >-
        python3 -m
        pip install
        build
        --user
    - name: Build a binary wheel and a source tarball
      run: python3 -m build
    - name: Store the distribution packages
      uses: actions/upload-artifact@v3
      with:
        name: python-package-distributions
        path: dist/

  publish-to-testpypi:
    name: Publish to TestPyPI
    needs:
    - build
    - run-local-tests
    runs-on: ubuntu-latest
    environment:
      name: testpypi
      url: https://test.pypi.org/p/airflow-clickhouse-plugin
    permissions:
      id-token: write  # IMPORTANT: mandatory for trusted publishing
    steps:
    - name: Download distribution packages
      uses: actions/download-artifact@v3
      with:
        name: python-package-distributions
        path: dist/
    - name: Publish to TestPyPI
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        repository-url: https://test.pypi.org/legacy/

  run-testpypi-tests:
    name: Test TestPyPI on ClickHouse
    runs-on: ubuntu-latest
    needs:
    - publish-to-testpypi
    services:
      clickhouse:
        image: clickhouse/clickhouse-server
        ports:
          - 9000/tcp
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: "3.12"
      - name: Install airflow-clickhouse-plugin from TestPyPI
        run: |
          python -m pip install --upgrade pip
          python -m pip install \
            --index-url https://test.pypi.org/simple \
            --extra-index-url https://pypi.org/simple \
            airflow-clickhouse-plugin[common.sql]==1.4.0 \
            --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.0/constraints-3.12.txt"
      - name: Run tests on ClickHouse server
        env:
          AIRFLOW_CONN_CLICKHOUSE_DEFAULT: "clickhouse://localhost:${{ job.services.clickhouse.ports['9000'] }}"
          # "src" is not added to PYTHONPATH to run tests using pip-installed version
        run: |
          cd tests
          python -m unittest

  publish-to-pypi:
    name: Publish to PyPI
    needs:
    - build
    - run-testpypi-tests
    runs-on: ubuntu-latest
    environment:
      name: pypi
      url: https://pypi.org/p/airflow-clickhouse-plugin
    permissions:
      id-token: write  # IMPORTANT: mandatory for trusted publishing
    steps:
    - name: Download distribution packages
      uses: actions/download-artifact@v3
      with:
        name: python-package-distributions
        path: dist/
    - name: Publish to PyPI
      uses: pypa/gh-action-pypi-publish@release/v1

  run-pypi-tests:
    name: Test PyPI on ClickHouse
    runs-on: ubuntu-latest
    needs:
      - publish-to-pypi
    services:
      clickhouse:
        image: clickhouse/clickhouse-server
        ports:
          - 9000/tcp
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: "3.12"
      - name: Install airflow-clickhouse-plugin from PyPI
        run: |
          python -m pip install --upgrade pip
          python -m pip install \
            airflow-clickhouse-plugin[common.sql]==1.4.0 \
            --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.0/constraints-3.12.txt"
      - name: Run tests on ClickHouse server
        env:
          AIRFLOW_CONN_CLICKHOUSE_DEFAULT: "clickhouse://localhost:${{ job.services.clickhouse.ports['9000'] }}"
          # "src" is not added to PYTHONPATH to run tests using pip-installed version
        run: |
          cd tests
          python -m unittest

  upload-to-github-release:
    name: Sign and upload to GitHub Release
    needs:
    - publish-to-pypi
    runs-on: ubuntu-latest
    permissions:
      contents: write  # IMPORTANT: mandatory for making GitHub Releases
      id-token: write  # IMPORTANT: mandatory for sigstore
    steps:
    - name: Download distribution packages
      uses: actions/download-artifact@v3
      with:
        name: python-package-distributions
        path: dist/
    - name: Sign with Sigstore
      uses: sigstore/gh-action-sigstore-python@v1.2.3
      with:
        inputs: >-
          ./dist/*.tar.gz
          ./dist/*.whl
    - name: Upload to GitHub Release
      env:
        GITHUB_TOKEN: ${{ github.token }}
      # Upload to GitHub Release using the `gh` CLI.
      # `dist/` contains the built packages, and the
      # sigstore-produced signatures and certificates.
      run: >-
        gh release upload
        '${{ github.ref_name }}' dist/**
        --repo '${{ github.repository }}'