Merge pull request #80 from bryzgaloff/pypi-publishing-fixes

PyPI publishing fixes
bryzgaloff · Apr 16, 2024 · 1e3fb48 · 1e3fb48
2 parents 1044dcf + 1b6179b
commit 1e3fb48
Show file tree

Hide file tree

Showing 2 changed files with 48 additions and 16 deletions.
diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml
@@ -6,11 +6,43 @@ on:
     tags:
     - "**"  # Push events to every tag including tags with slashes
 
+# How to publish a new version:
+# 1. Update all hardcoded versions in this file (automation TBD): plugin's, Airflow's, and Python's ones.
+#    Pick the latest Airflow's and Python's versions supported by the plugin.
+# 2. Publish a tag with a name equal to the plugin's current version and "v" prefix: e.g. v1.0.0.
+
 jobs:
+  run-local-tests:
+    name: Test locally on ClickHouse
+    runs-on: ubuntu-latest
+    services:
+      clickhouse:
+        image: clickhouse/clickhouse-server
+        ports:
+          - 9000/tcp
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install \
+          -r requirements.txt \
+          --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.0/constraints-3.12.txt"
+    - name: Run tests on ClickHouse server
+      env:
+        AIRFLOW_CONN_CLICKHOUSE_DEFAULT: "clickhouse://localhost:${{ job.services.clickhouse.ports['9000'] }}"
+        PYTHONPATH: "${{ github.workspace }}/src"  # for tests to import src/airflow_clickhouse_plugin
+      run: |
+        cd tests
+        python -m unittest
+
   build:
     name: Build distribution 📦
     runs-on: ubuntu-latest
-
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python
@@ -35,15 +67,13 @@ jobs:
     name: Publish to TestPyPI
     needs:
     - build
+    - run-local-tests
     runs-on: ubuntu-latest
-
     environment:
       name: testpypi
       url: https://test.pypi.org/p/airflow-clickhouse-plugin
-
     permissions:
       id-token: write  # IMPORTANT: mandatory for trusted publishing
-
     steps:
     - name: Download distribution packages
       uses: actions/download-artifact@v3
@@ -62,26 +92,28 @@ jobs:
     - publish-to-testpypi
     services:
       clickhouse:
-        image: yandex/clickhouse-server
+        image: clickhouse/clickhouse-server
         ports:
           - 9000/tcp
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.x"
+          python-version: "3.12"
       - name: Install airflow-clickhouse-plugin from TestPyPI
         run: |
           python -m pip install --upgrade pip
           python -m pip install \
             --index-url https://test.pypi.org/simple \
             --extra-index-url https://pypi.org/simple \
-            airflow-clickhouse-plugin[common.sql]==1.1.0
+            airflow-clickhouse-plugin[common.sql]==1.3.0 \
+            --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.0/constraints-3.12.txt"
       - name: Run tests on ClickHouse server
         env:
           AIRFLOW_CONN_CLICKHOUSE_DEFAULT: "clickhouse://localhost:${{ job.services.clickhouse.ports['9000'] }}"
+          # "src" is not added to PYTHONPATH to run tests using pip-installed version
         run: |
-          cd tests  # run tests using pip-installed version
+          cd tests
           python -m unittest
 
   publish-to-pypi:
@@ -95,7 +127,6 @@ jobs:
       url: https://pypi.org/p/airflow-clickhouse-plugin
     permissions:
       id-token: write  # IMPORTANT: mandatory for trusted publishing
-
     steps:
     - name: Download distribution packages
       uses: actions/download-artifact@v3
@@ -112,35 +143,36 @@ jobs:
       - publish-to-pypi
     services:
       clickhouse:
-        image: yandex/clickhouse-server
+        image: clickhouse/clickhouse-server
         ports:
           - 9000/tcp
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.x"
+          python-version: "3.12"
       - name: Install airflow-clickhouse-plugin from PyPI
         run: |
           python -m pip install --upgrade pip
-          python -m pip install airflow-clickhouse-plugin[common.sql]==1.1.0
+          python -m pip install \
+            airflow-clickhouse-plugin[common.sql]==1.3.0 \
+            --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.0/constraints-3.12.txt"
       - name: Run tests on ClickHouse server
         env:
           AIRFLOW_CONN_CLICKHOUSE_DEFAULT: "clickhouse://localhost:${{ job.services.clickhouse.ports['9000'] }}"
+          # "src" is not added to PYTHONPATH to run tests using pip-installed version
         run: |
-          cd tests  # run tests using pip-installed version
+          cd tests
           python -m unittest
 
   upload-to-github-release:
     name: Sign and upload to GitHub Release
     needs:
     - publish-to-pypi
     runs-on: ubuntu-latest
-
     permissions:
       contents: write  # IMPORTANT: mandatory for making GitHub Releases
       id-token: write  # IMPORTANT: mandatory for sigstore
-
     steps:
     - name: Download distribution packages
       uses: actions/download-artifact@v3

diff --git a/tests/unit/sensors/test_clickhouse.py b/tests/unit/sensors/test_clickhouse.py
@@ -79,7 +79,7 @@ def test_defaults(self):
 
     def test_failure(self):
         is_failure_mock = mock.Mock(return_value=True)
-        with self.assertRaisesRegexp(AirflowException, 'is_failure returned True'):
+        with self.assertRaisesRegex(AirflowException, 'is_failure returned True'):
             ClickHouseSensor(
                 task_id='test3',  # required by Airflow
                 sql='SELECT 3',