diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4f02b8a..e0a40a4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,6 +8,7 @@ on: - "tests/**" - ".github/workflows/tests.yml" - "requirements.txt" + - "pyproject.toml" pull_request: branches: [ "master" ] paths: @@ -15,102 +16,140 @@ on: - "tests/**" - ".github/workflows/tests.yml" - "requirements.txt" + - "pyproject.toml" env: PYTHONPATH: "src" # for tests to import src/airflow_clickhouse_plugin jobs: - unit-tests-no-common-sql: + unit-tests: runs-on: ubuntu-latest strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11"] airflow-version: ["2.0.2", "2.1.4", "2.2.5", "2.3.4", "2.4.3", "2.5.3", "2.6.3", "2.7.1"] + airflow-extras: ["", "[common.sql]"] + exclude: + # constraints files for these combinations are missing + - python-version: "3.10" + airflow-version: "2.0.2" + - python-version: "3.10" + airflow-version: "2.1.4" + - python-version: "3.10" + airflow-version: "2.2.5" + - python-version: "3.11" + airflow-version: "2.0.2" + - python-version: "3.11" + airflow-version: "2.1.4" + - python-version: "3.11" + airflow-version: "2.2.5" + - python-version: "3.11" + airflow-version: "2.3.4" + - python-version: "3.11" + airflow-version: "2.4.3" + - python-version: "3.11" + airflow-version: "2.5.3" + # common.sql constraint for these Airflow versions is <1.3.0 + # => misses SQLExecuteQueryOperator + - airflow-extras: "[common.sql]" + airflow-version: "2.0.2" + - airflow-extras: "[common.sql]" + airflow-version: "2.1.4" + - airflow-extras: "[common.sql]" + airflow-version: "2.2.5" + - airflow-extras: "[common.sql]" + airflow-version: "2.3.4" + - airflow-extras: "[common.sql]" + airflow-version: "2.4.3" + include: + - airflow-extras: "[common.sql]" + tests-pattern: "" + - airflow-extras: "" + tests-pattern: "-p test_clickhouse.py" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install -r requirements.txt --ignore-installed apache-airflow==${{ matrix.airflow-version }} + python -m pip install \ + -r requirements.txt \ + apache-airflow${{ matrix.airflow-extras }}==${{ matrix.airflow-version }} \ + --ignore-installed \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-${{ matrix.airflow-version }}/constraints-${{ matrix.python-version }}.txt" - name: Run unit tests run: | - python -m unittest discover -t tests -s unit -p test_clickhouse.py + python -m unittest discover -t tests -s unit ${{ matrix.tests-pattern }} - unit-tests-common-sql: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] - airflow-version: ["2.1.4", "2.2.5", "2.3.4", "2.4.3", "2.5.3", "2.6.3", "2.7.1"] - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt --ignore-installed apache-airflow[common.sql]==${{ matrix.airflow-version }} - - name: Run unit tests - run: | - python -m unittest discover -t tests -s unit - - integration-tests-no-common-sql: + integration-tests: runs-on: ubuntu-latest strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11"] airflow-version: ["2.0.2", "2.1.4", "2.2.5", "2.3.4", "2.4.3", "2.5.3", "2.6.3", "2.7.1"] + airflow-extras: ["", "[common.sql]"] + exclude: + # constraints files for these combinations are missing + - python-version: "3.10" + airflow-version: "2.0.2" + - python-version: "3.10" + airflow-version: "2.1.4" + - python-version: "3.10" + airflow-version: "2.2.5" + - python-version: "3.11" + airflow-version: "2.0.2" + - python-version: "3.11" + airflow-version: "2.1.4" + - python-version: "3.11" + airflow-version: "2.2.5" + - python-version: "3.11" + airflow-version: "2.3.4" + - python-version: "3.11" + airflow-version: "2.4.3" + - python-version: "3.11" + airflow-version: "2.5.3" + # common.sql constraint for these Airflow versions is <1.3.0 + # => misses SQLExecuteQueryOperator + - airflow-extras: "[common.sql]" + airflow-version: "2.0.2" + - airflow-extras: "[common.sql]" + airflow-version: "2.1.4" + - airflow-extras: "[common.sql]" + airflow-version: "2.2.5" + - airflow-extras: "[common.sql]" + airflow-version: "2.3.4" + - airflow-extras: "[common.sql]" + airflow-version: "2.4.3" + include: + - airflow-extras: "[common.sql]" + tests-pattern: "" + - airflow-extras: "" + tests-pattern: "-p test_clickhouse.py" services: clickhouse: image: yandex/clickhouse-server ports: - 9000/tcp steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt --ignore-installed apache-airflow==${{ matrix.airflow-version }} - - name: Run tests on ClickHouse server - env: - AIRFLOW_CONN_CLICKHOUSE_DEFAULT: "clickhouse://localhost:${{ job.services.clickhouse.ports['9000'] }}" - run: | - python -m unittest discover -t tests -s integration -p test_clickhouse.py - - integration-tests-common-sql: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] - airflow-version: ["2.1.4", "2.2.5", "2.3.4", "2.4.3", "2.5.3", "2.6.3", "2.7.1"] - services: - clickhouse: - image: yandex/clickhouse-server - ports: - - 9000/tcp - steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install -r requirements.txt --ignore-installed apache-airflow[common.sql]==${{ matrix.airflow-version }} + python -m pip install \ + -r requirements.txt \ + apache-airflow${{ matrix.airflow-extras }}==${{ matrix.airflow-version }} \ + --ignore-installed \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-${{ matrix.airflow-version }}/constraints-${{ matrix.python-version }}.txt" - name: Run tests on ClickHouse server env: AIRFLOW_CONN_CLICKHOUSE_DEFAULT: "clickhouse://localhost:${{ job.services.clickhouse.ports['9000'] }}" run: | - python -m unittest discover -t tests -s integration + python -m unittest discover -t tests -s integration ${{ matrix.tests-pattern }} diff --git a/README.md b/README.md index 0d52218..517e1c8 100644 --- a/README.md +++ b/README.md @@ -62,10 +62,7 @@ Dependencies: `apache-airflow-providers-common-sql` (usually pre-packed with Air ## Python and Airflow versions support -Different versions of the plugin support different combinations of Python and - Airflow versions. We _primarily_ support **Airflow 2.0+ and Python 3.8+**. - If you need to use the plugin with older Python-Airflow combinations, pick a - suitable plugin version: +Different versions of the plugin support different combinations of Python and Airflow versions. We _primarily_ support **Airflow 2.0+ and Python 3.8+**. If you need to use the plugin with older Python-Airflow combinations, pick a suitable plugin version: | airflow-clickhouse-plugin version | Airflow version | Python version | |-----------------------------------|-------------------------|--------------------| @@ -81,8 +78,9 @@ Different versions of the plugin support different combinations of Python and | \>=0.5.4,<0.6.0 | ~=1.10.6 | \>=2.7 or >=3.5.\* | | \>=0.5.0,<0.5.4 | ==1.10.6 | \>=2.7 or >=3.5.\* | -`~=` means compatible release, see [PEP 440][pep-440-compatible-releases] for an - explanation. +`~=` means compatible release, see [PEP 440][pep-440-compatible-releases] for an explanation. + +[DB API 2.0 functionality](#python-db-api-20-family) requires `apache-airflow>=2.2` and `apache-airflow-providers-common-sql>=1.3`: earlier versions are not supported. Previous versions of the plugin might require `pandas` extra: `pip install airflow-clickhouse-plugin[pandas]==0.11.0`. Check out earlier versions of `README.md` for details. diff --git a/pyproject.toml b/pyproject.toml index 5a90017..4a57f08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ Issues = "https://github.com/bryzgaloff/airflow-clickhouse-plugin/issues" [project.optional-dependencies] "common.sql" = [ "apache-airflow[common.sql]>=2.2.0,<2.8.0", + "apache-airflow-providers-common-sql>=1.3.0", # introduces SQLExecuteQueryOperator "clickhouse-driver>=0.2.1", ]