From 1aa9d077b51065c93f71537d68985b89eb388c5c Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 30 Dec 2024 12:37:48 +0100 Subject: [PATCH] Improve caching strategy across the board of CI workflow We are using various caches in our build and so far - due to the way how "standard" caching works, PRs from forks could not effectively use the cache from main Airflow repository - because caches are not shared with other repositories - so the PRs builds could only use cache effectively when they were rebased and continued running from the same fork. This PR improves caching strategy using "stash" action from the ASF. Unlike `cache` - the action uses artifacts to store cache, and that makes it possible for the stash action to use such cache uploaded from `main` canary builds in PRs coming from the fork. As part of this change all the places where setup-python was used and breeze installed afterwards were reviewed and updated to use only breeze installation action (it already installs python) and this action has been improved to use UV caching effectively. Overall this PR should decrease setup overhead for many jobs across the CI workflow. Follow-up after #45266 --- .github/actions/breeze/action.yml | 22 +++- .github/actions/install-pre-commit/action.yml | 46 ++++++-- .../prepare_breeze_and_image/action.yml | 7 +- .../prepare_single_ci_image/action.yml | 2 +- .../workflows/additional-ci-image-checks.yml | 6 +- .../workflows/additional-prod-image-tests.yml | 6 + .github/workflows/basic-tests.yml | 108 ++++++++++++------ .github/workflows/ci-image-build.yml | 7 +- .github/workflows/ci-image-checks.yml | 80 ++++++++++--- .github/workflows/ci.yml | 29 ++++- .github/workflows/codeql-analysis.yml | 2 + .github/workflows/finalize-tests.yml | 6 +- .github/workflows/generate-constraints.yml | 6 + .github/workflows/helm-tests.yml | 7 ++ .../workflows/integration-system-tests.yml | 7 ++ .github/workflows/k8s-tests.yml | 16 +-- .github/workflows/prod-image-build.yml | 13 +-- .github/workflows/push-image-cache.yml | 6 +- .github/workflows/release_dockerhub_image.yml | 5 + .github/workflows/run-unit-tests.yml | 5 + .github/workflows/special-tests.yml | 10 ++ .github/workflows/task-sdk-tests.yml | 5 + .github/workflows/test-provider-packages.yml | 6 + contributing-docs/testing/k8s_tests.rst | 34 +++--- dev/breeze/doc/05_test_commands.rst | 8 +- .../airflow_breeze/utils/kubernetes_utils.py | 7 +- .../src/airflow_breeze/utils/run_tests.py | 4 +- .../airflow_breeze/utils/virtualenv_utils.py | 12 +- dev/breeze/uv.lock | 56 ++++----- scripts/ci/pre_commit/helm_lint.py | 2 +- scripts/ci/pre_commit/kubeconform.py | 2 +- 31 files changed, 381 insertions(+), 151 deletions(-) diff --git a/.github/actions/breeze/action.yml b/.github/actions/breeze/action.yml index 69ebcc7c66e6e..cbb1642b97812 100644 --- a/.github/actions/breeze/action.yml +++ b/.github/actions/breeze/action.yml @@ -22,6 +22,10 @@ inputs: python-version: description: 'Python version to use' default: "3.9" + use-uv: + description: 'Whether to use uv tool' + required: "true" + type: "string" outputs: host-python-version: description: Python version used in host @@ -33,13 +37,11 @@ runs: uses: actions/setup-python@v5 with: python-version: ${{ inputs.python-version }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/pyproject.toml + # NOTE! Installing Breeze without using cache is FASTER than when using cache - uv is so fast and has + # so low overhead, that just running upload cache/restore cache is slower than installing it from scratch - name: "Install Breeze" shell: bash run: ./scripts/ci/install_breeze.sh - env: - PYTHON_VERSION: ${{ inputs.python-version }} - name: "Free space" shell: bash run: breeze ci free-space @@ -56,3 +58,15 @@ runs: run: breeze setup config --no-cheatsheet --no-asciiart env: AIRFLOW_SOURCES_ROOT: "${{ github.workspace }}" + - name: "Use uv " + shell: bash + run: breeze setup config --use-uv + env: + AIRFLOW_SOURCES_ROOT: "${{ github.workspace }}" + if: inputs.use-uv == 'true' + - name: "Don't use uv " + shell: bash + run: breeze setup config --no-use-uv + env: + AIRFLOW_SOURCES_ROOT: "${{ github.workspace }}" + if: inputs.use-uv != 'true' diff --git a/.github/actions/install-pre-commit/action.yml b/.github/actions/install-pre-commit/action.yml index 8c704e5367ba2..dc175fd53d432 100644 --- a/.github/actions/install-pre-commit/action.yml +++ b/.github/actions/install-pre-commit/action.yml @@ -19,18 +19,19 @@ name: 'Install pre-commit' description: 'Installs pre-commit and related packages' inputs: + # TODO(potiuk): automate update of these versions python-version: description: 'Python version to use' - default: 3.9 + default: "3.9" uv-version: description: 'uv version to use' - default: 0.5.5 + default: "0.5.13" pre-commit-version: description: 'pre-commit version to use' - default: 4.0.1 + default: "4.0.1" pre-commit-uv-version: description: 'pre-commit-uv version to use' - default: 4.1.4 + default: "4.1.4" runs: using: "composite" steps: @@ -40,10 +41,35 @@ runs: pip install uv==${{inputs.uv-version}} || true uv tool install pre-commit==${{inputs.pre-commit-version}} --with uv==${{inputs.uv-version}} \ --with pre-commit-uv==${{inputs.pre-commit-uv-version}} - - name: Cache pre-commit envs - uses: actions/cache@v4 + working-directory: ${{ github.workspace }} + # We need to use tar file with archive to restore all the permissions and symlinks + - name: "Delete ~.cache" + run: | + du ~/ --max-depth=2 + echo + echo Deleting ~/.cache + echo + rm -rf ~/.cache + echo + shell: bash + - name: "Restore pre-commit cache" + uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c with: - path: ~/.cache/pre-commit - key: "pre-commit-${{inputs.python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}" - restore-keys: | - pre-commit-${{inputs.python-version}}- + key: cache-pre-commit-v4-${{ inputs.python-version }}-${{ hashFiles('.pre-commit-config.yaml') }} + path: /tmp/ + id: restore-pre-commit-cache + - name: "Restore .cache from the tar file" + run: tar -C ~ -xzf /tmp/cache-pre-commit.tar.gz + shell: bash + if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true' + - name: "Show restored files" + run: | + echo "Restored files" + du ~/ --max-depth=2 + echo + shell: bash + if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true' + - name: Install pre-commit hooks + shell: bash + run: pre-commit install-hooks || (cat ~/.cache/pre-commit/pre-commit.log && exit 1) + working-directory: ${{ github.workspace }} diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml index 2f8757c14cddd..4cfec042c5550 100644 --- a/.github/actions/prepare_breeze_and_image/action.yml +++ b/.github/actions/prepare_breeze_and_image/action.yml @@ -28,6 +28,9 @@ inputs: platform: description: 'Platform for the build - linux/amd64 or linux/arm64' required: true + use-uv: + description: 'Whether to use uv' + required: true outputs: host-python-version: description: Python version used in host @@ -40,11 +43,13 @@ runs: shell: bash - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} id: breeze - name: "Restore ${{ inputs.image-type }} docker image ${{ inputs.platform }}:${{ inputs.python }}" uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c with: - key: "${{ inputs.image-type }}-image-save-${{ inputs.platform }}-${{ inputs.python }}" + key: ${{ inputs.image-type }}-image-save-${{ inputs.platform }}-${{ inputs.python }} path: "/tmp/" - name: "Load ${{ inputs.image-type }} image ${{ inputs.platform }}:${{ inputs.python }}" run: > diff --git a/.github/actions/prepare_single_ci_image/action.yml b/.github/actions/prepare_single_ci_image/action.yml index e44a2e62321cc..9cdb7c6288f27 100644 --- a/.github/actions/prepare_single_ci_image/action.yml +++ b/.github/actions/prepare_single_ci_image/action.yml @@ -38,7 +38,7 @@ runs: - name: "Restore CI docker images ${{ inputs.platform }}:${{ inputs.python }}" uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c with: - key: "ci-image-save-${{ inputs.platform }}-${{ inputs.python }}" + key: ci-image-save-${{ inputs.platform }}-${{ inputs.python }} path: "/tmp/" if: contains(inputs.python-versions-list-as-string, inputs.python) - name: "Load CI image ${{ inputs.platform }}:${{ inputs.python }}" diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index 40196a6e04296..35389089eccd0 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -111,7 +111,7 @@ jobs: python-versions: ${{ inputs.python-versions }} branch: ${{ inputs.branch }} constraints-branch: ${{ inputs.constraints-branch }} - use-uv: ${{ inputs.use-uv}} + use-uv: ${{ inputs.use-uv }} include-success-outputs: ${{ inputs.include-success-outputs }} docker-cache: ${{ inputs.docker-cache }} disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} @@ -143,6 +143,8 @@ jobs: run: ./scripts/ci/cleanup_docker.sh - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} - name: "Login to ghcr.io" run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - name: "Check that image builds quickly" @@ -167,7 +169,7 @@ jobs: # platform: "linux/arm64" # branch: ${{ inputs.branch }} # constraints-branch: ${{ inputs.constraints-branch }} -# use-uv: ${{ inputs.use-uv}} +# use-uv: ${{ inputs.use-uv }} # upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} # docker-cache: ${{ inputs.docker-cache }} # disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index 62dd723999a8c..038bd74e0fbaa 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -56,6 +56,10 @@ on: # yamllint disable-line rule:truthy description: "Which version of python should be used by default" required: true type: string + use-uv: + description: "Whether to use uv" + required: true + type: string jobs: prod-image-extra-checks-main: name: PROD image extra checks (main) @@ -117,6 +121,7 @@ jobs: platform: "linux/amd64" image-type: "prod" python: ${{ inputs.default-python-version }} + use-uv: ${{ inputs.use-uv }} - name: "Test examples of PROD image building" run: " cd ./docker_tests && \ @@ -150,6 +155,7 @@ jobs: platform: "linux/amd64" image-type: "prod" python: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} + use-uv: ${{ inputs.use-uv }} id: breeze - name: "Test docker-compose quick start" run: breeze testing docker-compose-tests diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 47f80f05b7ac7..be78285b00508 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -56,6 +56,10 @@ on: # yamllint disable-line rule:truthy description: "Whether to run only latest version checks (true/false)" required: true type: string + use-uv: + description: "Whether to use uv in the image" + required: true + type: string jobs: run-breeze-tests: timeout-minutes: 10 @@ -72,16 +76,12 @@ jobs: persist-credentials: false - name: "Cleanup docker" run: ./scripts/ci/cleanup_docker.sh - - uses: actions/setup-python@v5 + - name: "Install Breeze" + uses: ./.github/actions/breeze with: - python-version: "${{ inputs.default-python-version }}" - cache: 'pip' - cache-dependency-path: ./dev/breeze/pyproject.toml - - run: pip install --editable ./dev/breeze/ - - run: python -m pytest -n auto --color=yes + use-uv: ${{ inputs.use-uv }} + - run: uv tool run --from apache-airflow-breeze pytest -n auto --color=yes working-directory: ./dev/breeze/ - - tests-ui: timeout-minutes: 10 name: React UI tests @@ -108,15 +108,24 @@ jobs: node-version: 21 cache: 'pnpm' cache-dependency-path: 'airflow/ui/pnpm-lock.yaml' - - name: "Cache eslint" - uses: actions/cache@v4 + - name: "Restore eslint cache (ui)" + uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c with: - path: 'airflow/ui/node_modules' - key: ${{ runner.os }}-ui-node-modules-${{ hashFiles('airflow/ui/**/pnpm-lock.yaml') }} + path: airflow/ui/node_modules/ + key: cache-ui-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/ui/**/pnpm-lock.yaml') }} + id: restore-eslint-cache - run: cd airflow/ui && pnpm install --frozen-lockfile - run: cd airflow/ui && pnpm test env: FORCE_COLOR: 2 + - name: "Save eslint cache (ui)" + uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c + with: + path: airflow/ui/node_modules/ + key: cache-ui-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/ui/**/pnpm-lock.yaml') }} + if-no-files-found: 'error' + retention-days: '2' + if: steps.restore-eslint-cache.outputs.stash-hit != 'true' tests-www: timeout-minutes: 10 @@ -137,15 +146,50 @@ jobs: uses: actions/setup-node@v4 with: node-version: 21 - - name: "Cache eslint" - uses: actions/cache@v4 + - name: "Restore eslint cache (www)" + uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c with: - path: 'airflow/www/node_modules' - key: ${{ runner.os }}-www-node-modules-${{ hashFiles('airflow/www/**/yarn.lock') }} + path: airflow/www/node_modules/ + key: cache-www-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/www/**/yarn.lock') }} + id: restore-eslint-cache - run: yarn --cwd airflow/www/ install --frozen-lockfile --non-interactive - run: yarn --cwd airflow/www/ run test env: FORCE_COLOR: 2 + - name: "Save eslint cache (www)" + uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c + with: + path: airflow/www/node_modules/ + key: cache-www-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/www/**/yarn.lock') }} + if-no-files-found: 'error' + retention-days: '2' + if: steps.restore-eslint-cache.outputs.stash-hit != 'true' + + install-pre-commit: + timeout-minutes: 5 + name: "Install pre-commit for cache" + runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + env: + PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" + if: inputs.basic-checks-only == 'true' + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} + id: breeze + - name: "Install pre-commit" + uses: ./.github/actions/install-pre-commit + id: pre-commit + with: + python-version: ${{steps.breeze.outputs.host-python-version}} # Those checks are run if no image needs to be built for checks. This is for simple changes that # Do not touch any of the python code or any of the important files that might require building @@ -154,6 +198,7 @@ jobs: timeout-minutes: 30 name: "Static checks: basic checks only" runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + needs: install-pre-commit if: inputs.basic-checks-only == 'true' steps: - name: "Cleanup repo" @@ -165,20 +210,10 @@ jobs: persist-credentials: false - name: "Cleanup docker" run: ./scripts/ci/cleanup_docker.sh - - name: "Setup python" - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.default-python-version }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/pyproject.toml - - name: "Setup python" - uses: actions/setup-python@v5 - with: - python-version: "${{ inputs.default-python-version }}" - cache: 'pip' - cache-dependency-path: ./dev/breeze/pyproject.toml - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} id: breeze - name: "Install pre-commit" uses: ./.github/actions/install-pre-commit @@ -216,6 +251,7 @@ jobs: timeout-minutes: 45 name: "Upgrade checks" runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + needs: install-pre-commit env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" if: inputs.canary-run == 'true' && inputs.latest-versions-only != 'true' @@ -229,12 +265,16 @@ jobs: persist-credentials: false - name: "Cleanup docker" run: ./scripts/ci/cleanup_docker.sh - # Install python from scratch. No cache used. We always want to have fresh version of everything - - uses: actions/setup-python@v5 + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} + id: breeze + - name: "Install pre-commit" + uses: ./.github/actions/install-pre-commit + id: pre-commit with: - python-version: "${{ inputs.default-python-version }}" - - name: "Install latest pre-commit" - run: pip install pre-commit + python-version: ${{steps.breeze.outputs.host-python-version}} - name: "Autoupdate all pre-commits" run: pre-commit autoupdate - name: "Run automated upgrade for black" @@ -305,6 +345,8 @@ jobs: run: ./scripts/ci/cleanup_docker.sh - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} - name: "Cleanup dist files" run: rm -fv ./dist/* - name: Setup git for tagging diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index bed2233dfcb07..553a509e4bf11 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -114,7 +114,6 @@ jobs: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} - USE_UV: ${{ inputs.use-uv }} VERBOSE: "true" steps: - name: "Cleanup repo" @@ -128,6 +127,8 @@ jobs: run: ./scripts/ci/cleanup_docker.sh - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} - name: "Login to ghcr.io" run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - name: > @@ -159,8 +160,8 @@ jobs: - name: "Stash CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c with: - key: "ci-image-save-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + key: ci-image-save-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }} path: "/tmp/ci-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar" if-no-files-found: 'error' - retention-days: 2 + retention-days: '2' if: inputs.upload-image-artifact == 'true' diff --git a/.github/workflows/ci-image-checks.yml b/.github/workflows/ci-image-checks.yml index 8e0e5e71dd00a..0868f607e1ef8 100644 --- a/.github/workflows/ci-image-checks.yml +++ b/.github/workflows/ci-image-checks.yml @@ -104,12 +104,57 @@ on: # yamllint disable-line rule:truthy description: "Whether to run coverage or not (true/false)" required: true type: string + use-uv: + description: "Whether to use uv to build the image (true/false)" + required: true + type: string jobs: + install-pre-commit: + timeout-minutes: 5 + name: "Install pre-commit for cache" + runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + env: + PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" + if: inputs.basic-checks-only == 'false' + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} + id: breeze + - name: "Install pre-commit" + uses: ./.github/actions/install-pre-commit + id: pre-commit + with: + python-version: ${{steps.breeze.outputs.host-python-version}} + - name: "Prepare .tar file from pre-commit cache" + run: | + tar -C ~ -czf /tmp/cache-pre-commit.tar.gz .cache/pre-commit .cache/uv + shell: bash + if: inputs.canary-run == 'true' + - name: "Save pre-commit cache" + uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c + with: + # yamllint disable rule:line-length + key: cache-pre-commit-v4-${{ steps.breeze.outputs.host-python-version }}-${{ hashFiles('.pre-commit-config.yaml') }} + path: /tmp/cache-pre-commit.tar.gz + if-no-files-found: 'error' + retention-days: '2' + if: inputs.canary-run == 'true' + static-checks: timeout-minutes: 45 name: "Static checks" runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + needs: install-pre-commit env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" UPGRADE_TO_NEWER_DEPENDENCIES: "${{ inputs.upgrade-to-newer-dependencies }}" @@ -123,17 +168,12 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.default-python-version }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/pyproject.toml - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: platform: "linux/amd64" python: ${{ inputs.default-python-version }} + use-uv: ${{ inputs.use-uv }} id: breeze - name: "Install pre-commit" uses: ./.github/actions/install-pre-commit @@ -154,6 +194,7 @@ jobs: timeout-minutes: 45 name: "MyPy checks" runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + needs: install-pre-commit if: inputs.needs-mypy == 'true' strategy: fail-fast: false @@ -175,6 +216,7 @@ jobs: with: platform: "linux/amd64" python: ${{ inputs.default-python-version }} + use-uv: ${{ inputs.use-uv }} id: breeze - name: "Install pre-commit" uses: ./.github/actions/install-pre-commit @@ -221,24 +263,32 @@ jobs: with: platform: "linux/amd64" python: ${{ inputs.default-python-version }} - - uses: actions/cache@v4 - id: cache-doc-inventories + use-uv: ${{ inputs.use-uv }} + - name: "Restore docs inventory cache" + uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c with: path: ./docs/_inventory_cache/ - key: docs-inventory-${{ hashFiles('pyproject.toml;') }} - restore-keys: | - docs-inventory-${{ hashFiles('pyproject.toml;') }} - docs-inventory- + # TODO(potiuk): do better with determining the key + key: cache-docs-inventory-v1-${{ hashFiles('pyproject.toml') }} + id: restore-docs-inventory-cache - name: "Building docs with ${{ matrix.flag }} flag" run: > breeze build-docs ${{ inputs.docs-list-as-string }} ${{ matrix.flag }} + - name: "Save docs inventory cache" + uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c + with: + path: ./docs/_inventory_cache/ + key: cache-docs-inventory-v1-${{ hashFiles('pyproject.toml') }} + if-no-files-found: 'error' + retention-days: '2' + if: steps.restore-docs-inventory-cache != 'true' - name: "Upload build docs" uses: actions/upload-artifact@v4 with: name: airflow-docs path: './docs/_build' - retention-days: 7 - if-no-files-found: error + retention-days: '7' + if-no-files-found: 'error' if: matrix.flag == '--docs-only' publish-docs: @@ -283,6 +333,7 @@ jobs: with: platform: "linux/amd64" python: ${{ inputs.default-python-version }} + use-uv: ${{ inputs.use-uv }} - name: "Publish docs" run: > breeze release-management publish-docs --override-versioned --run-in-parallel @@ -355,6 +406,7 @@ jobs: with: platform: "linux/amd64" python: ${{ inputs.default-python-version }} + use-uv: ${{ inputs.use-uv }} - name: "Generate airflow python client" run: > breeze release-management prepare-python-client --package-format both diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04e97f46d75ac..9ddeec96d2a08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,6 +141,7 @@ jobs: test-groups: ${{ steps.selective-checks.outputs.test-groups }} testable-core-integrations: ${{ steps.selective-checks.outputs.testable-core-integrations }} testable-providers-integrations: ${{ steps.selective-checks.outputs.testable-providers-integrations }} + use-uv: ${{ steps.selective-checks.outputs.force-pip == 'true' && 'false' || 'true' }} upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} steps: - name: "Cleanup repo" @@ -160,6 +161,9 @@ jobs: persist-credentials: false - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} + id: breeze - name: "Get information about the Workflow" id: source-run-info run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT} @@ -171,7 +175,6 @@ jobs: PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}" COMMIT_REF: "${{ github.sha }}" VERBOSE: "false" - run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} - name: env run: printenv @@ -193,6 +196,8 @@ jobs: skip-pre-commits: ${{needs.build-info.outputs.skip-pre-commits}} canary-run: ${{needs.build-info.outputs.canary-run}} latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}} + use-uv: ${{needs.build-info.outputs.use-uv}} + build-ci-images: name: Build CI images needs: [build-info] @@ -211,7 +216,7 @@ jobs: upload-image-artifact: "true" python-versions: ${{ needs.build-info.outputs.python-versions }} branch: ${{ needs.build-info.outputs.default-branch }} - use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }} + use-uv: ${{ needs.build-info.outputs.use-uv }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} docker-cache: ${{ needs.build-info.outputs.docker-cache }} @@ -239,7 +244,7 @@ jobs: latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} - use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }} + use-uv: ${{ needs.build-info.outputs.use-uv }} generate-constraints: @@ -257,6 +262,7 @@ jobs: generate-no-providers-constraints: ${{ needs.build-info.outputs.canary-run }} chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} + use-uv: ${{ needs.build-info.outputs.use-uv }} ci-image-checks: name: "CI image checks" @@ -285,6 +291,7 @@ jobs: needs-api-codegen: ${{ needs.build-info.outputs.needs-api-codegen }} default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} + use-uv: ${{ needs.build-info.outputs.use-uv }} providers: name: "Provider packages tests" @@ -308,6 +315,7 @@ jobs: skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} python-versions: ${{ needs.build-info.outputs.python-versions }} providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }} + use-uv: ${{ needs.build-info.outputs.use-uv }} tests-helm: name: "Helm tests" @@ -322,6 +330,7 @@ jobs: runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} + use-uv: ${{ needs.build-info.outputs.use-uv }} if: > needs.build-info.outputs.needs-helm-tests == 'true' && needs.build-info.outputs.default-branch == 'main' && @@ -351,6 +360,7 @@ jobs: run-migration-tests: "true" run-coverage: ${{ needs.build-info.outputs.run-coverage }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} + use-uv: ${{ needs.build-info.outputs.use-uv }} if: needs.build-info.outputs.run-tests == 'true' tests-mysql: @@ -377,6 +387,7 @@ jobs: run-coverage: ${{ needs.build-info.outputs.run-coverage }} run-migration-tests: "true" debug-resources: ${{ needs.build-info.outputs.debug-resources }} + use-uv: ${{ needs.build-info.outputs.use-uv }} if: needs.build-info.outputs.run-tests == 'true' tests-sqlite: @@ -405,6 +416,7 @@ jobs: run-coverage: ${{ needs.build-info.outputs.run-coverage }} run-migration-tests: "true" debug-resources: ${{ needs.build-info.outputs.debug-resources }} + use-uv: ${{ needs.build-info.outputs.use-uv }} if: needs.build-info.outputs.run-tests == 'true' tests-non-db: @@ -432,6 +444,7 @@ jobs: include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} + use-uv: ${{ needs.build-info.outputs.use-uv }} if: needs.build-info.outputs.run-tests == 'true' tests-special: @@ -462,6 +475,7 @@ jobs: upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} + use-uv: ${{ needs.build-info.outputs.use-uv }} tests-integration-system: name: Integration and System Tests @@ -482,6 +496,7 @@ jobs: skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} + use-uv: ${{ needs.build-info.outputs.use-uv }} if: needs.build-info.outputs.run-tests == 'true' tests-with-lowest-direct-resolution: @@ -512,6 +527,7 @@ jobs: run-coverage: ${{ needs.build-info.outputs.run-coverage }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} monitor-delay-time-in-seconds: 120 + use-uv: ${{ needs.build-info.outputs.use-uv }} build-prod-images: name: Build PROD images @@ -533,7 +549,7 @@ jobs: python-versions: ${{ needs.build-info.outputs.python-versions }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} branch: ${{ needs.build-info.outputs.default-branch }} - use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }} + use-uv: ${{ needs.build-info.outputs.use-uv }} build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} @@ -555,6 +571,7 @@ jobs: disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} canary-run: ${{ needs.build-info.outputs.canary-run }} + use-uv: ${{ needs.build-info.outputs.use-uv }} if: needs.build-info.outputs.prod-image-build == 'true' tests-kubernetes: @@ -570,7 +587,7 @@ jobs: runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }} + use-uv: ${{ needs.build-info.outputs.use-uv }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} kubernetes-combos: ${{ needs.build-info.outputs.kubernetes-combos }} if: > @@ -590,6 +607,7 @@ jobs: default-python-version: ${{ needs.build-info.outputs.default-python-version }} python-versions: ${{ needs.build-info.outputs.python-versions }} run-task-sdk-tests: ${{ needs.build-info.outputs.run-task-sdk-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} if: > ( needs.build-info.outputs.run-task-sdk-tests == 'true' || needs.build-info.outputs.run-tests == 'true' && @@ -624,6 +642,7 @@ jobs: docker-cache: ${{ needs.build-info.outputs.docker-cache }} disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} canary-run: ${{ needs.build-info.outputs.canary-run }} + use-uv: ${{ needs.build-info.outputs.use-uv }} notify-slack-failure: name: "Notify Slack on Failure" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index ca54b398202c6..3f4d7221fc61e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -49,6 +49,8 @@ jobs: persist-credentials: false - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: "true" - name: "Get information about the Workflow" id: source-run-info run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT} diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index b8fd240235f10..17f467162ceca 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -68,6 +68,10 @@ on: # yamllint disable-line rule:truthy description: "Whether this is a canary run (true/false)" required: true type: string + use-uv: + description: "Whether to use uv to build the image (true/false)" + required: true + type: string jobs: update-constraints: runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} @@ -140,7 +144,7 @@ jobs: python-versions: ${{ inputs.python-versions }} branch: ${{ inputs.branch }} constraints-branch: ${{ inputs.constraints-branch }} - use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }} + use-uv: ${{ inputs.use-uv }} include-success-outputs: ${{ inputs.include-success-outputs }} docker-cache: ${{ inputs.docker-cache }} disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} diff --git a/.github/workflows/generate-constraints.yml b/.github/workflows/generate-constraints.yml index 332d32e3160d2..d865c34efc435 100644 --- a/.github/workflows/generate-constraints.yml +++ b/.github/workflows/generate-constraints.yml @@ -40,6 +40,10 @@ on: # yamllint disable-line rule:truthy description: "Whether to run in debug mode (true/false)" required: true type: string + use-uv: + description: "Whether to use uvloop (true/false)" + required: true + type: string jobs: generate-constraints: permissions: @@ -66,6 +70,8 @@ jobs: persist-credentials: false - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} id: breeze - name: "Prepare all CI images: ${{ inputs.python-versions-list-as-string}}" uses: ./.github/actions/prepare_all_ci_images diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index d7f3f0b4d5bf0..5e3143b8a9e28 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -36,6 +36,10 @@ on: # yamllint disable-line rule:truthy description: "Which version of python should be used by default" required: true type: string + use-uv: + description: "Whether to use uvloop (true/false)" + required: true + type: string jobs: tests-helm: timeout-minutes: 80 @@ -70,6 +74,7 @@ jobs: with: platform: "linux/amd64" python: ${{ inputs.default-python-version }} + use-uv: ${{ inputs.use-uv }} - name: "Helm Unit Tests: ${{ matrix.helm-test-package }}" run: breeze testing helm-tests --test-type "${{ matrix.helm-test-package }}" @@ -91,6 +96,8 @@ jobs: run: ./scripts/ci/cleanup_docker.sh - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} - name: Setup git for tagging run: | git config --global user.email "name@example.com" diff --git a/.github/workflows/integration-system-tests.yml b/.github/workflows/integration-system-tests.yml index 0a6fd1fb406c7..5d5773846daa6 100644 --- a/.github/workflows/integration-system-tests.yml +++ b/.github/workflows/integration-system-tests.yml @@ -60,6 +60,10 @@ on: # yamllint disable-line rule:truthy description: "Debug resources (true/false)" required: true type: string + use-uv: + description: "Whether to use uv" + required: true + type: string jobs: tests-core-integration: timeout-minutes: 130 @@ -95,6 +99,7 @@ jobs: with: platform: "linux/amd64" python: ${{ inputs.default-python-version }} + use-uv: ${{ inputs.use-uv }} - name: "Integration: core ${{ matrix.integration }}" # yamllint disable rule:line-length run: ./scripts/ci/testing/run_integration_tests_with_retry.sh core "${{ matrix.integration }}" @@ -141,6 +146,7 @@ jobs: with: platform: "linux/amd64" python: ${{ inputs.default-python-version }} + use-uv: ${{ inputs.use-uv }} - name: "Integration: providers ${{ matrix.integration }}" run: ./scripts/ci/testing/run_integration_tests_with_retry.sh providers "${{ matrix.integration }}" - name: "Post Tests success" @@ -182,6 +188,7 @@ jobs: with: platform: "linux/amd64" python: ${{ inputs.default-python-version }} + use-uv: ${{ inputs.use-uv }} - name: "System Tests" run: > ./scripts/ci/testing/run_system_tests.sh diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml index e83cf68fb0e6f..ca7519a5f36fd 100644 --- a/.github/workflows/k8s-tests.yml +++ b/.github/workflows/k8s-tests.yml @@ -50,7 +50,7 @@ on: # yamllint disable-line rule:truthy type: string jobs: tests-kubernetes: - timeout-minutes: 240 + timeout-minutes: 60 name: "K8S System:${{ matrix.executor }}-${{ matrix.kubernetes-combo }}-${{ matrix.use-standard-naming }}" runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} strategy: @@ -85,13 +85,9 @@ jobs: platform: ${{ inputs.platform }} image-type: "prod" python: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} - - name: "Cache bin folder with tools for kubernetes testing" - uses: actions/cache@v4 - with: - path: ".build/.k8s-env" - key: "\ - k8s-env-${{ steps.breeze.outputs.host-python-version }}-\ - ${{ hashFiles('scripts/ci/kubernetes/k8s_requirements.txt','hatch_build.py') }}" + use-uv: ${{ inputs.use-uv }} + id: breeze + # preparing k8s environment with uv takes < 15 seconds with `uv` - there is no point in caching it. - name: "\ Run complete K8S tests ${{ matrix.executor }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}-\ ${{env.KUBERNETES_VERSION}}-${{ matrix.use-standard-naming }}" @@ -110,7 +106,7 @@ jobs: kind-logs-${{ matrix.kubernetes-combo }}-${{ matrix.executor }}-\ ${{ matrix.use-standard-naming }}" path: /tmp/kind_logs_* - retention-days: 7 + retention-days: '7' - name: "\ Upload test resource logs on failure ${{ matrix.executor }}-${{ matrix.kubernetes-combo }}-\ ${{ matrix.use-standard-naming }}" @@ -121,7 +117,7 @@ jobs: k8s-test-resources-${{ matrix.kubernetes-combo }}-${{ matrix.executor }}-\ ${{ matrix.use-standard-naming }}" path: /tmp/k8s_test_resources_* - retention-days: 7 + retention-days: '7' - name: "Delete clusters just in case they are left" run: breeze k8s delete-cluster --all if: always() diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index 4ee3538429222..81e4346e6a0d5 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -133,16 +133,14 @@ jobs: - name: "Cleanup docker" run: ./scripts/ci/cleanup_docker.sh if: inputs.upload-package-artifact == 'true' - - uses: actions/setup-python@v5 - with: - python-version: "${{ inputs.default-python-version }}" - if: inputs.upload-package-artifact == 'true' - name: "Cleanup dist and context file" shell: bash run: rm -fv ./dist/* ./docker-context-files/* if: inputs.upload-package-artifact == 'true' - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} if: inputs.upload-package-artifact == 'true' - name: "Prepare providers packages" shell: bash @@ -206,7 +204,6 @@ jobs: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} - USE_UV: ${{ inputs.use-uv }} VERBOSE: "true" steps: - name: "Cleanup repo" @@ -220,6 +217,8 @@ jobs: run: ./scripts/ci/cleanup_docker.sh - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} - name: "Cleanup dist and context file" shell: bash run: rm -fv ./dist/* ./docker-context-files/* @@ -280,8 +279,8 @@ jobs: - name: "Stash PROD docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c with: - key: "prod-image-save-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + key: prod-image-save-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }} path: "/tmp/prod-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar" if-no-files-found: 'error' - retention-days: 2 + retention-days: '2' if: inputs.upload-image-artifact == 'true' diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index 004c97a647bd9..2d125e35066dc 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -111,7 +111,6 @@ jobs: INCLUDE_SUCCESS_OUTPUTS: "${{ inputs.include-success-outputs }}" INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python }}" - USE_UV: ${{ inputs.use-uv }} UPGRADE_TO_NEWER_DEPENDENCIES: "false" VERBOSE: "true" VERSION_SUFFIX_FOR_PYPI: "dev0" @@ -127,6 +126,8 @@ jobs: run: ./scripts/ci/cleanup_docker.sh - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} - name: Login to ghcr.io run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - name: "Push CI latest images: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} (linux/amd64 only)" @@ -176,7 +177,6 @@ jobs: INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python }}" UPGRADE_TO_NEWER_DEPENDENCIES: "false" - USE_UV: ${{ inputs.branch == 'main' && inputs.use-uv || 'false' }} VERBOSE: "true" VERSION_SUFFIX_FOR_PYPI: "dev0" if: inputs.include-prod-images == 'true' @@ -192,6 +192,8 @@ jobs: run: ./scripts/ci/cleanup_docker.sh - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} - name: "Cleanup dist and context file" run: rm -fv ./dist/* ./docker-context-files/* - name: "Download packages prepared as artifacts" diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml index 5ce1585131f76..f23fdc9974abe 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_dockerhub_image.yml @@ -63,11 +63,14 @@ jobs: run: ./scripts/ci/cleanup_docker.sh - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: "false" - name: Selective checks id: selective-checks env: VERBOSE: "false" run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} + release-images: timeout-minutes: 120 name: "Release images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}" @@ -99,6 +102,8 @@ jobs: run: ./scripts/ci/cleanup_docker.sh - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: "false" - name: Free space run: breeze ci free-space --answer yes - name: "Cleanup dist and context file" diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 3a9440e528eaf..609f369f80d80 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -112,6 +112,10 @@ on: # yamllint disable-line rule:truthy required: false default: 20 type: number + use-uv: + description: "Whether to use uv" + required: true + type: string jobs: tests: timeout-minutes: 120 @@ -163,6 +167,7 @@ jobs: with: platform: "linux/amd64" python: ${{ matrix.python-version }} + use-uv: ${{ inputs.use-uv }} - name: > Migration Tests: ${{ matrix.python-version }}:${{ env.PARALLEL_TEST_TYPES }} uses: ./.github/actions/migration_tests diff --git a/.github/workflows/special-tests.yml b/.github/workflows/special-tests.yml index d416d55575fb9..cca54524b7dca 100644 --- a/.github/workflows/special-tests.yml +++ b/.github/workflows/special-tests.yml @@ -76,6 +76,10 @@ on: # yamllint disable-line rule:truthy description: "Whether to debug resources or not (true/false)" required: true type: string + use-uv: + description: "Whether to use uv or not (true/false)" + required: true + type: string jobs: tests-min-sqlalchemy: @@ -100,6 +104,7 @@ jobs: providers-test-types-list-as-string: ${{ inputs.providers-test-types-list-as-string }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} + use-uv: ${{ inputs.use-uv }} tests-boto: name: "Latest Boto test" @@ -124,6 +129,7 @@ jobs: include-success-outputs: ${{ inputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} + use-uv: ${{ inputs.use-uv }} tests-pendulum-2: name: "Pendulum2 test" @@ -148,6 +154,7 @@ jobs: include-success-outputs: ${{ inputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} + use-uv: ${{ inputs.use-uv }} tests-quarantined: name: "Quarantined test" @@ -171,6 +178,7 @@ jobs: include-success-outputs: ${{ inputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} + use-uv: ${{ inputs.use-uv }} tests-arm-collection: name: "ARM Collection test" @@ -194,6 +202,7 @@ jobs: include-success-outputs: ${{ inputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} + use-uv: ${{ inputs.use-uv }} if: ${{ inputs.default-branch == 'main' }} tests-system: @@ -218,3 +227,4 @@ jobs: include-success-outputs: ${{ inputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} + use-uv: ${{ inputs.use-uv }} diff --git a/.github/workflows/task-sdk-tests.yml b/.github/workflows/task-sdk-tests.yml index 756a66e546479..61b9ba75b5bee 100644 --- a/.github/workflows/task-sdk-tests.yml +++ b/.github/workflows/task-sdk-tests.yml @@ -36,6 +36,10 @@ on: # yamllint disable-line rule:truthy description: "Whether to run Task SDK tests or not (true/false)" required: true type: string + use-uv: + description: "Whether to use uv to build the image (true/false)" + required: true + type: string jobs: task-sdk-tests: timeout-minutes: 80 @@ -66,6 +70,7 @@ jobs: with: platform: "linux/amd64" python: ${{ matrix.python-version }} + use-uv: ${{ inputs.use-uv }} - name: "Cleanup dist files" run: rm -fv ./dist/* - name: "Prepare Task SDK packages: wheel" diff --git a/.github/workflows/test-provider-packages.yml b/.github/workflows/test-provider-packages.yml index 7b2bc485528c4..3d79c6896b8aa 100644 --- a/.github/workflows/test-provider-packages.yml +++ b/.github/workflows/test-provider-packages.yml @@ -58,6 +58,10 @@ on: # yamllint disable-line rule:truthy description: "JSON-formatted array of Python versions to build images from" required: true type: string + use-uv: + description: "Whether to use uv" + required: true + type: string jobs: prepare-install-verify-provider-packages: timeout-minutes: 80 @@ -87,6 +91,7 @@ jobs: with: platform: "linux/amd64" python: ${{ inputs.default-python-version }} + use-uv: ${{ inputs.use-uv }} - name: "Cleanup dist files" run: rm -fv ./dist/* - name: "Prepare provider documentation" @@ -175,6 +180,7 @@ jobs: with: platform: "linux/amd64" python: ${{ matrix.python-version }} + use-uv: ${{ inputs.use-uv }} - name: "Cleanup dist files" run: rm -fv ./dist/* - name: "Prepare provider packages: wheel" diff --git a/contributing-docs/testing/k8s_tests.rst b/contributing-docs/testing/k8s_tests.rst index a9ba3151fe9bb..4891182f69923 100644 --- a/contributing-docs/testing/k8s_tests.rst +++ b/contributing-docs/testing/k8s_tests.rst @@ -34,7 +34,7 @@ Before running ``breeze k8s`` cluster commands you need to setup the environment by ``breeze k8s setup-env`` command. Breeze in this command makes sure to download tools that are needed to run k8s tests: Helm, Kind, Kubectl in the right versions and sets up a Python virtualenv that is needed to run the tests. All those tools and env are setup in -``.build/.k8s-env`` folder. You can activate this environment yourselves as usual by sourcing +``.build/k8s-env`` folder. You can activate this environment yourselves as usual by sourcing ``bin/activate`` script, but since we are supporting multiple clusters in the same installation it is best if you use ``breeze k8s shell`` with the right parameters specifying which cluster to use. @@ -129,8 +129,8 @@ Entering shell with Kubernetes Cluster This shell is prepared to run Kubernetes tests interactively. It has ``kubectl`` and ``kind`` cli tools available in the path, it has also activated virtualenv environment that allows you to run tests via pytest. -The virtualenv is available in ./.build/.k8s-env/ -The binaries are available in ``.build/.k8s-env/bin`` path. +The virtualenv is available in ./.build/k8s-env/ +The binaries are available in ``.build/k8s-env/bin`` path. .. code-block:: bash @@ -184,26 +184,26 @@ The typical session for tests with Kubernetes looks like follows: breeze k8s setup-env The first time you run it, it should result in creating the virtualenv and installing good versions -of kind, kubectl and helm. All of them are installed in ``./build/.k8s-env`` (binaries available in ``bin`` +of kind, kubectl and helm. All of them are installed in ``./build/k8s-env`` (binaries available in ``bin`` sub-folder of it). .. code-block:: text - Initializing K8S virtualenv in /Users/jarek/IdeaProjects/airflow/.build/.k8s-env - Reinstalling PIP version in /Users/jarek/IdeaProjects/airflow/.build/.k8s-env - Installing necessary packages in /Users/jarek/IdeaProjects/airflow/.build/.k8s-env + Initializing K8S virtualenv in /Users/jarek/IdeaProjects/airflow/.build/k8s-env + Reinstalling PIP version in /Users/jarek/IdeaProjects/airflow/.build/k8s-env + Installing necessary packages in /Users/jarek/IdeaProjects/airflow/.build/k8s-env The ``kind`` tool is not downloaded yet. Downloading 0.14.0 version. Downloading from: https://github.com/kubernetes-sigs/kind/releases/download/v0.14.0/kind-darwin-arm64 The ``kubectl`` tool is not downloaded yet. Downloading 1.24.3 version. Downloading from: https://storage.googleapis.com/kubernetes-release/release/v1.24.3/bin/darwin/arm64/kubectl The ``helm`` tool is not downloaded yet. Downloading 3.9.2 version. Downloading from: https://get.helm.sh/helm-v3.9.2-darwin-arm64.tar.gz - Extracting the darwin-arm64/helm to /Users/jarek/IdeaProjects/airflow/.build/.k8s-env/bin - Moving the helm to /Users/jarek/IdeaProjects/airflow/.build/.k8s-env/bin/helm + Extracting the darwin-arm64/helm to /Users/jarek/IdeaProjects/airflow/.build/k8s-env/bin + Moving the helm to /Users/jarek/IdeaProjects/airflow/.build/k8s-env/bin/helm This prepares the virtual environment for tests and downloads the right versions of the tools -to ``./build/.k8s-env`` +to ``./build/k8s-env`` 2. Create the KinD cluster: @@ -404,10 +404,10 @@ Should show the status of current KinD cluster. .. code-block:: text - K8S Virtualenv is initialized in /Users/jarek/IdeaProjects/airflow/.build/.k8s-env - Good version of kind installed: 0.14.0 in /Users/jarek/IdeaProjects/airflow/.build/.k8s-env/bin - Good version of kubectl installed: 1.25.0 in /Users/jarek/IdeaProjects/airflow/.build/.k8s-env/bin - Good version of helm installed: 3.9.2 in /Users/jarek/IdeaProjects/airflow/.build/.k8s-env/bin + K8S Virtualenv is initialized in /Users/jarek/IdeaProjects/airflow/.build/k8s-env + Good version of kind installed: 0.14.0 in /Users/jarek/IdeaProjects/airflow/.build/k8s-env/bin + Good version of kubectl installed: 1.25.0 in /Users/jarek/IdeaProjects/airflow/.build/k8s-env/bin + Good version of helm installed: 3.9.2 in /Users/jarek/IdeaProjects/airflow/.build/k8s-env/bin Stable repo is already added Uploading Airflow image ghcr.io/apache/airflow/main/prod/python3.9-kubernetes to cluster airflow-python-3.9-v1.24.2 Image: "ghcr.io/apache/airflow/main/prod/python3.9-kubernetes" with ID "sha256:fb6195f7c2c2ad97788a563a3fe9420bf3576c85575378d642cd7985aff97412" not yet present on node "airflow-python-3.9-v1.24.2-worker", loading... @@ -511,7 +511,7 @@ The virtualenv required will be created automatically when the scripts are run. Running tests with kind-airflow-python-3.9-v1.24.2 cluster. Command to run: pytest kubernetes_tests ========================================================================================= test session starts ========================================================================================== - platform darwin -- Python 3.9.9, pytest-6.2.5, py-1.11.0, pluggy-1.0.0 -- /Users/jarek/IdeaProjects/airflow/.build/.k8s-env/bin/python + platform darwin -- Python 3.9.9, pytest-6.2.5, py-1.11.0, pluggy-1.0.0 -- /Users/jarek/IdeaProjects/airflow/.build/k8s-env/bin/python cachedir: .pytest_cache rootdir: /Users/jarek/IdeaProjects/airflow/kubernetes_tests plugins: anyio-3.6.1, instafail-0.4.2, xdist-2.5.0, forked-1.4.0, timeouts-1.2.1, cov-3.0.0 @@ -525,7 +525,7 @@ The virtualenv required will be created automatically when the scripts are run. 8b) You can enter an interactive shell to run tests one-by-one -This enters the virtualenv in ``.build/.k8s-env`` folder: +This enters the virtualenv in ``.build/k8s-env`` folder: .. code-block:: bash @@ -561,7 +561,7 @@ tests using the standard IntelliJ Run/Debug feature. You just need a few steps: :alt: Kubernetes testing virtualenv The virtualenv is created in your "Airflow" source directory in the -``.build/.k8s-env`` folder and you have to find ``python`` binary and choose +``.build/k8s-env`` folder and you have to find ``python`` binary and choose it when selecting interpreter. 9b) Choose pytest as test runner: diff --git a/dev/breeze/doc/05_test_commands.rst b/dev/breeze/doc/05_test_commands.rst index 819ac80224525..d9eb0856bbd76 100644 --- a/dev/breeze/doc/05_test_commands.rst +++ b/dev/breeze/doc/05_test_commands.rst @@ -338,7 +338,7 @@ Kubernetes environment can be set with the ``breeze k8s setup-env`` command. It will create appropriate virtualenv to run tests and download the right set of tools to run the tests: ``kind``, ``kubectl`` and ``helm`` in the right versions. You can re-run the command when you want to make sure the expected versions of the tools are installed properly in the -virtualenv. The Virtualenv is available in ``.build/.k8s-env/bin`` subdirectory of your Airflow +virtualenv. The Virtualenv is available in ``.build/k8s-env/bin`` subdirectory of your Airflow installation. .. image:: ./images/output_k8s_setup-env.svg @@ -561,7 +561,7 @@ be created and airflow deployed to it before running the tests): (kind-airflow-python-3.9-v1.24.0:KubernetesExecutor)> pytest test_kubernetes_executor.py ================================================= test session starts ================================================= - platform linux -- Python 3.10.6, pytest-6.2.5, py-1.11.0, pluggy-1.0.0 -- /home/jarek/code/airflow/.build/.k8s-env/bin/python + platform linux -- Python 3.10.6, pytest-6.2.5, py-1.11.0, pluggy-1.0.0 -- /home/jarek/code/airflow/.build/k8s-env/bin/python cachedir: .pytest_cache rootdir: /home/jarek/code/airflow, configfile: pytest.ini plugins: anyio-3.6.1 @@ -571,8 +571,8 @@ be created and airflow deployed to it before running the tests): test_kubernetes_executor.py::TestKubernetesExecutor::test_integration_run_dag_with_scheduler_failure PASSED [100%] ================================================== warnings summary =================================================== - .build/.k8s-env/lib/python3.10/site-packages/_pytest/config/__init__.py:1233 - /home/jarek/code/airflow/.build/.k8s-env/lib/python3.10/site-packages/_pytest/config/__init__.py:1233: PytestConfigWarning: Unknown config option: asyncio_mode + .build/k8s-env/lib/python3.10/site-packages/_pytest/config/__init__.py:1233 + /home/jarek/code/airflow/.build/k8s-env/lib/python3.10/site-packages/_pytest/config/__init__.py:1233: PytestConfigWarning: Unknown config option: asyncio_mode self._warn_or_fail_if_strict(f"Unknown config option: {key}\n") diff --git a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py index b9bdc5302bdfc..8525d1e9acc50 100644 --- a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py @@ -51,7 +51,7 @@ from airflow_breeze.utils.shared_options import get_dry_run, get_verbose from airflow_breeze.utils.virtualenv_utils import create_pip_command, create_uv_command -K8S_ENV_PATH = BUILD_CACHE_DIR / ".k8s-env" +K8S_ENV_PATH = BUILD_CACHE_DIR / "k8s-env" K8S_CLUSTERS_PATH = BUILD_CACHE_DIR / ".k8s-clusters" K8S_BIN_BASE_PATH = K8S_ENV_PATH / "bin" KIND_BIN_PATH = K8S_BIN_BASE_PATH / "kind" @@ -305,8 +305,10 @@ def _requirements_changed() -> bool: def _install_packages_in_k8s_virtualenv(): if check_if_cache_exists("use_uv"): + get_console().print("[info]Using uv to install k8s env[/]") command = create_uv_command(PYTHON_BIN_PATH) else: + get_console().print("[info]Using pip to install k8s env[/]") command = create_pip_command(PYTHON_BIN_PATH) install_command_no_constraints = [ *command, @@ -406,7 +408,7 @@ def create_virtualenv(force_venv_setup: bool) -> RunCommandResult: f"{venv_command_result.stdout}\n{venv_command_result.stderr}" ) return venv_command_result - get_console().print(f"[info]Reinstalling PIP version in {K8S_ENV_PATH}") + get_console().print(f"[info]Reinstalling pip=={PIP_VERSION} in {K8S_ENV_PATH}") command = create_pip_command(PYTHON_BIN_PATH) pip_reinstall_result = run_command( [*command, "install", f"pip=={PIP_VERSION}"], @@ -419,6 +421,7 @@ def create_virtualenv(force_venv_setup: bool) -> RunCommandResult: f"{pip_reinstall_result.stdout}\n{pip_reinstall_result.stderr}" ) return pip_reinstall_result + get_console().print(f"[info]Reinstalling uv=={UV_VERSION} in {K8S_ENV_PATH}") uv_reinstall_result = run_command( [*command, "install", f"uv=={UV_VERSION}"], check=False, diff --git a/dev/breeze/src/airflow_breeze/utils/run_tests.py b/dev/breeze/src/airflow_breeze/utils/run_tests.py index d28194bbe0381..b5f50a5892cfa 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_tests.py +++ b/dev/breeze/src/airflow_breeze/utils/run_tests.py @@ -106,7 +106,9 @@ def run_docker_compose_tests( env["DOCKER_IMAGE"] = image_name if skip_docker_compose_deletion: env["SKIP_DOCKER_COMPOSE_DELETION"] = "true" - with create_temp_venv(pip_version=PIP_VERSION, requirements_file=DOCKER_TESTS_REQUIREMENTS) as py_exe: + with create_temp_venv( + pip_version=PIP_VERSION, uv_version=UV_VERSION, requirements_file=DOCKER_TESTS_REQUIREMENTS + ) as py_exe: command_result = run_command( [py_exe, "-m", "pytest", str(test_path), *pytest_args, *extra_pytest_args], env=env, diff --git a/dev/breeze/src/airflow_breeze/utils/virtualenv_utils.py b/dev/breeze/src/airflow_breeze/utils/virtualenv_utils.py index 3ececdd228461..04622c4f11fec 100644 --- a/dev/breeze/src/airflow_breeze/utils/virtualenv_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/virtualenv_utils.py @@ -59,13 +59,9 @@ def create_venv( if not python_path.exists(): get_console().print(f"\n[errors]Python interpreter is not exist in path {python_path}. Exiting!\n") sys.exit(1) - if check_if_cache_exists("use_uv"): - command = create_uv_command(python_path) - else: - command = create_pip_command(python_path) if pip_version: result = run_command( - [*command, "install", f"pip=={pip_version}", "-q"], + [python_path.as_posix(), "-m", "pip", "install", f"pip=={pip_version}", "-q"], check=False, capture_output=False, text=True, @@ -78,7 +74,7 @@ def create_venv( sys.exit(result.returncode) if uv_version: result = run_command( - [*command, "install", f"uv=={uv_version}", "-q"], + [python_path.as_posix(), "-m", "pip", "install", f"uv=={uv_version}", "-q"], check=False, capture_output=False, text=True, @@ -89,6 +85,10 @@ def create_venv( f"{result.stdout}\n{result.stderr}" ) sys.exit(result.returncode) + if check_if_cache_exists("use_uv"): + command = create_uv_command(python_path) + else: + command = create_pip_command(python_path) if requirements_file: requirements_file = Path(requirements_file).absolute().as_posix() result = run_command( diff --git a/dev/breeze/uv.lock b/dev/breeze/uv.lock index d400d08f40f58..9b98a8746a2ba 100644 --- a/dev/breeze/uv.lock +++ b/dev/breeze/uv.lock @@ -52,13 +52,14 @@ dependencies = [ { name = "semver" }, { name = "tabulate" }, { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "tqdm" }, { name = "twine" }, ] [package.metadata] requires-dist = [ { name = "black", specifier = ">=23.11.0" }, - { name = "click", specifier = ">=8.1.7" }, + { name = "click", specifier = ">=8.1.8" }, { name = "filelock", specifier = ">=3.13.0" }, { name = "flit", specifier = "==3.10.1" }, { name = "flit-core", specifier = "==3.10.1" }, @@ -66,8 +67,7 @@ requires-dist = [ { name = "google-api-python-client", specifier = ">=2.142.0" }, { name = "google-auth-httplib2", specifier = ">=0.2.0" }, { name = "google-auth-oauthlib", specifier = ">=1.2.0" }, - { name = "hatch", specifier = "==1.9.4" }, - { name = "importlib-resources", marker = "python_full_version < '3.9'", specifier = ">=5.2,!=6.2.0,!=6.3.0,!=6.3.1" }, + { name = "hatch", specifier = "==1.14.0" }, { name = "inputimeout", specifier = ">=1.0.4" }, { name = "jinja2", specifier = ">=3.1.0" }, { name = "jsonschema", specifier = ">=4.19.1" }, @@ -85,6 +85,7 @@ requires-dist = [ { name = "semver", specifier = ">=3.0.2" }, { name = "tabulate", specifier = ">=0.9.0" }, { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0.1" }, + { name = "tqdm", specifier = ">=4.67.1" }, { name = "twine", specifier = ">=4.0.2" }, ] @@ -326,14 +327,14 @@ wheels = [ [[package]] name = "click" -version = "8.1.7" +version = "8.1.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "platform_system == 'Windows'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/96/d3/f04c7bfcf5c1862a2a5b845c6b2b360488cf47af55dfa79c98f6a6bf98b5/click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de", size = 336121 } +sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593 } wheels = [ - { url = "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", size = 97941 }, + { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188 }, ] [[package]] @@ -412,15 +413,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", size = 587408 }, ] -[[package]] -name = "editables" -version = "0.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/37/4a/986d35164e2033ddfb44515168a281a7986e260d344cf369c3f52d4c3275/editables-0.5.tar.gz", hash = "sha256:309627d9b5c4adc0e668d8c6fa7bac1ba7c8c5d415c2d27f60f081f8e80d1de2", size = 14744 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/be/0f2f4a5e8adc114a02b63d92bf8edbfa24db6fc602fca83c885af2479e0e/editables-0.5-py3-none-any.whl", hash = "sha256:61e5ffa82629e0d8bfe09bc44a07db3c1ab8ed1ce78a6980732870f19b5e7d4c", size = 5098 }, -] - [[package]] name = "exceptiongroup" version = "1.2.2" @@ -592,7 +584,7 @@ wheels = [ [[package]] name = "hatch" -version = "1.9.4" +version = "1.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -608,29 +600,29 @@ dependencies = [ { name = "tomli-w" }, { name = "tomlkit" }, { name = "userpath" }, + { name = "uv" }, { name = "virtualenv" }, { name = "zstandard" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/17/98/63bf6c592b65f67201db292489053b86310cfb107eb095d345398e00cbd3/hatch-1.9.4.tar.gz", hash = "sha256:9bb7d1c4a7a51cc1f9e16394875c940b45fa84b698f0291529316b27d74e7f32", size = 689598 } +sdist = { url = "https://files.pythonhosted.org/packages/bc/15/b4e3d50d8177e6e8a243b24d9819e3807f7bfd3b2bebe7b5aef32a9c79cb/hatch-1.14.0.tar.gz", hash = "sha256:351e41bc6c72bc93cb98651212226e495b43549eee27c487832e459e5d0f0eda", size = 5188143 } wheels = [ - { url = "https://files.pythonhosted.org/packages/05/38/ba8f90264d19ed39851f37a22f2a4be8e9644a1203f114b16647f954bb02/hatch-1.9.4-py3-none-any.whl", hash = "sha256:461eb86b4b46249e38a9a621c7239e61285fd8e14b5a1b5a727c394893a25300", size = 110812 }, + { url = "https://files.pythonhosted.org/packages/85/c6/ad910cdb79600af0100b7c4f7093eb4b95a2b44e589e66b6b938b09cc6f9/hatch-1.14.0-py3-none-any.whl", hash = "sha256:b12c7a2f4aaf6db7180e35c476e1a2ad4ec7197c20c4332964599424d4918ded", size = 125763 }, ] [[package]] name = "hatchling" -version = "1.21.1" +version = "1.27.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "editables" }, { name = "packaging" }, { name = "pathspec" }, { name = "pluggy" }, { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "trove-classifiers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d8/a1/7dd1caa87c0b15c04c6291e25112e5d082cce02ee87f221a8be1d594f857/hatchling-1.21.1.tar.gz", hash = "sha256:bba440453a224e7d4478457fa2e8d8c3633765bafa02975a6b53b9bf917980bc", size = 58059 } +sdist = { url = "https://files.pythonhosted.org/packages/8f/8a/cc1debe3514da292094f1c3a700e4ca25442489731ef7c0814358816bb03/hatchling-1.27.0.tar.gz", hash = "sha256:971c296d9819abb3811112fc52c7a9751c8d381898f36533bb16f9791e941fd6", size = 54983 } wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/bb/40528a09a33845bd7fd75c33b3be7faec3b5c8f15f68a58931da67420fb9/hatchling-1.21.1-py3-none-any.whl", hash = "sha256:21e8c13f8458b219a91cb84e5b61c15bf786695d1c4fabc29e91e78f94bfe892", size = 76740 }, + { url = "https://files.pythonhosted.org/packages/08/e7/ae38d7a6dfba0533684e0b2136817d667588ae3ec984c1a4e5df5eb88482/hatchling-1.27.0-py3-none-any.whl", hash = "sha256:d3a2f3567c4f926ea39849cdf924c7e99e6686c9c8e288ae1037c8fa2a5d937b", size = 75794 }, ] [[package]] @@ -985,11 +977,11 @@ wheels = [ [[package]] name = "packaging" -version = "24.1" +version = "24.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/51/65/50db4dda066951078f0a96cf12f4b9ada6e4b811516bf0262c0f4f7064d4/packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002", size = 148788 } +sdist = { url = "https://files.pythonhosted.org/packages/d0/63/68dbb6eb2de9cb10ee4c9c14a0148804425e13c4fb20d61cce69f53106da/packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f", size = 163950 } wheels = [ - { url = "https://files.pythonhosted.org/packages/08/aa/cc0199a5f0ad350994d660967a8efb233fe0416e4639146c089643407ce6/packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124", size = 53985 }, + { url = "https://files.pythonhosted.org/packages/88/ef/eb23f262cca3c0c4eb7ab1933c3b1f03d021f2c48f54763065b6f0e321be/packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759", size = 65451 }, ] [[package]] @@ -1619,6 +1611,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/b6/a447b5e4ec71e13871be01ba81f5dfc9d0af7e473da256ff46bc0e24026f/tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde", size = 37955 }, ] +[[package]] +name = "tqdm" +version = "4.67.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540 }, +] + [[package]] name = "trove-classifiers" version = "2024.10.21.16" diff --git a/scripts/ci/pre_commit/helm_lint.py b/scripts/ci/pre_commit/helm_lint.py index 7663fb4672051..bb1150949aadb 100755 --- a/scripts/ci/pre_commit/helm_lint.py +++ b/scripts/ci/pre_commit/helm_lint.py @@ -33,7 +33,7 @@ sys.exit(res_setup.returncode) AIRFLOW_SOURCES_DIR = Path(__file__).parents[3].resolve() -HELM_BIN_PATH = AIRFLOW_SOURCES_DIR / ".build" / ".k8s-env" / "bin" / "helm" +HELM_BIN_PATH = AIRFLOW_SOURCES_DIR / ".build" / "k8s-env" / "bin" / "helm" result = subprocess.run( [os.fspath(HELM_BIN_PATH), "lint", ".", "-f", "values.yaml"], diff --git a/scripts/ci/pre_commit/kubeconform.py b/scripts/ci/pre_commit/kubeconform.py index 3bfccd2f8d251..2452c2158d53e 100755 --- a/scripts/ci/pre_commit/kubeconform.py +++ b/scripts/ci/pre_commit/kubeconform.py @@ -33,7 +33,7 @@ sys.exit(res_setup.returncode) AIRFLOW_SOURCES_DIR = Path(__file__).parents[3].resolve() -HELM_BIN_PATH = AIRFLOW_SOURCES_DIR / ".build" / ".k8s-env" / "bin" / "helm" +HELM_BIN_PATH = AIRFLOW_SOURCES_DIR / ".build" / "k8s-env" / "bin" / "helm" ps = subprocess.Popen( [os.fspath(HELM_BIN_PATH), "template", ".", "-f", "values.yaml"],