diff --git a/.github/actions/breeze/action.yml b/.github/actions/breeze/action.yml index 69ebcc7c66e6e..cbb1642b97812 100644 --- a/.github/actions/breeze/action.yml +++ b/.github/actions/breeze/action.yml @@ -22,6 +22,10 @@ inputs: python-version: description: 'Python version to use' default: "3.9" + use-uv: + description: 'Whether to use uv tool' + required: "true" + type: "string" outputs: host-python-version: description: Python version used in host @@ -33,13 +37,11 @@ runs: uses: actions/setup-python@v5 with: python-version: ${{ inputs.python-version }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/pyproject.toml + # NOTE! Installing Breeze without using cache is FASTER than when using cache - uv is so fast and has + # so low overhead, that just running upload cache/restore cache is slower than installing it from scratch - name: "Install Breeze" shell: bash run: ./scripts/ci/install_breeze.sh - env: - PYTHON_VERSION: ${{ inputs.python-version }} - name: "Free space" shell: bash run: breeze ci free-space @@ -56,3 +58,15 @@ runs: run: breeze setup config --no-cheatsheet --no-asciiart env: AIRFLOW_SOURCES_ROOT: "${{ github.workspace }}" + - name: "Use uv " + shell: bash + run: breeze setup config --use-uv + env: + AIRFLOW_SOURCES_ROOT: "${{ github.workspace }}" + if: inputs.use-uv == 'true' + - name: "Don't use uv " + shell: bash + run: breeze setup config --no-use-uv + env: + AIRFLOW_SOURCES_ROOT: "${{ github.workspace }}" + if: inputs.use-uv != 'true' diff --git a/.github/actions/install-pre-commit/action.yml b/.github/actions/install-pre-commit/action.yml index 8c704e5367ba2..dc175fd53d432 100644 --- a/.github/actions/install-pre-commit/action.yml +++ b/.github/actions/install-pre-commit/action.yml @@ -19,18 +19,19 @@ name: 'Install pre-commit' description: 'Installs pre-commit and related packages' inputs: + # TODO(potiuk): automate update of these versions python-version: description: 'Python version to use' - default: 3.9 + default: "3.9" uv-version: description: 'uv version to use' - default: 0.5.5 + default: "0.5.13" pre-commit-version: description: 'pre-commit version to use' - default: 4.0.1 + default: "4.0.1" pre-commit-uv-version: description: 'pre-commit-uv version to use' - default: 4.1.4 + default: "4.1.4" runs: using: "composite" steps: @@ -40,10 +41,35 @@ runs: pip install uv==${{inputs.uv-version}} || true uv tool install pre-commit==${{inputs.pre-commit-version}} --with uv==${{inputs.uv-version}} \ --with pre-commit-uv==${{inputs.pre-commit-uv-version}} - - name: Cache pre-commit envs - uses: actions/cache@v4 + working-directory: ${{ github.workspace }} + # We need to use tar file with archive to restore all the permissions and symlinks + - name: "Delete ~.cache" + run: | + du ~/ --max-depth=2 + echo + echo Deleting ~/.cache + echo + rm -rf ~/.cache + echo + shell: bash + - name: "Restore pre-commit cache" + uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c with: - path: ~/.cache/pre-commit - key: "pre-commit-${{inputs.python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}" - restore-keys: | - pre-commit-${{inputs.python-version}}- + key: cache-pre-commit-v4-${{ inputs.python-version }}-${{ hashFiles('.pre-commit-config.yaml') }} + path: /tmp/ + id: restore-pre-commit-cache + - name: "Restore .cache from the tar file" + run: tar -C ~ -xzf /tmp/cache-pre-commit.tar.gz + shell: bash + if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true' + - name: "Show restored files" + run: | + echo "Restored files" + du ~/ --max-depth=2 + echo + shell: bash + if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true' + - name: Install pre-commit hooks + shell: bash + run: pre-commit install-hooks || (cat ~/.cache/pre-commit/pre-commit.log && exit 1) + working-directory: ${{ github.workspace }} diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml index 2f8757c14cddd..4cfec042c5550 100644 --- a/.github/actions/prepare_breeze_and_image/action.yml +++ b/.github/actions/prepare_breeze_and_image/action.yml @@ -28,6 +28,9 @@ inputs: platform: description: 'Platform for the build - linux/amd64 or linux/arm64' required: true + use-uv: + description: 'Whether to use uv' + required: true outputs: host-python-version: description: Python version used in host @@ -40,11 +43,13 @@ runs: shell: bash - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} id: breeze - name: "Restore ${{ inputs.image-type }} docker image ${{ inputs.platform }}:${{ inputs.python }}" uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c with: - key: "${{ inputs.image-type }}-image-save-${{ inputs.platform }}-${{ inputs.python }}" + key: ${{ inputs.image-type }}-image-save-${{ inputs.platform }}-${{ inputs.python }} path: "/tmp/" - name: "Load ${{ inputs.image-type }} image ${{ inputs.platform }}:${{ inputs.python }}" run: > diff --git a/.github/actions/prepare_single_ci_image/action.yml b/.github/actions/prepare_single_ci_image/action.yml index e44a2e62321cc..9cdb7c6288f27 100644 --- a/.github/actions/prepare_single_ci_image/action.yml +++ b/.github/actions/prepare_single_ci_image/action.yml @@ -38,7 +38,7 @@ runs: - name: "Restore CI docker images ${{ inputs.platform }}:${{ inputs.python }}" uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c with: - key: "ci-image-save-${{ inputs.platform }}-${{ inputs.python }}" + key: ci-image-save-${{ inputs.platform }}-${{ inputs.python }} path: "/tmp/" if: contains(inputs.python-versions-list-as-string, inputs.python) - name: "Load CI image ${{ inputs.platform }}:${{ inputs.python }}" diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index 40196a6e04296..35389089eccd0 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -111,7 +111,7 @@ jobs: python-versions: ${{ inputs.python-versions }} branch: ${{ inputs.branch }} constraints-branch: ${{ inputs.constraints-branch }} - use-uv: ${{ inputs.use-uv}} + use-uv: ${{ inputs.use-uv }} include-success-outputs: ${{ inputs.include-success-outputs }} docker-cache: ${{ inputs.docker-cache }} disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} @@ -143,6 +143,8 @@ jobs: run: ./scripts/ci/cleanup_docker.sh - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} - name: "Login to ghcr.io" run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - name: "Check that image builds quickly" @@ -167,7 +169,7 @@ jobs: # platform: "linux/arm64" # branch: ${{ inputs.branch }} # constraints-branch: ${{ inputs.constraints-branch }} -# use-uv: ${{ inputs.use-uv}} +# use-uv: ${{ inputs.use-uv }} # upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} # docker-cache: ${{ inputs.docker-cache }} # disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index 62dd723999a8c..038bd74e0fbaa 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -56,6 +56,10 @@ on: # yamllint disable-line rule:truthy description: "Which version of python should be used by default" required: true type: string + use-uv: + description: "Whether to use uv" + required: true + type: string jobs: prod-image-extra-checks-main: name: PROD image extra checks (main) @@ -117,6 +121,7 @@ jobs: platform: "linux/amd64" image-type: "prod" python: ${{ inputs.default-python-version }} + use-uv: ${{ inputs.use-uv }} - name: "Test examples of PROD image building" run: " cd ./docker_tests && \ @@ -150,6 +155,7 @@ jobs: platform: "linux/amd64" image-type: "prod" python: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} + use-uv: ${{ inputs.use-uv }} id: breeze - name: "Test docker-compose quick start" run: breeze testing docker-compose-tests diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 47f80f05b7ac7..6c389efbe60f7 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -56,6 +56,10 @@ on: # yamllint disable-line rule:truthy description: "Whether to run only latest version checks (true/false)" required: true type: string + use-uv: + description: "Whether to use uv in the image" + required: true + type: string jobs: run-breeze-tests: timeout-minutes: 10 @@ -72,16 +76,12 @@ jobs: persist-credentials: false - name: "Cleanup docker" run: ./scripts/ci/cleanup_docker.sh - - uses: actions/setup-python@v5 + - name: "Install Breeze" + uses: ./.github/actions/breeze with: - python-version: "${{ inputs.default-python-version }}" - cache: 'pip' - cache-dependency-path: ./dev/breeze/pyproject.toml - - run: pip install --editable ./dev/breeze/ - - run: python -m pytest -n auto --color=yes + use-uv: ${{ inputs.use-uv }} + - run: uv tool run --from apache-airflow-breeze pytest -n auto --color=yes working-directory: ./dev/breeze/ - - tests-ui: timeout-minutes: 10 name: React UI tests @@ -108,15 +108,24 @@ jobs: node-version: 21 cache: 'pnpm' cache-dependency-path: 'airflow/ui/pnpm-lock.yaml' - - name: "Cache eslint" - uses: actions/cache@v4 + - name: "Restore eslint cache (ui)" + uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c with: - path: 'airflow/ui/node_modules' - key: ${{ runner.os }}-ui-node-modules-${{ hashFiles('airflow/ui/**/pnpm-lock.yaml') }} + path: airflow/ui/node_modules/ + key: cache-ui-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/ui/**/pnpm-lock.yaml') }} + id: restore-eslint-cache - run: cd airflow/ui && pnpm install --frozen-lockfile - run: cd airflow/ui && pnpm test env: FORCE_COLOR: 2 + - name: "Save eslint cache (ui)" + uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c + with: + path: airflow/ui/node_modules/ + key: cache-ui-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/ui/**/pnpm-lock.yaml') }} + if-no-files-found: 'error' + retention-days: '2' + if: steps.restore-eslint-cache.outputs.stash-hit != 'true' tests-www: timeout-minutes: 10 @@ -137,15 +146,66 @@ jobs: uses: actions/setup-node@v4 with: node-version: 21 - - name: "Cache eslint" - uses: actions/cache@v4 + - name: "Restore eslint cache (www)" + uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c with: - path: 'airflow/www/node_modules' - key: ${{ runner.os }}-www-node-modules-${{ hashFiles('airflow/www/**/yarn.lock') }} + path: airflow/www/node_modules/ + key: cache-www-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/www/**/yarn.lock') }} + id: restore-eslint-cache - run: yarn --cwd airflow/www/ install --frozen-lockfile --non-interactive - run: yarn --cwd airflow/www/ run test env: FORCE_COLOR: 2 + - name: "Save eslint cache (www)" + uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c + with: + path: airflow/www/node_modules/ + key: cache-www-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/www/**/yarn.lock') }} + if-no-files-found: 'error' + retention-days: '2' + if: steps.restore-eslint-cache.outputs.stash-hit != 'true' + + install-pre-commit: + timeout-minutes: 5 + name: "Install pre-commit for cache" + runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + env: + PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" + if: inputs.basic-checks-only == 'true' + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} + id: breeze + - name: "Install pre-commit" + uses: ./.github/actions/install-pre-commit + id: pre-commit + with: + python-version: ${{steps.breeze.outputs.host-python-version}} + - name: "Prepare .tar file from pre-commit cache" + run: | + tar -C ~ -czf /tmp/cache-pre-commit.tar.gz .cache/pre-commit .cache/uv + shell: bash + # Saving pre-commit cache should happen only in one job in the entire workflow - because otherwise + # it will cause 409 conflict errors - see https://github.com/actions/upload-artifact/issues/478 + # the way it works with airflow - even if the same action is in "ci-image-tests" the if condition + # above `if: inputs.basic-checks-only == 'true'` will prevent it from running in the other job + - name: "Save pre-commit[pre-commit] cache" + uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c + with: + # yamllint disable rule:line-length + key: cache-pre-commit-v4-${{ steps.breeze.outputs.host-python-version }}-${{ hashFiles('.pre-commit-config.yaml') }} + path: /tmp/cache-pre-commit.tar.gz + if-no-files-found: 'error' + retention-days: '2' # Those checks are run if no image needs to be built for checks. This is for simple changes that # Do not touch any of the python code or any of the important files that might require building @@ -154,6 +214,7 @@ jobs: timeout-minutes: 30 name: "Static checks: basic checks only" runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + needs: install-pre-commit if: inputs.basic-checks-only == 'true' steps: - name: "Cleanup repo" @@ -165,20 +226,10 @@ jobs: persist-credentials: false - name: "Cleanup docker" run: ./scripts/ci/cleanup_docker.sh - - name: "Setup python" - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.default-python-version }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/pyproject.toml - - name: "Setup python" - uses: actions/setup-python@v5 - with: - python-version: "${{ inputs.default-python-version }}" - cache: 'pip' - cache-dependency-path: ./dev/breeze/pyproject.toml - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} id: breeze - name: "Install pre-commit" uses: ./.github/actions/install-pre-commit @@ -216,6 +267,7 @@ jobs: timeout-minutes: 45 name: "Upgrade checks" runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + needs: install-pre-commit env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" if: inputs.canary-run == 'true' && inputs.latest-versions-only != 'true' @@ -229,12 +281,16 @@ jobs: persist-credentials: false - name: "Cleanup docker" run: ./scripts/ci/cleanup_docker.sh - # Install python from scratch. No cache used. We always want to have fresh version of everything - - uses: actions/setup-python@v5 + - name: "Install Breeze" + uses: ./.github/actions/breeze with: - python-version: "${{ inputs.default-python-version }}" - - name: "Install latest pre-commit" - run: pip install pre-commit + use-uv: ${{ inputs.use-uv }} + id: breeze + - name: "Install pre-commit" + uses: ./.github/actions/install-pre-commit + id: pre-commit + with: + python-version: ${{steps.breeze.outputs.host-python-version}} - name: "Autoupdate all pre-commits" run: pre-commit autoupdate - name: "Run automated upgrade for black" @@ -305,6 +361,8 @@ jobs: run: ./scripts/ci/cleanup_docker.sh - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} - name: "Cleanup dist files" run: rm -fv ./dist/* - name: Setup git for tagging diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index bed2233dfcb07..553a509e4bf11 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -114,7 +114,6 @@ jobs: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} - USE_UV: ${{ inputs.use-uv }} VERBOSE: "true" steps: - name: "Cleanup repo" @@ -128,6 +127,8 @@ jobs: run: ./scripts/ci/cleanup_docker.sh - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} - name: "Login to ghcr.io" run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - name: > @@ -159,8 +160,8 @@ jobs: - name: "Stash CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c with: - key: "ci-image-save-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + key: ci-image-save-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }} path: "/tmp/ci-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar" if-no-files-found: 'error' - retention-days: 2 + retention-days: '2' if: inputs.upload-image-artifact == 'true' diff --git a/.github/workflows/ci-image-checks.yml b/.github/workflows/ci-image-checks.yml index 8e0e5e71dd00a..24c843a094f8d 100644 --- a/.github/workflows/ci-image-checks.yml +++ b/.github/workflows/ci-image-checks.yml @@ -104,12 +104,59 @@ on: # yamllint disable-line rule:truthy description: "Whether to run coverage or not (true/false)" required: true type: string + use-uv: + description: "Whether to use uv to build the image (true/false)" + required: true + type: string jobs: + install-pre-commit: + timeout-minutes: 5 + name: "Install pre-commit for cache" + runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + env: + PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" + if: inputs.basic-checks-only == 'false' + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} + id: breeze + - name: "Install pre-commit" + uses: ./.github/actions/install-pre-commit + id: pre-commit + with: + python-version: ${{steps.breeze.outputs.host-python-version}} + - name: "Prepare .tar file from pre-commit cache" + run: | + tar -C ~ -czf /tmp/cache-pre-commit.tar.gz .cache/pre-commit .cache/uv + shell: bash + # Saving pre-commit cache should happen only in one job in the entire workflow - because otherwise + # it will cause 409 conflict errors - see https://github.com/actions/upload-artifact/issues/478 + # the way it works with airflow - even if the same action is in "basic-tests" the if condition + # above `if: inputs.basic-checks-only == 'false'` will prevent it from running in the other job + - name: "Save pre-commit cache" + uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c + with: + # yamllint disable rule:line-length + key: cache-pre-commit-v4-${{ steps.breeze.outputs.host-python-version }}-${{ hashFiles('.pre-commit-config.yaml') }} + path: /tmp/cache-pre-commit.tar.gz + if-no-files-found: 'error' + retention-days: '2' + static-checks: timeout-minutes: 45 name: "Static checks" runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + needs: install-pre-commit env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" UPGRADE_TO_NEWER_DEPENDENCIES: "${{ inputs.upgrade-to-newer-dependencies }}" @@ -123,17 +170,12 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.default-python-version }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/pyproject.toml - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: platform: "linux/amd64" python: ${{ inputs.default-python-version }} + use-uv: ${{ inputs.use-uv }} id: breeze - name: "Install pre-commit" uses: ./.github/actions/install-pre-commit @@ -154,6 +196,7 @@ jobs: timeout-minutes: 45 name: "MyPy checks" runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + needs: install-pre-commit if: inputs.needs-mypy == 'true' strategy: fail-fast: false @@ -175,6 +218,7 @@ jobs: with: platform: "linux/amd64" python: ${{ inputs.default-python-version }} + use-uv: ${{ inputs.use-uv }} id: breeze - name: "Install pre-commit" uses: ./.github/actions/install-pre-commit @@ -221,24 +265,32 @@ jobs: with: platform: "linux/amd64" python: ${{ inputs.default-python-version }} - - uses: actions/cache@v4 - id: cache-doc-inventories + use-uv: ${{ inputs.use-uv }} + - name: "Restore docs inventory cache" + uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c with: path: ./docs/_inventory_cache/ - key: docs-inventory-${{ hashFiles('pyproject.toml;') }} - restore-keys: | - docs-inventory-${{ hashFiles('pyproject.toml;') }} - docs-inventory- + # TODO(potiuk): do better with determining the key + key: cache-docs-inventory-v1-${{ hashFiles('pyproject.toml') }} + id: restore-docs-inventory-cache - name: "Building docs with ${{ matrix.flag }} flag" run: > breeze build-docs ${{ inputs.docs-list-as-string }} ${{ matrix.flag }} + - name: "Save docs inventory cache" + uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c + with: + path: ./docs/_inventory_cache/ + key: cache-docs-inventory-v1-${{ hashFiles('pyproject.toml') }} + if-no-files-found: 'error' + retention-days: '2' + if: steps.restore-docs-inventory-cache != 'true' - name: "Upload build docs" uses: actions/upload-artifact@v4 with: name: airflow-docs path: './docs/_build' - retention-days: 7 - if-no-files-found: error + retention-days: '7' + if-no-files-found: 'error' if: matrix.flag == '--docs-only' publish-docs: @@ -283,6 +335,7 @@ jobs: with: platform: "linux/amd64" python: ${{ inputs.default-python-version }} + use-uv: ${{ inputs.use-uv }} - name: "Publish docs" run: > breeze release-management publish-docs --override-versioned --run-in-parallel @@ -355,6 +408,7 @@ jobs: with: platform: "linux/amd64" python: ${{ inputs.default-python-version }} + use-uv: ${{ inputs.use-uv }} - name: "Generate airflow python client" run: > breeze release-management prepare-python-client --package-format both diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04e97f46d75ac..9ddeec96d2a08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,6 +141,7 @@ jobs: test-groups: ${{ steps.selective-checks.outputs.test-groups }} testable-core-integrations: ${{ steps.selective-checks.outputs.testable-core-integrations }} testable-providers-integrations: ${{ steps.selective-checks.outputs.testable-providers-integrations }} + use-uv: ${{ steps.selective-checks.outputs.force-pip == 'true' && 'false' || 'true' }} upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} steps: - name: "Cleanup repo" @@ -160,6 +161,9 @@ jobs: persist-credentials: false - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} + id: breeze - name: "Get information about the Workflow" id: source-run-info run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT} @@ -171,7 +175,6 @@ jobs: PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}" COMMIT_REF: "${{ github.sha }}" VERBOSE: "false" - run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} - name: env run: printenv @@ -193,6 +196,8 @@ jobs: skip-pre-commits: ${{needs.build-info.outputs.skip-pre-commits}} canary-run: ${{needs.build-info.outputs.canary-run}} latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}} + use-uv: ${{needs.build-info.outputs.use-uv}} + build-ci-images: name: Build CI images needs: [build-info] @@ -211,7 +216,7 @@ jobs: upload-image-artifact: "true" python-versions: ${{ needs.build-info.outputs.python-versions }} branch: ${{ needs.build-info.outputs.default-branch }} - use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }} + use-uv: ${{ needs.build-info.outputs.use-uv }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} docker-cache: ${{ needs.build-info.outputs.docker-cache }} @@ -239,7 +244,7 @@ jobs: latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} - use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }} + use-uv: ${{ needs.build-info.outputs.use-uv }} generate-constraints: @@ -257,6 +262,7 @@ jobs: generate-no-providers-constraints: ${{ needs.build-info.outputs.canary-run }} chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} + use-uv: ${{ needs.build-info.outputs.use-uv }} ci-image-checks: name: "CI image checks" @@ -285,6 +291,7 @@ jobs: needs-api-codegen: ${{ needs.build-info.outputs.needs-api-codegen }} default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} + use-uv: ${{ needs.build-info.outputs.use-uv }} providers: name: "Provider packages tests" @@ -308,6 +315,7 @@ jobs: skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} python-versions: ${{ needs.build-info.outputs.python-versions }} providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }} + use-uv: ${{ needs.build-info.outputs.use-uv }} tests-helm: name: "Helm tests" @@ -322,6 +330,7 @@ jobs: runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} + use-uv: ${{ needs.build-info.outputs.use-uv }} if: > needs.build-info.outputs.needs-helm-tests == 'true' && needs.build-info.outputs.default-branch == 'main' && @@ -351,6 +360,7 @@ jobs: run-migration-tests: "true" run-coverage: ${{ needs.build-info.outputs.run-coverage }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} + use-uv: ${{ needs.build-info.outputs.use-uv }} if: needs.build-info.outputs.run-tests == 'true' tests-mysql: @@ -377,6 +387,7 @@ jobs: run-coverage: ${{ needs.build-info.outputs.run-coverage }} run-migration-tests: "true" debug-resources: ${{ needs.build-info.outputs.debug-resources }} + use-uv: ${{ needs.build-info.outputs.use-uv }} if: needs.build-info.outputs.run-tests == 'true' tests-sqlite: @@ -405,6 +416,7 @@ jobs: run-coverage: ${{ needs.build-info.outputs.run-coverage }} run-migration-tests: "true" debug-resources: ${{ needs.build-info.outputs.debug-resources }} + use-uv: ${{ needs.build-info.outputs.use-uv }} if: needs.build-info.outputs.run-tests == 'true' tests-non-db: @@ -432,6 +444,7 @@ jobs: include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} + use-uv: ${{ needs.build-info.outputs.use-uv }} if: needs.build-info.outputs.run-tests == 'true' tests-special: @@ -462,6 +475,7 @@ jobs: upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} + use-uv: ${{ needs.build-info.outputs.use-uv }} tests-integration-system: name: Integration and System Tests @@ -482,6 +496,7 @@ jobs: skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} + use-uv: ${{ needs.build-info.outputs.use-uv }} if: needs.build-info.outputs.run-tests == 'true' tests-with-lowest-direct-resolution: @@ -512,6 +527,7 @@ jobs: run-coverage: ${{ needs.build-info.outputs.run-coverage }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} monitor-delay-time-in-seconds: 120 + use-uv: ${{ needs.build-info.outputs.use-uv }} build-prod-images: name: Build PROD images @@ -533,7 +549,7 @@ jobs: python-versions: ${{ needs.build-info.outputs.python-versions }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} branch: ${{ needs.build-info.outputs.default-branch }} - use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }} + use-uv: ${{ needs.build-info.outputs.use-uv }} build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} @@ -555,6 +571,7 @@ jobs: disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} canary-run: ${{ needs.build-info.outputs.canary-run }} + use-uv: ${{ needs.build-info.outputs.use-uv }} if: needs.build-info.outputs.prod-image-build == 'true' tests-kubernetes: @@ -570,7 +587,7 @@ jobs: runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }} + use-uv: ${{ needs.build-info.outputs.use-uv }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} kubernetes-combos: ${{ needs.build-info.outputs.kubernetes-combos }} if: > @@ -590,6 +607,7 @@ jobs: default-python-version: ${{ needs.build-info.outputs.default-python-version }} python-versions: ${{ needs.build-info.outputs.python-versions }} run-task-sdk-tests: ${{ needs.build-info.outputs.run-task-sdk-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} if: > ( needs.build-info.outputs.run-task-sdk-tests == 'true' || needs.build-info.outputs.run-tests == 'true' && @@ -624,6 +642,7 @@ jobs: docker-cache: ${{ needs.build-info.outputs.docker-cache }} disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} canary-run: ${{ needs.build-info.outputs.canary-run }} + use-uv: ${{ needs.build-info.outputs.use-uv }} notify-slack-failure: name: "Notify Slack on Failure" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index ca54b398202c6..3f4d7221fc61e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -49,6 +49,8 @@ jobs: persist-credentials: false - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: "true" - name: "Get information about the Workflow" id: source-run-info run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT} diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index b8fd240235f10..17f467162ceca 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -68,6 +68,10 @@ on: # yamllint disable-line rule:truthy description: "Whether this is a canary run (true/false)" required: true type: string + use-uv: + description: "Whether to use uv to build the image (true/false)" + required: true + type: string jobs: update-constraints: runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} @@ -140,7 +144,7 @@ jobs: python-versions: ${{ inputs.python-versions }} branch: ${{ inputs.branch }} constraints-branch: ${{ inputs.constraints-branch }} - use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }} + use-uv: ${{ inputs.use-uv }} include-success-outputs: ${{ inputs.include-success-outputs }} docker-cache: ${{ inputs.docker-cache }} disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} diff --git a/.github/workflows/generate-constraints.yml b/.github/workflows/generate-constraints.yml index 332d32e3160d2..d865c34efc435 100644 --- a/.github/workflows/generate-constraints.yml +++ b/.github/workflows/generate-constraints.yml @@ -40,6 +40,10 @@ on: # yamllint disable-line rule:truthy description: "Whether to run in debug mode (true/false)" required: true type: string + use-uv: + description: "Whether to use uvloop (true/false)" + required: true + type: string jobs: generate-constraints: permissions: @@ -66,6 +70,8 @@ jobs: persist-credentials: false - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} id: breeze - name: "Prepare all CI images: ${{ inputs.python-versions-list-as-string}}" uses: ./.github/actions/prepare_all_ci_images diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index d7f3f0b4d5bf0..5e3143b8a9e28 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -36,6 +36,10 @@ on: # yamllint disable-line rule:truthy description: "Which version of python should be used by default" required: true type: string + use-uv: + description: "Whether to use uvloop (true/false)" + required: true + type: string jobs: tests-helm: timeout-minutes: 80 @@ -70,6 +74,7 @@ jobs: with: platform: "linux/amd64" python: ${{ inputs.default-python-version }} + use-uv: ${{ inputs.use-uv }} - name: "Helm Unit Tests: ${{ matrix.helm-test-package }}" run: breeze testing helm-tests --test-type "${{ matrix.helm-test-package }}" @@ -91,6 +96,8 @@ jobs: run: ./scripts/ci/cleanup_docker.sh - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} - name: Setup git for tagging run: | git config --global user.email "name@example.com" diff --git a/.github/workflows/integration-system-tests.yml b/.github/workflows/integration-system-tests.yml index 0a6fd1fb406c7..5d5773846daa6 100644 --- a/.github/workflows/integration-system-tests.yml +++ b/.github/workflows/integration-system-tests.yml @@ -60,6 +60,10 @@ on: # yamllint disable-line rule:truthy description: "Debug resources (true/false)" required: true type: string + use-uv: + description: "Whether to use uv" + required: true + type: string jobs: tests-core-integration: timeout-minutes: 130 @@ -95,6 +99,7 @@ jobs: with: platform: "linux/amd64" python: ${{ inputs.default-python-version }} + use-uv: ${{ inputs.use-uv }} - name: "Integration: core ${{ matrix.integration }}" # yamllint disable rule:line-length run: ./scripts/ci/testing/run_integration_tests_with_retry.sh core "${{ matrix.integration }}" @@ -141,6 +146,7 @@ jobs: with: platform: "linux/amd64" python: ${{ inputs.default-python-version }} + use-uv: ${{ inputs.use-uv }} - name: "Integration: providers ${{ matrix.integration }}" run: ./scripts/ci/testing/run_integration_tests_with_retry.sh providers "${{ matrix.integration }}" - name: "Post Tests success" @@ -182,6 +188,7 @@ jobs: with: platform: "linux/amd64" python: ${{ inputs.default-python-version }} + use-uv: ${{ inputs.use-uv }} - name: "System Tests" run: > ./scripts/ci/testing/run_system_tests.sh diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml index e83cf68fb0e6f..43ae3c9ed8581 100644 --- a/.github/workflows/k8s-tests.yml +++ b/.github/workflows/k8s-tests.yml @@ -49,10 +49,56 @@ on: # yamllint disable-line rule:truthy required: true type: string jobs: + cache-k8s-environment: + timeout-minutes: 10 + name: "K8S System: prepare cached environment" + runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + VERBOSE: "true" + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} + id: breeze + - name: "Restore cache folder with tools for kubernetes testing" + uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c + with: + key: "\ + cache-k8s-env-v1-${{ steps.breeze.outputs.host-python-version }}-\ + ${{ hashFiles('scripts/ci/kubernetes/k8s_requirements.txt','hatch_build.py') }}" + path: ".build/k8s-env" + id: restore-k8s-env-cache + - name: "Setup K8S environment" + run: breeze k8s setup-env --force-venv-setup + env: + VERBOSE: "false" + - name: "Save cache folder with tools for kubernetes testing" + uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c + with: + key: "\ + cache-k8s-env-v1-${{ steps.breeze.outputs.host-python-version }}-\ + ${{ hashFiles('scripts/ci/kubernetes/k8s_requirements.txt','hatch_build.py') }}" + path: ".build/k8s-env" + if-no-files-found: 'error' + retention-days: '2' + if: steps.restore-k8s-env-cache.outputs.stash-hit != 'true' + tests-kubernetes: - timeout-minutes: 240 + timeout-minutes: 60 name: "K8S System:${{ matrix.executor }}-${{ matrix.kubernetes-combo }}-${{ matrix.use-standard-naming }}" runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + needs: cache-k8s-environment strategy: matrix: executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor] @@ -85,13 +131,16 @@ jobs: platform: ${{ inputs.platform }} image-type: "prod" python: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} - - name: "Cache bin folder with tools for kubernetes testing" - uses: actions/cache@v4 + use-uv: ${{ inputs.use-uv }} + id: breeze + - name: "Restore cache folder with tools for kubernetes testing" + uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c with: - path: ".build/.k8s-env" key: "\ - k8s-env-${{ steps.breeze.outputs.host-python-version }}-\ + cache-k8s-env-v1-${{ steps.breeze.outputs.host-python-version }}-\ ${{ hashFiles('scripts/ci/kubernetes/k8s_requirements.txt','hatch_build.py') }}" + path: ".build/k8s-env" + id: restore-k8s-env-cache - name: "\ Run complete K8S tests ${{ matrix.executor }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}-\ ${{env.KUBERNETES_VERSION}}-${{ matrix.use-standard-naming }}" @@ -110,7 +159,7 @@ jobs: kind-logs-${{ matrix.kubernetes-combo }}-${{ matrix.executor }}-\ ${{ matrix.use-standard-naming }}" path: /tmp/kind_logs_* - retention-days: 7 + retention-days: '7' - name: "\ Upload test resource logs on failure ${{ matrix.executor }}-${{ matrix.kubernetes-combo }}-\ ${{ matrix.use-standard-naming }}" @@ -121,7 +170,7 @@ jobs: k8s-test-resources-${{ matrix.kubernetes-combo }}-${{ matrix.executor }}-\ ${{ matrix.use-standard-naming }}" path: /tmp/k8s_test_resources_* - retention-days: 7 + retention-days: '7' - name: "Delete clusters just in case they are left" run: breeze k8s delete-cluster --all if: always() diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index 4ee3538429222..81e4346e6a0d5 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -133,16 +133,14 @@ jobs: - name: "Cleanup docker" run: ./scripts/ci/cleanup_docker.sh if: inputs.upload-package-artifact == 'true' - - uses: actions/setup-python@v5 - with: - python-version: "${{ inputs.default-python-version }}" - if: inputs.upload-package-artifact == 'true' - name: "Cleanup dist and context file" shell: bash run: rm -fv ./dist/* ./docker-context-files/* if: inputs.upload-package-artifact == 'true' - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} if: inputs.upload-package-artifact == 'true' - name: "Prepare providers packages" shell: bash @@ -206,7 +204,6 @@ jobs: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} - USE_UV: ${{ inputs.use-uv }} VERBOSE: "true" steps: - name: "Cleanup repo" @@ -220,6 +217,8 @@ jobs: run: ./scripts/ci/cleanup_docker.sh - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} - name: "Cleanup dist and context file" shell: bash run: rm -fv ./dist/* ./docker-context-files/* @@ -280,8 +279,8 @@ jobs: - name: "Stash PROD docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c with: - key: "prod-image-save-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + key: prod-image-save-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }} path: "/tmp/prod-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar" if-no-files-found: 'error' - retention-days: 2 + retention-days: '2' if: inputs.upload-image-artifact == 'true' diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index 004c97a647bd9..2d125e35066dc 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -111,7 +111,6 @@ jobs: INCLUDE_SUCCESS_OUTPUTS: "${{ inputs.include-success-outputs }}" INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python }}" - USE_UV: ${{ inputs.use-uv }} UPGRADE_TO_NEWER_DEPENDENCIES: "false" VERBOSE: "true" VERSION_SUFFIX_FOR_PYPI: "dev0" @@ -127,6 +126,8 @@ jobs: run: ./scripts/ci/cleanup_docker.sh - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} - name: Login to ghcr.io run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - name: "Push CI latest images: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} (linux/amd64 only)" @@ -176,7 +177,6 @@ jobs: INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python }}" UPGRADE_TO_NEWER_DEPENDENCIES: "false" - USE_UV: ${{ inputs.branch == 'main' && inputs.use-uv || 'false' }} VERBOSE: "true" VERSION_SUFFIX_FOR_PYPI: "dev0" if: inputs.include-prod-images == 'true' @@ -192,6 +192,8 @@ jobs: run: ./scripts/ci/cleanup_docker.sh - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} - name: "Cleanup dist and context file" run: rm -fv ./dist/* ./docker-context-files/* - name: "Download packages prepared as artifacts" diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml index 5ce1585131f76..f23fdc9974abe 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_dockerhub_image.yml @@ -63,11 +63,14 @@ jobs: run: ./scripts/ci/cleanup_docker.sh - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: "false" - name: Selective checks id: selective-checks env: VERBOSE: "false" run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} + release-images: timeout-minutes: 120 name: "Release images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}" @@ -99,6 +102,8 @@ jobs: run: ./scripts/ci/cleanup_docker.sh - name: "Install Breeze" uses: ./.github/actions/breeze + with: + use-uv: "false" - name: Free space run: breeze ci free-space --answer yes - name: "Cleanup dist and context file" diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 3a9440e528eaf..609f369f80d80 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -112,6 +112,10 @@ on: # yamllint disable-line rule:truthy required: false default: 20 type: number + use-uv: + description: "Whether to use uv" + required: true + type: string jobs: tests: timeout-minutes: 120 @@ -163,6 +167,7 @@ jobs: with: platform: "linux/amd64" python: ${{ matrix.python-version }} + use-uv: ${{ inputs.use-uv }} - name: > Migration Tests: ${{ matrix.python-version }}:${{ env.PARALLEL_TEST_TYPES }} uses: ./.github/actions/migration_tests diff --git a/.github/workflows/special-tests.yml b/.github/workflows/special-tests.yml index d416d55575fb9..cca54524b7dca 100644 --- a/.github/workflows/special-tests.yml +++ b/.github/workflows/special-tests.yml @@ -76,6 +76,10 @@ on: # yamllint disable-line rule:truthy description: "Whether to debug resources or not (true/false)" required: true type: string + use-uv: + description: "Whether to use uv or not (true/false)" + required: true + type: string jobs: tests-min-sqlalchemy: @@ -100,6 +104,7 @@ jobs: providers-test-types-list-as-string: ${{ inputs.providers-test-types-list-as-string }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} + use-uv: ${{ inputs.use-uv }} tests-boto: name: "Latest Boto test" @@ -124,6 +129,7 @@ jobs: include-success-outputs: ${{ inputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} + use-uv: ${{ inputs.use-uv }} tests-pendulum-2: name: "Pendulum2 test" @@ -148,6 +154,7 @@ jobs: include-success-outputs: ${{ inputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} + use-uv: ${{ inputs.use-uv }} tests-quarantined: name: "Quarantined test" @@ -171,6 +178,7 @@ jobs: include-success-outputs: ${{ inputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} + use-uv: ${{ inputs.use-uv }} tests-arm-collection: name: "ARM Collection test" @@ -194,6 +202,7 @@ jobs: include-success-outputs: ${{ inputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} + use-uv: ${{ inputs.use-uv }} if: ${{ inputs.default-branch == 'main' }} tests-system: @@ -218,3 +227,4 @@ jobs: include-success-outputs: ${{ inputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} + use-uv: ${{ inputs.use-uv }} diff --git a/.github/workflows/task-sdk-tests.yml b/.github/workflows/task-sdk-tests.yml index 756a66e546479..61b9ba75b5bee 100644 --- a/.github/workflows/task-sdk-tests.yml +++ b/.github/workflows/task-sdk-tests.yml @@ -36,6 +36,10 @@ on: # yamllint disable-line rule:truthy description: "Whether to run Task SDK tests or not (true/false)" required: true type: string + use-uv: + description: "Whether to use uv to build the image (true/false)" + required: true + type: string jobs: task-sdk-tests: timeout-minutes: 80 @@ -66,6 +70,7 @@ jobs: with: platform: "linux/amd64" python: ${{ matrix.python-version }} + use-uv: ${{ inputs.use-uv }} - name: "Cleanup dist files" run: rm -fv ./dist/* - name: "Prepare Task SDK packages: wheel" diff --git a/.github/workflows/test-provider-packages.yml b/.github/workflows/test-provider-packages.yml index 7b2bc485528c4..3d79c6896b8aa 100644 --- a/.github/workflows/test-provider-packages.yml +++ b/.github/workflows/test-provider-packages.yml @@ -58,6 +58,10 @@ on: # yamllint disable-line rule:truthy description: "JSON-formatted array of Python versions to build images from" required: true type: string + use-uv: + description: "Whether to use uv" + required: true + type: string jobs: prepare-install-verify-provider-packages: timeout-minutes: 80 @@ -87,6 +91,7 @@ jobs: with: platform: "linux/amd64" python: ${{ inputs.default-python-version }} + use-uv: ${{ inputs.use-uv }} - name: "Cleanup dist files" run: rm -fv ./dist/* - name: "Prepare provider documentation" @@ -175,6 +180,7 @@ jobs: with: platform: "linux/amd64" python: ${{ matrix.python-version }} + use-uv: ${{ inputs.use-uv }} - name: "Cleanup dist files" run: rm -fv ./dist/* - name: "Prepare provider packages: wheel" diff --git a/contributing-docs/testing/k8s_tests.rst b/contributing-docs/testing/k8s_tests.rst index a9ba3151fe9bb..4891182f69923 100644 --- a/contributing-docs/testing/k8s_tests.rst +++ b/contributing-docs/testing/k8s_tests.rst @@ -34,7 +34,7 @@ Before running ``breeze k8s`` cluster commands you need to setup the environment by ``breeze k8s setup-env`` command. Breeze in this command makes sure to download tools that are needed to run k8s tests: Helm, Kind, Kubectl in the right versions and sets up a Python virtualenv that is needed to run the tests. All those tools and env are setup in -``.build/.k8s-env`` folder. You can activate this environment yourselves as usual by sourcing +``.build/k8s-env`` folder. You can activate this environment yourselves as usual by sourcing ``bin/activate`` script, but since we are supporting multiple clusters in the same installation it is best if you use ``breeze k8s shell`` with the right parameters specifying which cluster to use. @@ -129,8 +129,8 @@ Entering shell with Kubernetes Cluster This shell is prepared to run Kubernetes tests interactively. It has ``kubectl`` and ``kind`` cli tools available in the path, it has also activated virtualenv environment that allows you to run tests via pytest. -The virtualenv is available in ./.build/.k8s-env/ -The binaries are available in ``.build/.k8s-env/bin`` path. +The virtualenv is available in ./.build/k8s-env/ +The binaries are available in ``.build/k8s-env/bin`` path. .. code-block:: bash @@ -184,26 +184,26 @@ The typical session for tests with Kubernetes looks like follows: breeze k8s setup-env The first time you run it, it should result in creating the virtualenv and installing good versions -of kind, kubectl and helm. All of them are installed in ``./build/.k8s-env`` (binaries available in ``bin`` +of kind, kubectl and helm. All of them are installed in ``./build/k8s-env`` (binaries available in ``bin`` sub-folder of it). .. code-block:: text - Initializing K8S virtualenv in /Users/jarek/IdeaProjects/airflow/.build/.k8s-env - Reinstalling PIP version in /Users/jarek/IdeaProjects/airflow/.build/.k8s-env - Installing necessary packages in /Users/jarek/IdeaProjects/airflow/.build/.k8s-env + Initializing K8S virtualenv in /Users/jarek/IdeaProjects/airflow/.build/k8s-env + Reinstalling PIP version in /Users/jarek/IdeaProjects/airflow/.build/k8s-env + Installing necessary packages in /Users/jarek/IdeaProjects/airflow/.build/k8s-env The ``kind`` tool is not downloaded yet. Downloading 0.14.0 version. Downloading from: https://github.com/kubernetes-sigs/kind/releases/download/v0.14.0/kind-darwin-arm64 The ``kubectl`` tool is not downloaded yet. Downloading 1.24.3 version. Downloading from: https://storage.googleapis.com/kubernetes-release/release/v1.24.3/bin/darwin/arm64/kubectl The ``helm`` tool is not downloaded yet. Downloading 3.9.2 version. Downloading from: https://get.helm.sh/helm-v3.9.2-darwin-arm64.tar.gz - Extracting the darwin-arm64/helm to /Users/jarek/IdeaProjects/airflow/.build/.k8s-env/bin - Moving the helm to /Users/jarek/IdeaProjects/airflow/.build/.k8s-env/bin/helm + Extracting the darwin-arm64/helm to /Users/jarek/IdeaProjects/airflow/.build/k8s-env/bin + Moving the helm to /Users/jarek/IdeaProjects/airflow/.build/k8s-env/bin/helm This prepares the virtual environment for tests and downloads the right versions of the tools -to ``./build/.k8s-env`` +to ``./build/k8s-env`` 2. Create the KinD cluster: @@ -404,10 +404,10 @@ Should show the status of current KinD cluster. .. code-block:: text - K8S Virtualenv is initialized in /Users/jarek/IdeaProjects/airflow/.build/.k8s-env - Good version of kind installed: 0.14.0 in /Users/jarek/IdeaProjects/airflow/.build/.k8s-env/bin - Good version of kubectl installed: 1.25.0 in /Users/jarek/IdeaProjects/airflow/.build/.k8s-env/bin - Good version of helm installed: 3.9.2 in /Users/jarek/IdeaProjects/airflow/.build/.k8s-env/bin + K8S Virtualenv is initialized in /Users/jarek/IdeaProjects/airflow/.build/k8s-env + Good version of kind installed: 0.14.0 in /Users/jarek/IdeaProjects/airflow/.build/k8s-env/bin + Good version of kubectl installed: 1.25.0 in /Users/jarek/IdeaProjects/airflow/.build/k8s-env/bin + Good version of helm installed: 3.9.2 in /Users/jarek/IdeaProjects/airflow/.build/k8s-env/bin Stable repo is already added Uploading Airflow image ghcr.io/apache/airflow/main/prod/python3.9-kubernetes to cluster airflow-python-3.9-v1.24.2 Image: "ghcr.io/apache/airflow/main/prod/python3.9-kubernetes" with ID "sha256:fb6195f7c2c2ad97788a563a3fe9420bf3576c85575378d642cd7985aff97412" not yet present on node "airflow-python-3.9-v1.24.2-worker", loading... @@ -511,7 +511,7 @@ The virtualenv required will be created automatically when the scripts are run. Running tests with kind-airflow-python-3.9-v1.24.2 cluster. Command to run: pytest kubernetes_tests ========================================================================================= test session starts ========================================================================================== - platform darwin -- Python 3.9.9, pytest-6.2.5, py-1.11.0, pluggy-1.0.0 -- /Users/jarek/IdeaProjects/airflow/.build/.k8s-env/bin/python + platform darwin -- Python 3.9.9, pytest-6.2.5, py-1.11.0, pluggy-1.0.0 -- /Users/jarek/IdeaProjects/airflow/.build/k8s-env/bin/python cachedir: .pytest_cache rootdir: /Users/jarek/IdeaProjects/airflow/kubernetes_tests plugins: anyio-3.6.1, instafail-0.4.2, xdist-2.5.0, forked-1.4.0, timeouts-1.2.1, cov-3.0.0 @@ -525,7 +525,7 @@ The virtualenv required will be created automatically when the scripts are run. 8b) You can enter an interactive shell to run tests one-by-one -This enters the virtualenv in ``.build/.k8s-env`` folder: +This enters the virtualenv in ``.build/k8s-env`` folder: .. code-block:: bash @@ -561,7 +561,7 @@ tests using the standard IntelliJ Run/Debug feature. You just need a few steps: :alt: Kubernetes testing virtualenv The virtualenv is created in your "Airflow" source directory in the -``.build/.k8s-env`` folder and you have to find ``python`` binary and choose +``.build/k8s-env`` folder and you have to find ``python`` binary and choose it when selecting interpreter. 9b) Choose pytest as test runner: diff --git a/dev/breeze/doc/05_test_commands.rst b/dev/breeze/doc/05_test_commands.rst index 819ac80224525..d9eb0856bbd76 100644 --- a/dev/breeze/doc/05_test_commands.rst +++ b/dev/breeze/doc/05_test_commands.rst @@ -338,7 +338,7 @@ Kubernetes environment can be set with the ``breeze k8s setup-env`` command. It will create appropriate virtualenv to run tests and download the right set of tools to run the tests: ``kind``, ``kubectl`` and ``helm`` in the right versions. You can re-run the command when you want to make sure the expected versions of the tools are installed properly in the -virtualenv. The Virtualenv is available in ``.build/.k8s-env/bin`` subdirectory of your Airflow +virtualenv. The Virtualenv is available in ``.build/k8s-env/bin`` subdirectory of your Airflow installation. .. image:: ./images/output_k8s_setup-env.svg @@ -561,7 +561,7 @@ be created and airflow deployed to it before running the tests): (kind-airflow-python-3.9-v1.24.0:KubernetesExecutor)> pytest test_kubernetes_executor.py ================================================= test session starts ================================================= - platform linux -- Python 3.10.6, pytest-6.2.5, py-1.11.0, pluggy-1.0.0 -- /home/jarek/code/airflow/.build/.k8s-env/bin/python + platform linux -- Python 3.10.6, pytest-6.2.5, py-1.11.0, pluggy-1.0.0 -- /home/jarek/code/airflow/.build/k8s-env/bin/python cachedir: .pytest_cache rootdir: /home/jarek/code/airflow, configfile: pytest.ini plugins: anyio-3.6.1 @@ -571,8 +571,8 @@ be created and airflow deployed to it before running the tests): test_kubernetes_executor.py::TestKubernetesExecutor::test_integration_run_dag_with_scheduler_failure PASSED [100%] ================================================== warnings summary =================================================== - .build/.k8s-env/lib/python3.10/site-packages/_pytest/config/__init__.py:1233 - /home/jarek/code/airflow/.build/.k8s-env/lib/python3.10/site-packages/_pytest/config/__init__.py:1233: PytestConfigWarning: Unknown config option: asyncio_mode + .build/k8s-env/lib/python3.10/site-packages/_pytest/config/__init__.py:1233 + /home/jarek/code/airflow/.build/k8s-env/lib/python3.10/site-packages/_pytest/config/__init__.py:1233: PytestConfigWarning: Unknown config option: asyncio_mode self._warn_or_fail_if_strict(f"Unknown config option: {key}\n") diff --git a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py index b9bdc5302bdfc..8525d1e9acc50 100644 --- a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py @@ -51,7 +51,7 @@ from airflow_breeze.utils.shared_options import get_dry_run, get_verbose from airflow_breeze.utils.virtualenv_utils import create_pip_command, create_uv_command -K8S_ENV_PATH = BUILD_CACHE_DIR / ".k8s-env" +K8S_ENV_PATH = BUILD_CACHE_DIR / "k8s-env" K8S_CLUSTERS_PATH = BUILD_CACHE_DIR / ".k8s-clusters" K8S_BIN_BASE_PATH = K8S_ENV_PATH / "bin" KIND_BIN_PATH = K8S_BIN_BASE_PATH / "kind" @@ -305,8 +305,10 @@ def _requirements_changed() -> bool: def _install_packages_in_k8s_virtualenv(): if check_if_cache_exists("use_uv"): + get_console().print("[info]Using uv to install k8s env[/]") command = create_uv_command(PYTHON_BIN_PATH) else: + get_console().print("[info]Using pip to install k8s env[/]") command = create_pip_command(PYTHON_BIN_PATH) install_command_no_constraints = [ *command, @@ -406,7 +408,7 @@ def create_virtualenv(force_venv_setup: bool) -> RunCommandResult: f"{venv_command_result.stdout}\n{venv_command_result.stderr}" ) return venv_command_result - get_console().print(f"[info]Reinstalling PIP version in {K8S_ENV_PATH}") + get_console().print(f"[info]Reinstalling pip=={PIP_VERSION} in {K8S_ENV_PATH}") command = create_pip_command(PYTHON_BIN_PATH) pip_reinstall_result = run_command( [*command, "install", f"pip=={PIP_VERSION}"], @@ -419,6 +421,7 @@ def create_virtualenv(force_venv_setup: bool) -> RunCommandResult: f"{pip_reinstall_result.stdout}\n{pip_reinstall_result.stderr}" ) return pip_reinstall_result + get_console().print(f"[info]Reinstalling uv=={UV_VERSION} in {K8S_ENV_PATH}") uv_reinstall_result = run_command( [*command, "install", f"uv=={UV_VERSION}"], check=False, diff --git a/dev/breeze/src/airflow_breeze/utils/virtualenv_utils.py b/dev/breeze/src/airflow_breeze/utils/virtualenv_utils.py index 3ececdd228461..7cd4b7dbd7198 100644 --- a/dev/breeze/src/airflow_breeze/utils/virtualenv_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/virtualenv_utils.py @@ -44,8 +44,12 @@ def create_venv( requirements_file: str | Path | None = None, ) -> str: venv_path = Path(venv_path).resolve().absolute() + if check_if_cache_exists("use_uv"): + cmd = ["uv", "venv"] + else: + cmd = [python or sys.executable, "-m", "venv"] venv_command_result = run_command( - [python or sys.executable, "-m", "venv", venv_path.as_posix()], + [*cmd, "venv", venv_path.as_posix()], check=False, capture_output=True, ) diff --git a/dev/breeze/uv.lock b/dev/breeze/uv.lock index d400d08f40f58..9b98a8746a2ba 100644 --- a/dev/breeze/uv.lock +++ b/dev/breeze/uv.lock @@ -52,13 +52,14 @@ dependencies = [ { name = "semver" }, { name = "tabulate" }, { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "tqdm" }, { name = "twine" }, ] [package.metadata] requires-dist = [ { name = "black", specifier = ">=23.11.0" }, - { name = "click", specifier = ">=8.1.7" }, + { name = "click", specifier = ">=8.1.8" }, { name = "filelock", specifier = ">=3.13.0" }, { name = "flit", specifier = "==3.10.1" }, { name = "flit-core", specifier = "==3.10.1" }, @@ -66,8 +67,7 @@ requires-dist = [ { name = "google-api-python-client", specifier = ">=2.142.0" }, { name = "google-auth-httplib2", specifier = ">=0.2.0" }, { name = "google-auth-oauthlib", specifier = ">=1.2.0" }, - { name = "hatch", specifier = "==1.9.4" }, - { name = "importlib-resources", marker = "python_full_version < '3.9'", specifier = ">=5.2,!=6.2.0,!=6.3.0,!=6.3.1" }, + { name = "hatch", specifier = "==1.14.0" }, { name = "inputimeout", specifier = ">=1.0.4" }, { name = "jinja2", specifier = ">=3.1.0" }, { name = "jsonschema", specifier = ">=4.19.1" }, @@ -85,6 +85,7 @@ requires-dist = [ { name = "semver", specifier = ">=3.0.2" }, { name = "tabulate", specifier = ">=0.9.0" }, { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0.1" }, + { name = "tqdm", specifier = ">=4.67.1" }, { name = "twine", specifier = ">=4.0.2" }, ] @@ -326,14 +327,14 @@ wheels = [ [[package]] name = "click" -version = "8.1.7" +version = "8.1.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "platform_system == 'Windows'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/96/d3/f04c7bfcf5c1862a2a5b845c6b2b360488cf47af55dfa79c98f6a6bf98b5/click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de", size = 336121 } +sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593 } wheels = [ - { url = "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", size = 97941 }, + { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188 }, ] [[package]] @@ -412,15 +413,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", size = 587408 }, ] -[[package]] -name = "editables" -version = "0.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/37/4a/986d35164e2033ddfb44515168a281a7986e260d344cf369c3f52d4c3275/editables-0.5.tar.gz", hash = "sha256:309627d9b5c4adc0e668d8c6fa7bac1ba7c8c5d415c2d27f60f081f8e80d1de2", size = 14744 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/be/0f2f4a5e8adc114a02b63d92bf8edbfa24db6fc602fca83c885af2479e0e/editables-0.5-py3-none-any.whl", hash = "sha256:61e5ffa82629e0d8bfe09bc44a07db3c1ab8ed1ce78a6980732870f19b5e7d4c", size = 5098 }, -] - [[package]] name = "exceptiongroup" version = "1.2.2" @@ -592,7 +584,7 @@ wheels = [ [[package]] name = "hatch" -version = "1.9.4" +version = "1.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -608,29 +600,29 @@ dependencies = [ { name = "tomli-w" }, { name = "tomlkit" }, { name = "userpath" }, + { name = "uv" }, { name = "virtualenv" }, { name = "zstandard" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/17/98/63bf6c592b65f67201db292489053b86310cfb107eb095d345398e00cbd3/hatch-1.9.4.tar.gz", hash = "sha256:9bb7d1c4a7a51cc1f9e16394875c940b45fa84b698f0291529316b27d74e7f32", size = 689598 } +sdist = { url = "https://files.pythonhosted.org/packages/bc/15/b4e3d50d8177e6e8a243b24d9819e3807f7bfd3b2bebe7b5aef32a9c79cb/hatch-1.14.0.tar.gz", hash = "sha256:351e41bc6c72bc93cb98651212226e495b43549eee27c487832e459e5d0f0eda", size = 5188143 } wheels = [ - { url = "https://files.pythonhosted.org/packages/05/38/ba8f90264d19ed39851f37a22f2a4be8e9644a1203f114b16647f954bb02/hatch-1.9.4-py3-none-any.whl", hash = "sha256:461eb86b4b46249e38a9a621c7239e61285fd8e14b5a1b5a727c394893a25300", size = 110812 }, + { url = "https://files.pythonhosted.org/packages/85/c6/ad910cdb79600af0100b7c4f7093eb4b95a2b44e589e66b6b938b09cc6f9/hatch-1.14.0-py3-none-any.whl", hash = "sha256:b12c7a2f4aaf6db7180e35c476e1a2ad4ec7197c20c4332964599424d4918ded", size = 125763 }, ] [[package]] name = "hatchling" -version = "1.21.1" +version = "1.27.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "editables" }, { name = "packaging" }, { name = "pathspec" }, { name = "pluggy" }, { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "trove-classifiers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d8/a1/7dd1caa87c0b15c04c6291e25112e5d082cce02ee87f221a8be1d594f857/hatchling-1.21.1.tar.gz", hash = "sha256:bba440453a224e7d4478457fa2e8d8c3633765bafa02975a6b53b9bf917980bc", size = 58059 } +sdist = { url = "https://files.pythonhosted.org/packages/8f/8a/cc1debe3514da292094f1c3a700e4ca25442489731ef7c0814358816bb03/hatchling-1.27.0.tar.gz", hash = "sha256:971c296d9819abb3811112fc52c7a9751c8d381898f36533bb16f9791e941fd6", size = 54983 } wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/bb/40528a09a33845bd7fd75c33b3be7faec3b5c8f15f68a58931da67420fb9/hatchling-1.21.1-py3-none-any.whl", hash = "sha256:21e8c13f8458b219a91cb84e5b61c15bf786695d1c4fabc29e91e78f94bfe892", size = 76740 }, + { url = "https://files.pythonhosted.org/packages/08/e7/ae38d7a6dfba0533684e0b2136817d667588ae3ec984c1a4e5df5eb88482/hatchling-1.27.0-py3-none-any.whl", hash = "sha256:d3a2f3567c4f926ea39849cdf924c7e99e6686c9c8e288ae1037c8fa2a5d937b", size = 75794 }, ] [[package]] @@ -985,11 +977,11 @@ wheels = [ [[package]] name = "packaging" -version = "24.1" +version = "24.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/51/65/50db4dda066951078f0a96cf12f4b9ada6e4b811516bf0262c0f4f7064d4/packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002", size = 148788 } +sdist = { url = "https://files.pythonhosted.org/packages/d0/63/68dbb6eb2de9cb10ee4c9c14a0148804425e13c4fb20d61cce69f53106da/packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f", size = 163950 } wheels = [ - { url = "https://files.pythonhosted.org/packages/08/aa/cc0199a5f0ad350994d660967a8efb233fe0416e4639146c089643407ce6/packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124", size = 53985 }, + { url = "https://files.pythonhosted.org/packages/88/ef/eb23f262cca3c0c4eb7ab1933c3b1f03d021f2c48f54763065b6f0e321be/packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759", size = 65451 }, ] [[package]] @@ -1619,6 +1611,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/b6/a447b5e4ec71e13871be01ba81f5dfc9d0af7e473da256ff46bc0e24026f/tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde", size = 37955 }, ] +[[package]] +name = "tqdm" +version = "4.67.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540 }, +] + [[package]] name = "trove-classifiers" version = "2024.10.21.16" diff --git a/scripts/ci/pre_commit/helm_lint.py b/scripts/ci/pre_commit/helm_lint.py index 7663fb4672051..bb1150949aadb 100755 --- a/scripts/ci/pre_commit/helm_lint.py +++ b/scripts/ci/pre_commit/helm_lint.py @@ -33,7 +33,7 @@ sys.exit(res_setup.returncode) AIRFLOW_SOURCES_DIR = Path(__file__).parents[3].resolve() -HELM_BIN_PATH = AIRFLOW_SOURCES_DIR / ".build" / ".k8s-env" / "bin" / "helm" +HELM_BIN_PATH = AIRFLOW_SOURCES_DIR / ".build" / "k8s-env" / "bin" / "helm" result = subprocess.run( [os.fspath(HELM_BIN_PATH), "lint", ".", "-f", "values.yaml"], diff --git a/scripts/ci/pre_commit/kubeconform.py b/scripts/ci/pre_commit/kubeconform.py index 3bfccd2f8d251..2452c2158d53e 100755 --- a/scripts/ci/pre_commit/kubeconform.py +++ b/scripts/ci/pre_commit/kubeconform.py @@ -33,7 +33,7 @@ sys.exit(res_setup.returncode) AIRFLOW_SOURCES_DIR = Path(__file__).parents[3].resolve() -HELM_BIN_PATH = AIRFLOW_SOURCES_DIR / ".build" / ".k8s-env" / "bin" / "helm" +HELM_BIN_PATH = AIRFLOW_SOURCES_DIR / ".build" / "k8s-env" / "bin" / "helm" ps = subprocess.Popen( [os.fspath(HELM_BIN_PATH), "template", ".", "-f", "values.yaml"],