Merge branch 'main' into ctx_prop_final

.github/actions/breeze/action.yml

@@ -22,6 +22,9 @@ inputs:
   python-version:
     description: 'Python version to use'
     default: "3.9"
+  use-uv:
+    description: 'Whether to use uv tool'
+    required: true
 outputs:
   host-python-version:
     description: Python version used in host
@@ -33,13 +36,11 @@ runs:
       uses: actions/setup-python@v5
       with:
         python-version: ${{ inputs.python-version }}
-        cache: 'pip'
-        cache-dependency-path: ./dev/breeze/pyproject.toml
+    # NOTE! Installing Breeze without using cache is FASTER than when using cache - uv is so fast and has
+    # so low overhead, that just running upload cache/restore cache is slower than installing it from scratch
     - name: "Install Breeze"
       shell: bash
       run: ./scripts/ci/install_breeze.sh
-      env:
-        PYTHON_VERSION: ${{ inputs.python-version }}
     - name: "Free space"
       shell: bash
       run: breeze ci free-space

.github/actions/install-pre-commit/action.yml

@@ -21,29 +21,58 @@ description: 'Installs pre-commit and related packages'
 inputs:
   python-version:
     description: 'Python version to use'
-    default: 3.9
+    default: "3.9"
   uv-version:
     description: 'uv version to use'
-    default: 0.5.5
+    default: "0.5.14"  # Keep this comment to allow automatic replacement of uv version
   pre-commit-version:
     description: 'pre-commit version to use'
-    default: 4.0.1
+    default: "4.0.1"  # Keep this comment to allow automatic replacement of pre-commit version
   pre-commit-uv-version:
     description: 'pre-commit-uv version to use'
-    default: 4.1.4
+    default: "4.1.4"  # Keep this comment to allow automatic replacement of pre-commit-uv version
 runs:
   using: "composite"
   steps:
     - name: Install pre-commit, uv, and pre-commit-uv
       shell: bash
+      env:
+        UV_VERSION: ${{inputs.uv-version}}
+        PRE_COMMIT_VERSION: ${{inputs.pre-commit-version}}
+        PRE_COMMIT_UV_VERSION: ${{inputs.pre-commit-uv-version}}
       run: |
-        pip install uv==${{inputs.uv-version}} || true
-        uv tool install pre-commit==${{inputs.pre-commit-version}} --with uv==${{inputs.uv-version}} \
-        --with pre-commit-uv==${{inputs.pre-commit-uv-version}}
-    - name: Cache pre-commit envs
-      uses: actions/cache@v4
+        pip install uv==${UV_VERSION} || true
+        uv tool install pre-commit==${PRE_COMMIT_VERSION} --with uv==${UV_VERSION} \
+        --with pre-commit-uv==${PRE_COMMIT_UV_VERSION}
+      working-directory: ${{ github.workspace }}
+    # We need to use tar file with archive to restore all the permissions and symlinks
+    - name: "Delete ~.cache"
+      run: |
+        du ~/ --max-depth=2
+        echo
+        echo Deleting ~/.cache
+        echo
+        rm -rf ~/.cache
+        echo
+      shell: bash
+    - name: "Restore pre-commit cache"
+      uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
       with:
-        path: ~/.cache/pre-commit
-        key: "pre-commit-${{inputs.python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
-        restore-keys: |
-          pre-commit-${{inputs.python-version}}-
+        key: cache-pre-commit-v4-${{ inputs.python-version }}-${{ hashFiles('.pre-commit-config.yaml') }}
+        path: /tmp/
+      id: restore-pre-commit-cache
+    - name: "Restore .cache from the tar file"
+      run: tar -C ~ -xzf /tmp/cache-pre-commit.tar.gz
+      shell: bash
+      if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true'
+    - name: "Show restored files"
+      run: |
+        echo "Restored files"
+        du ~/ --max-depth=2
+        echo
+      shell: bash
+      if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true'
+    - name: Install pre-commit hooks
+      shell: bash
+      run: pre-commit install-hooks || (cat ~/.cache/pre-commit/pre-commit.log && exit 1)
+      working-directory: ${{ github.workspace }}

.github/actions/prepare_all_ci_images/action.yml

@@ -25,10 +25,6 @@ inputs:
   platform:
     description: 'Platform for the build - linux/amd64 or linux/arm64'
     required: true
-outputs:
-  host-python-version:
-    description: Python version used in host
-    value: ${{ steps.breeze.outputs.host-python-version }}
 runs:
   using: "composite"
   steps:

.github/actions/prepare_breeze_and_image/action.yml

@@ -28,6 +28,9 @@ inputs:
   platform:
     description: 'Platform for the build - linux/amd64 or linux/arm64'
     required: true
+  use-uv:
+    description: 'Whether to use uv'
+    required: true
 outputs:
   host-python-version:
     description: Python version used in host
@@ -40,14 +43,20 @@ runs:
       shell: bash
     - name: "Install Breeze"
       uses: ./.github/actions/breeze
+      with:
+        use-uv: ${{ inputs.use-uv }}
       id: breeze
     - name: "Restore ${{ inputs.image-type }} docker image ${{ inputs.platform }}:${{ inputs.python }}"
       uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
       with:
-        key: "${{ inputs.image-type }}-image-save-${{ inputs.platform }}-${{ inputs.python }}"
+        key: ${{ inputs.image-type }}-image-save-${{ inputs.platform }}-${{ inputs.python }}
         path: "/tmp/"
     - name: "Load ${{ inputs.image-type }} image ${{ inputs.platform }}:${{ inputs.python }}"
+      env:
+        PLATFORM: ${{ inputs.platform }}
+        PYTHON: ${{ inputs.python }}
+        IMAGE_TYPE: ${{ inputs.image-type }}
       run: >
-        breeze ${{ inputs.image-type }}-image load
-        --platform ${{ inputs.platform }} --python ${{ inputs.python }}
+        breeze ${IMAGE_TYPE}-image load
+        --platform ${PLATFORM} --python ${PYTHON}
       shell: bash

.github/actions/prepare_single_ci_image/action.yml

@@ -38,10 +38,13 @@ runs:
     - name: "Restore CI docker images ${{ inputs.platform }}:${{ inputs.python }}"
       uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
       with:
-        key: "ci-image-save-${{ inputs.platform }}-${{ inputs.python }}"
+        key: ci-image-save-${{ inputs.platform }}-${{ inputs.python }}
         path: "/tmp/"
       if: contains(inputs.python-versions-list-as-string, inputs.python)
     - name: "Load CI image ${{ inputs.platform }}:${{ inputs.python }}"
-      run: breeze ci-image load --platform "${{ inputs.platform }}" --python "${{ inputs.python }}"
+      env:
+        PLATFORM: ${{ inputs.platform }}
+        PYTHON: ${{ inputs.python }}
+      run: breeze ci-image load --platform "${PLATFORM}" --python "${PYTHON}"
       shell: bash
       if: contains(inputs.python-versions-list-as-string, inputs.python)

.github/workflows/additional-ci-image-checks.yml

@@ -111,7 +111,7 @@ jobs:
       python-versions: ${{ inputs.python-versions }}
       branch: ${{ inputs.branch }}
       constraints-branch: ${{ inputs.constraints-branch }}
-      use-uv: ${{ inputs.use-uv}}
+      use-uv: ${{ inputs.use-uv }}
       include-success-outputs: ${{ inputs.include-success-outputs }}
       docker-cache: ${{ inputs.docker-cache }}
       disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }}
@@ -143,8 +143,13 @@ jobs:
         run: ./scripts/ci/cleanup_docker.sh
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
+        with:
+          use-uv: ${{ inputs.use-uv }}
       - name: "Login to ghcr.io"
-        run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+        env:
+          actor: ${{ github.actor }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: echo "$GITHUB_TOKEN" | docker login ghcr.io -u "$actor" --password-stdin
       - name: "Check that image builds quickly"
         run: breeze shell --max-time 600 --platform "linux/amd64"
 
@@ -160,14 +165,16 @@ jobs:
 #      packages: write
 #    secrets: inherit
 #    with:
+#      platform: "linux/arm64"
 #      push-image: "false"
+#      upload-image-artifact: "true"
+#      upload-mount-cache-artifact:  ${{ inputs.canary-run }}
 #      runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
 #      runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }}
 #      python-versions: ${{ inputs.python-versions }}
-#      platform: "linux/arm64"
 #      branch: ${{ inputs.branch }}
 #      constraints-branch: ${{ inputs.constraints-branch }}
-#      use-uv: ${{ inputs.use-uv}}
+#      use-uv: ${{ inputs.use-uv }}
 #      upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
 #      docker-cache: ${{ inputs.docker-cache }}
 #      disable-airflow-repo-cache:  ${{ inputs.disable-airflow-repo-cache }}

.github/workflows/additional-prod-image-tests.yml

@@ -56,6 +56,10 @@ on:  # yamllint disable-line rule:truthy
         description: "Which version of python should be used by default"
         required: true
         type: string
+      use-uv:
+        description: "Whether to use uv"
+        required: true
+        type: string
 jobs:
   prod-image-extra-checks-main:
     name: PROD image extra checks (main)
@@ -117,12 +121,17 @@ jobs:
           platform: "linux/amd64"
           image-type: "prod"
           python: ${{ inputs.default-python-version }}
+          use-uv: ${{ inputs.use-uv }}
       - name: "Test examples of PROD image building"
+        env:
+          GITHUB_REPOSITORY: ${{ github.repository }}
+          DEFAULT_BRANCH: ${{ inputs.default-branch }}
+          DEFAULT_PYTHON_VERSION: ${{ inputs.default-python-version }}
         run: "
           cd ./docker_tests && \
           python -m pip install -r requirements.txt && \
-          TEST_IMAGE=\"ghcr.io/${{ github.repository }}/${{ inputs.default-branch }}\
-          /prod/python${{ inputs.default-python-version }}\" \
+          TEST_IMAGE=\"ghcr.io/$GITHUB_REPOSITORY/$DEFAULT_BRANCH\
+          /prod/python$DEFAULT_PYTHON_VERSION\" \
           python -m pytest test_examples_of_prod_image_building.py -n auto --color=yes"
 
   test-docker-compose-quick-start:
@@ -150,6 +159,7 @@ jobs:
           platform: "linux/amd64"
           image-type: "prod"
           python: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
+          use-uv: ${{ inputs.use-uv }}
         id: breeze
       - name: "Test docker-compose quick start"
         run: breeze testing docker-compose-tests

.github/workflows/backport-cli.yml

@@ -64,24 +64,28 @@ jobs:
         id: execute-backport
         env:
           GH_AUTH: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          COMMIT_SHA: ${{ inputs.commit-sha }}
         run: |
           git config --global user.email "[email protected]"
           git config --global user.name "Your Name"
           set +e
           {
           echo 'cherry_picker_output<<EOF'
-          cherry_picker ${{ inputs.commit-sha }} ${{ inputs.target-branch }}
+          cherry_picker ${COMMIT_SHA} ${TARGET_BRANCH}
           echo EOF
           } >> "${GITHUB_OUTPUT}"
         continue-on-error: true
 
       - name: Parse backport output
         id: parse-backport-output
+        env:
+          CHERRY_PICKER_OUTPUT: ${{ steps.execute-backport.outputs.cherry_picker_output }}
         run: |
           set +e
-          echo "${{ steps.execute-backport.outputs.cherry_picker_output }}"
+          echo "${CHERRY_PICKER_OUTPUT}"
 
-          url=$(echo "${{ steps.execute-backport.outputs.cherry_picker_output }}" | \
+          url=$(echo "${CHERRY_PICKER_OUTPUT}" | \
               grep -o 'Backport PR created at https://[^ ]*' | \
               awk '{print $5}')
 
@@ -99,17 +103,20 @@ jobs:
           GH_TOKEN: ${{ github.token }}
           REPOSITORY: ${{ github.repository }}
           RUN_ID: ${{ github.run_id }}
+          COMMIT_SHA: ${{ inputs.commit-sha }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          BACKPORT_URL: ${{ steps.parse-backport-output.outputs.backport-url }}
         run: |
-          COMMIT_INFO_URL="https://api.github.com/repos/${{ github.repository }}/commits/"
-          COMMIT_INFO_URL="${COMMIT_INFO_URL}${{ inputs.commit-sha }}/pulls"
+          COMMIT_INFO_URL="https://api.github.com/repos/$REPOSITORY/commits/"
+          COMMIT_INFO_URL="${COMMIT_INFO_URL}$COMMIT_SHA/pulls"
 
           PR_NUMBER=$(gh api \
               -H "Accept: application/vnd.github+json" \
               -H "X-GitHub-Api-Version: 2022-11-28" \
-              /repos/${{ github.repository }}/commits/${{ inputs.commit-sha }}/pulls \
+              /repos/$REPOSITORY/commits/$COMMIT_SHA/pulls \
               --jq '.[0].number')
 
           python ./dev/backport/update_backport_status.py \
-              ${{ steps.parse-backport-output.outputs.backport-url }} \
-              ${{ inputs.commit-sha }} ${{ inputs.target-branch }} \
+              $BACKPORT_URL \
+              $COMMIT_SHA $TARGET_BRANCH \
               "$PR_NUMBER"