Speed up image building in CI by exporting and importing mount cache

During the build, cache of ``uv`` and ``pip`` is stored in a separate
"cache mount" volum that is mounted during the build. This cache mount
volume is preserved between builds and can be exported and imported to
speed up the build process in CI - where cache is stored as artifact and
can be imported in the next build.

This PR implements it:

* export and import commands are added to breeze to export/import cache
  mount content

* the cache mount content is stashed as artifact in the build
  after image is built and it is restored before the image is built

.github/workflows/additional-ci-image-checks.yml

@@ -162,11 +162,13 @@ jobs:
 #      packages: write
 #    secrets: inherit
 #    with:
+#      platform: "linux/arm64"
 #      push-image: "false"
+#      upload-image-artifact: "true"
+#      upload-mount-cache-artifact:  ${{ inputs.canary-run }}
 #      runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
 #      runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }}
 #      python-versions: ${{ inputs.python-versions }}
-#      platform: "linux/arm64"
 #      branch: ${{ inputs.branch }}
 #      constraints-branch: ${{ inputs.constraints-branch }}
 #      use-uv: ${{ inputs.use-uv }}

.github/workflows/ci-image-build.yml

@@ -56,6 +56,10 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to upload docker image artifact"
         required: true
         type: string
+      upload-mount-cache-artifact:
+        description: "Whether to upload mount-cache artifact"
+        required: true
+        type: string
       debian-version:
         description: "Base Debian distribution to use for the build (bookworm)"
         type: string
@@ -129,15 +133,24 @@ jobs:
         uses: ./.github/actions/breeze
         with:
           use-uv: ${{ inputs.use-uv }}
+      - name: "Restore ${{ inputs.image-type }} cache mount image ${{ inputs.platform }}:${{ inputs.python }}"
+        uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
+        with:
+          key: "ci-cache-mount-save-v2-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
+          path: "/tmp/"
+        id: restore-cache-mount
+      - name: "Import mount-cache ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
+        run: >
+          breeze ci-image import-mount-cache
+          --cache-file /tmp/ci-cache-mount-save-v2-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar.gz
+        if: steps.restore-cache-mount.outputs.stash-hit == 'true'
       - name: "Login to ghcr.io"
         run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
       - name: >
           Build ${{ inputs.push-image == 'true' && ' & push ' || '' }}
           ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }} image
         run: >
-          breeze ci-image build
-          --builder airflow_cache
-          --platform "${{ inputs.platform }}"
+          breeze ci-image build --platform "${{ inputs.platform }}"
         env:
           DOCKER_CACHE: ${{ inputs.docker-cache }}
           DISABLE_AIRFLOW_REPO_CACHE: ${{ inputs.disable-airflow-repo-cache }}
@@ -165,3 +178,16 @@ jobs:
           if-no-files-found: 'error'
           retention-days: '2'
         if: inputs.upload-image-artifact == 'true'
+      - name: "Export mount cache ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
+        run: >
+          breeze ci-image export-mount-cache
+          --cache-file /tmp/ci-cache-mount-save-v2-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar.gz
+        if: inputs.upload-mount-cache-artifact == 'true'
+      - name: "Stash cache mount ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
+        uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
+        with:
+          key: "ci-cache-mount-save-v2-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
+          path: "/tmp/ci-cache-mount-save-v2-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar.gz"
+          if-no-files-found: 'error'
+          retention-days: 2
+        if: inputs.upload-mount-cache-artifact == 'true'

.github/workflows/ci.yml

@@ -213,7 +213,8 @@ jobs:
       runs-on-as-json-self-hosted: ${{ needs.build-info.outputs.runs-on-as-json-self-hosted }}
       platform: "linux/amd64"
       push-image: "false"
-      upload-image-artifact: "true"
+      upload-image-artifact: ${{ needs.build-info.outputs.canary-run }}
+      upload-mount-cache-artifact: ${{ needs.build-info.outputs.canary-run }}
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       branch: ${{ needs.build-info.outputs.default-branch }}
       use-uv: ${{ needs.build-info.outputs.use-uv }}

dev/breeze/doc/06_managing_docker_images.rst

@@ -133,6 +133,28 @@ To load the image from specific job run (for example 12538475388), you can use t
   :width: 100%
   :alt: Breeze image artifacts
 
+Exporting and importing CI image cache mount
+............................................
+
+During the build, cache of ``uv`` and ``pip`` is stored in a separate "cache mount" volum that is mounted
+during the build. This cache mount volume is preserved between builds and can be exported and imported
+to speed up the build process in CI - where cache is stored as artifact and can be imported in the next
+build.
+
+These are all available flags of ``export-mount-cache`` command:
+
+.. image:: ./images/output_ci-image_export-mount-cache.svg
+  :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_ci-image_export-mount-cache.svg
+  :width: 100%
+  :alt: Breeze ci-image
+
+These are all available flags of ``import-mount-cache`` command:
+
+.. image:: ./images/output_ci-image_import-mount-cache.svg
+  :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_ci-image_import-mount-cache.svg
+  :width: 100%
+  :alt: Breeze ci-image import-mount-cache
+
 PROD Image tasks
 ----------------
 

dev/breeze/doc/images/output_ci-image.txt

@@ -1 +1 @@
-2039b80b810ff0ad9e9f505548254f82
+de6e1ee2eb602569a8cd1a3c6c4cbafe