diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index bed2233dfcb07..ce4aede7d1f23 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -128,6 +128,17 @@ jobs: run: ./scripts/ci/cleanup_docker.sh - name: "Install Breeze" uses: ./.github/actions/breeze + - name: "Restore ${{ inputs.image-type }} cache mount image ${{ inputs.platform }}:${{ inputs.python }}" + uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c + with: + key: "ci-cache-mount-save-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + path: "/tmp/" + id: restore-cache-mount + - name: "Import mount-cache ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + run: > + breeze ci-image import-mount-cache --cache-file + /tmp/ci-cache-mount-save-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar.gz + if: steps.restore-cache-mount.outputs.cache-hit == 'true' - name: "Login to ghcr.io" run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - name: > @@ -164,3 +175,16 @@ jobs: if-no-files-found: 'error' retention-days: 2 if: inputs.upload-image-artifact == 'true' + - name: "Export mount cache ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + run: > + breeze ci-image export-mount-cache --cache-file + /tmp/ci-cache-mount-save-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar.gz + if: inputs.upload-image-artifact == 'true' + - name: "Stash cache mount ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c + with: + key: "ci-cache-mount-save-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + path: "/tmp/ci-cache-mount-save-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar" + if-no-files-found: 'error' + retention-days: 2 + if: inputs.upload-image-artifact == 'true' diff --git a/dev/breeze/doc/06_managing_docker_images.rst b/dev/breeze/doc/06_managing_docker_images.rst index 89c302c12d1b8..2b263486783d2 100644 --- a/dev/breeze/doc/06_managing_docker_images.rst +++ b/dev/breeze/doc/06_managing_docker_images.rst @@ -133,6 +133,28 @@ To load the image from specific job run (for example 12538475388), you can use t :width: 100% :alt: Breeze image artifacts +Exporting and importing CI image cache mount +............................................ + +During the build, cache of ``uv`` and ``pip`` is stored in a separate "cache mount" volum that is mounted +during the build. This cache mount volume is preserved between builds and can be exported and imported +to speed up the build process in CI - where cache is stored as artifact and can be imported in the next +build. + +These are all available flags of ``export-mount-cache`` command: + +.. image:: ./images/output_ci-image_export-mount-cache.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_ci-image_export-mount-cache.svg + :width: 100% + :alt: Breeze ci-image + +These are all available flags of ``import-mount-cache`` command: + +.. image:: ./images/output_ci-image_import-mount-cache.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_ci-image_import-mount-cache.svg + :width: 100% + :alt: Breeze ci-image import-mount-cache + PROD Image tasks ---------------- diff --git a/dev/breeze/doc/images/output_ci-image.svg b/dev/breeze/doc/images/output_ci-image.svg index 6fc5b425c5c7a..2b8c1414c8105 100644 --- a/dev/breeze/doc/images/output_ci-image.svg +++ b/dev/breeze/doc/images/output_ci-image.svg @@ -1,4 +1,4 @@ - +