diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index 35389089eccd0..47d7b59b35fb5 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -162,11 +162,13 @@ jobs: # packages: write # secrets: inherit # with: +# platform: "linux/arm64" # push-image: "false" +# upload-image-artifact: "true" +# upload-mount-cache-artifact: ${{ inputs.canary-run }} # runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} # runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }} # python-versions: ${{ inputs.python-versions }} -# platform: "linux/arm64" # branch: ${{ inputs.branch }} # constraints-branch: ${{ inputs.constraints-branch }} # use-uv: ${{ inputs.use-uv }} diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 553a509e4bf11..fb44503be0502 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -56,6 +56,10 @@ on: # yamllint disable-line rule:truthy description: "Whether to upload docker image artifact" required: true type: string + upload-mount-cache-artifact: + description: "Whether to upload mount-cache artifact" + required: true + type: string debian-version: description: "Base Debian distribution to use for the build (bookworm)" type: string @@ -129,15 +133,24 @@ jobs: uses: ./.github/actions/breeze with: use-uv: ${{ inputs.use-uv }} + - name: "Restore ${{ inputs.image-type }} cache mount image ${{ inputs.platform }}:${{ inputs.python }}" + uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c + with: + key: "ci-cache-mount-save-v2-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + path: "/tmp/" + id: restore-cache-mount + - name: "Import mount-cache ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + run: > + breeze ci-image import-mount-cache + --cache-file /tmp/ci-cache-mount-save-v2-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar.gz + if: steps.restore-cache-mount.outputs.stash-hit == 'true' - name: "Login to ghcr.io" run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - name: > Build ${{ inputs.push-image == 'true' && ' & push ' || '' }} ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }} image run: > - breeze ci-image build - --builder airflow_cache - --platform "${{ inputs.platform }}" + breeze ci-image build --platform "${{ inputs.platform }}" env: DOCKER_CACHE: ${{ inputs.docker-cache }} DISABLE_AIRFLOW_REPO_CACHE: ${{ inputs.disable-airflow-repo-cache }} @@ -165,3 +178,16 @@ jobs: if-no-files-found: 'error' retention-days: '2' if: inputs.upload-image-artifact == 'true' + - name: "Export mount cache ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + run: > + breeze ci-image export-mount-cache + --cache-file /tmp/ci-cache-mount-save-v2-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar.gz + if: inputs.upload-mount-cache-artifact == 'true' + - name: "Stash cache mount ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c + with: + key: "ci-cache-mount-save-v2-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + path: "/tmp/ci-cache-mount-save-v2-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar.gz" + if-no-files-found: 'error' + retention-days: 2 + if: inputs.upload-mount-cache-artifact == 'true' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ddeec96d2a08..b0e5e9bde7b95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -213,7 +213,8 @@ jobs: runs-on-as-json-self-hosted: ${{ needs.build-info.outputs.runs-on-as-json-self-hosted }} platform: "linux/amd64" push-image: "false" - upload-image-artifact: "true" + upload-image-artifact: ${{ needs.build-info.outputs.canary-run }} + upload-mount-cache-artifact: ${{ needs.build-info.outputs.canary-run }} python-versions: ${{ needs.build-info.outputs.python-versions }} branch: ${{ needs.build-info.outputs.default-branch }} use-uv: ${{ needs.build-info.outputs.use-uv }} diff --git a/dev/breeze/doc/06_managing_docker_images.rst b/dev/breeze/doc/06_managing_docker_images.rst index 89c302c12d1b8..2b263486783d2 100644 --- a/dev/breeze/doc/06_managing_docker_images.rst +++ b/dev/breeze/doc/06_managing_docker_images.rst @@ -133,6 +133,28 @@ To load the image from specific job run (for example 12538475388), you can use t :width: 100% :alt: Breeze image artifacts +Exporting and importing CI image cache mount +............................................ + +During the build, cache of ``uv`` and ``pip`` is stored in a separate "cache mount" volum that is mounted +during the build. This cache mount volume is preserved between builds and can be exported and imported +to speed up the build process in CI - where cache is stored as artifact and can be imported in the next +build. + +These are all available flags of ``export-mount-cache`` command: + +.. image:: ./images/output_ci-image_export-mount-cache.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_ci-image_export-mount-cache.svg + :width: 100% + :alt: Breeze ci-image + +These are all available flags of ``import-mount-cache`` command: + +.. image:: ./images/output_ci-image_import-mount-cache.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_ci-image_import-mount-cache.svg + :width: 100% + :alt: Breeze ci-image import-mount-cache + PROD Image tasks ---------------- diff --git a/dev/breeze/doc/images/output_ci-image.svg b/dev/breeze/doc/images/output_ci-image.svg index 6fc5b425c5c7a..2b8c1414c8105 100644 --- a/dev/breeze/doc/images/output_ci-image.svg +++ b/dev/breeze/doc/images/output_ci-image.svg @@ -1,4 +1,4 @@ - +