Simplify caching mechanisms for CI and PROD images

For a long time we had used a sophisticated mechanism to speed up our CI jobs by building the images in "pull_request_target" workflow and pushing them to GitHub registry. That however had several drawbacks: * CI image was complex when it comes to layer setup (we had to pre- cache installed dependencies by installing them from branch tip * The pull_request_target is a very dangerous workflow, we had a number of security problems with it (and it's difficult to debug) * Caching of `pip` and `uv` was not used because it increased size of the image significantly This PR significantly improves the caching mechanisms for the images building of several advacements that were not possible before: * The upload-artifacts@v4 action and improved stash action developed by @assignUser and published in "apache/infrastructure-actions" allows us to store all images (8GB per run) in artifacts rather than in registry - so we can do the image build once and share it with all the jobs. * The uv speed is "enough" to allow occasional installation of Airlfow locally. This allows to utilize cache-mount and locally build uv cache, rather than rely on "remote" cache when we are building local images for breeze. The first time you build local breeze image it will take 2-5 more minutes (depending on your network speed, but because we can utilise cache mounts, every subsequent build should be very fast - even if all dependencies change. Using uv also allows to "always" reinstall airflow when you build the image even if single source file changed, because with cache it takes sub-seconds to reinstall airflow and all dependencies. * the cache mounts are not included in the image size, and since we can export and import images in CI in artifacts and we do not need to rebuild them, the images shared as compressed artifacts are relatively small (2GB) - cache of `uv` is around 4GB on top of that so sharing image built in the "build image" job with other jobs in the same workflow is fast.
apache · Dec 28, 2024 · da858e3 · da858e3
1 parent a22faa5
commit da858e3
Show file tree

Hide file tree

Showing 110 changed files with 1,179 additions and 2,689 deletions.
diff --git a/.github/actions/checkout_target_commit/action.yml b/.github/actions/checkout_target_commit/action.yml
diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml
@@ -16,12 +16,15 @@
 # under the License.
 #
 ---
-name: 'Prepare breeze && current python image'
-description: 'Installs breeze and pulls current python image'
+name: 'Prepare breeze && current image (CI/PROD)'
+description: 'Installs breeze and recreates current python image from artifact'
 inputs:
   pull-image-type:
-    description: 'Which image to pull'
+    description: 'Which image to create'
     default: CI
+  platform:
+    description: 'Platform for the build - amd64 or arm64'
+    required: true
 outputs:
   host-python-version:
     description: Python version used in host
@@ -35,11 +38,24 @@ runs:
     - name: Login to ghcr.io
       shell: bash
       run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-    - name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG }}
+    - name: "Restore CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
+      uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
+      with:
+        key: "ci-image-docker-export-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
+        path: "/tmp/"
+      if: inputs.pull-image-type == 'CI'
+    - name: Import CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
+      run: ./scripts/ci/import_docker_image.sh /tmp/ci-docker-image-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar
       shell: bash
-      run: breeze ci-image pull --tag-as-latest
       if: inputs.pull-image-type == 'CI'
-    - name: Pull PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG }}
+    - name: "Restore PROD docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
+      uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
+      with:
+        key: "ci-image-docker-export-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
+        path: "/tmp/"
+      if: inputs.pull-image-type == 'PROD'
+    - name: Import PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
+      run: >
+        ./scripts/ci/import_docker_image.sh /tmp/prod-docker-image-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar
       shell: bash
-      run: breeze prod-image pull --tag-as-latest
       if: inputs.pull-image-type == 'PROD'
diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml
@@ -32,10 +32,6 @@ on:  # yamllint disable-line rule:truthy
         description: "The array of labels (in json form) determining self-hosted runners."
         required: true
         type: string
-      image-tag:
-        description: "Tag to set for the image"
-        required: true
-        type: string
       python-versions:
         description: "The list of python versions (stringified JSON array) to run the tests on."
         required: true
@@ -103,8 +99,6 @@ jobs:
       contents: read
       # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs
       # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo.
-      # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the
-      # "in-workflow-build" condition
       packages: write
     secrets: inherit
     with:
@@ -113,7 +107,7 @@ jobs:
       cache-type: "Early"
       include-prod-images: "false"
       push-latest-images: "false"
-      platform: "linux/amd64"
+      platform: "amd64"
       python-versions: ${{ inputs.python-versions }}
       branch: ${{ inputs.branch }}
       constraints-branch: ${{ inputs.constraints-branch }}
@@ -159,7 +153,7 @@ jobs:
 #  # There is no point in running this one in "canary" run, because the above step is doing the
 #  # same build anyway.
 #  build-ci-arm-images:
-#    name: Build CI ARM images (in-workflow)
+#    name: Build CI ARM images
 #    uses: ./.github/workflows/ci-image-build.yml
 #    permissions:
 #      contents: read
@@ -169,9 +163,8 @@ jobs:
 #      push-image: "false"
 #      runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
 #      runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }}
-#      image-tag: ${{ inputs.image-tag }}
 #      python-versions: ${{ inputs.python-versions }}
-#      platform: "linux/arm64"
+#      platform: "arm64"
 #      branch: ${{ inputs.branch }}
 #      constraints-branch: ${{ inputs.constraints-branch }}
 #      use-uv: ${{ inputs.use-uv}}

diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml
@@ -32,10 +32,6 @@ on:  # yamllint disable-line rule:truthy
         description: "Branch used to construct constraints URL from."
         required: true
         type: string
-      image-tag:
-        description: "Tag to set for the image"
-        required: true
-        type: string
       upgrade-to-newer-dependencies:
         description: "Whether to upgrade to newer dependencies (true/false)"
         required: true
@@ -70,7 +66,6 @@ jobs:
       default-python-version: ${{ inputs.default-python-version }}
       branch: ${{ inputs.default-branch }}
       use-uv: "false"
-      image-tag: ${{ inputs.image-tag }}
       build-provider-packages: ${{ inputs.default-branch == 'main' }}
       upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
       chicken-egg-providers: ${{ inputs.chicken-egg-providers }}
@@ -88,7 +83,6 @@ jobs:
       default-python-version: ${{ inputs.default-python-version }}
       branch: ${{ inputs.default-branch }}
       use-uv: "false"
-      image-tag: ${{ inputs.image-tag }}
       build-provider-packages: ${{ inputs.default-branch == 'main' }}
       upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
       chicken-egg-providers: ${{ inputs.chicken-egg-providers }}
@@ -122,11 +116,10 @@ jobs:
       - name: Login to ghcr.io
         shell: bash
         run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-      - name: Pull PROD image ${{ inputs.default-python-version}}:${{ inputs.image-tag }}
-        run: breeze prod-image pull --tag-as-latest
+      - name: Pull PROD image ${{ inputs.default-python-version}}
+        run: breeze prod-image pull
         env:
           PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
-          IMAGE_TAG: "${{ inputs.image-tag }}"
       - name: "Setup python"
         uses: actions/setup-python@v5
         with:
@@ -138,15 +131,14 @@ jobs:
           cd ./docker_tests && \
           python -m pip install -r requirements.txt && \
           TEST_IMAGE=\"ghcr.io/${{ github.repository }}/${{ inputs.default-branch }}\
-          /prod/python${{ inputs.default-python-version }}:${{ inputs.image-tag }}\" \
+          /prod/python${{ inputs.default-python-version }}\" \
           python -m pytest test_examples_of_prod_image_building.py -n auto --color=yes"
 
   test-docker-compose-quick-start:
     timeout-minutes: 60
     name: "Docker-compose quick start with PROD image verifying"
     runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
     env:
-      IMAGE_TAG: "${{ inputs.image-tag }}"
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
       GITHUB_REPOSITORY: ${{ github.repository }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -161,14 +153,10 @@ jobs:
         with:
           fetch-depth: 2
           persist-credentials: false
-      - name: "Cleanup docker"
-        run: ./scripts/ci/cleanup_docker.sh
-      - name: "Install Breeze"
-        uses: ./.github/actions/breeze
-      - name: Login to ghcr.io
-        shell: bash
-        run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-      - name: "Pull image ${{ inputs.default-python-version}}:${{ inputs.image-tag }}"
-        run: breeze prod-image pull --tag-as-latest
+      - name: "Prepare breeze & CI image: ${{ inputs.default-python-version}}"
+        uses: ./.github/actions/prepare_breeze_and_image
+        with:
+          platform: "amd64"
+        id: breeze
       - name: "Test docker-compose quick start"
         run: breeze testing docker-compose-tests
diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml
@@ -288,7 +288,6 @@ jobs:
     runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
     env:
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
-      IMAGE_TAG: ${{ inputs.image-tag }}
       GITHUB_REPOSITORY: ${{ github.repository }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       GITHUB_USERNAME: ${{ github.actor }}