Simplify caching mechanisms for CI and PROD images

apache · Dec 28, 2024 · a70964f · a70964f
1 parent a22faa5
commit a70964f
Show file tree

Hide file tree

Showing 110 changed files with 1,175 additions and 2,689 deletions.
diff --git a/.github/actions/checkout_target_commit/action.yml b/.github/actions/checkout_target_commit/action.yml
diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml
@@ -16,12 +16,15 @@
 # under the License.
 #
 ---
-name: 'Prepare breeze && current python image'
-description: 'Installs breeze and pulls current python image'
+name: 'Prepare breeze && current image (CI/PROD)'
+description: 'Installs breeze and recreates current python image from artifact'
 inputs:
   pull-image-type:
-    description: 'Which image to pull'
+    description: 'Which image to create'
     default: CI
+  platform:
+    description: 'Platform for the build - amd64 or arm64'
+    required: true
 outputs:
   host-python-version:
     description: Python version used in host
@@ -35,11 +38,23 @@ runs:
     - name: Login to ghcr.io
       shell: bash
       run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-    - name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG }}
+    - name: Restore CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
+      uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
+      with:
+        key: "ci-image-docker-dump-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
+        path: "/tmp/docker-image.tar"
+      if: inputs.pull-image-type == 'CI'
+    - name: Import CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
+      run: ./scripts/ci/import_docker_image.sh
       shell: bash
-      run: breeze ci-image pull --tag-as-latest
       if: inputs.pull-image-type == 'CI'
-    - name: Pull PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG }}
+    - name: "Restore PROD docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
+      uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
+      with:
+        key: "ci-image-docker-dump-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
+        path: "/tmp/docker-image.tar"
+      if: inputs.pull-image-type == 'PROD'
+    - name: Import PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
+      run: ./scripts/ci/import_docker_image.sh
       shell: bash
-      run: breeze prod-image pull --tag-as-latest
       if: inputs.pull-image-type == 'PROD'
diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml
@@ -32,10 +32,6 @@ on:  # yamllint disable-line rule:truthy
         description: "The array of labels (in json form) determining self-hosted runners."
         required: true
         type: string
-      image-tag:
-        description: "Tag to set for the image"
-        required: true
-        type: string
       python-versions:
         description: "The list of python versions (stringified JSON array) to run the tests on."
         required: true
@@ -103,8 +99,6 @@ jobs:
       contents: read
       # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs
       # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo.
-      # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the
-      # "in-workflow-build" condition
       packages: write
     secrets: inherit
     with:
@@ -113,7 +107,7 @@ jobs:
       cache-type: "Early"
       include-prod-images: "false"
       push-latest-images: "false"
-      platform: "linux/amd64"
+      platform: "amd64"
       python-versions: ${{ inputs.python-versions }}
       branch: ${{ inputs.branch }}
       constraints-branch: ${{ inputs.constraints-branch }}
@@ -159,7 +153,7 @@ jobs:
 #  # There is no point in running this one in "canary" run, because the above step is doing the
 #  # same build anyway.
 #  build-ci-arm-images:
-#    name: Build CI ARM images (in-workflow)
+#    name: Build CI ARM images
 #    uses: ./.github/workflows/ci-image-build.yml
 #    permissions:
 #      contents: read
@@ -169,9 +163,8 @@ jobs:
 #      push-image: "false"
 #      runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
 #      runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }}
-#      image-tag: ${{ inputs.image-tag }}
 #      python-versions: ${{ inputs.python-versions }}
-#      platform: "linux/arm64"
+#      platform: "arm64"
 #      branch: ${{ inputs.branch }}
 #      constraints-branch: ${{ inputs.constraints-branch }}
 #      use-uv: ${{ inputs.use-uv}}

diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml
@@ -32,10 +32,6 @@ on:  # yamllint disable-line rule:truthy
         description: "Branch used to construct constraints URL from."
         required: true
         type: string
-      image-tag:
-        description: "Tag to set for the image"
-        required: true
-        type: string
       upgrade-to-newer-dependencies:
         description: "Whether to upgrade to newer dependencies (true/false)"
         required: true
@@ -70,7 +66,6 @@ jobs:
       default-python-version: ${{ inputs.default-python-version }}
       branch: ${{ inputs.default-branch }}
       use-uv: "false"
-      image-tag: ${{ inputs.image-tag }}
       build-provider-packages: ${{ inputs.default-branch == 'main' }}
       upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
       chicken-egg-providers: ${{ inputs.chicken-egg-providers }}
@@ -88,7 +83,6 @@ jobs:
       default-python-version: ${{ inputs.default-python-version }}
       branch: ${{ inputs.default-branch }}
       use-uv: "false"
-      image-tag: ${{ inputs.image-tag }}
       build-provider-packages: ${{ inputs.default-branch == 'main' }}
       upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
       chicken-egg-providers: ${{ inputs.chicken-egg-providers }}
@@ -122,11 +116,10 @@ jobs:
       - name: Login to ghcr.io
         shell: bash
         run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-      - name: Pull PROD image ${{ inputs.default-python-version}}:${{ inputs.image-tag }}
-        run: breeze prod-image pull --tag-as-latest
+      - name: Pull PROD image ${{ inputs.default-python-version}}
+        run: breeze prod-image pull
         env:
           PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
-          IMAGE_TAG: "${{ inputs.image-tag }}"
       - name: "Setup python"
         uses: actions/setup-python@v5
         with:
@@ -138,15 +131,14 @@ jobs:
           cd ./docker_tests && \
           python -m pip install -r requirements.txt && \
           TEST_IMAGE=\"ghcr.io/${{ github.repository }}/${{ inputs.default-branch }}\
-          /prod/python${{ inputs.default-python-version }}:${{ inputs.image-tag }}\" \
+          /prod/python${{ inputs.default-python-version }}\" \
           python -m pytest test_examples_of_prod_image_building.py -n auto --color=yes"
 
   test-docker-compose-quick-start:
     timeout-minutes: 60
     name: "Docker-compose quick start with PROD image verifying"
     runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
     env:
-      IMAGE_TAG: "${{ inputs.image-tag }}"
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
       GITHUB_REPOSITORY: ${{ github.repository }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -161,14 +153,10 @@ jobs:
         with:
           fetch-depth: 2
           persist-credentials: false
-      - name: "Cleanup docker"
-        run: ./scripts/ci/cleanup_docker.sh
-      - name: "Install Breeze"
-        uses: ./.github/actions/breeze
-      - name: Login to ghcr.io
-        shell: bash
-        run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-      - name: "Pull image ${{ inputs.default-python-version}}:${{ inputs.image-tag }}"
-        run: breeze prod-image pull --tag-as-latest
+      - name: "Prepare breeze & CI image: ${{ inputs.default-python-version}}"
+        uses: ./.github/actions/prepare_breeze_and_image
+        with:
+          platform: "amd64"
+        id: breeze
       - name: "Test docker-compose quick start"
         run: breeze testing docker-compose-tests
diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml
@@ -288,7 +288,6 @@ jobs:
     runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
     env:
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
-      IMAGE_TAG: ${{ inputs.image-tag }}
       GITHUB_REPOSITORY: ${{ github.repository }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       GITHUB_USERNAME: ${{ github.actor }}