Simplify caching mechanisms for CI and PROD images (regular PR)

.github/actions/post_tests_success/action.yml

@@ -33,7 +33,7 @@ runs:
     - name: "Upload artifact for warnings"
       uses: actions/upload-artifact@v4
       with:
-        name: test-warnings-${{env.JOB_ID}}
+        name: test-warnings-${{ env.JOB_ID }}
         path: ./files/warnings-*.txt
         retention-days: 7
         if-no-files-found: ignore
@@ -50,5 +50,5 @@ runs:
       if: env.ENABLE_COVERAGE == 'true' && env.TEST_TYPES != 'Helm' && inputs.python-version != '3.12'
       with:
         name: coverage-${{env.JOB_ID}}
-        flags: python-${{env.PYTHON_MAJOR_MINOR_VERSION}},${{env.BACKEND}}-${{env.BACKEND_VERSION}}
+        flags: python-${{ env.PYTHON_MAJOR_MINOR_VERSION }},${{ env.BACKEND }}-${{ env.BACKEND_VERSION }}
         directory: "./files/coverage-reports/"

.github/actions/prepare_all_ci_images/action.yml

@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: 'Prepare all CI images'
+description: 'Recreates current python CI images from artifacts for all python versions'
+inputs:
+  python-versions-list-as-string:
+    description: 'Stringified array of all Python versions to test - separated by spaces.'
+    required: true
+  platform:
+    description: 'Platform for the build - linux/amd64 or linux/arm64'
+    required: true
+outputs:
+  host-python-version:
+    description: Python version used in host
+    value: ${{ steps.breeze.outputs.host-python-version }}
+runs:
+  using: "composite"
+  steps:
+    - name: "Cleanup docker"
+      run: ./scripts/ci/cleanup_docker.sh
+      shell: bash
+    # TODO: Currently we cannot loop through the list of python versions and have dynamic list of
+    #       tasks. Instead we hardcode all possible python versions and they - but
+    #       this should be implemented in stash action as list of keys to download.
+    #       That includes 3.8 - 3.12 as we are backporting it to v2-10-test branch
+    #       This is captured in https://github.com/apache/airflow/issues/45268
+    - name: "Restore CI docker image ${{ inputs.platform }}:3.8"
+      uses: ./.github/actions/prepare_single_ci_image
+      with:
+        platform: ${{ inputs.platform }}
+        python: "3.8"
+        python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}
+    - name: "Restore CI docker image ${{ inputs.platform }}:3.9"
+      uses: ./.github/actions/prepare_single_ci_image
+      with:
+        platform: ${{ inputs.platform }}
+        python: "3.9"
+        python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}
+    - name: "Restore CI docker image ${{ inputs.platform }}:3.10"
+      uses: ./.github/actions/prepare_single_ci_image
+      with:
+        platform: ${{ inputs.platform }}
+        python: "3.10"
+        python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}
+    - name: "Restore CI docker image ${{ inputs.platform }}:3.11"
+      uses: ./.github/actions/prepare_single_ci_image
+      with:
+        platform: ${{ inputs.platform }}
+        python: "3.11"
+        python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}
+    - name: "Restore CI docker image ${{ inputs.platform }}:3.12"
+      uses: ./.github/actions/prepare_single_ci_image
+      with:
+        platform: ${{ inputs.platform }}
+        python: "3.12"
+        python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}

.github/actions/prepare_breeze_and_image/action.yml

@@ -16,30 +16,38 @@
 # under the License.
 #
 ---
-name: 'Prepare breeze && current python image'
-description: 'Installs breeze and pulls current python image'
+name: 'Prepare breeze && current image (CI or PROD)'
+description: 'Installs breeze and recreates current python image from artifact'
 inputs:
-  pull-image-type:
-    description: 'Which image to pull'
-    default: CI
+  python:
+    description: 'Python version for image to prepare'
+    required: true
+  image-type:
+    description: 'Which image type to prepare (ci/prod)'
+    default: "ci"
+  platform:
+    description: 'Platform for the build - linux/amd64 or linux/arm64'
+    required: true
 outputs:
   host-python-version:
     description: Python version used in host
     value: ${{ steps.breeze.outputs.host-python-version }}
 runs:
   using: "composite"
   steps:
+    - name: "Cleanup docker"
+      run: ./scripts/ci/cleanup_docker.sh
+      shell: bash
     - name: "Install Breeze"
       uses: ./.github/actions/breeze
       id: breeze
-    - name: Login to ghcr.io
-      shell: bash
-      run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-    - name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG }}
-      shell: bash
-      run: breeze ci-image pull --tag-as-latest
-      if: inputs.pull-image-type == 'CI'
-    - name: Pull PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG }}
+    - name: "Restore ${{ inputs.image-type }} docker image ${{ inputs.platform }}:${{ inputs.python }}"
+      uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
+      with:
+        key: "${{ inputs.image-type }}-image-save-${{ inputs.platform }}-${{ inputs.python }}"
+        path: "/tmp/"
+    - name: "Load ${{ inputs.image-type }} image ${{ inputs.platform }}:${{ inputs.python }}"
+      run: >
+        breeze ${{ inputs.image-type }}-image load
+        --platform ${{ inputs.platform }} --python ${{ inputs.python }}
       shell: bash
-      run: breeze prod-image pull --tag-as-latest
-      if: inputs.pull-image-type == 'PROD'

.github/actions/prepare_single_ci_image/action.yml

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: 'Prepare single CI image'
+description: >
+  Recreates current python image from artifacts (needed for the hard-coded actions calling all
+  possible Python versions in "prepare_all_ci_images" action. Hopefully we can get rid of it when
+  the https://github.com/apache/airflow/issues/45268 is resolved and we contribute capability of
+  downloading multiple keys to the stash action.
+inputs:
+  python:
+    description: 'Python version for image to prepare'
+    required: true
+  python-versions-list-as-string:
+    description: 'Stringified array of all Python versions to prepare - separated by spaces.'
+    required: true
+  platform:
+    description: 'Platform for the build - linux/amd64 or linux/arm64'
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: "Restore CI docker images ${{ inputs.platform }}:${{ inputs.python }}"
+      uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
+      with:
+        key: "ci-image-save-${{ inputs.platform }}-${{ inputs.python }}"
+        path: "/tmp/"
+      if: contains(inputs.python-versions-list-as-string, inputs.python)
+    - name: "Load CI image ${{ inputs.platform }}:${{ inputs.python }}"
+      run: breeze ci-image load --platform "${{ inputs.platform }}" --python "${{ inputs.python }}"
+      shell: bash
+      if: contains(inputs.python-versions-list-as-string, inputs.python)

.github/workflows/additional-ci-image-checks.yml

@@ -32,10 +32,6 @@ on:  # yamllint disable-line rule:truthy
         description: "The array of labels (in json form) determining self-hosted runners."
         required: true
         type: string
-      image-tag:
-        description: "Tag to set for the image"
-        required: true
-        type: string
       python-versions:
         description: "The list of python versions (stringified JSON array) to run the tests on."
         required: true
@@ -103,8 +99,6 @@ jobs:
       contents: read
       # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs
       # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo.
-      # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the
-      # "in-workflow-build" condition
       packages: write
     secrets: inherit
     with:
@@ -159,7 +153,7 @@ jobs:
 #  # There is no point in running this one in "canary" run, because the above step is doing the
 #  # same build anyway.
 #  build-ci-arm-images:
-#    name: Build CI ARM images (in-workflow)
+#    name: Build CI ARM images
 #    uses: ./.github/workflows/ci-image-build.yml
 #    permissions:
 #      contents: read
@@ -169,7 +163,6 @@ jobs:
 #      push-image: "false"
 #      runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
 #      runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }}
-#      image-tag: ${{ inputs.image-tag }}
 #      python-versions: ${{ inputs.python-versions }}
 #      platform: "linux/arm64"
 #      branch: ${{ inputs.branch }}