Skip to content

Commit 07b074f

Browse files
Owen-CH-LeungOwen-CH-Leung
authored
Merge branch 'main' into Add-write-log-feature-to-ElasticSearchTaskHandler
2 parents 445da1f + 019ed2b commit 07b074f

File tree

822 files changed

+31651
-15573
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

822 files changed

+31651
-15573
lines changed

‎.github/actions/breeze/action.yml

+5-4
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@ inputs:
2222
python-version:
2323
description: 'Python version to use'
2424
default: "3.9"
25+
use-uv:
26+
description: 'Whether to use uv tool'
27+
required: true
2528
outputs:
2629
host-python-version:
2730
description: Python version used in host
@@ -33,13 +36,11 @@ runs:
3336
uses: actions/setup-python@v5
3437
with:
3538
python-version: ${{ inputs.python-version }}
36-
cache: 'pip'
37-
cache-dependency-path: ./dev/breeze/pyproject.toml
39+
# NOTE! Installing Breeze without using cache is FASTER than when using cache - uv is so fast and has
40+
# so low overhead, that just running upload cache/restore cache is slower than installing it from scratch
3841
- name: "Install Breeze"
3942
shell: bash
4043
run: ./scripts/ci/install_breeze.sh
41-
env:
42-
PYTHON_VERSION: ${{ inputs.python-version }}
4344
- name: "Free space"
4445
shell: bash
4546
run: breeze ci free-space

‎.github/actions/checkout_target_commit/action.yml

-81
This file was deleted.

‎.github/actions/install-pre-commit/action.yml

+42-13
Original file line numberDiff line numberDiff line change
@@ -21,29 +21,58 @@ description: 'Installs pre-commit and related packages'
2121
inputs:
2222
python-version:
2323
description: 'Python version to use'
24-
default: 3.9
24+
default: "3.9"
2525
uv-version:
2626
description: 'uv version to use'
27-
default: 0.5.5
27+
default: "0.5.14" # Keep this comment to allow automatic replacement of uv version
2828
pre-commit-version:
2929
description: 'pre-commit version to use'
30-
default: 4.0.1
30+
default: "4.0.1" # Keep this comment to allow automatic replacement of pre-commit version
3131
pre-commit-uv-version:
3232
description: 'pre-commit-uv version to use'
33-
default: 4.1.4
33+
default: "4.1.4" # Keep this comment to allow automatic replacement of pre-commit-uv version
3434
runs:
3535
using: "composite"
3636
steps:
3737
- name: Install pre-commit, uv, and pre-commit-uv
3838
shell: bash
39+
env:
40+
UV_VERSION: ${{inputs.uv-version}}
41+
PRE_COMMIT_VERSION: ${{inputs.pre-commit-version}}
42+
PRE_COMMIT_UV_VERSION: ${{inputs.pre-commit-uv-version}}
3943
run: |
40-
pip install uv==${{inputs.uv-version}} || true
41-
uv tool install pre-commit==${{inputs.pre-commit-version}} --with uv==${{inputs.uv-version}} \
42-
--with pre-commit-uv==${{inputs.pre-commit-uv-version}}
43-
- name: Cache pre-commit envs
44-
uses: actions/cache@v4
44+
pip install uv==${UV_VERSION} || true
45+
uv tool install pre-commit==${PRE_COMMIT_VERSION} --with uv==${UV_VERSION} \
46+
--with pre-commit-uv==${PRE_COMMIT_UV_VERSION}
47+
working-directory: ${{ github.workspace }}
48+
# We need to use tar file with archive to restore all the permissions and symlinks
49+
- name: "Delete ~.cache"
50+
run: |
51+
du ~/ --max-depth=2
52+
echo
53+
echo Deleting ~/.cache
54+
echo
55+
rm -rf ~/.cache
56+
echo
57+
shell: bash
58+
- name: "Restore pre-commit cache"
59+
uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
4560
with:
46-
path: ~/.cache/pre-commit
47-
key: "pre-commit-${{inputs.python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
48-
restore-keys: |
49-
pre-commit-${{inputs.python-version}}-
61+
key: cache-pre-commit-v4-${{ inputs.python-version }}-${{ hashFiles('.pre-commit-config.yaml') }}
62+
path: /tmp/
63+
id: restore-pre-commit-cache
64+
- name: "Restore .cache from the tar file"
65+
run: tar -C ~ -xzf /tmp/cache-pre-commit.tar.gz
66+
shell: bash
67+
if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true'
68+
- name: "Show restored files"
69+
run: |
70+
echo "Restored files"
71+
du ~/ --max-depth=2
72+
echo
73+
shell: bash
74+
if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true'
75+
- name: Install pre-commit hooks
76+
shell: bash
77+
run: pre-commit install-hooks || (cat ~/.cache/pre-commit/pre-commit.log && exit 1)
78+
working-directory: ${{ github.workspace }}

‎.github/actions/post_tests_success/action.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ runs:
3333
- name: "Upload artifact for warnings"
3434
uses: actions/upload-artifact@v4
3535
with:
36-
name: test-warnings-${{env.JOB_ID}}
36+
name: test-warnings-${{ env.JOB_ID }}
3737
path: ./files/warnings-*.txt
3838
retention-days: 7
3939
if-no-files-found: ignore
@@ -50,5 +50,5 @@ runs:
5050
if: env.ENABLE_COVERAGE == 'true' && env.TEST_TYPES != 'Helm' && inputs.python-version != '3.12'
5151
with:
5252
name: coverage-${{env.JOB_ID}}
53-
flags: python-${{env.PYTHON_MAJOR_MINOR_VERSION}},${{env.BACKEND}}-${{env.BACKEND_VERSION}}
53+
flags: python-${{ env.PYTHON_MAJOR_MINOR_VERSION }},${{ env.BACKEND }}-${{ env.BACKEND_VERSION }}
5454
directory: "./files/coverage-reports/"

‎.github/actions/prepare_all_ci_images/action.yml

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
#
18+
---
19+
name: 'Prepare all CI images'
20+
description: 'Recreates current python CI images from artifacts for all python versions'
21+
inputs:
22+
python-versions-list-as-string:
23+
description: 'Stringified array of all Python versions to test - separated by spaces.'
24+
required: true
25+
platform:
26+
description: 'Platform for the build - linux/amd64 or linux/arm64'
27+
required: true
28+
runs:
29+
using: "composite"
30+
steps:
31+
- name: "Cleanup docker"
32+
run: ./scripts/ci/cleanup_docker.sh
33+
shell: bash
34+
# TODO: Currently we cannot loop through the list of python versions and have dynamic list of
35+
# tasks. Instead we hardcode all possible python versions and they - but
36+
# this should be implemented in stash action as list of keys to download.
37+
# That includes 3.8 - 3.12 as we are backporting it to v2-10-test branch
38+
# This is captured in https://github.com/apache/airflow/issues/45268
39+
- name: "Restore CI docker image ${{ inputs.platform }}:3.8"
40+
uses: ./.github/actions/prepare_single_ci_image
41+
with:
42+
platform: ${{ inputs.platform }}
43+
python: "3.8"
44+
python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}
45+
- name: "Restore CI docker image ${{ inputs.platform }}:3.9"
46+
uses: ./.github/actions/prepare_single_ci_image
47+
with:
48+
platform: ${{ inputs.platform }}
49+
python: "3.9"
50+
python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}
51+
- name: "Restore CI docker image ${{ inputs.platform }}:3.10"
52+
uses: ./.github/actions/prepare_single_ci_image
53+
with:
54+
platform: ${{ inputs.platform }}
55+
python: "3.10"
56+
python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}
57+
- name: "Restore CI docker image ${{ inputs.platform }}:3.11"
58+
uses: ./.github/actions/prepare_single_ci_image
59+
with:
60+
platform: ${{ inputs.platform }}
61+
python: "3.11"
62+
python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}
63+
- name: "Restore CI docker image ${{ inputs.platform }}:3.12"
64+
uses: ./.github/actions/prepare_single_ci_image
65+
with:
66+
platform: ${{ inputs.platform }}
67+
python: "3.12"
68+
python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}

‎.github/actions/prepare_breeze_and_image/action.yml

+35-13
Original file line numberDiff line numberDiff line change
@@ -16,30 +16,52 @@
1616
# under the License.
1717
#
1818
---
19-
name: 'Prepare breeze && current python image'
20-
description: 'Installs breeze and pulls current python image'
19+
name: 'Prepare breeze && current image (CI or PROD)'
20+
description: 'Installs breeze and recreates current python image from artifact'
2121
inputs:
22-
pull-image-type:
23-
description: 'Which image to pull'
24-
default: CI
22+
python:
23+
description: 'Python version for image to prepare'
24+
required: true
25+
image-type:
26+
description: 'Which image type to prepare (ci/prod)'
27+
default: "ci"
28+
platform:
29+
description: 'Platform for the build - linux/amd64 or linux/arm64'
30+
required: true
31+
use-uv:
32+
description: 'Whether to use uv'
33+
required: true
2534
outputs:
2635
host-python-version:
2736
description: Python version used in host
2837
value: ${{ steps.breeze.outputs.host-python-version }}
2938
runs:
3039
using: "composite"
3140
steps:
41+
- name: "Cleanup docker"
42+
run: ./scripts/ci/cleanup_docker.sh
43+
shell: bash
3244
- name: "Install Breeze"
3345
uses: ./.github/actions/breeze
46+
with:
47+
use-uv: ${{ inputs.use-uv }}
3448
id: breeze
35-
- name: Login to ghcr.io
49+
- name: Check free space
50+
run: df -H
3651
shell: bash
37-
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
38-
- name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG }}
52+
- name: Make /mnt/ directory writeable
53+
run: sudo chown -R ${USER} /mnt
3954
shell: bash
40-
run: breeze ci-image pull --tag-as-latest
41-
if: inputs.pull-image-type == 'CI'
42-
- name: Pull PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG }}
55+
- name: "Restore ${{ inputs.image-type }} docker image ${{ inputs.platform }}:${{ inputs.python }}"
56+
uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
57+
with:
58+
key: ${{ inputs.image-type }}-image-save-${{ inputs.platform }}-${{ inputs.python }}
59+
path: "/mnt/"
60+
- name: "Load ${{ inputs.image-type }} image ${{ inputs.platform }}:${{ inputs.python }}"
61+
env:
62+
PLATFORM: ${{ inputs.platform }}
63+
PYTHON: ${{ inputs.python }}
64+
IMAGE_TYPE: ${{ inputs.image-type }}
65+
run: >
66+
breeze ${IMAGE_TYPE}-image load --platform "${PLATFORM}" --python "${PYTHON}" --image-file-dir "/mnt"
4367
shell: bash
44-
run: breeze prod-image pull --tag-as-latest
45-
if: inputs.pull-image-type == 'PROD'

‎.github/actions/prepare_single_ci_image/action.yml

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
#
18+
---
19+
name: 'Prepare single CI image'
20+
description: >
21+
Recreates current python image from artifacts (needed for the hard-coded actions calling all
22+
possible Python versions in "prepare_all_ci_images" action. Hopefully we can get rid of it when
23+
the https://github.com/apache/airflow/issues/45268 is resolved and we contribute capability of
24+
downloading multiple keys to the stash action.
25+
inputs:
26+
python:
27+
description: 'Python version for image to prepare'
28+
required: true
29+
python-versions-list-as-string:
30+
description: 'Stringified array of all Python versions to prepare - separated by spaces.'
31+
required: true
32+
platform:
33+
description: 'Platform for the build - linux/amd64 or linux/arm64'
34+
required: true
35+
runs:
36+
using: "composite"
37+
steps:
38+
- name: Check free space
39+
run: df -H
40+
shell: bash
41+
- name: Make /mnt/ directory writeable
42+
run: sudo chown -R ${USER} /mnt
43+
shell: bash
44+
- name: "Restore CI docker images ${{ inputs.platform }}:${{ inputs.python }}"
45+
uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
46+
with:
47+
key: ci-image-save-${{ inputs.platform }}-${{ inputs.python }}
48+
path: "/mnt/"
49+
if: contains(inputs.python-versions-list-as-string, inputs.python)
50+
- name: "Load CI image ${{ inputs.platform }}:${{ inputs.python }}"
51+
env:
52+
PLATFORM: ${{ inputs.platform }}
53+
PYTHON: ${{ inputs.python }}
54+
run: breeze ci-image load --platform "${PLATFORM}" --python "${PYTHON}" --image-file-dir "/mnt/"
55+
shell: bash
56+
if: contains(inputs.python-versions-list-as-string, inputs.python)

0 commit comments

Comments
 (0)