Add Python 3.12 support (#36755)

Finally after a number of dependency upgrades we seem to be able to
upgrade to Python 3.12 (pending universal_pathlib 0.2.0 conversion)

Several providers are excluded from being installed and wait for
Python 3.12, but it should not block Airlfow's general 3.12 support.

Co-authored-by: dirrao <[email protected]>
(cherry picked from commit 76dee0b)

Author: potiuk

.github/workflows/ci.yml

@@ -129,6 +129,7 @@ jobs:
       is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }}
       is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }}
       is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }}
+      latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }}
       chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }}
       has-migrations: ${{ steps.selective-checks.outputs.has-migrations }}
       source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }}
@@ -208,22 +209,22 @@ jobs:
           matrix.platform == 'linux/arm64' && needs.build-info.outputs.canary-run == 'true'
           && needs.build-info.outputs.default-branch == 'main'
       - name: "Push CI cache ${{ matrix.platform }}"
-        run: >
-          breeze ci-image build
-          --builder airflow_cache
-          --prepare-buildx-cache
-          --run-in-parallel
-          --platform ${{ matrix.platform }}
+        run: |
+          for PYTHON in ${{needs.build-info.outputs.python-versions-list-as-string}}; do
+            breeze ci-image build --builder airflow_cache --prepare-buildx-cache \
+            --python ${PYTHON} --platform ${{ matrix.platform }}
+          done
         env:
-          DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
           COMMIT_SHA: ${{ github.sha }}
+          PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
         if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main'
       - name: "Push CI latest image ${{ matrix.platform }}"
-        run: >
-          breeze ci-image build
-          --tag-as-latest --push --run-in-parallel --platform ${{ matrix.platform }}
+        run: |
+          for PYTHON in ${{needs.build-info.outputs.python-versions-list-as-string}}; do
+            breeze ci-image build --tag-as-latest --push --run-in-parallel \
+            --python ${PYTHON} --platform ${{ matrix.platform }}
+          done
         env:
-          DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
           COMMIT_SHA: ${{ github.sha }}
         # We only push "amd" image as it is really only needed for any kind of automated builds in CI
         # and currently there is not an easy way to make multi-platform image from two separate builds
@@ -252,6 +253,8 @@ jobs:
       RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
       UPGRADE_TO_NEWER_DEPENDENCIES: false
       PLATFORM: "linux/amd64"
+      PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
+      PYTHON_VERSION: ${{needs.build-info.outputs.default-python-version}}
     if: >
       needs.build-info.outputs.canary-run == 'true'
       && needs.build-info.outputs.default-branch == 'main'
@@ -267,6 +270,7 @@ jobs:
       - name: "Check that image builds quickly"
         run: breeze shell --max-time 120
 
+
   build-ci-images:
     strategy:
       fail-fast: true
@@ -566,7 +570,9 @@ jobs:
       RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
       PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
       UPGRADE_TO_NEWER_DEPENDENCIES: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}"
-    if: needs.build-info.outputs.basic-checks-only == 'false'
+    if: >
+      needs.build-info.outputs.basic-checks-only == 'false' &&
+      needs.build-info.outputs.latest-versions-only != 'true'
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -604,7 +610,9 @@ jobs:
     env:
       RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
       PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
-    if: needs.build-info.outputs.canary-run == 'true'
+    if: >
+      needs.build-info.outputs.canary-run == 'true' &&
+      needs.build-info.outputs.latest-versions-only != 'true'
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -666,6 +674,7 @@ jobs:
       RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
       PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
       UPGRADE_TO_NEWER_DEPENDENCIES: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}"
+    if: needs.build-info.outputs.latest-versions-only != 'true'
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -760,7 +769,9 @@ jobs:
     name: "Build docs"
     runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
     needs: [build-info, wait-for-ci-images]
-    if: needs.build-info.outputs.docs-build == 'true'
+    if: >
+      needs.build-info.outputs.docs-build == 'true' &&
+      needs.build-info.outputs.latest-versions-only != 'true'
     env:
       RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
       PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
@@ -824,7 +835,9 @@ jobs:
     name: "Spellcheck docs"
     runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
     needs: [build-info, wait-for-ci-images]
-    if: needs.build-info.outputs.docs-build == 'true'
+    if: >
+      needs.build-info.outputs.docs-build == 'true' &&
+      needs.build-info.outputs.latest-versions-only != 'true'
     env:
       RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
       PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
@@ -858,7 +871,9 @@ jobs:
     env:
       RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
       PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
-    if: needs.build-info.outputs.skip-provider-tests != 'true'
+    if: >
+      needs.build-info.outputs.skip-provider-tests != 'true' &&
+      needs.build-info.outputs.latest-versions-only != 'true'
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -1085,7 +1100,9 @@ jobs:
       JOB_ID: "helm-tests"
       USE_XDIST: "true"
     if: >
-      needs.build-info.outputs.needs-helm-tests == 'true' && needs.build-info.outputs.default-branch == 'main'
+      needs.build-info.outputs.needs-helm-tests == 'true' &&
+      needs.build-info.outputs.default-branch == 'main' &&
+      needs.build-info.outputs.latest-versions-only != 'true'
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -1385,7 +1402,9 @@ jobs:
       JOB_ID: >
         pendulum-2-${{needs.build-info.outputs.default-python-version}}-
         ${{needs.build-info.outputs.default-postgres-version}}
-    if: needs.build-info.outputs.run-tests == 'true'
+    if: >
+      needs.build-info.outputs.run-tests == 'true' &&
+      needs.build-info.outputs.latest-versions-only != 'true'
     steps:
       - name: Cleanup repo
         shell: bash
@@ -1796,7 +1815,9 @@ jobs:
       JOB_ID: "no-db-pendulum-2-${{needs.build-info.outputs.default-python-version}}"
       ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
       DOWNGRADE_PENDULUM: "true"
-    if: needs.build-info.outputs.run-tests == 'true'
+    if: >
+      needs.build-info.outputs.run-tests == 'true' &&
+      needs.build-info.outputs.latest-versions-only != 'true'
     steps:
       - name: Cleanup repo
         shell: bash
@@ -2210,19 +2231,19 @@ jobs:
         run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
         if: matrix.platform == 'linux/arm64'
       - name: "Push CI cache ${{ matrix.platform }}"
-        run: >
-          breeze ci-image build
-          --builder airflow_cache
-          --prepare-buildx-cache
-          --run-in-parallel
-          --platform ${{ matrix.platform }}
+        run: |
+          for PYTHON in ${{needs.build-info.outputs.python-versions-list-as-string}}; do
+            breeze ci-image build --builder airflow_cache --prepare-buildx-cache \
+            --python ${PYTHON} --platform ${{ matrix.platform }}
+          done
         env:
-          DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
           COMMIT_SHA: ${{ github.sha }}
       - name: "Push CI latest image ${{ matrix.platform }}"
-        run: >
-          breeze ci-image build
-          --tag-as-latest --push --run-in-parallel --platform ${{ matrix.platform }}
+        run: |
+          for PYTHON in ${{needs.build-info.outputs.python-versions-list-as-string}}; do
+            breeze ci-image build --tag-as-latest --push \
+            --python ${PYTHON} --platform ${{ matrix.platform }}
+          done
         env:
           DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
           COMMIT_SHA: ${{ github.sha }}
@@ -2231,8 +2252,6 @@ jobs:
         run: cp -v --no-preserve=mode,ownership ./dist/*.whl ./docker-context-files
       - name: "Push PROD cache ${{ matrix.python-version }} ${{ matrix.platform }}"
         run: |
-          # Do not run parallel builds here as they often fail due to github token expiry issues similar to
-          # those described in https://github.com/moby/buildkit/issues/2367
           for python in ${{needs.build-info.outputs.python-versions-list-as-string}}; do
             breeze prod-image build --builder airflow_cache --install-packages-from-context \
             --airflow-constraints-mode constraints-source-providers --prepare-buildx-cache \
@@ -2241,9 +2260,11 @@ jobs:
         env:
           COMMIT_SHA: ${{ github.sha }}
       - name: "Push PROD latest image ${{ matrix.platform }}"
-        run: >
-          breeze prod-image build --tag-as-latest --install-packages-from-context
-          --push --run-in-parallel --platform ${{ matrix.platform }}
+        run: |
+          for python in ${{needs.build-info.outputs.python-versions-list-as-string}}; do
+            breeze prod-image build --tag-as-latest --install-packages-from-context \
+            --push --run-in-parallel --python ${PYTHON} --platform ${{ matrix.platform }}
+          done
         env:
           DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
           COMMIT_SHA: ${{ github.sha }}
@@ -2298,13 +2319,13 @@ jobs:
       - name: >
           Build CI ARM images ${{ needs.build-info.outputs.image-tag }}
           ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
-        run: >
-          breeze ci-image build --run-in-parallel --builder airflow_cache --platform "linux/arm64"
+        run: |
+          for python in ${{needs.build-info.outputs.python-versions-list-as-string}}; do
+            breeze ci-image build --builder airflow_cache --platform "linux/arm64" --python ${python}
+          done
         env:
           UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
           DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
-          PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
-          DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
           COMMIT_SHA: ${{ github.sha }}
       - name: "Stop ARM instance"
         run: ./scripts/ci/images/ci_stop_arm_instance.sh

.pre-commit-config.yaml

@@ -704,7 +704,7 @@ repos:
         name: Verify usage of Airflow deprecation classes in core
         entry: category=DeprecationWarning|category=PendingDeprecationWarning
         files: \.py$
-        exclude: ^airflow/configuration\.py$|^airflow/providers|^scripts/in_container/verify_providers\.py$
+        exclude: ^airflow/configuration\.py$|^airflow/providers|^scripts/in_container/verify_providers\.py$|^tests/.*$
         pass_filenames: true
       - id: check-provide-create-sessions-imports
         language: pygrep

Dockerfile

@@ -1528,7 +1528,7 @@ ARG USE_CONSTRAINTS_FOR_CONTEXT_PACKAGES="false"
 
 # By changing the epoch we can force reinstalling Airflow and pip all dependencies
 # It can also be overwritten manually by setting the AIRFLOW_CI_BUILD_EPOCH environment variable.
-ARG AIRFLOW_CI_BUILD_EPOCH="10"
+ARG AIRFLOW_CI_BUILD_EPOCH="11"
 ENV AIRFLOW_CI_BUILD_EPOCH=${AIRFLOW_CI_BUILD_EPOCH}
 
 

Dockerfile.ci

@@ -1108,7 +1108,7 @@ ARG PYTHON_BASE_IMAGE
 ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
 
 # By increasing this number we can do force build of all dependencies
-ARG DEPENDENCIES_EPOCH_NUMBER="10"
+ARG DEPENDENCIES_EPOCH_NUMBER="11"
 
 # Make sure noninteractive debian install is used and language variables set
 ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \

README.md

@@ -100,7 +100,7 @@ Apache Airflow is tested with:
 
 |             | Main version (dev)           | Stable version (2.8.2) |
 |-------------|------------------------------|------------------------|
-| Python      | 3.8, 3.9, 3.10, 3.11         | 3.8, 3.9, 3.10, 3.11   |
+| Python      | 3.8, 3.9, 3.10, 3.11, 3.12   | 3.8, 3.9, 3.10, 3.11   |
 | Platform    | AMD64/ARM64(\*)              | AMD64/ARM64(\*)        |
 | Kubernetes  | 1.25, 1.26, 1.27, 1.28, 1.29 | 1.25, 1.26, 1.27, 1.28 |
 | PostgreSQL  | 12, 13, 14, 15, 16           | 12, 13, 14, 15, 16     |

airflow/__init__.py

@@ -39,7 +39,12 @@
 # configuration is therefore initted early here, simply by importing it.
 from airflow import configuration, settings
 
-__all__ = ["__version__", "DAG", "Dataset", "XComArg"]
+__all__ = [
+    "__version__",
+    "DAG",
+    "Dataset",
+    "XComArg",
+]
 
 # Make `airflow` a namespace package, supporting installing
 # airflow.providers.* in different locations (i.e. one in site, and one in user

contributing-docs/07_local_virtualenv.rst

@@ -37,7 +37,7 @@ Required Software Packages
 Use system-level package managers like yum, apt-get for Linux, or
 Homebrew for macOS to install required software packages:
 
-* Python (One of: 3.8, 3.9, 3.10, 3.11)
+* Python (One of: 3.8, 3.9, 3.10, 3.11, 3.12)
 * MySQL 5.7+
 * libxml
 * helm (only for helm chart tests)
@@ -186,6 +186,8 @@ This is what it shows currently:
 +-------------+---------+----------+---------------------------------------------------------------+
 | airflow-311 | virtual | devel    | Environment with Python 3.11                                  |
 +-------------+---------+----------+---------------------------------------------------------------+
+| airflow-312 | virtual | devel    | Environment with Python 3.12                                  |
++-------------+---------+----------+---------------------------------------------------------------+
 
 The default env (if you have not used one explicitly) is ``default`` and it is a Python 3.8
 virtualenv for maximum compatibility with ``devel`` extra installed - this devel extra contains the minimum set

dev/README_RELEASE_AIRFLOW.md

@@ -822,7 +822,7 @@ the older branches, you should set the "skip" field to true.
 ## Verify production images
 
 ```shell script
-for PYTHON in 3.8 3.9 3.10 3.11
+for PYTHON in 3.8 3.9 3.10 3.11 3.12
 do
     docker pull apache/airflow:${VERSION}-python${PYTHON}
     breeze prod-image verify --image-name apache/airflow:${VERSION}-python${PYTHON}

dev/breeze/README.md

@@ -66,6 +66,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT.
 
 ---------------------------------------------------------------------------------------------------------
 
-Package config hash: e2db123fd25e40b515520fb9f6ed32a601fe85e6a22b9845343bc5c81e5785b472527ed3bf6d07521c512d2222f99877a1ce1911ac504a3a6af7b7866aa674cc
+Package config hash: 0c6255210f3c20a29aa24405412399cf3f6ff897658f13a0fb7628b4888d6bfe99f49bdb2aa68d9045e14179d4be33a02543c12e394395931024522431bf5dec
 
 ---------------------------------------------------------------------------------------------------------

dev/breeze/doc/ci/02_images.md

@@ -574,7 +574,7 @@ percent-encoded when you access them via UI (/ = %2F)
 
 - \<BRANCH\> might be either "main" or "v2-\*-test"
 - \<X.Y\> - Python version (Major + Minor).Should be one of \["3.8",
-  "3.9", "3.10", "3.11"\].
+  "3.9", "3.10", "3.11", "3.12" \].
 - \<COMMIT_SHA\> - full-length SHA of commit either from the tip of the
   branch (for pushes/schedule) or commit from the tip of the branch used
   for the PR.