Drop python3.8 support core and providers - v2.10 backport

Add newsfragment

(cherry picked from commit f0c14e7)
(cherry picked from commit 1638974)

Integrated changes from PR #43310

migrate last 3 commits from original PR

Fix pytests of breeze

Fix constraints problem with cloudant in python 3.9

(cherry picked from commit 4375185)

Fix static checks for providers and core

Fix selective checks

(cherry picked from commit 1b55c8e)

Drop Python 3.8 support in provider packages

(cherry picked from commit 5de5ca7)
(cherry picked from commit 526948f)

.github/actions/breeze/action.yml

@@ -21,9 +21,6 @@ description: 'Sets up Python and Breeze'
 inputs:
   python-version:
     description: 'Python version to use'
-    # Version of Python used for reproducibility of the packages built
-    # Python 3.8 tarfile produces different tarballs than Python 3.9+ tarfile that's why we are forcing
-    # Python 3.9 for all release preparation commands to make sure that the tarballs are reproducible
     default: "3.9"
 outputs:
   host-python-version:

.github/workflows/check-providers.yml

@@ -108,6 +108,10 @@ jobs:
         run: >
           breeze release-management generate-issue-content-providers
           --only-available-in-dist --disable-progress
+      - name: Remove Python 3.9-incompatible provider packages
+        run: |
+          echo "Removing Python 3.9-incompatible provider: cloudant"
+          rm -vf dist/apache_airflow_providers_cloudant*
       - name: "Generate source constraints from CI image"
         shell: bash
         run: >

.pre-commit-config.yaml

@@ -1034,13 +1034,6 @@ repos:
         files: \.py$
         exclude: ^.*/.*_vendor/
         additional_dependencies: ['rich>=12.4.4']
-      - id: check-compat-cache-on-methods
-        name: Check that compat cache do not use on class methods
-        entry: ./scripts/ci/pre_commit/compat_cache_on_methods.py
-        language: python
-        pass_filenames: true
-        files: ^airflow/.*\.py$
-        exclude: ^.*/.*_vendor/
       - id: check-code-deprecations
         name: Check deprecations categories in decorators
         entry: ./scripts/ci/pre_commit/check_deprecations.py

.readthedocs.yml

@@ -20,7 +20,7 @@ formats: []
 sphinx:
     configuration: docs/rtd-deprecation/conf.py
 python:
-    version: "3.8"
+    version: "3.9"
     install:
         - method: pip
           path: .

Dockerfile

@@ -47,7 +47,7 @@ ARG AIRFLOW_USER_HOME_DIR=/home/airflow
 # latest released version here
 ARG AIRFLOW_VERSION="2.9.3"
 
-ARG PYTHON_BASE_IMAGE="python:3.8-slim-bookworm"
+ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm"
 
 ARG AIRFLOW_PIP_VERSION=24.3.1
 ARG AIRFLOW_UV_VERSION=0.4.29

Dockerfile.ci

@@ -16,7 +16,7 @@
 #
 # WARNING: THIS DOCKERFILE IS NOT INTENDED FOR PRODUCTION USE OR DEPLOYMENT.
 #
-ARG PYTHON_BASE_IMAGE="python:3.8-slim-bookworm"
+ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm"
 
 ##############################################################################################
 # This is the script image where we keep all inlined bash scripts needed in other segments
@@ -810,7 +810,7 @@ chmod 1777 /tmp
 
 AIRFLOW_SOURCES=$(cd "${IN_CONTAINER_DIR}/../.." || exit 1; pwd)
 
-PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:=3.8}
+PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:=3.9}
 
 export AIRFLOW_HOME=${AIRFLOW_HOME:=${HOME}}
 

INSTALL

@@ -141,9 +141,7 @@ This is what it shows currently:
 ┏━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
 ┃ Name        ┃ Type    ┃ Description                                                   ┃
 ┡━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
-│ default     │ virtual │ Default environment with Python 3.8 for maximum compatibility │
-├─────────────┼─────────┼───────────────────────────────────────────────────────────────┤
-│ airflow-38  │ virtual │ Environment with Python 3.8. No devel installed.              │
+│ default     │ virtual │ Default environment with Python 3.9 for maximum compatibility │
 ├─────────────┼─────────┼───────────────────────────────────────────────────────────────┤
 │ airflow-39  │ virtual │ Environment with Python 3.9. No devel installed.              │
 ├─────────────┼─────────┼───────────────────────────────────────────────────────────────┤
@@ -154,7 +152,7 @@ This is what it shows currently:
 │ airflow-312 │ virtual │ Environment with Python 3.12. No devel installed              │
 └─────────────┴─────────┴───────────────────────────────────────────────────────────────┘
 
-The default env (if you have not used one explicitly) is `default` and it is a Python 3.8
+The default env (if you have not used one explicitly) is `default` and it is a Python 3.9
 virtualenv for maximum compatibility with `devel` extra installed - this devel extra contains the minimum set
 of dependencies and tools that should be used during unit testing of core Airflow and running all `airflow`
 CLI commands - without support for providers or databases.
@@ -228,15 +226,15 @@ to avoid "works-for-me" syndrome, where you use different versions of dependenci
 that are used in main CI tests and by other contributors.
 
 There are different constraint files for different Python versions. For example, this command will install
-all basic devel requirements and requirements of Google provider as last successfully tested for Python 3.8:
+all basic devel requirements and requirements of Google provider as last successfully tested for Python 3.9:
 
     pip install -e ".[devel,google]"" \
-      --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-3.8.txt"
+      --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-3.9.txt"
 
 Using the 'constraints-no-providers' constraint files, you can upgrade Airflow without paying attention to the provider's dependencies. This allows you to keep installed provider dependencies and install the latest supported ones using pure Airflow core.
 
 pip install -e ".[devel]" \
-  --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-no-providers-3.8.txt"
+  --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-no-providers-3.9.txt"
 
 Airflow extras
 ==============

README.md

@@ -99,7 +99,7 @@ Apache Airflow is tested with:
 
 |             | Main version (dev)           | Stable version (2.10.4)      |
 |-------------|------------------------------|------------------------------|
-| Python      | 3.9, 3.10, 3.11, 3.12        | 3.8, 3.9, 3.10, 3.11, 3.12   |
+| Python      | 3.9, 3.10, 3.11, 3.12        | 3.9, 3.10, 3.11, 3.12        |
 | Platform    | AMD64/ARM64(\*)              | AMD64/ARM64(\*)              |
 | Kubernetes  | 1.26, 1.27, 1.28, 1.29, 1.30 | 1.26, 1.27, 1.28, 1.29, 1.30 |
 | PostgreSQL  | 12, 13, 14, 15, 16           | 12, 13, 14, 15, 16           |
@@ -311,7 +311,7 @@ They are based on the official release schedule of Python and Kubernetes, nicely
 1. We drop support for Python and Kubernetes versions when they reach EOL. Except for Kubernetes, a
    version stays supported by Airflow if two major cloud providers still provide support for it. We drop
    support for those EOL versions in main right after EOL date, and it is effectively removed when we release
-   the first new MINOR (Or MAJOR if there is no new MINOR version) of Airflow. For example, for Python 3.8 it
+   the first new MINOR (Or MAJOR if there is no new MINOR version) of Airflow. For example, for Python 3.9 it
    means that we will drop support in main right after 27.06.2023, and the first MAJOR or MINOR version of
    Airflow released after will not have it.
 

airflow/cli/commands/connection_command.py

@@ -21,6 +21,7 @@
 import json
 import os
 import warnings
+from functools import cache
 from pathlib import Path
 from typing import Any
 from urllib.parse import urlsplit, urlunsplit
@@ -30,7 +31,6 @@
 
 from airflow.cli.simple_table import AirflowConsole
 from airflow.cli.utils import is_stdout, print_export_output
-from airflow.compat.functools import cache
 from airflow.configuration import conf
 from airflow.exceptions import AirflowNotFoundException
 from airflow.hooks.base import BaseHook

airflow/configuration.py

@@ -1885,9 +1885,7 @@ def load_providers_configuration(self):
                     )
         self._default_values = create_default_config_parser(self.configuration_description)
         # sensitive_config_values needs to be refreshed here. This is a cached_property, so we can delete
-        # the cached values, and it will be refreshed on next access. This has been an implementation
-        # detail in Python 3.8 but as of Python 3.9 it is documented behaviour.
-        # See https://docs.python.org/3/library/functools.html#functools.cached_property
+        # the cached values, and it will be refreshed on next access.
         try:
             del self.sensitive_config_values
         except AttributeError:

airflow/io/__init__.py

@@ -18,6 +18,7 @@
 
 import inspect
 import logging
+from functools import cache
 from typing import (
     TYPE_CHECKING,
     Callable,
@@ -26,7 +27,6 @@
 
 from fsspec.implementations.local import LocalFileSystem
 
-from airflow.compat.functools import cache
 from airflow.providers_manager import ProvidersManager
 from airflow.stats import Stats
 from airflow.utils.module_loading import import_string

airflow/models/taskinstance.py

@@ -31,6 +31,7 @@
 from contextlib import nullcontext
 from datetime import timedelta
 from enum import Enum
+from functools import cache
 from typing import TYPE_CHECKING, Any, Callable, Collection, Generator, Iterable, Mapping, Tuple
 from urllib.parse import quote
 
@@ -69,7 +70,6 @@
 
 from airflow import settings
 from airflow.api_internal.internal_api_call import InternalApiConfig, internal_api_call
-from airflow.compat.functools import cache
 from airflow.configuration import conf
 from airflow.datasets import Dataset, DatasetAlias
 from airflow.datasets.manager import dataset_manager

airflow/operators/python.py

@@ -30,13 +30,13 @@
 import warnings
 from abc import ABCMeta, abstractmethod
 from collections.abc import Container
+from functools import cache
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import TYPE_CHECKING, Any, Callable, Collection, Iterable, Mapping, NamedTuple, Sequence, cast
 
 import lazy_object_proxy
 
-from airflow.compat.functools import cache
 from airflow.exceptions import (
     AirflowConfigException,
     AirflowException,

airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst

@@ -454,7 +454,7 @@ If you have pre-commit installed, pre-commit will be run automatically on commit
 manually after commit, you can run it via ``breeze static-checks --last-commit`` some of the tests might fail
 because suspension of the provider might cause changes in the dependencies, so if you see errors about
 missing dependencies imports, non-usable classes etc., you will need to build the CI image locally
-via ``breeze build-image --python 3.8 --upgrade-to-newer-dependencies`` after the first pre-commit run
+via ``breeze build-image --python 3.9 --upgrade-to-newer-dependencies`` after the first pre-commit run
 and then run the static checks again.
 
 If you want to be absolutely sure to run all static checks you can always do this via

airflow/providers/amazon/aws/transfers/sql_to_s3.py

@@ -223,12 +223,9 @@ def _partition_dataframe(self, df: pd.DataFrame) -> Iterable[tuple[str, pd.DataF
         for group_label in (grouped_df := df.groupby(**self.groupby_kwargs)).groups:
             yield (
                 cast(str, group_label),
-                cast(
-                    "pd.DataFrame",
-                    grouped_df.get_group(group_label)
-                    .drop(random_column_name, axis=1, errors="ignore")
-                    .reset_index(drop=True),
-                ),
+                grouped_df.get_group(group_label)
+                .drop(random_column_name, axis=1, errors="ignore")
+                .reset_index(drop=True),
             )
 
     def _get_hook(self) -> DbApiHook:

airflow/providers/amazon/aws/utils/mixins.py

@@ -28,13 +28,12 @@
 from __future__ import annotations
 
 import warnings
-from functools import cached_property
+from functools import cache, cached_property
 from typing import Any, Generic, NamedTuple, TypeVar
 
 from deprecated import deprecated
 from typing_extensions import final
 
-from airflow.compat.functools import cache
 from airflow.exceptions import AirflowProviderDeprecationWarning
 from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook
 

airflow/providers/cloudant/provider.yaml

@@ -44,8 +44,15 @@ versions:
   - 1.0.0
 
 dependencies:
-  - apache-airflow>=2.7.0
-  - cloudant>=2.13.0
+  - apache-airflow>=2.8.0
+  - ibmcloudant>=0.9.1
+
+excluded-python-versions:
+  # ibmcloudant transitively brings in urllib3 2.x, but the snowflake provider has a dependency that pins
+  # urllib3 to 1.x on Python 3.9; thus we exclude those Python versions from taking the update
+  # to ibmcloudant.
+  # See #21004, #41555, and https://github.com/snowflakedb/snowflake-connector-python/issues/2016
+  - "3.9"
 
 integrations:
   - integration-name: IBM Cloudant

airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py

@@ -19,14 +19,14 @@
 import logging
 import secrets
 import string
+from functools import cache
 from typing import TYPE_CHECKING
 
 import pendulum
 from deprecated import deprecated
 from kubernetes.client.rest import ApiException
 from slugify import slugify
 
-from airflow.compat.functools import cache
 from airflow.configuration import conf
 from airflow.exceptions import AirflowProviderDeprecationWarning
 

airflow/providers/common/io/xcom/backend.py

@@ -19,12 +19,13 @@
 import contextlib
 import json
 import uuid
+from functools import cache
+from pathlib import Path
 from typing import TYPE_CHECKING, Any, TypeVar
 from urllib.parse import urlsplit
 
 import fsspec.utils
 
-from airflow.compat.functools import cache
 from airflow.configuration import conf
 from airflow.io.path import ObjectStoragePath
 from airflow.models.xcom import BaseXCom
@@ -40,22 +41,6 @@
 SECTION = "common.io"
 
 
-def _is_relative_to(o: ObjectStoragePath, other: ObjectStoragePath) -> bool:
-    """
-    Return whether or not this path is relative to the other path.
-
-    This is a port of the pathlib.Path.is_relative_to method. It is not available in python 3.8.
-    """
-    if hasattr(o, "is_relative_to"):
-        return o.is_relative_to(other)
-
-    try:
-        o.relative_to(other)
-        return True
-    except ValueError:
-        return False
-
-
 def _get_compression_suffix(compression: str) -> str:
     """
     Return the compression suffix for the given compression.
@@ -111,7 +96,7 @@ def _get_full_path(data: str) -> ObjectStoragePath:
             raise TypeError(f"Not a valid url: {data}") from None
 
         if url.scheme:
-            if not _is_relative_to(ObjectStoragePath(data), p):
+            if not Path.is_relative_to(ObjectStoragePath(data), p):
                 raise ValueError(f"Invalid key: {data}")
             return p / data.replace(str(p), "", 1).lstrip("/")
 

airflow/providers/openlineage/conf.py

@@ -35,7 +35,7 @@ def decorator(func):
 
     cache = decorator
 else:
-    from airflow.compat.functools import cache
+    from functools import cache
 from airflow.configuration import conf
 
 _CONFIG_SECTION = "openlineage"

airflow/serialization/serialized_objects.py

@@ -25,6 +25,7 @@
 import logging
 import warnings
 import weakref
+from functools import cache
 from inspect import signature
 from textwrap import dedent
 from typing import TYPE_CHECKING, Any, Collection, Iterable, Mapping, NamedTuple, Union, cast
@@ -36,7 +37,6 @@
 
 from airflow import macros
 from airflow.callbacks.callback_requests import DagCallbackRequest, SlaCallbackRequest, TaskCallbackRequest
-from airflow.compat.functools import cache
 from airflow.configuration import conf
 from airflow.datasets import (
     BaseDataset,