diff --git a/.github/actions/install-pre-commit/action.yml b/.github/actions/install-pre-commit/action.yml index 8ac0440ceae7f..30a3367710a92 100644 --- a/.github/actions/install-pre-commit/action.yml +++ b/.github/actions/install-pre-commit/action.yml @@ -24,7 +24,7 @@ inputs: default: "3.9" uv-version: description: 'uv version to use' - default: "0.5.14" # Keep this comment to allow automatic replacement of uv version + default: "0.5.20" # Keep this comment to allow automatic replacement of uv version pre-commit-version: description: 'pre-commit version to use' default: "4.0.1" # Keep this comment to allow automatic replacement of pre-commit version diff --git a/Dockerfile b/Dockerfile index fb82c882048c0..9b7e8a4391f3e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,7 +55,7 @@ ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm" # Also use `force pip` label on your PR to swap all places we use `uv` to `pip` ARG AIRFLOW_PIP_VERSION=24.3.1 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main" -ARG AIRFLOW_UV_VERSION=0.5.14 +ARG AIRFLOW_UV_VERSION=0.5.20 ARG AIRFLOW_USE_UV="false" ARG UV_HTTP_TIMEOUT="300" ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" diff --git a/Dockerfile.ci b/Dockerfile.ci index 5996ebe1ccb21..4e80ff1050abd 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1268,7 +1268,7 @@ COPY --from=scripts common.sh install_packaging_tools.sh install_additional_depe # Also use `force pip` label on your PR to swap all places we use `uv` to `pip` ARG AIRFLOW_PIP_VERSION=24.3.1 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main" -ARG AIRFLOW_UV_VERSION=0.5.14 +ARG AIRFLOW_UV_VERSION=0.5.20 # TODO(potiuk): automate with upgrade check (possibly) ARG AIRFLOW_PRE_COMMIT_VERSION="4.0.1" ARG AIRFLOW_PRE_COMMIT_UV_VERSION="4.1.4" diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index ba6af6ca11e13..5b99c94a4f33d 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -28,14 +28,6 @@ core: type: string example: ~ default: "{AIRFLOW_HOME}/dags" - dag_bundle_storage_path: - description: | - The folder where Airflow bundles can store files locally (if required). - By default, this is ``tempfile.gettempdir()/airflow``. This path must be absolute. - version_added: 3.0.0 - type: string - example: "`tempfile.gettempdir()/dag_bundles" - default: ~ hostname_callable: description: | Hostname by providing a path to a callable, which will resolve the hostname. @@ -2670,7 +2662,17 @@ dag_bundles: Configuration for the DAG bundles. This allows Airflow to load DAGs from different sources. options: - backends: + dag_bundle_storage_path: + description: | + String path to folder where Airflow bundles can store files locally. Not templated. + If no path is provided, Airflow will use ``Path(tempfile.gettempdir()) / "airflow"``. + This path must be absolute. + version_added: 3.0.0 + type: string + example: "/tmp/some-place" + default: ~ + + config_list: description: | List of backend configs. Must supply name, classpath, and kwargs for each backend. diff --git a/airflow/dag_processing/bundles/base.py b/airflow/dag_processing/bundles/base.py index da60f77cf4a96..9b55c0d4f0ecf 100644 --- a/airflow/dag_processing/bundles/base.py +++ b/airflow/dag_processing/bundles/base.py @@ -74,7 +74,7 @@ def _dag_bundle_root_storage_path(self) -> Path: This is the root path, shared by various bundles. Each bundle should have its own subdirectory. """ - if configured_location := conf.get("core", "dag_bundle_storage_path", fallback=None): + if configured_location := conf.get("dag_bundles", "dag_bundle_storage_path", fallback=None): return Path(configured_location) return Path(tempfile.gettempdir(), "airflow", "dag_bundles") diff --git a/airflow/dag_processing/bundles/manager.py b/airflow/dag_processing/bundles/manager.py index c5a2115b24f75..2aa8cf2303ddd 100644 --- a/airflow/dag_processing/bundles/manager.py +++ b/airflow/dag_processing/bundles/manager.py @@ -54,7 +54,7 @@ def parse_config(self) -> None: if self._bundle_config: return - backends = conf.getjson("dag_bundles", "backends") + backends = conf.getjson("dag_bundles", "config_list") if not backends: return diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md index 3d1d7d8b53eb7..84f71f34c3f1d 100644 --- a/dev/breeze/doc/ci/02_images.md +++ b/dev/breeze/doc/ci/02_images.md @@ -443,7 +443,7 @@ can be used for CI images: | `ADDITIONAL_DEV_APT_DEPS` | | Additional apt dev dependencies installed in the first part of the image | | `ADDITIONAL_DEV_APT_ENV` | | Additional env variables defined when installing dev deps | | `AIRFLOW_PIP_VERSION` | `24.3.1` | `pip` version used. | -| `AIRFLOW_UV_VERSION` | `0.5.14` | `uv` version used. | +| `AIRFLOW_UV_VERSION` | `0.5.20` | `uv` version used. | | `AIRFLOW_PRE_COMMIT_VERSION` | `4.0.1` | `pre-commit` version used. | | `AIRFLOW_PRE_COMMIT_UV_VERSION` | `4.1.4` | `pre-commit-uv` version used. | | `AIRFLOW_USE_UV` | `true` | Whether to use UV for installation. | diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 1dad40f7e5b9f..99e619240702b 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -234,7 +234,7 @@ class VersionedFile(NamedTuple): AIRFLOW_PIP_VERSION = "24.3.1" -AIRFLOW_UV_VERSION = "0.5.14" +AIRFLOW_UV_VERSION = "0.5.20" AIRFLOW_USE_UV = False # TODO: automate these as well WHEEL_VERSION = "0.44.0" diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index b18f8c4da3227..287611732567b 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -189,7 +189,7 @@ ALLOWED_INSTALL_MYSQL_CLIENT_TYPES = ["mariadb", "mysql"] PIP_VERSION = "24.3.1" -UV_VERSION = "0.5.14" +UV_VERSION = "0.5.20" DEFAULT_UV_HTTP_TIMEOUT = 300 DEFAULT_WSL2_HTTP_TIMEOUT = 900 diff --git a/providers/tests/fab/auth_manager/conftest.py b/providers/tests/fab/auth_manager/conftest.py index 9c61f7ab2dccc..4cb4b84a24cde 100644 --- a/providers/tests/fab/auth_manager/conftest.py +++ b/providers/tests/fab/auth_manager/conftest.py @@ -95,7 +95,7 @@ def _config_bundle(path_to_parse: Path | str): "kwargs": {"path": str(path_to_parse), "refresh_interval": 0}, } ] - with conf_vars({("dag_bundles", "backends"): json.dumps(bundle_config)}): + with conf_vars({("dag_bundles", "config_list"): json.dumps(bundle_config)}): yield return _config_bundle diff --git a/scripts/ci/install_breeze.sh b/scripts/ci/install_breeze.sh index a9a830ca31914..98c29d44d07c2 100755 --- a/scripts/ci/install_breeze.sh +++ b/scripts/ci/install_breeze.sh @@ -22,7 +22,7 @@ cd "$( dirname "${BASH_SOURCE[0]}" )/../../" PYTHON_ARG="" PIP_VERSION="24.3.1" -UV_VERSION="0.5.14" +UV_VERSION="0.5.20" if [[ ${PYTHON_VERSION=} != "" ]]; then PYTHON_ARG="--python=$(which python"${PYTHON_VERSION}") " fi diff --git a/scripts/tools/setup_breeze b/scripts/tools/setup_breeze index 8b3932c982008..272d56c89d64b 100755 --- a/scripts/tools/setup_breeze +++ b/scripts/tools/setup_breeze @@ -27,7 +27,7 @@ COLOR_YELLOW=$'\e[33m' COLOR_BLUE=$'\e[34m' COLOR_RESET=$'\e[0m' -UV_VERSION="0.5.14" +UV_VERSION="0.5.20" function manual_instructions() { echo diff --git a/task_sdk/tests/execution_time/test_supervisor.py b/task_sdk/tests/execution_time/test_supervisor.py index fb84713216625..59afa26dc2aa5 100644 --- a/task_sdk/tests/execution_time/test_supervisor.py +++ b/task_sdk/tests/execution_time/test_supervisor.py @@ -74,7 +74,7 @@ def lineno(): def local_dag_bundle_cfg(path, name="my-bundle"): return { - "AIRFLOW__DAG_BUNDLES__BACKENDS": json.dumps( + "AIRFLOW__DAG_BUNDLES__CONFIG_LIST": json.dumps( [ { "name": name, diff --git a/task_sdk/tests/execution_time/test_task_runner.py b/task_sdk/tests/execution_time/test_task_runner.py index 0e3698050b88e..60b39da455c69 100644 --- a/task_sdk/tests/execution_time/test_task_runner.py +++ b/task_sdk/tests/execution_time/test_task_runner.py @@ -130,7 +130,7 @@ def test_parse(test_dags_dir: Path, make_ti_context): with patch.dict( os.environ, { - "AIRFLOW__DAG_BUNDLES__BACKENDS": json.dumps( + "AIRFLOW__DAG_BUNDLES__CONFIG_LIST": json.dumps( [ { "name": "my-bundle", @@ -574,7 +574,7 @@ def test_dag_parsing_context(make_ti_context, mock_supervisor_comms, monkeypatch ] ) - monkeypatch.setenv("AIRFLOW__DAG_BUNDLES__BACKENDS", dag_bundle_val) + monkeypatch.setenv("AIRFLOW__DAG_BUNDLES__CONFIG_LIST", dag_bundle_val) ti, _ = startup() # Presence of `conditional_task` below means DAG ID is properly set in the parsing context! diff --git a/tests/conftest.py b/tests/conftest.py index fca82aee34b87..8082238808dd4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -111,7 +111,7 @@ def _config_bundle(path_to_parse: Path | str): "kwargs": {"path": str(path_to_parse), "refresh_interval": 0}, } ] - with conf_vars({("dag_bundles", "backends"): json.dumps(bundle_config)}): + with conf_vars({("dag_bundles", "config_list"): json.dumps(bundle_config)}): yield return _config_bundle diff --git a/tests/dag_processing/bundles/test_dag_bundle_manager.py b/tests/dag_processing/bundles/test_dag_bundle_manager.py index b0baa21c6f84c..26f8b045837eb 100644 --- a/tests/dag_processing/bundles/test_dag_bundle_manager.py +++ b/tests/dag_processing/bundles/test_dag_bundle_manager.py @@ -70,7 +70,7 @@ def test_parse_bundle_config(value, expected): """Test that bundle_configs are read from configuration.""" envs = {"AIRFLOW__CORE__LOAD_EXAMPLES": "False"} if value: - envs["AIRFLOW__DAG_BUNDLES__BACKENDS"] = value + envs["AIRFLOW__DAG_BUNDLES__CONFIG_LIST"] = value cm = nullcontext() exp_fail = False if isinstance(expected, str): @@ -108,7 +108,7 @@ def path(self): def test_get_bundle(): """Test that get_bundle builds and returns a bundle.""" - with patch.dict(os.environ, {"AIRFLOW__DAG_BUNDLES__BACKENDS": json.dumps(BASIC_BUNDLE_CONFIG)}): + with patch.dict(os.environ, {"AIRFLOW__DAG_BUNDLES__CONFIG_LIST": json.dumps(BASIC_BUNDLE_CONFIG)}): bundle_manager = DagBundlesManager() with pytest.raises(ValueError, match="'bundle-that-doesn't-exist' is not configured"): @@ -120,7 +120,7 @@ def test_get_bundle(): assert bundle.refresh_interval == 1 # And none for version also works! - with patch.dict(os.environ, {"AIRFLOW__DAG_BUNDLES__BACKENDS": json.dumps(BASIC_BUNDLE_CONFIG)}): + with patch.dict(os.environ, {"AIRFLOW__DAG_BUNDLES__CONFIG_LIST": json.dumps(BASIC_BUNDLE_CONFIG)}): bundle = bundle_manager.get_bundle(name="my-test-bundle") assert isinstance(bundle, BasicBundle) assert bundle.name == "my-test-bundle" @@ -144,7 +144,7 @@ def _get_bundle_names_and_active(): ) # Initial add - with patch.dict(os.environ, {"AIRFLOW__DAG_BUNDLES__BACKENDS": json.dumps(BASIC_BUNDLE_CONFIG)}): + with patch.dict(os.environ, {"AIRFLOW__DAG_BUNDLES__CONFIG_LIST": json.dumps(BASIC_BUNDLE_CONFIG)}): manager = DagBundlesManager() manager.sync_bundles_to_db() assert _get_bundle_names_and_active() == [("my-test-bundle", True)] @@ -156,13 +156,13 @@ def _get_bundle_names_and_active(): assert _get_bundle_names_and_active() == [("dags-folder", True), ("my-test-bundle", False)] # Re-enable one that reappears in config - with patch.dict(os.environ, {"AIRFLOW__DAG_BUNDLES__BACKENDS": json.dumps(BASIC_BUNDLE_CONFIG)}): + with patch.dict(os.environ, {"AIRFLOW__DAG_BUNDLES__CONFIG_LIST": json.dumps(BASIC_BUNDLE_CONFIG)}): manager = DagBundlesManager() manager.sync_bundles_to_db() assert _get_bundle_names_and_active() == [("dags-folder", False), ("my-test-bundle", True)] -@conf_vars({("dag_bundles", "backends"): json.dumps(BASIC_BUNDLE_CONFIG)}) +@conf_vars({("dag_bundles", "config_list"): json.dumps(BASIC_BUNDLE_CONFIG)}) @pytest.mark.parametrize("version", [None, "hello"]) def test_view_url(version): """Test that view_url calls the bundle's view_url method.""" @@ -185,6 +185,6 @@ def test_example_dags_bundle_added(): def test_example_dags_name_is_reserved(): reserved_name_config = [{"name": "example_dags"}] - with conf_vars({("dag_bundles", "backends"): json.dumps(reserved_name_config)}): + with conf_vars({("dag_bundles", "config_list"): json.dumps(reserved_name_config)}): with pytest.raises(AirflowConfigException, match="Bundle name 'example_dags' is a reserved name."): DagBundlesManager().parse_config() diff --git a/tests/dag_processing/test_dag_bundles.py b/tests/dag_processing/test_dag_bundles.py index 32b2277b68c54..6f6fb2c80f044 100644 --- a/tests/dag_processing/test_dag_bundles.py +++ b/tests/dag_processing/test_dag_bundles.py @@ -39,12 +39,12 @@ @pytest.fixture(autouse=True) def bundle_temp_dir(tmp_path): - with conf_vars({("core", "dag_bundle_storage_path"): str(tmp_path)}): + with conf_vars({("dag_bundles", "dag_bundle_storage_path"): str(tmp_path)}): yield tmp_path def test_default_dag_storage_path(): - with conf_vars({("core", "dag_bundle_storage_path"): ""}): + with conf_vars({("dag_bundles", "dag_bundle_storage_path"): ""}): bundle = LocalDagBundle(name="test", path="/hello") assert bundle._dag_bundle_root_storage_path == Path(tempfile.gettempdir(), "airflow", "dag_bundles") @@ -60,7 +60,7 @@ def get_current_version(self): def path(self): pass - with conf_vars({("core", "dag_bundle_storage_path"): None}): + with conf_vars({("dag_bundles", "dag_bundle_storage_path"): None}): bundle = BasicBundle(name="test") assert bundle._dag_bundle_root_storage_path == Path(tempfile.gettempdir(), "airflow", "dag_bundles") diff --git a/tests/dag_processing/test_manager.py b/tests/dag_processing/test_manager.py index 4ab55c24eefc0..68740c4601ba0 100644 --- a/tests/dag_processing/test_manager.py +++ b/tests/dag_processing/test_manager.py @@ -857,7 +857,7 @@ def test_bundles_are_refreshed(self): bundletwo.refresh_interval = 300 bundletwo.get_current_version.return_value = None - with conf_vars({("dag_bundles", "backends"): json.dumps(config)}): + with conf_vars({("dag_bundles", "config_list"): json.dumps(config)}): DagBundlesManager().sync_bundles_to_db() with mock.patch( "airflow.dag_processing.bundles.manager.DagBundlesManager" @@ -910,7 +910,7 @@ def test_bundles_versions_are_stored(self): mybundle.supports_versioning = True mybundle.get_current_version.return_value = "123" - with conf_vars({("dag_bundles", "backends"): json.dumps(config)}): + with conf_vars({("dag_bundles", "config_list"): json.dumps(config)}): DagBundlesManager().sync_bundles_to_db() with mock.patch( "airflow.dag_processing.bundles.manager.DagBundlesManager" diff --git a/tests/serialization/test_dag_serialization.py b/tests/serialization/test_dag_serialization.py index a67864fb1ca6b..2b5d4cce4c7bb 100644 --- a/tests/serialization/test_dag_serialization.py +++ b/tests/serialization/test_dag_serialization.py @@ -415,6 +415,13 @@ def setup_test_cases(self): ) ) + # Skip that test if latest botocore is used - it reads all example dags and in case latest botocore + # is upgraded to latest, usually aiobotocore can't be installed and some of the system tests will fail with + # import errors. + @pytest.mark.skipif( + os.environ.get("UPGRADE_BOTO", "") == "true", + reason="This test is skipped when latest botocore is installed", + ) @pytest.mark.db_test def test_serialization(self): """Serialization and deserialization should work for every DAG and Operator."""