diff --git a/.github/ISSUE_TEMPLATE/airflow_bug_report.yml b/.github/ISSUE_TEMPLATE/airflow_bug_report.yml index 6f8598730a3eef..5cb424a5a5f298 100644 --- a/.github/ISSUE_TEMPLATE/airflow_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/airflow_bug_report.yml @@ -25,7 +25,7 @@ body: the latest release or main to see if the issue is fixed before reporting it. multiple: false options: - - "2.10.3" + - "2.10.4" - "main (development)" - "Other Airflow 2 version (please specify below)" validations: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b411e9e4b10687..10c14bdbc80269 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -744,23 +744,17 @@ jobs: steps: - name: Notify Slack id: slack - uses: slackapi/slack-github-action@v1.27.0 + uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 with: - channel-id: 'internal-airflow-ci-cd' + method: chat.postMessage + token: ${{ env.SLACK_BOT_TOKEN }} # yamllint disable rule:line-length payload: | - { - "text": "🚨🕒 Scheduled CI Failure Alert 🕒🚨\n\n*Details:* ", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "🚨🕒 Scheduled CI Failure Alert 🕒🚨\n\n*Details:* " - } - } - ] - } + channel: "internal-airflow-ci-cd" + text: "🚨🕒 Scheduled CI Failure Alert 🕒🚨\n\n*Details:* " + blocks: + - type: "section" + text: + type: "mrkdwn" + text: "🚨🕒 Scheduled CI Failure Alert 🕒🚨\n\n*Details:* " # yamllint enable rule:line-length - env: - SLACK_BOT_TOKEN: ${{ env.SLACK_BOT_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 65e3a0895b2cfe..e06068a02bd39c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -491,21 +491,21 @@ repos: files: ^docs/apache-airflow/extra-packages-ref\.rst$|^hatch_build.py pass_filenames: false entry: ./scripts/ci/pre_commit/check_extra_packages_ref.py - additional_dependencies: ['rich>=12.4.4', 'hatchling==1.26.3', 'tabulate'] + additional_dependencies: ['rich>=12.4.4', 'hatchling==1.27.0', 'tabulate'] - id: check-hatch-build-order name: Check order of dependencies in hatch_build.py language: python files: ^hatch_build.py$ pass_filenames: false entry: ./scripts/ci/pre_commit/check_order_hatch_build.py - additional_dependencies: ['rich>=12.4.4', 'hatchling==1.26.3'] + additional_dependencies: ['rich>=12.4.4', 'hatchling==1.27.0'] - id: update-extras name: Update extras in documentation entry: ./scripts/ci/pre_commit/insert_extras.py language: python files: ^contributing-docs/12_airflow_dependencies_and_extras.rst$|^INSTALL$|^providers/src/airflow/providers/.*/provider\.yaml$|^Dockerfile.* pass_filenames: false - additional_dependencies: ['rich>=12.4.4', 'hatchling==1.26.3'] + additional_dependencies: ['rich>=12.4.4', 'hatchling==1.27.0'] - id: check-extras-order name: Check order of extras in Dockerfile entry: ./scripts/ci/pre_commit/check_order_dockerfile_extras.py diff --git a/Dockerfile b/Dockerfile index 11e54e2b195dfe..1926b02f41b360 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,7 @@ ARG AIRFLOW_UID="50000" ARG AIRFLOW_USER_HOME_DIR=/home/airflow # latest released version here -ARG AIRFLOW_VERSION="2.10.3" +ARG AIRFLOW_VERSION="2.10.4" ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm" @@ -55,7 +55,7 @@ ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm" # Also use `force pip` label on your PR to swap all places we use `uv` to `pip` ARG AIRFLOW_PIP_VERSION=24.3.1 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main" -ARG AIRFLOW_UV_VERSION=0.5.8 +ARG AIRFLOW_UV_VERSION=0.5.9 ARG AIRFLOW_USE_UV="false" ARG UV_HTTP_TIMEOUT="300" ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" diff --git a/Dockerfile.ci b/Dockerfile.ci index 8ec65de6998f0a..4517de85fec2cf 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1381,7 +1381,7 @@ RUN bash /scripts/docker/install_packaging_tools.sh; \ # Also use `force pip` label on your PR to swap all places we use `uv` to `pip` ARG AIRFLOW_PIP_VERSION=24.3.1 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main" -ARG AIRFLOW_UV_VERSION=0.5.8 +ARG AIRFLOW_UV_VERSION=0.5.9 # TODO(potiuk): automate with upgrade check (possibly) ARG AIRFLOW_PRE_COMMIT_VERSION="4.0.1" ARG AIRFLOW_PRE_COMMIT_UV_VERSION="4.1.4" diff --git a/PROVIDERS.rst b/PROVIDERS.rst index b874f70f0cf469..60ef8ef9139893 100644 --- a/PROVIDERS.rst +++ b/PROVIDERS.rst @@ -153,8 +153,8 @@ Airflow version to the next MINOR release, when 12 months passed since the first MINOR version of Airflow. For example this means that by default we upgrade the minimum version of Airflow supported by providers -to 2.8.0 in the first Provider's release after 18th of December 2024. 18th of December 2023 is the date when the -first ``PATCHLEVEL`` of 2.8 (2.8.0) has been released. +to 2.9.0 in the first Provider's release after 8th of April 2025. 8th of April 2024 is the date when the +first ``PATCHLEVEL`` of 2.9 (2.9.0) has been released. When we increase the minimum Airflow version, this is not a reason to bump ``MAJOR`` version of the providers (unless there are other breaking changes in the provider). The reason for that is that people who use diff --git a/README.md b/README.md index b639dc3024b611..64207dbe70d16d 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ Airflow is not a streaming solution, but it is often used to process real-time d Apache Airflow is tested with: -| | Main version (dev) | Stable version (2.10.3) | +| | Main version (dev) | Stable version (2.10.4) | |------------|----------------------------|----------------------------| | Python | 3.9, 3.10, 3.11, 3.12 | 3.8, 3.9, 3.10, 3.11, 3.12 | | Platform | AMD64/ARM64(\*) | AMD64/ARM64(\*) | @@ -175,15 +175,15 @@ them to the appropriate format and workflow that your tool requires. ```bash -pip install 'apache-airflow==2.10.3' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.3/constraints-3.9.txt" +pip install 'apache-airflow==2.10.4' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.4/constraints-3.9.txt" ``` 2. Installing with extras (i.e., postgres, google) ```bash -pip install 'apache-airflow[postgres,google]==2.10.3' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.3/constraints-3.9.txt" +pip install 'apache-airflow[postgres,google]==2.10.4' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.4/constraints-3.9.txt" ``` For information on installing provider packages, check @@ -288,7 +288,7 @@ Apache Airflow version life cycle: | Version | Current Patch/Minor | State | First Release | Limited Support | EOL/Terminated | |-----------|-----------------------|-----------|-----------------|-------------------|------------------| -| 2 | 2.10.3 | Supported | Dec 17, 2020 | TBD | TBD | +| 2 | 2.10.4 | Supported | Dec 17, 2020 | TBD | TBD | | 1.10 | 1.10.15 | EOL | Aug 27, 2018 | Dec 17, 2020 | June 17, 2021 | | 1.9 | 1.9.0 | EOL | Jan 03, 2018 | Aug 27, 2018 | Aug 27, 2018 | | 1.8 | 1.8.2 | EOL | Mar 19, 2017 | Jan 03, 2018 | Jan 03, 2018 | diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index a8b5a265f3afa4..6bcc06898df7a4 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -21,6 +21,52 @@ .. towncrier release notes start +Airflow 2.10.4 (2024-12-16) +--------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +TaskInstance ``priority_weight`` is capped in 32-bit signed integer ranges (#43611) +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +Some database engines are limited to 32-bit integer values. As some users reported errors in +weight rolled-over to negative values, we decided to cap the value to the 32-bit integer. Even +if internally in python smaller or larger values to 64 bit are supported, ``priority_weight`` is +capped and only storing values from -2147483648 to 2147483647. + +Bug Fixes +^^^^^^^^^ + +- Fix stats of dynamic mapped tasks after automatic retries of failed tasks (#44300) +- Fix wrong display of multi-line messages in the log after filtering (#44457) +- Allow "/" in metrics validator (#42934) (#44515) +- Fix gantt flickering (#44488) (#44517) +- Fix problem with inability to remove fields from Connection form (#40421) (#44442) +- Check pool_slots on partial task import instead of execution (#39724) (#42693) +- Avoid grouping task instance stats by try_number for dynamic mapped tasks (#44300) (#44319) +- Re-queue task when they are stuck in queued (#43520) (#44158) +- Suppress the warnings where we check for sensitive values (#44148) (#44167) +- Fix get_task_instance_try_details to return appropriate schema (#43830) (#44133) +- Log message source details are grouped (#43681) (#44070) +- Fix duplication of Task tries in the UI (#43891) (#43950) +- Add correct mime-type in OpenAPI spec (#43879) (#43901) +- Disable extra links button if link is null or empty (#43844) (#43851) +- Disable XCom list ordering by execution_date (#43680) (#43696) +- Fix venv numpy example which needs to be 1.26 at least to be working in Python 3.12 (#43659) +- Fix Try Selector in Mapped Tasks also on Index 0 (#43590) (#43591) +- Prevent using ``trigger_rule="always"`` in a dynamic mapped task (#43810) +- Prevent using ``trigger_rule=TriggerRule.ALWAYS`` in a task-generated mapping within bare tasks (#44751) + +Doc Only Changes +"""""""""""""""" +- Update XCom docs around containers/helm (#44570) (#44573) + +Miscellaneous +""""""""""""" +- Raise deprecation warning when accessing inlet or outlet events through str (#43922) + + Airflow 2.10.3 (2024-11-05) --------------------------- diff --git a/airflow/api_fastapi/common/exceptions.py b/airflow/api_fastapi/common/exceptions.py index 1e779a6097576a..061eec55d3d84e 100644 --- a/airflow/api_fastapi/common/exceptions.py +++ b/airflow/api_fastapi/common/exceptions.py @@ -18,6 +18,7 @@ from __future__ import annotations from abc import ABC, abstractmethod +from enum import Enum from typing import Generic, TypeVar from fastapi import HTTPException, Request, status @@ -38,26 +39,46 @@ def exception_handler(self, request: Request, exc: T): raise NotImplementedError +class _DatabaseDialect(Enum): + SQLITE = "sqlite" + MYSQL = "mysql" + POSTGRES = "postgres" + + class _UniqueConstraintErrorHandler(BaseErrorHandler[IntegrityError]): """Exception raised when trying to insert a duplicate value in a unique column.""" + unique_constraint_error_prefix_dict: dict[_DatabaseDialect, str] = { + _DatabaseDialect.SQLITE: "UNIQUE constraint failed", + _DatabaseDialect.MYSQL: "Duplicate entry", + _DatabaseDialect.POSTGRES: "violates unique constraint", + } + def __init__(self): super().__init__(IntegrityError) - self.unique_constraint_error_messages = [ - "UNIQUE constraint failed", # SQLite - "Duplicate entry", # MySQL - "violates unique constraint", # PostgreSQL - ] + self.dialect: _DatabaseDialect.value | None = None def exception_handler(self, request: Request, exc: IntegrityError): """Handle IntegrityError exception.""" - exc_orig_str = str(exc.orig) - if any(error_msg in exc_orig_str for error_msg in self.unique_constraint_error_messages): + if self._is_dialect_matched(exc): raise HTTPException( status_code=status.HTTP_409_CONFLICT, - detail="Unique constraint violation", + detail={ + "reason": "Unique constraint violation", + "statement": str(exc.statement), + "orig_error": str(exc.orig), + }, ) + def _is_dialect_matched(self, exc: IntegrityError) -> bool: + """Check if the exception matches the unique constraint error message for any dialect.""" + exc_orig_str = str(exc.orig) + for dialect, error_msg in self.unique_constraint_error_prefix_dict.items(): + if error_msg in exc_orig_str: + self.dialect = dialect + return True + return False + DatabaseErrorHandlers = [ _UniqueConstraintErrorHandler(), diff --git a/airflow/api_fastapi/core_api/datamodels/ui/structure.py b/airflow/api_fastapi/core_api/datamodels/ui/structure.py index 40c7b9dc0774f3..35874caa70c78d 100644 --- a/airflow/api_fastapi/core_api/datamodels/ui/structure.py +++ b/airflow/api_fastapi/core_api/datamodels/ui/structure.py @@ -28,6 +28,7 @@ class EdgeResponse(BaseModel): label: str | None = None source_id: str target_id: str + is_source_asset: bool | None = None class NodeResponse(BaseModel): @@ -41,6 +42,7 @@ class NodeResponse(BaseModel): setup_teardown_type: Literal["setup", "teardown"] | None = None type: Literal["join", "task", "asset-condition", "asset", "asset-alias", "dag", "sensor", "trigger"] operator: str | None = None + asset_condition_type: Literal["or-gate", "and-gate"] | None = None class StructureDataResponse(BaseModel): diff --git a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml index a36e45d4241a91..cf1260f5a8485e 100644 --- a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml @@ -7734,6 +7734,11 @@ components: target_id: type: string title: Target Id + is_source_asset: + anyOf: + - type: boolean + - type: 'null' + title: Is Source Asset type: object required: - source_id @@ -8079,6 +8084,14 @@ components: - type: string - type: 'null' title: Operator + asset_condition_type: + anyOf: + - type: string + enum: + - or-gate + - and-gate + - type: 'null' + title: Asset Condition Type type: object required: - id diff --git a/airflow/api_fastapi/core_api/routes/public/connections.py b/airflow/api_fastapi/core_api/routes/public/connections.py index ca3a5deca6a4dc..61fc76832c61db 100644 --- a/airflow/api_fastapi/core_api/routes/public/connections.py +++ b/airflow/api_fastapi/core_api/routes/public/connections.py @@ -121,7 +121,9 @@ def get_connections( @connections_router.post( "", status_code=status.HTTP_201_CREATED, - responses=create_openapi_http_exception_doc([status.HTTP_409_CONFLICT]), + responses=create_openapi_http_exception_doc( + [status.HTTP_409_CONFLICT] + ), # handled by global exception handler ) def post_connection( post_body: ConnectionBody, diff --git a/airflow/api_fastapi/core_api/routes/ui/structure.py b/airflow/api_fastapi/core_api/routes/ui/structure.py index c3b914508d42ff..417b711cf19244 100644 --- a/airflow/api_fastapi/core_api/routes/ui/structure.py +++ b/airflow/api_fastapi/core_api/routes/ui/structure.py @@ -22,6 +22,7 @@ from airflow.api_fastapi.common.router import AirflowRouter from airflow.api_fastapi.core_api.datamodels.ui.structure import StructureDataResponse from airflow.api_fastapi.core_api.openapi.exceptions import create_openapi_http_exception_doc +from airflow.api_fastapi.core_api.services.ui.structure import get_upstream_assets from airflow.models.serialized_dag import SerializedDagModel from airflow.utils.dag_edges import dag_edges from airflow.utils.task_group import task_group_to_dict @@ -71,17 +72,16 @@ def structure_data( for dependency_dag_id, dependencies in SerializedDagModel.get_dag_dependencies().items(): for dependency in dependencies: + # Dependencies not related to `dag_id` are ignored if dependency_dag_id != dag_id and dependency.target != dag_id: continue - # Add nodes - nodes.append( - { - "id": dependency.node_id, - "label": dependency.dependency_id, - "type": dependency.dependency_type, - } - ) + # upstream assets are handled by the `get_upstream_assets` function. + if dependency.target != dependency.dependency_type and dependency.dependency_type in [ + "asset-alias", + "asset", + ]: + continue # Add edges # start dependency @@ -96,6 +96,20 @@ def structure_data( ) and exit_node_ref: end_edges.append({"source_id": exit_node_ref["id"], "target_id": dependency.node_id}) - data["edges"] = start_edges + edges + end_edges + # Add nodes + nodes.append( + { + "id": dependency.node_id, + "label": dependency.dependency_id, + "type": dependency.dependency_type, + } + ) + + upstream_asset_nodes, upstream_asset_edges = get_upstream_assets( + dag.timetable.asset_condition, entry_node_ref["id"] + ) + + data["nodes"] += upstream_asset_nodes + data["edges"] = upstream_asset_edges + start_edges + edges + end_edges return StructureDataResponse(**data) diff --git a/airflow/api_fastapi/core_api/services/__init__.py b/airflow/api_fastapi/core_api/services/__init__.py new file mode 100644 index 00000000000000..13a83393a9124b --- /dev/null +++ b/airflow/api_fastapi/core_api/services/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/api_fastapi/core_api/services/ui/__init.py b/airflow/api_fastapi/core_api/services/ui/__init.py new file mode 100644 index 00000000000000..13a83393a9124b --- /dev/null +++ b/airflow/api_fastapi/core_api/services/ui/__init.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/api_fastapi/core_api/services/ui/structure.py b/airflow/api_fastapi/core_api/services/ui/structure.py new file mode 100644 index 00000000000000..72ee000b1f5a5c --- /dev/null +++ b/airflow/api_fastapi/core_api/services/ui/structure.py @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Private service for dag structure. + +:meta private: +""" + +from __future__ import annotations + +from airflow.sdk.definitions.asset import Asset, AssetAlias, AssetAll, AssetAny, BaseAsset + + +def get_upstream_assets( + asset_condition: BaseAsset, entry_node_ref: str, level=0 +) -> tuple[list[dict], list[dict]]: + edges: list[dict] = [] + nodes: list[dict] = [] + asset_condition_type: str | None = None + + assets: list[Asset | AssetAlias] = [] + + nested_expression: AssetAll | AssetAny | None = None + + if isinstance(asset_condition, AssetAny): + asset_condition_type = "or-gate" + + elif isinstance(asset_condition, AssetAll): + asset_condition_type = "and-gate" + + if hasattr(asset_condition, "objects"): + for obj in asset_condition.objects: + if isinstance(obj, (AssetAll, AssetAny)): + nested_expression = obj + elif isinstance(obj, (Asset, AssetAlias)): + assets.append(obj) + else: + raise TypeError(f"Unsupported type: {type(obj)}") + + if asset_condition_type and assets: + asset_condition_id = f"{asset_condition_type}-{level}" + edges.append( + { + "source_id": asset_condition_id, + "target_id": entry_node_ref, + "is_source_asset": level == 0, + } + ) + nodes.append( + { + "id": asset_condition_id, + "label": asset_condition_id, + "type": "asset-condition", + "asset_condition_type": asset_condition_type, + } + ) + + for asset in assets: + edges.append( + { + "source_id": asset.name, + "target_id": asset_condition_id, + } + ) + nodes.append( + { + "id": asset.name, + "label": asset.name, + "type": "asset-alias" if isinstance(asset, AssetAlias) else "asset", + } + ) + + if nested_expression is not None: + n, e = get_upstream_assets(nested_expression, asset_condition_id, level=level + 1) + + nodes = nodes + n + edges = edges + e + + return nodes, edges diff --git a/airflow/api_fastapi/execution_api/datamodels/taskinstance.py b/airflow/api_fastapi/execution_api/datamodels/taskinstance.py index bbc557d012463a..92a1e933dc93aa 100644 --- a/airflow/api_fastapi/execution_api/datamodels/taskinstance.py +++ b/airflow/api_fastapi/execution_api/datamodels/taskinstance.py @@ -25,7 +25,10 @@ from airflow.api_fastapi.common.types import UtcDateTime from airflow.api_fastapi.core_api.base import BaseModel +from airflow.api_fastapi.execution_api.datamodels.connection import ConnectionResponse +from airflow.api_fastapi.execution_api.datamodels.variable import VariableResponse from airflow.utils.state import IntermediateTIState, TaskInstanceState as TIState, TerminalTIState +from airflow.utils.types import DagRunType class TIEnterRunningPayload(BaseModel): @@ -94,9 +97,7 @@ def ti_state_discriminator(v: dict[str, str] | BaseModel) -> str: state = v.get("state") else: state = getattr(v, "state", None) - if state == TIState.RUNNING: - return str(state) - elif state in set(TerminalTIState): + if state in set(TerminalTIState): return "_terminal_" elif state == TIState.DEFERRED: return "deferred" @@ -107,7 +108,6 @@ def ti_state_discriminator(v: dict[str, str] | BaseModel) -> str: # and "_other_" is a catch-all for all other states that are not covered by the other schemas. TIStateUpdate = Annotated[ Union[ - Annotated[TIEnterRunningPayload, Tag("running")], Annotated[TITerminalStatePayload, Tag("_terminal_")], Annotated[TITargetStatePayload, Tag("_other_")], Annotated[TIDeferredStatePayload, Tag("deferred")], @@ -135,3 +135,34 @@ class TaskInstance(BaseModel): run_id: str try_number: int map_index: int | None = None + + +class DagRun(BaseModel): + """Schema for DagRun model with minimal required fields needed for Runtime.""" + + # TODO: `dag_id` and `run_id` are duplicated from TaskInstance + # See if we can avoid sending these fields from API server and instead + # use the TaskInstance data to get the DAG run information in the client (Task Execution Interface). + dag_id: str + run_id: str + + logical_date: UtcDateTime + data_interval_start: UtcDateTime | None + data_interval_end: UtcDateTime | None + start_date: UtcDateTime + end_date: UtcDateTime | None + run_type: DagRunType + conf: Annotated[dict[str, Any], Field(default_factory=dict)] + + +class TIRunContext(BaseModel): + """Response schema for TaskInstance run context.""" + + dag_run: DagRun + """DAG run information for the task instance.""" + + variables: Annotated[list[VariableResponse], Field(default_factory=list)] + """Variables that can be accessed by the task instance.""" + + connections: Annotated[list[ConnectionResponse], Field(default_factory=list)] + """Connections that can be accessed by the task instance.""" diff --git a/airflow/api_fastapi/execution_api/routes/task_instances.py b/airflow/api_fastapi/execution_api/routes/task_instances.py index e06798209c5daa..3a1545283e81b3 100644 --- a/airflow/api_fastapi/execution_api/routes/task_instances.py +++ b/airflow/api_fastapi/execution_api/routes/task_instances.py @@ -30,12 +30,15 @@ from airflow.api_fastapi.common.db.common import SessionDep from airflow.api_fastapi.common.router import AirflowRouter from airflow.api_fastapi.execution_api.datamodels.taskinstance import ( + DagRun, TIDeferredStatePayload, TIEnterRunningPayload, TIHeartbeatInfo, + TIRunContext, TIStateUpdate, TITerminalStatePayload, ) +from airflow.models.dagrun import DagRun as DR from airflow.models.taskinstance import TaskInstance as TI, _update_rtif from airflow.models.trigger import Trigger from airflow.utils import timezone @@ -48,6 +51,110 @@ log = logging.getLogger(__name__) +@router.patch( + "/{task_instance_id}/run", + status_code=status.HTTP_200_OK, + responses={ + status.HTTP_404_NOT_FOUND: {"description": "Task Instance not found"}, + status.HTTP_409_CONFLICT: {"description": "The TI is already in the requested state"}, + status.HTTP_422_UNPROCESSABLE_ENTITY: {"description": "Invalid payload for the state transition"}, + }, +) +def ti_run( + task_instance_id: UUID, ti_run_payload: Annotated[TIEnterRunningPayload, Body()], session: SessionDep +) -> TIRunContext: + """ + Run a TaskInstance. + + This endpoint is used to start a TaskInstance that is in the QUEUED state. + """ + # We only use UUID above for validation purposes + ti_id_str = str(task_instance_id) + + old = select(TI.state, TI.dag_id, TI.run_id).where(TI.id == ti_id_str).with_for_update() + try: + (previous_state, dag_id, run_id) = session.execute(old).one() + except NoResultFound: + log.error("Task Instance %s not found", ti_id_str) + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={ + "reason": "not_found", + "message": "Task Instance not found", + }, + ) + + # We exclude_unset to avoid updating fields that are not set in the payload + data = ti_run_payload.model_dump(exclude_unset=True) + + query = update(TI).where(TI.id == ti_id_str).values(data) + + # TODO: We will need to change this for other states like: + # reschedule, retry, defer etc. + if previous_state != State.QUEUED: + log.warning( + "Can not start Task Instance ('%s') in invalid state: %s", + ti_id_str, + previous_state, + ) + + # TODO: Pass a RFC 9457 compliant error message in "detail" field + # https://datatracker.ietf.org/doc/html/rfc9457 + # to provide more information about the error + # FastAPI will automatically convert this to a JSON response + # This might be added in FastAPI in https://github.com/fastapi/fastapi/issues/10370 + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "reason": "invalid_state", + "message": "TI was not in a state where it could be marked as running", + "previous_state": previous_state, + }, + ) + log.info("Task with %s state started on %s ", previous_state, ti_run_payload.hostname) + # Ensure there is no end date set. + query = query.values( + end_date=None, + hostname=ti_run_payload.hostname, + unixname=ti_run_payload.unixname, + pid=ti_run_payload.pid, + state=State.RUNNING, + ) + + try: + result = session.execute(query) + log.info("TI %s state updated: %s row(s) affected", ti_id_str, result.rowcount) + + dr = session.execute( + select( + DR.run_id, + DR.dag_id, + DR.data_interval_start, + DR.data_interval_end, + DR.start_date, + DR.end_date, + DR.run_type, + DR.conf, + DR.logical_date, + ).filter_by(dag_id=dag_id, run_id=run_id) + ).one_or_none() + + if not dr: + raise ValueError(f"DagRun with dag_id={dag_id} and run_id={run_id} not found.") + + return TIRunContext( + dag_run=DagRun.model_validate(dr, from_attributes=True), + # TODO: Add variables and connections that are needed (and has perms) for the task + variables=[], + connections=[], + ) + except SQLAlchemyError as e: + log.error("Error marking Task Instance state as running: %s", e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Database error occurred" + ) + + @router.patch( "/{task_instance_id}/state", status_code=status.HTTP_204_NO_CONTENT, @@ -92,37 +199,7 @@ def ti_update_state( query = update(TI).where(TI.id == ti_id_str).values(data) - if isinstance(ti_patch_payload, TIEnterRunningPayload): - if previous_state != State.QUEUED: - log.warning( - "Can not start Task Instance ('%s') in invalid state: %s", - ti_id_str, - previous_state, - ) - - # TODO: Pass a RFC 9457 compliant error message in "detail" field - # https://datatracker.ietf.org/doc/html/rfc9457 - # to provide more information about the error - # FastAPI will automatically convert this to a JSON response - # This might be added in FastAPI in https://github.com/fastapi/fastapi/issues/10370 - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail={ - "reason": "invalid_state", - "message": "TI was not in a state where it could be marked as running", - "previous_state": previous_state, - }, - ) - log.info("Task with %s state started on %s ", previous_state, ti_patch_payload.hostname) - # Ensure there is no end date set. - query = query.values( - end_date=None, - hostname=ti_patch_payload.hostname, - unixname=ti_patch_payload.unixname, - pid=ti_patch_payload.pid, - state=State.RUNNING, - ) - elif isinstance(ti_patch_payload, TITerminalStatePayload): + if isinstance(ti_patch_payload, TITerminalStatePayload): query = TI.duration_expression_update(ti_patch_payload.end_date, query, session.bind) elif isinstance(ti_patch_payload, TIDeferredStatePayload): # Calculate timeout if it was passed diff --git a/airflow/assets/manager.py b/airflow/assets/manager.py index 364d01607e5c4b..99de69176a878d 100644 --- a/airflow/assets/manager.py +++ b/airflow/assets/manager.py @@ -174,11 +174,6 @@ def register_asset_change( if alias_ref.dag.is_active and not alias_ref.dag.is_paused } - dags_to_reparse = dags_to_queue_from_asset_alias - dags_to_queue_from_asset - if dags_to_reparse: - file_locs = {dag.fileloc for dag in dags_to_reparse} - cls._send_dag_priority_parsing_request(file_locs, session) - cls.notify_asset_changed(asset=asset) Stats.incr("asset.updates") diff --git a/airflow/executors/executor_loader.py b/airflow/executors/executor_loader.py index 9355fbb26a5aea..2651718bbad235 100644 --- a/airflow/executors/executor_loader.py +++ b/airflow/executors/executor_loader.py @@ -19,6 +19,7 @@ from __future__ import annotations import logging +import os from typing import TYPE_CHECKING from airflow.exceptions import AirflowConfigException, UnknownExecutorException diff --git a/airflow/models/dag.py b/airflow/models/dag.py index 6ccb1221ac5c01..1decc2db683ea6 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -2280,7 +2280,7 @@ def dag_ready(dag_id: str, cond: BaseAsset, statuses: dict) -> bool | None: # we may be dealing with old version. In that case, # just wait for the dag to be reserialized. try: - return cond.evaluate(statuses) + return cond.evaluate(statuses, session=session) except AttributeError: log.warning("dag '%s' has old serialization; skipping DAG run creation.", dag_id) return None diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index 15ac435ff7d7e9..bf324a297dd381 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -1010,7 +1010,6 @@ def get_triggering_events() -> dict[str, list[AssetEvent]]: # * KNOWN_CONTEXT_KEYS in airflow/utils/context.py # * Table in docs/apache-airflow/templates-ref.rst context: dict[str, Any] = { - "conf": conf, "dag": dag, "dag_run": dag_run, "data_interval_end": timezone.coerce_datetime(data_interval.end), diff --git a/airflow/reproducible_build.yaml b/airflow/reproducible_build.yaml index 5f2d030fd0b636..7f7f3298ef46fa 100644 --- a/airflow/reproducible_build.yaml +++ b/airflow/reproducible_build.yaml @@ -1,2 +1,2 @@ -release-notes-hash: c68f3fa23f84c7fc270d73baaa2cc18d -source-date-epoch: 1732690252 +release-notes-hash: f1d91d32ade6da6eedd24362610d5f84 +source-date-epoch: 1734354109 diff --git a/airflow/timetables/base.py b/airflow/timetables/base.py index b80a6323a8c18b..c719f92437be14 100644 --- a/airflow/timetables/base.py +++ b/airflow/timetables/base.py @@ -24,6 +24,7 @@ if TYPE_CHECKING: from pendulum import DateTime + from sqlalchemy.orm import Session from airflow.sdk.definitions.asset import Asset, AssetAlias from airflow.serialization.dag_dependency import DagDependency @@ -52,7 +53,7 @@ def __and__(self, other: BaseAsset) -> BaseAsset: def as_expression(self) -> Any: return None - def evaluate(self, statuses: dict[str, bool]) -> bool: + def evaluate(self, statuses: dict[str, bool], *, session: Session | None = None) -> bool: return False def iter_assets(self) -> Iterator[tuple[AssetUniqueKey, Asset]]: diff --git a/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow/ui/openapi-gen/requests/schemas.gen.ts index 41e309d694c9bb..696a9011d0122f 100644 --- a/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -2664,6 +2664,17 @@ export const $EdgeResponse = { type: "string", title: "Target Id", }, + is_source_asset: { + anyOf: [ + { + type: "boolean", + }, + { + type: "null", + }, + ], + title: "Is Source Asset", + }, }, type: "object", required: ["source_id", "target_id"], @@ -3208,6 +3219,18 @@ export const $NodeResponse = { ], title: "Operator", }, + asset_condition_type: { + anyOf: [ + { + type: "string", + enum: ["or-gate", "and-gate"], + }, + { + type: "null", + }, + ], + title: "Asset Condition Type", + }, }, type: "object", required: ["id", "label", "type"], diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts index c178f78608e307..f667e06de0994e 100644 --- a/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow/ui/openapi-gen/requests/types.gen.ts @@ -643,6 +643,7 @@ export type EdgeResponse = { label?: string | null; source_id: string; target_id: string; + is_source_asset?: boolean | null; }; /** @@ -783,6 +784,7 @@ export type NodeResponse = { | "sensor" | "trigger"; operator?: string | null; + asset_condition_type?: "or-gate" | "and-gate" | null; }; export type type = diff --git a/airflow/utils/context.py b/airflow/utils/context.py index 67f31a3f2b6b46..c6cf2db498532c 100644 --- a/airflow/utils/context.py +++ b/airflow/utils/context.py @@ -67,7 +67,6 @@ # * Context in airflow/utils/context.pyi. # * Table in docs/apache-airflow/templates-ref.rst KNOWN_CONTEXT_KEYS: set[str] = { - "conf", "conn", "dag", "dag_run", diff --git a/airflow/utils/context.pyi b/airflow/utils/context.pyi index b2fdc98a7459a9..4696e48c75d6f8 100644 --- a/airflow/utils/context.pyi +++ b/airflow/utils/context.pyi @@ -32,7 +32,6 @@ from typing import Any, overload from pendulum import DateTime from sqlalchemy.orm import Session -from airflow.configuration import AirflowConfigParser from airflow.models.asset import AssetEvent from airflow.models.baseoperator import BaseOperator from airflow.models.dag import DAG @@ -100,7 +99,6 @@ class InletEventsAccessors(Mapping[Asset | AssetAlias, InletEventsAccessor]): # * KNOWN_CONTEXT_KEYS in airflow/utils/context.py # * Table in docs/apache-airflow/templates-ref.rst class Context(TypedDict, total=False): - conf: AirflowConfigParser conn: Any dag: DAG dag_run: DagRun | DagRunPydantic diff --git a/airflow/www/security_appless.py b/airflow/www/security_appless.py index f6f7f261be989d..7996ed7ae7b61f 100644 --- a/airflow/www/security_appless.py +++ b/airflow/www/security_appless.py @@ -21,7 +21,7 @@ from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride if TYPE_CHECKING: - from flask_session import Session + from sqlalchemy.orm import Session class FakeAppBuilder: diff --git a/airflow/www/static/js/main.js b/airflow/www/static/js/main.js index c60827c7a362bb..b3dcabd5d47f30 100644 --- a/airflow/www/static/js/main.js +++ b/airflow/www/static/js/main.js @@ -321,4 +321,8 @@ $(document).ready(() => { // Global Tooltip selector $(".js-tooltip").tooltip(); + + // Turn off autocomplete for login form + $("#username:input")[0].autocomplete = "off"; + $("#password:input")[0].autocomplete = "off"; }); diff --git a/chart/Chart.yaml b/chart/Chart.yaml index b5db1fe351f405..ac74165e1eb09d 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -20,7 +20,7 @@ apiVersion: v2 name: airflow version: 1.16.0-dev -appVersion: 2.10.3 +appVersion: 2.10.4 description: The official Helm chart to deploy Apache Airflow, a platform to programmatically author, schedule, and monitor workflows home: https://airflow.apache.org/ @@ -47,23 +47,23 @@ annotations: url: https://airflow.apache.org/docs/helm-chart/1.16.0/ artifacthub.io/screenshots: | - title: DAGs View - url: https://airflow.apache.org/docs/apache-airflow/2.10.3/_images/dags.png + url: https://airflow.apache.org/docs/apache-airflow/2.10.4/_images/dags.png - title: Datasets View - url: https://airflow.apache.org/docs/apache-airflow/2.10.3/_images/datasets.png + url: https://airflow.apache.org/docs/apache-airflow/2.10.4/_images/datasets.png - title: Grid View - url: https://airflow.apache.org/docs/apache-airflow/2.10.3/_images/grid.png + url: https://airflow.apache.org/docs/apache-airflow/2.10.4/_images/grid.png - title: Graph View - url: https://airflow.apache.org/docs/apache-airflow/2.10.3/_images/graph.png + url: https://airflow.apache.org/docs/apache-airflow/2.10.4/_images/graph.png - title: Calendar View - url: https://airflow.apache.org/docs/apache-airflow/2.10.3/_images/calendar.png + url: https://airflow.apache.org/docs/apache-airflow/2.10.4/_images/calendar.png - title: Variable View - url: https://airflow.apache.org/docs/apache-airflow/2.10.3/_images/variable_hidden.png + url: https://airflow.apache.org/docs/apache-airflow/2.10.4/_images/variable_hidden.png - title: Gantt Chart - url: https://airflow.apache.org/docs/apache-airflow/2.10.3/_images/gantt.png + url: https://airflow.apache.org/docs/apache-airflow/2.10.4/_images/gantt.png - title: Task Duration - url: https://airflow.apache.org/docs/apache-airflow/2.10.3/_images/duration.png + url: https://airflow.apache.org/docs/apache-airflow/2.10.4/_images/duration.png - title: Code View - url: https://airflow.apache.org/docs/apache-airflow/2.10.3/_images/code.png + url: https://airflow.apache.org/docs/apache-airflow/2.10.4/_images/code.png artifacthub.io/changes: | - description: Add git-sync container lifecycle hooks kind: added diff --git a/chart/newsfragments/43698.significant.rst b/chart/newsfragments/43698.significant.rst deleted file mode 100644 index 0120e973baca13..00000000000000 --- a/chart/newsfragments/43698.significant.rst +++ /dev/null @@ -1,3 +0,0 @@ -Default Airflow image is updated to ``2.10.3`` - -The default Airflow image that is used with the Chart is now ``2.10.3``, previously it was ``2.10.2``. diff --git a/chart/newsfragments/44959.significant.rst b/chart/newsfragments/44959.significant.rst new file mode 100644 index 00000000000000..af264fe91cbad7 --- /dev/null +++ b/chart/newsfragments/44959.significant.rst @@ -0,0 +1,3 @@ +Default Airflow image is updated to ``2.10.4`` + +The default Airflow image that is used with the Chart is now ``2.10.4``, previously it was ``2.10.2``. diff --git a/chart/values.schema.json b/chart/values.schema.json index 813a08e7170fbd..52939b48c5c98a 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -78,7 +78,7 @@ "defaultAirflowTag": { "description": "Default airflow tag to deploy.", "type": "string", - "default": "2.10.3", + "default": "2.10.4", "x-docsSection": "Common" }, "defaultAirflowDigest": { @@ -93,7 +93,7 @@ "airflowVersion": { "description": "Airflow version (Used to make some decisions based on Airflow Version being deployed).", "type": "string", - "default": "2.10.3", + "default": "2.10.4", "x-docsSection": "Common" }, "securityContext": { diff --git a/chart/values.yaml b/chart/values.yaml index 89aebfeb86bdc2..adf68c3a194d3a 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -68,13 +68,13 @@ airflowHome: /opt/airflow defaultAirflowRepository: apache/airflow # Default airflow tag to deploy -defaultAirflowTag: "2.10.3" +defaultAirflowTag: "2.10.4" # Default airflow digest. If specified, it takes precedence over tag defaultAirflowDigest: ~ # Airflow version (Used to make some decisions based on Airflow Version being deployed) -airflowVersion: "2.10.3" +airflowVersion: "2.10.4" # Images images: diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 8043517831dbf6..9010c3b54d9748 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -16,7 +16,7 @@ # under the License. [build-system] -requires = ["hatchling==1.26.3"] +requires = ["hatchling==1.27.0"] build-backend = "hatchling.build" [project] diff --git a/contributing-docs/testing/unit_tests.rst b/contributing-docs/testing/unit_tests.rst index 9ad3d95d0dccdd..e3a9ba16a332ef 100644 --- a/contributing-docs/testing/unit_tests.rst +++ b/contributing-docs/testing/unit_tests.rst @@ -1029,7 +1029,7 @@ Our CI runs provider tests for providers with previous compatible airflow releas if the providers still work when installed for older airflow versions. The back-compatibility tests based on the configuration specified in the -``BASE_PROVIDERS_COMPATIBILITY_CHECKS`` constant in the ``./dev/breeze/src/airflow_breeze/global_constants.py`` +``PROVIDERS_COMPATIBILITY_TESTS_MATRIX`` constant in the ``./dev/breeze/src/airflow_breeze/global_constants.py`` file - where we specify: * Python version @@ -1185,7 +1185,7 @@ Herr id how to reproduce it. breeze release-management generate-constraints --airflow-constraints-mode constraints-source-providers --answer yes 4. Remove providers that are not compatible with Airflow version installed by default. You can look up - the incompatible providers in the ``BASE_PROVIDERS_COMPATIBILITY_CHECKS`` constant in the + the incompatible providers in the ``PROVIDERS_COMPATIBILITY_TESTS_MATRIX`` constant in the ``./dev/breeze/src/airflow_breeze/global_constants.py`` file. 5. Enter breeze environment, installing selected airflow version and the provider packages prepared from main diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md index 0afd99c77b6fbb..f8701f7fb9c919 100644 --- a/dev/README_RELEASE_PROVIDER_PACKAGES.md +++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md @@ -90,9 +90,8 @@ the versions of Airflow that are not applicable anymore. searching and replacing old version occurrences with newer one. For example 2.8.0 to 2.9.0 3. Update minimum airflow version for all packages, you should modify `MIN_AIRFLOW_VERSION` -in `src/airflow_breeze/utils/packages.py` and run the `prepare-provider-documentation` -command with the `--only-min-version-update` flag. This will only update the min version in -the `__init__.py` files and package documentation without bumping the provider versions. +in `src/airflow_breeze/utils/packages.py` and run the `breeze release-management prepare-provider-documentation --only-min-version-update` +This will only update the min version in the `__init__.py` files and package documentation without bumping the provider versions. 4. Remove `AIRFLOW_V_2_X_PLUS` in all tests (review and update skipif and other conditional behaviour and test_compat.py, where X is the TARGET version we change to. For example diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md index 1d8327ea7754ac..60972293abc2fd 100644 --- a/dev/breeze/doc/ci/02_images.md +++ b/dev/breeze/doc/ci/02_images.md @@ -448,7 +448,7 @@ can be used for CI images: | `ADDITIONAL_DEV_APT_DEPS` | | Additional apt dev dependencies installed in the first part of the image | | `ADDITIONAL_DEV_APT_ENV` | | Additional env variables defined when installing dev deps | | `AIRFLOW_PIP_VERSION` | `24.3.1` | PIP version used. | -| `AIRFLOW_UV_VERSION` | `0.5.8` | UV version used. | +| `AIRFLOW_UV_VERSION` | `0.5.9` | UV version used. | | `AIRFLOW_USE_UV` | `true` | Whether to use UV for installation. | | `PIP_PROGRESS_BAR` | `on` | Progress bar for PIP installation | diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index a536f416c6d8a3..acb143be5d81e9 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -235,7 +235,7 @@ class VersionedFile(NamedTuple): AIRFLOW_PIP_VERSION = "24.3.1" -AIRFLOW_UV_VERSION = "0.5.8" +AIRFLOW_UV_VERSION = "0.5.9" AIRFLOW_USE_UV = False # TODO: automate these as well WHEEL_VERSION = "0.44.0" diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 8d7e7353eec50a..477ad4c0a81c2d 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -208,7 +208,7 @@ ALLOWED_INSTALL_MYSQL_CLIENT_TYPES = ["mariadb", "mysql"] PIP_VERSION = "24.3.1" -UV_VERSION = "0.5.8" +UV_VERSION = "0.5.9" DEFAULT_UV_HTTP_TIMEOUT = 300 DEFAULT_WSL2_HTTP_TIMEOUT = 900 @@ -611,12 +611,6 @@ def get_airflow_extras(): PROVIDERS_COMPATIBILITY_TESTS_MATRIX: list[dict[str, str | list[str]]] = [ - { - "python-version": "3.9", - "airflow-version": "2.8.4", - "remove-providers": "cloudant fab edge", - "run-tests": "true", - }, { "python-version": "3.9", "airflow-version": "2.9.3", diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py b/dev/breeze/src/airflow_breeze/utils/packages.py index f568d91d7196ca..b4ceecbee5632e 100644 --- a/dev/breeze/src/airflow_breeze/utils/packages.py +++ b/dev/breeze/src/airflow_breeze/utils/packages.py @@ -51,7 +51,7 @@ from airflow_breeze.utils.version_utils import remove_local_version_suffix from airflow_breeze.utils.versions import get_version_tag, strip_leading_zeros_from_version -MIN_AIRFLOW_VERSION = "2.8.0" +MIN_AIRFLOW_VERSION = "2.9.0" HTTPS_REMOTE = "apache-https-for-providers" LONG_PROVIDERS_PREFIX = "apache-airflow-providers-" diff --git a/dev/breeze/tests/test_packages.py b/dev/breeze/tests/test_packages.py index f7b6c17ab11fa9..d7391376f1b6ee 100644 --- a/dev/breeze/tests/test_packages.py +++ b/dev/breeze/tests/test_packages.py @@ -104,7 +104,7 @@ def test_get_long_package_name(): def test_get_provider_requirements(): # update me when asana dependencies change - assert get_provider_requirements("asana") == ["apache-airflow>=2.8.0", "asana>=0.10,<4.0.0"] + assert get_provider_requirements("asana") == ["apache-airflow>=2.9.0", "asana>=0.10,<4.0.0"] def test_get_removed_providers(): @@ -210,7 +210,7 @@ def test_get_documentation_package_path(): "beta0", """ "apache-airflow-providers-common-sql>=1.20.0b0", - "apache-airflow>=2.8.0b0", + "apache-airflow>=2.9.0b0", "asyncpg>=0.30.0", "psycopg2-binary>=2.9.4", """, @@ -221,7 +221,7 @@ def test_get_documentation_package_path(): "", """ "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "asyncpg>=0.30.0", "psycopg2-binary>=2.9.4", """, @@ -437,7 +437,7 @@ def test_validate_provider_info_with_schema(): @pytest.mark.parametrize( "provider_id, min_version", [ - ("amazon", "2.8.0"), + ("amazon", "2.9.0"), ("fab", "3.0.0.dev0"), ], ) @@ -502,7 +502,7 @@ def test_provider_jinja_context(): "CHANGELOG_RELATIVE_PATH": "../../providers/src/airflow/providers/amazon", "SUPPORTED_PYTHON_VERSIONS": ["3.9", "3.10", "3.11", "3.12"], "PLUGINS": [], - "MIN_AIRFLOW_VERSION": "2.8.0", + "MIN_AIRFLOW_VERSION": "2.9.0", "PROVIDER_REMOVED": False, "PROVIDER_INFO": provider_info, } diff --git a/docker_tests/requirements.txt b/docker_tests/requirements.txt index bcd9cd0655bb48..12bb045a85dd71 100644 --- a/docker_tests/requirements.txt +++ b/docker_tests/requirements.txt @@ -3,4 +3,4 @@ pytest-xdist # Requests 3 if it will be released, will be heavily breaking. requests>=2.27.0,<3 python-on-whales>=0.70.0 -hatchling==1.26.3 +hatchling==1.27.0 diff --git a/docs/apache-airflow-providers-edge/edge_executor.rst b/docs/apache-airflow-providers-edge/edge_executor.rst index d5cb5edb4aaabc..04286cc404c4a0 100644 --- a/docs/apache-airflow-providers-edge/edge_executor.rst +++ b/docs/apache-airflow-providers-edge/edge_executor.rst @@ -305,6 +305,3 @@ The following features are known missing and will be implemented in increments: - Describe more details on deployment options and tuning - Provide scripts and guides to install edge components as service (systemd) - Extend Helm-Chart for needed support - While it is in not-ready state, a wheel release package must be manually built from source tree - via ``breeze release-management prepare-provider-packages --include-not-ready-providers edge`` - and then installed via pip from the generated wheel file. diff --git a/docs/apache-airflow-providers-google/operators/cloud/vertex_ai.rst b/docs/apache-airflow-providers-google/operators/cloud/vertex_ai.rst index 173b23dfa30027..12b86d25d91963 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/vertex_ai.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/vertex_ai.rst @@ -644,6 +644,38 @@ The operator returns the cached content response in :ref:`XCom ` :start-after: [START how_to_cloud_vertex_ai_generate_from_cached_content_operator] :end-before: [END how_to_cloud_vertex_ai_generate_from_cached_content_operator] +Interacting with Vertex AI Feature Store +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To get a feature view sync job you can use +:class:`~airflow.providers.google.cloud.operators.vertex_ai.feature_store.GetFeatureViewSyncOperator`. +The operator returns sync job results in :ref:`XCom ` under ``return_value`` key. + +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_feature_store.py + :language: python + :dedent: 4 + :start-after: [START how_to_cloud_vertex_ai_feature_store_get_feature_view_sync_operator] + :end-before: [END how_to_cloud_vertex_ai_feature_store_get_feature_view_sync_operator] + +To sync a feature view you can use +:class:`~airflow.providers.google.cloud.operators.vertex_ai.feature_store.SyncFeatureViewOperator`. +The operator returns the sync job name in :ref:`XCom ` under ``return_value`` key. + +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_feature_store.py + :language: python + :dedent: 4 + :start-after: [START how_to_cloud_vertex_ai_feature_store_sync_feature_view_operator] + :end-before: [END how_to_cloud_vertex_ai_feature_store_sync_feature_view_operator] + +To check if Feature View Sync succeeded you can use +:class:`~airflow.providers.google.cloud.sensors.vertex_ai.FeatureViewSyncSensor`. + +.. exampleinclude:: /../../providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_feature_store.py + :language: python + :dedent: 4 + :start-after: [START how_to_cloud_vertex_ai_feature_store_feature_view_sync_sensor] + :end-before: [END how_to_cloud_vertex_ai_feature_store_feature_view_sync_sensor] + Reference ^^^^^^^^^ diff --git a/docs/apache-airflow/installation/supported-versions.rst b/docs/apache-airflow/installation/supported-versions.rst index 723825389ea077..233ff0bf3ecd37 100644 --- a/docs/apache-airflow/installation/supported-versions.rst +++ b/docs/apache-airflow/installation/supported-versions.rst @@ -29,7 +29,7 @@ Apache Airflow® version life cycle: ========= ===================== ========= =============== ================= ================ Version Current Patch/Minor State First Release Limited Support EOL/Terminated ========= ===================== ========= =============== ================= ================ -2 2.10.3 Supported Dec 17, 2020 TBD TBD +2 2.10.4 Supported Dec 17, 2020 TBD TBD 1.10 1.10.15 EOL Aug 27, 2018 Dec 17, 2020 June 17, 2021 1.9 1.9.0 EOL Jan 03, 2018 Aug 27, 2018 Aug 27, 2018 1.8 1.8.2 EOL Mar 19, 2017 Jan 03, 2018 Jan 03, 2018 diff --git a/docs/apache-airflow/templates-ref.rst b/docs/apache-airflow/templates-ref.rst index 78e5c17f9925be..cf7b015141b94c 100644 --- a/docs/apache-airflow/templates-ref.rst +++ b/docs/apache-airflow/templates-ref.rst @@ -79,8 +79,6 @@ Variable Type Description ``{{ conn }}`` Airflow connections. See `Airflow Connections in Templates`_ below. ``{{ task_instance_key_str }}`` str | A unique, human-readable key to the task instance. The format is | ``{dag_id}__{task_id}__{ds_nodash}``. -``{{ conf }}`` AirflowConfigParser | The full configuration object representing the content of your - | ``airflow.cfg``. See :mod:`airflow.configuration.conf`. ``{{ run_id }}`` str The currently running :class:`~airflow.models.dagrun.DagRun` run ID. ``{{ dag_run }}`` DagRun The currently running :class:`~airflow.models.dagrun.DagRun`. ``{{ test_mode }}`` bool Whether the task instance was run by the ``airflow test`` CLI. diff --git a/generated/PYPI_README.md b/generated/PYPI_README.md index 76ef8113cbcc76..7628f5d5444481 100644 --- a/generated/PYPI_README.md +++ b/generated/PYPI_README.md @@ -54,7 +54,7 @@ Use Airflow to author workflows as directed acyclic graphs (DAGs) of tasks. The Apache Airflow is tested with: -| | Main version (dev) | Stable version (2.10.3) | +| | Main version (dev) | Stable version (2.10.4) | |------------|----------------------------|----------------------------| | Python | 3.9, 3.10, 3.11, 3.12 | 3.8, 3.9, 3.10, 3.11, 3.12 | | Platform | AMD64/ARM64(\*) | AMD64/ARM64(\*) | @@ -128,15 +128,15 @@ them to the appropriate format and workflow that your tool requires. ```bash -pip install 'apache-airflow==2.10.3' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.3/constraints-3.9.txt" +pip install 'apache-airflow==2.10.4' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.4/constraints-3.9.txt" ``` 2. Installing with extras (i.e., postgres, google) ```bash -pip install 'apache-airflow[postgres,google]==2.10.3' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.3/constraints-3.9.txt" +pip install 'apache-airflow[postgres,google]==2.10.4' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.4/constraints-3.9.txt" ``` For information on installing provider packages, check diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index fe11c5804eb429..7a9307ade337f0 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -2,7 +2,7 @@ "airbyte": { "deps": [ "airbyte-api>=0.52.0", - "apache-airflow>=2.8.0" + "apache-airflow>=2.9.0" ], "devel-deps": [], "plugins": [], @@ -14,7 +14,7 @@ "deps": [ "alibabacloud_adb20211201>=1.0.0", "alibabacloud_tea_openapi>=0.3.7", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "oss2>=2.14.0" ], "devel-deps": [], @@ -29,7 +29,7 @@ "apache-airflow-providers-common-compat>=1.3.0", "apache-airflow-providers-common-sql>=1.20.0", "apache-airflow-providers-http", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "asgiref>=2.3.0", "boto3>=1.34.90", "botocore>=1.34.90", @@ -74,7 +74,7 @@ }, "apache.beam": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "apache-beam>=2.53.0", "pyarrow>=14.0.1" ], @@ -91,7 +91,7 @@ }, "apache.cassandra": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "cassandra-driver>=3.29.1" ], "devel-deps": [], @@ -103,7 +103,7 @@ "apache.drill": { "deps": [ "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "sqlalchemy-drill>=1.1.0" ], "devel-deps": [], @@ -117,7 +117,7 @@ "apache.druid": { "deps": [ "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "pydruid>=0.4.1" ], "devel-deps": [], @@ -132,7 +132,7 @@ "apache.flink": { "deps": [ "apache-airflow-providers-cncf-kubernetes>=5.1.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "cryptography>=41.0.0" ], "devel-deps": [], @@ -145,7 +145,7 @@ }, "apache.hdfs": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "hdfs[avro,dataframe,kerberos]>=2.5.4;python_version<\"3.12\"", "hdfs[avro,dataframe,kerberos]>=2.7.3;python_version>=\"3.12\"", "pandas>=1.5.3,<2.2;python_version<\"3.9\"", @@ -160,7 +160,7 @@ "apache.hive": { "deps": [ "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "hmsclient>=0.1.0", "jmespath>=0.7.0", "pandas>=1.5.3,<2.2;python_version<\"3.9\"", @@ -189,7 +189,7 @@ }, "apache.iceberg": { "deps": [ - "apache-airflow>=2.8.0" + "apache-airflow>=2.9.0" ], "devel-deps": [ "pyiceberg>=0.5.0" @@ -202,7 +202,7 @@ "apache.impala": { "deps": [ "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "impyla>=0.18.0,<1.0" ], "devel-deps": [ @@ -217,7 +217,7 @@ }, "apache.kafka": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "asgiref>=2.3.0", "confluent-kafka>=2.3.0" ], @@ -229,7 +229,7 @@ }, "apache.kylin": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "kylinpy>=2.7.0" ], "devel-deps": [], @@ -242,7 +242,7 @@ "deps": [ "aiohttp>=3.9.2", "apache-airflow-providers-http", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "asgiref>=2.3.0" ], "devel-deps": [], @@ -255,7 +255,7 @@ }, "apache.pig": { "deps": [ - "apache-airflow>=2.8.0" + "apache-airflow>=2.9.0" ], "devel-deps": [], "plugins": [], @@ -266,7 +266,7 @@ "apache.pinot": { "deps": [ "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "pinotdb>=5.1.0" ], "devel-deps": [], @@ -279,7 +279,7 @@ }, "apache.spark": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "grpcio-status>=1.59.0", "pyspark>=3.1.3" ], @@ -294,7 +294,7 @@ }, "apprise": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "apprise>=1.8.0" ], "devel-deps": [], @@ -305,7 +305,7 @@ }, "arangodb": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "python-arango>=7.3.2" ], "devel-deps": [], @@ -316,7 +316,7 @@ }, "asana": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "asana>=0.10,<4.0.0" ], "devel-deps": [], @@ -327,7 +327,7 @@ }, "atlassian.jira": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "atlassian-python-api>3.41.10" ], "devel-deps": [], @@ -338,7 +338,7 @@ }, "celery": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "celery[redis]>=5.3.0,<6,!=5.3.3,!=5.3.2", "flower>=1.0.0", "google-re2>=1.0" @@ -353,7 +353,7 @@ }, "cloudant": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "ibmcloudant==0.9.1 ; python_version >= \"3.10\"" ], "devel-deps": [], @@ -367,7 +367,7 @@ "cncf.kubernetes": { "deps": [ "aiofiles>=23.2.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "asgiref>=3.5.2", "cryptography>=41.0.0", "google-re2>=1.0", @@ -382,7 +382,7 @@ }, "cohere": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "cohere>=4.37,<5" ], "devel-deps": [], @@ -393,7 +393,7 @@ }, "common.compat": { "deps": [ - "apache-airflow>=2.8.0" + "apache-airflow>=2.9.0" ], "devel-deps": [], "plugins": [], @@ -406,7 +406,7 @@ }, "common.io": { "deps": [ - "apache-airflow>=2.8.0" + "apache-airflow>=2.9.0" ], "devel-deps": [], "plugins": [], @@ -419,7 +419,7 @@ }, "common.sql": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "more-itertools>=9.0.0", "sqlparse>=0.5.1" ], @@ -435,7 +435,7 @@ "deps": [ "aiohttp>=3.9.2, <4", "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "databricks-sql-connector>=3.0.0", "mergedeep>=1.3.4", "pandas>=1.5.3,<2.2;python_version<\"3.9\"", @@ -460,7 +460,7 @@ }, "datadog": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "datadog>=0.14.0" ], "devel-deps": [], @@ -473,7 +473,7 @@ "deps": [ "aiohttp>=3.9.2", "apache-airflow-providers-http", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "asgiref>=2.3.0" ], "devel-deps": [], @@ -488,7 +488,7 @@ "dingding": { "deps": [ "apache-airflow-providers-http", - "apache-airflow>=2.8.0" + "apache-airflow>=2.9.0" ], "devel-deps": [], "plugins": [], @@ -501,7 +501,7 @@ "discord": { "deps": [ "apache-airflow-providers-http", - "apache-airflow>=2.8.0" + "apache-airflow>=2.9.0" ], "devel-deps": [], "plugins": [], @@ -513,7 +513,7 @@ }, "docker": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "docker>=7.1.0", "python-dotenv>=0.21.0" ], @@ -546,7 +546,7 @@ "elasticsearch": { "deps": [ "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "elasticsearch>=8.10,<9" ], "devel-deps": [], @@ -560,7 +560,7 @@ "exasol": { "deps": [ "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "pandas>=1.5.3,<2.2;python_version<\"3.9\"", "pandas>=2.1.2,<2.2;python_version>=\"3.9\"", "pyexasol>=0.5.1" @@ -595,7 +595,7 @@ }, "facebook": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "facebook-business>=15.0.2" ], "devel-deps": [], @@ -606,7 +606,7 @@ }, "ftp": { "deps": [ - "apache-airflow>=2.8.0" + "apache-airflow>=2.9.0" ], "devel-deps": [], "plugins": [], @@ -620,7 +620,7 @@ "github": { "deps": [ "PyGithub>=2.1.1", - "apache-airflow>=2.8.0" + "apache-airflow>=2.9.0" ], "devel-deps": [], "plugins": [], @@ -633,7 +633,7 @@ "PyOpenSSL>=23.0.0", "apache-airflow-providers-common-compat>=1.2.1", "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "asgiref>=3.5.2", "dill>=0.2.3", "gcloud-aio-auth>=5.2.0", @@ -646,7 +646,7 @@ "google-api-python-client>=2.0.2", "google-auth-httplib2>=0.0.1", "google-auth>=2.29.0", - "google-cloud-aiplatform>=1.70.0", + "google-cloud-aiplatform>=1.73.0", "google-cloud-automl>=2.12.0", "google-cloud-batch>=0.13.0", "google-cloud-bigquery-datatransfer>=3.13.0", @@ -725,7 +725,7 @@ }, "grpc": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "google-auth-httplib2>=0.0.1", "google-auth>=1.0.0, <3.0.0", "grpcio>=1.59.0" @@ -738,7 +738,7 @@ }, "hashicorp": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "hvac>=1.1.0" ], "devel-deps": [], @@ -752,7 +752,7 @@ "http": { "deps": [ "aiohttp>=3.9.2,!=3.11.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "asgiref>=2.3.0", "requests-toolbelt>=0.4.0", "requests>=2.27.0,<3" @@ -765,7 +765,7 @@ }, "imap": { "deps": [ - "apache-airflow>=2.8.0" + "apache-airflow>=2.9.0" ], "devel-deps": [], "plugins": [], @@ -775,7 +775,7 @@ }, "influxdb": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "influxdb-client>=1.19.0", "requests>=2.27.0,<3" ], @@ -788,7 +788,7 @@ "jdbc": { "deps": [ "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "jaydebeapi>=1.1.1" ], "devel-deps": [], @@ -801,7 +801,7 @@ }, "jenkins": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "python-jenkins>=1.0.0" ], "devel-deps": [], @@ -814,7 +814,7 @@ "deps": [ "adal>=1.2.7", "adlfs>=2023.10.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "azure-batch>=8.0.0", "azure-cosmos>=4.6.0", "azure-datalake-store>=0.0.45", @@ -856,7 +856,7 @@ "microsoft.mssql": { "deps": [ "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "methodtools>=0.4.7", "pymssql>=2.3.0" ], @@ -870,7 +870,7 @@ }, "microsoft.psrp": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "pypsrp>=0.8.0" ], "devel-deps": [], @@ -881,7 +881,7 @@ }, "microsoft.winrm": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "pywinrm>=0.4" ], "devel-deps": [], @@ -892,7 +892,7 @@ }, "mongo": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "dnspython>=1.13.0", "pymongo>=4.0.0" ], @@ -908,7 +908,7 @@ "deps": [ "aiomysql>=0.2.0", "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "mysql-connector-python>=8.0.29", "mysqlclient>=1.4.0; sys_platform != 'darwin'" ], @@ -927,7 +927,7 @@ }, "neo4j": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "neo4j>=4.2.1" ], "devel-deps": [], @@ -939,7 +939,7 @@ "odbc": { "deps": [ "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "pyodbc>=5.0.0" ], "devel-deps": [], @@ -952,7 +952,7 @@ }, "openai": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "openai[datalib]>=1.32.0" ], "devel-deps": [], @@ -963,7 +963,7 @@ }, "openfaas": { "deps": [ - "apache-airflow>=2.8.0" + "apache-airflow>=2.9.0" ], "devel-deps": [], "plugins": [], @@ -975,7 +975,7 @@ "deps": [ "apache-airflow-providers-common-compat>=1.2.1", "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "attrs>=22.2", "openlineage-integration-common>=1.24.2", "openlineage-python>=1.24.2" @@ -996,7 +996,7 @@ }, "opensearch": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "opensearch-py>=2.2.0" ], "devel-deps": [], @@ -1007,7 +1007,7 @@ }, "opsgenie": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "opsgenie-sdk>=2.1.5" ], "devel-deps": [], @@ -1019,7 +1019,7 @@ "oracle": { "deps": [ "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "oracledb>=2.0.0" ], "devel-deps": [], @@ -1032,7 +1032,7 @@ }, "pagerduty": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "pdpyras>=4.2.0" ], "devel-deps": [], @@ -1043,7 +1043,7 @@ }, "papermill": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "ipykernel>=6.29.4", "pandas>=1.5.3,<2.2;python_version<\"3.9\"", "pandas>=2.1.2,<2.2;python_version>=\"3.9\"", @@ -1059,7 +1059,7 @@ "pgvector": { "deps": [ "apache-airflow-providers-postgres>=5.7.1", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "pgvector!=0.3.0" ], "devel-deps": [], @@ -1073,7 +1073,7 @@ }, "pinecone": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "pinecone-client>=3.1.0" ], "devel-deps": [], @@ -1085,7 +1085,7 @@ "postgres": { "deps": [ "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "asyncpg>=0.30.0", "psycopg2-binary>=2.9.4" ], @@ -1102,7 +1102,7 @@ "presto": { "deps": [ "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "pandas>=1.5.3,<2.2;python_version<\"3.9\"", "pandas>=2.1.2,<2.2;python_version>=\"3.9\"", "presto-python-client>=0.8.4" @@ -1118,7 +1118,7 @@ }, "qdrant": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "qdrant_client>=1.10.1" ], "devel-deps": [], @@ -1129,7 +1129,7 @@ }, "redis": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "redis>=4.5.2,!=4.5.5,!=5.0.2" ], "devel-deps": [], @@ -1140,7 +1140,7 @@ }, "salesforce": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "pandas>=1.5.3,<2.2;python_version<\"3.9\"", "pandas>=2.1.2,<2.2;python_version>=\"3.9\"", "simple-salesforce>=1.0.0" @@ -1153,7 +1153,7 @@ }, "samba": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "smbprotocol>=1.5.0" ], "devel-deps": [], @@ -1167,7 +1167,7 @@ "segment": { "deps": [ "analytics-python>=1.2.9", - "apache-airflow>=2.8.0" + "apache-airflow>=2.9.0" ], "devel-deps": [], "plugins": [], @@ -1177,7 +1177,7 @@ }, "sendgrid": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "sendgrid>=6.0.0" ], "devel-deps": [], @@ -1189,7 +1189,7 @@ "sftp": { "deps": [ "apache-airflow-providers-ssh>=2.1.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "asyncssh>=2.12.0", "paramiko>=2.9.0" ], @@ -1205,7 +1205,7 @@ }, "singularity": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "spython>=0.0.56" ], "devel-deps": [], @@ -1217,7 +1217,7 @@ "slack": { "deps": [ "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "slack_sdk>=3.19.0" ], "devel-deps": [], @@ -1230,7 +1230,7 @@ }, "smtp": { "deps": [ - "apache-airflow>=2.8.0" + "apache-airflow>=2.9.0" ], "devel-deps": [], "plugins": [], @@ -1242,7 +1242,7 @@ "deps": [ "apache-airflow-providers-common-compat>=1.1.0", "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "pandas>=1.5.3,<2.2;python_version<\"3.9\"", "pandas>=2.1.2,<2.2;python_version>=\"3.9\"", "pyarrow>=14.0.1", @@ -1264,7 +1264,7 @@ "deps": [ "aiosqlite>=0.20.0", "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0" + "apache-airflow>=2.9.0" ], "devel-deps": [], "plugins": [], @@ -1276,7 +1276,7 @@ }, "ssh": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "paramiko>=2.9.0", "sshtunnel>=0.3.2" ], @@ -1289,7 +1289,7 @@ "standard": { "deps": [ "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0" + "apache-airflow>=2.9.0" ], "devel-deps": [], "plugins": [], @@ -1299,7 +1299,7 @@ }, "tableau": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "tableauserverclient>=0.25" ], "devel-deps": [], @@ -1310,7 +1310,7 @@ }, "telegram": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "python-telegram-bot>=20.2" ], "devel-deps": [], @@ -1322,7 +1322,7 @@ "teradata": { "deps": [ "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "teradatasql>=17.20.0.28", "teradatasqlalchemy>=17.20.0.0" ], @@ -1339,7 +1339,7 @@ "trino": { "deps": [ "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "pandas>=1.5.3,<2.2;python_version<\"3.9\"", "pandas>=2.1.2,<2.2;python_version>=\"3.9\"", "trino>=0.318.0" @@ -1357,7 +1357,7 @@ "vertica": { "deps": [ "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "vertica-python>=0.6.0" ], "devel-deps": [], @@ -1370,7 +1370,7 @@ }, "weaviate": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "httpx>=0.25.0", "pandas>=1.5.3,<2.2;python_version<\"3.9\"", "pandas>=2.1.2,<2.2;python_version>=\"3.9\"", @@ -1384,7 +1384,7 @@ }, "yandex": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "yandex-query-client>=0.1.4", "yandexcloud>=0.308.0" ], @@ -1397,7 +1397,7 @@ "ydb": { "deps": [ "apache-airflow-providers-common-sql>=1.20.0", - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "ydb-dbapi>=0.1.0", "ydb>=3.18.8" ], @@ -1411,7 +1411,7 @@ }, "zendesk": { "deps": [ - "apache-airflow>=2.8.0", + "apache-airflow>=2.9.0", "zenpy>=2.0.40" ], "devel-deps": [], diff --git a/newsfragments/44820.significant.rst b/newsfragments/44820.significant.rst new file mode 100644 index 00000000000000..61b7c968ff333b --- /dev/null +++ b/newsfragments/44820.significant.rst @@ -0,0 +1,38 @@ +Removed ``conf`` from the Task template context + +The ``conf`` variable, which provided access to the full Airflow configuration (``airflow.cfg``), has been +removed from the Task (Jinja2) template context for security and simplicity. If you +need specific configuration values in your tasks, retrieve them explicitly in your DAG or task code +using the ``airflow.configuration.conf`` module. + +For users retrieving the webserver URL (e.g., to include log links in task or callbacks), one of the +most common use-case, use the ``ti.log_url`` property available in the ``TaskInstance`` context instead. + +Example: + +.. code-block:: python + + PythonOperator( + task_id="my_task", + python_callable=my_task_callable, + on_failure_callback=SmtpNotifier( + from_email="example@example.com", + to="example@example.com", + subject="Task {{ ti.task_id }} failed", + html_content="Task {{ ti.task_id }} failed. Log URL: {{ ti.log_url }}", + ), + ) + +* Types of change + + * [x] DAG changes + * [ ] Config changes + * [ ] API changes + * [ ] CLI changes + * [ ] Behaviour changes + * [ ] Plugin changes + * [ ] Dependency changes + +* Migration rules needed + + * Remove context key ``conf`` diff --git a/providers/src/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst b/providers/src/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst index 867f45edf9ff03..b5f918149ebd2e 100644 --- a/providers/src/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst +++ b/providers/src/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst @@ -130,7 +130,7 @@ Add chicken-egg-provider to compatibility checks ................................................ Providers that have "min-airflow-version" set to the new, upcoming versions should be excluded in -all previous versions of compatibility check matrix in ``BASE_PROVIDERS_COMPATIBILITY_CHECKS`` in +all previous versions of compatibility check matrix in ``PROVIDERS_COMPATIBILITY_TESTS_MATRIX`` in ``src/airflow_breeze/global_constants.py``. Please add it to all previous versions Add chicken-egg-provider to constraint generation diff --git a/providers/src/airflow/providers/airbyte/__init__.py b/providers/src/airflow/providers/airbyte/__init__.py index 816983815f0251..40b12bf3fc8683 100644 --- a/providers/src/airflow/providers/airbyte/__init__.py +++ b/providers/src/airflow/providers/airbyte/__init__.py @@ -32,8 +32,8 @@ __version__ = "4.0.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-airbyte:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-airbyte:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/airbyte/provider.yaml b/providers/src/airflow/providers/airbyte/provider.yaml index 7db89834ecf5b9..62683871728b81 100644 --- a/providers/src/airflow/providers/airbyte/provider.yaml +++ b/providers/src/airflow/providers/airbyte/provider.yaml @@ -50,7 +50,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - airbyte-api>=0.52.0 integrations: diff --git a/providers/src/airflow/providers/alibaba/CHANGELOG.rst b/providers/src/airflow/providers/alibaba/CHANGELOG.rst index 6926f10a86362a..3dd8a237f58c2c 100644 --- a/providers/src/airflow/providers/alibaba/CHANGELOG.rst +++ b/providers/src/airflow/providers/alibaba/CHANGELOG.rst @@ -26,6 +26,20 @@ Changelog --------- + + +.. warning:: + All deprecated classes, parameters and features have been removed from the alibaba provider package. + The following breaking changes were introduced: + + * Operators + * Remove ``get_hook`` method from ``AnalyticDBSparkBaseOperator``. Use ``self.hook`` instead. + + * sensors + * Remove ``get_hook`` method from ``AnalyticDBSparkSensor``. Use ``self.hook`` instead. + + + 2.9.1 ..... diff --git a/providers/src/airflow/providers/alibaba/__init__.py b/providers/src/airflow/providers/alibaba/__init__.py index b301da374ff67c..4c580fbab6f3a4 100644 --- a/providers/src/airflow/providers/alibaba/__init__.py +++ b/providers/src/airflow/providers/alibaba/__init__.py @@ -32,8 +32,8 @@ __version__ = "2.9.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-alibaba:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-alibaba:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/alibaba/cloud/operators/analyticdb_spark.py b/providers/src/airflow/providers/alibaba/cloud/operators/analyticdb_spark.py index 324b715ddff334..3cf414230f519f 100644 --- a/providers/src/airflow/providers/alibaba/cloud/operators/analyticdb_spark.py +++ b/providers/src/airflow/providers/alibaba/cloud/operators/analyticdb_spark.py @@ -22,9 +22,7 @@ from functools import cached_property from typing import TYPE_CHECKING, Any -from deprecated.classic import deprecated - -from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning +from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.alibaba.cloud.hooks.analyticdb_spark import AnalyticDBSparkHook, AppState @@ -56,11 +54,6 @@ def hook(self) -> AnalyticDBSparkHook: """Get valid hook.""" return AnalyticDBSparkHook(adb_spark_conn_id=self._adb_spark_conn_id, region=self._region) - @deprecated(reason="use `hook` property instead.", category=AirflowProviderDeprecationWarning) - def get_hook(self) -> AnalyticDBSparkHook: - """Get valid hook.""" - return self.hook - def execute(self, context: Context) -> Any: ... def monitor_application(self): diff --git a/providers/src/airflow/providers/alibaba/cloud/sensors/analyticdb_spark.py b/providers/src/airflow/providers/alibaba/cloud/sensors/analyticdb_spark.py index afbbeef4d9275f..705c1fddab7f70 100644 --- a/providers/src/airflow/providers/alibaba/cloud/sensors/analyticdb_spark.py +++ b/providers/src/airflow/providers/alibaba/cloud/sensors/analyticdb_spark.py @@ -21,9 +21,6 @@ from functools import cached_property from typing import TYPE_CHECKING, Any -from deprecated.classic import deprecated - -from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.alibaba.cloud.hooks.analyticdb_spark import AnalyticDBSparkHook, AppState from airflow.sensors.base import BaseSensorOperator @@ -60,11 +57,6 @@ def hook(self) -> AnalyticDBSparkHook: """Get valid hook.""" return AnalyticDBSparkHook(adb_spark_conn_id=self._adb_spark_conn_id, region=self._region) - @deprecated(reason="use `hook` property instead.", category=AirflowProviderDeprecationWarning) - def get_hook(self) -> AnalyticDBSparkHook: - """Get valid hook.""" - return self.hook - def poke(self, context: Context) -> bool: app_id = self.app_id diff --git a/providers/src/airflow/providers/alibaba/provider.yaml b/providers/src/airflow/providers/alibaba/provider.yaml index fa6d9461db0dc7..7d8c38b9ad51fb 100644 --- a/providers/src/airflow/providers/alibaba/provider.yaml +++ b/providers/src/airflow/providers/alibaba/provider.yaml @@ -51,7 +51,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - oss2>=2.14.0 - alibabacloud_adb20211201>=1.0.0 - alibabacloud_tea_openapi>=0.3.7 diff --git a/providers/src/airflow/providers/amazon/__init__.py b/providers/src/airflow/providers/amazon/__init__.py index 0388ca0969f339..39bade51082b3e 100644 --- a/providers/src/airflow/providers/amazon/__init__.py +++ b/providers/src/airflow/providers/amazon/__init__.py @@ -32,8 +32,8 @@ __version__ = "9.1.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-amazon:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-amazon:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/amazon/provider.yaml b/providers/src/airflow/providers/amazon/provider.yaml index 1f598fca6a6f41..493d5b15928f10 100644 --- a/providers/src/airflow/providers/amazon/provider.yaml +++ b/providers/src/airflow/providers/amazon/provider.yaml @@ -93,7 +93,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-compat>=1.3.0 - apache-airflow-providers-common-sql>=1.20.0 - apache-airflow-providers-http diff --git a/providers/src/airflow/providers/apache/beam/__init__.py b/providers/src/airflow/providers/apache/beam/__init__.py index 74bc85eddfc011..1e2f46029b4830 100644 --- a/providers/src/airflow/providers/apache/beam/__init__.py +++ b/providers/src/airflow/providers/apache/beam/__init__.py @@ -32,8 +32,8 @@ __version__ = "5.9.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-apache-beam:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-apache-beam:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/apache/beam/provider.yaml b/providers/src/airflow/providers/apache/beam/provider.yaml index 8f90db785691b5..22e7f68d7bc429 100644 --- a/providers/src/airflow/providers/apache/beam/provider.yaml +++ b/providers/src/airflow/providers/apache/beam/provider.yaml @@ -63,7 +63,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 # Apache Beam > 2.53.0 and pyarrow > 14.0.1 fix https://nvd.nist.gov/vuln/detail/CVE-2023-47248. - apache-beam>=2.53.0 - pyarrow>=14.0.1 diff --git a/providers/src/airflow/providers/apache/cassandra/__init__.py b/providers/src/airflow/providers/apache/cassandra/__init__.py index 1e3027d34b3b50..1e4eaa5585daee 100644 --- a/providers/src/airflow/providers/apache/cassandra/__init__.py +++ b/providers/src/airflow/providers/apache/cassandra/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.6.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-apache-cassandra:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-apache-cassandra:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/apache/cassandra/provider.yaml b/providers/src/airflow/providers/apache/cassandra/provider.yaml index 7f2ea49fbbfda7..9a43bea9bc318d 100644 --- a/providers/src/airflow/providers/apache/cassandra/provider.yaml +++ b/providers/src/airflow/providers/apache/cassandra/provider.yaml @@ -47,7 +47,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - cassandra-driver>=3.29.1 integrations: diff --git a/providers/src/airflow/providers/apache/drill/__init__.py b/providers/src/airflow/providers/apache/drill/__init__.py index 698775a1cf494c..45f166c86bb9b1 100644 --- a/providers/src/airflow/providers/apache/drill/__init__.py +++ b/providers/src/airflow/providers/apache/drill/__init__.py @@ -32,8 +32,8 @@ __version__ = "2.8.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-apache-drill:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-apache-drill:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/apache/drill/provider.yaml b/providers/src/airflow/providers/apache/drill/provider.yaml index fe1f97d9fb5158..ffc1e78f8b7e4e 100644 --- a/providers/src/airflow/providers/apache/drill/provider.yaml +++ b/providers/src/airflow/providers/apache/drill/provider.yaml @@ -53,7 +53,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-sql>=1.20.0 - sqlalchemy-drill>=1.1.0 diff --git a/providers/src/airflow/providers/apache/druid/__init__.py b/providers/src/airflow/providers/apache/druid/__init__.py index e7d5b2f9b71eb8..2fef9da020df41 100644 --- a/providers/src/airflow/providers/apache/druid/__init__.py +++ b/providers/src/airflow/providers/apache/druid/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.12.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-apache-druid:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-apache-druid:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/apache/druid/provider.yaml b/providers/src/airflow/providers/apache/druid/provider.yaml index ee068c811cfa9d..62bd4b0a5a23b9 100644 --- a/providers/src/airflow/providers/apache/druid/provider.yaml +++ b/providers/src/airflow/providers/apache/druid/provider.yaml @@ -60,7 +60,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-sql>=1.20.0 - pydruid>=0.4.1 diff --git a/providers/src/airflow/providers/apache/flink/__init__.py b/providers/src/airflow/providers/apache/flink/__init__.py index d0f2ccd875c70d..64bdc0f8c21f90 100644 --- a/providers/src/airflow/providers/apache/flink/__init__.py +++ b/providers/src/airflow/providers/apache/flink/__init__.py @@ -32,8 +32,8 @@ __version__ = "1.5.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-apache-flink:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-apache-flink:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/apache/flink/provider.yaml b/providers/src/airflow/providers/apache/flink/provider.yaml index 8a089a22894bf7..80a7b03ac4c1c0 100644 --- a/providers/src/airflow/providers/apache/flink/provider.yaml +++ b/providers/src/airflow/providers/apache/flink/provider.yaml @@ -40,7 +40,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - cryptography>=41.0.0 - apache-airflow-providers-cncf-kubernetes>=5.1.0 diff --git a/providers/src/airflow/providers/apache/hdfs/__init__.py b/providers/src/airflow/providers/apache/hdfs/__init__.py index f37ec37384a8db..01a1b1d5cdcdbd 100644 --- a/providers/src/airflow/providers/apache/hdfs/__init__.py +++ b/providers/src/airflow/providers/apache/hdfs/__init__.py @@ -32,8 +32,8 @@ __version__ = "4.6.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-apache-hdfs:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-apache-hdfs:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/apache/hdfs/provider.yaml b/providers/src/airflow/providers/apache/hdfs/provider.yaml index f8c514e9c99f2a..2b2e5839d0db04 100644 --- a/providers/src/airflow/providers/apache/hdfs/provider.yaml +++ b/providers/src/airflow/providers/apache/hdfs/provider.yaml @@ -56,7 +56,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - hdfs[avro,dataframe,kerberos]>=2.5.4;python_version<"3.12" - hdfs[avro,dataframe,kerberos]>=2.7.3;python_version>="3.12" - pandas>=2.1.2,<2.2;python_version>="3.9" diff --git a/providers/src/airflow/providers/apache/hive/__init__.py b/providers/src/airflow/providers/apache/hive/__init__.py index 7e4d45f58422f5..03da613dd89627 100644 --- a/providers/src/airflow/providers/apache/hive/__init__.py +++ b/providers/src/airflow/providers/apache/hive/__init__.py @@ -32,8 +32,8 @@ __version__ = "8.2.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-apache-hive:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-apache-hive:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/apache/hive/provider.yaml b/providers/src/airflow/providers/apache/hive/provider.yaml index 7329f4f8491f87..92667f2d30ab10 100644 --- a/providers/src/airflow/providers/apache/hive/provider.yaml +++ b/providers/src/airflow/providers/apache/hive/provider.yaml @@ -73,7 +73,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-sql>=1.20.0 - hmsclient>=0.1.0 # In pandas 2.2 minimal version of the sqlalchemy is 2.0 diff --git a/providers/src/airflow/providers/apache/iceberg/__init__.py b/providers/src/airflow/providers/apache/iceberg/__init__.py index e71ea486fc917e..09defb6c16f9d0 100644 --- a/providers/src/airflow/providers/apache/iceberg/__init__.py +++ b/providers/src/airflow/providers/apache/iceberg/__init__.py @@ -32,8 +32,8 @@ __version__ = "1.1.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-apache-iceberg:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-apache-iceberg:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/apache/iceberg/provider.yaml b/providers/src/airflow/providers/apache/iceberg/provider.yaml index 3dd930b0eaba18..dd0d7402240717 100644 --- a/providers/src/airflow/providers/apache/iceberg/provider.yaml +++ b/providers/src/airflow/providers/apache/iceberg/provider.yaml @@ -29,7 +29,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 devel-dependencies: - pyiceberg>=0.5.0 diff --git a/providers/src/airflow/providers/apache/impala/__init__.py b/providers/src/airflow/providers/apache/impala/__init__.py index 61b8964ac12ae1..56a1a5cf28080f 100644 --- a/providers/src/airflow/providers/apache/impala/__init__.py +++ b/providers/src/airflow/providers/apache/impala/__init__.py @@ -32,8 +32,8 @@ __version__ = "1.5.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-apache-impala:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-apache-impala:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/apache/impala/provider.yaml b/providers/src/airflow/providers/apache/impala/provider.yaml index f9a55e08f94197..b4e9cd868920d0 100644 --- a/providers/src/airflow/providers/apache/impala/provider.yaml +++ b/providers/src/airflow/providers/apache/impala/provider.yaml @@ -43,7 +43,7 @@ versions: dependencies: - impyla>=0.18.0,<1.0 - apache-airflow-providers-common-sql>=1.20.0 - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 additional-extras: - name: kerberos diff --git a/providers/src/airflow/providers/apache/kafka/__init__.py b/providers/src/airflow/providers/apache/kafka/__init__.py index b7050a89dab8e1..3eda7193db54fb 100644 --- a/providers/src/airflow/providers/apache/kafka/__init__.py +++ b/providers/src/airflow/providers/apache/kafka/__init__.py @@ -32,8 +32,8 @@ __version__ = "1.6.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-apache-kafka:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-apache-kafka:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/apache/kafka/provider.yaml b/providers/src/airflow/providers/apache/kafka/provider.yaml index 23961ec3c89f5c..89d41a5c32a3b0 100644 --- a/providers/src/airflow/providers/apache/kafka/provider.yaml +++ b/providers/src/airflow/providers/apache/kafka/provider.yaml @@ -39,7 +39,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - asgiref>=2.3.0 - confluent-kafka>=2.3.0 diff --git a/providers/src/airflow/providers/apache/kylin/__init__.py b/providers/src/airflow/providers/apache/kylin/__init__.py index fa7850f8d16096..deae72e2b9b52d 100644 --- a/providers/src/airflow/providers/apache/kylin/__init__.py +++ b/providers/src/airflow/providers/apache/kylin/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.7.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-apache-kylin:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-apache-kylin:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/apache/kylin/provider.yaml b/providers/src/airflow/providers/apache/kylin/provider.yaml index 610934f9ca8429..3405f514893fcd 100644 --- a/providers/src/airflow/providers/apache/kylin/provider.yaml +++ b/providers/src/airflow/providers/apache/kylin/provider.yaml @@ -45,7 +45,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - kylinpy>=2.7.0 integrations: diff --git a/providers/src/airflow/providers/apache/livy/__init__.py b/providers/src/airflow/providers/apache/livy/__init__.py index fe978a34bc54fa..f9add3e071390b 100644 --- a/providers/src/airflow/providers/apache/livy/__init__.py +++ b/providers/src/airflow/providers/apache/livy/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.9.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-apache-livy:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-apache-livy:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/apache/livy/provider.yaml b/providers/src/airflow/providers/apache/livy/provider.yaml index 31f4b7f1a14947..9a3453c35df394 100644 --- a/providers/src/airflow/providers/apache/livy/provider.yaml +++ b/providers/src/airflow/providers/apache/livy/provider.yaml @@ -56,7 +56,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-http - aiohttp>=3.9.2 - asgiref>=2.3.0 diff --git a/providers/src/airflow/providers/apache/pig/__init__.py b/providers/src/airflow/providers/apache/pig/__init__.py index 544cea7657618f..345844846a89cc 100644 --- a/providers/src/airflow/providers/apache/pig/__init__.py +++ b/providers/src/airflow/providers/apache/pig/__init__.py @@ -32,8 +32,8 @@ __version__ = "4.5.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-apache-pig:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-apache-pig:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/apache/pig/provider.yaml b/providers/src/airflow/providers/apache/pig/provider.yaml index 3ce8397d2d4442..ccc8da59d03721 100644 --- a/providers/src/airflow/providers/apache/pig/provider.yaml +++ b/providers/src/airflow/providers/apache/pig/provider.yaml @@ -44,7 +44,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 integrations: - integration-name: Apache Pig diff --git a/providers/src/airflow/providers/apache/pinot/__init__.py b/providers/src/airflow/providers/apache/pinot/__init__.py index f3c024b362073d..9b335f626b1c32 100644 --- a/providers/src/airflow/providers/apache/pinot/__init__.py +++ b/providers/src/airflow/providers/apache/pinot/__init__.py @@ -32,8 +32,8 @@ __version__ = "4.5.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-apache-pinot:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-apache-pinot:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/apache/pinot/provider.yaml b/providers/src/airflow/providers/apache/pinot/provider.yaml index b495d52272ad71..b96c17ba9ae071 100644 --- a/providers/src/airflow/providers/apache/pinot/provider.yaml +++ b/providers/src/airflow/providers/apache/pinot/provider.yaml @@ -53,7 +53,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-sql>=1.20.0 - pinotdb>=5.1.0 diff --git a/providers/src/airflow/providers/apache/spark/__init__.py b/providers/src/airflow/providers/apache/spark/__init__.py index 45eb39d5bb5795..e18a9dc4f1d418 100644 --- a/providers/src/airflow/providers/apache/spark/__init__.py +++ b/providers/src/airflow/providers/apache/spark/__init__.py @@ -32,8 +32,8 @@ __version__ = "4.11.3" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-apache-spark:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-apache-spark:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/apache/spark/provider.yaml b/providers/src/airflow/providers/apache/spark/provider.yaml index 74526dea6471d2..9d7c98caf40fd7 100644 --- a/providers/src/airflow/providers/apache/spark/provider.yaml +++ b/providers/src/airflow/providers/apache/spark/provider.yaml @@ -65,7 +65,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - pyspark>=3.1.3 - grpcio-status>=1.59.0 diff --git a/providers/src/airflow/providers/apprise/__init__.py b/providers/src/airflow/providers/apprise/__init__.py index ab059aecb221ac..f86b0bac637c8b 100644 --- a/providers/src/airflow/providers/apprise/__init__.py +++ b/providers/src/airflow/providers/apprise/__init__.py @@ -32,8 +32,8 @@ __version__ = "1.4.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-apprise:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-apprise:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/apprise/provider.yaml b/providers/src/airflow/providers/apprise/provider.yaml index bf4388a5b5df4d..ddbd46e8b80701 100644 --- a/providers/src/airflow/providers/apprise/provider.yaml +++ b/providers/src/airflow/providers/apprise/provider.yaml @@ -47,7 +47,7 @@ integrations: tags: [software] dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apprise>=1.8.0 hooks: diff --git a/providers/src/airflow/providers/arangodb/__init__.py b/providers/src/airflow/providers/arangodb/__init__.py index cc7e2447261490..a7a7b0c41aa1da 100644 --- a/providers/src/airflow/providers/arangodb/__init__.py +++ b/providers/src/airflow/providers/arangodb/__init__.py @@ -32,8 +32,8 @@ __version__ = "2.6.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-arangodb:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-arangodb:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/arangodb/provider.yaml b/providers/src/airflow/providers/arangodb/provider.yaml index e5a480bc6a89b2..e4ff5e3d065c62 100644 --- a/providers/src/airflow/providers/arangodb/provider.yaml +++ b/providers/src/airflow/providers/arangodb/provider.yaml @@ -22,7 +22,7 @@ description: | `ArangoDB `__ dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - python-arango>=7.3.2 state: ready diff --git a/providers/src/airflow/providers/asana/__init__.py b/providers/src/airflow/providers/asana/__init__.py index f8b8eda494fa8a..5ab8bea12934e9 100644 --- a/providers/src/airflow/providers/asana/__init__.py +++ b/providers/src/airflow/providers/asana/__init__.py @@ -32,8 +32,8 @@ __version__ = "2.6.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-asana:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-asana:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/asana/provider.yaml b/providers/src/airflow/providers/asana/provider.yaml index 40f7131462cc4b..353379b9c8ca18 100644 --- a/providers/src/airflow/providers/asana/provider.yaml +++ b/providers/src/airflow/providers/asana/provider.yaml @@ -44,7 +44,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 # The Asana provider uses Asana Python API https://github.com/Asana/python-asana/ and version # 4.* of it introduced 31.07.2023 is heavily incompatible with previous versions # (new way of generating API client from specification has been used). Until we convert to the new API diff --git a/providers/src/airflow/providers/atlassian/jira/__init__.py b/providers/src/airflow/providers/atlassian/jira/__init__.py index 1d37ccd74913a7..6acf59b4f112f7 100644 --- a/providers/src/airflow/providers/atlassian/jira/__init__.py +++ b/providers/src/airflow/providers/atlassian/jira/__init__.py @@ -32,8 +32,8 @@ __version__ = "2.7.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-atlassian-jira:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-atlassian-jira:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/atlassian/jira/provider.yaml b/providers/src/airflow/providers/atlassian/jira/provider.yaml index 0a505256e7cb8d..0d6fc430b948dc 100644 --- a/providers/src/airflow/providers/atlassian/jira/provider.yaml +++ b/providers/src/airflow/providers/atlassian/jira/provider.yaml @@ -42,7 +42,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - atlassian-python-api>3.41.10 integrations: diff --git a/providers/src/airflow/providers/celery/__init__.py b/providers/src/airflow/providers/celery/__init__.py index e92b2684c88809..e955bc9d03d1c2 100644 --- a/providers/src/airflow/providers/celery/__init__.py +++ b/providers/src/airflow/providers/celery/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.8.5" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-celery:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-celery:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/celery/provider.yaml b/providers/src/airflow/providers/celery/provider.yaml index daa143cf552890..ca27d600424ca3 100644 --- a/providers/src/airflow/providers/celery/provider.yaml +++ b/providers/src/airflow/providers/celery/provider.yaml @@ -62,7 +62,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 # The Celery is known to introduce problems when upgraded to a MAJOR version. Airflow Core # Uses Celery for CeleryExecutor, and we also know that Kubernetes Python client follows SemVer # (https://docs.celeryq.dev/en/stable/contributing.html?highlight=semver#versions). diff --git a/providers/src/airflow/providers/cloudant/__init__.py b/providers/src/airflow/providers/cloudant/__init__.py index 3bc6d532a9b8ba..91c3b1245a3950 100644 --- a/providers/src/airflow/providers/cloudant/__init__.py +++ b/providers/src/airflow/providers/cloudant/__init__.py @@ -32,8 +32,8 @@ __version__ = "4.0.3" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-cloudant:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-cloudant:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/cloudant/provider.yaml b/providers/src/airflow/providers/cloudant/provider.yaml index 05ab8f0f439c07..aad4ceec60c705 100644 --- a/providers/src/airflow/providers/cloudant/provider.yaml +++ b/providers/src/airflow/providers/cloudant/provider.yaml @@ -49,7 +49,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 # Even though 3.9 is excluded below, we need to make this python_version aware so that `uv` can generate a # full lock file when building lock file from provider sources - 'ibmcloudant==0.9.1 ; python_version >= "3.10"' diff --git a/providers/src/airflow/providers/cncf/kubernetes/__init__.py b/providers/src/airflow/providers/cncf/kubernetes/__init__.py index 33eafb097ed106..c901b0db5c3280 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/__init__.py +++ b/providers/src/airflow/providers/cncf/kubernetes/__init__.py @@ -32,8 +32,8 @@ __version__ = "10.1.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-cncf-kubernetes:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-cncf-kubernetes:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py b/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py index 56c3a786795a7b..bc5699361bac0d 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py +++ b/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py @@ -244,6 +244,7 @@ def clear_not_launched_queued_tasks(self, session: Session = NEW_SESSION) -> Non from airflow.executors.executor_loader import ExecutorLoader default_executor = str(ExecutorLoader.get_default_executor_name()) + default_executor = default_executor.strip(":") with Stats.timer("kubernetes_executor.clear_not_launched_queued_tasks.duration"): self.log.debug("Clearing tasks that have not been launched") diff --git a/providers/src/airflow/providers/cncf/kubernetes/provider.yaml b/providers/src/airflow/providers/cncf/kubernetes/provider.yaml index c52090faa06acf..56bd73f827eaab 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/provider.yaml +++ b/providers/src/airflow/providers/cncf/kubernetes/provider.yaml @@ -97,7 +97,7 @@ versions: dependencies: - aiofiles>=23.2.0 - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - asgiref>=3.5.2 - cryptography>=41.0.0 # The Kubernetes API is known to introduce problems when upgraded to a MAJOR version. Airflow Core diff --git a/providers/src/airflow/providers/cohere/__init__.py b/providers/src/airflow/providers/cohere/__init__.py index 3c375df277e3dd..83cd2e62eb5e67 100644 --- a/providers/src/airflow/providers/cohere/__init__.py +++ b/providers/src/airflow/providers/cohere/__init__.py @@ -32,8 +32,8 @@ __version__ = "1.3.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-cohere:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-cohere:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/cohere/provider.yaml b/providers/src/airflow/providers/cohere/provider.yaml index e5fa6ebb027179..325dff3842dccf 100644 --- a/providers/src/airflow/providers/cohere/provider.yaml +++ b/providers/src/airflow/providers/cohere/provider.yaml @@ -45,7 +45,7 @@ integrations: tags: [software] dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - cohere>=4.37,<5 hooks: diff --git a/providers/src/airflow/providers/common/compat/__init__.py b/providers/src/airflow/providers/common/compat/__init__.py index 5bda470e7b0741..21133a52bb0837 100644 --- a/providers/src/airflow/providers/common/compat/__init__.py +++ b/providers/src/airflow/providers/common/compat/__init__.py @@ -32,8 +32,8 @@ __version__ = "1.3.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-common-compat:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-common-compat:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/common/compat/provider.yaml b/providers/src/airflow/providers/common/compat/provider.yaml index 9ed101d8b37d52..34be19b27b665b 100644 --- a/providers/src/airflow/providers/common/compat/provider.yaml +++ b/providers/src/airflow/providers/common/compat/provider.yaml @@ -33,7 +33,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 integrations: - integration-name: Common Compat diff --git a/providers/src/airflow/providers/common/io/__init__.py b/providers/src/airflow/providers/common/io/__init__.py index ac05e1f448dc7b..9e8fadeae514dc 100644 --- a/providers/src/airflow/providers/common/io/__init__.py +++ b/providers/src/airflow/providers/common/io/__init__.py @@ -32,8 +32,8 @@ __version__ = "1.4.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-common-io:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-common-io:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/common/io/provider.yaml b/providers/src/airflow/providers/common/io/provider.yaml index c32f046eb929a1..10b36f0cf89c01 100644 --- a/providers/src/airflow/providers/common/io/provider.yaml +++ b/providers/src/airflow/providers/common/io/provider.yaml @@ -37,7 +37,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 integrations: - integration-name: Common IO diff --git a/providers/src/airflow/providers/common/sql/__init__.py b/providers/src/airflow/providers/common/sql/__init__.py index f6669bc885482b..00659c67195dc0 100644 --- a/providers/src/airflow/providers/common/sql/__init__.py +++ b/providers/src/airflow/providers/common/sql/__init__.py @@ -32,8 +32,8 @@ __version__ = "1.20.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-common-sql:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-common-sql:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/common/sql/hooks/handlers.py b/providers/src/airflow/providers/common/sql/hooks/handlers.py index b399dc0023f6f8..e1b5fa33b91e7e 100644 --- a/providers/src/airflow/providers/common/sql/hooks/handlers.py +++ b/providers/src/airflow/providers/common/sql/hooks/handlers.py @@ -19,7 +19,7 @@ from collections.abc import Iterable -def return_single_query_results(sql: str | Iterable[str], return_last: bool, split_statements: bool): +def return_single_query_results(sql: str | Iterable[str], return_last: bool, split_statements: bool | None): """ Determine when results of single query only should be returned. diff --git a/providers/src/airflow/providers/common/sql/provider.yaml b/providers/src/airflow/providers/common/sql/provider.yaml index 64d16d593e856d..4abc9575ead115 100644 --- a/providers/src/airflow/providers/common/sql/provider.yaml +++ b/providers/src/airflow/providers/common/sql/provider.yaml @@ -64,7 +64,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - sqlparse>=0.5.1 - more-itertools>=9.0.0 diff --git a/providers/src/airflow/providers/databricks/__init__.py b/providers/src/airflow/providers/databricks/__init__.py index f4270920318063..9d22f47179f8e8 100644 --- a/providers/src/airflow/providers/databricks/__init__.py +++ b/providers/src/airflow/providers/databricks/__init__.py @@ -32,8 +32,8 @@ __version__ = "6.13.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-databricks:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-databricks:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/databricks/provider.yaml b/providers/src/airflow/providers/databricks/provider.yaml index ef9d8a5dcc6299..f694c884f89930 100644 --- a/providers/src/airflow/providers/databricks/provider.yaml +++ b/providers/src/airflow/providers/databricks/provider.yaml @@ -72,7 +72,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-sql>=1.20.0 - requests>=2.27.0,<3 - databricks-sql-connector>=3.0.0 diff --git a/providers/src/airflow/providers/datadog/__init__.py b/providers/src/airflow/providers/datadog/__init__.py index 29c01e18b71c17..580399f97135ca 100644 --- a/providers/src/airflow/providers/datadog/__init__.py +++ b/providers/src/airflow/providers/datadog/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.7.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-datadog:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-datadog:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/datadog/provider.yaml b/providers/src/airflow/providers/datadog/provider.yaml index 80cbbe9c6ecab8..ac2a6794e80896 100644 --- a/providers/src/airflow/providers/datadog/provider.yaml +++ b/providers/src/airflow/providers/datadog/provider.yaml @@ -47,7 +47,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - datadog>=0.14.0 integrations: diff --git a/providers/src/airflow/providers/dbt/cloud/__init__.py b/providers/src/airflow/providers/dbt/cloud/__init__.py index 18018a34893acd..094a82e3fd0039 100644 --- a/providers/src/airflow/providers/dbt/cloud/__init__.py +++ b/providers/src/airflow/providers/dbt/cloud/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.11.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-dbt-cloud:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-dbt-cloud:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/dbt/cloud/provider.yaml b/providers/src/airflow/providers/dbt/cloud/provider.yaml index aad2e418005e92..dffd1f06dc76ba 100644 --- a/providers/src/airflow/providers/dbt/cloud/provider.yaml +++ b/providers/src/airflow/providers/dbt/cloud/provider.yaml @@ -59,7 +59,7 @@ versions: - 1.0.1 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-http - asgiref>=2.3.0 - aiohttp>=3.9.2 diff --git a/providers/src/airflow/providers/dingding/__init__.py b/providers/src/airflow/providers/dingding/__init__.py index 568e2f3d8aeded..52bab2206d54b5 100644 --- a/providers/src/airflow/providers/dingding/__init__.py +++ b/providers/src/airflow/providers/dingding/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.6.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-dingding:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-dingding:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/dingding/provider.yaml b/providers/src/airflow/providers/dingding/provider.yaml index caa0f2b93dc43c..270f9db093fae2 100644 --- a/providers/src/airflow/providers/dingding/provider.yaml +++ b/providers/src/airflow/providers/dingding/provider.yaml @@ -44,7 +44,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-http integrations: diff --git a/providers/src/airflow/providers/discord/__init__.py b/providers/src/airflow/providers/discord/__init__.py index 94a385dee1d4ea..ada5ddc7852890 100644 --- a/providers/src/airflow/providers/discord/__init__.py +++ b/providers/src/airflow/providers/discord/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.8.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-discord:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-discord:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/discord/provider.yaml b/providers/src/airflow/providers/discord/provider.yaml index 250e3ac87476aa..d77877a1da42d1 100644 --- a/providers/src/airflow/providers/discord/provider.yaml +++ b/providers/src/airflow/providers/discord/provider.yaml @@ -47,7 +47,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-http integrations: diff --git a/providers/src/airflow/providers/docker/__init__.py b/providers/src/airflow/providers/docker/__init__.py index 7512a8042f5938..1b551c7d874e97 100644 --- a/providers/src/airflow/providers/docker/__init__.py +++ b/providers/src/airflow/providers/docker/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.14.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-docker:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-docker:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/docker/provider.yaml b/providers/src/airflow/providers/docker/provider.yaml index 23cfdec100e6c8..21c98a3fd102a1 100644 --- a/providers/src/airflow/providers/docker/provider.yaml +++ b/providers/src/airflow/providers/docker/provider.yaml @@ -73,7 +73,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - docker>=7.1.0 - python-dotenv>=0.21.0 diff --git a/providers/src/airflow/providers/edge/CHANGELOG.rst b/providers/src/airflow/providers/edge/CHANGELOG.rst index f9d9816d34acfd..af4df15918cd33 100644 --- a/providers/src/airflow/providers/edge/CHANGELOG.rst +++ b/providers/src/airflow/providers/edge/CHANGELOG.rst @@ -27,6 +27,14 @@ Changelog --------- +0.9.6pre0 +......... + +Misc +~~~~ + +* ``Replace null value in log file chunk with question mark to fix exception by pushing log into DB.`` + 0.9.5pre0 ......... diff --git a/providers/src/airflow/providers/edge/__init__.py b/providers/src/airflow/providers/edge/__init__.py index 72aaf60f364995..7c0490c20785e9 100644 --- a/providers/src/airflow/providers/edge/__init__.py +++ b/providers/src/airflow/providers/edge/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "0.9.5pre0" +__version__ = "0.9.6pre0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.10.0" diff --git a/providers/src/airflow/providers/edge/cli/edge_command.py b/providers/src/airflow/providers/edge/cli/edge_command.py index 8e89c835252f30..115923e981fb78 100644 --- a/providers/src/airflow/providers/edge/cli/edge_command.py +++ b/providers/src/airflow/providers/edge/cli/edge_command.py @@ -275,7 +275,8 @@ def check_running_jobs(self) -> None: read_data = logfile.read() job.logsize += len(read_data) # backslashreplace to keep not decoded characters and not raising exception - log_data = read_data.decode(errors="backslashreplace") + # replace null with question mark to fix issue during DB push + log_data = read_data.decode(errors="backslashreplace").replace("\x00", "\ufffd") while True: chunk_data = log_data[:push_log_chunk_size] log_data = log_data[push_log_chunk_size:] diff --git a/providers/src/airflow/providers/edge/provider.yaml b/providers/src/airflow/providers/edge/provider.yaml index ac64d35691bf0f..f6b0457c07d7ea 100644 --- a/providers/src/airflow/providers/edge/provider.yaml +++ b/providers/src/airflow/providers/edge/provider.yaml @@ -27,7 +27,7 @@ source-date-epoch: 1729683247 # note that those versions are maintained by release manager - do not update them manually versions: - - 0.9.5pre0 + - 0.9.6pre0 dependencies: - apache-airflow>=2.10.0 diff --git a/providers/src/airflow/providers/elasticsearch/__init__.py b/providers/src/airflow/providers/elasticsearch/__init__.py index 1f0d663e16df15..2a5a478aaa4274 100644 --- a/providers/src/airflow/providers/elasticsearch/__init__.py +++ b/providers/src/airflow/providers/elasticsearch/__init__.py @@ -32,8 +32,8 @@ __version__ = "5.5.3" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-elasticsearch:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-elasticsearch:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/elasticsearch/provider.yaml b/providers/src/airflow/providers/elasticsearch/provider.yaml index 5385b3b5c250ba..8685f849851f3f 100644 --- a/providers/src/airflow/providers/elasticsearch/provider.yaml +++ b/providers/src/airflow/providers/elasticsearch/provider.yaml @@ -70,7 +70,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-sql>=1.20.0 - elasticsearch>=8.10,<9 diff --git a/providers/src/airflow/providers/exasol/__init__.py b/providers/src/airflow/providers/exasol/__init__.py index 8845aa1cb8dc5d..f321cb2526f319 100644 --- a/providers/src/airflow/providers/exasol/__init__.py +++ b/providers/src/airflow/providers/exasol/__init__.py @@ -32,8 +32,8 @@ __version__ = "4.6.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-exasol:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-exasol:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/exasol/provider.yaml b/providers/src/airflow/providers/exasol/provider.yaml index 4b2754333a5e4b..1f0f15dd2b2f51 100644 --- a/providers/src/airflow/providers/exasol/provider.yaml +++ b/providers/src/airflow/providers/exasol/provider.yaml @@ -61,7 +61,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-sql>=1.20.0 - pyexasol>=0.5.1 # In pandas 2.2 minimal version of the sqlalchemy is 2.0 diff --git a/providers/src/airflow/providers/facebook/__init__.py b/providers/src/airflow/providers/facebook/__init__.py index 5e2c2e7e419dbb..2fa380b0887b5a 100644 --- a/providers/src/airflow/providers/facebook/__init__.py +++ b/providers/src/airflow/providers/facebook/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.6.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-facebook:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-facebook:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/facebook/provider.yaml b/providers/src/airflow/providers/facebook/provider.yaml index 6bcaa88404d3e5..13973000a1d2de 100644 --- a/providers/src/airflow/providers/facebook/provider.yaml +++ b/providers/src/airflow/providers/facebook/provider.yaml @@ -49,7 +49,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - facebook-business>=15.0.2 integrations: diff --git a/providers/src/airflow/providers/ftp/__init__.py b/providers/src/airflow/providers/ftp/__init__.py index 22928a70a24fab..560c65e00c2689 100644 --- a/providers/src/airflow/providers/ftp/__init__.py +++ b/providers/src/airflow/providers/ftp/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.11.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-ftp:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-ftp:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/ftp/provider.yaml b/providers/src/airflow/providers/ftp/provider.yaml index a9394d62c579e2..eae17afdd6b377 100644 --- a/providers/src/airflow/providers/ftp/provider.yaml +++ b/providers/src/airflow/providers/ftp/provider.yaml @@ -55,7 +55,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 integrations: - integration-name: File Transfer Protocol (FTP) diff --git a/providers/src/airflow/providers/github/__init__.py b/providers/src/airflow/providers/github/__init__.py index 453a74a4c4f966..da0044213db36b 100644 --- a/providers/src/airflow/providers/github/__init__.py +++ b/providers/src/airflow/providers/github/__init__.py @@ -32,8 +32,8 @@ __version__ = "2.7.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-github:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-github:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/github/provider.yaml b/providers/src/airflow/providers/github/provider.yaml index eaa025adcba486..75c47f5de6b941 100644 --- a/providers/src/airflow/providers/github/provider.yaml +++ b/providers/src/airflow/providers/github/provider.yaml @@ -23,7 +23,7 @@ description: | `GitHub `__ dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - PyGithub>=2.1.1 state: ready diff --git a/providers/src/airflow/providers/google/__init__.py b/providers/src/airflow/providers/google/__init__.py index 15abb0c12f63e8..a1e268748d315f 100644 --- a/providers/src/airflow/providers/google/__init__.py +++ b/providers/src/airflow/providers/google/__init__.py @@ -32,8 +32,8 @@ __version__ = "11.1.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-google:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-google:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/google/cloud/hooks/dataproc.py b/providers/src/airflow/providers/google/cloud/hooks/dataproc.py index b704c23107e717..6cc57128bab4bf 100644 --- a/providers/src/airflow/providers/google/cloud/hooks/dataproc.py +++ b/providers/src/airflow/providers/google/cloud/hooks/dataproc.py @@ -116,13 +116,17 @@ def add_args(self, args: list[str] | None = None) -> None: if args is not None: self.job["job"][self.job_type]["args"] = args - def add_query(self, query: str) -> None: + def add_query(self, query: str | list[str]) -> None: """ - Set query for Dataproc job. + Add query for Dataproc job. :param query: query for the job. """ - self.job["job"][self.job_type]["query_list"] = {"queries": [query]} + queries = self.job["job"][self.job_type].setdefault("query_list", {"queries": []})["queries"] + if isinstance(query, str): + queries.append(query) + elif isinstance(query, list): + queries.extend(query) def add_query_uri(self, query_uri: str) -> None: """ diff --git a/providers/src/airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py b/providers/src/airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py new file mode 100644 index 00000000000000..69c7b69f8dad56 --- /dev/null +++ b/providers/src/airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py @@ -0,0 +1,147 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""This module contains a Google Cloud Vertex AI Feature Store hook.""" + +from __future__ import annotations + +from google.api_core.client_options import ClientOptions +from google.cloud.aiplatform_v1beta1 import ( + FeatureOnlineStoreAdminServiceClient, +) + +from airflow.exceptions import AirflowException +from airflow.providers.google.common.consts import CLIENT_INFO +from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook + + +class FeatureStoreHook(GoogleBaseHook): + """ + Hook for interacting with Google Cloud Vertex AI Feature Store. + + This hook provides an interface to manage Feature Store resources in Vertex AI, + including feature views and their synchronization operations. It handles authentication + and provides methods for common Feature Store operations. + + :param gcp_conn_id: The connection ID to use for connecting to Google Cloud Platform. + Defaults to 'google_cloud_default'. + :param impersonation_chain: Optional service account to impersonate using short-term + credentials. Can be either a single account or a chain of accounts required to + get the access_token of the last account in the list, which will be impersonated + in the request. If set as a string, the account must grant the originating account + the Service Account Token Creator IAM role. If set as a sequence, the identities + from the list must grant Service Account Token Creator IAM role to the directly + preceding identity, with first account from the list granting this role to the + originating account. + """ + + def get_feature_online_store_admin_service_client( + self, + location: str | None = None, + ) -> FeatureOnlineStoreAdminServiceClient: + """ + Create and returns a FeatureOnlineStoreAdminServiceClient object. + + This method initializes a client for interacting with the Feature Store API, + handling proper endpoint configuration based on the specified location. + + :param location: Optional. The Google Cloud region where the service is located. + If provided and not 'global', the client will be configured to use the + region-specific API endpoint. + """ + if location and location != "global": + client_options = ClientOptions(api_endpoint=f"{location}-aiplatform.googleapis.com:443") + else: + client_options = ClientOptions() + return FeatureOnlineStoreAdminServiceClient( + credentials=self.get_credentials(), client_info=CLIENT_INFO, client_options=client_options + ) + + def get_feature_view_sync( + self, + location: str, + feature_view_sync_name: str, + ) -> dict: + """ + Retrieve the status and details of a Feature View synchronization operation. + + This method fetches information about a specific feature view sync operation, + including its current status, timing information, and synchronization metrics. + + :param location: The Google Cloud region where the feature store is located + (e.g., 'us-central1', 'us-east1'). + :param feature_view_sync_name: The full resource name of the feature view + sync operation to retrieve. + """ + client = self.get_feature_online_store_admin_service_client(location) + + try: + response = client.get_feature_view_sync(name=feature_view_sync_name) + + report = { + "name": feature_view_sync_name, + "start_time": int(response.run_time.start_time.seconds), + } + + if hasattr(response.run_time, "end_time") and response.run_time.end_time.seconds: + report["end_time"] = int(response.run_time.end_time.seconds) + report["sync_summary"] = { + "row_synced": int(response.sync_summary.row_synced), + "total_slot": int(response.sync_summary.total_slot), + } + + return report + + except Exception as e: + self.log.error("Failed to get feature view sync: %s", str(e)) + raise AirflowException(str(e)) + + @GoogleBaseHook.fallback_to_default_project_id + def sync_feature_view( + self, + location: str, + feature_online_store_id: str, + feature_view_id: str, + project_id: str = PROVIDE_PROJECT_ID, + ) -> str: + """ + Initiate a synchronization operation for a Feature View. + + This method triggers a sync operation that updates the online serving data + for a feature view based on the latest data in the underlying batch source. + The sync operation ensures that the online feature values are up-to-date + for real-time serving. + + :param location: The Google Cloud region where the feature store is located + (e.g., 'us-central1', 'us-east1'). + :param feature_online_store_id: The ID of the online feature store that + contains the feature view to be synchronized. + :param feature_view_id: The ID of the feature view to synchronize. + :param project_id: The ID of the Google Cloud project that contains the + feature store. If not provided, will attempt to determine from the + environment. + """ + client = self.get_feature_online_store_admin_service_client(location) + feature_view = f"projects/{project_id}/locations/{location}/featureOnlineStores/{feature_online_store_id}/featureViews/{feature_view_id}" + + try: + response = client.sync_feature_view(feature_view=feature_view) + + return str(response.feature_view_sync) + + except Exception as e: + self.log.error("Failed to sync feature view: %s", str(e)) + raise AirflowException(str(e)) diff --git a/providers/src/airflow/providers/google/cloud/operators/vertex_ai/feature_store.py b/providers/src/airflow/providers/google/cloud/operators/vertex_ai/feature_store.py new file mode 100644 index 00000000000000..318ff25a3a9157 --- /dev/null +++ b/providers/src/airflow/providers/google/cloud/operators/vertex_ai/feature_store.py @@ -0,0 +1,163 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""This module contains Google Vertex AI Feature Store operators.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any + +from airflow.providers.google.cloud.hooks.vertex_ai.feature_store import FeatureStoreHook +from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator + +if TYPE_CHECKING: + from airflow.utils.context import Context + + +class SyncFeatureViewOperator(GoogleCloudBaseOperator): + """ + Initiate a synchronization operation for a Feature View in Vertex AI Feature Store. + + This operator triggers a sync operation that updates the online serving data for a feature view + based on the latest data in the underlying batch source. The sync operation ensures that + the online feature values are up-to-date for real-time serving. + + :param project_id: Required. The ID of the Google Cloud project that contains the feature store. + This is used to identify which project's resources to interact with. + :param location: Required. The location of the feature store (e.g., 'us-central1', 'us-east1'). + This specifies the Google Cloud region where the feature store resources are located. + :param feature_online_store_id: Required. The ID of the online feature store that contains + the feature view to be synchronized. This store serves as the online serving layer. + :param feature_view_id: Required. The ID of the feature view to synchronize. This identifies + the specific view that needs to have its online values updated from the batch source. + :param gcp_conn_id: The connection ID to use for connecting to Google Cloud Platform. + Defaults to 'google_cloud_default'. + :param impersonation_chain: Optional service account to impersonate using short-term + credentials. Can be either a single account or a chain of accounts required to + get the access_token of the last account in the list, which will be impersonated + in the request. If set as a string, the account must grant the originating account + the Service Account Token Creator IAM role. If set as a sequence, the identities + from the list must grant Service Account Token Creator IAM role to the directly + preceding identity, with first account from the list granting this role to the + originating account. + """ + + template_fields: Sequence[str] = ( + "project_id", + "location", + "feature_online_store_id", + "feature_view_id", + ) + + def __init__( + self, + *, + project_id: str, + location: str, + feature_online_store_id: str, + feature_view_id: str, + gcp_conn_id: str = "google_cloud_default", + impersonation_chain: str | Sequence[str] | None = None, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.project_id = project_id + self.location = location + self.feature_online_store_id = feature_online_store_id + self.feature_view_id = feature_view_id + self.gcp_conn_id = gcp_conn_id + self.impersonation_chain = impersonation_chain + + def execute(self, context: Context) -> str: + """Execute the feature view sync operation.""" + self.hook = FeatureStoreHook( + gcp_conn_id=self.gcp_conn_id, + impersonation_chain=self.impersonation_chain, + ) + self.log.info("Submitting Feature View sync job now...") + response = self.hook.sync_feature_view( + project_id=self.project_id, + location=self.location, + feature_online_store_id=self.feature_online_store_id, + feature_view_id=self.feature_view_id, + ) + self.log.info("Retrieved Feature View sync: %s", response) + + return response + + +class GetFeatureViewSyncOperator(GoogleCloudBaseOperator): + """ + Retrieve the status and details of a Feature View synchronization operation. + + This operator fetches information about a specific feature view sync operation, + including its current status, timing information, and synchronization metrics. + It's typically used to monitor the progress of a sync operation initiated by + the SyncFeatureViewOperator. + + :param location: Required. The location of the feature store (e.g., 'us-central1', 'us-east1'). + This specifies the Google Cloud region where the feature store resources are located. + :param feature_view_sync_name: Required. The full resource name of the feature view + sync operation to retrieve. This is typically the return value from a + SyncFeatureViewOperator execution. + :param gcp_conn_id: The connection ID to use for connecting to Google Cloud Platform. + Defaults to 'google_cloud_default'. + :param impersonation_chain: Optional service account to impersonate using short-term + credentials. Can be either a single account or a chain of accounts required to + get the access_token of the last account in the list, which will be impersonated + in the request. If set as a string, the account must grant the originating account + the Service Account Token Creator IAM role. If set as a sequence, the identities + from the list must grant Service Account Token Creator IAM role to the directly + preceding identity, with first account from the list granting this role to the + originating account. + """ + + template_fields: Sequence[str] = ( + "location", + "feature_view_sync_name", + ) + + def __init__( + self, + *, + location: str, + feature_view_sync_name: str, + gcp_conn_id: str = "google_cloud_default", + impersonation_chain: str | Sequence[str] | None = None, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.location = location + self.feature_view_sync_name = feature_view_sync_name + self.gcp_conn_id = gcp_conn_id + self.impersonation_chain = impersonation_chain + + def execute(self, context: Context) -> dict[str, Any]: + """Execute the get feature view sync operation.""" + self.hook = FeatureStoreHook( + gcp_conn_id=self.gcp_conn_id, + impersonation_chain=self.impersonation_chain, + ) + self.log.info("Retrieving Feature View sync job now...") + response = self.hook.get_feature_view_sync( + location=self.location, feature_view_sync_name=self.feature_view_sync_name + ) + self.log.info("Retrieved Feature View sync: %s", self.feature_view_sync_name) + self.log.info(response) + + return response diff --git a/providers/src/airflow/providers/google/cloud/sensors/vertex_ai/__init__.py b/providers/src/airflow/providers/google/cloud/sensors/vertex_ai/__init__.py new file mode 100644 index 00000000000000..13a83393a9124b --- /dev/null +++ b/providers/src/airflow/providers/google/cloud/sensors/vertex_ai/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/src/airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py b/providers/src/airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py new file mode 100644 index 00000000000000..88503db2e0cb6b --- /dev/null +++ b/providers/src/airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""This module contains a Vertex AI Feature Store sensor.""" + +from __future__ import annotations + +import time +from collections.abc import Sequence +from typing import TYPE_CHECKING + +from airflow.exceptions import AirflowException +from airflow.providers.google.cloud.hooks.vertex_ai.feature_store import FeatureStoreHook +from airflow.sensors.base import BaseSensorOperator + +if TYPE_CHECKING: + from airflow.utils.context import Context + + +class FeatureViewSyncSensor(BaseSensorOperator): + """ + Sensor to monitor the state of a Vertex AI Feature View sync operation. + + :param feature_view_sync_name: The name of the feature view sync operation to monitor. (templated) + :param location: Required. The Cloud region in which to handle the request. (templated) + :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform. + :param wait_timeout: How many seconds to wait for sync to complete. + :param impersonation_chain: Optional service account to impersonate using short-term + credentials. + """ + + template_fields: Sequence[str] = ("location", "feature_view_sync_name") + ui_color = "#f0eee4" + + def __init__( + self, + *, + feature_view_sync_name: str, + location: str, + gcp_conn_id: str = "google_cloud_default", + wait_timeout: int | None = None, + impersonation_chain: str | Sequence[str] | None = None, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.feature_view_sync_name = feature_view_sync_name + self.location = location + self.gcp_conn_id = gcp_conn_id + self.wait_timeout = wait_timeout + self.impersonation_chain = impersonation_chain + self.start_sensor_time: float | None = None + + def execute(self, context: Context) -> None: + self.start_sensor_time = time.monotonic() + super().execute(context) + + def _duration(self): + return time.monotonic() - self.start_sensor_time + + def poke(self, context: Context) -> bool: + hook = FeatureStoreHook( + gcp_conn_id=self.gcp_conn_id, + impersonation_chain=self.impersonation_chain, + ) + + try: + response = hook.get_feature_view_sync( + location=self.location, + feature_view_sync_name=self.feature_view_sync_name, + ) + + # Check if the sync has completed by verifying end_time exists + if response.get("end_time", 0) > 0: + self.log.info( + "Feature View sync %s completed. Rows synced: %d, Total slots: %d", + self.feature_view_sync_name, + int(response.get("sync_summary", "").get("row_synced", "")), + int(response.get("sync_summary", "").get("total_slot", "")), + ) + return True + + if self.wait_timeout and self._duration() > self.wait_timeout: + raise AirflowException( + f"Timeout: Feature View sync {self.feature_view_sync_name} " + f"not completed after {self.wait_timeout}s" + ) + + self.log.info("Waiting for Feature View sync %s to complete.", self.feature_view_sync_name) + return False + + except Exception as e: + if self.wait_timeout and self._duration() > self.wait_timeout: + raise AirflowException( + f"Timeout: Feature View sync {self.feature_view_sync_name} " + f"not completed after {self.wait_timeout}s" + ) + self.log.info("Error checking sync status, will retry: %s", str(e)) + return False diff --git a/providers/src/airflow/providers/google/provider.yaml b/providers/src/airflow/providers/google/provider.yaml index 61fd6f9b98a740..442e1cecccce94 100644 --- a/providers/src/airflow/providers/google/provider.yaml +++ b/providers/src/airflow/providers/google/provider.yaml @@ -100,7 +100,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-compat>=1.2.1 - apache-airflow-providers-common-sql>=1.20.0 - asgiref>=3.5.2 @@ -118,7 +118,7 @@ dependencies: - google-api-python-client>=2.0.2 - google-auth>=2.29.0 - google-auth-httplib2>=0.0.1 - - google-cloud-aiplatform>=1.70.0 + - google-cloud-aiplatform>=1.73.0 - google-cloud-automl>=2.12.0 # Excluded versions contain bug https://github.com/apache/airflow/issues/39541 which is resolved in 3.24.0 - google-cloud-bigquery>=3.4.0,!=3.21.*,!=3.22.0,!=3.23.* @@ -693,6 +693,7 @@ operators: - airflow.providers.google.cloud.operators.vertex_ai.model_service - airflow.providers.google.cloud.operators.vertex_ai.pipeline_job - airflow.providers.google.cloud.operators.vertex_ai.generative_model + - airflow.providers.google.cloud.operators.vertex_ai.feature_store - integration-name: Google Looker python-modules: - airflow.providers.google.cloud.operators.looker @@ -743,6 +744,9 @@ sensors: - integration-name: Google Cloud Pub/Sub python-modules: - airflow.providers.google.cloud.sensors.pubsub + - integration-name: Google Vertex AI + python-modules: + - airflow.providers.google.cloud.sensors.vertex_ai.feature_store - integration-name: Google Cloud Workflows python-modules: - airflow.providers.google.cloud.sensors.workflows @@ -963,6 +967,7 @@ hooks: - airflow.providers.google.cloud.hooks.vertex_ai.pipeline_job - airflow.providers.google.cloud.hooks.vertex_ai.generative_model - airflow.providers.google.cloud.hooks.vertex_ai.prediction_service + - airflow.providers.google.cloud.hooks.vertex_ai.feature_store - integration-name: Google Looker python-modules: - airflow.providers.google.cloud.hooks.looker diff --git a/providers/src/airflow/providers/grpc/__init__.py b/providers/src/airflow/providers/grpc/__init__.py index 0806e492d06c6f..cba0a87726d724 100644 --- a/providers/src/airflow/providers/grpc/__init__.py +++ b/providers/src/airflow/providers/grpc/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.6.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-grpc:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-grpc:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/grpc/provider.yaml b/providers/src/airflow/providers/grpc/provider.yaml index 66d44073114bb7..1ed98fc8318464 100644 --- a/providers/src/airflow/providers/grpc/provider.yaml +++ b/providers/src/airflow/providers/grpc/provider.yaml @@ -46,7 +46,7 @@ versions: - 1.0.1 - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 # Google has very clear rules on what dependencies should be used. All the limits below # follow strict guidelines of Google Libraries as quoted here: # While this issue is open, dependents of google-api-core, google-cloud-core. and google-auth diff --git a/providers/src/airflow/providers/hashicorp/__init__.py b/providers/src/airflow/providers/hashicorp/__init__.py index 9ae09d6f00b356..d451667b27bb99 100644 --- a/providers/src/airflow/providers/hashicorp/__init__.py +++ b/providers/src/airflow/providers/hashicorp/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.8.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-hashicorp:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-hashicorp:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/hashicorp/provider.yaml b/providers/src/airflow/providers/hashicorp/provider.yaml index 74a979c20e214f..af3a4fa80d66d3 100644 --- a/providers/src/airflow/providers/hashicorp/provider.yaml +++ b/providers/src/airflow/providers/hashicorp/provider.yaml @@ -56,7 +56,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - hvac>=1.1.0 integrations: diff --git a/providers/src/airflow/providers/http/__init__.py b/providers/src/airflow/providers/http/__init__.py index fc8e23ba0591c4..1fb4509d959f05 100644 --- a/providers/src/airflow/providers/http/__init__.py +++ b/providers/src/airflow/providers/http/__init__.py @@ -32,8 +32,8 @@ __version__ = "4.13.3" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-http:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-http:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/http/provider.yaml b/providers/src/airflow/providers/http/provider.yaml index 2714fdf1539fd6..740b0836698243 100644 --- a/providers/src/airflow/providers/http/provider.yaml +++ b/providers/src/airflow/providers/http/provider.yaml @@ -63,7 +63,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 # The 2.26.0 release of requests got rid of the chardet LGPL mandatory dependency, allowing us to # release it as a requirement for airflow - requests>=2.27.0,<3 diff --git a/providers/src/airflow/providers/imap/__init__.py b/providers/src/airflow/providers/imap/__init__.py index b4b17d91c6ce36..5fa1a1b03d5073 100644 --- a/providers/src/airflow/providers/imap/__init__.py +++ b/providers/src/airflow/providers/imap/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.7.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-imap:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-imap:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/imap/provider.yaml b/providers/src/airflow/providers/imap/provider.yaml index 988a29341f8769..99783e4161b292 100644 --- a/providers/src/airflow/providers/imap/provider.yaml +++ b/providers/src/airflow/providers/imap/provider.yaml @@ -51,7 +51,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 integrations: - integration-name: Internet Message Access Protocol (IMAP) diff --git a/providers/src/airflow/providers/influxdb/__init__.py b/providers/src/airflow/providers/influxdb/__init__.py index 6bbd2cb92dab24..a343bd88fca946 100644 --- a/providers/src/airflow/providers/influxdb/__init__.py +++ b/providers/src/airflow/providers/influxdb/__init__.py @@ -32,8 +32,8 @@ __version__ = "2.7.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-influxdb:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-influxdb:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/influxdb/provider.yaml b/providers/src/airflow/providers/influxdb/provider.yaml index 723f56b350bae1..2539f2479a07a7 100644 --- a/providers/src/airflow/providers/influxdb/provider.yaml +++ b/providers/src/airflow/providers/influxdb/provider.yaml @@ -24,7 +24,7 @@ description: | `InfluxDB `__ dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - influxdb-client>=1.19.0 - requests>=2.27.0,<3 diff --git a/providers/src/airflow/providers/jdbc/__init__.py b/providers/src/airflow/providers/jdbc/__init__.py index fd6e33f7745a07..96f6c2acc555ff 100644 --- a/providers/src/airflow/providers/jdbc/__init__.py +++ b/providers/src/airflow/providers/jdbc/__init__.py @@ -32,8 +32,8 @@ __version__ = "4.5.3" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-jdbc:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-jdbc:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/jdbc/provider.yaml b/providers/src/airflow/providers/jdbc/provider.yaml index 28f746afb1c5ac..adc894031417cf 100644 --- a/providers/src/airflow/providers/jdbc/provider.yaml +++ b/providers/src/airflow/providers/jdbc/provider.yaml @@ -55,7 +55,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-sql>=1.20.0 - jaydebeapi>=1.1.1 diff --git a/providers/src/airflow/providers/jenkins/__init__.py b/providers/src/airflow/providers/jenkins/__init__.py index 5ac32f8289d4cc..5ac72623bb9f7a 100644 --- a/providers/src/airflow/providers/jenkins/__init__.py +++ b/providers/src/airflow/providers/jenkins/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.7.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-jenkins:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-jenkins:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/jenkins/provider.yaml b/providers/src/airflow/providers/jenkins/provider.yaml index 7e20c041c18c21..ddc663264df31f 100644 --- a/providers/src/airflow/providers/jenkins/provider.yaml +++ b/providers/src/airflow/providers/jenkins/provider.yaml @@ -54,7 +54,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - python-jenkins>=1.0.0 integrations: diff --git a/providers/src/airflow/providers/microsoft/azure/__init__.py b/providers/src/airflow/providers/microsoft/azure/__init__.py index 12b4fec787cfa6..1bb574d0798025 100644 --- a/providers/src/airflow/providers/microsoft/azure/__init__.py +++ b/providers/src/airflow/providers/microsoft/azure/__init__.py @@ -32,8 +32,8 @@ __version__ = "11.1.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-microsoft-azure:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-microsoft-azure:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/microsoft/azure/provider.yaml b/providers/src/airflow/providers/microsoft/azure/provider.yaml index 6b54d6fdac701f..8db33d42ff3baf 100644 --- a/providers/src/airflow/providers/microsoft/azure/provider.yaml +++ b/providers/src/airflow/providers/microsoft/azure/provider.yaml @@ -86,7 +86,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - adlfs>=2023.10.0 - azure-batch>=8.0.0 - azure-cosmos>=4.6.0 diff --git a/providers/src/airflow/providers/microsoft/mssql/__init__.py b/providers/src/airflow/providers/microsoft/mssql/__init__.py index 66a28651c10a1a..21ddcb9338c235 100644 --- a/providers/src/airflow/providers/microsoft/mssql/__init__.py +++ b/providers/src/airflow/providers/microsoft/mssql/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.9.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-microsoft-mssql:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-microsoft-mssql:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/microsoft/mssql/provider.yaml b/providers/src/airflow/providers/microsoft/mssql/provider.yaml index de96f1500f7458..2be4edf0852262 100644 --- a/providers/src/airflow/providers/microsoft/mssql/provider.yaml +++ b/providers/src/airflow/providers/microsoft/mssql/provider.yaml @@ -56,7 +56,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-sql>=1.20.0 - pymssql>=2.3.0 # The methodtools dependency can be removed with min airflow version >=2.9.1 diff --git a/providers/src/airflow/providers/microsoft/psrp/__init__.py b/providers/src/airflow/providers/microsoft/psrp/__init__.py index 2b06cb0e15bcd6..6a65db8ea4201a 100644 --- a/providers/src/airflow/providers/microsoft/psrp/__init__.py +++ b/providers/src/airflow/providers/microsoft/psrp/__init__.py @@ -32,8 +32,8 @@ __version__ = "2.8.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-microsoft-psrp:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-microsoft-psrp:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/microsoft/psrp/provider.yaml b/providers/src/airflow/providers/microsoft/psrp/provider.yaml index ae29eb92fa2032..d089c55ce06c8d 100644 --- a/providers/src/airflow/providers/microsoft/psrp/provider.yaml +++ b/providers/src/airflow/providers/microsoft/psrp/provider.yaml @@ -49,7 +49,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - pypsrp>=0.8.0 integrations: diff --git a/providers/src/airflow/providers/microsoft/winrm/__init__.py b/providers/src/airflow/providers/microsoft/winrm/__init__.py index c940e194d14579..2fcfc1ceb8602e 100644 --- a/providers/src/airflow/providers/microsoft/winrm/__init__.py +++ b/providers/src/airflow/providers/microsoft/winrm/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.6.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-microsoft-winrm:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-microsoft-winrm:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/microsoft/winrm/provider.yaml b/providers/src/airflow/providers/microsoft/winrm/provider.yaml index bbab2a63c62b76..530b469ecf2af5 100644 --- a/providers/src/airflow/providers/microsoft/winrm/provider.yaml +++ b/providers/src/airflow/providers/microsoft/winrm/provider.yaml @@ -49,7 +49,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - pywinrm>=0.4 integrations: diff --git a/providers/src/airflow/providers/mongo/__init__.py b/providers/src/airflow/providers/mongo/__init__.py index ff762b502b2688..f969d460250f56 100644 --- a/providers/src/airflow/providers/mongo/__init__.py +++ b/providers/src/airflow/providers/mongo/__init__.py @@ -32,8 +32,8 @@ __version__ = "4.2.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-mongo:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-mongo:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/mongo/provider.yaml b/providers/src/airflow/providers/mongo/provider.yaml index db3d1fc578b4f1..9dc63803a9ce9e 100644 --- a/providers/src/airflow/providers/mongo/provider.yaml +++ b/providers/src/airflow/providers/mongo/provider.yaml @@ -53,7 +53,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - dnspython>=1.13.0 - pymongo>=4.0.0 diff --git a/providers/src/airflow/providers/mysql/__init__.py b/providers/src/airflow/providers/mysql/__init__.py index a8a39577cd56d9..7016bc25585594 100644 --- a/providers/src/airflow/providers/mysql/__init__.py +++ b/providers/src/airflow/providers/mysql/__init__.py @@ -32,8 +32,8 @@ __version__ = "5.7.4" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-mysql:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-mysql:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/mysql/provider.yaml b/providers/src/airflow/providers/mysql/provider.yaml index fae601addd3def..8c58c5023464c9 100644 --- a/providers/src/airflow/providers/mysql/provider.yaml +++ b/providers/src/airflow/providers/mysql/provider.yaml @@ -69,7 +69,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-sql>=1.20.0 # The mysqlclient package creates friction when installing on MacOS as it needs pkg-config to # Install and compile, and it's really only used by MySQL provider, so we can skip it on MacOS diff --git a/providers/src/airflow/providers/neo4j/__init__.py b/providers/src/airflow/providers/neo4j/__init__.py index 99d14644d4a56c..55c5a37a997804 100644 --- a/providers/src/airflow/providers/neo4j/__init__.py +++ b/providers/src/airflow/providers/neo4j/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.7.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-neo4j:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-neo4j:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/neo4j/provider.yaml b/providers/src/airflow/providers/neo4j/provider.yaml index 990bb45b514a00..d283dd7d4e224d 100644 --- a/providers/src/airflow/providers/neo4j/provider.yaml +++ b/providers/src/airflow/providers/neo4j/provider.yaml @@ -49,7 +49,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - neo4j>=4.2.1 integrations: diff --git a/providers/src/airflow/providers/odbc/__init__.py b/providers/src/airflow/providers/odbc/__init__.py index 370a73a68f50e3..dedcd26d337ef4 100644 --- a/providers/src/airflow/providers/odbc/__init__.py +++ b/providers/src/airflow/providers/odbc/__init__.py @@ -32,8 +32,8 @@ __version__ = "4.8.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-odbc:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-odbc:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/odbc/provider.yaml b/providers/src/airflow/providers/odbc/provider.yaml index 6c8d16fcc3f87f..9c29ac86f176ce 100644 --- a/providers/src/airflow/providers/odbc/provider.yaml +++ b/providers/src/airflow/providers/odbc/provider.yaml @@ -56,7 +56,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-sql>=1.20.0 - pyodbc>=5.0.0 diff --git a/providers/src/airflow/providers/openai/__init__.py b/providers/src/airflow/providers/openai/__init__.py index df5afee767e0a9..db176bb5501aa7 100644 --- a/providers/src/airflow/providers/openai/__init__.py +++ b/providers/src/airflow/providers/openai/__init__.py @@ -32,8 +32,8 @@ __version__ = "1.4.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-openai:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-openai:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/openai/provider.yaml b/providers/src/airflow/providers/openai/provider.yaml index 3e8bcb9e886730..2f32f9aff999cc 100644 --- a/providers/src/airflow/providers/openai/provider.yaml +++ b/providers/src/airflow/providers/openai/provider.yaml @@ -45,7 +45,7 @@ integrations: tags: [software] dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - openai[datalib]>=1.32.0 hooks: diff --git a/providers/src/airflow/providers/openfaas/__init__.py b/providers/src/airflow/providers/openfaas/__init__.py index 23c23fb4876fb3..53bd197c57285f 100644 --- a/providers/src/airflow/providers/openfaas/__init__.py +++ b/providers/src/airflow/providers/openfaas/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.6.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-openfaas:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-openfaas:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/openfaas/provider.yaml b/providers/src/airflow/providers/openfaas/provider.yaml index 3c647be58aaedf..505e2651ff16b0 100644 --- a/providers/src/airflow/providers/openfaas/provider.yaml +++ b/providers/src/airflow/providers/openfaas/provider.yaml @@ -43,7 +43,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 integrations: - integration-name: OpenFaaS diff --git a/providers/src/airflow/providers/openlineage/__init__.py b/providers/src/airflow/providers/openlineage/__init__.py index 50ebe2c4cd6dca..50243446130a3c 100644 --- a/providers/src/airflow/providers/openlineage/__init__.py +++ b/providers/src/airflow/providers/openlineage/__init__.py @@ -32,8 +32,8 @@ __version__ = "1.14.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-openlineage:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-openlineage:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/openlineage/provider.yaml b/providers/src/airflow/providers/openlineage/provider.yaml index c9eaa5206992b2..efe6d4d7344fc7 100644 --- a/providers/src/airflow/providers/openlineage/provider.yaml +++ b/providers/src/airflow/providers/openlineage/provider.yaml @@ -51,7 +51,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-sql>=1.20.0 - apache-airflow-providers-common-compat>=1.2.1 - attrs>=22.2 diff --git a/providers/src/airflow/providers/opensearch/__init__.py b/providers/src/airflow/providers/opensearch/__init__.py index 457d4fdeaaf701..c5366889c03d57 100644 --- a/providers/src/airflow/providers/opensearch/__init__.py +++ b/providers/src/airflow/providers/opensearch/__init__.py @@ -32,8 +32,8 @@ __version__ = "1.5.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-opensearch:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-opensearch:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/opensearch/provider.yaml b/providers/src/airflow/providers/opensearch/provider.yaml index e5b3027ebef67e..534f6c0239559e 100644 --- a/providers/src/airflow/providers/opensearch/provider.yaml +++ b/providers/src/airflow/providers/opensearch/provider.yaml @@ -36,7 +36,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - opensearch-py>=2.2.0 integrations: diff --git a/providers/src/airflow/providers/opsgenie/__init__.py b/providers/src/airflow/providers/opsgenie/__init__.py index 32fd3e62d3c917..810cd8aa124d52 100644 --- a/providers/src/airflow/providers/opsgenie/__init__.py +++ b/providers/src/airflow/providers/opsgenie/__init__.py @@ -32,8 +32,8 @@ __version__ = "5.7.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-opsgenie:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-opsgenie:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/opsgenie/provider.yaml b/providers/src/airflow/providers/opsgenie/provider.yaml index 0e6fdf9f0413dd..180930d33cdc6b 100644 --- a/providers/src/airflow/providers/opsgenie/provider.yaml +++ b/providers/src/airflow/providers/opsgenie/provider.yaml @@ -48,7 +48,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - opsgenie-sdk>=2.1.5 integrations: diff --git a/providers/src/airflow/providers/oracle/__init__.py b/providers/src/airflow/providers/oracle/__init__.py index a57bacc0c8b791..f1b34e0d352ebe 100644 --- a/providers/src/airflow/providers/oracle/__init__.py +++ b/providers/src/airflow/providers/oracle/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.12.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-oracle:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-oracle:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/oracle/provider.yaml b/providers/src/airflow/providers/oracle/provider.yaml index f883e87c7145b6..0c54d35d1905a3 100644 --- a/providers/src/airflow/providers/oracle/provider.yaml +++ b/providers/src/airflow/providers/oracle/provider.yaml @@ -61,7 +61,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-sql>=1.20.0 - oracledb>=2.0.0 diff --git a/providers/src/airflow/providers/pagerduty/__init__.py b/providers/src/airflow/providers/pagerduty/__init__.py index f0cbb3e46d7203..15f7855b37b4c3 100644 --- a/providers/src/airflow/providers/pagerduty/__init__.py +++ b/providers/src/airflow/providers/pagerduty/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.8.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-pagerduty:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-pagerduty:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/pagerduty/provider.yaml b/providers/src/airflow/providers/pagerduty/provider.yaml index de0899dcdfe8da..06920c1a8879e3 100644 --- a/providers/src/airflow/providers/pagerduty/provider.yaml +++ b/providers/src/airflow/providers/pagerduty/provider.yaml @@ -51,7 +51,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - pdpyras>=4.2.0 integrations: diff --git a/providers/src/airflow/providers/papermill/__init__.py b/providers/src/airflow/providers/papermill/__init__.py index b1e48dd9c0d98b..aff56418f1a56d 100644 --- a/providers/src/airflow/providers/papermill/__init__.py +++ b/providers/src/airflow/providers/papermill/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.8.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-papermill:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-papermill:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/papermill/provider.yaml b/providers/src/airflow/providers/papermill/provider.yaml index f793a71cc42abd..dbf98e552ee7a6 100644 --- a/providers/src/airflow/providers/papermill/provider.yaml +++ b/providers/src/airflow/providers/papermill/provider.yaml @@ -53,7 +53,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - papermill[all]>=2.6.0 - scrapbook[all]>=0.5.0 - ipykernel>=6.29.4 diff --git a/providers/src/airflow/providers/pgvector/__init__.py b/providers/src/airflow/providers/pgvector/__init__.py index fbfb782e94e1a7..db7851f54bce95 100644 --- a/providers/src/airflow/providers/pgvector/__init__.py +++ b/providers/src/airflow/providers/pgvector/__init__.py @@ -32,8 +32,8 @@ __version__ = "1.3.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-pgvector:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-pgvector:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/pgvector/provider.yaml b/providers/src/airflow/providers/pgvector/provider.yaml index eb992edf078b37..a2cf05d30af7c0 100644 --- a/providers/src/airflow/providers/pgvector/provider.yaml +++ b/providers/src/airflow/providers/pgvector/provider.yaml @@ -43,7 +43,7 @@ integrations: tags: [software] dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-postgres>=5.7.1 # setting !=0.3.0 version due to https://github.com/pgvector/pgvector-python/issues/79 # observed in 0.3.0. diff --git a/providers/src/airflow/providers/pinecone/__init__.py b/providers/src/airflow/providers/pinecone/__init__.py index 2e5eab468cd4b0..99ad18cce2c7c0 100644 --- a/providers/src/airflow/providers/pinecone/__init__.py +++ b/providers/src/airflow/providers/pinecone/__init__.py @@ -32,8 +32,8 @@ __version__ = "2.1.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-pinecone:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-pinecone:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/pinecone/provider.yaml b/providers/src/airflow/providers/pinecone/provider.yaml index 8b330fa0fa1f4e..a5155af3e65143 100644 --- a/providers/src/airflow/providers/pinecone/provider.yaml +++ b/providers/src/airflow/providers/pinecone/provider.yaml @@ -45,7 +45,7 @@ integrations: tags: [software] dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - pinecone-client>=3.1.0 hooks: diff --git a/providers/src/airflow/providers/postgres/__init__.py b/providers/src/airflow/providers/postgres/__init__.py index 9c6e5eba38a1b6..e082807f81a2af 100644 --- a/providers/src/airflow/providers/postgres/__init__.py +++ b/providers/src/airflow/providers/postgres/__init__.py @@ -32,8 +32,8 @@ __version__ = "5.14.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-postgres:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-postgres:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/postgres/provider.yaml b/providers/src/airflow/providers/postgres/provider.yaml index 7e79f85c578fd6..10d3d5bf2cf401 100644 --- a/providers/src/airflow/providers/postgres/provider.yaml +++ b/providers/src/airflow/providers/postgres/provider.yaml @@ -67,7 +67,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-sql>=1.20.0 - psycopg2-binary>=2.9.4 - asyncpg>=0.30.0 diff --git a/providers/src/airflow/providers/presto/__init__.py b/providers/src/airflow/providers/presto/__init__.py index 21003b2c152ac9..fc007f5e011b5d 100644 --- a/providers/src/airflow/providers/presto/__init__.py +++ b/providers/src/airflow/providers/presto/__init__.py @@ -32,8 +32,8 @@ __version__ = "5.7.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-presto:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-presto:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/presto/provider.yaml b/providers/src/airflow/providers/presto/provider.yaml index 1128fb54bd6cac..56f8ffcf026800 100644 --- a/providers/src/airflow/providers/presto/provider.yaml +++ b/providers/src/airflow/providers/presto/provider.yaml @@ -62,7 +62,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-sql>=1.20.0 - presto-python-client>=0.8.4 # In pandas 2.2 minimal version of the sqlalchemy is 2.0 diff --git a/providers/src/airflow/providers/qdrant/__init__.py b/providers/src/airflow/providers/qdrant/__init__.py index ebab8060774678..3354bbe99cffed 100644 --- a/providers/src/airflow/providers/qdrant/__init__.py +++ b/providers/src/airflow/providers/qdrant/__init__.py @@ -32,8 +32,8 @@ __version__ = "1.2.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-qdrant:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-qdrant:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/qdrant/provider.yaml b/providers/src/airflow/providers/qdrant/provider.yaml index 3f86123faf67f9..5a190e88b04961 100644 --- a/providers/src/airflow/providers/qdrant/provider.yaml +++ b/providers/src/airflow/providers/qdrant/provider.yaml @@ -43,7 +43,7 @@ integrations: dependencies: - qdrant_client>=1.10.1 - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 hooks: - integration-name: Qdrant diff --git a/providers/src/airflow/providers/redis/__init__.py b/providers/src/airflow/providers/redis/__init__.py index 9d71e73648c328..f19bf85e2bdb9d 100644 --- a/providers/src/airflow/providers/redis/__init__.py +++ b/providers/src/airflow/providers/redis/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.8.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-redis:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-redis:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/redis/provider.yaml b/providers/src/airflow/providers/redis/provider.yaml index 63b019f274c127..d57c020d99e32e 100644 --- a/providers/src/airflow/providers/redis/provider.yaml +++ b/providers/src/airflow/providers/redis/provider.yaml @@ -49,7 +49,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 # 5.0.2 excluded due to breaking changes which fixed in https://github.com/redis/redis-py/pull/3176 - redis>=4.5.2,!=4.5.5,!=5.0.2 diff --git a/providers/src/airflow/providers/salesforce/__init__.py b/providers/src/airflow/providers/salesforce/__init__.py index 0cf2ba8f97a1b6..3f0ca926f3078b 100644 --- a/providers/src/airflow/providers/salesforce/__init__.py +++ b/providers/src/airflow/providers/salesforce/__init__.py @@ -32,8 +32,8 @@ __version__ = "5.8.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-salesforce:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-salesforce:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/salesforce/provider.yaml b/providers/src/airflow/providers/salesforce/provider.yaml index dd5147d2ffee16..bfb4732fa3c7ae 100644 --- a/providers/src/airflow/providers/salesforce/provider.yaml +++ b/providers/src/airflow/providers/salesforce/provider.yaml @@ -58,7 +58,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - simple-salesforce>=1.0.0 # In pandas 2.2 minimal version of the sqlalchemy is 2.0 # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies diff --git a/providers/src/airflow/providers/samba/__init__.py b/providers/src/airflow/providers/samba/__init__.py index 4f4d0ee2915da5..e1761adb8ef04e 100644 --- a/providers/src/airflow/providers/samba/__init__.py +++ b/providers/src/airflow/providers/samba/__init__.py @@ -32,8 +32,8 @@ __version__ = "4.8.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-samba:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-samba:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/samba/provider.yaml b/providers/src/airflow/providers/samba/provider.yaml index 4e681228bffe30..65bc3260d7d75f 100644 --- a/providers/src/airflow/providers/samba/provider.yaml +++ b/providers/src/airflow/providers/samba/provider.yaml @@ -47,7 +47,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - smbprotocol>=1.5.0 integrations: diff --git a/providers/src/airflow/providers/segment/__init__.py b/providers/src/airflow/providers/segment/__init__.py index a51e77748eec45..3b958abf2adf22 100644 --- a/providers/src/airflow/providers/segment/__init__.py +++ b/providers/src/airflow/providers/segment/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.6.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-segment:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-segment:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/segment/provider.yaml b/providers/src/airflow/providers/segment/provider.yaml index 30c5e75beb647d..62904cfa4bbe64 100644 --- a/providers/src/airflow/providers/segment/provider.yaml +++ b/providers/src/airflow/providers/segment/provider.yaml @@ -43,7 +43,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - analytics-python>=1.2.9 integrations: diff --git a/providers/src/airflow/providers/sendgrid/__init__.py b/providers/src/airflow/providers/sendgrid/__init__.py index c6342ee4144df8..965e521dc966c6 100644 --- a/providers/src/airflow/providers/sendgrid/__init__.py +++ b/providers/src/airflow/providers/sendgrid/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.6.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-sendgrid:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-sendgrid:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/sendgrid/provider.yaml b/providers/src/airflow/providers/sendgrid/provider.yaml index 7a64ef4c2de1a3..e5cf9c8e8e0a5d 100644 --- a/providers/src/airflow/providers/sendgrid/provider.yaml +++ b/providers/src/airflow/providers/sendgrid/provider.yaml @@ -22,7 +22,7 @@ description: | `Sendgrid `__ dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - sendgrid>=6.0.0 state: ready diff --git a/providers/src/airflow/providers/sftp/__init__.py b/providers/src/airflow/providers/sftp/__init__.py index 78a423f73c0273..8d41727dda995b 100644 --- a/providers/src/airflow/providers/sftp/__init__.py +++ b/providers/src/airflow/providers/sftp/__init__.py @@ -32,8 +32,8 @@ __version__ = "4.11.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-sftp:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-sftp:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/sftp/provider.yaml b/providers/src/airflow/providers/sftp/provider.yaml index abf990cc4190e0..192ec08c6addb0 100644 --- a/providers/src/airflow/providers/sftp/provider.yaml +++ b/providers/src/airflow/providers/sftp/provider.yaml @@ -67,7 +67,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-ssh>=2.1.0 - paramiko>=2.9.0 - asyncssh>=2.12.0 diff --git a/providers/src/airflow/providers/singularity/__init__.py b/providers/src/airflow/providers/singularity/__init__.py index 366f0cc44518a5..7e1e78f6cad1cc 100644 --- a/providers/src/airflow/providers/singularity/__init__.py +++ b/providers/src/airflow/providers/singularity/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.6.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-singularity:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-singularity:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/singularity/provider.yaml b/providers/src/airflow/providers/singularity/provider.yaml index 3d36879f50ef78..92bd2958d53d46 100644 --- a/providers/src/airflow/providers/singularity/provider.yaml +++ b/providers/src/airflow/providers/singularity/provider.yaml @@ -45,7 +45,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - spython>=0.0.56 integrations: diff --git a/providers/src/airflow/providers/slack/__init__.py b/providers/src/airflow/providers/slack/__init__.py index adf18e2554884e..13ae096a699c68 100644 --- a/providers/src/airflow/providers/slack/__init__.py +++ b/providers/src/airflow/providers/slack/__init__.py @@ -32,8 +32,8 @@ __version__ = "8.9.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-slack:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-slack:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/slack/provider.yaml b/providers/src/airflow/providers/slack/provider.yaml index 732cecccd24ffa..417047673762a0 100644 --- a/providers/src/airflow/providers/slack/provider.yaml +++ b/providers/src/airflow/providers/slack/provider.yaml @@ -66,7 +66,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-sql>=1.20.0 - slack_sdk>=3.19.0 diff --git a/providers/src/airflow/providers/smtp/__init__.py b/providers/src/airflow/providers/smtp/__init__.py index ab800c4733356c..0727f6d18c67ba 100644 --- a/providers/src/airflow/providers/smtp/__init__.py +++ b/providers/src/airflow/providers/smtp/__init__.py @@ -32,8 +32,8 @@ __version__ = "1.8.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-smtp:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-smtp:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/smtp/provider.yaml b/providers/src/airflow/providers/smtp/provider.yaml index a7bc766595dc58..6510cb50cd3c5a 100644 --- a/providers/src/airflow/providers/smtp/provider.yaml +++ b/providers/src/airflow/providers/smtp/provider.yaml @@ -44,7 +44,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 integrations: - integration-name: Simple Mail Transfer Protocol (SMTP) diff --git a/providers/src/airflow/providers/snowflake/__init__.py b/providers/src/airflow/providers/snowflake/__init__.py index d0b9091e825162..d90ac13f93aeff 100644 --- a/providers/src/airflow/providers/snowflake/__init__.py +++ b/providers/src/airflow/providers/snowflake/__init__.py @@ -32,8 +32,8 @@ __version__ = "5.8.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-snowflake:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-snowflake:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/snowflake/provider.yaml b/providers/src/airflow/providers/snowflake/provider.yaml index 4009e0d6036ba2..0e28ac0b63cd62 100644 --- a/providers/src/airflow/providers/snowflake/provider.yaml +++ b/providers/src/airflow/providers/snowflake/provider.yaml @@ -80,7 +80,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-compat>=1.1.0 - apache-airflow-providers-common-sql>=1.20.0 # In pandas 2.2 minimal version of the sqlalchemy is 2.0 diff --git a/providers/src/airflow/providers/sqlite/__init__.py b/providers/src/airflow/providers/sqlite/__init__.py index e89b6638acc296..dcdedf8a7db57d 100644 --- a/providers/src/airflow/providers/sqlite/__init__.py +++ b/providers/src/airflow/providers/sqlite/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.9.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-sqlite:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-sqlite:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/sqlite/provider.yaml b/providers/src/airflow/providers/sqlite/provider.yaml index 3ab2417cf2c06f..777862fa1ddf78 100644 --- a/providers/src/airflow/providers/sqlite/provider.yaml +++ b/providers/src/airflow/providers/sqlite/provider.yaml @@ -56,7 +56,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - aiosqlite>=0.20.0 - apache-airflow-providers-common-sql>=1.20.0 diff --git a/providers/src/airflow/providers/ssh/__init__.py b/providers/src/airflow/providers/ssh/__init__.py index ca5d09b1846932..cf61ba7cc4694f 100644 --- a/providers/src/airflow/providers/ssh/__init__.py +++ b/providers/src/airflow/providers/ssh/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.14.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-ssh:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-ssh:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/ssh/provider.yaml b/providers/src/airflow/providers/ssh/provider.yaml index 4f64433717da22..85aefe90818802 100644 --- a/providers/src/airflow/providers/ssh/provider.yaml +++ b/providers/src/airflow/providers/ssh/provider.yaml @@ -64,7 +64,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - paramiko>=2.9.0 - sshtunnel>=0.3.2 diff --git a/providers/src/airflow/providers/standard/__init__.py b/providers/src/airflow/providers/standard/__init__.py index 164dc7203aa1a6..5a03aa6c194779 100644 --- a/providers/src/airflow/providers/standard/__init__.py +++ b/providers/src/airflow/providers/standard/__init__.py @@ -32,8 +32,8 @@ __version__ = "0.0.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-standard:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-standard:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/standard/provider.yaml b/providers/src/airflow/providers/standard/provider.yaml index 661f3f4f855e41..baa54c3eca291b 100644 --- a/providers/src/airflow/providers/standard/provider.yaml +++ b/providers/src/airflow/providers/standard/provider.yaml @@ -28,7 +28,7 @@ versions: - 0.0.1 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-sql>=1.20.0 integrations: diff --git a/providers/src/airflow/providers/tableau/__init__.py b/providers/src/airflow/providers/tableau/__init__.py index a3bd7cd6af747c..0714c3a896e85a 100644 --- a/providers/src/airflow/providers/tableau/__init__.py +++ b/providers/src/airflow/providers/tableau/__init__.py @@ -32,8 +32,8 @@ __version__ = "4.6.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-tableau:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-tableau:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/tableau/provider.yaml b/providers/src/airflow/providers/tableau/provider.yaml index e9aeff2f455cbf..24746353180675 100644 --- a/providers/src/airflow/providers/tableau/provider.yaml +++ b/providers/src/airflow/providers/tableau/provider.yaml @@ -54,7 +54,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - tableauserverclient>=0.25 integrations: diff --git a/providers/src/airflow/providers/telegram/__init__.py b/providers/src/airflow/providers/telegram/__init__.py index be38ca0d2fc770..f0d7effbc6bbe0 100644 --- a/providers/src/airflow/providers/telegram/__init__.py +++ b/providers/src/airflow/providers/telegram/__init__.py @@ -32,8 +32,8 @@ __version__ = "4.6.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-telegram:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-telegram:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/telegram/provider.yaml b/providers/src/airflow/providers/telegram/provider.yaml index 3bbcf3753b967a..043767d96a0772 100644 --- a/providers/src/airflow/providers/telegram/provider.yaml +++ b/providers/src/airflow/providers/telegram/provider.yaml @@ -49,7 +49,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - python-telegram-bot>=20.2 integrations: diff --git a/providers/src/airflow/providers/teradata/__init__.py b/providers/src/airflow/providers/teradata/__init__.py index 8a1af78207ad00..e75d3e2e357fd0 100644 --- a/providers/src/airflow/providers/teradata/__init__.py +++ b/providers/src/airflow/providers/teradata/__init__.py @@ -32,8 +32,8 @@ __version__ = "2.6.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-teradata:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-teradata:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/teradata/provider.yaml b/providers/src/airflow/providers/teradata/provider.yaml index 6ac23b1c340f79..b0f405b38b10e3 100644 --- a/providers/src/airflow/providers/teradata/provider.yaml +++ b/providers/src/airflow/providers/teradata/provider.yaml @@ -36,7 +36,7 @@ versions: - 2.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-sql>=1.20.0 - teradatasqlalchemy>=17.20.0.0 - teradatasql>=17.20.0.28 diff --git a/providers/src/airflow/providers/trino/__init__.py b/providers/src/airflow/providers/trino/__init__.py index 785eca4dd8f0dc..b886d97ab91456 100644 --- a/providers/src/airflow/providers/trino/__init__.py +++ b/providers/src/airflow/providers/trino/__init__.py @@ -32,8 +32,8 @@ __version__ = "5.9.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-trino:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-trino:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/trino/provider.yaml b/providers/src/airflow/providers/trino/provider.yaml index 960ce1c6c71020..57e5da90061294 100644 --- a/providers/src/airflow/providers/trino/provider.yaml +++ b/providers/src/airflow/providers/trino/provider.yaml @@ -65,7 +65,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-sql>=1.20.0 # In pandas 2.2 minimal version of the sqlalchemy is 2.0 # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies diff --git a/providers/src/airflow/providers/vertica/__init__.py b/providers/src/airflow/providers/vertica/__init__.py index 16409ae96367b7..04994d906219da 100644 --- a/providers/src/airflow/providers/vertica/__init__.py +++ b/providers/src/airflow/providers/vertica/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.9.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-vertica:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-vertica:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/vertica/provider.yaml b/providers/src/airflow/providers/vertica/provider.yaml index 8e55440be6fd18..5dcf73f4278f5e 100644 --- a/providers/src/airflow/providers/vertica/provider.yaml +++ b/providers/src/airflow/providers/vertica/provider.yaml @@ -54,7 +54,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-sql>=1.20.0 - vertica-python>=0.6.0 diff --git a/providers/src/airflow/providers/weaviate/__init__.py b/providers/src/airflow/providers/weaviate/__init__.py index 5547395b7cb54d..c0ec0ed7ed651f 100644 --- a/providers/src/airflow/providers/weaviate/__init__.py +++ b/providers/src/airflow/providers/weaviate/__init__.py @@ -32,8 +32,8 @@ __version__ = "2.1.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-weaviate:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-weaviate:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/weaviate/provider.yaml b/providers/src/airflow/providers/weaviate/provider.yaml index 8328d2c61474a6..3c3cc1c751c560 100644 --- a/providers/src/airflow/providers/weaviate/provider.yaml +++ b/providers/src/airflow/providers/weaviate/provider.yaml @@ -49,7 +49,7 @@ integrations: tags: [software] dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - httpx>=0.25.0 - weaviate-client>=4.4.0 # In pandas 2.2 minimal version of the sqlalchemy is 2.0 diff --git a/providers/src/airflow/providers/yandex/__init__.py b/providers/src/airflow/providers/yandex/__init__.py index 0cfe8989d883f9..13dfa2f97ee40b 100644 --- a/providers/src/airflow/providers/yandex/__init__.py +++ b/providers/src/airflow/providers/yandex/__init__.py @@ -32,8 +32,8 @@ __version__ = "3.12.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-yandex:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-yandex:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/yandex/provider.yaml b/providers/src/airflow/providers/yandex/provider.yaml index 000781f28de13f..74ed85a2cf065a 100644 --- a/providers/src/airflow/providers/yandex/provider.yaml +++ b/providers/src/airflow/providers/yandex/provider.yaml @@ -52,7 +52,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - yandexcloud>=0.308.0 - yandex-query-client>=0.1.4 diff --git a/providers/src/airflow/providers/ydb/__init__.py b/providers/src/airflow/providers/ydb/__init__.py index 4646e586805f30..7b678744a76f3a 100644 --- a/providers/src/airflow/providers/ydb/__init__.py +++ b/providers/src/airflow/providers/ydb/__init__.py @@ -32,8 +32,8 @@ __version__ = "2.0.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-ydb:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-ydb:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/ydb/provider.yaml b/providers/src/airflow/providers/ydb/provider.yaml index b04b3a048d15b2..8b2d3040908262 100644 --- a/providers/src/airflow/providers/ydb/provider.yaml +++ b/providers/src/airflow/providers/ydb/provider.yaml @@ -33,7 +33,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - apache-airflow-providers-common-sql>=1.20.0 - ydb>=3.18.8 - ydb-dbapi>=0.1.0 diff --git a/providers/src/airflow/providers/zendesk/__init__.py b/providers/src/airflow/providers/zendesk/__init__.py index 1f4fb6efc489f0..319d02f6d17cb5 100644 --- a/providers/src/airflow/providers/zendesk/__init__.py +++ b/providers/src/airflow/providers/zendesk/__init__.py @@ -32,8 +32,8 @@ __version__ = "4.8.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.8.0" + "2.9.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-zendesk:{__version__}` needs Apache Airflow 2.8.0+" + f"The package `apache-airflow-providers-zendesk:{__version__}` needs Apache Airflow 2.9.0+" ) diff --git a/providers/src/airflow/providers/zendesk/provider.yaml b/providers/src/airflow/providers/zendesk/provider.yaml index eb552c323bbbdc..645b4e056b8593 100644 --- a/providers/src/airflow/providers/zendesk/provider.yaml +++ b/providers/src/airflow/providers/zendesk/provider.yaml @@ -47,7 +47,7 @@ versions: - 1.0.0 dependencies: - - apache-airflow>=2.8.0 + - apache-airflow>=2.9.0 - zenpy>=2.0.40 integrations: diff --git a/providers/tests/google/cloud/hooks/test_dataproc.py b/providers/tests/google/cloud/hooks/test_dataproc.py index b15d7855454e97..1e0349d8bfdcd4 100644 --- a/providers/tests/google/cloud/hooks/test_dataproc.py +++ b/providers/tests/google/cloud/hooks/test_dataproc.py @@ -1097,9 +1097,16 @@ def test_add_args(self): assert args == self.builder.job["job"][self.job_type]["args"] def test_add_query(self): - query = ["query"] - self.builder.add_query(query) - assert self.builder.job["job"][self.job_type]["query_list"] == {"queries": [query]} + query1 = "query1" + self.builder.add_query(query1) + query2 = "query2" + self.builder.add_query(query2) + assert self.builder.job["job"][self.job_type]["query_list"] == {"queries": [query1, query2]} + new_queries = ["query3", "query4"] + self.builder.add_query(new_queries) + assert self.builder.job["job"][self.job_type]["query_list"] == { + "queries": [query1, query2] + new_queries + } def test_add_query_uri(self): query_uri = "query_uri" diff --git a/providers/tests/google/cloud/hooks/vertex_ai/test_feature_store.py b/providers/tests/google/cloud/hooks/vertex_ai/test_feature_store.py new file mode 100644 index 00000000000000..aff2a9612bf0ae --- /dev/null +++ b/providers/tests/google/cloud/hooks/vertex_ai/test_feature_store.py @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from unittest import mock + +from airflow.providers.google.cloud.hooks.vertex_ai.feature_store import FeatureStoreHook + +from providers.tests.google.cloud.utils.base_gcp_mock import ( + mock_base_gcp_hook_default_project_id, +) + +BASE_STRING = "airflow.providers.google.common.hooks.base_google.{}" +FEATURE_STORE_STRING = "airflow.providers.google.cloud.hooks.vertex_ai.feature_store.{}" + +TEST_GCP_CONN_ID = "test-gcp-conn-id" +TEST_PROJECT_ID = "test-project" +TEST_LOCATION = "us-central1" +TEST_FEATURE_ONLINE_STORE_ID = "test-store" +TEST_FEATURE_VIEW_ID = "test-view" +TEST_FEATURE_VIEW = f"projects/{TEST_PROJECT_ID}/locations/{TEST_LOCATION}/featureOnlineStores/{TEST_FEATURE_ONLINE_STORE_ID}/featureViews/{TEST_FEATURE_VIEW_ID}" +TEST_FEATURE_VIEW_SYNC_NAME = f"{TEST_FEATURE_VIEW}/featureViewSyncs/sync123" + + +class TestFeatureStoreHook: + def setup_method(self): + with mock.patch( + BASE_STRING.format("GoogleBaseHook.__init__"), new=mock_base_gcp_hook_default_project_id + ): + self.hook = FeatureStoreHook(gcp_conn_id=TEST_GCP_CONN_ID) + + @mock.patch(FEATURE_STORE_STRING.format("FeatureOnlineStoreAdminServiceClient"), autospec=True) + @mock.patch(BASE_STRING.format("GoogleBaseHook.get_credentials")) + def test_get_feature_online_store_admin_service_client(self, mock_get_credentials, mock_client): + self.hook.get_feature_online_store_admin_service_client(location=TEST_LOCATION) + mock_client.assert_called_once_with( + credentials=mock_get_credentials.return_value, client_info=mock.ANY, client_options=mock.ANY + ) + client_options = mock_client.call_args[1]["client_options"] + assert client_options.api_endpoint == f"{TEST_LOCATION}-aiplatform.googleapis.com:443" + + mock_client.reset_mock() + self.hook.get_feature_online_store_admin_service_client() + mock_client.assert_called_once_with( + credentials=mock_get_credentials.return_value, client_info=mock.ANY, client_options=mock.ANY + ) + client_options = mock_client.call_args[1]["client_options"] + assert not client_options.api_endpoint + + @mock.patch(FEATURE_STORE_STRING.format("FeatureStoreHook.get_feature_online_store_admin_service_client")) + def test_get_feature_view_sync(self, mock_client_getter): + mock_client = mock.MagicMock() + mock_client_getter.return_value = mock_client + + # Create a mock response with the expected structure + mock_response = mock.MagicMock() + mock_response.run_time.start_time.seconds = 1 + mock_response.run_time.end_time.seconds = 1 + mock_response.sync_summary.row_synced = 1 + mock_response.sync_summary.total_slot = 1 + + mock_client.get_feature_view_sync.return_value = mock_response + + expected_result = { + "name": TEST_FEATURE_VIEW_SYNC_NAME, + "start_time": 1, + "end_time": 1, + "sync_summary": {"row_synced": 1, "total_slot": 1}, + } + + result = self.hook.get_feature_view_sync( + location=TEST_LOCATION, + feature_view_sync_name=TEST_FEATURE_VIEW_SYNC_NAME, + ) + + mock_client.get_feature_view_sync.assert_called_once_with(name=TEST_FEATURE_VIEW_SYNC_NAME) + assert result == expected_result + + @mock.patch(FEATURE_STORE_STRING.format("FeatureStoreHook.get_feature_online_store_admin_service_client")) + def test_sync_feature_view(self, mock_client_getter): + mock_client = mock.MagicMock() + mock_client_getter.return_value = mock_client + + # Create a mock response with the expected structure + mock_response = mock.MagicMock() + mock_response.feature_view_sync = "test-sync-operation-name" + mock_client.sync_feature_view.return_value = mock_response + + result = self.hook.sync_feature_view( + project_id=TEST_PROJECT_ID, + location=TEST_LOCATION, + feature_online_store_id=TEST_FEATURE_ONLINE_STORE_ID, + feature_view_id=TEST_FEATURE_VIEW_ID, + ) + + mock_client.sync_feature_view.assert_called_once_with(feature_view=TEST_FEATURE_VIEW) + assert result == "test-sync-operation-name" diff --git a/providers/tests/google/cloud/operators/vertex_ai/test_feature_store.py b/providers/tests/google/cloud/operators/vertex_ai/test_feature_store.py new file mode 100644 index 00000000000000..5340b69d720092 --- /dev/null +++ b/providers/tests/google/cloud/operators/vertex_ai/test_feature_store.py @@ -0,0 +1,119 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from unittest import mock + +from airflow.providers.google.cloud.operators.vertex_ai.feature_store import ( + GetFeatureViewSyncOperator, + SyncFeatureViewOperator, +) + +VERTEX_AI_PATH = "airflow.providers.google.cloud.operators.vertex_ai.{}" + +TASK_ID = "test_task_id" +GCP_PROJECT = "test-project" +GCP_LOCATION = "us-central1" +GCP_CONN_ID = "test-conn" +IMPERSONATION_CHAIN = ["ACCOUNT_1", "ACCOUNT_2", "ACCOUNT_3"] +FEATURE_ONLINE_STORE_ID = "test-store" +FEATURE_VIEW_ID = "test-view" +FEATURE_VIEW_SYNC_NAME = f"projects/{GCP_PROJECT}/locations/{GCP_LOCATION}/featureOnlineStores/{FEATURE_ONLINE_STORE_ID}/featureViews/{FEATURE_VIEW_ID}/featureViewSyncs/sync123" + + +class TestSyncFeatureViewOperator: + @mock.patch(VERTEX_AI_PATH.format("feature_store.FeatureStoreHook")) + def test_execute(self, mock_hook_class): + # Create the mock hook and set up its return value + mock_hook = mock.MagicMock() + mock_hook_class.return_value = mock_hook + + # Set up the return value for sync_feature_view to match the hook implementation + mock_hook.sync_feature_view.return_value = FEATURE_VIEW_SYNC_NAME + + op = SyncFeatureViewOperator( + task_id=TASK_ID, + project_id=GCP_PROJECT, + location=GCP_LOCATION, + feature_online_store_id=FEATURE_ONLINE_STORE_ID, + feature_view_id=FEATURE_VIEW_ID, + gcp_conn_id=GCP_CONN_ID, + impersonation_chain=IMPERSONATION_CHAIN, + ) + + response = op.execute(context={"ti": mock.MagicMock()}) + + # Verify hook initialization + mock_hook_class.assert_called_once_with( + gcp_conn_id=GCP_CONN_ID, + impersonation_chain=IMPERSONATION_CHAIN, + ) + + # Verify hook method call + mock_hook.sync_feature_view.assert_called_once_with( + project_id=GCP_PROJECT, + location=GCP_LOCATION, + feature_online_store_id=FEATURE_ONLINE_STORE_ID, + feature_view_id=FEATURE_VIEW_ID, + ) + + # Verify response matches expected value + assert response == FEATURE_VIEW_SYNC_NAME + + +class TestGetFeatureViewSyncOperator: + @mock.patch(VERTEX_AI_PATH.format("feature_store.FeatureStoreHook")) + def test_execute(self, mock_hook_class): + # Create the mock hook and set up expected response + mock_hook = mock.MagicMock() + mock_hook_class.return_value = mock_hook + + expected_response = { + "name": FEATURE_VIEW_SYNC_NAME, + "start_time": 1000, + "end_time": 2000, + "sync_summary": {"row_synced": 500, "total_slot": 4}, + } + + # Set up the return value for get_feature_view_sync to match the hook implementation + mock_hook.get_feature_view_sync.return_value = expected_response + + op = GetFeatureViewSyncOperator( + task_id=TASK_ID, + location=GCP_LOCATION, + feature_view_sync_name=FEATURE_VIEW_SYNC_NAME, + gcp_conn_id=GCP_CONN_ID, + impersonation_chain=IMPERSONATION_CHAIN, + ) + + response = op.execute(context={"ti": mock.MagicMock()}) + + # Verify hook initialization + mock_hook_class.assert_called_once_with( + gcp_conn_id=GCP_CONN_ID, + impersonation_chain=IMPERSONATION_CHAIN, + ) + + # Verify hook method call + mock_hook.get_feature_view_sync.assert_called_once_with( + location=GCP_LOCATION, + feature_view_sync_name=FEATURE_VIEW_SYNC_NAME, + ) + + # Verify response matches expected structure + assert response == expected_response diff --git a/providers/tests/google/cloud/sensors/test_vertex_ai.py b/providers/tests/google/cloud/sensors/test_vertex_ai.py new file mode 100644 index 00000000000000..5e3171e9885100 --- /dev/null +++ b/providers/tests/google/cloud/sensors/test_vertex_ai.py @@ -0,0 +1,148 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock +from unittest.mock import Mock + +import pytest + +from airflow.exceptions import AirflowException +from airflow.providers.google.cloud.sensors.vertex_ai.feature_store import FeatureViewSyncSensor + +TASK_ID = "test-task" +GCP_CONN_ID = "test-conn" +GCP_LOCATION = "us-central1" +FEATURE_VIEW_SYNC_NAME = "projects/123/locations/us-central1/featureViews/test-view/operations/sync-123" +TIMEOUT = 120 + + +class TestFeatureViewSyncSensor: + def create_sync_response(self, end_time=None, row_synced=None, total_slot=None): + response = {} + if end_time is not None: + response["end_time"] = end_time + if row_synced is not None and total_slot is not None: + response["sync_summary"] = {"row_synced": str(row_synced), "total_slot": str(total_slot)} + return response + + @mock.patch("airflow.providers.google.cloud.sensors.vertex_ai.feature_store.FeatureStoreHook") + def test_sync_completed(self, mock_hook): + mock_hook.return_value.get_feature_view_sync.return_value = self.create_sync_response( + end_time=1234567890, row_synced=1000, total_slot=5 + ) + + sensor = FeatureViewSyncSensor( + task_id=TASK_ID, + feature_view_sync_name=FEATURE_VIEW_SYNC_NAME, + location=GCP_LOCATION, + gcp_conn_id=GCP_CONN_ID, + timeout=TIMEOUT, + ) + ret = sensor.poke(context={}) + + mock_hook.return_value.get_feature_view_sync.assert_called_once_with( + location=GCP_LOCATION, + feature_view_sync_name=FEATURE_VIEW_SYNC_NAME, + ) + assert ret + + @mock.patch("airflow.providers.google.cloud.sensors.vertex_ai.feature_store.FeatureStoreHook") + def test_sync_running(self, mock_hook): + mock_hook.return_value.get_feature_view_sync.return_value = self.create_sync_response( + end_time=0, row_synced=0, total_slot=5 + ) + + sensor = FeatureViewSyncSensor( + task_id=TASK_ID, + feature_view_sync_name=FEATURE_VIEW_SYNC_NAME, + location=GCP_LOCATION, + gcp_conn_id=GCP_CONN_ID, + timeout=TIMEOUT, + ) + ret = sensor.poke(context={}) + + mock_hook.return_value.get_feature_view_sync.assert_called_once_with( + location=GCP_LOCATION, + feature_view_sync_name=FEATURE_VIEW_SYNC_NAME, + ) + assert not ret + + @mock.patch("airflow.providers.google.cloud.sensors.vertex_ai.feature_store.FeatureStoreHook") + def test_sync_error_with_retry(self, mock_hook): + mock_hook.return_value.get_feature_view_sync.side_effect = Exception("API Error") + + sensor = FeatureViewSyncSensor( + task_id=TASK_ID, + feature_view_sync_name=FEATURE_VIEW_SYNC_NAME, + location=GCP_LOCATION, + gcp_conn_id=GCP_CONN_ID, + timeout=TIMEOUT, + ) + ret = sensor.poke(context={}) + + mock_hook.return_value.get_feature_view_sync.assert_called_once_with( + location=GCP_LOCATION, + feature_view_sync_name=FEATURE_VIEW_SYNC_NAME, + ) + assert not ret + + @mock.patch("airflow.providers.google.cloud.sensors.vertex_ai.feature_store.FeatureStoreHook") + def test_timeout_during_running(self, mock_hook): + mock_hook.return_value.get_feature_view_sync.return_value = self.create_sync_response( + end_time=0, row_synced=0, total_slot=5 + ) + + sensor = FeatureViewSyncSensor( + task_id=TASK_ID, + feature_view_sync_name=FEATURE_VIEW_SYNC_NAME, + location=GCP_LOCATION, + gcp_conn_id=GCP_CONN_ID, + timeout=TIMEOUT, + wait_timeout=300, + ) + + sensor._duration = Mock() + sensor._duration.return_value = 301 + + with pytest.raises( + AirflowException, + match=f"Timeout: Feature View sync {FEATURE_VIEW_SYNC_NAME} not completed after 300s", + ): + sensor.poke(context={}) + + @mock.patch("airflow.providers.google.cloud.sensors.vertex_ai.feature_store.FeatureStoreHook") + def test_timeout_during_error(self, mock_hook): + mock_hook.return_value.get_feature_view_sync.side_effect = Exception("API Error") + + sensor = FeatureViewSyncSensor( + task_id=TASK_ID, + feature_view_sync_name=FEATURE_VIEW_SYNC_NAME, + location=GCP_LOCATION, + gcp_conn_id=GCP_CONN_ID, + timeout=TIMEOUT, + wait_timeout=300, + ) + + sensor._duration = Mock() + sensor._duration.return_value = 301 + + with pytest.raises( + AirflowException, + match=f"Timeout: Feature View sync {FEATURE_VIEW_SYNC_NAME} not completed after 300s", + ): + sensor.poke(context={}) diff --git a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_feature_store.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_feature_store.py new file mode 100644 index 00000000000000..3d0794fa85cbd7 --- /dev/null +++ b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_feature_store.py @@ -0,0 +1,90 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Example Airflow DAG for Google Vertex AI Feature Store operations. +""" + +from __future__ import annotations + +import os +from datetime import datetime + +from airflow import DAG +from airflow.providers.google.cloud.operators.vertex_ai.feature_store import ( + GetFeatureViewSyncOperator, + SyncFeatureViewOperator, +) +from airflow.providers.google.cloud.sensors.vertex_ai.feature_store import FeatureViewSyncSensor + +PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default") +DAG_ID = "vertex_ai_feature_store_dag" +REGION = "us-central1" + +FEATURE_ONLINE_STORE_ID = "my_feature_online_store_unique" +FEATURE_VIEW_ID = "feature_view_publications" + +with DAG( + dag_id=DAG_ID, + description="Sample DAG with Vertex AI Feature Store operations.", + schedule="@once", + start_date=datetime(2024, 1, 1), + catchup=False, + tags=["example", "vertex_ai", "feature_store"], +) as dag: + # [START how_to_cloud_vertex_ai_feature_store_sync_feature_view_operator] + sync_task = SyncFeatureViewOperator( + task_id="sync_task", + project_id=PROJECT_ID, + location=REGION, + feature_online_store_id=FEATURE_ONLINE_STORE_ID, + feature_view_id=FEATURE_VIEW_ID, + ) + # [END how_to_cloud_vertex_ai_feature_store_sync_feature_view_operator] + + # [START how_to_cloud_vertex_ai_feature_store_feature_view_sync_sensor] + wait_for_sync = FeatureViewSyncSensor( + task_id="wait_for_sync", + location=REGION, + feature_view_sync_name="{{ task_instance.xcom_pull(task_ids='sync_task', key='return_value')}}", + poke_interval=60, # Check every minute + timeout=600, # Timeout after 10 minutes + mode="reschedule", + ) + # [END how_to_cloud_vertex_ai_feature_store_feature_view_sync_sensor] + + # [START how_to_cloud_vertex_ai_feature_store_get_feature_view_sync_operator] + get_task = GetFeatureViewSyncOperator( + task_id="get_task", + location=REGION, + feature_view_sync_name="{{ task_instance.xcom_pull(task_ids='sync_task', key='return_value')}}", + ) + # [END how_to_cloud_vertex_ai_feature_store_get_feature_view_sync_operator] + + sync_task >> wait_for_sync >> get_task + + from tests_common.test_utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "tearDown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/pyproject.toml b/pyproject.toml index a13fff7916a06e..73238f6e0949e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ requires = [ "GitPython==3.1.43", "gitdb==4.0.11", - "hatchling==1.26.3", + "hatchling==1.27.0", "packaging==24.2", "pathspec==0.12.1", "pluggy==1.5.0", diff --git a/scripts/ci/install_breeze.sh b/scripts/ci/install_breeze.sh index efac778b6e6e73..6a7d34d756cfc3 100755 --- a/scripts/ci/install_breeze.sh +++ b/scripts/ci/install_breeze.sh @@ -22,7 +22,7 @@ cd "$( dirname "${BASH_SOURCE[0]}" )/../../" PYTHON_ARG="" PIP_VERSION="24.3.1" -UV_VERSION="0.5.8" +UV_VERSION="0.5.9" if [[ ${PYTHON_VERSION=} != "" ]]; then PYTHON_ARG="--python=$(which python"${PYTHON_VERSION}") " fi diff --git a/scripts/ci/pre_commit/supported_versions.py b/scripts/ci/pre_commit/supported_versions.py index 93e5245d980120..8524f237dc9931 100755 --- a/scripts/ci/pre_commit/supported_versions.py +++ b/scripts/ci/pre_commit/supported_versions.py @@ -27,7 +27,7 @@ HEADERS = ("Version", "Current Patch/Minor", "State", "First Release", "Limited Support", "EOL/Terminated") SUPPORTED_VERSIONS = ( - ("2", "2.10.3", "Supported", "Dec 17, 2020", "TBD", "TBD"), + ("2", "2.10.4", "Supported", "Dec 17, 2020", "TBD", "TBD"), ("1.10", "1.10.15", "EOL", "Aug 27, 2018", "Dec 17, 2020", "June 17, 2021"), ("1.9", "1.9.0", "EOL", "Jan 03, 2018", "Aug 27, 2018", "Aug 27, 2018"), ("1.8", "1.8.2", "EOL", "Mar 19, 2017", "Jan 03, 2018", "Jan 03, 2018"), diff --git a/scripts/tools/setup_breeze b/scripts/tools/setup_breeze index cc6afcda41613b..278f1ecd9f83a8 100755 --- a/scripts/tools/setup_breeze +++ b/scripts/tools/setup_breeze @@ -27,7 +27,7 @@ COLOR_YELLOW=$'\e[33m' COLOR_BLUE=$'\e[34m' COLOR_RESET=$'\e[0m' -UV_VERSION="0.5.8" +UV_VERSION="0.5.9" function manual_instructions() { echo diff --git a/task_sdk/src/airflow/sdk/api/client.py b/task_sdk/src/airflow/sdk/api/client.py index 568eb3c90bd86d..5f08f2a6242c44 100644 --- a/task_sdk/src/airflow/sdk/api/client.py +++ b/task_sdk/src/airflow/sdk/api/client.py @@ -30,10 +30,12 @@ from airflow.sdk import __version__ from airflow.sdk.api.datamodels._generated import ( ConnectionResponse, + DagRunType, TerminalTIState, TIDeferredStatePayload, TIEnterRunningPayload, TIHeartbeatInfo, + TIRunContext, TITerminalStatePayload, ValidationError as RemoteValidationError, VariablePostBody, @@ -110,11 +112,12 @@ class TaskInstanceOperations: def __init__(self, client: Client): self.client = client - def start(self, id: uuid.UUID, pid: int, when: datetime): + def start(self, id: uuid.UUID, pid: int, when: datetime) -> TIRunContext: """Tell the API server that this TI has started running.""" body = TIEnterRunningPayload(pid=pid, hostname=get_hostname(), unixname=getuser(), start_date=when) - self.client.patch(f"task-instances/{id}/state", content=body.model_dump_json()) + resp = self.client.patch(f"task-instances/{id}/run", content=body.model_dump_json()) + return TIRunContext.model_validate_json(resp.read()) def finish(self, id: uuid.UUID, state: TerminalTIState, when: datetime): """Tell the API server that this TI has reached a terminal state.""" @@ -218,7 +221,23 @@ def auth_flow(self, request: httpx.Request): # This exists as a aid for debugging or local running via the `dry_run` argument to Client. It doesn't make # sense for returning connections etc. def noop_handler(request: httpx.Request) -> httpx.Response: - log.debug("Dry-run request", method=request.method, path=request.url.path) + path = request.url.path + log.debug("Dry-run request", method=request.method, path=path) + + if path.startswith("/task-instances/") and path.endswith("/run"): + # Return a fake context + return httpx.Response( + 200, + json={ + "dag_run": { + "dag_id": "test_dag", + "run_id": "test_run", + "logical_date": "2021-01-01T00:00:00Z", + "start_date": "2021-01-01T00:00:00Z", + "run_type": DagRunType.MANUAL, + }, + }, + ) return httpx.Response(200, json={"text": "Hello, world!"}) diff --git a/task_sdk/src/airflow/sdk/api/datamodels/_generated.py b/task_sdk/src/airflow/sdk/api/datamodels/_generated.py index 37659ffcc1be67..5a103e78fc0ffa 100644 --- a/task_sdk/src/airflow/sdk/api/datamodels/_generated.py +++ b/task_sdk/src/airflow/sdk/api/datamodels/_generated.py @@ -44,6 +44,17 @@ class ConnectionResponse(BaseModel): extra: Annotated[str | None, Field(title="Extra")] = None +class DagRunType(str, Enum): + """ + Class with DagRun types. + """ + + BACKFILL = "backfill" + SCHEDULED = "scheduled" + MANUAL = "manual" + ASSET_TRIGGERED = "asset_triggered" + + class IntermediateTIState(str, Enum): """ States that a Task Instance can be in that indicate it is not yet in a terminal or running state. @@ -159,10 +170,36 @@ class TaskInstance(BaseModel): map_index: Annotated[int | None, Field(title="Map Index")] = None +class DagRun(BaseModel): + """ + Schema for DagRun model with minimal required fields needed for Runtime. + """ + + dag_id: Annotated[str, Field(title="Dag Id")] + run_id: Annotated[str, Field(title="Run Id")] + logical_date: Annotated[datetime, Field(title="Logical Date")] + data_interval_start: Annotated[datetime | None, Field(title="Data Interval Start")] = None + data_interval_end: Annotated[datetime | None, Field(title="Data Interval End")] = None + start_date: Annotated[datetime, Field(title="Start Date")] + end_date: Annotated[datetime | None, Field(title="End Date")] = None + run_type: DagRunType + conf: Annotated[dict[str, Any] | None, Field(title="Conf")] = None + + class HTTPValidationError(BaseModel): detail: Annotated[list[ValidationError] | None, Field(title="Detail")] = None +class TIRunContext(BaseModel): + """ + Response schema for TaskInstance run context. + """ + + dag_run: DagRun + variables: Annotated[list[VariableResponse] | None, Field(title="Variables")] = None + connections: Annotated[list[ConnectionResponse] | None, Field(title="Connections")] = None + + class TITerminalStatePayload(BaseModel): """ Schema for updating TaskInstance to a terminal state (e.g., SUCCESS or FAILED). diff --git a/task_sdk/src/airflow/sdk/definitions/asset/__init__.py b/task_sdk/src/airflow/sdk/definitions/asset/__init__.py index a0beb24150f7d4..c7d3906cf2e566 100644 --- a/task_sdk/src/airflow/sdk/definitions/asset/__init__.py +++ b/task_sdk/src/airflow/sdk/definitions/asset/__init__.py @@ -17,6 +17,7 @@ from __future__ import annotations +import contextlib import logging import operator import os @@ -32,6 +33,8 @@ from collections.abc import Iterable, Iterator from urllib.parse import SplitResult + from sqlalchemy.orm import Session + from airflow.models.asset import AssetModel from airflow.triggers.base import BaseTrigger @@ -227,7 +230,7 @@ def as_expression(self) -> Any: """ raise NotImplementedError - def evaluate(self, statuses: dict[str, bool]) -> bool: + def evaluate(self, statuses: dict[str, bool], *, session: Session | None = None) -> bool: raise NotImplementedError def iter_assets(self) -> Iterator[tuple[AssetUniqueKey, Asset]]: @@ -385,7 +388,7 @@ def iter_assets(self) -> Iterator[tuple[AssetUniqueKey, Asset]]: def iter_asset_aliases(self) -> Iterator[tuple[str, AssetAlias]]: return iter(()) - def evaluate(self, statuses: dict[str, bool]) -> bool: + def evaluate(self, statuses: dict[str, bool], *, session: Session | None = None) -> bool: return statuses.get(self.uri, False) def iter_dag_dependencies(self, *, source: str, target: str) -> Iterator[DagDependency]: @@ -428,11 +431,11 @@ class AssetAlias(BaseAsset): name: str = attrs.field(validator=_validate_non_empty_identifier) group: str = attrs.field(kw_only=True, default="asset", validator=_validate_identifier) - def _resolve_assets(self) -> list[Asset]: + def _resolve_assets(self, session: Session | None = None) -> list[Asset]: from airflow.models.asset import expand_alias_to_assets from airflow.utils.session import create_session - with create_session() as session: + with contextlib.nullcontext(session) if session else create_session() as session: asset_models = expand_alias_to_assets(self.name, session) return [m.to_public() for m in asset_models] @@ -444,8 +447,8 @@ def as_expression(self) -> Any: """ return {"alias": {"name": self.name, "group": self.group}} - def evaluate(self, statuses: dict[str, bool]) -> bool: - return any(x.evaluate(statuses=statuses) for x in self._resolve_assets()) + def evaluate(self, statuses: dict[str, bool], *, session: Session | None = None) -> bool: + return any(x.evaluate(statuses=statuses, session=session) for x in self._resolve_assets(session)) def iter_assets(self) -> Iterator[tuple[AssetUniqueKey, Asset]]: return iter(()) @@ -495,8 +498,8 @@ def __init__(self, *objects: BaseAsset) -> None: raise TypeError("expect asset expressions in condition") self.objects = objects - def evaluate(self, statuses: dict[str, bool]) -> bool: - return self.agg_func(x.evaluate(statuses=statuses) for x in self.objects) + def evaluate(self, statuses: dict[str, bool], *, session: Session | None = None) -> bool: + return self.agg_func(x.evaluate(statuses=statuses, session=session) for x in self.objects) def iter_assets(self) -> Iterator[tuple[AssetUniqueKey, Asset]]: seen: set[AssetUniqueKey] = set() # We want to keep the first instance. diff --git a/task_sdk/src/airflow/sdk/definitions/asset/decorators.py b/task_sdk/src/airflow/sdk/definitions/asset/decorators.py index 1cb1ea4e316968..531e097fd99f88 100644 --- a/task_sdk/src/airflow/sdk/definitions/asset/decorators.py +++ b/task_sdk/src/airflow/sdk/definitions/asset/decorators.py @@ -28,6 +28,8 @@ if TYPE_CHECKING: from collections.abc import Callable, Collection, Iterator, Mapping + from sqlalchemy.orm import Session + from airflow.io.path import ObjectStoragePath from airflow.models.param import ParamsDict from airflow.sdk.definitions.asset import AssetAlias, AssetUniqueKey @@ -120,8 +122,8 @@ def __attrs_post_init__(self) -> None: with self._source.create_dag(dag_id=self._function.__name__): _AssetMainOperator.from_definition(self) - def evaluate(self, statuses: dict[str, bool]) -> bool: - return all(o.evaluate(statuses=statuses) for o in self._source.outlets) + def evaluate(self, statuses: dict[str, bool], *, session: Session | None = None) -> bool: + return all(o.evaluate(statuses=statuses, session=session) for o in self._source.outlets) def iter_assets(self) -> Iterator[tuple[AssetUniqueKey, Asset]]: for o in self._source.outlets: diff --git a/task_sdk/src/airflow/sdk/definitions/baseoperator.py b/task_sdk/src/airflow/sdk/definitions/baseoperator.py index 9b1f64d970e64e..aaaee72a25df15 100644 --- a/task_sdk/src/airflow/sdk/definitions/baseoperator.py +++ b/task_sdk/src/airflow/sdk/definitions/baseoperator.py @@ -732,7 +732,7 @@ def __init__( f"Invalid arguments were passed to {self.__class__.__name__} (task_id: {task_id}). " f"Invalid arguments were:\n**kwargs: {kwargs}", ) - validate_key(task_id) + validate_key(self.task_id) self.owner = owner self.email = email diff --git a/task_sdk/src/airflow/sdk/execution_time/comms.py b/task_sdk/src/airflow/sdk/execution_time/comms.py index 9e6093a092da0d..03f92c549fd754 100644 --- a/task_sdk/src/airflow/sdk/execution_time/comms.py +++ b/task_sdk/src/airflow/sdk/execution_time/comms.py @@ -54,6 +54,7 @@ TaskInstance, TerminalTIState, TIDeferredStatePayload, + TIRunContext, VariableResponse, XComResponse, ) @@ -70,6 +71,7 @@ class StartupDetails(BaseModel): Responses will come back on stdin """ + ti_context: TIRunContext type: Literal["StartupDetails"] = "StartupDetails" diff --git a/task_sdk/src/airflow/sdk/execution_time/supervisor.py b/task_sdk/src/airflow/sdk/execution_time/supervisor.py index 677030b7bdce22..589cae56434c4b 100644 --- a/task_sdk/src/airflow/sdk/execution_time/supervisor.py +++ b/task_sdk/src/airflow/sdk/execution_time/supervisor.py @@ -397,7 +397,7 @@ def _on_child_started(self, ti: TaskInstance, path: str | os.PathLike[str], requ # We've forked, but the task won't start doing anything until we send it the StartupDetails # message. But before we do that, we need to tell the server it's started (so it has the chance to # tell us "no, stop!" for any reason) - self.client.task_instances.start(ti.id, self.pid, datetime.now(tz=timezone.utc)) + ti_context = self.client.task_instances.start(ti.id, self.pid, datetime.now(tz=timezone.utc)) self._last_successful_heartbeat = time.monotonic() except Exception: # On any error kill that subprocess! @@ -408,6 +408,7 @@ def _on_child_started(self, ti: TaskInstance, path: str | os.PathLike[str], requ ti=ti, file=os.fspath(path), requests_fd=requests_fd, + ti_context=ti_context, ) # Send the message to tell the process what it needs to execute diff --git a/task_sdk/src/airflow/sdk/execution_time/task_runner.py b/task_sdk/src/airflow/sdk/execution_time/task_runner.py index 5aca25f590e5e8..92f400d46e2bb7 100644 --- a/task_sdk/src/airflow/sdk/execution_time/task_runner.py +++ b/task_sdk/src/airflow/sdk/execution_time/task_runner.py @@ -23,13 +23,13 @@ import sys from datetime import datetime, timezone from io import FileIO -from typing import TYPE_CHECKING, Any, Generic, TextIO, TypeVar +from typing import TYPE_CHECKING, Annotated, Any, Generic, TextIO, TypeVar import attrs import structlog -from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter +from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter -from airflow.sdk.api.datamodels._generated import TaskInstance, TerminalTIState +from airflow.sdk.api.datamodels._generated import TaskInstance, TerminalTIState, TIRunContext from airflow.sdk.definitions.baseoperator import BaseOperator from airflow.sdk.execution_time.comms import ( DeferTask, @@ -48,9 +48,13 @@ class RuntimeTaskInstance(TaskInstance): model_config = ConfigDict(arbitrary_types_allowed=True) task: BaseOperator + _ti_context_from_server: Annotated[TIRunContext | None, Field(repr=False)] = None + """The Task Instance context from the API server, if any.""" def get_template_context(self): + # TODO: Assess if we need to it through airflow.utils.timezone.coerce_datetime() context: dict[str, Any] = { + # From the Task Execution interface "dag": self.task.dag, "inlets": self.task.inlets, "map_index_template": self.task.map_index_template, @@ -59,15 +63,9 @@ def get_template_context(self): "task": self.task, "task_instance": self, "ti": self, - # "dag_run": dag_run, - # "data_interval_end": timezone.coerce_datetime(data_interval.end), - # "data_interval_start": timezone.coerce_datetime(data_interval.start), # "outlet_events": OutletEventAccessors(), - # "ds": ds, - # "ds_nodash": ds_nodash, # "expanded_ti_count": expanded_ti_count, # "inlet_events": InletEventsAccessors(task.inlets, session=session), - # "logical_date": logical_date, # "macros": macros, # "params": validated_params, # "prev_data_interval_start_success": get_prev_data_interval_start_success(), @@ -77,15 +75,36 @@ def get_template_context(self): # "task_instance_key_str": f"{task.dag_id}__{task.task_id}__{ds_nodash}", # "test_mode": task_instance.test_mode, # "triggering_asset_events": lazy_object_proxy.Proxy(get_triggering_events), - # "ts": ts, - # "ts_nodash": ts_nodash, - # "ts_nodash_with_tz": ts_nodash_with_tz, # "var": { # "json": VariableAccessor(deserialize_json=True), # "value": VariableAccessor(deserialize_json=False), # }, # "conn": ConnectionAccessor(), } + if self._ti_context_from_server: + dag_run = self._ti_context_from_server.dag_run + + logical_date = dag_run.logical_date + ds = logical_date.strftime("%Y-%m-%d") + ds_nodash = ds.replace("-", "") + ts = logical_date.isoformat() + ts_nodash = logical_date.strftime("%Y%m%dT%H%M%S") + ts_nodash_with_tz = ts.replace("-", "").replace(":", "") + + context_from_server = { + # TODO: Assess if we need to pass these through timezone.coerce_datetime + "dag_run": dag_run, + "data_interval_end": dag_run.data_interval_end, + "data_interval_start": dag_run.data_interval_start, + "logical_date": logical_date, + "ds": ds, + "ds_nodash": ds_nodash, + "task_instance_key_str": f"{self.task.dag_id}__{self.task.task_id}__{ds_nodash}", + "ts": ts, + "ts_nodash": ts_nodash, + "ts_nodash_with_tz": ts_nodash_with_tz, + } + context.update(context_from_server) return context @@ -113,7 +132,11 @@ def parse(what: StartupDetails) -> RuntimeTaskInstance: if not isinstance(task, BaseOperator): raise TypeError(f"task is of the wrong type, got {type(task)}, wanted {BaseOperator}") - return RuntimeTaskInstance.model_construct(**what.ti.model_dump(exclude_unset=True), task=task) + return RuntimeTaskInstance.model_construct( + **what.ti.model_dump(exclude_unset=True), + task=task, + _ti_context_from_server=what.ti_context, + ) SendMsgType = TypeVar("SendMsgType", bound=BaseModel) diff --git a/task_sdk/tests/api/test_client.py b/task_sdk/tests/api/test_client.py index d10531ba1bb4fb..346c3adfcc1371 100644 --- a/task_sdk/tests/api/test_client.py +++ b/task_sdk/tests/api/test_client.py @@ -94,23 +94,31 @@ class TestTaskInstanceOperations: response parsing. """ - def test_task_instance_start(self): + def test_task_instance_start(self, make_ti_context): # Simulate a successful response from the server that starts a task ti_id = uuid6.uuid7() + start_date = "2024-10-31T12:00:00Z" + ti_context = make_ti_context( + start_date=start_date, + logical_date="2024-10-31T12:00:00Z", + run_type="manual", + ) def handle_request(request: httpx.Request) -> httpx.Response: - if request.url.path == f"/task-instances/{ti_id}/state": + if request.url.path == f"/task-instances/{ti_id}/run": actual_body = json.loads(request.read()) assert actual_body["pid"] == 100 - assert actual_body["start_date"] == "2024-10-31T12:00:00Z" + assert actual_body["start_date"] == start_date assert actual_body["state"] == "running" return httpx.Response( - status_code=204, + status_code=200, + json=ti_context.model_dump(mode="json"), ) return httpx.Response(status_code=400, json={"detail": "Bad Request"}) client = make_client(transport=httpx.MockTransport(handle_request)) - client.task_instances.start(ti_id, 100, "2024-10-31T12:00:00Z") + resp = client.task_instances.start(ti_id, 100, start_date) + assert resp == ti_context @pytest.mark.parametrize("state", [state for state in TerminalTIState]) def test_task_instance_finish(self, state): diff --git a/task_sdk/tests/conftest.py b/task_sdk/tests/conftest.py index 04e94008842d3f..25d0a1b0061b6c 100644 --- a/task_sdk/tests/conftest.py +++ b/task_sdk/tests/conftest.py @@ -19,7 +19,7 @@ import logging import os from pathlib import Path -from typing import TYPE_CHECKING, NoReturn +from typing import TYPE_CHECKING, Any, NoReturn, Protocol import pytest @@ -29,8 +29,12 @@ os.environ["_AIRFLOW_SKIP_DB_TESTS"] = "true" if TYPE_CHECKING: + from datetime import datetime + from structlog.typing import EventDict, WrappedLogger + from airflow.sdk.api.datamodels._generated import TIRunContext + @pytest.hookimpl() def pytest_addhooks(pluginmanager: pytest.PytestPluginManager): @@ -116,7 +120,7 @@ def _disable_ol_plugin(): # The OpenLineage plugin imports setproctitle, and that now causes (C) level thread calls, which on Py # 3.12+ issues a warning when os.fork happens. So for this plugin we disable it - # And we load plugins when setting the priorty_weight field + # And we load plugins when setting the priority_weight field import airflow.plugins_manager old = airflow.plugins_manager.plugins @@ -128,3 +132,85 @@ def _disable_ol_plugin(): yield airflow.plugins_manager.plugins = None + + +class MakeTIContextCallable(Protocol): + def __call__( + self, + dag_id: str = ..., + run_id: str = ..., + logical_date: str | datetime = ..., + data_interval_start: str | datetime = ..., + data_interval_end: str | datetime = ..., + start_date: str | datetime = ..., + run_type: str = ..., + ) -> TIRunContext: ... + + +class MakeTIContextDictCallable(Protocol): + def __call__( + self, + dag_id: str = ..., + run_id: str = ..., + logical_date: str = ..., + data_interval_start: str | datetime = ..., + data_interval_end: str | datetime = ..., + start_date: str | datetime = ..., + run_type: str = ..., + ) -> dict[str, Any]: ... + + +@pytest.fixture +def make_ti_context() -> MakeTIContextCallable: + """Factory for creating TIRunContext objects.""" + from airflow.sdk.api.datamodels._generated import DagRun, TIRunContext + + def _make_context( + dag_id: str = "test_dag", + run_id: str = "test_run", + logical_date: str | datetime = "2024-12-01T01:00:00Z", + data_interval_start: str | datetime = "2024-12-01T00:00:00Z", + data_interval_end: str | datetime = "2024-12-01T01:00:00Z", + start_date: str | datetime = "2024-12-01T01:00:00Z", + run_type: str = "manual", + ) -> TIRunContext: + return TIRunContext( + dag_run=DagRun( + dag_id=dag_id, + run_id=run_id, + logical_date=logical_date, # type: ignore + data_interval_start=data_interval_start, # type: ignore + data_interval_end=data_interval_end, # type: ignore + start_date=start_date, # type: ignore + run_type=run_type, # type: ignore + ) + ) + + return _make_context + + +@pytest.fixture +def make_ti_context_dict(make_ti_context: MakeTIContextCallable) -> MakeTIContextDictCallable: + """Factory for creating context dictionaries suited for API Server response.""" + + def _make_context_dict( + dag_id: str = "test_dag", + run_id: str = "test_run", + logical_date: str | datetime = "2024-12-01T00:00:00Z", + data_interval_start: str | datetime = "2024-12-01T00:00:00Z", + data_interval_end: str | datetime = "2024-12-01T01:00:00Z", + start_date: str | datetime = "2024-12-01T00:00:00Z", + run_type: str = "manual", + ) -> dict[str, Any]: + context = make_ti_context( + dag_id=dag_id, + run_id=run_id, + logical_date=logical_date, + data_interval_start=data_interval_start, + data_interval_end=data_interval_end, + start_date=start_date, + run_type=run_type, + ) + return context.model_dump(exclude_unset=True, mode="json") + + return _make_context_dict diff --git a/task_sdk/tests/defintions/test_asset.py b/task_sdk/tests/defintions/test_asset.py index afdfb37a5fd5d6..82225a619664dd 100644 --- a/task_sdk/tests/defintions/test_asset.py +++ b/task_sdk/tests/defintions/test_asset.py @@ -514,11 +514,11 @@ def test_as_expression(self, request: pytest.FixtureRequest, alias_fixture_name) def test_evalute_empty(self, asset_alias_1, asset): assert asset_alias_1.evaluate({asset.uri: True}) is False - assert asset_alias_1._resolve_assets.mock_calls == [mock.call()] + assert asset_alias_1._resolve_assets.mock_calls == [mock.call(None)] def test_evalute_resolved(self, resolved_asset_alias_2, asset): assert resolved_asset_alias_2.evaluate({asset.uri: True}) is True - assert resolved_asset_alias_2._resolve_assets.mock_calls == [mock.call()] + assert resolved_asset_alias_2._resolve_assets.mock_calls == [mock.call(None)] class TestAssetSubclasses: diff --git a/task_sdk/tests/execution_time/test_supervisor.py b/task_sdk/tests/execution_time/test_supervisor.py index 406b2ee26996fd..70f9e264864082 100644 --- a/task_sdk/tests/execution_time/test_supervisor.py +++ b/task_sdk/tests/execution_time/test_supervisor.py @@ -254,7 +254,7 @@ def test_run_simple_dag(self, test_dags_dir, captured_logs, time_machine): try_number=1, ) # Assert Exit Code is 0 - assert supervise(ti=ti, dag_path=dagfile_path, token="", server="", dry_run=True) == 0 + assert supervise(ti=ti, dag_path=dagfile_path, token="", server="", dry_run=True) == 0, captured_logs # We should have a log from the task! assert { @@ -265,7 +265,9 @@ def test_run_simple_dag(self, test_dags_dir, captured_logs, time_machine): "timestamp": "2024-11-07T12:34:56.078901Z", } in captured_logs - def test_supervise_handles_deferred_task(self, test_dags_dir, captured_logs, time_machine, mocker): + def test_supervise_handles_deferred_task( + self, test_dags_dir, captured_logs, time_machine, mocker, make_ti_context + ): """ Test that the supervisor handles a deferred task correctly. @@ -281,12 +283,13 @@ def test_supervise_handles_deferred_task(self, test_dags_dir, captured_logs, tim # Create a mock client to assert calls to the client # We assume the implementation of the client is correct and only need to check the calls mock_client = mocker.Mock(spec=sdk_client.Client) + mock_client.task_instances.start.return_value = make_ti_context() instant = tz.datetime(2024, 11, 7, 12, 34, 56, 0) time_machine.move_to(instant, tick=False) # Assert supervisor runs the task successfully - assert supervise(ti=ti, dag_path=dagfile_path, token="", client=mock_client) == 0 + assert supervise(ti=ti, dag_path=dagfile_path, token="", client=mock_client) == 0, captured_logs # Validate calls to the client mock_client.task_instances.start.assert_called_once_with(ti.id, mocker.ANY, mocker.ANY) @@ -320,7 +323,7 @@ def test_supervisor_handles_already_running_task(self): # The API Server would return a 409 Conflict status code if the TI is not # in a "queued" state. def handle_request(request: httpx.Request) -> httpx.Response: - if request.url.path == f"/task-instances/{ti.id}/state": + if request.url.path == f"/task-instances/{ti.id}/run": return httpx.Response( 409, json={ @@ -345,7 +348,7 @@ def handle_request(request: httpx.Request) -> httpx.Response: } @pytest.mark.parametrize("captured_logs", [logging.ERROR], indirect=True, ids=["log_level=error"]) - def test_state_conflict_on_heartbeat(self, captured_logs, monkeypatch, mocker): + def test_state_conflict_on_heartbeat(self, captured_logs, monkeypatch, mocker, make_ti_context_dict): """ Test that ensures that the Supervisor does not cause the task to fail if the Task Instance is no longer in the running state. Instead, it logs the error and terminates the task process if it @@ -383,7 +386,9 @@ def handle_request(request: httpx.Request) -> httpx.Response: "current_state": "success", }, ) - # Return a 204 for all other requests like the initial call to mark the task as running + elif request.url.path == f"/task-instances/{ti_id}/run": + return httpx.Response(200, json=make_ti_context_dict()) + # Return a 204 for all other requests return httpx.Response(status_code=204) proc = WatchedSubprocess.start( diff --git a/task_sdk/tests/execution_time/test_task_runner.py b/task_sdk/tests/execution_time/test_task_runner.py index c9755c252bbe68..2b812c92a7338e 100644 --- a/task_sdk/tests/execution_time/test_task_runner.py +++ b/task_sdk/tests/execution_time/test_task_runner.py @@ -94,7 +94,12 @@ def test_recv_StartupDetails(self): w.makefile("wb").write( b'{"type":"StartupDetails", "ti": {' b'"id": "4d828a62-a417-4936-a7a6-2b3fabacecab", "task_id": "a", "try_number": 1, "run_id": "b", "dag_id": "c" }, ' - b'"file": "/dev/null", "requests_fd": ' + str(w2.fileno()).encode("ascii") + b"}\n" + b'"ti_context":{"dag_run":{"dag_id":"c","run_id":"b","logical_date":"2024-12-01T01:00:00Z",' + b'"data_interval_start":"2024-12-01T00:00:00Z","data_interval_end":"2024-12-01T01:00:00Z",' + b'"start_date":"2024-12-01T01:00:00Z","end_date":null,"run_type":"manual","conf":null},' + b'"variables":null,"connections":null},"file": "/dev/null", "requests_fd": ' + + str(w2.fileno()).encode("ascii") + + b"}\n" ) decoder = CommsDecoder(input=r.makefile("r")) @@ -112,12 +117,13 @@ def test_recv_StartupDetails(self): assert decoder.request_socket.fileno() == w2.fileno() -def test_parse(test_dags_dir: Path): +def test_parse(test_dags_dir: Path, make_ti_context): """Test that checks parsing of a basic dag with an un-mocked parse.""" what = StartupDetails( ti=TaskInstance(id=uuid7(), task_id="a", dag_id="super_basic", run_id="c", try_number=1), file=str(test_dags_dir / "super_basic.py"), requests_fd=0, + ti_context=make_ti_context(), ) ti = parse(what) @@ -128,12 +134,13 @@ def test_parse(test_dags_dir: Path): assert isinstance(ti.task.dag, DAG) -def test_run_basic(time_machine, mocked_parse): +def test_run_basic(time_machine, mocked_parse, make_ti_context): """Test running a basic task.""" what = StartupDetails( ti=TaskInstance(id=uuid7(), task_id="hello", dag_id="super_basic_run", run_id="c", try_number=1), file="", requests_fd=0, + ti_context=make_ti_context(), ) instant = timezone.datetime(2024, 12, 3, 10, 0) @@ -150,7 +157,7 @@ def test_run_basic(time_machine, mocked_parse): ) -def test_run_deferred_basic(time_machine, mocked_parse): +def test_run_deferred_basic(time_machine, mocked_parse, make_ti_context): """Test that a task can transition to a deferred state.""" import datetime @@ -169,6 +176,7 @@ def test_run_deferred_basic(time_machine, mocked_parse): ti=TaskInstance(id=uuid7(), task_id="async", dag_id="basic_deferred_run", run_id="c", try_number=1), file="", requests_fd=0, + ti_context=make_ti_context(), ) # Expected DeferTask @@ -194,7 +202,7 @@ def test_run_deferred_basic(time_machine, mocked_parse): mock_supervisor_comms.send_request.assert_called_once_with(msg=expected_defer_task, log=mock.ANY) -def test_run_basic_skipped(time_machine, mocked_parse): +def test_run_basic_skipped(time_machine, mocked_parse, make_ti_context): """Test running a basic task that marks itself skipped.""" from airflow.providers.standard.operators.python import PythonOperator @@ -209,6 +217,7 @@ def test_run_basic_skipped(time_machine, mocked_parse): ti=TaskInstance(id=uuid7(), task_id="skip", dag_id="basic_skipped", run_id="c", try_number=1), file="", requests_fd=0, + ti_context=make_ti_context(), ) ti = mocked_parse(what, "basic_skipped", task) @@ -226,7 +235,7 @@ def test_run_basic_skipped(time_machine, mocked_parse): ) -def test_startup_basic_templated_dag(mocked_parse): +def test_startup_basic_templated_dag(mocked_parse, make_ti_context): """Test running a DAG with templated task.""" from airflow.providers.standard.operators.bash import BashOperator @@ -241,6 +250,7 @@ def test_startup_basic_templated_dag(mocked_parse): ), file="", requests_fd=0, + ti_context=make_ti_context(), ) mocked_parse(what, "basic_templated_dag", task) @@ -288,7 +298,9 @@ def test_startup_basic_templated_dag(mocked_parse): ), ], ) -def test_startup_dag_with_templated_fields(mocked_parse, task_params, expected_rendered_fields): +def test_startup_dag_with_templated_fields( + mocked_parse, task_params, expected_rendered_fields, make_ti_context +): """Test startup of a DAG with various templated fields.""" class CustomOperator(BaseOperator): @@ -305,6 +317,7 @@ def __init__(self, *args, **kwargs): ti=TaskInstance(id=uuid7(), task_id="templated_task", dag_id="basic_dag", run_id="c", try_number=1), file="", requests_fd=0, + ti_context=make_ti_context(), ) mocked_parse(what, "basic_dag", task) @@ -318,3 +331,73 @@ def __init__(self, *args, **kwargs): msg=SetRenderedFields(rendered_fields=expected_rendered_fields), log=mock.ANY, ) + + +class TestRuntimeTaskInstance: + def test_get_context_without_ti_context_from_server(self, mocked_parse, make_ti_context): + """Test get_template_context without ti_context_from_server.""" + + task = BaseOperator(task_id="hello") + + ti_id = uuid7() + ti = TaskInstance( + id=ti_id, task_id=task.task_id, dag_id="basic_task", run_id="test_run", try_number=1 + ) + + what = StartupDetails(ti=ti, file="", requests_fd=0, ti_context=make_ti_context()) + runtime_ti = mocked_parse(what, ti.dag_id, task) + context = runtime_ti.get_template_context() + + # Verify the context keys and values + assert context == { + "dag": runtime_ti.task.dag, + "inlets": task.inlets, + "map_index_template": task.map_index_template, + "outlets": task.outlets, + "run_id": "test_run", + "task": task, + "task_instance": runtime_ti, + "ti": runtime_ti, + } + + def test_get_context_with_ti_context_from_server(self, mocked_parse, make_ti_context): + """Test the context keys are added when sent from API server (mocked)""" + from airflow.utils import timezone + + ti = TaskInstance(id=uuid7(), task_id="hello", dag_id="basic_task", run_id="test_run", try_number=1) + + task = BaseOperator(task_id=ti.task_id) + + ti_context = make_ti_context(dag_id=ti.dag_id, run_id=ti.run_id) + what = StartupDetails(ti=ti, file="", requests_fd=0, ti_context=ti_context) + + runtime_ti = mocked_parse(what, ti.dag_id, task) + + # Assume the context is sent from the API server + # `task_sdk/tests/api/test_client.py::test_task_instance_start` checks the context is received + # from the API server + runtime_ti._ti_context_from_server = ti_context + dr = ti_context.dag_run + + context = runtime_ti.get_template_context() + + assert context == { + "dag": runtime_ti.task.dag, + "inlets": task.inlets, + "map_index_template": task.map_index_template, + "outlets": task.outlets, + "run_id": "test_run", + "task": task, + "task_instance": runtime_ti, + "ti": runtime_ti, + "dag_run": dr, + "data_interval_end": timezone.datetime(2024, 12, 1, 1, 0, 0), + "data_interval_start": timezone.datetime(2024, 12, 1, 0, 0, 0), + "logical_date": timezone.datetime(2024, 12, 1, 1, 0, 0), + "ds": "2024-12-01", + "ds_nodash": "20241201", + "task_instance_key_str": "basic_task__hello__20241201", + "ts": "2024-12-01T01:00:00+00:00", + "ts_nodash": "20241201T010000", + "ts_nodash_with_tz": "20241201T010000+0000", + } diff --git a/tests/api_fastapi/common/__init__.py b/tests/api_fastapi/common/__init__.py new file mode 100644 index 00000000000000..13a83393a9124b --- /dev/null +++ b/tests/api_fastapi/common/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/api_fastapi/common/test_exceptions.py b/tests/api_fastapi/common/test_exceptions.py new file mode 100644 index 00000000000000..e296f2320430c0 --- /dev/null +++ b/tests/api_fastapi/common/test_exceptions.py @@ -0,0 +1,236 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest +from fastapi import HTTPException, status +from sqlalchemy.exc import IntegrityError + +from airflow.api_fastapi.common.exceptions import _DatabaseDialect, _UniqueConstraintErrorHandler +from airflow.configuration import conf +from airflow.models import DagRun, Pool, Variable +from airflow.utils.session import provide_session +from airflow.utils.state import DagRunState + +from tests_common.test_utils.db import clear_db_connections, clear_db_dags, clear_db_pools, clear_db_runs + +pytestmark = pytest.mark.db_test + +CURRENT_DATABASE_DIALECT = conf.get_mandatory_value("database", "sql_alchemy_conn").lower() +TEST_POOL = "test_pool" +TEST_VARIABLE_KEY = "test_key" +PYTEST_MARKS_DB_DIALECT = [ + { + "condition": CURRENT_DATABASE_DIALECT.startswith(_DatabaseDialect.MYSQL.value) + or CURRENT_DATABASE_DIALECT.startswith(_DatabaseDialect.POSTGRES.value), + "reason": f"Test for {_DatabaseDialect.SQLITE.value} only", + }, + { + "condition": CURRENT_DATABASE_DIALECT.startswith(_DatabaseDialect.SQLITE.value) + or CURRENT_DATABASE_DIALECT.startswith(_DatabaseDialect.POSTGRES.value), + "reason": f"Test for {_DatabaseDialect.MYSQL.value} only", + }, + { + "condition": CURRENT_DATABASE_DIALECT.startswith(_DatabaseDialect.SQLITE.value) + or CURRENT_DATABASE_DIALECT.startswith(_DatabaseDialect.MYSQL.value), + "reason": f"Test for {_DatabaseDialect.POSTGRES.value} only", + }, +] + + +def generate_test_cases_parametrize( + test_cases: list[str], expected_exceptions_with_dialects: list[list[HTTPException]] +): + """Generate cross product of test cases for parametrize with different database dialects.""" + generated_test_cases = [] + for test_case, expected_exception_with_dialects in zip(test_cases, expected_exceptions_with_dialects): + for mark, expected_exception in zip(PYTEST_MARKS_DB_DIALECT, expected_exception_with_dialects): + generated_test_cases.append( + pytest.param( + test_case, + expected_exception, + id=f"{mark['reason']} - {test_case}", + marks=pytest.mark.skipif(**mark), # type: ignore + ) + ) + return generated_test_cases + + +def get_unique_constraint_error_prefix(): + """Get unique constraint error prefix based on current database dialect.""" + if CURRENT_DATABASE_DIALECT.startswith(_DatabaseDialect.SQLITE.value): + return _UniqueConstraintErrorHandler.unique_constraint_error_prefix_dict[_DatabaseDialect.SQLITE] + if CURRENT_DATABASE_DIALECT.startswith(_DatabaseDialect.MYSQL.value): + return _UniqueConstraintErrorHandler.unique_constraint_error_prefix_dict[_DatabaseDialect.MYSQL] + if CURRENT_DATABASE_DIALECT.startswith(_DatabaseDialect.POSTGRES.value): + return _UniqueConstraintErrorHandler.unique_constraint_error_prefix_dict[_DatabaseDialect.POSTGRES] + return "" + + +class TestUniqueConstraintErrorHandler: + unique_constraint_error_handler = _UniqueConstraintErrorHandler() + + @pytest.fixture(autouse=True) + def setup(self) -> None: + clear_db_connections(add_default_connections_back=False) + clear_db_pools() + clear_db_runs() + clear_db_dags() + + def teardown_method(self) -> None: + clear_db_connections() + clear_db_pools() + clear_db_runs() + clear_db_dags() + + @pytest.mark.parametrize( + "table, expected_exception", + generate_test_cases_parametrize( + ["Pool", "Variable"], + [ + [ # Pool + HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "reason": "Unique constraint violation", + "statement": "INSERT INTO slot_pool (pool, slots, description, include_deferred) VALUES (?, ?, ?, ?)", + "orig_error": "UNIQUE constraint failed: slot_pool.pool", + }, + ), + HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "reason": "Unique constraint violation", + "statement": "INSERT INTO slot_pool (pool, slots, description, include_deferred) VALUES (%s, %s, %s, %s)", + "orig_error": "(1062, \"Duplicate entry 'test_pool' for key 'slot_pool.slot_pool_pool_uq'\")", + }, + ), + HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "reason": "Unique constraint violation", + "statement": "INSERT INTO slot_pool (pool, slots, description, include_deferred) VALUES (%(pool)s, %(slots)s, %(description)s, %(include_deferred)s) RETURNING slot_pool.id", + "orig_error": 'duplicate key value violates unique constraint "slot_pool_pool_uq"\nDETAIL: Key (pool)=(test_pool) already exists.\n', + }, + ), + ], + [ # Variable + HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "reason": "Unique constraint violation", + "statement": 'INSERT INTO variable ("key", val, description, is_encrypted) VALUES (?, ?, ?, ?)', + "orig_error": "UNIQUE constraint failed: variable.key", + }, + ), + HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "reason": "Unique constraint violation", + "statement": "INSERT INTO variable (`key`, val, description, is_encrypted) VALUES (%s, %s, %s, %s)", + "orig_error": "(1062, \"Duplicate entry 'test_key' for key 'variable.variable_key_uq'\")", + }, + ), + HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "reason": "Unique constraint violation", + "statement": "INSERT INTO variable (key, val, description, is_encrypted) VALUES (%(key)s, %(val)s, %(description)s, %(is_encrypted)s) RETURNING variable.id", + "orig_error": 'duplicate key value violates unique constraint "variable_key_uq"\nDETAIL: Key (key)=(test_key) already exists.\n', + }, + ), + ], + ], + ), + ) + @provide_session + def test_handle_single_column_unique_constraint_error(self, session, table, expected_exception) -> None: + # Take Pool and Variable tables as test cases + if table == "Pool": + session.add(Pool(pool=TEST_POOL, slots=1, description="test pool", include_deferred=False)) + session.add(Pool(pool=TEST_POOL, slots=1, description="test pool", include_deferred=False)) + elif table == "Variable": + session.add(Variable(key=TEST_VARIABLE_KEY, val="test_val")) + session.add(Variable(key=TEST_VARIABLE_KEY, val="test_val")) + + with pytest.raises(IntegrityError) as exeinfo_integrity_error: + session.commit() + + with pytest.raises(HTTPException) as exeinfo_response_error: + self.unique_constraint_error_handler.exception_handler(None, exeinfo_integrity_error.value) # type: ignore + + assert exeinfo_response_error.value.status_code == expected_exception.status_code + assert exeinfo_response_error.value.detail == expected_exception.detail + + @pytest.mark.parametrize( + "table, expected_exception", + generate_test_cases_parametrize( + ["DagRun"], + [ + [ # DagRun + HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "reason": "Unique constraint violation", + "statement": "INSERT INTO dag_run (dag_id, queued_at, logical_date, start_date, end_date, state, run_id, creating_job_id, external_trigger, run_type, triggered_by, conf, data_interval_start, data_interval_end, last_scheduling_decision, log_template_id, updated_at, clear_number, backfill_id, dag_version_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, (SELECT max(log_template.id) AS max_1 \nFROM log_template), ?, ?, ?, ?)", + "orig_error": "UNIQUE constraint failed: dag_run.dag_id, dag_run.run_id", + }, + ), + HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "reason": "Unique constraint violation", + "statement": "INSERT INTO dag_run (dag_id, queued_at, logical_date, start_date, end_date, state, run_id, creating_job_id, external_trigger, run_type, triggered_by, conf, data_interval_start, data_interval_end, last_scheduling_decision, log_template_id, updated_at, clear_number, backfill_id, dag_version_id) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, (SELECT max(log_template.id) AS max_1 \nFROM log_template), %s, %s, %s, %s)", + "orig_error": "(1062, \"Duplicate entry 'test_dag_id-test_run_id' for key 'dag_run.dag_run_dag_id_run_id_key'\")", + }, + ), + HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "reason": "Unique constraint violation", + "statement": "INSERT INTO dag_run (dag_id, queued_at, logical_date, start_date, end_date, state, run_id, creating_job_id, external_trigger, run_type, triggered_by, conf, data_interval_start, data_interval_end, last_scheduling_decision, log_template_id, updated_at, clear_number, backfill_id, dag_version_id) VALUES (%(dag_id)s, %(queued_at)s, %(logical_date)s, %(start_date)s, %(end_date)s, %(state)s, %(run_id)s, %(creating_job_id)s, %(external_trigger)s, %(run_type)s, %(triggered_by)s, %(conf)s, %(data_interval_start)s, %(data_interval_end)s, %(last_scheduling_decision)s, (SELECT max(log_template.id) AS max_1 \nFROM log_template), %(updated_at)s, %(clear_number)s, %(backfill_id)s, %(dag_version_id)s) RETURNING dag_run.id", + "orig_error": 'duplicate key value violates unique constraint "dag_run_dag_id_run_id_key"\nDETAIL: Key (dag_id, run_id)=(test_dag_id, test_run_id) already exists.\n', + }, + ), + ], + ], + ), + ) + @provide_session + def test_handle_multiple_columns_unique_constraint_error( + self, session, table, expected_exception + ) -> None: + if table == "DagRun": + session.add( + DagRun( + dag_id="test_dag_id", run_id="test_run_id", run_type="manual", state=DagRunState.RUNNING + ) + ) + session.add( + DagRun( + dag_id="test_dag_id", run_id="test_run_id", run_type="manual", state=DagRunState.RUNNING + ) + ) + + with pytest.raises(IntegrityError) as exeinfo_integrity_error: + session.commit() + + with pytest.raises(HTTPException) as exeinfo_response_error: + self.unique_constraint_error_handler.exception_handler(None, exeinfo_integrity_error.value) # type: ignore + + assert exeinfo_response_error.value.status_code == expected_exception.status_code + assert exeinfo_response_error.value.detail == expected_exception.detail diff --git a/tests/api_fastapi/core_api/routes/public/test_connections.py b/tests/api_fastapi/core_api/routes/public/test_connections.py index 9d6cfd9fb4fff4..0cada4b94297c3 100644 --- a/tests/api_fastapi/core_api/routes/public/test_connections.py +++ b/tests/api_fastapi/core_api/routes/public/test_connections.py @@ -246,9 +246,9 @@ def test_post_should_respond_already_exist(self, test_client, body): # Another request response = test_client.post("/public/connections", json=body) assert response.status_code == 409 - assert response.json() == { - "detail": "Unique constraint violation", - } + response_json = response.json() + assert "detail" in response_json + assert list(response_json["detail"].keys()) == ["reason", "statement", "orig_error"] @pytest.mark.enable_redact @pytest.mark.parametrize( @@ -396,9 +396,9 @@ def test_post_should_respond_already_exist(self, test_client, body): # Another request response = test_client.post("/public/connections/bulk", json=body) assert response.status_code == 409 - assert response.json() == { - "detail": "Unique constraint violation", - } + response_json = response.json() + assert "detail" in response_json + assert list(response_json["detail"].keys()) == ["reason", "statement", "orig_error"] @pytest.mark.enable_redact @pytest.mark.parametrize( @@ -649,7 +649,6 @@ def test_patch_should_respond_400(self, test_client, body): self.create_connection() response = test_client.patch(f"/public/connections/{TEST_CONN_ID}", json=body) assert response.status_code == 400 - print(response.json()) assert response.json() == { "detail": "The connection_id in the request body does not match the URL parameter", } diff --git a/tests/api_fastapi/core_api/routes/public/test_dag_run.py b/tests/api_fastapi/core_api/routes/public/test_dag_run.py index f996800a8ee384..c3679d2d106e73 100644 --- a/tests/api_fastapi/core_api/routes/public/test_dag_run.py +++ b/tests/api_fastapi/core_api/routes/public/test_dag_run.py @@ -1353,4 +1353,6 @@ def test_response_404(self, test_client): def test_response_409(self, test_client): response = test_client.post(f"/public/dags/{DAG1_ID}/dagRuns", json={"dag_run_id": DAG1_RUN1_ID}) assert response.status_code == 409 - assert response.json()["detail"] == "Unique constraint violation" + response_json = response.json() + assert "detail" in response_json + assert list(response_json["detail"].keys()) == ["reason", "statement", "orig_error"] diff --git a/tests/api_fastapi/core_api/routes/public/test_pools.py b/tests/api_fastapi/core_api/routes/public/test_pools.py index 5b16052c3c86f1..2a7b767d2b851f 100644 --- a/tests/api_fastapi/core_api/routes/public/test_pools.py +++ b/tests/api_fastapi/core_api/routes/public/test_pools.py @@ -349,7 +349,7 @@ def test_should_respond_200(self, test_client, session, body, expected_status_co "deferred_slots": 0, }, 409, - {"detail": "Unique constraint violation"}, + None, ), ], ) @@ -371,7 +371,13 @@ def test_should_response_409( assert session.query(Pool).count() == n_pools + 1 response = test_client.post("/public/pools", json=body) assert response.status_code == second_expected_status_code - assert response.json() == second_expected_response + if second_expected_status_code == 201: + assert response.json() == second_expected_response + else: + response_json = response.json() + assert "detail" in response_json + assert list(response_json["detail"].keys()) == ["reason", "statement", "orig_error"] + assert session.query(Pool).count() == n_pools + 1 @@ -425,7 +431,7 @@ class TestPostPools(TestPoolsEndpoint): ] }, 409, - {"detail": "Unique constraint violation"}, + None, ), ( { @@ -435,7 +441,7 @@ class TestPostPools(TestPoolsEndpoint): ] }, 409, - {"detail": "Unique constraint violation"}, + None, ), ( { @@ -445,7 +451,7 @@ class TestPostPools(TestPoolsEndpoint): ] }, 409, - {"detail": "Unique constraint violation"}, + None, ), ], ) @@ -454,8 +460,11 @@ def test_post_pools(self, test_client, session, body, expected_status_code, expe n_pools = session.query(Pool).count() response = test_client.post("/public/pools/bulk", json=body) assert response.status_code == expected_status_code - assert response.json() == expected_response if expected_status_code == 201: + response.json() == expected_response assert session.query(Pool).count() == n_pools + 2 else: + response_json = response.json() + assert "detail" in response_json + assert list(response_json["detail"].keys()) == ["reason", "statement", "orig_error"] assert session.query(Pool).count() == n_pools diff --git a/tests/api_fastapi/core_api/routes/ui/test_structure.py b/tests/api_fastapi/core_api/routes/ui/test_structure.py index 0596b0f43e0feb..8efc6dc9d12b00 100644 --- a/tests/api_fastapi/core_api/routes/ui/test_structure.py +++ b/tests/api_fastapi/core_api/routes/ui/test_structure.py @@ -90,12 +90,14 @@ class TestStructureDataEndpoint: "edges": [ { "is_setup_teardown": None, + "is_source_asset": None, "label": None, "source_id": "external_task_sensor", "target_id": "task_2", }, { "is_setup_teardown": None, + "is_source_asset": None, "label": None, "source_id": "task_1", "target_id": "external_task_sensor", @@ -103,6 +105,7 @@ class TestStructureDataEndpoint: ], "nodes": [ { + "asset_condition_type": None, "children": None, "id": "task_1", "is_mapped": None, @@ -113,6 +116,7 @@ class TestStructureDataEndpoint: "operator": "EmptyOperator", }, { + "asset_condition_type": None, "children": None, "id": "external_task_sensor", "is_mapped": None, @@ -123,6 +127,7 @@ class TestStructureDataEndpoint: "operator": "ExternalTaskSensor", }, { + "asset_condition_type": None, "children": None, "id": "task_2", "is_mapped": None, @@ -155,6 +160,7 @@ class TestStructureDataEndpoint: "edges": [], "nodes": [ { + "asset_condition_type": None, "children": None, "id": "task_1", "is_mapped": None, @@ -177,50 +183,65 @@ class TestStructureDataEndpoint: { "is_setup_teardown": None, "label": None, - "source_id": "trigger:external_trigger:test_dag_id:trigger_dag_run_operator", + "source_id": "and-gate-0", "target_id": "task_1", + "is_source_asset": True, }, { "is_setup_teardown": None, "label": None, - "source_id": "asset:asset1", - "target_id": "task_1", + "source_id": "asset1", + "target_id": "and-gate-0", + "is_source_asset": None, }, { "is_setup_teardown": None, "label": None, - "source_id": "asset:asset2", - "target_id": "task_1", + "source_id": "asset2", + "target_id": "and-gate-0", + "is_source_asset": None, }, { "is_setup_teardown": None, "label": None, - "source_id": "asset-alias:example-alias", + "source_id": "example-alias", + "target_id": "and-gate-0", + "is_source_asset": None, + }, + { + "is_setup_teardown": None, + "label": None, + "source_id": "trigger:external_trigger:test_dag_id:trigger_dag_run_operator", "target_id": "task_1", + "is_source_asset": None, }, { "is_setup_teardown": None, "label": None, "source_id": "sensor:test_dag_id:test_dag_id:external_task_sensor", "target_id": "task_1", + "is_source_asset": None, }, { "is_setup_teardown": None, "label": None, "source_id": "external_task_sensor", "target_id": "task_2", + "is_source_asset": None, }, { "is_setup_teardown": None, "label": None, "source_id": "task_1", "target_id": "external_task_sensor", + "is_source_asset": None, }, { "is_setup_teardown": None, "label": None, "source_id": "task_2", "target_id": "asset:s3://dataset-bucket/example.csv", + "is_source_asset": None, }, ], "nodes": [ @@ -233,6 +254,7 @@ class TestStructureDataEndpoint: "setup_teardown_type": None, "type": "task", "operator": "EmptyOperator", + "asset_condition_type": None, }, { "children": None, @@ -243,6 +265,7 @@ class TestStructureDataEndpoint: "setup_teardown_type": None, "type": "task", "operator": "ExternalTaskSensor", + "asset_condition_type": None, }, { "children": None, @@ -253,6 +276,7 @@ class TestStructureDataEndpoint: "setup_teardown_type": None, "type": "task", "operator": "EmptyOperator", + "asset_condition_type": None, }, { "children": None, @@ -263,56 +287,73 @@ class TestStructureDataEndpoint: "setup_teardown_type": None, "type": "trigger", "operator": None, + "asset_condition_type": None, }, { "children": None, - "id": "asset:asset1", + "id": "asset:s3://dataset-bucket/example.csv", "is_mapped": None, - "label": "asset1", + "label": "s3://dataset-bucket/example.csv", "tooltip": None, "setup_teardown_type": None, "type": "asset", "operator": None, + "asset_condition_type": None, }, { "children": None, - "id": "asset:asset2", + "id": "sensor:test_dag_id:test_dag_id:external_task_sensor", "is_mapped": None, - "label": "asset2", + "label": "external_task_sensor", "tooltip": None, "setup_teardown_type": None, - "type": "asset", + "type": "sensor", "operator": None, + "asset_condition_type": None, }, { "children": None, - "id": "asset-alias:example-alias", + "id": "and-gate-0", "is_mapped": None, - "label": "example-alias", + "label": "and-gate-0", "tooltip": None, "setup_teardown_type": None, - "type": "asset-alias", + "type": "asset-condition", "operator": None, + "asset_condition_type": "and-gate", }, { "children": None, - "id": "asset:s3://dataset-bucket/example.csv", + "id": "asset1", "is_mapped": None, - "label": "s3://dataset-bucket/example.csv", + "label": "asset1", "tooltip": None, "setup_teardown_type": None, "type": "asset", "operator": None, + "asset_condition_type": None, }, { "children": None, - "id": "sensor:test_dag_id:test_dag_id:external_task_sensor", + "id": "asset2", "is_mapped": None, - "label": "external_task_sensor", + "label": "asset2", "tooltip": None, "setup_teardown_type": None, - "type": "sensor", + "type": "asset", + "operator": None, + "asset_condition_type": None, + }, + { + "children": None, + "id": "example-alias", + "is_mapped": None, + "label": "example-alias", + "tooltip": None, + "setup_teardown_type": None, + "type": "asset-alias", "operator": None, + "asset_condition_type": None, }, ], "arrange": "LR", @@ -323,6 +364,7 @@ class TestStructureDataEndpoint: { "edges": [ { + "is_source_asset": None, "is_setup_teardown": None, "label": None, "source_id": "trigger_dag_run_operator", @@ -331,6 +373,7 @@ class TestStructureDataEndpoint: ], "nodes": [ { + "asset_condition_type": None, "children": None, "id": "trigger_dag_run_operator", "is_mapped": None, @@ -341,6 +384,7 @@ class TestStructureDataEndpoint: "operator": "TriggerDagRunOperator", }, { + "asset_condition_type": None, "children": None, "id": "trigger:external_trigger:test_dag_id:trigger_dag_run_operator", "is_mapped": None, diff --git a/tests/api_fastapi/execution_api/routes/test_task_instances.py b/tests/api_fastapi/execution_api/routes/test_task_instances.py index 15e56bbc587103..e67d82a718cd60 100644 --- a/tests/api_fastapi/execution_api/routes/test_task_instances.py +++ b/tests/api_fastapi/execution_api/routes/test_task_instances.py @@ -39,40 +39,58 @@ DEFAULT_END_DATE = timezone.parse("2024-10-31T12:00:00Z") -class TestTIUpdateState: +class TestTIRunState: def setup_method(self): clear_db_runs() def teardown_method(self): clear_db_runs() - def test_ti_update_state_to_running(self, client, session, create_task_instance): + def test_ti_run_state_to_running(self, client, session, create_task_instance, time_machine): """ Test that the Task Instance state is updated to running when the Task Instance is in a state where it can be marked as running. """ + instant_str = "2024-09-30T12:00:00Z" + instant = timezone.parse(instant_str) + time_machine.move_to(instant, tick=False) ti = create_task_instance( - task_id="test_ti_update_state_to_running", + task_id="test_ti_run_state_to_running", state=State.QUEUED, session=session, + start_date=instant, ) session.commit() response = client.patch( - f"/execution/task-instances/{ti.id}/state", + f"/execution/task-instances/{ti.id}/run", json={ "state": "running", "hostname": "random-hostname", "unixname": "random-unixname", "pid": 100, - "start_date": "2024-10-31T12:00:00Z", + "start_date": instant_str, }, ) - assert response.status_code == 204 - assert response.text == "" + assert response.status_code == 200 + assert response.json() == { + "dag_run": { + "dag_id": "dag", + "run_id": "test", + "logical_date": instant_str, + "data_interval_start": instant.subtract(days=1).to_iso8601_string(), + "data_interval_end": instant_str, + "start_date": instant_str, + "end_date": None, + "run_type": "manual", + "conf": {}, + }, + "variables": [], + "connections": [], + } # Refresh the Task Instance from the database so that we can check the updated values session.refresh(ti) @@ -80,10 +98,10 @@ def test_ti_update_state_to_running(self, client, session, create_task_instance) assert ti.hostname == "random-hostname" assert ti.unixname == "random-unixname" assert ti.pid == 100 - assert ti.start_date.isoformat() == "2024-10-31T12:00:00+00:00" + assert ti.start_date == instant @pytest.mark.parametrize("initial_ti_state", [s for s in TaskInstanceState if s != State.QUEUED]) - def test_ti_update_state_conflict_if_not_queued( + def test_ti_run_state_conflict_if_not_queued( self, client, session, create_task_instance, initial_ti_state ): """ @@ -91,13 +109,13 @@ def test_ti_update_state_conflict_if_not_queued( running. In this case, the Task Instance is first in NONE state so it cannot be marked as running. """ ti = create_task_instance( - task_id="test_ti_update_state_conflict_if_not_queued", + task_id="test_ti_run_state_conflict_if_not_queued", state=initial_ti_state, ) session.commit() response = client.patch( - f"/execution/task-instances/{ti.id}/state", + f"/execution/task-instances/{ti.id}/run", json={ "state": "running", "hostname": "random-hostname", @@ -118,6 +136,14 @@ def test_ti_update_state_conflict_if_not_queued( assert session.scalar(select(TaskInstance.state).where(TaskInstance.id == ti.id)) == initial_ti_state + +class TestTIUpdateState: + def setup_method(self): + clear_db_runs() + + def teardown_method(self): + clear_db_runs() + @pytest.mark.parametrize( ("state", "end_date", "expected_state"), [ @@ -160,7 +186,7 @@ def test_ti_update_state_not_found(self, client, session): task_instance_id = "0182e924-0f1e-77e6-ab50-e977118bc139" # Pre-condition: the Task Instance does not exist - assert session.scalar(select(TaskInstance.id).where(TaskInstance.id == task_instance_id)) is None + assert session.get(TaskInstance, task_instance_id) is None payload = {"state": "success", "end_date": "2024-10-31T12:30:00Z"} @@ -171,6 +197,26 @@ def test_ti_update_state_not_found(self, client, session): "message": "Task Instance not found", } + def test_ti_update_state_running_errors(self, client, session, create_task_instance, time_machine): + """ + Test that a 422 error is returned when the Task Instance state is RUNNING in the payload. + + Task should be set to Running state via the /execution/task-instances/{task_instance_id}/run endpoint. + """ + + ti = create_task_instance( + task_id="test_ti_update_state_running_errors", + state=State.QUEUED, + session=session, + start_date=DEFAULT_START_DATE, + ) + + session.commit() + + response = client.patch(f"/execution/task-instances/{ti.id}/state", json={"state": "running"}) + + assert response.status_code == 422 + def test_ti_update_state_database_error(self, client, session, create_task_instance): """ Test that a database error is handled correctly when updating the Task Instance state. @@ -181,17 +227,14 @@ def test_ti_update_state_database_error(self, client, session, create_task_insta ) session.commit() payload = { - "state": "running", - "hostname": "random-hostname", - "unixname": "random-unixname", - "pid": 100, - "start_date": "2024-10-31T12:00:00Z", + "state": "success", + "end_date": "2024-10-31T12:00:00Z", } with mock.patch( "airflow.api_fastapi.common.db.common.Session.execute", side_effect=[ - mock.Mock(one=lambda: ("queued",)), # First call returns "queued" + mock.Mock(one=lambda: ("running",)), # First call returns "queued" SQLAlchemyError("Database error"), # Second call raises an error ], ): @@ -334,7 +377,7 @@ def test_ti_heartbeat_non_existent_task(self, client, session, create_task_insta task_instance_id = "0182e924-0f1e-77e6-ab50-e977118bc139" # Pre-condition: the Task Instance does not exist - assert session.scalar(select(TaskInstance.id).where(TaskInstance.id == task_instance_id)) is None + assert session.get(TaskInstance, task_instance_id) is None response = client.put( f"/execution/task-instances/{task_instance_id}/heartbeat", diff --git a/tests/assets/test_manager.py b/tests/assets/test_manager.py index afbbfc23adee1e..b47dae7f2e7f1e 100644 --- a/tests/assets/test_manager.py +++ b/tests/assets/test_manager.py @@ -34,7 +34,6 @@ DagScheduleAssetReference, ) from airflow.models.dag import DagModel -from airflow.models.dagbag import DagPriorityParsingRequest from airflow.sdk.definitions.asset import Asset, AssetAlias from tests.listeners import asset_listener @@ -139,7 +138,6 @@ def test_register_asset_change_with_alias(self, session, dag_maker, mock_task_in # Ensure we've created an asset assert session.query(AssetEvent).filter_by(asset_id=asm.id).count() == 1 assert session.query(AssetDagRunQueue).count() == 2 - assert session.query(DagPriorityParsingRequest).count() == 2 def test_register_asset_change_no_downstreams(self, session, mock_task_instance): asset_manager = AssetManager() diff --git a/tests/models/test_baseoperator.py b/tests/models/test_baseoperator.py index e95866d95a5e95..2c598edc777acc 100644 --- a/tests/models/test_baseoperator.py +++ b/tests/models/test_baseoperator.py @@ -395,6 +395,33 @@ def test_chain(self): assert [op2] == tgop3.get_direct_relatives(upstream=False) assert [op2] == tgop4.get_direct_relatives(upstream=False) + def test_baseoperator_raises_exception_when_task_id_plus_taskgroup_id_exceeds_250_chars(self): + """Test exception is raised when operator task id + taskgroup id > 250 chars.""" + dag = DAG(dag_id="foo", schedule=None, start_date=datetime.now()) + + tg1 = TaskGroup("A" * 20, dag=dag) + with pytest.raises(ValueError, match="The key has to be less than 250 characters"): + BaseOperator(task_id="1" * 250, task_group=tg1, dag=dag) + + def test_baseoperator_with_task_id_and_taskgroup_id_less_than_250_chars(self): + """Test exception is not raised when operator task id + taskgroup id < 250 chars.""" + dag = DAG(dag_id="foo", schedule=None, start_date=datetime.now()) + + tg1 = TaskGroup("A" * 10, dag=dag) + try: + BaseOperator(task_id="1" * 239, task_group=tg1, dag=dag) + except Exception as e: + pytest.fail(f"Exception raised: {e}") + + def test_baseoperator_with_task_id_less_than_250_chars(self): + """Test exception is not raised when operator task id < 250 chars.""" + dag = DAG(dag_id="foo", schedule=None, start_date=datetime.now()) + + try: + BaseOperator(task_id="1" * 249, dag=dag) + except Exception as e: + pytest.fail(f"Exception raised: {e}") + def test_chain_linear(self): dag = DAG(dag_id="test_chain_linear", schedule=None, start_date=datetime.now())