Merge branch 'main' into move-arangodb-to-new-provider-strcucture

.editorconfig

@@ -40,6 +40,9 @@ indent_size = 2
 [*.ts]
 indent_size = 2
 
+[*.tsx]
+indent_size = 2
+
 [*.css]
 indent_size = 2
 

.github/boring-cyborg.yml

@@ -34,10 +34,7 @@ labelPRBasedOnFilePath:
     - providers/apache/beam/**
 
   provider:apache-cassandra:
-    - providers/src/airflow/providers/apache/cassandra/**/*
-    - docs/apache-airflow-providers-apache-cassandra/**/*
-    - providers/tests/apache/cassandra/**/*
-    - providers/tests/system/apache/cassandra/**/*
+    - providers/apache/cassandra/**
 
   provider:apache-drill:
     - providers/src/airflow/providers/apache/drill/**/*
@@ -82,10 +79,7 @@ labelPRBasedOnFilePath:
     - providers/apache/kylin/**
 
   provider:apache-livy:
-    - providers/src/airflow/providers/apache/livy/**/*
-    - docs/apache-airflow-providers-apache-livy/**/*
-    - providers/tests/apache/livy/**/*
-    - providers/tests/system/apache/livy/**/*
+    - providers/apache/livy/**
 
   provider:apache-pig:
     - providers/apache/pig/**
@@ -97,9 +91,7 @@ labelPRBasedOnFilePath:
     - providers/apache/spark/**
 
   provider:apprise:
-    - providers/src/airflow/providers/apprise/**/*
-    - docs/apache-airflow-providers-apprise/**/*
-    - providers/tests/apprise/**/*
+    - providers/apprise/**
 
   provider:arangodb:
     - providers/arangodb/**
@@ -108,9 +100,7 @@ labelPRBasedOnFilePath:
     - providers/asana/**
 
   provider:atlassian-jira:
-    - providers/src/airflow/providers/atlassian/jira/**/*
-    - docs/apache-airflow-providers-atlassian-jira/**/*
-    - providers/tests/atlassian/jira/**/*
+    - providers/atlassian/jira/**
 
   provider:celery:
     - providers/celery/**
@@ -193,10 +183,7 @@ labelPRBasedOnFilePath:
     - providers/facebook/**
 
   provider:ftp:
-    - providers/src/airflow/providers/ftp/**/*
-    - docs/apache-airflow-providers-ftp/**/*
-    - providers/tests/ftp/**/*
-    - providers/tests/system/ftp/**/*
+    - providers/ftp/**
 
   provider:github:
     - providers/src/airflow/providers/github/**/*
@@ -225,10 +212,7 @@ labelPRBasedOnFilePath:
     - providers/imap/**
 
   provider:influxdb:
-    - providers/src/airflow/providers/influxdb/**/*
-    - docs/apache-airflow-providers-influxdb/**/*
-    - providers/tests/influxdb/**/*
-    - providers/tests/system/influxdb/**/*
+    - providers/influxdb/**
 
   provider:jdbc:
     - providers/src/airflow/providers/jdbc/**/*
@@ -345,10 +329,7 @@ labelPRBasedOnFilePath:
     - providers/singularity/**
 
   provider:slack:
-    - providers/src/airflow/providers/slack/**/*
-    - docs/apache-airflow-providers-slack/**/*
-    - providers/tests/slack/**/*
-    - providers/tests/system/slack/**/*
+    - providers/slack/**
 
   provider:smtp:
     - providers/src/airflow/providers/smtp/**/*
@@ -377,10 +358,7 @@ labelPRBasedOnFilePath:
     - providers/teradata/**
 
   provider:trino:
-    - providers/src/airflow/providers/trino/**/*
-    - docs/apache-airflow-providers-trino/**/*
-    - providers/tests/trino/**/*
-    - providers/tests/system/trino/**/*
+    - providers/trino/**
 
   provider:vertica:
     - providers/vertica/**

.github/workflows/basic-tests.yml

@@ -85,7 +85,7 @@ jobs:
       - run: uv tool run --from apache-airflow-breeze pytest -n auto --color=yes
         working-directory: ./dev/breeze/
   tests-ui:
-    timeout-minutes: 10
+    timeout-minutes: 15
     name: React UI tests
     runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
     if: inputs.run-ui-tests == 'true'
@@ -115,7 +115,7 @@ jobs:
         with:
           path: airflow/ui/node_modules/
           key: cache-ui-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/ui/**/pnpm-lock.yaml') }}
-        id: restore-eslint-cache
+        id: restore-eslint-cache-ui
       - run: cd airflow/ui && pnpm install --frozen-lockfile
       - run: cd airflow/ui && pnpm test
         env:
@@ -127,7 +127,29 @@ jobs:
           key: cache-ui-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/ui/**/pnpm-lock.yaml') }}
           if-no-files-found: 'error'
           retention-days: '2'
-        if: steps.restore-eslint-cache.outputs.stash-hit != 'true'
+        if: steps.restore-eslint-cache-ui.outputs.stash-hit != 'true'
+      - name: "Restore eslint cache (simple auth manager UI)"
+        uses: apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
+        with:
+          path: airflow/auth/managers/simple/ui/node_modules/
+          key: >
+            cache-simple-am-ui-node-modules-v1-
+            ${{ runner.os }}-${{ hashFiles('airflow/auth/managers/simple/ui/**/pnpm-lock.yaml') }}
+        id: restore-eslint-cache-simple-am-ui
+      - run: cd airflow/auth/managers/simple/ui && pnpm install --frozen-lockfile
+      - run: cd airflow/auth/managers/simple/ui && pnpm test
+        env:
+          FORCE_COLOR: 2
+      - name: "Save eslint cache (ui)"
+        uses: apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
+        with:
+          path: airflow/auth/managers/simple/ui/node_modules/
+          key: >
+            cache-simple-am-ui-node-modules-v1-
+            ${{ runner.os }}-${{ hashFiles('airflow/auth/managers/simple/ui/**/pnpm-lock.yaml') }}
+          if-no-files-found: 'error'
+          retention-days: '2'
+        if: steps.restore-eslint-cache-simple-am-ui.outputs.stash-hit != 'true'
 
   tests-www:
     timeout-minutes: 10

.pre-commit-config.yaml

@@ -627,7 +627,7 @@ repos:
           ^providers/src/airflow/providers/fab/www/static/css/bootstrap-theme.css$|
           ^providers/src/airflow/providers/amazon/aws/hooks/emr.py$|
           ^providers/src/airflow/providers/amazon/aws/operators/emr.py$|
-          ^providers/src/airflow/providers/apache/cassandra/hooks/cassandra.py$|
+          ^providers/apache/cassandra/src/airflow/providers/apache/cassandra/hooks/cassandra.py$|
           ^providers/src/airflow/providers/apache/hive/operators/hive_stats.py$|
           ^providers/src/airflow/providers/apache/hive/transfers/vertica_to_hive.py$|
           ^providers/apache/spark/src/airflow/providers/apache/spark/decorators/|

Dockerfile.ci

@@ -935,17 +935,10 @@ function check_boto_upgrade() {
     # We need to include few dependencies to pass pip check with other dependencies:
     #   * oss2 as dependency as otherwise jmespath will be bumped (sync with alibaba provider)
     #   * cryptography is kept for snowflake-connector-python limitation (sync with snowflake provider)
-    #   * requests needs to be limited to be compatible with apache beam (sync with apache-beam provider)
-    #   * yandexcloud requirements for requests does not match those of apache.beam and latest botocore
-    #   Both requests and yandexcloud exclusion above might be removed after
-    #   https://github.com/apache/beam/issues/32080 is addressed
-    #   This is already addressed and planned for 2.59.0 release.
-    #   When you remove yandexcloud and opensearch from the above list, you can also remove the
-    #   optional providers_dependencies exclusions from "test_example_dags.py" in "tests/always".
     set -x
     # shellcheck disable=SC2086
     ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore \
-       "oss2>=2.14.0" "cryptography<43.0.0" "requests!=2.32.*,<3.0.0,>=2.24.0"
+       "oss2>=2.14.0" "cryptography<43.0.0" "opensearch-py"
     set +x
     pip check
 }

airflow/api_fastapi/core_api/base.py

@@ -27,3 +27,15 @@ class BaseModel(PydanticBaseModel):
     """
 
     model_config = ConfigDict(from_attributes=True, populate_by_name=True)
+
+
+class StrictBaseModel(BaseModel):
+    """
+    StrictBaseModel is a base Pydantic model for REST API that does not allow any extra fields.
+
+    Use this class for models that should not have any extra fields in the payload.
+
+    :meta private:
+    """
+
+    model_config = ConfigDict(from_attributes=True, populate_by_name=True, extra="forbid")

airflow/api_fastapi/core_api/datamodels/assets.py

@@ -21,19 +21,19 @@
 
 from pydantic import Field, field_validator
 
-from airflow.api_fastapi.core_api.base import BaseModel
+from airflow.api_fastapi.core_api.base import BaseModel, StrictBaseModel
 from airflow.utils.log.secrets_masker import redact
 
 
-class DagScheduleAssetReference(BaseModel):
+class DagScheduleAssetReference(StrictBaseModel):
     """DAG schedule reference serializer for assets."""
 
     dag_id: str
     created_at: datetime
     updated_at: datetime
 
 
-class TaskOutletAssetReference(BaseModel):
+class TaskOutletAssetReference(StrictBaseModel):
     """Task outlet reference serializer for assets."""
 
     dag_id: str
@@ -84,7 +84,7 @@ class AssetAliasCollectionResponse(BaseModel):
     total_entries: int
 
 
-class DagRunAssetReference(BaseModel):
+class DagRunAssetReference(StrictBaseModel):
     """DAGRun serializer for asset responses."""
 
     run_id: str
@@ -141,7 +141,7 @@ class QueuedEventCollectionResponse(BaseModel):
     total_entries: int
 
 
-class CreateAssetEventsBody(BaseModel):
+class CreateAssetEventsBody(StrictBaseModel):
     """Create asset events request."""
 
     asset_id: int

airflow/api_fastapi/core_api/datamodels/backfills.py

@@ -19,11 +19,11 @@
 
 from datetime import datetime
 
-from airflow.api_fastapi.core_api.base import BaseModel
+from airflow.api_fastapi.core_api.base import BaseModel, StrictBaseModel
 from airflow.models.backfill import ReprocessBehavior
 
 
-class BackfillPostBody(BaseModel):
+class BackfillPostBody(StrictBaseModel):
     """Object used for create backfill request."""
 
     dag_id: str

airflow/api_fastapi/core_api/datamodels/common.py

@@ -27,7 +27,7 @@
 
 from pydantic import Discriminator, Field, Tag
 
-from airflow.api_fastapi.core_api.base import BaseModel
+from airflow.api_fastapi.core_api.base import BaseModel, StrictBaseModel
 
 # Common Bulk Data Models
 T = TypeVar("T")
@@ -57,7 +57,7 @@ class BulkActionNotOnExistence(enum.Enum):
     SKIP = "skip"
 
 
-class BulkBaseAction(BaseModel, Generic[T]):
+class BulkBaseAction(StrictBaseModel, Generic[T]):
     """Base class for bulk actions."""
 
     action: BulkAction = Field(..., description="The action to be performed on the entities.")
@@ -88,7 +88,7 @@ def _action_discriminator(action: Any) -> str:
     return BulkAction(action["action"]).value
 
 
-class BulkBody(BaseModel, Generic[T]):
+class BulkBody(StrictBaseModel, Generic[T]):
     """Serializer for bulk entity operations."""
 
     actions: list[

airflow/api_fastapi/core_api/datamodels/config.py

@@ -16,10 +16,10 @@
 # under the License.
 from __future__ import annotations
 
-from airflow.api_fastapi.core_api.base import BaseModel
+from airflow.api_fastapi.core_api.base import StrictBaseModel
 
 
-class ConfigOption(BaseModel):
+class ConfigOption(StrictBaseModel):
     """Config option."""
 
     key: str
@@ -32,7 +32,7 @@ def text_format(self):
         return f"{self.key} = {self.value}"
 
 
-class ConfigSection(BaseModel):
+class ConfigSection(StrictBaseModel):
     """Config Section Schema."""
 
     name: str
@@ -53,7 +53,7 @@ def text_format(self):
         return f"[{self.name}]\n" + "\n".join(option.text_format for option in self.options) + "\n"
 
 
-class Config(BaseModel):
+class Config(StrictBaseModel):
     """List of config sections with their options."""
 
     sections: list[ConfigSection]

airflow/api_fastapi/core_api/datamodels/connections.py

@@ -22,7 +22,7 @@
 from pydantic import Field, field_validator
 from pydantic_core.core_schema import ValidationInfo
 
-from airflow.api_fastapi.core_api.base import BaseModel
+from airflow.api_fastapi.core_api.base import BaseModel, StrictBaseModel
 from airflow.utils.log.secrets_masker import redact
 
 
@@ -76,7 +76,7 @@ class ConnectionTestResponse(BaseModel):
 
 
 # Request Models
-class ConnectionBody(BaseModel):
+class ConnectionBody(StrictBaseModel):
     """Connection Serializer for requests body."""
 
     connection_id: str = Field(serialization_alias="conn_id", max_length=200, pattern=r"^[\w.-]+$")

airflow/api_fastapi/core_api/datamodels/dag_run.py

@@ -22,7 +22,7 @@
 
 from pydantic import AwareDatetime, Field, NonNegativeInt, computed_field, model_validator
 
-from airflow.api_fastapi.core_api.base import BaseModel
+from airflow.api_fastapi.core_api.base import BaseModel, StrictBaseModel
 from airflow.models import DagRun
 from airflow.utils import timezone
 from airflow.utils.state import DagRunState
@@ -37,14 +37,14 @@ class DAGRunPatchStates(str, Enum):
     FAILED = DagRunState.FAILED
 
 
-class DAGRunPatchBody(BaseModel):
+class DAGRunPatchBody(StrictBaseModel):
     """DAG Run Serializer for PATCH requests."""
 
     state: DAGRunPatchStates | None = None
     note: str | None = Field(None, max_length=1000)
 
 
-class DAGRunClearBody(BaseModel):
+class DAGRunClearBody(StrictBaseModel):
     """DAG Run serializer for clear endpoint body."""
 
     dry_run: bool = True
@@ -78,7 +78,7 @@ class DAGRunCollectionResponse(BaseModel):
     total_entries: int
 
 
-class TriggerDAGRunPostBody(BaseModel):
+class TriggerDAGRunPostBody(StrictBaseModel):
     """Trigger DAG Run Serializer for POST body."""
 
     dag_run_id: str | None = None
@@ -109,7 +109,7 @@ def logical_date(self) -> datetime:
         return timezone.utcnow()
 
 
-class DAGRunsBatchBody(BaseModel):
+class DAGRunsBatchBody(StrictBaseModel):
     """List DAG Runs body for batch endpoint."""
 
     order_by: str | None = None

airflow/api_fastapi/core_api/datamodels/dags.py

@@ -31,7 +31,7 @@
     field_validator,
 )
 
-from airflow.api_fastapi.core_api.base import BaseModel
+from airflow.api_fastapi.core_api.base import BaseModel, StrictBaseModel
 from airflow.api_fastapi.core_api.datamodels.dag_tags import DagTagResponse
 from airflow.configuration import conf
 
@@ -92,7 +92,7 @@ def file_token(self) -> str:
         return serializer.dumps(self.fileloc)
 
 
-class DAGPatchBody(BaseModel):
+class DAGPatchBody(StrictBaseModel):
     """Dag Serializer for updatable bodies."""
 
     is_paused: bool