From 5c905b2a2d94861b38db1bc6eaf6346fa828fefe Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Sun, 12 Nov 2023 13:55:18 +0100 Subject: [PATCH 001/286] feat: Implement `auth_kwargs` parameter in Http Connection --- airflow/providers/http/hooks/http.py | 13 +++++++++---- tests/providers/http/hooks/test_http.py | 24 ++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 5b6063a3f7d7c..f19ba8cd7f7fe 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -48,7 +48,6 @@ class HttpHook(BaseHook): :param tcp_keep_alive_count: The TCP Keep Alive count parameter (corresponds to ``socket.TCP_KEEPCNT``) :param tcp_keep_alive_interval: The TCP Keep Alive interval parameter (corresponds to ``socket.TCP_KEEPINTVL``) - :param auth_args: extra arguments used to initialize the auth_type if different than default HTTPBasicAuth """ conn_name_attr = "http_conn_id" @@ -107,13 +106,19 @@ def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: if conn.port: self.base_url += f":{conn.port}" - if conn.login: - session.auth = self.auth_type(conn.login, conn.password) + + conn_extra: dict = conn.extra_dejson.copy() + auth_args: list[str | None] = [conn.login, conn.password] + auth_kwargs: dict[str, Any] = conn_extra.pop("auth_kwargs", {}) + + if any(auth_args) or auth_kwargs: + session.auth = self.auth_type(*auth_args, **auth_kwargs) elif self._auth_type: session.auth = self.auth_type() + if conn.extra: try: - session.headers.update(conn.extra_dejson) + session.headers.update(conn_extra) except TypeError: self.log.warning("Connection to %s has invalid extra field.", conn.host) if headers: diff --git a/tests/providers/http/hooks/test_http.py b/tests/providers/http/hooks/test_http.py index 617009d5750d6..1d6d639cdd377 100644 --- a/tests/providers/http/hooks/test_http.py +++ b/tests/providers/http/hooks/test_http.py @@ -60,6 +60,11 @@ def get_airflow_connection_with_login_and_password(unused_conn_id=None): ) +class CustomAuthBase(HTTPBasicAuth): + def __init__(self, username: str, password: str, endpoint: str): + super().__init__(username, password) + + class TestHttpHook: """Test get, post and raise_for_status""" @@ -268,6 +273,25 @@ def test_connection_without_host(self, mock_get_connection): hook.get_conn({}) assert hook.base_url == "http://" + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_connection): + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + login="username", + password="pass", + extra='{"auth_kwargs": {"endpoint": "http://localhost"}}', + ) + mock_get_connection.return_value = conn + + hook = HttpHook(auth_type=CustomAuthBase) + session = hook.get_conn({}) + + auth.assert_called_once_with("username", "pass", endpoint="http://localhost") + assert "auth_kwargs" not in session.headers + @pytest.mark.parametrize("method", ["GET", "POST"]) def test_json_request(self, method, requests_mock): obj1 = {"a": 1, "b": "abc", "c": [1, 2, {"d": 10}]} From 1c7fcabdf33aa61d4d0051512e6de52d419722cf Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Thu, 23 Nov 2023 22:29:25 +0100 Subject: [PATCH 002/286] feat: Make auth_type configurable from Connection --- airflow/providers/http/hooks/http.py | 34 ++++++++++++++++++---------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index f19ba8cd7f7fe..b0dd00e399974 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -18,6 +18,7 @@ from __future__ import annotations import asyncio +from importlib import import_module from typing import TYPE_CHECKING, Any, Callable import aiohttp @@ -70,20 +71,13 @@ def __init__( self.method = method.upper() self.base_url: str = "" self._retry_obj: Callable[..., Any] - self._auth_type: Any = auth_type + self._is_auth_type_setup: bool = auth_type is not None + self.auth_type: Any = auth_type or HTTPBasicAuth self.tcp_keep_alive = tcp_keep_alive self.keep_alive_idle = tcp_keep_alive_idle self.keep_alive_count = tcp_keep_alive_count self.keep_alive_interval = tcp_keep_alive_interval - @property - def auth_type(self): - return self._auth_type or HTTPBasicAuth - - @auth_type.setter - def auth_type(self, v): - self._auth_type = v - # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: @@ -108,13 +102,15 @@ def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: self.base_url += f":{conn.port}" conn_extra: dict = conn.extra_dejson.copy() + conn_auth_type: Any = self._conn_auth_type(conn_extra=conn_extra) + auth_type = self.auth_type or conn_auth_type auth_args: list[str | None] = [conn.login, conn.password] auth_kwargs: dict[str, Any] = conn_extra.pop("auth_kwargs", {}) if any(auth_args) or auth_kwargs: - session.auth = self.auth_type(*auth_args, **auth_kwargs) - elif self._auth_type: - session.auth = self.auth_type() + session.auth = auth_type(*auth_args, **auth_kwargs) + elif self._is_auth_type_setup: + session.auth = auth_type() if conn.extra: try: @@ -126,6 +122,20 @@ def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: return session + def _conn_auth_type(self, conn_extra: dict) -> Any: + """Load auth_type module from extra Connection parameters.""" + module_name: str | None = conn_extra.pop("auth_type", None) + if module_name: + try: + module = import_module(module_name) + self._is_auth_type_setup = True + self.log.info("Loaded auth_type: %s", module_name) + return module + except ImportError as error: + self.log.debug("Cannot import auth_type '%s' due to: %s", error) + raise AirflowException(error) + return None + def run( self, endpoint: str | None = None, From 4d43f63c7b0e582210d13870d45f7fcfa64b4208 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 24 Nov 2023 00:25:53 +0100 Subject: [PATCH 003/286] fix: Correctly use auth_type from Connection --- airflow/providers/http/hooks/http.py | 15 +++++------ tests/providers/http/hooks/test_http.py | 36 ++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index b0dd00e399974..eb530ec98f465 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -18,7 +18,6 @@ from __future__ import annotations import asyncio -from importlib import import_module from typing import TYPE_CHECKING, Any, Callable import aiohttp @@ -31,6 +30,7 @@ from airflow.exceptions import AirflowException from airflow.hooks.base import BaseHook +from airflow.utils.module_loading import import_string if TYPE_CHECKING: from aiohttp.client_reqrep import ClientResponse @@ -72,7 +72,7 @@ def __init__( self.base_url: str = "" self._retry_obj: Callable[..., Any] self._is_auth_type_setup: bool = auth_type is not None - self.auth_type: Any = auth_type or HTTPBasicAuth + self.auth_type: Any = auth_type self.tcp_keep_alive = tcp_keep_alive self.keep_alive_idle = tcp_keep_alive_idle self.keep_alive_count = tcp_keep_alive_count @@ -102,8 +102,8 @@ def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: self.base_url += f":{conn.port}" conn_extra: dict = conn.extra_dejson.copy() - conn_auth_type: Any = self._conn_auth_type(conn_extra=conn_extra) - auth_type = self.auth_type or conn_auth_type + conn_auth_type_name: str | None = conn_extra.pop("auth_type", None) + auth_type: Any = self.auth_type or self._conn_auth_type(conn_auth_type_name) or HTTPBasicAuth auth_args: list[str | None] = [conn.login, conn.password] auth_kwargs: dict[str, Any] = conn_extra.pop("auth_kwargs", {}) @@ -112,7 +112,7 @@ def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: elif self._is_auth_type_setup: session.auth = auth_type() - if conn.extra: + if conn_extra: try: session.headers.update(conn_extra) except TypeError: @@ -122,12 +122,11 @@ def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: return session - def _conn_auth_type(self, conn_extra: dict) -> Any: + def _conn_auth_type(self, module_name: str | None) -> Any: """Load auth_type module from extra Connection parameters.""" - module_name: str | None = conn_extra.pop("auth_type", None) if module_name: try: - module = import_module(module_name) + module = import_string(module_name) self._is_auth_type_setup = True self.log.info("Loaded auth_type: %s", module_name) return module diff --git a/tests/providers/http/hooks/test_http.py b/tests/providers/http/hooks/test_http.py index 1d6d639cdd377..0e5162c5c0b65 100644 --- a/tests/providers/http/hooks/test_http.py +++ b/tests/providers/http/hooks/test_http.py @@ -282,7 +282,7 @@ def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_conne conn_type="http", login="username", password="pass", - extra='{"auth_kwargs": {"endpoint": "http://localhost"}}', + extra='{"x-header": 0, "auth_kwargs": {"endpoint": "http://localhost"}}', ) mock_get_connection.return_value = conn @@ -291,6 +291,40 @@ def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_conne auth.assert_called_once_with("username", "pass", endpoint="http://localhost") assert "auth_kwargs" not in session.headers + assert "x-header" in session.headers + + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_header_and_auth_type(self, auth, mock_get_connection): + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + login="username", + password="pass", + extra='{"x-header": 0, "auth_type": "tests.providers.http.hooks.test_http.CustomAuthBase"}', + ) + mock_get_connection.return_value = conn + + session = HttpHook().get_conn({}) + auth.assert_called_once_with("username", "pass") + assert isinstance(session.auth, CustomAuthBase) + assert "auth_type" not in session.headers + assert "x-header" in session.headers + + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_auth_type_and_no_credentials(self, auth, mock_get_connection): + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + extra='{"auth_type": "tests.providers.http.hooks.test_http.CustomAuthBase"}', + ) + mock_get_connection.return_value = conn + + HttpHook().get_conn({}) + auth.assert_called_once() @pytest.mark.parametrize("method", ["GET", "POST"]) def test_json_request(self, method, requests_mock): From d5d3b3ebafa75d02876d7796b55462e9076029ef Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 24 Nov 2023 01:55:44 +0100 Subject: [PATCH 004/286] feat: Add Connection documentation --- airflow/providers/http/hooks/http.py | 6 ++++++ .../apache-airflow-providers-http/connections/http.rst | 10 +++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index eb530ec98f465..33ab8484e622b 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -39,6 +39,12 @@ class HttpHook(BaseHook): """Interact with HTTP servers. + To configure the auth_type, in addition to the `auth_type` parameter, you can also: + * set the `auth_type` parameter in the Connection settings. + * define extra parameters used to instantiate the `auth_type` class, in the Connection settings. + + See :doc:`/connections/http` for full documentation. + :param method: the API method to be called :param http_conn_id: :ref:`http connection` that has the base API url i.e https://www.google.com/ and optional authentication credentials. Default diff --git a/docs/apache-airflow-providers-http/connections/http.rst b/docs/apache-airflow-providers-http/connections/http.rst index 41856cefee7d8..53a90db51bc06 100644 --- a/docs/apache-airflow-providers-http/connections/http.rst +++ b/docs/apache-airflow-providers-http/connections/http.rst @@ -54,7 +54,15 @@ Schema (optional) Specify the service type etc: http/https. Extras (optional) - Specify headers in json format. + Any key / value parameters supplied here will be added to the headers in json format. + + Additionally there a few special optional keywords that are handled separately. + + - ``auth_type`` + * The path to the Authentication class to attach the request.Session object. Typically a subclass of + ``request.auth.AuthBase``. This is omitted if the HttpHook is declared with a ``auth_type``. + - ``auth_kwargs`` + * Extra key-value parameters used to instantiate the ``auth_type`` class. When specifying the connection in environment variable you should specify it using URI syntax. From e9848f56f6b287f435bbff126a439e748f6f03cd Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 5 Dec 2023 23:14:43 +0100 Subject: [PATCH 005/286] feat: Make available auth_types configurable from airflow config --- airflow/providers/http/hooks/http.py | 55 +++++++++++++++---- airflow/providers/http/provider.yaml | 15 +++++ .../configurations-ref.rst | 18 ++++++ docs/apache-airflow-providers-http/index.rst | 1 + tests/providers/http/hooks/test_http.py | 28 +++++++++- 5 files changed, 105 insertions(+), 12 deletions(-) create mode 100644 docs/apache-airflow-providers-http/configurations-ref.rst diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 33ab8484e622b..37187b415c0da 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -28,6 +28,7 @@ from requests.auth import HTTPBasicAuth from requests_toolbelt.adapters.socket_options import TCPKeepAliveAdapter +from airflow.compat.functools import cache from airflow.exceptions import AirflowException from airflow.hooks.base import BaseHook from airflow.utils.module_loading import import_string @@ -36,12 +37,37 @@ from aiohttp.client_reqrep import ClientResponse +DEFAULT_AUTH_TYPES = frozenset( + { + "request.auth.HTTPBasicAuth", + "request.auth.HTTPProxyAuth", + "request.auth.HTTPDigestAuth", + } +) + + +@cache +def get_auth_types() -> frozenset[str]: + """Get comma-separated extra auth_types from airflow config. + + Those auth_types can then be used in Connection configuration. + """ + from airflow.configuration import conf + + auth_types = DEFAULT_AUTH_TYPES.copy() + extra_auth_types = conf.get("http", "extra_auth_types", fallback=None) + if extra_auth_types: + auth_types |= frozenset({field.strip() for field in extra_auth_types.split(",")}) + return auth_types + + class HttpHook(BaseHook): """Interact with HTTP servers. To configure the auth_type, in addition to the `auth_type` parameter, you can also: * set the `auth_type` parameter in the Connection settings. - * define extra parameters used to instantiate the `auth_type` class, in the Connection settings. + * define extra parameters passed to the `auth_type` class via the `auth_kwargs`, in the Connection + settings. The class will be instantiated with those parameters. See :doc:`/connections/http` for full documentation. @@ -109,7 +135,7 @@ def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: conn_extra: dict = conn.extra_dejson.copy() conn_auth_type_name: str | None = conn_extra.pop("auth_type", None) - auth_type: Any = self.auth_type or self._conn_auth_type(conn_auth_type_name) or HTTPBasicAuth + auth_type: Any = self.auth_type or self._load_conn_auth_type(conn_auth_type_name) or HTTPBasicAuth auth_args: list[str | None] = [conn.login, conn.password] auth_kwargs: dict[str, Any] = conn_extra.pop("auth_kwargs", {}) @@ -128,17 +154,24 @@ def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: return session - def _conn_auth_type(self, module_name: str | None) -> Any: + def _load_conn_auth_type(self, module_name: str | None) -> Any: """Load auth_type module from extra Connection parameters.""" if module_name: - try: - module = import_string(module_name) - self._is_auth_type_setup = True - self.log.info("Loaded auth_type: %s", module_name) - return module - except ImportError as error: - self.log.debug("Cannot import auth_type '%s' due to: %s", error) - raise AirflowException(error) + if module_name in get_auth_types(): + try: + module = import_string(module_name) + self._is_auth_type_setup = True + self.log.info("Loaded auth_type: %s", module_name) + return module + except ImportError as error: + self.log.debug("Cannot import auth_type '%s' due to: %s", error) + raise AirflowException(error) + else: + self.log.warning( + "Skipping import of auth_type '%s'. The class should be listed in " + "'extra_auth_types' config of the http provider.", + module_name, + ) return None def run( diff --git a/airflow/providers/http/provider.yaml b/airflow/providers/http/provider.yaml index d193955808473..47982186f142b 100644 --- a/airflow/providers/http/provider.yaml +++ b/airflow/providers/http/provider.yaml @@ -89,3 +89,18 @@ triggers: connection-types: - hook-class-name: airflow.providers.http.hooks.http.HttpHook connection-type: http + +config: + http: + description: "Options for Http provider." + options: + extra_auth_types: + description: | + A comma separated list of auth_type classes, which can be used to + configure Http Connections in Airflow's UI. This list restricts which + classes can be arbitrary imported, and protects from dependency + injections. + type: string + version_added: 4.8.0 + example: "requests_kerberos.HTTPKerberosAuth,any.other.custom.HTTPAuth" + default: ~ diff --git a/docs/apache-airflow-providers-http/configurations-ref.rst b/docs/apache-airflow-providers-http/configurations-ref.rst new file mode 100644 index 0000000000000..5885c9d91b6e8 --- /dev/null +++ b/docs/apache-airflow-providers-http/configurations-ref.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../exts/includes/providers-configurations-ref.rst diff --git a/docs/apache-airflow-providers-http/index.rst b/docs/apache-airflow-providers-http/index.rst index 5730d2969793a..1f19700066a23 100644 --- a/docs/apache-airflow-providers-http/index.rst +++ b/docs/apache-airflow-providers-http/index.rst @@ -42,6 +42,7 @@ :maxdepth: 1 :caption: References + Configuration Python API <_api/airflow/providers/http/index> .. toctree:: diff --git a/tests/providers/http/hooks/test_http.py b/tests/providers/http/hooks/test_http.py index 0e5162c5c0b65..870169b00dbe6 100644 --- a/tests/providers/http/hooks/test_http.py +++ b/tests/providers/http/hooks/test_http.py @@ -34,7 +34,7 @@ from airflow.exceptions import AirflowException from airflow.models import Connection -from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook +from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook, get_auth_types @pytest.fixture @@ -65,6 +65,9 @@ def __init__(self, username: str, password: str, endpoint: str): super().__init__(username, password) +@mock.patch.dict( + "os.environ", AIRFLOW__HTTP__EXTRA_AUTH_TYPES="tests.providers.http.hooks.test_http.CustomAuthBase" +) class TestHttpHook: """Test get, post and raise_for_status""" @@ -293,6 +296,29 @@ def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_conne assert "auth_kwargs" not in session.headers assert "x-header" in session.headers + def test_available_connection_auth_types(self): + auth_types = get_auth_types() + assert auth_types == frozenset( + { + "request.auth.HTTPBasicAuth", + "request.auth.HTTPProxyAuth", + "request.auth.HTTPDigestAuth", + "tests.providers.http.hooks.test_http.CustomAuthBase", + } + ) + + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + def test_connection_with_invalid_auth_type_get_skipped(self, mock_get_connection, caplog): + auth_type: str = "auth_type.class.not.available.for.Import" + conn = Connection( + conn_id="http_default", + conn_type="http", + extra=f'{{"auth_type": "{auth_type}"}}', + ) + mock_get_connection.return_value = conn + HttpHook().get_conn({}) + assert f"Skipping import of auth_type '{auth_type}'." in caplog.text + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") def test_connection_with_extra_header_and_auth_type(self, auth, mock_get_connection): From 6580407e9f744b36c27dae902328b47558b752da Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Wed, 6 Dec 2023 22:24:25 +0100 Subject: [PATCH 006/286] feat: Add fields for auth config and header config in Http Connection form --- airflow/providers/http/hooks/http.py | 24 ++++++++++++++++++++++++ airflow/providers_manager.py | 4 ++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 37187b415c0da..67a565432d2b7 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -110,6 +110,30 @@ def __init__( self.keep_alive_count = tcp_keep_alive_count self.keep_alive_interval = tcp_keep_alive_interval + @classmethod + def get_connection_form_widgets(cls) -> dict[str, Any]: + """Return connection widgets to add to connection form.""" + from flask_babel import lazy_gettext + from wtforms.fields import SelectField, TextAreaField + + auth_types_choices = frozenset({""}) | get_auth_types() + return { + "auth_type": SelectField( + lazy_gettext("Auth type"), + choices=[(clazz, clazz) for clazz in auth_types_choices] + ), + "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs")), + "extra_headers": TextAreaField(lazy_gettext("Extra Headers")) + } + + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + """Return custom field behaviour.""" + return { + "hidden_fields": ["extra"], + "relabeling": {} + } + # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py index 979bd2ecf17e0..e825f8e72533b 100644 --- a/airflow/providers_manager.py +++ b/airflow/providers_manager.py @@ -918,7 +918,7 @@ def _import_hook( :param package_name: provider package - only needed in case connection_type is missing : return """ - from wtforms import BooleanField, IntegerField, PasswordField, StringField + from wtforms import BooleanField, IntegerField, PasswordField, StringField, TextAreaField, SelectField if connection_type is None and hook_class_name is None: raise ValueError("Either connection_type or hook_class_name must be set") @@ -938,7 +938,7 @@ def _import_hook( raise ValueError( f"Provider package name is not set when hook_class_name ({hook_class_name}) is used" ) - allowed_field_classes = [IntegerField, PasswordField, StringField, BooleanField] + allowed_field_classes = [IntegerField, PasswordField, StringField, BooleanField, TextAreaField, SelectField] hook_class = _correctness_check(package_name, hook_class_name, provider_info) if hook_class is None: return None From 82038563f5eaf31330606b67363fe4d4f4fe927f Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Thu, 7 Dec 2023 08:48:01 +0100 Subject: [PATCH 007/286] fix: Correctly apply styling to extra fields --- airflow/providers/http/hooks/http.py | 13 ++++++------- airflow/providers_manager.py | 11 +++++++++-- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 67a565432d2b7..9008b084a737d 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -113,6 +113,7 @@ def __init__( @classmethod def get_connection_form_widgets(cls) -> dict[str, Any]: """Return connection widgets to add to connection form.""" + from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget, Select2Widget from flask_babel import lazy_gettext from wtforms.fields import SelectField, TextAreaField @@ -120,19 +121,17 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: return { "auth_type": SelectField( lazy_gettext("Auth type"), - choices=[(clazz, clazz) for clazz in auth_types_choices] + choices=[(clazz, clazz) for clazz in auth_types_choices], + widget=Select2Widget(), ), - "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs")), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers")) + "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), + "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } @classmethod def get_ui_field_behaviour(cls) -> dict[str, Any]: """Return custom field behaviour.""" - return { - "hidden_fields": ["extra"], - "relabeling": {} - } + return {"hidden_fields": ["extra"], "relabeling": {}} # headers may be passed through directly or in the "extra" field in the connection # definition diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py index e825f8e72533b..c078c8cddb9eb 100644 --- a/airflow/providers_manager.py +++ b/airflow/providers_manager.py @@ -918,7 +918,7 @@ def _import_hook( :param package_name: provider package - only needed in case connection_type is missing : return """ - from wtforms import BooleanField, IntegerField, PasswordField, StringField, TextAreaField, SelectField + from wtforms import BooleanField, IntegerField, PasswordField, SelectField, StringField, TextAreaField if connection_type is None and hook_class_name is None: raise ValueError("Either connection_type or hook_class_name must be set") @@ -938,7 +938,14 @@ def _import_hook( raise ValueError( f"Provider package name is not set when hook_class_name ({hook_class_name}) is used" ) - allowed_field_classes = [IntegerField, PasswordField, StringField, BooleanField, TextAreaField, SelectField] + allowed_field_classes = [ + IntegerField, + PasswordField, + StringField, + BooleanField, + TextAreaField, + SelectField, + ] hook_class = _correctness_check(package_name, hook_class_name, provider_info) if hook_class is None: return None From 70f92a7e1a2304815675c4e1c77a7d082933ca61 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 19 Dec 2023 01:05:47 +0100 Subject: [PATCH 008/286] feat: Implement simplistic collapsable textarea for "extra" --- airflow/providers/http/hooks/http.py | 9 +++----- airflow/www/forms.py | 26 +++++++++++++++++++++++- airflow/www/static/js/connection_form.js | 4 ++-- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 9008b084a737d..3f3f2418eac61 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -25,6 +25,8 @@ import tenacity from aiohttp import ClientResponseError from asgiref.sync import sync_to_async +from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget +from markupsafe import Markup from requests.auth import HTTPBasicAuth from requests_toolbelt.adapters.socket_options import TCPKeepAliveAdapter @@ -125,14 +127,9 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: widget=Select2Widget(), ), "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), + "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()) } - @classmethod - def get_ui_field_behaviour(cls) -> dict[str, Any]: - """Return custom field behaviour.""" - return {"hidden_fields": ["extra"], "relabeling": {}} - # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: diff --git a/airflow/www/forms.py b/airflow/www/forms.py index 8a8f69cf4455c..170da52d9859b 100644 --- a/airflow/www/forms.py +++ b/airflow/www/forms.py @@ -32,6 +32,7 @@ from flask_appbuilder.forms import DynamicForm from flask_babel import lazy_gettext from flask_wtf import FlaskForm +from markupsafe import Markup from wtforms import widgets from wtforms.fields import Field, IntegerField, PasswordField, SelectField, StringField, TextAreaField from wtforms.validators import InputRequired, Optional @@ -194,6 +195,29 @@ def populate_obj(self, item): field.populate_obj(item, name) +class BS3AccordionTextAreaFieldWidget(BS3TextAreaFieldWidget): + + @staticmethod + def _make_collapsable_panel(field: Field, content: Markup) -> str: + collapsable_id: str = f"collapsable_{field.id}" + return f""" +
+
+

+ +

+
+ +
+ """ + + def __call__(self, field, **kwargs): + text_area = super(BS3TextAreaFieldWidget, self).__call__(field, **kwargs) + return self._make_collapsable_panel(field=field, content=text_area) + + @cache def create_connection_form_class() -> type[DynamicForm]: """Create a form class for editing and adding Connection. @@ -240,7 +264,7 @@ def process(self, formdata=None, obj=None, **kwargs): login = StringField(lazy_gettext("Login"), widget=BS3TextFieldWidget()) password = PasswordField(lazy_gettext("Password"), widget=BS3PasswordFieldWidget()) port = IntegerField(lazy_gettext("Port"), validators=[Optional()], widget=BS3TextFieldWidget()) - extra = TextAreaField(lazy_gettext("Extra"), widget=BS3TextAreaFieldWidget()) + extra = TextAreaField(lazy_gettext("Extra"), widget=BS3AccordionTextAreaFieldWidget()) for key, value in providers_manager.connection_form_widgets.items(): setattr(ConnectionForm, key, value.field) diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index e59ef9a1501c8..ccb36a29f08e0 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -83,7 +83,7 @@ function restoreFieldBehaviours() { } ); - Array.from(document.querySelectorAll(".form-control")).forEach((elem) => { + Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach((elem) => { // eslint-disable-next-line no-param-reassign elem.placeholder = ""; elem.parentElement.parentElement.classList.remove("hide"); @@ -101,7 +101,7 @@ function applyFieldBehaviours(connection) { if (Array.isArray(connection.hidden_fields)) { connection.hidden_fields.forEach((field) => { document - .getElementById(field) + .querySelector(`label[for='${field}']`) .parentElement.parentElement.classList.add("hide"); }); } From db2a8594f2b0e48a7f05b468371c1c8be39a476a Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Wed, 20 Dec 2023 16:21:08 +0100 Subject: [PATCH 009/286] fix: express clearly empty frozenset creation Goal is to have an empty default choice --- airflow/providers/http/hooks/http.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 3f3f2418eac61..fe8d7f6a5492d 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -119,7 +119,8 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: from flask_babel import lazy_gettext from wtforms.fields import SelectField, TextAreaField - auth_types_choices = frozenset({""}) | get_auth_types() + default_auth_type = frozenset({""}) + auth_types_choices = default_auth_type | get_auth_types() return { "auth_type": SelectField( lazy_gettext("Auth type"), From d14c80f346b7f3a06fe6ba1ceb739fcfd52ccdb6 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Wed, 20 Dec 2023 16:43:29 +0100 Subject: [PATCH 010/286] feat: Refactor Accordion TextArea to use wtform utils --- airflow/providers/http/hooks/http.py | 4 +-- airflow/www/forms.py | 36 ++++++++++++++++++------ airflow/www/static/js/connection_form.js | 12 ++++---- 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index fe8d7f6a5492d..5e3dda7ca70cd 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -25,8 +25,6 @@ import tenacity from aiohttp import ClientResponseError from asgiref.sync import sync_to_async -from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget -from markupsafe import Markup from requests.auth import HTTPBasicAuth from requests_toolbelt.adapters.socket_options import TCPKeepAliveAdapter @@ -128,7 +126,7 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: widget=Select2Widget(), ), "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()) + "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } # headers may be passed through directly or in the "extra" field in the connection diff --git a/airflow/www/forms.py b/airflow/www/forms.py index 170da52d9859b..ceb453fd2eba7 100644 --- a/airflow/www/forms.py +++ b/airflow/www/forms.py @@ -20,7 +20,7 @@ import datetime import json import operator -from typing import Iterator +from typing import TYPE_CHECKING, Iterator import pendulum from flask_appbuilder.fieldwidgets import ( @@ -32,10 +32,10 @@ from flask_appbuilder.forms import DynamicForm from flask_babel import lazy_gettext from flask_wtf import FlaskForm -from markupsafe import Markup from wtforms import widgets from wtforms.fields import Field, IntegerField, PasswordField, SelectField, StringField, TextAreaField from wtforms.validators import InputRequired, Optional +from wtforms.widgets.core import html_params from airflow.compat.functools import cache from airflow.configuration import conf @@ -50,6 +50,9 @@ BS3TextFieldROWidget, ) +if TYPE_CHECKING: + from markupsafe import Markup + class DateTimeWithTimezoneField(Field): """A text field which stores a `datetime.datetime` matching a format.""" @@ -196,18 +199,33 @@ def populate_obj(self, item): class BS3AccordionTextAreaFieldWidget(BS3TextAreaFieldWidget): + """Collapsable Text Area Field Widget. + + Text Area Field encapsulated into an accordion, which allows to "Show" and + "Hide" the field within a form. + """ @staticmethod - def _make_collapsable_panel(field: Field, content: Markup) -> str: + def _make_accordion_panel(field: Field, content: Markup) -> str: collapsable_id: str = f"collapsable_{field.id}" return f""" -
-
-

- +
+
+

+ Show

- @@ -215,7 +233,7 @@ def _make_collapsable_panel(field: Field, content: Markup) -> str: def __call__(self, field, **kwargs): text_area = super(BS3TextAreaFieldWidget, self).__call__(field, **kwargs) - return self._make_collapsable_panel(field=field, content=text_area) + return self._make_accordion_panel(field=field, content=text_area) @cache diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index ccb36a29f08e0..00de949d1acef 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -83,11 +83,13 @@ function restoreFieldBehaviours() { } ); - Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach((elem) => { - // eslint-disable-next-line no-param-reassign - elem.placeholder = ""; - elem.parentElement.parentElement.classList.remove("hide"); - }); + Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach( + (elem) => { + // eslint-disable-next-line no-param-reassign + elem.placeholder = ""; + elem.parentElement.parentElement.classList.remove("hide"); + } + ); } /** From 49c692eba8f0ffea3a73b7b88656b0a037892b15 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Thu, 21 Dec 2023 13:04:19 +0100 Subject: [PATCH 011/286] feat: Implement 'collapse_extra' field behavior --- ...stomized_form_field_behaviours.schema.json | 4 +++ airflow/providers/http/hooks/http.py | 4 +++ airflow/www/forms.py | 35 +++++++++++-------- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/airflow/customized_form_field_behaviours.schema.json b/airflow/customized_form_field_behaviours.schema.json index 78791a87886c1..fa5ace958c5e8 100644 --- a/airflow/customized_form_field_behaviours.schema.json +++ b/airflow/customized_form_field_behaviours.schema.json @@ -22,6 +22,10 @@ "additionalProperties": { "type": "string" } + }, + "collapse_extra": { + "type": "boolean", + "description": "Collapse the 'Extra' field." } }, "additionalProperties": true, diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 5e3dda7ca70cd..1906553930a5b 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -129,6 +129,10 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + return {"hidden_fields": [], "relabeling": {}, "collapse_extra": True} + # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: diff --git a/airflow/www/forms.py b/airflow/www/forms.py index ceb453fd2eba7..cee5d0ba5eb07 100644 --- a/airflow/www/forms.py +++ b/airflow/www/forms.py @@ -198,16 +198,18 @@ def populate_obj(self, item): field.populate_obj(item, name) -class BS3AccordionTextAreaFieldWidget(BS3TextAreaFieldWidget): - """Collapsable Text Area Field Widget. +class BS3CollapsibleTextAreaFieldWidget(BS3TextAreaFieldWidget): + """Collapsible Text Area Field Widget. - Text Area Field encapsulated into an accordion, which allows to "Show" and + Text Area Field encapsulated into an accordion/collapsible, which allows to "Show" and "Hide" the field within a form. """ - @staticmethod - def _make_accordion_panel(field: Field, content: Markup) -> str: - collapsable_id: str = f"collapsable_{field.id}" + def __init__(self, default_expanded: bool = True): + self._expanded = default_expanded + + def _make_accordion_panel(self, field: Field, content: Markup) -> str: + collapsible_id: str = f"collapsible_{field.id}" return f"""
@@ -215,16 +217,21 @@ def _make_accordion_panel(field: Field, content: Markup) -> str: Show + aria_expanded=self._expanded, + aria_controls=collapsible_id, + href='#' + collapsible_id + )}> + Show +

{content.__html__()}
@@ -282,7 +289,7 @@ def process(self, formdata=None, obj=None, **kwargs): login = StringField(lazy_gettext("Login"), widget=BS3TextFieldWidget()) password = PasswordField(lazy_gettext("Password"), widget=BS3PasswordFieldWidget()) port = IntegerField(lazy_gettext("Port"), validators=[Optional()], widget=BS3TextFieldWidget()) - extra = TextAreaField(lazy_gettext("Extra"), widget=BS3AccordionTextAreaFieldWidget()) + extra = TextAreaField(lazy_gettext("Extra"), widget=BS3CollapsibleTextAreaFieldWidget()) for key, value in providers_manager.connection_form_widgets.items(): setattr(ConnectionForm, key, value.field) From 09e50574610d0af8de1807dddeac4bc315433927 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Sat, 30 Dec 2023 16:43:47 +0100 Subject: [PATCH 012/286] feat: Implement parameterizable behavior for collapsible field --- ...stomized_form_field_behaviours.schema.json | 19 +++++- airflow/providers/http/hooks/http.py | 2 +- airflow/www/forms.py | 62 ++++++++++++------- airflow/www/static/css/connection.css | 23 +++++++ airflow/www/static/js/connection_form.js | 49 ++++++++++++--- .../www/templates/airflow/conn_create.html | 2 +- airflow/www/templates/airflow/conn_edit.html | 1 + airflow/www/webpack.config.js | 1 + 8 files changed, 125 insertions(+), 34 deletions(-) create mode 100644 airflow/www/static/css/connection.css diff --git a/airflow/customized_form_field_behaviours.schema.json b/airflow/customized_form_field_behaviours.schema.json index fa5ace958c5e8..8aa05945ebb01 100644 --- a/airflow/customized_form_field_behaviours.schema.json +++ b/airflow/customized_form_field_behaviours.schema.json @@ -23,9 +23,22 @@ "type": "string" } }, - "collapse_extra": { - "type": "boolean", - "description": "Collapse the 'Extra' field." + "collapsible_fields": { + "description": "List of collapsed fields for the hook, with their properties.", + "type": "object", + "patternProperties": { + "\"^.*$\"": { + "description": "Name of the field to enable collapsing.", + "type": "object", + "properties": { + "expanded": { + "description": "Set the default state of the field as expanded.", + "default": true, + "type": "boolean" + } + } + } + } } }, "additionalProperties": true, diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 1906553930a5b..69e003fa21333 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -131,7 +131,7 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: @classmethod def get_ui_field_behaviour(cls) -> dict[str, Any]: - return {"hidden_fields": [], "relabeling": {}, "collapse_extra": True} + return {"hidden_fields": [], "relabeling": {}, "collapsible_fields": {"extra": {"expanded": False}}} # headers may be passed through directly or in the "extra" field in the connection # definition diff --git a/airflow/www/forms.py b/airflow/www/forms.py index cee5d0ba5eb07..31d8d37765a4d 100644 --- a/airflow/www/forms.py +++ b/airflow/www/forms.py @@ -198,26 +198,30 @@ def populate_obj(self, item): field.populate_obj(item, name) -class BS3CollapsibleTextAreaFieldWidget(BS3TextAreaFieldWidget): - """Collapsible Text Area Field Widget. +class BS3CollapsibleWidgetMixin: + """Implement an accordion/collapsible, which allows to hide the field.""" - Text Area Field encapsulated into an accordion/collapsible, which allows to "Show" and - "Hide" the field within a form. - """ - - def __init__(self, default_expanded: bool = True): - self._expanded = default_expanded + _collapsible: bool = False # Hide the accordion button and collapsible panel + _expanded: bool = True # Display the collapsible content - def _make_accordion_panel(self, field: Field, content: Markup) -> str: + @classmethod + def _make_accordion_panel(cls, field: Field, content: Markup) -> str: collapsible_id: str = f"collapsible_{field.id}" return f""" -
-
+
+

@@ -228,10 +232,10 @@ def _make_accordion_panel(self, field: Field, content: Markup) -> str:
{content.__html__()}
@@ -239,8 +243,20 @@ def _make_accordion_panel(self, field: Field, content: Markup) -> str: """ def __call__(self, field, **kwargs): - text_area = super(BS3TextAreaFieldWidget, self).__call__(field, **kwargs) - return self._make_accordion_panel(field=field, content=text_area) + content = super().__call__(field, **kwargs) + return self._make_accordion_panel(field=field, content=content) + + +class BS3CollapsibleTextAreaFieldWidget(BS3CollapsibleWidgetMixin, BS3TextAreaFieldWidget): + """Collapsible TextArea Field Widget.""" + + +class BS3CollapsibleTextFieldWidget(BS3CollapsibleWidgetMixin, BS3TextFieldWidget): + """Collapsible Text Field Widget.""" + + +class BS3CollapsiblePasswordFieldWidget(BS3CollapsibleWidgetMixin, BS3PasswordFieldWidget): + """Collapsible Password Field Widget.""" @cache @@ -283,12 +299,14 @@ def process(self, formdata=None, obj=None, **kwargs): "corresponding Airflow Provider Package." ), ) - description = StringField(lazy_gettext("Description"), widget=BS3TextAreaFieldWidget()) - host = StringField(lazy_gettext("Host"), widget=BS3TextFieldWidget()) - schema = StringField(lazy_gettext("Schema"), widget=BS3TextFieldWidget()) - login = StringField(lazy_gettext("Login"), widget=BS3TextFieldWidget()) - password = PasswordField(lazy_gettext("Password"), widget=BS3PasswordFieldWidget()) - port = IntegerField(lazy_gettext("Port"), validators=[Optional()], widget=BS3TextFieldWidget()) + description = StringField(lazy_gettext("Description"), widget=BS3CollapsibleTextAreaFieldWidget()) + host = StringField(lazy_gettext("Host"), widget=BS3CollapsibleTextFieldWidget()) + schema = StringField(lazy_gettext("Schema"), widget=BS3CollapsibleTextFieldWidget()) + login = StringField(lazy_gettext("Login"), widget=BS3CollapsibleTextFieldWidget()) + password = PasswordField(lazy_gettext("Password"), widget=BS3CollapsiblePasswordFieldWidget()) + port = IntegerField( + lazy_gettext("Port"), validators=[Optional()], widget=BS3CollapsibleTextFieldWidget() + ) extra = TextAreaField(lazy_gettext("Extra"), widget=BS3CollapsibleTextAreaFieldWidget()) for key, value in providers_manager.connection_form_widgets.items(): diff --git a/airflow/www/static/css/connection.css b/airflow/www/static/css/connection.css new file mode 100644 index 0000000000000..78edf0db5d4dc --- /dev/null +++ b/airflow/www/static/css/connection.css @@ -0,0 +1,23 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +.panel-invisible { + margin: 0; + border: 0; +} diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index 00de949d1acef..431cec815f77e 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -83,13 +83,28 @@ function restoreFieldBehaviours() { } ); - Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach( - (elem) => { - // eslint-disable-next-line no-param-reassign - elem.placeholder = ""; - elem.parentElement.parentElement.classList.remove("hide"); - } - ); + Array.from(document.querySelectorAll(".form-control")).forEach((elem) => { + // eslint-disable-next-line no-param-reassign + elem.placeholder = ""; + elem.parentElement.parentElement.classList.remove("hide"); + }); + + Array.from(document.querySelectorAll(".form-panel")).forEach((elem) => { + elem.parentElement.parentElement.classList.remove("hide"); + + elem.classList.add("panel-invisible"); + const panelHeader = elem.children[0]; + panelHeader.classList.add("hidden"); + panelHeader.firstElementChild.firstElementChild.setAttribute( + "aria-expanded", + "true" + ); + + const collapsible = elem.children[1]; + collapsible.setAttribute("aria-expanded", "true"); + collapsible.classList.add("in"); + collapsible.style.height = null; + }); } /** @@ -122,6 +137,26 @@ function applyFieldBehaviours(connection) { document.getElementById(field).placeholder = placeholder; }); } + + if (connection.collapsible_fields) { + Object.entries(connection.collapsible_fields).forEach((entry) => { + const [field, properties] = entry; + + const collapsibleController = document.getElementById( + `control_collapsible_${field}` + ); + const panelHeader = collapsibleController.parentElement.parentElement; + panelHeader.classList.remove("hidden"); + panelHeader.parentElement.classList.remove("panel-invisible"); + + if (properties.expanded === false) { + const collapsible = document.getElementById(`collapsible_${field}`); + collapsible.classList.remove("in"); + collapsible.setAttribute("aria-expanded", "false"); + collapsibleController.setAttribute("aria-expanded", "false"); + } + }); + } } } diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index ac92b967f7e34..307450b05d16b 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -25,7 +25,7 @@ - + {# required for codemirror #} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 653b0dd3ce07f..11ebd6c4cb436 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -25,6 +25,7 @@ + {# required for codemirror #} diff --git a/airflow/www/webpack.config.js b/airflow/www/webpack.config.js index 6ac1f3a208890..db2796fac1199 100644 --- a/airflow/www/webpack.config.js +++ b/airflow/www/webpack.config.js @@ -60,6 +60,7 @@ const config = { airflowDefaultTheme: `${CSS_DIR}/bootstrap-theme.css`, connectionForm: `${JS_DIR}/connection_form.js`, chart: [`${CSS_DIR}/chart.css`], + connection: [`${CSS_DIR}/connection.css`], dag: `${JS_DIR}/dag.js`, dagDependencies: `${JS_DIR}/dag_dependencies.js`, dags: [`${CSS_DIR}/dags.css`, `${JS_DIR}/dags.js`], From 2c11e76e17f462a0a45c48c85641e77ad398c987 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Sat, 30 Dec 2023 17:13:52 +0100 Subject: [PATCH 013/286] feat: Deprecate headers params passed directly in 'Extra' field --- airflow/providers/http/hooks/http.py | 17 ++++++++++++++--- tests/providers/http/hooks/test_http.py | 13 +++++++++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 69e003fa21333..5bb6eb3d1b949 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -18,6 +18,7 @@ from __future__ import annotations import asyncio +import warnings from typing import TYPE_CHECKING, Any, Callable import aiohttp @@ -29,7 +30,7 @@ from requests_toolbelt.adapters.socket_options import TCPKeepAliveAdapter from airflow.compat.functools import cache -from airflow.exceptions import AirflowException +from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.hooks.base import BaseHook from airflow.utils.module_loading import import_string @@ -126,7 +127,7 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: widget=Select2Widget(), ), "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), + "headers": TextAreaField(lazy_gettext("Headers"), widget=BS3TextAreaFieldWidget()), } @classmethod @@ -168,8 +169,18 @@ def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: session.auth = auth_type() if conn_extra: + headers_params: dict = conn_extra.pop("headers", {}) + if conn_extra: + warnings.warn( + "Passing headers parameters directly in 'Extra' field is deprecated, and " + "will be removed in a future version of the Http provider. Use the 'Headers' " + "field instead.", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + headers_params = {**conn_extra, **headers_params} try: - session.headers.update(conn_extra) + session.headers.update(headers_params) except TypeError: self.log.warning("Connection to %s has invalid extra field.", conn.host) if headers: diff --git a/tests/providers/http/hooks/test_http.py b/tests/providers/http/hooks/test_http.py index 870169b00dbe6..c76da82901653 100644 --- a/tests/providers/http/hooks/test_http.py +++ b/tests/providers/http/hooks/test_http.py @@ -47,7 +47,12 @@ def aioresponse(): def get_airflow_connection(unused_conn_id=None): - return Connection(conn_id="http_default", conn_type="http", host="test:8080/", extra='{"bearer": "test"}') + return Connection( + conn_id="http_default", + conn_type="http", + host="test:8080/", + extra='{"bearer":"test","headers":{"some":"header"}}', + ) def get_airflow_connection_with_port(unused_conn_id=None): @@ -124,8 +129,12 @@ def test_hook_contains_header_from_extra_field(self): with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): expected_conn = get_airflow_connection() conn = self.get_hook.get_conn() - assert dict(conn.headers, **json.loads(expected_conn.extra)) == conn.headers + + conn_extra: dict = json.loads(expected_conn.extra) + headers = dict(conn.headers, **conn_extra.pop("headers", {}), **conn_extra) + assert headers == conn.headers assert conn.headers.get("bearer") == "test" + assert conn.headers.get("some") == "header" @mock.patch("requests.Request") def test_hook_with_method_in_lowercase(self, mock_requests): From da4ef3b19c9a36a409bad7c5d5b3e5f333a654cb Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 5 Jan 2024 06:48:16 +0100 Subject: [PATCH 014/286] revert: Remove collapsible field --- ...stomized_form_field_behaviours.schema.json | 17 ---- airflow/providers/http/hooks/http.py | 4 - airflow/www/forms.py | 83 ++----------------- airflow/www/static/css/connection.css | 23 ----- airflow/www/static/js/connection_form.js | 39 +-------- .../www/templates/airflow/conn_create.html | 1 - airflow/www/templates/airflow/conn_edit.html | 1 - airflow/www/webpack.config.js | 1 - 8 files changed, 9 insertions(+), 160 deletions(-) delete mode 100644 airflow/www/static/css/connection.css diff --git a/airflow/customized_form_field_behaviours.schema.json b/airflow/customized_form_field_behaviours.schema.json index 8aa05945ebb01..78791a87886c1 100644 --- a/airflow/customized_form_field_behaviours.schema.json +++ b/airflow/customized_form_field_behaviours.schema.json @@ -22,23 +22,6 @@ "additionalProperties": { "type": "string" } - }, - "collapsible_fields": { - "description": "List of collapsed fields for the hook, with their properties.", - "type": "object", - "patternProperties": { - "\"^.*$\"": { - "description": "Name of the field to enable collapsing.", - "type": "object", - "properties": { - "expanded": { - "description": "Set the default state of the field as expanded.", - "default": true, - "type": "boolean" - } - } - } - } } }, "additionalProperties": true, diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 5bb6eb3d1b949..9bf2769c2d107 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -130,10 +130,6 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: "headers": TextAreaField(lazy_gettext("Headers"), widget=BS3TextAreaFieldWidget()), } - @classmethod - def get_ui_field_behaviour(cls) -> dict[str, Any]: - return {"hidden_fields": [], "relabeling": {}, "collapsible_fields": {"extra": {"expanded": False}}} - # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: diff --git a/airflow/www/forms.py b/airflow/www/forms.py index 31d8d37765a4d..8a8f69cf4455c 100644 --- a/airflow/www/forms.py +++ b/airflow/www/forms.py @@ -20,7 +20,7 @@ import datetime import json import operator -from typing import TYPE_CHECKING, Iterator +from typing import Iterator import pendulum from flask_appbuilder.fieldwidgets import ( @@ -35,7 +35,6 @@ from wtforms import widgets from wtforms.fields import Field, IntegerField, PasswordField, SelectField, StringField, TextAreaField from wtforms.validators import InputRequired, Optional -from wtforms.widgets.core import html_params from airflow.compat.functools import cache from airflow.configuration import conf @@ -50,9 +49,6 @@ BS3TextFieldROWidget, ) -if TYPE_CHECKING: - from markupsafe import Markup - class DateTimeWithTimezoneField(Field): """A text field which stores a `datetime.datetime` matching a format.""" @@ -198,67 +194,6 @@ def populate_obj(self, item): field.populate_obj(item, name) -class BS3CollapsibleWidgetMixin: - """Implement an accordion/collapsible, which allows to hide the field.""" - - _collapsible: bool = False # Hide the accordion button and collapsible panel - _expanded: bool = True # Display the collapsible content - - @classmethod - def _make_accordion_panel(cls, field: Field, content: Markup) -> str: - collapsible_id: str = f"collapsible_{field.id}" - return f""" -
- -
- {content.__html__()} -
-
- """ - - def __call__(self, field, **kwargs): - content = super().__call__(field, **kwargs) - return self._make_accordion_panel(field=field, content=content) - - -class BS3CollapsibleTextAreaFieldWidget(BS3CollapsibleWidgetMixin, BS3TextAreaFieldWidget): - """Collapsible TextArea Field Widget.""" - - -class BS3CollapsibleTextFieldWidget(BS3CollapsibleWidgetMixin, BS3TextFieldWidget): - """Collapsible Text Field Widget.""" - - -class BS3CollapsiblePasswordFieldWidget(BS3CollapsibleWidgetMixin, BS3PasswordFieldWidget): - """Collapsible Password Field Widget.""" - - @cache def create_connection_form_class() -> type[DynamicForm]: """Create a form class for editing and adding Connection. @@ -299,15 +234,13 @@ def process(self, formdata=None, obj=None, **kwargs): "corresponding Airflow Provider Package." ), ) - description = StringField(lazy_gettext("Description"), widget=BS3CollapsibleTextAreaFieldWidget()) - host = StringField(lazy_gettext("Host"), widget=BS3CollapsibleTextFieldWidget()) - schema = StringField(lazy_gettext("Schema"), widget=BS3CollapsibleTextFieldWidget()) - login = StringField(lazy_gettext("Login"), widget=BS3CollapsibleTextFieldWidget()) - password = PasswordField(lazy_gettext("Password"), widget=BS3CollapsiblePasswordFieldWidget()) - port = IntegerField( - lazy_gettext("Port"), validators=[Optional()], widget=BS3CollapsibleTextFieldWidget() - ) - extra = TextAreaField(lazy_gettext("Extra"), widget=BS3CollapsibleTextAreaFieldWidget()) + description = StringField(lazy_gettext("Description"), widget=BS3TextAreaFieldWidget()) + host = StringField(lazy_gettext("Host"), widget=BS3TextFieldWidget()) + schema = StringField(lazy_gettext("Schema"), widget=BS3TextFieldWidget()) + login = StringField(lazy_gettext("Login"), widget=BS3TextFieldWidget()) + password = PasswordField(lazy_gettext("Password"), widget=BS3PasswordFieldWidget()) + port = IntegerField(lazy_gettext("Port"), validators=[Optional()], widget=BS3TextFieldWidget()) + extra = TextAreaField(lazy_gettext("Extra"), widget=BS3TextAreaFieldWidget()) for key, value in providers_manager.connection_form_widgets.items(): setattr(ConnectionForm, key, value.field) diff --git a/airflow/www/static/css/connection.css b/airflow/www/static/css/connection.css deleted file mode 100644 index 78edf0db5d4dc..0000000000000 --- a/airflow/www/static/css/connection.css +++ /dev/null @@ -1,23 +0,0 @@ -/*! - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -.panel-invisible { - margin: 0; - border: 0; -} diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index 431cec815f77e..e59ef9a1501c8 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -88,23 +88,6 @@ function restoreFieldBehaviours() { elem.placeholder = ""; elem.parentElement.parentElement.classList.remove("hide"); }); - - Array.from(document.querySelectorAll(".form-panel")).forEach((elem) => { - elem.parentElement.parentElement.classList.remove("hide"); - - elem.classList.add("panel-invisible"); - const panelHeader = elem.children[0]; - panelHeader.classList.add("hidden"); - panelHeader.firstElementChild.firstElementChild.setAttribute( - "aria-expanded", - "true" - ); - - const collapsible = elem.children[1]; - collapsible.setAttribute("aria-expanded", "true"); - collapsible.classList.add("in"); - collapsible.style.height = null; - }); } /** @@ -118,7 +101,7 @@ function applyFieldBehaviours(connection) { if (Array.isArray(connection.hidden_fields)) { connection.hidden_fields.forEach((field) => { document - .querySelector(`label[for='${field}']`) + .getElementById(field) .parentElement.parentElement.classList.add("hide"); }); } @@ -137,26 +120,6 @@ function applyFieldBehaviours(connection) { document.getElementById(field).placeholder = placeholder; }); } - - if (connection.collapsible_fields) { - Object.entries(connection.collapsible_fields).forEach((entry) => { - const [field, properties] = entry; - - const collapsibleController = document.getElementById( - `control_collapsible_${field}` - ); - const panelHeader = collapsibleController.parentElement.parentElement; - panelHeader.classList.remove("hidden"); - panelHeader.parentElement.classList.remove("panel-invisible"); - - if (properties.expanded === false) { - const collapsible = document.getElementById(`collapsible_${field}`); - collapsible.classList.remove("in"); - collapsible.setAttribute("aria-expanded", "false"); - collapsibleController.setAttribute("aria-expanded", "false"); - } - }); - } } } diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index 307450b05d16b..fb3e188949b66 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -25,7 +25,6 @@ - {# required for codemirror #} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 11ebd6c4cb436..653b0dd3ce07f 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -25,7 +25,6 @@ - {# required for codemirror #} diff --git a/airflow/www/webpack.config.js b/airflow/www/webpack.config.js index db2796fac1199..6ac1f3a208890 100644 --- a/airflow/www/webpack.config.js +++ b/airflow/www/webpack.config.js @@ -60,7 +60,6 @@ const config = { airflowDefaultTheme: `${CSS_DIR}/bootstrap-theme.css`, connectionForm: `${JS_DIR}/connection_form.js`, chart: [`${CSS_DIR}/chart.css`], - connection: [`${CSS_DIR}/connection.css`], dag: `${JS_DIR}/dag.js`, dagDependencies: `${JS_DIR}/dag_dependencies.js`, dags: [`${CSS_DIR}/dags.css`, `${JS_DIR}/dags.js`], From 7393d26fe6fe3bf3a75396b5d345fa4e0e24adb5 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 5 Jan 2024 08:44:10 +0100 Subject: [PATCH 015/286] feat: add 'Extra' deprecation warning in Connection UI --- airflow/providers/http/hooks/http.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 9bf2769c2d107..85436c41f5737 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -127,7 +127,16 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: widget=Select2Widget(), ), "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), - "headers": TextAreaField(lazy_gettext("Headers"), widget=BS3TextAreaFieldWidget()), + "headers": TextAreaField( + lazy_gettext("Headers"), + description=( + "Warning: " + "Passing headers parameters directly in 'Extra' field is deprecated, and " + "will be removed in a future version of the Http provider. Use the 'Headers' " + "field instead." + ), + widget=BS3TextAreaFieldWidget(), + ), } # headers may be passed through directly or in the "extra" field in the connection From 6e98e4dede6d1278186af8d7c79ea8960fec8d8d Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 5 Jan 2024 08:47:09 +0100 Subject: [PATCH 016/286] fix: set the default value for "auth_type" as empty string SelectField expects a string as value. The default of select choice cannot be None. --- airflow/providers/http/hooks/http.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 85436c41f5737..400941a3a5f37 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -118,13 +118,14 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: from flask_babel import lazy_gettext from wtforms.fields import SelectField, TextAreaField - default_auth_type = frozenset({""}) - auth_types_choices = default_auth_type | get_auth_types() + default_auth_type: str = "" + auth_types_choices = frozenset({default_auth_type}) | get_auth_types() return { "auth_type": SelectField( lazy_gettext("Auth type"), choices=[(clazz, clazz) for clazz in auth_types_choices], widget=Select2Widget(), + default=default_auth_type ), "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), "headers": TextAreaField( From 5306499d8a99789d99edb2073e223d59406787ee Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 5 Jan 2024 08:48:11 +0100 Subject: [PATCH 017/286] fix: Use Livy hook to test invalid extra removal --- tests/www/views/test_views_connection.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/www/views/test_views_connection.py b/tests/www/views/test_views_connection.py index 98902db286c38..944487bef43ef 100644 --- a/tests/www/views/test_views_connection.py +++ b/tests/www/views/test_views_connection.py @@ -427,8 +427,12 @@ def test_process_form_invalid_extra_removed(admin_client): """ Test that when an invalid json `extra` is passed in the form, it is removed and _not_ saved over the existing extras. + + Note: This can only be tested with a Hook which does not have any custom fields (otherwise + the custom fields override the extra data when editing a Connection). Thus, this is currently + tested with livy. """ - conn_details = {"conn_id": "test_conn", "conn_type": "http"} + conn_details = {"conn_id": "test_conn", "conn_type": "livy"} conn = Connection(**conn_details, extra='{"foo": "bar"}') conn.id = 1 From b07a8cf1e627d974cbd3ec540060adf80521bc81 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Mon, 8 Jan 2024 16:40:49 +0100 Subject: [PATCH 018/286] feat: Implement CodeMirrorField for providers --- .../default_webserver_config.py | 7 ++ airflow/providers/http/hooks/http.py | 79 ++++++++++++------- airflow/providers/http/provider.yaml | 3 +- airflow/providers_manager.py | 2 + airflow/utils/json.py | 10 +++ airflow/www/app.py | 3 + .../www/templates/airflow/conn_create.html | 1 + airflow/www/templates/airflow/conn_edit.html | 1 + setup.cfg | 1 + 9 files changed, 77 insertions(+), 30 deletions(-) diff --git a/airflow/config_templates/default_webserver_config.py b/airflow/config_templates/default_webserver_config.py index 5522c0676b26c..f9a1eaea16e4d 100644 --- a/airflow/config_templates/default_webserver_config.py +++ b/airflow/config_templates/default_webserver_config.py @@ -34,6 +34,13 @@ WTF_CSRF_ENABLED = True WTF_CSRF_TIME_LIMIT = None +# Flask CodeMirror config +CODEMIRROR_LANGUAGES = ["javascript"] +# CODEMIRROR_THEME = '3024-day' +# CODEMIRROR_ADDONS = ( +# ('ADDON_DIR','ADDON_NAME'), +# ) + # ---------------------------------------------------- # AUTHENTICATION CONFIG # ---------------------------------------------------- diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 400941a3a5f37..7b57739bb50ea 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -32,6 +32,7 @@ from airflow.compat.functools import cache from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.hooks.base import BaseHook +from airflow.utils.json import none_safe_loads from airflow.utils.module_loading import import_string if TYPE_CHECKING: @@ -40,9 +41,9 @@ DEFAULT_AUTH_TYPES = frozenset( { - "request.auth.HTTPBasicAuth", - "request.auth.HTTPProxyAuth", - "request.auth.HTTPDigestAuth", + "requests.auth.HTTPBasicAuth", + "requests.auth.HTTPProxyAuth", + "requests.auth.HTTPDigestAuth", } ) @@ -113,10 +114,13 @@ def __init__( @classmethod def get_connection_form_widgets(cls) -> dict[str, Any]: - """Return connection widgets to add to connection form.""" - from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget, Select2Widget + """Return connection widgets to add to the connection form.""" + from flask_appbuilder.fieldwidgets import Select2Widget from flask_babel import lazy_gettext - from wtforms.fields import SelectField, TextAreaField + from flask_codemirror.fields import CodeMirrorField + from wtforms.fields import SelectField + + from airflow.www.validators import ValidJson default_auth_type: str = "" auth_types_choices = frozenset({default_auth_type}) | get_auth_types() @@ -125,18 +129,24 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: lazy_gettext("Auth type"), choices=[(clazz, clazz) for clazz in auth_types_choices], widget=Select2Widget(), - default=default_auth_type + default=default_auth_type, + ), + "auth_kwargs": CodeMirrorField( + lazy_gettext("Auth kwargs"), + validators=[ValidJson()], + language={"name": "javascript", "json": True}, + config={"gutters": ["CodeMirror-lint-markers"], "lineNumbers": True, "lint": True}, ), - "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), - "headers": TextAreaField( + "headers": CodeMirrorField( lazy_gettext("Headers"), + validators=[ValidJson()], + language={"name": "javascript", "json": True}, + config={"gutters": ["CodeMirror-lint-markers"], "lineNumbers": True, "lint": True}, description=( - "Warning: " - "Passing headers parameters directly in 'Extra' field is deprecated, and " + "Warning: Passing headers parameters directly in 'Extra' field is deprecated, and " "will be removed in a future version of the Http provider. Use the 'Headers' " "field instead." ), - widget=BS3TextAreaFieldWidget(), ), } @@ -163,30 +173,24 @@ def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: if conn.port: self.base_url += f":{conn.port}" - conn_extra: dict = conn.extra_dejson.copy() - conn_auth_type_name: str | None = conn_extra.pop("auth_type", None) - auth_type: Any = self.auth_type or self._load_conn_auth_type(conn_auth_type_name) or HTTPBasicAuth + conn_extra: dict = self._parse_extra(conn_extra=conn.extra_dejson) auth_args: list[str | None] = [conn.login, conn.password] - auth_kwargs: dict[str, Any] = conn_extra.pop("auth_kwargs", {}) + auth_kwargs: dict[str, Any] = conn_extra["auth_kwargs"] + auth_type: Any = ( + self.auth_type + or self._load_conn_auth_type(module_name=conn_extra["auth_type"]) + or HTTPBasicAuth + ) if any(auth_args) or auth_kwargs: session.auth = auth_type(*auth_args, **auth_kwargs) elif self._is_auth_type_setup: session.auth = auth_type() - if conn_extra: - headers_params: dict = conn_extra.pop("headers", {}) - if conn_extra: - warnings.warn( - "Passing headers parameters directly in 'Extra' field is deprecated, and " - "will be removed in a future version of the Http provider. Use the 'Headers' " - "field instead.", - AirflowProviderDeprecationWarning, - stacklevel=2, - ) - headers_params = {**conn_extra, **headers_params} + extra_headers = conn_extra["headers"] + if extra_headers: try: - session.headers.update(headers_params) + session.headers.update(extra_headers) except TypeError: self.log.warning("Connection to %s has invalid extra field.", conn.host) if headers: @@ -194,6 +198,25 @@ def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: return session + @staticmethod + def _parse_extra(conn_extra: dict) -> dict: + extra = conn_extra.copy() + auth_type: str | None = extra.pop("auth_type", None) + auth_kwargs: dict = none_safe_loads(extra.pop("auth_kwargs", None), default={}) + headers: dict = none_safe_loads(extra.pop("headers", None), default={}) + + if extra: + warnings.warn( + "Passing headers parameters directly in 'Extra' field is deprecated, and " + "will be removed in a future version of the Http provider. Use the 'Headers' " + "field instead.", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + headers = {**extra, **headers} + + return {"auth_type": auth_type, "auth_kwargs": auth_kwargs, "headers": headers} + def _load_conn_auth_type(self, module_name: str | None) -> Any: """Load auth_type module from extra Connection parameters.""" if module_name: diff --git a/airflow/providers/http/provider.yaml b/airflow/providers/http/provider.yaml index 47982186f142b..3e990fc720700 100644 --- a/airflow/providers/http/provider.yaml +++ b/airflow/providers/http/provider.yaml @@ -98,8 +98,7 @@ config: description: | A comma separated list of auth_type classes, which can be used to configure Http Connections in Airflow's UI. This list restricts which - classes can be arbitrary imported, and protects from dependency - injections. + classes can be arbitrary imported to prevent dependency injections. type: string version_added: 4.8.0 example: "requests_kerberos.HTTPKerberosAuth,any.other.custom.HTTPAuth" diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py index c078c8cddb9eb..c50903d1961ed 100644 --- a/airflow/providers_manager.py +++ b/airflow/providers_manager.py @@ -918,6 +918,7 @@ def _import_hook( :param package_name: provider package - only needed in case connection_type is missing : return """ + from flask_codemirror.fields import CodeMirrorField from wtforms import BooleanField, IntegerField, PasswordField, SelectField, StringField, TextAreaField if connection_type is None and hook_class_name is None: @@ -945,6 +946,7 @@ def _import_hook( BooleanField, TextAreaField, SelectField, + CodeMirrorField, ] hook_class = _correctness_check(package_name, hook_class_name, provider_info) if hook_class is None: diff --git a/airflow/utils/json.py b/airflow/utils/json.py index 4d89e340c1cd4..fc58375134171 100644 --- a/airflow/utils/json.py +++ b/airflow/utils/json.py @@ -122,5 +122,15 @@ def orm_object_hook(dct: dict) -> object: return deserialize(dct, False) +def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> dict | None: + """Safely loads JSON. + + Returns None by default if the given object is None. + """ + if obj is not None: + return json.loads(obj) + return default + + # backwards compatibility AirflowJsonEncoder = WebEncoder diff --git a/airflow/www/app.py b/airflow/www/app.py index 749efe8912c03..a02e8793c2454 100644 --- a/airflow/www/app.py +++ b/airflow/www/app.py @@ -22,6 +22,7 @@ from flask import Flask from flask_appbuilder import SQLA +from flask_codemirror import CodeMirror from flask_wtf.csrf import CSRFProtect from markupsafe import Markup from sqlalchemy.engine.url import make_url @@ -130,6 +131,8 @@ def create_app(config=None, testing=False): csrf.init_app(flask_app) + CodeMirror(flask_app) + init_wsgi_middleware(flask_app) db = SQLA() diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index fb3e188949b66..8e3d8db0d5e00 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -26,6 +26,7 @@ {# required for codemirror #} +{{ codemirror.include_codemirror() }} {% endblock %} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 653b0dd3ce07f..174bfa164c4c4 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -26,6 +26,7 @@ {# required for codemirror #} +{{ codemirror.include_codemirror() }} {% endblock %} diff --git a/setup.cfg b/setup.cfg index 6d7bb58e6beec..e92a8c2a0ceea 100644 --- a/setup.cfg +++ b/setup.cfg @@ -105,6 +105,7 @@ install_requires = flask-login>=0.6.2 flask-session>=0.4.0 flask-wtf>=0.15 + flask-codemirror>=1.3 fsspec>=2023.10.0 google-re2>=1.0 gunicorn>=20.1.0 From 53672f039011d740af67a70657292df07cc14c03 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Mon, 8 Jan 2024 19:21:34 +0100 Subject: [PATCH 019/286] revert: Remove CodeMirror from providers --- .../config_templates/default_webserver_config.py | 7 ------- airflow/providers/http/hooks/http.py | 16 +++++++--------- airflow/providers_manager.py | 2 -- airflow/www/app.py | 3 --- airflow/www/templates/airflow/conn_create.html | 1 - airflow/www/templates/airflow/conn_edit.html | 1 - setup.cfg | 1 - 7 files changed, 7 insertions(+), 24 deletions(-) diff --git a/airflow/config_templates/default_webserver_config.py b/airflow/config_templates/default_webserver_config.py index f9a1eaea16e4d..5522c0676b26c 100644 --- a/airflow/config_templates/default_webserver_config.py +++ b/airflow/config_templates/default_webserver_config.py @@ -34,13 +34,6 @@ WTF_CSRF_ENABLED = True WTF_CSRF_TIME_LIMIT = None -# Flask CodeMirror config -CODEMIRROR_LANGUAGES = ["javascript"] -# CODEMIRROR_THEME = '3024-day' -# CODEMIRROR_ADDONS = ( -# ('ADDON_DIR','ADDON_NAME'), -# ) - # ---------------------------------------------------- # AUTHENTICATION CONFIG # ---------------------------------------------------- diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 7b57739bb50ea..77c0617ba0d2e 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -115,10 +115,9 @@ def __init__( @classmethod def get_connection_form_widgets(cls) -> dict[str, Any]: """Return connection widgets to add to the connection form.""" - from flask_appbuilder.fieldwidgets import Select2Widget + from flask_appbuilder.fieldwidgets import Select2Widget, BS3TextAreaFieldWidget from flask_babel import lazy_gettext - from flask_codemirror.fields import CodeMirrorField - from wtforms.fields import SelectField + from wtforms.fields import SelectField, TextAreaField from airflow.www.validators import ValidJson @@ -131,17 +130,15 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: widget=Select2Widget(), default=default_auth_type, ), - "auth_kwargs": CodeMirrorField( + "auth_kwargs": TextAreaField( lazy_gettext("Auth kwargs"), validators=[ValidJson()], - language={"name": "javascript", "json": True}, - config={"gutters": ["CodeMirror-lint-markers"], "lineNumbers": True, "lint": True}, + widget=BS3TextAreaFieldWidget() ), - "headers": CodeMirrorField( + "headers": TextAreaField( lazy_gettext("Headers"), validators=[ValidJson()], - language={"name": "javascript", "json": True}, - config={"gutters": ["CodeMirror-lint-markers"], "lineNumbers": True, "lint": True}, + widget=BS3TextAreaFieldWidget(), description=( "Warning: Passing headers parameters directly in 'Extra' field is deprecated, and " "will be removed in a future version of the Http provider. Use the 'Headers' " @@ -174,6 +171,7 @@ def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: self.base_url += f":{conn.port}" conn_extra: dict = self._parse_extra(conn_extra=conn.extra_dejson) + print(conn_extra) auth_args: list[str | None] = [conn.login, conn.password] auth_kwargs: dict[str, Any] = conn_extra["auth_kwargs"] auth_type: Any = ( diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py index c50903d1961ed..c078c8cddb9eb 100644 --- a/airflow/providers_manager.py +++ b/airflow/providers_manager.py @@ -918,7 +918,6 @@ def _import_hook( :param package_name: provider package - only needed in case connection_type is missing : return """ - from flask_codemirror.fields import CodeMirrorField from wtforms import BooleanField, IntegerField, PasswordField, SelectField, StringField, TextAreaField if connection_type is None and hook_class_name is None: @@ -946,7 +945,6 @@ def _import_hook( BooleanField, TextAreaField, SelectField, - CodeMirrorField, ] hook_class = _correctness_check(package_name, hook_class_name, provider_info) if hook_class is None: diff --git a/airflow/www/app.py b/airflow/www/app.py index a02e8793c2454..749efe8912c03 100644 --- a/airflow/www/app.py +++ b/airflow/www/app.py @@ -22,7 +22,6 @@ from flask import Flask from flask_appbuilder import SQLA -from flask_codemirror import CodeMirror from flask_wtf.csrf import CSRFProtect from markupsafe import Markup from sqlalchemy.engine.url import make_url @@ -131,8 +130,6 @@ def create_app(config=None, testing=False): csrf.init_app(flask_app) - CodeMirror(flask_app) - init_wsgi_middleware(flask_app) db = SQLA() diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index 8e3d8db0d5e00..fb3e188949b66 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -26,7 +26,6 @@ {# required for codemirror #} -{{ codemirror.include_codemirror() }} {% endblock %} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 174bfa164c4c4..653b0dd3ce07f 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -26,7 +26,6 @@ {# required for codemirror #} -{{ codemirror.include_codemirror() }} {% endblock %} diff --git a/setup.cfg b/setup.cfg index e92a8c2a0ceea..6d7bb58e6beec 100644 --- a/setup.cfg +++ b/setup.cfg @@ -105,7 +105,6 @@ install_requires = flask-login>=0.6.2 flask-session>=0.4.0 flask-wtf>=0.15 - flask-codemirror>=1.3 fsspec>=2023.10.0 google-re2>=1.0 gunicorn>=20.1.0 From 099b9ad2b14b475ac94a7ef82c23156c76f0d9be Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Mon, 8 Jan 2024 20:47:50 +0100 Subject: [PATCH 020/286] feat: Add documentation --- airflow/providers/http/hooks/http.py | 13 +- airflow/utils/json.py | 2 +- .../connections/http.rst | 112 ++++++++++++++---- .../img/connection_auth_kwargs.png | Bin 0 -> 9623 bytes .../img/connection_auth_type.png | Bin 0 -> 14199 bytes .../img/connection_headers.png | Bin 0 -> 5256 bytes .../img/connection_username_password.png | Bin 0 -> 4761 bytes 7 files changed, 96 insertions(+), 31 deletions(-) create mode 100644 docs/apache-airflow-providers-http/img/connection_auth_kwargs.png create mode 100644 docs/apache-airflow-providers-http/img/connection_auth_type.png create mode 100644 docs/apache-airflow-providers-http/img/connection_headers.png create mode 100644 docs/apache-airflow-providers-http/img/connection_username_password.png diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 77c0617ba0d2e..1346a2f9ad223 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -19,7 +19,7 @@ import asyncio import warnings -from typing import TYPE_CHECKING, Any, Callable +from typing import TYPE_CHECKING, Any, Callable, cast import aiohttp import requests @@ -115,7 +115,7 @@ def __init__( @classmethod def get_connection_form_widgets(cls) -> dict[str, Any]: """Return connection widgets to add to the connection form.""" - from flask_appbuilder.fieldwidgets import Select2Widget, BS3TextAreaFieldWidget + from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget, Select2Widget from flask_babel import lazy_gettext from wtforms.fields import SelectField, TextAreaField @@ -131,9 +131,7 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: default=default_auth_type, ), "auth_kwargs": TextAreaField( - lazy_gettext("Auth kwargs"), - validators=[ValidJson()], - widget=BS3TextAreaFieldWidget() + lazy_gettext("Auth kwargs"), validators=[ValidJson()], widget=BS3TextAreaFieldWidget() ), "headers": TextAreaField( lazy_gettext("Headers"), @@ -171,7 +169,6 @@ def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: self.base_url += f":{conn.port}" conn_extra: dict = self._parse_extra(conn_extra=conn.extra_dejson) - print(conn_extra) auth_args: list[str | None] = [conn.login, conn.password] auth_kwargs: dict[str, Any] = conn_extra["auth_kwargs"] auth_type: Any = ( @@ -200,8 +197,8 @@ def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: def _parse_extra(conn_extra: dict) -> dict: extra = conn_extra.copy() auth_type: str | None = extra.pop("auth_type", None) - auth_kwargs: dict = none_safe_loads(extra.pop("auth_kwargs", None), default={}) - headers: dict = none_safe_loads(extra.pop("headers", None), default={}) + auth_kwargs = cast(dict, none_safe_loads(extra.pop("auth_kwargs", None), default={})) + headers = cast(dict, none_safe_loads(extra.pop("headers", None), default={})) if extra: warnings.warn( diff --git a/airflow/utils/json.py b/airflow/utils/json.py index fc58375134171..e27e4d1b77857 100644 --- a/airflow/utils/json.py +++ b/airflow/utils/json.py @@ -122,7 +122,7 @@ def orm_object_hook(dct: dict) -> object: return deserialize(dct, False) -def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> dict | None: +def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> Any: """Safely loads JSON. Returns None by default if the given object is None. diff --git a/docs/apache-airflow-providers-http/connections/http.rst b/docs/apache-airflow-providers-http/connections/http.rst index 53a90db51bc06..2613755d2a141 100644 --- a/docs/apache-airflow-providers-http/connections/http.rst +++ b/docs/apache-airflow-providers-http/connections/http.rst @@ -24,53 +24,121 @@ HTTP Connection The HTTP connection enables connections to HTTP services. -Authenticating with HTTP ------------------------- - -Login and Password authentication can be used along with any authentication method using headers. -Headers can be given in json format in the Extras field. - Default Connection IDs ---------------------- The HTTP operators and hooks use ``http_default`` by default. +Authentication +-------------- + + .. _auth_basic: + +Authenticating via Basic auth +............................. +The simplest way to authenticate is to specify a *Login* and *Password* in the +Connection. + +.. image:: /img/connection_username_password.png + +By default, when a *Login* or *Password* is provided, the HTTP operators and +Hooks will perform a basic authentication via the +``requests.auth.HTTPBasicAuth`` class. + +Authenticating via Headers +.......................... +If :ref:`Basic authentication` is not enough, you can also add +*Headers* to the requests performed by the HTTP operators and Hooks. + +Headers can be passed in json format in the *Headers* field: + +.. image:: /img/connection_headers.png + +.. note:: Login and Password authentication can be used along custom Headers. + +Authenticating via Auth class +............................. +For more complex use-cases, you can inject a Auth class into the HTTP operators +and Hooks via the *Auth type* setting. This is particularly useful when you +need token refresh or advanced authentication methods like kerberos, oauth, ... + +.. image:: /img/connection_auth_type.png + +By default, only `requests Auth classes `_ +are available. But you can install any classes based on ``requests.auth.AuthBase`` +into your Airflow instance (via pip install), and then specify those classes in +``extra_auth_types`` :doc:`configuration setting<../configurations-ref>` to +make them available in the Connection UI. + +If the Auth class requires more than a *Username* and a *Password*, you can +pass extra keywords arguments with the *Auth kwargs* setting. + +Example with the ``HTTPKerberosAuth`` from `requests-kerberos `_ : + +.. image:: /img/connection_auth_kwargs.png + +.. tip:: + + You probably don't need to write an entire custom HttpOperator or HttpHook + to customize the connection. Simply extend the ``requests.auth.AuthBase`` + class and configure a Connection with it. + Configuring the Connection -------------------------- +Via the Admin panel +................... + +Configuring the Connection via the Airflow Admin panel offers more +possibilities than via :ref:`environment variables`. + Login (optional) - Specify the login for the http service you would like to connect too. + The login (username) of the http service you would like to connect too. + If provided, by default, the HttpHook perform a Basic authentication. Password (optional) - Specify the password for the http service you would like to connect too. + The password of the http service you would like to connect too. + If provided, by default, the HttpHook perform a Basic authentication. Host (optional) Specify the entire url or the base of the url for the service. Port (optional) - Specify a port number if applicable. + A port number if applicable. Schema (optional) - Specify the service type etc: http/https. + The service type. E.g: http/https. + +Auth type (optional) + Python class used by the HttpHook (and the underlying requests library) to + authenticate. If provided, the *Login* and *Password* are passed as the two + first arguments to this class. If *Login* and/or *Password* are provided + without any Auth type, the HttpHook will by default perform a basic + authentication via the ``requests.auth.HTTPBasicAuth`` class. + + Extra classes can be added via the ``extra_auth_types`` + :doc:`configuration setting<../configurations-ref>`. -Extras (optional) - Any key / value parameters supplied here will be added to the headers in json format. +Auth kwargs (optional) + Extra key-value parameters passed to the Auth type class. - Additionally there a few special optional keywords that are handled separately. +Headers (optional) + Extra key-value parameters added to the Headers in JSON format. - - ``auth_type`` - * The path to the Authentication class to attach the request.Session object. Typically a subclass of - ``request.auth.AuthBase``. This is omitted if the HttpHook is declared with a ``auth_type``. - - ``auth_kwargs`` - * Extra key-value parameters used to instantiate the ``auth_type`` class. +Extras (optional - deprecated) + *Deprecated*: Specify headers in json format. + + .. _env-variable: + +Via environment variable +........................ When specifying the connection in environment variable you should specify it using URI syntax. -Note that all components of the URI should be URL-encoded. - -For example: +.. note:: All components of the URI should be **URL-encoded**. .. code-block:: bash + :caption: Example: - export AIRFLOW_CONN_HTTP_DEFAULT='http://username:password@servvice.com:80/https?headers=header' + export AIRFLOW_CONN_HTTP_DEFAULT='http://username:password@service.com:80/https?headers=header' diff --git a/docs/apache-airflow-providers-http/img/connection_auth_kwargs.png b/docs/apache-airflow-providers-http/img/connection_auth_kwargs.png new file mode 100644 index 0000000000000000000000000000000000000000..7023c3a7a072f965f9dd053f77c5a5d64b396f47 GIT binary patch literal 9623 zcmcI~2{_c>+jom(n~(ixI5hS2r&k-xWWM^15qC|U8&@yI zp4VbIoNz&&;S|%$vllM#zw{D3bLeCYFTFO4DogYQZ3Q#Q(MB#`@|PP{w$;0sjC|gQ zwZ00``u#3Nh5MrM%WwU6MOC!GEa0bjzF~#+9_k$RUt@1t_`%yIg(IZq`(>tL>=B>- z{$n_3=<@gXa@sCopHqhy8ve(KHqs61*aGb{L`~`|6OlkDb>1ArkdEJf(u%A*3sMN|@KU37UWK z`S!;+d!t~o=`Y)wx6bxn)?k%F?VSe!d^L0ZZ$r)AA%p$*$)o=^r})M@ajS#xngony z*Lg06eEks5LtY{{Vqn>ksZNFGX*0{b@+ z#0=Qog@Hj;9>jhwU?syRst>MH=7w=oH_sMKnFK}Mwee}-tenP+6yF*CgOf6#l9m~nV&Lm0k^g3ly*;YJZL)1?XSHiuScftr z#fh-AL>l<*5_9^a$4@YkHx|3Pz$brZ?SLRnt1aptkF^SCGF4So9T@qEx}W?S9Hhje zjy(hdWv_M_&Bzq|=dy@X#CzWgO-|?oQ*sLLA1p%)S2ocGlYcv~*k1^Hr`qWbrW`D1 ze39xdiiP$)c~TF*B%`J~(|0HvDL6}UXmC$AX=?Uck=2jS&$IIKgwo{Qg;dPIWDXvG zMpEM11PV^VaExN#`GO17fs{SQ=DebuB3m@@iB@cQBh_QnXofNT+W;eRTnE;5qA}H_ z{zc{+2E+#+k7QZLTNt<2>=yczPV0#hS6B%9kd3{-YcCyL!Ul_+k; zp0DzSG_^E_A8QO|+-h}b+8Fg8y{FmTOUkQv9CZ=kv~^}^Lc*}YvMAAFoY;}%=|t1SKU^tU@;Fdz77c@R*1UA%Eu_Y3tz$}<))Yv z-3pd<>}&eTERFRZ4--c09NoKpl|RsR_t%vKf8p&x_J~HJ$(xT zo$oS|$Z5$+e80CnXSF^*nB_j*ndQH~H}>(#(cCVJdOvxk)i2Uz0R!(`MoYbx$5G&7 zbo(K0GjYpF_4f8Q`5Qg(SozU`tNz$D_S>4h8^zdEijt6gb!toDK<=$u;A)fy!Y`h(--=f&9P7^^!9I4 zS6lByogoK?=((~MUPch>kn9^#u;HhK@Ta@&)yh^MbQOJm+xd0L-}4_>pNLW6Rf${5 zn7B7A{qPGNpZ|_4r;-v{iFBo2Xh>#+3G;|{E(gSV34$r{E%Y!9`-0-)9E>o*$tJE)bX!7Ho6BjelTH=PL$J?c1 ztcS}V=!m@Z84KnZ_Y8M`u0~g~R_#1AF0&rzJU`m7zjtE99k_*)p*%{1*Zcfkv+5{6 zR8O`>jGTCSrfnv=TK~22s0ZyRx?UYb5c6=Y2V+m39wWHzVWSoL=?cN?3qvg)87ltu zzh9h(5?Ru&?C-2D*OsZIjn3QJB{cg>n)^q~LN{gBOxrEsTCs>YYEnTcVzXZ93bGh0 zzrFi`%hISd+fZlbYVcBS(%oG>bOD1C|O-DZ0gNyeZ7| z;uZHP-86$>`0iguVGr-XX5S>WNe=s()OWpcM50@!i6`t8{wEP90XuQUY@7_G<` zMX#lM_TFlzVc))-lM#s5Zg<<^h7V_c#6Zqm$~pbCRB0|7%Gw{i@E|bt-6%$}x^(MT z-Qu_`f0Gqj1Z!{SyJoQ}*|2k|Qd9r z>FA|&f!~@IC8ok-O}>jCIHvPUqjzf_FCfwq>*MTI=x)nt{8ZFBRE`|3^>UW9s6CB; z+3O(2M`ZC5LsVYC9G_8Wt~q;JO}+n&`_z}P(KFTfvQsHJbB!C-1>(qCCFW<61a)mJ zfOn$ExQ(<4t2>BmWijT^^%ryB)2zikhe00HJII3zq&#Kg*d$5IyFLcNnc?t;1aLJx zoJ_6;UlJf$ak1T*Vm)Q#Qln0~QLgivXwu@@_BCrF#V znBPwJa3cfanoY|cE6bmaJ@LQff|QJNDw<>XEWazOvKK!+1>2;7y$rtLFc@1}@5Cq^ z5Oh90X3=-{nICF79SBuUIwF7??)EGxo70_>P)j;b9!5H#)Hl_1 zveiRRBuu#?H!9SwzaroBq4NfU|RlwHPbf$H1%Avp5&z!DSXF0SkUDbIQ zN5eK3l*mwE#zO9}ZBKBzYRHFZBD9^0n9#M-o()i^yOx{safWYyQzNa#c6BHxXF}Ja z=TzVIm+=V@q@7h6h!}-c`>^rC(p-m*cP7Kja zXnQ)RsZ+5HuBV_@p$s+68ylhGc+%wfh=gMoM5{ix-n#blmFo4>aIpk}F%GGZQlEpa z3wlgv9Xm2)uZ6;2Eq1EQT8`gz3s5uOC*K-(z)}<-x^bclCkeB(~dQ^KZsXG93)}Y@3u(u2tR+;ef__lr)x~c zs@fSnBczc+RPD}83uSqz>jXlZFN{@i>ojr-JW{l0sO6iiBS_stR=1xb>wi>xSK!1U zL2LdH!5&D=3-{dfOHz~^B!x^?4C$#}Wk7NbjRY>)20)`_Rah=>~yAj`Jqb1^m7l^l&k_AB-56!+#;7VCgf_!uFmkuic!7h?*gZt{)_2! z_c`Me!sS*?m7~g@XM9#~KLmRnseyl4WPAbV2*VohK#Q(PPtg}6O__^XUmF|M=ox4_ z=%bFpNYb{Qkrt(|FI1%aC8ceb?mr5CW!NDnU+Hzvvb;Ewb1u|6KlmS)o9_9`>qjTe z>2Sb~h^;ekty=tgZxaZOj%!mK5;{L2XR53y=c+y=nxBOWYq!@&;XMU942~iy{FPe6 zyH-AoE3SQaeEVrK6-tzPE!`imsL7?9x|XY91sBHJ&nU2JU$VP2Oh>`kp37-@+oHC zk8(4QJ%7dN;5ZA?yJa_tui}60sE-N^DUojZym`q_c#yXxfwn!* zgmwDa&!7ie?kNx?9j05>#_w!&(xPXh0=JQ?~35`m$S9)v?zSr?L@Jvwk6s^tZy$#4l(*9EbNxA;-XvBZZ zJM4>a9U6S-(>`mn37JGEW``@B@@_U>)9IcoaYH#25(MHuq%}@Bv>r`vc9*gF{Hzo_ ztrjf-1^GrHd~Esv&v8iJWujnQmEGS)rsXzTrWCmTT&M=K>$5pq^mZjKbwRcG_#i0{ zfE=s+z1^`uX62_(G^}}hy}enmDimbd2;eNfz$pLz)4-zJZ$tX;`ilQv992nE^em~| zsdDTeuea4{iRPJbZCL9w%c`!f9+@nk{Lvbh>j46?{%Ox(%zI{)9jQk@y9HHhY_?V= zmn%Y<`8vD8TfSzghk8XOtN3F~Yup}S-6pknm{tX~42Nu}xxeoOa&sZu%@1&;QipGa zr4Kt&cFsmQbnH4uJE2wwtYbbANF-d~*m4VBm-tpDZ^(?k zHMx6NMFkVJ4EgcPr!Y6`!COlszKg+~>58EXL+|?UM0ZK6SRzO3EfB(V#u>i9YB&iy z+jDO{rep=N-anpeGyg?7 z=xZ?-BKBfEud<()7eI@R0KDQUtRpZ6fVyUH}^g+ zA&J<5vq7al<_9Y_;67axeB6J>$cR-vbfP7u<>%lUw;YO}iBpPyexU4}#1II8f$xD* z$AX~q(iWkn%&HyJnf`ab-?)rD1Hk;|O1mWf`8kco9Id$V%C`-B3crMLH}0aRAIuQ4 ztolnVawp}4`j*=z8=5wP&H~yPNyaw}mLODimeiA#(LPLa1Ji&3C92VPegiJXgD9V{ zZP=Phl}l%rr8$E|jj4If7UYj_jSVQ1=PL-sX79Xsg>;iy_O`nGnp>UhN<&x(wq9ICsLozy*^cqcG&t(C$ zw(3eS6H>X((`ZL!e|Nv}%^m60idosmUy?UptWNLS35jbtq*4&9VH)fg4QGKFLxO$d zv0A1l;Rzf&M80I35<5}#UW z07qEE3+8>ihCzIrKDSjVSYl0kzd0V^s(hHis&i{~dP3QGuw3x>{GfETq3UI<0E++S zeScG4ZpkOQHS9cV=}930(7W#D6lmPP)D*^=r4hqRDFwI=mRYxi05;jOOES&ZOT9Jw z_}Uz2Vy+JR&;;ke@mjCl&50P-T^hT^dNMMt(_5HxoZA{70@?o>n`h3iK9?27D%{); z5l?FI{^1wEe#!dRu&)gzBq=$wj; z)iaW6`y2jzOc_i2*3r-dWV2@DpJ^Ya>d0TpbL3eNfRzC+LN9{H6b>TpMTfMz6PM$tC8tK0pe9(V zWZi{A+S51rlxIFrf+EDM06iHSw9ORnz`wauasS>?#Fa9pwquuw*@W?@sHY^sI_~}i zf}oipHjV&;k>wHRa<4Mz(Ie@Ke|7o!o9O;>34 zkKrMUHE3o}eR(nbFfi5OwW+4zR|;5y&C(Aij$p_OhEj1n*RbZOILHY*3;eWdQLLMF zIzIqXr-UP15p;Z4=lG5sJ|a+S{f`0U?8DygI7ok#>peObJ9_ox*Z{!*NTTR%I!8Jr zoohK=y(ypFaQtG1%>AaP-S--3_=Bu3aC~Or`dS>`r}LBijlc;X{AF#f5s)&^kDF(H zq63J2NT1$3#}L2nE4Gt|6DWm>PsYonhr1tA&mW0{Uv>`Pmp~YAh%l~BcSQ93e3!w* zCC9HnA;P8}!W8}1Xs~`a-4G?cF#=hbf{)M**Lgs$_&#EGnM7t(tl&Btk1>w>D)$sS zCJ`1#^!{;avbEK9q&!z=!WWsM;4#l~BO1HkuY-Inte?z+fBkC%xn#`vMJ>rS$9dpF}PO3F$M%_W;tTB|<`*t{}BY>T@%ppMeYVE~+I1h-1NxRg>A zg5`xDhwt((K#!h4BD{vHbfy*bUP2j>bZVPSi=GQU@XHJOIEX;Y=jZ3f73fqi3~Hbs zJ_v*!iFJDV`}gmk%UAppegu^@#0vx-|Fl)`jWBCcG+xhI$Wk&-tT`ukxAL_Nv&IR& z=tc7SPo5i0e7w`=s<^n|k9D0IGmaEgMs+O?Cok`{w#JRSN?4TVoIa!#J6o7X3c+NtLyj1m2J54Ku8NfltG`MtV znaa}wplZ*Z`o;I;)LyE!LlFt3=GB6X*25^3zua(;o(%7)7n>=Bz{7r9$kLJ*(zeYv zBYZg!s$ZR9A{?Jf#XI*wHYbwYy#1mR6_{AkouA{cF!s>rP(b)1q0|aLZrx|JApsah zY*OcV^+l(Pow|*r2WWdhp~nJYaEk%? zc(l&jd1Ip?ShzVxwAiKnL#p-ESB9yku(uPAh#?l5GUlDhSoN322%dY{mTZ{FZwcWRVk?*D1 zZD}EU3|?6}w%<>4@ooLK!~Clz7o(bcdGa_&gXAOOvnbNLKy5{fSDQSY$Khg08TCz9 zTi*ocTRh2E|2i8o%C?8av8LM5$JR2-vJ+)wxk^`8sE#>?=i+bHRzc42X}8u~ar43Eg8c3QY!gT3r0AERgA;z=Q0 z`NE*^n^T?j3fHmX3l|MKNnS^Dj`rBR*cdFc-_u1&qqi%&ZSQ(-P0B44ziabM(grE^ z!DiAeNi7D)na=C|qP5a!^1?e;uEXu+4>r`fk!P1@i=gO%o>vY;(}53Tu0M zToR*oBGz2u(Wj2Qa6;iNrxf}hh4;)P%5IHBU50lLPJNHfAr}G<^paXNj6c(#@6N7n zr$*>U*E|5%iz|Qmq418iOeb%$aEsr4k3D_16oWvrY!ecm-n0Gs>Ux>ZsfPBbV{pio z6YrISEJVD7dpGrPdg@skj-Ny1 z{7ovWvmTr*cA`t6Gw+`G`19@!xp>6y(>&spb^@U8{XIJT7(A5OpDQqQZ@OAYZm4KD3SsW($Znu zGtq6Y-#O7U$fsp!K@0bn!q&v7G!o7}XbhrvEw!%YbVb|I$Dz-Ae*L;Fsu=R@yjCHG zjS-pQi$pRBalV#}D6>jdUMnE)ezRxq^ldz&#(}6KbPGa>OuK0Q(FD=kO|CR;!RJ7N z!`7U$%OdyuG|C^}iGu#}&v5%a^IUEr{oSa9p3d-xYUg3D(U>#4UQC&EcLzKXp9#f< z#Cof#$TO}A;vNIHX$3oJ;n2(PPmN07yQG%}|FGx76v_FV!l_0)4-Y9 zbZ(yy2$hl0RkKsOEr@u@@})EoC@9erb-#b^YBISA>7}u#NA_lB@Mr5xrwnWd;ey+# z20lZ)kK|L4s8<2ycE2u6F4Fv`y26p~))|+6R=8Dr^*- z0)Ui`ajGj~xM}9rbq6JwpbIDxr z1LgG^!M34#v@h0uTAwQN9FW{JDl zwK;rq6m-JzND9E@Az(+%w_tn#IHZ%NLisRsJ^2RA7PXJ)!SoQFBne2v-nL^}5;Kwu z!LImtp6)0u6F^R$F#=ye1-}vOcS0%PJO>vawYnOtUl{A8dato{)<1Qb^%U#?-d(A^kiECYHjx0rcd9SJ7b{! z96pARQQ6}ZGRQC&#%4{n0d>qeBAz0K0tqb`SkvsiMas*105&;C05l&37UD#`U;plS zkz`u$=bdK2-wlW$YHw$20A>eHGz(~1xg|P4*m&JD{ty)8tw6bcSE_ zMU`0^_8k2*IdeY&@s&mUpaz!pDjj4V33WB}NdWpqm&*MT1praXlYM8j_^#2jHgpcjBpYOx2IWu@IqX04SL z9wn~mzqeyrKGm6S#(-oOGi5RRkt1Oq8HKNHKL)pdIVnRegA))!qI|@hXA!GQQdB1u zQ_Dp9vNZ6>>v*g*5QGXQnQ$E-e8(M|y;?CL8sP^!$YPvmu%YS}M1nI68ZIT}ipD_z z<6&R_HEiY+_&?mxSyv_W)t<$uuEafyjWcL~qm^d{zNFfORVpsdZ&4t2i|UnX^n zc@C8S32ILd(FYlJDXr?Jlkx;9RvFVAoLVAjYGtzh;(LJ{k}GNccJ<^JDh$}n6Fohq zwawFU91anWxNV7+{r$5?SXnQ;pGHwh8A>DMlzA9KA2ZWjW%u5XuA=^%L8%+hdNA+KUxJ+NQ^~; zTn9UHP(jy0^`zM4#$I$mj(&?!TSuq#wsw~CIzANk)`;z1NF&A3EomfU0tp(xzd z2ZcTVDiW>l`g~X*OFz&oTQJg7kzk+Sm8sM2upK1z0}1W_Bs2f_)jcE%Uk(~1PJpBb z;gB%@-1RSw06?<831r+ngG}CA#fXIes>*|VZUs2-Rx8-B;om}jF0P}m*bEr=Onkuk8g0E@GN8TH(>+$dxn|e`V?;qmYie-%1%QuKYo} zTHWJ?Mv(qiDu63;ZrM_cbD)O-WOK$3K5Gm~5*BR-9R~*&jIxmP5IZpcKlL$yg*J^v zz)AeA@PpaC=ZfDQxY6Vw(ZrJSs;e)5$+)5@cBlOOfi0X=TV&u^t3gWDPBoA=7(7ER zQ(FfL|KmeSgG(WWbN`Rs8w9{eDN`;Vv=|hht9ZJrw?>o5+so|!QQ4NeB${+x0%*88q;*~Y KT8Y}dfd2)j<)#Y& literal 0 HcmV?d00001 diff --git a/docs/apache-airflow-providers-http/img/connection_auth_type.png b/docs/apache-airflow-providers-http/img/connection_auth_type.png new file mode 100644 index 0000000000000000000000000000000000000000..52eb584e5ccf6463273c9b0d35171944d451af9e GIT binary patch literal 14199 zcmch8XIPWlwl2%HEh~tXqEt~qIwDQF282i_fCvGiAiYU%0u~|)Qlt}_l+cUx63P;! z6RNa8U?H85AP_(Zf%_%6_Bnf>bI)_{k9+^{0QtT-#~gFaG2eHLH}7?|)fhmmAUZla zhKCRpC>`BD5Ww%PXHEe>*-vEj)6q#QK2*7H;A2j}h4>qeWG!xxl`vB)-qyt4sieB$ zK!I^4{_6to9tVb=yr>fRj*oBX{%=hGK(XWx`L7FCMn$QE{n{j#%tES+jZ;z>wWBLy z)|Y&zJ5tO8c4S2zN-5sF;b`ymw>G~46VlOrV)=>wah3~pf%ezEbrt@9(0&^@tq1yz z&SF6ae1<#t0(k%VoqP=R-39QOm9E&Dc(-5w)ot1MK?$B)ipa)5&(OV0dve4x-+#J} zB|8atk~=s`G4{FqT4dbO+0(-W>d2h}*iREaZ1QOCh6GLh?;mc517iaZl3If2%-h1l zD4Cs1${fNWe1K1kTjTDRT-en3)V}zPm6+SKa(|U(%9$0!Mn{7)`&xTj;C)6W<=vi) zolM3Pbk9y2ote1PDHv>eF+1GCX)ZlyF|dG*IAL&+X6#VdTzezy?+)@swvKC{%slpb zrFbGljB3@g>)}~$r`{=%rzK~HPU*6J?cVp#x&9IDxvS}W-Ky%BiHEjpg<_xETUakU zI2MVy8Jw##>yf={`3_k4z}$f`*x4&8r*jp4zw*m)yAE56ToooKJ77@F?n<-w}U^m}xbX2m=)o+xvHvuLI6^WI#x3k`GC@K=GXzz0#)m5{mr@G%S7JkMK z>+txD-5Vq1_)jLoThg( z7wLwInsm|}hEu?K*|K}0JT(H*Y+nn#Sut@#s_yyOgQh~xw^=U9tmjg<8Q8ol9c+{K zMTKIe|7PSGi!!T5&3L#F`-{`mjZ)wp5#lDW5Lm!9g2}xZQ?lZ|-buazT4lagh8L;9 zbA~%W>ey@L?cIT5*=3dO-w3I~G}nveTmNDAMV`P_yz+T}?7Y)jZC2)BwUIoqYz1cL4D z?KR!qE9$)0IsdjnOH0cO*t*Sx_hsc?Jj2JM5F9he4m59bqhMIouVF&mt~f7Xu~Qm>K>&E8%#QO46*8mprfkZQUl<|REZ^47lDpg(M^^u zf6Y8J95}o`MmtpJZ9f#1mACh@$~l}ZE~{+*%L9f7(RNqG0X2ZD_s{uxSYj#L@>U|KplpF}}~g!L$O zDK#Zg?W3_^x|c-@>fGM*Vn(32k!f*+Vb^xB2&1$v5KeS-{I$?!3ya>OvWoVBr~KbW zFLu&4utgN%;qecr+5%z%_#`j{9o;>X62_s#2@$E(E6NT+#bS~`g1AhA)8;^&kVz$jkGX}Ji=?z^CT&{zT%^0!USH2>Qtg;DQEn9`?751^YuVZ5Pkes*W^W+@ zekpPiYrXMGGY$*{Om(ciqvKi8a*})`yUe9Dr|zB}!W`Z=SDCu@0U$>CB~~#XtG`~o z>JQ7)5f(dalOq?JcFZR$#uoIgRO#6C}*EwY6Dekdix!B-8D2XStOGu#Ak1 zmvyX8+jvs|Idj;iKdmYl-91^idA1#w1g+%2NcD_-c#zYh zTc_ql8GHuMvi^)VPPpfYOTs3#1MCkE&J4^{ zSohs_x9*qfSK2M|-9Ri6@XsX@;Zaxa;>*(g%A5C|kjRVJk;1_Rx|b0v-B3tqXy{rm z2a^cWDyv0pY0qHFRy8u~j;gq=M;aVtmlHwk2;!_4^`7qfTHTFVbv3w8K!K`d-2r{dE{w1Clu9ak=bM!;GTt()^PhF4HYISblQOftp!vINoYwDf)+a${@;ZIXgOtQK!!Pb^=J}}0 zv)HtS&P4MJyx!ZXv-OJdKNlX(U_IB)u(OFndp)T6jjj^z9S3eIFTW%1O*DmN+rmDD zyExFxkWo?H%uHg%zTcpbcp+m8ZmQB3|LU#aAB+6ik@}kaySD4fwN@^mBF6 zon2*ssI50D0OyIf={|}BJJ$$fx<7`oh+>BiP3}cPL0Tz4eLa%l@gh}A#jpfFmh#Vx z(K7DVlUpLDcU{i0kGmWl62{;JE23Caio8ty;g{b}uldICQZSr%9tY)i_ka0|jcz~@ zNH7bFi&(QFsNSng<*#90T*J=hirzjx&5T2=^81rYDUC1~L_6i~(|A!hrx~xwmfGat zPCU3Hf};Y@mVae``*7zW>7XT3ZA`-uv==0YEym0Q^rpYM>8F({bDN3_{T=}~jNf%6 z;xaRDx44cH69dwWf4-k{$oa8sX)&}@Gyxdey5GUcEW-d5b!tS`WwwQ@x5!wWK(ryk zpW-Yv4=tb`0X01nEHS)EuV0@a?+#>$CTk>!RKFWd0*P{NHrZ29SV&moB;@2c zrG;`ZX(Pku&k2_j9@JYvb~CQtm%W}y>=@ivm4%mf3BN7I+s-;{gk9Tx>{pyw^`0bu zfXLb2H$>*28QRej|3rcVi~XtrEVd((s{+;KPWyeXNpjUz2^tYp4f-Clt+)x0?{ zRiXGd=c7QKY$3$dpL=@h8J9aEYwp6)VU04I4B`jdV{72S2>-K821K|<&^~{iZMD}M zcXXs3hTG^lt|`@}FI8dkJ3Rb1I>J~SDk?8n?|p^Oh6Sv^wmRHv!wvD?wmMSoW@C5s zNTHG1(US}DV1ei4ju zH#pej(G+pW!x zXR877dLv8m!{6wZ?*dpDNT6HgnT(+^;<-ALn`&WdWGIsk-OG|V0H8ca>59?VP?S0V zMQ$BK5ol*mk9Zifh{T&y=PKznL)4HNj3-_a6W&81_-z_mzA!Wy148M_r78w=6E{&% zKGR(YfUQ1}5h&e8ILJ^knQcL9v4sMKpl^h*snRG3c;8Jzb@Z{ZrFD3!W~-m{db1?;dSA|vm6i1uo0R;c z11)k``}jXW?(lrc>d`AOOQ-NU7!~L(i|qmjxr>szEqc{69FtSuG!Z){f&5;e8WSY# zOA-SDithaI-y1pl*u>I4yqJIldmTgf_%gL~XgFxXtP@F0@`HgSlR~ko?zshD7$1G3 z5&=58;(>pA87IUYUMIPuji8Z>Oy%yQd|XE~S$<4fDhd0RS3ha|Q=H*gaa)_&(9~$bwnlupzK$? z;WNuh+8bEOIhp^7_fSK};X`p5Enppj5m7_W&gV}&1NPxETlpz{eV9vuy*&oky&8*( zk_yG%TU=O3lJh1e0@v?x9Qm0^j5d1W<2lGV0&gWAjV2mf+Jt9F#*ZeNSUQB4;e60l z7BoIs42=V01Q1Ao%y@3UXD~6dXnhnj{-+^I2tg|45eTk)`ppSP`E_cOGnM-tdJD@+ z+yBe}&VW@%A2qC;G%~&t4qWd5hwD?v9@`Bgkewwe+|1H4E8oTTc9o~+dj@!Meo=uG zQ(PRs1~*0Hp{Hk6qXMv#1Ic#1cA?g`lRf|eW z`V4MK`NhQbIuxmvH zO$>Y+FDMYu*VB6n@DwWDq8nI32_r5Yr=wp{+&TfK9V{Rhd!MFs4&b zUY-%ya3<{#=o}_mX@BYdR>e8*;7;5ps zMlmrlQQ=EWLV}H(yE_KJ9~j_{{@aQFOqtY88yz>XbPTVOT=bfA3DPVP^Gl{qoD^lB z6UE#_Zv$Gr>f0MBNBfC8-awBx?5qi@J9C*!Vs>7yj%r3E;JB_+uWQO3g~| zBmuL5tIVDHRGBK(ZaR;1)Tc&&y+`%TSJvZo%21v1IE&^<*$}{o7aV~90FGOt4rvii z+k<6xswax9${A$?4{95QfzJn6sCBRB)RT3AK1m4S=H{mP^yyO{YLpHL0)CkIPak?H zP&D34MPM+P**5CfrJs&7f&E!_>W+Q|^J;JpdTy8UVDH%<+5bA%Fp1HTL}^3rn=}hM zJ!lIddD8x(g&EzQFxu?L_F7XT$bv@h={kj|_6o_X^OHms4^ofM^rhfmPajv0o>9r` z%g3J>uOHXmDrjCgfx2*97D{5EdW@M5aeN(gInum&k6-2Z{d!I3y#38Yd5NgRf`bz_qLWC~Q0<*87VZJL?7Y zIX?<70QD`Mt&x5D*$Ls^DZhz>-Db+gG$-Py1X6(|OzSA$-$cG-IHh(yGgP)><(t#i zT8%$UzL~{lP5MaQu0y$BZaArOF=bEMQcB6=FvZDe&7HmGJ4rdADo$;$m+FEcGME}` zh%^Dj;euLoa`Tbxu=0|PhB@r{cuIev&+-AWvy08j`rG#`fTl8^B?u!9*AwJ0mT*G( zL}2y68rtAe>S0f;*`iYT3~MT(bFjOVJ>CR~W9p@2e+k@_Ldq3lH4J`+*f)e=B{Fi>M^c62Rh;ForEx zUk~y@S8wvAU!`uUADp&_1F5J1m=dRW9ZGUBS&KzWd01gR#t)Qt6!C1yF_9E< zTi%ELnHcZHfP_#&QPS!@FYLo|py8SP&$S6YI;0c{JgN%+wp4GBB8v6IFdBWE9LrzY zT0LkTTo8Du2=nEYrF@h2U&H_GpQ}OMFj^mP_Czz>+I0G&YdmH<{jh;}P(JUUJ;Wm< zX^Mq`_KdxS`k9qy_e2DuRQiXNW*U!>vvcXCEfDi0@6NQ*IqgZGjSRAx>)o-~N2AF> zwUEh(9utlYaNqbT9n59(G%((@m z+z7*tdvny!gKdR@x#h}buS3nx7nRt>ybD$_N?KFS8d-t*F%^LaV-wijH9aL-z>=D1 zk)I)_!Hu~v7-<#v+rDe5nU4Wo{uQ66?TTBaiqp@gO z_!1xD2U8k7TG;}aT2t4HpFI!#XLPhDEkWJ*-rQy$@AFu{oIjbbBL_Fn&s8Ji-Y+e5 zUnD3l^fRh^<8BMQFo5J~?t5F-j9)LvtQNys@dJr^SRn1d_!fZ7tew7>9pQrsUd)lx zrTqAQB~q~X=Dcw$6abfy+iCMyrQm;g5B!kP0 zE6FAv#;_Uj1-;|L{I_rXH8SQM&to>9FWFUhq!5fuOm&ogc%}t@bzoq{^_lIArN@XP zu==3<4>Dc^?J)A8*Dz{Xp6Ptg`RNZLWO5|eS|4*fc%_6*RUB$?s&@Fu3hcJ{Avs`Z z5Zc*j?oA25wMEcO{Nc(J2hQ|-)dDd}01LnZCxTkzEu0s%9Z*r3(Lc6Ex~|Frmozzd zsc=G$=$C%rC+o*m8HTiW+ryt|lyGRKlq}0-jpg-gBT~>)0p|JY+)O%1j%*8P?hM?m zrH^$eV8!FVSsWUGJwJE)(JR8>HGRE|r2F63ySI94{0Y85g-7BBKNF!Kr{^?A@kzG> z>zat-{uko|TV;z{;q-D4Elk7BeHQL7Xi5f`BOf$TEugu4rSDv4o% z+4$4d*1pSbn9;i)sBgqeq@QDGOn$O33;QTm6z){}MkjA%tlVBeo+P#Q6;^*9bgJsp(tD6<#s)d5T#sK3G-B@#SLY};&Yl1Z zR2$s1#oZ)GEkzVm?~w?$r(%dYZHcv$4tf@iys6sTX^4}p<-bci=4UilR57}RWE;)q z_CH>Kh($f@(@bo}E5~(JV=)DT1eu zFVp>c7s~T7J=D~R7L(79t=^pTFS)L+CgYASSaFpMd$3ga72=J1RdH^7DSN0DSDN;w z#D8s{11%S8-L?!6WA;ik*p3-Gi<5;pH*C*|`TN)0-JT*CIX=z&&Nt%E%zW0*$ud5@ zJjlj@e_pWA`SxTVo>ZW z2FEB*9~IR+*rS@OohGr~wB-eu_45hI)_uP7BH7z04vat`7Iz4fQgBV=v-U_fmQ)cw zzR&4(asfMQ#;|2}IWHmEQh){nIMgC8J|1gY=Qg34^0V&h{^kWT&NUJ}zm!f`;Njs} z972Uz_a;VS*lxTf>y%D9;_645I{YH`Q{=Ayt3d$loNW#1#g{hPQ#(^>fvXSQ30O)0 zCNGMw)pAdDuMe3}MS8x7_+t6rI-f zBW<|+N4jk-VbMMu>ET>9HW)quRgdQV*`sGifnJZoIE^aIP0-E)^+#=C%t-Dxrzbi_ zwqyeK4Ls`9ql;nHD!DGx4R&xfpgbY0L8@_vByy;uEBBH5U4?XyeXcsL_%hgTS0 z`q?+>l5CR}*xe_A6!)*Dq=f?a&ooOTXMn0`(~pKENrxPY2zfeaUBRES-VA@zguH`l zv+_H7dzN$RyL_WG{bIk)UZXBQ318ZSneT#+FDXcDC`>oIz{4hg2Z|Upno5r|2D)bN zK@rB{Hsjz2ky$5f2mXA;d+Sz*gtT<>jga$`Tw^86gF^8-9T8QC_g)v>Q13zS0koh1 zC-7GK?>hU>V9g#XVQXAwoJEh^HlM++ElZA$jv|x9tyLD@d8KdK1ZSh2A&CAz6Hbi} z-CNHHt;_^k3J3_S32`>U@uS7z&g(sIp^){Dp_h$TgHcgU zIs$r!Qmz&r=CJi7k;w+T7z(@O+%nF4=6&e3W=5p6OG)a<>`OWo8LHua!$Rf_e10iY zUmccbO49>Gw^RLByAG?rx(&sFF9nJkfbxsvT)(S)KT0~aHLRnAT}s7N`~^_yF)DTc z3I>v*z+Hpu%R7BQzmnQA+AgtL6Jk%zJ@F~1`PLdC<@eJ&HE2d?*^do1F4cstZYIIZ zt@cRrH=sQySKo^<`S_zF2)OSJ&gFSLCrz*TE-vtlO|01R&zqNuZ{tfbYsM9s!-{dN z^-b2Z!Gn81p?-y&X`#v!(IlNYR+!n3DIan=+$W-Qiq^h3sPCq&_6@v!jcLlr^({FRjDgaG4>(>3#VFungj6;q$)n?k}mkc>mPt(!u zbJDW7hc|z=F>BLoX|r{ZN{Aj6PzEMlmo9-vsz(ERiGhIDjRs5X-DX=(ZVkO;A!weD z6FMZS&qf}3j4{|PzAwqb;UvT#4$jr%7GzC8`MMjJpU(9GK~TN?3_M=3wLx|*PJ|yp z@)2-iOno%yw(-4f+0U;Lc|VJ+S7MG4qh~pM(!*kOwhF-a?jt3zeN7Mep?6ChxLy?3 z4xh*7u2))b{9K5hK9Avblhp^=c|+7H5=rbN+282y&CpUTf+&J$F-tZ$u^i$hKs;T2 zbmItU3$JKoY@b)>jyroOqMpAjY2V1nB<$yGGN#(Xbp$|rq?A*ESxq|=r{56}0MgD= z5EuxtZVW@|`UK<_0xdlM@1plOK;B&LBpG4YP8bDUysIe@0|`^XcUoZy8V&|^zi=M-s4A;sOX$*T1;1rsBG${FT<*_FLG(*32Fi^tP=L&u-1+kM0cR5J&DC$3AbA zb$lE)q*f7n@Oq9=qnb1$?fgMxjHvAjI6)EtE{+2uGqSRxAJ*?`uKU-@jlMp)MR+}V ziKw_JN-E0A$cIGhw0|fo9btMbVySq@hvm>oFJvy?v7=n4-c$n@Y4_an3KT-Gg%4>E z8MaK9^X`kkzSN&ylc1CMBWcfL?bc4&*47s0bkvsC7Ta7jy*3jNhp`(c3l6?$5k z=4VNUNo+k>&j$$>Cuidpr}Rd0H23Zxzhsm3#TEg~4KXQwn5*At^ORr0g=p#yC>zM$ z)$w3r6e-ktQ^>6RYYEGOb23q3czc^e#&$sWn@Ko-2Tq zA$-e&bQ?!(L2Om~3^>M6`*HLl4$^$MY2nJE78SH){#|tdF{7f)C;L+62s0BVpcOw6 zKQ7hgqcjE9)ZtP8YWI$F|;t2j#oW$&Oq3^0AN#jT~vUssBB; z0dO6)y)qH;Ki3ccPk|D3%KztR?EiieJ~W3?V6Qy6cX)Q9^ZE-_0OwKrHUQ?!_y%a3 zFn}ndwF2E%mZhc@7}!KXXCYeC6wt87!*{c#$xMQQ6+uczQx%IF9f#3(C}}=ZMKCl_ zeWVD%n|fmt5)?cRU$Ve&U*>&b?F?X^@2(X5@*ZUy=ZzW6Qp?1Xg5iP2_UZjVQ??}x zOX5qv2!KV({|wC_<+oRCX9uRCd7nQIOf>~eE`~jf>HXk7vNP#25&cDP<2G@)Y^VIkA9cy#;qZ$rlS`T)ceFnke z&wgAZ*ZL^}(vVOQqle|5J@LyVOtLfqpI1^MW!e71@o1Egqm^n;AHtFr)WW!UPpZgGLP%*k2QOYL6z{t2lUxYidIpQj^?iUYjo_aX z{qRV^cT>e9X*poT=6lNBz$rj31^q4=n|w&wPikm5EuNe$sPr!m<;F-1MlSsqh}L&O z#_i{4N*N)P6Jr6S?8juw18qAy(i>BR3JevMw(j1qXi1e5t;56(-kuAXaGYtvnIA69 zcU4~)bSNq=mIOpa@BX6z7}Xv8N|fQVRrItnlO7dg1~f*OJiid9eJB)51)`cqJ8aaB zmd(a8#iYjB$-^H`tN$6~)m<;D0-Enp!p55wl*u?-{8(wlZsGb|p0pY^6EdR|C(RW^ z%GjIy`s`x%?swF0qF6Vzu>f+hp7lZ~(@r@*Tw7{5M=P9DZEQru6}YG0^$p|AZU;3{ z$XkD0?#)D;2i;id?gzR!q`<#Z6+|g*yVaz|oA-s3B2xIfEptSl{>Ln{4<2$$f4-s+ zL=Xn;d5|CZcb_kXbM58>@+mKmBy%*-k_T`BpgR1#@G@&xyMGv?3LKWd{uQ2q>zmZ0 zZp$_U%Bui4aK0Th#H0&v0S(SNBDQK+F_$VX+F}+pe`Yg56?&DoMNi8ueSLp2?q{<~N}iH0>+K6G|4+q()-t?5Ww z0>uGfu%lK-X?6RJ?1xYg=Imwp3n6blX-o3|jbT zfKhMahF>F=qu)%;D$47R$$C!*4~{r+dIaUBZ1IGdUvmKRRTg>I=D;>V5trU9^|7h- z&1r^8)W^JMBn|#Jgv(7uHnkZ~Gt7WZG)aucV`VeJ%D*EW>zZgm_kSVYfv&{{kX!TRvA1pP#c1X0u5<26+ zC60q8_xoB9N-C%UToLdD$&T*70?eNenC#`;dDhxV_5!lq= zE)LqFDs>iQCH!*$yZLBnZI)=64-lCM3uZt^x1CE0cus#;6kgr?f;1Si(>O%559CIz zf7RF7so|L&*J)Q-qJpvM4ggkVMXOWmd+Xg?Ztqj_0LA0UW59NVRSM{W%{8ZEQ1}T{{VdS^2f@WOxT$~Ql_ET!Tq)Km$ zN6#}evjm9D&fWD(Yo_cMQsjnd_ZsZ@UK>V5Ex*$@SyB_1t85?O3X=bMvD8=@fU%v- zY9^;u(M&)=%-YA6io;Ccs^ZG35if4kQ}Xq59qT@kN6-5b9?df|jylGkSGwYq9@G6h z$3ibqE$DS-17HLJHxj|rvsZZo=9n)4SEc52eD_Z^ zzZ1~CL-I*RaiHU@M*9qC!8V3BRhX=I5X~2hrB4F)s~OEd_xOXSRndDZQM-rE?*KT1 zdr!+Au7UuN127v)JXls;fLh?fSJp5Nz5M`uN=Ii4^H=BQIH2}!2eJSK2lgIH5TUNF zP92f%LWDfE`@ZbD0w@bUA3z}?C~wq2mqXW$ zD7&nV57fdN$@8CIfG~MxLKUu1wF1V1LT_83oyCIL49K8=l+PK_V=8f7W?hvVH)s{k z>{3-ri|}ws)tyNB9I;B64xb@EBQux^dY7dDLH#Im@~C45P^{6{)qMi!!ZAQKtrtKN z7(g)HTjekY03stTZS4q5oS;#rY14~e-^v;P%GFc9QQTh7x@G?#-)S*5HFfOhp7jGC zUOyI4T#N^Y!a*o3q8Gyq3d;ac7TUETkWwEm`0k)Kod39eK9TaAy|Ap9ZONW0q5;0H z;<#r02MX(1q@c!mnab>I9vXtGs_BjI!XQVNppPH_)D_FFO;Qeg_^ZLrKG$AF{VI$z zAO$x#W2c$o9IsnoMw4wAa|7bib?~A!y&Y_xMcKK1fU5IQ-=w%0AW+483D4Pw^L@~L@K5TQ%5qoBkcfTgBV@d z8$1Db-em8ucP&ZgnD@H6MW10%5iF78Tzr9AdnM(4W>lAibruW97F-z$94=vZ z6Av+cIG@RI>(~jdVM;lkbEe_GW`G6H3&0S|rJdeOv{(6p-o)BhM>SVR!Oz;66iK}> zf#|S-o3FX?d<>IHq3v?;SK-6N(QtROi`lk9w66j^KQZUp1NBX{^;>p6jE~D`dr=X&N!8)Q|&Rw z-nmQl&ZSP#*HgWpo_pyQfT?22_wS@RiC&!@ki(wjZiBN{SwO2klxF<#}@Q|-ZX zGoR0{Iv(poI9Z%)4=jKc9lY+;sV%!nd+25#pyA^XegiCJ{^<8tXYX~N@jjOa96P#) M542TEz)xQPFTow~N&o-= literal 0 HcmV?d00001 diff --git a/docs/apache-airflow-providers-http/img/connection_headers.png b/docs/apache-airflow-providers-http/img/connection_headers.png new file mode 100644 index 0000000000000000000000000000000000000000..413e9bbb38864faf0dd5664703b9089ff0b7bd5e GIT binary patch literal 5256 zcmb7Ic{r5o`yXfn{-V})%5hC16>~W0*`i5pPc5~&!_Z*&@TQfZG(MG_BoO+`NXB8 z9cLtFwPB4YYQ)t9B8V#!9>!GTtzxn9z61NxOR26(i5#r%Yd8mX zpspJ8#$8$syuMdWkp$lPLfgu|3wA>n2RozP0kP;M(D-3DJV7{eSD?`7DFLe$svtVK zQXrI18VCt+$j?P)C&oNc2yK;mP)&FT>;wtK#y-+W5L@X#Lh$T(nAMSE;3&rnyAcEC z=B`9Y)31Gasqo@r&2Lc$Ksrplcbu4A)+;|xiiy6^5E1hE2n=@Smv23B#hBxd`M!T# z;oFxEnNSws+K+=nu|$kS1<~tp{^0_Dt>)tIEzT4=aOBWXNS{k_ety2;UKs4bA;@|O zqI_48U5@qP!-su9fX5X;*!ZA*_1&YXJa?|LQ*sBZ*nIV=EK&jnJF5asx!*D8 zqD|Tiyz1lgM;Zu9wrlLxa<^2*#&T(F&im)*@7WOyvr6k48XC^!Q6Dt=gTuhh;JSW! z(nzC#D9CHZ`xnj`gs`zBwwg({QRK?w@WqL7$}3N5ZJ|8t-809c6d6RJKWPi@wu&2x zcEr;P^YfK*^h2JGK+AEsXA3O%nX@t9lWTgD%@8mrgimB`Rxh%jl#)356_u5Rr}Hff z?kZUoZp_tbcLuA?4A$sUGj)_w4(KQjf$QcHa|a|64;?m;j&!dcTY@2ScBl@KMh*uy zaWRImoXw@-$oX-~>Tsk#A-+3jdAhG?1YE7D&CiQIA7B5B6iA|ZE=_j%Y;(4J-riS3 zzCK?Y?8CVhzotq4@DLf){qv5U(jL%R7dBit5tFhO#p1X{FQ4_;Y!)_YAF(n-l;GOZ z54)J*k0Tn12Z%>5n?=l&Q35k1hNBj#dRTh-(qK?$L7sVH;mw`Gp&QXVV0s4J)6kcP zj*}~sP)CV~x22ddX`Y|i<5PF7mTn3IloxEAW*u(6*i%(x&7R-&?tm0 zc^u>|u8Vn;W}COTY;i)gElS_{pk}(GerhFG)QC&e}LIYN^nK_x>Q`K6N@g{EVKwmRXK^9NZ>d z^HiGgx_1W4yW}9TxSH^Ygbw?&xJcU6R>z?bT%}2jLma$IRh2MOqZ71LnRK|p9M{JW zTX5+0%A?Yc`38ou2VZrDA~|&5x<%F%n~Y%OursJ-)Y7YY=%p)3vUlBA^97XL;cpAk zc(&GQ1>T3=j!M_wHJF$#QGbgZ7$D)Tv;p)1kgr^3(sXt&WWY)R!f?yFA<>ogRGAmHtO$}!F+-icSz0A;Zz(U~Nh3w5b9 z3lT|Ew2h5Tj9;7%s{yr*ZvrbxJI7E~#ci6CBa`P!UKp@s$;&)2y=T@5aP7IlEEP<4 zreFPNg5DMOrzkrGj>X()qMUWUWvZ=t;$AzPR&9RSvd^vQ8s=PG_iG+RVKD-8s+6=s zzmiAYm~Y6K7}Pf)>Uu6cDGy*Iv`Va0_-eZ_ZL}NiU+`>yQz}2aF^XKA7*0>vD|g>= zuY7vCrXSkPZG1xNMx#Rkt#I&Jg1qB310fI?_5Jkvn%G#*Mv1}xf(B5A4fY?lCT)^C zdw#zuKfx-nN&m!Jo+h5uHr$Sq!ygOsWGu_YtmPRtF06bv)$djH*vG+t4s@1wynw3; z`r)r5k<@I;;@T_c%1o&w&uSa?yVcpw5I1jH)u%~E9}UlqCF_C`zBH3n#&`N(LNLH>GgW*=?Jnd{y>_P zUfD%8If`a>6G9aex(h3Jp>gxKzSQKtBwCRGR=-mqnNfB|J#{3kS$nbjDHV@67Fq+# zja|k;FwrEzSQyLcVLu5#xa`IbM@&=ezkm89GeA7gw*9fxu|U*S-wj{Fl$e4&sAe}? zZStrst<#j1$2sQs$wcv3${l=1sl23{r6_%7FueHgPYRywmNI3VeazLgAor=W%-M@P zlk!}kaC+Pf?;d_8?x;dKvSG1fAUq@K(FodiIy$8+^RpO*J}|I&C(}v#9Viw{gB(Uhx^wD6z@c9;^~di z&VD}}jHPJVTg2isr^Fht`-aMn3>l99h`|eZbP$a2a$z4io4Br9$D6Vb(rxR?sorr0 zH+KrQr^I_ik&~_1YaZKG=&QIq^vpp?il5sNlhvLi&la6#)zG`it9-e!Uy#s)Jy=j_ z2ZG$PopwtK{sOuix%Gns#yG3_Umw^8p;7u_3zvif#+)?oCd}&7rOnI&1Iny#t z**6xC8=!a3^uJ;w%f@ln7NbX-;tk)sc#DV5MyL(3o2Lb(j~^10hwCAfje{riwU{uP z$(M~9@4>=8QWXD#YHJhVSS;43DQ=g~pS?CVVbh)vDwz3D1|h2@v9E2L%3@G@Zt&~~ zyt)2B!?Ez}vvUtHH7Yh4T4>VxyV7n5?8w#n-BUqZN3PDHK}WXhDZ*$AyA>jg&HX`n zM3fX1D4V1nnqKa6)6U_6E%X2va#y}WE5mQ0Ro_owJ5&8XmR1q6bv!H~oE&@E5Oky@SUzc)d}XKWJyRt_1;{R^`9_2p4PRVDHZO5d0FAFS3fT5Q*y zTI@98!ggX5`xK)dG)jX`_o)t9+~8e)p@W$Td!@Rap2@EK9{-p{d}~K?0G35(ogWJUxF=^gdUdFEF?O|X zWW2NbTK{jGS&Udp80iSWarKc(u-BNZukco-=@vl0Op55O@~SHEh|+X=c_@u?Cj%s> zD0XYwEr&=a1B^VJ=_B5m)UPb+-V#{67M-%7%$u?R0Pv@e9(^S)5DJ{?!I)4lzp2Cy zu*3MYMAKV>(qs+N)5sO%V6kFroqLcnHVR?^75TlzNjGI{9AH z?_c<|XX>QL%G^HTMM;&N>MeFa=B|(3)@Fa`%C`!7$p@M(mq&mM*ZfOB3F(dIfsgc;t6Edc<_QlVP9D>>+G z9(9a-`*sm8%$|EAAi~rBugTHw_wUnX__s%eV}BraTQF%V>uvWltjO2WGvA3dxIB`# z4G-^45R?i0p&MVq)CF}e*~W@6R$_#(sKai!0^@_q|>wch2;jFQ|)-Vb2mh+Ain#B!$aSK zgM%NxfB(MRZ>W|n^4EvyQ{Lj0enZ)Fq-_ox5-+~6@E)2cz{)=Tzg_tc)BZ1Z3{-_s zfV<`zR)@ctG;t4Od@A2E#Twi_4PhQ2LU4~K7~b^s?6YoAsJp>6Y=by@>0cbZva(Vy zbml|wSW6P**3uV71{~N}nJuoUsQ5;m_%9^@2pIJXO?#mJ`&B6TzIZWPx5g}3+T|0^ZBmz$t5%Ur1?3w22x^7u=(PV>RJdI`5- z^5^Wpb43lPll8hFUv~BI-zdBe2yHOv7u@*7&++0#R;vR_BVT|)XF5ik6MH~$bg;6r z;<`C&{WBWUsQwm}f+pItHnl}m?H-;4%917|w!CQhqGSC%M#99CRVu=2-ZvfdRh#nV zN?ooPuB$*F`~PZaU`p;UwSw5cL^-vBGu!X4KoNq$wxIO?Z{>})&HIxWyH+xol?#nJ NWnqUZF}w83zX2P`_&Wdq literal 0 HcmV?d00001 diff --git a/docs/apache-airflow-providers-http/img/connection_username_password.png b/docs/apache-airflow-providers-http/img/connection_username_password.png new file mode 100644 index 0000000000000000000000000000000000000000..6e36e77dd4cb48f62107a3f654648587bddc58e7 GIT binary patch literal 4761 zcmchbcU03^o5!P|;P@gUqabyRBPa|Y0trZS6j4z?q$vUcqA-y*h?E3EU@S;qMnF0u z(g{VRlYmN5T9iFsdW&WcE+Gf+@j6Uopb@@XX=e3HCon2biA)vxaT*S!RJ3xgOLI`DzV z&2xKseS&`%p+IgvaCdjl&(1!j0f*;S(Sz>pj$4ZtC$~^wE!F+d(mk)c>gXq@hPy(@Cb59eAf4h1K$Ny zvvJ3q1Oy^s9O<$3lCe(nuy`l_}xUI_%)8AUWqzr*n+Y^pZitg6I zLeC6Vdf;g*nKB{9kSgh{rD%npDisdYg zw&r?`wq!pR5I#1!Ia65DJ<=ROmi8NpeRfDXd7>*VMgQd1EP~tCFV)(LekLrNK0(UJ z;64kL!JLjdt|rG;K@e?IRf<=N>*m|~;XX4xJwrpbb9|6jLlSbOdxrb-<@r)Hl|Lku zyO1VGn%Ip;hs?Xuv|qAGiK-`cS1CxX8)2p6e}u}Kw7*m#Z9=8Okk%#kX?ol(W)Ij@ zlH{JYR#{wXOxW4e(fJ+3JGGBc$+`i)#D>a>it$Cci%E-AKl-?<{o8xj(k&%2#4@|$ z&Y0hFb#>LdvNb;%qi-L$ooANSouuZnytG6=vz=grQ-eUp&667^+Ts;a&Eeyy+mplP z2)Z6>J#QV(@yaZ;E-`w%|Inw_`o#~ckGMri>-DNP@j*W-?5{1u(y{EEOQDCd z%aDe-g91V~+FvTQx9nnX%l#QJ+Y0W?mTJXJoTg+7bT^2j%$0sEm}s`W-f+_M>G&A(XNP6 zho85#oVVSEs>>*1(3=JGdmuMWvJbe)B$MM})-O$7P?K?$P{UZzf@bkm%iDH!_FaQn zhEX?C;GRb0924Wmp0aTPh@kax+&-9Ua5Tg5Qs|rG5W{YvgBwrWhnDlebNt z9hk5$ax&VOrDic^M?xu%yB!#9bM%BUKTL%7=;$PNdwnv3k%%U1^n}gMY*br3(fIgG zsC`o*#JFQM-+%GF069VFZaBj@1(vPlJu#LcZ<4x(=s#+iw1*#Zp(Y+$s)gOKpy^Mi zhh1JEh{`|oY+<3$n<1MnKkys2Z9Ux)@ z%BUKpVONr&Z|;;k8*i)*dNQ|f3*??JoyK&EP}~>~JsA>|8_DWt1a3eeJ-N5|VI}jl zGMR+Q*(I5YLM_6Y15OHtRX9Q~gnh||_Pq@YxK;hNF=yGk z|BgOMZ*$IA0Jii?4~WJH?1^JPu2w2Qf;Vo4H_G)Dd=BXh=iVR}^TRHM+Dgw^$;8e` zP$DG*Rzz~=_oft=tOs#9Y;VtPoe-f5S=C96a!oeMBG2Z&RVeC*f6spIFLmbFo?@NX z-$(3iy%C@2DEf0N_~e%_5<`V$yEaGhHzJNH86j{SAA4;j8F&G|99)9ZRFv$`dcdBi zu9uI*4r}Vk6o&h_d#IO&T@D>(j-DTBFx|o~vD0mqTH;;PV4m3X-3Q_Y2ELXyjh<)exI zAI=l-``$WPgJko zCOa?C2+evBNE_aoFXhY&d7nehVw^muQmF~`&aPISZY>J6*5|qedh&Tac5AJy9(QH( zm0IPMsCqSB_E1ax)UWj`*Btm3XMM|gi9PO4$$nVHgXxSCWr!5{^Jx6wl2xuh-mjxl z>sJSySG1r52UEhrsa5?&_IB9Q zkOnAEI3>)e{ID|3v6d+A3&aAo0YTqdcwwrOj>!D3vE=2>L>6#hcMp(G$SV=x=see* zezA@FgKI(7{j0FEvooJSAk=~-`Y5$jtM)OXL5uDODAYAE75gF&V31wiqv=~Re=1=G z$f=$(BhUK2el;TiLuQO3QAjt*f0xSq=BS^D*ykxWI+KMvgkV9i;VDWRx9}jNfL%DaGjA|W{vQ$NpS3m4=I-6QJZ}L}~wP_n}m)v?XSj1Nx_PDiK^uj29eT?=~Mn-f>J)|u$Y0Q?t#MSwG^ghl$AI*YQh zPGZ4#`&YAys;bE{_ed{f(V#TgRDd`-wlxn=(T4zRi8N6v&%LPE2D@vdn9hvyJOK)0 zo5bNQwNxHk*?5Vn)$syP6ln|k`s~Ko5m3|5!!&g$uh4iW@cxxT)+;#8UzfUEN6*nH zv?yuiGB};hJ>V9!_DASU zA8w_fYPQUpz`4E(xJHivGHtDklg(P_KGgdyG^%6DtOoA5MV+o&AQo=N*DjlLvwkSu zGld5EG;sN|JgWQ6HPoknI{`GAsz0FpaO|ptUZB5(wy)cH05C!;>Dbi)^kQzahB0rOS2)^^!pMBGqd>KLLc`E3qWiy%&Z<__TUz@VEwbOwXrxKJ5u3YvNnYXhWJ7bY-C*PQ~_Dzw@4q zcEWY^tbhD%ABytj1gg=ULuR4O$zh`M%fx*LCniZXf$I-xzzlPdW0oDL2W@J&Z{NOk z$17TQhaYX%CTXaz%nr&Mr=BHx4{sWI&DHu?bJMNu(Wi{(rNl`=i|iWidjZ+8*_hcn zTwe!ktI{1fRmN8>#Q2FIXJe)P=pUDckPlW`Vq_64XHt3^j*Y=&;I`3a(iH&Xk_p(& zRZbq87O-IFSJJoAH)Dyoi{Du~p#oD;jJFbo1WPg__Q4!Y$a6zDX6NJSxqW8aS7EDRj9^xD&NQP6Uc9bgIU$EzSuy7XkUDeWJ73?Nt>GqqxAGePKGm z5R9MD)q&%jw?d1gs+`u}vRGu(?95C154u4x6d?7k=NC@JpLJXuexIL}mF2ABTxfv- z*xT6+5O2abH*6Z%;@j@Ch9He$Y&03*q@cESMj*l(~?m8SC=FfD5o|Ix9xc zsD!xj0DRn^TkDCQ2pXG$oMrmnPw4oHL(KIfaA_uK=i|9Tm-f@spV3u+%1~@-kPTXM z-1b%NI^M9!GB+J35wTwy-(h+ShP3?`9cn=y1(?>As)tqJN~zsf7_rHMBRAgFb9$Z$ zp|w9G&miaP*bk;e`$`ZX9ZflcFmH-%&o|G_5jko22F%Ej4`_{%HO;ybVCW39*?lDqn1(u<{qD}X_YW2lcjDZtPyo^xHQn2o zV|6)CWJLAN{p;M+`8lb3EggwO?8X=;Mh~+xz~*eSC$A*D7JJ_@a`|AyqMX6$TUst1 z-FH@|vZjY>ywQlQSk){!=dO3DdTBA}DsKmgj0#@{qC13d(_6)PxV|>P53KaFMIeum z#4>N@@x$y~{zh2+mOga1(hc=BBI)8+aCBm&f};?~S92I1gGFyw-@CR!mgOcs{v9mw zSgXNV(bgh+f`Wij{0agP0++y&v?a~&`%-~MAxp`hbPV!2D#7{mX21p%0%^%#S95Ea&2 zqmQilODU|X*w>#nCFptQWHS?)&D*z2t6lTnPS;E>`|9+mBR5z>5m-R2$vMVp$@58? zC)nXvUdvIDk7z0aZ4iF6t&|#3ZEs`7Mj;%!@ot)dR zs}K|U;waI$QPms)nQDDfWZ3Y19^^!(0-7FonL^qnn`Ig1NNf9!4JdmBb|h<|m+$G} z>NBCyG2cr6D5(*fov^^Dmgl~HEB>&fF>K${(N2r99r$UmauCfl({Zv~mmEp-e^j_J z1lg+J(TCUF{4mwW6U5E=&e{GZ$=7-u7Wv&+Y%^LZMLr`C+K}Y*^~2NV!phd@<90tm zrJ#vn@XMEf6^%GjRyfl%wR+Q*2 zIy+?1^PStpdc&<^FC^2Ikm9U=TSukp{J%jB&r7>^DgT*=YK7eXx}-UiZLwfo1*9JQ2&Vsd b1GBE*^^2vH0QXn|ulCwy6NBQ54!`{!fAs15 literal 0 HcmV?d00001 From 35a26d83be25e1147f7e9e175d1d9ead0abd1048 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 08:18:23 +0100 Subject: [PATCH 021/286] feat: Implement auth_type and auth_kwargs in the AsyncHttpHook --- airflow/providers/http/hooks/http.py | 223 ++++++++++++++---------- airflow/utils/json.py | 10 -- tests/providers/http/hooks/test_http.py | 7 +- 3 files changed, 136 insertions(+), 104 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 1346a2f9ad223..76ded12276682 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -18,7 +18,9 @@ from __future__ import annotations import asyncio +import json import warnings +from logging import Logger from typing import TYPE_CHECKING, Any, Callable, cast import aiohttp @@ -32,7 +34,6 @@ from airflow.compat.functools import cache from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.hooks.base import BaseHook -from airflow.utils.json import none_safe_loads from airflow.utils.module_loading import import_string if TYPE_CHECKING: @@ -44,6 +45,7 @@ "requests.auth.HTTPBasicAuth", "requests.auth.HTTPProxyAuth", "requests.auth.HTTPDigestAuth", + "aiohttp.BasicAuth" } ) @@ -63,54 +65,35 @@ def get_auth_types() -> frozenset[str]: return auth_types -class HttpHook(BaseHook): - """Interact with HTTP servers. +def json_safe_loads(obj: str | dict | None, default: Any = None) -> Any: + """Safely loads optional JSON. - To configure the auth_type, in addition to the `auth_type` parameter, you can also: - * set the `auth_type` parameter in the Connection settings. - * define extra parameters passed to the `auth_type` class via the `auth_kwargs`, in the Connection - settings. The class will be instantiated with those parameters. + Returns 'default' (None) if the object is None. + Return the object as-is if it is a dictionary. - See :doc:`/connections/http` for full documentation. - - :param method: the API method to be called - :param http_conn_id: :ref:`http connection` that has the base - API url i.e https://www.google.com/ and optional authentication credentials. Default - headers can also be specified in the Extra field in json format. - :param auth_type: The auth type for the service - :param tcp_keep_alive: Enable TCP Keep Alive for the connection. - :param tcp_keep_alive_idle: The TCP Keep Alive Idle parameter (corresponds to ``socket.TCP_KEEPIDLE``). - :param tcp_keep_alive_count: The TCP Keep Alive count parameter (corresponds to ``socket.TCP_KEEPCNT``) - :param tcp_keep_alive_interval: The TCP Keep Alive interval parameter (corresponds to - ``socket.TCP_KEEPINTVL``) + This method is used to parse parameters passed in 'extra' into dict. + Those parameters can be None (when they are omitted), dict (when the Connection + is created via the API) or str (when Connection is created via the UI). """ + if isinstance(obj, dict): + return obj + if obj is not None: + return json.loads(obj) + return default - conn_name_attr = "http_conn_id" - default_conn_name = "http_default" - conn_type = "http" - hook_name = "HTTP" - def __init__( - self, - method: str = "POST", - http_conn_id: str = default_conn_name, - auth_type: Any = None, - tcp_keep_alive: bool = True, - tcp_keep_alive_idle: int = 120, - tcp_keep_alive_count: int = 20, - tcp_keep_alive_interval: int = 30, - ) -> None: - super().__init__() - self.http_conn_id = http_conn_id - self.method = method.upper() - self.base_url: str = "" - self._retry_obj: Callable[..., Any] - self._is_auth_type_setup: bool = auth_type is not None - self.auth_type: Any = auth_type - self.tcp_keep_alive = tcp_keep_alive - self.keep_alive_idle = tcp_keep_alive_idle - self.keep_alive_count = tcp_keep_alive_count - self.keep_alive_interval = tcp_keep_alive_interval +class HttpHookMixin: + """Common superclass for the HttpHook and HttpAsyncHook. + + Implements methods to create a Connection. + """ + + default_auth_type: Any + http_conn_id: str + base_url: str + auth_type: Any + get_connection: Callable + log: Logger @classmethod def get_connection_form_widgets(cls) -> dict[str, Any]: @@ -145,14 +128,15 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: ), } - # headers may be passed through directly or in the "extra" field in the connection - # definition - def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: - """Create a Requests HTTP session. + def load_connection_settings(self, *, headers: dict[Any, Any] | None = None) -> tuple[dict, Any]: + """Load and update the class with Connection Settings. - :param headers: additional headers to be passed through as a dictionary + Load the settings from the Connection and update the class. + Returns the headers and auth which are later passed into a request.Session + (for the HttpHook) or an aiohttp.Session (for the AsyncHttpHook). """ - session = requests.Session() + _headers = {} + _auth = None if self.http_conn_id: conn = self.get_connection(self.http_conn_id) @@ -169,36 +153,42 @@ def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: self.base_url += f":{conn.port}" conn_extra: dict = self._parse_extra(conn_extra=conn.extra_dejson) + print(conn_extra) auth_args: list[str | None] = [conn.login, conn.password] auth_kwargs: dict[str, Any] = conn_extra["auth_kwargs"] auth_type: Any = ( self.auth_type or self._load_conn_auth_type(module_name=conn_extra["auth_type"]) - or HTTPBasicAuth + or self.default_auth_type ) if any(auth_args) or auth_kwargs: - session.auth = auth_type(*auth_args, **auth_kwargs) + _auth = auth_type(*auth_args, **auth_kwargs) elif self._is_auth_type_setup: - session.auth = auth_type() + _auth = auth_type() extra_headers = conn_extra["headers"] if extra_headers: try: - session.headers.update(extra_headers) + _headers.update(extra_headers) except TypeError: self.log.warning("Connection to %s has invalid extra field.", conn.host) if headers: - session.headers.update(headers) + _headers.update(headers) - return session + return _headers, _auth @staticmethod def _parse_extra(conn_extra: dict) -> dict: + """Parse the settings from 'extra' into dict. + + The "auth_kwargs" and "headers" data from TextAreaField are returned as + string via the 'extra' field. This method converts the data to dict. + """ extra = conn_extra.copy() auth_type: str | None = extra.pop("auth_type", None) - auth_kwargs = cast(dict, none_safe_loads(extra.pop("auth_kwargs", None), default={})) - headers = cast(dict, none_safe_loads(extra.pop("headers", None), default={})) + auth_kwargs = cast(dict, json_safe_loads(extra.pop("auth_kwargs", None), default={})) + headers = cast(dict, json_safe_loads(extra.pop("headers", None), default={})) if extra: warnings.warn( @@ -213,7 +203,11 @@ def _parse_extra(conn_extra: dict) -> dict: return {"auth_type": auth_type, "auth_kwargs": auth_kwargs, "headers": headers} def _load_conn_auth_type(self, module_name: str | None) -> Any: - """Load auth_type module from extra Connection parameters.""" + """Load auth_type module from extra Connection parameters. + + Check if the auth_type module is listed in 'extra_auth_types' and load it. + This method protects against the execution of random modules. + """ if module_name: if module_name in get_auth_types(): try: @@ -232,6 +226,74 @@ def _load_conn_auth_type(self, module_name: str | None) -> Any: ) return None + +class HttpHook(HttpHookMixin, BaseHook): + """Interact with HTTP servers. + + To configure the auth_type, in addition to the `auth_type` parameter, you can also: + * set the `auth_type` parameter in the Connection settings. + * define extra parameters passed to the `auth_type` class via the `auth_kwargs`, in the Connection + settings. The class will be instantiated with those parameters. + + See :doc:`/connections/http` for full documentation. + + :param method: the API method to be called + :param http_conn_id: :ref:`http connection` that has the base + API url i.e https://www.google.com/ and optional authentication credentials. Default + headers can also be specified in the Extra field in json format. + :param auth_type: The auth type for the service + :param tcp_keep_alive: Enable TCP Keep Alive for the connection. + :param tcp_keep_alive_idle: The TCP Keep Alive Idle parameter (corresponds to ``socket.TCP_KEEPIDLE``). + :param tcp_keep_alive_count: The TCP Keep Alive count parameter (corresponds to ``socket.TCP_KEEPCNT``) + :param tcp_keep_alive_interval: The TCP Keep Alive interval parameter (corresponds to + ``socket.TCP_KEEPINTVL``) + """ + + conn_name_attr = "http_conn_id" + default_conn_name = "http_default" + conn_type = "http" + hook_name = "HTTP" + default_auth_type = HTTPBasicAuth + + def __init__( + self, + method: str = "POST", + http_conn_id: str = default_conn_name, + auth_type: Any = None, + tcp_keep_alive: bool = True, + tcp_keep_alive_idle: int = 120, + tcp_keep_alive_count: int = 20, + tcp_keep_alive_interval: int = 30, + ) -> None: + super().__init__() + self.http_conn_id = http_conn_id + self.method = method.upper() + self.base_url: str = "" + self._retry_obj: Callable[..., Any] + self._is_auth_type_setup: bool = auth_type is not None + self.auth_type: Any = auth_type + self.tcp_keep_alive = tcp_keep_alive + self.keep_alive_idle = tcp_keep_alive_idle + self.keep_alive_count = tcp_keep_alive_count + self.keep_alive_interval = tcp_keep_alive_interval + + @classmethod + def get_connection_form_widgets(cls) -> dict[str, Any]: + return super().get_connection_form_widgets() + + def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: + """Create a Requests HTTP session. + + :param headers: Additional headers to be passed through as a dictionary. + Note: Headers may also be passed in the "Headers" field in the Connection definition + """ + headers, auth = self.load_connection_settings(headers=headers) + + session = requests.Session() + session.auth = auth + session.headers.update(headers) + return session + def run( self, endpoint: str | None = None, @@ -252,7 +314,6 @@ def run( For example, ``run(json=obj)`` is passed as ``requests.Request(json=obj)`` """ extra_options = extra_options or {} - session = self.get_conn(headers) url = self.url_from_endpoint(endpoint) @@ -372,7 +433,7 @@ def test_connection(self): return False, str(e) -class HttpAsyncHook(BaseHook): +class HttpAsyncHook(HttpHookMixin, BaseHook): """Interact with HTTP servers asynchronously. :param method: the API method to be called @@ -386,12 +447,13 @@ class HttpAsyncHook(BaseHook): default_conn_name = "http_default" conn_type = "http" hook_name = "HTTP" + default_auth_type = aiohttp.BasicAuth def __init__( self, method: str = "POST", http_conn_id: str = default_conn_name, - auth_type: Any = aiohttp.BasicAuth, + auth_type: Any = None, retry_limit: int = 3, retry_delay: float = 1.0, ) -> None: @@ -399,12 +461,17 @@ def __init__( self.method = method.upper() self.base_url: str = "" self._retry_obj: Callable[..., Any] + self._is_auth_type_setup: bool = auth_type is not None self.auth_type: Any = auth_type if retry_limit < 1: raise ValueError("Retry limit must be greater than equal to 1") self.retry_limit = retry_limit self.retry_delay = retry_delay + @classmethod + def get_connection_form_widgets(cls) -> dict[str, Any]: + return super().get_connection_form_widgets() + async def run( self, endpoint: str | None = None, @@ -417,39 +484,13 @@ async def run( :param endpoint: Endpoint to be called, i.e. ``resource/v1/query?``. :param data: Payload to be uploaded or request parameters. :param headers: Additional headers to be passed through as a dict. + Note: Headers may also be passed in the "Headers" field in the Connection definition. :param extra_options: Additional kwargs to pass when creating a request. For example, ``run(json=obj)`` is passed as ``aiohttp.ClientSession().get(json=obj)``. """ extra_options = extra_options or {} - - # headers may be passed through directly or in the "extra" field in the connection - # definition - _headers = {} - auth = None - - if self.http_conn_id: - conn = await sync_to_async(self.get_connection)(self.http_conn_id) - - if conn.host and "://" in conn.host: - self.base_url = conn.host - else: - # schema defaults to HTTP - schema = conn.schema if conn.schema else "http" - host = conn.host if conn.host else "" - self.base_url = schema + "://" + host - - if conn.port: - self.base_url += f":{conn.port}" - if conn.login: - auth = self.auth_type(conn.login, conn.password) - if conn.extra: - try: - _headers.update(conn.extra_dejson) - except TypeError: - self.log.warning("Connection to %s has invalid extra field.", conn.host) - if headers: - _headers.update(headers) + headers, auth = await sync_to_async(self.load_connection_settings)(headers) base_url = (self.base_url or "").rstrip("/") endpoint = (endpoint or "").lstrip("/") @@ -478,7 +519,7 @@ async def run( url, json=data if self.method in ("POST", "PUT", "PATCH") else None, params=data if self.method == "GET" else None, - headers=_headers, + headers=headers, auth=auth, **extra_options, ) diff --git a/airflow/utils/json.py b/airflow/utils/json.py index e27e4d1b77857..4d89e340c1cd4 100644 --- a/airflow/utils/json.py +++ b/airflow/utils/json.py @@ -122,15 +122,5 @@ def orm_object_hook(dct: dict) -> object: return deserialize(dct, False) -def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> Any: - """Safely loads JSON. - - Returns None by default if the given object is None. - """ - if obj is not None: - return json.loads(obj) - return default - - # backwards compatibility AirflowJsonEncoder = WebEncoder diff --git a/tests/providers/http/hooks/test_http.py b/tests/providers/http/hooks/test_http.py index c76da82901653..add81d8cd324a 100644 --- a/tests/providers/http/hooks/test_http.py +++ b/tests/providers/http/hooks/test_http.py @@ -309,9 +309,10 @@ def test_available_connection_auth_types(self): auth_types = get_auth_types() assert auth_types == frozenset( { - "request.auth.HTTPBasicAuth", - "request.auth.HTTPProxyAuth", - "request.auth.HTTPDigestAuth", + "requests.auth.HTTPBasicAuth", + "requests.auth.HTTPProxyAuth", + "requests.auth.HTTPDigestAuth", + "aiohttp.BasicAuth", "tests.providers.http.hooks.test_http.CustomAuthBase", } ) From 26bf63feefa2a5f9c8838e898fefa60c4dd2ee8d Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 19:04:06 +0100 Subject: [PATCH 022/286] feat: Add tests --- airflow/providers/http/hooks/http.py | 5 ++--- tests/providers/http/hooks/test_http.py | 28 ++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 76ded12276682..00231eb64f9c6 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -45,7 +45,7 @@ "requests.auth.HTTPBasicAuth", "requests.auth.HTTPProxyAuth", "requests.auth.HTTPDigestAuth", - "aiohttp.BasicAuth" + "aiohttp.BasicAuth", } ) @@ -153,7 +153,6 @@ def load_connection_settings(self, *, headers: dict[Any, Any] | None = None) -> self.base_url += f":{conn.port}" conn_extra: dict = self._parse_extra(conn_extra=conn.extra_dejson) - print(conn_extra) auth_args: list[str | None] = [conn.login, conn.password] auth_kwargs: dict[str, Any] = conn_extra["auth_kwargs"] auth_type: Any = ( @@ -490,7 +489,7 @@ async def run( ``aiohttp.ClientSession().get(json=obj)``. """ extra_options = extra_options or {} - headers, auth = await sync_to_async(self.load_connection_settings)(headers) + headers, auth = await sync_to_async(self.load_connection_settings)(headers=headers) base_url = (self.base_url or "").rstrip("/") endpoint = (endpoint or "").lstrip("/") diff --git a/tests/providers/http/hooks/test_http.py b/tests/providers/http/hooks/test_http.py index add81d8cd324a..62f729f1db4ef 100644 --- a/tests/providers/http/hooks/test_http.py +++ b/tests/providers/http/hooks/test_http.py @@ -362,6 +362,32 @@ def test_connection_with_extra_auth_type_and_no_credentials(self, auth, mock_get HttpHook().get_conn({}) auth.assert_called_once() + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_string_headers_and_auth_kwargs(self, auth, mock_get_connection): + """When passed via the UI, the 'headers' and 'auth_kwargs' fields' data is + saved as string. + """ + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + login="username", + password="pass", + extra=r""" + {"auth_kwargs": "{\r\n \"endpoint\": \"http://localhost\"\r\n}", + "headers": "{\r\n \"some\": \"headers\"\r\n}"} + """, + ) + mock_get_connection.return_value = conn + + hook = HttpHook(auth_type=CustomAuthBase) + session = hook.get_conn({}) + + auth.assert_called_once_with("username", "pass", endpoint="http://localhost") + assert "auth_kwargs" not in session.headers + assert "some" in session.headers + @pytest.mark.parametrize("method", ["GET", "POST"]) def test_json_request(self, method, requests_mock): obj1 = {"a": 1, "b": "abc", "c": [1, 2, {"d": 10}]} @@ -602,7 +628,7 @@ async def test_async_post_request_with_error_code(self, aioresponse): async def test_async_request_uses_connection_extra(self, aioresponse): """Test api call asynchronously with a connection that has extra field.""" - connection_extra = {"bearer": "test"} + connection_extra = {"bearer": "test", "some": "header"} aioresponse.post( "http://test:8080/v1/test", From bf7fe2f03d80c5dd163f837d4b83f033daa3954c Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 21:04:22 +0100 Subject: [PATCH 023/286] fix: Add header and auth into FakeSession test object --- tests/providers/http/sensors/test_http.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/providers/http/sensors/test_http.py b/tests/providers/http/sensors/test_http.py index f842ea91fcd40..4a75b50384238 100644 --- a/tests/providers/http/sensors/test_http.py +++ b/tests/providers/http/sensors/test_http.py @@ -265,10 +265,14 @@ def resp_check(_): class FakeSession: + """Mock requests.Session object.""" + def __init__(self): self.response = requests.Response() self.response.status_code = 200 self.response._content = "apache/airflow".encode("ascii", "ignore") + self.headers = {} + self.auth = None def send(self, *args, **kwargs): return self.response From 3130e054749c2bc226a986df9f36e69592048539 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 21:14:54 +0100 Subject: [PATCH 024/286] fix: Use default BasicAuth in LivyAsyncHook --- airflow/providers/apache/livy/hooks/livy.py | 1 + airflow/providers/http/hooks/http.py | 2 +- docs/spelling_wordlist.txt | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/airflow/providers/apache/livy/hooks/livy.py b/airflow/providers/apache/livy/hooks/livy.py index ff2f9faac34e1..c2b983661cda1 100644 --- a/airflow/providers/apache/livy/hooks/livy.py +++ b/airflow/providers/apache/livy/hooks/livy.py @@ -489,6 +489,7 @@ def __init__( extra_headers: dict[str, Any] | None = None, ) -> None: super().__init__(http_conn_id=livy_conn_id) + self.auth_type = self.default_auth_type self.extra_headers = extra_headers or {} self.extra_options = extra_options or {} diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 00231eb64f9c6..d44913596188c 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -133,7 +133,7 @@ def load_connection_settings(self, *, headers: dict[Any, Any] | None = None) -> Load the settings from the Connection and update the class. Returns the headers and auth which are later passed into a request.Session - (for the HttpHook) or an aiohttp.Session (for the AsyncHttpHook). + (for the HttpHook) or an aiohttp.Session (for the HttpAsyncHook). """ _headers = {} _auth = None diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 191c787751028..020b2f3100009 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -21,6 +21,7 @@ afterall AgentKey aio aiobotocore +aiohttp AioSession aiplatform Airbnb From de77a7eb102422450fce7adaa2f41096dfeee42f Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 08:53:13 +0200 Subject: [PATCH 025/286] fix: Forgot to put self before invoking _url_from_endpoint --- airflow/providers/http/hooks/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index df965a4e7f2cf..e456c592e3169 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -347,7 +347,7 @@ def run( session = self.get_conn(headers) - url = _url_from_endpoint(self.base_url, endpoint) + url = self._url_from_endpoint(self.base_url, endpoint) if self.tcp_keep_alive: keep_alive_adapter = TCPKeepAliveAdapter( @@ -521,7 +521,7 @@ async def run( session_conf = self._process_session_conf(session_conf) session_conf.update(extra_options) - url = _url_from_endpoint(self.base_url, endpoint) + url = self._url_from_endpoint(self.base_url, endpoint) async with aiohttp.ClientSession() as session: if self.method == "GET": From 1b65b4dad09db695cb02bbaf5b0170b5aefbcb1b Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 09:03:54 +0200 Subject: [PATCH 026/286] refactor: Removed json_safe_loads method as suggested by Joffrey and see what happens --- airflow/providers/http/hooks/http.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index e456c592e3169..b29c1a6e79415 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -65,23 +65,6 @@ def get_auth_types() -> frozenset[str]: return auth_types -def json_safe_loads(obj: str | dict | None, default: Any = None) -> Any: - """Safely loads optional JSON. - - Returns 'default' (None) if the object is None. - Return the object as-is if it is a dictionary. - - This method is used to parse parameters passed in 'extra' into dict. - Those parameters can be None (when they are omitted), dict (when the Connection - is created via the API) or str (when Connection is created via the UI). - """ - if isinstance(obj, dict): - return obj - if obj is not None: - return json.loads(obj) - return default - - class HttpHookMixin: """Common superclass for the HttpHook and HttpAsyncHook. @@ -205,8 +188,8 @@ def _parse_extra(conn_extra: dict) -> dict: session_conf["cert"] = cert session_conf["max_redirects"] = extra.pop("max_redirects", DEFAULT_REDIRECT_LIMIT) auth_type: str | None = extra.pop("auth_type", None) - auth_kwargs = cast(dict, json_safe_loads(extra.pop("auth_kwargs", None), default={})) - headers = cast(dict, json_safe_loads(extra.pop("headers", None), default={})) + auth_kwargs = extra.pop("auth_kwargs", {}) + headers = extra.pop("headers", {}) if extra: warnings.warn( From 83f6648e8c247e05123ffc86cdc2c4744504f126 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 13:11:47 +0200 Subject: [PATCH 027/286] refactor: Fixed some static checks in http provider --- airflow/providers/http/hooks/http.py | 21 ++++++++++----------- tests/providers/http/hooks/test_http.py | 8 ++------ 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index b29c1a6e79415..e69affc1689cf 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -18,10 +18,9 @@ from __future__ import annotations import asyncio -import json import warnings from logging import Logger -from typing import TYPE_CHECKING, Any, Callable, cast +from typing import TYPE_CHECKING, Any, Callable import aiohttp import requests @@ -139,10 +138,7 @@ def load_connection_settings(self, *, headers: dict[Any, Any] | None = None) -> _session_conf = conn_extra["session_conf"] auth_args: list[str | None] = [conn.login, conn.password] auth_kwargs: dict[str, Any] = conn_extra["auth_kwargs"] - auth_type: Any = ( - self.auth_type - or self._load_conn_auth_type(module_name=conn_extra["auth_type"]) - ) + auth_type: Any = self.auth_type or self._load_conn_auth_type(module_name=conn_extra["auth_type"]) if auth_type: if any(auth_args) or auth_kwargs: @@ -177,7 +173,9 @@ def _parse_extra(conn_extra: dict) -> dict: ) # ignore this as timeout is only accepted in request method of Session if timeout is not None: session_conf["timeout"] = timeout - allow_redirects = extra.pop("allow_redirects", None) # ignore this as only max_redirects is accepted in Session + allow_redirects = extra.pop( + "allow_redirects", None + ) # ignore this as only max_redirects is accepted in Session if allow_redirects is not None: session_conf["allow_redirects"] = allow_redirects session_conf["proxies"] = extra.pop("proxies", extra.pop("proxy", {})) @@ -433,11 +431,12 @@ def run_with_advanced_retry(self, _retry_args: dict[Any, Any], *args: Any, **kwa # TODO: remove ignore type when https://github.com/jd/tenacity/issues/428 is resolved return self._retry_obj(self.run, *args, **kwargs) # type: ignore + @staticmethod def _url_from_endpoint(base_url: str | None, endpoint: str | None) -> str: - """Combine base url with endpoint.""" - if base_url and not base_url.endswith("/") and endpoint and not endpoint.startswith("/"): - return f"{base_url}/{endpoint}" - return (base_url or "") + (endpoint or "") + """Combine base url with endpoint.""" + if base_url and not base_url.endswith("/") and endpoint and not endpoint.startswith("/"): + return f"{base_url}/{endpoint}" + return (base_url or "") + (endpoint or "") def test_connection(self): """Test HTTP Connection.""" diff --git a/tests/providers/http/hooks/test_http.py b/tests/providers/http/hooks/test_http.py index 6671c4f998cc8..bbe5e5167b26b 100644 --- a/tests/providers/http/hooks/test_http.py +++ b/tests/providers/http/hooks/test_http.py @@ -701,9 +701,7 @@ async def test_async_request_uses_connection_extra(self, aioresponse): """Test api call asynchronously with a connection that has extra field.""" connection_extra = {"bearer": "test", "some": "header"} - airflow_connection = get_airflow_connection_with_extra( - extra=connection_extra - ) + airflow_connection = get_airflow_connection_with_extra(extra=connection_extra) aioresponse.post( "http://test:8080/v1/test", @@ -764,9 +762,7 @@ def test_load_connection_settings(self): "allow_redirects": False, "max_redirects": 3, } - airflow_connection = get_airflow_connection_with_extra( - extra=extra - ) + airflow_connection = get_airflow_connection_with_extra(extra=extra) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=airflow_connection): headers, auth, session_conf = HttpAsyncHook().load_connection_settings(headers={"bearer": "test"}) From c6dd1c3aa37c16ef9b823842905a0eecc020c48a Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 13:29:01 +0200 Subject: [PATCH 028/286] refactor: Moved _url_from_endpoint method to HttpHookMixin --- airflow/providers/http/hooks/http.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index e69affc1689cf..70a1313091f8b 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -230,6 +230,13 @@ def _load_conn_auth_type(self, module_name: str | None) -> Any: ) return None + @staticmethod + def _url_from_endpoint(base_url: str | None, endpoint: str | None) -> str: + """Combine base url with endpoint.""" + if base_url and not base_url.endswith("/") and endpoint and not endpoint.startswith("/"): + return f"{base_url}/{endpoint}" + return (base_url or "") + (endpoint or "") + class HttpHook(HttpHookMixin, BaseHook): """Interact with HTTP servers. @@ -431,13 +438,6 @@ def run_with_advanced_retry(self, _retry_args: dict[Any, Any], *args: Any, **kwa # TODO: remove ignore type when https://github.com/jd/tenacity/issues/428 is resolved return self._retry_obj(self.run, *args, **kwargs) # type: ignore - @staticmethod - def _url_from_endpoint(base_url: str | None, endpoint: str | None) -> str: - """Combine base url with endpoint.""" - if base_url and not base_url.endswith("/") and endpoint and not endpoint.startswith("/"): - return f"{base_url}/{endpoint}" - return (base_url or "") + (endpoint or "") - def test_connection(self): """Test HTTP Connection.""" try: From 0c02a96c12d1e189681f9e11e6f28183ef16c940 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 13:32:10 +0200 Subject: [PATCH 029/286] refactor: Moved default_auth_type to HttpHookMixin --- airflow/providers/http/hooks/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 70a1313091f8b..88ccd409613e5 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -73,6 +73,7 @@ class HttpHookMixin: http_conn_id: str base_url: str auth_type: Any + default_auth_type = HTTPBasicAuth get_connection: Callable log: Logger @@ -264,7 +265,6 @@ class HttpHook(HttpHookMixin, BaseHook): default_conn_name = "http_default" conn_type = "http" hook_name = "HTTP" - default_auth_type = HTTPBasicAuth def __init__( self, From 78f0475603a0f7cf9ac75f33cbba001e901643eb Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 14:54:55 +0200 Subject: [PATCH 030/286] refactor: Removed print statements and unused json param in AsyncHttpHook --- airflow/providers/http/hooks/http.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 88ccd409613e5..24b4d91a53b17 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -484,7 +484,6 @@ async def run( self, endpoint: str | None = None, data: dict[str, Any] | str | None = None, - json: dict[str, Any] | str | None = None, headers: dict[str, Any] | None = None, extra_options: dict[str, Any] | None = None, ) -> ClientResponse: @@ -524,7 +523,6 @@ async def run( raise AirflowException(f"Unexpected HTTP Method: {self.method}") for attempt in range(1, 1 + self.retry_limit): - print(f"headers: {headers}") response = await request_func( url, json=data if self.method in ("POST", "PUT", "PATCH") else None, @@ -564,7 +562,6 @@ def _process_session_conf(cls, session_conf: dict) -> dict: verify = session_conf.pop("verify") if verify is not None: session_conf["verify_ssl"] = verify - print(f"session_conf: {session_conf}") return session_conf def _retryable_error_async(self, exception: ClientResponseError) -> bool: From 0a7defd91865cd491f3381d9ff32be3ff7a61572 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 16:32:18 +0200 Subject: [PATCH 031/286] refactor: Updated url_from_endpoint back to same implementation as in main branch but still defined in HttpHookMixin --- airflow/providers/http/hooks/http.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 24b4d91a53b17..075af4d89f9a7 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -231,12 +231,11 @@ def _load_conn_auth_type(self, module_name: str | None) -> Any: ) return None - @staticmethod - def _url_from_endpoint(base_url: str | None, endpoint: str | None) -> str: + def url_from_endpoint(self, endpoint: str | None) -> str: """Combine base url with endpoint.""" - if base_url and not base_url.endswith("/") and endpoint and not endpoint.startswith("/"): - return f"{base_url}/{endpoint}" - return (base_url or "") + (endpoint or "") + if self.base_url and not self.base_url.endswith("/") and endpoint and not endpoint.startswith("/"): + return self.base_url + "/" + endpoint + return (self.base_url or "") + (endpoint or "") class HttpHook(HttpHookMixin, BaseHook): @@ -309,7 +308,12 @@ def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: session.verify = session_conf["verify"] session.cert = session_conf.get("cert") session.max_redirects = session_conf["max_redirects"] - session.headers.update(headers) + + try: + session.headers.update(headers) + except TypeError: + self.log.warning("Connection to %s has invalid extra field.", self.http_conn_id) + return session def run( @@ -335,7 +339,7 @@ def run( session = self.get_conn(headers) - url = self._url_from_endpoint(self.base_url, endpoint) + url = self.url_from_endpoint(endpoint) if self.tcp_keep_alive: keep_alive_adapter = TCPKeepAliveAdapter( @@ -502,7 +506,7 @@ async def run( session_conf = self._process_session_conf(session_conf) session_conf.update(extra_options) - url = self._url_from_endpoint(self.base_url, endpoint) + url = self.url_from_endpoint(endpoint) async with aiohttp.ClientSession() as session: if self.method == "GET": From 0e1c15967fc70d90d0c64b2a043b320931ca40fa Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 16:36:22 +0200 Subject: [PATCH 032/286] refactor: Removed docstring for removed json parameter in run method of HttpAsyncHook --- airflow/providers/http/hooks/http.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 075af4d89f9a7..0e4c60171c2b1 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -495,7 +495,6 @@ async def run( :param endpoint: Endpoint to be called, i.e. ``resource/v1/query?``. :param data: Payload to be uploaded or request parameters. - :param json: Payload to be uploaded as JSON. :param headers: Additional headers to be passed through as a dict. :param extra_options: Additional kwargs to pass when creating a request. For example, ``run(json=obj)`` is passed as From 2bafed7cf6462b62fc5b3a7eb3d9bbb0be9b7f37 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 16:38:39 +0200 Subject: [PATCH 033/286] refactor: Aligned HttpTrigger with version from main branch --- airflow/providers/http/triggers/http.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/airflow/providers/http/triggers/http.py b/airflow/providers/http/triggers/http.py index 59a484b081e04..e64c7ee7c1e1f 100644 --- a/airflow/providers/http/triggers/http.py +++ b/airflow/providers/http/triggers/http.py @@ -71,7 +71,7 @@ def __init__( self.extra_options = extra_options def serialize(self) -> tuple[str, dict[str, Any]]: - """Serialize HttpTrigger arguments and classpath.""" + """Serializes HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpTrigger", { @@ -86,7 +86,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Make a series of asynchronous http calls via a http hook.""" + """Makes a series of asynchronous http calls via an http hook.""" hook = HttpAsyncHook( method=self.method, http_conn_id=self.http_conn_id, @@ -161,7 +161,7 @@ def __init__( self.poke_interval = poke_interval def serialize(self) -> tuple[str, dict[str, Any]]: - """Serialize HttpTrigger arguments and classpath.""" + """Serializes HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpSensorTrigger", { @@ -175,7 +175,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Make a series of asynchronous http calls via an http hook.""" + """Makes a series of asynchronous http calls via an http hook.""" hook = self._get_async_hook() while True: try: @@ -186,7 +186,6 @@ async def run(self) -> AsyncIterator[TriggerEvent]: extra_options=self.extra_options, ) yield TriggerEvent(True) - return except AirflowException as exc: if str(exc).startswith("404"): await asyncio.sleep(self.poke_interval) From e09af7052f16440c2d7a69766a65dc60ea9e1b57 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 16:41:05 +0200 Subject: [PATCH 034/286] refactor: Removed test_trigger_on_post_with_data from TestHttpTrigger as it was also removed in main branch --- tests/providers/http/triggers/test_http.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tests/providers/http/triggers/test_http.py b/tests/providers/http/triggers/test_http.py index a4f2559876fae..b013a16fba10f 100644 --- a/tests/providers/http/triggers/test_http.py +++ b/tests/providers/http/triggers/test_http.py @@ -138,18 +138,3 @@ async def test_convert_response(self, client_response): assert response.encoding == client_response.get_encoding() assert response.reason == client_response.reason assert dict(response.cookies) == dict(client_response.cookies) - - @pytest.mark.db_test - @pytest.mark.asyncio - @mock.patch("aiohttp.client.ClientSession.post") - async def test_trigger_on_post_with_data(self, mock_http_post, trigger): - """ - Test that HttpTrigger fires the correct event in case of an error. - """ - generator = trigger.run() - await generator.asend(None) - mock_http_post.assert_called_once() - _, kwargs = mock_http_post.call_args - assert kwargs["data"] == TEST_DATA - assert kwargs["json"] is None - assert kwargs["params"] is None From 522e3f2140734c816ff10173d26fd7974f487d94 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 17:03:29 +0200 Subject: [PATCH 035/286] refactor: Changed docstrings in HttpTrigger to imperative mode --- airflow/providers/http/triggers/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/http/triggers/http.py b/airflow/providers/http/triggers/http.py index e64c7ee7c1e1f..9f9dea559eb92 100644 --- a/airflow/providers/http/triggers/http.py +++ b/airflow/providers/http/triggers/http.py @@ -71,7 +71,7 @@ def __init__( self.extra_options = extra_options def serialize(self) -> tuple[str, dict[str, Any]]: - """Serializes HttpTrigger arguments and classpath.""" + """Serialize HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpTrigger", { @@ -86,7 +86,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Makes a series of asynchronous http calls via an http hook.""" + """Make a series of asynchronous http calls via a http hook.""" hook = HttpAsyncHook( method=self.method, http_conn_id=self.http_conn_id, From 050747cf04041afe413ea71b605f8d2b2349430f Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 11 Apr 2024 12:15:06 +0200 Subject: [PATCH 036/286] refactor: Updated docstrings of serialize and run method of HttpTrigger --- airflow/providers/http/triggers/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/http/triggers/http.py b/airflow/providers/http/triggers/http.py index 9f9dea559eb92..675651df4762b 100644 --- a/airflow/providers/http/triggers/http.py +++ b/airflow/providers/http/triggers/http.py @@ -161,7 +161,7 @@ def __init__( self.poke_interval = poke_interval def serialize(self) -> tuple[str, dict[str, Any]]: - """Serializes HttpTrigger arguments and classpath.""" + """Serialize HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpSensorTrigger", { @@ -175,7 +175,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Makes a series of asynchronous http calls via an http hook.""" + """Make a series of asynchronous http calls via a http hook.""" hook = self._get_async_hook() while True: try: From b921e5d0556f818b232b02c08cea2a9151e656f6 Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 3 May 2024 11:23:28 +0200 Subject: [PATCH 037/286] fix: Fixed HTTP tests --- airflow/providers/http/hooks/http.py | 4 +- tests/providers/http/hooks/test_http.py | 82 +++++++++++++------------ 2 files changed, 45 insertions(+), 41 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 22e80b0388a79..dee31ff51b319 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -567,9 +567,7 @@ def _process_session_conf(cls, session_conf: dict) -> dict: verify = session_conf.pop("verify") if verify is not None: session_conf["verify_ssl"] = verify - trust_env = session_conf.pop("trust_env") - if trust_env is not None: - session_conf["trust_env"] = trust_env + session_conf.pop("trust_env") return session_conf def _retryable_error_async(self, exception: ClientResponseError) -> bool: diff --git a/tests/providers/http/hooks/test_http.py b/tests/providers/http/hooks/test_http.py index 264a5b67d7e85..fd8893ffc9ba5 100644 --- a/tests/providers/http/hooks/test_http.py +++ b/tests/providers/http/hooks/test_http.py @@ -48,7 +48,9 @@ def aioresponse(): def get_airflow_connection(conn_id: str = "http_default"): - return Connection(conn_id=conn_id, conn_type="http", host="test:8080/", extra='{"bearer": "test"}') + return Connection( + conn_id=conn_id, conn_type="http", host="test:8080/", extra='{"headers": {"bearer": "test"}}' + ) def get_airflow_connection_with_extra(extra: dict): @@ -127,7 +129,9 @@ def test_get_request_do_not_raise_for_status_if_check_response_is_false(self, re assert resp.text == '{"status":{"status": 404}}' def test_hook_contains_header_from_extra_field(self): - airflow_connection = get_airflow_connection_with_extra(extra={"bearer": "test", "some": "header"}) + airflow_connection = get_airflow_connection_with_extra( + extra={"headers": {"bearer": "test", "some": "header"}} + ) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=airflow_connection): expected_conn = get_airflow_connection() conn = self.get_hook.get_conn() @@ -139,7 +143,9 @@ def test_hook_contains_header_from_extra_field(self): assert conn.headers.get("some") == "header" def test_hook_ignore_max_redirects_from_extra_field_as_header(self): - airflow_connection = get_airflow_connection_with_extra(extra={"bearer": "test", "max_redirects": 3}) + airflow_connection = get_airflow_connection_with_extra( + extra={"headers": {"bearer": "test"}, "max_redirects": 3} + ) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=airflow_connection): expected_conn = airflow_connection() conn = self.get_hook.get_conn() @@ -155,7 +161,7 @@ def test_hook_ignore_max_redirects_from_extra_field_as_header(self): def test_hook_ignore_proxies_from_extra_field_as_header(self): airflow_connection = get_airflow_connection_with_extra( - extra={"bearer": "test", "proxies": {"http": "http://proxy:80", "https": "https://proxy:80"}} + extra={"headers": {"bearer": "test"}, "proxies": {"http": "http://proxy:80", "https": "https://proxy:80"}} ) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=airflow_connection): expected_conn = airflow_connection() @@ -171,7 +177,9 @@ def test_hook_ignore_proxies_from_extra_field_as_header(self): assert conn.trust_env is True def test_hook_ignore_verify_from_extra_field_as_header(self): - airflow_connection = get_airflow_connection_with_extra(extra={"bearer": "test", "verify": False}) + airflow_connection = get_airflow_connection_with_extra( + extra={"headers": {"bearer": "test"}, "verify": False} + ) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=airflow_connection): expected_conn = airflow_connection() conn = self.get_hook.get_conn() @@ -186,7 +194,9 @@ def test_hook_ignore_verify_from_extra_field_as_header(self): assert conn.trust_env is True def test_hook_ignore_cert_from_extra_field_as_header(self): - airflow_connection = get_airflow_connection_with_extra(extra={"bearer": "test", "cert": "cert.crt"}) + airflow_connection = get_airflow_connection_with_extra( + extra={"headers": {"bearer": "test"}, "cert": "cert.crt"} + ) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=airflow_connection): expected_conn = airflow_connection() conn = self.get_hook.get_conn() @@ -201,7 +211,9 @@ def test_hook_ignore_cert_from_extra_field_as_header(self): assert conn.trust_env is True def test_hook_ignore_trust_env_from_extra_field_as_header(self): - airflow_connection = get_airflow_connection_with_extra(extra={"bearer": "test", "trust_env": False}) + airflow_connection = get_airflow_connection_with_extra( + extra={"headers": {"bearer": "test"}, "trust_env": False} + ) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=airflow_connection): expected_conn = airflow_connection() conn = self.get_hook.get_conn() @@ -373,7 +385,7 @@ def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_conne conn_type="http", login="username", password="pass", - extra='{"x-header": 0, "auth_kwargs": {"endpoint": "http://localhost"}}', + extra='{"headers": {"x-header": 0}, "auth_kwargs": {"endpoint": "http://localhost"}}', ) mock_get_connection.return_value = conn @@ -417,7 +429,7 @@ def test_connection_with_extra_header_and_auth_type(self, auth, mock_get_connect conn_type="http", login="username", password="pass", - extra='{"x-header": 0, "auth_type": "tests.providers.http.hooks.test_http.CustomAuthBase"}', + extra='{"headers": {"x-header": 0}, "auth_type": "tests.providers.http.hooks.test_http.CustomAuthBase"}', ) mock_get_connection.return_value = conn @@ -453,9 +465,9 @@ def test_connection_with_string_headers_and_auth_kwargs(self, auth, mock_get_con conn_type="http", login="username", password="pass", - extra=r""" - {"auth_kwargs": "{\r\n \"endpoint\": \"http://localhost\"\r\n}", - "headers": "{\r\n \"some\": \"headers\"\r\n}"} + extra=""" + {"auth_kwargs": {\r\n "endpoint": "http://localhost"\r\n}, + "headers": {\r\n "some": "headers"\r\n}} """, ) mock_get_connection.return_value = conn @@ -720,7 +732,7 @@ async def test_async_request_uses_connection_extra(self, aioresponse): """Test api call asynchronously with a connection that has extra field.""" connection_extra = {"bearer": "test", "some": "header"} - airflow_connection = get_airflow_connection_with_extra(extra=connection_extra) + airflow_connection = get_airflow_connection_with_extra(extra={"headers": connection_extra}) aioresponse.post( "http://test:8080/v1/test", @@ -745,7 +757,7 @@ async def test_async_request_uses_connection_extra_with_requests_parameters(self proxy = {"http": "http://proxy:80", "https": "https://proxy:80"} airflow_connection = get_airflow_connection_with_extra( extra={ - **connection_extra, + **{"headers": connection_extra}, **{ "proxies": proxy, "timeout": 60, @@ -770,36 +782,30 @@ async def test_async_request_uses_connection_extra_with_requests_parameters(self assert mocked_function.call_args.kwargs.get("verify_ssl") is False assert mocked_function.call_args.kwargs.get("allow_redirects") is False assert mocked_function.call_args.kwargs.get("max_redirects") == 3 - assert mocked_function.call_args.kwargs.get("trust_env") is False - - def test_process_extra_options_from_connection(self): - extra_options = {} - proxy = {"http": "http://proxy:80", "https": "https://proxy:80"} - conn = get_airflow_connection_with_extra( - extra={ - "bearer": "test", - "stream": True, - "cert": "cert.crt", - "proxies": proxy, - "timeout": 60, - "verify": False, - "allow_redirects": False, - "max_redirects": 3, - "trust_env": False, - } - )() - - actual = HttpAsyncHook._process_extra_options_from_connection(conn=conn, extra_options=extra_options) + assert mocked_function.call_args.kwargs.get("trust_env") is None - assert extra_options == { - "proxy": proxy, + def test_parse_extra(self): + proxy = {"http": "http://proxy:80", "https": "https://proxy:80"} + headers = {"bearer": "test"} + session_conf = { + "stream": True, + "cert": "cert.crt", + "proxies": proxy, "timeout": 60, - "verify_ssl": False, + "verify": False, "allow_redirects": False, "max_redirects": 3, "trust_env": False, } - assert actual == {"bearer": "test"} + + actual = HttpAsyncHook._parse_extra(conn_extra={**{"headers": headers}, **session_conf}) + + assert actual == { + "auth_type": None, + "auth_kwargs": {}, + "session_conf": session_conf, + "headers": headers + } @pytest.mark.asyncio async def test_build_request_url_from_connection(self): From 5919523ef7f9aff9cac17380acab6c4e23de44c7 Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 3 May 2024 11:44:54 +0200 Subject: [PATCH 038/286] fix: Fixed static checks on HTTP tests --- tests/providers/http/hooks/test_http.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/providers/http/hooks/test_http.py b/tests/providers/http/hooks/test_http.py index fd8893ffc9ba5..90ae81969c429 100644 --- a/tests/providers/http/hooks/test_http.py +++ b/tests/providers/http/hooks/test_http.py @@ -161,7 +161,10 @@ def test_hook_ignore_max_redirects_from_extra_field_as_header(self): def test_hook_ignore_proxies_from_extra_field_as_header(self): airflow_connection = get_airflow_connection_with_extra( - extra={"headers": {"bearer": "test"}, "proxies": {"http": "http://proxy:80", "https": "https://proxy:80"}} + extra={ + "headers": {"bearer": "test"}, + "proxies": {"http": "http://proxy:80", "https": "https://proxy:80"}, + } ) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=airflow_connection): expected_conn = airflow_connection() @@ -804,7 +807,7 @@ def test_parse_extra(self): "auth_type": None, "auth_kwargs": {}, "session_conf": session_conf, - "headers": headers + "headers": headers, } @pytest.mark.asyncio From 2c247545e819be82f48a964ff8c2ad822541a506 Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 3 May 2024 14:29:19 +0200 Subject: [PATCH 039/286] refactor: Mvoed get_connection_form_widgets method from HttpHookMixin to HttpHook --- airflow/providers/http/hooks/http.py | 66 ++++++++++++++-------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index dee31ff51b319..04b0bf7e386dc 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -77,39 +77,6 @@ class HttpHookMixin: get_connection: Callable log: Logger - @classmethod - def get_connection_form_widgets(cls) -> dict[str, Any]: - """Return connection widgets to add to the connection form.""" - from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget, Select2Widget - from flask_babel import lazy_gettext - from wtforms.fields import SelectField, TextAreaField - - from airflow.www.validators import ValidJson - - default_auth_type: str = "" - auth_types_choices = frozenset({default_auth_type}) | get_auth_types() - return { - "auth_type": SelectField( - lazy_gettext("Auth type"), - choices=[(clazz, clazz) for clazz in auth_types_choices], - widget=Select2Widget(), - default=default_auth_type, - ), - "auth_kwargs": TextAreaField( - lazy_gettext("Auth kwargs"), validators=[ValidJson()], widget=BS3TextAreaFieldWidget() - ), - "headers": TextAreaField( - lazy_gettext("Headers"), - validators=[ValidJson()], - widget=BS3TextAreaFieldWidget(), - description=( - "Warning: Passing headers parameters directly in 'Extra' field is deprecated, and " - "will be removed in a future version of the Http provider. Use the 'Headers' " - "field instead." - ), - ), - } - def load_connection_settings(self, *, headers: dict[Any, Any] | None = None) -> tuple[dict, Any, dict]: """Load and update the class with Connection Settings. @@ -266,6 +233,39 @@ class HttpHook(HttpHookMixin, BaseHook): conn_type = "http" hook_name = "HTTP" + @classmethod + def get_connection_form_widgets(cls) -> dict[str, Any]: + """Return connection widgets to add to the connection form.""" + from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget, Select2Widget + from flask_babel import lazy_gettext + from wtforms.fields import SelectField, TextAreaField + + from airflow.www.validators import ValidJson + + default_auth_type: str = "" + auth_types_choices = frozenset({default_auth_type}) | get_auth_types() + return { + "auth_type": SelectField( + lazy_gettext("Auth type"), + choices=[(clazz, clazz) for clazz in auth_types_choices], + widget=Select2Widget(), + default=default_auth_type, + ), + "auth_kwargs": TextAreaField( + lazy_gettext("Auth kwargs"), validators=[ValidJson()], widget=BS3TextAreaFieldWidget() + ), + "headers": TextAreaField( + lazy_gettext("Headers"), + validators=[ValidJson()], + widget=BS3TextAreaFieldWidget(), + description=( + "Warning: Passing headers parameters directly in 'Extra' field is deprecated, and " + "will be removed in a future version of the Http provider. Use the 'Headers' " + "field instead." + ), + ), + } + def __init__( self, method: str = "POST", From 513bdad514331a0ef774095a868259d6300ef346 Mon Sep 17 00:00:00 2001 From: David Blain Date: Mon, 6 May 2024 10:47:06 +0200 Subject: [PATCH 040/286] refactor: Changed auth_type field in Http connection ui to string field instead of select field to make it appear again in UI --- airflow/providers/http/hooks/http.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 04b0bf7e386dc..ec661c6c56396 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -236,25 +236,18 @@ class HttpHook(HttpHookMixin, BaseHook): @classmethod def get_connection_form_widgets(cls) -> dict[str, Any]: """Return connection widgets to add to the connection form.""" - from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget, Select2Widget + from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget, BS3TextFieldWidget from flask_babel import lazy_gettext - from wtforms.fields import SelectField, TextAreaField + from wtforms import StringField from airflow.www.validators import ValidJson - default_auth_type: str = "" - auth_types_choices = frozenset({default_auth_type}) | get_auth_types() return { - "auth_type": SelectField( - lazy_gettext("Auth type"), - choices=[(clazz, clazz) for clazz in auth_types_choices], - widget=Select2Widget(), - default=default_auth_type, - ), - "auth_kwargs": TextAreaField( + "extra__http__auth_type": StringField(lazy_gettext("Auth type"), widget=BS3TextFieldWidget()), + "extra__http__auth_kwargs": StringField( lazy_gettext("Auth kwargs"), validators=[ValidJson()], widget=BS3TextAreaFieldWidget() ), - "headers": TextAreaField( + "extra__http__headers": StringField( lazy_gettext("Headers"), validators=[ValidJson()], widget=BS3TextAreaFieldWidget(), From fb2d2c2006028e00f7d542f2216b05503f6ce3b1 Mon Sep 17 00:00:00 2001 From: David Blain Date: Mon, 6 May 2024 11:13:14 +0200 Subject: [PATCH 041/286] refactor: Added HTTPKerberosAuth as possible AUTH_TYPE in HttpHook --- airflow/providers/http/hooks/http.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index ec661c6c56396..d3d9125b95f71 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -44,6 +44,7 @@ "requests.auth.HTTPBasicAuth", "requests.auth.HTTPProxyAuth", "requests.auth.HTTPDigestAuth", + "requests_kerberos.HTTPKerberosAuth", "aiohttp.BasicAuth", } ) From e1b7e5c3ab800d9a8467b31aab501adfba03c2bd Mon Sep 17 00:00:00 2001 From: David Blain Date: Mon, 6 May 2024 13:09:59 +0200 Subject: [PATCH 042/286] refactor: Moved get_connection_form_widgets method from HttpHook to HttpHookMixin --- airflow/providers/apache/livy/hooks/livy.py | 4 ++ airflow/providers/http/hooks/http.py | 51 +++++++++++---------- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/airflow/providers/apache/livy/hooks/livy.py b/airflow/providers/apache/livy/hooks/livy.py index 0ea9a6fa8a620..a12fc1266412d 100644 --- a/airflow/providers/apache/livy/hooks/livy.py +++ b/airflow/providers/apache/livy/hooks/livy.py @@ -80,6 +80,10 @@ class LivyHook(HttpHook, LoggingMixin): conn_type = "livy" hook_name = "Apache Livy" + @classmethod + def get_connection_form_widgets(cls) -> dict[str, Any]: + return super().get_connection_form_widgets() + def __init__( self, livy_conn_id: str = default_conn_name, diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index d3d9125b95f71..ef38728dc855f 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -72,12 +72,39 @@ class HttpHookMixin: """ http_conn_id: str + conn_type: str base_url: str auth_type: Any default_auth_type = HTTPBasicAuth get_connection: Callable log: Logger + @classmethod + def get_connection_form_widgets(cls) -> dict[str, Any]: + """Return connection widgets to add to the connection form.""" + from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget, BS3TextFieldWidget + from flask_babel import lazy_gettext + from wtforms import StringField + + from airflow.www.validators import ValidJson + + return { + f"extra__{cls.conn_type}__auth_type": StringField(lazy_gettext("Auth type"), widget=BS3TextFieldWidget()), + f"extra__{cls.conn_type}__auth_kwargs": StringField( + lazy_gettext("Auth kwargs"), validators=[ValidJson()], widget=BS3TextAreaFieldWidget() + ), + f"extra__{cls.conn_type}__headers": StringField( + lazy_gettext("Headers"), + validators=[ValidJson()], + widget=BS3TextAreaFieldWidget(), + description=( + "Warning: Passing headers parameters directly in 'Extra' field is deprecated, and " + "will be removed in a future version of the Http provider. Use the 'Headers' " + "field instead." + ), + ), + } + def load_connection_settings(self, *, headers: dict[Any, Any] | None = None) -> tuple[dict, Any, dict]: """Load and update the class with Connection Settings. @@ -236,29 +263,7 @@ class HttpHook(HttpHookMixin, BaseHook): @classmethod def get_connection_form_widgets(cls) -> dict[str, Any]: - """Return connection widgets to add to the connection form.""" - from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget, BS3TextFieldWidget - from flask_babel import lazy_gettext - from wtforms import StringField - - from airflow.www.validators import ValidJson - - return { - "extra__http__auth_type": StringField(lazy_gettext("Auth type"), widget=BS3TextFieldWidget()), - "extra__http__auth_kwargs": StringField( - lazy_gettext("Auth kwargs"), validators=[ValidJson()], widget=BS3TextAreaFieldWidget() - ), - "extra__http__headers": StringField( - lazy_gettext("Headers"), - validators=[ValidJson()], - widget=BS3TextAreaFieldWidget(), - description=( - "Warning: Passing headers parameters directly in 'Extra' field is deprecated, and " - "will be removed in a future version of the Http provider. Use the 'Headers' " - "field instead." - ), - ), - } + return super().get_connection_form_widgets() def __init__( self, From c50ed8cfa04c9005c178a6e94e26177887d27dc9 Mon Sep 17 00:00:00 2001 From: David Blain Date: Mon, 6 May 2024 13:15:40 +0200 Subject: [PATCH 043/286] refactor: Pass auth_type parameter from LivyHook to constructor of HttpHook as it has also this parameter instead of redefining the same field --- airflow/providers/apache/livy/hooks/livy.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/airflow/providers/apache/livy/hooks/livy.py b/airflow/providers/apache/livy/hooks/livy.py index a12fc1266412d..c4e25cac57a76 100644 --- a/airflow/providers/apache/livy/hooks/livy.py +++ b/airflow/providers/apache/livy/hooks/livy.py @@ -91,11 +91,9 @@ def __init__( extra_headers: dict[str, Any] | None = None, auth_type: Any | None = None, ) -> None: - super().__init__(http_conn_id=livy_conn_id) + super().__init__(http_conn_id=livy_conn_id, auth_type=auth_type) self.extra_headers = extra_headers or {} self.extra_options = extra_options or {} - if auth_type: - self.auth_type = auth_type def get_conn(self, headers: dict[str, Any] | None = None) -> Any: """ From a3cd370928195836c73d969474957330da1834e9 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 7 May 2024 10:09:51 +0200 Subject: [PATCH 044/286] refactor: Added other remaining connection properties in get_connection_form_widgets method of HttpHookMixin --- airflow/providers/http/hooks/http.py | 63 ++++++++++++++++++++++------ 1 file changed, 51 insertions(+), 12 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index ef38728dc855f..74ff0a9673bd2 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -18,7 +18,10 @@ from __future__ import annotations import asyncio +import json import warnings +from contextlib import suppress +from json import JSONDecodeError from logging import Logger from typing import TYPE_CHECKING, Any, Callable @@ -65,6 +68,24 @@ def get_auth_types() -> frozenset[str]: return auth_types +def json_loads(value: str | dict | None, default: dict | None = None) -> dict: + """Safely loads optional JSON. + + Returns 'default' (None) if the object is None. + Return the object as-is if it is a dictionary. + + This method is used to parse parameters passed in 'extra' into dict. + Those parameters can be None (when they are omitted), dict (when the Connection + is created via the API) or str (when Connection is created via the UI). + """ + if isinstance(value, dict): + return value + if value is not None: + with suppress(JSONDecodeError): + return json.loads(value) + return default | {} + + class HttpHookMixin: """Common superclass for the HttpHook and HttpAsyncHook. @@ -74,7 +95,6 @@ class HttpHookMixin: http_conn_id: str conn_type: str base_url: str - auth_type: Any default_auth_type = HTTPBasicAuth get_connection: Callable log: Logger @@ -82,20 +102,35 @@ class HttpHookMixin: @classmethod def get_connection_form_widgets(cls) -> dict[str, Any]: """Return connection widgets to add to the connection form.""" - from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget, BS3TextFieldWidget + from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget, BS3TextFieldWidget, Select2Widget from flask_babel import lazy_gettext - from wtforms import StringField + from wtforms.fields import BooleanField, SelectField, StringField, TextAreaField - from airflow.www.validators import ValidJson + default_auth_type: str = "" + auth_types_choices = frozenset({default_auth_type}) | get_auth_types() return { - f"extra__{cls.conn_type}__auth_type": StringField(lazy_gettext("Auth type"), widget=BS3TextFieldWidget()), - f"extra__{cls.conn_type}__auth_kwargs": StringField( - lazy_gettext("Auth kwargs"), validators=[ValidJson()], widget=BS3TextAreaFieldWidget() + "timeout": StringField(lazy_gettext("Timeout"), widget=BS3TextFieldWidget()), + "allow_redirects": BooleanField(lazy_gettext("Allow redirects"), default=True), + "proxies": TextAreaField(lazy_gettext("Proxies"), widget=BS3TextAreaFieldWidget()), + "stream": BooleanField(lazy_gettext("Stream"), default=False), + "verify": BooleanField(lazy_gettext("Verify"), default=True), + "trust_env": BooleanField(lazy_gettext("Trust env"), default=True), + "cert": StringField(lazy_gettext("Cert"), widget=BS3TextFieldWidget()), + "max_redirects": StringField( + lazy_gettext("Max redirects"), widget=BS3TextFieldWidget(), default=DEFAULT_REDIRECT_LIMIT + ), + "auth_type": SelectField( + lazy_gettext("Auth type"), + choices=[(clazz, clazz) for clazz in auth_types_choices], + widget=Select2Widget(), + default=default_auth_type, ), - f"extra__{cls.conn_type}__headers": StringField( + "auth_kwargs": TextAreaField( + lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget() + ), + "headers": TextAreaField( lazy_gettext("Headers"), - validators=[ValidJson()], widget=BS3TextAreaFieldWidget(), description=( "Warning: Passing headers parameters directly in 'Extra' field is deprecated, and " @@ -136,7 +171,11 @@ def load_connection_settings(self, *, headers: dict[Any, Any] | None = None) -> auth_kwargs: dict[str, Any] = conn_extra["auth_kwargs"] auth_type: Any = self.auth_type or self._load_conn_auth_type(module_name=conn_extra["auth_type"]) + self.log.info("auth_type: %s", auth_type) + if auth_type: + self.log.info("auth_args: %s", auth_args) + self.log.info("auth_kwargs: %s", auth_kwargs) if any(auth_args) or auth_kwargs: _auth = auth_type(*auth_args, **auth_kwargs) elif conn.login: @@ -174,7 +213,7 @@ def _parse_extra(conn_extra: dict) -> dict: ) # ignore this as only max_redirects is accepted in Session if allow_redirects is not None: session_conf["allow_redirects"] = allow_redirects - session_conf["proxies"] = extra.pop("proxies", extra.pop("proxy", {})) + session_conf["proxies"] = json_loads(extra.pop("proxies", extra.pop("proxy", {}))) session_conf["stream"] = extra.pop("stream", False) session_conf["verify"] = extra.pop("verify", extra.pop("verify_ssl", True)) session_conf["trust_env"] = extra.pop("trust_env", True) @@ -183,8 +222,8 @@ def _parse_extra(conn_extra: dict) -> dict: session_conf["cert"] = cert session_conf["max_redirects"] = extra.pop("max_redirects", DEFAULT_REDIRECT_LIMIT) auth_type: str | None = extra.pop("auth_type", None) - auth_kwargs = extra.pop("auth_kwargs", {}) - headers = extra.pop("headers", {}) + auth_kwargs = json_loads(extra.pop("auth_kwargs", {})) + headers = json_loads(extra.pop("headers", {})) if extra: warnings.warn( From 026fa3152f358b566f0d655bf3378432a2414019 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 7 May 2024 10:11:00 +0200 Subject: [PATCH 045/286] refactor: Updated Javascript to render all textarea's in connection with CodeMirror --- airflow/www/static/js/connection_form.js | 34 +++++++++++++----------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index e59ef9a1501c8..bfe0b3b05dd1d 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -362,20 +362,24 @@ $(document).ready(() => { // Initialize the form by setting a connection type. changeConnType(connTypeElem.value); - // Change conn.extra TextArea widget to CodeMirror - const textArea = document.getElementById("extra"); - editor = CodeMirror.fromTextArea(textArea, { - mode: { name: "javascript", json: true }, - gutters: ["CodeMirror-lint-markers"], - lineWrapping: true, - lint: true, - }); + // Get all textarea elements + const textAreas= document.getElementsByTagName("textarea"); + + Array.from(textAreas).forEach(textArea => { + // Change TextArea widget to CodeMirror + editor = CodeMirror.fromTextArea(textArea, { + mode: { name: "javascript", json: true }, + gutters: ["CodeMirror-lint-markers"], + lineWrapping: true, + lint: true, + }); - // beautify JSON but only if it is not equal to default value of empty string - const jsonData = editor.getValue(); - if (jsonData !== "") { - const data = JSON.parse(jsonData); - const formattedData = JSON.stringify(data, null, 2); - editor.setValue(formattedData); - } + // beautify JSON but only if it is not equal to default value of empty string + const jsonData = editor.getValue(); + if (jsonData !== "") { + const data = JSON.parse(jsonData); + const formattedData = JSON.stringify(data, null, 2); + editor.setValue(formattedData); + } + }); }); From af246ad3c9c548260c5fa70dbc2b0007344d5dd7 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 7 May 2024 12:32:17 +0200 Subject: [PATCH 046/286] refactor: Added debug logging statement and fixed error logging statement when import_string fails for auth_type --- airflow/providers/http/hooks/http.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 74ff0a9673bd2..1cf9585ac470a 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -171,11 +171,9 @@ def load_connection_settings(self, *, headers: dict[Any, Any] | None = None) -> auth_kwargs: dict[str, Any] = conn_extra["auth_kwargs"] auth_type: Any = self.auth_type or self._load_conn_auth_type(module_name=conn_extra["auth_type"]) - self.log.info("auth_type: %s", auth_type) + self.log.debug("auth_type: %s", auth_type) if auth_type: - self.log.info("auth_args: %s", auth_args) - self.log.info("auth_kwargs: %s", auth_kwargs) if any(auth_args) or auth_kwargs: _auth = auth_type(*auth_args, **auth_kwargs) elif conn.login: @@ -192,10 +190,13 @@ def load_connection_settings(self, *, headers: dict[Any, Any] | None = None) -> if headers: _headers.update(headers) + self.log.debug("headers: %s", _headers) + self.log.debug("auth: %s", _auth) + self.log.debug("session_conf: %s", _session_conf) + return _headers, _auth, _session_conf - @staticmethod - def _parse_extra(conn_extra: dict) -> dict: + def _parse_extra(self, conn_extra: dict) -> dict: """Parse the settings from 'extra' into dict. The "auth_kwargs" and "headers" data from TextAreaField are returned as @@ -256,14 +257,13 @@ def _load_conn_auth_type(self, module_name: str | None) -> Any: self.log.info("Loaded auth_type: %s", module_name) return module except ImportError as error: - self.log.debug("Cannot import auth_type '%s' due to: %s", error) + self.log.error("Cannot import auth_type '%s' due to: %s", module_name, error) raise AirflowException(error) - else: - self.log.warning( - "Skipping import of auth_type '%s'. The class should be listed in " - "'extra_auth_types' config of the http provider.", - module_name, - ) + self.log.warning( + "Skipping import of auth_type '%s'. The class should be listed in " + "'extra_auth_types' config of the http provider.", + module_name, + ) return None def url_from_endpoint(self, endpoint: str | None) -> str: From 4669d460317f104c3196a7bfd5a6d1adb1a4882a Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 7 May 2024 13:13:33 +0200 Subject: [PATCH 047/286] refactor: Updated http tests changed some headers with real key/values --- tests/providers/http/hooks/test_http.py | 93 +++++++++++++++---------- 1 file changed, 55 insertions(+), 38 deletions(-) diff --git a/tests/providers/http/hooks/test_http.py b/tests/providers/http/hooks/test_http.py index 90ae81969c429..e3b51234c875b 100644 --- a/tests/providers/http/hooks/test_http.py +++ b/tests/providers/http/hooks/test_http.py @@ -48,8 +48,10 @@ def aioresponse(): def get_airflow_connection(conn_id: str = "http_default"): + extra = ("{\"headers\": {\r\n \"Content-Type\": \"application/json\",\r\n \"X-Requested-By\": " + "\"Airflow\"\r\n}}") return Connection( - conn_id=conn_id, conn_type="http", host="test:8080/", extra='{"headers": {"bearer": "test"}}' + conn_id=conn_id, conn_type="http", host="test:8080/", extra=extra ) @@ -130,7 +132,7 @@ def test_get_request_do_not_raise_for_status_if_check_response_is_false(self, re def test_hook_contains_header_from_extra_field(self): airflow_connection = get_airflow_connection_with_extra( - extra={"headers": {"bearer": "test", "some": "header"}} + extra={"headers": {"Content-Type": "application/json", "X-Requested-By": "Airflow"}} ) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=airflow_connection): expected_conn = get_airflow_connection() @@ -139,19 +141,22 @@ def test_hook_contains_header_from_extra_field(self): conn_extra: dict = json.loads(expected_conn.extra) headers = dict(conn.headers, **conn_extra.pop("headers", {}), **conn_extra) assert headers == conn.headers - assert conn.headers.get("bearer") == "test" - assert conn.headers.get("some") == "header" + assert conn.headers["Content-Type"] == "application/json" + assert conn.headers["X-Requested-By"] == "Airflow" def test_hook_ignore_max_redirects_from_extra_field_as_header(self): airflow_connection = get_airflow_connection_with_extra( - extra={"headers": {"bearer": "test"}, "max_redirects": 3} + extra={ + "headers": {"Content-Type": "application/json", "X-Requested-By": "Airflow"}, + "max_redirects": 3, + } ) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=airflow_connection): expected_conn = airflow_connection() conn = self.get_hook.get_conn() assert dict(conn.headers, **json.loads(expected_conn.extra)) != conn.headers - assert conn.headers.get("bearer") == "test" - assert conn.headers.get("allow_redirects") is None + assert conn.headers["Content-Type"] == "application/json" + assert conn.headers["X-Requested-By"] == "Airflow" assert conn.proxies == {} assert conn.stream is False assert conn.verify is True @@ -162,7 +167,7 @@ def test_hook_ignore_max_redirects_from_extra_field_as_header(self): def test_hook_ignore_proxies_from_extra_field_as_header(self): airflow_connection = get_airflow_connection_with_extra( extra={ - "headers": {"bearer": "test"}, + "headers": {"Content-Type": "application/json", "X-Requested-By": "Airflow"}, "proxies": {"http": "http://proxy:80", "https": "https://proxy:80"}, } ) @@ -170,8 +175,8 @@ def test_hook_ignore_proxies_from_extra_field_as_header(self): expected_conn = airflow_connection() conn = self.get_hook.get_conn() assert dict(conn.headers, **json.loads(expected_conn.extra)) != conn.headers - assert conn.headers.get("bearer") == "test" - assert conn.headers.get("proxies") is None + assert conn.headers["Content-Type"] == "application/json" + assert conn.headers["X-Requested-By"] == "Airflow" assert conn.proxies == {"http": "http://proxy:80", "https": "https://proxy:80"} assert conn.stream is False assert conn.verify is True @@ -181,14 +186,17 @@ def test_hook_ignore_proxies_from_extra_field_as_header(self): def test_hook_ignore_verify_from_extra_field_as_header(self): airflow_connection = get_airflow_connection_with_extra( - extra={"headers": {"bearer": "test"}, "verify": False} + extra={ + "headers": {"Content-Type": "application/json", "X-Requested-By": "Airflow"}, + "verify": False, + } ) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=airflow_connection): expected_conn = airflow_connection() conn = self.get_hook.get_conn() assert dict(conn.headers, **json.loads(expected_conn.extra)) != conn.headers - assert conn.headers.get("bearer") == "test" - assert conn.headers.get("verify") is None + assert conn.headers["Content-Type"] == "application/json" + assert conn.headers["X-Requested-By"] == "Airflow" assert conn.proxies == {} assert conn.stream is False assert conn.verify is False @@ -198,14 +206,17 @@ def test_hook_ignore_verify_from_extra_field_as_header(self): def test_hook_ignore_cert_from_extra_field_as_header(self): airflow_connection = get_airflow_connection_with_extra( - extra={"headers": {"bearer": "test"}, "cert": "cert.crt"} + extra={ + "headers": {"Content-Type": "application/json", "X-Requested-By": "Airflow"}, + "cert": "cert.crt", + } ) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=airflow_connection): expected_conn = airflow_connection() conn = self.get_hook.get_conn() assert dict(conn.headers, **json.loads(expected_conn.extra)) != conn.headers - assert conn.headers.get("bearer") == "test" - assert conn.headers.get("cert") is None + assert conn.headers["Content-Type"] == "application/json" + assert conn.headers["X-Requested-By"] == "Airflow" assert conn.proxies == {} assert conn.stream is False assert conn.verify is True @@ -215,14 +226,17 @@ def test_hook_ignore_cert_from_extra_field_as_header(self): def test_hook_ignore_trust_env_from_extra_field_as_header(self): airflow_connection = get_airflow_connection_with_extra( - extra={"headers": {"bearer": "test"}, "trust_env": False} + extra={ + "headers": {"Content-Type": "application/json", "X-Requested-By": "Airflow"}, + "trust_env": False, + } ) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=airflow_connection): expected_conn = airflow_connection() conn = self.get_hook.get_conn() assert dict(conn.headers, **json.loads(expected_conn.extra)) != conn.headers - assert conn.headers.get("bearer") == "test" - assert conn.headers.get("cert") is None + assert conn.headers["Content-Type"] == "application/json" + assert conn.headers["X-Requested-By"] == "Airflow" assert conn.proxies == {} assert conn.stream is False assert conn.verify is True @@ -244,18 +258,20 @@ def test_hook_with_method_in_lowercase(self, mock_requests): @pytest.mark.db_test def test_hook_uses_provided_header(self): - conn = self.get_hook.get_conn(headers={"bearer": "newT0k3n"}) - assert conn.headers.get("bearer") == "newT0k3n" + conn = self.get_hook.get_conn(headers={"Content-Type": "text/html"}) + assert conn.headers["Content-Type"] == "text/html" + assert conn.headers.get("X-Requested-By") is None @pytest.mark.db_test def test_hook_has_no_header_from_extra(self): conn = self.get_hook.get_conn() - assert conn.headers.get("bearer") is None + assert conn.headers.get("Content-Type") is None + assert conn.headers.get("X-Requested-By") is None def test_hooks_header_from_extra_is_overridden(self): with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): - conn = self.get_hook.get_conn(headers={"bearer": "newT0k3n"}) - assert conn.headers.get("bearer") == "newT0k3n" + conn = self.get_hook.get_conn(headers={"Content-Type": "text/html"}) + assert conn.headers["Content-Type"] == "text/html" def test_post_request(self, requests_mock): requests_mock.post( @@ -332,8 +348,9 @@ def run_and_return(unused_session, prepped_request, unused_extra_options, **kwar with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): prepared_request = self.get_hook.run("v1/test", headers={"some_other_header": "test"}) actual = dict(prepared_request.headers) - assert actual.get("bearer") == "test" - assert actual.get("some_other_header") == "test" + assert actual["Content-Type"] == "application/json" + assert actual["X-Requested-By"] == "Airflow" + assert actual["some_other_header"] == "test" @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") def test_http_connection(self, mock_get_connection): @@ -406,6 +423,7 @@ def test_available_connection_auth_types(self): "requests.auth.HTTPBasicAuth", "requests.auth.HTTPProxyAuth", "requests.auth.HTTPDigestAuth", + "requests_kerberos.HTTPKerberosAuth", "aiohttp.BasicAuth", "tests.providers.http.hooks.test_http.CustomAuthBase", } @@ -733,9 +751,8 @@ async def test_async_post_request_with_error_code(self, aioresponse): @pytest.mark.asyncio async def test_async_request_uses_connection_extra(self, aioresponse): """Test api call asynchronously with a connection that has extra field.""" - - connection_extra = {"bearer": "test", "some": "header"} - airflow_connection = get_airflow_connection_with_extra(extra={"headers": connection_extra}) + headers = {"Content-Type": "application/json", "X-Requested-By": "Airflow"} + airflow_connection = get_airflow_connection_with_extra(extra={"headers": headers}) aioresponse.post( "http://test:8080/v1/test", @@ -748,19 +765,19 @@ async def test_async_request_uses_connection_extra(self, aioresponse): hook = HttpAsyncHook() with mock.patch("aiohttp.ClientSession.post", new_callable=mock.AsyncMock) as mocked_function: await hook.run("v1/test") - headers = mocked_function.call_args.kwargs.get("headers") + _headers = mocked_function.call_args.kwargs.get("headers") assert all( - key in headers and headers[key] == value for key, value in connection_extra.items() + key in _headers and _headers[key] == value for key, value in headers.items() ) @pytest.mark.asyncio async def test_async_request_uses_connection_extra_with_requests_parameters(self): """Test api call asynchronously with a connection that has extra field.""" - connection_extra = {"bearer": "test"} + headers = {"Content-Type": "application/json", "X-Requested-By": "Airflow"} proxy = {"http": "http://proxy:80", "https": "https://proxy:80"} airflow_connection = get_airflow_connection_with_extra( extra={ - **{"headers": connection_extra}, + **{"headers": headers}, **{ "proxies": proxy, "timeout": 60, @@ -776,9 +793,9 @@ async def test_async_request_uses_connection_extra_with_requests_parameters(self hook = HttpAsyncHook() with mock.patch("aiohttp.ClientSession.post", new_callable=mock.AsyncMock) as mocked_function: await hook.run("v1/test") - headers = mocked_function.call_args.kwargs.get("headers") + _headers = mocked_function.call_args.kwargs.get("headers") assert all( - key in headers and headers[key] == value for key, value in connection_extra.items() + key in headers and headers[key] == value for key, value in _headers.items() ) assert mocked_function.call_args.kwargs.get("proxy") == proxy assert mocked_function.call_args.kwargs.get("timeout") == 60 @@ -789,7 +806,7 @@ async def test_async_request_uses_connection_extra_with_requests_parameters(self def test_parse_extra(self): proxy = {"http": "http://proxy:80", "https": "https://proxy:80"} - headers = {"bearer": "test"} + headers = "{\r\n \"Content-Type\": \"application/json\",\r\n \"X-Requested-By\": \"Airflow\"\r\n}" session_conf = { "stream": True, "cert": "cert.crt", @@ -801,13 +818,13 @@ def test_parse_extra(self): "trust_env": False, } - actual = HttpAsyncHook._parse_extra(conn_extra={**{"headers": headers}, **session_conf}) + actual = HttpAsyncHook()._parse_extra(conn_extra={**{"headers": headers}, **session_conf}) assert actual == { "auth_type": None, "auth_kwargs": {}, "session_conf": session_conf, - "headers": headers, + "headers": json.loads(headers), } @pytest.mark.asyncio From 7ca9393cd38ba727c94a350de1c4aceaa551ea1f Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 7 May 2024 15:30:29 +0200 Subject: [PATCH 048/286] refactor: Refactored some assertions in http tests --- tests/providers/http/hooks/test_http.py | 27 +++++++++++++------------ 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tests/providers/http/hooks/test_http.py b/tests/providers/http/hooks/test_http.py index e3b51234c875b..b8775e30025e8 100644 --- a/tests/providers/http/hooks/test_http.py +++ b/tests/providers/http/hooks/test_http.py @@ -37,6 +37,8 @@ from airflow.models import Connection from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook, get_auth_types +DEFAULT_HEADERS = "{\r\n \"Content-Type\": \"application/json\",\r\n \"X-Requested-By\": \"Airflow\"\r\n}" + @pytest.fixture def aioresponse(): @@ -48,10 +50,8 @@ def aioresponse(): def get_airflow_connection(conn_id: str = "http_default"): - extra = ("{\"headers\": {\r\n \"Content-Type\": \"application/json\",\r\n \"X-Requested-By\": " - "\"Airflow\"\r\n}}") return Connection( - conn_id=conn_id, conn_type="http", host="test:8080/", extra=extra + conn_id=conn_id, conn_type="http", host="test:8080/", extra={"headers": DEFAULT_HEADERS} ) @@ -132,14 +132,14 @@ def test_get_request_do_not_raise_for_status_if_check_response_is_false(self, re def test_hook_contains_header_from_extra_field(self): airflow_connection = get_airflow_connection_with_extra( - extra={"headers": {"Content-Type": "application/json", "X-Requested-By": "Airflow"}} + extra={"headers": DEFAULT_HEADERS} ) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=airflow_connection): expected_conn = get_airflow_connection() conn = self.get_hook.get_conn() conn_extra: dict = json.loads(expected_conn.extra) - headers = dict(conn.headers, **conn_extra.pop("headers", {}), **conn_extra) + headers = dict(conn.headers, **json.loads(conn_extra["headers"])) assert headers == conn.headers assert conn.headers["Content-Type"] == "application/json" assert conn.headers["X-Requested-By"] == "Airflow" @@ -147,7 +147,7 @@ def test_hook_contains_header_from_extra_field(self): def test_hook_ignore_max_redirects_from_extra_field_as_header(self): airflow_connection = get_airflow_connection_with_extra( extra={ - "headers": {"Content-Type": "application/json", "X-Requested-By": "Airflow"}, + "headers": DEFAULT_HEADERS, "max_redirects": 3, } ) @@ -167,7 +167,7 @@ def test_hook_ignore_max_redirects_from_extra_field_as_header(self): def test_hook_ignore_proxies_from_extra_field_as_header(self): airflow_connection = get_airflow_connection_with_extra( extra={ - "headers": {"Content-Type": "application/json", "X-Requested-By": "Airflow"}, + "headers": DEFAULT_HEADERS, "proxies": {"http": "http://proxy:80", "https": "https://proxy:80"}, } ) @@ -207,7 +207,7 @@ def test_hook_ignore_verify_from_extra_field_as_header(self): def test_hook_ignore_cert_from_extra_field_as_header(self): airflow_connection = get_airflow_connection_with_extra( extra={ - "headers": {"Content-Type": "application/json", "X-Requested-By": "Airflow"}, + "headers": DEFAULT_HEADERS, "cert": "cert.crt", } ) @@ -227,7 +227,7 @@ def test_hook_ignore_cert_from_extra_field_as_header(self): def test_hook_ignore_trust_env_from_extra_field_as_header(self): airflow_connection = get_airflow_connection_with_extra( extra={ - "headers": {"Content-Type": "application/json", "X-Requested-By": "Airflow"}, + "headers": DEFAULT_HEADERS, "trust_env": False, } ) @@ -486,9 +486,9 @@ def test_connection_with_string_headers_and_auth_kwargs(self, auth, mock_get_con conn_type="http", login="username", password="pass", - extra=""" - {"auth_kwargs": {\r\n "endpoint": "http://localhost"\r\n}, - "headers": {\r\n "some": "headers"\r\n}} + extra=f""" + {{"auth_kwargs": {{\r\n "endpoint": "http://localhost"\r\n}}, + "headers": {DEFAULT_HEADERS}}} """, ) mock_get_connection.return_value = conn @@ -498,7 +498,8 @@ def test_connection_with_string_headers_and_auth_kwargs(self, auth, mock_get_con auth.assert_called_once_with("username", "pass", endpoint="http://localhost") assert "auth_kwargs" not in session.headers - assert "some" in session.headers + assert session.headers["Content-Type"] == "application/json" + assert session.headers["X-Requested-By"] == "Airflow" @pytest.mark.parametrize("method", ["GET", "POST"]) def test_json_request(self, method, requests_mock): From 3c5246ccc127f6320a95cffee580b12f69c9b236 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 7 May 2024 18:25:19 +0200 Subject: [PATCH 049/286] refactor: Reformatted Javascript like expected by static checks --- airflow/www/static/js/connection_form.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index bfe0b3b05dd1d..adc56f2cd3e7e 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -363,9 +363,9 @@ $(document).ready(() => { changeConnType(connTypeElem.value); // Get all textarea elements - const textAreas= document.getElementsByTagName("textarea"); + const textAreas = document.getElementsByTagName("textarea"); - Array.from(textAreas).forEach(textArea => { + Array.from(textAreas).forEach((textArea) => { // Change TextArea widget to CodeMirror editor = CodeMirror.fromTextArea(textArea, { mode: { name: "javascript", json: true }, From c15c7057fa713d509cea5b5e9380d98ee592b2d6 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 7 May 2024 18:25:37 +0200 Subject: [PATCH 050/286] refactor: Reformatted HTTP unit tests like expected by static checks --- tests/providers/http/hooks/test_http.py | 29 ++++++++++--------------- 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/tests/providers/http/hooks/test_http.py b/tests/providers/http/hooks/test_http.py index b8775e30025e8..8e5dfaa4b1a2b 100644 --- a/tests/providers/http/hooks/test_http.py +++ b/tests/providers/http/hooks/test_http.py @@ -37,7 +37,8 @@ from airflow.models import Connection from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook, get_auth_types -DEFAULT_HEADERS = "{\r\n \"Content-Type\": \"application/json\",\r\n \"X-Requested-By\": \"Airflow\"\r\n}" +DEFAULT_HEADERS_AS_STRING = '{\r\n "Content-Type": "application/json",\r\n "X-Requested-By": "Airflow"\r\n}' +DEFAULT_HEADERS = json.loads(DEFAULT_HEADERS_AS_STRING) @pytest.fixture @@ -131,23 +132,20 @@ def test_get_request_do_not_raise_for_status_if_check_response_is_false(self, re assert resp.text == '{"status":{"status": 404}}' def test_hook_contains_header_from_extra_field(self): - airflow_connection = get_airflow_connection_with_extra( - extra={"headers": DEFAULT_HEADERS} - ) + airflow_connection = get_airflow_connection_with_extra(extra={"headers": DEFAULT_HEADERS_AS_STRING}) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=airflow_connection): expected_conn = get_airflow_connection() conn = self.get_hook.get_conn() conn_extra: dict = json.loads(expected_conn.extra) - headers = dict(conn.headers, **json.loads(conn_extra["headers"])) - assert headers == conn.headers + assert dict(conn.headers, **conn_extra["headers"]) == conn.headers assert conn.headers["Content-Type"] == "application/json" assert conn.headers["X-Requested-By"] == "Airflow" def test_hook_ignore_max_redirects_from_extra_field_as_header(self): airflow_connection = get_airflow_connection_with_extra( extra={ - "headers": DEFAULT_HEADERS, + "headers": DEFAULT_HEADERS_AS_STRING, "max_redirects": 3, } ) @@ -488,7 +486,7 @@ def test_connection_with_string_headers_and_auth_kwargs(self, auth, mock_get_con password="pass", extra=f""" {{"auth_kwargs": {{\r\n "endpoint": "http://localhost"\r\n}}, - "headers": {DEFAULT_HEADERS}}} + "headers": {DEFAULT_HEADERS_AS_STRING}}} """, ) mock_get_connection.return_value = conn @@ -767,9 +765,7 @@ async def test_async_request_uses_connection_extra(self, aioresponse): with mock.patch("aiohttp.ClientSession.post", new_callable=mock.AsyncMock) as mocked_function: await hook.run("v1/test") _headers = mocked_function.call_args.kwargs.get("headers") - assert all( - key in _headers and _headers[key] == value for key, value in headers.items() - ) + assert all(key in headers and headers[key] == value for key, value in _headers.items()) @pytest.mark.asyncio async def test_async_request_uses_connection_extra_with_requests_parameters(self): @@ -778,7 +774,7 @@ async def test_async_request_uses_connection_extra_with_requests_parameters(self proxy = {"http": "http://proxy:80", "https": "https://proxy:80"} airflow_connection = get_airflow_connection_with_extra( extra={ - **{"headers": headers}, + **{"headers": DEFAULT_HEADERS}, **{ "proxies": proxy, "timeout": 60, @@ -795,9 +791,7 @@ async def test_async_request_uses_connection_extra_with_requests_parameters(self with mock.patch("aiohttp.ClientSession.post", new_callable=mock.AsyncMock) as mocked_function: await hook.run("v1/test") _headers = mocked_function.call_args.kwargs.get("headers") - assert all( - key in headers and headers[key] == value for key, value in _headers.items() - ) + assert all(key in headers and headers[key] == value for key, value in _headers.items()) assert mocked_function.call_args.kwargs.get("proxy") == proxy assert mocked_function.call_args.kwargs.get("timeout") == 60 assert mocked_function.call_args.kwargs.get("verify_ssl") is False @@ -807,7 +801,6 @@ async def test_async_request_uses_connection_extra_with_requests_parameters(self def test_parse_extra(self): proxy = {"http": "http://proxy:80", "https": "https://proxy:80"} - headers = "{\r\n \"Content-Type\": \"application/json\",\r\n \"X-Requested-By\": \"Airflow\"\r\n}" session_conf = { "stream": True, "cert": "cert.crt", @@ -819,13 +812,13 @@ def test_parse_extra(self): "trust_env": False, } - actual = HttpAsyncHook()._parse_extra(conn_extra={**{"headers": headers}, **session_conf}) + actual = HttpAsyncHook()._parse_extra(conn_extra={**{"headers": DEFAULT_HEADERS_AS_STRING}, **session_conf}) assert actual == { "auth_type": None, "auth_kwargs": {}, "session_conf": session_conf, - "headers": json.loads(headers), + "headers": DEFAULT_HEADERS, } @pytest.mark.asyncio From 1a41944ee6291e8ef92dfb07e4dee200b2276ff1 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 7 May 2024 18:28:40 +0200 Subject: [PATCH 051/286] refactor: Re-added auth_type to HttpHookMixin --- airflow/providers/http/hooks/http.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 1cf9585ac470a..8c79e5fda9004 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -83,7 +83,7 @@ def json_loads(value: str | dict | None, default: dict | None = None) -> dict: if value is not None: with suppress(JSONDecodeError): return json.loads(value) - return default | {} + return default or {} class HttpHookMixin: @@ -95,6 +95,7 @@ class HttpHookMixin: http_conn_id: str conn_type: str base_url: str + auth_type: Any default_auth_type = HTTPBasicAuth get_connection: Callable log: Logger From 4f3d2abc9e6553a78dd2ae9e7c86be28a920da6c Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 7 May 2024 19:34:37 +0200 Subject: [PATCH 052/286] refactor: Enhanced extra_dejson property to allow load string escaped nested json structures --- airflow/models/connection.py | 16 ++++++--- airflow/providers/apache/livy/hooks/livy.py | 4 +++ airflow/providers/http/hooks/http.py | 39 ++++++++------------- 3 files changed, 30 insertions(+), 29 deletions(-) diff --git a/airflow/models/connection.py b/airflow/models/connection.py index 6e1435ebfad65..1c03a78e32a1a 100644 --- a/airflow/models/connection.py +++ b/airflow/models/connection.py @@ -20,6 +20,7 @@ import json import logging import warnings +from contextlib import suppress from json import JSONDecodeError from typing import Any from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit @@ -474,18 +475,23 @@ def test_connection(self): @property def extra_dejson(self) -> dict: """Returns the extra property by deserializing json.""" - obj = {} + extra = {} + if self.extra: try: - obj = json.loads(self.extra) - + for key, value in json.loads(self.extra).items(): + if isinstance(value, str): + with suppress(JSONDecodeError): + extra[key] = json.loads(value) + else: + extra[key] = value except JSONDecodeError: self.log.exception("Failed parsing the json for conn_id %s", self.conn_id) # Mask sensitive keys from this list - mask_secret(obj) + mask_secret(extra) - return obj + return extra @classmethod def get_connection_from_secrets(cls, conn_id: str) -> Connection: diff --git a/airflow/providers/apache/livy/hooks/livy.py b/airflow/providers/apache/livy/hooks/livy.py index c4e25cac57a76..c68e0cca39735 100644 --- a/airflow/providers/apache/livy/hooks/livy.py +++ b/airflow/providers/apache/livy/hooks/livy.py @@ -84,6 +84,10 @@ class LivyHook(HttpHook, LoggingMixin): def get_connection_form_widgets(cls) -> dict[str, Any]: return super().get_connection_form_widgets() + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + return super().get_ui_field_behaviour() + def __init__( self, livy_conn_id: str = default_conn_name, diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 8c79e5fda9004..ef57963f141f0 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -18,10 +18,7 @@ from __future__ import annotations import asyncio -import json import warnings -from contextlib import suppress -from json import JSONDecodeError from logging import Logger from typing import TYPE_CHECKING, Any, Callable @@ -68,24 +65,6 @@ def get_auth_types() -> frozenset[str]: return auth_types -def json_loads(value: str | dict | None, default: dict | None = None) -> dict: - """Safely loads optional JSON. - - Returns 'default' (None) if the object is None. - Return the object as-is if it is a dictionary. - - This method is used to parse parameters passed in 'extra' into dict. - Those parameters can be None (when they are omitted), dict (when the Connection - is created via the API) or str (when Connection is created via the UI). - """ - if isinstance(value, dict): - return value - if value is not None: - with suppress(JSONDecodeError): - return json.loads(value) - return default or {} - - class HttpHookMixin: """Common superclass for the HttpHook and HttpAsyncHook. @@ -141,6 +120,14 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: ), } + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + """Return custom UI field behaviour for Hive Client Wrapper connection.""" + return { + "hidden_fields": ["extra"], + "relabeling": {}, + } + def load_connection_settings(self, *, headers: dict[Any, Any] | None = None) -> tuple[dict, Any, dict]: """Load and update the class with Connection Settings. @@ -215,7 +202,7 @@ def _parse_extra(self, conn_extra: dict) -> dict: ) # ignore this as only max_redirects is accepted in Session if allow_redirects is not None: session_conf["allow_redirects"] = allow_redirects - session_conf["proxies"] = json_loads(extra.pop("proxies", extra.pop("proxy", {}))) + session_conf["proxies"] = extra.pop("proxies", extra.pop("proxy", {})) session_conf["stream"] = extra.pop("stream", False) session_conf["verify"] = extra.pop("verify", extra.pop("verify_ssl", True)) session_conf["trust_env"] = extra.pop("trust_env", True) @@ -224,8 +211,8 @@ def _parse_extra(self, conn_extra: dict) -> dict: session_conf["cert"] = cert session_conf["max_redirects"] = extra.pop("max_redirects", DEFAULT_REDIRECT_LIMIT) auth_type: str | None = extra.pop("auth_type", None) - auth_kwargs = json_loads(extra.pop("auth_kwargs", {})) - headers = json_loads(extra.pop("headers", {})) + auth_kwargs = extra.pop("auth_kwargs", {}) + headers = extra.pop("headers", {}) if extra: warnings.warn( @@ -305,6 +292,10 @@ class HttpHook(HttpHookMixin, BaseHook): def get_connection_form_widgets(cls) -> dict[str, Any]: return super().get_connection_form_widgets() + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + return super().get_ui_field_behaviour() + def __init__( self, method: str = "POST", From b7382f15ece81e812600381f159118ddb65d42c5 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 7 May 2024 20:06:38 +0200 Subject: [PATCH 053/286] fix: Make sure value if set even if nested string structure can't be deserialized as json --- airflow/models/connection.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/airflow/models/connection.py b/airflow/models/connection.py index 1c03a78e32a1a..9e61373c9f757 100644 --- a/airflow/models/connection.py +++ b/airflow/models/connection.py @@ -480,11 +480,10 @@ def extra_dejson(self) -> dict: if self.extra: try: for key, value in json.loads(self.extra).items(): + extra[key] = value if isinstance(value, str): with suppress(JSONDecodeError): extra[key] = json.loads(value) - else: - extra[key] = value except JSONDecodeError: self.log.exception("Failed parsing the json for conn_id %s", self.conn_id) From e58df223b4a141391f9a86127e9b0c6d116a1e70 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 7 May 2024 20:47:52 +0200 Subject: [PATCH 054/286] refactor: Fixed 2 additional static check remarks --- airflow/providers/http/hooks/http.py | 4 +--- tests/providers/http/hooks/test_http.py | 4 +++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index ef57963f141f0..f5e475db6d7d8 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -106,9 +106,7 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: widget=Select2Widget(), default=default_auth_type, ), - "auth_kwargs": TextAreaField( - lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget() - ), + "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), "headers": TextAreaField( lazy_gettext("Headers"), widget=BS3TextAreaFieldWidget(), diff --git a/tests/providers/http/hooks/test_http.py b/tests/providers/http/hooks/test_http.py index 8e5dfaa4b1a2b..20c54c562ae01 100644 --- a/tests/providers/http/hooks/test_http.py +++ b/tests/providers/http/hooks/test_http.py @@ -812,7 +812,9 @@ def test_parse_extra(self): "trust_env": False, } - actual = HttpAsyncHook()._parse_extra(conn_extra={**{"headers": DEFAULT_HEADERS_AS_STRING}, **session_conf}) + actual = HttpAsyncHook()._parse_extra( + conn_extra={**{"headers": DEFAULT_HEADERS_AS_STRING}, **session_conf} + ) assert actual == { "auth_type": None, From 8bc23f0a11c72e0c58a16cbbf3c894fe4db92e3e Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 8 May 2024 09:57:56 +0200 Subject: [PATCH 055/286] refactor: Added test for extra_dejson property of Connection --- airflow/models/connection.py | 3 ++- tests/models/test_connection.py | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/airflow/models/connection.py b/airflow/models/connection.py index 9e61373c9f757..48936edb7edce 100644 --- a/airflow/models/connection.py +++ b/airflow/models/connection.py @@ -21,6 +21,7 @@ import logging import warnings from contextlib import suppress +from functools import cached_property from json import JSONDecodeError from typing import Any from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit @@ -472,7 +473,7 @@ def test_connection(self): return status, message - @property + @cached_property def extra_dejson(self) -> dict: """Returns the extra property by deserializing json.""" extra = {} diff --git a/tests/models/test_connection.py b/tests/models/test_connection.py index 21e5682c8d8a2..77100c9802cc0 100644 --- a/tests/models/test_connection.py +++ b/tests/models/test_connection.py @@ -250,3 +250,25 @@ def test_get_uri(self, connection, expected_uri): # string works as expected. def test_sanitize_conn_id(self, connection, expected_conn_id): assert connection.conn_id == expected_conn_id + + def test_extra_dejson(self): + extra = ('{"trust_env": false, "verify": false, "stream": true, "headers":' + '{\r\n "Content-Type": "application/json",\r\n "X-Requested-By": "Airflow"\r\n}}') + connection = Connection( + conn_id="pokeapi", + conn_type="http", + login="user", + password="pass", + host="https://pokeapi.co/", + port=100, + schema="https", + extra=extra, + ), + + assert connection.extra_dejson == { + "trust_env": False, + "verify": False, + "stream": True, + "headers": {"Content-Type": "application", "X-Requested-By": "Airflow"} + } + From fc4d0fa4b911caf19c7422171604a4c28586795e Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 8 May 2024 10:24:55 +0200 Subject: [PATCH 056/286] fix: Fixed the test of extra_dejson method on Connection --- tests/models/test_connection.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/models/test_connection.py b/tests/models/test_connection.py index 77100c9802cc0..126323eb443a3 100644 --- a/tests/models/test_connection.py +++ b/tests/models/test_connection.py @@ -263,12 +263,11 @@ def test_extra_dejson(self): port=100, schema="https", extra=extra, - ), + ) assert connection.extra_dejson == { "trust_env": False, "verify": False, "stream": True, - "headers": {"Content-Type": "application", "X-Requested-By": "Airflow"} + "headers": {"Content-Type": "application/json", "X-Requested-By": "Airflow"}, } - From b80cd8876ad7fa375d39d293c272817049ef5fb3 Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 8 May 2024 10:35:45 +0200 Subject: [PATCH 057/286] refactor: Do not apply the CodeMirror on the description textarea --- airflow/www/static/js/connection_form.js | 28 +++++++++++++----------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index adc56f2cd3e7e..9efcf99db7615 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -366,20 +366,22 @@ $(document).ready(() => { const textAreas = document.getElementsByTagName("textarea"); Array.from(textAreas).forEach((textArea) => { - // Change TextArea widget to CodeMirror - editor = CodeMirror.fromTextArea(textArea, { - mode: { name: "javascript", json: true }, - gutters: ["CodeMirror-lint-markers"], - lineWrapping: true, - lint: true, - }); + if (textArea.id !== "description") { + // Change TextArea widget to CodeMirror + editor = CodeMirror.fromTextArea(textArea, { + mode: {name: "javascript", json: true}, + gutters: ["CodeMirror-lint-markers"], + lineWrapping: true, + lint: true, + }); - // beautify JSON but only if it is not equal to default value of empty string - const jsonData = editor.getValue(); - if (jsonData !== "") { - const data = JSON.parse(jsonData); - const formattedData = JSON.stringify(data, null, 2); - editor.setValue(formattedData); + // beautify JSON but only if it is not equal to default value of empty string + const jsonData = editor.getValue(); + if (jsonData !== "") { + const data = JSON.parse(jsonData); + const formattedData = JSON.stringify(data, null, 2); + editor.setValue(formattedData); + } } }); }); From 27cf2e88ecb5414132fe944dd183eddabd7cc123 Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 8 May 2024 10:37:34 +0200 Subject: [PATCH 058/286] refactor: editor variable can also be a const --- airflow/www/static/js/connection_form.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index 9efcf99db7615..3d59f19b12529 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -368,7 +368,7 @@ $(document).ready(() => { Array.from(textAreas).forEach((textArea) => { if (textArea.id !== "description") { // Change TextArea widget to CodeMirror - editor = CodeMirror.fromTextArea(textArea, { + const editor = CodeMirror.fromTextArea(textArea, { mode: {name: "javascript", json: true}, gutters: ["CodeMirror-lint-markers"], lineWrapping: true, From 227f90ecd593f1df77d823dedd8ec6a0813fdefc Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 8 May 2024 10:59:31 +0200 Subject: [PATCH 059/286] refactor: Applied 2 static checks changes --- airflow/www/static/js/connection_form.js | 2 +- tests/models/test_connection.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index 3d59f19b12529..ec248a80bc99f 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -369,7 +369,7 @@ $(document).ready(() => { if (textArea.id !== "description") { // Change TextArea widget to CodeMirror const editor = CodeMirror.fromTextArea(textArea, { - mode: {name: "javascript", json: true}, + mode: { name: "javascript", json: true }, gutters: ["CodeMirror-lint-markers"], lineWrapping: true, lint: true, diff --git a/tests/models/test_connection.py b/tests/models/test_connection.py index 126323eb443a3..3f7504713f9c4 100644 --- a/tests/models/test_connection.py +++ b/tests/models/test_connection.py @@ -252,8 +252,10 @@ def test_sanitize_conn_id(self, connection, expected_conn_id): assert connection.conn_id == expected_conn_id def test_extra_dejson(self): - extra = ('{"trust_env": false, "verify": false, "stream": true, "headers":' - '{\r\n "Content-Type": "application/json",\r\n "X-Requested-By": "Airflow"\r\n}}') + extra = ( + '{"trust_env": false, "verify": false, "stream": true, "headers":' + '{\r\n "Content-Type": "application/json",\r\n "X-Requested-By": "Airflow"\r\n}}' + ) connection = Connection( conn_id="pokeapi", conn_type="http", From d8a0c2e816d9a00a9ed9a52db04c6b0adca46436 Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 8 May 2024 11:43:57 +0200 Subject: [PATCH 060/286] refactor: Removed const from editor --- airflow/www/static/js/connection_form.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index ec248a80bc99f..48014c41f3393 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -368,7 +368,7 @@ $(document).ready(() => { Array.from(textAreas).forEach((textArea) => { if (textArea.id !== "description") { // Change TextArea widget to CodeMirror - const editor = CodeMirror.fromTextArea(textArea, { + editor = CodeMirror.fromTextArea(textArea, { mode: { name: "javascript", json: true }, gutters: ["CodeMirror-lint-markers"], lineWrapping: true, From 889676614af28a3d2ee91691b989f1c99f753f88 Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 8 May 2024 16:02:23 +0200 Subject: [PATCH 061/286] refactor: Changed conn_type to ftp in test_process_form_invalid_extra_removed as http as livy do now also have custom fields --- tests/www/views/test_views_connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/www/views/test_views_connection.py b/tests/www/views/test_views_connection.py index 583bad3b0d27f..4008fa0f76c7b 100644 --- a/tests/www/views/test_views_connection.py +++ b/tests/www/views/test_views_connection.py @@ -431,9 +431,9 @@ def test_process_form_invalid_extra_removed(admin_client): Note: This can only be tested with a Hook which does not have any custom fields (otherwise the custom fields override the extra data when editing a Connection). Thus, this is currently - tested with livy. + tested with ftp. """ - conn_details = {"conn_id": "test_conn", "conn_type": "livy"} + conn_details = {"conn_id": "test_conn", "conn_type": "ftp"} conn = Connection(**conn_details, extra='{"foo": "bar"}') conn.id = 1 From 106fdd02e4b8938592931445468999215857bfae Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 8 May 2024 20:45:41 +0200 Subject: [PATCH 062/286] refactor: Extracted get_extra_dejson method from extra_dejson property so the caller can choose if nested structures should also be deserialized or not --- airflow/models/connection.py | 28 +++++++++++++++++++--------- airflow/providers/http/hooks/http.py | 2 +- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/airflow/models/connection.py b/airflow/models/connection.py index 48936edb7edce..81c9596d9befe 100644 --- a/airflow/models/connection.py +++ b/airflow/models/connection.py @@ -21,7 +21,6 @@ import logging import warnings from contextlib import suppress -from functools import cached_property from json import JSONDecodeError from typing import Any from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit @@ -473,18 +472,24 @@ def test_connection(self): return status, message - @cached_property - def extra_dejson(self) -> dict: - """Returns the extra property by deserializing json.""" + def get_extra_dejson(self, nested: bool = False) -> dict: + """ + Returns the extra property by deserializing json. + + :param nested: Determines whether nested structures are also deserialized into json (default False). + """ extra = {} if self.extra: try: - for key, value in json.loads(self.extra).items(): - extra[key] = value - if isinstance(value, str): - with suppress(JSONDecodeError): - extra[key] = json.loads(value) + if nested: + for key, value in json.loads(self.extra).items(): + extra[key] = value + if isinstance(value, str): + with suppress(JSONDecodeError): + extra[key] = json.loads(value) + else: + extra = json.loads(self.extra) except JSONDecodeError: self.log.exception("Failed parsing the json for conn_id %s", self.conn_id) @@ -493,6 +498,11 @@ def extra_dejson(self) -> dict: return extra + @property + def extra_dejson(self) -> dict: + """Returns the extra property by deserializing json.""" + return self.get_extra_dejson() + @classmethod def get_connection_from_secrets(cls, conn_id: str) -> Connection: """ diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index f5e475db6d7d8..4ca8ab6a33aab 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -151,7 +151,7 @@ def load_connection_settings(self, *, headers: dict[Any, Any] | None = None) -> if conn.port: self.base_url += f":{conn.port}" - conn_extra: dict = self._parse_extra(conn_extra=conn.extra_dejson) + conn_extra: dict = self._parse_extra(conn_extra=conn.get_extra_dejson(nested=True)) _session_conf = conn_extra["session_conf"] auth_args: list[str | None] = [conn.login, conn.password] auth_kwargs: dict[str, Any] = conn_extra["auth_kwargs"] From e70c70ad6fd5c1de7313ad9301fc954edcd3b115 Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 8 May 2024 21:08:31 +0200 Subject: [PATCH 063/286] refactor: Changed docstring of get_extra_dejson method in imperative mode --- airflow/models/connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/models/connection.py b/airflow/models/connection.py index 81c9596d9befe..95855e15fe76e 100644 --- a/airflow/models/connection.py +++ b/airflow/models/connection.py @@ -474,9 +474,9 @@ def test_connection(self): def get_extra_dejson(self, nested: bool = False) -> dict: """ - Returns the extra property by deserializing json. + Deserialize extra property to JSON. - :param nested: Determines whether nested structures are also deserialized into json (default False). + :param nested: Determines whether nested structures are also deserialized into JSON (default False). """ extra = {} From 2667681c3e9fed9e4ef77cabbe34a42fcb18fca4 Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 18 Jul 2024 15:32:40 +0200 Subject: [PATCH 064/286] refactor: Updated HttpHook --- airflow/providers/http/hooks/http.py | 44 ++++++++++++++++++---------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 4ca8ab6a33aab..02cedfc4836d4 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -39,6 +39,7 @@ if TYPE_CHECKING: from aiohttp.client_reqrep import ClientResponse + DEFAULT_AUTH_TYPES = frozenset( { "requests.auth.HTTPBasicAuth", @@ -260,7 +261,8 @@ def url_from_endpoint(self, endpoint: str | None) -> str: class HttpHook(HttpHookMixin, BaseHook): - """Interact with HTTP servers. + """ + Interact with HTTP servers. To configure the auth_type, in addition to the `auth_type` parameter, you can also: * set the `auth_type` parameter in the Connection settings. @@ -279,6 +281,7 @@ class HttpHook(HttpHookMixin, BaseHook): :param tcp_keep_alive_count: The TCP Keep Alive count parameter (corresponds to ``socket.TCP_KEEPCNT``) :param tcp_keep_alive_interval: The TCP Keep Alive interval parameter (corresponds to ``socket.TCP_KEEPINTVL``) + :param auth_args: extra arguments used to initialize the auth_type if different than default HTTPBasicAuth """ conn_name_attr = "http_conn_id" @@ -318,10 +321,11 @@ def __init__( # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: - """Create a Requests HTTP session. + """ + Create a Requests HTTP session. - :param headers: Additional headers to be passed through as a dictionary. - Note: Headers may also be passed in the "Headers" field in the Connection definition + :param headers: additional headers to be passed through as a dictionary. + Note: Headers may also be passed in the "Headers" field in the Connection definition """ headers, auth, session_conf = self.load_connection_settings(headers=headers) @@ -354,7 +358,8 @@ def run( extra_options: dict[str, Any] | None = None, **request_kwargs: Any, ) -> Any: - r"""Perform the request. + r""" + Perform the request. :param endpoint: the endpoint to be called i.e. resource/v1/query? :param data: payload to be uploaded or request parameters @@ -391,7 +396,8 @@ def run( return self.run_and_check(session, prepped_request, extra_options) def check_response(self, response: requests.Response) -> None: - """Check the status code and raise on failure. + """ + Check the status code and raise on failure. :param response: A requests response object. :raise AirflowException: If the response contains a status code not @@ -410,7 +416,8 @@ def run_and_check( prepped_request: requests.PreparedRequest, extra_options: dict[Any, Any], ) -> Any: - """Grab extra options, actually run the request, and check the result. + """ + Grab extra options, actually run the request, and check the result. :param session: the session to be used to execute the request :param prepped_request: the prepared request generated in run() @@ -447,7 +454,8 @@ def run_and_check( raise ex def run_with_advanced_retry(self, _retry_args: dict[Any, Any], *args: Any, **kwargs: Any) -> Any: - """Run the hook with retry. + """ + Run the hook with retry. This is useful for connectors which might be disturbed by intermittent issues and should not instantly fail. @@ -482,7 +490,8 @@ def test_connection(self): class HttpAsyncHook(HttpHookMixin, BaseHook): - """Interact with HTTP servers asynchronously. + """ + Interact with HTTP servers asynchronously. :param method: the API method to be called :param http_conn_id: http connection id that has the base @@ -518,20 +527,23 @@ async def run( self, endpoint: str | None = None, data: dict[str, Any] | str | None = None, + json: dict[str, Any] | str | None = None, headers: dict[str, Any] | None = None, extra_options: dict[str, Any] | None = None, ) -> ClientResponse: - """Perform an asynchronous HTTP request call. + """ + Perform an asynchronous HTTP request call. :param endpoint: Endpoint to be called, i.e. ``resource/v1/query?``. :param data: Payload to be uploaded or request parameters. + :param json: Payload to be uploaded as JSON. :param headers: Additional headers to be passed through as a dict. :param extra_options: Additional kwargs to pass when creating a request. For example, ``run(json=obj)`` is passed as ``aiohttp.ClientSession().get(json=obj)``. """ extra_options = extra_options or {} - headers, auth, session_conf = await sync_to_async(self.load_connection_settings)(headers=headers) + _headers, auth, session_conf = await sync_to_async(self.load_connection_settings)(headers=headers) session_conf = self._process_session_conf(session_conf) session_conf.update(extra_options) @@ -558,11 +570,12 @@ async def run( for attempt in range(1, 1 + self.retry_limit): response = await request_func( url, - json=data if self.method in ("POST", "PUT", "PATCH") else None, params=data if self.method == "GET" else None, - headers=headers, + data=data if self.method in ("POST", "PUT", "PATCH") else None, + json=json, + headers=_headers, auth=auth, - **session_conf, + **extra_options, ) try: response.raise_for_status() @@ -599,7 +612,8 @@ def _process_session_conf(cls, session_conf: dict) -> dict: return session_conf def _retryable_error_async(self, exception: ClientResponseError) -> bool: - """Determine whether an exception may successful on a subsequent attempt. + """ + Determine whether an exception may successful on a subsequent attempt. It considers the following to be retryable: - requests_exceptions.ConnectionError From 81919bc0fbf07bf6c05bdc6d6211faf97b570580 Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 18 Jul 2024 18:47:49 +0200 Subject: [PATCH 065/286] refactor: Reformatted some docstrings in HttpHook --- airflow/providers/http/hooks/http.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 02cedfc4836d4..51d1c70effc37 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -53,7 +53,8 @@ @cache def get_auth_types() -> frozenset[str]: - """Get comma-separated extra auth_types from airflow config. + """ + Get comma-separated extra auth_types from airflow config. Those auth_types can then be used in Connection configuration. """ @@ -67,7 +68,8 @@ def get_auth_types() -> frozenset[str]: class HttpHookMixin: - """Common superclass for the HttpHook and HttpAsyncHook. + """ + Common superclass for the HttpHook and HttpAsyncHook. Implements methods to create a Connection. """ @@ -128,7 +130,8 @@ def get_ui_field_behaviour(cls) -> dict[str, Any]: } def load_connection_settings(self, *, headers: dict[Any, Any] | None = None) -> tuple[dict, Any, dict]: - """Load and update the class with Connection Settings. + """ + Load and update the class with Connection Settings. Load the settings from the Connection and update the class. Returns the headers and auth which are later passed into a request.Session @@ -184,7 +187,8 @@ def load_connection_settings(self, *, headers: dict[Any, Any] | None = None) -> return _headers, _auth, _session_conf def _parse_extra(self, conn_extra: dict) -> dict: - """Parse the settings from 'extra' into dict. + """ + Parse the settings from 'extra' into dict. The "auth_kwargs" and "headers" data from TextAreaField are returned as string via the 'extra' field. This method converts the data to dict. @@ -231,7 +235,8 @@ def _parse_extra(self, conn_extra: dict) -> dict: } def _load_conn_auth_type(self, module_name: str | None) -> Any: - """Load auth_type module from extra Connection parameters. + """ + Load auth_type module from extra Connection parameters. Check if the auth_type module is listed in 'extra_auth_types' and load it. This method protects against the execution of random modules. From 078d75e6f42a6f504ec3db5e04f20b18af052c28 Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 18 Jul 2024 20:14:20 +0200 Subject: [PATCH 066/286] refactor: HttpHook now uses patched version of Connection + added test which checks when this patched class has to be removed so we don't forget --- airflow/providers/http/__init__.py | 14 +++++ airflow/providers/http/hooks/http.py | 75 ++++++++++++++++++++++++- tests/providers/http/hooks/test_http.py | 9 +++ 3 files changed, 97 insertions(+), 1 deletion(-) diff --git a/airflow/providers/http/__init__.py b/airflow/providers/http/__init__.py index 26166e3218be0..40b23f64a5945 100644 --- a/airflow/providers/http/__init__.py +++ b/airflow/providers/http/__init__.py @@ -37,3 +37,17 @@ raise RuntimeError( f"The package `apache-airflow-providers-http:{__version__}` needs Apache Airflow 2.7.0+" ) + + +def airflow_dependency_version(): + import re + import yaml + + from os.path import join, dirname + + with open(join(dirname(__file__), "provider.yaml"), encoding="utf-8") as file: + for dependency in yaml.safe_load(file)["dependencies"]: + if dependency.startswith('apache-airflow'): + match = re.search(r'>=([\d\.]+)', dependency) + if match: + return packaging.version.parse(match.group(1)) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 51d1c70effc37..4e15549bb9102 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -19,10 +19,14 @@ import asyncio import warnings +from contextlib import suppress +from functools import cached_property +from json import JSONDecodeError from logging import Logger from typing import TYPE_CHECKING, Any, Callable import aiohttp +import packaging.version import requests import tenacity from aiohttp import ClientResponseError @@ -34,6 +38,10 @@ from airflow.compat.functools import cache from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.hooks.base import BaseHook +from airflow.models import Connection +from airflow.providers.http import airflow_dependency_version +from airflow.utils import json +from airflow.utils.log.secrets_masker import mask_secret from airflow.utils.module_loading import import_string if TYPE_CHECKING: @@ -67,6 +75,49 @@ def get_auth_types() -> frozenset[str]: return auth_types +class ConnectionWithExtra(Connection): + def __init__( + self, + conn_id: str | None = None, + conn_type: str | None = None, + description: str | None = None, + host: str | None = None, + login: str | None = None, + password: str | None = None, + schema: str | None = None, + port: int | None = None, + extra: str | dict | None = None, + uri: str | None = None, + ): + super().__init__(conn_id, conn_type, description, host, login, password, schema, port, extra, uri) + + def get_extra_dejson(self, nested: bool = False) -> dict: + """ + Deserialize extra property to JSON. + + :param nested: Determines whether nested structures are also deserialized into JSON (default False). + """ + extra = {} + + if self.extra: + try: + if nested: + for key, value in json.loads(self.extra).items(): + extra[key] = value + if isinstance(value, str): + with suppress(JSONDecodeError): + extra[key] = json.loads(value) + else: + extra = json.loads(self.extra) + except JSONDecodeError: + self.log.exception("Failed parsing the json for conn_id %s", self.conn_id) + + # Mask sensitive keys from this list + mask_secret(extra) + + return extra + + class HttpHookMixin: """ Common superclass for the HttpHook and HttpAsyncHook. @@ -129,6 +180,28 @@ def get_ui_field_behaviour(cls) -> dict[str, Any]: "relabeling": {}, } + @cached_property + def connection(self) -> Connection: + """ + This method calls the original get_connection method from the BaseHook and returns a patched version + of the Connection class which also has the get_extra_dejson method that has been added in Apache + Airflow since 2.10.0. Once the provider depends on Airflow version 2.10.0 or higher, this method and + the ConnectionWithExtra class can be removed. + """ + conn = BaseHook.get_connection(conn_id=self.http_conn_id) + + return ConnectionWithExtra( + conn_id=self.http_conn_id, + conn_type=conn.conn_type, + description=conn.description, + host=conn.host, + login=conn.login, + password=conn.password, + schema=conn.schema, + port=conn.port, + extra=conn.extra, + ) + def load_connection_settings(self, *, headers: dict[Any, Any] | None = None) -> tuple[dict, Any, dict]: """ Load and update the class with Connection Settings. @@ -142,7 +215,7 @@ def load_connection_settings(self, *, headers: dict[Any, Any] | None = None) -> _session_conf = {} if self.http_conn_id: - conn = self.get_connection(self.http_conn_id) + conn = self.connection if conn.host and "://" in conn.host: self.base_url = conn.host diff --git a/tests/providers/http/hooks/test_http.py b/tests/providers/http/hooks/test_http.py index 20c54c562ae01..8825acdb08516 100644 --- a/tests/providers/http/hooks/test_http.py +++ b/tests/providers/http/hooks/test_http.py @@ -25,6 +25,7 @@ from http import HTTPStatus from unittest import mock +import packaging.version import pytest import requests import tenacity @@ -35,6 +36,7 @@ from airflow.exceptions import AirflowException from airflow.models import Connection +from airflow.providers.http import airflow_dependency_version from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook, get_auth_types DEFAULT_HEADERS_AS_STRING = '{\r\n "Content-Type": "application/json",\r\n "X-Requested-By": "Airflow"\r\n}' @@ -676,6 +678,13 @@ def test_url_from_endpoint(self, base_url: str, endpoint: str, expected_url: str hook.base_url = base_url assert hook.url_from_endpoint(endpoint) == expected_url + def test_airflow_dependency_version(self): + if airflow_dependency_version() >= packaging.version.parse("2.10.0"): + raise RuntimeError( + "The class ConnectionWithExtra can be removed from the HttpHook since the get_extra_dejson" + "method is now available on the Connection class since Apache Airflow 2.10.0+" + ) + class TestHttpAsyncHook: @pytest.mark.asyncio From d1da151043d098ca6ebd178b723b1a241c4bb95b Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 18 Jul 2024 20:56:42 +0200 Subject: [PATCH 067/286] refactor: Fixed some static checks --- airflow/providers/http/__init__.py | 8 ++++---- airflow/providers/http/hooks/http.py | 2 -- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/airflow/providers/http/__init__.py b/airflow/providers/http/__init__.py index 40b23f64a5945..91045a3af3884 100644 --- a/airflow/providers/http/__init__.py +++ b/airflow/providers/http/__init__.py @@ -41,13 +41,13 @@ def airflow_dependency_version(): import re - import yaml + from os.path import dirname, join - from os.path import join, dirname + import yaml with open(join(dirname(__file__), "provider.yaml"), encoding="utf-8") as file: for dependency in yaml.safe_load(file)["dependencies"]: - if dependency.startswith('apache-airflow'): - match = re.search(r'>=([\d\.]+)', dependency) + if dependency.startswith("apache-airflow"): + match = re.search(r">=([\d\.]+)", dependency) if match: return packaging.version.parse(match.group(1)) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 4e15549bb9102..7c5f6ae5e63f9 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -26,7 +26,6 @@ from typing import TYPE_CHECKING, Any, Callable import aiohttp -import packaging.version import requests import tenacity from aiohttp import ClientResponseError @@ -39,7 +38,6 @@ from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.hooks.base import BaseHook from airflow.models import Connection -from airflow.providers.http import airflow_dependency_version from airflow.utils import json from airflow.utils.log.secrets_masker import mask_secret from airflow.utils.module_loading import import_string From 94ad512a70a867eda5c58f8c872c7de3b6640595 Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 18 Jul 2024 20:58:59 +0200 Subject: [PATCH 068/286] refactor: Fixed json import HttpHook --- airflow/providers/http/hooks/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 7c5f6ae5e63f9..bb87c196abb12 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -18,6 +18,7 @@ from __future__ import annotations import asyncio +import json import warnings from contextlib import suppress from functools import cached_property @@ -38,7 +39,6 @@ from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.hooks.base import BaseHook from airflow.models import Connection -from airflow.utils import json from airflow.utils.log.secrets_masker import mask_secret from airflow.utils.module_loading import import_string From 1a1606c6e689c4594fe5c5adc0cdf647a6c031be Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 19 Jul 2024 11:30:33 +0200 Subject: [PATCH 069/286] docs: Updated docstrings --- airflow/providers/http/hooks/http.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index bb87c196abb12..a7f4dda94e171 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -74,6 +74,12 @@ def get_auth_types() -> frozenset[str]: class ConnectionWithExtra(Connection): + """ + Patched Connection class added for backward compatibility. + + Implements the get_extra_dejson method which was added in the Connection class in Airflow 2.10.0. + This patched class must be removed once the http provider depends on Airflow 2.10.0 or higher. + """ def __init__( self, conn_id: str | None = None, @@ -181,6 +187,8 @@ def get_ui_field_behaviour(cls) -> dict[str, Any]: @cached_property def connection(self) -> Connection: """ + Return a cached connection property. + This method calls the original get_connection method from the BaseHook and returns a patched version of the Connection class which also has the get_extra_dejson method that has been added in Apache Airflow since 2.10.0. Once the provider depends on Airflow version 2.10.0 or higher, this method and From ecc46195b7ba87c105e2e369d4eb8466c41b75da Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 19 Jul 2024 12:33:16 +0200 Subject: [PATCH 070/286] docs: Fixed docstring ConnectionWithExtra --- airflow/providers/http/hooks/http.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index a7f4dda94e171..05eab253ddb91 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -75,10 +75,10 @@ def get_auth_types() -> frozenset[str]: class ConnectionWithExtra(Connection): """ - Patched Connection class added for backward compatibility. + Patched Connection class added for backward compatibility. - Implements the get_extra_dejson method which was added in the Connection class in Airflow 2.10.0. - This patched class must be removed once the http provider depends on Airflow 2.10.0 or higher. + Implements the get_extra_dejson method which was added in the Connection class in Airflow 2.10.0. + This patched class must be removed once the http provider depends on Airflow 2.10.0 or higher. """ def __init__( self, From 43fad6f4d12d307dc2f85453d36e997a2dba7fba Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 19 Jul 2024 14:50:43 +0200 Subject: [PATCH 071/286] docs: Added newline after docstring constructor --- airflow/providers/http/hooks/http.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 05eab253ddb91..a59eda097dae1 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -80,6 +80,7 @@ class ConnectionWithExtra(Connection): Implements the get_extra_dejson method which was added in the Connection class in Airflow 2.10.0. This patched class must be removed once the http provider depends on Airflow 2.10.0 or higher. """ + def __init__( self, conn_id: str | None = None, From 91f6fad3f48f4902d01495dc46e8345904fde074 Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 1 Aug 2024 12:30:10 +0200 Subject: [PATCH 072/286] docs: Quoted the get_extra_dejson method in docstring to avoid spelling errors --- airflow/providers/http/hooks/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index a59eda097dae1..dc476b24ec4f0 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -77,7 +77,7 @@ class ConnectionWithExtra(Connection): """ Patched Connection class added for backward compatibility. - Implements the get_extra_dejson method which was added in the Connection class in Airflow 2.10.0. + Implements the `get_extra_dejson` method which was added in the Connection class in Airflow 2.10.0. This patched class must be removed once the http provider depends on Airflow 2.10.0 or higher. """ @@ -191,7 +191,7 @@ def connection(self) -> Connection: Return a cached connection property. This method calls the original get_connection method from the BaseHook and returns a patched version - of the Connection class which also has the get_extra_dejson method that has been added in Apache + of the Connection class which also has the `get_extra_dejson` method that has been added in Apache Airflow since 2.10.0. Once the provider depends on Airflow version 2.10.0 or higher, this method and the ConnectionWithExtra class can be removed. """ From c9c2ec4ecf4b13108f43ca947d1ea550aea6ba79 Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 10 Jan 2025 16:22:23 +0100 Subject: [PATCH 073/286] refactor: Refactored HttpHook without mixin to have less difference with current main version --- .../src/airflow/providers/http/hooks/http.py | 535 ++++++++---------- providers/tests/http/hooks/test_http.py | 26 +- 2 files changed, 241 insertions(+), 320 deletions(-) diff --git a/providers/src/airflow/providers/http/hooks/http.py b/providers/src/airflow/providers/http/hooks/http.py index dc476b24ec4f0..acf315941b465 100644 --- a/providers/src/airflow/providers/http/hooks/http.py +++ b/providers/src/airflow/providers/http/hooks/http.py @@ -21,29 +21,28 @@ import json import warnings from contextlib import suppress -from functools import cached_property +from functools import cache from json import JSONDecodeError -from logging import Logger from typing import TYPE_CHECKING, Any, Callable +from urllib.parse import urlparse import aiohttp import requests import tenacity from aiohttp import ClientResponseError -from asgiref.sync import sync_to_async -from requests.auth import HTTPBasicAuth -from requests.models import DEFAULT_REDIRECT_LIMIT -from requests_toolbelt.adapters.socket_options import TCPKeepAliveAdapter - -from airflow.compat.functools import cache from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.hooks.base import BaseHook -from airflow.models import Connection -from airflow.utils.log.secrets_masker import mask_secret from airflow.utils.module_loading import import_string +from asgiref.sync import sync_to_async +from requests.auth import HTTPBasicAuth, AuthBase +from requests.models import DEFAULT_REDIRECT_LIMIT +from requests_toolbelt.adapters.socket_options import TCPKeepAliveAdapter if TYPE_CHECKING: from aiohttp.client_reqrep import ClientResponse + from requests.adapters import HTTPAdapter + + from airflow.models import Connection DEFAULT_AUTH_TYPES = frozenset( @@ -57,86 +56,51 @@ ) -@cache -def get_auth_types() -> frozenset[str]: - """ - Get comma-separated extra auth_types from airflow config. - - Those auth_types can then be used in Connection configuration. - """ - from airflow.configuration import conf +def _url_from_endpoint(base_url: str | None, endpoint: str | None) -> str: + """Combine base url with endpoint.""" + if base_url and not base_url.endswith("/") and endpoint and not endpoint.startswith("/"): + return f"{base_url}/{endpoint}" + return (base_url or "") + (endpoint or "") - auth_types = DEFAULT_AUTH_TYPES.copy() - extra_auth_types = conf.get("http", "extra_auth_types", fallback=None) - if extra_auth_types: - auth_types |= frozenset({field.strip() for field in extra_auth_types.split(",")}) - return auth_types - -class ConnectionWithExtra(Connection): +class HttpHook(BaseHook): """ - Patched Connection class added for backward compatibility. + Interact with HTTP servers. - Implements the `get_extra_dejson` method which was added in the Connection class in Airflow 2.10.0. - This patched class must be removed once the http provider depends on Airflow 2.10.0 or higher. + :param method: the API method to be called + :param http_conn_id: :ref:`http connection` that has the base + API url i.e https://www.google.com/ and optional authentication credentials. Default + headers can also be specified in the Extra field in json format. + :param auth_type: The auth type for the service + :param adapter: An optional instance of `requests.adapters.HTTPAdapter` to mount for the session. + :param tcp_keep_alive: Enable TCP Keep Alive for the connection. + :param tcp_keep_alive_idle: The TCP Keep Alive Idle parameter (corresponds to ``socket.TCP_KEEPIDLE``). + :param tcp_keep_alive_count: The TCP Keep Alive count parameter (corresponds to ``socket.TCP_KEEPCNT``) + :param tcp_keep_alive_interval: The TCP Keep Alive interval parameter (corresponds to + ``socket.TCP_KEEPINTVL``) + :param auth_args: extra arguments used to initialize the auth_type if different than default HTTPBasicAuth """ - def __init__( - self, - conn_id: str | None = None, - conn_type: str | None = None, - description: str | None = None, - host: str | None = None, - login: str | None = None, - password: str | None = None, - schema: str | None = None, - port: int | None = None, - extra: str | dict | None = None, - uri: str | None = None, - ): - super().__init__(conn_id, conn_type, description, host, login, password, schema, port, extra, uri) - - def get_extra_dejson(self, nested: bool = False) -> dict: - """ - Deserialize extra property to JSON. + conn_name_attr = "http_conn_id" + default_conn_name = "http_default" + conn_type = "http" + hook_name = "HTTP" - :param nested: Determines whether nested structures are also deserialized into JSON (default False). + @classmethod + @cache + def get_auth_types(cls) -> frozenset[str]: """ - extra = {} - - if self.extra: - try: - if nested: - for key, value in json.loads(self.extra).items(): - extra[key] = value - if isinstance(value, str): - with suppress(JSONDecodeError): - extra[key] = json.loads(value) - else: - extra = json.loads(self.extra) - except JSONDecodeError: - self.log.exception("Failed parsing the json for conn_id %s", self.conn_id) - - # Mask sensitive keys from this list - mask_secret(extra) + Get comma-separated extra auth_types from airflow config. - return extra - - -class HttpHookMixin: - """ - Common superclass for the HttpHook and HttpAsyncHook. - - Implements methods to create a Connection. - """ + Those auth_types can then be used in Connection configuration. + """ + from airflow.configuration import conf - http_conn_id: str - conn_type: str - base_url: str - auth_type: Any - default_auth_type = HTTPBasicAuth - get_connection: Callable - log: Logger + auth_types = DEFAULT_AUTH_TYPES.copy() + extra_auth_types = conf.get("http", "extra_auth_types", fallback=None) + if extra_auth_types: + auth_types |= frozenset({field.strip() for field in extra_auth_types.split(",")}) + return auth_types @classmethod def get_connection_form_widgets(cls) -> dict[str, Any]: @@ -146,7 +110,7 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: from wtforms.fields import BooleanField, SelectField, StringField, TextAreaField default_auth_type: str = "" - auth_types_choices = frozenset({default_auth_type}) | get_auth_types() + auth_types_choices = frozenset({default_auth_type}) | cls.get_auth_types() return { "timeout": StringField(lazy_gettext("Timeout"), widget=BS3TextFieldWidget()), @@ -185,134 +149,74 @@ def get_ui_field_behaviour(cls) -> dict[str, Any]: "relabeling": {}, } - @cached_property - def connection(self) -> Connection: - """ - Return a cached connection property. + def __init__( + self, + method: str = "POST", + http_conn_id: str = default_conn_name, + auth_type: Any = None, + tcp_keep_alive: bool = True, + tcp_keep_alive_idle: int = 120, + tcp_keep_alive_count: int = 20, + tcp_keep_alive_interval: int = 30, + adapter: HTTPAdapter | None = None, + ) -> None: + super().__init__() + self.http_conn_id = http_conn_id + self.method = method.upper() + self.base_url: str = "" + self._retry_obj: Callable[..., Any] + self.auth_type: Any = auth_type - This method calls the original get_connection method from the BaseHook and returns a patched version - of the Connection class which also has the `get_extra_dejson` method that has been added in Apache - Airflow since 2.10.0. Once the provider depends on Airflow version 2.10.0 or higher, this method and - the ConnectionWithExtra class can be removed. - """ - conn = BaseHook.get_connection(conn_id=self.http_conn_id) - - return ConnectionWithExtra( - conn_id=self.http_conn_id, - conn_type=conn.conn_type, - description=conn.description, - host=conn.host, - login=conn.login, - password=conn.password, - schema=conn.schema, - port=conn.port, - extra=conn.extra, - ) + # If no adapter is provided, use TCPKeepAliveAdapter (default behavior) + self.adapter = adapter + if tcp_keep_alive and adapter is None: + self.keep_alive_adapter = TCPKeepAliveAdapter( + idle=tcp_keep_alive_idle, + count=tcp_keep_alive_count, + interval=tcp_keep_alive_interval, + ) + else: + self.keep_alive_adapter = None - def load_connection_settings(self, *, headers: dict[Any, Any] | None = None) -> tuple[dict, Any, dict]: + # headers may be passed through directly or in the "extra" field in the connection + # definition + def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: """ - Load and update the class with Connection Settings. + Create a Requests HTTP session. - Load the settings from the Connection and update the class. - Returns the headers and auth which are later passed into a request.Session - (for the HttpHook) or an aiohttp.Session (for the HttpAsyncHook). + :param headers: Additional headers to be passed through as a dictionary. + :return: A configured requests.Session object. """ - _headers = {} - _auth = None - _session_conf = {} - - if self.http_conn_id: - conn = self.connection - - if conn.host and "://" in conn.host: - self.base_url = conn.host - else: - # schema defaults to HTTP - schema = conn.schema if conn.schema else "http" - host = conn.host if conn.host else "" - self.base_url = f"{schema}://{host}" - - if conn.port: - self.base_url += f":{conn.port}" - - conn_extra: dict = self._parse_extra(conn_extra=conn.get_extra_dejson(nested=True)) - _session_conf = conn_extra["session_conf"] - auth_args: list[str | None] = [conn.login, conn.password] - auth_kwargs: dict[str, Any] = conn_extra["auth_kwargs"] - auth_type: Any = self.auth_type or self._load_conn_auth_type(module_name=conn_extra["auth_type"]) - - self.log.debug("auth_type: %s", auth_type) - - if auth_type: - if any(auth_args) or auth_kwargs: - _auth = auth_type(*auth_args, **auth_kwargs) - elif conn.login: - _auth = auth_type(conn.login, conn.password) - else: - _auth = auth_type() - - extra_headers = conn_extra["headers"] - if extra_headers: - try: - _headers.update(extra_headers) - except TypeError: - self.log.warning("Connection to %s has invalid extra field.", conn.host) + session = requests.Session() + connection = self.get_connection(self.http_conn_id) + self._set_base_url(connection) + session = self._configure_session_from_auth(session, connection) + if connection.extra: + session = self._configure_session_from_extra(session, connection) + session = self._configure_session_from_mount_adapters(session) if headers: - _headers.update(headers) - - self.log.debug("headers: %s", _headers) - self.log.debug("auth: %s", _auth) - self.log.debug("session_conf: %s", _session_conf) - - return _headers, _auth, _session_conf - - def _parse_extra(self, conn_extra: dict) -> dict: - """ - Parse the settings from 'extra' into dict. - - The "auth_kwargs" and "headers" data from TextAreaField are returned as - string via the 'extra' field. This method converts the data to dict. - """ - extra = conn_extra.copy() - session_conf = {} - timeout = extra.pop( - "timeout", None - ) # ignore this as timeout is only accepted in request method of Session - if timeout is not None: - session_conf["timeout"] = timeout - allow_redirects = extra.pop( - "allow_redirects", None - ) # ignore this as only max_redirects is accepted in Session - if allow_redirects is not None: - session_conf["allow_redirects"] = allow_redirects - session_conf["proxies"] = extra.pop("proxies", extra.pop("proxy", {})) - session_conf["stream"] = extra.pop("stream", False) - session_conf["verify"] = extra.pop("verify", extra.pop("verify_ssl", True)) - session_conf["trust_env"] = extra.pop("trust_env", True) - cert = extra.pop("cert", None) - if cert is not None: - session_conf["cert"] = cert - session_conf["max_redirects"] = extra.pop("max_redirects", DEFAULT_REDIRECT_LIMIT) - auth_type: str | None = extra.pop("auth_type", None) - auth_kwargs = extra.pop("auth_kwargs", {}) - headers = extra.pop("headers", {}) - - if extra: - warnings.warn( - "Passing headers parameters directly in 'Extra' field is deprecated, and " - "will be removed in a future version of the Http provider. Use the 'Headers' " - "field instead.", - AirflowProviderDeprecationWarning, - stacklevel=2, - ) - headers = {**extra, **headers} + session.headers.update(headers) + return session - return { - "auth_type": auth_type, - "auth_kwargs": auth_kwargs, - "session_conf": session_conf, - "headers": headers, - } + def _set_base_url(self, connection: Connection) -> None: + host = connection.host or "" + schema = connection.schema or "http" + # RFC 3986 (https://www.rfc-editor.org/rfc/rfc3986.html#page-16) + if "://" in host: + self.base_url = host + else: + self.base_url = f"{schema}://{host}" if host else f"{schema}://" + if connection.port: + self.base_url = f"{self.base_url}:{connection.port}" + parsed = urlparse(self.base_url) + if not parsed.scheme: + raise ValueError(f"Invalid base URL: Missing scheme in {self.base_url}") + + def _configure_session_from_auth( + self, session: requests.Session, connection: Connection + ) -> requests.Session: + session.auth = self._extract_auth(connection) + return session def _load_conn_auth_type(self, module_name: str | None) -> Any: """ @@ -322,7 +226,7 @@ def _load_conn_auth_type(self, module_name: str | None) -> Any: This method protects against the execution of random modules. """ if module_name: - if module_name in get_auth_types(): + if module_name in self.get_auth_types(): try: module = import_string(module_name) self._is_auth_type_setup = True @@ -338,101 +242,74 @@ def _load_conn_auth_type(self, module_name: str | None) -> Any: ) return None - def url_from_endpoint(self, endpoint: str | None) -> str: - """Combine base url with endpoint.""" - if self.base_url and not self.base_url.endswith("/") and endpoint and not endpoint.startswith("/"): - return self.base_url + "/" + endpoint - return (self.base_url or "") + (endpoint or "") + def _extract_auth(self, connection: Connection) -> AuthBase | None: + extra = connection.extra_dejson + auth_type: Any = self.auth_type or self._load_conn_auth_type(module_name=extra.get("auth_type")) + auth_kwargs = extra.get("auth_kwargs", {}) + self.log.debug("auth_type: %s", auth_type) + self.log.debug("auth_kwargs: %s", auth_kwargs) -class HttpHook(HttpHookMixin, BaseHook): - """ - Interact with HTTP servers. + if auth_type: + auth_args: list[str | None] = [connection.login, connection.password] - To configure the auth_type, in addition to the `auth_type` parameter, you can also: - * set the `auth_type` parameter in the Connection settings. - * define extra parameters passed to the `auth_type` class via the `auth_kwargs`, in the Connection - settings. The class will be instantiated with those parameters. - - See :doc:`/connections/http` for full documentation. - - :param method: the API method to be called - :param http_conn_id: :ref:`http connection` that has the base - API url i.e https://www.google.com/ and optional authentication credentials. Default - headers can also be specified in the Extra field in json format. - :param auth_type: The auth type for the service - :param tcp_keep_alive: Enable TCP Keep Alive for the connection. - :param tcp_keep_alive_idle: The TCP Keep Alive Idle parameter (corresponds to ``socket.TCP_KEEPIDLE``). - :param tcp_keep_alive_count: The TCP Keep Alive count parameter (corresponds to ``socket.TCP_KEEPCNT``) - :param tcp_keep_alive_interval: The TCP Keep Alive interval parameter (corresponds to - ``socket.TCP_KEEPINTVL``) - :param auth_args: extra arguments used to initialize the auth_type if different than default HTTPBasicAuth - """ - - conn_name_attr = "http_conn_id" - default_conn_name = "http_default" - conn_type = "http" - hook_name = "HTTP" - - @classmethod - def get_connection_form_widgets(cls) -> dict[str, Any]: - return super().get_connection_form_widgets() - - @classmethod - def get_ui_field_behaviour(cls) -> dict[str, Any]: - return super().get_ui_field_behaviour() - - def __init__( - self, - method: str = "POST", - http_conn_id: str = default_conn_name, - auth_type: Any = None, - tcp_keep_alive: bool = True, - tcp_keep_alive_idle: int = 120, - tcp_keep_alive_count: int = 20, - tcp_keep_alive_interval: int = 30, - ) -> None: - super().__init__() - self.http_conn_id = http_conn_id - self.method = method.upper() - self.base_url: str = "" - self._retry_obj: Callable[..., Any] - self.auth_type: Any = auth_type - self.tcp_keep_alive = tcp_keep_alive - self.keep_alive_idle = tcp_keep_alive_idle - self.keep_alive_count = tcp_keep_alive_count - self.keep_alive_interval = tcp_keep_alive_interval + self.log.debug("auth_args: %s", auth_args) - # headers may be passed through directly or in the "extra" field in the connection - # definition - def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: - """ - Create a Requests HTTP session. - - :param headers: additional headers to be passed through as a dictionary. - Note: Headers may also be passed in the "Headers" field in the Connection definition - """ - headers, auth, session_conf = self.load_connection_settings(headers=headers) + if any(auth_args): + if auth_kwargs: + _auth = auth_type(*auth_args, **auth_kwargs) + else: + return auth_type(*auth_args) + else: + return auth_type() + return None - session_conf.pop( - "timeout", None - ) # ignore this as timeout is only accepted in request method of Session - session_conf.pop("allow_redirects", None) # ignore this as only max_redirects is accepted in Session + def _configure_session_from_extra( + self, session: requests.Session, connection: Connection + ) -> requests.Session: + extra = connection.extra_dejson + extra.pop("timeout", None) + extra.pop("allow_redirects", None) + extra.pop("auth_type", None) + extra.pop("auth_kwargs", None) + headers = extra.pop("headers", {}) + if isinstance(headers, str): + with suppress(JSONDecodeError): + headers = json.loads(headers) + session.proxies = extra.pop("proxies", extra.pop("proxy", {})) + session.stream = extra.pop("stream", False) + session.verify = extra.pop("verify", extra.pop("verify_ssl", True)) + session.cert = extra.pop("cert", None) + session.max_redirects = extra.pop("max_redirects", DEFAULT_REDIRECT_LIMIT) + session.trust_env = extra.pop("trust_env", True) - session = requests.Session() - session.auth = auth - session.proxies = session_conf["proxies"] - session.stream = session_conf["stream"] - session.verify = session_conf["verify"] - session.cert = session_conf.get("cert") - session.max_redirects = session_conf["max_redirects"] - session.trust_env = session_conf["trust_env"] + if extra: + warnings.warn( + "Passing headers parameters directly in 'Extra' field is deprecated, and " + "will be removed in a future version of the Http provider. Use the 'Headers' " + "field instead.", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + headers = {**extra, **headers} try: session.headers.update(headers) except TypeError: - self.log.warning("Connection to %s has invalid extra field.", self.http_conn_id) + self.log.warning("Connection to %s has invalid extra field.", connection.host) + return session + def _configure_session_from_mount_adapters(self, session: requests.Session) -> requests.Session: + scheme = urlparse(self.base_url).scheme + if not scheme: + raise ValueError( + f"Cannot mount adapters: {self.base_url} does not include a valid scheme (http or https)." + ) + if self.adapter: + session.mount(f"{scheme}://", self.adapter) + elif self.keep_alive_adapter: + session.mount("http://", self.keep_alive_adapter) + session.mount("https://", self.keep_alive_adapter) return session def run( @@ -461,11 +338,6 @@ def run( url = self.url_from_endpoint(endpoint) - if self.tcp_keep_alive: - keep_alive_adapter = TCPKeepAliveAdapter( - idle=self.keep_alive_idle, count=self.keep_alive_count, interval=self.keep_alive_interval - ) - session.mount(url, keep_alive_adapter) if self.method == "GET": # GET uses params req = requests.Request(self.method, url, params=data, headers=headers, **request_kwargs) @@ -565,6 +437,10 @@ def run_with_advanced_retry(self, _retry_args: dict[Any, Any], *args: Any, **kwa # TODO: remove ignore type when https://github.com/jd/tenacity/issues/428 is resolved return self._retry_obj(self.run, *args, **kwargs) # type: ignore + def url_from_endpoint(self, endpoint: str | None) -> str: + """Combine base url with endpoint.""" + return _url_from_endpoint(base_url=self.base_url, endpoint=endpoint) + def test_connection(self): """Test HTTP Connection.""" try: @@ -574,7 +450,7 @@ def test_connection(self): return False, str(e) -class HttpAsyncHook(HttpHookMixin, BaseHook): +class HttpAsyncHook(BaseHook): """ Interact with HTTP servers asynchronously. @@ -594,7 +470,7 @@ def __init__( self, method: str = "POST", http_conn_id: str = default_conn_name, - auth_type: Any = None, + auth_type: Any = aiohttp.BasicAuth, retry_limit: int = 3, retry_delay: float = 1.0, ) -> None: @@ -628,11 +504,38 @@ async def run( ``aiohttp.ClientSession().get(json=obj)``. """ extra_options = extra_options or {} - _headers, auth, session_conf = await sync_to_async(self.load_connection_settings)(headers=headers) - session_conf = self._process_session_conf(session_conf) - session_conf.update(extra_options) - url = self.url_from_endpoint(endpoint) + # headers may be passed through directly or in the "extra" field in the connection + # definition + _headers = {} + auth = None + + if self.http_conn_id: + conn = await sync_to_async(self.get_connection)(self.http_conn_id) + + if conn.host and "://" in conn.host: + self.base_url = conn.host + else: + # schema defaults to HTTP + schema = conn.schema if conn.schema else "http" + host = conn.host if conn.host else "" + self.base_url = schema + "://" + host + + if conn.port: + self.base_url += f":{conn.port}" + if conn.login: + auth = self.auth_type(conn.login, conn.password) + if conn.extra: + extra = self._process_extra_options_from_connection(conn=conn, extra_options=extra_options) + + try: + _headers.update(extra) + except TypeError: + self.log.warning("Connection to %s has invalid extra field.", conn.host) + if headers: + _headers.update(headers) + + url = _url_from_endpoint(self.base_url, endpoint) async with aiohttp.ClientSession() as session: if self.method == "GET": @@ -684,17 +587,30 @@ async def run( raise NotImplementedError # should not reach this, but makes mypy happy @classmethod - def _process_session_conf(cls, session_conf: dict) -> dict: - session_conf.pop("stream", None) - session_conf.pop("cert", None) - proxies = session_conf.pop("proxies") - if proxies is not None: - session_conf["proxy"] = proxies - verify = session_conf.pop("verify") - if verify is not None: - session_conf["verify_ssl"] = verify - session_conf.pop("trust_env") - return session_conf + def _process_extra_options_from_connection(cls, conn: Connection, extra_options: dict) -> dict: + extra = conn.extra_dejson + extra.pop("stream", None) + extra.pop("cert", None) + proxies = extra.pop("proxies", extra.pop("proxy", None)) + timeout = extra.pop("timeout", None) + verify_ssl = extra.pop("verify", extra.pop("verify_ssl", None)) + allow_redirects = extra.pop("allow_redirects", None) + max_redirects = extra.pop("max_redirects", None) + trust_env = extra.pop("trust_env", None) + + if proxies is not None and "proxy" not in extra_options: + extra_options["proxy"] = proxies + if timeout is not None and "timeout" not in extra_options: + extra_options["timeout"] = timeout + if verify_ssl is not None and "verify_ssl" not in extra_options: + extra_options["verify_ssl"] = verify_ssl + if allow_redirects is not None and "allow_redirects" not in extra_options: + extra_options["allow_redirects"] = allow_redirects + if max_redirects is not None and "max_redirects" not in extra_options: + extra_options["max_redirects"] = max_redirects + if trust_env is not None and "trust_env" not in extra_options: + extra_options["trust_env"] = trust_env + return extra def _retryable_error_async(self, exception: ClientResponseError) -> bool: """ @@ -713,5 +629,4 @@ def _retryable_error_async(self, exception: ClientResponseError) -> bool: if exception.status == 413: # don't retry for payload Too Large return False - return exception.status >= 500 diff --git a/providers/tests/http/hooks/test_http.py b/providers/tests/http/hooks/test_http.py index 01918bfa4ad31..6caaef379de0e 100644 --- a/providers/tests/http/hooks/test_http.py +++ b/providers/tests/http/hooks/test_http.py @@ -37,7 +37,7 @@ from airflow.exceptions import AirflowException from airflow.models import Connection from airflow.providers.http import airflow_dependency_version -from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook, get_auth_types +from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook DEFAULT_HEADERS_AS_STRING = '{\r\n "Content-Type": "application/json",\r\n "X-Requested-By": "Airflow"\r\n}' DEFAULT_HEADERS = json.loads(DEFAULT_HEADERS_AS_STRING) @@ -397,7 +397,7 @@ def test_connection_without_host(self, mock_get_connection): assert hook.base_url == "http://" @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") - @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + @mock.patch("providers.tests.http.hooks.test_http.CustomAuthBase.__init__") def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_connection): auth.return_value = None conn = Connection( @@ -417,7 +417,7 @@ def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_conne assert "x-header" in session.headers def test_available_connection_auth_types(self): - auth_types = get_auth_types() + auth_types = HttpHook.get_auth_types() assert auth_types == frozenset( { "requests.auth.HTTPBasicAuth", @@ -441,18 +441,20 @@ def test_connection_with_invalid_auth_type_get_skipped(self, mock_get_connection HttpHook().get_conn({}) assert f"Skipping import of auth_type '{auth_type}'." in caplog.text + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_auth_types") @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") - @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") - def test_connection_with_extra_header_and_auth_type(self, auth, mock_get_connection): + @mock.patch("providers.tests.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_header_and_auth_type(self, auth, mock_get_connection, mock_get_auth_types): auth.return_value = None conn = Connection( conn_id="http_default", conn_type="http", login="username", password="pass", - extra='{"headers": {"x-header": 0}, "auth_type": "tests.providers.http.hooks.test_http.CustomAuthBase"}', + extra='{"headers": {"x-header": 0}, "auth_type": "providers.tests.http.hooks.test_http.CustomAuthBase"}', ) mock_get_connection.return_value = conn + mock_get_auth_types.return_value = frozenset(["providers.tests.http.hooks.test_http.CustomAuthBase"]) session = HttpHook().get_conn({}) auth.assert_called_once_with("username", "pass") @@ -460,22 +462,26 @@ def test_connection_with_extra_header_and_auth_type(self, auth, mock_get_connect assert "auth_type" not in session.headers assert "x-header" in session.headers + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_auth_types") @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") - @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") - def test_connection_with_extra_auth_type_and_no_credentials(self, auth, mock_get_connection): + @mock.patch("providers.tests.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_auth_type_and_no_credentials(self, auth, mock_get_connection, mock_get_auth_types): auth.return_value = None conn = Connection( conn_id="http_default", conn_type="http", - extra='{"auth_type": "tests.providers.http.hooks.test_http.CustomAuthBase"}', + login="username", + password="secret", + extra='{"auth_type": "providers.tests.http.hooks.test_http.CustomAuthBase"}', ) mock_get_connection.return_value = conn + mock_get_auth_types.return_value = frozenset(["providers.tests.http.hooks.test_http.CustomAuthBase"]) HttpHook().get_conn({}) auth.assert_called_once() @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") - @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + @mock.patch("providers.tests.http.hooks.test_http.CustomAuthBase.__init__") def test_connection_with_string_headers_and_auth_kwargs(self, auth, mock_get_connection): """When passed via the UI, the 'headers' and 'auth_kwargs' fields' data is saved as string. From 8e1d9ebfbf1f2c7afc6b41fe1562d0a4f571e96d Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 10 Jan 2025 18:50:50 +0100 Subject: [PATCH 074/286] refactor: Reformatted imports HttpHook --- providers/src/airflow/providers/http/hooks/http.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/providers/src/airflow/providers/http/hooks/http.py b/providers/src/airflow/providers/http/hooks/http.py index 5fdf8383baa51..0edbcbbfdc865 100644 --- a/providers/src/airflow/providers/http/hooks/http.py +++ b/providers/src/airflow/providers/http/hooks/http.py @@ -30,14 +30,15 @@ import requests import tenacity from aiohttp import ClientResponseError -from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning -from airflow.hooks.base import BaseHook -from airflow.utils.module_loading import import_string from asgiref.sync import sync_to_async -from requests.auth import HTTPBasicAuth, AuthBase +from requests.auth import AuthBase from requests.models import DEFAULT_REDIRECT_LIMIT from requests_toolbelt.adapters.socket_options import TCPKeepAliveAdapter +from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning +from airflow.hooks.base import BaseHook +from airflow.utils.module_loading import import_string + if TYPE_CHECKING: from aiohttp.client_reqrep import ClientResponse from requests.adapters import HTTPAdapter @@ -232,7 +233,7 @@ def _load_conn_auth_type(self, module_name: str | None) -> Any: self._is_auth_type_setup = True self.log.info("Loaded auth_type: %s", module_name) return module - except ImportError as error: + except Exception as error: self.log.error("Cannot import auth_type '%s' due to: %s", module_name, error) raise AirflowException(error) self.log.warning( From 8d02ea982b41a11ecbc806108fdb4c0f84c3e889 Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 10 Jan 2025 19:02:57 +0100 Subject: [PATCH 075/286] refactor: Re-added missing except in HttpHook --- providers/src/airflow/providers/http/hooks/http.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/providers/src/airflow/providers/http/hooks/http.py b/providers/src/airflow/providers/http/hooks/http.py index 0edbcbbfdc865..7c451bd7fbae8 100644 --- a/providers/src/airflow/providers/http/hooks/http.py +++ b/providers/src/airflow/providers/http/hooks/http.py @@ -296,6 +296,8 @@ def _configure_session_from_extra( try: session.headers.update(headers) + except TypeError: + self.log.warning("Connection to %s has invalid headers field.", connection.host) return session def _set_base_url(self, connection: Connection) -> None: From 03a8beccf73c3b01e76d1960d01df0e0bce57c29 Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 10 Jan 2025 19:04:19 +0100 Subject: [PATCH 076/286] refactor: Fixed indentation TestHttpHook --- providers/tests/http/hooks/test_http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/providers/tests/http/hooks/test_http.py b/providers/tests/http/hooks/test_http.py index a64eef1dbb60e..c02e54c0fdd42 100644 --- a/providers/tests/http/hooks/test_http.py +++ b/providers/tests/http/hooks/test_http.py @@ -705,8 +705,8 @@ def test_custom_adapter(self): assert isinstance( session.adapters["https://"], type(custom_adapter) ), "Custom HTTPS adapter not correctly mounted" - - def test_airflow_dependency_version(self): + + def test_airflow_dependency_version(self): if airflow_dependency_version() >= packaging.version.parse("2.10.0"): raise RuntimeError( "The class ConnectionWithExtra can be removed from the HttpHook since the get_extra_dejson" From c20f7bdda42e60e0614c75d85e738efa3d14552c Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 10 Jan 2025 19:11:49 +0100 Subject: [PATCH 077/286] refactor: Changed checks related to Airflow dependency for http provider --- providers/src/airflow/providers/http/hooks/http.py | 4 ++++ providers/tests/http/hooks/test_http.py | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/providers/src/airflow/providers/http/hooks/http.py b/providers/src/airflow/providers/http/hooks/http.py index 7c451bd7fbae8..de80b4298e408 100644 --- a/providers/src/airflow/providers/http/hooks/http.py +++ b/providers/src/airflow/providers/http/hooks/http.py @@ -268,15 +268,19 @@ def _extract_auth(self, connection: Connection) -> AuthBase | None: def _configure_session_from_extra( self, session: requests.Session, connection: Connection ) -> requests.Session: + # TODO: once http provider depends on Airflow 2.10.0, use get_extra_dejson(True) instead extra = connection.extra_dejson extra.pop("timeout", None) extra.pop("allow_redirects", None) extra.pop("auth_type", None) extra.pop("auth_kwargs", None) headers = extra.pop("headers", {}) + + # TODO: once http provider depends on Airflow 2.10.0, we can remove this checked section below if isinstance(headers, str): with suppress(JSONDecodeError): headers = json.loads(headers) + session.proxies = extra.pop("proxies", extra.pop("proxy", {})) session.stream = extra.pop("stream", False) session.verify = extra.pop("verify", extra.pop("verify_ssl", True)) diff --git a/providers/tests/http/hooks/test_http.py b/providers/tests/http/hooks/test_http.py index c02e54c0fdd42..4eb9972892088 100644 --- a/providers/tests/http/hooks/test_http.py +++ b/providers/tests/http/hooks/test_http.py @@ -709,8 +709,9 @@ def test_custom_adapter(self): def test_airflow_dependency_version(self): if airflow_dependency_version() >= packaging.version.parse("2.10.0"): raise RuntimeError( - "The class ConnectionWithExtra can be removed from the HttpHook since the get_extra_dejson" - "method is now available on the Connection class since Apache Airflow 2.10.0+" + f"The method {HttpHook._configure_session_from_extra.__name__} from the " + f"{HttpHook.__name__} should be refactored since the {Connection.get_extra_dejson.__name__} " + f"method is now available on the {Connection.__name__} class since Apache Airflow 2.10.0" ) From dfcd97bc89f7dff4d644aac3cc01c18ef411d389 Mon Sep 17 00:00:00 2001 From: David Blain Date: Sat, 11 Jan 2025 13:19:21 +0100 Subject: [PATCH 078/286] refactor: Reformatted TestHttpHook --- providers/tests/http/hooks/test_http.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/providers/tests/http/hooks/test_http.py b/providers/tests/http/hooks/test_http.py index 4eb9972892088..f19dbfe0dc083 100644 --- a/providers/tests/http/hooks/test_http.py +++ b/providers/tests/http/hooks/test_http.py @@ -465,7 +465,9 @@ def test_connection_with_extra_header_and_auth_type(self, auth, mock_get_connect @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_auth_types") @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") @mock.patch("providers.tests.http.hooks.test_http.CustomAuthBase.__init__") - def test_connection_with_extra_auth_type_and_no_credentials(self, auth, mock_get_connection, mock_get_auth_types): + def test_connection_with_extra_auth_type_and_no_credentials( + self, auth, mock_get_connection, mock_get_auth_types + ): auth.return_value = None conn = Connection( conn_id="http_default", From 3b0ff43a7e1cb09895658cc5569b19ea3937355e Mon Sep 17 00:00:00 2001 From: David Blain Date: Sat, 11 Jan 2025 13:22:02 +0100 Subject: [PATCH 079/286] refactor: Refactored test_airflow_dependency_version in TestHttpHook --- .../src/airflow/providers/http/__init__.py | 14 -------------- providers/tests/http/hooks/test_http.py | 17 ++++++++++++++++- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/providers/src/airflow/providers/http/__init__.py b/providers/src/airflow/providers/http/__init__.py index 8d48e3592956a..037ab58fbdc20 100644 --- a/providers/src/airflow/providers/http/__init__.py +++ b/providers/src/airflow/providers/http/__init__.py @@ -37,17 +37,3 @@ raise RuntimeError( f"The package `apache-airflow-providers-http:{__version__}` needs Apache Airflow 2.9.0+" ) - - -def airflow_dependency_version(): - import re - from os.path import dirname, join - - import yaml - - with open(join(dirname(__file__), "provider.yaml"), encoding="utf-8") as file: - for dependency in yaml.safe_load(file)["dependencies"]: - if dependency.startswith("apache-airflow"): - match = re.search(r">=([\d\.]+)", dependency) - if match: - return packaging.version.parse(match.group(1)) diff --git a/providers/tests/http/hooks/test_http.py b/providers/tests/http/hooks/test_http.py index f19dbfe0dc083..576c39090ed03 100644 --- a/providers/tests/http/hooks/test_http.py +++ b/providers/tests/http/hooks/test_http.py @@ -36,9 +36,10 @@ from airflow.exceptions import AirflowException from airflow.models import Connection -from airflow.providers.http import airflow_dependency_version from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook +from airflow.providers import http + DEFAULT_HEADERS_AS_STRING = '{\r\n "Content-Type": "application/json",\r\n "X-Requested-By": "Airflow"\r\n}' DEFAULT_HEADERS = json.loads(DEFAULT_HEADERS_AS_STRING) @@ -73,6 +74,20 @@ def get_airflow_connection_with_login_and_password(conn_id: str = "http_default" return Connection(conn_id=conn_id, conn_type="http", host="test.com", login="username", password="pass") +def airflow_dependency_version(): + import re + from os.path import dirname, join + + import yaml + + with open(join(dirname(http.__file__), "provider.yaml"), encoding="utf-8") as file: + for dependency in yaml.safe_load(file)["dependencies"]: + if dependency.startswith("apache-airflow"): + match = re.search(r">=([\d\.]+)", dependency) + if match: + return packaging.version.parse(match.group(1)) + + class CustomAuthBase(HTTPBasicAuth): def __init__(self, username: str, password: str, endpoint: str): super().__init__(username, password) From ef9f00aaa244ba0be2b27dd5fcd923ec3894184e Mon Sep 17 00:00:00 2001 From: David Blain Date: Sat, 11 Jan 2025 13:30:00 +0100 Subject: [PATCH 080/286] refactor: Removed default_auth_type from LivyHook --- providers/src/airflow/providers/apache/livy/hooks/livy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/providers/src/airflow/providers/apache/livy/hooks/livy.py b/providers/src/airflow/providers/apache/livy/hooks/livy.py index 8f10a04903c75..611634e85649c 100644 --- a/providers/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/src/airflow/providers/apache/livy/hooks/livy.py @@ -497,7 +497,6 @@ def __init__( extra_headers: dict[str, Any] | None = None, ) -> None: super().__init__(http_conn_id=livy_conn_id) - self.auth_type = self.default_auth_type self.extra_headers = extra_headers or {} self.extra_options = extra_options or {} From 1a60887f0f263d0bb9dcb4510c37340d3f1f9ec9 Mon Sep 17 00:00:00 2001 From: David Blain Date: Sat, 11 Jan 2025 13:33:04 +0100 Subject: [PATCH 081/286] refactor: Reformatted and removed duplicate _set_base_url method from HttpHook --- .../src/airflow/providers/http/hooks/http.py | 72 ++++++++----------- 1 file changed, 29 insertions(+), 43 deletions(-) diff --git a/providers/src/airflow/providers/http/hooks/http.py b/providers/src/airflow/providers/http/hooks/http.py index de80b4298e408..432c7dcf4c7ab 100644 --- a/providers/src/airflow/providers/http/hooks/http.py +++ b/providers/src/airflow/providers/http/hooks/http.py @@ -87,6 +87,35 @@ class HttpHook(BaseHook): conn_type = "http" hook_name = "HTTP" + def __init__( + self, + method: str = "POST", + http_conn_id: str = default_conn_name, + auth_type: Any = None, + tcp_keep_alive: bool = True, + tcp_keep_alive_idle: int = 120, + tcp_keep_alive_count: int = 20, + tcp_keep_alive_interval: int = 30, + adapter: HTTPAdapter | None = None, + ) -> None: + super().__init__() + self.http_conn_id = http_conn_id + self.method = method.upper() + self.base_url: str = "" + self._retry_obj: Callable[..., Any] + self.auth_type: Any = auth_type + + # If no adapter is provided, use TCPKeepAliveAdapter (default behavior) + self.adapter = adapter + if tcp_keep_alive and adapter is None: + self.keep_alive_adapter = TCPKeepAliveAdapter( + idle=tcp_keep_alive_idle, + count=tcp_keep_alive_count, + interval=tcp_keep_alive_interval, + ) + else: + self.keep_alive_adapter = None + @classmethod @cache def get_auth_types(cls) -> frozenset[str]: @@ -150,35 +179,6 @@ def get_ui_field_behaviour(cls) -> dict[str, Any]: "relabeling": {}, } - def __init__( - self, - method: str = "POST", - http_conn_id: str = default_conn_name, - auth_type: Any = None, - tcp_keep_alive: bool = True, - tcp_keep_alive_idle: int = 120, - tcp_keep_alive_count: int = 20, - tcp_keep_alive_interval: int = 30, - adapter: HTTPAdapter | None = None, - ) -> None: - super().__init__() - self.http_conn_id = http_conn_id - self.method = method.upper() - self.base_url: str = "" - self._retry_obj: Callable[..., Any] - self.auth_type: Any = auth_type - - # If no adapter is provided, use TCPKeepAliveAdapter (default behavior) - self.adapter = adapter - if tcp_keep_alive and adapter is None: - self.keep_alive_adapter = TCPKeepAliveAdapter( - idle=tcp_keep_alive_idle, - count=tcp_keep_alive_count, - interval=tcp_keep_alive_interval, - ) - else: - self.keep_alive_adapter = None - # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: @@ -304,20 +304,6 @@ def _configure_session_from_extra( self.log.warning("Connection to %s has invalid headers field.", connection.host) return session - def _set_base_url(self, connection: Connection) -> None: - host = connection.host or "" - schema = connection.schema or "http" - # RFC 3986 (https://www.rfc-editor.org/rfc/rfc3986.html#page-16) - if "://" in host: - self.base_url = host - else: - self.base_url = f"{schema}://{host}" if host else f"{schema}://" - if connection.port: - self.base_url = f"{self.base_url}:{connection.port}" - parsed = urlparse(self.base_url) - if not parsed.scheme: - raise ValueError(f"Invalid base URL: Missing scheme in {self.base_url}") - def _configure_session_from_mount_adapters(self, session: requests.Session) -> requests.Session: scheme = urlparse(self.base_url).scheme if not scheme: From f57fe39526f27027a5216f0d16d33083671207cf Mon Sep 17 00:00:00 2001 From: David Blain Date: Sat, 11 Jan 2025 14:01:35 +0100 Subject: [PATCH 082/286] refactor: Reorganized imports TestHttpHook --- providers/tests/http/hooks/test_http.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/providers/tests/http/hooks/test_http.py b/providers/tests/http/hooks/test_http.py index 576c39090ed03..5cdfe0c858581 100644 --- a/providers/tests/http/hooks/test_http.py +++ b/providers/tests/http/hooks/test_http.py @@ -36,9 +36,8 @@ from airflow.exceptions import AirflowException from airflow.models import Connection -from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook - from airflow.providers import http +from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook DEFAULT_HEADERS_AS_STRING = '{\r\n "Content-Type": "application/json",\r\n "X-Requested-By": "Airflow"\r\n}' DEFAULT_HEADERS = json.loads(DEFAULT_HEADERS_AS_STRING) From 6f102c067d4160a7a30829812e5bbf42e050c9f8 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 14 Jan 2025 11:41:02 +0100 Subject: [PATCH 083/286] refactor: Moved import AuthBase to type checking block --- providers/src/airflow/providers/http/hooks/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/src/airflow/providers/http/hooks/http.py b/providers/src/airflow/providers/http/hooks/http.py index 432c7dcf4c7ab..be20ea8aaba39 100644 --- a/providers/src/airflow/providers/http/hooks/http.py +++ b/providers/src/airflow/providers/http/hooks/http.py @@ -31,7 +31,6 @@ import tenacity from aiohttp import ClientResponseError from asgiref.sync import sync_to_async -from requests.auth import AuthBase from requests.models import DEFAULT_REDIRECT_LIMIT from requests_toolbelt.adapters.socket_options import TCPKeepAliveAdapter @@ -42,6 +41,7 @@ if TYPE_CHECKING: from aiohttp.client_reqrep import ClientResponse from requests.adapters import HTTPAdapter + from requests.auth import AuthBase from airflow.models import Connection From f0a00ffbe3d309be3e6923f7f069c69fef9cd686 Mon Sep 17 00:00:00 2001 From: David Blain Date: Sun, 26 Jan 2025 13:47:16 +0100 Subject: [PATCH 084/286] refactor: Reorganized imports TestHttpHook --- providers/tests/http/hooks/test_http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/tests/http/hooks/test_http.py b/providers/tests/http/hooks/test_http.py index 7096da46ca652..20a3f59434d93 100644 --- a/providers/tests/http/hooks/test_http.py +++ b/providers/tests/http/hooks/test_http.py @@ -25,8 +25,8 @@ from http import HTTPStatus from unittest import mock -import packaging.version import aiohttp +import packaging.version import pytest import requests import tenacity From 8f4300180a6e8723203dfb9174c24aee0909ea58 Mon Sep 17 00:00:00 2001 From: David Blain Date: Sun, 26 Jan 2025 13:48:48 +0100 Subject: [PATCH 085/286] refactor: Reorganized imports HttpHook --- providers/src/airflow/providers/http/hooks/http.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/providers/src/airflow/providers/http/hooks/http.py b/providers/src/airflow/providers/http/hooks/http.py index cc6e4cf10b1e0..31e5fa7f3c22d 100644 --- a/providers/src/airflow/providers/http/hooks/http.py +++ b/providers/src/airflow/providers/http/hooks/http.py @@ -17,7 +17,6 @@ # under the License. from __future__ import annotations -import asyncio import json import warnings from contextlib import suppress @@ -36,8 +35,8 @@ from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.hooks.base import BaseHook -from airflow.utils.module_loading import import_string from airflow.providers.http.exceptions import HttpErrorException, HttpMethodException +from airflow.utils.module_loading import import_string if TYPE_CHECKING: from aiohttp.client_reqrep import ClientResponse From affa175c74e2c3a3f21a7a3b9ba5a66cfbd771fc Mon Sep 17 00:00:00 2001 From: David Blain Date: Sun, 26 Jan 2025 13:53:45 +0100 Subject: [PATCH 086/286] refactor: Only test airflow dependency of http provider when running on Airflow 2.10 or higher --- providers/tests/http/hooks/test_http.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/providers/tests/http/hooks/test_http.py b/providers/tests/http/hooks/test_http.py index 20a3f59434d93..2f82d498fc007 100644 --- a/providers/tests/http/hooks/test_http.py +++ b/providers/tests/http/hooks/test_http.py @@ -40,6 +40,8 @@ from airflow.providers import http from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook +from tests_common.test_utils.version_compat import AIRFLOW_V_2_10_PLUS + DEFAULT_HEADERS_AS_STRING = '{\r\n "Content-Type": "application/json",\r\n "X-Requested-By": "Airflow"\r\n}' DEFAULT_HEADERS = json.loads(DEFAULT_HEADERS_AS_STRING) @@ -723,6 +725,7 @@ def test_custom_adapter(self): session.adapters["https://"], type(custom_adapter) ), "Custom HTTPS adapter not correctly mounted" + @pytest.mark.skipif(not AIRFLOW_V_2_10_PLUS, reason="Lambda parameters works in Airflow >= 2.10.0") def test_airflow_dependency_version(self): if airflow_dependency_version() >= packaging.version.parse("2.10.0"): raise RuntimeError( From eb00ddf52580ad7a97d134ba1da496746bc20af7 Mon Sep 17 00:00:00 2001 From: David Blain Date: Sun, 26 Jan 2025 14:13:45 +0100 Subject: [PATCH 087/286] refactor: Updated unit tests for HttpHook and ignore deprecation warnings --- providers/tests/http/hooks/test_http.py | 520 +++++++++--------------- 1 file changed, 193 insertions(+), 327 deletions(-) diff --git a/providers/tests/http/hooks/test_http.py b/providers/tests/http/hooks/test_http.py index 2f82d498fc007..0f9d0bb240518 100644 --- a/providers/tests/http/hooks/test_http.py +++ b/providers/tests/http/hooks/test_http.py @@ -22,11 +22,11 @@ import json import logging import os +import warnings from http import HTTPStatus from unittest import mock import aiohttp -import packaging.version import pytest import requests import tenacity @@ -35,16 +35,10 @@ from requests.auth import AuthBase, HTTPBasicAuth from requests.models import DEFAULT_REDIRECT_LIMIT -from airflow.exceptions import AirflowException +from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models import Connection -from airflow.providers import http from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook -from tests_common.test_utils.version_compat import AIRFLOW_V_2_10_PLUS - -DEFAULT_HEADERS_AS_STRING = '{\r\n "Content-Type": "application/json",\r\n "X-Requested-By": "Airflow"\r\n}' -DEFAULT_HEADERS = json.loads(DEFAULT_HEADERS_AS_STRING) - @pytest.fixture def aioresponse(): @@ -56,9 +50,7 @@ def aioresponse(): def get_airflow_connection(conn_id: str = "http_default"): - return Connection( - conn_id=conn_id, conn_type="http", host="test:8080/", extra={"headers": DEFAULT_HEADERS} - ) + return Connection(conn_id=conn_id, conn_type="http", host="test:8080/", extra='{"bearer": "test"}') def get_airflow_connection_with_extra(extra: dict): @@ -76,28 +68,6 @@ def get_airflow_connection_with_login_and_password(conn_id: str = "http_default" return Connection(conn_id=conn_id, conn_type="http", host="test.com", login="username", password="pass") -def airflow_dependency_version(): - import re - from os.path import dirname, join - - import yaml - - with open(join(dirname(http.__file__), "provider.yaml"), encoding="utf-8") as file: - for dependency in yaml.safe_load(file)["dependencies"]: - if dependency.startswith("apache-airflow"): - match = re.search(r">=([\d\.]+)", dependency) - if match: - return packaging.version.parse(match.group(1)) - - -class CustomAuthBase(HTTPBasicAuth): - def __init__(self, username: str, password: str, endpoint: str): - super().__init__(username, password) - - -@mock.patch.dict( - "os.environ", AIRFLOW__HTTP__EXTRA_AUTH_TYPES="tests.providers.http.hooks.test_http.CustomAuthBase" -) class TestHttpHook: """Test get, post and raise_for_status""" @@ -112,12 +82,14 @@ def setup_method(self): self.post_hook = HttpHook(method="POST") def test_raise_for_status_with_200(self, requests_mock): - requests_mock.get( - "http://test:8080/v1/test", status_code=200, text='{"status":{"status": 200}}', reason="OK" - ) - with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): - resp = self.get_hook.run("v1/test") - assert resp.text == '{"status":{"status": 200}}' + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + requests_mock.get( + "http://test:8080/v1/test", status_code=200, text='{"status":{"status": 200}}', reason="OK" + ) + with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): + resp = self.get_hook.run("v1/test") + assert resp.text == '{"status":{"status": 200}}' @mock.patch("requests.Request") @mock.patch("requests.Session") @@ -139,127 +111,114 @@ def test_get_request_with_port(self, mock_session, mock_request): mock_request.reset_mock() def test_get_request_do_not_raise_for_status_if_check_response_is_false(self, requests_mock): - requests_mock.get( - "http://test:8080/v1/test", - status_code=404, - text='{"status":{"status": 404}}', - reason="Bad request", - ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + requests_mock.get( + "http://test:8080/v1/test", + status_code=404, + text='{"status":{"status": 404}}', + reason="Bad request", + ) - with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): - resp = self.get_hook.run("v1/test", extra_options={"check_response": False}) - assert resp.text == '{"status":{"status": 404}}' + with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): + resp = self.get_hook.run("v1/test", extra_options={"check_response": False}) + assert resp.text == '{"status":{"status": 404}}' def test_hook_contains_header_from_extra_field(self): - airflow_connection = get_airflow_connection_with_extra(extra={"headers": DEFAULT_HEADERS_AS_STRING}) - with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=airflow_connection): - expected_conn = get_airflow_connection() - conn = self.get_hook.get_conn() - - conn_extra: dict = json.loads(expected_conn.extra) - assert dict(conn.headers, **conn_extra["headers"]) == conn.headers - assert conn.headers["Content-Type"] == "application/json" - assert conn.headers["X-Requested-By"] == "Airflow" + with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + expected_conn = get_airflow_connection() + conn = self.get_hook.get_conn() + assert dict(conn.headers, **json.loads(expected_conn.extra)) == conn.headers + assert conn.headers.get("bearer") == "test" def test_hook_ignore_max_redirects_from_extra_field_as_header(self): - airflow_connection = get_airflow_connection_with_extra( - extra={ - "headers": DEFAULT_HEADERS_AS_STRING, - "max_redirects": 3, - } - ) + airflow_connection = get_airflow_connection_with_extra(extra={"bearer": "test", "max_redirects": 3}) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=airflow_connection): - expected_conn = airflow_connection() - conn = self.get_hook.get_conn() - assert dict(conn.headers, **json.loads(expected_conn.extra)) != conn.headers - assert conn.headers["Content-Type"] == "application/json" - assert conn.headers["X-Requested-By"] == "Airflow" - assert conn.proxies == {} - assert conn.stream is False - assert conn.verify is True - assert conn.cert is None - assert conn.max_redirects == 3 - assert conn.trust_env is True + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + expected_conn = airflow_connection() + conn = self.get_hook.get_conn() + assert dict(conn.headers, **json.loads(expected_conn.extra)) != conn.headers + assert conn.headers.get("bearer") == "test" + assert conn.headers.get("allow_redirects") is None + assert conn.proxies == {} + assert conn.stream is False + assert conn.verify is True + assert conn.cert is None + assert conn.max_redirects == 3 + assert conn.trust_env is True def test_hook_ignore_proxies_from_extra_field_as_header(self): airflow_connection = get_airflow_connection_with_extra( - extra={ - "headers": DEFAULT_HEADERS, - "proxies": {"http": "http://proxy:80", "https": "https://proxy:80"}, - } + extra={"bearer": "test", "proxies": {"http": "http://proxy:80", "https": "https://proxy:80"}} ) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=airflow_connection): - expected_conn = airflow_connection() - conn = self.get_hook.get_conn() - assert dict(conn.headers, **json.loads(expected_conn.extra)) != conn.headers - assert conn.headers["Content-Type"] == "application/json" - assert conn.headers["X-Requested-By"] == "Airflow" - assert conn.proxies == {"http": "http://proxy:80", "https": "https://proxy:80"} - assert conn.stream is False - assert conn.verify is True - assert conn.cert is None - assert conn.max_redirects == DEFAULT_REDIRECT_LIMIT - assert conn.trust_env is True + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + expected_conn = airflow_connection() + conn = self.get_hook.get_conn() + assert dict(conn.headers, **json.loads(expected_conn.extra)) != conn.headers + assert conn.headers.get("bearer") == "test" + assert conn.headers.get("proxies") is None + assert conn.proxies == {"http": "http://proxy:80", "https": "https://proxy:80"} + assert conn.stream is False + assert conn.verify is True + assert conn.cert is None + assert conn.max_redirects == DEFAULT_REDIRECT_LIMIT + assert conn.trust_env is True def test_hook_ignore_verify_from_extra_field_as_header(self): - airflow_connection = get_airflow_connection_with_extra( - extra={ - "headers": {"Content-Type": "application/json", "X-Requested-By": "Airflow"}, - "verify": False, - } - ) + airflow_connection = get_airflow_connection_with_extra(extra={"bearer": "test", "verify": False}) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=airflow_connection): - expected_conn = airflow_connection() - conn = self.get_hook.get_conn() - assert dict(conn.headers, **json.loads(expected_conn.extra)) != conn.headers - assert conn.headers["Content-Type"] == "application/json" - assert conn.headers["X-Requested-By"] == "Airflow" - assert conn.proxies == {} - assert conn.stream is False - assert conn.verify is False - assert conn.cert is None - assert conn.max_redirects == DEFAULT_REDIRECT_LIMIT - assert conn.trust_env is True + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + expected_conn = airflow_connection() + conn = self.get_hook.get_conn() + assert dict(conn.headers, **json.loads(expected_conn.extra)) != conn.headers + assert conn.headers.get("bearer") == "test" + assert conn.headers.get("verify") is None + assert conn.proxies == {} + assert conn.stream is False + assert conn.verify is False + assert conn.cert is None + assert conn.max_redirects == DEFAULT_REDIRECT_LIMIT + assert conn.trust_env is True def test_hook_ignore_cert_from_extra_field_as_header(self): - airflow_connection = get_airflow_connection_with_extra( - extra={ - "headers": DEFAULT_HEADERS, - "cert": "cert.crt", - } - ) + airflow_connection = get_airflow_connection_with_extra(extra={"bearer": "test", "cert": "cert.crt"}) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=airflow_connection): - expected_conn = airflow_connection() - conn = self.get_hook.get_conn() - assert dict(conn.headers, **json.loads(expected_conn.extra)) != conn.headers - assert conn.headers["Content-Type"] == "application/json" - assert conn.headers["X-Requested-By"] == "Airflow" - assert conn.proxies == {} - assert conn.stream is False - assert conn.verify is True - assert conn.cert == "cert.crt" - assert conn.max_redirects == DEFAULT_REDIRECT_LIMIT - assert conn.trust_env is True + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + expected_conn = airflow_connection() + conn = self.get_hook.get_conn() + assert dict(conn.headers, **json.loads(expected_conn.extra)) != conn.headers + assert conn.headers.get("bearer") == "test" + assert conn.headers.get("cert") is None + assert conn.proxies == {} + assert conn.stream is False + assert conn.verify is True + assert conn.cert == "cert.crt" + assert conn.max_redirects == DEFAULT_REDIRECT_LIMIT + assert conn.trust_env is True def test_hook_ignore_trust_env_from_extra_field_as_header(self): - airflow_connection = get_airflow_connection_with_extra( - extra={ - "headers": DEFAULT_HEADERS, - "trust_env": False, - } - ) + airflow_connection = get_airflow_connection_with_extra(extra={"bearer": "test", "trust_env": False}) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=airflow_connection): - expected_conn = airflow_connection() - conn = self.get_hook.get_conn() - assert dict(conn.headers, **json.loads(expected_conn.extra)) != conn.headers - assert conn.headers["Content-Type"] == "application/json" - assert conn.headers["X-Requested-By"] == "Airflow" - assert conn.proxies == {} - assert conn.stream is False - assert conn.verify is True - assert conn.cert is None - assert conn.max_redirects == DEFAULT_REDIRECT_LIMIT - assert conn.trust_env is False + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + expected_conn = airflow_connection() + conn = self.get_hook.get_conn() + assert dict(conn.headers, **json.loads(expected_conn.extra)) != conn.headers + assert conn.headers.get("bearer") == "test" + assert conn.headers.get("cert") is None + assert conn.proxies == {} + assert conn.stream is False + assert conn.verify is True + assert conn.cert is None + assert conn.max_redirects == DEFAULT_REDIRECT_LIMIT + assert conn.trust_env is False @mock.patch("requests.Request") def test_hook_with_method_in_lowercase(self, mock_requests): @@ -275,20 +234,20 @@ def test_hook_with_method_in_lowercase(self, mock_requests): @pytest.mark.db_test def test_hook_uses_provided_header(self): - conn = self.get_hook.get_conn(headers={"Content-Type": "text/html"}) - assert conn.headers["Content-Type"] == "text/html" - assert conn.headers.get("X-Requested-By") is None + conn = self.get_hook.get_conn(headers={"bearer": "newT0k3n"}) + assert conn.headers.get("bearer") == "newT0k3n" @pytest.mark.db_test def test_hook_has_no_header_from_extra(self): conn = self.get_hook.get_conn() - assert conn.headers.get("Content-Type") is None - assert conn.headers.get("X-Requested-By") is None + assert conn.headers.get("bearer") is None def test_hooks_header_from_extra_is_overridden(self): with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): - conn = self.get_hook.get_conn(headers={"Content-Type": "text/html"}) - assert conn.headers["Content-Type"] == "text/html" + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + conn = self.get_hook.get_conn(headers={"bearer": "newT0k3n"}) + assert conn.headers.get("bearer") == "newT0k3n" def test_post_request(self, requests_mock): requests_mock.post( @@ -296,8 +255,10 @@ def test_post_request(self, requests_mock): ) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): - resp = self.post_hook.run("v1/test") - assert resp.status_code == 200 + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + resp = self.post_hook.run("v1/test") + assert resp.status_code == 200 def test_post_request_with_error_code(self, requests_mock): requests_mock.post( @@ -309,7 +270,9 @@ def test_post_request_with_error_code(self, requests_mock): with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): with pytest.raises(AirflowException): - self.post_hook.run("v1/test") + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + self.post_hook.run("v1/test") def test_post_request_do_not_raise_for_status_if_check_response_is_false(self, requests_mock): requests_mock.post( @@ -320,8 +283,10 @@ def test_post_request_do_not_raise_for_status_if_check_response_is_false(self, r ) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): - resp = self.post_hook.run("v1/test", extra_options={"check_response": False}) - assert resp.status_code == 418 + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + resp = self.post_hook.run("v1/test", extra_options={"check_response": False}) + assert resp.status_code == 418 @pytest.mark.db_test @mock.patch("airflow.providers.http.hooks.http.requests.Session") @@ -351,8 +316,10 @@ def test_run_with_advanced_retry(self, requests_mock): reraise=True, ) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): - response = self.get_hook.run_with_advanced_retry(endpoint="v1/test", _retry_args=retry_args) - assert isinstance(response, requests.Response) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + response = self.get_hook.run_with_advanced_retry(endpoint="v1/test", _retry_args=retry_args) + assert isinstance(response, requests.Response) def test_header_from_extra_and_run_method_are_merged(self): def run_and_return(unused_session, prepped_request, unused_extra_options, **kwargs): @@ -363,11 +330,12 @@ def run_and_return(unused_session, prepped_request, unused_extra_options, **kwar "airflow.providers.http.hooks.http.HttpHook.run_and_check", side_effect=run_and_return ): with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): - prepared_request = self.get_hook.run("v1/test", headers={"some_other_header": "test"}) - actual = dict(prepared_request.headers) - assert actual["Content-Type"] == "application/json" - assert actual["X-Requested-By"] == "Airflow" - assert actual["some_other_header"] == "test" + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + prepared_request = self.get_hook.run("v1/test", headers={"some_other_header": "test"}) + actual = dict(prepared_request.headers) + assert actual.get("bearer") == "test" + assert actual.get("some_other_header") == "test" @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") def test_http_connection(self, mock_get_connection): @@ -413,119 +381,6 @@ def test_connection_without_host(self, mock_get_connection): hook.get_conn({}) assert hook.base_url == "http://" - @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") - @mock.patch("providers.tests.http.hooks.test_http.CustomAuthBase.__init__") - def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_connection): - auth.return_value = None - conn = Connection( - conn_id="http_default", - conn_type="http", - login="username", - password="pass", - extra='{"headers": {"x-header": 0}, "auth_kwargs": {"endpoint": "http://localhost"}}', - ) - mock_get_connection.return_value = conn - - hook = HttpHook(auth_type=CustomAuthBase) - session = hook.get_conn({}) - - auth.assert_called_once_with("username", "pass", endpoint="http://localhost") - assert "auth_kwargs" not in session.headers - assert "x-header" in session.headers - - def test_available_connection_auth_types(self): - auth_types = HttpHook.get_auth_types() - assert auth_types == frozenset( - { - "requests.auth.HTTPBasicAuth", - "requests.auth.HTTPProxyAuth", - "requests.auth.HTTPDigestAuth", - "requests_kerberos.HTTPKerberosAuth", - "aiohttp.BasicAuth", - "tests.providers.http.hooks.test_http.CustomAuthBase", - } - ) - - @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") - def test_connection_with_invalid_auth_type_get_skipped(self, mock_get_connection, caplog): - auth_type: str = "auth_type.class.not.available.for.Import" - conn = Connection( - conn_id="http_default", - conn_type="http", - extra=f'{{"auth_type": "{auth_type}"}}', - ) - mock_get_connection.return_value = conn - HttpHook().get_conn({}) - assert f"Skipping import of auth_type '{auth_type}'." in caplog.text - - @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_auth_types") - @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") - @mock.patch("providers.tests.http.hooks.test_http.CustomAuthBase.__init__") - def test_connection_with_extra_header_and_auth_type(self, auth, mock_get_connection, mock_get_auth_types): - auth.return_value = None - conn = Connection( - conn_id="http_default", - conn_type="http", - login="username", - password="pass", - extra='{"headers": {"x-header": 0}, "auth_type": "providers.tests.http.hooks.test_http.CustomAuthBase"}', - ) - mock_get_connection.return_value = conn - mock_get_auth_types.return_value = frozenset(["providers.tests.http.hooks.test_http.CustomAuthBase"]) - - session = HttpHook().get_conn({}) - auth.assert_called_once_with("username", "pass") - assert isinstance(session.auth, CustomAuthBase) - assert "auth_type" not in session.headers - assert "x-header" in session.headers - - @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_auth_types") - @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") - @mock.patch("providers.tests.http.hooks.test_http.CustomAuthBase.__init__") - def test_connection_with_extra_auth_type_and_no_credentials( - self, auth, mock_get_connection, mock_get_auth_types - ): - auth.return_value = None - conn = Connection( - conn_id="http_default", - conn_type="http", - login="username", - password="secret", - extra='{"auth_type": "providers.tests.http.hooks.test_http.CustomAuthBase"}', - ) - mock_get_connection.return_value = conn - mock_get_auth_types.return_value = frozenset(["providers.tests.http.hooks.test_http.CustomAuthBase"]) - - HttpHook().get_conn({}) - auth.assert_called_once() - - @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") - @mock.patch("providers.tests.http.hooks.test_http.CustomAuthBase.__init__") - def test_connection_with_string_headers_and_auth_kwargs(self, auth, mock_get_connection): - """When passed via the UI, the 'headers' and 'auth_kwargs' fields' data is - saved as string. - """ - auth.return_value = None - conn = Connection( - conn_id="http_default", - conn_type="http", - login="username", - password="pass", - extra=f""" - {{"auth_kwargs": {{\r\n "endpoint": "http://localhost"\r\n}}, - "headers": {DEFAULT_HEADERS_AS_STRING}}} - """, - ) - mock_get_connection.return_value = conn - - hook = HttpHook(auth_type=CustomAuthBase) - session = hook.get_conn({}) - - auth.assert_called_once_with("username", "pass", endpoint="http://localhost") - assert "auth_kwargs" not in session.headers - assert session.headers["Content-Type"] == "application/json" - assert session.headers["X-Requested-By"] == "Airflow" - @pytest.mark.parametrize("method", ["GET", "POST"]) def test_json_request(self, method, requests_mock): obj1 = {"a": 1, "b": "abc", "c": [1, 2, {"d": 10}]} @@ -536,8 +391,10 @@ def match_obj1(request): requests_mock.request(method=method, url="//test:8080/v1/test", additional_matcher=match_obj1) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): - # will raise NoMockAddress exception if obj1 != request.json() - HttpHook(method=method).run("v1/test", json=obj1) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + # will raise NoMockAddress exception if obj1 != request.json() + HttpHook(method=method).run("v1/test", json=obj1) @mock.patch("airflow.providers.http.hooks.http.requests.Session.send") def test_verify_set_to_true_by_default(self, mock_session_send): @@ -612,35 +469,43 @@ def test_verify_false_parameter_overwrites_set_requests_ca_bundle_env_var(self, def test_connection_success(self, requests_mock): requests_mock.get("http://test:8080", status_code=200, json={"status": {"status": 200}}, reason="OK") with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): - status, msg = self.get_hook.test_connection() - assert status is True - assert msg == "Connection successfully tested" + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + status, msg = self.get_hook.test_connection() + assert status is True + assert msg == "Connection successfully tested" def test_connection_failure(self, requests_mock): requests_mock.get( "http://test:8080", status_code=500, json={"message": "internal server error"}, reason="NOT_OK" ) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): - status, msg = self.get_hook.test_connection() - assert status is False - assert msg == "500:NOT_OK" + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + status, msg = self.get_hook.test_connection() + assert status is False + assert msg == "500:NOT_OK" @mock.patch("requests.auth.AuthBase.__init__") def test_loginless_custom_auth_initialized_with_no_args(self, auth): with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): - auth.return_value = None - hook = HttpHook("GET", "http_default", AuthBase) - hook.get_conn() - auth.assert_called_once_with() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + auth.return_value = None + hook = HttpHook("GET", "http_default", AuthBase) + hook.get_conn() + auth.assert_called_once_with() @mock.patch("requests.auth.AuthBase.__init__") def test_loginless_custom_auth_initialized_with_args(self, auth): with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): - auth.return_value = None - auth_with_args = functools.partial(AuthBase, "test_arg") - hook = HttpHook("GET", "http_default", auth_with_args) - hook.get_conn() - auth.assert_called_once_with("test_arg") + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + auth.return_value = None + auth_with_args = functools.partial(AuthBase, "test_arg") + hook = HttpHook("GET", "http_default", auth_with_args) + hook.get_conn() + auth.assert_called_once_with("test_arg") @mock.patch("requests.auth.HTTPBasicAuth.__init__") def test_login_password_basic_auth_initialized(self, auth): @@ -648,18 +513,22 @@ def test_login_password_basic_auth_initialized(self, auth): "airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection_with_login_and_password, ): - auth.return_value = None - hook = HttpHook("GET", "http_default", HTTPBasicAuth) - hook.get_conn() - auth.assert_called_once_with("username", "pass") + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + auth.return_value = None + hook = HttpHook("GET", "http_default", HTTPBasicAuth) + hook.get_conn() + auth.assert_called_once_with("username", "pass") @mock.patch("requests.auth.HTTPBasicAuth.__init__") def test_default_auth_not_initialized(self, auth): with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): - auth.return_value = None - hook = HttpHook("GET", "http_default") - hook.get_conn() - auth.assert_not_called() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + auth.return_value = None + hook = HttpHook("GET", "http_default") + hook.get_conn() + auth.assert_not_called() def test_keep_alive_enabled(self): with ( @@ -725,15 +594,6 @@ def test_custom_adapter(self): session.adapters["https://"], type(custom_adapter) ), "Custom HTTPS adapter not correctly mounted" - @pytest.mark.skipif(not AIRFLOW_V_2_10_PLUS, reason="Lambda parameters works in Airflow >= 2.10.0") - def test_airflow_dependency_version(self): - if airflow_dependency_version() >= packaging.version.parse("2.10.0"): - raise RuntimeError( - f"The method {HttpHook._configure_session_from_extra.__name__} from the " - f"{HttpHook.__name__} should be refactored since the {Connection.get_extra_dejson.__name__} " - f"method is now available on the {Connection.__name__} class since Apache Airflow 2.10.0" - ) - class TestHttpAsyncHook: @pytest.mark.asyncio @@ -821,8 +681,8 @@ async def test_async_post_request_with_error_code(self): @pytest.mark.asyncio async def test_async_request_uses_connection_extra(self): """Test api call asynchronously with a connection that has extra field.""" - headers = {"Content-Type": "application/json", "X-Requested-By": "Airflow"} - airflow_connection = get_airflow_connection_with_extra(extra={"headers": headers}) + + connection_extra = {"bearer": "test"} with aioresponses() as m: m.post( @@ -846,11 +706,11 @@ async def test_async_request_uses_connection_extra(self): @pytest.mark.asyncio async def test_async_request_uses_connection_extra_with_requests_parameters(self): """Test api call asynchronously with a connection that has extra field.""" - headers = {"Content-Type": "application/json", "X-Requested-By": "Airflow"} + connection_extra = {"bearer": "test"} proxy = {"http": "http://proxy:80", "https": "https://proxy:80"} airflow_connection = get_airflow_connection_with_extra( extra={ - **{"headers": DEFAULT_HEADERS}, + **connection_extra, **{ "proxies": proxy, "timeout": 60, @@ -864,6 +724,7 @@ async def test_async_request_uses_connection_extra_with_requests_parameters(self with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=airflow_connection): hook = HttpAsyncHook() + with aioresponses() as m: m.post( "http://test:8080/v1/test", @@ -887,29 +748,34 @@ async def test_async_request_uses_connection_extra_with_requests_parameters(self assert mocked_function.call_args.kwargs.get("max_redirects") == 3 assert mocked_function.call_args.kwargs.get("trust_env") is False - def test_parse_extra(self): + def test_process_extra_options_from_connection(self): + extra_options = {} proxy = {"http": "http://proxy:80", "https": "https://proxy:80"} - session_conf = { - "stream": True, - "cert": "cert.crt", - "proxies": proxy, + conn = get_airflow_connection_with_extra( + extra={ + "bearer": "test", + "stream": True, + "cert": "cert.crt", + "proxies": proxy, + "timeout": 60, + "verify": False, + "allow_redirects": False, + "max_redirects": 3, + "trust_env": False, + } + )() + + actual = HttpAsyncHook._process_extra_options_from_connection(conn=conn, extra_options=extra_options) + + assert extra_options == { + "proxy": proxy, "timeout": 60, - "verify": False, + "verify_ssl": False, "allow_redirects": False, "max_redirects": 3, "trust_env": False, } - - actual = HttpAsyncHook()._parse_extra( - conn_extra={**{"headers": DEFAULT_HEADERS_AS_STRING}, **session_conf} - ) - - assert actual == { - "auth_type": None, - "auth_kwargs": {}, - "session_conf": session_conf, - "headers": DEFAULT_HEADERS, - } + assert actual == {"bearer": "test"} @pytest.mark.asyncio async def test_build_request_url_from_connection(self): From 97f4606bdcbc300faa93fede01fd93590773c693 Mon Sep 17 00:00:00 2001 From: David Blain Date: Mon, 27 Jan 2025 20:09:06 +0100 Subject: [PATCH 088/286] refactor: Re-added missing auth test cases for HttpHook --- providers/tests/http/hooks/test_http.py | 122 ++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/providers/tests/http/hooks/test_http.py b/providers/tests/http/hooks/test_http.py index 0f9d0bb240518..79a2af5169d22 100644 --- a/providers/tests/http/hooks/test_http.py +++ b/providers/tests/http/hooks/test_http.py @@ -68,6 +68,14 @@ def get_airflow_connection_with_login_and_password(conn_id: str = "http_default" return Connection(conn_id=conn_id, conn_type="http", host="test.com", login="username", password="pass") +class CustomAuthBase(HTTPBasicAuth): + def __init__(self, username: str, password: str, endpoint: str): + super().__init__(username, password) + + +@mock.patch.dict( + "os.environ", AIRFLOW__HTTP__EXTRA_AUTH_TYPES="tests.providers.http.hooks.test_http.CustomAuthBase" +) class TestHttpHook: """Test get, post and raise_for_status""" @@ -381,6 +389,120 @@ def test_connection_without_host(self, mock_get_connection): hook.get_conn({}) assert hook.base_url == "http://" + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("providers.tests.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_connection): + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + login="username", + password="pass", + extra='{"headers": {"x-header": 0}, "auth_kwargs": {"endpoint": "http://localhost"}}', + ) + mock_get_connection.return_value = conn + + hook = HttpHook(auth_type=CustomAuthBase) + session = hook.get_conn({}) + + auth.assert_called_once_with("username", "pass", endpoint="http://localhost") + assert "auth_kwargs" not in session.headers + assert "x-header" in session.headers + + def test_available_connection_auth_types(self): + auth_types = HttpHook.get_auth_types() + assert auth_types == frozenset( + { + "requests.auth.HTTPBasicAuth", + "requests.auth.HTTPProxyAuth", + "requests.auth.HTTPDigestAuth", + "requests_kerberos.HTTPKerberosAuth", + "aiohttp.BasicAuth", + "tests.providers.http.hooks.test_http.CustomAuthBase", + } + ) + + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + def test_connection_with_invalid_auth_type_get_skipped(self, mock_get_connection, caplog): + auth_type: str = "auth_type.class.not.available.for.Import" + conn = Connection( + conn_id="http_default", + conn_type="http", + extra=f'{{"auth_type": "{auth_type}"}}', + ) + mock_get_connection.return_value = conn + HttpHook().get_conn({}) + assert f"Skipping import of auth_type '{auth_type}'." in caplog.text + + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_auth_types") + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("providers.tests.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_header_and_auth_type(self, auth, mock_get_connection, mock_get_auth_types): + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + login="username", + password="pass", + extra='{"headers": {"x-header": 0}, "auth_type": "providers.tests.http.hooks.test_http.CustomAuthBase"}', + ) + mock_get_connection.return_value = conn + mock_get_auth_types.return_value = frozenset(["providers.tests.http.hooks.test_http.CustomAuthBase"]) + + session = HttpHook().get_conn({}) + auth.assert_called_once_with("username", "pass") + assert isinstance(session.auth, CustomAuthBase) + assert "auth_type" not in session.headers + assert "x-header" in session.headers + + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_auth_types") + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("providers.tests.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_auth_type_and_no_credentials( + self, auth, mock_get_connection, mock_get_auth_types + ): + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + extra='{"headers": {"x-header": 0}, "auth_type": "providers.tests.http.hooks.test_http.CustomAuthBase"}', + ) + mock_get_connection.return_value = conn + mock_get_auth_types.return_value = frozenset(["providers.tests.http.hooks.test_http.CustomAuthBase"]) + + session = HttpHook().get_conn({}) + auth.assert_called_once() + assert isinstance(session.auth, CustomAuthBase) + assert "auth_type" not in session.headers + assert "x-header" in session.headers + + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("providers.tests.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_string_headers_and_auth_kwargs(self, auth, mock_get_connection): + """When passed via the UI, the 'headers' and 'auth_kwargs' fields' data is + saved as string. + """ + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + login="username", + password="pass", + extra=f""" + {{"auth_kwargs": {{\r\n "endpoint": "http://localhost"\r\n}}, + "headers": ""}} + """, + ) + mock_get_connection.return_value = conn + + hook = HttpHook(auth_type=CustomAuthBase) + session = hook.get_conn({}) + + auth.assert_called_once_with("username", "pass", endpoint="http://localhost") + assert "auth_kwargs" not in session.headers + assert session.headers["Content-Type"] == "application/json" + assert session.headers["X-Requested-By"] == "Airflow" + @pytest.mark.parametrize("method", ["GET", "POST"]) def test_json_request(self, method, requests_mock): obj1 = {"a": 1, "b": "abc", "c": [1, 2, {"d": 10}]} From 9c3ab29e0d8d91f06d6bc9a039c3a270dd1c398a Mon Sep 17 00:00:00 2001 From: David Blain Date: Mon, 27 Jan 2025 20:10:14 +0100 Subject: [PATCH 089/286] refactor: Fixed test_post_request_with_error_code in TestHttpHook --- providers/tests/http/hooks/test_http.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/providers/tests/http/hooks/test_http.py b/providers/tests/http/hooks/test_http.py index 79a2af5169d22..921d53d53c3af 100644 --- a/providers/tests/http/hooks/test_http.py +++ b/providers/tests/http/hooks/test_http.py @@ -277,9 +277,9 @@ def test_post_request_with_error_code(self, requests_mock): ) with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): - with pytest.raises(AirflowException): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=AirflowProviderDeprecationWarning) + with pytest.raises(AirflowException): self.post_hook.run("v1/test") def test_post_request_do_not_raise_for_status_if_check_response_is_false(self, requests_mock): From 7648028d417a04a9302342a669d83fc36804b33f Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Sun, 12 Nov 2023 13:55:18 +0100 Subject: [PATCH 090/286] feat: Implement `auth_kwargs` parameter in Http Connection --- .../provider_tests/http/hooks/test_http.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index 82a1ff9765156..e1a380e4588e5 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -67,6 +67,11 @@ def get_airflow_connection_with_login_and_password(conn_id: str = "http_default" return Connection(conn_id=conn_id, conn_type="http", host="test.com", login="username", password="pass") +class CustomAuthBase(HTTPBasicAuth): + def __init__(self, username: str, password: str, endpoint: str): + super().__init__(username, password) + + class TestHttpHook: """Test get, post and raise_for_status""" @@ -352,6 +357,25 @@ def test_connection_without_host(self, mock_get_connection): hook.get_conn({}) assert hook.base_url == "http://" + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_connection): + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + login="username", + password="pass", + extra='{"auth_kwargs": {"endpoint": "http://localhost"}}', + ) + mock_get_connection.return_value = conn + + hook = HttpHook(auth_type=CustomAuthBase) + session = hook.get_conn({}) + + auth.assert_called_once_with("username", "pass", endpoint="http://localhost") + assert "auth_kwargs" not in session.headers + @pytest.mark.parametrize("method", ["GET", "POST"]) def test_json_request(self, method, requests_mock): obj1 = {"a": 1, "b": "abc", "c": [1, 2, {"d": 10}]} From a8dbd95a085ab2902e2c4f015ee0f9fd854bb602 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 24 Nov 2023 00:25:53 +0100 Subject: [PATCH 091/286] fix: Correctly use auth_type from Connection --- .../provider_tests/http/hooks/test_http.py | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index e1a380e4588e5..802dd0ef15c39 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -366,7 +366,7 @@ def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_conne conn_type="http", login="username", password="pass", - extra='{"auth_kwargs": {"endpoint": "http://localhost"}}', + extra='{"x-header": 0, "auth_kwargs": {"endpoint": "http://localhost"}}', ) mock_get_connection.return_value = conn @@ -375,6 +375,40 @@ def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_conne auth.assert_called_once_with("username", "pass", endpoint="http://localhost") assert "auth_kwargs" not in session.headers + assert "x-header" in session.headers + + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_header_and_auth_type(self, auth, mock_get_connection): + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + login="username", + password="pass", + extra='{"x-header": 0, "auth_type": "tests.providers.http.hooks.test_http.CustomAuthBase"}', + ) + mock_get_connection.return_value = conn + + session = HttpHook().get_conn({}) + auth.assert_called_once_with("username", "pass") + assert isinstance(session.auth, CustomAuthBase) + assert "auth_type" not in session.headers + assert "x-header" in session.headers + + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_auth_type_and_no_credentials(self, auth, mock_get_connection): + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + extra='{"auth_type": "tests.providers.http.hooks.test_http.CustomAuthBase"}', + ) + mock_get_connection.return_value = conn + + HttpHook().get_conn({}) + auth.assert_called_once() @pytest.mark.parametrize("method", ["GET", "POST"]) def test_json_request(self, method, requests_mock): From e740d35e30916791deb59169a5a49de105d187ba Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 24 Nov 2023 01:55:44 +0100 Subject: [PATCH 092/286] feat: Add Connection documentation --- providers/http/src/airflow/providers/http/hooks/http.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index b22a01f8283db..eb63e27ee0925 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -51,6 +51,12 @@ class HttpHook(BaseHook): """ Interact with HTTP servers. + To configure the auth_type, in addition to the `auth_type` parameter, you can also: + * set the `auth_type` parameter in the Connection settings. + * define extra parameters used to instantiate the `auth_type` class, in the Connection settings. + + See :doc:`/connections/http` for full documentation. + :param method: the API method to be called :param http_conn_id: :ref:`http connection` that has the base API url i.e https://www.google.com/ and optional authentication credentials. Default From e4a226dc5dec3cf6f13beb3770a371ed064b62fb Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 5 Dec 2023 23:14:43 +0100 Subject: [PATCH 093/286] feat: Make available auth_types configurable from airflow config --- providers/http/docs/configurations-ref.rst | 0 providers/http/docs/index.rst | 1 + providers/http/provider.yaml | 15 ++++++++++ .../provider_tests/http/hooks/test_http.py | 28 ++++++++++++++++++- 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 providers/http/docs/configurations-ref.rst diff --git a/providers/http/docs/configurations-ref.rst b/providers/http/docs/configurations-ref.rst new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/providers/http/docs/index.rst b/providers/http/docs/index.rst index 49745912f879b..7fc650ea8f50f 100644 --- a/providers/http/docs/index.rst +++ b/providers/http/docs/index.rst @@ -42,6 +42,7 @@ :maxdepth: 1 :caption: References + Configuration Python API <_api/airflow/providers/http/index> .. toctree:: diff --git a/providers/http/provider.yaml b/providers/http/provider.yaml index dee0796c04891..7a991044ea801 100644 --- a/providers/http/provider.yaml +++ b/providers/http/provider.yaml @@ -93,3 +93,18 @@ triggers: connection-types: - hook-class-name: airflow.providers.http.hooks.http.HttpHook connection-type: http + +config: + http: + description: "Options for Http provider." + options: + extra_auth_types: + description: | + A comma separated list of auth_type classes, which can be used to + configure Http Connections in Airflow's UI. This list restricts which + classes can be arbitrary imported, and protects from dependency + injections. + type: string + version_added: 4.8.0 + example: "requests_kerberos.HTTPKerberosAuth,any.other.custom.HTTPAuth" + default: ~ diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index 802dd0ef15c39..fe6b8f882f2b7 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -36,7 +36,7 @@ from airflow.exceptions import AirflowException from airflow.models import Connection -from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook +from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook, get_auth_types @pytest.fixture @@ -72,6 +72,9 @@ def __init__(self, username: str, password: str, endpoint: str): super().__init__(username, password) +@mock.patch.dict( + "os.environ", AIRFLOW__HTTP__EXTRA_AUTH_TYPES="tests.providers.http.hooks.test_http.CustomAuthBase" +) class TestHttpHook: """Test get, post and raise_for_status""" @@ -377,6 +380,29 @@ def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_conne assert "auth_kwargs" not in session.headers assert "x-header" in session.headers + def test_available_connection_auth_types(self): + auth_types = get_auth_types() + assert auth_types == frozenset( + { + "request.auth.HTTPBasicAuth", + "request.auth.HTTPProxyAuth", + "request.auth.HTTPDigestAuth", + "tests.providers.http.hooks.test_http.CustomAuthBase", + } + ) + + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + def test_connection_with_invalid_auth_type_get_skipped(self, mock_get_connection, caplog): + auth_type: str = "auth_type.class.not.available.for.Import" + conn = Connection( + conn_id="http_default", + conn_type="http", + extra=f'{{"auth_type": "{auth_type}"}}', + ) + mock_get_connection.return_value = conn + HttpHook().get_conn({}) + assert f"Skipping import of auth_type '{auth_type}'." in caplog.text + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") def test_connection_with_extra_header_and_auth_type(self, auth, mock_get_connection): From 92a38971561ae6b6f1cd67cb18faf33449b3b4b2 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Wed, 6 Dec 2023 22:24:25 +0100 Subject: [PATCH 094/286] feat: Add fields for auth config and header config in Http Connection form --- .../src/airflow/providers/http/hooks/http.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index eb63e27ee0925..31a0bc5cfc8a5 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -113,6 +113,30 @@ def auth_type(self): def auth_type(self, v): self._auth_type = v + @classmethod + def get_connection_form_widgets(cls) -> dict[str, Any]: + """Return connection widgets to add to connection form.""" + from flask_babel import lazy_gettext + from wtforms.fields import SelectField, TextAreaField + + auth_types_choices = frozenset({""}) | get_auth_types() + return { + "auth_type": SelectField( + lazy_gettext("Auth type"), + choices=[(clazz, clazz) for clazz in auth_types_choices] + ), + "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs")), + "extra_headers": TextAreaField(lazy_gettext("Extra Headers")) + } + + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + """Return custom field behaviour.""" + return { + "hidden_fields": ["extra"], + "relabeling": {} + } + # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: From a92d3401e609f23c0dc588f3a3ad5668eb606bcb Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Thu, 7 Dec 2023 08:48:01 +0100 Subject: [PATCH 095/286] fix: Correctly apply styling to extra fields --- .../http/src/airflow/providers/http/hooks/http.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 31a0bc5cfc8a5..538bc1e6e38cb 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -116,6 +116,7 @@ def auth_type(self, v): @classmethod def get_connection_form_widgets(cls) -> dict[str, Any]: """Return connection widgets to add to connection form.""" + from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget, Select2Widget from flask_babel import lazy_gettext from wtforms.fields import SelectField, TextAreaField @@ -123,19 +124,17 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: return { "auth_type": SelectField( lazy_gettext("Auth type"), - choices=[(clazz, clazz) for clazz in auth_types_choices] + choices=[(clazz, clazz) for clazz in auth_types_choices], + widget=Select2Widget(), ), - "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs")), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers")) + "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), + "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } @classmethod def get_ui_field_behaviour(cls) -> dict[str, Any]: """Return custom field behaviour.""" - return { - "hidden_fields": ["extra"], - "relabeling": {} - } + return {"hidden_fields": ["extra"], "relabeling": {}} # headers may be passed through directly or in the "extra" field in the connection # definition From 9532528fdd5fd47ff7a18b00ba55b5514637fdcc Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 19 Dec 2023 01:05:47 +0100 Subject: [PATCH 096/286] feat: Implement simplistic collapsable textarea for "extra" --- airflow/www/forms.py | 26 ++++++++++++++++++- airflow/www/static/js/connection_form.js | 4 +-- .../src/airflow/providers/http/hooks/http.py | 9 +++---- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/airflow/www/forms.py b/airflow/www/forms.py index 7028e2026e449..b69184c6590c2 100644 --- a/airflow/www/forms.py +++ b/airflow/www/forms.py @@ -33,6 +33,7 @@ from flask_appbuilder.forms import DynamicForm from flask_babel import lazy_gettext from flask_wtf import FlaskForm +from markupsafe import Markup from wtforms import widgets from wtforms.fields import Field, IntegerField, PasswordField, SelectField, StringField, TextAreaField from wtforms.validators import InputRequired, Optional @@ -176,6 +177,29 @@ def populate_obj(self, item): field.populate_obj(item, name) +class BS3AccordionTextAreaFieldWidget(BS3TextAreaFieldWidget): + + @staticmethod + def _make_collapsable_panel(field: Field, content: Markup) -> str: + collapsable_id: str = f"collapsable_{field.id}" + return f""" +
+
+

+ +

+
+ +
+ """ + + def __call__(self, field, **kwargs): + text_area = super(BS3TextAreaFieldWidget, self).__call__(field, **kwargs) + return self._make_collapsable_panel(field=field, content=text_area) + + @cache def create_connection_form_class() -> type[DynamicForm]: """ @@ -223,7 +247,7 @@ def process(self, formdata=None, obj=None, **kwargs): login = StringField(lazy_gettext("Login"), widget=BS3TextFieldWidget()) password = PasswordField(lazy_gettext("Password"), widget=BS3PasswordFieldWidget()) port = IntegerField(lazy_gettext("Port"), validators=[Optional()], widget=BS3TextFieldWidget()) - extra = TextAreaField(lazy_gettext("Extra"), widget=BS3TextAreaFieldWidget()) + extra = TextAreaField(lazy_gettext("Extra"), widget=BS3AccordionTextAreaFieldWidget()) for key, value in providers_manager.connection_form_widgets.items(): setattr(ConnectionForm, key, value.field) diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index d039fc7275462..1c97e00803174 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -83,7 +83,7 @@ function restoreFieldBehaviours() { } ); - Array.from(document.querySelectorAll(".form-control")).forEach((elem) => { + Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach((elem) => { // eslint-disable-next-line no-param-reassign elem.placeholder = ""; elem.parentElement.parentElement.classList.remove("hide"); @@ -101,7 +101,7 @@ function applyFieldBehaviours(connection) { if (Array.isArray(connection.hidden_fields)) { connection.hidden_fields.forEach((field) => { document - .getElementById(field) + .querySelector(`label[for='${field}']`) .parentElement.parentElement.classList.add("hide"); }); } diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 538bc1e6e38cb..7268c13f8764d 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -25,6 +25,8 @@ import tenacity from aiohttp import ClientResponseError from asgiref.sync import sync_to_async +from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget +from markupsafe import Markup from requests.auth import HTTPBasicAuth from requests.models import DEFAULT_REDIRECT_LIMIT from requests_toolbelt.adapters.socket_options import TCPKeepAliveAdapter @@ -128,14 +130,9 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: widget=Select2Widget(), ), "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), + "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()) } - @classmethod - def get_ui_field_behaviour(cls) -> dict[str, Any]: - """Return custom field behaviour.""" - return {"hidden_fields": ["extra"], "relabeling": {}} - # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: From 83358448f9fe96c8d3d28f187b17eea7e90f880b Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Wed, 20 Dec 2023 16:21:08 +0100 Subject: [PATCH 097/286] fix: express clearly empty frozenset creation Goal is to have an empty default choice --- providers/http/src/airflow/providers/http/hooks/http.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 7268c13f8764d..0f6bea66bf624 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -122,7 +122,8 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: from flask_babel import lazy_gettext from wtforms.fields import SelectField, TextAreaField - auth_types_choices = frozenset({""}) | get_auth_types() + default_auth_type = frozenset({""}) + auth_types_choices = default_auth_type | get_auth_types() return { "auth_type": SelectField( lazy_gettext("Auth type"), From 85b71a87878ce6f4b4aff59afdca891822662b37 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Wed, 20 Dec 2023 16:43:29 +0100 Subject: [PATCH 098/286] feat: Refactor Accordion TextArea to use wtform utils --- airflow/www/static/js/connection_form.js | 12 +++++++----- .../http/src/airflow/providers/http/hooks/http.py | 4 +--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index 1c97e00803174..119fe39daae54 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -83,11 +83,13 @@ function restoreFieldBehaviours() { } ); - Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach((elem) => { - // eslint-disable-next-line no-param-reassign - elem.placeholder = ""; - elem.parentElement.parentElement.classList.remove("hide"); - }); + Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach( + (elem) => { + // eslint-disable-next-line no-param-reassign + elem.placeholder = ""; + elem.parentElement.parentElement.classList.remove("hide"); + } + ); } /** diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 0f6bea66bf624..77563d2eb1a81 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -25,8 +25,6 @@ import tenacity from aiohttp import ClientResponseError from asgiref.sync import sync_to_async -from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget -from markupsafe import Markup from requests.auth import HTTPBasicAuth from requests.models import DEFAULT_REDIRECT_LIMIT from requests_toolbelt.adapters.socket_options import TCPKeepAliveAdapter @@ -131,7 +129,7 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: widget=Select2Widget(), ), "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()) + "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } # headers may be passed through directly or in the "extra" field in the connection From 4fc54be0bf8d9f6cb951f047df1e5fdebeee6cfc Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Thu, 21 Dec 2023 13:04:19 +0100 Subject: [PATCH 099/286] feat: Implement 'collapse_extra' field behavior --- airflow/customized_form_field_behaviours.schema.json | 4 ++++ providers/http/src/airflow/providers/http/hooks/http.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/airflow/customized_form_field_behaviours.schema.json b/airflow/customized_form_field_behaviours.schema.json index 78791a87886c1..fa5ace958c5e8 100644 --- a/airflow/customized_form_field_behaviours.schema.json +++ b/airflow/customized_form_field_behaviours.schema.json @@ -22,6 +22,10 @@ "additionalProperties": { "type": "string" } + }, + "collapse_extra": { + "type": "boolean", + "description": "Collapse the 'Extra' field." } }, "additionalProperties": true, diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 77563d2eb1a81..2ddd9ca434efe 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -132,6 +132,10 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + return {"hidden_fields": [], "relabeling": {}, "collapse_extra": True} + # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: From c28df9211721ed3df388129e58653e2baf00d82b Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Sat, 30 Dec 2023 16:43:47 +0100 Subject: [PATCH 100/286] feat: Implement parameterizable behavior for collapsible field --- ...stomized_form_field_behaviours.schema.json | 19 +++++-- airflow/www/static/css/connection.css | 23 +++++++++ airflow/www/static/js/connection_form.js | 49 ++++++++++++++++--- .../www/templates/airflow/conn_create.html | 2 +- airflow/www/templates/airflow/conn_edit.html | 1 + airflow/www/webpack.config.js | 1 + .../src/airflow/providers/http/hooks/http.py | 2 +- 7 files changed, 85 insertions(+), 12 deletions(-) create mode 100644 airflow/www/static/css/connection.css diff --git a/airflow/customized_form_field_behaviours.schema.json b/airflow/customized_form_field_behaviours.schema.json index fa5ace958c5e8..8aa05945ebb01 100644 --- a/airflow/customized_form_field_behaviours.schema.json +++ b/airflow/customized_form_field_behaviours.schema.json @@ -23,9 +23,22 @@ "type": "string" } }, - "collapse_extra": { - "type": "boolean", - "description": "Collapse the 'Extra' field." + "collapsible_fields": { + "description": "List of collapsed fields for the hook, with their properties.", + "type": "object", + "patternProperties": { + "\"^.*$\"": { + "description": "Name of the field to enable collapsing.", + "type": "object", + "properties": { + "expanded": { + "description": "Set the default state of the field as expanded.", + "default": true, + "type": "boolean" + } + } + } + } } }, "additionalProperties": true, diff --git a/airflow/www/static/css/connection.css b/airflow/www/static/css/connection.css new file mode 100644 index 0000000000000..78edf0db5d4dc --- /dev/null +++ b/airflow/www/static/css/connection.css @@ -0,0 +1,23 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +.panel-invisible { + margin: 0; + border: 0; +} diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index 119fe39daae54..5c60638caf488 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -83,13 +83,28 @@ function restoreFieldBehaviours() { } ); - Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach( - (elem) => { - // eslint-disable-next-line no-param-reassign - elem.placeholder = ""; - elem.parentElement.parentElement.classList.remove("hide"); - } - ); + Array.from(document.querySelectorAll(".form-control")).forEach((elem) => { + // eslint-disable-next-line no-param-reassign + elem.placeholder = ""; + elem.parentElement.parentElement.classList.remove("hide"); + }); + + Array.from(document.querySelectorAll(".form-panel")).forEach((elem) => { + elem.parentElement.parentElement.classList.remove("hide"); + + elem.classList.add("panel-invisible"); + const panelHeader = elem.children[0]; + panelHeader.classList.add("hidden"); + panelHeader.firstElementChild.firstElementChild.setAttribute( + "aria-expanded", + "true" + ); + + const collapsible = elem.children[1]; + collapsible.setAttribute("aria-expanded", "true"); + collapsible.classList.add("in"); + collapsible.style.height = null; + }); } /** @@ -122,6 +137,26 @@ function applyFieldBehaviours(connection) { document.getElementById(field).placeholder = placeholder; }); } + + if (connection.collapsible_fields) { + Object.entries(connection.collapsible_fields).forEach((entry) => { + const [field, properties] = entry; + + const collapsibleController = document.getElementById( + `control_collapsible_${field}` + ); + const panelHeader = collapsibleController.parentElement.parentElement; + panelHeader.classList.remove("hidden"); + panelHeader.parentElement.classList.remove("panel-invisible"); + + if (properties.expanded === false) { + const collapsible = document.getElementById(`collapsible_${field}`); + collapsible.classList.remove("in"); + collapsible.setAttribute("aria-expanded", "false"); + collapsibleController.setAttribute("aria-expanded", "false"); + } + }); + } } } diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index ac92b967f7e34..307450b05d16b 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -25,7 +25,7 @@ - + {# required for codemirror #} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 653b0dd3ce07f..11ebd6c4cb436 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -25,6 +25,7 @@ + {# required for codemirror #} diff --git a/airflow/www/webpack.config.js b/airflow/www/webpack.config.js index 9d5800f783f50..ad1a7098e0803 100644 --- a/airflow/www/webpack.config.js +++ b/airflow/www/webpack.config.js @@ -60,6 +60,7 @@ const config = { airflowDefaultTheme: `${CSS_DIR}/bootstrap-theme.css`, connectionForm: `${JS_DIR}/connection_form.js`, chart: [`${CSS_DIR}/chart.css`], + connection: [`${CSS_DIR}/connection.css`], dag: `${JS_DIR}/dag.js`, dagDependencies: `${JS_DIR}/dag_dependencies.js`, dags: [`${CSS_DIR}/dags.css`, `${JS_DIR}/dags.js`], diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 2ddd9ca434efe..192e57c1d70fc 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -134,7 +134,7 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: @classmethod def get_ui_field_behaviour(cls) -> dict[str, Any]: - return {"hidden_fields": [], "relabeling": {}, "collapse_extra": True} + return {"hidden_fields": [], "relabeling": {}, "collapsible_fields": {"extra": {"expanded": False}}} # headers may be passed through directly or in the "extra" field in the connection # definition From e3a8544dabe1e055a3eabdff982285215d48f2ec Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 5 Jan 2024 06:48:16 +0100 Subject: [PATCH 101/286] revert: Remove collapsible field --- ...stomized_form_field_behaviours.schema.json | 17 -------- airflow/www/static/css/connection.css | 23 ----------- airflow/www/static/js/connection_form.js | 39 +------------------ .../www/templates/airflow/conn_create.html | 1 - airflow/www/templates/airflow/conn_edit.html | 1 - airflow/www/webpack.config.js | 1 - .../src/airflow/providers/http/hooks/http.py | 4 -- 7 files changed, 1 insertion(+), 85 deletions(-) delete mode 100644 airflow/www/static/css/connection.css diff --git a/airflow/customized_form_field_behaviours.schema.json b/airflow/customized_form_field_behaviours.schema.json index 8aa05945ebb01..78791a87886c1 100644 --- a/airflow/customized_form_field_behaviours.schema.json +++ b/airflow/customized_form_field_behaviours.schema.json @@ -22,23 +22,6 @@ "additionalProperties": { "type": "string" } - }, - "collapsible_fields": { - "description": "List of collapsed fields for the hook, with their properties.", - "type": "object", - "patternProperties": { - "\"^.*$\"": { - "description": "Name of the field to enable collapsing.", - "type": "object", - "properties": { - "expanded": { - "description": "Set the default state of the field as expanded.", - "default": true, - "type": "boolean" - } - } - } - } } }, "additionalProperties": true, diff --git a/airflow/www/static/css/connection.css b/airflow/www/static/css/connection.css deleted file mode 100644 index 78edf0db5d4dc..0000000000000 --- a/airflow/www/static/css/connection.css +++ /dev/null @@ -1,23 +0,0 @@ -/*! - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -.panel-invisible { - margin: 0; - border: 0; -} diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index 5c60638caf488..d039fc7275462 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -88,23 +88,6 @@ function restoreFieldBehaviours() { elem.placeholder = ""; elem.parentElement.parentElement.classList.remove("hide"); }); - - Array.from(document.querySelectorAll(".form-panel")).forEach((elem) => { - elem.parentElement.parentElement.classList.remove("hide"); - - elem.classList.add("panel-invisible"); - const panelHeader = elem.children[0]; - panelHeader.classList.add("hidden"); - panelHeader.firstElementChild.firstElementChild.setAttribute( - "aria-expanded", - "true" - ); - - const collapsible = elem.children[1]; - collapsible.setAttribute("aria-expanded", "true"); - collapsible.classList.add("in"); - collapsible.style.height = null; - }); } /** @@ -118,7 +101,7 @@ function applyFieldBehaviours(connection) { if (Array.isArray(connection.hidden_fields)) { connection.hidden_fields.forEach((field) => { document - .querySelector(`label[for='${field}']`) + .getElementById(field) .parentElement.parentElement.classList.add("hide"); }); } @@ -137,26 +120,6 @@ function applyFieldBehaviours(connection) { document.getElementById(field).placeholder = placeholder; }); } - - if (connection.collapsible_fields) { - Object.entries(connection.collapsible_fields).forEach((entry) => { - const [field, properties] = entry; - - const collapsibleController = document.getElementById( - `control_collapsible_${field}` - ); - const panelHeader = collapsibleController.parentElement.parentElement; - panelHeader.classList.remove("hidden"); - panelHeader.parentElement.classList.remove("panel-invisible"); - - if (properties.expanded === false) { - const collapsible = document.getElementById(`collapsible_${field}`); - collapsible.classList.remove("in"); - collapsible.setAttribute("aria-expanded", "false"); - collapsibleController.setAttribute("aria-expanded", "false"); - } - }); - } } } diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index 307450b05d16b..fb3e188949b66 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -25,7 +25,6 @@ - {# required for codemirror #} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 11ebd6c4cb436..653b0dd3ce07f 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -25,7 +25,6 @@ - {# required for codemirror #} diff --git a/airflow/www/webpack.config.js b/airflow/www/webpack.config.js index ad1a7098e0803..9d5800f783f50 100644 --- a/airflow/www/webpack.config.js +++ b/airflow/www/webpack.config.js @@ -60,7 +60,6 @@ const config = { airflowDefaultTheme: `${CSS_DIR}/bootstrap-theme.css`, connectionForm: `${JS_DIR}/connection_form.js`, chart: [`${CSS_DIR}/chart.css`], - connection: [`${CSS_DIR}/connection.css`], dag: `${JS_DIR}/dag.js`, dagDependencies: `${JS_DIR}/dag_dependencies.js`, dags: [`${CSS_DIR}/dags.css`, `${JS_DIR}/dags.js`], diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 192e57c1d70fc..77563d2eb1a81 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -132,10 +132,6 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } - @classmethod - def get_ui_field_behaviour(cls) -> dict[str, Any]: - return {"hidden_fields": [], "relabeling": {}, "collapsible_fields": {"extra": {"expanded": False}}} - # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: From 21a29ec278b5795862a7642db31eaf6b40173fab Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 5 Jan 2024 08:47:09 +0100 Subject: [PATCH 102/286] fix: set the default value for "auth_type" as empty string SelectField expects a string as value. The default of select choice cannot be None. --- providers/http/src/airflow/providers/http/hooks/http.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 77563d2eb1a81..9cf5d4983dbc5 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -120,13 +120,14 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: from flask_babel import lazy_gettext from wtforms.fields import SelectField, TextAreaField - default_auth_type = frozenset({""}) - auth_types_choices = default_auth_type | get_auth_types() + default_auth_type: str = "" + auth_types_choices = frozenset({default_auth_type}) | get_auth_types() return { "auth_type": SelectField( lazy_gettext("Auth type"), choices=[(clazz, clazz) for clazz in auth_types_choices], widget=Select2Widget(), + default=default_auth_type ), "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), From 4f074842eca61e58d3b3c14915a2fc0017576ff0 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 5 Jan 2024 08:48:11 +0100 Subject: [PATCH 103/286] fix: Use Livy hook to test invalid extra removal --- tests/www/views/test_views_connection.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/www/views/test_views_connection.py b/tests/www/views/test_views_connection.py index f9a4efd11c15b..1e21dc4856ed1 100644 --- a/tests/www/views/test_views_connection.py +++ b/tests/www/views/test_views_connection.py @@ -459,8 +459,12 @@ def test_process_form_invalid_extra_removed(admin_client): """ Test that when an invalid json `extra` is passed in the form, it is removed and _not_ saved over the existing extras. + + Note: This can only be tested with a Hook which does not have any custom fields (otherwise + the custom fields override the extra data when editing a Connection). Thus, this is currently + tested with livy. """ - conn_details = {"conn_id": "test_conn", "conn_type": "http"} + conn_details = {"conn_id": "test_conn", "conn_type": "livy"} conn = Connection(**conn_details, extra='{"foo": "bar"}') conn.id = 1 From 88f53c14e5d29a79959f29298b781bb4c0235d95 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Mon, 8 Jan 2024 16:40:49 +0100 Subject: [PATCH 104/286] feat: Implement CodeMirrorField for providers --- airflow/config_templates/default_webserver_config.py | 7 +++++++ airflow/utils/json.py | 10 ++++++++++ airflow/www/app.py | 3 +++ airflow/www/templates/airflow/conn_create.html | 1 + airflow/www/templates/airflow/conn_edit.html | 1 + providers/http/provider.yaml | 3 +-- setup.cfg | 0 7 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 setup.cfg diff --git a/airflow/config_templates/default_webserver_config.py b/airflow/config_templates/default_webserver_config.py index 4ad8ee6743f39..85b9d4d2c8dbb 100644 --- a/airflow/config_templates/default_webserver_config.py +++ b/airflow/config_templates/default_webserver_config.py @@ -35,6 +35,13 @@ WTF_CSRF_ENABLED = True WTF_CSRF_TIME_LIMIT = None +# Flask CodeMirror config +CODEMIRROR_LANGUAGES = ["javascript"] +# CODEMIRROR_THEME = '3024-day' +# CODEMIRROR_ADDONS = ( +# ('ADDON_DIR','ADDON_NAME'), +# ) + # ---------------------------------------------------- # AUTHENTICATION CONFIG (specific to FAB auth manager) # ---------------------------------------------------- diff --git a/airflow/utils/json.py b/airflow/utils/json.py index a8846282899f3..9622f4c0a6b30 100644 --- a/airflow/utils/json.py +++ b/airflow/utils/json.py @@ -123,5 +123,15 @@ def orm_object_hook(dct: dict) -> object: return deserialize(dct, False) +def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> dict | None: + """Safely loads JSON. + + Returns None by default if the given object is None. + """ + if obj is not None: + return json.loads(obj) + return default + + # backwards compatibility AirflowJsonEncoder = WebEncoder diff --git a/airflow/www/app.py b/airflow/www/app.py index 91b23875dcda1..0b38a1fb1dd7a 100644 --- a/airflow/www/app.py +++ b/airflow/www/app.py @@ -22,6 +22,7 @@ from flask import Flask from flask_appbuilder import SQLA +from flask_codemirror import CodeMirror from flask_wtf.csrf import CSRFProtect from markupsafe import Markup from sqlalchemy.engine.url import make_url @@ -125,6 +126,8 @@ def create_app(config=None, testing=False): csrf.init_app(flask_app) + CodeMirror(flask_app) + init_wsgi_middleware(flask_app) db = SQLA() diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index fb3e188949b66..8e3d8db0d5e00 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -26,6 +26,7 @@ {# required for codemirror #} +{{ codemirror.include_codemirror() }} {% endblock %} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 653b0dd3ce07f..174bfa164c4c4 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -26,6 +26,7 @@ {# required for codemirror #} +{{ codemirror.include_codemirror() }} {% endblock %} diff --git a/providers/http/provider.yaml b/providers/http/provider.yaml index 7a991044ea801..bb0214e7c6e66 100644 --- a/providers/http/provider.yaml +++ b/providers/http/provider.yaml @@ -102,8 +102,7 @@ config: description: | A comma separated list of auth_type classes, which can be used to configure Http Connections in Airflow's UI. This list restricts which - classes can be arbitrary imported, and protects from dependency - injections. + classes can be arbitrary imported to prevent dependency injections. type: string version_added: 4.8.0 example: "requests_kerberos.HTTPKerberosAuth,any.other.custom.HTTPAuth" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000000..e69de29bb2d1d From 2c4169d65223e636f2301088285ae467abedb67d Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Mon, 8 Jan 2024 19:21:34 +0100 Subject: [PATCH 105/286] revert: Remove CodeMirror from providers --- airflow/config_templates/default_webserver_config.py | 7 ------- airflow/www/app.py | 3 --- airflow/www/templates/airflow/conn_create.html | 1 - airflow/www/templates/airflow/conn_edit.html | 1 - 4 files changed, 12 deletions(-) diff --git a/airflow/config_templates/default_webserver_config.py b/airflow/config_templates/default_webserver_config.py index 85b9d4d2c8dbb..4ad8ee6743f39 100644 --- a/airflow/config_templates/default_webserver_config.py +++ b/airflow/config_templates/default_webserver_config.py @@ -35,13 +35,6 @@ WTF_CSRF_ENABLED = True WTF_CSRF_TIME_LIMIT = None -# Flask CodeMirror config -CODEMIRROR_LANGUAGES = ["javascript"] -# CODEMIRROR_THEME = '3024-day' -# CODEMIRROR_ADDONS = ( -# ('ADDON_DIR','ADDON_NAME'), -# ) - # ---------------------------------------------------- # AUTHENTICATION CONFIG (specific to FAB auth manager) # ---------------------------------------------------- diff --git a/airflow/www/app.py b/airflow/www/app.py index 0b38a1fb1dd7a..91b23875dcda1 100644 --- a/airflow/www/app.py +++ b/airflow/www/app.py @@ -22,7 +22,6 @@ from flask import Flask from flask_appbuilder import SQLA -from flask_codemirror import CodeMirror from flask_wtf.csrf import CSRFProtect from markupsafe import Markup from sqlalchemy.engine.url import make_url @@ -126,8 +125,6 @@ def create_app(config=None, testing=False): csrf.init_app(flask_app) - CodeMirror(flask_app) - init_wsgi_middleware(flask_app) db = SQLA() diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index 8e3d8db0d5e00..fb3e188949b66 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -26,7 +26,6 @@ {# required for codemirror #} -{{ codemirror.include_codemirror() }} {% endblock %} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 174bfa164c4c4..653b0dd3ce07f 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -26,7 +26,6 @@ {# required for codemirror #} -{{ codemirror.include_codemirror() }} {% endblock %} From 5bbcbe31363a8aa5ee61ce38791c88eb5ab8757a Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Mon, 8 Jan 2024 20:47:50 +0100 Subject: [PATCH 106/286] feat: Add documentation --- airflow/utils/json.py | 2 +- .../img/connection_auth_kwargs.png | Bin 0 -> 9623 bytes .../img/connection_auth_type.png | Bin 0 -> 14199 bytes .../img/connection_headers.png | Bin 0 -> 5256 bytes .../img/connection_username_password.png | Bin 0 -> 4761 bytes 5 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 docs/apache-airflow-providers-http/img/connection_auth_kwargs.png create mode 100644 docs/apache-airflow-providers-http/img/connection_auth_type.png create mode 100644 docs/apache-airflow-providers-http/img/connection_headers.png create mode 100644 docs/apache-airflow-providers-http/img/connection_username_password.png diff --git a/airflow/utils/json.py b/airflow/utils/json.py index 9622f4c0a6b30..eb3cd40941197 100644 --- a/airflow/utils/json.py +++ b/airflow/utils/json.py @@ -123,7 +123,7 @@ def orm_object_hook(dct: dict) -> object: return deserialize(dct, False) -def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> dict | None: +def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> Any: """Safely loads JSON. Returns None by default if the given object is None. diff --git a/docs/apache-airflow-providers-http/img/connection_auth_kwargs.png b/docs/apache-airflow-providers-http/img/connection_auth_kwargs.png new file mode 100644 index 0000000000000000000000000000000000000000..7023c3a7a072f965f9dd053f77c5a5d64b396f47 GIT binary patch literal 9623 zcmcI~2{_c>+jom(n~(ixI5hS2r&k-xWWM^15qC|U8&@yI zp4VbIoNz&&;S|%$vllM#zw{D3bLeCYFTFO4DogYQZ3Q#Q(MB#`@|PP{w$;0sjC|gQ zwZ00``u#3Nh5MrM%WwU6MOC!GEa0bjzF~#+9_k$RUt@1t_`%yIg(IZq`(>tL>=B>- z{$n_3=<@gXa@sCopHqhy8ve(KHqs61*aGb{L`~`|6OlkDb>1ArkdEJf(u%A*3sMN|@KU37UWK z`S!;+d!t~o=`Y)wx6bxn)?k%F?VSe!d^L0ZZ$r)AA%p$*$)o=^r})M@ajS#xngony z*Lg06eEks5LtY{{Vqn>ksZNFGX*0{b@+ z#0=Qog@Hj;9>jhwU?syRst>MH=7w=oH_sMKnFK}Mwee}-tenP+6yF*CgOf6#l9m~nV&Lm0k^g3ly*;YJZL)1?XSHiuScftr z#fh-AL>l<*5_9^a$4@YkHx|3Pz$brZ?SLRnt1aptkF^SCGF4So9T@qEx}W?S9Hhje zjy(hdWv_M_&Bzq|=dy@X#CzWgO-|?oQ*sLLA1p%)S2ocGlYcv~*k1^Hr`qWbrW`D1 ze39xdiiP$)c~TF*B%`J~(|0HvDL6}UXmC$AX=?Uck=2jS&$IIKgwo{Qg;dPIWDXvG zMpEM11PV^VaExN#`GO17fs{SQ=DebuB3m@@iB@cQBh_QnXofNT+W;eRTnE;5qA}H_ z{zc{+2E+#+k7QZLTNt<2>=yczPV0#hS6B%9kd3{-YcCyL!Ul_+k; zp0DzSG_^E_A8QO|+-h}b+8Fg8y{FmTOUkQv9CZ=kv~^}^Lc*}YvMAAFoY;}%=|t1SKU^tU@;Fdz77c@R*1UA%Eu_Y3tz$}<))Yv z-3pd<>}&eTERFRZ4--c09NoKpl|RsR_t%vKf8p&x_J~HJ$(xT zo$oS|$Z5$+e80CnXSF^*nB_j*ndQH~H}>(#(cCVJdOvxk)i2Uz0R!(`MoYbx$5G&7 zbo(K0GjYpF_4f8Q`5Qg(SozU`tNz$D_S>4h8^zdEijt6gb!toDK<=$u;A)fy!Y`h(--=f&9P7^^!9I4 zS6lByogoK?=((~MUPch>kn9^#u;HhK@Ta@&)yh^MbQOJm+xd0L-}4_>pNLW6Rf${5 zn7B7A{qPGNpZ|_4r;-v{iFBo2Xh>#+3G;|{E(gSV34$r{E%Y!9`-0-)9E>o*$tJE)bX!7Ho6BjelTH=PL$J?c1 ztcS}V=!m@Z84KnZ_Y8M`u0~g~R_#1AF0&rzJU`m7zjtE99k_*)p*%{1*Zcfkv+5{6 zR8O`>jGTCSrfnv=TK~22s0ZyRx?UYb5c6=Y2V+m39wWHzVWSoL=?cN?3qvg)87ltu zzh9h(5?Ru&?C-2D*OsZIjn3QJB{cg>n)^q~LN{gBOxrEsTCs>YYEnTcVzXZ93bGh0 zzrFi`%hISd+fZlbYVcBS(%oG>bOD1C|O-DZ0gNyeZ7| z;uZHP-86$>`0iguVGr-XX5S>WNe=s()OWpcM50@!i6`t8{wEP90XuQUY@7_G<` zMX#lM_TFlzVc))-lM#s5Zg<<^h7V_c#6Zqm$~pbCRB0|7%Gw{i@E|bt-6%$}x^(MT z-Qu_`f0Gqj1Z!{SyJoQ}*|2k|Qd9r z>FA|&f!~@IC8ok-O}>jCIHvPUqjzf_FCfwq>*MTI=x)nt{8ZFBRE`|3^>UW9s6CB; z+3O(2M`ZC5LsVYC9G_8Wt~q;JO}+n&`_z}P(KFTfvQsHJbB!C-1>(qCCFW<61a)mJ zfOn$ExQ(<4t2>BmWijT^^%ryB)2zikhe00HJII3zq&#Kg*d$5IyFLcNnc?t;1aLJx zoJ_6;UlJf$ak1T*Vm)Q#Qln0~QLgivXwu@@_BCrF#V znBPwJa3cfanoY|cE6bmaJ@LQff|QJNDw<>XEWazOvKK!+1>2;7y$rtLFc@1}@5Cq^ z5Oh90X3=-{nICF79SBuUIwF7??)EGxo70_>P)j;b9!5H#)Hl_1 zveiRRBuu#?H!9SwzaroBq4NfU|RlwHPbf$H1%Avp5&z!DSXF0SkUDbIQ zN5eK3l*mwE#zO9}ZBKBzYRHFZBD9^0n9#M-o()i^yOx{safWYyQzNa#c6BHxXF}Ja z=TzVIm+=V@q@7h6h!}-c`>^rC(p-m*cP7Kja zXnQ)RsZ+5HuBV_@p$s+68ylhGc+%wfh=gMoM5{ix-n#blmFo4>aIpk}F%GGZQlEpa z3wlgv9Xm2)uZ6;2Eq1EQT8`gz3s5uOC*K-(z)}<-x^bclCkeB(~dQ^KZsXG93)}Y@3u(u2tR+;ef__lr)x~c zs@fSnBczc+RPD}83uSqz>jXlZFN{@i>ojr-JW{l0sO6iiBS_stR=1xb>wi>xSK!1U zL2LdH!5&D=3-{dfOHz~^B!x^?4C$#}Wk7NbjRY>)20)`_Rah=>~yAj`Jqb1^m7l^l&k_AB-56!+#;7VCgf_!uFmkuic!7h?*gZt{)_2! z_c`Me!sS*?m7~g@XM9#~KLmRnseyl4WPAbV2*VohK#Q(PPtg}6O__^XUmF|M=ox4_ z=%bFpNYb{Qkrt(|FI1%aC8ceb?mr5CW!NDnU+Hzvvb;Ewb1u|6KlmS)o9_9`>qjTe z>2Sb~h^;ekty=tgZxaZOj%!mK5;{L2XR53y=c+y=nxBOWYq!@&;XMU942~iy{FPe6 zyH-AoE3SQaeEVrK6-tzPE!`imsL7?9x|XY91sBHJ&nU2JU$VP2Oh>`kp37-@+oHC zk8(4QJ%7dN;5ZA?yJa_tui}60sE-N^DUojZym`q_c#yXxfwn!* zgmwDa&!7ie?kNx?9j05>#_w!&(xPXh0=JQ?~35`m$S9)v?zSr?L@Jvwk6s^tZy$#4l(*9EbNxA;-XvBZZ zJM4>a9U6S-(>`mn37JGEW``@B@@_U>)9IcoaYH#25(MHuq%}@Bv>r`vc9*gF{Hzo_ ztrjf-1^GrHd~Esv&v8iJWujnQmEGS)rsXzTrWCmTT&M=K>$5pq^mZjKbwRcG_#i0{ zfE=s+z1^`uX62_(G^}}hy}enmDimbd2;eNfz$pLz)4-zJZ$tX;`ilQv992nE^em~| zsdDTeuea4{iRPJbZCL9w%c`!f9+@nk{Lvbh>j46?{%Ox(%zI{)9jQk@y9HHhY_?V= zmn%Y<`8vD8TfSzghk8XOtN3F~Yup}S-6pknm{tX~42Nu}xxeoOa&sZu%@1&;QipGa zr4Kt&cFsmQbnH4uJE2wwtYbbANF-d~*m4VBm-tpDZ^(?k zHMx6NMFkVJ4EgcPr!Y6`!COlszKg+~>58EXL+|?UM0ZK6SRzO3EfB(V#u>i9YB&iy z+jDO{rep=N-anpeGyg?7 z=xZ?-BKBfEud<()7eI@R0KDQUtRpZ6fVyUH}^g+ zA&J<5vq7al<_9Y_;67axeB6J>$cR-vbfP7u<>%lUw;YO}iBpPyexU4}#1II8f$xD* z$AX~q(iWkn%&HyJnf`ab-?)rD1Hk;|O1mWf`8kco9Id$V%C`-B3crMLH}0aRAIuQ4 ztolnVawp}4`j*=z8=5wP&H~yPNyaw}mLODimeiA#(LPLa1Ji&3C92VPegiJXgD9V{ zZP=Phl}l%rr8$E|jj4If7UYj_jSVQ1=PL-sX79Xsg>;iy_O`nGnp>UhN<&x(wq9ICsLozy*^cqcG&t(C$ zw(3eS6H>X((`ZL!e|Nv}%^m60idosmUy?UptWNLS35jbtq*4&9VH)fg4QGKFLxO$d zv0A1l;Rzf&M80I35<5}#UW z07qEE3+8>ihCzIrKDSjVSYl0kzd0V^s(hHis&i{~dP3QGuw3x>{GfETq3UI<0E++S zeScG4ZpkOQHS9cV=}930(7W#D6lmPP)D*^=r4hqRDFwI=mRYxi05;jOOES&ZOT9Jw z_}Uz2Vy+JR&;;ke@mjCl&50P-T^hT^dNMMt(_5HxoZA{70@?o>n`h3iK9?27D%{); z5l?FI{^1wEe#!dRu&)gzBq=$wj; z)iaW6`y2jzOc_i2*3r-dWV2@DpJ^Ya>d0TpbL3eNfRzC+LN9{H6b>TpMTfMz6PM$tC8tK0pe9(V zWZi{A+S51rlxIFrf+EDM06iHSw9ORnz`wauasS>?#Fa9pwquuw*@W?@sHY^sI_~}i zf}oipHjV&;k>wHRa<4Mz(Ie@Ke|7o!o9O;>34 zkKrMUHE3o}eR(nbFfi5OwW+4zR|;5y&C(Aij$p_OhEj1n*RbZOILHY*3;eWdQLLMF zIzIqXr-UP15p;Z4=lG5sJ|a+S{f`0U?8DygI7ok#>peObJ9_ox*Z{!*NTTR%I!8Jr zoohK=y(ypFaQtG1%>AaP-S--3_=Bu3aC~Or`dS>`r}LBijlc;X{AF#f5s)&^kDF(H zq63J2NT1$3#}L2nE4Gt|6DWm>PsYonhr1tA&mW0{Uv>`Pmp~YAh%l~BcSQ93e3!w* zCC9HnA;P8}!W8}1Xs~`a-4G?cF#=hbf{)M**Lgs$_&#EGnM7t(tl&Btk1>w>D)$sS zCJ`1#^!{;avbEK9q&!z=!WWsM;4#l~BO1HkuY-Inte?z+fBkC%xn#`vMJ>rS$9dpF}PO3F$M%_W;tTB|<`*t{}BY>T@%ppMeYVE~+I1h-1NxRg>A zg5`xDhwt((K#!h4BD{vHbfy*bUP2j>bZVPSi=GQU@XHJOIEX;Y=jZ3f73fqi3~Hbs zJ_v*!iFJDV`}gmk%UAppegu^@#0vx-|Fl)`jWBCcG+xhI$Wk&-tT`ukxAL_Nv&IR& z=tc7SPo5i0e7w`=s<^n|k9D0IGmaEgMs+O?Cok`{w#JRSN?4TVoIa!#J6o7X3c+NtLyj1m2J54Ku8NfltG`MtV znaa}wplZ*Z`o;I;)LyE!LlFt3=GB6X*25^3zua(;o(%7)7n>=Bz{7r9$kLJ*(zeYv zBYZg!s$ZR9A{?Jf#XI*wHYbwYy#1mR6_{AkouA{cF!s>rP(b)1q0|aLZrx|JApsah zY*OcV^+l(Pow|*r2WWdhp~nJYaEk%? zc(l&jd1Ip?ShzVxwAiKnL#p-ESB9yku(uPAh#?l5GUlDhSoN322%dY{mTZ{FZwcWRVk?*D1 zZD}EU3|?6}w%<>4@ooLK!~Clz7o(bcdGa_&gXAOOvnbNLKy5{fSDQSY$Khg08TCz9 zTi*ocTRh2E|2i8o%C?8av8LM5$JR2-vJ+)wxk^`8sE#>?=i+bHRzc42X}8u~ar43Eg8c3QY!gT3r0AERgA;z=Q0 z`NE*^n^T?j3fHmX3l|MKNnS^Dj`rBR*cdFc-_u1&qqi%&ZSQ(-P0B44ziabM(grE^ z!DiAeNi7D)na=C|qP5a!^1?e;uEXu+4>r`fk!P1@i=gO%o>vY;(}53Tu0M zToR*oBGz2u(Wj2Qa6;iNrxf}hh4;)P%5IHBU50lLPJNHfAr}G<^paXNj6c(#@6N7n zr$*>U*E|5%iz|Qmq418iOeb%$aEsr4k3D_16oWvrY!ecm-n0Gs>Ux>ZsfPBbV{pio z6YrISEJVD7dpGrPdg@skj-Ny1 z{7ovWvmTr*cA`t6Gw+`G`19@!xp>6y(>&spb^@U8{XIJT7(A5OpDQqQZ@OAYZm4KD3SsW($Znu zGtq6Y-#O7U$fsp!K@0bn!q&v7G!o7}XbhrvEw!%YbVb|I$Dz-Ae*L;Fsu=R@yjCHG zjS-pQi$pRBalV#}D6>jdUMnE)ezRxq^ldz&#(}6KbPGa>OuK0Q(FD=kO|CR;!RJ7N z!`7U$%OdyuG|C^}iGu#}&v5%a^IUEr{oSa9p3d-xYUg3D(U>#4UQC&EcLzKXp9#f< z#Cof#$TO}A;vNIHX$3oJ;n2(PPmN07yQG%}|FGx76v_FV!l_0)4-Y9 zbZ(yy2$hl0RkKsOEr@u@@})EoC@9erb-#b^YBISA>7}u#NA_lB@Mr5xrwnWd;ey+# z20lZ)kK|L4s8<2ycE2u6F4Fv`y26p~))|+6R=8Dr^*- z0)Ui`ajGj~xM}9rbq6JwpbIDxr z1LgG^!M34#v@h0uTAwQN9FW{JDl zwK;rq6m-JzND9E@Az(+%w_tn#IHZ%NLisRsJ^2RA7PXJ)!SoQFBne2v-nL^}5;Kwu z!LImtp6)0u6F^R$F#=ye1-}vOcS0%PJO>vawYnOtUl{A8dato{)<1Qb^%U#?-d(A^kiECYHjx0rcd9SJ7b{! z96pARQQ6}ZGRQC&#%4{n0d>qeBAz0K0tqb`SkvsiMas*105&;C05l&37UD#`U;plS zkz`u$=bdK2-wlW$YHw$20A>eHGz(~1xg|P4*m&JD{ty)8tw6bcSE_ zMU`0^_8k2*IdeY&@s&mUpaz!pDjj4V33WB}NdWpqm&*MT1praXlYM8j_^#2jHgpcjBpYOx2IWu@IqX04SL z9wn~mzqeyrKGm6S#(-oOGi5RRkt1Oq8HKNHKL)pdIVnRegA))!qI|@hXA!GQQdB1u zQ_Dp9vNZ6>>v*g*5QGXQnQ$E-e8(M|y;?CL8sP^!$YPvmu%YS}M1nI68ZIT}ipD_z z<6&R_HEiY+_&?mxSyv_W)t<$uuEafyjWcL~qm^d{zNFfORVpsdZ&4t2i|UnX^n zc@C8S32ILd(FYlJDXr?Jlkx;9RvFVAoLVAjYGtzh;(LJ{k}GNccJ<^JDh$}n6Fohq zwawFU91anWxNV7+{r$5?SXnQ;pGHwh8A>DMlzA9KA2ZWjW%u5XuA=^%L8%+hdNA+KUxJ+NQ^~; zTn9UHP(jy0^`zM4#$I$mj(&?!TSuq#wsw~CIzANk)`;z1NF&A3EomfU0tp(xzd z2ZcTVDiW>l`g~X*OFz&oTQJg7kzk+Sm8sM2upK1z0}1W_Bs2f_)jcE%Uk(~1PJpBb z;gB%@-1RSw06?<831r+ngG}CA#fXIes>*|VZUs2-Rx8-B;om}jF0P}m*bEr=Onkuk8g0E@GN8TH(>+$dxn|e`V?;qmYie-%1%QuKYo} zTHWJ?Mv(qiDu63;ZrM_cbD)O-WOK$3K5Gm~5*BR-9R~*&jIxmP5IZpcKlL$yg*J^v zz)AeA@PpaC=ZfDQxY6Vw(ZrJSs;e)5$+)5@cBlOOfi0X=TV&u^t3gWDPBoA=7(7ER zQ(FfL|KmeSgG(WWbN`Rs8w9{eDN`;Vv=|hht9ZJrw?>o5+so|!QQ4NeB${+x0%*88q;*~Y KT8Y}dfd2)j<)#Y& literal 0 HcmV?d00001 diff --git a/docs/apache-airflow-providers-http/img/connection_auth_type.png b/docs/apache-airflow-providers-http/img/connection_auth_type.png new file mode 100644 index 0000000000000000000000000000000000000000..52eb584e5ccf6463273c9b0d35171944d451af9e GIT binary patch literal 14199 zcmch8XIPWlwl2%HEh~tXqEt~qIwDQF282i_fCvGiAiYU%0u~|)Qlt}_l+cUx63P;! z6RNa8U?H85AP_(Zf%_%6_Bnf>bI)_{k9+^{0QtT-#~gFaG2eHLH}7?|)fhmmAUZla zhKCRpC>`BD5Ww%PXHEe>*-vEj)6q#QK2*7H;A2j}h4>qeWG!xxl`vB)-qyt4sieB$ zK!I^4{_6to9tVb=yr>fRj*oBX{%=hGK(XWx`L7FCMn$QE{n{j#%tES+jZ;z>wWBLy z)|Y&zJ5tO8c4S2zN-5sF;b`ymw>G~46VlOrV)=>wah3~pf%ezEbrt@9(0&^@tq1yz z&SF6ae1<#t0(k%VoqP=R-39QOm9E&Dc(-5w)ot1MK?$B)ipa)5&(OV0dve4x-+#J} zB|8atk~=s`G4{FqT4dbO+0(-W>d2h}*iREaZ1QOCh6GLh?;mc517iaZl3If2%-h1l zD4Cs1${fNWe1K1kTjTDRT-en3)V}zPm6+SKa(|U(%9$0!Mn{7)`&xTj;C)6W<=vi) zolM3Pbk9y2ote1PDHv>eF+1GCX)ZlyF|dG*IAL&+X6#VdTzezy?+)@swvKC{%slpb zrFbGljB3@g>)}~$r`{=%rzK~HPU*6J?cVp#x&9IDxvS}W-Ky%BiHEjpg<_xETUakU zI2MVy8Jw##>yf={`3_k4z}$f`*x4&8r*jp4zw*m)yAE56ToooKJ77@F?n<-w}U^m}xbX2m=)o+xvHvuLI6^WI#x3k`GC@K=GXzz0#)m5{mr@G%S7JkMK z>+txD-5Vq1_)jLoThg( z7wLwInsm|}hEu?K*|K}0JT(H*Y+nn#Sut@#s_yyOgQh~xw^=U9tmjg<8Q8ol9c+{K zMTKIe|7PSGi!!T5&3L#F`-{`mjZ)wp5#lDW5Lm!9g2}xZQ?lZ|-buazT4lagh8L;9 zbA~%W>ey@L?cIT5*=3dO-w3I~G}nveTmNDAMV`P_yz+T}?7Y)jZC2)BwUIoqYz1cL4D z?KR!qE9$)0IsdjnOH0cO*t*Sx_hsc?Jj2JM5F9he4m59bqhMIouVF&mt~f7Xu~Qm>K>&E8%#QO46*8mprfkZQUl<|REZ^47lDpg(M^^u zf6Y8J95}o`MmtpJZ9f#1mACh@$~l}ZE~{+*%L9f7(RNqG0X2ZD_s{uxSYj#L@>U|KplpF}}~g!L$O zDK#Zg?W3_^x|c-@>fGM*Vn(32k!f*+Vb^xB2&1$v5KeS-{I$?!3ya>OvWoVBr~KbW zFLu&4utgN%;qecr+5%z%_#`j{9o;>X62_s#2@$E(E6NT+#bS~`g1AhA)8;^&kVz$jkGX}Ji=?z^CT&{zT%^0!USH2>Qtg;DQEn9`?751^YuVZ5Pkes*W^W+@ zekpPiYrXMGGY$*{Om(ciqvKi8a*})`yUe9Dr|zB}!W`Z=SDCu@0U$>CB~~#XtG`~o z>JQ7)5f(dalOq?JcFZR$#uoIgRO#6C}*EwY6Dekdix!B-8D2XStOGu#Ak1 zmvyX8+jvs|Idj;iKdmYl-91^idA1#w1g+%2NcD_-c#zYh zTc_ql8GHuMvi^)VPPpfYOTs3#1MCkE&J4^{ zSohs_x9*qfSK2M|-9Ri6@XsX@;Zaxa;>*(g%A5C|kjRVJk;1_Rx|b0v-B3tqXy{rm z2a^cWDyv0pY0qHFRy8u~j;gq=M;aVtmlHwk2;!_4^`7qfTHTFVbv3w8K!K`d-2r{dE{w1Clu9ak=bM!;GTt()^PhF4HYISblQOftp!vINoYwDf)+a${@;ZIXgOtQK!!Pb^=J}}0 zv)HtS&P4MJyx!ZXv-OJdKNlX(U_IB)u(OFndp)T6jjj^z9S3eIFTW%1O*DmN+rmDD zyExFxkWo?H%uHg%zTcpbcp+m8ZmQB3|LU#aAB+6ik@}kaySD4fwN@^mBF6 zon2*ssI50D0OyIf={|}BJJ$$fx<7`oh+>BiP3}cPL0Tz4eLa%l@gh}A#jpfFmh#Vx z(K7DVlUpLDcU{i0kGmWl62{;JE23Caio8ty;g{b}uldICQZSr%9tY)i_ka0|jcz~@ zNH7bFi&(QFsNSng<*#90T*J=hirzjx&5T2=^81rYDUC1~L_6i~(|A!hrx~xwmfGat zPCU3Hf};Y@mVae``*7zW>7XT3ZA`-uv==0YEym0Q^rpYM>8F({bDN3_{T=}~jNf%6 z;xaRDx44cH69dwWf4-k{$oa8sX)&}@Gyxdey5GUcEW-d5b!tS`WwwQ@x5!wWK(ryk zpW-Yv4=tb`0X01nEHS)EuV0@a?+#>$CTk>!RKFWd0*P{NHrZ29SV&moB;@2c zrG;`ZX(Pku&k2_j9@JYvb~CQtm%W}y>=@ivm4%mf3BN7I+s-;{gk9Tx>{pyw^`0bu zfXLb2H$>*28QRej|3rcVi~XtrEVd((s{+;KPWyeXNpjUz2^tYp4f-Clt+)x0?{ zRiXGd=c7QKY$3$dpL=@h8J9aEYwp6)VU04I4B`jdV{72S2>-K821K|<&^~{iZMD}M zcXXs3hTG^lt|`@}FI8dkJ3Rb1I>J~SDk?8n?|p^Oh6Sv^wmRHv!wvD?wmMSoW@C5s zNTHG1(US}DV1ei4ju zH#pej(G+pW!x zXR877dLv8m!{6wZ?*dpDNT6HgnT(+^;<-ALn`&WdWGIsk-OG|V0H8ca>59?VP?S0V zMQ$BK5ol*mk9Zifh{T&y=PKznL)4HNj3-_a6W&81_-z_mzA!Wy148M_r78w=6E{&% zKGR(YfUQ1}5h&e8ILJ^knQcL9v4sMKpl^h*snRG3c;8Jzb@Z{ZrFD3!W~-m{db1?;dSA|vm6i1uo0R;c z11)k``}jXW?(lrc>d`AOOQ-NU7!~L(i|qmjxr>szEqc{69FtSuG!Z){f&5;e8WSY# zOA-SDithaI-y1pl*u>I4yqJIldmTgf_%gL~XgFxXtP@F0@`HgSlR~ko?zshD7$1G3 z5&=58;(>pA87IUYUMIPuji8Z>Oy%yQd|XE~S$<4fDhd0RS3ha|Q=H*gaa)_&(9~$bwnlupzK$? z;WNuh+8bEOIhp^7_fSK};X`p5Enppj5m7_W&gV}&1NPxETlpz{eV9vuy*&oky&8*( zk_yG%TU=O3lJh1e0@v?x9Qm0^j5d1W<2lGV0&gWAjV2mf+Jt9F#*ZeNSUQB4;e60l z7BoIs42=V01Q1Ao%y@3UXD~6dXnhnj{-+^I2tg|45eTk)`ppSP`E_cOGnM-tdJD@+ z+yBe}&VW@%A2qC;G%~&t4qWd5hwD?v9@`Bgkewwe+|1H4E8oTTc9o~+dj@!Meo=uG zQ(PRs1~*0Hp{Hk6qXMv#1Ic#1cA?g`lRf|eW z`V4MK`NhQbIuxmvH zO$>Y+FDMYu*VB6n@DwWDq8nI32_r5Yr=wp{+&TfK9V{Rhd!MFs4&b zUY-%ya3<{#=o}_mX@BYdR>e8*;7;5ps zMlmrlQQ=EWLV}H(yE_KJ9~j_{{@aQFOqtY88yz>XbPTVOT=bfA3DPVP^Gl{qoD^lB z6UE#_Zv$Gr>f0MBNBfC8-awBx?5qi@J9C*!Vs>7yj%r3E;JB_+uWQO3g~| zBmuL5tIVDHRGBK(ZaR;1)Tc&&y+`%TSJvZo%21v1IE&^<*$}{o7aV~90FGOt4rvii z+k<6xswax9${A$?4{95QfzJn6sCBRB)RT3AK1m4S=H{mP^yyO{YLpHL0)CkIPak?H zP&D34MPM+P**5CfrJs&7f&E!_>W+Q|^J;JpdTy8UVDH%<+5bA%Fp1HTL}^3rn=}hM zJ!lIddD8x(g&EzQFxu?L_F7XT$bv@h={kj|_6o_X^OHms4^ofM^rhfmPajv0o>9r` z%g3J>uOHXmDrjCgfx2*97D{5EdW@M5aeN(gInum&k6-2Z{d!I3y#38Yd5NgRf`bz_qLWC~Q0<*87VZJL?7Y zIX?<70QD`Mt&x5D*$Ls^DZhz>-Db+gG$-Py1X6(|OzSA$-$cG-IHh(yGgP)><(t#i zT8%$UzL~{lP5MaQu0y$BZaArOF=bEMQcB6=FvZDe&7HmGJ4rdADo$;$m+FEcGME}` zh%^Dj;euLoa`Tbxu=0|PhB@r{cuIev&+-AWvy08j`rG#`fTl8^B?u!9*AwJ0mT*G( zL}2y68rtAe>S0f;*`iYT3~MT(bFjOVJ>CR~W9p@2e+k@_Ldq3lH4J`+*f)e=B{Fi>M^c62Rh;ForEx zUk~y@S8wvAU!`uUADp&_1F5J1m=dRW9ZGUBS&KzWd01gR#t)Qt6!C1yF_9E< zTi%ELnHcZHfP_#&QPS!@FYLo|py8SP&$S6YI;0c{JgN%+wp4GBB8v6IFdBWE9LrzY zT0LkTTo8Du2=nEYrF@h2U&H_GpQ}OMFj^mP_Czz>+I0G&YdmH<{jh;}P(JUUJ;Wm< zX^Mq`_KdxS`k9qy_e2DuRQiXNW*U!>vvcXCEfDi0@6NQ*IqgZGjSRAx>)o-~N2AF> zwUEh(9utlYaNqbT9n59(G%((@m z+z7*tdvny!gKdR@x#h}buS3nx7nRt>ybD$_N?KFS8d-t*F%^LaV-wijH9aL-z>=D1 zk)I)_!Hu~v7-<#v+rDe5nU4Wo{uQ66?TTBaiqp@gO z_!1xD2U8k7TG;}aT2t4HpFI!#XLPhDEkWJ*-rQy$@AFu{oIjbbBL_Fn&s8Ji-Y+e5 zUnD3l^fRh^<8BMQFo5J~?t5F-j9)LvtQNys@dJr^SRn1d_!fZ7tew7>9pQrsUd)lx zrTqAQB~q~X=Dcw$6abfy+iCMyrQm;g5B!kP0 zE6FAv#;_Uj1-;|L{I_rXH8SQM&to>9FWFUhq!5fuOm&ogc%}t@bzoq{^_lIArN@XP zu==3<4>Dc^?J)A8*Dz{Xp6Ptg`RNZLWO5|eS|4*fc%_6*RUB$?s&@Fu3hcJ{Avs`Z z5Zc*j?oA25wMEcO{Nc(J2hQ|-)dDd}01LnZCxTkzEu0s%9Z*r3(Lc6Ex~|Frmozzd zsc=G$=$C%rC+o*m8HTiW+ryt|lyGRKlq}0-jpg-gBT~>)0p|JY+)O%1j%*8P?hM?m zrH^$eV8!FVSsWUGJwJE)(JR8>HGRE|r2F63ySI94{0Y85g-7BBKNF!Kr{^?A@kzG> z>zat-{uko|TV;z{;q-D4Elk7BeHQL7Xi5f`BOf$TEugu4rSDv4o% z+4$4d*1pSbn9;i)sBgqeq@QDGOn$O33;QTm6z){}MkjA%tlVBeo+P#Q6;^*9bgJsp(tD6<#s)d5T#sK3G-B@#SLY};&Yl1Z zR2$s1#oZ)GEkzVm?~w?$r(%dYZHcv$4tf@iys6sTX^4}p<-bci=4UilR57}RWE;)q z_CH>Kh($f@(@bo}E5~(JV=)DT1eu zFVp>c7s~T7J=D~R7L(79t=^pTFS)L+CgYASSaFpMd$3ga72=J1RdH^7DSN0DSDN;w z#D8s{11%S8-L?!6WA;ik*p3-Gi<5;pH*C*|`TN)0-JT*CIX=z&&Nt%E%zW0*$ud5@ zJjlj@e_pWA`SxTVo>ZW z2FEB*9~IR+*rS@OohGr~wB-eu_45hI)_uP7BH7z04vat`7Iz4fQgBV=v-U_fmQ)cw zzR&4(asfMQ#;|2}IWHmEQh){nIMgC8J|1gY=Qg34^0V&h{^kWT&NUJ}zm!f`;Njs} z972Uz_a;VS*lxTf>y%D9;_645I{YH`Q{=Ayt3d$loNW#1#g{hPQ#(^>fvXSQ30O)0 zCNGMw)pAdDuMe3}MS8x7_+t6rI-f zBW<|+N4jk-VbMMu>ET>9HW)quRgdQV*`sGifnJZoIE^aIP0-E)^+#=C%t-Dxrzbi_ zwqyeK4Ls`9ql;nHD!DGx4R&xfpgbY0L8@_vByy;uEBBH5U4?XyeXcsL_%hgTS0 z`q?+>l5CR}*xe_A6!)*Dq=f?a&ooOTXMn0`(~pKENrxPY2zfeaUBRES-VA@zguH`l zv+_H7dzN$RyL_WG{bIk)UZXBQ318ZSneT#+FDXcDC`>oIz{4hg2Z|Upno5r|2D)bN zK@rB{Hsjz2ky$5f2mXA;d+Sz*gtT<>jga$`Tw^86gF^8-9T8QC_g)v>Q13zS0koh1 zC-7GK?>hU>V9g#XVQXAwoJEh^HlM++ElZA$jv|x9tyLD@d8KdK1ZSh2A&CAz6Hbi} z-CNHHt;_^k3J3_S32`>U@uS7z&g(sIp^){Dp_h$TgHcgU zIs$r!Qmz&r=CJi7k;w+T7z(@O+%nF4=6&e3W=5p6OG)a<>`OWo8LHua!$Rf_e10iY zUmccbO49>Gw^RLByAG?rx(&sFF9nJkfbxsvT)(S)KT0~aHLRnAT}s7N`~^_yF)DTc z3I>v*z+Hpu%R7BQzmnQA+AgtL6Jk%zJ@F~1`PLdC<@eJ&HE2d?*^do1F4cstZYIIZ zt@cRrH=sQySKo^<`S_zF2)OSJ&gFSLCrz*TE-vtlO|01R&zqNuZ{tfbYsM9s!-{dN z^-b2Z!Gn81p?-y&X`#v!(IlNYR+!n3DIan=+$W-Qiq^h3sPCq&_6@v!jcLlr^({FRjDgaG4>(>3#VFungj6;q$)n?k}mkc>mPt(!u zbJDW7hc|z=F>BLoX|r{ZN{Aj6PzEMlmo9-vsz(ERiGhIDjRs5X-DX=(ZVkO;A!weD z6FMZS&qf}3j4{|PzAwqb;UvT#4$jr%7GzC8`MMjJpU(9GK~TN?3_M=3wLx|*PJ|yp z@)2-iOno%yw(-4f+0U;Lc|VJ+S7MG4qh~pM(!*kOwhF-a?jt3zeN7Mep?6ChxLy?3 z4xh*7u2))b{9K5hK9Avblhp^=c|+7H5=rbN+282y&CpUTf+&J$F-tZ$u^i$hKs;T2 zbmItU3$JKoY@b)>jyroOqMpAjY2V1nB<$yGGN#(Xbp$|rq?A*ESxq|=r{56}0MgD= z5EuxtZVW@|`UK<_0xdlM@1plOK;B&LBpG4YP8bDUysIe@0|`^XcUoZy8V&|^zi=M-s4A;sOX$*T1;1rsBG${FT<*_FLG(*32Fi^tP=L&u-1+kM0cR5J&DC$3AbA zb$lE)q*f7n@Oq9=qnb1$?fgMxjHvAjI6)EtE{+2uGqSRxAJ*?`uKU-@jlMp)MR+}V ziKw_JN-E0A$cIGhw0|fo9btMbVySq@hvm>oFJvy?v7=n4-c$n@Y4_an3KT-Gg%4>E z8MaK9^X`kkzSN&ylc1CMBWcfL?bc4&*47s0bkvsC7Ta7jy*3jNhp`(c3l6?$5k z=4VNUNo+k>&j$$>Cuidpr}Rd0H23Zxzhsm3#TEg~4KXQwn5*At^ORr0g=p#yC>zM$ z)$w3r6e-ktQ^>6RYYEGOb23q3czc^e#&$sWn@Ko-2Tq zA$-e&bQ?!(L2Om~3^>M6`*HLl4$^$MY2nJE78SH){#|tdF{7f)C;L+62s0BVpcOw6 zKQ7hgqcjE9)ZtP8YWI$F|;t2j#oW$&Oq3^0AN#jT~vUssBB; z0dO6)y)qH;Ki3ccPk|D3%KztR?EiieJ~W3?V6Qy6cX)Q9^ZE-_0OwKrHUQ?!_y%a3 zFn}ndwF2E%mZhc@7}!KXXCYeC6wt87!*{c#$xMQQ6+uczQx%IF9f#3(C}}=ZMKCl_ zeWVD%n|fmt5)?cRU$Ve&U*>&b?F?X^@2(X5@*ZUy=ZzW6Qp?1Xg5iP2_UZjVQ??}x zOX5qv2!KV({|wC_<+oRCX9uRCd7nQIOf>~eE`~jf>HXk7vNP#25&cDP<2G@)Y^VIkA9cy#;qZ$rlS`T)ceFnke z&wgAZ*ZL^}(vVOQqle|5J@LyVOtLfqpI1^MW!e71@o1Egqm^n;AHtFr)WW!UPpZgGLP%*k2QOYL6z{t2lUxYidIpQj^?iUYjo_aX z{qRV^cT>e9X*poT=6lNBz$rj31^q4=n|w&wPikm5EuNe$sPr!m<;F-1MlSsqh}L&O z#_i{4N*N)P6Jr6S?8juw18qAy(i>BR3JevMw(j1qXi1e5t;56(-kuAXaGYtvnIA69 zcU4~)bSNq=mIOpa@BX6z7}Xv8N|fQVRrItnlO7dg1~f*OJiid9eJB)51)`cqJ8aaB zmd(a8#iYjB$-^H`tN$6~)m<;D0-Enp!p55wl*u?-{8(wlZsGb|p0pY^6EdR|C(RW^ z%GjIy`s`x%?swF0qF6Vzu>f+hp7lZ~(@r@*Tw7{5M=P9DZEQru6}YG0^$p|AZU;3{ z$XkD0?#)D;2i;id?gzR!q`<#Z6+|g*yVaz|oA-s3B2xIfEptSl{>Ln{4<2$$f4-s+ zL=Xn;d5|CZcb_kXbM58>@+mKmBy%*-k_T`BpgR1#@G@&xyMGv?3LKWd{uQ2q>zmZ0 zZp$_U%Bui4aK0Th#H0&v0S(SNBDQK+F_$VX+F}+pe`Yg56?&DoMNi8ueSLp2?q{<~N}iH0>+K6G|4+q()-t?5Ww z0>uGfu%lK-X?6RJ?1xYg=Imwp3n6blX-o3|jbT zfKhMahF>F=qu)%;D$47R$$C!*4~{r+dIaUBZ1IGdUvmKRRTg>I=D;>V5trU9^|7h- z&1r^8)W^JMBn|#Jgv(7uHnkZ~Gt7WZG)aucV`VeJ%D*EW>zZgm_kSVYfv&{{kX!TRvA1pP#c1X0u5<26+ zC60q8_xoB9N-C%UToLdD$&T*70?eNenC#`;dDhxV_5!lq= zE)LqFDs>iQCH!*$yZLBnZI)=64-lCM3uZt^x1CE0cus#;6kgr?f;1Si(>O%559CIz zf7RF7so|L&*J)Q-qJpvM4ggkVMXOWmd+Xg?Ztqj_0LA0UW59NVRSM{W%{8ZEQ1}T{{VdS^2f@WOxT$~Ql_ET!Tq)Km$ zN6#}evjm9D&fWD(Yo_cMQsjnd_ZsZ@UK>V5Ex*$@SyB_1t85?O3X=bMvD8=@fU%v- zY9^;u(M&)=%-YA6io;Ccs^ZG35if4kQ}Xq59qT@kN6-5b9?df|jylGkSGwYq9@G6h z$3ibqE$DS-17HLJHxj|rvsZZo=9n)4SEc52eD_Z^ zzZ1~CL-I*RaiHU@M*9qC!8V3BRhX=I5X~2hrB4F)s~OEd_xOXSRndDZQM-rE?*KT1 zdr!+Au7UuN127v)JXls;fLh?fSJp5Nz5M`uN=Ii4^H=BQIH2}!2eJSK2lgIH5TUNF zP92f%LWDfE`@ZbD0w@bUA3z}?C~wq2mqXW$ zD7&nV57fdN$@8CIfG~MxLKUu1wF1V1LT_83oyCIL49K8=l+PK_V=8f7W?hvVH)s{k z>{3-ri|}ws)tyNB9I;B64xb@EBQux^dY7dDLH#Im@~C45P^{6{)qMi!!ZAQKtrtKN z7(g)HTjekY03stTZS4q5oS;#rY14~e-^v;P%GFc9QQTh7x@G?#-)S*5HFfOhp7jGC zUOyI4T#N^Y!a*o3q8Gyq3d;ac7TUETkWwEm`0k)Kod39eK9TaAy|Ap9ZONW0q5;0H z;<#r02MX(1q@c!mnab>I9vXtGs_BjI!XQVNppPH_)D_FFO;Qeg_^ZLrKG$AF{VI$z zAO$x#W2c$o9IsnoMw4wAa|7bib?~A!y&Y_xMcKK1fU5IQ-=w%0AW+483D4Pw^L@~L@K5TQ%5qoBkcfTgBV@d z8$1Db-em8ucP&ZgnD@H6MW10%5iF78Tzr9AdnM(4W>lAibruW97F-z$94=vZ z6Av+cIG@RI>(~jdVM;lkbEe_GW`G6H3&0S|rJdeOv{(6p-o)BhM>SVR!Oz;66iK}> zf#|S-o3FX?d<>IHq3v?;SK-6N(QtROi`lk9w66j^KQZUp1NBX{^;>p6jE~D`dr=X&N!8)Q|&Rw z-nmQl&ZSP#*HgWpo_pyQfT?22_wS@RiC&!@ki(wjZiBN{SwO2klxF<#}@Q|-ZX zGoR0{Iv(poI9Z%)4=jKc9lY+;sV%!nd+25#pyA^XegiCJ{^<8tXYX~N@jjOa96P#) M542TEz)xQPFTow~N&o-= literal 0 HcmV?d00001 diff --git a/docs/apache-airflow-providers-http/img/connection_headers.png b/docs/apache-airflow-providers-http/img/connection_headers.png new file mode 100644 index 0000000000000000000000000000000000000000..413e9bbb38864faf0dd5664703b9089ff0b7bd5e GIT binary patch literal 5256 zcmb7Ic{r5o`yXfn{-V})%5hC16>~W0*`i5pPc5~&!_Z*&@TQfZG(MG_BoO+`NXB8 z9cLtFwPB4YYQ)t9B8V#!9>!GTtzxn9z61NxOR26(i5#r%Yd8mX zpspJ8#$8$syuMdWkp$lPLfgu|3wA>n2RozP0kP;M(D-3DJV7{eSD?`7DFLe$svtVK zQXrI18VCt+$j?P)C&oNc2yK;mP)&FT>;wtK#y-+W5L@X#Lh$T(nAMSE;3&rnyAcEC z=B`9Y)31Gasqo@r&2Lc$Ksrplcbu4A)+;|xiiy6^5E1hE2n=@Smv23B#hBxd`M!T# z;oFxEnNSws+K+=nu|$kS1<~tp{^0_Dt>)tIEzT4=aOBWXNS{k_ety2;UKs4bA;@|O zqI_48U5@qP!-su9fX5X;*!ZA*_1&YXJa?|LQ*sBZ*nIV=EK&jnJF5asx!*D8 zqD|Tiyz1lgM;Zu9wrlLxa<^2*#&T(F&im)*@7WOyvr6k48XC^!Q6Dt=gTuhh;JSW! z(nzC#D9CHZ`xnj`gs`zBwwg({QRK?w@WqL7$}3N5ZJ|8t-809c6d6RJKWPi@wu&2x zcEr;P^YfK*^h2JGK+AEsXA3O%nX@t9lWTgD%@8mrgimB`Rxh%jl#)356_u5Rr}Hff z?kZUoZp_tbcLuA?4A$sUGj)_w4(KQjf$QcHa|a|64;?m;j&!dcTY@2ScBl@KMh*uy zaWRImoXw@-$oX-~>Tsk#A-+3jdAhG?1YE7D&CiQIA7B5B6iA|ZE=_j%Y;(4J-riS3 zzCK?Y?8CVhzotq4@DLf){qv5U(jL%R7dBit5tFhO#p1X{FQ4_;Y!)_YAF(n-l;GOZ z54)J*k0Tn12Z%>5n?=l&Q35k1hNBj#dRTh-(qK?$L7sVH;mw`Gp&QXVV0s4J)6kcP zj*}~sP)CV~x22ddX`Y|i<5PF7mTn3IloxEAW*u(6*i%(x&7R-&?tm0 zc^u>|u8Vn;W}COTY;i)gElS_{pk}(GerhFG)QC&e}LIYN^nK_x>Q`K6N@g{EVKwmRXK^9NZ>d z^HiGgx_1W4yW}9TxSH^Ygbw?&xJcU6R>z?bT%}2jLma$IRh2MOqZ71LnRK|p9M{JW zTX5+0%A?Yc`38ou2VZrDA~|&5x<%F%n~Y%OursJ-)Y7YY=%p)3vUlBA^97XL;cpAk zc(&GQ1>T3=j!M_wHJF$#QGbgZ7$D)Tv;p)1kgr^3(sXt&WWY)R!f?yFA<>ogRGAmHtO$}!F+-icSz0A;Zz(U~Nh3w5b9 z3lT|Ew2h5Tj9;7%s{yr*ZvrbxJI7E~#ci6CBa`P!UKp@s$;&)2y=T@5aP7IlEEP<4 zreFPNg5DMOrzkrGj>X()qMUWUWvZ=t;$AzPR&9RSvd^vQ8s=PG_iG+RVKD-8s+6=s zzmiAYm~Y6K7}Pf)>Uu6cDGy*Iv`Va0_-eZ_ZL}NiU+`>yQz}2aF^XKA7*0>vD|g>= zuY7vCrXSkPZG1xNMx#Rkt#I&Jg1qB310fI?_5Jkvn%G#*Mv1}xf(B5A4fY?lCT)^C zdw#zuKfx-nN&m!Jo+h5uHr$Sq!ygOsWGu_YtmPRtF06bv)$djH*vG+t4s@1wynw3; z`r)r5k<@I;;@T_c%1o&w&uSa?yVcpw5I1jH)u%~E9}UlqCF_C`zBH3n#&`N(LNLH>GgW*=?Jnd{y>_P zUfD%8If`a>6G9aex(h3Jp>gxKzSQKtBwCRGR=-mqnNfB|J#{3kS$nbjDHV@67Fq+# zja|k;FwrEzSQyLcVLu5#xa`IbM@&=ezkm89GeA7gw*9fxu|U*S-wj{Fl$e4&sAe}? zZStrst<#j1$2sQs$wcv3${l=1sl23{r6_%7FueHgPYRywmNI3VeazLgAor=W%-M@P zlk!}kaC+Pf?;d_8?x;dKvSG1fAUq@K(FodiIy$8+^RpO*J}|I&C(}v#9Viw{gB(Uhx^wD6z@c9;^~di z&VD}}jHPJVTg2isr^Fht`-aMn3>l99h`|eZbP$a2a$z4io4Br9$D6Vb(rxR?sorr0 zH+KrQr^I_ik&~_1YaZKG=&QIq^vpp?il5sNlhvLi&la6#)zG`it9-e!Uy#s)Jy=j_ z2ZG$PopwtK{sOuix%Gns#yG3_Umw^8p;7u_3zvif#+)?oCd}&7rOnI&1Iny#t z**6xC8=!a3^uJ;w%f@ln7NbX-;tk)sc#DV5MyL(3o2Lb(j~^10hwCAfje{riwU{uP z$(M~9@4>=8QWXD#YHJhVSS;43DQ=g~pS?CVVbh)vDwz3D1|h2@v9E2L%3@G@Zt&~~ zyt)2B!?Ez}vvUtHH7Yh4T4>VxyV7n5?8w#n-BUqZN3PDHK}WXhDZ*$AyA>jg&HX`n zM3fX1D4V1nnqKa6)6U_6E%X2va#y}WE5mQ0Ro_owJ5&8XmR1q6bv!H~oE&@E5Oky@SUzc)d}XKWJyRt_1;{R^`9_2p4PRVDHZO5d0FAFS3fT5Q*y zTI@98!ggX5`xK)dG)jX`_o)t9+~8e)p@W$Td!@Rap2@EK9{-p{d}~K?0G35(ogWJUxF=^gdUdFEF?O|X zWW2NbTK{jGS&Udp80iSWarKc(u-BNZukco-=@vl0Op55O@~SHEh|+X=c_@u?Cj%s> zD0XYwEr&=a1B^VJ=_B5m)UPb+-V#{67M-%7%$u?R0Pv@e9(^S)5DJ{?!I)4lzp2Cy zu*3MYMAKV>(qs+N)5sO%V6kFroqLcnHVR?^75TlzNjGI{9AH z?_c<|XX>QL%G^HTMM;&N>MeFa=B|(3)@Fa`%C`!7$p@M(mq&mM*ZfOB3F(dIfsgc;t6Edc<_QlVP9D>>+G z9(9a-`*sm8%$|EAAi~rBugTHw_wUnX__s%eV}BraTQF%V>uvWltjO2WGvA3dxIB`# z4G-^45R?i0p&MVq)CF}e*~W@6R$_#(sKai!0^@_q|>wch2;jFQ|)-Vb2mh+Ain#B!$aSK zgM%NxfB(MRZ>W|n^4EvyQ{Lj0enZ)Fq-_ox5-+~6@E)2cz{)=Tzg_tc)BZ1Z3{-_s zfV<`zR)@ctG;t4Od@A2E#Twi_4PhQ2LU4~K7~b^s?6YoAsJp>6Y=by@>0cbZva(Vy zbml|wSW6P**3uV71{~N}nJuoUsQ5;m_%9^@2pIJXO?#mJ`&B6TzIZWPx5g}3+T|0^ZBmz$t5%Ur1?3w22x^7u=(PV>RJdI`5- z^5^Wpb43lPll8hFUv~BI-zdBe2yHOv7u@*7&++0#R;vR_BVT|)XF5ik6MH~$bg;6r z;<`C&{WBWUsQwm}f+pItHnl}m?H-;4%917|w!CQhqGSC%M#99CRVu=2-ZvfdRh#nV zN?ooPuB$*F`~PZaU`p;UwSw5cL^-vBGu!X4KoNq$wxIO?Z{>})&HIxWyH+xol?#nJ NWnqUZF}w83zX2P`_&Wdq literal 0 HcmV?d00001 diff --git a/docs/apache-airflow-providers-http/img/connection_username_password.png b/docs/apache-airflow-providers-http/img/connection_username_password.png new file mode 100644 index 0000000000000000000000000000000000000000..6e36e77dd4cb48f62107a3f654648587bddc58e7 GIT binary patch literal 4761 zcmchbcU03^o5!P|;P@gUqabyRBPa|Y0trZS6j4z?q$vUcqA-y*h?E3EU@S;qMnF0u z(g{VRlYmN5T9iFsdW&WcE+Gf+@j6Uopb@@XX=e3HCon2biA)vxaT*S!RJ3xgOLI`DzV z&2xKseS&`%p+IgvaCdjl&(1!j0f*;S(Sz>pj$4ZtC$~^wE!F+d(mk)c>gXq@hPy(@Cb59eAf4h1K$Ny zvvJ3q1Oy^s9O<$3lCe(nuy`l_}xUI_%)8AUWqzr*n+Y^pZitg6I zLeC6Vdf;g*nKB{9kSgh{rD%npDisdYg zw&r?`wq!pR5I#1!Ia65DJ<=ROmi8NpeRfDXd7>*VMgQd1EP~tCFV)(LekLrNK0(UJ z;64kL!JLjdt|rG;K@e?IRf<=N>*m|~;XX4xJwrpbb9|6jLlSbOdxrb-<@r)Hl|Lku zyO1VGn%Ip;hs?Xuv|qAGiK-`cS1CxX8)2p6e}u}Kw7*m#Z9=8Okk%#kX?ol(W)Ij@ zlH{JYR#{wXOxW4e(fJ+3JGGBc$+`i)#D>a>it$Cci%E-AKl-?<{o8xj(k&%2#4@|$ z&Y0hFb#>LdvNb;%qi-L$ooANSouuZnytG6=vz=grQ-eUp&667^+Ts;a&Eeyy+mplP z2)Z6>J#QV(@yaZ;E-`w%|Inw_`o#~ckGMri>-DNP@j*W-?5{1u(y{EEOQDCd z%aDe-g91V~+FvTQx9nnX%l#QJ+Y0W?mTJXJoTg+7bT^2j%$0sEm}s`W-f+_M>G&A(XNP6 zho85#oVVSEs>>*1(3=JGdmuMWvJbe)B$MM})-O$7P?K?$P{UZzf@bkm%iDH!_FaQn zhEX?C;GRb0924Wmp0aTPh@kax+&-9Ua5Tg5Qs|rG5W{YvgBwrWhnDlebNt z9hk5$ax&VOrDic^M?xu%yB!#9bM%BUKTL%7=;$PNdwnv3k%%U1^n}gMY*br3(fIgG zsC`o*#JFQM-+%GF069VFZaBj@1(vPlJu#LcZ<4x(=s#+iw1*#Zp(Y+$s)gOKpy^Mi zhh1JEh{`|oY+<3$n<1MnKkys2Z9Ux)@ z%BUKpVONr&Z|;;k8*i)*dNQ|f3*??JoyK&EP}~>~JsA>|8_DWt1a3eeJ-N5|VI}jl zGMR+Q*(I5YLM_6Y15OHtRX9Q~gnh||_Pq@YxK;hNF=yGk z|BgOMZ*$IA0Jii?4~WJH?1^JPu2w2Qf;Vo4H_G)Dd=BXh=iVR}^TRHM+Dgw^$;8e` zP$DG*Rzz~=_oft=tOs#9Y;VtPoe-f5S=C96a!oeMBG2Z&RVeC*f6spIFLmbFo?@NX z-$(3iy%C@2DEf0N_~e%_5<`V$yEaGhHzJNH86j{SAA4;j8F&G|99)9ZRFv$`dcdBi zu9uI*4r}Vk6o&h_d#IO&T@D>(j-DTBFx|o~vD0mqTH;;PV4m3X-3Q_Y2ELXyjh<)exI zAI=l-``$WPgJko zCOa?C2+evBNE_aoFXhY&d7nehVw^muQmF~`&aPISZY>J6*5|qedh&Tac5AJy9(QH( zm0IPMsCqSB_E1ax)UWj`*Btm3XMM|gi9PO4$$nVHgXxSCWr!5{^Jx6wl2xuh-mjxl z>sJSySG1r52UEhrsa5?&_IB9Q zkOnAEI3>)e{ID|3v6d+A3&aAo0YTqdcwwrOj>!D3vE=2>L>6#hcMp(G$SV=x=see* zezA@FgKI(7{j0FEvooJSAk=~-`Y5$jtM)OXL5uDODAYAE75gF&V31wiqv=~Re=1=G z$f=$(BhUK2el;TiLuQO3QAjt*f0xSq=BS^D*ykxWI+KMvgkV9i;VDWRx9}jNfL%DaGjA|W{vQ$NpS3m4=I-6QJZ}L}~wP_n}m)v?XSj1Nx_PDiK^uj29eT?=~Mn-f>J)|u$Y0Q?t#MSwG^ghl$AI*YQh zPGZ4#`&YAys;bE{_ed{f(V#TgRDd`-wlxn=(T4zRi8N6v&%LPE2D@vdn9hvyJOK)0 zo5bNQwNxHk*?5Vn)$syP6ln|k`s~Ko5m3|5!!&g$uh4iW@cxxT)+;#8UzfUEN6*nH zv?yuiGB};hJ>V9!_DASU zA8w_fYPQUpz`4E(xJHivGHtDklg(P_KGgdyG^%6DtOoA5MV+o&AQo=N*DjlLvwkSu zGld5EG;sN|JgWQ6HPoknI{`GAsz0FpaO|ptUZB5(wy)cH05C!;>Dbi)^kQzahB0rOS2)^^!pMBGqd>KLLc`E3qWiy%&Z<__TUz@VEwbOwXrxKJ5u3YvNnYXhWJ7bY-C*PQ~_Dzw@4q zcEWY^tbhD%ABytj1gg=ULuR4O$zh`M%fx*LCniZXf$I-xzzlPdW0oDL2W@J&Z{NOk z$17TQhaYX%CTXaz%nr&Mr=BHx4{sWI&DHu?bJMNu(Wi{(rNl`=i|iWidjZ+8*_hcn zTwe!ktI{1fRmN8>#Q2FIXJe)P=pUDckPlW`Vq_64XHt3^j*Y=&;I`3a(iH&Xk_p(& zRZbq87O-IFSJJoAH)Dyoi{Du~p#oD;jJFbo1WPg__Q4!Y$a6zDX6NJSxqW8aS7EDRj9^xD&NQP6Uc9bgIU$EzSuy7XkUDeWJ73?Nt>GqqxAGePKGm z5R9MD)q&%jw?d1gs+`u}vRGu(?95C154u4x6d?7k=NC@JpLJXuexIL}mF2ABTxfv- z*xT6+5O2abH*6Z%;@j@Ch9He$Y&03*q@cESMj*l(~?m8SC=FfD5o|Ix9xc zsD!xj0DRn^TkDCQ2pXG$oMrmnPw4oHL(KIfaA_uK=i|9Tm-f@spV3u+%1~@-kPTXM z-1b%NI^M9!GB+J35wTwy-(h+ShP3?`9cn=y1(?>As)tqJN~zsf7_rHMBRAgFb9$Z$ zp|w9G&miaP*bk;e`$`ZX9ZflcFmH-%&o|G_5jko22F%Ej4`_{%HO;ybVCW39*?lDqn1(u<{qD}X_YW2lcjDZtPyo^xHQn2o zV|6)CWJLAN{p;M+`8lb3EggwO?8X=;Mh~+xz~*eSC$A*D7JJ_@a`|AyqMX6$TUst1 z-FH@|vZjY>ywQlQSk){!=dO3DdTBA}DsKmgj0#@{qC13d(_6)PxV|>P53KaFMIeum z#4>N@@x$y~{zh2+mOga1(hc=BBI)8+aCBm&f};?~S92I1gGFyw-@CR!mgOcs{v9mw zSgXNV(bgh+f`Wij{0agP0++y&v?a~&`%-~MAxp`hbPV!2D#7{mX21p%0%^%#S95Ea&2 zqmQilODU|X*w>#nCFptQWHS?)&D*z2t6lTnPS;E>`|9+mBR5z>5m-R2$vMVp$@58? zC)nXvUdvIDk7z0aZ4iF6t&|#3ZEs`7Mj;%!@ot)dR zs}K|U;waI$QPms)nQDDfWZ3Y19^^!(0-7FonL^qnn`Ig1NNf9!4JdmBb|h<|m+$G} z>NBCyG2cr6D5(*fov^^Dmgl~HEB>&fF>K${(N2r99r$UmauCfl({Zv~mmEp-e^j_J z1lg+J(TCUF{4mwW6U5E=&e{GZ$=7-u7Wv&+Y%^LZMLr`C+K}Y*^~2NV!phd@<90tm zrJ#vn@XMEf6^%GjRyfl%wR+Q*2 zIy+?1^PStpdc&<^FC^2Ikm9U=TSukp{J%jB&r7>^DgT*=YK7eXx}-UiZLwfo1*9JQ2&Vsd b1GBE*^^2vH0QXn|ulCwy6NBQ54!`{!fAs15 literal 0 HcmV?d00001 From 20f759534b8cedcd6fc49b8069796158faa17b6c Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 08:18:23 +0100 Subject: [PATCH 107/286] feat: Implement auth_type and auth_kwargs in the AsyncHttpHook --- airflow/utils/json.py | 10 ---------- .../http/tests/provider_tests/http/hooks/test_http.py | 7 ++++--- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/airflow/utils/json.py b/airflow/utils/json.py index eb3cd40941197..a8846282899f3 100644 --- a/airflow/utils/json.py +++ b/airflow/utils/json.py @@ -123,15 +123,5 @@ def orm_object_hook(dct: dict) -> object: return deserialize(dct, False) -def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> Any: - """Safely loads JSON. - - Returns None by default if the given object is None. - """ - if obj is not None: - return json.loads(obj) - return default - - # backwards compatibility AirflowJsonEncoder = WebEncoder diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index fe6b8f882f2b7..1e944dda1d090 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -384,9 +384,10 @@ def test_available_connection_auth_types(self): auth_types = get_auth_types() assert auth_types == frozenset( { - "request.auth.HTTPBasicAuth", - "request.auth.HTTPProxyAuth", - "request.auth.HTTPDigestAuth", + "requests.auth.HTTPBasicAuth", + "requests.auth.HTTPProxyAuth", + "requests.auth.HTTPDigestAuth", + "aiohttp.BasicAuth", "tests.providers.http.hooks.test_http.CustomAuthBase", } ) From 7274a694e96f3eecf4e8dea3d5d627b8514fc9af Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 19:04:06 +0100 Subject: [PATCH 108/286] feat: Add tests --- .../provider_tests/http/hooks/test_http.py | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index 1e944dda1d090..b6f43995b979d 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -437,6 +437,32 @@ def test_connection_with_extra_auth_type_and_no_credentials(self, auth, mock_get HttpHook().get_conn({}) auth.assert_called_once() + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_string_headers_and_auth_kwargs(self, auth, mock_get_connection): + """When passed via the UI, the 'headers' and 'auth_kwargs' fields' data is + saved as string. + """ + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + login="username", + password="pass", + extra=r""" + {"auth_kwargs": "{\r\n \"endpoint\": \"http://localhost\"\r\n}", + "headers": "{\r\n \"some\": \"headers\"\r\n}"} + """, + ) + mock_get_connection.return_value = conn + + hook = HttpHook(auth_type=CustomAuthBase) + session = hook.get_conn({}) + + auth.assert_called_once_with("username", "pass", endpoint="http://localhost") + assert "auth_kwargs" not in session.headers + assert "some" in session.headers + @pytest.mark.parametrize("method", ["GET", "POST"]) def test_json_request(self, method, requests_mock): obj1 = {"a": 1, "b": "abc", "c": [1, 2, {"d": 10}]} @@ -724,7 +750,7 @@ async def test_async_post_request_with_error_code(self): async def test_async_request_uses_connection_extra(self): """Test api call asynchronously with a connection that has extra field.""" - connection_extra = {"bearer": "test"} + connection_extra = {"bearer": "test", "some": "header"} with aioresponses() as m: m.post( From 8e77abd4ef69577e64c15dd1373d539921caf4c6 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 21:04:22 +0100 Subject: [PATCH 109/286] fix: Add header and auth into FakeSession test object --- providers/http/tests/provider_tests/http/sensors/test_http.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/providers/http/tests/provider_tests/http/sensors/test_http.py b/providers/http/tests/provider_tests/http/sensors/test_http.py index 78a11e15bb7c1..47af8f49c48cf 100644 --- a/providers/http/tests/provider_tests/http/sensors/test_http.py +++ b/providers/http/tests/provider_tests/http/sensors/test_http.py @@ -238,10 +238,14 @@ def resp_check(_): class FakeSession: + """Mock requests.Session object.""" + def __init__(self): self.response = requests.Response() self.response.status_code = 200 self.response._content = "apache/airflow".encode("ascii", "ignore") + self.headers = {} + self.auth = None def send(self, *args, **kwargs): return self.response From d4612015d180b0e12246db395002f626fdb8ddcc Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 21:14:54 +0100 Subject: [PATCH 110/286] fix: Use default BasicAuth in LivyAsyncHook --- docs/spelling_wordlist.txt | 1 + providers/src/airflow/providers/apache/livy/hooks/livy.py | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 2b35ed5bc2dd9..7aae2582f24b1 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -25,6 +25,7 @@ afterall AgentKey aio aiobotocore +aiohttp AioSession aiplatform Airbnb diff --git a/providers/src/airflow/providers/apache/livy/hooks/livy.py b/providers/src/airflow/providers/apache/livy/hooks/livy.py index 3eec9599457fe..ec50a57b87ea4 100644 --- a/providers/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/src/airflow/providers/apache/livy/hooks/livy.py @@ -491,6 +491,7 @@ def __init__( extra_headers: dict[str, Any] | None = None, ) -> None: super().__init__(http_conn_id=livy_conn_id) + self.auth_type = self.default_auth_type self.extra_headers = extra_headers or {} self.extra_options = extra_options or {} From 2a658464d4b515145c1356de4ed528d57a01700a Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 16:36:22 +0200 Subject: [PATCH 111/286] refactor: Removed docstring for removed json parameter in run method of HttpAsyncHook --- providers/http/src/airflow/providers/http/hooks/http.py | 1 - 1 file changed, 1 deletion(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 9cf5d4983dbc5..932aeb31d29c0 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -397,7 +397,6 @@ async def run( :param endpoint: Endpoint to be called, i.e. ``resource/v1/query?``. :param data: Payload to be uploaded or request parameters. - :param json: Payload to be uploaded as JSON. :param headers: Additional headers to be passed through as a dict. :param extra_options: Additional kwargs to pass when creating a request. For example, ``run(json=obj)`` is passed as From 53baee62c64ee3f91f12326d0a918f2d653a1ba9 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 16:38:39 +0200 Subject: [PATCH 112/286] refactor: Aligned HttpTrigger with version from main branch --- .../http/src/airflow/providers/http/triggers/http.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/providers/http/src/airflow/providers/http/triggers/http.py b/providers/http/src/airflow/providers/http/triggers/http.py index d25d3a55cfb5b..ec9780bdeab49 100644 --- a/providers/http/src/airflow/providers/http/triggers/http.py +++ b/providers/http/src/airflow/providers/http/triggers/http.py @@ -73,7 +73,7 @@ def __init__( self.extra_options = extra_options def serialize(self) -> tuple[str, dict[str, Any]]: - """Serialize HttpTrigger arguments and classpath.""" + """Serializes HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpTrigger", { @@ -88,7 +88,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Make a series of asynchronous http calls via a http hook.""" + """Makes a series of asynchronous http calls via an http hook.""" hook = HttpAsyncHook( method=self.method, http_conn_id=self.http_conn_id, @@ -165,7 +165,7 @@ def __init__( self.poke_interval = poke_interval def serialize(self) -> tuple[str, dict[str, Any]]: - """Serialize HttpTrigger arguments and classpath.""" + """Serializes HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpSensorTrigger", { @@ -180,7 +180,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Make a series of asynchronous http calls via an http hook.""" + """Makes a series of asynchronous http calls via an http hook.""" hook = self._get_async_hook() while True: try: @@ -193,7 +193,6 @@ async def run(self) -> AsyncIterator[TriggerEvent]: extra_options=self.extra_options, ) yield TriggerEvent(True) - return except AirflowException as exc: if str(exc).startswith("404"): await asyncio.sleep(self.poke_interval) From 31a03bd6eaac6b3a20824dad9a2e0e10d646b255 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 17:03:29 +0200 Subject: [PATCH 113/286] refactor: Changed docstrings in HttpTrigger to imperative mode --- providers/http/src/airflow/providers/http/triggers/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/providers/http/src/airflow/providers/http/triggers/http.py b/providers/http/src/airflow/providers/http/triggers/http.py index ec9780bdeab49..5975389830f36 100644 --- a/providers/http/src/airflow/providers/http/triggers/http.py +++ b/providers/http/src/airflow/providers/http/triggers/http.py @@ -73,7 +73,7 @@ def __init__( self.extra_options = extra_options def serialize(self) -> tuple[str, dict[str, Any]]: - """Serializes HttpTrigger arguments and classpath.""" + """Serialize HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpTrigger", { @@ -88,7 +88,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Makes a series of asynchronous http calls via an http hook.""" + """Make a series of asynchronous http calls via a http hook.""" hook = HttpAsyncHook( method=self.method, http_conn_id=self.http_conn_id, From b72a36169057909d8af45e9516b6fe15e1911a6c Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 11 Apr 2024 12:15:06 +0200 Subject: [PATCH 114/286] refactor: Updated docstrings of serialize and run method of HttpTrigger --- providers/http/src/airflow/providers/http/triggers/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/providers/http/src/airflow/providers/http/triggers/http.py b/providers/http/src/airflow/providers/http/triggers/http.py index 5975389830f36..d30f41990f5b0 100644 --- a/providers/http/src/airflow/providers/http/triggers/http.py +++ b/providers/http/src/airflow/providers/http/triggers/http.py @@ -165,7 +165,7 @@ def __init__( self.poke_interval = poke_interval def serialize(self) -> tuple[str, dict[str, Any]]: - """Serializes HttpTrigger arguments and classpath.""" + """Serialize HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpSensorTrigger", { @@ -180,7 +180,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Makes a series of asynchronous http calls via an http hook.""" + """Make a series of asynchronous http calls via a http hook.""" hook = self._get_async_hook() while True: try: From 6b891447552d6fbc4a9d06ca3fcf7b28af46d8d0 Mon Sep 17 00:00:00 2001 From: David Blain Date: Mon, 6 May 2024 13:09:59 +0200 Subject: [PATCH 115/286] refactor: Moved get_connection_form_widgets method from HttpHook to HttpHookMixin --- providers/src/airflow/providers/apache/livy/hooks/livy.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/providers/src/airflow/providers/apache/livy/hooks/livy.py b/providers/src/airflow/providers/apache/livy/hooks/livy.py index ec50a57b87ea4..66978696f7b2a 100644 --- a/providers/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/src/airflow/providers/apache/livy/hooks/livy.py @@ -81,6 +81,10 @@ class LivyHook(HttpHook, LoggingMixin): conn_type = "livy" hook_name = "Apache Livy" + @classmethod + def get_connection_form_widgets(cls) -> dict[str, Any]: + return super().get_connection_form_widgets() + def __init__( self, livy_conn_id: str = default_conn_name, From 8552a97dd14593b58c02a7841caf7be9529103da Mon Sep 17 00:00:00 2001 From: David Blain Date: Mon, 6 May 2024 13:15:40 +0200 Subject: [PATCH 116/286] refactor: Pass auth_type parameter from LivyHook to constructor of HttpHook as it has also this parameter instead of redefining the same field --- providers/src/airflow/providers/apache/livy/hooks/livy.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/providers/src/airflow/providers/apache/livy/hooks/livy.py b/providers/src/airflow/providers/apache/livy/hooks/livy.py index 66978696f7b2a..9d4ed40c4f7f8 100644 --- a/providers/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/src/airflow/providers/apache/livy/hooks/livy.py @@ -92,11 +92,9 @@ def __init__( extra_headers: dict[str, Any] | None = None, auth_type: Any | None = None, ) -> None: - super().__init__(http_conn_id=livy_conn_id) + super().__init__(http_conn_id=livy_conn_id, auth_type=auth_type) self.extra_headers = extra_headers or {} self.extra_options = extra_options or {} - if auth_type: - self.auth_type = auth_type def get_conn(self, headers: dict[str, Any] | None = None) -> Any: """ From dad4f28a3cd82177b5c1687193794d4126de042c Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 7 May 2024 19:34:37 +0200 Subject: [PATCH 117/286] refactor: Enhanced extra_dejson property to allow load string escaped nested json structures --- airflow/models/connection.py | 2 +- providers/src/airflow/providers/apache/livy/hooks/livy.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/airflow/models/connection.py b/airflow/models/connection.py index 01df1626657da..19aeaf9e63089 100644 --- a/airflow/models/connection.py +++ b/airflow/models/connection.py @@ -432,7 +432,7 @@ def get_extra_dejson(self, nested: bool = False) -> dict: self.log.exception("Failed parsing the json for conn_id %s", self.conn_id) # Mask sensitive keys from this list - mask_secret(extra) + mask_secret(obj) return extra diff --git a/providers/src/airflow/providers/apache/livy/hooks/livy.py b/providers/src/airflow/providers/apache/livy/hooks/livy.py index 9d4ed40c4f7f8..8f10a04903c75 100644 --- a/providers/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/src/airflow/providers/apache/livy/hooks/livy.py @@ -85,6 +85,10 @@ class LivyHook(HttpHook, LoggingMixin): def get_connection_form_widgets(cls) -> dict[str, Any]: return super().get_connection_form_widgets() + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + return super().get_ui_field_behaviour() + def __init__( self, livy_conn_id: str = default_conn_name, From d435a62b387d89c8ebc75d3a7e9a9e6f007046ec Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 8 May 2024 16:02:23 +0200 Subject: [PATCH 118/286] refactor: Changed conn_type to ftp in test_process_form_invalid_extra_removed as http as livy do now also have custom fields --- tests/www/views/test_views_connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/www/views/test_views_connection.py b/tests/www/views/test_views_connection.py index 1e21dc4856ed1..19a36c0ac6b39 100644 --- a/tests/www/views/test_views_connection.py +++ b/tests/www/views/test_views_connection.py @@ -462,9 +462,9 @@ def test_process_form_invalid_extra_removed(admin_client): Note: This can only be tested with a Hook which does not have any custom fields (otherwise the custom fields override the extra data when editing a Connection). Thus, this is currently - tested with livy. + tested with ftp. """ - conn_details = {"conn_id": "test_conn", "conn_type": "livy"} + conn_details = {"conn_id": "test_conn", "conn_type": "ftp"} conn = Connection(**conn_details, extra='{"foo": "bar"}') conn.id = 1 From ce11702be5dc158da8517df99b608a1954aa37d6 Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 18 Jul 2024 20:14:20 +0200 Subject: [PATCH 119/286] refactor: HttpHook now uses patched version of Connection + added test which checks when this patched class has to be removed so we don't forget --- .../src/airflow/providers/apache/druid/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/providers/src/airflow/providers/apache/druid/__init__.py b/providers/src/airflow/providers/apache/druid/__init__.py index 7585be9880dc1..d00c3c8da7757 100644 --- a/providers/src/airflow/providers/apache/druid/__init__.py +++ b/providers/src/airflow/providers/apache/druid/__init__.py @@ -37,3 +37,17 @@ raise RuntimeError( f"The package `apache-airflow-providers-apache-druid:{__version__}` needs Apache Airflow 2.9.0+" ) + + +def airflow_dependency_version(): + import re + import yaml + + from os.path import join, dirname + + with open(join(dirname(__file__), "provider.yaml"), encoding="utf-8") as file: + for dependency in yaml.safe_load(file)["dependencies"]: + if dependency.startswith('apache-airflow'): + match = re.search(r'>=([\d\.]+)', dependency) + if match: + return packaging.version.parse(match.group(1)) From 5642c9a8176ccbd7123c76c2372f8d4fe3b78f4a Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 18 Jul 2024 20:56:42 +0200 Subject: [PATCH 120/286] refactor: Fixed some static checks --- providers/src/airflow/providers/apache/druid/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/providers/src/airflow/providers/apache/druid/__init__.py b/providers/src/airflow/providers/apache/druid/__init__.py index d00c3c8da7757..18870506f868b 100644 --- a/providers/src/airflow/providers/apache/druid/__init__.py +++ b/providers/src/airflow/providers/apache/druid/__init__.py @@ -41,13 +41,13 @@ def airflow_dependency_version(): import re - import yaml + from os.path import dirname, join - from os.path import join, dirname + import yaml with open(join(dirname(__file__), "provider.yaml"), encoding="utf-8") as file: for dependency in yaml.safe_load(file)["dependencies"]: - if dependency.startswith('apache-airflow'): - match = re.search(r'>=([\d\.]+)', dependency) + if dependency.startswith("apache-airflow"): + match = re.search(r">=([\d\.]+)", dependency) if match: return packaging.version.parse(match.group(1)) From 2d7e5824b72956b8ec2f66fa389d735737a2df0e Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 28 Jan 2025 11:33:11 +0100 Subject: [PATCH 121/286] docs: Updated documentation --- .../configurations-ref.rst | 18 ------------------ providers/http/docs/configurations-ref.rst | 18 ++++++++++++++++++ .../img/connection_auth_kwargs.png | Bin .../connections}/img/connection_auth_type.png | Bin .../connections}/img/connection_headers.png | Bin .../img/connection_username_password.png | Bin 6 files changed, 18 insertions(+), 18 deletions(-) delete mode 100644 docs/apache-airflow-providers-http/configurations-ref.rst rename {docs/apache-airflow-providers-http => providers/http/docs/connections}/img/connection_auth_kwargs.png (100%) rename {docs/apache-airflow-providers-http => providers/http/docs/connections}/img/connection_auth_type.png (100%) rename {docs/apache-airflow-providers-http => providers/http/docs/connections}/img/connection_headers.png (100%) rename {docs/apache-airflow-providers-http => providers/http/docs/connections}/img/connection_username_password.png (100%) diff --git a/docs/apache-airflow-providers-http/configurations-ref.rst b/docs/apache-airflow-providers-http/configurations-ref.rst deleted file mode 100644 index 5885c9d91b6e8..0000000000000 --- a/docs/apache-airflow-providers-http/configurations-ref.rst +++ /dev/null @@ -1,18 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -.. include:: ../exts/includes/providers-configurations-ref.rst diff --git a/providers/http/docs/configurations-ref.rst b/providers/http/docs/configurations-ref.rst index e69de29bb2d1d..5885c9d91b6e8 100644 --- a/providers/http/docs/configurations-ref.rst +++ b/providers/http/docs/configurations-ref.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../exts/includes/providers-configurations-ref.rst diff --git a/docs/apache-airflow-providers-http/img/connection_auth_kwargs.png b/providers/http/docs/connections/img/connection_auth_kwargs.png similarity index 100% rename from docs/apache-airflow-providers-http/img/connection_auth_kwargs.png rename to providers/http/docs/connections/img/connection_auth_kwargs.png diff --git a/docs/apache-airflow-providers-http/img/connection_auth_type.png b/providers/http/docs/connections/img/connection_auth_type.png similarity index 100% rename from docs/apache-airflow-providers-http/img/connection_auth_type.png rename to providers/http/docs/connections/img/connection_auth_type.png diff --git a/docs/apache-airflow-providers-http/img/connection_headers.png b/providers/http/docs/connections/img/connection_headers.png similarity index 100% rename from docs/apache-airflow-providers-http/img/connection_headers.png rename to providers/http/docs/connections/img/connection_headers.png diff --git a/docs/apache-airflow-providers-http/img/connection_username_password.png b/providers/http/docs/connections/img/connection_username_password.png similarity index 100% rename from docs/apache-airflow-providers-http/img/connection_username_password.png rename to providers/http/docs/connections/img/connection_username_password.png From bb67c14a35c631a37b3b26dbab45dc85bf290ebf Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 28 Jan 2025 13:45:55 +0100 Subject: [PATCH 122/286] docs: Fixed path to images --- providers/http/docs/connections/http.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/providers/http/docs/connections/http.rst b/providers/http/docs/connections/http.rst index 3bc3d951b5b5d..a5cf74e090b06 100644 --- a/providers/http/docs/connections/http.rst +++ b/providers/http/docs/connections/http.rst @@ -39,7 +39,7 @@ Authenticating via Basic auth The simplest way to authenticate is to specify a *Login* and *Password* in the Connection. -.. image:: /img/connection_username_password.png +.. image:: img/connection_username_password.png By default, when a *Login* or *Password* is provided, the HTTP operators and Hooks will perform a basic authentication via the @@ -52,7 +52,7 @@ If :ref:`Basic authentication` is not enough, you can also add Headers can be passed in json format in the *Headers* field: -.. image:: /img/connection_headers.png +.. image:: img/connection_headers.png .. note:: Login and Password authentication can be used along custom Headers. @@ -62,7 +62,7 @@ For more complex use-cases, you can inject a Auth class into the HTTP operators and Hooks via the *Auth type* setting. This is particularly useful when you need token refresh or advanced authentication methods like kerberos, oauth, ... -.. image:: /img/connection_auth_type.png +.. image:: img/connection_auth_type.png By default, only `requests Auth classes `_ are available. But you can install any classes based on ``requests.auth.AuthBase`` @@ -75,7 +75,7 @@ pass extra keywords arguments with the *Auth kwargs* setting. Example with the ``HTTPKerberosAuth`` from `requests-kerberos `_ : -.. image:: /img/connection_auth_kwargs.png +.. image:: img/connection_auth_kwargs.png .. tip:: From c0b579f1dafff7538d8bcea304fc789a9c871b76 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 28 Jan 2025 13:52:50 +0100 Subject: [PATCH 123/286] refactor: Fixed some static checks --- .../providers/http/get_provider_info.py | 14 +++++++++++++ .../src/airflow/providers/http/hooks/http.py | 20 ------------------- .../provider_tests/http/hooks/test_http.py | 8 ++++---- 3 files changed, 18 insertions(+), 24 deletions(-) diff --git a/providers/http/src/airflow/providers/http/get_provider_info.py b/providers/http/src/airflow/providers/http/get_provider_info.py index f0f387c63ec88..94e04fdc0ccc0 100644 --- a/providers/http/src/airflow/providers/http/get_provider_info.py +++ b/providers/http/src/airflow/providers/http/get_provider_info.py @@ -103,6 +103,20 @@ def get_provider_info(): "connection-types": [ {"hook-class-name": "airflow.providers.http.hooks.http.HttpHook", "connection-type": "http"} ], + "config": { + "http": { + "description": "Options for Http provider.", + "options": { + "extra_auth_types": { + "description": "A comma separated list of auth_type classes, which can be used to\nconfigure Http Connections in Airflow's UI. This list restricts which\nclasses can be arbitrary imported to prevent dependency injections.\n", + "type": "string", + "version_added": "4.8.0", + "example": "requests_kerberos.HTTPKerberosAuth,any.other.custom.HTTPAuth", + "default": None, + } + }, + } + }, "dependencies": [ "apache-airflow>=2.9.0", "requests>=2.27.0,<3", diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 27275dd1d4a98..c0dc2670fc850 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -185,26 +185,6 @@ def get_ui_field_behaviour(cls) -> dict[str, Any]: "relabeling": {}, } - @classmethod - def get_connection_form_widgets(cls) -> dict[str, Any]: - """Return connection widgets to add to connection form.""" - from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget, Select2Widget - from flask_babel import lazy_gettext - from wtforms.fields import SelectField, TextAreaField - - default_auth_type: str = "" - auth_types_choices = frozenset({default_auth_type}) | get_auth_types() - return { - "auth_type": SelectField( - lazy_gettext("Auth type"), - choices=[(clazz, clazz) for clazz in auth_types_choices], - widget=Select2Widget(), - default=default_auth_type - ), - "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), - } - # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index 16882b6c7cca3..a31aa92a31133 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -488,10 +488,10 @@ def test_connection_with_string_headers_and_auth_kwargs(self, auth, mock_get_con conn_type="http", login="username", password="pass", - extra=f""" - {{"auth_kwargs": {{\r\n "endpoint": "http://localhost"\r\n}}, - "headers": ""}} - """, + extra=""" + {"auth_kwargs": {\r\n "endpoint": "http://localhost"\r\n}, + "headers": ""} + """, ) mock_get_connection.return_value = conn From 6aea6355c23dbd10ce09c61d46171e4dc146b3d4 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 28 Jan 2025 13:54:05 +0100 Subject: [PATCH 124/286] refactor: Removed airflow_dependency_version --- .../src/airflow/providers/apache/druid/__init__.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/providers/src/airflow/providers/apache/druid/__init__.py b/providers/src/airflow/providers/apache/druid/__init__.py index 18870506f868b..7585be9880dc1 100644 --- a/providers/src/airflow/providers/apache/druid/__init__.py +++ b/providers/src/airflow/providers/apache/druid/__init__.py @@ -37,17 +37,3 @@ raise RuntimeError( f"The package `apache-airflow-providers-apache-druid:{__version__}` needs Apache Airflow 2.9.0+" ) - - -def airflow_dependency_version(): - import re - from os.path import dirname, join - - import yaml - - with open(join(dirname(__file__), "provider.yaml"), encoding="utf-8") as file: - for dependency in yaml.safe_load(file)["dependencies"]: - if dependency.startswith("apache-airflow"): - match = re.search(r">=([\d\.]+)", dependency) - if match: - return packaging.version.parse(match.group(1)) From d84e40c699184d21c001010c05180cd3e390b4ed Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 28 Jan 2025 14:40:47 +0100 Subject: [PATCH 125/286] refactor: Removed BS3AccordionTextAreaFieldWidget in forms --- airflow/www/forms.py | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/airflow/www/forms.py b/airflow/www/forms.py index b69184c6590c2..7028e2026e449 100644 --- a/airflow/www/forms.py +++ b/airflow/www/forms.py @@ -33,7 +33,6 @@ from flask_appbuilder.forms import DynamicForm from flask_babel import lazy_gettext from flask_wtf import FlaskForm -from markupsafe import Markup from wtforms import widgets from wtforms.fields import Field, IntegerField, PasswordField, SelectField, StringField, TextAreaField from wtforms.validators import InputRequired, Optional @@ -177,29 +176,6 @@ def populate_obj(self, item): field.populate_obj(item, name) -class BS3AccordionTextAreaFieldWidget(BS3TextAreaFieldWidget): - - @staticmethod - def _make_collapsable_panel(field: Field, content: Markup) -> str: - collapsable_id: str = f"collapsable_{field.id}" - return f""" -
-
-

- -

-
- -
- """ - - def __call__(self, field, **kwargs): - text_area = super(BS3TextAreaFieldWidget, self).__call__(field, **kwargs) - return self._make_collapsable_panel(field=field, content=text_area) - - @cache def create_connection_form_class() -> type[DynamicForm]: """ @@ -247,7 +223,7 @@ def process(self, formdata=None, obj=None, **kwargs): login = StringField(lazy_gettext("Login"), widget=BS3TextFieldWidget()) password = PasswordField(lazy_gettext("Password"), widget=BS3PasswordFieldWidget()) port = IntegerField(lazy_gettext("Port"), validators=[Optional()], widget=BS3TextFieldWidget()) - extra = TextAreaField(lazy_gettext("Extra"), widget=BS3AccordionTextAreaFieldWidget()) + extra = TextAreaField(lazy_gettext("Extra"), widget=BS3TextAreaFieldWidget()) for key, value in providers_manager.connection_form_widgets.items(): setattr(ConnectionForm, key, value.field) From 9c2eb4cfb4fc353b86c1b22210843dd1df4a577b Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 28 Jan 2025 14:42:28 +0100 Subject: [PATCH 126/286] refactor: Fixed get_extra_dejson method in Connection --- airflow/models/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/models/connection.py b/airflow/models/connection.py index 19aeaf9e63089..01df1626657da 100644 --- a/airflow/models/connection.py +++ b/airflow/models/connection.py @@ -432,7 +432,7 @@ def get_extra_dejson(self, nested: bool = False) -> dict: self.log.exception("Failed parsing the json for conn_id %s", self.conn_id) # Mask sensitive keys from this list - mask_secret(obj) + mask_secret(extra) return extra From 5f194880f89f40f67e968d10952f7a6805835c4e Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 28 Jan 2025 15:18:18 +0100 Subject: [PATCH 127/286] refactor: Deleted setup.cfg --- setup.cfg | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index e69de29bb2d1d..0000000000000 From 336d0827a7b55551741c998d5c795e09b0c81331 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 28 Jan 2025 17:22:16 +0100 Subject: [PATCH 128/286] refactor: Refactored common code for auth used in HttpHook and AsyncHttpHook --- .../src/airflow/providers/http/hooks/http.py | 92 +++++++++---------- .../providers/apache/livy/hooks/livy.py | 4 +- 2 files changed, 45 insertions(+), 51 deletions(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index c0dc2670fc850..62a00668b37ba 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -64,6 +64,46 @@ def _url_from_endpoint(base_url: str | None, endpoint: str | None) -> str: return (base_url or "") + (endpoint or "") +def _load_conn_auth_type(module_name: str | None) -> Any: + """ + Load auth_type module from extra Connection parameters. + + Check if the auth_type module is listed in 'extra_auth_types' and load it. + This method protects against the execution of random modules. + """ + if module_name: + if module_name in HttpHook.get_auth_types(): + try: + module = import_string(module_name) + return module + except Exception as error: + raise AirflowException(error) + warnings.warn( + f"Skipping import of auth_type '{module_name}'. The class should be listed in " + "'extra_auth_types' config of the http provider." + ) + return None + + +def _extract_auth(connection: Connection, auth_type: Any) -> AuthBase | None: + extra = connection.extra_dejson + auth_type = auth_type or _load_conn_auth_type(module_name=extra.get("auth_type")) + + if auth_type: + auth_args: list[str | None] = [connection.login, connection.password] + + if any(auth_args): + auth_kwargs = extra.get("auth_kwargs", {}) + + if auth_kwargs: + _auth = auth_type(*auth_args, **auth_kwargs) + else: + return auth_type(*auth_args) + else: + return auth_type() + return None + + class HttpHook(BaseHook): """ Interact with HTTP servers. @@ -222,55 +262,9 @@ def _set_base_url(self, connection: Connection) -> None: def _configure_session_from_auth( self, session: requests.Session, connection: Connection ) -> requests.Session: - session.auth = self._extract_auth(connection) + session.auth = _extract_auth(connection, self.auth_type) return session - def _load_conn_auth_type(self, module_name: str | None) -> Any: - """ - Load auth_type module from extra Connection parameters. - - Check if the auth_type module is listed in 'extra_auth_types' and load it. - This method protects against the execution of random modules. - """ - if module_name: - if module_name in self.get_auth_types(): - try: - module = import_string(module_name) - self._is_auth_type_setup = True - self.log.info("Loaded auth_type: %s", module_name) - return module - except Exception as error: - self.log.error("Cannot import auth_type '%s' due to: %s", module_name, error) - raise AirflowException(error) - self.log.warning( - "Skipping import of auth_type '%s'. The class should be listed in " - "'extra_auth_types' config of the http provider.", - module_name, - ) - return None - - def _extract_auth(self, connection: Connection) -> AuthBase | None: - extra = connection.extra_dejson - auth_type: Any = self.auth_type or self._load_conn_auth_type(module_name=extra.get("auth_type")) - auth_kwargs = extra.get("auth_kwargs", {}) - - self.log.debug("auth_type: %s", auth_type) - self.log.debug("auth_kwargs: %s", auth_kwargs) - - if auth_type: - auth_args: list[str | None] = [connection.login, connection.password] - - self.log.debug("auth_args: %s", auth_args) - - if any(auth_args): - if auth_kwargs: - _auth = auth_type(*auth_args, **auth_kwargs) - else: - return auth_type(*auth_args) - else: - return auth_type() - return None - def _configure_session_from_extra( self, session: requests.Session, connection: Connection ) -> requests.Session: @@ -481,7 +475,7 @@ def __init__( self, method: str = "POST", http_conn_id: str = default_conn_name, - auth_type: Any = aiohttp.BasicAuth, + auth_type: Any = None, retry_limit: int = 3, retry_delay: float = 1.0, ) -> None: @@ -535,7 +529,7 @@ async def run( if conn.port: self.base_url += f":{conn.port}" if conn.login: - auth = self.auth_type(conn.login, conn.password) + auth = _extract_auth(conn, self.auth_type) if conn.extra: extra = self._process_extra_options_from_connection(conn=conn, extra_options=extra_options) diff --git a/providers/src/airflow/providers/apache/livy/hooks/livy.py b/providers/src/airflow/providers/apache/livy/hooks/livy.py index bcea2d27b5192..10d42043ca385 100644 --- a/providers/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/src/airflow/providers/apache/livy/hooks/livy.py @@ -495,9 +495,9 @@ def __init__( livy_conn_id: str = default_conn_name, extra_options: dict[str, Any] | None = None, extra_headers: dict[str, Any] | None = None, + auth_type: Any | None = None, ) -> None: - super().__init__(http_conn_id=livy_conn_id) - self.auth_type = self.default_auth_type + super().__init__(http_conn_id=livy_conn_id, auth_type=auth_type) self.method = "POST" self.extra_headers = extra_headers or {} self.extra_options = extra_options or {} From a8e71edd411b543c8e02301e4a6e0214906311b7 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 28 Jan 2025 17:42:14 +0100 Subject: [PATCH 129/286] refactor: Added RuntimeWarning and stacklevel --- providers/http/src/airflow/providers/http/hooks/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 62a00668b37ba..e9b85ec5ad4f2 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -80,7 +80,7 @@ def _load_conn_auth_type(module_name: str | None) -> Any: raise AirflowException(error) warnings.warn( f"Skipping import of auth_type '{module_name}'. The class should be listed in " - "'extra_auth_types' config of the http provider." + "'extra_auth_types' config of the http provider.", RuntimeWarning, stacklevel=2 ) return None From 24bbb79462e36cec0365e504fe7c14914bfc6521 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 28 Jan 2025 20:55:34 +0100 Subject: [PATCH 130/286] refactor: Deprecated the _do_api_call_async of the LivyAsyncHook and delegate it to run_method which in turns delegates to the HttpAsyncHook which is a more DRY solution and avoid code duplication which is almost similar between LivyHook and HttpHook --- .../src/airflow/providers/http/hooks/http.py | 11 +- .../providers/apache/livy/hooks/livy.py | 109 ++---------- .../tests/apache/livy/hooks/test_livy.py | 157 ++---------------- 3 files changed, 40 insertions(+), 237 deletions(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index e9b85ec5ad4f2..59f198741bc62 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -17,6 +17,7 @@ # under the License. from __future__ import annotations +import asyncio import json import warnings from contextlib import suppress @@ -80,7 +81,9 @@ def _load_conn_auth_type(module_name: str | None) -> Any: raise AirflowException(error) warnings.warn( f"Skipping import of auth_type '{module_name}'. The class should be listed in " - "'extra_auth_types' config of the http provider.", RuntimeWarning, stacklevel=2 + "'extra_auth_types' config of the http provider.", + RuntimeWarning, + stacklevel=2, ) return None @@ -583,10 +586,10 @@ async def run( # In this case, the user probably made a mistake. # Don't retry. raise HttpErrorException(f"{e.status}:{e.message}") - else: - return response - raise NotImplementedError # should not reach this, but makes mypy happy + await asyncio.sleep(self.retry_delay) + + return response @classmethod def _process_extra_options_from_connection(cls, conn: Connection, extra_options: dict) -> dict: diff --git a/providers/src/airflow/providers/apache/livy/hooks/livy.py b/providers/src/airflow/providers/apache/livy/hooks/livy.py index 10d42043ca385..c5af2e1cfc11d 100644 --- a/providers/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/src/airflow/providers/apache/livy/hooks/livy.py @@ -18,24 +18,19 @@ from __future__ import annotations -import asyncio import json import re +import warnings from collections.abc import Sequence from enum import Enum -from typing import TYPE_CHECKING, Any +from typing import Any import aiohttp import requests -from aiohttp import ClientResponseError -from asgiref.sync import sync_to_async - from airflow.exceptions import AirflowException +from airflow.providers.http.exceptions import HttpErrorException from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook -if TYPE_CHECKING: - from airflow.models import Connection - class BatchState(Enum): """Batch session states.""" @@ -509,89 +504,9 @@ async def _do_api_call_async( headers: dict[str, Any] | None = None, extra_options: dict[str, Any] | None = None, ) -> Any: - """ - Perform an asynchronous HTTP request call. - - :param endpoint: the endpoint to be called i.e. resource/v1/query? - :param data: payload to be uploaded or request parameters - :param headers: additional headers to be passed through as a dictionary - :param extra_options: Additional kwargs to pass when creating a request. - For example, ``run(json=obj)`` is passed as ``aiohttp.ClientSession().get(json=obj)`` - """ - extra_options = extra_options or {} - - # headers may be passed through directly or in the "extra" field in the connection - # definition - _headers = {} - auth = None + warnings.warn("The '_do_api_call_async' method is deprecated, use 'run_method' instead", DeprecationWarning, stacklevel=2) - if self.http_conn_id: - conn = await sync_to_async(self.get_connection)(self.http_conn_id) - - self.base_url = self._generate_base_url(conn) - if conn.login: - auth = self.auth_type(conn.login, conn.password) - if conn.extra: - try: - _headers.update(conn.extra_dejson) - except TypeError: - self.log.warning("Connection to %s has invalid extra field.", conn.host) - if headers: - _headers.update(headers) - - if self.base_url and not self.base_url.endswith("/") and endpoint and not endpoint.startswith("/"): - url = self.base_url + "/" + endpoint - else: - url = (self.base_url or "") + (endpoint or "") - - async with aiohttp.ClientSession() as session: - if self.method == "GET": - request_func = session.get - elif self.method == "POST": - request_func = session.post - elif self.method == "PATCH": - request_func = session.patch - else: - return {"Response": f"Unexpected HTTP Method: {self.method}", "status": "error"} - - for attempt_num in range(1, 1 + self.retry_limit): - response = await request_func( - url, - json=data if self.method in ("POST", "PATCH") else None, - params=data if self.method == "GET" else None, - headers=headers, - auth=auth, - **extra_options, - ) - try: - response.raise_for_status() - return await response.json() - except ClientResponseError as e: - self.log.warning( - "[Try %d of %d] Request to %s failed.", - attempt_num, - self.retry_limit, - url, - ) - if not self._retryable_error_async(e) or attempt_num == self.retry_limit: - self.log.exception("HTTP error, status code: %s", e.status) - # In this case, the user probably made a mistake. - # Don't retry. - return {"Response": {e.message}, "Status Code": {e.status}, "status": "error"} - - await asyncio.sleep(self.retry_delay) - - def _generate_base_url(self, conn: Connection) -> str: - if conn.host and "://" in conn.host: - base_url: str = conn.host - else: - # schema defaults to HTTP - schema = conn.schema if conn.schema else "http" - host = conn.host if conn.host else "" - base_url = f"{schema}://{host}" - if conn.port: - base_url = f"{base_url}:{conn.port}" - return base_url + return await self.run_method(endpoint=endpoint, method=self.method, data=data, headers=headers, extra_options=extra_options) async def run_method( self, @@ -599,6 +514,7 @@ async def run_method( method: str = "GET", data: Any | None = None, headers: dict[str, Any] | None = None, + extra_options: dict[str, Any] | None = None, ) -> Any: """ Wrap HttpAsyncHook; allows to change method on the same HttpAsyncHook. @@ -615,7 +531,18 @@ async def run_method( back_method = self.method self.method = method try: - result = await self._do_api_call_async(endpoint, data, headers, self.extra_options) + async with aiohttp.ClientSession() as session: + try: + result = await super().run( + session=session, + endpoint=endpoint, + data=data, + headers=headers, + extra_options=extra_options or self.extra_options, + ) + except HttpErrorException as e: + status, message = str(e).split(":", 1) + return {"Response": {message}, "Status Code": {status}, "status": "error"} finally: self.method = back_method return {"status": "success", "response": result} diff --git a/providers/tests/apache/livy/hooks/test_livy.py b/providers/tests/apache/livy/hooks/test_livy.py index 6cb6dd911600c..0e48539f04148 100644 --- a/providers/tests/apache/livy/hooks/test_livy.py +++ b/providers/tests/apache/livy/hooks/test_livy.py @@ -504,160 +504,33 @@ async def test_dump_batch_logs_error(self, mock_get_batch_logs): assert log_dump == {"id": 1, "log": ["mock_log_1", "mock_log_2"]} @pytest.mark.asyncio - @mock.patch("airflow.providers.apache.livy.hooks.livy.LivyAsyncHook._do_api_call_async") - async def test_run_method_success(self, mock_do_api_call_async): + @mock.patch("airflow.providers.http.hooks.http.HttpAsyncHook.run") + async def test_run_method_success(self, mock_run): """Asserts the run_method for success response.""" - mock_do_api_call_async.return_value = {"status": "error", "response": {"id": 1}} + mock_run.return_value = {"status": "error", "response": {"id": 1}} hook = LivyAsyncHook(livy_conn_id=LIVY_CONN_ID) response = await hook.run_method("localhost", "GET") assert response["status"] == "success" @pytest.mark.asyncio - @mock.patch("airflow.providers.apache.livy.hooks.livy.LivyAsyncHook._do_api_call_async") - async def test_run_method_error(self, mock_do_api_call_async): + @mock.patch("airflow.providers.http.hooks.http.HttpAsyncHook.run") + async def test_run_method_error(self, mock_run): """Asserts the run_method for error response.""" - mock_do_api_call_async.return_value = {"status": "error", "response": {"id": 1}} + mock_run.return_value = {"status": "error", "response": {"id": 1}} hook = LivyAsyncHook(livy_conn_id=LIVY_CONN_ID) response = await hook.run_method("localhost", "abc") assert response == {"status": "error", "response": "Invalid http method abc"} @pytest.mark.asyncio - @mock.patch("airflow.providers.apache.livy.hooks.livy.aiohttp.ClientSession") - @mock.patch("airflow.providers.apache.livy.hooks.livy.LivyAsyncHook.get_connection") - async def test_do_api_call_async_post_method_with_success(self, mock_get_connection, mock_session): - """Asserts the _do_api_call_async for success response for POST method.""" - - async def mock_fun(arg1, arg2, arg3, arg4): - return {"status": "success"} - - mock_session.return_value.__aexit__.return_value = mock_fun - mock_session.return_value.__aenter__.return_value.post = AsyncMock() - mock_session.return_value.__aenter__.return_value.post.return_value.json = AsyncMock( - return_value={"status": "success"} - ) - GET_RUN_ENDPOINT = "api/jobs/runs/get" - hook = LivyAsyncHook(livy_conn_id=LIVY_CONN_ID) - hook.http_conn_id = mock_get_connection - hook.http_conn_id.host = "https://localhost" - hook.http_conn_id.login = "login" - hook.http_conn_id.password = "PASSWORD" - response = await hook._do_api_call_async(GET_RUN_ENDPOINT) - assert response == {"status": "success"} - - @pytest.mark.asyncio - @mock.patch("airflow.providers.apache.livy.hooks.livy.aiohttp.ClientSession") - @mock.patch("airflow.providers.apache.livy.hooks.livy.LivyAsyncHook.get_connection") - async def test_do_api_call_async_get_method_with_success(self, mock_get_connection, mock_session): - """Asserts the _do_api_call_async for GET method.""" - - async def mock_fun(arg1, arg2, arg3, arg4): - return {"status": "success"} - - mock_session.return_value.__aexit__.return_value = mock_fun - mock_session.return_value.__aenter__.return_value.get = AsyncMock() - mock_session.return_value.__aenter__.return_value.get.return_value.json = AsyncMock( - return_value={"status": "success"} - ) - GET_RUN_ENDPOINT = "api/jobs/runs/get" - hook = LivyAsyncHook(livy_conn_id=LIVY_CONN_ID) - hook.method = "GET" - hook.http_conn_id = mock_get_connection - hook.http_conn_id.host = "test.com" - hook.http_conn_id.login = "login" - hook.http_conn_id.password = "PASSWORD" - hook.http_conn_id.extra_dejson = "" - response = await hook._do_api_call_async(GET_RUN_ENDPOINT) - assert response == {"status": "success"} - - @pytest.mark.asyncio - @mock.patch("airflow.providers.apache.livy.hooks.livy.aiohttp.ClientSession") - @mock.patch("airflow.providers.apache.livy.hooks.livy.LivyAsyncHook.get_connection") - async def test_do_api_call_async_patch_method_with_success(self, mock_get_connection, mock_session): - """Asserts the _do_api_call_async for PATCH method.""" - - async def mock_fun(arg1, arg2, arg3, arg4): - return {"status": "success"} - - mock_session.return_value.__aexit__.return_value = mock_fun - mock_session.return_value.__aenter__.return_value.patch = AsyncMock() - mock_session.return_value.__aenter__.return_value.patch.return_value.json = AsyncMock( - return_value={"status": "success"} - ) - GET_RUN_ENDPOINT = "api/jobs/runs/get" - hook = LivyAsyncHook(livy_conn_id=LIVY_CONN_ID) - hook.method = "PATCH" - hook.http_conn_id = mock_get_connection - hook.http_conn_id.host = "test.com" - hook.http_conn_id.login = "login" - hook.http_conn_id.password = "PASSWORD" - hook.http_conn_id.extra_dejson = "" - response = await hook._do_api_call_async(GET_RUN_ENDPOINT) - assert response == {"status": "success"} - - @pytest.mark.asyncio - @mock.patch("airflow.providers.apache.livy.hooks.livy.aiohttp.ClientSession") - @mock.patch("airflow.providers.apache.livy.hooks.livy.LivyAsyncHook.get_connection") - async def test_do_api_call_async_unexpected_method_error(self, mock_get_connection, mock_session): - """Asserts the _do_api_call_async for unexpected method error""" - GET_RUN_ENDPOINT = "api/jobs/runs/get" - hook = LivyAsyncHook(livy_conn_id=LIVY_CONN_ID) - hook.method = "abc" - hook.http_conn_id = mock_get_connection - hook.http_conn_id.host = "test.com" - hook.http_conn_id.login = "login" - hook.http_conn_id.password = "PASSWORD" - hook.http_conn_id.extra_dejson = "" - response = await hook._do_api_call_async(endpoint=GET_RUN_ENDPOINT, headers={}) - assert response == {"Response": "Unexpected HTTP Method: abc", "status": "error"} - - @pytest.mark.asyncio - @mock.patch("airflow.providers.apache.livy.hooks.livy.aiohttp.ClientSession") - @mock.patch("airflow.providers.apache.livy.hooks.livy.LivyAsyncHook.get_connection") - async def test_do_api_call_async_with_type_error(self, mock_get_connection, mock_session): - """Asserts the _do_api_call_async for TypeError.""" - - async def mock_fun(arg1, arg2, arg3, arg4): - return {"random value"} + @mock.patch("airflow.providers.http.hooks.http.HttpAsyncHook.run") + async def test_run_method_http_error(self, mock_run): + """Asserts the run_method for error response.""" + from airflow.providers.http.exceptions import HttpErrorException - mock_session.return_value.__aexit__.return_value = mock_fun - mock_session.return_value.__aenter__.return_value.patch.return_value.json.return_value = {} + mock_run.side_effect = HttpErrorException("404:Unauthorized") hook = LivyAsyncHook(livy_conn_id=LIVY_CONN_ID) - hook.method = "PATCH" - hook.retry_limit = 1 - hook.retry_delay = 1 - hook.http_conn_id = mock_get_connection - with pytest.raises(TypeError): - await hook._do_api_call_async(endpoint="", data="test", headers=mock_fun, extra_options=mock_fun) - - @pytest.mark.asyncio - @mock.patch("airflow.providers.apache.livy.hooks.livy.aiohttp.ClientSession") - @mock.patch("airflow.providers.apache.livy.hooks.livy.LivyAsyncHook.get_connection") - async def test_do_api_call_async_with_client_response_error(self, mock_get_connection, mock_session): - """Asserts the _do_api_call_async for Client Response Error.""" - - async def mock_fun(arg1, arg2, arg3, arg4): - return {"random value"} - - mock_session.return_value.__aexit__.return_value = mock_fun - mock_session.return_value.__aenter__.return_value.patch = AsyncMock() - mock_session.return_value.__aenter__.return_value.patch.return_value.json.side_effect = ( - ClientResponseError( - request_info=RequestInfo(url="example.com", method="PATCH", headers=multidict.CIMultiDict()), - status=500, - history=[], - ) - ) - GET_RUN_ENDPOINT = "" - hook = LivyAsyncHook(livy_conn_id="livy_default") - hook.method = "PATCH" - hook.base_url = "" - hook.http_conn_id = mock_get_connection - hook.http_conn_id.host = "test.com" - hook.http_conn_id.login = "login" - hook.http_conn_id.password = "PASSWORD" - hook.http_conn_id.extra_dejson = "" - response = await hook._do_api_call_async(GET_RUN_ENDPOINT) - assert response["status"] == "error" + response = await hook.run_method("localhost", "POST") + assert response == {"Response": {"Unauthorized"}, "Status Code": {"404"}, "status": "error"} def set_conn(self): db.merge_conn( @@ -687,9 +560,9 @@ def test_build_get_hook(self): for conn_id, expected in connection_url_mapping.items(): hook = LivyAsyncHook(livy_conn_id=conn_id) - response_conn: Connection = hook.get_connection(conn_id=conn_id) + response_conn: Connection = hook.get_conn() assert isinstance(response_conn, Connection) - assert hook._generate_base_url(response_conn) == expected + assert hook.base_url == expected def test_build_body(self): # minimal request From 07e56c736cb65b8f8249fecb0c9b5da6e7c5c514 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 28 Jan 2025 21:13:33 +0100 Subject: [PATCH 131/286] refactor: Fixed static types --- .../src/airflow/providers/apache/livy/hooks/livy.py | 11 +++++++++-- providers/tests/apache/livy/hooks/test_livy.py | 4 +--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/providers/src/airflow/providers/apache/livy/hooks/livy.py b/providers/src/airflow/providers/apache/livy/hooks/livy.py index c5af2e1cfc11d..b4ef4d63c0a11 100644 --- a/providers/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/src/airflow/providers/apache/livy/hooks/livy.py @@ -27,6 +27,7 @@ import aiohttp import requests + from airflow.exceptions import AirflowException from airflow.providers.http.exceptions import HttpErrorException from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook @@ -504,9 +505,15 @@ async def _do_api_call_async( headers: dict[str, Any] | None = None, extra_options: dict[str, Any] | None = None, ) -> Any: - warnings.warn("The '_do_api_call_async' method is deprecated, use 'run_method' instead", DeprecationWarning, stacklevel=2) + warnings.warn( + "The '_do_api_call_async' method is deprecated, use 'run_method' instead", + DeprecationWarning, + stacklevel=2, + ) - return await self.run_method(endpoint=endpoint, method=self.method, data=data, headers=headers, extra_options=extra_options) + return await self.run_method( + endpoint=endpoint, method=self.method, data=data, headers=headers, extra_options=extra_options + ) async def run_method( self, diff --git a/providers/tests/apache/livy/hooks/test_livy.py b/providers/tests/apache/livy/hooks/test_livy.py index 0e48539f04148..02fdd790b297f 100644 --- a/providers/tests/apache/livy/hooks/test_livy.py +++ b/providers/tests/apache/livy/hooks/test_livy.py @@ -18,12 +18,10 @@ import json from unittest import mock -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import MagicMock, patch -import multidict import pytest import requests -from aiohttp import ClientResponseError, RequestInfo from requests.exceptions import RequestException from airflow.exceptions import AirflowException From 72d21584b41b83d6962739718294cb62ff5edb6c Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 28 Jan 2025 21:18:55 +0100 Subject: [PATCH 132/286] refactor: Refactored run_method of LivyAsyncHook --- .../providers/apache/livy/hooks/livy.py | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/providers/src/airflow/providers/apache/livy/hooks/livy.py b/providers/src/airflow/providers/apache/livy/hooks/livy.py index b4ef4d63c0a11..d6ac68f1234b8 100644 --- a/providers/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/src/airflow/providers/apache/livy/hooks/livy.py @@ -25,7 +25,6 @@ from enum import Enum from typing import Any -import aiohttp import requests from airflow.exceptions import AirflowException @@ -530,6 +529,7 @@ async def run_method( :param endpoint: endpoint :param data: request payload :param headers: headers + :param extra_options: Additional kwargs to pass when creating a request. :return: http response """ if method not in ("GET", "POST", "PUT", "DELETE", "HEAD"): @@ -538,18 +538,15 @@ async def run_method( back_method = self.method self.method = method try: - async with aiohttp.ClientSession() as session: - try: - result = await super().run( - session=session, - endpoint=endpoint, - data=data, - headers=headers, - extra_options=extra_options or self.extra_options, - ) - except HttpErrorException as e: - status, message = str(e).split(":", 1) - return {"Response": {message}, "Status Code": {status}, "status": "error"} + result = await super().run( + endpoint=endpoint, + data=data, + headers=headers, + extra_options=extra_options or self.extra_options, + ) + except HttpErrorException as e: + status, message = str(e).split(":", 1) + return {"Response": {message}, "Status Code": {status}, "status": "error"} finally: self.method = back_method return {"status": "success", "response": result} From 5e08337285dad12c3ef42cc8d5d3950a4c840b31 Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 29 Jan 2025 10:28:50 +0100 Subject: [PATCH 133/286] refactor: Raise AirflowProviderDeprecationWarning instead of DeprecationWarning when calling deprecated _do_api_call_async method and added unit test --- .../src/airflow/providers/apache/livy/hooks/livy.py | 4 ++-- providers/tests/apache/livy/hooks/test_livy.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/providers/src/airflow/providers/apache/livy/hooks/livy.py b/providers/src/airflow/providers/apache/livy/hooks/livy.py index d6ac68f1234b8..0b6879750f393 100644 --- a/providers/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/src/airflow/providers/apache/livy/hooks/livy.py @@ -27,7 +27,7 @@ import requests -from airflow.exceptions import AirflowException +from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.providers.http.exceptions import HttpErrorException from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook @@ -506,7 +506,7 @@ async def _do_api_call_async( ) -> Any: warnings.warn( "The '_do_api_call_async' method is deprecated, use 'run_method' instead", - DeprecationWarning, + AirflowProviderDeprecationWarning, stacklevel=2, ) diff --git a/providers/tests/apache/livy/hooks/test_livy.py b/providers/tests/apache/livy/hooks/test_livy.py index 02fdd790b297f..5fc474e63df19 100644 --- a/providers/tests/apache/livy/hooks/test_livy.py +++ b/providers/tests/apache/livy/hooks/test_livy.py @@ -501,6 +501,18 @@ async def test_dump_batch_logs_error(self, mock_get_batch_logs): log_dump = await hook.dump_batch_logs(BATCH_ID) assert log_dump == {"id": 1, "log": ["mock_log_1", "mock_log_2"]} + @pytest.mark.asyncio + @mock.patch("airflow.providers.http.hooks.http.HttpAsyncHook.run") + async def test_do_api_call_async_gives_deprecation_warning(self, mock_run): + """Asserts the run_method for success response.""" + from airflow.exceptions import AirflowProviderDeprecationWarning + + mock_run.return_value = {"status": "error", "response": {"id": 1}} + hook = LivyAsyncHook(livy_conn_id=LIVY_CONN_ID) + with pytest.warns(AirflowProviderDeprecationWarning, match="deprecated"): + response = await hook._do_api_call_async("localhost") + assert response["status"] == "success" + @pytest.mark.asyncio @mock.patch("airflow.providers.http.hooks.http.HttpAsyncHook.run") async def test_run_method_success(self, mock_run): From eae6600514d9784d751c36e9750ce8f5426508c4 Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 29 Jan 2025 10:30:36 +0100 Subject: [PATCH 134/286] refactor: Raise NotImplementedError in HttpHook run for mypy --- providers/http/src/airflow/providers/http/hooks/http.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 59f198741bc62..a76c6081d09ec 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -591,6 +591,8 @@ async def run( return response + raise NotImplementedError # should not reach this, but makes mypy happy + @classmethod def _process_extra_options_from_connection(cls, conn: Connection, extra_options: dict) -> dict: extra = conn.extra_dejson From 8767e6eda6bacf74194c317de3c749378da84166 Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 29 Jan 2025 10:32:54 +0100 Subject: [PATCH 135/286] refactor: Pass empty string if endpoint is None --- providers/src/airflow/providers/apache/livy/hooks/livy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/src/airflow/providers/apache/livy/hooks/livy.py b/providers/src/airflow/providers/apache/livy/hooks/livy.py index 0b6879750f393..b09e6332911af 100644 --- a/providers/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/src/airflow/providers/apache/livy/hooks/livy.py @@ -511,7 +511,7 @@ async def _do_api_call_async( ) return await self.run_method( - endpoint=endpoint, method=self.method, data=data, headers=headers, extra_options=extra_options + endpoint=endpoint or "", method=self.method, data=data, headers=headers, extra_options=extra_options ) async def run_method( From a2f9708e7a1760585d5e343b64f0e2093e64de01 Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 29 Jan 2025 11:02:12 +0100 Subject: [PATCH 136/286] refactor: Mock connection instead of persisting them in database --- .../tests/apache/livy/hooks/test_livy.py | 85 ++++++++++--------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/providers/tests/apache/livy/hooks/test_livy.py b/providers/tests/apache/livy/hooks/test_livy.py index 5fc474e63df19..bf17ecfad89e5 100644 --- a/providers/tests/apache/livy/hooks/test_livy.py +++ b/providers/tests/apache/livy/hooks/test_livy.py @@ -22,14 +22,11 @@ import pytest import requests -from requests.exceptions import RequestException - from airflow.exceptions import AirflowException from airflow.models import Connection from airflow.providers.apache.livy.hooks.livy import BatchState, LivyAsyncHook, LivyHook from airflow.utils import db - -from tests_common.test_utils.db import clear_db_connections +from requests.exceptions import RequestException LIVY_CONN_ID = LivyHook.default_conn_name DEFAULT_CONN_ID = LivyHook.default_conn_name @@ -49,41 +46,32 @@ pytest.param("forty two", id="invalid string"), pytest.param({"a": "b"}, id="dictionary"), ] - - -@pytest.mark.db_test -class TestLivyDbHook: - @classmethod - def setup_class(cls): - clear_db_connections(add_default_connections_back=False) - db.merge_conn( - Connection( +CONNECTIONS: dict[str, Connection] = { + DEFAULT_CONN_ID: Connection( conn_id=DEFAULT_CONN_ID, conn_type="http", host=DEFAULT_HOST, schema=DEFAULT_SCHEMA, port=DEFAULT_PORT, - ) - ) - db.merge_conn(Connection(conn_id="default_port", conn_type="http", host="http://host")) - db.merge_conn(Connection(conn_id="default_protocol", conn_type="http", host="host")) - db.merge_conn(Connection(conn_id="port_set", host="host", conn_type="http", port=1234)) - db.merge_conn(Connection(conn_id="schema_set", host="host", conn_type="http", schema="https")) - db.merge_conn( - Connection(conn_id="dont_override_schema", conn_type="http", host="http://host", schema="https") - ) - db.merge_conn(Connection(conn_id="missing_host", conn_type="http", port=1234)) - db.merge_conn(Connection(conn_id="invalid_uri", uri="http://invalid_uri:4321")) - db.merge_conn( - Connection( - conn_id="with_credentials", login="login", password="secret", conn_type="http", host="host" - ) - ) + ), + "default_port": Connection(conn_id="default_port", conn_type="http", host="http://host"), + "default_protocol": Connection(conn_id="default_protocol", conn_type="http", host="host"), + "port_set": Connection(conn_id="port_set", host="host", conn_type="http", port=1234), + "schema_set": Connection(conn_id="schema_set", host="host", conn_type="http", schema="https"), + "dont_override_schema": Connection(conn_id="dont_override_schema", conn_type="http", host="http://host", schema="https"), + "missing_host": Connection(conn_id="missing_host", conn_type="http", port=1234), + "invalid_uri": Connection(conn_id="invalid_uri", uri="http://invalid_uri:4321"), + "with_credentials": Connection( + conn_id="with_credentials", login="login", password="secret", conn_type="http", host="host" + ), +} + + +def get_connection(conn_id: str) -> Connection: + return CONNECTIONS[conn_id] - @classmethod - def teardown_class(cls): - clear_db_connections(add_default_connections_back=True) +class TestLivyDbHook: @pytest.mark.db_test @pytest.mark.parametrize( "conn_id, expected", @@ -95,10 +83,15 @@ def teardown_class(cls): pytest.param("dont_override_schema", "http://host", id="ignore-defined-schema"), ], ) + #@mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_connection) def test_build_get_hook(self, conn_id, expected): - hook = LivyHook(livy_conn_id=conn_id) - hook.get_conn() - assert hook.base_url == expected + with patch( + "airflow.hooks.base.BaseHook.get_connection", + side_effect=get_connection, + ): + hook = LivyHook(livy_conn_id=conn_id) + hook.get_conn() + assert hook.base_url == expected @pytest.mark.skip("Inherited HttpHook does not handle missing hostname") def test_missing_host(self): @@ -307,8 +300,12 @@ def test_post_batch_calls_get_conn_if_no_batch_id(self, mock_get_conn, mock_run_ mock_get_conn.assert_not_called() def test_invalid_uri(self): - with pytest.raises(RequestException): - LivyHook(livy_conn_id="invalid_uri").post_batch(file="sparkapp") + with patch( + "airflow.hooks.base.BaseHook.get_connection", + side_effect=get_connection, + ): + with pytest.raises(RequestException): + LivyHook(livy_conn_id="invalid_uri").post_batch(file="sparkapp") def test_get_batch_state_success(self, requests_mock): running = BatchState.RUNNING @@ -416,15 +413,19 @@ def test_extra_headers(self, requests_mock): hook.post_batch(file="sparkapp") def test_alternate_auth_type(self): - auth_type = MagicMock() + with patch( + "airflow.hooks.base.BaseHook.get_connection", + side_effect=get_connection, + ): + auth_type = MagicMock() - hook = LivyHook(livy_conn_id="with_credentials", auth_type=auth_type) + hook = LivyHook(livy_conn_id="with_credentials", auth_type=auth_type) - auth_type.assert_not_called() + auth_type.assert_not_called() - hook.get_conn() + hook.get_conn() - auth_type.assert_called_once_with("login", "secret") + auth_type.assert_called_once_with("login", "secret") class TestLivyAsyncHook: From 17af2fd090e3b6cf3c19338b9e0cee1fd78f8fad Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 29 Jan 2025 11:04:55 +0100 Subject: [PATCH 137/286] refactor: Reformatted _do_api_call_async --- providers/src/airflow/providers/apache/livy/hooks/livy.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/providers/src/airflow/providers/apache/livy/hooks/livy.py b/providers/src/airflow/providers/apache/livy/hooks/livy.py index b09e6332911af..8b2019e29afdc 100644 --- a/providers/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/src/airflow/providers/apache/livy/hooks/livy.py @@ -511,7 +511,11 @@ async def _do_api_call_async( ) return await self.run_method( - endpoint=endpoint or "", method=self.method, data=data, headers=headers, extra_options=extra_options + endpoint=endpoint or "", + method=self.method, + data=data, + headers=headers, + extra_options=extra_options, ) async def run_method( From 0c7fd4e81cd590dfce9a596220acd5f41a3ccffa Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Sun, 12 Nov 2023 13:55:18 +0100 Subject: [PATCH 138/286] feat: Implement `auth_kwargs` parameter in Http Connection --- .../provider_tests/http/hooks/test_http.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index 82a1ff9765156..e1a380e4588e5 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -67,6 +67,11 @@ def get_airflow_connection_with_login_and_password(conn_id: str = "http_default" return Connection(conn_id=conn_id, conn_type="http", host="test.com", login="username", password="pass") +class CustomAuthBase(HTTPBasicAuth): + def __init__(self, username: str, password: str, endpoint: str): + super().__init__(username, password) + + class TestHttpHook: """Test get, post and raise_for_status""" @@ -352,6 +357,25 @@ def test_connection_without_host(self, mock_get_connection): hook.get_conn({}) assert hook.base_url == "http://" + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_connection): + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + login="username", + password="pass", + extra='{"auth_kwargs": {"endpoint": "http://localhost"}}', + ) + mock_get_connection.return_value = conn + + hook = HttpHook(auth_type=CustomAuthBase) + session = hook.get_conn({}) + + auth.assert_called_once_with("username", "pass", endpoint="http://localhost") + assert "auth_kwargs" not in session.headers + @pytest.mark.parametrize("method", ["GET", "POST"]) def test_json_request(self, method, requests_mock): obj1 = {"a": 1, "b": "abc", "c": [1, 2, {"d": 10}]} From b28d3f645e19469cf570a027970c94a5208c7f46 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 24 Nov 2023 00:25:53 +0100 Subject: [PATCH 139/286] fix: Correctly use auth_type from Connection --- .../provider_tests/http/hooks/test_http.py | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index e1a380e4588e5..802dd0ef15c39 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -366,7 +366,7 @@ def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_conne conn_type="http", login="username", password="pass", - extra='{"auth_kwargs": {"endpoint": "http://localhost"}}', + extra='{"x-header": 0, "auth_kwargs": {"endpoint": "http://localhost"}}', ) mock_get_connection.return_value = conn @@ -375,6 +375,40 @@ def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_conne auth.assert_called_once_with("username", "pass", endpoint="http://localhost") assert "auth_kwargs" not in session.headers + assert "x-header" in session.headers + + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_header_and_auth_type(self, auth, mock_get_connection): + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + login="username", + password="pass", + extra='{"x-header": 0, "auth_type": "tests.providers.http.hooks.test_http.CustomAuthBase"}', + ) + mock_get_connection.return_value = conn + + session = HttpHook().get_conn({}) + auth.assert_called_once_with("username", "pass") + assert isinstance(session.auth, CustomAuthBase) + assert "auth_type" not in session.headers + assert "x-header" in session.headers + + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_auth_type_and_no_credentials(self, auth, mock_get_connection): + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + extra='{"auth_type": "tests.providers.http.hooks.test_http.CustomAuthBase"}', + ) + mock_get_connection.return_value = conn + + HttpHook().get_conn({}) + auth.assert_called_once() @pytest.mark.parametrize("method", ["GET", "POST"]) def test_json_request(self, method, requests_mock): From f20caccc7b8e39eee07d54471be47ccfce696390 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 24 Nov 2023 01:55:44 +0100 Subject: [PATCH 140/286] feat: Add Connection documentation --- providers/http/src/airflow/providers/http/hooks/http.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index b22a01f8283db..eb63e27ee0925 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -51,6 +51,12 @@ class HttpHook(BaseHook): """ Interact with HTTP servers. + To configure the auth_type, in addition to the `auth_type` parameter, you can also: + * set the `auth_type` parameter in the Connection settings. + * define extra parameters used to instantiate the `auth_type` class, in the Connection settings. + + See :doc:`/connections/http` for full documentation. + :param method: the API method to be called :param http_conn_id: :ref:`http connection` that has the base API url i.e https://www.google.com/ and optional authentication credentials. Default From 0223d7a4ef8f94baec91ac05bff2d8d67ce56bed Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 5 Dec 2023 23:14:43 +0100 Subject: [PATCH 141/286] feat: Make available auth_types configurable from airflow config --- providers/http/docs/configurations-ref.rst | 0 providers/http/docs/index.rst | 1 + providers/http/provider.yaml | 15 ++++++++++ .../provider_tests/http/hooks/test_http.py | 28 ++++++++++++++++++- 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 providers/http/docs/configurations-ref.rst diff --git a/providers/http/docs/configurations-ref.rst b/providers/http/docs/configurations-ref.rst new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/providers/http/docs/index.rst b/providers/http/docs/index.rst index 49745912f879b..7fc650ea8f50f 100644 --- a/providers/http/docs/index.rst +++ b/providers/http/docs/index.rst @@ -42,6 +42,7 @@ :maxdepth: 1 :caption: References + Configuration Python API <_api/airflow/providers/http/index> .. toctree:: diff --git a/providers/http/provider.yaml b/providers/http/provider.yaml index dee0796c04891..7a991044ea801 100644 --- a/providers/http/provider.yaml +++ b/providers/http/provider.yaml @@ -93,3 +93,18 @@ triggers: connection-types: - hook-class-name: airflow.providers.http.hooks.http.HttpHook connection-type: http + +config: + http: + description: "Options for Http provider." + options: + extra_auth_types: + description: | + A comma separated list of auth_type classes, which can be used to + configure Http Connections in Airflow's UI. This list restricts which + classes can be arbitrary imported, and protects from dependency + injections. + type: string + version_added: 4.8.0 + example: "requests_kerberos.HTTPKerberosAuth,any.other.custom.HTTPAuth" + default: ~ diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index 802dd0ef15c39..fe6b8f882f2b7 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -36,7 +36,7 @@ from airflow.exceptions import AirflowException from airflow.models import Connection -from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook +from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook, get_auth_types @pytest.fixture @@ -72,6 +72,9 @@ def __init__(self, username: str, password: str, endpoint: str): super().__init__(username, password) +@mock.patch.dict( + "os.environ", AIRFLOW__HTTP__EXTRA_AUTH_TYPES="tests.providers.http.hooks.test_http.CustomAuthBase" +) class TestHttpHook: """Test get, post and raise_for_status""" @@ -377,6 +380,29 @@ def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_conne assert "auth_kwargs" not in session.headers assert "x-header" in session.headers + def test_available_connection_auth_types(self): + auth_types = get_auth_types() + assert auth_types == frozenset( + { + "request.auth.HTTPBasicAuth", + "request.auth.HTTPProxyAuth", + "request.auth.HTTPDigestAuth", + "tests.providers.http.hooks.test_http.CustomAuthBase", + } + ) + + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + def test_connection_with_invalid_auth_type_get_skipped(self, mock_get_connection, caplog): + auth_type: str = "auth_type.class.not.available.for.Import" + conn = Connection( + conn_id="http_default", + conn_type="http", + extra=f'{{"auth_type": "{auth_type}"}}', + ) + mock_get_connection.return_value = conn + HttpHook().get_conn({}) + assert f"Skipping import of auth_type '{auth_type}'." in caplog.text + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") def test_connection_with_extra_header_and_auth_type(self, auth, mock_get_connection): From 91f9b00be3e416aebfa9a6e79f532ab19af9d2b1 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Wed, 6 Dec 2023 22:24:25 +0100 Subject: [PATCH 142/286] feat: Add fields for auth config and header config in Http Connection form --- .../src/airflow/providers/http/hooks/http.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index eb63e27ee0925..31a0bc5cfc8a5 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -113,6 +113,30 @@ def auth_type(self): def auth_type(self, v): self._auth_type = v + @classmethod + def get_connection_form_widgets(cls) -> dict[str, Any]: + """Return connection widgets to add to connection form.""" + from flask_babel import lazy_gettext + from wtforms.fields import SelectField, TextAreaField + + auth_types_choices = frozenset({""}) | get_auth_types() + return { + "auth_type": SelectField( + lazy_gettext("Auth type"), + choices=[(clazz, clazz) for clazz in auth_types_choices] + ), + "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs")), + "extra_headers": TextAreaField(lazy_gettext("Extra Headers")) + } + + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + """Return custom field behaviour.""" + return { + "hidden_fields": ["extra"], + "relabeling": {} + } + # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: From eaf8c376f22a8fad025ce0dee725792364d0196c Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Thu, 7 Dec 2023 08:48:01 +0100 Subject: [PATCH 143/286] fix: Correctly apply styling to extra fields --- .../http/src/airflow/providers/http/hooks/http.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 31a0bc5cfc8a5..538bc1e6e38cb 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -116,6 +116,7 @@ def auth_type(self, v): @classmethod def get_connection_form_widgets(cls) -> dict[str, Any]: """Return connection widgets to add to connection form.""" + from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget, Select2Widget from flask_babel import lazy_gettext from wtforms.fields import SelectField, TextAreaField @@ -123,19 +124,17 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: return { "auth_type": SelectField( lazy_gettext("Auth type"), - choices=[(clazz, clazz) for clazz in auth_types_choices] + choices=[(clazz, clazz) for clazz in auth_types_choices], + widget=Select2Widget(), ), - "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs")), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers")) + "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), + "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } @classmethod def get_ui_field_behaviour(cls) -> dict[str, Any]: """Return custom field behaviour.""" - return { - "hidden_fields": ["extra"], - "relabeling": {} - } + return {"hidden_fields": ["extra"], "relabeling": {}} # headers may be passed through directly or in the "extra" field in the connection # definition From 5fcb44aea31bb104b98ec68bdcc4c64e5856550a Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 19 Dec 2023 01:05:47 +0100 Subject: [PATCH 144/286] feat: Implement simplistic collapsable textarea for "extra" --- airflow/www/forms.py | 26 ++++++++++++++++++- airflow/www/static/js/connection_form.js | 4 +-- .../src/airflow/providers/http/hooks/http.py | 9 +++---- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/airflow/www/forms.py b/airflow/www/forms.py index 7028e2026e449..b69184c6590c2 100644 --- a/airflow/www/forms.py +++ b/airflow/www/forms.py @@ -33,6 +33,7 @@ from flask_appbuilder.forms import DynamicForm from flask_babel import lazy_gettext from flask_wtf import FlaskForm +from markupsafe import Markup from wtforms import widgets from wtforms.fields import Field, IntegerField, PasswordField, SelectField, StringField, TextAreaField from wtforms.validators import InputRequired, Optional @@ -176,6 +177,29 @@ def populate_obj(self, item): field.populate_obj(item, name) +class BS3AccordionTextAreaFieldWidget(BS3TextAreaFieldWidget): + + @staticmethod + def _make_collapsable_panel(field: Field, content: Markup) -> str: + collapsable_id: str = f"collapsable_{field.id}" + return f""" +
+
+

+ +

+
+ +
+ """ + + def __call__(self, field, **kwargs): + text_area = super(BS3TextAreaFieldWidget, self).__call__(field, **kwargs) + return self._make_collapsable_panel(field=field, content=text_area) + + @cache def create_connection_form_class() -> type[DynamicForm]: """ @@ -223,7 +247,7 @@ def process(self, formdata=None, obj=None, **kwargs): login = StringField(lazy_gettext("Login"), widget=BS3TextFieldWidget()) password = PasswordField(lazy_gettext("Password"), widget=BS3PasswordFieldWidget()) port = IntegerField(lazy_gettext("Port"), validators=[Optional()], widget=BS3TextFieldWidget()) - extra = TextAreaField(lazy_gettext("Extra"), widget=BS3TextAreaFieldWidget()) + extra = TextAreaField(lazy_gettext("Extra"), widget=BS3AccordionTextAreaFieldWidget()) for key, value in providers_manager.connection_form_widgets.items(): setattr(ConnectionForm, key, value.field) diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index d039fc7275462..1c97e00803174 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -83,7 +83,7 @@ function restoreFieldBehaviours() { } ); - Array.from(document.querySelectorAll(".form-control")).forEach((elem) => { + Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach((elem) => { // eslint-disable-next-line no-param-reassign elem.placeholder = ""; elem.parentElement.parentElement.classList.remove("hide"); @@ -101,7 +101,7 @@ function applyFieldBehaviours(connection) { if (Array.isArray(connection.hidden_fields)) { connection.hidden_fields.forEach((field) => { document - .getElementById(field) + .querySelector(`label[for='${field}']`) .parentElement.parentElement.classList.add("hide"); }); } diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 538bc1e6e38cb..7268c13f8764d 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -25,6 +25,8 @@ import tenacity from aiohttp import ClientResponseError from asgiref.sync import sync_to_async +from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget +from markupsafe import Markup from requests.auth import HTTPBasicAuth from requests.models import DEFAULT_REDIRECT_LIMIT from requests_toolbelt.adapters.socket_options import TCPKeepAliveAdapter @@ -128,14 +130,9 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: widget=Select2Widget(), ), "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), + "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()) } - @classmethod - def get_ui_field_behaviour(cls) -> dict[str, Any]: - """Return custom field behaviour.""" - return {"hidden_fields": ["extra"], "relabeling": {}} - # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: From c284414e2b635ce9619c1d450d0d961292c084cd Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Wed, 20 Dec 2023 16:21:08 +0100 Subject: [PATCH 145/286] fix: express clearly empty frozenset creation Goal is to have an empty default choice --- providers/http/src/airflow/providers/http/hooks/http.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 7268c13f8764d..0f6bea66bf624 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -122,7 +122,8 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: from flask_babel import lazy_gettext from wtforms.fields import SelectField, TextAreaField - auth_types_choices = frozenset({""}) | get_auth_types() + default_auth_type = frozenset({""}) + auth_types_choices = default_auth_type | get_auth_types() return { "auth_type": SelectField( lazy_gettext("Auth type"), From ef7c0700a63298b80f271d393152cf39441b277b Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Wed, 20 Dec 2023 16:43:29 +0100 Subject: [PATCH 146/286] feat: Refactor Accordion TextArea to use wtform utils --- airflow/www/static/js/connection_form.js | 12 +++++++----- .../http/src/airflow/providers/http/hooks/http.py | 4 +--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index 1c97e00803174..119fe39daae54 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -83,11 +83,13 @@ function restoreFieldBehaviours() { } ); - Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach((elem) => { - // eslint-disable-next-line no-param-reassign - elem.placeholder = ""; - elem.parentElement.parentElement.classList.remove("hide"); - }); + Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach( + (elem) => { + // eslint-disable-next-line no-param-reassign + elem.placeholder = ""; + elem.parentElement.parentElement.classList.remove("hide"); + } + ); } /** diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 0f6bea66bf624..77563d2eb1a81 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -25,8 +25,6 @@ import tenacity from aiohttp import ClientResponseError from asgiref.sync import sync_to_async -from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget -from markupsafe import Markup from requests.auth import HTTPBasicAuth from requests.models import DEFAULT_REDIRECT_LIMIT from requests_toolbelt.adapters.socket_options import TCPKeepAliveAdapter @@ -131,7 +129,7 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: widget=Select2Widget(), ), "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()) + "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } # headers may be passed through directly or in the "extra" field in the connection From 138f296180b7500208df5fbf3ea2688f91027146 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Thu, 21 Dec 2023 13:04:19 +0100 Subject: [PATCH 147/286] feat: Implement 'collapse_extra' field behavior --- airflow/customized_form_field_behaviours.schema.json | 4 ++++ providers/http/src/airflow/providers/http/hooks/http.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/airflow/customized_form_field_behaviours.schema.json b/airflow/customized_form_field_behaviours.schema.json index 78791a87886c1..fa5ace958c5e8 100644 --- a/airflow/customized_form_field_behaviours.schema.json +++ b/airflow/customized_form_field_behaviours.schema.json @@ -22,6 +22,10 @@ "additionalProperties": { "type": "string" } + }, + "collapse_extra": { + "type": "boolean", + "description": "Collapse the 'Extra' field." } }, "additionalProperties": true, diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 77563d2eb1a81..2ddd9ca434efe 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -132,6 +132,10 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + return {"hidden_fields": [], "relabeling": {}, "collapse_extra": True} + # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: From 1f0a171460a61b71f17e51bec740b26252a16445 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Sat, 30 Dec 2023 16:43:47 +0100 Subject: [PATCH 148/286] feat: Implement parameterizable behavior for collapsible field --- ...stomized_form_field_behaviours.schema.json | 19 +++++-- airflow/www/static/css/connection.css | 23 +++++++++ airflow/www/static/js/connection_form.js | 49 ++++++++++++++++--- .../www/templates/airflow/conn_create.html | 2 +- airflow/www/templates/airflow/conn_edit.html | 1 + airflow/www/webpack.config.js | 1 + .../src/airflow/providers/http/hooks/http.py | 2 +- 7 files changed, 85 insertions(+), 12 deletions(-) create mode 100644 airflow/www/static/css/connection.css diff --git a/airflow/customized_form_field_behaviours.schema.json b/airflow/customized_form_field_behaviours.schema.json index fa5ace958c5e8..8aa05945ebb01 100644 --- a/airflow/customized_form_field_behaviours.schema.json +++ b/airflow/customized_form_field_behaviours.schema.json @@ -23,9 +23,22 @@ "type": "string" } }, - "collapse_extra": { - "type": "boolean", - "description": "Collapse the 'Extra' field." + "collapsible_fields": { + "description": "List of collapsed fields for the hook, with their properties.", + "type": "object", + "patternProperties": { + "\"^.*$\"": { + "description": "Name of the field to enable collapsing.", + "type": "object", + "properties": { + "expanded": { + "description": "Set the default state of the field as expanded.", + "default": true, + "type": "boolean" + } + } + } + } } }, "additionalProperties": true, diff --git a/airflow/www/static/css/connection.css b/airflow/www/static/css/connection.css new file mode 100644 index 0000000000000..78edf0db5d4dc --- /dev/null +++ b/airflow/www/static/css/connection.css @@ -0,0 +1,23 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +.panel-invisible { + margin: 0; + border: 0; +} diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index 119fe39daae54..5c60638caf488 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -83,13 +83,28 @@ function restoreFieldBehaviours() { } ); - Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach( - (elem) => { - // eslint-disable-next-line no-param-reassign - elem.placeholder = ""; - elem.parentElement.parentElement.classList.remove("hide"); - } - ); + Array.from(document.querySelectorAll(".form-control")).forEach((elem) => { + // eslint-disable-next-line no-param-reassign + elem.placeholder = ""; + elem.parentElement.parentElement.classList.remove("hide"); + }); + + Array.from(document.querySelectorAll(".form-panel")).forEach((elem) => { + elem.parentElement.parentElement.classList.remove("hide"); + + elem.classList.add("panel-invisible"); + const panelHeader = elem.children[0]; + panelHeader.classList.add("hidden"); + panelHeader.firstElementChild.firstElementChild.setAttribute( + "aria-expanded", + "true" + ); + + const collapsible = elem.children[1]; + collapsible.setAttribute("aria-expanded", "true"); + collapsible.classList.add("in"); + collapsible.style.height = null; + }); } /** @@ -122,6 +137,26 @@ function applyFieldBehaviours(connection) { document.getElementById(field).placeholder = placeholder; }); } + + if (connection.collapsible_fields) { + Object.entries(connection.collapsible_fields).forEach((entry) => { + const [field, properties] = entry; + + const collapsibleController = document.getElementById( + `control_collapsible_${field}` + ); + const panelHeader = collapsibleController.parentElement.parentElement; + panelHeader.classList.remove("hidden"); + panelHeader.parentElement.classList.remove("panel-invisible"); + + if (properties.expanded === false) { + const collapsible = document.getElementById(`collapsible_${field}`); + collapsible.classList.remove("in"); + collapsible.setAttribute("aria-expanded", "false"); + collapsibleController.setAttribute("aria-expanded", "false"); + } + }); + } } } diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index ac92b967f7e34..307450b05d16b 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -25,7 +25,7 @@ - + {# required for codemirror #} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 653b0dd3ce07f..11ebd6c4cb436 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -25,6 +25,7 @@ + {# required for codemirror #} diff --git a/airflow/www/webpack.config.js b/airflow/www/webpack.config.js index 9d5800f783f50..ad1a7098e0803 100644 --- a/airflow/www/webpack.config.js +++ b/airflow/www/webpack.config.js @@ -60,6 +60,7 @@ const config = { airflowDefaultTheme: `${CSS_DIR}/bootstrap-theme.css`, connectionForm: `${JS_DIR}/connection_form.js`, chart: [`${CSS_DIR}/chart.css`], + connection: [`${CSS_DIR}/connection.css`], dag: `${JS_DIR}/dag.js`, dagDependencies: `${JS_DIR}/dag_dependencies.js`, dags: [`${CSS_DIR}/dags.css`, `${JS_DIR}/dags.js`], diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 2ddd9ca434efe..192e57c1d70fc 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -134,7 +134,7 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: @classmethod def get_ui_field_behaviour(cls) -> dict[str, Any]: - return {"hidden_fields": [], "relabeling": {}, "collapse_extra": True} + return {"hidden_fields": [], "relabeling": {}, "collapsible_fields": {"extra": {"expanded": False}}} # headers may be passed through directly or in the "extra" field in the connection # definition From 38aeb730b7d5cbdde0978da59c88b06882b6ed85 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 5 Jan 2024 06:48:16 +0100 Subject: [PATCH 149/286] revert: Remove collapsible field --- ...stomized_form_field_behaviours.schema.json | 17 -------- airflow/www/static/css/connection.css | 23 ----------- airflow/www/static/js/connection_form.js | 39 +------------------ .../www/templates/airflow/conn_create.html | 1 - airflow/www/templates/airflow/conn_edit.html | 1 - airflow/www/webpack.config.js | 1 - .../src/airflow/providers/http/hooks/http.py | 4 -- 7 files changed, 1 insertion(+), 85 deletions(-) delete mode 100644 airflow/www/static/css/connection.css diff --git a/airflow/customized_form_field_behaviours.schema.json b/airflow/customized_form_field_behaviours.schema.json index 8aa05945ebb01..78791a87886c1 100644 --- a/airflow/customized_form_field_behaviours.schema.json +++ b/airflow/customized_form_field_behaviours.schema.json @@ -22,23 +22,6 @@ "additionalProperties": { "type": "string" } - }, - "collapsible_fields": { - "description": "List of collapsed fields for the hook, with their properties.", - "type": "object", - "patternProperties": { - "\"^.*$\"": { - "description": "Name of the field to enable collapsing.", - "type": "object", - "properties": { - "expanded": { - "description": "Set the default state of the field as expanded.", - "default": true, - "type": "boolean" - } - } - } - } } }, "additionalProperties": true, diff --git a/airflow/www/static/css/connection.css b/airflow/www/static/css/connection.css deleted file mode 100644 index 78edf0db5d4dc..0000000000000 --- a/airflow/www/static/css/connection.css +++ /dev/null @@ -1,23 +0,0 @@ -/*! - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -.panel-invisible { - margin: 0; - border: 0; -} diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index 5c60638caf488..d039fc7275462 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -88,23 +88,6 @@ function restoreFieldBehaviours() { elem.placeholder = ""; elem.parentElement.parentElement.classList.remove("hide"); }); - - Array.from(document.querySelectorAll(".form-panel")).forEach((elem) => { - elem.parentElement.parentElement.classList.remove("hide"); - - elem.classList.add("panel-invisible"); - const panelHeader = elem.children[0]; - panelHeader.classList.add("hidden"); - panelHeader.firstElementChild.firstElementChild.setAttribute( - "aria-expanded", - "true" - ); - - const collapsible = elem.children[1]; - collapsible.setAttribute("aria-expanded", "true"); - collapsible.classList.add("in"); - collapsible.style.height = null; - }); } /** @@ -118,7 +101,7 @@ function applyFieldBehaviours(connection) { if (Array.isArray(connection.hidden_fields)) { connection.hidden_fields.forEach((field) => { document - .querySelector(`label[for='${field}']`) + .getElementById(field) .parentElement.parentElement.classList.add("hide"); }); } @@ -137,26 +120,6 @@ function applyFieldBehaviours(connection) { document.getElementById(field).placeholder = placeholder; }); } - - if (connection.collapsible_fields) { - Object.entries(connection.collapsible_fields).forEach((entry) => { - const [field, properties] = entry; - - const collapsibleController = document.getElementById( - `control_collapsible_${field}` - ); - const panelHeader = collapsibleController.parentElement.parentElement; - panelHeader.classList.remove("hidden"); - panelHeader.parentElement.classList.remove("panel-invisible"); - - if (properties.expanded === false) { - const collapsible = document.getElementById(`collapsible_${field}`); - collapsible.classList.remove("in"); - collapsible.setAttribute("aria-expanded", "false"); - collapsibleController.setAttribute("aria-expanded", "false"); - } - }); - } } } diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index 307450b05d16b..fb3e188949b66 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -25,7 +25,6 @@ - {# required for codemirror #} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 11ebd6c4cb436..653b0dd3ce07f 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -25,7 +25,6 @@ - {# required for codemirror #} diff --git a/airflow/www/webpack.config.js b/airflow/www/webpack.config.js index ad1a7098e0803..9d5800f783f50 100644 --- a/airflow/www/webpack.config.js +++ b/airflow/www/webpack.config.js @@ -60,7 +60,6 @@ const config = { airflowDefaultTheme: `${CSS_DIR}/bootstrap-theme.css`, connectionForm: `${JS_DIR}/connection_form.js`, chart: [`${CSS_DIR}/chart.css`], - connection: [`${CSS_DIR}/connection.css`], dag: `${JS_DIR}/dag.js`, dagDependencies: `${JS_DIR}/dag_dependencies.js`, dags: [`${CSS_DIR}/dags.css`, `${JS_DIR}/dags.js`], diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 192e57c1d70fc..77563d2eb1a81 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -132,10 +132,6 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } - @classmethod - def get_ui_field_behaviour(cls) -> dict[str, Any]: - return {"hidden_fields": [], "relabeling": {}, "collapsible_fields": {"extra": {"expanded": False}}} - # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: From 35913380bc6043d6ef93b2389200c4034ca768fb Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 5 Jan 2024 08:47:09 +0100 Subject: [PATCH 150/286] fix: set the default value for "auth_type" as empty string SelectField expects a string as value. The default of select choice cannot be None. --- providers/http/src/airflow/providers/http/hooks/http.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 77563d2eb1a81..9cf5d4983dbc5 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -120,13 +120,14 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: from flask_babel import lazy_gettext from wtforms.fields import SelectField, TextAreaField - default_auth_type = frozenset({""}) - auth_types_choices = default_auth_type | get_auth_types() + default_auth_type: str = "" + auth_types_choices = frozenset({default_auth_type}) | get_auth_types() return { "auth_type": SelectField( lazy_gettext("Auth type"), choices=[(clazz, clazz) for clazz in auth_types_choices], widget=Select2Widget(), + default=default_auth_type ), "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), From a12d810792003d0a01c652dac1e98a59c061a343 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 5 Jan 2024 08:48:11 +0100 Subject: [PATCH 151/286] fix: Use Livy hook to test invalid extra removal --- tests/www/views/test_views_connection.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/www/views/test_views_connection.py b/tests/www/views/test_views_connection.py index f9a4efd11c15b..1e21dc4856ed1 100644 --- a/tests/www/views/test_views_connection.py +++ b/tests/www/views/test_views_connection.py @@ -459,8 +459,12 @@ def test_process_form_invalid_extra_removed(admin_client): """ Test that when an invalid json `extra` is passed in the form, it is removed and _not_ saved over the existing extras. + + Note: This can only be tested with a Hook which does not have any custom fields (otherwise + the custom fields override the extra data when editing a Connection). Thus, this is currently + tested with livy. """ - conn_details = {"conn_id": "test_conn", "conn_type": "http"} + conn_details = {"conn_id": "test_conn", "conn_type": "livy"} conn = Connection(**conn_details, extra='{"foo": "bar"}') conn.id = 1 From f8449a1f623e7dfbe5c031ac1cf734b2dd3b9ee9 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Mon, 8 Jan 2024 16:40:49 +0100 Subject: [PATCH 152/286] feat: Implement CodeMirrorField for providers --- airflow/config_templates/default_webserver_config.py | 7 +++++++ airflow/utils/json.py | 10 ++++++++++ airflow/www/app.py | 3 +++ airflow/www/templates/airflow/conn_create.html | 1 + airflow/www/templates/airflow/conn_edit.html | 1 + providers/http/provider.yaml | 3 +-- setup.cfg | 0 7 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 setup.cfg diff --git a/airflow/config_templates/default_webserver_config.py b/airflow/config_templates/default_webserver_config.py index 4ad8ee6743f39..85b9d4d2c8dbb 100644 --- a/airflow/config_templates/default_webserver_config.py +++ b/airflow/config_templates/default_webserver_config.py @@ -35,6 +35,13 @@ WTF_CSRF_ENABLED = True WTF_CSRF_TIME_LIMIT = None +# Flask CodeMirror config +CODEMIRROR_LANGUAGES = ["javascript"] +# CODEMIRROR_THEME = '3024-day' +# CODEMIRROR_ADDONS = ( +# ('ADDON_DIR','ADDON_NAME'), +# ) + # ---------------------------------------------------- # AUTHENTICATION CONFIG (specific to FAB auth manager) # ---------------------------------------------------- diff --git a/airflow/utils/json.py b/airflow/utils/json.py index a8846282899f3..9622f4c0a6b30 100644 --- a/airflow/utils/json.py +++ b/airflow/utils/json.py @@ -123,5 +123,15 @@ def orm_object_hook(dct: dict) -> object: return deserialize(dct, False) +def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> dict | None: + """Safely loads JSON. + + Returns None by default if the given object is None. + """ + if obj is not None: + return json.loads(obj) + return default + + # backwards compatibility AirflowJsonEncoder = WebEncoder diff --git a/airflow/www/app.py b/airflow/www/app.py index 2656045e84ba5..c85d82793f678 100644 --- a/airflow/www/app.py +++ b/airflow/www/app.py @@ -22,6 +22,7 @@ from flask import Flask from flask_appbuilder import SQLA +from flask_codemirror import CodeMirror from flask_wtf.csrf import CSRFProtect from markupsafe import Markup from sqlalchemy.engine.url import make_url @@ -127,6 +128,8 @@ def create_app(config=None, testing=False): csrf.init_app(flask_app) + CodeMirror(flask_app) + init_wsgi_middleware(flask_app) db = SQLA() diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index fb3e188949b66..8e3d8db0d5e00 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -26,6 +26,7 @@ {# required for codemirror #} +{{ codemirror.include_codemirror() }} {% endblock %} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 653b0dd3ce07f..174bfa164c4c4 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -26,6 +26,7 @@ {# required for codemirror #} +{{ codemirror.include_codemirror() }} {% endblock %} diff --git a/providers/http/provider.yaml b/providers/http/provider.yaml index 7a991044ea801..bb0214e7c6e66 100644 --- a/providers/http/provider.yaml +++ b/providers/http/provider.yaml @@ -102,8 +102,7 @@ config: description: | A comma separated list of auth_type classes, which can be used to configure Http Connections in Airflow's UI. This list restricts which - classes can be arbitrary imported, and protects from dependency - injections. + classes can be arbitrary imported to prevent dependency injections. type: string version_added: 4.8.0 example: "requests_kerberos.HTTPKerberosAuth,any.other.custom.HTTPAuth" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000000..e69de29bb2d1d From 04c208e7698c118911883db2e4f9ed70801a0f98 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Mon, 8 Jan 2024 19:21:34 +0100 Subject: [PATCH 153/286] revert: Remove CodeMirror from providers --- airflow/config_templates/default_webserver_config.py | 7 ------- airflow/www/app.py | 3 --- airflow/www/templates/airflow/conn_create.html | 1 - airflow/www/templates/airflow/conn_edit.html | 1 - 4 files changed, 12 deletions(-) diff --git a/airflow/config_templates/default_webserver_config.py b/airflow/config_templates/default_webserver_config.py index 85b9d4d2c8dbb..4ad8ee6743f39 100644 --- a/airflow/config_templates/default_webserver_config.py +++ b/airflow/config_templates/default_webserver_config.py @@ -35,13 +35,6 @@ WTF_CSRF_ENABLED = True WTF_CSRF_TIME_LIMIT = None -# Flask CodeMirror config -CODEMIRROR_LANGUAGES = ["javascript"] -# CODEMIRROR_THEME = '3024-day' -# CODEMIRROR_ADDONS = ( -# ('ADDON_DIR','ADDON_NAME'), -# ) - # ---------------------------------------------------- # AUTHENTICATION CONFIG (specific to FAB auth manager) # ---------------------------------------------------- diff --git a/airflow/www/app.py b/airflow/www/app.py index c85d82793f678..2656045e84ba5 100644 --- a/airflow/www/app.py +++ b/airflow/www/app.py @@ -22,7 +22,6 @@ from flask import Flask from flask_appbuilder import SQLA -from flask_codemirror import CodeMirror from flask_wtf.csrf import CSRFProtect from markupsafe import Markup from sqlalchemy.engine.url import make_url @@ -128,8 +127,6 @@ def create_app(config=None, testing=False): csrf.init_app(flask_app) - CodeMirror(flask_app) - init_wsgi_middleware(flask_app) db = SQLA() diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index 8e3d8db0d5e00..fb3e188949b66 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -26,7 +26,6 @@ {# required for codemirror #} -{{ codemirror.include_codemirror() }} {% endblock %} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 174bfa164c4c4..653b0dd3ce07f 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -26,7 +26,6 @@ {# required for codemirror #} -{{ codemirror.include_codemirror() }} {% endblock %} From c8e0888c46451ff34eb914e3c9a4efdb5f2aefce Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Mon, 8 Jan 2024 20:47:50 +0100 Subject: [PATCH 154/286] feat: Add documentation --- airflow/utils/json.py | 2 +- .../img/connection_auth_kwargs.png | Bin 0 -> 9623 bytes .../img/connection_auth_type.png | Bin 0 -> 14199 bytes .../img/connection_headers.png | Bin 0 -> 5256 bytes .../img/connection_username_password.png | Bin 0 -> 4761 bytes 5 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 docs/apache-airflow-providers-http/img/connection_auth_kwargs.png create mode 100644 docs/apache-airflow-providers-http/img/connection_auth_type.png create mode 100644 docs/apache-airflow-providers-http/img/connection_headers.png create mode 100644 docs/apache-airflow-providers-http/img/connection_username_password.png diff --git a/airflow/utils/json.py b/airflow/utils/json.py index 9622f4c0a6b30..eb3cd40941197 100644 --- a/airflow/utils/json.py +++ b/airflow/utils/json.py @@ -123,7 +123,7 @@ def orm_object_hook(dct: dict) -> object: return deserialize(dct, False) -def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> dict | None: +def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> Any: """Safely loads JSON. Returns None by default if the given object is None. diff --git a/docs/apache-airflow-providers-http/img/connection_auth_kwargs.png b/docs/apache-airflow-providers-http/img/connection_auth_kwargs.png new file mode 100644 index 0000000000000000000000000000000000000000..7023c3a7a072f965f9dd053f77c5a5d64b396f47 GIT binary patch literal 9623 zcmcI~2{_c>+jom(n~(ixI5hS2r&k-xWWM^15qC|U8&@yI zp4VbIoNz&&;S|%$vllM#zw{D3bLeCYFTFO4DogYQZ3Q#Q(MB#`@|PP{w$;0sjC|gQ zwZ00``u#3Nh5MrM%WwU6MOC!GEa0bjzF~#+9_k$RUt@1t_`%yIg(IZq`(>tL>=B>- z{$n_3=<@gXa@sCopHqhy8ve(KHqs61*aGb{L`~`|6OlkDb>1ArkdEJf(u%A*3sMN|@KU37UWK z`S!;+d!t~o=`Y)wx6bxn)?k%F?VSe!d^L0ZZ$r)AA%p$*$)o=^r})M@ajS#xngony z*Lg06eEks5LtY{{Vqn>ksZNFGX*0{b@+ z#0=Qog@Hj;9>jhwU?syRst>MH=7w=oH_sMKnFK}Mwee}-tenP+6yF*CgOf6#l9m~nV&Lm0k^g3ly*;YJZL)1?XSHiuScftr z#fh-AL>l<*5_9^a$4@YkHx|3Pz$brZ?SLRnt1aptkF^SCGF4So9T@qEx}W?S9Hhje zjy(hdWv_M_&Bzq|=dy@X#CzWgO-|?oQ*sLLA1p%)S2ocGlYcv~*k1^Hr`qWbrW`D1 ze39xdiiP$)c~TF*B%`J~(|0HvDL6}UXmC$AX=?Uck=2jS&$IIKgwo{Qg;dPIWDXvG zMpEM11PV^VaExN#`GO17fs{SQ=DebuB3m@@iB@cQBh_QnXofNT+W;eRTnE;5qA}H_ z{zc{+2E+#+k7QZLTNt<2>=yczPV0#hS6B%9kd3{-YcCyL!Ul_+k; zp0DzSG_^E_A8QO|+-h}b+8Fg8y{FmTOUkQv9CZ=kv~^}^Lc*}YvMAAFoY;}%=|t1SKU^tU@;Fdz77c@R*1UA%Eu_Y3tz$}<))Yv z-3pd<>}&eTERFRZ4--c09NoKpl|RsR_t%vKf8p&x_J~HJ$(xT zo$oS|$Z5$+e80CnXSF^*nB_j*ndQH~H}>(#(cCVJdOvxk)i2Uz0R!(`MoYbx$5G&7 zbo(K0GjYpF_4f8Q`5Qg(SozU`tNz$D_S>4h8^zdEijt6gb!toDK<=$u;A)fy!Y`h(--=f&9P7^^!9I4 zS6lByogoK?=((~MUPch>kn9^#u;HhK@Ta@&)yh^MbQOJm+xd0L-}4_>pNLW6Rf${5 zn7B7A{qPGNpZ|_4r;-v{iFBo2Xh>#+3G;|{E(gSV34$r{E%Y!9`-0-)9E>o*$tJE)bX!7Ho6BjelTH=PL$J?c1 ztcS}V=!m@Z84KnZ_Y8M`u0~g~R_#1AF0&rzJU`m7zjtE99k_*)p*%{1*Zcfkv+5{6 zR8O`>jGTCSrfnv=TK~22s0ZyRx?UYb5c6=Y2V+m39wWHzVWSoL=?cN?3qvg)87ltu zzh9h(5?Ru&?C-2D*OsZIjn3QJB{cg>n)^q~LN{gBOxrEsTCs>YYEnTcVzXZ93bGh0 zzrFi`%hISd+fZlbYVcBS(%oG>bOD1C|O-DZ0gNyeZ7| z;uZHP-86$>`0iguVGr-XX5S>WNe=s()OWpcM50@!i6`t8{wEP90XuQUY@7_G<` zMX#lM_TFlzVc))-lM#s5Zg<<^h7V_c#6Zqm$~pbCRB0|7%Gw{i@E|bt-6%$}x^(MT z-Qu_`f0Gqj1Z!{SyJoQ}*|2k|Qd9r z>FA|&f!~@IC8ok-O}>jCIHvPUqjzf_FCfwq>*MTI=x)nt{8ZFBRE`|3^>UW9s6CB; z+3O(2M`ZC5LsVYC9G_8Wt~q;JO}+n&`_z}P(KFTfvQsHJbB!C-1>(qCCFW<61a)mJ zfOn$ExQ(<4t2>BmWijT^^%ryB)2zikhe00HJII3zq&#Kg*d$5IyFLcNnc?t;1aLJx zoJ_6;UlJf$ak1T*Vm)Q#Qln0~QLgivXwu@@_BCrF#V znBPwJa3cfanoY|cE6bmaJ@LQff|QJNDw<>XEWazOvKK!+1>2;7y$rtLFc@1}@5Cq^ z5Oh90X3=-{nICF79SBuUIwF7??)EGxo70_>P)j;b9!5H#)Hl_1 zveiRRBuu#?H!9SwzaroBq4NfU|RlwHPbf$H1%Avp5&z!DSXF0SkUDbIQ zN5eK3l*mwE#zO9}ZBKBzYRHFZBD9^0n9#M-o()i^yOx{safWYyQzNa#c6BHxXF}Ja z=TzVIm+=V@q@7h6h!}-c`>^rC(p-m*cP7Kja zXnQ)RsZ+5HuBV_@p$s+68ylhGc+%wfh=gMoM5{ix-n#blmFo4>aIpk}F%GGZQlEpa z3wlgv9Xm2)uZ6;2Eq1EQT8`gz3s5uOC*K-(z)}<-x^bclCkeB(~dQ^KZsXG93)}Y@3u(u2tR+;ef__lr)x~c zs@fSnBczc+RPD}83uSqz>jXlZFN{@i>ojr-JW{l0sO6iiBS_stR=1xb>wi>xSK!1U zL2LdH!5&D=3-{dfOHz~^B!x^?4C$#}Wk7NbjRY>)20)`_Rah=>~yAj`Jqb1^m7l^l&k_AB-56!+#;7VCgf_!uFmkuic!7h?*gZt{)_2! z_c`Me!sS*?m7~g@XM9#~KLmRnseyl4WPAbV2*VohK#Q(PPtg}6O__^XUmF|M=ox4_ z=%bFpNYb{Qkrt(|FI1%aC8ceb?mr5CW!NDnU+Hzvvb;Ewb1u|6KlmS)o9_9`>qjTe z>2Sb~h^;ekty=tgZxaZOj%!mK5;{L2XR53y=c+y=nxBOWYq!@&;XMU942~iy{FPe6 zyH-AoE3SQaeEVrK6-tzPE!`imsL7?9x|XY91sBHJ&nU2JU$VP2Oh>`kp37-@+oHC zk8(4QJ%7dN;5ZA?yJa_tui}60sE-N^DUojZym`q_c#yXxfwn!* zgmwDa&!7ie?kNx?9j05>#_w!&(xPXh0=JQ?~35`m$S9)v?zSr?L@Jvwk6s^tZy$#4l(*9EbNxA;-XvBZZ zJM4>a9U6S-(>`mn37JGEW``@B@@_U>)9IcoaYH#25(MHuq%}@Bv>r`vc9*gF{Hzo_ ztrjf-1^GrHd~Esv&v8iJWujnQmEGS)rsXzTrWCmTT&M=K>$5pq^mZjKbwRcG_#i0{ zfE=s+z1^`uX62_(G^}}hy}enmDimbd2;eNfz$pLz)4-zJZ$tX;`ilQv992nE^em~| zsdDTeuea4{iRPJbZCL9w%c`!f9+@nk{Lvbh>j46?{%Ox(%zI{)9jQk@y9HHhY_?V= zmn%Y<`8vD8TfSzghk8XOtN3F~Yup}S-6pknm{tX~42Nu}xxeoOa&sZu%@1&;QipGa zr4Kt&cFsmQbnH4uJE2wwtYbbANF-d~*m4VBm-tpDZ^(?k zHMx6NMFkVJ4EgcPr!Y6`!COlszKg+~>58EXL+|?UM0ZK6SRzO3EfB(V#u>i9YB&iy z+jDO{rep=N-anpeGyg?7 z=xZ?-BKBfEud<()7eI@R0KDQUtRpZ6fVyUH}^g+ zA&J<5vq7al<_9Y_;67axeB6J>$cR-vbfP7u<>%lUw;YO}iBpPyexU4}#1II8f$xD* z$AX~q(iWkn%&HyJnf`ab-?)rD1Hk;|O1mWf`8kco9Id$V%C`-B3crMLH}0aRAIuQ4 ztolnVawp}4`j*=z8=5wP&H~yPNyaw}mLODimeiA#(LPLa1Ji&3C92VPegiJXgD9V{ zZP=Phl}l%rr8$E|jj4If7UYj_jSVQ1=PL-sX79Xsg>;iy_O`nGnp>UhN<&x(wq9ICsLozy*^cqcG&t(C$ zw(3eS6H>X((`ZL!e|Nv}%^m60idosmUy?UptWNLS35jbtq*4&9VH)fg4QGKFLxO$d zv0A1l;Rzf&M80I35<5}#UW z07qEE3+8>ihCzIrKDSjVSYl0kzd0V^s(hHis&i{~dP3QGuw3x>{GfETq3UI<0E++S zeScG4ZpkOQHS9cV=}930(7W#D6lmPP)D*^=r4hqRDFwI=mRYxi05;jOOES&ZOT9Jw z_}Uz2Vy+JR&;;ke@mjCl&50P-T^hT^dNMMt(_5HxoZA{70@?o>n`h3iK9?27D%{); z5l?FI{^1wEe#!dRu&)gzBq=$wj; z)iaW6`y2jzOc_i2*3r-dWV2@DpJ^Ya>d0TpbL3eNfRzC+LN9{H6b>TpMTfMz6PM$tC8tK0pe9(V zWZi{A+S51rlxIFrf+EDM06iHSw9ORnz`wauasS>?#Fa9pwquuw*@W?@sHY^sI_~}i zf}oipHjV&;k>wHRa<4Mz(Ie@Ke|7o!o9O;>34 zkKrMUHE3o}eR(nbFfi5OwW+4zR|;5y&C(Aij$p_OhEj1n*RbZOILHY*3;eWdQLLMF zIzIqXr-UP15p;Z4=lG5sJ|a+S{f`0U?8DygI7ok#>peObJ9_ox*Z{!*NTTR%I!8Jr zoohK=y(ypFaQtG1%>AaP-S--3_=Bu3aC~Or`dS>`r}LBijlc;X{AF#f5s)&^kDF(H zq63J2NT1$3#}L2nE4Gt|6DWm>PsYonhr1tA&mW0{Uv>`Pmp~YAh%l~BcSQ93e3!w* zCC9HnA;P8}!W8}1Xs~`a-4G?cF#=hbf{)M**Lgs$_&#EGnM7t(tl&Btk1>w>D)$sS zCJ`1#^!{;avbEK9q&!z=!WWsM;4#l~BO1HkuY-Inte?z+fBkC%xn#`vMJ>rS$9dpF}PO3F$M%_W;tTB|<`*t{}BY>T@%ppMeYVE~+I1h-1NxRg>A zg5`xDhwt((K#!h4BD{vHbfy*bUP2j>bZVPSi=GQU@XHJOIEX;Y=jZ3f73fqi3~Hbs zJ_v*!iFJDV`}gmk%UAppegu^@#0vx-|Fl)`jWBCcG+xhI$Wk&-tT`ukxAL_Nv&IR& z=tc7SPo5i0e7w`=s<^n|k9D0IGmaEgMs+O?Cok`{w#JRSN?4TVoIa!#J6o7X3c+NtLyj1m2J54Ku8NfltG`MtV znaa}wplZ*Z`o;I;)LyE!LlFt3=GB6X*25^3zua(;o(%7)7n>=Bz{7r9$kLJ*(zeYv zBYZg!s$ZR9A{?Jf#XI*wHYbwYy#1mR6_{AkouA{cF!s>rP(b)1q0|aLZrx|JApsah zY*OcV^+l(Pow|*r2WWdhp~nJYaEk%? zc(l&jd1Ip?ShzVxwAiKnL#p-ESB9yku(uPAh#?l5GUlDhSoN322%dY{mTZ{FZwcWRVk?*D1 zZD}EU3|?6}w%<>4@ooLK!~Clz7o(bcdGa_&gXAOOvnbNLKy5{fSDQSY$Khg08TCz9 zTi*ocTRh2E|2i8o%C?8av8LM5$JR2-vJ+)wxk^`8sE#>?=i+bHRzc42X}8u~ar43Eg8c3QY!gT3r0AERgA;z=Q0 z`NE*^n^T?j3fHmX3l|MKNnS^Dj`rBR*cdFc-_u1&qqi%&ZSQ(-P0B44ziabM(grE^ z!DiAeNi7D)na=C|qP5a!^1?e;uEXu+4>r`fk!P1@i=gO%o>vY;(}53Tu0M zToR*oBGz2u(Wj2Qa6;iNrxf}hh4;)P%5IHBU50lLPJNHfAr}G<^paXNj6c(#@6N7n zr$*>U*E|5%iz|Qmq418iOeb%$aEsr4k3D_16oWvrY!ecm-n0Gs>Ux>ZsfPBbV{pio z6YrISEJVD7dpGrPdg@skj-Ny1 z{7ovWvmTr*cA`t6Gw+`G`19@!xp>6y(>&spb^@U8{XIJT7(A5OpDQqQZ@OAYZm4KD3SsW($Znu zGtq6Y-#O7U$fsp!K@0bn!q&v7G!o7}XbhrvEw!%YbVb|I$Dz-Ae*L;Fsu=R@yjCHG zjS-pQi$pRBalV#}D6>jdUMnE)ezRxq^ldz&#(}6KbPGa>OuK0Q(FD=kO|CR;!RJ7N z!`7U$%OdyuG|C^}iGu#}&v5%a^IUEr{oSa9p3d-xYUg3D(U>#4UQC&EcLzKXp9#f< z#Cof#$TO}A;vNIHX$3oJ;n2(PPmN07yQG%}|FGx76v_FV!l_0)4-Y9 zbZ(yy2$hl0RkKsOEr@u@@})EoC@9erb-#b^YBISA>7}u#NA_lB@Mr5xrwnWd;ey+# z20lZ)kK|L4s8<2ycE2u6F4Fv`y26p~))|+6R=8Dr^*- z0)Ui`ajGj~xM}9rbq6JwpbIDxr z1LgG^!M34#v@h0uTAwQN9FW{JDl zwK;rq6m-JzND9E@Az(+%w_tn#IHZ%NLisRsJ^2RA7PXJ)!SoQFBne2v-nL^}5;Kwu z!LImtp6)0u6F^R$F#=ye1-}vOcS0%PJO>vawYnOtUl{A8dato{)<1Qb^%U#?-d(A^kiECYHjx0rcd9SJ7b{! z96pARQQ6}ZGRQC&#%4{n0d>qeBAz0K0tqb`SkvsiMas*105&;C05l&37UD#`U;plS zkz`u$=bdK2-wlW$YHw$20A>eHGz(~1xg|P4*m&JD{ty)8tw6bcSE_ zMU`0^_8k2*IdeY&@s&mUpaz!pDjj4V33WB}NdWpqm&*MT1praXlYM8j_^#2jHgpcjBpYOx2IWu@IqX04SL z9wn~mzqeyrKGm6S#(-oOGi5RRkt1Oq8HKNHKL)pdIVnRegA))!qI|@hXA!GQQdB1u zQ_Dp9vNZ6>>v*g*5QGXQnQ$E-e8(M|y;?CL8sP^!$YPvmu%YS}M1nI68ZIT}ipD_z z<6&R_HEiY+_&?mxSyv_W)t<$uuEafyjWcL~qm^d{zNFfORVpsdZ&4t2i|UnX^n zc@C8S32ILd(FYlJDXr?Jlkx;9RvFVAoLVAjYGtzh;(LJ{k}GNccJ<^JDh$}n6Fohq zwawFU91anWxNV7+{r$5?SXnQ;pGHwh8A>DMlzA9KA2ZWjW%u5XuA=^%L8%+hdNA+KUxJ+NQ^~; zTn9UHP(jy0^`zM4#$I$mj(&?!TSuq#wsw~CIzANk)`;z1NF&A3EomfU0tp(xzd z2ZcTVDiW>l`g~X*OFz&oTQJg7kzk+Sm8sM2upK1z0}1W_Bs2f_)jcE%Uk(~1PJpBb z;gB%@-1RSw06?<831r+ngG}CA#fXIes>*|VZUs2-Rx8-B;om}jF0P}m*bEr=Onkuk8g0E@GN8TH(>+$dxn|e`V?;qmYie-%1%QuKYo} zTHWJ?Mv(qiDu63;ZrM_cbD)O-WOK$3K5Gm~5*BR-9R~*&jIxmP5IZpcKlL$yg*J^v zz)AeA@PpaC=ZfDQxY6Vw(ZrJSs;e)5$+)5@cBlOOfi0X=TV&u^t3gWDPBoA=7(7ER zQ(FfL|KmeSgG(WWbN`Rs8w9{eDN`;Vv=|hht9ZJrw?>o5+so|!QQ4NeB${+x0%*88q;*~Y KT8Y}dfd2)j<)#Y& literal 0 HcmV?d00001 diff --git a/docs/apache-airflow-providers-http/img/connection_auth_type.png b/docs/apache-airflow-providers-http/img/connection_auth_type.png new file mode 100644 index 0000000000000000000000000000000000000000..52eb584e5ccf6463273c9b0d35171944d451af9e GIT binary patch literal 14199 zcmch8XIPWlwl2%HEh~tXqEt~qIwDQF282i_fCvGiAiYU%0u~|)Qlt}_l+cUx63P;! z6RNa8U?H85AP_(Zf%_%6_Bnf>bI)_{k9+^{0QtT-#~gFaG2eHLH}7?|)fhmmAUZla zhKCRpC>`BD5Ww%PXHEe>*-vEj)6q#QK2*7H;A2j}h4>qeWG!xxl`vB)-qyt4sieB$ zK!I^4{_6to9tVb=yr>fRj*oBX{%=hGK(XWx`L7FCMn$QE{n{j#%tES+jZ;z>wWBLy z)|Y&zJ5tO8c4S2zN-5sF;b`ymw>G~46VlOrV)=>wah3~pf%ezEbrt@9(0&^@tq1yz z&SF6ae1<#t0(k%VoqP=R-39QOm9E&Dc(-5w)ot1MK?$B)ipa)5&(OV0dve4x-+#J} zB|8atk~=s`G4{FqT4dbO+0(-W>d2h}*iREaZ1QOCh6GLh?;mc517iaZl3If2%-h1l zD4Cs1${fNWe1K1kTjTDRT-en3)V}zPm6+SKa(|U(%9$0!Mn{7)`&xTj;C)6W<=vi) zolM3Pbk9y2ote1PDHv>eF+1GCX)ZlyF|dG*IAL&+X6#VdTzezy?+)@swvKC{%slpb zrFbGljB3@g>)}~$r`{=%rzK~HPU*6J?cVp#x&9IDxvS}W-Ky%BiHEjpg<_xETUakU zI2MVy8Jw##>yf={`3_k4z}$f`*x4&8r*jp4zw*m)yAE56ToooKJ77@F?n<-w}U^m}xbX2m=)o+xvHvuLI6^WI#x3k`GC@K=GXzz0#)m5{mr@G%S7JkMK z>+txD-5Vq1_)jLoThg( z7wLwInsm|}hEu?K*|K}0JT(H*Y+nn#Sut@#s_yyOgQh~xw^=U9tmjg<8Q8ol9c+{K zMTKIe|7PSGi!!T5&3L#F`-{`mjZ)wp5#lDW5Lm!9g2}xZQ?lZ|-buazT4lagh8L;9 zbA~%W>ey@L?cIT5*=3dO-w3I~G}nveTmNDAMV`P_yz+T}?7Y)jZC2)BwUIoqYz1cL4D z?KR!qE9$)0IsdjnOH0cO*t*Sx_hsc?Jj2JM5F9he4m59bqhMIouVF&mt~f7Xu~Qm>K>&E8%#QO46*8mprfkZQUl<|REZ^47lDpg(M^^u zf6Y8J95}o`MmtpJZ9f#1mACh@$~l}ZE~{+*%L9f7(RNqG0X2ZD_s{uxSYj#L@>U|KplpF}}~g!L$O zDK#Zg?W3_^x|c-@>fGM*Vn(32k!f*+Vb^xB2&1$v5KeS-{I$?!3ya>OvWoVBr~KbW zFLu&4utgN%;qecr+5%z%_#`j{9o;>X62_s#2@$E(E6NT+#bS~`g1AhA)8;^&kVz$jkGX}Ji=?z^CT&{zT%^0!USH2>Qtg;DQEn9`?751^YuVZ5Pkes*W^W+@ zekpPiYrXMGGY$*{Om(ciqvKi8a*})`yUe9Dr|zB}!W`Z=SDCu@0U$>CB~~#XtG`~o z>JQ7)5f(dalOq?JcFZR$#uoIgRO#6C}*EwY6Dekdix!B-8D2XStOGu#Ak1 zmvyX8+jvs|Idj;iKdmYl-91^idA1#w1g+%2NcD_-c#zYh zTc_ql8GHuMvi^)VPPpfYOTs3#1MCkE&J4^{ zSohs_x9*qfSK2M|-9Ri6@XsX@;Zaxa;>*(g%A5C|kjRVJk;1_Rx|b0v-B3tqXy{rm z2a^cWDyv0pY0qHFRy8u~j;gq=M;aVtmlHwk2;!_4^`7qfTHTFVbv3w8K!K`d-2r{dE{w1Clu9ak=bM!;GTt()^PhF4HYISblQOftp!vINoYwDf)+a${@;ZIXgOtQK!!Pb^=J}}0 zv)HtS&P4MJyx!ZXv-OJdKNlX(U_IB)u(OFndp)T6jjj^z9S3eIFTW%1O*DmN+rmDD zyExFxkWo?H%uHg%zTcpbcp+m8ZmQB3|LU#aAB+6ik@}kaySD4fwN@^mBF6 zon2*ssI50D0OyIf={|}BJJ$$fx<7`oh+>BiP3}cPL0Tz4eLa%l@gh}A#jpfFmh#Vx z(K7DVlUpLDcU{i0kGmWl62{;JE23Caio8ty;g{b}uldICQZSr%9tY)i_ka0|jcz~@ zNH7bFi&(QFsNSng<*#90T*J=hirzjx&5T2=^81rYDUC1~L_6i~(|A!hrx~xwmfGat zPCU3Hf};Y@mVae``*7zW>7XT3ZA`-uv==0YEym0Q^rpYM>8F({bDN3_{T=}~jNf%6 z;xaRDx44cH69dwWf4-k{$oa8sX)&}@Gyxdey5GUcEW-d5b!tS`WwwQ@x5!wWK(ryk zpW-Yv4=tb`0X01nEHS)EuV0@a?+#>$CTk>!RKFWd0*P{NHrZ29SV&moB;@2c zrG;`ZX(Pku&k2_j9@JYvb~CQtm%W}y>=@ivm4%mf3BN7I+s-;{gk9Tx>{pyw^`0bu zfXLb2H$>*28QRej|3rcVi~XtrEVd((s{+;KPWyeXNpjUz2^tYp4f-Clt+)x0?{ zRiXGd=c7QKY$3$dpL=@h8J9aEYwp6)VU04I4B`jdV{72S2>-K821K|<&^~{iZMD}M zcXXs3hTG^lt|`@}FI8dkJ3Rb1I>J~SDk?8n?|p^Oh6Sv^wmRHv!wvD?wmMSoW@C5s zNTHG1(US}DV1ei4ju zH#pej(G+pW!x zXR877dLv8m!{6wZ?*dpDNT6HgnT(+^;<-ALn`&WdWGIsk-OG|V0H8ca>59?VP?S0V zMQ$BK5ol*mk9Zifh{T&y=PKznL)4HNj3-_a6W&81_-z_mzA!Wy148M_r78w=6E{&% zKGR(YfUQ1}5h&e8ILJ^knQcL9v4sMKpl^h*snRG3c;8Jzb@Z{ZrFD3!W~-m{db1?;dSA|vm6i1uo0R;c z11)k``}jXW?(lrc>d`AOOQ-NU7!~L(i|qmjxr>szEqc{69FtSuG!Z){f&5;e8WSY# zOA-SDithaI-y1pl*u>I4yqJIldmTgf_%gL~XgFxXtP@F0@`HgSlR~ko?zshD7$1G3 z5&=58;(>pA87IUYUMIPuji8Z>Oy%yQd|XE~S$<4fDhd0RS3ha|Q=H*gaa)_&(9~$bwnlupzK$? z;WNuh+8bEOIhp^7_fSK};X`p5Enppj5m7_W&gV}&1NPxETlpz{eV9vuy*&oky&8*( zk_yG%TU=O3lJh1e0@v?x9Qm0^j5d1W<2lGV0&gWAjV2mf+Jt9F#*ZeNSUQB4;e60l z7BoIs42=V01Q1Ao%y@3UXD~6dXnhnj{-+^I2tg|45eTk)`ppSP`E_cOGnM-tdJD@+ z+yBe}&VW@%A2qC;G%~&t4qWd5hwD?v9@`Bgkewwe+|1H4E8oTTc9o~+dj@!Meo=uG zQ(PRs1~*0Hp{Hk6qXMv#1Ic#1cA?g`lRf|eW z`V4MK`NhQbIuxmvH zO$>Y+FDMYu*VB6n@DwWDq8nI32_r5Yr=wp{+&TfK9V{Rhd!MFs4&b zUY-%ya3<{#=o}_mX@BYdR>e8*;7;5ps zMlmrlQQ=EWLV}H(yE_KJ9~j_{{@aQFOqtY88yz>XbPTVOT=bfA3DPVP^Gl{qoD^lB z6UE#_Zv$Gr>f0MBNBfC8-awBx?5qi@J9C*!Vs>7yj%r3E;JB_+uWQO3g~| zBmuL5tIVDHRGBK(ZaR;1)Tc&&y+`%TSJvZo%21v1IE&^<*$}{o7aV~90FGOt4rvii z+k<6xswax9${A$?4{95QfzJn6sCBRB)RT3AK1m4S=H{mP^yyO{YLpHL0)CkIPak?H zP&D34MPM+P**5CfrJs&7f&E!_>W+Q|^J;JpdTy8UVDH%<+5bA%Fp1HTL}^3rn=}hM zJ!lIddD8x(g&EzQFxu?L_F7XT$bv@h={kj|_6o_X^OHms4^ofM^rhfmPajv0o>9r` z%g3J>uOHXmDrjCgfx2*97D{5EdW@M5aeN(gInum&k6-2Z{d!I3y#38Yd5NgRf`bz_qLWC~Q0<*87VZJL?7Y zIX?<70QD`Mt&x5D*$Ls^DZhz>-Db+gG$-Py1X6(|OzSA$-$cG-IHh(yGgP)><(t#i zT8%$UzL~{lP5MaQu0y$BZaArOF=bEMQcB6=FvZDe&7HmGJ4rdADo$;$m+FEcGME}` zh%^Dj;euLoa`Tbxu=0|PhB@r{cuIev&+-AWvy08j`rG#`fTl8^B?u!9*AwJ0mT*G( zL}2y68rtAe>S0f;*`iYT3~MT(bFjOVJ>CR~W9p@2e+k@_Ldq3lH4J`+*f)e=B{Fi>M^c62Rh;ForEx zUk~y@S8wvAU!`uUADp&_1F5J1m=dRW9ZGUBS&KzWd01gR#t)Qt6!C1yF_9E< zTi%ELnHcZHfP_#&QPS!@FYLo|py8SP&$S6YI;0c{JgN%+wp4GBB8v6IFdBWE9LrzY zT0LkTTo8Du2=nEYrF@h2U&H_GpQ}OMFj^mP_Czz>+I0G&YdmH<{jh;}P(JUUJ;Wm< zX^Mq`_KdxS`k9qy_e2DuRQiXNW*U!>vvcXCEfDi0@6NQ*IqgZGjSRAx>)o-~N2AF> zwUEh(9utlYaNqbT9n59(G%((@m z+z7*tdvny!gKdR@x#h}buS3nx7nRt>ybD$_N?KFS8d-t*F%^LaV-wijH9aL-z>=D1 zk)I)_!Hu~v7-<#v+rDe5nU4Wo{uQ66?TTBaiqp@gO z_!1xD2U8k7TG;}aT2t4HpFI!#XLPhDEkWJ*-rQy$@AFu{oIjbbBL_Fn&s8Ji-Y+e5 zUnD3l^fRh^<8BMQFo5J~?t5F-j9)LvtQNys@dJr^SRn1d_!fZ7tew7>9pQrsUd)lx zrTqAQB~q~X=Dcw$6abfy+iCMyrQm;g5B!kP0 zE6FAv#;_Uj1-;|L{I_rXH8SQM&to>9FWFUhq!5fuOm&ogc%}t@bzoq{^_lIArN@XP zu==3<4>Dc^?J)A8*Dz{Xp6Ptg`RNZLWO5|eS|4*fc%_6*RUB$?s&@Fu3hcJ{Avs`Z z5Zc*j?oA25wMEcO{Nc(J2hQ|-)dDd}01LnZCxTkzEu0s%9Z*r3(Lc6Ex~|Frmozzd zsc=G$=$C%rC+o*m8HTiW+ryt|lyGRKlq}0-jpg-gBT~>)0p|JY+)O%1j%*8P?hM?m zrH^$eV8!FVSsWUGJwJE)(JR8>HGRE|r2F63ySI94{0Y85g-7BBKNF!Kr{^?A@kzG> z>zat-{uko|TV;z{;q-D4Elk7BeHQL7Xi5f`BOf$TEugu4rSDv4o% z+4$4d*1pSbn9;i)sBgqeq@QDGOn$O33;QTm6z){}MkjA%tlVBeo+P#Q6;^*9bgJsp(tD6<#s)d5T#sK3G-B@#SLY};&Yl1Z zR2$s1#oZ)GEkzVm?~w?$r(%dYZHcv$4tf@iys6sTX^4}p<-bci=4UilR57}RWE;)q z_CH>Kh($f@(@bo}E5~(JV=)DT1eu zFVp>c7s~T7J=D~R7L(79t=^pTFS)L+CgYASSaFpMd$3ga72=J1RdH^7DSN0DSDN;w z#D8s{11%S8-L?!6WA;ik*p3-Gi<5;pH*C*|`TN)0-JT*CIX=z&&Nt%E%zW0*$ud5@ zJjlj@e_pWA`SxTVo>ZW z2FEB*9~IR+*rS@OohGr~wB-eu_45hI)_uP7BH7z04vat`7Iz4fQgBV=v-U_fmQ)cw zzR&4(asfMQ#;|2}IWHmEQh){nIMgC8J|1gY=Qg34^0V&h{^kWT&NUJ}zm!f`;Njs} z972Uz_a;VS*lxTf>y%D9;_645I{YH`Q{=Ayt3d$loNW#1#g{hPQ#(^>fvXSQ30O)0 zCNGMw)pAdDuMe3}MS8x7_+t6rI-f zBW<|+N4jk-VbMMu>ET>9HW)quRgdQV*`sGifnJZoIE^aIP0-E)^+#=C%t-Dxrzbi_ zwqyeK4Ls`9ql;nHD!DGx4R&xfpgbY0L8@_vByy;uEBBH5U4?XyeXcsL_%hgTS0 z`q?+>l5CR}*xe_A6!)*Dq=f?a&ooOTXMn0`(~pKENrxPY2zfeaUBRES-VA@zguH`l zv+_H7dzN$RyL_WG{bIk)UZXBQ318ZSneT#+FDXcDC`>oIz{4hg2Z|Upno5r|2D)bN zK@rB{Hsjz2ky$5f2mXA;d+Sz*gtT<>jga$`Tw^86gF^8-9T8QC_g)v>Q13zS0koh1 zC-7GK?>hU>V9g#XVQXAwoJEh^HlM++ElZA$jv|x9tyLD@d8KdK1ZSh2A&CAz6Hbi} z-CNHHt;_^k3J3_S32`>U@uS7z&g(sIp^){Dp_h$TgHcgU zIs$r!Qmz&r=CJi7k;w+T7z(@O+%nF4=6&e3W=5p6OG)a<>`OWo8LHua!$Rf_e10iY zUmccbO49>Gw^RLByAG?rx(&sFF9nJkfbxsvT)(S)KT0~aHLRnAT}s7N`~^_yF)DTc z3I>v*z+Hpu%R7BQzmnQA+AgtL6Jk%zJ@F~1`PLdC<@eJ&HE2d?*^do1F4cstZYIIZ zt@cRrH=sQySKo^<`S_zF2)OSJ&gFSLCrz*TE-vtlO|01R&zqNuZ{tfbYsM9s!-{dN z^-b2Z!Gn81p?-y&X`#v!(IlNYR+!n3DIan=+$W-Qiq^h3sPCq&_6@v!jcLlr^({FRjDgaG4>(>3#VFungj6;q$)n?k}mkc>mPt(!u zbJDW7hc|z=F>BLoX|r{ZN{Aj6PzEMlmo9-vsz(ERiGhIDjRs5X-DX=(ZVkO;A!weD z6FMZS&qf}3j4{|PzAwqb;UvT#4$jr%7GzC8`MMjJpU(9GK~TN?3_M=3wLx|*PJ|yp z@)2-iOno%yw(-4f+0U;Lc|VJ+S7MG4qh~pM(!*kOwhF-a?jt3zeN7Mep?6ChxLy?3 z4xh*7u2))b{9K5hK9Avblhp^=c|+7H5=rbN+282y&CpUTf+&J$F-tZ$u^i$hKs;T2 zbmItU3$JKoY@b)>jyroOqMpAjY2V1nB<$yGGN#(Xbp$|rq?A*ESxq|=r{56}0MgD= z5EuxtZVW@|`UK<_0xdlM@1plOK;B&LBpG4YP8bDUysIe@0|`^XcUoZy8V&|^zi=M-s4A;sOX$*T1;1rsBG${FT<*_FLG(*32Fi^tP=L&u-1+kM0cR5J&DC$3AbA zb$lE)q*f7n@Oq9=qnb1$?fgMxjHvAjI6)EtE{+2uGqSRxAJ*?`uKU-@jlMp)MR+}V ziKw_JN-E0A$cIGhw0|fo9btMbVySq@hvm>oFJvy?v7=n4-c$n@Y4_an3KT-Gg%4>E z8MaK9^X`kkzSN&ylc1CMBWcfL?bc4&*47s0bkvsC7Ta7jy*3jNhp`(c3l6?$5k z=4VNUNo+k>&j$$>Cuidpr}Rd0H23Zxzhsm3#TEg~4KXQwn5*At^ORr0g=p#yC>zM$ z)$w3r6e-ktQ^>6RYYEGOb23q3czc^e#&$sWn@Ko-2Tq zA$-e&bQ?!(L2Om~3^>M6`*HLl4$^$MY2nJE78SH){#|tdF{7f)C;L+62s0BVpcOw6 zKQ7hgqcjE9)ZtP8YWI$F|;t2j#oW$&Oq3^0AN#jT~vUssBB; z0dO6)y)qH;Ki3ccPk|D3%KztR?EiieJ~W3?V6Qy6cX)Q9^ZE-_0OwKrHUQ?!_y%a3 zFn}ndwF2E%mZhc@7}!KXXCYeC6wt87!*{c#$xMQQ6+uczQx%IF9f#3(C}}=ZMKCl_ zeWVD%n|fmt5)?cRU$Ve&U*>&b?F?X^@2(X5@*ZUy=ZzW6Qp?1Xg5iP2_UZjVQ??}x zOX5qv2!KV({|wC_<+oRCX9uRCd7nQIOf>~eE`~jf>HXk7vNP#25&cDP<2G@)Y^VIkA9cy#;qZ$rlS`T)ceFnke z&wgAZ*ZL^}(vVOQqle|5J@LyVOtLfqpI1^MW!e71@o1Egqm^n;AHtFr)WW!UPpZgGLP%*k2QOYL6z{t2lUxYidIpQj^?iUYjo_aX z{qRV^cT>e9X*poT=6lNBz$rj31^q4=n|w&wPikm5EuNe$sPr!m<;F-1MlSsqh}L&O z#_i{4N*N)P6Jr6S?8juw18qAy(i>BR3JevMw(j1qXi1e5t;56(-kuAXaGYtvnIA69 zcU4~)bSNq=mIOpa@BX6z7}Xv8N|fQVRrItnlO7dg1~f*OJiid9eJB)51)`cqJ8aaB zmd(a8#iYjB$-^H`tN$6~)m<;D0-Enp!p55wl*u?-{8(wlZsGb|p0pY^6EdR|C(RW^ z%GjIy`s`x%?swF0qF6Vzu>f+hp7lZ~(@r@*Tw7{5M=P9DZEQru6}YG0^$p|AZU;3{ z$XkD0?#)D;2i;id?gzR!q`<#Z6+|g*yVaz|oA-s3B2xIfEptSl{>Ln{4<2$$f4-s+ zL=Xn;d5|CZcb_kXbM58>@+mKmBy%*-k_T`BpgR1#@G@&xyMGv?3LKWd{uQ2q>zmZ0 zZp$_U%Bui4aK0Th#H0&v0S(SNBDQK+F_$VX+F}+pe`Yg56?&DoMNi8ueSLp2?q{<~N}iH0>+K6G|4+q()-t?5Ww z0>uGfu%lK-X?6RJ?1xYg=Imwp3n6blX-o3|jbT zfKhMahF>F=qu)%;D$47R$$C!*4~{r+dIaUBZ1IGdUvmKRRTg>I=D;>V5trU9^|7h- z&1r^8)W^JMBn|#Jgv(7uHnkZ~Gt7WZG)aucV`VeJ%D*EW>zZgm_kSVYfv&{{kX!TRvA1pP#c1X0u5<26+ zC60q8_xoB9N-C%UToLdD$&T*70?eNenC#`;dDhxV_5!lq= zE)LqFDs>iQCH!*$yZLBnZI)=64-lCM3uZt^x1CE0cus#;6kgr?f;1Si(>O%559CIz zf7RF7so|L&*J)Q-qJpvM4ggkVMXOWmd+Xg?Ztqj_0LA0UW59NVRSM{W%{8ZEQ1}T{{VdS^2f@WOxT$~Ql_ET!Tq)Km$ zN6#}evjm9D&fWD(Yo_cMQsjnd_ZsZ@UK>V5Ex*$@SyB_1t85?O3X=bMvD8=@fU%v- zY9^;u(M&)=%-YA6io;Ccs^ZG35if4kQ}Xq59qT@kN6-5b9?df|jylGkSGwYq9@G6h z$3ibqE$DS-17HLJHxj|rvsZZo=9n)4SEc52eD_Z^ zzZ1~CL-I*RaiHU@M*9qC!8V3BRhX=I5X~2hrB4F)s~OEd_xOXSRndDZQM-rE?*KT1 zdr!+Au7UuN127v)JXls;fLh?fSJp5Nz5M`uN=Ii4^H=BQIH2}!2eJSK2lgIH5TUNF zP92f%LWDfE`@ZbD0w@bUA3z}?C~wq2mqXW$ zD7&nV57fdN$@8CIfG~MxLKUu1wF1V1LT_83oyCIL49K8=l+PK_V=8f7W?hvVH)s{k z>{3-ri|}ws)tyNB9I;B64xb@EBQux^dY7dDLH#Im@~C45P^{6{)qMi!!ZAQKtrtKN z7(g)HTjekY03stTZS4q5oS;#rY14~e-^v;P%GFc9QQTh7x@G?#-)S*5HFfOhp7jGC zUOyI4T#N^Y!a*o3q8Gyq3d;ac7TUETkWwEm`0k)Kod39eK9TaAy|Ap9ZONW0q5;0H z;<#r02MX(1q@c!mnab>I9vXtGs_BjI!XQVNppPH_)D_FFO;Qeg_^ZLrKG$AF{VI$z zAO$x#W2c$o9IsnoMw4wAa|7bib?~A!y&Y_xMcKK1fU5IQ-=w%0AW+483D4Pw^L@~L@K5TQ%5qoBkcfTgBV@d z8$1Db-em8ucP&ZgnD@H6MW10%5iF78Tzr9AdnM(4W>lAibruW97F-z$94=vZ z6Av+cIG@RI>(~jdVM;lkbEe_GW`G6H3&0S|rJdeOv{(6p-o)BhM>SVR!Oz;66iK}> zf#|S-o3FX?d<>IHq3v?;SK-6N(QtROi`lk9w66j^KQZUp1NBX{^;>p6jE~D`dr=X&N!8)Q|&Rw z-nmQl&ZSP#*HgWpo_pyQfT?22_wS@RiC&!@ki(wjZiBN{SwO2klxF<#}@Q|-ZX zGoR0{Iv(poI9Z%)4=jKc9lY+;sV%!nd+25#pyA^XegiCJ{^<8tXYX~N@jjOa96P#) M542TEz)xQPFTow~N&o-= literal 0 HcmV?d00001 diff --git a/docs/apache-airflow-providers-http/img/connection_headers.png b/docs/apache-airflow-providers-http/img/connection_headers.png new file mode 100644 index 0000000000000000000000000000000000000000..413e9bbb38864faf0dd5664703b9089ff0b7bd5e GIT binary patch literal 5256 zcmb7Ic{r5o`yXfn{-V})%5hC16>~W0*`i5pPc5~&!_Z*&@TQfZG(MG_BoO+`NXB8 z9cLtFwPB4YYQ)t9B8V#!9>!GTtzxn9z61NxOR26(i5#r%Yd8mX zpspJ8#$8$syuMdWkp$lPLfgu|3wA>n2RozP0kP;M(D-3DJV7{eSD?`7DFLe$svtVK zQXrI18VCt+$j?P)C&oNc2yK;mP)&FT>;wtK#y-+W5L@X#Lh$T(nAMSE;3&rnyAcEC z=B`9Y)31Gasqo@r&2Lc$Ksrplcbu4A)+;|xiiy6^5E1hE2n=@Smv23B#hBxd`M!T# z;oFxEnNSws+K+=nu|$kS1<~tp{^0_Dt>)tIEzT4=aOBWXNS{k_ety2;UKs4bA;@|O zqI_48U5@qP!-su9fX5X;*!ZA*_1&YXJa?|LQ*sBZ*nIV=EK&jnJF5asx!*D8 zqD|Tiyz1lgM;Zu9wrlLxa<^2*#&T(F&im)*@7WOyvr6k48XC^!Q6Dt=gTuhh;JSW! z(nzC#D9CHZ`xnj`gs`zBwwg({QRK?w@WqL7$}3N5ZJ|8t-809c6d6RJKWPi@wu&2x zcEr;P^YfK*^h2JGK+AEsXA3O%nX@t9lWTgD%@8mrgimB`Rxh%jl#)356_u5Rr}Hff z?kZUoZp_tbcLuA?4A$sUGj)_w4(KQjf$QcHa|a|64;?m;j&!dcTY@2ScBl@KMh*uy zaWRImoXw@-$oX-~>Tsk#A-+3jdAhG?1YE7D&CiQIA7B5B6iA|ZE=_j%Y;(4J-riS3 zzCK?Y?8CVhzotq4@DLf){qv5U(jL%R7dBit5tFhO#p1X{FQ4_;Y!)_YAF(n-l;GOZ z54)J*k0Tn12Z%>5n?=l&Q35k1hNBj#dRTh-(qK?$L7sVH;mw`Gp&QXVV0s4J)6kcP zj*}~sP)CV~x22ddX`Y|i<5PF7mTn3IloxEAW*u(6*i%(x&7R-&?tm0 zc^u>|u8Vn;W}COTY;i)gElS_{pk}(GerhFG)QC&e}LIYN^nK_x>Q`K6N@g{EVKwmRXK^9NZ>d z^HiGgx_1W4yW}9TxSH^Ygbw?&xJcU6R>z?bT%}2jLma$IRh2MOqZ71LnRK|p9M{JW zTX5+0%A?Yc`38ou2VZrDA~|&5x<%F%n~Y%OursJ-)Y7YY=%p)3vUlBA^97XL;cpAk zc(&GQ1>T3=j!M_wHJF$#QGbgZ7$D)Tv;p)1kgr^3(sXt&WWY)R!f?yFA<>ogRGAmHtO$}!F+-icSz0A;Zz(U~Nh3w5b9 z3lT|Ew2h5Tj9;7%s{yr*ZvrbxJI7E~#ci6CBa`P!UKp@s$;&)2y=T@5aP7IlEEP<4 zreFPNg5DMOrzkrGj>X()qMUWUWvZ=t;$AzPR&9RSvd^vQ8s=PG_iG+RVKD-8s+6=s zzmiAYm~Y6K7}Pf)>Uu6cDGy*Iv`Va0_-eZ_ZL}NiU+`>yQz}2aF^XKA7*0>vD|g>= zuY7vCrXSkPZG1xNMx#Rkt#I&Jg1qB310fI?_5Jkvn%G#*Mv1}xf(B5A4fY?lCT)^C zdw#zuKfx-nN&m!Jo+h5uHr$Sq!ygOsWGu_YtmPRtF06bv)$djH*vG+t4s@1wynw3; z`r)r5k<@I;;@T_c%1o&w&uSa?yVcpw5I1jH)u%~E9}UlqCF_C`zBH3n#&`N(LNLH>GgW*=?Jnd{y>_P zUfD%8If`a>6G9aex(h3Jp>gxKzSQKtBwCRGR=-mqnNfB|J#{3kS$nbjDHV@67Fq+# zja|k;FwrEzSQyLcVLu5#xa`IbM@&=ezkm89GeA7gw*9fxu|U*S-wj{Fl$e4&sAe}? zZStrst<#j1$2sQs$wcv3${l=1sl23{r6_%7FueHgPYRywmNI3VeazLgAor=W%-M@P zlk!}kaC+Pf?;d_8?x;dKvSG1fAUq@K(FodiIy$8+^RpO*J}|I&C(}v#9Viw{gB(Uhx^wD6z@c9;^~di z&VD}}jHPJVTg2isr^Fht`-aMn3>l99h`|eZbP$a2a$z4io4Br9$D6Vb(rxR?sorr0 zH+KrQr^I_ik&~_1YaZKG=&QIq^vpp?il5sNlhvLi&la6#)zG`it9-e!Uy#s)Jy=j_ z2ZG$PopwtK{sOuix%Gns#yG3_Umw^8p;7u_3zvif#+)?oCd}&7rOnI&1Iny#t z**6xC8=!a3^uJ;w%f@ln7NbX-;tk)sc#DV5MyL(3o2Lb(j~^10hwCAfje{riwU{uP z$(M~9@4>=8QWXD#YHJhVSS;43DQ=g~pS?CVVbh)vDwz3D1|h2@v9E2L%3@G@Zt&~~ zyt)2B!?Ez}vvUtHH7Yh4T4>VxyV7n5?8w#n-BUqZN3PDHK}WXhDZ*$AyA>jg&HX`n zM3fX1D4V1nnqKa6)6U_6E%X2va#y}WE5mQ0Ro_owJ5&8XmR1q6bv!H~oE&@E5Oky@SUzc)d}XKWJyRt_1;{R^`9_2p4PRVDHZO5d0FAFS3fT5Q*y zTI@98!ggX5`xK)dG)jX`_o)t9+~8e)p@W$Td!@Rap2@EK9{-p{d}~K?0G35(ogWJUxF=^gdUdFEF?O|X zWW2NbTK{jGS&Udp80iSWarKc(u-BNZukco-=@vl0Op55O@~SHEh|+X=c_@u?Cj%s> zD0XYwEr&=a1B^VJ=_B5m)UPb+-V#{67M-%7%$u?R0Pv@e9(^S)5DJ{?!I)4lzp2Cy zu*3MYMAKV>(qs+N)5sO%V6kFroqLcnHVR?^75TlzNjGI{9AH z?_c<|XX>QL%G^HTMM;&N>MeFa=B|(3)@Fa`%C`!7$p@M(mq&mM*ZfOB3F(dIfsgc;t6Edc<_QlVP9D>>+G z9(9a-`*sm8%$|EAAi~rBugTHw_wUnX__s%eV}BraTQF%V>uvWltjO2WGvA3dxIB`# z4G-^45R?i0p&MVq)CF}e*~W@6R$_#(sKai!0^@_q|>wch2;jFQ|)-Vb2mh+Ain#B!$aSK zgM%NxfB(MRZ>W|n^4EvyQ{Lj0enZ)Fq-_ox5-+~6@E)2cz{)=Tzg_tc)BZ1Z3{-_s zfV<`zR)@ctG;t4Od@A2E#Twi_4PhQ2LU4~K7~b^s?6YoAsJp>6Y=by@>0cbZva(Vy zbml|wSW6P**3uV71{~N}nJuoUsQ5;m_%9^@2pIJXO?#mJ`&B6TzIZWPx5g}3+T|0^ZBmz$t5%Ur1?3w22x^7u=(PV>RJdI`5- z^5^Wpb43lPll8hFUv~BI-zdBe2yHOv7u@*7&++0#R;vR_BVT|)XF5ik6MH~$bg;6r z;<`C&{WBWUsQwm}f+pItHnl}m?H-;4%917|w!CQhqGSC%M#99CRVu=2-ZvfdRh#nV zN?ooPuB$*F`~PZaU`p;UwSw5cL^-vBGu!X4KoNq$wxIO?Z{>})&HIxWyH+xol?#nJ NWnqUZF}w83zX2P`_&Wdq literal 0 HcmV?d00001 diff --git a/docs/apache-airflow-providers-http/img/connection_username_password.png b/docs/apache-airflow-providers-http/img/connection_username_password.png new file mode 100644 index 0000000000000000000000000000000000000000..6e36e77dd4cb48f62107a3f654648587bddc58e7 GIT binary patch literal 4761 zcmchbcU03^o5!P|;P@gUqabyRBPa|Y0trZS6j4z?q$vUcqA-y*h?E3EU@S;qMnF0u z(g{VRlYmN5T9iFsdW&WcE+Gf+@j6Uopb@@XX=e3HCon2biA)vxaT*S!RJ3xgOLI`DzV z&2xKseS&`%p+IgvaCdjl&(1!j0f*;S(Sz>pj$4ZtC$~^wE!F+d(mk)c>gXq@hPy(@Cb59eAf4h1K$Ny zvvJ3q1Oy^s9O<$3lCe(nuy`l_}xUI_%)8AUWqzr*n+Y^pZitg6I zLeC6Vdf;g*nKB{9kSgh{rD%npDisdYg zw&r?`wq!pR5I#1!Ia65DJ<=ROmi8NpeRfDXd7>*VMgQd1EP~tCFV)(LekLrNK0(UJ z;64kL!JLjdt|rG;K@e?IRf<=N>*m|~;XX4xJwrpbb9|6jLlSbOdxrb-<@r)Hl|Lku zyO1VGn%Ip;hs?Xuv|qAGiK-`cS1CxX8)2p6e}u}Kw7*m#Z9=8Okk%#kX?ol(W)Ij@ zlH{JYR#{wXOxW4e(fJ+3JGGBc$+`i)#D>a>it$Cci%E-AKl-?<{o8xj(k&%2#4@|$ z&Y0hFb#>LdvNb;%qi-L$ooANSouuZnytG6=vz=grQ-eUp&667^+Ts;a&Eeyy+mplP z2)Z6>J#QV(@yaZ;E-`w%|Inw_`o#~ckGMri>-DNP@j*W-?5{1u(y{EEOQDCd z%aDe-g91V~+FvTQx9nnX%l#QJ+Y0W?mTJXJoTg+7bT^2j%$0sEm}s`W-f+_M>G&A(XNP6 zho85#oVVSEs>>*1(3=JGdmuMWvJbe)B$MM})-O$7P?K?$P{UZzf@bkm%iDH!_FaQn zhEX?C;GRb0924Wmp0aTPh@kax+&-9Ua5Tg5Qs|rG5W{YvgBwrWhnDlebNt z9hk5$ax&VOrDic^M?xu%yB!#9bM%BUKTL%7=;$PNdwnv3k%%U1^n}gMY*br3(fIgG zsC`o*#JFQM-+%GF069VFZaBj@1(vPlJu#LcZ<4x(=s#+iw1*#Zp(Y+$s)gOKpy^Mi zhh1JEh{`|oY+<3$n<1MnKkys2Z9Ux)@ z%BUKpVONr&Z|;;k8*i)*dNQ|f3*??JoyK&EP}~>~JsA>|8_DWt1a3eeJ-N5|VI}jl zGMR+Q*(I5YLM_6Y15OHtRX9Q~gnh||_Pq@YxK;hNF=yGk z|BgOMZ*$IA0Jii?4~WJH?1^JPu2w2Qf;Vo4H_G)Dd=BXh=iVR}^TRHM+Dgw^$;8e` zP$DG*Rzz~=_oft=tOs#9Y;VtPoe-f5S=C96a!oeMBG2Z&RVeC*f6spIFLmbFo?@NX z-$(3iy%C@2DEf0N_~e%_5<`V$yEaGhHzJNH86j{SAA4;j8F&G|99)9ZRFv$`dcdBi zu9uI*4r}Vk6o&h_d#IO&T@D>(j-DTBFx|o~vD0mqTH;;PV4m3X-3Q_Y2ELXyjh<)exI zAI=l-``$WPgJko zCOa?C2+evBNE_aoFXhY&d7nehVw^muQmF~`&aPISZY>J6*5|qedh&Tac5AJy9(QH( zm0IPMsCqSB_E1ax)UWj`*Btm3XMM|gi9PO4$$nVHgXxSCWr!5{^Jx6wl2xuh-mjxl z>sJSySG1r52UEhrsa5?&_IB9Q zkOnAEI3>)e{ID|3v6d+A3&aAo0YTqdcwwrOj>!D3vE=2>L>6#hcMp(G$SV=x=see* zezA@FgKI(7{j0FEvooJSAk=~-`Y5$jtM)OXL5uDODAYAE75gF&V31wiqv=~Re=1=G z$f=$(BhUK2el;TiLuQO3QAjt*f0xSq=BS^D*ykxWI+KMvgkV9i;VDWRx9}jNfL%DaGjA|W{vQ$NpS3m4=I-6QJZ}L}~wP_n}m)v?XSj1Nx_PDiK^uj29eT?=~Mn-f>J)|u$Y0Q?t#MSwG^ghl$AI*YQh zPGZ4#`&YAys;bE{_ed{f(V#TgRDd`-wlxn=(T4zRi8N6v&%LPE2D@vdn9hvyJOK)0 zo5bNQwNxHk*?5Vn)$syP6ln|k`s~Ko5m3|5!!&g$uh4iW@cxxT)+;#8UzfUEN6*nH zv?yuiGB};hJ>V9!_DASU zA8w_fYPQUpz`4E(xJHivGHtDklg(P_KGgdyG^%6DtOoA5MV+o&AQo=N*DjlLvwkSu zGld5EG;sN|JgWQ6HPoknI{`GAsz0FpaO|ptUZB5(wy)cH05C!;>Dbi)^kQzahB0rOS2)^^!pMBGqd>KLLc`E3qWiy%&Z<__TUz@VEwbOwXrxKJ5u3YvNnYXhWJ7bY-C*PQ~_Dzw@4q zcEWY^tbhD%ABytj1gg=ULuR4O$zh`M%fx*LCniZXf$I-xzzlPdW0oDL2W@J&Z{NOk z$17TQhaYX%CTXaz%nr&Mr=BHx4{sWI&DHu?bJMNu(Wi{(rNl`=i|iWidjZ+8*_hcn zTwe!ktI{1fRmN8>#Q2FIXJe)P=pUDckPlW`Vq_64XHt3^j*Y=&;I`3a(iH&Xk_p(& zRZbq87O-IFSJJoAH)Dyoi{Du~p#oD;jJFbo1WPg__Q4!Y$a6zDX6NJSxqW8aS7EDRj9^xD&NQP6Uc9bgIU$EzSuy7XkUDeWJ73?Nt>GqqxAGePKGm z5R9MD)q&%jw?d1gs+`u}vRGu(?95C154u4x6d?7k=NC@JpLJXuexIL}mF2ABTxfv- z*xT6+5O2abH*6Z%;@j@Ch9He$Y&03*q@cESMj*l(~?m8SC=FfD5o|Ix9xc zsD!xj0DRn^TkDCQ2pXG$oMrmnPw4oHL(KIfaA_uK=i|9Tm-f@spV3u+%1~@-kPTXM z-1b%NI^M9!GB+J35wTwy-(h+ShP3?`9cn=y1(?>As)tqJN~zsf7_rHMBRAgFb9$Z$ zp|w9G&miaP*bk;e`$`ZX9ZflcFmH-%&o|G_5jko22F%Ej4`_{%HO;ybVCW39*?lDqn1(u<{qD}X_YW2lcjDZtPyo^xHQn2o zV|6)CWJLAN{p;M+`8lb3EggwO?8X=;Mh~+xz~*eSC$A*D7JJ_@a`|AyqMX6$TUst1 z-FH@|vZjY>ywQlQSk){!=dO3DdTBA}DsKmgj0#@{qC13d(_6)PxV|>P53KaFMIeum z#4>N@@x$y~{zh2+mOga1(hc=BBI)8+aCBm&f};?~S92I1gGFyw-@CR!mgOcs{v9mw zSgXNV(bgh+f`Wij{0agP0++y&v?a~&`%-~MAxp`hbPV!2D#7{mX21p%0%^%#S95Ea&2 zqmQilODU|X*w>#nCFptQWHS?)&D*z2t6lTnPS;E>`|9+mBR5z>5m-R2$vMVp$@58? zC)nXvUdvIDk7z0aZ4iF6t&|#3ZEs`7Mj;%!@ot)dR zs}K|U;waI$QPms)nQDDfWZ3Y19^^!(0-7FonL^qnn`Ig1NNf9!4JdmBb|h<|m+$G} z>NBCyG2cr6D5(*fov^^Dmgl~HEB>&fF>K${(N2r99r$UmauCfl({Zv~mmEp-e^j_J z1lg+J(TCUF{4mwW6U5E=&e{GZ$=7-u7Wv&+Y%^LZMLr`C+K}Y*^~2NV!phd@<90tm zrJ#vn@XMEf6^%GjRyfl%wR+Q*2 zIy+?1^PStpdc&<^FC^2Ikm9U=TSukp{J%jB&r7>^DgT*=YK7eXx}-UiZLwfo1*9JQ2&Vsd b1GBE*^^2vH0QXn|ulCwy6NBQ54!`{!fAs15 literal 0 HcmV?d00001 From 79efb5e44fe8facb19e8028a7bb86217a7eb5a0d Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 08:18:23 +0100 Subject: [PATCH 155/286] feat: Implement auth_type and auth_kwargs in the AsyncHttpHook --- airflow/utils/json.py | 10 ---------- .../http/tests/provider_tests/http/hooks/test_http.py | 7 ++++--- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/airflow/utils/json.py b/airflow/utils/json.py index eb3cd40941197..a8846282899f3 100644 --- a/airflow/utils/json.py +++ b/airflow/utils/json.py @@ -123,15 +123,5 @@ def orm_object_hook(dct: dict) -> object: return deserialize(dct, False) -def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> Any: - """Safely loads JSON. - - Returns None by default if the given object is None. - """ - if obj is not None: - return json.loads(obj) - return default - - # backwards compatibility AirflowJsonEncoder = WebEncoder diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index fe6b8f882f2b7..1e944dda1d090 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -384,9 +384,10 @@ def test_available_connection_auth_types(self): auth_types = get_auth_types() assert auth_types == frozenset( { - "request.auth.HTTPBasicAuth", - "request.auth.HTTPProxyAuth", - "request.auth.HTTPDigestAuth", + "requests.auth.HTTPBasicAuth", + "requests.auth.HTTPProxyAuth", + "requests.auth.HTTPDigestAuth", + "aiohttp.BasicAuth", "tests.providers.http.hooks.test_http.CustomAuthBase", } ) From 0247bc25e0f75eacd530c1ef4e4de26fb1c4613c Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 19:04:06 +0100 Subject: [PATCH 156/286] feat: Add tests --- .../provider_tests/http/hooks/test_http.py | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index 1e944dda1d090..b6f43995b979d 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -437,6 +437,32 @@ def test_connection_with_extra_auth_type_and_no_credentials(self, auth, mock_get HttpHook().get_conn({}) auth.assert_called_once() + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_string_headers_and_auth_kwargs(self, auth, mock_get_connection): + """When passed via the UI, the 'headers' and 'auth_kwargs' fields' data is + saved as string. + """ + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + login="username", + password="pass", + extra=r""" + {"auth_kwargs": "{\r\n \"endpoint\": \"http://localhost\"\r\n}", + "headers": "{\r\n \"some\": \"headers\"\r\n}"} + """, + ) + mock_get_connection.return_value = conn + + hook = HttpHook(auth_type=CustomAuthBase) + session = hook.get_conn({}) + + auth.assert_called_once_with("username", "pass", endpoint="http://localhost") + assert "auth_kwargs" not in session.headers + assert "some" in session.headers + @pytest.mark.parametrize("method", ["GET", "POST"]) def test_json_request(self, method, requests_mock): obj1 = {"a": 1, "b": "abc", "c": [1, 2, {"d": 10}]} @@ -724,7 +750,7 @@ async def test_async_post_request_with_error_code(self): async def test_async_request_uses_connection_extra(self): """Test api call asynchronously with a connection that has extra field.""" - connection_extra = {"bearer": "test"} + connection_extra = {"bearer": "test", "some": "header"} with aioresponses() as m: m.post( From 775ae616796063057d5b1d12f73cdc8d138edb41 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 21:04:22 +0100 Subject: [PATCH 157/286] fix: Add header and auth into FakeSession test object --- providers/http/tests/provider_tests/http/sensors/test_http.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/providers/http/tests/provider_tests/http/sensors/test_http.py b/providers/http/tests/provider_tests/http/sensors/test_http.py index 78a11e15bb7c1..47af8f49c48cf 100644 --- a/providers/http/tests/provider_tests/http/sensors/test_http.py +++ b/providers/http/tests/provider_tests/http/sensors/test_http.py @@ -238,10 +238,14 @@ def resp_check(_): class FakeSession: + """Mock requests.Session object.""" + def __init__(self): self.response = requests.Response() self.response.status_code = 200 self.response._content = "apache/airflow".encode("ascii", "ignore") + self.headers = {} + self.auth = None def send(self, *args, **kwargs): return self.response From aa89fdbc635226be56e8ef119debf3038f30186c Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 21:14:54 +0100 Subject: [PATCH 158/286] fix: Use default BasicAuth in LivyAsyncHook --- docs/spelling_wordlist.txt | 1 + .../livy/src/airflow/providers/apache/livy/hooks/livy.py | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 2b35ed5bc2dd9..7aae2582f24b1 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -25,6 +25,7 @@ afterall AgentKey aio aiobotocore +aiohttp AioSession aiplatform Airbnb diff --git a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py index 934985befbc91..f73fa5d87da92 100644 --- a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py @@ -87,9 +87,8 @@ def __init__( extra_headers: dict[str, Any] | None = None, auth_type: Any | None = None, ) -> None: - super().__init__() + super().__init__(http_conn_id=livy_conn_id) self.method = "POST" - self.http_conn_id = livy_conn_id self.extra_headers = extra_headers or {} self.extra_options = extra_options or {} if auth_type: @@ -491,9 +490,9 @@ def __init__( extra_options: dict[str, Any] | None = None, extra_headers: dict[str, Any] | None = None, ) -> None: - super().__init__() + super().__init__(http_conn_id=livy_conn_id) self.method = "POST" - self.http_conn_id = livy_conn_id + self.auth_type = self.default_auth_type self.extra_headers = extra_headers or {} self.extra_options = extra_options or {} From 70eee06d8628683bfd391f73f4a006a3a9c010f6 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 16:36:22 +0200 Subject: [PATCH 159/286] refactor: Removed docstring for removed json parameter in run method of HttpAsyncHook --- providers/http/src/airflow/providers/http/hooks/http.py | 1 - 1 file changed, 1 deletion(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 9cf5d4983dbc5..932aeb31d29c0 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -397,7 +397,6 @@ async def run( :param endpoint: Endpoint to be called, i.e. ``resource/v1/query?``. :param data: Payload to be uploaded or request parameters. - :param json: Payload to be uploaded as JSON. :param headers: Additional headers to be passed through as a dict. :param extra_options: Additional kwargs to pass when creating a request. For example, ``run(json=obj)`` is passed as From 6a30c527477442fd2471a2080fe692b1ca1bb237 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 16:38:39 +0200 Subject: [PATCH 160/286] refactor: Aligned HttpTrigger with version from main branch --- .../http/src/airflow/providers/http/triggers/http.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/providers/http/src/airflow/providers/http/triggers/http.py b/providers/http/src/airflow/providers/http/triggers/http.py index d25d3a55cfb5b..ec9780bdeab49 100644 --- a/providers/http/src/airflow/providers/http/triggers/http.py +++ b/providers/http/src/airflow/providers/http/triggers/http.py @@ -73,7 +73,7 @@ def __init__( self.extra_options = extra_options def serialize(self) -> tuple[str, dict[str, Any]]: - """Serialize HttpTrigger arguments and classpath.""" + """Serializes HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpTrigger", { @@ -88,7 +88,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Make a series of asynchronous http calls via a http hook.""" + """Makes a series of asynchronous http calls via an http hook.""" hook = HttpAsyncHook( method=self.method, http_conn_id=self.http_conn_id, @@ -165,7 +165,7 @@ def __init__( self.poke_interval = poke_interval def serialize(self) -> tuple[str, dict[str, Any]]: - """Serialize HttpTrigger arguments and classpath.""" + """Serializes HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpSensorTrigger", { @@ -180,7 +180,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Make a series of asynchronous http calls via an http hook.""" + """Makes a series of asynchronous http calls via an http hook.""" hook = self._get_async_hook() while True: try: @@ -193,7 +193,6 @@ async def run(self) -> AsyncIterator[TriggerEvent]: extra_options=self.extra_options, ) yield TriggerEvent(True) - return except AirflowException as exc: if str(exc).startswith("404"): await asyncio.sleep(self.poke_interval) From beab41f2d1b29320a84246148ee419d5399da9f6 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 17:03:29 +0200 Subject: [PATCH 161/286] refactor: Changed docstrings in HttpTrigger to imperative mode --- providers/http/src/airflow/providers/http/triggers/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/providers/http/src/airflow/providers/http/triggers/http.py b/providers/http/src/airflow/providers/http/triggers/http.py index ec9780bdeab49..5975389830f36 100644 --- a/providers/http/src/airflow/providers/http/triggers/http.py +++ b/providers/http/src/airflow/providers/http/triggers/http.py @@ -73,7 +73,7 @@ def __init__( self.extra_options = extra_options def serialize(self) -> tuple[str, dict[str, Any]]: - """Serializes HttpTrigger arguments and classpath.""" + """Serialize HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpTrigger", { @@ -88,7 +88,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Makes a series of asynchronous http calls via an http hook.""" + """Make a series of asynchronous http calls via a http hook.""" hook = HttpAsyncHook( method=self.method, http_conn_id=self.http_conn_id, From feece9ada0f334800b7589be81b5c574106448fb Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 11 Apr 2024 12:15:06 +0200 Subject: [PATCH 162/286] refactor: Updated docstrings of serialize and run method of HttpTrigger --- providers/http/src/airflow/providers/http/triggers/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/providers/http/src/airflow/providers/http/triggers/http.py b/providers/http/src/airflow/providers/http/triggers/http.py index 5975389830f36..d30f41990f5b0 100644 --- a/providers/http/src/airflow/providers/http/triggers/http.py +++ b/providers/http/src/airflow/providers/http/triggers/http.py @@ -165,7 +165,7 @@ def __init__( self.poke_interval = poke_interval def serialize(self) -> tuple[str, dict[str, Any]]: - """Serializes HttpTrigger arguments and classpath.""" + """Serialize HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpSensorTrigger", { @@ -180,7 +180,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Makes a series of asynchronous http calls via an http hook.""" + """Make a series of asynchronous http calls via a http hook.""" hook = self._get_async_hook() while True: try: From 42486f6dd413232d78002b8565939d2a65abfc60 Mon Sep 17 00:00:00 2001 From: David Blain Date: Mon, 6 May 2024 13:09:59 +0200 Subject: [PATCH 163/286] refactor: Moved get_connection_form_widgets method from HttpHook to HttpHookMixin --- .../livy/src/airflow/providers/apache/livy/hooks/livy.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py index f73fa5d87da92..70cf4190e63aa 100644 --- a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py @@ -80,6 +80,10 @@ class LivyHook(HttpHook): conn_type = "livy" hook_name = "Apache Livy" + @classmethod + def get_connection_form_widgets(cls) -> dict[str, Any]: + return super().get_connection_form_widgets() + def __init__( self, livy_conn_id: str = default_conn_name, From 0575391fa3b315e49e6d78ef68b908caf506befc Mon Sep 17 00:00:00 2001 From: David Blain Date: Mon, 6 May 2024 13:15:40 +0200 Subject: [PATCH 164/286] refactor: Pass auth_type parameter from LivyHook to constructor of HttpHook as it has also this parameter instead of redefining the same field --- .../livy/src/airflow/providers/apache/livy/hooks/livy.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py index 70cf4190e63aa..c4e740d5a855f 100644 --- a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py @@ -91,12 +91,10 @@ def __init__( extra_headers: dict[str, Any] | None = None, auth_type: Any | None = None, ) -> None: - super().__init__(http_conn_id=livy_conn_id) + super().__init__(http_conn_id=livy_conn_id, auth_type=auth_type) self.method = "POST" self.extra_headers = extra_headers or {} self.extra_options = extra_options or {} - if auth_type: - self.auth_type = auth_type def get_conn(self, headers: dict[str, Any] | None = None) -> Any: """ From 2bafca4584fc0b932eb7393978f720ab83bb1faa Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 7 May 2024 19:34:37 +0200 Subject: [PATCH 165/286] refactor: Enhanced extra_dejson property to allow load string escaped nested json structures --- airflow/models/connection.py | 2 +- .../livy/src/airflow/providers/apache/livy/hooks/livy.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/airflow/models/connection.py b/airflow/models/connection.py index 01df1626657da..19aeaf9e63089 100644 --- a/airflow/models/connection.py +++ b/airflow/models/connection.py @@ -432,7 +432,7 @@ def get_extra_dejson(self, nested: bool = False) -> dict: self.log.exception("Failed parsing the json for conn_id %s", self.conn_id) # Mask sensitive keys from this list - mask_secret(extra) + mask_secret(obj) return extra diff --git a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py index c4e740d5a855f..408aac0818c63 100644 --- a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py @@ -84,6 +84,10 @@ class LivyHook(HttpHook): def get_connection_form_widgets(cls) -> dict[str, Any]: return super().get_connection_form_widgets() + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + return super().get_ui_field_behaviour() + def __init__( self, livy_conn_id: str = default_conn_name, From 8d12d3835c1a6a5f15faf68ec17258454fb846d5 Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 8 May 2024 16:02:23 +0200 Subject: [PATCH 166/286] refactor: Changed conn_type to ftp in test_process_form_invalid_extra_removed as http as livy do now also have custom fields --- tests/www/views/test_views_connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/www/views/test_views_connection.py b/tests/www/views/test_views_connection.py index 1e21dc4856ed1..19a36c0ac6b39 100644 --- a/tests/www/views/test_views_connection.py +++ b/tests/www/views/test_views_connection.py @@ -462,9 +462,9 @@ def test_process_form_invalid_extra_removed(admin_client): Note: This can only be tested with a Hook which does not have any custom fields (otherwise the custom fields override the extra data when editing a Connection). Thus, this is currently - tested with livy. + tested with ftp. """ - conn_details = {"conn_id": "test_conn", "conn_type": "livy"} + conn_details = {"conn_id": "test_conn", "conn_type": "ftp"} conn = Connection(**conn_details, extra='{"foo": "bar"}') conn.id = 1 From a1e621734108a6fb5d934aa960bd42caebb881d1 Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 18 Jul 2024 20:14:20 +0200 Subject: [PATCH 167/286] refactor: HttpHook now uses patched version of Connection + added test which checks when this patched class has to be removed so we don't forget --- .../src/airflow/providers/apache/druid/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/providers/src/airflow/providers/apache/druid/__init__.py b/providers/src/airflow/providers/apache/druid/__init__.py index 7585be9880dc1..d00c3c8da7757 100644 --- a/providers/src/airflow/providers/apache/druid/__init__.py +++ b/providers/src/airflow/providers/apache/druid/__init__.py @@ -37,3 +37,17 @@ raise RuntimeError( f"The package `apache-airflow-providers-apache-druid:{__version__}` needs Apache Airflow 2.9.0+" ) + + +def airflow_dependency_version(): + import re + import yaml + + from os.path import join, dirname + + with open(join(dirname(__file__), "provider.yaml"), encoding="utf-8") as file: + for dependency in yaml.safe_load(file)["dependencies"]: + if dependency.startswith('apache-airflow'): + match = re.search(r'>=([\d\.]+)', dependency) + if match: + return packaging.version.parse(match.group(1)) From fc75485b738bc4505a9b8808c2f671b35b8d0298 Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 18 Jul 2024 20:56:42 +0200 Subject: [PATCH 168/286] refactor: Fixed some static checks --- providers/src/airflow/providers/apache/druid/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/providers/src/airflow/providers/apache/druid/__init__.py b/providers/src/airflow/providers/apache/druid/__init__.py index d00c3c8da7757..18870506f868b 100644 --- a/providers/src/airflow/providers/apache/druid/__init__.py +++ b/providers/src/airflow/providers/apache/druid/__init__.py @@ -41,13 +41,13 @@ def airflow_dependency_version(): import re - import yaml + from os.path import dirname, join - from os.path import join, dirname + import yaml with open(join(dirname(__file__), "provider.yaml"), encoding="utf-8") as file: for dependency in yaml.safe_load(file)["dependencies"]: - if dependency.startswith('apache-airflow'): - match = re.search(r'>=([\d\.]+)', dependency) + if dependency.startswith("apache-airflow"): + match = re.search(r">=([\d\.]+)", dependency) if match: return packaging.version.parse(match.group(1)) From 747f061e8fd1b035490373e2ce9b12096293d6c2 Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 30 Jan 2025 12:18:22 +0100 Subject: [PATCH 169/286] refactor: Removed assignation of method with POST value as it is already the default in the parent HttpHook --- .../livy/src/airflow/providers/apache/livy/hooks/livy.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py index 575c7f9fdf2cd..ae5ef805eba22 100644 --- a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py @@ -23,7 +23,7 @@ import warnings from collections.abc import Sequence from enum import Enum -from typing import TYPE_CHECKING, Any +from typing import Any import requests @@ -31,9 +31,6 @@ from airflow.providers.http.exceptions import HttpErrorException from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook -if TYPE_CHECKING: - from airflow.models import Connection - class BatchState(Enum): """Batch session states.""" @@ -94,7 +91,6 @@ def __init__( auth_type: Any | None = None, ) -> None: super().__init__(http_conn_id=livy_conn_id, auth_type=auth_type) - self.method = "POST" self.extra_headers = extra_headers or {} self.extra_options = extra_options or {} @@ -496,7 +492,6 @@ def __init__( auth_type: Any | None = None, ) -> None: super().__init__(http_conn_id=livy_conn_id, auth_type=auth_type) - self.method = "POST" self.extra_headers = extra_headers or {} self.extra_options = extra_options or {} From 9b20f78fa5dacc145790c501f81f5174d33e180a Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 30 Jan 2025 16:54:04 +0100 Subject: [PATCH 170/286] refactor: Formatted forms --- airflow/www/forms.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airflow/www/forms.py b/airflow/www/forms.py index b69184c6590c2..2a79684122fd2 100644 --- a/airflow/www/forms.py +++ b/airflow/www/forms.py @@ -178,7 +178,6 @@ def populate_obj(self, item): class BS3AccordionTextAreaFieldWidget(BS3TextAreaFieldWidget): - @staticmethod def _make_collapsable_panel(field: Field, content: Markup) -> str: collapsable_id: str = f"collapsable_{field.id}" From 147fa031e06ee818282878462e199a8c6d9eaf3c Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 30 Jan 2025 16:58:29 +0100 Subject: [PATCH 171/286] refactor: Cleaned up files --- .../apache/livy/hooks/test_livy.py | 20 ++++++----- .../src/airflow/providers/http/hooks/http.py | 36 +++++-------------- .../providers/apache/druid/__init__.py | 14 -------- setup.cfg | 0 4 files changed, 19 insertions(+), 51 deletions(-) delete mode 100644 setup.cfg diff --git a/providers/apache/livy/tests/provider_tests/apache/livy/hooks/test_livy.py b/providers/apache/livy/tests/provider_tests/apache/livy/hooks/test_livy.py index bf17ecfad89e5..239fa01053be7 100644 --- a/providers/apache/livy/tests/provider_tests/apache/livy/hooks/test_livy.py +++ b/providers/apache/livy/tests/provider_tests/apache/livy/hooks/test_livy.py @@ -22,11 +22,12 @@ import pytest import requests +from requests.exceptions import RequestException + from airflow.exceptions import AirflowException from airflow.models import Connection from airflow.providers.apache.livy.hooks.livy import BatchState, LivyAsyncHook, LivyHook from airflow.utils import db -from requests.exceptions import RequestException LIVY_CONN_ID = LivyHook.default_conn_name DEFAULT_CONN_ID = LivyHook.default_conn_name @@ -48,17 +49,19 @@ ] CONNECTIONS: dict[str, Connection] = { DEFAULT_CONN_ID: Connection( - conn_id=DEFAULT_CONN_ID, - conn_type="http", - host=DEFAULT_HOST, - schema=DEFAULT_SCHEMA, - port=DEFAULT_PORT, - ), + conn_id=DEFAULT_CONN_ID, + conn_type="http", + host=DEFAULT_HOST, + schema=DEFAULT_SCHEMA, + port=DEFAULT_PORT, + ), "default_port": Connection(conn_id="default_port", conn_type="http", host="http://host"), "default_protocol": Connection(conn_id="default_protocol", conn_type="http", host="host"), "port_set": Connection(conn_id="port_set", host="host", conn_type="http", port=1234), "schema_set": Connection(conn_id="schema_set", host="host", conn_type="http", schema="https"), - "dont_override_schema": Connection(conn_id="dont_override_schema", conn_type="http", host="http://host", schema="https"), + "dont_override_schema": Connection( + conn_id="dont_override_schema", conn_type="http", host="http://host", schema="https" + ), "missing_host": Connection(conn_id="missing_host", conn_type="http", port=1234), "invalid_uri": Connection(conn_id="invalid_uri", uri="http://invalid_uri:4321"), "with_credentials": Connection( @@ -83,7 +86,6 @@ class TestLivyDbHook: pytest.param("dont_override_schema", "http://host", id="ignore-defined-schema"), ], ) - #@mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_connection) def test_build_get_hook(self, conn_id, expected): with patch( "airflow.hooks.base.BaseHook.get_connection", diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index bbbc94381598c..72ca10d796465 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -181,6 +181,14 @@ def get_auth_types(cls) -> frozenset[str]: auth_types |= frozenset({field.strip() for field in extra_auth_types.split(",")}) return auth_types + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + """Return custom UI field behaviour for Hive Client Wrapper connection.""" + return { + "hidden_fields": ["extra"], + "relabeling": {}, + } + @classmethod def get_connection_form_widgets(cls) -> dict[str, Any]: """Return connection widgets to add to the connection form.""" @@ -220,34 +228,6 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: ), } - @classmethod - def get_ui_field_behaviour(cls) -> dict[str, Any]: - """Return custom UI field behaviour for Hive Client Wrapper connection.""" - return { - "hidden_fields": ["extra"], - "relabeling": {}, - } - - @classmethod - def get_connection_form_widgets(cls) -> dict[str, Any]: - """Return connection widgets to add to connection form.""" - from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget, Select2Widget - from flask_babel import lazy_gettext - from wtforms.fields import SelectField, TextAreaField - - default_auth_type: str = "" - auth_types_choices = frozenset({default_auth_type}) | get_auth_types() - return { - "auth_type": SelectField( - lazy_gettext("Auth type"), - choices=[(clazz, clazz) for clazz in auth_types_choices], - widget=Select2Widget(), - default=default_auth_type - ), - "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), - } - # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: diff --git a/providers/src/airflow/providers/apache/druid/__init__.py b/providers/src/airflow/providers/apache/druid/__init__.py index 18870506f868b..7585be9880dc1 100644 --- a/providers/src/airflow/providers/apache/druid/__init__.py +++ b/providers/src/airflow/providers/apache/druid/__init__.py @@ -37,17 +37,3 @@ raise RuntimeError( f"The package `apache-airflow-providers-apache-druid:{__version__}` needs Apache Airflow 2.9.0+" ) - - -def airflow_dependency_version(): - import re - from os.path import dirname, join - - import yaml - - with open(join(dirname(__file__), "provider.yaml"), encoding="utf-8") as file: - for dependency in yaml.safe_load(file)["dependencies"]: - if dependency.startswith("apache-airflow"): - match = re.search(r">=([\d\.]+)", dependency) - if match: - return packaging.version.parse(match.group(1)) diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index e69de29bb2d1d..0000000000000 From 25aba007dd70864f6a20492c4fe2e4e601320b72 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Sun, 12 Nov 2023 13:55:18 +0100 Subject: [PATCH 172/286] feat: Implement `auth_kwargs` parameter in Http Connection --- .../provider_tests/http/hooks/test_http.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index 82a1ff9765156..e1a380e4588e5 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -67,6 +67,11 @@ def get_airflow_connection_with_login_and_password(conn_id: str = "http_default" return Connection(conn_id=conn_id, conn_type="http", host="test.com", login="username", password="pass") +class CustomAuthBase(HTTPBasicAuth): + def __init__(self, username: str, password: str, endpoint: str): + super().__init__(username, password) + + class TestHttpHook: """Test get, post and raise_for_status""" @@ -352,6 +357,25 @@ def test_connection_without_host(self, mock_get_connection): hook.get_conn({}) assert hook.base_url == "http://" + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_connection): + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + login="username", + password="pass", + extra='{"auth_kwargs": {"endpoint": "http://localhost"}}', + ) + mock_get_connection.return_value = conn + + hook = HttpHook(auth_type=CustomAuthBase) + session = hook.get_conn({}) + + auth.assert_called_once_with("username", "pass", endpoint="http://localhost") + assert "auth_kwargs" not in session.headers + @pytest.mark.parametrize("method", ["GET", "POST"]) def test_json_request(self, method, requests_mock): obj1 = {"a": 1, "b": "abc", "c": [1, 2, {"d": 10}]} From 70df6a275bcba8bcdadd36e180de301bb70c54dc Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 24 Nov 2023 00:25:53 +0100 Subject: [PATCH 173/286] fix: Correctly use auth_type from Connection --- .../provider_tests/http/hooks/test_http.py | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index e1a380e4588e5..802dd0ef15c39 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -366,7 +366,7 @@ def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_conne conn_type="http", login="username", password="pass", - extra='{"auth_kwargs": {"endpoint": "http://localhost"}}', + extra='{"x-header": 0, "auth_kwargs": {"endpoint": "http://localhost"}}', ) mock_get_connection.return_value = conn @@ -375,6 +375,40 @@ def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_conne auth.assert_called_once_with("username", "pass", endpoint="http://localhost") assert "auth_kwargs" not in session.headers + assert "x-header" in session.headers + + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_header_and_auth_type(self, auth, mock_get_connection): + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + login="username", + password="pass", + extra='{"x-header": 0, "auth_type": "tests.providers.http.hooks.test_http.CustomAuthBase"}', + ) + mock_get_connection.return_value = conn + + session = HttpHook().get_conn({}) + auth.assert_called_once_with("username", "pass") + assert isinstance(session.auth, CustomAuthBase) + assert "auth_type" not in session.headers + assert "x-header" in session.headers + + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_auth_type_and_no_credentials(self, auth, mock_get_connection): + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + extra='{"auth_type": "tests.providers.http.hooks.test_http.CustomAuthBase"}', + ) + mock_get_connection.return_value = conn + + HttpHook().get_conn({}) + auth.assert_called_once() @pytest.mark.parametrize("method", ["GET", "POST"]) def test_json_request(self, method, requests_mock): From 5eddbc7adff3139a022c74c4532f33ddd293f491 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 24 Nov 2023 01:55:44 +0100 Subject: [PATCH 174/286] feat: Add Connection documentation --- providers/http/src/airflow/providers/http/hooks/http.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index b22a01f8283db..eb63e27ee0925 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -51,6 +51,12 @@ class HttpHook(BaseHook): """ Interact with HTTP servers. + To configure the auth_type, in addition to the `auth_type` parameter, you can also: + * set the `auth_type` parameter in the Connection settings. + * define extra parameters used to instantiate the `auth_type` class, in the Connection settings. + + See :doc:`/connections/http` for full documentation. + :param method: the API method to be called :param http_conn_id: :ref:`http connection` that has the base API url i.e https://www.google.com/ and optional authentication credentials. Default From 62aec5baa49bd106bf6e68f31fdf9ecf61c160de Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 5 Dec 2023 23:14:43 +0100 Subject: [PATCH 175/286] feat: Make available auth_types configurable from airflow config --- providers/http/docs/configurations-ref.rst | 0 providers/http/docs/index.rst | 1 + providers/http/provider.yaml | 15 ++++++++++ .../provider_tests/http/hooks/test_http.py | 28 ++++++++++++++++++- 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 providers/http/docs/configurations-ref.rst diff --git a/providers/http/docs/configurations-ref.rst b/providers/http/docs/configurations-ref.rst new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/providers/http/docs/index.rst b/providers/http/docs/index.rst index 49745912f879b..7fc650ea8f50f 100644 --- a/providers/http/docs/index.rst +++ b/providers/http/docs/index.rst @@ -42,6 +42,7 @@ :maxdepth: 1 :caption: References + Configuration Python API <_api/airflow/providers/http/index> .. toctree:: diff --git a/providers/http/provider.yaml b/providers/http/provider.yaml index dee0796c04891..7a991044ea801 100644 --- a/providers/http/provider.yaml +++ b/providers/http/provider.yaml @@ -93,3 +93,18 @@ triggers: connection-types: - hook-class-name: airflow.providers.http.hooks.http.HttpHook connection-type: http + +config: + http: + description: "Options for Http provider." + options: + extra_auth_types: + description: | + A comma separated list of auth_type classes, which can be used to + configure Http Connections in Airflow's UI. This list restricts which + classes can be arbitrary imported, and protects from dependency + injections. + type: string + version_added: 4.8.0 + example: "requests_kerberos.HTTPKerberosAuth,any.other.custom.HTTPAuth" + default: ~ diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index 802dd0ef15c39..fe6b8f882f2b7 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -36,7 +36,7 @@ from airflow.exceptions import AirflowException from airflow.models import Connection -from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook +from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook, get_auth_types @pytest.fixture @@ -72,6 +72,9 @@ def __init__(self, username: str, password: str, endpoint: str): super().__init__(username, password) +@mock.patch.dict( + "os.environ", AIRFLOW__HTTP__EXTRA_AUTH_TYPES="tests.providers.http.hooks.test_http.CustomAuthBase" +) class TestHttpHook: """Test get, post and raise_for_status""" @@ -377,6 +380,29 @@ def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_conne assert "auth_kwargs" not in session.headers assert "x-header" in session.headers + def test_available_connection_auth_types(self): + auth_types = get_auth_types() + assert auth_types == frozenset( + { + "request.auth.HTTPBasicAuth", + "request.auth.HTTPProxyAuth", + "request.auth.HTTPDigestAuth", + "tests.providers.http.hooks.test_http.CustomAuthBase", + } + ) + + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + def test_connection_with_invalid_auth_type_get_skipped(self, mock_get_connection, caplog): + auth_type: str = "auth_type.class.not.available.for.Import" + conn = Connection( + conn_id="http_default", + conn_type="http", + extra=f'{{"auth_type": "{auth_type}"}}', + ) + mock_get_connection.return_value = conn + HttpHook().get_conn({}) + assert f"Skipping import of auth_type '{auth_type}'." in caplog.text + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") def test_connection_with_extra_header_and_auth_type(self, auth, mock_get_connection): From c047baf12705c25e1d95bb3ac662db77c9409b70 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Wed, 6 Dec 2023 22:24:25 +0100 Subject: [PATCH 176/286] feat: Add fields for auth config and header config in Http Connection form --- .../src/airflow/providers/http/hooks/http.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index eb63e27ee0925..31a0bc5cfc8a5 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -113,6 +113,30 @@ def auth_type(self): def auth_type(self, v): self._auth_type = v + @classmethod + def get_connection_form_widgets(cls) -> dict[str, Any]: + """Return connection widgets to add to connection form.""" + from flask_babel import lazy_gettext + from wtforms.fields import SelectField, TextAreaField + + auth_types_choices = frozenset({""}) | get_auth_types() + return { + "auth_type": SelectField( + lazy_gettext("Auth type"), + choices=[(clazz, clazz) for clazz in auth_types_choices] + ), + "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs")), + "extra_headers": TextAreaField(lazy_gettext("Extra Headers")) + } + + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + """Return custom field behaviour.""" + return { + "hidden_fields": ["extra"], + "relabeling": {} + } + # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: From 0ea96a82e3599b28e014322ef7b71f9f6af22a70 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Thu, 7 Dec 2023 08:48:01 +0100 Subject: [PATCH 177/286] fix: Correctly apply styling to extra fields --- .../http/src/airflow/providers/http/hooks/http.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 31a0bc5cfc8a5..538bc1e6e38cb 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -116,6 +116,7 @@ def auth_type(self, v): @classmethod def get_connection_form_widgets(cls) -> dict[str, Any]: """Return connection widgets to add to connection form.""" + from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget, Select2Widget from flask_babel import lazy_gettext from wtforms.fields import SelectField, TextAreaField @@ -123,19 +124,17 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: return { "auth_type": SelectField( lazy_gettext("Auth type"), - choices=[(clazz, clazz) for clazz in auth_types_choices] + choices=[(clazz, clazz) for clazz in auth_types_choices], + widget=Select2Widget(), ), - "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs")), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers")) + "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), + "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } @classmethod def get_ui_field_behaviour(cls) -> dict[str, Any]: """Return custom field behaviour.""" - return { - "hidden_fields": ["extra"], - "relabeling": {} - } + return {"hidden_fields": ["extra"], "relabeling": {}} # headers may be passed through directly or in the "extra" field in the connection # definition From 53c6fa8425b6341bbcefcc3823567a537befc4b7 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 19 Dec 2023 01:05:47 +0100 Subject: [PATCH 178/286] feat: Implement simplistic collapsable textarea for "extra" --- airflow/www/forms.py | 26 ++++++++++++++++++- airflow/www/static/js/connection_form.js | 4 +-- .../src/airflow/providers/http/hooks/http.py | 9 +++---- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/airflow/www/forms.py b/airflow/www/forms.py index 7028e2026e449..b69184c6590c2 100644 --- a/airflow/www/forms.py +++ b/airflow/www/forms.py @@ -33,6 +33,7 @@ from flask_appbuilder.forms import DynamicForm from flask_babel import lazy_gettext from flask_wtf import FlaskForm +from markupsafe import Markup from wtforms import widgets from wtforms.fields import Field, IntegerField, PasswordField, SelectField, StringField, TextAreaField from wtforms.validators import InputRequired, Optional @@ -176,6 +177,29 @@ def populate_obj(self, item): field.populate_obj(item, name) +class BS3AccordionTextAreaFieldWidget(BS3TextAreaFieldWidget): + + @staticmethod + def _make_collapsable_panel(field: Field, content: Markup) -> str: + collapsable_id: str = f"collapsable_{field.id}" + return f""" +
+
+

+ +

+
+ +
+ """ + + def __call__(self, field, **kwargs): + text_area = super(BS3TextAreaFieldWidget, self).__call__(field, **kwargs) + return self._make_collapsable_panel(field=field, content=text_area) + + @cache def create_connection_form_class() -> type[DynamicForm]: """ @@ -223,7 +247,7 @@ def process(self, formdata=None, obj=None, **kwargs): login = StringField(lazy_gettext("Login"), widget=BS3TextFieldWidget()) password = PasswordField(lazy_gettext("Password"), widget=BS3PasswordFieldWidget()) port = IntegerField(lazy_gettext("Port"), validators=[Optional()], widget=BS3TextFieldWidget()) - extra = TextAreaField(lazy_gettext("Extra"), widget=BS3TextAreaFieldWidget()) + extra = TextAreaField(lazy_gettext("Extra"), widget=BS3AccordionTextAreaFieldWidget()) for key, value in providers_manager.connection_form_widgets.items(): setattr(ConnectionForm, key, value.field) diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index d039fc7275462..1c97e00803174 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -83,7 +83,7 @@ function restoreFieldBehaviours() { } ); - Array.from(document.querySelectorAll(".form-control")).forEach((elem) => { + Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach((elem) => { // eslint-disable-next-line no-param-reassign elem.placeholder = ""; elem.parentElement.parentElement.classList.remove("hide"); @@ -101,7 +101,7 @@ function applyFieldBehaviours(connection) { if (Array.isArray(connection.hidden_fields)) { connection.hidden_fields.forEach((field) => { document - .getElementById(field) + .querySelector(`label[for='${field}']`) .parentElement.parentElement.classList.add("hide"); }); } diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 538bc1e6e38cb..7268c13f8764d 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -25,6 +25,8 @@ import tenacity from aiohttp import ClientResponseError from asgiref.sync import sync_to_async +from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget +from markupsafe import Markup from requests.auth import HTTPBasicAuth from requests.models import DEFAULT_REDIRECT_LIMIT from requests_toolbelt.adapters.socket_options import TCPKeepAliveAdapter @@ -128,14 +130,9 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: widget=Select2Widget(), ), "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), + "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()) } - @classmethod - def get_ui_field_behaviour(cls) -> dict[str, Any]: - """Return custom field behaviour.""" - return {"hidden_fields": ["extra"], "relabeling": {}} - # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: From a6cec2ebfddd26583b1cbdf1e104d67a0b77f02b Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Wed, 20 Dec 2023 16:21:08 +0100 Subject: [PATCH 179/286] fix: express clearly empty frozenset creation Goal is to have an empty default choice --- providers/http/src/airflow/providers/http/hooks/http.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 7268c13f8764d..0f6bea66bf624 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -122,7 +122,8 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: from flask_babel import lazy_gettext from wtforms.fields import SelectField, TextAreaField - auth_types_choices = frozenset({""}) | get_auth_types() + default_auth_type = frozenset({""}) + auth_types_choices = default_auth_type | get_auth_types() return { "auth_type": SelectField( lazy_gettext("Auth type"), From c6a9d244f6f45dcd62490561a3e3054ac18ae632 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Wed, 20 Dec 2023 16:43:29 +0100 Subject: [PATCH 180/286] feat: Refactor Accordion TextArea to use wtform utils --- airflow/www/static/js/connection_form.js | 12 +++++++----- .../http/src/airflow/providers/http/hooks/http.py | 4 +--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index 1c97e00803174..119fe39daae54 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -83,11 +83,13 @@ function restoreFieldBehaviours() { } ); - Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach((elem) => { - // eslint-disable-next-line no-param-reassign - elem.placeholder = ""; - elem.parentElement.parentElement.classList.remove("hide"); - }); + Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach( + (elem) => { + // eslint-disable-next-line no-param-reassign + elem.placeholder = ""; + elem.parentElement.parentElement.classList.remove("hide"); + } + ); } /** diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 0f6bea66bf624..77563d2eb1a81 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -25,8 +25,6 @@ import tenacity from aiohttp import ClientResponseError from asgiref.sync import sync_to_async -from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget -from markupsafe import Markup from requests.auth import HTTPBasicAuth from requests.models import DEFAULT_REDIRECT_LIMIT from requests_toolbelt.adapters.socket_options import TCPKeepAliveAdapter @@ -131,7 +129,7 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: widget=Select2Widget(), ), "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()) + "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } # headers may be passed through directly or in the "extra" field in the connection From 895c7584e1fed709b03ac725fd3d53567259c107 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Thu, 21 Dec 2023 13:04:19 +0100 Subject: [PATCH 181/286] feat: Implement 'collapse_extra' field behavior --- airflow/customized_form_field_behaviours.schema.json | 4 ++++ providers/http/src/airflow/providers/http/hooks/http.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/airflow/customized_form_field_behaviours.schema.json b/airflow/customized_form_field_behaviours.schema.json index 78791a87886c1..fa5ace958c5e8 100644 --- a/airflow/customized_form_field_behaviours.schema.json +++ b/airflow/customized_form_field_behaviours.schema.json @@ -22,6 +22,10 @@ "additionalProperties": { "type": "string" } + }, + "collapse_extra": { + "type": "boolean", + "description": "Collapse the 'Extra' field." } }, "additionalProperties": true, diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 77563d2eb1a81..2ddd9ca434efe 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -132,6 +132,10 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + return {"hidden_fields": [], "relabeling": {}, "collapse_extra": True} + # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: From a46b74dbc4fede4003f4d004beb99739e403c15f Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Sat, 30 Dec 2023 16:43:47 +0100 Subject: [PATCH 182/286] feat: Implement parameterizable behavior for collapsible field --- ...stomized_form_field_behaviours.schema.json | 19 +++++-- airflow/www/static/css/connection.css | 23 +++++++++ airflow/www/static/js/connection_form.js | 49 ++++++++++++++++--- .../www/templates/airflow/conn_create.html | 2 +- airflow/www/templates/airflow/conn_edit.html | 1 + airflow/www/webpack.config.js | 1 + .../src/airflow/providers/http/hooks/http.py | 2 +- 7 files changed, 85 insertions(+), 12 deletions(-) create mode 100644 airflow/www/static/css/connection.css diff --git a/airflow/customized_form_field_behaviours.schema.json b/airflow/customized_form_field_behaviours.schema.json index fa5ace958c5e8..8aa05945ebb01 100644 --- a/airflow/customized_form_field_behaviours.schema.json +++ b/airflow/customized_form_field_behaviours.schema.json @@ -23,9 +23,22 @@ "type": "string" } }, - "collapse_extra": { - "type": "boolean", - "description": "Collapse the 'Extra' field." + "collapsible_fields": { + "description": "List of collapsed fields for the hook, with their properties.", + "type": "object", + "patternProperties": { + "\"^.*$\"": { + "description": "Name of the field to enable collapsing.", + "type": "object", + "properties": { + "expanded": { + "description": "Set the default state of the field as expanded.", + "default": true, + "type": "boolean" + } + } + } + } } }, "additionalProperties": true, diff --git a/airflow/www/static/css/connection.css b/airflow/www/static/css/connection.css new file mode 100644 index 0000000000000..78edf0db5d4dc --- /dev/null +++ b/airflow/www/static/css/connection.css @@ -0,0 +1,23 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +.panel-invisible { + margin: 0; + border: 0; +} diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index 119fe39daae54..5c60638caf488 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -83,13 +83,28 @@ function restoreFieldBehaviours() { } ); - Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach( - (elem) => { - // eslint-disable-next-line no-param-reassign - elem.placeholder = ""; - elem.parentElement.parentElement.classList.remove("hide"); - } - ); + Array.from(document.querySelectorAll(".form-control")).forEach((elem) => { + // eslint-disable-next-line no-param-reassign + elem.placeholder = ""; + elem.parentElement.parentElement.classList.remove("hide"); + }); + + Array.from(document.querySelectorAll(".form-panel")).forEach((elem) => { + elem.parentElement.parentElement.classList.remove("hide"); + + elem.classList.add("panel-invisible"); + const panelHeader = elem.children[0]; + panelHeader.classList.add("hidden"); + panelHeader.firstElementChild.firstElementChild.setAttribute( + "aria-expanded", + "true" + ); + + const collapsible = elem.children[1]; + collapsible.setAttribute("aria-expanded", "true"); + collapsible.classList.add("in"); + collapsible.style.height = null; + }); } /** @@ -122,6 +137,26 @@ function applyFieldBehaviours(connection) { document.getElementById(field).placeholder = placeholder; }); } + + if (connection.collapsible_fields) { + Object.entries(connection.collapsible_fields).forEach((entry) => { + const [field, properties] = entry; + + const collapsibleController = document.getElementById( + `control_collapsible_${field}` + ); + const panelHeader = collapsibleController.parentElement.parentElement; + panelHeader.classList.remove("hidden"); + panelHeader.parentElement.classList.remove("panel-invisible"); + + if (properties.expanded === false) { + const collapsible = document.getElementById(`collapsible_${field}`); + collapsible.classList.remove("in"); + collapsible.setAttribute("aria-expanded", "false"); + collapsibleController.setAttribute("aria-expanded", "false"); + } + }); + } } } diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index ac92b967f7e34..307450b05d16b 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -25,7 +25,7 @@ - + {# required for codemirror #} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 653b0dd3ce07f..11ebd6c4cb436 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -25,6 +25,7 @@ + {# required for codemirror #} diff --git a/airflow/www/webpack.config.js b/airflow/www/webpack.config.js index 9d5800f783f50..ad1a7098e0803 100644 --- a/airflow/www/webpack.config.js +++ b/airflow/www/webpack.config.js @@ -60,6 +60,7 @@ const config = { airflowDefaultTheme: `${CSS_DIR}/bootstrap-theme.css`, connectionForm: `${JS_DIR}/connection_form.js`, chart: [`${CSS_DIR}/chart.css`], + connection: [`${CSS_DIR}/connection.css`], dag: `${JS_DIR}/dag.js`, dagDependencies: `${JS_DIR}/dag_dependencies.js`, dags: [`${CSS_DIR}/dags.css`, `${JS_DIR}/dags.js`], diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 2ddd9ca434efe..192e57c1d70fc 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -134,7 +134,7 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: @classmethod def get_ui_field_behaviour(cls) -> dict[str, Any]: - return {"hidden_fields": [], "relabeling": {}, "collapse_extra": True} + return {"hidden_fields": [], "relabeling": {}, "collapsible_fields": {"extra": {"expanded": False}}} # headers may be passed through directly or in the "extra" field in the connection # definition From fe05f442f5d602177b5e972bd2306c3cf43d9d5a Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 5 Jan 2024 06:48:16 +0100 Subject: [PATCH 183/286] revert: Remove collapsible field --- ...stomized_form_field_behaviours.schema.json | 17 -------- airflow/www/static/css/connection.css | 23 ----------- airflow/www/static/js/connection_form.js | 39 +------------------ .../www/templates/airflow/conn_create.html | 1 - airflow/www/templates/airflow/conn_edit.html | 1 - airflow/www/webpack.config.js | 1 - .../src/airflow/providers/http/hooks/http.py | 4 -- 7 files changed, 1 insertion(+), 85 deletions(-) delete mode 100644 airflow/www/static/css/connection.css diff --git a/airflow/customized_form_field_behaviours.schema.json b/airflow/customized_form_field_behaviours.schema.json index 8aa05945ebb01..78791a87886c1 100644 --- a/airflow/customized_form_field_behaviours.schema.json +++ b/airflow/customized_form_field_behaviours.schema.json @@ -22,23 +22,6 @@ "additionalProperties": { "type": "string" } - }, - "collapsible_fields": { - "description": "List of collapsed fields for the hook, with their properties.", - "type": "object", - "patternProperties": { - "\"^.*$\"": { - "description": "Name of the field to enable collapsing.", - "type": "object", - "properties": { - "expanded": { - "description": "Set the default state of the field as expanded.", - "default": true, - "type": "boolean" - } - } - } - } } }, "additionalProperties": true, diff --git a/airflow/www/static/css/connection.css b/airflow/www/static/css/connection.css deleted file mode 100644 index 78edf0db5d4dc..0000000000000 --- a/airflow/www/static/css/connection.css +++ /dev/null @@ -1,23 +0,0 @@ -/*! - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -.panel-invisible { - margin: 0; - border: 0; -} diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index 5c60638caf488..d039fc7275462 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -88,23 +88,6 @@ function restoreFieldBehaviours() { elem.placeholder = ""; elem.parentElement.parentElement.classList.remove("hide"); }); - - Array.from(document.querySelectorAll(".form-panel")).forEach((elem) => { - elem.parentElement.parentElement.classList.remove("hide"); - - elem.classList.add("panel-invisible"); - const panelHeader = elem.children[0]; - panelHeader.classList.add("hidden"); - panelHeader.firstElementChild.firstElementChild.setAttribute( - "aria-expanded", - "true" - ); - - const collapsible = elem.children[1]; - collapsible.setAttribute("aria-expanded", "true"); - collapsible.classList.add("in"); - collapsible.style.height = null; - }); } /** @@ -118,7 +101,7 @@ function applyFieldBehaviours(connection) { if (Array.isArray(connection.hidden_fields)) { connection.hidden_fields.forEach((field) => { document - .querySelector(`label[for='${field}']`) + .getElementById(field) .parentElement.parentElement.classList.add("hide"); }); } @@ -137,26 +120,6 @@ function applyFieldBehaviours(connection) { document.getElementById(field).placeholder = placeholder; }); } - - if (connection.collapsible_fields) { - Object.entries(connection.collapsible_fields).forEach((entry) => { - const [field, properties] = entry; - - const collapsibleController = document.getElementById( - `control_collapsible_${field}` - ); - const panelHeader = collapsibleController.parentElement.parentElement; - panelHeader.classList.remove("hidden"); - panelHeader.parentElement.classList.remove("panel-invisible"); - - if (properties.expanded === false) { - const collapsible = document.getElementById(`collapsible_${field}`); - collapsible.classList.remove("in"); - collapsible.setAttribute("aria-expanded", "false"); - collapsibleController.setAttribute("aria-expanded", "false"); - } - }); - } } } diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index 307450b05d16b..fb3e188949b66 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -25,7 +25,6 @@ - {# required for codemirror #} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 11ebd6c4cb436..653b0dd3ce07f 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -25,7 +25,6 @@ - {# required for codemirror #} diff --git a/airflow/www/webpack.config.js b/airflow/www/webpack.config.js index ad1a7098e0803..9d5800f783f50 100644 --- a/airflow/www/webpack.config.js +++ b/airflow/www/webpack.config.js @@ -60,7 +60,6 @@ const config = { airflowDefaultTheme: `${CSS_DIR}/bootstrap-theme.css`, connectionForm: `${JS_DIR}/connection_form.js`, chart: [`${CSS_DIR}/chart.css`], - connection: [`${CSS_DIR}/connection.css`], dag: `${JS_DIR}/dag.js`, dagDependencies: `${JS_DIR}/dag_dependencies.js`, dags: [`${CSS_DIR}/dags.css`, `${JS_DIR}/dags.js`], diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 192e57c1d70fc..77563d2eb1a81 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -132,10 +132,6 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } - @classmethod - def get_ui_field_behaviour(cls) -> dict[str, Any]: - return {"hidden_fields": [], "relabeling": {}, "collapsible_fields": {"extra": {"expanded": False}}} - # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: From 8369687345d015fa38cae2f69d251eceb4cff900 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 5 Jan 2024 08:47:09 +0100 Subject: [PATCH 184/286] fix: set the default value for "auth_type" as empty string SelectField expects a string as value. The default of select choice cannot be None. --- providers/http/src/airflow/providers/http/hooks/http.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 77563d2eb1a81..9cf5d4983dbc5 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -120,13 +120,14 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: from flask_babel import lazy_gettext from wtforms.fields import SelectField, TextAreaField - default_auth_type = frozenset({""}) - auth_types_choices = default_auth_type | get_auth_types() + default_auth_type: str = "" + auth_types_choices = frozenset({default_auth_type}) | get_auth_types() return { "auth_type": SelectField( lazy_gettext("Auth type"), choices=[(clazz, clazz) for clazz in auth_types_choices], widget=Select2Widget(), + default=default_auth_type ), "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), From 181600162b210c78f763f7d9cfe2fc6e9edeb11a Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 5 Jan 2024 08:48:11 +0100 Subject: [PATCH 185/286] fix: Use Livy hook to test invalid extra removal --- tests/www/views/test_views_connection.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/www/views/test_views_connection.py b/tests/www/views/test_views_connection.py index f9a4efd11c15b..1e21dc4856ed1 100644 --- a/tests/www/views/test_views_connection.py +++ b/tests/www/views/test_views_connection.py @@ -459,8 +459,12 @@ def test_process_form_invalid_extra_removed(admin_client): """ Test that when an invalid json `extra` is passed in the form, it is removed and _not_ saved over the existing extras. + + Note: This can only be tested with a Hook which does not have any custom fields (otherwise + the custom fields override the extra data when editing a Connection). Thus, this is currently + tested with livy. """ - conn_details = {"conn_id": "test_conn", "conn_type": "http"} + conn_details = {"conn_id": "test_conn", "conn_type": "livy"} conn = Connection(**conn_details, extra='{"foo": "bar"}') conn.id = 1 From 836f338140ef092c034fa14306b7240217d581af Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Mon, 8 Jan 2024 16:40:49 +0100 Subject: [PATCH 186/286] feat: Implement CodeMirrorField for providers --- airflow/config_templates/default_webserver_config.py | 7 +++++++ airflow/utils/json.py | 10 ++++++++++ airflow/www/app.py | 3 +++ airflow/www/templates/airflow/conn_create.html | 1 + airflow/www/templates/airflow/conn_edit.html | 1 + providers/http/provider.yaml | 3 +-- setup.cfg | 0 7 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 setup.cfg diff --git a/airflow/config_templates/default_webserver_config.py b/airflow/config_templates/default_webserver_config.py index 4ad8ee6743f39..85b9d4d2c8dbb 100644 --- a/airflow/config_templates/default_webserver_config.py +++ b/airflow/config_templates/default_webserver_config.py @@ -35,6 +35,13 @@ WTF_CSRF_ENABLED = True WTF_CSRF_TIME_LIMIT = None +# Flask CodeMirror config +CODEMIRROR_LANGUAGES = ["javascript"] +# CODEMIRROR_THEME = '3024-day' +# CODEMIRROR_ADDONS = ( +# ('ADDON_DIR','ADDON_NAME'), +# ) + # ---------------------------------------------------- # AUTHENTICATION CONFIG (specific to FAB auth manager) # ---------------------------------------------------- diff --git a/airflow/utils/json.py b/airflow/utils/json.py index a8846282899f3..9622f4c0a6b30 100644 --- a/airflow/utils/json.py +++ b/airflow/utils/json.py @@ -123,5 +123,15 @@ def orm_object_hook(dct: dict) -> object: return deserialize(dct, False) +def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> dict | None: + """Safely loads JSON. + + Returns None by default if the given object is None. + """ + if obj is not None: + return json.loads(obj) + return default + + # backwards compatibility AirflowJsonEncoder = WebEncoder diff --git a/airflow/www/app.py b/airflow/www/app.py index 2656045e84ba5..c85d82793f678 100644 --- a/airflow/www/app.py +++ b/airflow/www/app.py @@ -22,6 +22,7 @@ from flask import Flask from flask_appbuilder import SQLA +from flask_codemirror import CodeMirror from flask_wtf.csrf import CSRFProtect from markupsafe import Markup from sqlalchemy.engine.url import make_url @@ -127,6 +128,8 @@ def create_app(config=None, testing=False): csrf.init_app(flask_app) + CodeMirror(flask_app) + init_wsgi_middleware(flask_app) db = SQLA() diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index fb3e188949b66..8e3d8db0d5e00 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -26,6 +26,7 @@ {# required for codemirror #} +{{ codemirror.include_codemirror() }} {% endblock %} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 653b0dd3ce07f..174bfa164c4c4 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -26,6 +26,7 @@ {# required for codemirror #} +{{ codemirror.include_codemirror() }} {% endblock %} diff --git a/providers/http/provider.yaml b/providers/http/provider.yaml index 7a991044ea801..bb0214e7c6e66 100644 --- a/providers/http/provider.yaml +++ b/providers/http/provider.yaml @@ -102,8 +102,7 @@ config: description: | A comma separated list of auth_type classes, which can be used to configure Http Connections in Airflow's UI. This list restricts which - classes can be arbitrary imported, and protects from dependency - injections. + classes can be arbitrary imported to prevent dependency injections. type: string version_added: 4.8.0 example: "requests_kerberos.HTTPKerberosAuth,any.other.custom.HTTPAuth" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000000..e69de29bb2d1d From eeea37c97f5448176ce5ae3b04638e97269d1560 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Mon, 8 Jan 2024 19:21:34 +0100 Subject: [PATCH 187/286] revert: Remove CodeMirror from providers --- airflow/config_templates/default_webserver_config.py | 7 ------- airflow/www/app.py | 3 --- airflow/www/templates/airflow/conn_create.html | 1 - airflow/www/templates/airflow/conn_edit.html | 1 - 4 files changed, 12 deletions(-) diff --git a/airflow/config_templates/default_webserver_config.py b/airflow/config_templates/default_webserver_config.py index 85b9d4d2c8dbb..4ad8ee6743f39 100644 --- a/airflow/config_templates/default_webserver_config.py +++ b/airflow/config_templates/default_webserver_config.py @@ -35,13 +35,6 @@ WTF_CSRF_ENABLED = True WTF_CSRF_TIME_LIMIT = None -# Flask CodeMirror config -CODEMIRROR_LANGUAGES = ["javascript"] -# CODEMIRROR_THEME = '3024-day' -# CODEMIRROR_ADDONS = ( -# ('ADDON_DIR','ADDON_NAME'), -# ) - # ---------------------------------------------------- # AUTHENTICATION CONFIG (specific to FAB auth manager) # ---------------------------------------------------- diff --git a/airflow/www/app.py b/airflow/www/app.py index c85d82793f678..2656045e84ba5 100644 --- a/airflow/www/app.py +++ b/airflow/www/app.py @@ -22,7 +22,6 @@ from flask import Flask from flask_appbuilder import SQLA -from flask_codemirror import CodeMirror from flask_wtf.csrf import CSRFProtect from markupsafe import Markup from sqlalchemy.engine.url import make_url @@ -128,8 +127,6 @@ def create_app(config=None, testing=False): csrf.init_app(flask_app) - CodeMirror(flask_app) - init_wsgi_middleware(flask_app) db = SQLA() diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index 8e3d8db0d5e00..fb3e188949b66 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -26,7 +26,6 @@ {# required for codemirror #} -{{ codemirror.include_codemirror() }} {% endblock %} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 174bfa164c4c4..653b0dd3ce07f 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -26,7 +26,6 @@ {# required for codemirror #} -{{ codemirror.include_codemirror() }} {% endblock %} From 34743895779c5c24e68e37a22488844b53091ae7 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Mon, 8 Jan 2024 20:47:50 +0100 Subject: [PATCH 188/286] feat: Add documentation --- airflow/utils/json.py | 2 +- .../img/connection_auth_kwargs.png | Bin 0 -> 9623 bytes .../img/connection_auth_type.png | Bin 0 -> 14199 bytes .../img/connection_headers.png | Bin 0 -> 5256 bytes .../img/connection_username_password.png | Bin 0 -> 4761 bytes 5 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 docs/apache-airflow-providers-http/img/connection_auth_kwargs.png create mode 100644 docs/apache-airflow-providers-http/img/connection_auth_type.png create mode 100644 docs/apache-airflow-providers-http/img/connection_headers.png create mode 100644 docs/apache-airflow-providers-http/img/connection_username_password.png diff --git a/airflow/utils/json.py b/airflow/utils/json.py index 9622f4c0a6b30..eb3cd40941197 100644 --- a/airflow/utils/json.py +++ b/airflow/utils/json.py @@ -123,7 +123,7 @@ def orm_object_hook(dct: dict) -> object: return deserialize(dct, False) -def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> dict | None: +def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> Any: """Safely loads JSON. Returns None by default if the given object is None. diff --git a/docs/apache-airflow-providers-http/img/connection_auth_kwargs.png b/docs/apache-airflow-providers-http/img/connection_auth_kwargs.png new file mode 100644 index 0000000000000000000000000000000000000000..7023c3a7a072f965f9dd053f77c5a5d64b396f47 GIT binary patch literal 9623 zcmcI~2{_c>+jom(n~(ixI5hS2r&k-xWWM^15qC|U8&@yI zp4VbIoNz&&;S|%$vllM#zw{D3bLeCYFTFO4DogYQZ3Q#Q(MB#`@|PP{w$;0sjC|gQ zwZ00``u#3Nh5MrM%WwU6MOC!GEa0bjzF~#+9_k$RUt@1t_`%yIg(IZq`(>tL>=B>- z{$n_3=<@gXa@sCopHqhy8ve(KHqs61*aGb{L`~`|6OlkDb>1ArkdEJf(u%A*3sMN|@KU37UWK z`S!;+d!t~o=`Y)wx6bxn)?k%F?VSe!d^L0ZZ$r)AA%p$*$)o=^r})M@ajS#xngony z*Lg06eEks5LtY{{Vqn>ksZNFGX*0{b@+ z#0=Qog@Hj;9>jhwU?syRst>MH=7w=oH_sMKnFK}Mwee}-tenP+6yF*CgOf6#l9m~nV&Lm0k^g3ly*;YJZL)1?XSHiuScftr z#fh-AL>l<*5_9^a$4@YkHx|3Pz$brZ?SLRnt1aptkF^SCGF4So9T@qEx}W?S9Hhje zjy(hdWv_M_&Bzq|=dy@X#CzWgO-|?oQ*sLLA1p%)S2ocGlYcv~*k1^Hr`qWbrW`D1 ze39xdiiP$)c~TF*B%`J~(|0HvDL6}UXmC$AX=?Uck=2jS&$IIKgwo{Qg;dPIWDXvG zMpEM11PV^VaExN#`GO17fs{SQ=DebuB3m@@iB@cQBh_QnXofNT+W;eRTnE;5qA}H_ z{zc{+2E+#+k7QZLTNt<2>=yczPV0#hS6B%9kd3{-YcCyL!Ul_+k; zp0DzSG_^E_A8QO|+-h}b+8Fg8y{FmTOUkQv9CZ=kv~^}^Lc*}YvMAAFoY;}%=|t1SKU^tU@;Fdz77c@R*1UA%Eu_Y3tz$}<))Yv z-3pd<>}&eTERFRZ4--c09NoKpl|RsR_t%vKf8p&x_J~HJ$(xT zo$oS|$Z5$+e80CnXSF^*nB_j*ndQH~H}>(#(cCVJdOvxk)i2Uz0R!(`MoYbx$5G&7 zbo(K0GjYpF_4f8Q`5Qg(SozU`tNz$D_S>4h8^zdEijt6gb!toDK<=$u;A)fy!Y`h(--=f&9P7^^!9I4 zS6lByogoK?=((~MUPch>kn9^#u;HhK@Ta@&)yh^MbQOJm+xd0L-}4_>pNLW6Rf${5 zn7B7A{qPGNpZ|_4r;-v{iFBo2Xh>#+3G;|{E(gSV34$r{E%Y!9`-0-)9E>o*$tJE)bX!7Ho6BjelTH=PL$J?c1 ztcS}V=!m@Z84KnZ_Y8M`u0~g~R_#1AF0&rzJU`m7zjtE99k_*)p*%{1*Zcfkv+5{6 zR8O`>jGTCSrfnv=TK~22s0ZyRx?UYb5c6=Y2V+m39wWHzVWSoL=?cN?3qvg)87ltu zzh9h(5?Ru&?C-2D*OsZIjn3QJB{cg>n)^q~LN{gBOxrEsTCs>YYEnTcVzXZ93bGh0 zzrFi`%hISd+fZlbYVcBS(%oG>bOD1C|O-DZ0gNyeZ7| z;uZHP-86$>`0iguVGr-XX5S>WNe=s()OWpcM50@!i6`t8{wEP90XuQUY@7_G<` zMX#lM_TFlzVc))-lM#s5Zg<<^h7V_c#6Zqm$~pbCRB0|7%Gw{i@E|bt-6%$}x^(MT z-Qu_`f0Gqj1Z!{SyJoQ}*|2k|Qd9r z>FA|&f!~@IC8ok-O}>jCIHvPUqjzf_FCfwq>*MTI=x)nt{8ZFBRE`|3^>UW9s6CB; z+3O(2M`ZC5LsVYC9G_8Wt~q;JO}+n&`_z}P(KFTfvQsHJbB!C-1>(qCCFW<61a)mJ zfOn$ExQ(<4t2>BmWijT^^%ryB)2zikhe00HJII3zq&#Kg*d$5IyFLcNnc?t;1aLJx zoJ_6;UlJf$ak1T*Vm)Q#Qln0~QLgivXwu@@_BCrF#V znBPwJa3cfanoY|cE6bmaJ@LQff|QJNDw<>XEWazOvKK!+1>2;7y$rtLFc@1}@5Cq^ z5Oh90X3=-{nICF79SBuUIwF7??)EGxo70_>P)j;b9!5H#)Hl_1 zveiRRBuu#?H!9SwzaroBq4NfU|RlwHPbf$H1%Avp5&z!DSXF0SkUDbIQ zN5eK3l*mwE#zO9}ZBKBzYRHFZBD9^0n9#M-o()i^yOx{safWYyQzNa#c6BHxXF}Ja z=TzVIm+=V@q@7h6h!}-c`>^rC(p-m*cP7Kja zXnQ)RsZ+5HuBV_@p$s+68ylhGc+%wfh=gMoM5{ix-n#blmFo4>aIpk}F%GGZQlEpa z3wlgv9Xm2)uZ6;2Eq1EQT8`gz3s5uOC*K-(z)}<-x^bclCkeB(~dQ^KZsXG93)}Y@3u(u2tR+;ef__lr)x~c zs@fSnBczc+RPD}83uSqz>jXlZFN{@i>ojr-JW{l0sO6iiBS_stR=1xb>wi>xSK!1U zL2LdH!5&D=3-{dfOHz~^B!x^?4C$#}Wk7NbjRY>)20)`_Rah=>~yAj`Jqb1^m7l^l&k_AB-56!+#;7VCgf_!uFmkuic!7h?*gZt{)_2! z_c`Me!sS*?m7~g@XM9#~KLmRnseyl4WPAbV2*VohK#Q(PPtg}6O__^XUmF|M=ox4_ z=%bFpNYb{Qkrt(|FI1%aC8ceb?mr5CW!NDnU+Hzvvb;Ewb1u|6KlmS)o9_9`>qjTe z>2Sb~h^;ekty=tgZxaZOj%!mK5;{L2XR53y=c+y=nxBOWYq!@&;XMU942~iy{FPe6 zyH-AoE3SQaeEVrK6-tzPE!`imsL7?9x|XY91sBHJ&nU2JU$VP2Oh>`kp37-@+oHC zk8(4QJ%7dN;5ZA?yJa_tui}60sE-N^DUojZym`q_c#yXxfwn!* zgmwDa&!7ie?kNx?9j05>#_w!&(xPXh0=JQ?~35`m$S9)v?zSr?L@Jvwk6s^tZy$#4l(*9EbNxA;-XvBZZ zJM4>a9U6S-(>`mn37JGEW``@B@@_U>)9IcoaYH#25(MHuq%}@Bv>r`vc9*gF{Hzo_ ztrjf-1^GrHd~Esv&v8iJWujnQmEGS)rsXzTrWCmTT&M=K>$5pq^mZjKbwRcG_#i0{ zfE=s+z1^`uX62_(G^}}hy}enmDimbd2;eNfz$pLz)4-zJZ$tX;`ilQv992nE^em~| zsdDTeuea4{iRPJbZCL9w%c`!f9+@nk{Lvbh>j46?{%Ox(%zI{)9jQk@y9HHhY_?V= zmn%Y<`8vD8TfSzghk8XOtN3F~Yup}S-6pknm{tX~42Nu}xxeoOa&sZu%@1&;QipGa zr4Kt&cFsmQbnH4uJE2wwtYbbANF-d~*m4VBm-tpDZ^(?k zHMx6NMFkVJ4EgcPr!Y6`!COlszKg+~>58EXL+|?UM0ZK6SRzO3EfB(V#u>i9YB&iy z+jDO{rep=N-anpeGyg?7 z=xZ?-BKBfEud<()7eI@R0KDQUtRpZ6fVyUH}^g+ zA&J<5vq7al<_9Y_;67axeB6J>$cR-vbfP7u<>%lUw;YO}iBpPyexU4}#1II8f$xD* z$AX~q(iWkn%&HyJnf`ab-?)rD1Hk;|O1mWf`8kco9Id$V%C`-B3crMLH}0aRAIuQ4 ztolnVawp}4`j*=z8=5wP&H~yPNyaw}mLODimeiA#(LPLa1Ji&3C92VPegiJXgD9V{ zZP=Phl}l%rr8$E|jj4If7UYj_jSVQ1=PL-sX79Xsg>;iy_O`nGnp>UhN<&x(wq9ICsLozy*^cqcG&t(C$ zw(3eS6H>X((`ZL!e|Nv}%^m60idosmUy?UptWNLS35jbtq*4&9VH)fg4QGKFLxO$d zv0A1l;Rzf&M80I35<5}#UW z07qEE3+8>ihCzIrKDSjVSYl0kzd0V^s(hHis&i{~dP3QGuw3x>{GfETq3UI<0E++S zeScG4ZpkOQHS9cV=}930(7W#D6lmPP)D*^=r4hqRDFwI=mRYxi05;jOOES&ZOT9Jw z_}Uz2Vy+JR&;;ke@mjCl&50P-T^hT^dNMMt(_5HxoZA{70@?o>n`h3iK9?27D%{); z5l?FI{^1wEe#!dRu&)gzBq=$wj; z)iaW6`y2jzOc_i2*3r-dWV2@DpJ^Ya>d0TpbL3eNfRzC+LN9{H6b>TpMTfMz6PM$tC8tK0pe9(V zWZi{A+S51rlxIFrf+EDM06iHSw9ORnz`wauasS>?#Fa9pwquuw*@W?@sHY^sI_~}i zf}oipHjV&;k>wHRa<4Mz(Ie@Ke|7o!o9O;>34 zkKrMUHE3o}eR(nbFfi5OwW+4zR|;5y&C(Aij$p_OhEj1n*RbZOILHY*3;eWdQLLMF zIzIqXr-UP15p;Z4=lG5sJ|a+S{f`0U?8DygI7ok#>peObJ9_ox*Z{!*NTTR%I!8Jr zoohK=y(ypFaQtG1%>AaP-S--3_=Bu3aC~Or`dS>`r}LBijlc;X{AF#f5s)&^kDF(H zq63J2NT1$3#}L2nE4Gt|6DWm>PsYonhr1tA&mW0{Uv>`Pmp~YAh%l~BcSQ93e3!w* zCC9HnA;P8}!W8}1Xs~`a-4G?cF#=hbf{)M**Lgs$_&#EGnM7t(tl&Btk1>w>D)$sS zCJ`1#^!{;avbEK9q&!z=!WWsM;4#l~BO1HkuY-Inte?z+fBkC%xn#`vMJ>rS$9dpF}PO3F$M%_W;tTB|<`*t{}BY>T@%ppMeYVE~+I1h-1NxRg>A zg5`xDhwt((K#!h4BD{vHbfy*bUP2j>bZVPSi=GQU@XHJOIEX;Y=jZ3f73fqi3~Hbs zJ_v*!iFJDV`}gmk%UAppegu^@#0vx-|Fl)`jWBCcG+xhI$Wk&-tT`ukxAL_Nv&IR& z=tc7SPo5i0e7w`=s<^n|k9D0IGmaEgMs+O?Cok`{w#JRSN?4TVoIa!#J6o7X3c+NtLyj1m2J54Ku8NfltG`MtV znaa}wplZ*Z`o;I;)LyE!LlFt3=GB6X*25^3zua(;o(%7)7n>=Bz{7r9$kLJ*(zeYv zBYZg!s$ZR9A{?Jf#XI*wHYbwYy#1mR6_{AkouA{cF!s>rP(b)1q0|aLZrx|JApsah zY*OcV^+l(Pow|*r2WWdhp~nJYaEk%? zc(l&jd1Ip?ShzVxwAiKnL#p-ESB9yku(uPAh#?l5GUlDhSoN322%dY{mTZ{FZwcWRVk?*D1 zZD}EU3|?6}w%<>4@ooLK!~Clz7o(bcdGa_&gXAOOvnbNLKy5{fSDQSY$Khg08TCz9 zTi*ocTRh2E|2i8o%C?8av8LM5$JR2-vJ+)wxk^`8sE#>?=i+bHRzc42X}8u~ar43Eg8c3QY!gT3r0AERgA;z=Q0 z`NE*^n^T?j3fHmX3l|MKNnS^Dj`rBR*cdFc-_u1&qqi%&ZSQ(-P0B44ziabM(grE^ z!DiAeNi7D)na=C|qP5a!^1?e;uEXu+4>r`fk!P1@i=gO%o>vY;(}53Tu0M zToR*oBGz2u(Wj2Qa6;iNrxf}hh4;)P%5IHBU50lLPJNHfAr}G<^paXNj6c(#@6N7n zr$*>U*E|5%iz|Qmq418iOeb%$aEsr4k3D_16oWvrY!ecm-n0Gs>Ux>ZsfPBbV{pio z6YrISEJVD7dpGrPdg@skj-Ny1 z{7ovWvmTr*cA`t6Gw+`G`19@!xp>6y(>&spb^@U8{XIJT7(A5OpDQqQZ@OAYZm4KD3SsW($Znu zGtq6Y-#O7U$fsp!K@0bn!q&v7G!o7}XbhrvEw!%YbVb|I$Dz-Ae*L;Fsu=R@yjCHG zjS-pQi$pRBalV#}D6>jdUMnE)ezRxq^ldz&#(}6KbPGa>OuK0Q(FD=kO|CR;!RJ7N z!`7U$%OdyuG|C^}iGu#}&v5%a^IUEr{oSa9p3d-xYUg3D(U>#4UQC&EcLzKXp9#f< z#Cof#$TO}A;vNIHX$3oJ;n2(PPmN07yQG%}|FGx76v_FV!l_0)4-Y9 zbZ(yy2$hl0RkKsOEr@u@@})EoC@9erb-#b^YBISA>7}u#NA_lB@Mr5xrwnWd;ey+# z20lZ)kK|L4s8<2ycE2u6F4Fv`y26p~))|+6R=8Dr^*- z0)Ui`ajGj~xM}9rbq6JwpbIDxr z1LgG^!M34#v@h0uTAwQN9FW{JDl zwK;rq6m-JzND9E@Az(+%w_tn#IHZ%NLisRsJ^2RA7PXJ)!SoQFBne2v-nL^}5;Kwu z!LImtp6)0u6F^R$F#=ye1-}vOcS0%PJO>vawYnOtUl{A8dato{)<1Qb^%U#?-d(A^kiECYHjx0rcd9SJ7b{! z96pARQQ6}ZGRQC&#%4{n0d>qeBAz0K0tqb`SkvsiMas*105&;C05l&37UD#`U;plS zkz`u$=bdK2-wlW$YHw$20A>eHGz(~1xg|P4*m&JD{ty)8tw6bcSE_ zMU`0^_8k2*IdeY&@s&mUpaz!pDjj4V33WB}NdWpqm&*MT1praXlYM8j_^#2jHgpcjBpYOx2IWu@IqX04SL z9wn~mzqeyrKGm6S#(-oOGi5RRkt1Oq8HKNHKL)pdIVnRegA))!qI|@hXA!GQQdB1u zQ_Dp9vNZ6>>v*g*5QGXQnQ$E-e8(M|y;?CL8sP^!$YPvmu%YS}M1nI68ZIT}ipD_z z<6&R_HEiY+_&?mxSyv_W)t<$uuEafyjWcL~qm^d{zNFfORVpsdZ&4t2i|UnX^n zc@C8S32ILd(FYlJDXr?Jlkx;9RvFVAoLVAjYGtzh;(LJ{k}GNccJ<^JDh$}n6Fohq zwawFU91anWxNV7+{r$5?SXnQ;pGHwh8A>DMlzA9KA2ZWjW%u5XuA=^%L8%+hdNA+KUxJ+NQ^~; zTn9UHP(jy0^`zM4#$I$mj(&?!TSuq#wsw~CIzANk)`;z1NF&A3EomfU0tp(xzd z2ZcTVDiW>l`g~X*OFz&oTQJg7kzk+Sm8sM2upK1z0}1W_Bs2f_)jcE%Uk(~1PJpBb z;gB%@-1RSw06?<831r+ngG}CA#fXIes>*|VZUs2-Rx8-B;om}jF0P}m*bEr=Onkuk8g0E@GN8TH(>+$dxn|e`V?;qmYie-%1%QuKYo} zTHWJ?Mv(qiDu63;ZrM_cbD)O-WOK$3K5Gm~5*BR-9R~*&jIxmP5IZpcKlL$yg*J^v zz)AeA@PpaC=ZfDQxY6Vw(ZrJSs;e)5$+)5@cBlOOfi0X=TV&u^t3gWDPBoA=7(7ER zQ(FfL|KmeSgG(WWbN`Rs8w9{eDN`;Vv=|hht9ZJrw?>o5+so|!QQ4NeB${+x0%*88q;*~Y KT8Y}dfd2)j<)#Y& literal 0 HcmV?d00001 diff --git a/docs/apache-airflow-providers-http/img/connection_auth_type.png b/docs/apache-airflow-providers-http/img/connection_auth_type.png new file mode 100644 index 0000000000000000000000000000000000000000..52eb584e5ccf6463273c9b0d35171944d451af9e GIT binary patch literal 14199 zcmch8XIPWlwl2%HEh~tXqEt~qIwDQF282i_fCvGiAiYU%0u~|)Qlt}_l+cUx63P;! z6RNa8U?H85AP_(Zf%_%6_Bnf>bI)_{k9+^{0QtT-#~gFaG2eHLH}7?|)fhmmAUZla zhKCRpC>`BD5Ww%PXHEe>*-vEj)6q#QK2*7H;A2j}h4>qeWG!xxl`vB)-qyt4sieB$ zK!I^4{_6to9tVb=yr>fRj*oBX{%=hGK(XWx`L7FCMn$QE{n{j#%tES+jZ;z>wWBLy z)|Y&zJ5tO8c4S2zN-5sF;b`ymw>G~46VlOrV)=>wah3~pf%ezEbrt@9(0&^@tq1yz z&SF6ae1<#t0(k%VoqP=R-39QOm9E&Dc(-5w)ot1MK?$B)ipa)5&(OV0dve4x-+#J} zB|8atk~=s`G4{FqT4dbO+0(-W>d2h}*iREaZ1QOCh6GLh?;mc517iaZl3If2%-h1l zD4Cs1${fNWe1K1kTjTDRT-en3)V}zPm6+SKa(|U(%9$0!Mn{7)`&xTj;C)6W<=vi) zolM3Pbk9y2ote1PDHv>eF+1GCX)ZlyF|dG*IAL&+X6#VdTzezy?+)@swvKC{%slpb zrFbGljB3@g>)}~$r`{=%rzK~HPU*6J?cVp#x&9IDxvS}W-Ky%BiHEjpg<_xETUakU zI2MVy8Jw##>yf={`3_k4z}$f`*x4&8r*jp4zw*m)yAE56ToooKJ77@F?n<-w}U^m}xbX2m=)o+xvHvuLI6^WI#x3k`GC@K=GXzz0#)m5{mr@G%S7JkMK z>+txD-5Vq1_)jLoThg( z7wLwInsm|}hEu?K*|K}0JT(H*Y+nn#Sut@#s_yyOgQh~xw^=U9tmjg<8Q8ol9c+{K zMTKIe|7PSGi!!T5&3L#F`-{`mjZ)wp5#lDW5Lm!9g2}xZQ?lZ|-buazT4lagh8L;9 zbA~%W>ey@L?cIT5*=3dO-w3I~G}nveTmNDAMV`P_yz+T}?7Y)jZC2)BwUIoqYz1cL4D z?KR!qE9$)0IsdjnOH0cO*t*Sx_hsc?Jj2JM5F9he4m59bqhMIouVF&mt~f7Xu~Qm>K>&E8%#QO46*8mprfkZQUl<|REZ^47lDpg(M^^u zf6Y8J95}o`MmtpJZ9f#1mACh@$~l}ZE~{+*%L9f7(RNqG0X2ZD_s{uxSYj#L@>U|KplpF}}~g!L$O zDK#Zg?W3_^x|c-@>fGM*Vn(32k!f*+Vb^xB2&1$v5KeS-{I$?!3ya>OvWoVBr~KbW zFLu&4utgN%;qecr+5%z%_#`j{9o;>X62_s#2@$E(E6NT+#bS~`g1AhA)8;^&kVz$jkGX}Ji=?z^CT&{zT%^0!USH2>Qtg;DQEn9`?751^YuVZ5Pkes*W^W+@ zekpPiYrXMGGY$*{Om(ciqvKi8a*})`yUe9Dr|zB}!W`Z=SDCu@0U$>CB~~#XtG`~o z>JQ7)5f(dalOq?JcFZR$#uoIgRO#6C}*EwY6Dekdix!B-8D2XStOGu#Ak1 zmvyX8+jvs|Idj;iKdmYl-91^idA1#w1g+%2NcD_-c#zYh zTc_ql8GHuMvi^)VPPpfYOTs3#1MCkE&J4^{ zSohs_x9*qfSK2M|-9Ri6@XsX@;Zaxa;>*(g%A5C|kjRVJk;1_Rx|b0v-B3tqXy{rm z2a^cWDyv0pY0qHFRy8u~j;gq=M;aVtmlHwk2;!_4^`7qfTHTFVbv3w8K!K`d-2r{dE{w1Clu9ak=bM!;GTt()^PhF4HYISblQOftp!vINoYwDf)+a${@;ZIXgOtQK!!Pb^=J}}0 zv)HtS&P4MJyx!ZXv-OJdKNlX(U_IB)u(OFndp)T6jjj^z9S3eIFTW%1O*DmN+rmDD zyExFxkWo?H%uHg%zTcpbcp+m8ZmQB3|LU#aAB+6ik@}kaySD4fwN@^mBF6 zon2*ssI50D0OyIf={|}BJJ$$fx<7`oh+>BiP3}cPL0Tz4eLa%l@gh}A#jpfFmh#Vx z(K7DVlUpLDcU{i0kGmWl62{;JE23Caio8ty;g{b}uldICQZSr%9tY)i_ka0|jcz~@ zNH7bFi&(QFsNSng<*#90T*J=hirzjx&5T2=^81rYDUC1~L_6i~(|A!hrx~xwmfGat zPCU3Hf};Y@mVae``*7zW>7XT3ZA`-uv==0YEym0Q^rpYM>8F({bDN3_{T=}~jNf%6 z;xaRDx44cH69dwWf4-k{$oa8sX)&}@Gyxdey5GUcEW-d5b!tS`WwwQ@x5!wWK(ryk zpW-Yv4=tb`0X01nEHS)EuV0@a?+#>$CTk>!RKFWd0*P{NHrZ29SV&moB;@2c zrG;`ZX(Pku&k2_j9@JYvb~CQtm%W}y>=@ivm4%mf3BN7I+s-;{gk9Tx>{pyw^`0bu zfXLb2H$>*28QRej|3rcVi~XtrEVd((s{+;KPWyeXNpjUz2^tYp4f-Clt+)x0?{ zRiXGd=c7QKY$3$dpL=@h8J9aEYwp6)VU04I4B`jdV{72S2>-K821K|<&^~{iZMD}M zcXXs3hTG^lt|`@}FI8dkJ3Rb1I>J~SDk?8n?|p^Oh6Sv^wmRHv!wvD?wmMSoW@C5s zNTHG1(US}DV1ei4ju zH#pej(G+pW!x zXR877dLv8m!{6wZ?*dpDNT6HgnT(+^;<-ALn`&WdWGIsk-OG|V0H8ca>59?VP?S0V zMQ$BK5ol*mk9Zifh{T&y=PKznL)4HNj3-_a6W&81_-z_mzA!Wy148M_r78w=6E{&% zKGR(YfUQ1}5h&e8ILJ^knQcL9v4sMKpl^h*snRG3c;8Jzb@Z{ZrFD3!W~-m{db1?;dSA|vm6i1uo0R;c z11)k``}jXW?(lrc>d`AOOQ-NU7!~L(i|qmjxr>szEqc{69FtSuG!Z){f&5;e8WSY# zOA-SDithaI-y1pl*u>I4yqJIldmTgf_%gL~XgFxXtP@F0@`HgSlR~ko?zshD7$1G3 z5&=58;(>pA87IUYUMIPuji8Z>Oy%yQd|XE~S$<4fDhd0RS3ha|Q=H*gaa)_&(9~$bwnlupzK$? z;WNuh+8bEOIhp^7_fSK};X`p5Enppj5m7_W&gV}&1NPxETlpz{eV9vuy*&oky&8*( zk_yG%TU=O3lJh1e0@v?x9Qm0^j5d1W<2lGV0&gWAjV2mf+Jt9F#*ZeNSUQB4;e60l z7BoIs42=V01Q1Ao%y@3UXD~6dXnhnj{-+^I2tg|45eTk)`ppSP`E_cOGnM-tdJD@+ z+yBe}&VW@%A2qC;G%~&t4qWd5hwD?v9@`Bgkewwe+|1H4E8oTTc9o~+dj@!Meo=uG zQ(PRs1~*0Hp{Hk6qXMv#1Ic#1cA?g`lRf|eW z`V4MK`NhQbIuxmvH zO$>Y+FDMYu*VB6n@DwWDq8nI32_r5Yr=wp{+&TfK9V{Rhd!MFs4&b zUY-%ya3<{#=o}_mX@BYdR>e8*;7;5ps zMlmrlQQ=EWLV}H(yE_KJ9~j_{{@aQFOqtY88yz>XbPTVOT=bfA3DPVP^Gl{qoD^lB z6UE#_Zv$Gr>f0MBNBfC8-awBx?5qi@J9C*!Vs>7yj%r3E;JB_+uWQO3g~| zBmuL5tIVDHRGBK(ZaR;1)Tc&&y+`%TSJvZo%21v1IE&^<*$}{o7aV~90FGOt4rvii z+k<6xswax9${A$?4{95QfzJn6sCBRB)RT3AK1m4S=H{mP^yyO{YLpHL0)CkIPak?H zP&D34MPM+P**5CfrJs&7f&E!_>W+Q|^J;JpdTy8UVDH%<+5bA%Fp1HTL}^3rn=}hM zJ!lIddD8x(g&EzQFxu?L_F7XT$bv@h={kj|_6o_X^OHms4^ofM^rhfmPajv0o>9r` z%g3J>uOHXmDrjCgfx2*97D{5EdW@M5aeN(gInum&k6-2Z{d!I3y#38Yd5NgRf`bz_qLWC~Q0<*87VZJL?7Y zIX?<70QD`Mt&x5D*$Ls^DZhz>-Db+gG$-Py1X6(|OzSA$-$cG-IHh(yGgP)><(t#i zT8%$UzL~{lP5MaQu0y$BZaArOF=bEMQcB6=FvZDe&7HmGJ4rdADo$;$m+FEcGME}` zh%^Dj;euLoa`Tbxu=0|PhB@r{cuIev&+-AWvy08j`rG#`fTl8^B?u!9*AwJ0mT*G( zL}2y68rtAe>S0f;*`iYT3~MT(bFjOVJ>CR~W9p@2e+k@_Ldq3lH4J`+*f)e=B{Fi>M^c62Rh;ForEx zUk~y@S8wvAU!`uUADp&_1F5J1m=dRW9ZGUBS&KzWd01gR#t)Qt6!C1yF_9E< zTi%ELnHcZHfP_#&QPS!@FYLo|py8SP&$S6YI;0c{JgN%+wp4GBB8v6IFdBWE9LrzY zT0LkTTo8Du2=nEYrF@h2U&H_GpQ}OMFj^mP_Czz>+I0G&YdmH<{jh;}P(JUUJ;Wm< zX^Mq`_KdxS`k9qy_e2DuRQiXNW*U!>vvcXCEfDi0@6NQ*IqgZGjSRAx>)o-~N2AF> zwUEh(9utlYaNqbT9n59(G%((@m z+z7*tdvny!gKdR@x#h}buS3nx7nRt>ybD$_N?KFS8d-t*F%^LaV-wijH9aL-z>=D1 zk)I)_!Hu~v7-<#v+rDe5nU4Wo{uQ66?TTBaiqp@gO z_!1xD2U8k7TG;}aT2t4HpFI!#XLPhDEkWJ*-rQy$@AFu{oIjbbBL_Fn&s8Ji-Y+e5 zUnD3l^fRh^<8BMQFo5J~?t5F-j9)LvtQNys@dJr^SRn1d_!fZ7tew7>9pQrsUd)lx zrTqAQB~q~X=Dcw$6abfy+iCMyrQm;g5B!kP0 zE6FAv#;_Uj1-;|L{I_rXH8SQM&to>9FWFUhq!5fuOm&ogc%}t@bzoq{^_lIArN@XP zu==3<4>Dc^?J)A8*Dz{Xp6Ptg`RNZLWO5|eS|4*fc%_6*RUB$?s&@Fu3hcJ{Avs`Z z5Zc*j?oA25wMEcO{Nc(J2hQ|-)dDd}01LnZCxTkzEu0s%9Z*r3(Lc6Ex~|Frmozzd zsc=G$=$C%rC+o*m8HTiW+ryt|lyGRKlq}0-jpg-gBT~>)0p|JY+)O%1j%*8P?hM?m zrH^$eV8!FVSsWUGJwJE)(JR8>HGRE|r2F63ySI94{0Y85g-7BBKNF!Kr{^?A@kzG> z>zat-{uko|TV;z{;q-D4Elk7BeHQL7Xi5f`BOf$TEugu4rSDv4o% z+4$4d*1pSbn9;i)sBgqeq@QDGOn$O33;QTm6z){}MkjA%tlVBeo+P#Q6;^*9bgJsp(tD6<#s)d5T#sK3G-B@#SLY};&Yl1Z zR2$s1#oZ)GEkzVm?~w?$r(%dYZHcv$4tf@iys6sTX^4}p<-bci=4UilR57}RWE;)q z_CH>Kh($f@(@bo}E5~(JV=)DT1eu zFVp>c7s~T7J=D~R7L(79t=^pTFS)L+CgYASSaFpMd$3ga72=J1RdH^7DSN0DSDN;w z#D8s{11%S8-L?!6WA;ik*p3-Gi<5;pH*C*|`TN)0-JT*CIX=z&&Nt%E%zW0*$ud5@ zJjlj@e_pWA`SxTVo>ZW z2FEB*9~IR+*rS@OohGr~wB-eu_45hI)_uP7BH7z04vat`7Iz4fQgBV=v-U_fmQ)cw zzR&4(asfMQ#;|2}IWHmEQh){nIMgC8J|1gY=Qg34^0V&h{^kWT&NUJ}zm!f`;Njs} z972Uz_a;VS*lxTf>y%D9;_645I{YH`Q{=Ayt3d$loNW#1#g{hPQ#(^>fvXSQ30O)0 zCNGMw)pAdDuMe3}MS8x7_+t6rI-f zBW<|+N4jk-VbMMu>ET>9HW)quRgdQV*`sGifnJZoIE^aIP0-E)^+#=C%t-Dxrzbi_ zwqyeK4Ls`9ql;nHD!DGx4R&xfpgbY0L8@_vByy;uEBBH5U4?XyeXcsL_%hgTS0 z`q?+>l5CR}*xe_A6!)*Dq=f?a&ooOTXMn0`(~pKENrxPY2zfeaUBRES-VA@zguH`l zv+_H7dzN$RyL_WG{bIk)UZXBQ318ZSneT#+FDXcDC`>oIz{4hg2Z|Upno5r|2D)bN zK@rB{Hsjz2ky$5f2mXA;d+Sz*gtT<>jga$`Tw^86gF^8-9T8QC_g)v>Q13zS0koh1 zC-7GK?>hU>V9g#XVQXAwoJEh^HlM++ElZA$jv|x9tyLD@d8KdK1ZSh2A&CAz6Hbi} z-CNHHt;_^k3J3_S32`>U@uS7z&g(sIp^){Dp_h$TgHcgU zIs$r!Qmz&r=CJi7k;w+T7z(@O+%nF4=6&e3W=5p6OG)a<>`OWo8LHua!$Rf_e10iY zUmccbO49>Gw^RLByAG?rx(&sFF9nJkfbxsvT)(S)KT0~aHLRnAT}s7N`~^_yF)DTc z3I>v*z+Hpu%R7BQzmnQA+AgtL6Jk%zJ@F~1`PLdC<@eJ&HE2d?*^do1F4cstZYIIZ zt@cRrH=sQySKo^<`S_zF2)OSJ&gFSLCrz*TE-vtlO|01R&zqNuZ{tfbYsM9s!-{dN z^-b2Z!Gn81p?-y&X`#v!(IlNYR+!n3DIan=+$W-Qiq^h3sPCq&_6@v!jcLlr^({FRjDgaG4>(>3#VFungj6;q$)n?k}mkc>mPt(!u zbJDW7hc|z=F>BLoX|r{ZN{Aj6PzEMlmo9-vsz(ERiGhIDjRs5X-DX=(ZVkO;A!weD z6FMZS&qf}3j4{|PzAwqb;UvT#4$jr%7GzC8`MMjJpU(9GK~TN?3_M=3wLx|*PJ|yp z@)2-iOno%yw(-4f+0U;Lc|VJ+S7MG4qh~pM(!*kOwhF-a?jt3zeN7Mep?6ChxLy?3 z4xh*7u2))b{9K5hK9Avblhp^=c|+7H5=rbN+282y&CpUTf+&J$F-tZ$u^i$hKs;T2 zbmItU3$JKoY@b)>jyroOqMpAjY2V1nB<$yGGN#(Xbp$|rq?A*ESxq|=r{56}0MgD= z5EuxtZVW@|`UK<_0xdlM@1plOK;B&LBpG4YP8bDUysIe@0|`^XcUoZy8V&|^zi=M-s4A;sOX$*T1;1rsBG${FT<*_FLG(*32Fi^tP=L&u-1+kM0cR5J&DC$3AbA zb$lE)q*f7n@Oq9=qnb1$?fgMxjHvAjI6)EtE{+2uGqSRxAJ*?`uKU-@jlMp)MR+}V ziKw_JN-E0A$cIGhw0|fo9btMbVySq@hvm>oFJvy?v7=n4-c$n@Y4_an3KT-Gg%4>E z8MaK9^X`kkzSN&ylc1CMBWcfL?bc4&*47s0bkvsC7Ta7jy*3jNhp`(c3l6?$5k z=4VNUNo+k>&j$$>Cuidpr}Rd0H23Zxzhsm3#TEg~4KXQwn5*At^ORr0g=p#yC>zM$ z)$w3r6e-ktQ^>6RYYEGOb23q3czc^e#&$sWn@Ko-2Tq zA$-e&bQ?!(L2Om~3^>M6`*HLl4$^$MY2nJE78SH){#|tdF{7f)C;L+62s0BVpcOw6 zKQ7hgqcjE9)ZtP8YWI$F|;t2j#oW$&Oq3^0AN#jT~vUssBB; z0dO6)y)qH;Ki3ccPk|D3%KztR?EiieJ~W3?V6Qy6cX)Q9^ZE-_0OwKrHUQ?!_y%a3 zFn}ndwF2E%mZhc@7}!KXXCYeC6wt87!*{c#$xMQQ6+uczQx%IF9f#3(C}}=ZMKCl_ zeWVD%n|fmt5)?cRU$Ve&U*>&b?F?X^@2(X5@*ZUy=ZzW6Qp?1Xg5iP2_UZjVQ??}x zOX5qv2!KV({|wC_<+oRCX9uRCd7nQIOf>~eE`~jf>HXk7vNP#25&cDP<2G@)Y^VIkA9cy#;qZ$rlS`T)ceFnke z&wgAZ*ZL^}(vVOQqle|5J@LyVOtLfqpI1^MW!e71@o1Egqm^n;AHtFr)WW!UPpZgGLP%*k2QOYL6z{t2lUxYidIpQj^?iUYjo_aX z{qRV^cT>e9X*poT=6lNBz$rj31^q4=n|w&wPikm5EuNe$sPr!m<;F-1MlSsqh}L&O z#_i{4N*N)P6Jr6S?8juw18qAy(i>BR3JevMw(j1qXi1e5t;56(-kuAXaGYtvnIA69 zcU4~)bSNq=mIOpa@BX6z7}Xv8N|fQVRrItnlO7dg1~f*OJiid9eJB)51)`cqJ8aaB zmd(a8#iYjB$-^H`tN$6~)m<;D0-Enp!p55wl*u?-{8(wlZsGb|p0pY^6EdR|C(RW^ z%GjIy`s`x%?swF0qF6Vzu>f+hp7lZ~(@r@*Tw7{5M=P9DZEQru6}YG0^$p|AZU;3{ z$XkD0?#)D;2i;id?gzR!q`<#Z6+|g*yVaz|oA-s3B2xIfEptSl{>Ln{4<2$$f4-s+ zL=Xn;d5|CZcb_kXbM58>@+mKmBy%*-k_T`BpgR1#@G@&xyMGv?3LKWd{uQ2q>zmZ0 zZp$_U%Bui4aK0Th#H0&v0S(SNBDQK+F_$VX+F}+pe`Yg56?&DoMNi8ueSLp2?q{<~N}iH0>+K6G|4+q()-t?5Ww z0>uGfu%lK-X?6RJ?1xYg=Imwp3n6blX-o3|jbT zfKhMahF>F=qu)%;D$47R$$C!*4~{r+dIaUBZ1IGdUvmKRRTg>I=D;>V5trU9^|7h- z&1r^8)W^JMBn|#Jgv(7uHnkZ~Gt7WZG)aucV`VeJ%D*EW>zZgm_kSVYfv&{{kX!TRvA1pP#c1X0u5<26+ zC60q8_xoB9N-C%UToLdD$&T*70?eNenC#`;dDhxV_5!lq= zE)LqFDs>iQCH!*$yZLBnZI)=64-lCM3uZt^x1CE0cus#;6kgr?f;1Si(>O%559CIz zf7RF7so|L&*J)Q-qJpvM4ggkVMXOWmd+Xg?Ztqj_0LA0UW59NVRSM{W%{8ZEQ1}T{{VdS^2f@WOxT$~Ql_ET!Tq)Km$ zN6#}evjm9D&fWD(Yo_cMQsjnd_ZsZ@UK>V5Ex*$@SyB_1t85?O3X=bMvD8=@fU%v- zY9^;u(M&)=%-YA6io;Ccs^ZG35if4kQ}Xq59qT@kN6-5b9?df|jylGkSGwYq9@G6h z$3ibqE$DS-17HLJHxj|rvsZZo=9n)4SEc52eD_Z^ zzZ1~CL-I*RaiHU@M*9qC!8V3BRhX=I5X~2hrB4F)s~OEd_xOXSRndDZQM-rE?*KT1 zdr!+Au7UuN127v)JXls;fLh?fSJp5Nz5M`uN=Ii4^H=BQIH2}!2eJSK2lgIH5TUNF zP92f%LWDfE`@ZbD0w@bUA3z}?C~wq2mqXW$ zD7&nV57fdN$@8CIfG~MxLKUu1wF1V1LT_83oyCIL49K8=l+PK_V=8f7W?hvVH)s{k z>{3-ri|}ws)tyNB9I;B64xb@EBQux^dY7dDLH#Im@~C45P^{6{)qMi!!ZAQKtrtKN z7(g)HTjekY03stTZS4q5oS;#rY14~e-^v;P%GFc9QQTh7x@G?#-)S*5HFfOhp7jGC zUOyI4T#N^Y!a*o3q8Gyq3d;ac7TUETkWwEm`0k)Kod39eK9TaAy|Ap9ZONW0q5;0H z;<#r02MX(1q@c!mnab>I9vXtGs_BjI!XQVNppPH_)D_FFO;Qeg_^ZLrKG$AF{VI$z zAO$x#W2c$o9IsnoMw4wAa|7bib?~A!y&Y_xMcKK1fU5IQ-=w%0AW+483D4Pw^L@~L@K5TQ%5qoBkcfTgBV@d z8$1Db-em8ucP&ZgnD@H6MW10%5iF78Tzr9AdnM(4W>lAibruW97F-z$94=vZ z6Av+cIG@RI>(~jdVM;lkbEe_GW`G6H3&0S|rJdeOv{(6p-o)BhM>SVR!Oz;66iK}> zf#|S-o3FX?d<>IHq3v?;SK-6N(QtROi`lk9w66j^KQZUp1NBX{^;>p6jE~D`dr=X&N!8)Q|&Rw z-nmQl&ZSP#*HgWpo_pyQfT?22_wS@RiC&!@ki(wjZiBN{SwO2klxF<#}@Q|-ZX zGoR0{Iv(poI9Z%)4=jKc9lY+;sV%!nd+25#pyA^XegiCJ{^<8tXYX~N@jjOa96P#) M542TEz)xQPFTow~N&o-= literal 0 HcmV?d00001 diff --git a/docs/apache-airflow-providers-http/img/connection_headers.png b/docs/apache-airflow-providers-http/img/connection_headers.png new file mode 100644 index 0000000000000000000000000000000000000000..413e9bbb38864faf0dd5664703b9089ff0b7bd5e GIT binary patch literal 5256 zcmb7Ic{r5o`yXfn{-V})%5hC16>~W0*`i5pPc5~&!_Z*&@TQfZG(MG_BoO+`NXB8 z9cLtFwPB4YYQ)t9B8V#!9>!GTtzxn9z61NxOR26(i5#r%Yd8mX zpspJ8#$8$syuMdWkp$lPLfgu|3wA>n2RozP0kP;M(D-3DJV7{eSD?`7DFLe$svtVK zQXrI18VCt+$j?P)C&oNc2yK;mP)&FT>;wtK#y-+W5L@X#Lh$T(nAMSE;3&rnyAcEC z=B`9Y)31Gasqo@r&2Lc$Ksrplcbu4A)+;|xiiy6^5E1hE2n=@Smv23B#hBxd`M!T# z;oFxEnNSws+K+=nu|$kS1<~tp{^0_Dt>)tIEzT4=aOBWXNS{k_ety2;UKs4bA;@|O zqI_48U5@qP!-su9fX5X;*!ZA*_1&YXJa?|LQ*sBZ*nIV=EK&jnJF5asx!*D8 zqD|Tiyz1lgM;Zu9wrlLxa<^2*#&T(F&im)*@7WOyvr6k48XC^!Q6Dt=gTuhh;JSW! z(nzC#D9CHZ`xnj`gs`zBwwg({QRK?w@WqL7$}3N5ZJ|8t-809c6d6RJKWPi@wu&2x zcEr;P^YfK*^h2JGK+AEsXA3O%nX@t9lWTgD%@8mrgimB`Rxh%jl#)356_u5Rr}Hff z?kZUoZp_tbcLuA?4A$sUGj)_w4(KQjf$QcHa|a|64;?m;j&!dcTY@2ScBl@KMh*uy zaWRImoXw@-$oX-~>Tsk#A-+3jdAhG?1YE7D&CiQIA7B5B6iA|ZE=_j%Y;(4J-riS3 zzCK?Y?8CVhzotq4@DLf){qv5U(jL%R7dBit5tFhO#p1X{FQ4_;Y!)_YAF(n-l;GOZ z54)J*k0Tn12Z%>5n?=l&Q35k1hNBj#dRTh-(qK?$L7sVH;mw`Gp&QXVV0s4J)6kcP zj*}~sP)CV~x22ddX`Y|i<5PF7mTn3IloxEAW*u(6*i%(x&7R-&?tm0 zc^u>|u8Vn;W}COTY;i)gElS_{pk}(GerhFG)QC&e}LIYN^nK_x>Q`K6N@g{EVKwmRXK^9NZ>d z^HiGgx_1W4yW}9TxSH^Ygbw?&xJcU6R>z?bT%}2jLma$IRh2MOqZ71LnRK|p9M{JW zTX5+0%A?Yc`38ou2VZrDA~|&5x<%F%n~Y%OursJ-)Y7YY=%p)3vUlBA^97XL;cpAk zc(&GQ1>T3=j!M_wHJF$#QGbgZ7$D)Tv;p)1kgr^3(sXt&WWY)R!f?yFA<>ogRGAmHtO$}!F+-icSz0A;Zz(U~Nh3w5b9 z3lT|Ew2h5Tj9;7%s{yr*ZvrbxJI7E~#ci6CBa`P!UKp@s$;&)2y=T@5aP7IlEEP<4 zreFPNg5DMOrzkrGj>X()qMUWUWvZ=t;$AzPR&9RSvd^vQ8s=PG_iG+RVKD-8s+6=s zzmiAYm~Y6K7}Pf)>Uu6cDGy*Iv`Va0_-eZ_ZL}NiU+`>yQz}2aF^XKA7*0>vD|g>= zuY7vCrXSkPZG1xNMx#Rkt#I&Jg1qB310fI?_5Jkvn%G#*Mv1}xf(B5A4fY?lCT)^C zdw#zuKfx-nN&m!Jo+h5uHr$Sq!ygOsWGu_YtmPRtF06bv)$djH*vG+t4s@1wynw3; z`r)r5k<@I;;@T_c%1o&w&uSa?yVcpw5I1jH)u%~E9}UlqCF_C`zBH3n#&`N(LNLH>GgW*=?Jnd{y>_P zUfD%8If`a>6G9aex(h3Jp>gxKzSQKtBwCRGR=-mqnNfB|J#{3kS$nbjDHV@67Fq+# zja|k;FwrEzSQyLcVLu5#xa`IbM@&=ezkm89GeA7gw*9fxu|U*S-wj{Fl$e4&sAe}? zZStrst<#j1$2sQs$wcv3${l=1sl23{r6_%7FueHgPYRywmNI3VeazLgAor=W%-M@P zlk!}kaC+Pf?;d_8?x;dKvSG1fAUq@K(FodiIy$8+^RpO*J}|I&C(}v#9Viw{gB(Uhx^wD6z@c9;^~di z&VD}}jHPJVTg2isr^Fht`-aMn3>l99h`|eZbP$a2a$z4io4Br9$D6Vb(rxR?sorr0 zH+KrQr^I_ik&~_1YaZKG=&QIq^vpp?il5sNlhvLi&la6#)zG`it9-e!Uy#s)Jy=j_ z2ZG$PopwtK{sOuix%Gns#yG3_Umw^8p;7u_3zvif#+)?oCd}&7rOnI&1Iny#t z**6xC8=!a3^uJ;w%f@ln7NbX-;tk)sc#DV5MyL(3o2Lb(j~^10hwCAfje{riwU{uP z$(M~9@4>=8QWXD#YHJhVSS;43DQ=g~pS?CVVbh)vDwz3D1|h2@v9E2L%3@G@Zt&~~ zyt)2B!?Ez}vvUtHH7Yh4T4>VxyV7n5?8w#n-BUqZN3PDHK}WXhDZ*$AyA>jg&HX`n zM3fX1D4V1nnqKa6)6U_6E%X2va#y}WE5mQ0Ro_owJ5&8XmR1q6bv!H~oE&@E5Oky@SUzc)d}XKWJyRt_1;{R^`9_2p4PRVDHZO5d0FAFS3fT5Q*y zTI@98!ggX5`xK)dG)jX`_o)t9+~8e)p@W$Td!@Rap2@EK9{-p{d}~K?0G35(ogWJUxF=^gdUdFEF?O|X zWW2NbTK{jGS&Udp80iSWarKc(u-BNZukco-=@vl0Op55O@~SHEh|+X=c_@u?Cj%s> zD0XYwEr&=a1B^VJ=_B5m)UPb+-V#{67M-%7%$u?R0Pv@e9(^S)5DJ{?!I)4lzp2Cy zu*3MYMAKV>(qs+N)5sO%V6kFroqLcnHVR?^75TlzNjGI{9AH z?_c<|XX>QL%G^HTMM;&N>MeFa=B|(3)@Fa`%C`!7$p@M(mq&mM*ZfOB3F(dIfsgc;t6Edc<_QlVP9D>>+G z9(9a-`*sm8%$|EAAi~rBugTHw_wUnX__s%eV}BraTQF%V>uvWltjO2WGvA3dxIB`# z4G-^45R?i0p&MVq)CF}e*~W@6R$_#(sKai!0^@_q|>wch2;jFQ|)-Vb2mh+Ain#B!$aSK zgM%NxfB(MRZ>W|n^4EvyQ{Lj0enZ)Fq-_ox5-+~6@E)2cz{)=Tzg_tc)BZ1Z3{-_s zfV<`zR)@ctG;t4Od@A2E#Twi_4PhQ2LU4~K7~b^s?6YoAsJp>6Y=by@>0cbZva(Vy zbml|wSW6P**3uV71{~N}nJuoUsQ5;m_%9^@2pIJXO?#mJ`&B6TzIZWPx5g}3+T|0^ZBmz$t5%Ur1?3w22x^7u=(PV>RJdI`5- z^5^Wpb43lPll8hFUv~BI-zdBe2yHOv7u@*7&++0#R;vR_BVT|)XF5ik6MH~$bg;6r z;<`C&{WBWUsQwm}f+pItHnl}m?H-;4%917|w!CQhqGSC%M#99CRVu=2-ZvfdRh#nV zN?ooPuB$*F`~PZaU`p;UwSw5cL^-vBGu!X4KoNq$wxIO?Z{>})&HIxWyH+xol?#nJ NWnqUZF}w83zX2P`_&Wdq literal 0 HcmV?d00001 diff --git a/docs/apache-airflow-providers-http/img/connection_username_password.png b/docs/apache-airflow-providers-http/img/connection_username_password.png new file mode 100644 index 0000000000000000000000000000000000000000..6e36e77dd4cb48f62107a3f654648587bddc58e7 GIT binary patch literal 4761 zcmchbcU03^o5!P|;P@gUqabyRBPa|Y0trZS6j4z?q$vUcqA-y*h?E3EU@S;qMnF0u z(g{VRlYmN5T9iFsdW&WcE+Gf+@j6Uopb@@XX=e3HCon2biA)vxaT*S!RJ3xgOLI`DzV z&2xKseS&`%p+IgvaCdjl&(1!j0f*;S(Sz>pj$4ZtC$~^wE!F+d(mk)c>gXq@hPy(@Cb59eAf4h1K$Ny zvvJ3q1Oy^s9O<$3lCe(nuy`l_}xUI_%)8AUWqzr*n+Y^pZitg6I zLeC6Vdf;g*nKB{9kSgh{rD%npDisdYg zw&r?`wq!pR5I#1!Ia65DJ<=ROmi8NpeRfDXd7>*VMgQd1EP~tCFV)(LekLrNK0(UJ z;64kL!JLjdt|rG;K@e?IRf<=N>*m|~;XX4xJwrpbb9|6jLlSbOdxrb-<@r)Hl|Lku zyO1VGn%Ip;hs?Xuv|qAGiK-`cS1CxX8)2p6e}u}Kw7*m#Z9=8Okk%#kX?ol(W)Ij@ zlH{JYR#{wXOxW4e(fJ+3JGGBc$+`i)#D>a>it$Cci%E-AKl-?<{o8xj(k&%2#4@|$ z&Y0hFb#>LdvNb;%qi-L$ooANSouuZnytG6=vz=grQ-eUp&667^+Ts;a&Eeyy+mplP z2)Z6>J#QV(@yaZ;E-`w%|Inw_`o#~ckGMri>-DNP@j*W-?5{1u(y{EEOQDCd z%aDe-g91V~+FvTQx9nnX%l#QJ+Y0W?mTJXJoTg+7bT^2j%$0sEm}s`W-f+_M>G&A(XNP6 zho85#oVVSEs>>*1(3=JGdmuMWvJbe)B$MM})-O$7P?K?$P{UZzf@bkm%iDH!_FaQn zhEX?C;GRb0924Wmp0aTPh@kax+&-9Ua5Tg5Qs|rG5W{YvgBwrWhnDlebNt z9hk5$ax&VOrDic^M?xu%yB!#9bM%BUKTL%7=;$PNdwnv3k%%U1^n}gMY*br3(fIgG zsC`o*#JFQM-+%GF069VFZaBj@1(vPlJu#LcZ<4x(=s#+iw1*#Zp(Y+$s)gOKpy^Mi zhh1JEh{`|oY+<3$n<1MnKkys2Z9Ux)@ z%BUKpVONr&Z|;;k8*i)*dNQ|f3*??JoyK&EP}~>~JsA>|8_DWt1a3eeJ-N5|VI}jl zGMR+Q*(I5YLM_6Y15OHtRX9Q~gnh||_Pq@YxK;hNF=yGk z|BgOMZ*$IA0Jii?4~WJH?1^JPu2w2Qf;Vo4H_G)Dd=BXh=iVR}^TRHM+Dgw^$;8e` zP$DG*Rzz~=_oft=tOs#9Y;VtPoe-f5S=C96a!oeMBG2Z&RVeC*f6spIFLmbFo?@NX z-$(3iy%C@2DEf0N_~e%_5<`V$yEaGhHzJNH86j{SAA4;j8F&G|99)9ZRFv$`dcdBi zu9uI*4r}Vk6o&h_d#IO&T@D>(j-DTBFx|o~vD0mqTH;;PV4m3X-3Q_Y2ELXyjh<)exI zAI=l-``$WPgJko zCOa?C2+evBNE_aoFXhY&d7nehVw^muQmF~`&aPISZY>J6*5|qedh&Tac5AJy9(QH( zm0IPMsCqSB_E1ax)UWj`*Btm3XMM|gi9PO4$$nVHgXxSCWr!5{^Jx6wl2xuh-mjxl z>sJSySG1r52UEhrsa5?&_IB9Q zkOnAEI3>)e{ID|3v6d+A3&aAo0YTqdcwwrOj>!D3vE=2>L>6#hcMp(G$SV=x=see* zezA@FgKI(7{j0FEvooJSAk=~-`Y5$jtM)OXL5uDODAYAE75gF&V31wiqv=~Re=1=G z$f=$(BhUK2el;TiLuQO3QAjt*f0xSq=BS^D*ykxWI+KMvgkV9i;VDWRx9}jNfL%DaGjA|W{vQ$NpS3m4=I-6QJZ}L}~wP_n}m)v?XSj1Nx_PDiK^uj29eT?=~Mn-f>J)|u$Y0Q?t#MSwG^ghl$AI*YQh zPGZ4#`&YAys;bE{_ed{f(V#TgRDd`-wlxn=(T4zRi8N6v&%LPE2D@vdn9hvyJOK)0 zo5bNQwNxHk*?5Vn)$syP6ln|k`s~Ko5m3|5!!&g$uh4iW@cxxT)+;#8UzfUEN6*nH zv?yuiGB};hJ>V9!_DASU zA8w_fYPQUpz`4E(xJHivGHtDklg(P_KGgdyG^%6DtOoA5MV+o&AQo=N*DjlLvwkSu zGld5EG;sN|JgWQ6HPoknI{`GAsz0FpaO|ptUZB5(wy)cH05C!;>Dbi)^kQzahB0rOS2)^^!pMBGqd>KLLc`E3qWiy%&Z<__TUz@VEwbOwXrxKJ5u3YvNnYXhWJ7bY-C*PQ~_Dzw@4q zcEWY^tbhD%ABytj1gg=ULuR4O$zh`M%fx*LCniZXf$I-xzzlPdW0oDL2W@J&Z{NOk z$17TQhaYX%CTXaz%nr&Mr=BHx4{sWI&DHu?bJMNu(Wi{(rNl`=i|iWidjZ+8*_hcn zTwe!ktI{1fRmN8>#Q2FIXJe)P=pUDckPlW`Vq_64XHt3^j*Y=&;I`3a(iH&Xk_p(& zRZbq87O-IFSJJoAH)Dyoi{Du~p#oD;jJFbo1WPg__Q4!Y$a6zDX6NJSxqW8aS7EDRj9^xD&NQP6Uc9bgIU$EzSuy7XkUDeWJ73?Nt>GqqxAGePKGm z5R9MD)q&%jw?d1gs+`u}vRGu(?95C154u4x6d?7k=NC@JpLJXuexIL}mF2ABTxfv- z*xT6+5O2abH*6Z%;@j@Ch9He$Y&03*q@cESMj*l(~?m8SC=FfD5o|Ix9xc zsD!xj0DRn^TkDCQ2pXG$oMrmnPw4oHL(KIfaA_uK=i|9Tm-f@spV3u+%1~@-kPTXM z-1b%NI^M9!GB+J35wTwy-(h+ShP3?`9cn=y1(?>As)tqJN~zsf7_rHMBRAgFb9$Z$ zp|w9G&miaP*bk;e`$`ZX9ZflcFmH-%&o|G_5jko22F%Ej4`_{%HO;ybVCW39*?lDqn1(u<{qD}X_YW2lcjDZtPyo^xHQn2o zV|6)CWJLAN{p;M+`8lb3EggwO?8X=;Mh~+xz~*eSC$A*D7JJ_@a`|AyqMX6$TUst1 z-FH@|vZjY>ywQlQSk){!=dO3DdTBA}DsKmgj0#@{qC13d(_6)PxV|>P53KaFMIeum z#4>N@@x$y~{zh2+mOga1(hc=BBI)8+aCBm&f};?~S92I1gGFyw-@CR!mgOcs{v9mw zSgXNV(bgh+f`Wij{0agP0++y&v?a~&`%-~MAxp`hbPV!2D#7{mX21p%0%^%#S95Ea&2 zqmQilODU|X*w>#nCFptQWHS?)&D*z2t6lTnPS;E>`|9+mBR5z>5m-R2$vMVp$@58? zC)nXvUdvIDk7z0aZ4iF6t&|#3ZEs`7Mj;%!@ot)dR zs}K|U;waI$QPms)nQDDfWZ3Y19^^!(0-7FonL^qnn`Ig1NNf9!4JdmBb|h<|m+$G} z>NBCyG2cr6D5(*fov^^Dmgl~HEB>&fF>K${(N2r99r$UmauCfl({Zv~mmEp-e^j_J z1lg+J(TCUF{4mwW6U5E=&e{GZ$=7-u7Wv&+Y%^LZMLr`C+K}Y*^~2NV!phd@<90tm zrJ#vn@XMEf6^%GjRyfl%wR+Q*2 zIy+?1^PStpdc&<^FC^2Ikm9U=TSukp{J%jB&r7>^DgT*=YK7eXx}-UiZLwfo1*9JQ2&Vsd b1GBE*^^2vH0QXn|ulCwy6NBQ54!`{!fAs15 literal 0 HcmV?d00001 From a51b8af6d8cf16b7774590a85304ab99b2442bc9 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 08:18:23 +0100 Subject: [PATCH 189/286] feat: Implement auth_type and auth_kwargs in the AsyncHttpHook --- airflow/utils/json.py | 10 ---------- .../http/tests/provider_tests/http/hooks/test_http.py | 7 ++++--- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/airflow/utils/json.py b/airflow/utils/json.py index eb3cd40941197..a8846282899f3 100644 --- a/airflow/utils/json.py +++ b/airflow/utils/json.py @@ -123,15 +123,5 @@ def orm_object_hook(dct: dict) -> object: return deserialize(dct, False) -def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> Any: - """Safely loads JSON. - - Returns None by default if the given object is None. - """ - if obj is not None: - return json.loads(obj) - return default - - # backwards compatibility AirflowJsonEncoder = WebEncoder diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index fe6b8f882f2b7..1e944dda1d090 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -384,9 +384,10 @@ def test_available_connection_auth_types(self): auth_types = get_auth_types() assert auth_types == frozenset( { - "request.auth.HTTPBasicAuth", - "request.auth.HTTPProxyAuth", - "request.auth.HTTPDigestAuth", + "requests.auth.HTTPBasicAuth", + "requests.auth.HTTPProxyAuth", + "requests.auth.HTTPDigestAuth", + "aiohttp.BasicAuth", "tests.providers.http.hooks.test_http.CustomAuthBase", } ) From 594a9c4abaa2f262f1a89d1f2f3843a40fe989bc Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 19:04:06 +0100 Subject: [PATCH 190/286] feat: Add tests --- .../provider_tests/http/hooks/test_http.py | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index 1e944dda1d090..b6f43995b979d 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -437,6 +437,32 @@ def test_connection_with_extra_auth_type_and_no_credentials(self, auth, mock_get HttpHook().get_conn({}) auth.assert_called_once() + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_string_headers_and_auth_kwargs(self, auth, mock_get_connection): + """When passed via the UI, the 'headers' and 'auth_kwargs' fields' data is + saved as string. + """ + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + login="username", + password="pass", + extra=r""" + {"auth_kwargs": "{\r\n \"endpoint\": \"http://localhost\"\r\n}", + "headers": "{\r\n \"some\": \"headers\"\r\n}"} + """, + ) + mock_get_connection.return_value = conn + + hook = HttpHook(auth_type=CustomAuthBase) + session = hook.get_conn({}) + + auth.assert_called_once_with("username", "pass", endpoint="http://localhost") + assert "auth_kwargs" not in session.headers + assert "some" in session.headers + @pytest.mark.parametrize("method", ["GET", "POST"]) def test_json_request(self, method, requests_mock): obj1 = {"a": 1, "b": "abc", "c": [1, 2, {"d": 10}]} @@ -724,7 +750,7 @@ async def test_async_post_request_with_error_code(self): async def test_async_request_uses_connection_extra(self): """Test api call asynchronously with a connection that has extra field.""" - connection_extra = {"bearer": "test"} + connection_extra = {"bearer": "test", "some": "header"} with aioresponses() as m: m.post( From e2b78aff7fb8777b55683d057a3afc7990bdafab Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 21:04:22 +0100 Subject: [PATCH 191/286] fix: Add header and auth into FakeSession test object --- providers/http/tests/provider_tests/http/sensors/test_http.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/providers/http/tests/provider_tests/http/sensors/test_http.py b/providers/http/tests/provider_tests/http/sensors/test_http.py index 78a11e15bb7c1..47af8f49c48cf 100644 --- a/providers/http/tests/provider_tests/http/sensors/test_http.py +++ b/providers/http/tests/provider_tests/http/sensors/test_http.py @@ -238,10 +238,14 @@ def resp_check(_): class FakeSession: + """Mock requests.Session object.""" + def __init__(self): self.response = requests.Response() self.response.status_code = 200 self.response._content = "apache/airflow".encode("ascii", "ignore") + self.headers = {} + self.auth = None def send(self, *args, **kwargs): return self.response From 30c8e371f1cbef31b2e8d94c39f81cc62956c4f6 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 21:14:54 +0100 Subject: [PATCH 192/286] fix: Use default BasicAuth in LivyAsyncHook --- docs/spelling_wordlist.txt | 1 + .../livy/src/airflow/providers/apache/livy/hooks/livy.py | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 2b35ed5bc2dd9..7aae2582f24b1 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -25,6 +25,7 @@ afterall AgentKey aio aiobotocore +aiohttp AioSession aiplatform Airbnb diff --git a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py index 934985befbc91..f73fa5d87da92 100644 --- a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py @@ -87,9 +87,8 @@ def __init__( extra_headers: dict[str, Any] | None = None, auth_type: Any | None = None, ) -> None: - super().__init__() + super().__init__(http_conn_id=livy_conn_id) self.method = "POST" - self.http_conn_id = livy_conn_id self.extra_headers = extra_headers or {} self.extra_options = extra_options or {} if auth_type: @@ -491,9 +490,9 @@ def __init__( extra_options: dict[str, Any] | None = None, extra_headers: dict[str, Any] | None = None, ) -> None: - super().__init__() + super().__init__(http_conn_id=livy_conn_id) self.method = "POST" - self.http_conn_id = livy_conn_id + self.auth_type = self.default_auth_type self.extra_headers = extra_headers or {} self.extra_options = extra_options or {} From 8429a8222f73c6b5139db41e2e3ed2e9e20a049e Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 16:36:22 +0200 Subject: [PATCH 193/286] refactor: Removed docstring for removed json parameter in run method of HttpAsyncHook --- providers/http/src/airflow/providers/http/hooks/http.py | 1 - 1 file changed, 1 deletion(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 9cf5d4983dbc5..932aeb31d29c0 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -397,7 +397,6 @@ async def run( :param endpoint: Endpoint to be called, i.e. ``resource/v1/query?``. :param data: Payload to be uploaded or request parameters. - :param json: Payload to be uploaded as JSON. :param headers: Additional headers to be passed through as a dict. :param extra_options: Additional kwargs to pass when creating a request. For example, ``run(json=obj)`` is passed as From 3ce2f41e26b5f3331b094873ee1a84275193556d Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 16:38:39 +0200 Subject: [PATCH 194/286] refactor: Aligned HttpTrigger with version from main branch --- .../http/src/airflow/providers/http/triggers/http.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/providers/http/src/airflow/providers/http/triggers/http.py b/providers/http/src/airflow/providers/http/triggers/http.py index d25d3a55cfb5b..ec9780bdeab49 100644 --- a/providers/http/src/airflow/providers/http/triggers/http.py +++ b/providers/http/src/airflow/providers/http/triggers/http.py @@ -73,7 +73,7 @@ def __init__( self.extra_options = extra_options def serialize(self) -> tuple[str, dict[str, Any]]: - """Serialize HttpTrigger arguments and classpath.""" + """Serializes HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpTrigger", { @@ -88,7 +88,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Make a series of asynchronous http calls via a http hook.""" + """Makes a series of asynchronous http calls via an http hook.""" hook = HttpAsyncHook( method=self.method, http_conn_id=self.http_conn_id, @@ -165,7 +165,7 @@ def __init__( self.poke_interval = poke_interval def serialize(self) -> tuple[str, dict[str, Any]]: - """Serialize HttpTrigger arguments and classpath.""" + """Serializes HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpSensorTrigger", { @@ -180,7 +180,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Make a series of asynchronous http calls via an http hook.""" + """Makes a series of asynchronous http calls via an http hook.""" hook = self._get_async_hook() while True: try: @@ -193,7 +193,6 @@ async def run(self) -> AsyncIterator[TriggerEvent]: extra_options=self.extra_options, ) yield TriggerEvent(True) - return except AirflowException as exc: if str(exc).startswith("404"): await asyncio.sleep(self.poke_interval) From 7b6b9db1dc2c12d37c0a730f9399483d371412a6 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 17:03:29 +0200 Subject: [PATCH 195/286] refactor: Changed docstrings in HttpTrigger to imperative mode --- providers/http/src/airflow/providers/http/triggers/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/providers/http/src/airflow/providers/http/triggers/http.py b/providers/http/src/airflow/providers/http/triggers/http.py index ec9780bdeab49..5975389830f36 100644 --- a/providers/http/src/airflow/providers/http/triggers/http.py +++ b/providers/http/src/airflow/providers/http/triggers/http.py @@ -73,7 +73,7 @@ def __init__( self.extra_options = extra_options def serialize(self) -> tuple[str, dict[str, Any]]: - """Serializes HttpTrigger arguments and classpath.""" + """Serialize HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpTrigger", { @@ -88,7 +88,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Makes a series of asynchronous http calls via an http hook.""" + """Make a series of asynchronous http calls via a http hook.""" hook = HttpAsyncHook( method=self.method, http_conn_id=self.http_conn_id, From 92c0a11e6dd74d1b28a4eb76fc3f59b1ba3c10f7 Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 11 Apr 2024 12:15:06 +0200 Subject: [PATCH 196/286] refactor: Updated docstrings of serialize and run method of HttpTrigger --- providers/http/src/airflow/providers/http/triggers/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/providers/http/src/airflow/providers/http/triggers/http.py b/providers/http/src/airflow/providers/http/triggers/http.py index 5975389830f36..d30f41990f5b0 100644 --- a/providers/http/src/airflow/providers/http/triggers/http.py +++ b/providers/http/src/airflow/providers/http/triggers/http.py @@ -165,7 +165,7 @@ def __init__( self.poke_interval = poke_interval def serialize(self) -> tuple[str, dict[str, Any]]: - """Serializes HttpTrigger arguments and classpath.""" + """Serialize HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpSensorTrigger", { @@ -180,7 +180,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Makes a series of asynchronous http calls via an http hook.""" + """Make a series of asynchronous http calls via a http hook.""" hook = self._get_async_hook() while True: try: From a82481d096e634bb0d88fcef40cb327b91271d2c Mon Sep 17 00:00:00 2001 From: David Blain Date: Mon, 6 May 2024 13:09:59 +0200 Subject: [PATCH 197/286] refactor: Moved get_connection_form_widgets method from HttpHook to HttpHookMixin --- .../livy/src/airflow/providers/apache/livy/hooks/livy.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py index f73fa5d87da92..70cf4190e63aa 100644 --- a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py @@ -80,6 +80,10 @@ class LivyHook(HttpHook): conn_type = "livy" hook_name = "Apache Livy" + @classmethod + def get_connection_form_widgets(cls) -> dict[str, Any]: + return super().get_connection_form_widgets() + def __init__( self, livy_conn_id: str = default_conn_name, From a489d2fa66e74d7b6ee4760fd33ceffb500c071c Mon Sep 17 00:00:00 2001 From: David Blain Date: Mon, 6 May 2024 13:15:40 +0200 Subject: [PATCH 198/286] refactor: Pass auth_type parameter from LivyHook to constructor of HttpHook as it has also this parameter instead of redefining the same field --- .../livy/src/airflow/providers/apache/livy/hooks/livy.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py index 70cf4190e63aa..c4e740d5a855f 100644 --- a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py @@ -91,12 +91,10 @@ def __init__( extra_headers: dict[str, Any] | None = None, auth_type: Any | None = None, ) -> None: - super().__init__(http_conn_id=livy_conn_id) + super().__init__(http_conn_id=livy_conn_id, auth_type=auth_type) self.method = "POST" self.extra_headers = extra_headers or {} self.extra_options = extra_options or {} - if auth_type: - self.auth_type = auth_type def get_conn(self, headers: dict[str, Any] | None = None) -> Any: """ From f8aa894141800dec5d6c2c64901e9353a11029b7 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 7 May 2024 19:34:37 +0200 Subject: [PATCH 199/286] refactor: Enhanced extra_dejson property to allow load string escaped nested json structures --- airflow/models/connection.py | 2 +- .../livy/src/airflow/providers/apache/livy/hooks/livy.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/airflow/models/connection.py b/airflow/models/connection.py index 01df1626657da..19aeaf9e63089 100644 --- a/airflow/models/connection.py +++ b/airflow/models/connection.py @@ -432,7 +432,7 @@ def get_extra_dejson(self, nested: bool = False) -> dict: self.log.exception("Failed parsing the json for conn_id %s", self.conn_id) # Mask sensitive keys from this list - mask_secret(extra) + mask_secret(obj) return extra diff --git a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py index c4e740d5a855f..408aac0818c63 100644 --- a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py @@ -84,6 +84,10 @@ class LivyHook(HttpHook): def get_connection_form_widgets(cls) -> dict[str, Any]: return super().get_connection_form_widgets() + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + return super().get_ui_field_behaviour() + def __init__( self, livy_conn_id: str = default_conn_name, From 8fbad3bcfbc6d6736af8ce39e94cddce001db025 Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 8 May 2024 16:02:23 +0200 Subject: [PATCH 200/286] refactor: Changed conn_type to ftp in test_process_form_invalid_extra_removed as http as livy do now also have custom fields --- tests/www/views/test_views_connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/www/views/test_views_connection.py b/tests/www/views/test_views_connection.py index 1e21dc4856ed1..19a36c0ac6b39 100644 --- a/tests/www/views/test_views_connection.py +++ b/tests/www/views/test_views_connection.py @@ -462,9 +462,9 @@ def test_process_form_invalid_extra_removed(admin_client): Note: This can only be tested with a Hook which does not have any custom fields (otherwise the custom fields override the extra data when editing a Connection). Thus, this is currently - tested with livy. + tested with ftp. """ - conn_details = {"conn_id": "test_conn", "conn_type": "livy"} + conn_details = {"conn_id": "test_conn", "conn_type": "ftp"} conn = Connection(**conn_details, extra='{"foo": "bar"}') conn.id = 1 From 76f659547b9077c72a4ab5605770df004522b3ce Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 18 Jul 2024 20:14:20 +0200 Subject: [PATCH 201/286] refactor: HttpHook now uses patched version of Connection + added test which checks when this patched class has to be removed so we don't forget --- .../src/airflow/providers/apache/druid/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py b/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py index 7585be9880dc1..d00c3c8da7757 100644 --- a/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py +++ b/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py @@ -37,3 +37,17 @@ raise RuntimeError( f"The package `apache-airflow-providers-apache-druid:{__version__}` needs Apache Airflow 2.9.0+" ) + + +def airflow_dependency_version(): + import re + import yaml + + from os.path import join, dirname + + with open(join(dirname(__file__), "provider.yaml"), encoding="utf-8") as file: + for dependency in yaml.safe_load(file)["dependencies"]: + if dependency.startswith('apache-airflow'): + match = re.search(r'>=([\d\.]+)', dependency) + if match: + return packaging.version.parse(match.group(1)) From 2c495f9ba322117aca8c87be84b37567d2b7e5be Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 18 Jul 2024 20:56:42 +0200 Subject: [PATCH 202/286] refactor: Fixed some static checks --- .../druid/src/airflow/providers/apache/druid/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py b/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py index d00c3c8da7757..18870506f868b 100644 --- a/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py +++ b/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py @@ -41,13 +41,13 @@ def airflow_dependency_version(): import re - import yaml + from os.path import dirname, join - from os.path import join, dirname + import yaml with open(join(dirname(__file__), "provider.yaml"), encoding="utf-8") as file: for dependency in yaml.safe_load(file)["dependencies"]: - if dependency.startswith('apache-airflow'): - match = re.search(r'>=([\d\.]+)', dependency) + if dependency.startswith("apache-airflow"): + match = re.search(r">=([\d\.]+)", dependency) if match: return packaging.version.parse(match.group(1)) From 14ec36db2df7867155d2cbf3bdd1f695491ccc90 Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 31 Jan 2025 13:40:35 +0100 Subject: [PATCH 203/286] refactor: Remove wrong changes from main --- .../providers/apache/druid/__init__.py | 14 ------------- .../src/airflow/providers/http/hooks/http.py | 20 ------------------- setup.cfg | 0 3 files changed, 34 deletions(-) delete mode 100644 setup.cfg diff --git a/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py b/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py index 18870506f868b..7585be9880dc1 100644 --- a/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py +++ b/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py @@ -37,17 +37,3 @@ raise RuntimeError( f"The package `apache-airflow-providers-apache-druid:{__version__}` needs Apache Airflow 2.9.0+" ) - - -def airflow_dependency_version(): - import re - from os.path import dirname, join - - import yaml - - with open(join(dirname(__file__), "provider.yaml"), encoding="utf-8") as file: - for dependency in yaml.safe_load(file)["dependencies"]: - if dependency.startswith("apache-airflow"): - match = re.search(r">=([\d\.]+)", dependency) - if match: - return packaging.version.parse(match.group(1)) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 0a785930de79b..72ca10d796465 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -228,26 +228,6 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: ), } - @classmethod - def get_connection_form_widgets(cls) -> dict[str, Any]: - """Return connection widgets to add to connection form.""" - from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget, Select2Widget - from flask_babel import lazy_gettext - from wtforms.fields import SelectField, TextAreaField - - default_auth_type: str = "" - auth_types_choices = frozenset({default_auth_type}) | get_auth_types() - return { - "auth_type": SelectField( - lazy_gettext("Auth type"), - choices=[(clazz, clazz) for clazz in auth_types_choices], - widget=Select2Widget(), - default=default_auth_type - ), - "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), - } - # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index e69de29bb2d1d..0000000000000 From 57082427a38582855f3aa56c589d13ceb287ae67 Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 31 Jan 2025 13:41:25 +0100 Subject: [PATCH 204/286] refactor: fixed get_extra_dejson --- airflow/models/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/models/connection.py b/airflow/models/connection.py index 19aeaf9e63089..01df1626657da 100644 --- a/airflow/models/connection.py +++ b/airflow/models/connection.py @@ -432,7 +432,7 @@ def get_extra_dejson(self, nested: bool = False) -> dict: self.log.exception("Failed parsing the json for conn_id %s", self.conn_id) # Mask sensitive keys from this list - mask_secret(obj) + mask_secret(extra) return extra From 8f8f95d64a39280f53fc9b995923d8fb2eef4a8a Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 31 Jan 2025 14:21:13 +0100 Subject: [PATCH 205/286] refactor: Removed accordion widget as this won't be implemented in Airflow 2.x anyway now Airflow 3.x is coming up so better wait --- airflow/www/forms.py | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/airflow/www/forms.py b/airflow/www/forms.py index 2a79684122fd2..7028e2026e449 100644 --- a/airflow/www/forms.py +++ b/airflow/www/forms.py @@ -33,7 +33,6 @@ from flask_appbuilder.forms import DynamicForm from flask_babel import lazy_gettext from flask_wtf import FlaskForm -from markupsafe import Markup from wtforms import widgets from wtforms.fields import Field, IntegerField, PasswordField, SelectField, StringField, TextAreaField from wtforms.validators import InputRequired, Optional @@ -177,28 +176,6 @@ def populate_obj(self, item): field.populate_obj(item, name) -class BS3AccordionTextAreaFieldWidget(BS3TextAreaFieldWidget): - @staticmethod - def _make_collapsable_panel(field: Field, content: Markup) -> str: - collapsable_id: str = f"collapsable_{field.id}" - return f""" -
-
-

- -

-
- -
- """ - - def __call__(self, field, **kwargs): - text_area = super(BS3TextAreaFieldWidget, self).__call__(field, **kwargs) - return self._make_collapsable_panel(field=field, content=text_area) - - @cache def create_connection_form_class() -> type[DynamicForm]: """ @@ -246,7 +223,7 @@ def process(self, formdata=None, obj=None, **kwargs): login = StringField(lazy_gettext("Login"), widget=BS3TextFieldWidget()) password = PasswordField(lazy_gettext("Password"), widget=BS3PasswordFieldWidget()) port = IntegerField(lazy_gettext("Port"), validators=[Optional()], widget=BS3TextFieldWidget()) - extra = TextAreaField(lazy_gettext("Extra"), widget=BS3AccordionTextAreaFieldWidget()) + extra = TextAreaField(lazy_gettext("Extra"), widget=BS3TextAreaFieldWidget()) for key, value in providers_manager.connection_form_widgets.items(): setattr(ConnectionForm, key, value.field) From 9ae2eee5803f3941c11a9ca0a4347517a5cf4477 Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 31 Jan 2025 14:28:08 +0100 Subject: [PATCH 206/286] refactor: Removed obsolete http provider docs under docs --- .../img/connection_auth_kwargs.png | Bin 9623 -> 0 bytes .../img/connection_auth_type.png | Bin 14199 -> 0 bytes .../img/connection_headers.png | Bin 5256 -> 0 bytes .../img/connection_username_password.png | Bin 4761 -> 0 bytes 4 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 docs/apache-airflow-providers-http/img/connection_auth_kwargs.png delete mode 100644 docs/apache-airflow-providers-http/img/connection_auth_type.png delete mode 100644 docs/apache-airflow-providers-http/img/connection_headers.png delete mode 100644 docs/apache-airflow-providers-http/img/connection_username_password.png diff --git a/docs/apache-airflow-providers-http/img/connection_auth_kwargs.png b/docs/apache-airflow-providers-http/img/connection_auth_kwargs.png deleted file mode 100644 index 7023c3a7a072f965f9dd053f77c5a5d64b396f47..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9623 zcmcI~2{_c>+jom(n~(ixI5hS2r&k-xWWM^15qC|U8&@yI zp4VbIoNz&&;S|%$vllM#zw{D3bLeCYFTFO4DogYQZ3Q#Q(MB#`@|PP{w$;0sjC|gQ zwZ00``u#3Nh5MrM%WwU6MOC!GEa0bjzF~#+9_k$RUt@1t_`%yIg(IZq`(>tL>=B>- z{$n_3=<@gXa@sCopHqhy8ve(KHqs61*aGb{L`~`|6OlkDb>1ArkdEJf(u%A*3sMN|@KU37UWK z`S!;+d!t~o=`Y)wx6bxn)?k%F?VSe!d^L0ZZ$r)AA%p$*$)o=^r})M@ajS#xngony z*Lg06eEks5LtY{{Vqn>ksZNFGX*0{b@+ z#0=Qog@Hj;9>jhwU?syRst>MH=7w=oH_sMKnFK}Mwee}-tenP+6yF*CgOf6#l9m~nV&Lm0k^g3ly*;YJZL)1?XSHiuScftr z#fh-AL>l<*5_9^a$4@YkHx|3Pz$brZ?SLRnt1aptkF^SCGF4So9T@qEx}W?S9Hhje zjy(hdWv_M_&Bzq|=dy@X#CzWgO-|?oQ*sLLA1p%)S2ocGlYcv~*k1^Hr`qWbrW`D1 ze39xdiiP$)c~TF*B%`J~(|0HvDL6}UXmC$AX=?Uck=2jS&$IIKgwo{Qg;dPIWDXvG zMpEM11PV^VaExN#`GO17fs{SQ=DebuB3m@@iB@cQBh_QnXofNT+W;eRTnE;5qA}H_ z{zc{+2E+#+k7QZLTNt<2>=yczPV0#hS6B%9kd3{-YcCyL!Ul_+k; zp0DzSG_^E_A8QO|+-h}b+8Fg8y{FmTOUkQv9CZ=kv~^}^Lc*}YvMAAFoY;}%=|t1SKU^tU@;Fdz77c@R*1UA%Eu_Y3tz$}<))Yv z-3pd<>}&eTERFRZ4--c09NoKpl|RsR_t%vKf8p&x_J~HJ$(xT zo$oS|$Z5$+e80CnXSF^*nB_j*ndQH~H}>(#(cCVJdOvxk)i2Uz0R!(`MoYbx$5G&7 zbo(K0GjYpF_4f8Q`5Qg(SozU`tNz$D_S>4h8^zdEijt6gb!toDK<=$u;A)fy!Y`h(--=f&9P7^^!9I4 zS6lByogoK?=((~MUPch>kn9^#u;HhK@Ta@&)yh^MbQOJm+xd0L-}4_>pNLW6Rf${5 zn7B7A{qPGNpZ|_4r;-v{iFBo2Xh>#+3G;|{E(gSV34$r{E%Y!9`-0-)9E>o*$tJE)bX!7Ho6BjelTH=PL$J?c1 ztcS}V=!m@Z84KnZ_Y8M`u0~g~R_#1AF0&rzJU`m7zjtE99k_*)p*%{1*Zcfkv+5{6 zR8O`>jGTCSrfnv=TK~22s0ZyRx?UYb5c6=Y2V+m39wWHzVWSoL=?cN?3qvg)87ltu zzh9h(5?Ru&?C-2D*OsZIjn3QJB{cg>n)^q~LN{gBOxrEsTCs>YYEnTcVzXZ93bGh0 zzrFi`%hISd+fZlbYVcBS(%oG>bOD1C|O-DZ0gNyeZ7| z;uZHP-86$>`0iguVGr-XX5S>WNe=s()OWpcM50@!i6`t8{wEP90XuQUY@7_G<` zMX#lM_TFlzVc))-lM#s5Zg<<^h7V_c#6Zqm$~pbCRB0|7%Gw{i@E|bt-6%$}x^(MT z-Qu_`f0Gqj1Z!{SyJoQ}*|2k|Qd9r z>FA|&f!~@IC8ok-O}>jCIHvPUqjzf_FCfwq>*MTI=x)nt{8ZFBRE`|3^>UW9s6CB; z+3O(2M`ZC5LsVYC9G_8Wt~q;JO}+n&`_z}P(KFTfvQsHJbB!C-1>(qCCFW<61a)mJ zfOn$ExQ(<4t2>BmWijT^^%ryB)2zikhe00HJII3zq&#Kg*d$5IyFLcNnc?t;1aLJx zoJ_6;UlJf$ak1T*Vm)Q#Qln0~QLgivXwu@@_BCrF#V znBPwJa3cfanoY|cE6bmaJ@LQff|QJNDw<>XEWazOvKK!+1>2;7y$rtLFc@1}@5Cq^ z5Oh90X3=-{nICF79SBuUIwF7??)EGxo70_>P)j;b9!5H#)Hl_1 zveiRRBuu#?H!9SwzaroBq4NfU|RlwHPbf$H1%Avp5&z!DSXF0SkUDbIQ zN5eK3l*mwE#zO9}ZBKBzYRHFZBD9^0n9#M-o()i^yOx{safWYyQzNa#c6BHxXF}Ja z=TzVIm+=V@q@7h6h!}-c`>^rC(p-m*cP7Kja zXnQ)RsZ+5HuBV_@p$s+68ylhGc+%wfh=gMoM5{ix-n#blmFo4>aIpk}F%GGZQlEpa z3wlgv9Xm2)uZ6;2Eq1EQT8`gz3s5uOC*K-(z)}<-x^bclCkeB(~dQ^KZsXG93)}Y@3u(u2tR+;ef__lr)x~c zs@fSnBczc+RPD}83uSqz>jXlZFN{@i>ojr-JW{l0sO6iiBS_stR=1xb>wi>xSK!1U zL2LdH!5&D=3-{dfOHz~^B!x^?4C$#}Wk7NbjRY>)20)`_Rah=>~yAj`Jqb1^m7l^l&k_AB-56!+#;7VCgf_!uFmkuic!7h?*gZt{)_2! z_c`Me!sS*?m7~g@XM9#~KLmRnseyl4WPAbV2*VohK#Q(PPtg}6O__^XUmF|M=ox4_ z=%bFpNYb{Qkrt(|FI1%aC8ceb?mr5CW!NDnU+Hzvvb;Ewb1u|6KlmS)o9_9`>qjTe z>2Sb~h^;ekty=tgZxaZOj%!mK5;{L2XR53y=c+y=nxBOWYq!@&;XMU942~iy{FPe6 zyH-AoE3SQaeEVrK6-tzPE!`imsL7?9x|XY91sBHJ&nU2JU$VP2Oh>`kp37-@+oHC zk8(4QJ%7dN;5ZA?yJa_tui}60sE-N^DUojZym`q_c#yXxfwn!* zgmwDa&!7ie?kNx?9j05>#_w!&(xPXh0=JQ?~35`m$S9)v?zSr?L@Jvwk6s^tZy$#4l(*9EbNxA;-XvBZZ zJM4>a9U6S-(>`mn37JGEW``@B@@_U>)9IcoaYH#25(MHuq%}@Bv>r`vc9*gF{Hzo_ ztrjf-1^GrHd~Esv&v8iJWujnQmEGS)rsXzTrWCmTT&M=K>$5pq^mZjKbwRcG_#i0{ zfE=s+z1^`uX62_(G^}}hy}enmDimbd2;eNfz$pLz)4-zJZ$tX;`ilQv992nE^em~| zsdDTeuea4{iRPJbZCL9w%c`!f9+@nk{Lvbh>j46?{%Ox(%zI{)9jQk@y9HHhY_?V= zmn%Y<`8vD8TfSzghk8XOtN3F~Yup}S-6pknm{tX~42Nu}xxeoOa&sZu%@1&;QipGa zr4Kt&cFsmQbnH4uJE2wwtYbbANF-d~*m4VBm-tpDZ^(?k zHMx6NMFkVJ4EgcPr!Y6`!COlszKg+~>58EXL+|?UM0ZK6SRzO3EfB(V#u>i9YB&iy z+jDO{rep=N-anpeGyg?7 z=xZ?-BKBfEud<()7eI@R0KDQUtRpZ6fVyUH}^g+ zA&J<5vq7al<_9Y_;67axeB6J>$cR-vbfP7u<>%lUw;YO}iBpPyexU4}#1II8f$xD* z$AX~q(iWkn%&HyJnf`ab-?)rD1Hk;|O1mWf`8kco9Id$V%C`-B3crMLH}0aRAIuQ4 ztolnVawp}4`j*=z8=5wP&H~yPNyaw}mLODimeiA#(LPLa1Ji&3C92VPegiJXgD9V{ zZP=Phl}l%rr8$E|jj4If7UYj_jSVQ1=PL-sX79Xsg>;iy_O`nGnp>UhN<&x(wq9ICsLozy*^cqcG&t(C$ zw(3eS6H>X((`ZL!e|Nv}%^m60idosmUy?UptWNLS35jbtq*4&9VH)fg4QGKFLxO$d zv0A1l;Rzf&M80I35<5}#UW z07qEE3+8>ihCzIrKDSjVSYl0kzd0V^s(hHis&i{~dP3QGuw3x>{GfETq3UI<0E++S zeScG4ZpkOQHS9cV=}930(7W#D6lmPP)D*^=r4hqRDFwI=mRYxi05;jOOES&ZOT9Jw z_}Uz2Vy+JR&;;ke@mjCl&50P-T^hT^dNMMt(_5HxoZA{70@?o>n`h3iK9?27D%{); z5l?FI{^1wEe#!dRu&)gzBq=$wj; z)iaW6`y2jzOc_i2*3r-dWV2@DpJ^Ya>d0TpbL3eNfRzC+LN9{H6b>TpMTfMz6PM$tC8tK0pe9(V zWZi{A+S51rlxIFrf+EDM06iHSw9ORnz`wauasS>?#Fa9pwquuw*@W?@sHY^sI_~}i zf}oipHjV&;k>wHRa<4Mz(Ie@Ke|7o!o9O;>34 zkKrMUHE3o}eR(nbFfi5OwW+4zR|;5y&C(Aij$p_OhEj1n*RbZOILHY*3;eWdQLLMF zIzIqXr-UP15p;Z4=lG5sJ|a+S{f`0U?8DygI7ok#>peObJ9_ox*Z{!*NTTR%I!8Jr zoohK=y(ypFaQtG1%>AaP-S--3_=Bu3aC~Or`dS>`r}LBijlc;X{AF#f5s)&^kDF(H zq63J2NT1$3#}L2nE4Gt|6DWm>PsYonhr1tA&mW0{Uv>`Pmp~YAh%l~BcSQ93e3!w* zCC9HnA;P8}!W8}1Xs~`a-4G?cF#=hbf{)M**Lgs$_&#EGnM7t(tl&Btk1>w>D)$sS zCJ`1#^!{;avbEK9q&!z=!WWsM;4#l~BO1HkuY-Inte?z+fBkC%xn#`vMJ>rS$9dpF}PO3F$M%_W;tTB|<`*t{}BY>T@%ppMeYVE~+I1h-1NxRg>A zg5`xDhwt((K#!h4BD{vHbfy*bUP2j>bZVPSi=GQU@XHJOIEX;Y=jZ3f73fqi3~Hbs zJ_v*!iFJDV`}gmk%UAppegu^@#0vx-|Fl)`jWBCcG+xhI$Wk&-tT`ukxAL_Nv&IR& z=tc7SPo5i0e7w`=s<^n|k9D0IGmaEgMs+O?Cok`{w#JRSN?4TVoIa!#J6o7X3c+NtLyj1m2J54Ku8NfltG`MtV znaa}wplZ*Z`o;I;)LyE!LlFt3=GB6X*25^3zua(;o(%7)7n>=Bz{7r9$kLJ*(zeYv zBYZg!s$ZR9A{?Jf#XI*wHYbwYy#1mR6_{AkouA{cF!s>rP(b)1q0|aLZrx|JApsah zY*OcV^+l(Pow|*r2WWdhp~nJYaEk%? zc(l&jd1Ip?ShzVxwAiKnL#p-ESB9yku(uPAh#?l5GUlDhSoN322%dY{mTZ{FZwcWRVk?*D1 zZD}EU3|?6}w%<>4@ooLK!~Clz7o(bcdGa_&gXAOOvnbNLKy5{fSDQSY$Khg08TCz9 zTi*ocTRh2E|2i8o%C?8av8LM5$JR2-vJ+)wxk^`8sE#>?=i+bHRzc42X}8u~ar43Eg8c3QY!gT3r0AERgA;z=Q0 z`NE*^n^T?j3fHmX3l|MKNnS^Dj`rBR*cdFc-_u1&qqi%&ZSQ(-P0B44ziabM(grE^ z!DiAeNi7D)na=C|qP5a!^1?e;uEXu+4>r`fk!P1@i=gO%o>vY;(}53Tu0M zToR*oBGz2u(Wj2Qa6;iNrxf}hh4;)P%5IHBU50lLPJNHfAr}G<^paXNj6c(#@6N7n zr$*>U*E|5%iz|Qmq418iOeb%$aEsr4k3D_16oWvrY!ecm-n0Gs>Ux>ZsfPBbV{pio z6YrISEJVD7dpGrPdg@skj-Ny1 z{7ovWvmTr*cA`t6Gw+`G`19@!xp>6y(>&spb^@U8{XIJT7(A5OpDQqQZ@OAYZm4KD3SsW($Znu zGtq6Y-#O7U$fsp!K@0bn!q&v7G!o7}XbhrvEw!%YbVb|I$Dz-Ae*L;Fsu=R@yjCHG zjS-pQi$pRBalV#}D6>jdUMnE)ezRxq^ldz&#(}6KbPGa>OuK0Q(FD=kO|CR;!RJ7N z!`7U$%OdyuG|C^}iGu#}&v5%a^IUEr{oSa9p3d-xYUg3D(U>#4UQC&EcLzKXp9#f< z#Cof#$TO}A;vNIHX$3oJ;n2(PPmN07yQG%}|FGx76v_FV!l_0)4-Y9 zbZ(yy2$hl0RkKsOEr@u@@})EoC@9erb-#b^YBISA>7}u#NA_lB@Mr5xrwnWd;ey+# z20lZ)kK|L4s8<2ycE2u6F4Fv`y26p~))|+6R=8Dr^*- z0)Ui`ajGj~xM}9rbq6JwpbIDxr z1LgG^!M34#v@h0uTAwQN9FW{JDl zwK;rq6m-JzND9E@Az(+%w_tn#IHZ%NLisRsJ^2RA7PXJ)!SoQFBne2v-nL^}5;Kwu z!LImtp6)0u6F^R$F#=ye1-}vOcS0%PJO>vawYnOtUl{A8dato{)<1Qb^%U#?-d(A^kiECYHjx0rcd9SJ7b{! z96pARQQ6}ZGRQC&#%4{n0d>qeBAz0K0tqb`SkvsiMas*105&;C05l&37UD#`U;plS zkz`u$=bdK2-wlW$YHw$20A>eHGz(~1xg|P4*m&JD{ty)8tw6bcSE_ zMU`0^_8k2*IdeY&@s&mUpaz!pDjj4V33WB}NdWpqm&*MT1praXlYM8j_^#2jHgpcjBpYOx2IWu@IqX04SL z9wn~mzqeyrKGm6S#(-oOGi5RRkt1Oq8HKNHKL)pdIVnRegA))!qI|@hXA!GQQdB1u zQ_Dp9vNZ6>>v*g*5QGXQnQ$E-e8(M|y;?CL8sP^!$YPvmu%YS}M1nI68ZIT}ipD_z z<6&R_HEiY+_&?mxSyv_W)t<$uuEafyjWcL~qm^d{zNFfORVpsdZ&4t2i|UnX^n zc@C8S32ILd(FYlJDXr?Jlkx;9RvFVAoLVAjYGtzh;(LJ{k}GNccJ<^JDh$}n6Fohq zwawFU91anWxNV7+{r$5?SXnQ;pGHwh8A>DMlzA9KA2ZWjW%u5XuA=^%L8%+hdNA+KUxJ+NQ^~; zTn9UHP(jy0^`zM4#$I$mj(&?!TSuq#wsw~CIzANk)`;z1NF&A3EomfU0tp(xzd z2ZcTVDiW>l`g~X*OFz&oTQJg7kzk+Sm8sM2upK1z0}1W_Bs2f_)jcE%Uk(~1PJpBb z;gB%@-1RSw06?<831r+ngG}CA#fXIes>*|VZUs2-Rx8-B;om}jF0P}m*bEr=Onkuk8g0E@GN8TH(>+$dxn|e`V?;qmYie-%1%QuKYo} zTHWJ?Mv(qiDu63;ZrM_cbD)O-WOK$3K5Gm~5*BR-9R~*&jIxmP5IZpcKlL$yg*J^v zz)AeA@PpaC=ZfDQxY6Vw(ZrJSs;e)5$+)5@cBlOOfi0X=TV&u^t3gWDPBoA=7(7ER zQ(FfL|KmeSgG(WWbN`Rs8w9{eDN`;Vv=|hht9ZJrw?>o5+so|!QQ4NeB${+x0%*88q;*~Y KT8Y}dfd2)j<)#Y& diff --git a/docs/apache-airflow-providers-http/img/connection_auth_type.png b/docs/apache-airflow-providers-http/img/connection_auth_type.png deleted file mode 100644 index 52eb584e5ccf6463273c9b0d35171944d451af9e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14199 zcmch8XIPWlwl2%HEh~tXqEt~qIwDQF282i_fCvGiAiYU%0u~|)Qlt}_l+cUx63P;! z6RNa8U?H85AP_(Zf%_%6_Bnf>bI)_{k9+^{0QtT-#~gFaG2eHLH}7?|)fhmmAUZla zhKCRpC>`BD5Ww%PXHEe>*-vEj)6q#QK2*7H;A2j}h4>qeWG!xxl`vB)-qyt4sieB$ zK!I^4{_6to9tVb=yr>fRj*oBX{%=hGK(XWx`L7FCMn$QE{n{j#%tES+jZ;z>wWBLy z)|Y&zJ5tO8c4S2zN-5sF;b`ymw>G~46VlOrV)=>wah3~pf%ezEbrt@9(0&^@tq1yz z&SF6ae1<#t0(k%VoqP=R-39QOm9E&Dc(-5w)ot1MK?$B)ipa)5&(OV0dve4x-+#J} zB|8atk~=s`G4{FqT4dbO+0(-W>d2h}*iREaZ1QOCh6GLh?;mc517iaZl3If2%-h1l zD4Cs1${fNWe1K1kTjTDRT-en3)V}zPm6+SKa(|U(%9$0!Mn{7)`&xTj;C)6W<=vi) zolM3Pbk9y2ote1PDHv>eF+1GCX)ZlyF|dG*IAL&+X6#VdTzezy?+)@swvKC{%slpb zrFbGljB3@g>)}~$r`{=%rzK~HPU*6J?cVp#x&9IDxvS}W-Ky%BiHEjpg<_xETUakU zI2MVy8Jw##>yf={`3_k4z}$f`*x4&8r*jp4zw*m)yAE56ToooKJ77@F?n<-w}U^m}xbX2m=)o+xvHvuLI6^WI#x3k`GC@K=GXzz0#)m5{mr@G%S7JkMK z>+txD-5Vq1_)jLoThg( z7wLwInsm|}hEu?K*|K}0JT(H*Y+nn#Sut@#s_yyOgQh~xw^=U9tmjg<8Q8ol9c+{K zMTKIe|7PSGi!!T5&3L#F`-{`mjZ)wp5#lDW5Lm!9g2}xZQ?lZ|-buazT4lagh8L;9 zbA~%W>ey@L?cIT5*=3dO-w3I~G}nveTmNDAMV`P_yz+T}?7Y)jZC2)BwUIoqYz1cL4D z?KR!qE9$)0IsdjnOH0cO*t*Sx_hsc?Jj2JM5F9he4m59bqhMIouVF&mt~f7Xu~Qm>K>&E8%#QO46*8mprfkZQUl<|REZ^47lDpg(M^^u zf6Y8J95}o`MmtpJZ9f#1mACh@$~l}ZE~{+*%L9f7(RNqG0X2ZD_s{uxSYj#L@>U|KplpF}}~g!L$O zDK#Zg?W3_^x|c-@>fGM*Vn(32k!f*+Vb^xB2&1$v5KeS-{I$?!3ya>OvWoVBr~KbW zFLu&4utgN%;qecr+5%z%_#`j{9o;>X62_s#2@$E(E6NT+#bS~`g1AhA)8;^&kVz$jkGX}Ji=?z^CT&{zT%^0!USH2>Qtg;DQEn9`?751^YuVZ5Pkes*W^W+@ zekpPiYrXMGGY$*{Om(ciqvKi8a*})`yUe9Dr|zB}!W`Z=SDCu@0U$>CB~~#XtG`~o z>JQ7)5f(dalOq?JcFZR$#uoIgRO#6C}*EwY6Dekdix!B-8D2XStOGu#Ak1 zmvyX8+jvs|Idj;iKdmYl-91^idA1#w1g+%2NcD_-c#zYh zTc_ql8GHuMvi^)VPPpfYOTs3#1MCkE&J4^{ zSohs_x9*qfSK2M|-9Ri6@XsX@;Zaxa;>*(g%A5C|kjRVJk;1_Rx|b0v-B3tqXy{rm z2a^cWDyv0pY0qHFRy8u~j;gq=M;aVtmlHwk2;!_4^`7qfTHTFVbv3w8K!K`d-2r{dE{w1Clu9ak=bM!;GTt()^PhF4HYISblQOftp!vINoYwDf)+a${@;ZIXgOtQK!!Pb^=J}}0 zv)HtS&P4MJyx!ZXv-OJdKNlX(U_IB)u(OFndp)T6jjj^z9S3eIFTW%1O*DmN+rmDD zyExFxkWo?H%uHg%zTcpbcp+m8ZmQB3|LU#aAB+6ik@}kaySD4fwN@^mBF6 zon2*ssI50D0OyIf={|}BJJ$$fx<7`oh+>BiP3}cPL0Tz4eLa%l@gh}A#jpfFmh#Vx z(K7DVlUpLDcU{i0kGmWl62{;JE23Caio8ty;g{b}uldICQZSr%9tY)i_ka0|jcz~@ zNH7bFi&(QFsNSng<*#90T*J=hirzjx&5T2=^81rYDUC1~L_6i~(|A!hrx~xwmfGat zPCU3Hf};Y@mVae``*7zW>7XT3ZA`-uv==0YEym0Q^rpYM>8F({bDN3_{T=}~jNf%6 z;xaRDx44cH69dwWf4-k{$oa8sX)&}@Gyxdey5GUcEW-d5b!tS`WwwQ@x5!wWK(ryk zpW-Yv4=tb`0X01nEHS)EuV0@a?+#>$CTk>!RKFWd0*P{NHrZ29SV&moB;@2c zrG;`ZX(Pku&k2_j9@JYvb~CQtm%W}y>=@ivm4%mf3BN7I+s-;{gk9Tx>{pyw^`0bu zfXLb2H$>*28QRej|3rcVi~XtrEVd((s{+;KPWyeXNpjUz2^tYp4f-Clt+)x0?{ zRiXGd=c7QKY$3$dpL=@h8J9aEYwp6)VU04I4B`jdV{72S2>-K821K|<&^~{iZMD}M zcXXs3hTG^lt|`@}FI8dkJ3Rb1I>J~SDk?8n?|p^Oh6Sv^wmRHv!wvD?wmMSoW@C5s zNTHG1(US}DV1ei4ju zH#pej(G+pW!x zXR877dLv8m!{6wZ?*dpDNT6HgnT(+^;<-ALn`&WdWGIsk-OG|V0H8ca>59?VP?S0V zMQ$BK5ol*mk9Zifh{T&y=PKznL)4HNj3-_a6W&81_-z_mzA!Wy148M_r78w=6E{&% zKGR(YfUQ1}5h&e8ILJ^knQcL9v4sMKpl^h*snRG3c;8Jzb@Z{ZrFD3!W~-m{db1?;dSA|vm6i1uo0R;c z11)k``}jXW?(lrc>d`AOOQ-NU7!~L(i|qmjxr>szEqc{69FtSuG!Z){f&5;e8WSY# zOA-SDithaI-y1pl*u>I4yqJIldmTgf_%gL~XgFxXtP@F0@`HgSlR~ko?zshD7$1G3 z5&=58;(>pA87IUYUMIPuji8Z>Oy%yQd|XE~S$<4fDhd0RS3ha|Q=H*gaa)_&(9~$bwnlupzK$? z;WNuh+8bEOIhp^7_fSK};X`p5Enppj5m7_W&gV}&1NPxETlpz{eV9vuy*&oky&8*( zk_yG%TU=O3lJh1e0@v?x9Qm0^j5d1W<2lGV0&gWAjV2mf+Jt9F#*ZeNSUQB4;e60l z7BoIs42=V01Q1Ao%y@3UXD~6dXnhnj{-+^I2tg|45eTk)`ppSP`E_cOGnM-tdJD@+ z+yBe}&VW@%A2qC;G%~&t4qWd5hwD?v9@`Bgkewwe+|1H4E8oTTc9o~+dj@!Meo=uG zQ(PRs1~*0Hp{Hk6qXMv#1Ic#1cA?g`lRf|eW z`V4MK`NhQbIuxmvH zO$>Y+FDMYu*VB6n@DwWDq8nI32_r5Yr=wp{+&TfK9V{Rhd!MFs4&b zUY-%ya3<{#=o}_mX@BYdR>e8*;7;5ps zMlmrlQQ=EWLV}H(yE_KJ9~j_{{@aQFOqtY88yz>XbPTVOT=bfA3DPVP^Gl{qoD^lB z6UE#_Zv$Gr>f0MBNBfC8-awBx?5qi@J9C*!Vs>7yj%r3E;JB_+uWQO3g~| zBmuL5tIVDHRGBK(ZaR;1)Tc&&y+`%TSJvZo%21v1IE&^<*$}{o7aV~90FGOt4rvii z+k<6xswax9${A$?4{95QfzJn6sCBRB)RT3AK1m4S=H{mP^yyO{YLpHL0)CkIPak?H zP&D34MPM+P**5CfrJs&7f&E!_>W+Q|^J;JpdTy8UVDH%<+5bA%Fp1HTL}^3rn=}hM zJ!lIddD8x(g&EzQFxu?L_F7XT$bv@h={kj|_6o_X^OHms4^ofM^rhfmPajv0o>9r` z%g3J>uOHXmDrjCgfx2*97D{5EdW@M5aeN(gInum&k6-2Z{d!I3y#38Yd5NgRf`bz_qLWC~Q0<*87VZJL?7Y zIX?<70QD`Mt&x5D*$Ls^DZhz>-Db+gG$-Py1X6(|OzSA$-$cG-IHh(yGgP)><(t#i zT8%$UzL~{lP5MaQu0y$BZaArOF=bEMQcB6=FvZDe&7HmGJ4rdADo$;$m+FEcGME}` zh%^Dj;euLoa`Tbxu=0|PhB@r{cuIev&+-AWvy08j`rG#`fTl8^B?u!9*AwJ0mT*G( zL}2y68rtAe>S0f;*`iYT3~MT(bFjOVJ>CR~W9p@2e+k@_Ldq3lH4J`+*f)e=B{Fi>M^c62Rh;ForEx zUk~y@S8wvAU!`uUADp&_1F5J1m=dRW9ZGUBS&KzWd01gR#t)Qt6!C1yF_9E< zTi%ELnHcZHfP_#&QPS!@FYLo|py8SP&$S6YI;0c{JgN%+wp4GBB8v6IFdBWE9LrzY zT0LkTTo8Du2=nEYrF@h2U&H_GpQ}OMFj^mP_Czz>+I0G&YdmH<{jh;}P(JUUJ;Wm< zX^Mq`_KdxS`k9qy_e2DuRQiXNW*U!>vvcXCEfDi0@6NQ*IqgZGjSRAx>)o-~N2AF> zwUEh(9utlYaNqbT9n59(G%((@m z+z7*tdvny!gKdR@x#h}buS3nx7nRt>ybD$_N?KFS8d-t*F%^LaV-wijH9aL-z>=D1 zk)I)_!Hu~v7-<#v+rDe5nU4Wo{uQ66?TTBaiqp@gO z_!1xD2U8k7TG;}aT2t4HpFI!#XLPhDEkWJ*-rQy$@AFu{oIjbbBL_Fn&s8Ji-Y+e5 zUnD3l^fRh^<8BMQFo5J~?t5F-j9)LvtQNys@dJr^SRn1d_!fZ7tew7>9pQrsUd)lx zrTqAQB~q~X=Dcw$6abfy+iCMyrQm;g5B!kP0 zE6FAv#;_Uj1-;|L{I_rXH8SQM&to>9FWFUhq!5fuOm&ogc%}t@bzoq{^_lIArN@XP zu==3<4>Dc^?J)A8*Dz{Xp6Ptg`RNZLWO5|eS|4*fc%_6*RUB$?s&@Fu3hcJ{Avs`Z z5Zc*j?oA25wMEcO{Nc(J2hQ|-)dDd}01LnZCxTkzEu0s%9Z*r3(Lc6Ex~|Frmozzd zsc=G$=$C%rC+o*m8HTiW+ryt|lyGRKlq}0-jpg-gBT~>)0p|JY+)O%1j%*8P?hM?m zrH^$eV8!FVSsWUGJwJE)(JR8>HGRE|r2F63ySI94{0Y85g-7BBKNF!Kr{^?A@kzG> z>zat-{uko|TV;z{;q-D4Elk7BeHQL7Xi5f`BOf$TEugu4rSDv4o% z+4$4d*1pSbn9;i)sBgqeq@QDGOn$O33;QTm6z){}MkjA%tlVBeo+P#Q6;^*9bgJsp(tD6<#s)d5T#sK3G-B@#SLY};&Yl1Z zR2$s1#oZ)GEkzVm?~w?$r(%dYZHcv$4tf@iys6sTX^4}p<-bci=4UilR57}RWE;)q z_CH>Kh($f@(@bo}E5~(JV=)DT1eu zFVp>c7s~T7J=D~R7L(79t=^pTFS)L+CgYASSaFpMd$3ga72=J1RdH^7DSN0DSDN;w z#D8s{11%S8-L?!6WA;ik*p3-Gi<5;pH*C*|`TN)0-JT*CIX=z&&Nt%E%zW0*$ud5@ zJjlj@e_pWA`SxTVo>ZW z2FEB*9~IR+*rS@OohGr~wB-eu_45hI)_uP7BH7z04vat`7Iz4fQgBV=v-U_fmQ)cw zzR&4(asfMQ#;|2}IWHmEQh){nIMgC8J|1gY=Qg34^0V&h{^kWT&NUJ}zm!f`;Njs} z972Uz_a;VS*lxTf>y%D9;_645I{YH`Q{=Ayt3d$loNW#1#g{hPQ#(^>fvXSQ30O)0 zCNGMw)pAdDuMe3}MS8x7_+t6rI-f zBW<|+N4jk-VbMMu>ET>9HW)quRgdQV*`sGifnJZoIE^aIP0-E)^+#=C%t-Dxrzbi_ zwqyeK4Ls`9ql;nHD!DGx4R&xfpgbY0L8@_vByy;uEBBH5U4?XyeXcsL_%hgTS0 z`q?+>l5CR}*xe_A6!)*Dq=f?a&ooOTXMn0`(~pKENrxPY2zfeaUBRES-VA@zguH`l zv+_H7dzN$RyL_WG{bIk)UZXBQ318ZSneT#+FDXcDC`>oIz{4hg2Z|Upno5r|2D)bN zK@rB{Hsjz2ky$5f2mXA;d+Sz*gtT<>jga$`Tw^86gF^8-9T8QC_g)v>Q13zS0koh1 zC-7GK?>hU>V9g#XVQXAwoJEh^HlM++ElZA$jv|x9tyLD@d8KdK1ZSh2A&CAz6Hbi} z-CNHHt;_^k3J3_S32`>U@uS7z&g(sIp^){Dp_h$TgHcgU zIs$r!Qmz&r=CJi7k;w+T7z(@O+%nF4=6&e3W=5p6OG)a<>`OWo8LHua!$Rf_e10iY zUmccbO49>Gw^RLByAG?rx(&sFF9nJkfbxsvT)(S)KT0~aHLRnAT}s7N`~^_yF)DTc z3I>v*z+Hpu%R7BQzmnQA+AgtL6Jk%zJ@F~1`PLdC<@eJ&HE2d?*^do1F4cstZYIIZ zt@cRrH=sQySKo^<`S_zF2)OSJ&gFSLCrz*TE-vtlO|01R&zqNuZ{tfbYsM9s!-{dN z^-b2Z!Gn81p?-y&X`#v!(IlNYR+!n3DIan=+$W-Qiq^h3sPCq&_6@v!jcLlr^({FRjDgaG4>(>3#VFungj6;q$)n?k}mkc>mPt(!u zbJDW7hc|z=F>BLoX|r{ZN{Aj6PzEMlmo9-vsz(ERiGhIDjRs5X-DX=(ZVkO;A!weD z6FMZS&qf}3j4{|PzAwqb;UvT#4$jr%7GzC8`MMjJpU(9GK~TN?3_M=3wLx|*PJ|yp z@)2-iOno%yw(-4f+0U;Lc|VJ+S7MG4qh~pM(!*kOwhF-a?jt3zeN7Mep?6ChxLy?3 z4xh*7u2))b{9K5hK9Avblhp^=c|+7H5=rbN+282y&CpUTf+&J$F-tZ$u^i$hKs;T2 zbmItU3$JKoY@b)>jyroOqMpAjY2V1nB<$yGGN#(Xbp$|rq?A*ESxq|=r{56}0MgD= z5EuxtZVW@|`UK<_0xdlM@1plOK;B&LBpG4YP8bDUysIe@0|`^XcUoZy8V&|^zi=M-s4A;sOX$*T1;1rsBG${FT<*_FLG(*32Fi^tP=L&u-1+kM0cR5J&DC$3AbA zb$lE)q*f7n@Oq9=qnb1$?fgMxjHvAjI6)EtE{+2uGqSRxAJ*?`uKU-@jlMp)MR+}V ziKw_JN-E0A$cIGhw0|fo9btMbVySq@hvm>oFJvy?v7=n4-c$n@Y4_an3KT-Gg%4>E z8MaK9^X`kkzSN&ylc1CMBWcfL?bc4&*47s0bkvsC7Ta7jy*3jNhp`(c3l6?$5k z=4VNUNo+k>&j$$>Cuidpr}Rd0H23Zxzhsm3#TEg~4KXQwn5*At^ORr0g=p#yC>zM$ z)$w3r6e-ktQ^>6RYYEGOb23q3czc^e#&$sWn@Ko-2Tq zA$-e&bQ?!(L2Om~3^>M6`*HLl4$^$MY2nJE78SH){#|tdF{7f)C;L+62s0BVpcOw6 zKQ7hgqcjE9)ZtP8YWI$F|;t2j#oW$&Oq3^0AN#jT~vUssBB; z0dO6)y)qH;Ki3ccPk|D3%KztR?EiieJ~W3?V6Qy6cX)Q9^ZE-_0OwKrHUQ?!_y%a3 zFn}ndwF2E%mZhc@7}!KXXCYeC6wt87!*{c#$xMQQ6+uczQx%IF9f#3(C}}=ZMKCl_ zeWVD%n|fmt5)?cRU$Ve&U*>&b?F?X^@2(X5@*ZUy=ZzW6Qp?1Xg5iP2_UZjVQ??}x zOX5qv2!KV({|wC_<+oRCX9uRCd7nQIOf>~eE`~jf>HXk7vNP#25&cDP<2G@)Y^VIkA9cy#;qZ$rlS`T)ceFnke z&wgAZ*ZL^}(vVOQqle|5J@LyVOtLfqpI1^MW!e71@o1Egqm^n;AHtFr)WW!UPpZgGLP%*k2QOYL6z{t2lUxYidIpQj^?iUYjo_aX z{qRV^cT>e9X*poT=6lNBz$rj31^q4=n|w&wPikm5EuNe$sPr!m<;F-1MlSsqh}L&O z#_i{4N*N)P6Jr6S?8juw18qAy(i>BR3JevMw(j1qXi1e5t;56(-kuAXaGYtvnIA69 zcU4~)bSNq=mIOpa@BX6z7}Xv8N|fQVRrItnlO7dg1~f*OJiid9eJB)51)`cqJ8aaB zmd(a8#iYjB$-^H`tN$6~)m<;D0-Enp!p55wl*u?-{8(wlZsGb|p0pY^6EdR|C(RW^ z%GjIy`s`x%?swF0qF6Vzu>f+hp7lZ~(@r@*Tw7{5M=P9DZEQru6}YG0^$p|AZU;3{ z$XkD0?#)D;2i;id?gzR!q`<#Z6+|g*yVaz|oA-s3B2xIfEptSl{>Ln{4<2$$f4-s+ zL=Xn;d5|CZcb_kXbM58>@+mKmBy%*-k_T`BpgR1#@G@&xyMGv?3LKWd{uQ2q>zmZ0 zZp$_U%Bui4aK0Th#H0&v0S(SNBDQK+F_$VX+F}+pe`Yg56?&DoMNi8ueSLp2?q{<~N}iH0>+K6G|4+q()-t?5Ww z0>uGfu%lK-X?6RJ?1xYg=Imwp3n6blX-o3|jbT zfKhMahF>F=qu)%;D$47R$$C!*4~{r+dIaUBZ1IGdUvmKRRTg>I=D;>V5trU9^|7h- z&1r^8)W^JMBn|#Jgv(7uHnkZ~Gt7WZG)aucV`VeJ%D*EW>zZgm_kSVYfv&{{kX!TRvA1pP#c1X0u5<26+ zC60q8_xoB9N-C%UToLdD$&T*70?eNenC#`;dDhxV_5!lq= zE)LqFDs>iQCH!*$yZLBnZI)=64-lCM3uZt^x1CE0cus#;6kgr?f;1Si(>O%559CIz zf7RF7so|L&*J)Q-qJpvM4ggkVMXOWmd+Xg?Ztqj_0LA0UW59NVRSM{W%{8ZEQ1}T{{VdS^2f@WOxT$~Ql_ET!Tq)Km$ zN6#}evjm9D&fWD(Yo_cMQsjnd_ZsZ@UK>V5Ex*$@SyB_1t85?O3X=bMvD8=@fU%v- zY9^;u(M&)=%-YA6io;Ccs^ZG35if4kQ}Xq59qT@kN6-5b9?df|jylGkSGwYq9@G6h z$3ibqE$DS-17HLJHxj|rvsZZo=9n)4SEc52eD_Z^ zzZ1~CL-I*RaiHU@M*9qC!8V3BRhX=I5X~2hrB4F)s~OEd_xOXSRndDZQM-rE?*KT1 zdr!+Au7UuN127v)JXls;fLh?fSJp5Nz5M`uN=Ii4^H=BQIH2}!2eJSK2lgIH5TUNF zP92f%LWDfE`@ZbD0w@bUA3z}?C~wq2mqXW$ zD7&nV57fdN$@8CIfG~MxLKUu1wF1V1LT_83oyCIL49K8=l+PK_V=8f7W?hvVH)s{k z>{3-ri|}ws)tyNB9I;B64xb@EBQux^dY7dDLH#Im@~C45P^{6{)qMi!!ZAQKtrtKN z7(g)HTjekY03stTZS4q5oS;#rY14~e-^v;P%GFc9QQTh7x@G?#-)S*5HFfOhp7jGC zUOyI4T#N^Y!a*o3q8Gyq3d;ac7TUETkWwEm`0k)Kod39eK9TaAy|Ap9ZONW0q5;0H z;<#r02MX(1q@c!mnab>I9vXtGs_BjI!XQVNppPH_)D_FFO;Qeg_^ZLrKG$AF{VI$z zAO$x#W2c$o9IsnoMw4wAa|7bib?~A!y&Y_xMcKK1fU5IQ-=w%0AW+483D4Pw^L@~L@K5TQ%5qoBkcfTgBV@d z8$1Db-em8ucP&ZgnD@H6MW10%5iF78Tzr9AdnM(4W>lAibruW97F-z$94=vZ z6Av+cIG@RI>(~jdVM;lkbEe_GW`G6H3&0S|rJdeOv{(6p-o)BhM>SVR!Oz;66iK}> zf#|S-o3FX?d<>IHq3v?;SK-6N(QtROi`lk9w66j^KQZUp1NBX{^;>p6jE~D`dr=X&N!8)Q|&Rw z-nmQl&ZSP#*HgWpo_pyQfT?22_wS@RiC&!@ki(wjZiBN{SwO2klxF<#}@Q|-ZX zGoR0{Iv(poI9Z%)4=jKc9lY+;sV%!nd+25#pyA^XegiCJ{^<8tXYX~N@jjOa96P#) M542TEz)xQPFTow~N&o-= diff --git a/docs/apache-airflow-providers-http/img/connection_headers.png b/docs/apache-airflow-providers-http/img/connection_headers.png deleted file mode 100644 index 413e9bbb38864faf0dd5664703b9089ff0b7bd5e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5256 zcmb7Ic{r5o`yXfn{-V})%5hC16>~W0*`i5pPc5~&!_Z*&@TQfZG(MG_BoO+`NXB8 z9cLtFwPB4YYQ)t9B8V#!9>!GTtzxn9z61NxOR26(i5#r%Yd8mX zpspJ8#$8$syuMdWkp$lPLfgu|3wA>n2RozP0kP;M(D-3DJV7{eSD?`7DFLe$svtVK zQXrI18VCt+$j?P)C&oNc2yK;mP)&FT>;wtK#y-+W5L@X#Lh$T(nAMSE;3&rnyAcEC z=B`9Y)31Gasqo@r&2Lc$Ksrplcbu4A)+;|xiiy6^5E1hE2n=@Smv23B#hBxd`M!T# z;oFxEnNSws+K+=nu|$kS1<~tp{^0_Dt>)tIEzT4=aOBWXNS{k_ety2;UKs4bA;@|O zqI_48U5@qP!-su9fX5X;*!ZA*_1&YXJa?|LQ*sBZ*nIV=EK&jnJF5asx!*D8 zqD|Tiyz1lgM;Zu9wrlLxa<^2*#&T(F&im)*@7WOyvr6k48XC^!Q6Dt=gTuhh;JSW! z(nzC#D9CHZ`xnj`gs`zBwwg({QRK?w@WqL7$}3N5ZJ|8t-809c6d6RJKWPi@wu&2x zcEr;P^YfK*^h2JGK+AEsXA3O%nX@t9lWTgD%@8mrgimB`Rxh%jl#)356_u5Rr}Hff z?kZUoZp_tbcLuA?4A$sUGj)_w4(KQjf$QcHa|a|64;?m;j&!dcTY@2ScBl@KMh*uy zaWRImoXw@-$oX-~>Tsk#A-+3jdAhG?1YE7D&CiQIA7B5B6iA|ZE=_j%Y;(4J-riS3 zzCK?Y?8CVhzotq4@DLf){qv5U(jL%R7dBit5tFhO#p1X{FQ4_;Y!)_YAF(n-l;GOZ z54)J*k0Tn12Z%>5n?=l&Q35k1hNBj#dRTh-(qK?$L7sVH;mw`Gp&QXVV0s4J)6kcP zj*}~sP)CV~x22ddX`Y|i<5PF7mTn3IloxEAW*u(6*i%(x&7R-&?tm0 zc^u>|u8Vn;W}COTY;i)gElS_{pk}(GerhFG)QC&e}LIYN^nK_x>Q`K6N@g{EVKwmRXK^9NZ>d z^HiGgx_1W4yW}9TxSH^Ygbw?&xJcU6R>z?bT%}2jLma$IRh2MOqZ71LnRK|p9M{JW zTX5+0%A?Yc`38ou2VZrDA~|&5x<%F%n~Y%OursJ-)Y7YY=%p)3vUlBA^97XL;cpAk zc(&GQ1>T3=j!M_wHJF$#QGbgZ7$D)Tv;p)1kgr^3(sXt&WWY)R!f?yFA<>ogRGAmHtO$}!F+-icSz0A;Zz(U~Nh3w5b9 z3lT|Ew2h5Tj9;7%s{yr*ZvrbxJI7E~#ci6CBa`P!UKp@s$;&)2y=T@5aP7IlEEP<4 zreFPNg5DMOrzkrGj>X()qMUWUWvZ=t;$AzPR&9RSvd^vQ8s=PG_iG+RVKD-8s+6=s zzmiAYm~Y6K7}Pf)>Uu6cDGy*Iv`Va0_-eZ_ZL}NiU+`>yQz}2aF^XKA7*0>vD|g>= zuY7vCrXSkPZG1xNMx#Rkt#I&Jg1qB310fI?_5Jkvn%G#*Mv1}xf(B5A4fY?lCT)^C zdw#zuKfx-nN&m!Jo+h5uHr$Sq!ygOsWGu_YtmPRtF06bv)$djH*vG+t4s@1wynw3; z`r)r5k<@I;;@T_c%1o&w&uSa?yVcpw5I1jH)u%~E9}UlqCF_C`zBH3n#&`N(LNLH>GgW*=?Jnd{y>_P zUfD%8If`a>6G9aex(h3Jp>gxKzSQKtBwCRGR=-mqnNfB|J#{3kS$nbjDHV@67Fq+# zja|k;FwrEzSQyLcVLu5#xa`IbM@&=ezkm89GeA7gw*9fxu|U*S-wj{Fl$e4&sAe}? zZStrst<#j1$2sQs$wcv3${l=1sl23{r6_%7FueHgPYRywmNI3VeazLgAor=W%-M@P zlk!}kaC+Pf?;d_8?x;dKvSG1fAUq@K(FodiIy$8+^RpO*J}|I&C(}v#9Viw{gB(Uhx^wD6z@c9;^~di z&VD}}jHPJVTg2isr^Fht`-aMn3>l99h`|eZbP$a2a$z4io4Br9$D6Vb(rxR?sorr0 zH+KrQr^I_ik&~_1YaZKG=&QIq^vpp?il5sNlhvLi&la6#)zG`it9-e!Uy#s)Jy=j_ z2ZG$PopwtK{sOuix%Gns#yG3_Umw^8p;7u_3zvif#+)?oCd}&7rOnI&1Iny#t z**6xC8=!a3^uJ;w%f@ln7NbX-;tk)sc#DV5MyL(3o2Lb(j~^10hwCAfje{riwU{uP z$(M~9@4>=8QWXD#YHJhVSS;43DQ=g~pS?CVVbh)vDwz3D1|h2@v9E2L%3@G@Zt&~~ zyt)2B!?Ez}vvUtHH7Yh4T4>VxyV7n5?8w#n-BUqZN3PDHK}WXhDZ*$AyA>jg&HX`n zM3fX1D4V1nnqKa6)6U_6E%X2va#y}WE5mQ0Ro_owJ5&8XmR1q6bv!H~oE&@E5Oky@SUzc)d}XKWJyRt_1;{R^`9_2p4PRVDHZO5d0FAFS3fT5Q*y zTI@98!ggX5`xK)dG)jX`_o)t9+~8e)p@W$Td!@Rap2@EK9{-p{d}~K?0G35(ogWJUxF=^gdUdFEF?O|X zWW2NbTK{jGS&Udp80iSWarKc(u-BNZukco-=@vl0Op55O@~SHEh|+X=c_@u?Cj%s> zD0XYwEr&=a1B^VJ=_B5m)UPb+-V#{67M-%7%$u?R0Pv@e9(^S)5DJ{?!I)4lzp2Cy zu*3MYMAKV>(qs+N)5sO%V6kFroqLcnHVR?^75TlzNjGI{9AH z?_c<|XX>QL%G^HTMM;&N>MeFa=B|(3)@Fa`%C`!7$p@M(mq&mM*ZfOB3F(dIfsgc;t6Edc<_QlVP9D>>+G z9(9a-`*sm8%$|EAAi~rBugTHw_wUnX__s%eV}BraTQF%V>uvWltjO2WGvA3dxIB`# z4G-^45R?i0p&MVq)CF}e*~W@6R$_#(sKai!0^@_q|>wch2;jFQ|)-Vb2mh+Ain#B!$aSK zgM%NxfB(MRZ>W|n^4EvyQ{Lj0enZ)Fq-_ox5-+~6@E)2cz{)=Tzg_tc)BZ1Z3{-_s zfV<`zR)@ctG;t4Od@A2E#Twi_4PhQ2LU4~K7~b^s?6YoAsJp>6Y=by@>0cbZva(Vy zbml|wSW6P**3uV71{~N}nJuoUsQ5;m_%9^@2pIJXO?#mJ`&B6TzIZWPx5g}3+T|0^ZBmz$t5%Ur1?3w22x^7u=(PV>RJdI`5- z^5^Wpb43lPll8hFUv~BI-zdBe2yHOv7u@*7&++0#R;vR_BVT|)XF5ik6MH~$bg;6r z;<`C&{WBWUsQwm}f+pItHnl}m?H-;4%917|w!CQhqGSC%M#99CRVu=2-ZvfdRh#nV zN?ooPuB$*F`~PZaU`p;UwSw5cL^-vBGu!X4KoNq$wxIO?Z{>})&HIxWyH+xol?#nJ NWnqUZF}w83zX2P`_&Wdq diff --git a/docs/apache-airflow-providers-http/img/connection_username_password.png b/docs/apache-airflow-providers-http/img/connection_username_password.png deleted file mode 100644 index 6e36e77dd4cb48f62107a3f654648587bddc58e7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4761 zcmchbcU03^o5!P|;P@gUqabyRBPa|Y0trZS6j4z?q$vUcqA-y*h?E3EU@S;qMnF0u z(g{VRlYmN5T9iFsdW&WcE+Gf+@j6Uopb@@XX=e3HCon2biA)vxaT*S!RJ3xgOLI`DzV z&2xKseS&`%p+IgvaCdjl&(1!j0f*;S(Sz>pj$4ZtC$~^wE!F+d(mk)c>gXq@hPy(@Cb59eAf4h1K$Ny zvvJ3q1Oy^s9O<$3lCe(nuy`l_}xUI_%)8AUWqzr*n+Y^pZitg6I zLeC6Vdf;g*nKB{9kSgh{rD%npDisdYg zw&r?`wq!pR5I#1!Ia65DJ<=ROmi8NpeRfDXd7>*VMgQd1EP~tCFV)(LekLrNK0(UJ z;64kL!JLjdt|rG;K@e?IRf<=N>*m|~;XX4xJwrpbb9|6jLlSbOdxrb-<@r)Hl|Lku zyO1VGn%Ip;hs?Xuv|qAGiK-`cS1CxX8)2p6e}u}Kw7*m#Z9=8Okk%#kX?ol(W)Ij@ zlH{JYR#{wXOxW4e(fJ+3JGGBc$+`i)#D>a>it$Cci%E-AKl-?<{o8xj(k&%2#4@|$ z&Y0hFb#>LdvNb;%qi-L$ooANSouuZnytG6=vz=grQ-eUp&667^+Ts;a&Eeyy+mplP z2)Z6>J#QV(@yaZ;E-`w%|Inw_`o#~ckGMri>-DNP@j*W-?5{1u(y{EEOQDCd z%aDe-g91V~+FvTQx9nnX%l#QJ+Y0W?mTJXJoTg+7bT^2j%$0sEm}s`W-f+_M>G&A(XNP6 zho85#oVVSEs>>*1(3=JGdmuMWvJbe)B$MM})-O$7P?K?$P{UZzf@bkm%iDH!_FaQn zhEX?C;GRb0924Wmp0aTPh@kax+&-9Ua5Tg5Qs|rG5W{YvgBwrWhnDlebNt z9hk5$ax&VOrDic^M?xu%yB!#9bM%BUKTL%7=;$PNdwnv3k%%U1^n}gMY*br3(fIgG zsC`o*#JFQM-+%GF069VFZaBj@1(vPlJu#LcZ<4x(=s#+iw1*#Zp(Y+$s)gOKpy^Mi zhh1JEh{`|oY+<3$n<1MnKkys2Z9Ux)@ z%BUKpVONr&Z|;;k8*i)*dNQ|f3*??JoyK&EP}~>~JsA>|8_DWt1a3eeJ-N5|VI}jl zGMR+Q*(I5YLM_6Y15OHtRX9Q~gnh||_Pq@YxK;hNF=yGk z|BgOMZ*$IA0Jii?4~WJH?1^JPu2w2Qf;Vo4H_G)Dd=BXh=iVR}^TRHM+Dgw^$;8e` zP$DG*Rzz~=_oft=tOs#9Y;VtPoe-f5S=C96a!oeMBG2Z&RVeC*f6spIFLmbFo?@NX z-$(3iy%C@2DEf0N_~e%_5<`V$yEaGhHzJNH86j{SAA4;j8F&G|99)9ZRFv$`dcdBi zu9uI*4r}Vk6o&h_d#IO&T@D>(j-DTBFx|o~vD0mqTH;;PV4m3X-3Q_Y2ELXyjh<)exI zAI=l-``$WPgJko zCOa?C2+evBNE_aoFXhY&d7nehVw^muQmF~`&aPISZY>J6*5|qedh&Tac5AJy9(QH( zm0IPMsCqSB_E1ax)UWj`*Btm3XMM|gi9PO4$$nVHgXxSCWr!5{^Jx6wl2xuh-mjxl z>sJSySG1r52UEhrsa5?&_IB9Q zkOnAEI3>)e{ID|3v6d+A3&aAo0YTqdcwwrOj>!D3vE=2>L>6#hcMp(G$SV=x=see* zezA@FgKI(7{j0FEvooJSAk=~-`Y5$jtM)OXL5uDODAYAE75gF&V31wiqv=~Re=1=G z$f=$(BhUK2el;TiLuQO3QAjt*f0xSq=BS^D*ykxWI+KMvgkV9i;VDWRx9}jNfL%DaGjA|W{vQ$NpS3m4=I-6QJZ}L}~wP_n}m)v?XSj1Nx_PDiK^uj29eT?=~Mn-f>J)|u$Y0Q?t#MSwG^ghl$AI*YQh zPGZ4#`&YAys;bE{_ed{f(V#TgRDd`-wlxn=(T4zRi8N6v&%LPE2D@vdn9hvyJOK)0 zo5bNQwNxHk*?5Vn)$syP6ln|k`s~Ko5m3|5!!&g$uh4iW@cxxT)+;#8UzfUEN6*nH zv?yuiGB};hJ>V9!_DASU zA8w_fYPQUpz`4E(xJHivGHtDklg(P_KGgdyG^%6DtOoA5MV+o&AQo=N*DjlLvwkSu zGld5EG;sN|JgWQ6HPoknI{`GAsz0FpaO|ptUZB5(wy)cH05C!;>Dbi)^kQzahB0rOS2)^^!pMBGqd>KLLc`E3qWiy%&Z<__TUz@VEwbOwXrxKJ5u3YvNnYXhWJ7bY-C*PQ~_Dzw@4q zcEWY^tbhD%ABytj1gg=ULuR4O$zh`M%fx*LCniZXf$I-xzzlPdW0oDL2W@J&Z{NOk z$17TQhaYX%CTXaz%nr&Mr=BHx4{sWI&DHu?bJMNu(Wi{(rNl`=i|iWidjZ+8*_hcn zTwe!ktI{1fRmN8>#Q2FIXJe)P=pUDckPlW`Vq_64XHt3^j*Y=&;I`3a(iH&Xk_p(& zRZbq87O-IFSJJoAH)Dyoi{Du~p#oD;jJFbo1WPg__Q4!Y$a6zDX6NJSxqW8aS7EDRj9^xD&NQP6Uc9bgIU$EzSuy7XkUDeWJ73?Nt>GqqxAGePKGm z5R9MD)q&%jw?d1gs+`u}vRGu(?95C154u4x6d?7k=NC@JpLJXuexIL}mF2ABTxfv- z*xT6+5O2abH*6Z%;@j@Ch9He$Y&03*q@cESMj*l(~?m8SC=FfD5o|Ix9xc zsD!xj0DRn^TkDCQ2pXG$oMrmnPw4oHL(KIfaA_uK=i|9Tm-f@spV3u+%1~@-kPTXM z-1b%NI^M9!GB+J35wTwy-(h+ShP3?`9cn=y1(?>As)tqJN~zsf7_rHMBRAgFb9$Z$ zp|w9G&miaP*bk;e`$`ZX9ZflcFmH-%&o|G_5jko22F%Ej4`_{%HO;ybVCW39*?lDqn1(u<{qD}X_YW2lcjDZtPyo^xHQn2o zV|6)CWJLAN{p;M+`8lb3EggwO?8X=;Mh~+xz~*eSC$A*D7JJ_@a`|AyqMX6$TUst1 z-FH@|vZjY>ywQlQSk){!=dO3DdTBA}DsKmgj0#@{qC13d(_6)PxV|>P53KaFMIeum z#4>N@@x$y~{zh2+mOga1(hc=BBI)8+aCBm&f};?~S92I1gGFyw-@CR!mgOcs{v9mw zSgXNV(bgh+f`Wij{0agP0++y&v?a~&`%-~MAxp`hbPV!2D#7{mX21p%0%^%#S95Ea&2 zqmQilODU|X*w>#nCFptQWHS?)&D*z2t6lTnPS;E>`|9+mBR5z>5m-R2$vMVp$@58? zC)nXvUdvIDk7z0aZ4iF6t&|#3ZEs`7Mj;%!@ot)dR zs}K|U;waI$QPms)nQDDfWZ3Y19^^!(0-7FonL^qnn`Ig1NNf9!4JdmBb|h<|m+$G} z>NBCyG2cr6D5(*fov^^Dmgl~HEB>&fF>K${(N2r99r$UmauCfl({Zv~mmEp-e^j_J z1lg+J(TCUF{4mwW6U5E=&e{GZ$=7-u7Wv&+Y%^LZMLr`C+K}Y*^~2NV!phd@<90tm zrJ#vn@XMEf6^%GjRyfl%wR+Q*2 zIy+?1^PStpdc&<^FC^2Ikm9U=TSukp{J%jB&r7>^DgT*=YK7eXx}-UiZLwfo1*9JQ2&Vsd b1GBE*^^2vH0QXn|ulCwy6NBQ54!`{!fAs15 From 6a51249ebad03affecfd891ca993917b43689691 Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 31 Jan 2025 21:17:39 +0100 Subject: [PATCH 207/286] refactor: Fixed tests in TestHttpHook --- .../provider_tests/http/hooks/test_http.py | 40 +++++++++---------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index a31aa92a31133..f1fe8e033e9c9 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -74,7 +74,7 @@ def __init__(self, username: str, password: str, endpoint: str): @mock.patch.dict( - "os.environ", AIRFLOW__HTTP__EXTRA_AUTH_TYPES="tests.providers.http.hooks.test_http.CustomAuthBase" + "os.environ", AIRFLOW__HTTP__EXTRA_AUTH_TYPES="provider_tests.http.hooks.test_http.CustomAuthBase" ) class TestHttpHook: """Test get, post and raise_for_status""" @@ -390,7 +390,7 @@ def test_connection_without_host(self, mock_get_connection): assert hook.base_url == "http://" @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") - @mock.patch("providers.tests.http.hooks.test_http.CustomAuthBase.__init__") + @mock.patch("provider_tests.http.hooks.test_http.CustomAuthBase.__init__") def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_connection): auth.return_value = None conn = Connection( @@ -418,12 +418,12 @@ def test_available_connection_auth_types(self): "requests.auth.HTTPDigestAuth", "requests_kerberos.HTTPKerberosAuth", "aiohttp.BasicAuth", - "tests.providers.http.hooks.test_http.CustomAuthBase", + "provider_tests.http.hooks.test_http.CustomAuthBase", } ) @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") - def test_connection_with_invalid_auth_type_get_skipped(self, mock_get_connection, caplog): + def test_connection_with_invalid_auth_type_get_skipped(self, mock_get_connection): auth_type: str = "auth_type.class.not.available.for.Import" conn = Connection( conn_id="http_default", @@ -431,53 +431,51 @@ def test_connection_with_invalid_auth_type_get_skipped(self, mock_get_connection extra=f'{{"auth_type": "{auth_type}"}}', ) mock_get_connection.return_value = conn - HttpHook().get_conn({}) - assert f"Skipping import of auth_type '{auth_type}'." in caplog.text + with pytest.warns(RuntimeWarning, match=f"Skipping import of auth_type '{auth_type}'."): + HttpHook().get_conn({}) - @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_auth_types") @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") - @mock.patch("providers.tests.http.hooks.test_http.CustomAuthBase.__init__") - def test_connection_with_extra_header_and_auth_type(self, auth, mock_get_connection, mock_get_auth_types): + @mock.patch("provider_tests.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_header_and_auth_type(self, auth, mock_get_connection): auth.return_value = None conn = Connection( conn_id="http_default", conn_type="http", login="username", password="pass", - extra='{"headers": {"x-header": 0}, "auth_type": "providers.tests.http.hooks.test_http.CustomAuthBase"}', + extra='{"headers": {"x-header": 0}, "auth_type": "provider_tests.http.hooks.test_http.CustomAuthBase"}', ) mock_get_connection.return_value = conn - mock_get_auth_types.return_value = frozenset(["providers.tests.http.hooks.test_http.CustomAuthBase"]) session = HttpHook().get_conn({}) auth.assert_called_once_with("username", "pass") assert isinstance(session.auth, CustomAuthBase) assert "auth_type" not in session.headers assert "x-header" in session.headers + assert session.headers["x-header"] == 0 - @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_auth_types") @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") - @mock.patch("providers.tests.http.hooks.test_http.CustomAuthBase.__init__") + @mock.patch("provider_tests.http.hooks.test_http.CustomAuthBase.__init__") def test_connection_with_extra_auth_type_and_no_credentials( - self, auth, mock_get_connection, mock_get_auth_types + self, auth, mock_get_connection ): auth.return_value = None conn = Connection( conn_id="http_default", conn_type="http", - extra='{"headers": {"x-header": 0}, "auth_type": "providers.tests.http.hooks.test_http.CustomAuthBase"}', + extra='{"headers": {"x-header": 0}, "auth_type": "provider_tests.http.hooks.test_http.CustomAuthBase"}', ) mock_get_connection.return_value = conn - mock_get_auth_types.return_value = frozenset(["providers.tests.http.hooks.test_http.CustomAuthBase"]) session = HttpHook().get_conn({}) auth.assert_called_once() assert isinstance(session.auth, CustomAuthBase) assert "auth_type" not in session.headers assert "x-header" in session.headers + assert session.headers["x-header"] == 0 @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") - @mock.patch("providers.tests.http.hooks.test_http.CustomAuthBase.__init__") + @mock.patch("provider_tests.http.hooks.test_http.CustomAuthBase.__init__") def test_connection_with_string_headers_and_auth_kwargs(self, auth, mock_get_connection): """When passed via the UI, the 'headers' and 'auth_kwargs' fields' data is saved as string. @@ -490,7 +488,7 @@ def test_connection_with_string_headers_and_auth_kwargs(self, auth, mock_get_con password="pass", extra=""" {"auth_kwargs": {\r\n "endpoint": "http://localhost"\r\n}, - "headers": ""} + "headers": {"x-header": 0}} """, ) mock_get_connection.return_value = conn @@ -499,9 +497,9 @@ def test_connection_with_string_headers_and_auth_kwargs(self, auth, mock_get_con session = hook.get_conn({}) auth.assert_called_once_with("username", "pass", endpoint="http://localhost") - assert "auth_kwargs" not in session.headers - assert session.headers["Content-Type"] == "application/json" - assert session.headers["X-Requested-By"] == "Airflow" + assert "auth_type" not in session.headers + assert "x-header" in session.headers + assert session.headers["x-header"] == 0 @pytest.mark.parametrize("method", ["GET", "POST"]) def test_json_request(self, method, requests_mock): From 899f3eb501906bc40d99020f14fd78f10d333f3e Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 31 Jan 2025 21:43:07 +0100 Subject: [PATCH 208/286] refactor: Reformatted TestHttpHook --- providers/http/tests/provider_tests/http/hooks/test_http.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index f1fe8e033e9c9..1296033d439cf 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -456,9 +456,7 @@ def test_connection_with_extra_header_and_auth_type(self, auth, mock_get_connect @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") @mock.patch("provider_tests.http.hooks.test_http.CustomAuthBase.__init__") - def test_connection_with_extra_auth_type_and_no_credentials( - self, auth, mock_get_connection - ): + def test_connection_with_extra_auth_type_and_no_credentials(self, auth, mock_get_connection): auth.return_value = None conn = Connection( conn_id="http_default", From 0ad8f1a7a708b9e13128c6eb7a2426488af50309 Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 31 Jan 2025 21:44:51 +0100 Subject: [PATCH 209/286] refactor: Ignore method type --- .../livy/src/airflow/providers/apache/livy/hooks/livy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py index ae5ef805eba22..bf406dcf9fb9d 100644 --- a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py @@ -130,7 +130,7 @@ def run_method( if not self.extra_options: self.extra_options = {"check_response": False} - back_method = self.method + back_method = self.method # type: ignore self.method = method try: if retry_args: @@ -510,7 +510,7 @@ async def _do_api_call_async( return await self.run_method( endpoint=endpoint or "", - method=self.method, + method=self.method, # type: ignore data=data, headers=headers, extra_options=extra_options, @@ -537,7 +537,7 @@ async def run_method( if method not in ("GET", "POST", "PUT", "DELETE", "HEAD"): return {"status": "error", "response": f"Invalid http method {method}"} - back_method = self.method + back_method = self.method # type: ignore self.method = method try: result = await super().run( From b04e8590fed3caece1a3ed1f01eb35458e31f66c Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 31 Jan 2025 21:57:28 +0100 Subject: [PATCH 210/286] refactor: Changed http conn type to HTTP --- .../http/tests/provider_tests/http/hooks/test_http.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index 1296033d439cf..7fd65984369a5 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -50,22 +50,22 @@ def aioresponse(): def get_airflow_connection(conn_id: str = "http_default"): - return Connection(conn_id=conn_id, conn_type="http", host="test:8080/", extra='{"bearer": "test"}') + return Connection(conn_id=conn_id, conn_type="HTTP", host="test:8080/", extra='{"bearer": "test"}') def get_airflow_connection_with_extra(extra: dict): def inner(conn_id: str = "http_default"): - return Connection(conn_id=conn_id, conn_type="http", host="test:8080/", extra=json.dumps(extra)) + return Connection(conn_id=conn_id, conn_type="HTTP", host="test:8080/", extra=json.dumps(extra)) return inner def get_airflow_connection_with_port(conn_id: str = "http_default"): - return Connection(conn_id=conn_id, conn_type="http", host="test.com", port=1234) + return Connection(conn_id=conn_id, conn_type="HTTP", host="test.com", port=1234) def get_airflow_connection_with_login_and_password(conn_id: str = "http_default"): - return Connection(conn_id=conn_id, conn_type="http", host="test.com", login="username", password="pass") + return Connection(conn_id=conn_id, conn_type="HTTP", host="test.com", login="username", password="pass") class CustomAuthBase(HTTPBasicAuth): From 3574a804026b7d753b2e140e48f807496a5a9b3f Mon Sep 17 00:00:00 2001 From: David Blain Date: Mon, 3 Feb 2025 11:10:10 +0100 Subject: [PATCH 211/286] Revert "refactor: Changed http conn type to HTTP" This reverts commit b04e8590fed3caece1a3ed1f01eb35458e31f66c. --- .../http/tests/provider_tests/http/hooks/test_http.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index 7fd65984369a5..1296033d439cf 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -50,22 +50,22 @@ def aioresponse(): def get_airflow_connection(conn_id: str = "http_default"): - return Connection(conn_id=conn_id, conn_type="HTTP", host="test:8080/", extra='{"bearer": "test"}') + return Connection(conn_id=conn_id, conn_type="http", host="test:8080/", extra='{"bearer": "test"}') def get_airflow_connection_with_extra(extra: dict): def inner(conn_id: str = "http_default"): - return Connection(conn_id=conn_id, conn_type="HTTP", host="test:8080/", extra=json.dumps(extra)) + return Connection(conn_id=conn_id, conn_type="http", host="test:8080/", extra=json.dumps(extra)) return inner def get_airflow_connection_with_port(conn_id: str = "http_default"): - return Connection(conn_id=conn_id, conn_type="HTTP", host="test.com", port=1234) + return Connection(conn_id=conn_id, conn_type="http", host="test.com", port=1234) def get_airflow_connection_with_login_and_password(conn_id: str = "http_default"): - return Connection(conn_id=conn_id, conn_type="HTTP", host="test.com", login="username", password="pass") + return Connection(conn_id=conn_id, conn_type="http", host="test.com", login="username", password="pass") class CustomAuthBase(HTTPBasicAuth): From 40145f873b5633ba02853b8d4711d4cba7c499b4 Mon Sep 17 00:00:00 2001 From: David Blain Date: Mon, 3 Feb 2025 11:28:38 +0100 Subject: [PATCH 212/286] refactor: Fixed AsyncLivyHook --- .../airflow/providers/apache/livy/hooks/livy.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py index bf406dcf9fb9d..815435fa597d2 100644 --- a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py @@ -25,6 +25,7 @@ from enum import Enum from typing import Any +import aiohttp import requests from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning @@ -540,12 +541,14 @@ async def run_method( back_method = self.method # type: ignore self.method = method try: - result = await super().run( - endpoint=endpoint, - data=data, - headers=headers, - extra_options=extra_options or self.extra_options, - ) + async with aiohttp.ClientSession() as session: + result = await super().run( + session=session, + endpoint=endpoint, + data=data, + headers=headers, + extra_options=extra_options or self.extra_options, + ) except HttpErrorException as e: status, message = str(e).split(":", 1) return {"Response": {message}, "Status Code": {status}, "status": "error"} From 115102fa01602360386d21da581d4aadf6f4a3f2 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Sun, 12 Nov 2023 13:55:18 +0100 Subject: [PATCH 213/286] feat: Implement `auth_kwargs` parameter in Http Connection --- .../provider_tests/http/hooks/test_http.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index 82a1ff9765156..e1a380e4588e5 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -67,6 +67,11 @@ def get_airflow_connection_with_login_and_password(conn_id: str = "http_default" return Connection(conn_id=conn_id, conn_type="http", host="test.com", login="username", password="pass") +class CustomAuthBase(HTTPBasicAuth): + def __init__(self, username: str, password: str, endpoint: str): + super().__init__(username, password) + + class TestHttpHook: """Test get, post and raise_for_status""" @@ -352,6 +357,25 @@ def test_connection_without_host(self, mock_get_connection): hook.get_conn({}) assert hook.base_url == "http://" + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_connection): + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + login="username", + password="pass", + extra='{"auth_kwargs": {"endpoint": "http://localhost"}}', + ) + mock_get_connection.return_value = conn + + hook = HttpHook(auth_type=CustomAuthBase) + session = hook.get_conn({}) + + auth.assert_called_once_with("username", "pass", endpoint="http://localhost") + assert "auth_kwargs" not in session.headers + @pytest.mark.parametrize("method", ["GET", "POST"]) def test_json_request(self, method, requests_mock): obj1 = {"a": 1, "b": "abc", "c": [1, 2, {"d": 10}]} From 5a2edbebe391b64a189cd63585c5bd9b0bf0e8fe Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 24 Nov 2023 00:25:53 +0100 Subject: [PATCH 214/286] fix: Correctly use auth_type from Connection --- .../provider_tests/http/hooks/test_http.py | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index e1a380e4588e5..802dd0ef15c39 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -366,7 +366,7 @@ def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_conne conn_type="http", login="username", password="pass", - extra='{"auth_kwargs": {"endpoint": "http://localhost"}}', + extra='{"x-header": 0, "auth_kwargs": {"endpoint": "http://localhost"}}', ) mock_get_connection.return_value = conn @@ -375,6 +375,40 @@ def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_conne auth.assert_called_once_with("username", "pass", endpoint="http://localhost") assert "auth_kwargs" not in session.headers + assert "x-header" in session.headers + + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_header_and_auth_type(self, auth, mock_get_connection): + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + login="username", + password="pass", + extra='{"x-header": 0, "auth_type": "tests.providers.http.hooks.test_http.CustomAuthBase"}', + ) + mock_get_connection.return_value = conn + + session = HttpHook().get_conn({}) + auth.assert_called_once_with("username", "pass") + assert isinstance(session.auth, CustomAuthBase) + assert "auth_type" not in session.headers + assert "x-header" in session.headers + + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_auth_type_and_no_credentials(self, auth, mock_get_connection): + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + extra='{"auth_type": "tests.providers.http.hooks.test_http.CustomAuthBase"}', + ) + mock_get_connection.return_value = conn + + HttpHook().get_conn({}) + auth.assert_called_once() @pytest.mark.parametrize("method", ["GET", "POST"]) def test_json_request(self, method, requests_mock): From c77aea1c933b2d31fa96f3301c6f7fa5bbb6f900 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 24 Nov 2023 01:55:44 +0100 Subject: [PATCH 215/286] feat: Add Connection documentation --- providers/http/src/airflow/providers/http/hooks/http.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index b22a01f8283db..eb63e27ee0925 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -51,6 +51,12 @@ class HttpHook(BaseHook): """ Interact with HTTP servers. + To configure the auth_type, in addition to the `auth_type` parameter, you can also: + * set the `auth_type` parameter in the Connection settings. + * define extra parameters used to instantiate the `auth_type` class, in the Connection settings. + + See :doc:`/connections/http` for full documentation. + :param method: the API method to be called :param http_conn_id: :ref:`http connection` that has the base API url i.e https://www.google.com/ and optional authentication credentials. Default From 987a5807a38c7900cd4c76e5533489d38717dce8 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 5 Dec 2023 23:14:43 +0100 Subject: [PATCH 216/286] feat: Make available auth_types configurable from airflow config --- providers/http/docs/configurations-ref.rst | 0 providers/http/docs/index.rst | 1 + providers/http/provider.yaml | 15 ++++++++++ .../provider_tests/http/hooks/test_http.py | 28 ++++++++++++++++++- 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 providers/http/docs/configurations-ref.rst diff --git a/providers/http/docs/configurations-ref.rst b/providers/http/docs/configurations-ref.rst new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/providers/http/docs/index.rst b/providers/http/docs/index.rst index fd873092b915e..099c4213c4b10 100644 --- a/providers/http/docs/index.rst +++ b/providers/http/docs/index.rst @@ -42,6 +42,7 @@ :maxdepth: 1 :caption: References + Configuration Python API <_api/airflow/providers/http/index> .. toctree:: diff --git a/providers/http/provider.yaml b/providers/http/provider.yaml index dee0796c04891..7a991044ea801 100644 --- a/providers/http/provider.yaml +++ b/providers/http/provider.yaml @@ -93,3 +93,18 @@ triggers: connection-types: - hook-class-name: airflow.providers.http.hooks.http.HttpHook connection-type: http + +config: + http: + description: "Options for Http provider." + options: + extra_auth_types: + description: | + A comma separated list of auth_type classes, which can be used to + configure Http Connections in Airflow's UI. This list restricts which + classes can be arbitrary imported, and protects from dependency + injections. + type: string + version_added: 4.8.0 + example: "requests_kerberos.HTTPKerberosAuth,any.other.custom.HTTPAuth" + default: ~ diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index 802dd0ef15c39..fe6b8f882f2b7 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -36,7 +36,7 @@ from airflow.exceptions import AirflowException from airflow.models import Connection -from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook +from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook, get_auth_types @pytest.fixture @@ -72,6 +72,9 @@ def __init__(self, username: str, password: str, endpoint: str): super().__init__(username, password) +@mock.patch.dict( + "os.environ", AIRFLOW__HTTP__EXTRA_AUTH_TYPES="tests.providers.http.hooks.test_http.CustomAuthBase" +) class TestHttpHook: """Test get, post and raise_for_status""" @@ -377,6 +380,29 @@ def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_conne assert "auth_kwargs" not in session.headers assert "x-header" in session.headers + def test_available_connection_auth_types(self): + auth_types = get_auth_types() + assert auth_types == frozenset( + { + "request.auth.HTTPBasicAuth", + "request.auth.HTTPProxyAuth", + "request.auth.HTTPDigestAuth", + "tests.providers.http.hooks.test_http.CustomAuthBase", + } + ) + + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + def test_connection_with_invalid_auth_type_get_skipped(self, mock_get_connection, caplog): + auth_type: str = "auth_type.class.not.available.for.Import" + conn = Connection( + conn_id="http_default", + conn_type="http", + extra=f'{{"auth_type": "{auth_type}"}}', + ) + mock_get_connection.return_value = conn + HttpHook().get_conn({}) + assert f"Skipping import of auth_type '{auth_type}'." in caplog.text + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") def test_connection_with_extra_header_and_auth_type(self, auth, mock_get_connection): From dc03c44b048582e55fd7c144e69ddde686eb450f Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Wed, 6 Dec 2023 22:24:25 +0100 Subject: [PATCH 217/286] feat: Add fields for auth config and header config in Http Connection form --- .../src/airflow/providers/http/hooks/http.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index eb63e27ee0925..31a0bc5cfc8a5 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -113,6 +113,30 @@ def auth_type(self): def auth_type(self, v): self._auth_type = v + @classmethod + def get_connection_form_widgets(cls) -> dict[str, Any]: + """Return connection widgets to add to connection form.""" + from flask_babel import lazy_gettext + from wtforms.fields import SelectField, TextAreaField + + auth_types_choices = frozenset({""}) | get_auth_types() + return { + "auth_type": SelectField( + lazy_gettext("Auth type"), + choices=[(clazz, clazz) for clazz in auth_types_choices] + ), + "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs")), + "extra_headers": TextAreaField(lazy_gettext("Extra Headers")) + } + + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + """Return custom field behaviour.""" + return { + "hidden_fields": ["extra"], + "relabeling": {} + } + # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: From 7977c40926505d9970a02adb777684f3779f9691 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Thu, 7 Dec 2023 08:48:01 +0100 Subject: [PATCH 218/286] fix: Correctly apply styling to extra fields --- .../http/src/airflow/providers/http/hooks/http.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 31a0bc5cfc8a5..538bc1e6e38cb 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -116,6 +116,7 @@ def auth_type(self, v): @classmethod def get_connection_form_widgets(cls) -> dict[str, Any]: """Return connection widgets to add to connection form.""" + from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget, Select2Widget from flask_babel import lazy_gettext from wtforms.fields import SelectField, TextAreaField @@ -123,19 +124,17 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: return { "auth_type": SelectField( lazy_gettext("Auth type"), - choices=[(clazz, clazz) for clazz in auth_types_choices] + choices=[(clazz, clazz) for clazz in auth_types_choices], + widget=Select2Widget(), ), - "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs")), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers")) + "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), + "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } @classmethod def get_ui_field_behaviour(cls) -> dict[str, Any]: """Return custom field behaviour.""" - return { - "hidden_fields": ["extra"], - "relabeling": {} - } + return {"hidden_fields": ["extra"], "relabeling": {}} # headers may be passed through directly or in the "extra" field in the connection # definition From 408ff506fd898bac58947a405b00781f95022da3 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 19 Dec 2023 01:05:47 +0100 Subject: [PATCH 219/286] feat: Implement simplistic collapsable textarea for "extra" --- airflow/www/forms.py | 26 ++++++++++++++++++- airflow/www/static/js/connection_form.js | 4 +-- .../src/airflow/providers/http/hooks/http.py | 9 +++---- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/airflow/www/forms.py b/airflow/www/forms.py index 7028e2026e449..b69184c6590c2 100644 --- a/airflow/www/forms.py +++ b/airflow/www/forms.py @@ -33,6 +33,7 @@ from flask_appbuilder.forms import DynamicForm from flask_babel import lazy_gettext from flask_wtf import FlaskForm +from markupsafe import Markup from wtforms import widgets from wtforms.fields import Field, IntegerField, PasswordField, SelectField, StringField, TextAreaField from wtforms.validators import InputRequired, Optional @@ -176,6 +177,29 @@ def populate_obj(self, item): field.populate_obj(item, name) +class BS3AccordionTextAreaFieldWidget(BS3TextAreaFieldWidget): + + @staticmethod + def _make_collapsable_panel(field: Field, content: Markup) -> str: + collapsable_id: str = f"collapsable_{field.id}" + return f""" +
+
+

+ +

+
+ +
+ """ + + def __call__(self, field, **kwargs): + text_area = super(BS3TextAreaFieldWidget, self).__call__(field, **kwargs) + return self._make_collapsable_panel(field=field, content=text_area) + + @cache def create_connection_form_class() -> type[DynamicForm]: """ @@ -223,7 +247,7 @@ def process(self, formdata=None, obj=None, **kwargs): login = StringField(lazy_gettext("Login"), widget=BS3TextFieldWidget()) password = PasswordField(lazy_gettext("Password"), widget=BS3PasswordFieldWidget()) port = IntegerField(lazy_gettext("Port"), validators=[Optional()], widget=BS3TextFieldWidget()) - extra = TextAreaField(lazy_gettext("Extra"), widget=BS3TextAreaFieldWidget()) + extra = TextAreaField(lazy_gettext("Extra"), widget=BS3AccordionTextAreaFieldWidget()) for key, value in providers_manager.connection_form_widgets.items(): setattr(ConnectionForm, key, value.field) diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index d039fc7275462..1c97e00803174 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -83,7 +83,7 @@ function restoreFieldBehaviours() { } ); - Array.from(document.querySelectorAll(".form-control")).forEach((elem) => { + Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach((elem) => { // eslint-disable-next-line no-param-reassign elem.placeholder = ""; elem.parentElement.parentElement.classList.remove("hide"); @@ -101,7 +101,7 @@ function applyFieldBehaviours(connection) { if (Array.isArray(connection.hidden_fields)) { connection.hidden_fields.forEach((field) => { document - .getElementById(field) + .querySelector(`label[for='${field}']`) .parentElement.parentElement.classList.add("hide"); }); } diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 538bc1e6e38cb..7268c13f8764d 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -25,6 +25,8 @@ import tenacity from aiohttp import ClientResponseError from asgiref.sync import sync_to_async +from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget +from markupsafe import Markup from requests.auth import HTTPBasicAuth from requests.models import DEFAULT_REDIRECT_LIMIT from requests_toolbelt.adapters.socket_options import TCPKeepAliveAdapter @@ -128,14 +130,9 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: widget=Select2Widget(), ), "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), + "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()) } - @classmethod - def get_ui_field_behaviour(cls) -> dict[str, Any]: - """Return custom field behaviour.""" - return {"hidden_fields": ["extra"], "relabeling": {}} - # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: From d0c4155786045b91363176642aa5e5b8473fbb28 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Wed, 20 Dec 2023 16:21:08 +0100 Subject: [PATCH 220/286] fix: express clearly empty frozenset creation Goal is to have an empty default choice --- providers/http/src/airflow/providers/http/hooks/http.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 7268c13f8764d..0f6bea66bf624 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -122,7 +122,8 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: from flask_babel import lazy_gettext from wtforms.fields import SelectField, TextAreaField - auth_types_choices = frozenset({""}) | get_auth_types() + default_auth_type = frozenset({""}) + auth_types_choices = default_auth_type | get_auth_types() return { "auth_type": SelectField( lazy_gettext("Auth type"), From c86c136fb9134ed97b433d4abc48e3c4ad95d986 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Wed, 20 Dec 2023 16:43:29 +0100 Subject: [PATCH 221/286] feat: Refactor Accordion TextArea to use wtform utils --- airflow/www/static/js/connection_form.js | 12 +++++++----- .../http/src/airflow/providers/http/hooks/http.py | 4 +--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index 1c97e00803174..119fe39daae54 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -83,11 +83,13 @@ function restoreFieldBehaviours() { } ); - Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach((elem) => { - // eslint-disable-next-line no-param-reassign - elem.placeholder = ""; - elem.parentElement.parentElement.classList.remove("hide"); - }); + Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach( + (elem) => { + // eslint-disable-next-line no-param-reassign + elem.placeholder = ""; + elem.parentElement.parentElement.classList.remove("hide"); + } + ); } /** diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 0f6bea66bf624..77563d2eb1a81 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -25,8 +25,6 @@ import tenacity from aiohttp import ClientResponseError from asgiref.sync import sync_to_async -from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget -from markupsafe import Markup from requests.auth import HTTPBasicAuth from requests.models import DEFAULT_REDIRECT_LIMIT from requests_toolbelt.adapters.socket_options import TCPKeepAliveAdapter @@ -131,7 +129,7 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: widget=Select2Widget(), ), "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()) + "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } # headers may be passed through directly or in the "extra" field in the connection From 7f7aa3104ffb6c5520a7cb85f1bdd7a28372c7a9 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Thu, 21 Dec 2023 13:04:19 +0100 Subject: [PATCH 222/286] feat: Implement 'collapse_extra' field behavior --- airflow/customized_form_field_behaviours.schema.json | 4 ++++ providers/http/src/airflow/providers/http/hooks/http.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/airflow/customized_form_field_behaviours.schema.json b/airflow/customized_form_field_behaviours.schema.json index 78791a87886c1..fa5ace958c5e8 100644 --- a/airflow/customized_form_field_behaviours.schema.json +++ b/airflow/customized_form_field_behaviours.schema.json @@ -22,6 +22,10 @@ "additionalProperties": { "type": "string" } + }, + "collapse_extra": { + "type": "boolean", + "description": "Collapse the 'Extra' field." } }, "additionalProperties": true, diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 77563d2eb1a81..2ddd9ca434efe 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -132,6 +132,10 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + return {"hidden_fields": [], "relabeling": {}, "collapse_extra": True} + # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: From f8e7c03d8041056efae5421a2fa2aa7eebb75d82 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Sat, 30 Dec 2023 16:43:47 +0100 Subject: [PATCH 223/286] feat: Implement parameterizable behavior for collapsible field --- ...stomized_form_field_behaviours.schema.json | 19 +++++-- airflow/www/static/css/connection.css | 23 +++++++++ airflow/www/static/js/connection_form.js | 49 ++++++++++++++++--- .../www/templates/airflow/conn_create.html | 2 +- airflow/www/templates/airflow/conn_edit.html | 1 + airflow/www/webpack.config.js | 1 + .../src/airflow/providers/http/hooks/http.py | 2 +- 7 files changed, 85 insertions(+), 12 deletions(-) create mode 100644 airflow/www/static/css/connection.css diff --git a/airflow/customized_form_field_behaviours.schema.json b/airflow/customized_form_field_behaviours.schema.json index fa5ace958c5e8..8aa05945ebb01 100644 --- a/airflow/customized_form_field_behaviours.schema.json +++ b/airflow/customized_form_field_behaviours.schema.json @@ -23,9 +23,22 @@ "type": "string" } }, - "collapse_extra": { - "type": "boolean", - "description": "Collapse the 'Extra' field." + "collapsible_fields": { + "description": "List of collapsed fields for the hook, with their properties.", + "type": "object", + "patternProperties": { + "\"^.*$\"": { + "description": "Name of the field to enable collapsing.", + "type": "object", + "properties": { + "expanded": { + "description": "Set the default state of the field as expanded.", + "default": true, + "type": "boolean" + } + } + } + } } }, "additionalProperties": true, diff --git a/airflow/www/static/css/connection.css b/airflow/www/static/css/connection.css new file mode 100644 index 0000000000000..78edf0db5d4dc --- /dev/null +++ b/airflow/www/static/css/connection.css @@ -0,0 +1,23 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +.panel-invisible { + margin: 0; + border: 0; +} diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index 119fe39daae54..5c60638caf488 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -83,13 +83,28 @@ function restoreFieldBehaviours() { } ); - Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach( - (elem) => { - // eslint-disable-next-line no-param-reassign - elem.placeholder = ""; - elem.parentElement.parentElement.classList.remove("hide"); - } - ); + Array.from(document.querySelectorAll(".form-control")).forEach((elem) => { + // eslint-disable-next-line no-param-reassign + elem.placeholder = ""; + elem.parentElement.parentElement.classList.remove("hide"); + }); + + Array.from(document.querySelectorAll(".form-panel")).forEach((elem) => { + elem.parentElement.parentElement.classList.remove("hide"); + + elem.classList.add("panel-invisible"); + const panelHeader = elem.children[0]; + panelHeader.classList.add("hidden"); + panelHeader.firstElementChild.firstElementChild.setAttribute( + "aria-expanded", + "true" + ); + + const collapsible = elem.children[1]; + collapsible.setAttribute("aria-expanded", "true"); + collapsible.classList.add("in"); + collapsible.style.height = null; + }); } /** @@ -122,6 +137,26 @@ function applyFieldBehaviours(connection) { document.getElementById(field).placeholder = placeholder; }); } + + if (connection.collapsible_fields) { + Object.entries(connection.collapsible_fields).forEach((entry) => { + const [field, properties] = entry; + + const collapsibleController = document.getElementById( + `control_collapsible_${field}` + ); + const panelHeader = collapsibleController.parentElement.parentElement; + panelHeader.classList.remove("hidden"); + panelHeader.parentElement.classList.remove("panel-invisible"); + + if (properties.expanded === false) { + const collapsible = document.getElementById(`collapsible_${field}`); + collapsible.classList.remove("in"); + collapsible.setAttribute("aria-expanded", "false"); + collapsibleController.setAttribute("aria-expanded", "false"); + } + }); + } } } diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index ac92b967f7e34..307450b05d16b 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -25,7 +25,7 @@ - + {# required for codemirror #} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 653b0dd3ce07f..11ebd6c4cb436 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -25,6 +25,7 @@ + {# required for codemirror #} diff --git a/airflow/www/webpack.config.js b/airflow/www/webpack.config.js index 9d5800f783f50..ad1a7098e0803 100644 --- a/airflow/www/webpack.config.js +++ b/airflow/www/webpack.config.js @@ -60,6 +60,7 @@ const config = { airflowDefaultTheme: `${CSS_DIR}/bootstrap-theme.css`, connectionForm: `${JS_DIR}/connection_form.js`, chart: [`${CSS_DIR}/chart.css`], + connection: [`${CSS_DIR}/connection.css`], dag: `${JS_DIR}/dag.js`, dagDependencies: `${JS_DIR}/dag_dependencies.js`, dags: [`${CSS_DIR}/dags.css`, `${JS_DIR}/dags.js`], diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 2ddd9ca434efe..192e57c1d70fc 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -134,7 +134,7 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: @classmethod def get_ui_field_behaviour(cls) -> dict[str, Any]: - return {"hidden_fields": [], "relabeling": {}, "collapse_extra": True} + return {"hidden_fields": [], "relabeling": {}, "collapsible_fields": {"extra": {"expanded": False}}} # headers may be passed through directly or in the "extra" field in the connection # definition From 96b0b2a5186ed35128eff52bc5c9f88d841b49f7 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 5 Jan 2024 06:48:16 +0100 Subject: [PATCH 224/286] revert: Remove collapsible field --- ...stomized_form_field_behaviours.schema.json | 17 -------- airflow/www/static/css/connection.css | 23 ----------- airflow/www/static/js/connection_form.js | 39 +------------------ .../www/templates/airflow/conn_create.html | 1 - airflow/www/templates/airflow/conn_edit.html | 1 - airflow/www/webpack.config.js | 1 - .../src/airflow/providers/http/hooks/http.py | 4 -- 7 files changed, 1 insertion(+), 85 deletions(-) delete mode 100644 airflow/www/static/css/connection.css diff --git a/airflow/customized_form_field_behaviours.schema.json b/airflow/customized_form_field_behaviours.schema.json index 8aa05945ebb01..78791a87886c1 100644 --- a/airflow/customized_form_field_behaviours.schema.json +++ b/airflow/customized_form_field_behaviours.schema.json @@ -22,23 +22,6 @@ "additionalProperties": { "type": "string" } - }, - "collapsible_fields": { - "description": "List of collapsed fields for the hook, with their properties.", - "type": "object", - "patternProperties": { - "\"^.*$\"": { - "description": "Name of the field to enable collapsing.", - "type": "object", - "properties": { - "expanded": { - "description": "Set the default state of the field as expanded.", - "default": true, - "type": "boolean" - } - } - } - } } }, "additionalProperties": true, diff --git a/airflow/www/static/css/connection.css b/airflow/www/static/css/connection.css deleted file mode 100644 index 78edf0db5d4dc..0000000000000 --- a/airflow/www/static/css/connection.css +++ /dev/null @@ -1,23 +0,0 @@ -/*! - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -.panel-invisible { - margin: 0; - border: 0; -} diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index 5c60638caf488..d039fc7275462 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -88,23 +88,6 @@ function restoreFieldBehaviours() { elem.placeholder = ""; elem.parentElement.parentElement.classList.remove("hide"); }); - - Array.from(document.querySelectorAll(".form-panel")).forEach((elem) => { - elem.parentElement.parentElement.classList.remove("hide"); - - elem.classList.add("panel-invisible"); - const panelHeader = elem.children[0]; - panelHeader.classList.add("hidden"); - panelHeader.firstElementChild.firstElementChild.setAttribute( - "aria-expanded", - "true" - ); - - const collapsible = elem.children[1]; - collapsible.setAttribute("aria-expanded", "true"); - collapsible.classList.add("in"); - collapsible.style.height = null; - }); } /** @@ -118,7 +101,7 @@ function applyFieldBehaviours(connection) { if (Array.isArray(connection.hidden_fields)) { connection.hidden_fields.forEach((field) => { document - .querySelector(`label[for='${field}']`) + .getElementById(field) .parentElement.parentElement.classList.add("hide"); }); } @@ -137,26 +120,6 @@ function applyFieldBehaviours(connection) { document.getElementById(field).placeholder = placeholder; }); } - - if (connection.collapsible_fields) { - Object.entries(connection.collapsible_fields).forEach((entry) => { - const [field, properties] = entry; - - const collapsibleController = document.getElementById( - `control_collapsible_${field}` - ); - const panelHeader = collapsibleController.parentElement.parentElement; - panelHeader.classList.remove("hidden"); - panelHeader.parentElement.classList.remove("panel-invisible"); - - if (properties.expanded === false) { - const collapsible = document.getElementById(`collapsible_${field}`); - collapsible.classList.remove("in"); - collapsible.setAttribute("aria-expanded", "false"); - collapsibleController.setAttribute("aria-expanded", "false"); - } - }); - } } } diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index 307450b05d16b..fb3e188949b66 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -25,7 +25,6 @@ - {# required for codemirror #} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 11ebd6c4cb436..653b0dd3ce07f 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -25,7 +25,6 @@ - {# required for codemirror #} diff --git a/airflow/www/webpack.config.js b/airflow/www/webpack.config.js index ad1a7098e0803..9d5800f783f50 100644 --- a/airflow/www/webpack.config.js +++ b/airflow/www/webpack.config.js @@ -60,7 +60,6 @@ const config = { airflowDefaultTheme: `${CSS_DIR}/bootstrap-theme.css`, connectionForm: `${JS_DIR}/connection_form.js`, chart: [`${CSS_DIR}/chart.css`], - connection: [`${CSS_DIR}/connection.css`], dag: `${JS_DIR}/dag.js`, dagDependencies: `${JS_DIR}/dag_dependencies.js`, dags: [`${CSS_DIR}/dags.css`, `${JS_DIR}/dags.js`], diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 192e57c1d70fc..77563d2eb1a81 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -132,10 +132,6 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } - @classmethod - def get_ui_field_behaviour(cls) -> dict[str, Any]: - return {"hidden_fields": [], "relabeling": {}, "collapsible_fields": {"extra": {"expanded": False}}} - # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: From 326f2ddd3794832be80e63b752b2dbfb646db40b Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 5 Jan 2024 08:47:09 +0100 Subject: [PATCH 225/286] fix: set the default value for "auth_type" as empty string SelectField expects a string as value. The default of select choice cannot be None. --- providers/http/src/airflow/providers/http/hooks/http.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 77563d2eb1a81..9cf5d4983dbc5 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -120,13 +120,14 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: from flask_babel import lazy_gettext from wtforms.fields import SelectField, TextAreaField - default_auth_type = frozenset({""}) - auth_types_choices = default_auth_type | get_auth_types() + default_auth_type: str = "" + auth_types_choices = frozenset({default_auth_type}) | get_auth_types() return { "auth_type": SelectField( lazy_gettext("Auth type"), choices=[(clazz, clazz) for clazz in auth_types_choices], widget=Select2Widget(), + default=default_auth_type ), "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), From 69a78e513bcc58f50b62f8cf22962106f7de6ce8 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 5 Jan 2024 08:48:11 +0100 Subject: [PATCH 226/286] fix: Use Livy hook to test invalid extra removal --- tests/www/views/test_views_connection.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/www/views/test_views_connection.py b/tests/www/views/test_views_connection.py index f9a4efd11c15b..1e21dc4856ed1 100644 --- a/tests/www/views/test_views_connection.py +++ b/tests/www/views/test_views_connection.py @@ -459,8 +459,12 @@ def test_process_form_invalid_extra_removed(admin_client): """ Test that when an invalid json `extra` is passed in the form, it is removed and _not_ saved over the existing extras. + + Note: This can only be tested with a Hook which does not have any custom fields (otherwise + the custom fields override the extra data when editing a Connection). Thus, this is currently + tested with livy. """ - conn_details = {"conn_id": "test_conn", "conn_type": "http"} + conn_details = {"conn_id": "test_conn", "conn_type": "livy"} conn = Connection(**conn_details, extra='{"foo": "bar"}') conn.id = 1 From e99fa2b7f5345413b48181281593eb84994dd861 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Mon, 8 Jan 2024 16:40:49 +0100 Subject: [PATCH 227/286] feat: Implement CodeMirrorField for providers --- airflow/config_templates/default_webserver_config.py | 7 +++++++ airflow/utils/json.py | 10 ++++++++++ airflow/www/app.py | 3 +++ airflow/www/templates/airflow/conn_create.html | 1 + airflow/www/templates/airflow/conn_edit.html | 1 + providers/http/provider.yaml | 3 +-- setup.cfg | 0 7 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 setup.cfg diff --git a/airflow/config_templates/default_webserver_config.py b/airflow/config_templates/default_webserver_config.py index 4ad8ee6743f39..85b9d4d2c8dbb 100644 --- a/airflow/config_templates/default_webserver_config.py +++ b/airflow/config_templates/default_webserver_config.py @@ -35,6 +35,13 @@ WTF_CSRF_ENABLED = True WTF_CSRF_TIME_LIMIT = None +# Flask CodeMirror config +CODEMIRROR_LANGUAGES = ["javascript"] +# CODEMIRROR_THEME = '3024-day' +# CODEMIRROR_ADDONS = ( +# ('ADDON_DIR','ADDON_NAME'), +# ) + # ---------------------------------------------------- # AUTHENTICATION CONFIG (specific to FAB auth manager) # ---------------------------------------------------- diff --git a/airflow/utils/json.py b/airflow/utils/json.py index a8846282899f3..9622f4c0a6b30 100644 --- a/airflow/utils/json.py +++ b/airflow/utils/json.py @@ -123,5 +123,15 @@ def orm_object_hook(dct: dict) -> object: return deserialize(dct, False) +def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> dict | None: + """Safely loads JSON. + + Returns None by default if the given object is None. + """ + if obj is not None: + return json.loads(obj) + return default + + # backwards compatibility AirflowJsonEncoder = WebEncoder diff --git a/airflow/www/app.py b/airflow/www/app.py index 2656045e84ba5..c85d82793f678 100644 --- a/airflow/www/app.py +++ b/airflow/www/app.py @@ -22,6 +22,7 @@ from flask import Flask from flask_appbuilder import SQLA +from flask_codemirror import CodeMirror from flask_wtf.csrf import CSRFProtect from markupsafe import Markup from sqlalchemy.engine.url import make_url @@ -127,6 +128,8 @@ def create_app(config=None, testing=False): csrf.init_app(flask_app) + CodeMirror(flask_app) + init_wsgi_middleware(flask_app) db = SQLA() diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index fb3e188949b66..8e3d8db0d5e00 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -26,6 +26,7 @@ {# required for codemirror #} +{{ codemirror.include_codemirror() }} {% endblock %} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 653b0dd3ce07f..174bfa164c4c4 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -26,6 +26,7 @@ {# required for codemirror #} +{{ codemirror.include_codemirror() }} {% endblock %} diff --git a/providers/http/provider.yaml b/providers/http/provider.yaml index 7a991044ea801..bb0214e7c6e66 100644 --- a/providers/http/provider.yaml +++ b/providers/http/provider.yaml @@ -102,8 +102,7 @@ config: description: | A comma separated list of auth_type classes, which can be used to configure Http Connections in Airflow's UI. This list restricts which - classes can be arbitrary imported, and protects from dependency - injections. + classes can be arbitrary imported to prevent dependency injections. type: string version_added: 4.8.0 example: "requests_kerberos.HTTPKerberosAuth,any.other.custom.HTTPAuth" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000000..e69de29bb2d1d From 48d60407419ecc7f6d3a43272aa2375cc3ff5905 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Mon, 8 Jan 2024 19:21:34 +0100 Subject: [PATCH 228/286] revert: Remove CodeMirror from providers --- airflow/config_templates/default_webserver_config.py | 7 ------- airflow/www/app.py | 3 --- airflow/www/templates/airflow/conn_create.html | 1 - airflow/www/templates/airflow/conn_edit.html | 1 - 4 files changed, 12 deletions(-) diff --git a/airflow/config_templates/default_webserver_config.py b/airflow/config_templates/default_webserver_config.py index 85b9d4d2c8dbb..4ad8ee6743f39 100644 --- a/airflow/config_templates/default_webserver_config.py +++ b/airflow/config_templates/default_webserver_config.py @@ -35,13 +35,6 @@ WTF_CSRF_ENABLED = True WTF_CSRF_TIME_LIMIT = None -# Flask CodeMirror config -CODEMIRROR_LANGUAGES = ["javascript"] -# CODEMIRROR_THEME = '3024-day' -# CODEMIRROR_ADDONS = ( -# ('ADDON_DIR','ADDON_NAME'), -# ) - # ---------------------------------------------------- # AUTHENTICATION CONFIG (specific to FAB auth manager) # ---------------------------------------------------- diff --git a/airflow/www/app.py b/airflow/www/app.py index c85d82793f678..2656045e84ba5 100644 --- a/airflow/www/app.py +++ b/airflow/www/app.py @@ -22,7 +22,6 @@ from flask import Flask from flask_appbuilder import SQLA -from flask_codemirror import CodeMirror from flask_wtf.csrf import CSRFProtect from markupsafe import Markup from sqlalchemy.engine.url import make_url @@ -128,8 +127,6 @@ def create_app(config=None, testing=False): csrf.init_app(flask_app) - CodeMirror(flask_app) - init_wsgi_middleware(flask_app) db = SQLA() diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index 8e3d8db0d5e00..fb3e188949b66 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -26,7 +26,6 @@ {# required for codemirror #} -{{ codemirror.include_codemirror() }} {% endblock %} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 174bfa164c4c4..653b0dd3ce07f 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -26,7 +26,6 @@ {# required for codemirror #} -{{ codemirror.include_codemirror() }} {% endblock %} From 6c82276d6eb86b6da06feb5c4daab48d7bef879f Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Mon, 8 Jan 2024 20:47:50 +0100 Subject: [PATCH 229/286] feat: Add documentation --- airflow/utils/json.py | 2 +- .../img/connection_auth_kwargs.png | Bin 0 -> 9623 bytes .../img/connection_auth_type.png | Bin 0 -> 14199 bytes .../img/connection_headers.png | Bin 0 -> 5256 bytes .../img/connection_username_password.png | Bin 0 -> 4761 bytes 5 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 docs/apache-airflow-providers-http/img/connection_auth_kwargs.png create mode 100644 docs/apache-airflow-providers-http/img/connection_auth_type.png create mode 100644 docs/apache-airflow-providers-http/img/connection_headers.png create mode 100644 docs/apache-airflow-providers-http/img/connection_username_password.png diff --git a/airflow/utils/json.py b/airflow/utils/json.py index 9622f4c0a6b30..eb3cd40941197 100644 --- a/airflow/utils/json.py +++ b/airflow/utils/json.py @@ -123,7 +123,7 @@ def orm_object_hook(dct: dict) -> object: return deserialize(dct, False) -def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> dict | None: +def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> Any: """Safely loads JSON. Returns None by default if the given object is None. diff --git a/docs/apache-airflow-providers-http/img/connection_auth_kwargs.png b/docs/apache-airflow-providers-http/img/connection_auth_kwargs.png new file mode 100644 index 0000000000000000000000000000000000000000..7023c3a7a072f965f9dd053f77c5a5d64b396f47 GIT binary patch literal 9623 zcmcI~2{_c>+jom(n~(ixI5hS2r&k-xWWM^15qC|U8&@yI zp4VbIoNz&&;S|%$vllM#zw{D3bLeCYFTFO4DogYQZ3Q#Q(MB#`@|PP{w$;0sjC|gQ zwZ00``u#3Nh5MrM%WwU6MOC!GEa0bjzF~#+9_k$RUt@1t_`%yIg(IZq`(>tL>=B>- z{$n_3=<@gXa@sCopHqhy8ve(KHqs61*aGb{L`~`|6OlkDb>1ArkdEJf(u%A*3sMN|@KU37UWK z`S!;+d!t~o=`Y)wx6bxn)?k%F?VSe!d^L0ZZ$r)AA%p$*$)o=^r})M@ajS#xngony z*Lg06eEks5LtY{{Vqn>ksZNFGX*0{b@+ z#0=Qog@Hj;9>jhwU?syRst>MH=7w=oH_sMKnFK}Mwee}-tenP+6yF*CgOf6#l9m~nV&Lm0k^g3ly*;YJZL)1?XSHiuScftr z#fh-AL>l<*5_9^a$4@YkHx|3Pz$brZ?SLRnt1aptkF^SCGF4So9T@qEx}W?S9Hhje zjy(hdWv_M_&Bzq|=dy@X#CzWgO-|?oQ*sLLA1p%)S2ocGlYcv~*k1^Hr`qWbrW`D1 ze39xdiiP$)c~TF*B%`J~(|0HvDL6}UXmC$AX=?Uck=2jS&$IIKgwo{Qg;dPIWDXvG zMpEM11PV^VaExN#`GO17fs{SQ=DebuB3m@@iB@cQBh_QnXofNT+W;eRTnE;5qA}H_ z{zc{+2E+#+k7QZLTNt<2>=yczPV0#hS6B%9kd3{-YcCyL!Ul_+k; zp0DzSG_^E_A8QO|+-h}b+8Fg8y{FmTOUkQv9CZ=kv~^}^Lc*}YvMAAFoY;}%=|t1SKU^tU@;Fdz77c@R*1UA%Eu_Y3tz$}<))Yv z-3pd<>}&eTERFRZ4--c09NoKpl|RsR_t%vKf8p&x_J~HJ$(xT zo$oS|$Z5$+e80CnXSF^*nB_j*ndQH~H}>(#(cCVJdOvxk)i2Uz0R!(`MoYbx$5G&7 zbo(K0GjYpF_4f8Q`5Qg(SozU`tNz$D_S>4h8^zdEijt6gb!toDK<=$u;A)fy!Y`h(--=f&9P7^^!9I4 zS6lByogoK?=((~MUPch>kn9^#u;HhK@Ta@&)yh^MbQOJm+xd0L-}4_>pNLW6Rf${5 zn7B7A{qPGNpZ|_4r;-v{iFBo2Xh>#+3G;|{E(gSV34$r{E%Y!9`-0-)9E>o*$tJE)bX!7Ho6BjelTH=PL$J?c1 ztcS}V=!m@Z84KnZ_Y8M`u0~g~R_#1AF0&rzJU`m7zjtE99k_*)p*%{1*Zcfkv+5{6 zR8O`>jGTCSrfnv=TK~22s0ZyRx?UYb5c6=Y2V+m39wWHzVWSoL=?cN?3qvg)87ltu zzh9h(5?Ru&?C-2D*OsZIjn3QJB{cg>n)^q~LN{gBOxrEsTCs>YYEnTcVzXZ93bGh0 zzrFi`%hISd+fZlbYVcBS(%oG>bOD1C|O-DZ0gNyeZ7| z;uZHP-86$>`0iguVGr-XX5S>WNe=s()OWpcM50@!i6`t8{wEP90XuQUY@7_G<` zMX#lM_TFlzVc))-lM#s5Zg<<^h7V_c#6Zqm$~pbCRB0|7%Gw{i@E|bt-6%$}x^(MT z-Qu_`f0Gqj1Z!{SyJoQ}*|2k|Qd9r z>FA|&f!~@IC8ok-O}>jCIHvPUqjzf_FCfwq>*MTI=x)nt{8ZFBRE`|3^>UW9s6CB; z+3O(2M`ZC5LsVYC9G_8Wt~q;JO}+n&`_z}P(KFTfvQsHJbB!C-1>(qCCFW<61a)mJ zfOn$ExQ(<4t2>BmWijT^^%ryB)2zikhe00HJII3zq&#Kg*d$5IyFLcNnc?t;1aLJx zoJ_6;UlJf$ak1T*Vm)Q#Qln0~QLgivXwu@@_BCrF#V znBPwJa3cfanoY|cE6bmaJ@LQff|QJNDw<>XEWazOvKK!+1>2;7y$rtLFc@1}@5Cq^ z5Oh90X3=-{nICF79SBuUIwF7??)EGxo70_>P)j;b9!5H#)Hl_1 zveiRRBuu#?H!9SwzaroBq4NfU|RlwHPbf$H1%Avp5&z!DSXF0SkUDbIQ zN5eK3l*mwE#zO9}ZBKBzYRHFZBD9^0n9#M-o()i^yOx{safWYyQzNa#c6BHxXF}Ja z=TzVIm+=V@q@7h6h!}-c`>^rC(p-m*cP7Kja zXnQ)RsZ+5HuBV_@p$s+68ylhGc+%wfh=gMoM5{ix-n#blmFo4>aIpk}F%GGZQlEpa z3wlgv9Xm2)uZ6;2Eq1EQT8`gz3s5uOC*K-(z)}<-x^bclCkeB(~dQ^KZsXG93)}Y@3u(u2tR+;ef__lr)x~c zs@fSnBczc+RPD}83uSqz>jXlZFN{@i>ojr-JW{l0sO6iiBS_stR=1xb>wi>xSK!1U zL2LdH!5&D=3-{dfOHz~^B!x^?4C$#}Wk7NbjRY>)20)`_Rah=>~yAj`Jqb1^m7l^l&k_AB-56!+#;7VCgf_!uFmkuic!7h?*gZt{)_2! z_c`Me!sS*?m7~g@XM9#~KLmRnseyl4WPAbV2*VohK#Q(PPtg}6O__^XUmF|M=ox4_ z=%bFpNYb{Qkrt(|FI1%aC8ceb?mr5CW!NDnU+Hzvvb;Ewb1u|6KlmS)o9_9`>qjTe z>2Sb~h^;ekty=tgZxaZOj%!mK5;{L2XR53y=c+y=nxBOWYq!@&;XMU942~iy{FPe6 zyH-AoE3SQaeEVrK6-tzPE!`imsL7?9x|XY91sBHJ&nU2JU$VP2Oh>`kp37-@+oHC zk8(4QJ%7dN;5ZA?yJa_tui}60sE-N^DUojZym`q_c#yXxfwn!* zgmwDa&!7ie?kNx?9j05>#_w!&(xPXh0=JQ?~35`m$S9)v?zSr?L@Jvwk6s^tZy$#4l(*9EbNxA;-XvBZZ zJM4>a9U6S-(>`mn37JGEW``@B@@_U>)9IcoaYH#25(MHuq%}@Bv>r`vc9*gF{Hzo_ ztrjf-1^GrHd~Esv&v8iJWujnQmEGS)rsXzTrWCmTT&M=K>$5pq^mZjKbwRcG_#i0{ zfE=s+z1^`uX62_(G^}}hy}enmDimbd2;eNfz$pLz)4-zJZ$tX;`ilQv992nE^em~| zsdDTeuea4{iRPJbZCL9w%c`!f9+@nk{Lvbh>j46?{%Ox(%zI{)9jQk@y9HHhY_?V= zmn%Y<`8vD8TfSzghk8XOtN3F~Yup}S-6pknm{tX~42Nu}xxeoOa&sZu%@1&;QipGa zr4Kt&cFsmQbnH4uJE2wwtYbbANF-d~*m4VBm-tpDZ^(?k zHMx6NMFkVJ4EgcPr!Y6`!COlszKg+~>58EXL+|?UM0ZK6SRzO3EfB(V#u>i9YB&iy z+jDO{rep=N-anpeGyg?7 z=xZ?-BKBfEud<()7eI@R0KDQUtRpZ6fVyUH}^g+ zA&J<5vq7al<_9Y_;67axeB6J>$cR-vbfP7u<>%lUw;YO}iBpPyexU4}#1II8f$xD* z$AX~q(iWkn%&HyJnf`ab-?)rD1Hk;|O1mWf`8kco9Id$V%C`-B3crMLH}0aRAIuQ4 ztolnVawp}4`j*=z8=5wP&H~yPNyaw}mLODimeiA#(LPLa1Ji&3C92VPegiJXgD9V{ zZP=Phl}l%rr8$E|jj4If7UYj_jSVQ1=PL-sX79Xsg>;iy_O`nGnp>UhN<&x(wq9ICsLozy*^cqcG&t(C$ zw(3eS6H>X((`ZL!e|Nv}%^m60idosmUy?UptWNLS35jbtq*4&9VH)fg4QGKFLxO$d zv0A1l;Rzf&M80I35<5}#UW z07qEE3+8>ihCzIrKDSjVSYl0kzd0V^s(hHis&i{~dP3QGuw3x>{GfETq3UI<0E++S zeScG4ZpkOQHS9cV=}930(7W#D6lmPP)D*^=r4hqRDFwI=mRYxi05;jOOES&ZOT9Jw z_}Uz2Vy+JR&;;ke@mjCl&50P-T^hT^dNMMt(_5HxoZA{70@?o>n`h3iK9?27D%{); z5l?FI{^1wEe#!dRu&)gzBq=$wj; z)iaW6`y2jzOc_i2*3r-dWV2@DpJ^Ya>d0TpbL3eNfRzC+LN9{H6b>TpMTfMz6PM$tC8tK0pe9(V zWZi{A+S51rlxIFrf+EDM06iHSw9ORnz`wauasS>?#Fa9pwquuw*@W?@sHY^sI_~}i zf}oipHjV&;k>wHRa<4Mz(Ie@Ke|7o!o9O;>34 zkKrMUHE3o}eR(nbFfi5OwW+4zR|;5y&C(Aij$p_OhEj1n*RbZOILHY*3;eWdQLLMF zIzIqXr-UP15p;Z4=lG5sJ|a+S{f`0U?8DygI7ok#>peObJ9_ox*Z{!*NTTR%I!8Jr zoohK=y(ypFaQtG1%>AaP-S--3_=Bu3aC~Or`dS>`r}LBijlc;X{AF#f5s)&^kDF(H zq63J2NT1$3#}L2nE4Gt|6DWm>PsYonhr1tA&mW0{Uv>`Pmp~YAh%l~BcSQ93e3!w* zCC9HnA;P8}!W8}1Xs~`a-4G?cF#=hbf{)M**Lgs$_&#EGnM7t(tl&Btk1>w>D)$sS zCJ`1#^!{;avbEK9q&!z=!WWsM;4#l~BO1HkuY-Inte?z+fBkC%xn#`vMJ>rS$9dpF}PO3F$M%_W;tTB|<`*t{}BY>T@%ppMeYVE~+I1h-1NxRg>A zg5`xDhwt((K#!h4BD{vHbfy*bUP2j>bZVPSi=GQU@XHJOIEX;Y=jZ3f73fqi3~Hbs zJ_v*!iFJDV`}gmk%UAppegu^@#0vx-|Fl)`jWBCcG+xhI$Wk&-tT`ukxAL_Nv&IR& z=tc7SPo5i0e7w`=s<^n|k9D0IGmaEgMs+O?Cok`{w#JRSN?4TVoIa!#J6o7X3c+NtLyj1m2J54Ku8NfltG`MtV znaa}wplZ*Z`o;I;)LyE!LlFt3=GB6X*25^3zua(;o(%7)7n>=Bz{7r9$kLJ*(zeYv zBYZg!s$ZR9A{?Jf#XI*wHYbwYy#1mR6_{AkouA{cF!s>rP(b)1q0|aLZrx|JApsah zY*OcV^+l(Pow|*r2WWdhp~nJYaEk%? zc(l&jd1Ip?ShzVxwAiKnL#p-ESB9yku(uPAh#?l5GUlDhSoN322%dY{mTZ{FZwcWRVk?*D1 zZD}EU3|?6}w%<>4@ooLK!~Clz7o(bcdGa_&gXAOOvnbNLKy5{fSDQSY$Khg08TCz9 zTi*ocTRh2E|2i8o%C?8av8LM5$JR2-vJ+)wxk^`8sE#>?=i+bHRzc42X}8u~ar43Eg8c3QY!gT3r0AERgA;z=Q0 z`NE*^n^T?j3fHmX3l|MKNnS^Dj`rBR*cdFc-_u1&qqi%&ZSQ(-P0B44ziabM(grE^ z!DiAeNi7D)na=C|qP5a!^1?e;uEXu+4>r`fk!P1@i=gO%o>vY;(}53Tu0M zToR*oBGz2u(Wj2Qa6;iNrxf}hh4;)P%5IHBU50lLPJNHfAr}G<^paXNj6c(#@6N7n zr$*>U*E|5%iz|Qmq418iOeb%$aEsr4k3D_16oWvrY!ecm-n0Gs>Ux>ZsfPBbV{pio z6YrISEJVD7dpGrPdg@skj-Ny1 z{7ovWvmTr*cA`t6Gw+`G`19@!xp>6y(>&spb^@U8{XIJT7(A5OpDQqQZ@OAYZm4KD3SsW($Znu zGtq6Y-#O7U$fsp!K@0bn!q&v7G!o7}XbhrvEw!%YbVb|I$Dz-Ae*L;Fsu=R@yjCHG zjS-pQi$pRBalV#}D6>jdUMnE)ezRxq^ldz&#(}6KbPGa>OuK0Q(FD=kO|CR;!RJ7N z!`7U$%OdyuG|C^}iGu#}&v5%a^IUEr{oSa9p3d-xYUg3D(U>#4UQC&EcLzKXp9#f< z#Cof#$TO}A;vNIHX$3oJ;n2(PPmN07yQG%}|FGx76v_FV!l_0)4-Y9 zbZ(yy2$hl0RkKsOEr@u@@})EoC@9erb-#b^YBISA>7}u#NA_lB@Mr5xrwnWd;ey+# z20lZ)kK|L4s8<2ycE2u6F4Fv`y26p~))|+6R=8Dr^*- z0)Ui`ajGj~xM}9rbq6JwpbIDxr z1LgG^!M34#v@h0uTAwQN9FW{JDl zwK;rq6m-JzND9E@Az(+%w_tn#IHZ%NLisRsJ^2RA7PXJ)!SoQFBne2v-nL^}5;Kwu z!LImtp6)0u6F^R$F#=ye1-}vOcS0%PJO>vawYnOtUl{A8dato{)<1Qb^%U#?-d(A^kiECYHjx0rcd9SJ7b{! z96pARQQ6}ZGRQC&#%4{n0d>qeBAz0K0tqb`SkvsiMas*105&;C05l&37UD#`U;plS zkz`u$=bdK2-wlW$YHw$20A>eHGz(~1xg|P4*m&JD{ty)8tw6bcSE_ zMU`0^_8k2*IdeY&@s&mUpaz!pDjj4V33WB}NdWpqm&*MT1praXlYM8j_^#2jHgpcjBpYOx2IWu@IqX04SL z9wn~mzqeyrKGm6S#(-oOGi5RRkt1Oq8HKNHKL)pdIVnRegA))!qI|@hXA!GQQdB1u zQ_Dp9vNZ6>>v*g*5QGXQnQ$E-e8(M|y;?CL8sP^!$YPvmu%YS}M1nI68ZIT}ipD_z z<6&R_HEiY+_&?mxSyv_W)t<$uuEafyjWcL~qm^d{zNFfORVpsdZ&4t2i|UnX^n zc@C8S32ILd(FYlJDXr?Jlkx;9RvFVAoLVAjYGtzh;(LJ{k}GNccJ<^JDh$}n6Fohq zwawFU91anWxNV7+{r$5?SXnQ;pGHwh8A>DMlzA9KA2ZWjW%u5XuA=^%L8%+hdNA+KUxJ+NQ^~; zTn9UHP(jy0^`zM4#$I$mj(&?!TSuq#wsw~CIzANk)`;z1NF&A3EomfU0tp(xzd z2ZcTVDiW>l`g~X*OFz&oTQJg7kzk+Sm8sM2upK1z0}1W_Bs2f_)jcE%Uk(~1PJpBb z;gB%@-1RSw06?<831r+ngG}CA#fXIes>*|VZUs2-Rx8-B;om}jF0P}m*bEr=Onkuk8g0E@GN8TH(>+$dxn|e`V?;qmYie-%1%QuKYo} zTHWJ?Mv(qiDu63;ZrM_cbD)O-WOK$3K5Gm~5*BR-9R~*&jIxmP5IZpcKlL$yg*J^v zz)AeA@PpaC=ZfDQxY6Vw(ZrJSs;e)5$+)5@cBlOOfi0X=TV&u^t3gWDPBoA=7(7ER zQ(FfL|KmeSgG(WWbN`Rs8w9{eDN`;Vv=|hht9ZJrw?>o5+so|!QQ4NeB${+x0%*88q;*~Y KT8Y}dfd2)j<)#Y& literal 0 HcmV?d00001 diff --git a/docs/apache-airflow-providers-http/img/connection_auth_type.png b/docs/apache-airflow-providers-http/img/connection_auth_type.png new file mode 100644 index 0000000000000000000000000000000000000000..52eb584e5ccf6463273c9b0d35171944d451af9e GIT binary patch literal 14199 zcmch8XIPWlwl2%HEh~tXqEt~qIwDQF282i_fCvGiAiYU%0u~|)Qlt}_l+cUx63P;! z6RNa8U?H85AP_(Zf%_%6_Bnf>bI)_{k9+^{0QtT-#~gFaG2eHLH}7?|)fhmmAUZla zhKCRpC>`BD5Ww%PXHEe>*-vEj)6q#QK2*7H;A2j}h4>qeWG!xxl`vB)-qyt4sieB$ zK!I^4{_6to9tVb=yr>fRj*oBX{%=hGK(XWx`L7FCMn$QE{n{j#%tES+jZ;z>wWBLy z)|Y&zJ5tO8c4S2zN-5sF;b`ymw>G~46VlOrV)=>wah3~pf%ezEbrt@9(0&^@tq1yz z&SF6ae1<#t0(k%VoqP=R-39QOm9E&Dc(-5w)ot1MK?$B)ipa)5&(OV0dve4x-+#J} zB|8atk~=s`G4{FqT4dbO+0(-W>d2h}*iREaZ1QOCh6GLh?;mc517iaZl3If2%-h1l zD4Cs1${fNWe1K1kTjTDRT-en3)V}zPm6+SKa(|U(%9$0!Mn{7)`&xTj;C)6W<=vi) zolM3Pbk9y2ote1PDHv>eF+1GCX)ZlyF|dG*IAL&+X6#VdTzezy?+)@swvKC{%slpb zrFbGljB3@g>)}~$r`{=%rzK~HPU*6J?cVp#x&9IDxvS}W-Ky%BiHEjpg<_xETUakU zI2MVy8Jw##>yf={`3_k4z}$f`*x4&8r*jp4zw*m)yAE56ToooKJ77@F?n<-w}U^m}xbX2m=)o+xvHvuLI6^WI#x3k`GC@K=GXzz0#)m5{mr@G%S7JkMK z>+txD-5Vq1_)jLoThg( z7wLwInsm|}hEu?K*|K}0JT(H*Y+nn#Sut@#s_yyOgQh~xw^=U9tmjg<8Q8ol9c+{K zMTKIe|7PSGi!!T5&3L#F`-{`mjZ)wp5#lDW5Lm!9g2}xZQ?lZ|-buazT4lagh8L;9 zbA~%W>ey@L?cIT5*=3dO-w3I~G}nveTmNDAMV`P_yz+T}?7Y)jZC2)BwUIoqYz1cL4D z?KR!qE9$)0IsdjnOH0cO*t*Sx_hsc?Jj2JM5F9he4m59bqhMIouVF&mt~f7Xu~Qm>K>&E8%#QO46*8mprfkZQUl<|REZ^47lDpg(M^^u zf6Y8J95}o`MmtpJZ9f#1mACh@$~l}ZE~{+*%L9f7(RNqG0X2ZD_s{uxSYj#L@>U|KplpF}}~g!L$O zDK#Zg?W3_^x|c-@>fGM*Vn(32k!f*+Vb^xB2&1$v5KeS-{I$?!3ya>OvWoVBr~KbW zFLu&4utgN%;qecr+5%z%_#`j{9o;>X62_s#2@$E(E6NT+#bS~`g1AhA)8;^&kVz$jkGX}Ji=?z^CT&{zT%^0!USH2>Qtg;DQEn9`?751^YuVZ5Pkes*W^W+@ zekpPiYrXMGGY$*{Om(ciqvKi8a*})`yUe9Dr|zB}!W`Z=SDCu@0U$>CB~~#XtG`~o z>JQ7)5f(dalOq?JcFZR$#uoIgRO#6C}*EwY6Dekdix!B-8D2XStOGu#Ak1 zmvyX8+jvs|Idj;iKdmYl-91^idA1#w1g+%2NcD_-c#zYh zTc_ql8GHuMvi^)VPPpfYOTs3#1MCkE&J4^{ zSohs_x9*qfSK2M|-9Ri6@XsX@;Zaxa;>*(g%A5C|kjRVJk;1_Rx|b0v-B3tqXy{rm z2a^cWDyv0pY0qHFRy8u~j;gq=M;aVtmlHwk2;!_4^`7qfTHTFVbv3w8K!K`d-2r{dE{w1Clu9ak=bM!;GTt()^PhF4HYISblQOftp!vINoYwDf)+a${@;ZIXgOtQK!!Pb^=J}}0 zv)HtS&P4MJyx!ZXv-OJdKNlX(U_IB)u(OFndp)T6jjj^z9S3eIFTW%1O*DmN+rmDD zyExFxkWo?H%uHg%zTcpbcp+m8ZmQB3|LU#aAB+6ik@}kaySD4fwN@^mBF6 zon2*ssI50D0OyIf={|}BJJ$$fx<7`oh+>BiP3}cPL0Tz4eLa%l@gh}A#jpfFmh#Vx z(K7DVlUpLDcU{i0kGmWl62{;JE23Caio8ty;g{b}uldICQZSr%9tY)i_ka0|jcz~@ zNH7bFi&(QFsNSng<*#90T*J=hirzjx&5T2=^81rYDUC1~L_6i~(|A!hrx~xwmfGat zPCU3Hf};Y@mVae``*7zW>7XT3ZA`-uv==0YEym0Q^rpYM>8F({bDN3_{T=}~jNf%6 z;xaRDx44cH69dwWf4-k{$oa8sX)&}@Gyxdey5GUcEW-d5b!tS`WwwQ@x5!wWK(ryk zpW-Yv4=tb`0X01nEHS)EuV0@a?+#>$CTk>!RKFWd0*P{NHrZ29SV&moB;@2c zrG;`ZX(Pku&k2_j9@JYvb~CQtm%W}y>=@ivm4%mf3BN7I+s-;{gk9Tx>{pyw^`0bu zfXLb2H$>*28QRej|3rcVi~XtrEVd((s{+;KPWyeXNpjUz2^tYp4f-Clt+)x0?{ zRiXGd=c7QKY$3$dpL=@h8J9aEYwp6)VU04I4B`jdV{72S2>-K821K|<&^~{iZMD}M zcXXs3hTG^lt|`@}FI8dkJ3Rb1I>J~SDk?8n?|p^Oh6Sv^wmRHv!wvD?wmMSoW@C5s zNTHG1(US}DV1ei4ju zH#pej(G+pW!x zXR877dLv8m!{6wZ?*dpDNT6HgnT(+^;<-ALn`&WdWGIsk-OG|V0H8ca>59?VP?S0V zMQ$BK5ol*mk9Zifh{T&y=PKznL)4HNj3-_a6W&81_-z_mzA!Wy148M_r78w=6E{&% zKGR(YfUQ1}5h&e8ILJ^knQcL9v4sMKpl^h*snRG3c;8Jzb@Z{ZrFD3!W~-m{db1?;dSA|vm6i1uo0R;c z11)k``}jXW?(lrc>d`AOOQ-NU7!~L(i|qmjxr>szEqc{69FtSuG!Z){f&5;e8WSY# zOA-SDithaI-y1pl*u>I4yqJIldmTgf_%gL~XgFxXtP@F0@`HgSlR~ko?zshD7$1G3 z5&=58;(>pA87IUYUMIPuji8Z>Oy%yQd|XE~S$<4fDhd0RS3ha|Q=H*gaa)_&(9~$bwnlupzK$? z;WNuh+8bEOIhp^7_fSK};X`p5Enppj5m7_W&gV}&1NPxETlpz{eV9vuy*&oky&8*( zk_yG%TU=O3lJh1e0@v?x9Qm0^j5d1W<2lGV0&gWAjV2mf+Jt9F#*ZeNSUQB4;e60l z7BoIs42=V01Q1Ao%y@3UXD~6dXnhnj{-+^I2tg|45eTk)`ppSP`E_cOGnM-tdJD@+ z+yBe}&VW@%A2qC;G%~&t4qWd5hwD?v9@`Bgkewwe+|1H4E8oTTc9o~+dj@!Meo=uG zQ(PRs1~*0Hp{Hk6qXMv#1Ic#1cA?g`lRf|eW z`V4MK`NhQbIuxmvH zO$>Y+FDMYu*VB6n@DwWDq8nI32_r5Yr=wp{+&TfK9V{Rhd!MFs4&b zUY-%ya3<{#=o}_mX@BYdR>e8*;7;5ps zMlmrlQQ=EWLV}H(yE_KJ9~j_{{@aQFOqtY88yz>XbPTVOT=bfA3DPVP^Gl{qoD^lB z6UE#_Zv$Gr>f0MBNBfC8-awBx?5qi@J9C*!Vs>7yj%r3E;JB_+uWQO3g~| zBmuL5tIVDHRGBK(ZaR;1)Tc&&y+`%TSJvZo%21v1IE&^<*$}{o7aV~90FGOt4rvii z+k<6xswax9${A$?4{95QfzJn6sCBRB)RT3AK1m4S=H{mP^yyO{YLpHL0)CkIPak?H zP&D34MPM+P**5CfrJs&7f&E!_>W+Q|^J;JpdTy8UVDH%<+5bA%Fp1HTL}^3rn=}hM zJ!lIddD8x(g&EzQFxu?L_F7XT$bv@h={kj|_6o_X^OHms4^ofM^rhfmPajv0o>9r` z%g3J>uOHXmDrjCgfx2*97D{5EdW@M5aeN(gInum&k6-2Z{d!I3y#38Yd5NgRf`bz_qLWC~Q0<*87VZJL?7Y zIX?<70QD`Mt&x5D*$Ls^DZhz>-Db+gG$-Py1X6(|OzSA$-$cG-IHh(yGgP)><(t#i zT8%$UzL~{lP5MaQu0y$BZaArOF=bEMQcB6=FvZDe&7HmGJ4rdADo$;$m+FEcGME}` zh%^Dj;euLoa`Tbxu=0|PhB@r{cuIev&+-AWvy08j`rG#`fTl8^B?u!9*AwJ0mT*G( zL}2y68rtAe>S0f;*`iYT3~MT(bFjOVJ>CR~W9p@2e+k@_Ldq3lH4J`+*f)e=B{Fi>M^c62Rh;ForEx zUk~y@S8wvAU!`uUADp&_1F5J1m=dRW9ZGUBS&KzWd01gR#t)Qt6!C1yF_9E< zTi%ELnHcZHfP_#&QPS!@FYLo|py8SP&$S6YI;0c{JgN%+wp4GBB8v6IFdBWE9LrzY zT0LkTTo8Du2=nEYrF@h2U&H_GpQ}OMFj^mP_Czz>+I0G&YdmH<{jh;}P(JUUJ;Wm< zX^Mq`_KdxS`k9qy_e2DuRQiXNW*U!>vvcXCEfDi0@6NQ*IqgZGjSRAx>)o-~N2AF> zwUEh(9utlYaNqbT9n59(G%((@m z+z7*tdvny!gKdR@x#h}buS3nx7nRt>ybD$_N?KFS8d-t*F%^LaV-wijH9aL-z>=D1 zk)I)_!Hu~v7-<#v+rDe5nU4Wo{uQ66?TTBaiqp@gO z_!1xD2U8k7TG;}aT2t4HpFI!#XLPhDEkWJ*-rQy$@AFu{oIjbbBL_Fn&s8Ji-Y+e5 zUnD3l^fRh^<8BMQFo5J~?t5F-j9)LvtQNys@dJr^SRn1d_!fZ7tew7>9pQrsUd)lx zrTqAQB~q~X=Dcw$6abfy+iCMyrQm;g5B!kP0 zE6FAv#;_Uj1-;|L{I_rXH8SQM&to>9FWFUhq!5fuOm&ogc%}t@bzoq{^_lIArN@XP zu==3<4>Dc^?J)A8*Dz{Xp6Ptg`RNZLWO5|eS|4*fc%_6*RUB$?s&@Fu3hcJ{Avs`Z z5Zc*j?oA25wMEcO{Nc(J2hQ|-)dDd}01LnZCxTkzEu0s%9Z*r3(Lc6Ex~|Frmozzd zsc=G$=$C%rC+o*m8HTiW+ryt|lyGRKlq}0-jpg-gBT~>)0p|JY+)O%1j%*8P?hM?m zrH^$eV8!FVSsWUGJwJE)(JR8>HGRE|r2F63ySI94{0Y85g-7BBKNF!Kr{^?A@kzG> z>zat-{uko|TV;z{;q-D4Elk7BeHQL7Xi5f`BOf$TEugu4rSDv4o% z+4$4d*1pSbn9;i)sBgqeq@QDGOn$O33;QTm6z){}MkjA%tlVBeo+P#Q6;^*9bgJsp(tD6<#s)d5T#sK3G-B@#SLY};&Yl1Z zR2$s1#oZ)GEkzVm?~w?$r(%dYZHcv$4tf@iys6sTX^4}p<-bci=4UilR57}RWE;)q z_CH>Kh($f@(@bo}E5~(JV=)DT1eu zFVp>c7s~T7J=D~R7L(79t=^pTFS)L+CgYASSaFpMd$3ga72=J1RdH^7DSN0DSDN;w z#D8s{11%S8-L?!6WA;ik*p3-Gi<5;pH*C*|`TN)0-JT*CIX=z&&Nt%E%zW0*$ud5@ zJjlj@e_pWA`SxTVo>ZW z2FEB*9~IR+*rS@OohGr~wB-eu_45hI)_uP7BH7z04vat`7Iz4fQgBV=v-U_fmQ)cw zzR&4(asfMQ#;|2}IWHmEQh){nIMgC8J|1gY=Qg34^0V&h{^kWT&NUJ}zm!f`;Njs} z972Uz_a;VS*lxTf>y%D9;_645I{YH`Q{=Ayt3d$loNW#1#g{hPQ#(^>fvXSQ30O)0 zCNGMw)pAdDuMe3}MS8x7_+t6rI-f zBW<|+N4jk-VbMMu>ET>9HW)quRgdQV*`sGifnJZoIE^aIP0-E)^+#=C%t-Dxrzbi_ zwqyeK4Ls`9ql;nHD!DGx4R&xfpgbY0L8@_vByy;uEBBH5U4?XyeXcsL_%hgTS0 z`q?+>l5CR}*xe_A6!)*Dq=f?a&ooOTXMn0`(~pKENrxPY2zfeaUBRES-VA@zguH`l zv+_H7dzN$RyL_WG{bIk)UZXBQ318ZSneT#+FDXcDC`>oIz{4hg2Z|Upno5r|2D)bN zK@rB{Hsjz2ky$5f2mXA;d+Sz*gtT<>jga$`Tw^86gF^8-9T8QC_g)v>Q13zS0koh1 zC-7GK?>hU>V9g#XVQXAwoJEh^HlM++ElZA$jv|x9tyLD@d8KdK1ZSh2A&CAz6Hbi} z-CNHHt;_^k3J3_S32`>U@uS7z&g(sIp^){Dp_h$TgHcgU zIs$r!Qmz&r=CJi7k;w+T7z(@O+%nF4=6&e3W=5p6OG)a<>`OWo8LHua!$Rf_e10iY zUmccbO49>Gw^RLByAG?rx(&sFF9nJkfbxsvT)(S)KT0~aHLRnAT}s7N`~^_yF)DTc z3I>v*z+Hpu%R7BQzmnQA+AgtL6Jk%zJ@F~1`PLdC<@eJ&HE2d?*^do1F4cstZYIIZ zt@cRrH=sQySKo^<`S_zF2)OSJ&gFSLCrz*TE-vtlO|01R&zqNuZ{tfbYsM9s!-{dN z^-b2Z!Gn81p?-y&X`#v!(IlNYR+!n3DIan=+$W-Qiq^h3sPCq&_6@v!jcLlr^({FRjDgaG4>(>3#VFungj6;q$)n?k}mkc>mPt(!u zbJDW7hc|z=F>BLoX|r{ZN{Aj6PzEMlmo9-vsz(ERiGhIDjRs5X-DX=(ZVkO;A!weD z6FMZS&qf}3j4{|PzAwqb;UvT#4$jr%7GzC8`MMjJpU(9GK~TN?3_M=3wLx|*PJ|yp z@)2-iOno%yw(-4f+0U;Lc|VJ+S7MG4qh~pM(!*kOwhF-a?jt3zeN7Mep?6ChxLy?3 z4xh*7u2))b{9K5hK9Avblhp^=c|+7H5=rbN+282y&CpUTf+&J$F-tZ$u^i$hKs;T2 zbmItU3$JKoY@b)>jyroOqMpAjY2V1nB<$yGGN#(Xbp$|rq?A*ESxq|=r{56}0MgD= z5EuxtZVW@|`UK<_0xdlM@1plOK;B&LBpG4YP8bDUysIe@0|`^XcUoZy8V&|^zi=M-s4A;sOX$*T1;1rsBG${FT<*_FLG(*32Fi^tP=L&u-1+kM0cR5J&DC$3AbA zb$lE)q*f7n@Oq9=qnb1$?fgMxjHvAjI6)EtE{+2uGqSRxAJ*?`uKU-@jlMp)MR+}V ziKw_JN-E0A$cIGhw0|fo9btMbVySq@hvm>oFJvy?v7=n4-c$n@Y4_an3KT-Gg%4>E z8MaK9^X`kkzSN&ylc1CMBWcfL?bc4&*47s0bkvsC7Ta7jy*3jNhp`(c3l6?$5k z=4VNUNo+k>&j$$>Cuidpr}Rd0H23Zxzhsm3#TEg~4KXQwn5*At^ORr0g=p#yC>zM$ z)$w3r6e-ktQ^>6RYYEGOb23q3czc^e#&$sWn@Ko-2Tq zA$-e&bQ?!(L2Om~3^>M6`*HLl4$^$MY2nJE78SH){#|tdF{7f)C;L+62s0BVpcOw6 zKQ7hgqcjE9)ZtP8YWI$F|;t2j#oW$&Oq3^0AN#jT~vUssBB; z0dO6)y)qH;Ki3ccPk|D3%KztR?EiieJ~W3?V6Qy6cX)Q9^ZE-_0OwKrHUQ?!_y%a3 zFn}ndwF2E%mZhc@7}!KXXCYeC6wt87!*{c#$xMQQ6+uczQx%IF9f#3(C}}=ZMKCl_ zeWVD%n|fmt5)?cRU$Ve&U*>&b?F?X^@2(X5@*ZUy=ZzW6Qp?1Xg5iP2_UZjVQ??}x zOX5qv2!KV({|wC_<+oRCX9uRCd7nQIOf>~eE`~jf>HXk7vNP#25&cDP<2G@)Y^VIkA9cy#;qZ$rlS`T)ceFnke z&wgAZ*ZL^}(vVOQqle|5J@LyVOtLfqpI1^MW!e71@o1Egqm^n;AHtFr)WW!UPpZgGLP%*k2QOYL6z{t2lUxYidIpQj^?iUYjo_aX z{qRV^cT>e9X*poT=6lNBz$rj31^q4=n|w&wPikm5EuNe$sPr!m<;F-1MlSsqh}L&O z#_i{4N*N)P6Jr6S?8juw18qAy(i>BR3JevMw(j1qXi1e5t;56(-kuAXaGYtvnIA69 zcU4~)bSNq=mIOpa@BX6z7}Xv8N|fQVRrItnlO7dg1~f*OJiid9eJB)51)`cqJ8aaB zmd(a8#iYjB$-^H`tN$6~)m<;D0-Enp!p55wl*u?-{8(wlZsGb|p0pY^6EdR|C(RW^ z%GjIy`s`x%?swF0qF6Vzu>f+hp7lZ~(@r@*Tw7{5M=P9DZEQru6}YG0^$p|AZU;3{ z$XkD0?#)D;2i;id?gzR!q`<#Z6+|g*yVaz|oA-s3B2xIfEptSl{>Ln{4<2$$f4-s+ zL=Xn;d5|CZcb_kXbM58>@+mKmBy%*-k_T`BpgR1#@G@&xyMGv?3LKWd{uQ2q>zmZ0 zZp$_U%Bui4aK0Th#H0&v0S(SNBDQK+F_$VX+F}+pe`Yg56?&DoMNi8ueSLp2?q{<~N}iH0>+K6G|4+q()-t?5Ww z0>uGfu%lK-X?6RJ?1xYg=Imwp3n6blX-o3|jbT zfKhMahF>F=qu)%;D$47R$$C!*4~{r+dIaUBZ1IGdUvmKRRTg>I=D;>V5trU9^|7h- z&1r^8)W^JMBn|#Jgv(7uHnkZ~Gt7WZG)aucV`VeJ%D*EW>zZgm_kSVYfv&{{kX!TRvA1pP#c1X0u5<26+ zC60q8_xoB9N-C%UToLdD$&T*70?eNenC#`;dDhxV_5!lq= zE)LqFDs>iQCH!*$yZLBnZI)=64-lCM3uZt^x1CE0cus#;6kgr?f;1Si(>O%559CIz zf7RF7so|L&*J)Q-qJpvM4ggkVMXOWmd+Xg?Ztqj_0LA0UW59NVRSM{W%{8ZEQ1}T{{VdS^2f@WOxT$~Ql_ET!Tq)Km$ zN6#}evjm9D&fWD(Yo_cMQsjnd_ZsZ@UK>V5Ex*$@SyB_1t85?O3X=bMvD8=@fU%v- zY9^;u(M&)=%-YA6io;Ccs^ZG35if4kQ}Xq59qT@kN6-5b9?df|jylGkSGwYq9@G6h z$3ibqE$DS-17HLJHxj|rvsZZo=9n)4SEc52eD_Z^ zzZ1~CL-I*RaiHU@M*9qC!8V3BRhX=I5X~2hrB4F)s~OEd_xOXSRndDZQM-rE?*KT1 zdr!+Au7UuN127v)JXls;fLh?fSJp5Nz5M`uN=Ii4^H=BQIH2}!2eJSK2lgIH5TUNF zP92f%LWDfE`@ZbD0w@bUA3z}?C~wq2mqXW$ zD7&nV57fdN$@8CIfG~MxLKUu1wF1V1LT_83oyCIL49K8=l+PK_V=8f7W?hvVH)s{k z>{3-ri|}ws)tyNB9I;B64xb@EBQux^dY7dDLH#Im@~C45P^{6{)qMi!!ZAQKtrtKN z7(g)HTjekY03stTZS4q5oS;#rY14~e-^v;P%GFc9QQTh7x@G?#-)S*5HFfOhp7jGC zUOyI4T#N^Y!a*o3q8Gyq3d;ac7TUETkWwEm`0k)Kod39eK9TaAy|Ap9ZONW0q5;0H z;<#r02MX(1q@c!mnab>I9vXtGs_BjI!XQVNppPH_)D_FFO;Qeg_^ZLrKG$AF{VI$z zAO$x#W2c$o9IsnoMw4wAa|7bib?~A!y&Y_xMcKK1fU5IQ-=w%0AW+483D4Pw^L@~L@K5TQ%5qoBkcfTgBV@d z8$1Db-em8ucP&ZgnD@H6MW10%5iF78Tzr9AdnM(4W>lAibruW97F-z$94=vZ z6Av+cIG@RI>(~jdVM;lkbEe_GW`G6H3&0S|rJdeOv{(6p-o)BhM>SVR!Oz;66iK}> zf#|S-o3FX?d<>IHq3v?;SK-6N(QtROi`lk9w66j^KQZUp1NBX{^;>p6jE~D`dr=X&N!8)Q|&Rw z-nmQl&ZSP#*HgWpo_pyQfT?22_wS@RiC&!@ki(wjZiBN{SwO2klxF<#}@Q|-ZX zGoR0{Iv(poI9Z%)4=jKc9lY+;sV%!nd+25#pyA^XegiCJ{^<8tXYX~N@jjOa96P#) M542TEz)xQPFTow~N&o-= literal 0 HcmV?d00001 diff --git a/docs/apache-airflow-providers-http/img/connection_headers.png b/docs/apache-airflow-providers-http/img/connection_headers.png new file mode 100644 index 0000000000000000000000000000000000000000..413e9bbb38864faf0dd5664703b9089ff0b7bd5e GIT binary patch literal 5256 zcmb7Ic{r5o`yXfn{-V})%5hC16>~W0*`i5pPc5~&!_Z*&@TQfZG(MG_BoO+`NXB8 z9cLtFwPB4YYQ)t9B8V#!9>!GTtzxn9z61NxOR26(i5#r%Yd8mX zpspJ8#$8$syuMdWkp$lPLfgu|3wA>n2RozP0kP;M(D-3DJV7{eSD?`7DFLe$svtVK zQXrI18VCt+$j?P)C&oNc2yK;mP)&FT>;wtK#y-+W5L@X#Lh$T(nAMSE;3&rnyAcEC z=B`9Y)31Gasqo@r&2Lc$Ksrplcbu4A)+;|xiiy6^5E1hE2n=@Smv23B#hBxd`M!T# z;oFxEnNSws+K+=nu|$kS1<~tp{^0_Dt>)tIEzT4=aOBWXNS{k_ety2;UKs4bA;@|O zqI_48U5@qP!-su9fX5X;*!ZA*_1&YXJa?|LQ*sBZ*nIV=EK&jnJF5asx!*D8 zqD|Tiyz1lgM;Zu9wrlLxa<^2*#&T(F&im)*@7WOyvr6k48XC^!Q6Dt=gTuhh;JSW! z(nzC#D9CHZ`xnj`gs`zBwwg({QRK?w@WqL7$}3N5ZJ|8t-809c6d6RJKWPi@wu&2x zcEr;P^YfK*^h2JGK+AEsXA3O%nX@t9lWTgD%@8mrgimB`Rxh%jl#)356_u5Rr}Hff z?kZUoZp_tbcLuA?4A$sUGj)_w4(KQjf$QcHa|a|64;?m;j&!dcTY@2ScBl@KMh*uy zaWRImoXw@-$oX-~>Tsk#A-+3jdAhG?1YE7D&CiQIA7B5B6iA|ZE=_j%Y;(4J-riS3 zzCK?Y?8CVhzotq4@DLf){qv5U(jL%R7dBit5tFhO#p1X{FQ4_;Y!)_YAF(n-l;GOZ z54)J*k0Tn12Z%>5n?=l&Q35k1hNBj#dRTh-(qK?$L7sVH;mw`Gp&QXVV0s4J)6kcP zj*}~sP)CV~x22ddX`Y|i<5PF7mTn3IloxEAW*u(6*i%(x&7R-&?tm0 zc^u>|u8Vn;W}COTY;i)gElS_{pk}(GerhFG)QC&e}LIYN^nK_x>Q`K6N@g{EVKwmRXK^9NZ>d z^HiGgx_1W4yW}9TxSH^Ygbw?&xJcU6R>z?bT%}2jLma$IRh2MOqZ71LnRK|p9M{JW zTX5+0%A?Yc`38ou2VZrDA~|&5x<%F%n~Y%OursJ-)Y7YY=%p)3vUlBA^97XL;cpAk zc(&GQ1>T3=j!M_wHJF$#QGbgZ7$D)Tv;p)1kgr^3(sXt&WWY)R!f?yFA<>ogRGAmHtO$}!F+-icSz0A;Zz(U~Nh3w5b9 z3lT|Ew2h5Tj9;7%s{yr*ZvrbxJI7E~#ci6CBa`P!UKp@s$;&)2y=T@5aP7IlEEP<4 zreFPNg5DMOrzkrGj>X()qMUWUWvZ=t;$AzPR&9RSvd^vQ8s=PG_iG+RVKD-8s+6=s zzmiAYm~Y6K7}Pf)>Uu6cDGy*Iv`Va0_-eZ_ZL}NiU+`>yQz}2aF^XKA7*0>vD|g>= zuY7vCrXSkPZG1xNMx#Rkt#I&Jg1qB310fI?_5Jkvn%G#*Mv1}xf(B5A4fY?lCT)^C zdw#zuKfx-nN&m!Jo+h5uHr$Sq!ygOsWGu_YtmPRtF06bv)$djH*vG+t4s@1wynw3; z`r)r5k<@I;;@T_c%1o&w&uSa?yVcpw5I1jH)u%~E9}UlqCF_C`zBH3n#&`N(LNLH>GgW*=?Jnd{y>_P zUfD%8If`a>6G9aex(h3Jp>gxKzSQKtBwCRGR=-mqnNfB|J#{3kS$nbjDHV@67Fq+# zja|k;FwrEzSQyLcVLu5#xa`IbM@&=ezkm89GeA7gw*9fxu|U*S-wj{Fl$e4&sAe}? zZStrst<#j1$2sQs$wcv3${l=1sl23{r6_%7FueHgPYRywmNI3VeazLgAor=W%-M@P zlk!}kaC+Pf?;d_8?x;dKvSG1fAUq@K(FodiIy$8+^RpO*J}|I&C(}v#9Viw{gB(Uhx^wD6z@c9;^~di z&VD}}jHPJVTg2isr^Fht`-aMn3>l99h`|eZbP$a2a$z4io4Br9$D6Vb(rxR?sorr0 zH+KrQr^I_ik&~_1YaZKG=&QIq^vpp?il5sNlhvLi&la6#)zG`it9-e!Uy#s)Jy=j_ z2ZG$PopwtK{sOuix%Gns#yG3_Umw^8p;7u_3zvif#+)?oCd}&7rOnI&1Iny#t z**6xC8=!a3^uJ;w%f@ln7NbX-;tk)sc#DV5MyL(3o2Lb(j~^10hwCAfje{riwU{uP z$(M~9@4>=8QWXD#YHJhVSS;43DQ=g~pS?CVVbh)vDwz3D1|h2@v9E2L%3@G@Zt&~~ zyt)2B!?Ez}vvUtHH7Yh4T4>VxyV7n5?8w#n-BUqZN3PDHK}WXhDZ*$AyA>jg&HX`n zM3fX1D4V1nnqKa6)6U_6E%X2va#y}WE5mQ0Ro_owJ5&8XmR1q6bv!H~oE&@E5Oky@SUzc)d}XKWJyRt_1;{R^`9_2p4PRVDHZO5d0FAFS3fT5Q*y zTI@98!ggX5`xK)dG)jX`_o)t9+~8e)p@W$Td!@Rap2@EK9{-p{d}~K?0G35(ogWJUxF=^gdUdFEF?O|X zWW2NbTK{jGS&Udp80iSWarKc(u-BNZukco-=@vl0Op55O@~SHEh|+X=c_@u?Cj%s> zD0XYwEr&=a1B^VJ=_B5m)UPb+-V#{67M-%7%$u?R0Pv@e9(^S)5DJ{?!I)4lzp2Cy zu*3MYMAKV>(qs+N)5sO%V6kFroqLcnHVR?^75TlzNjGI{9AH z?_c<|XX>QL%G^HTMM;&N>MeFa=B|(3)@Fa`%C`!7$p@M(mq&mM*ZfOB3F(dIfsgc;t6Edc<_QlVP9D>>+G z9(9a-`*sm8%$|EAAi~rBugTHw_wUnX__s%eV}BraTQF%V>uvWltjO2WGvA3dxIB`# z4G-^45R?i0p&MVq)CF}e*~W@6R$_#(sKai!0^@_q|>wch2;jFQ|)-Vb2mh+Ain#B!$aSK zgM%NxfB(MRZ>W|n^4EvyQ{Lj0enZ)Fq-_ox5-+~6@E)2cz{)=Tzg_tc)BZ1Z3{-_s zfV<`zR)@ctG;t4Od@A2E#Twi_4PhQ2LU4~K7~b^s?6YoAsJp>6Y=by@>0cbZva(Vy zbml|wSW6P**3uV71{~N}nJuoUsQ5;m_%9^@2pIJXO?#mJ`&B6TzIZWPx5g}3+T|0^ZBmz$t5%Ur1?3w22x^7u=(PV>RJdI`5- z^5^Wpb43lPll8hFUv~BI-zdBe2yHOv7u@*7&++0#R;vR_BVT|)XF5ik6MH~$bg;6r z;<`C&{WBWUsQwm}f+pItHnl}m?H-;4%917|w!CQhqGSC%M#99CRVu=2-ZvfdRh#nV zN?ooPuB$*F`~PZaU`p;UwSw5cL^-vBGu!X4KoNq$wxIO?Z{>})&HIxWyH+xol?#nJ NWnqUZF}w83zX2P`_&Wdq literal 0 HcmV?d00001 diff --git a/docs/apache-airflow-providers-http/img/connection_username_password.png b/docs/apache-airflow-providers-http/img/connection_username_password.png new file mode 100644 index 0000000000000000000000000000000000000000..6e36e77dd4cb48f62107a3f654648587bddc58e7 GIT binary patch literal 4761 zcmchbcU03^o5!P|;P@gUqabyRBPa|Y0trZS6j4z?q$vUcqA-y*h?E3EU@S;qMnF0u z(g{VRlYmN5T9iFsdW&WcE+Gf+@j6Uopb@@XX=e3HCon2biA)vxaT*S!RJ3xgOLI`DzV z&2xKseS&`%p+IgvaCdjl&(1!j0f*;S(Sz>pj$4ZtC$~^wE!F+d(mk)c>gXq@hPy(@Cb59eAf4h1K$Ny zvvJ3q1Oy^s9O<$3lCe(nuy`l_}xUI_%)8AUWqzr*n+Y^pZitg6I zLeC6Vdf;g*nKB{9kSgh{rD%npDisdYg zw&r?`wq!pR5I#1!Ia65DJ<=ROmi8NpeRfDXd7>*VMgQd1EP~tCFV)(LekLrNK0(UJ z;64kL!JLjdt|rG;K@e?IRf<=N>*m|~;XX4xJwrpbb9|6jLlSbOdxrb-<@r)Hl|Lku zyO1VGn%Ip;hs?Xuv|qAGiK-`cS1CxX8)2p6e}u}Kw7*m#Z9=8Okk%#kX?ol(W)Ij@ zlH{JYR#{wXOxW4e(fJ+3JGGBc$+`i)#D>a>it$Cci%E-AKl-?<{o8xj(k&%2#4@|$ z&Y0hFb#>LdvNb;%qi-L$ooANSouuZnytG6=vz=grQ-eUp&667^+Ts;a&Eeyy+mplP z2)Z6>J#QV(@yaZ;E-`w%|Inw_`o#~ckGMri>-DNP@j*W-?5{1u(y{EEOQDCd z%aDe-g91V~+FvTQx9nnX%l#QJ+Y0W?mTJXJoTg+7bT^2j%$0sEm}s`W-f+_M>G&A(XNP6 zho85#oVVSEs>>*1(3=JGdmuMWvJbe)B$MM})-O$7P?K?$P{UZzf@bkm%iDH!_FaQn zhEX?C;GRb0924Wmp0aTPh@kax+&-9Ua5Tg5Qs|rG5W{YvgBwrWhnDlebNt z9hk5$ax&VOrDic^M?xu%yB!#9bM%BUKTL%7=;$PNdwnv3k%%U1^n}gMY*br3(fIgG zsC`o*#JFQM-+%GF069VFZaBj@1(vPlJu#LcZ<4x(=s#+iw1*#Zp(Y+$s)gOKpy^Mi zhh1JEh{`|oY+<3$n<1MnKkys2Z9Ux)@ z%BUKpVONr&Z|;;k8*i)*dNQ|f3*??JoyK&EP}~>~JsA>|8_DWt1a3eeJ-N5|VI}jl zGMR+Q*(I5YLM_6Y15OHtRX9Q~gnh||_Pq@YxK;hNF=yGk z|BgOMZ*$IA0Jii?4~WJH?1^JPu2w2Qf;Vo4H_G)Dd=BXh=iVR}^TRHM+Dgw^$;8e` zP$DG*Rzz~=_oft=tOs#9Y;VtPoe-f5S=C96a!oeMBG2Z&RVeC*f6spIFLmbFo?@NX z-$(3iy%C@2DEf0N_~e%_5<`V$yEaGhHzJNH86j{SAA4;j8F&G|99)9ZRFv$`dcdBi zu9uI*4r}Vk6o&h_d#IO&T@D>(j-DTBFx|o~vD0mqTH;;PV4m3X-3Q_Y2ELXyjh<)exI zAI=l-``$WPgJko zCOa?C2+evBNE_aoFXhY&d7nehVw^muQmF~`&aPISZY>J6*5|qedh&Tac5AJy9(QH( zm0IPMsCqSB_E1ax)UWj`*Btm3XMM|gi9PO4$$nVHgXxSCWr!5{^Jx6wl2xuh-mjxl z>sJSySG1r52UEhrsa5?&_IB9Q zkOnAEI3>)e{ID|3v6d+A3&aAo0YTqdcwwrOj>!D3vE=2>L>6#hcMp(G$SV=x=see* zezA@FgKI(7{j0FEvooJSAk=~-`Y5$jtM)OXL5uDODAYAE75gF&V31wiqv=~Re=1=G z$f=$(BhUK2el;TiLuQO3QAjt*f0xSq=BS^D*ykxWI+KMvgkV9i;VDWRx9}jNfL%DaGjA|W{vQ$NpS3m4=I-6QJZ}L}~wP_n}m)v?XSj1Nx_PDiK^uj29eT?=~Mn-f>J)|u$Y0Q?t#MSwG^ghl$AI*YQh zPGZ4#`&YAys;bE{_ed{f(V#TgRDd`-wlxn=(T4zRi8N6v&%LPE2D@vdn9hvyJOK)0 zo5bNQwNxHk*?5Vn)$syP6ln|k`s~Ko5m3|5!!&g$uh4iW@cxxT)+;#8UzfUEN6*nH zv?yuiGB};hJ>V9!_DASU zA8w_fYPQUpz`4E(xJHivGHtDklg(P_KGgdyG^%6DtOoA5MV+o&AQo=N*DjlLvwkSu zGld5EG;sN|JgWQ6HPoknI{`GAsz0FpaO|ptUZB5(wy)cH05C!;>Dbi)^kQzahB0rOS2)^^!pMBGqd>KLLc`E3qWiy%&Z<__TUz@VEwbOwXrxKJ5u3YvNnYXhWJ7bY-C*PQ~_Dzw@4q zcEWY^tbhD%ABytj1gg=ULuR4O$zh`M%fx*LCniZXf$I-xzzlPdW0oDL2W@J&Z{NOk z$17TQhaYX%CTXaz%nr&Mr=BHx4{sWI&DHu?bJMNu(Wi{(rNl`=i|iWidjZ+8*_hcn zTwe!ktI{1fRmN8>#Q2FIXJe)P=pUDckPlW`Vq_64XHt3^j*Y=&;I`3a(iH&Xk_p(& zRZbq87O-IFSJJoAH)Dyoi{Du~p#oD;jJFbo1WPg__Q4!Y$a6zDX6NJSxqW8aS7EDRj9^xD&NQP6Uc9bgIU$EzSuy7XkUDeWJ73?Nt>GqqxAGePKGm z5R9MD)q&%jw?d1gs+`u}vRGu(?95C154u4x6d?7k=NC@JpLJXuexIL}mF2ABTxfv- z*xT6+5O2abH*6Z%;@j@Ch9He$Y&03*q@cESMj*l(~?m8SC=FfD5o|Ix9xc zsD!xj0DRn^TkDCQ2pXG$oMrmnPw4oHL(KIfaA_uK=i|9Tm-f@spV3u+%1~@-kPTXM z-1b%NI^M9!GB+J35wTwy-(h+ShP3?`9cn=y1(?>As)tqJN~zsf7_rHMBRAgFb9$Z$ zp|w9G&miaP*bk;e`$`ZX9ZflcFmH-%&o|G_5jko22F%Ej4`_{%HO;ybVCW39*?lDqn1(u<{qD}X_YW2lcjDZtPyo^xHQn2o zV|6)CWJLAN{p;M+`8lb3EggwO?8X=;Mh~+xz~*eSC$A*D7JJ_@a`|AyqMX6$TUst1 z-FH@|vZjY>ywQlQSk){!=dO3DdTBA}DsKmgj0#@{qC13d(_6)PxV|>P53KaFMIeum z#4>N@@x$y~{zh2+mOga1(hc=BBI)8+aCBm&f};?~S92I1gGFyw-@CR!mgOcs{v9mw zSgXNV(bgh+f`Wij{0agP0++y&v?a~&`%-~MAxp`hbPV!2D#7{mX21p%0%^%#S95Ea&2 zqmQilODU|X*w>#nCFptQWHS?)&D*z2t6lTnPS;E>`|9+mBR5z>5m-R2$vMVp$@58? zC)nXvUdvIDk7z0aZ4iF6t&|#3ZEs`7Mj;%!@ot)dR zs}K|U;waI$QPms)nQDDfWZ3Y19^^!(0-7FonL^qnn`Ig1NNf9!4JdmBb|h<|m+$G} z>NBCyG2cr6D5(*fov^^Dmgl~HEB>&fF>K${(N2r99r$UmauCfl({Zv~mmEp-e^j_J z1lg+J(TCUF{4mwW6U5E=&e{GZ$=7-u7Wv&+Y%^LZMLr`C+K}Y*^~2NV!phd@<90tm zrJ#vn@XMEf6^%GjRyfl%wR+Q*2 zIy+?1^PStpdc&<^FC^2Ikm9U=TSukp{J%jB&r7>^DgT*=YK7eXx}-UiZLwfo1*9JQ2&Vsd b1GBE*^^2vH0QXn|ulCwy6NBQ54!`{!fAs15 literal 0 HcmV?d00001 From 9c9edb05138f5fa46077e679848c7d2c5f9a378d Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 08:18:23 +0100 Subject: [PATCH 230/286] feat: Implement auth_type and auth_kwargs in the AsyncHttpHook --- airflow/utils/json.py | 10 ---------- .../http/tests/provider_tests/http/hooks/test_http.py | 7 ++++--- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/airflow/utils/json.py b/airflow/utils/json.py index eb3cd40941197..a8846282899f3 100644 --- a/airflow/utils/json.py +++ b/airflow/utils/json.py @@ -123,15 +123,5 @@ def orm_object_hook(dct: dict) -> object: return deserialize(dct, False) -def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> Any: - """Safely loads JSON. - - Returns None by default if the given object is None. - """ - if obj is not None: - return json.loads(obj) - return default - - # backwards compatibility AirflowJsonEncoder = WebEncoder diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index fe6b8f882f2b7..1e944dda1d090 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -384,9 +384,10 @@ def test_available_connection_auth_types(self): auth_types = get_auth_types() assert auth_types == frozenset( { - "request.auth.HTTPBasicAuth", - "request.auth.HTTPProxyAuth", - "request.auth.HTTPDigestAuth", + "requests.auth.HTTPBasicAuth", + "requests.auth.HTTPProxyAuth", + "requests.auth.HTTPDigestAuth", + "aiohttp.BasicAuth", "tests.providers.http.hooks.test_http.CustomAuthBase", } ) From cc8b30f818b3294c11be91a155728f7f079b76d3 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 19:04:06 +0100 Subject: [PATCH 231/286] feat: Add tests --- .../provider_tests/http/hooks/test_http.py | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index 1e944dda1d090..b6f43995b979d 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -437,6 +437,32 @@ def test_connection_with_extra_auth_type_and_no_credentials(self, auth, mock_get HttpHook().get_conn({}) auth.assert_called_once() + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_string_headers_and_auth_kwargs(self, auth, mock_get_connection): + """When passed via the UI, the 'headers' and 'auth_kwargs' fields' data is + saved as string. + """ + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + login="username", + password="pass", + extra=r""" + {"auth_kwargs": "{\r\n \"endpoint\": \"http://localhost\"\r\n}", + "headers": "{\r\n \"some\": \"headers\"\r\n}"} + """, + ) + mock_get_connection.return_value = conn + + hook = HttpHook(auth_type=CustomAuthBase) + session = hook.get_conn({}) + + auth.assert_called_once_with("username", "pass", endpoint="http://localhost") + assert "auth_kwargs" not in session.headers + assert "some" in session.headers + @pytest.mark.parametrize("method", ["GET", "POST"]) def test_json_request(self, method, requests_mock): obj1 = {"a": 1, "b": "abc", "c": [1, 2, {"d": 10}]} @@ -724,7 +750,7 @@ async def test_async_post_request_with_error_code(self): async def test_async_request_uses_connection_extra(self): """Test api call asynchronously with a connection that has extra field.""" - connection_extra = {"bearer": "test"} + connection_extra = {"bearer": "test", "some": "header"} with aioresponses() as m: m.post( From 721a2e140b212bcc5ef5632e64c44b3443c01a1b Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 21:04:22 +0100 Subject: [PATCH 232/286] fix: Add header and auth into FakeSession test object --- providers/http/tests/provider_tests/http/sensors/test_http.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/providers/http/tests/provider_tests/http/sensors/test_http.py b/providers/http/tests/provider_tests/http/sensors/test_http.py index 78a11e15bb7c1..47af8f49c48cf 100644 --- a/providers/http/tests/provider_tests/http/sensors/test_http.py +++ b/providers/http/tests/provider_tests/http/sensors/test_http.py @@ -238,10 +238,14 @@ def resp_check(_): class FakeSession: + """Mock requests.Session object.""" + def __init__(self): self.response = requests.Response() self.response.status_code = 200 self.response._content = "apache/airflow".encode("ascii", "ignore") + self.headers = {} + self.auth = None def send(self, *args, **kwargs): return self.response From 9508efcfe27bc78555363a0e006b8e820089f6f6 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 21:14:54 +0100 Subject: [PATCH 233/286] fix: Use default BasicAuth in LivyAsyncHook --- docs/spelling_wordlist.txt | 1 + .../livy/src/airflow/providers/apache/livy/hooks/livy.py | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index e9574a359266f..581d5c30a761b 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -25,6 +25,7 @@ afterall AgentKey aio aiobotocore +aiohttp AioSession aiplatform Airbnb diff --git a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py index f185ccdfbe33b..e9e1a902ae610 100644 --- a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py @@ -93,9 +93,8 @@ def __init__( auth_type: Any | None = None, endpoint_prefix: str | None = None, ) -> None: - super().__init__() + super().__init__(http_conn_id=livy_conn_id) self.method = "POST" - self.http_conn_id = livy_conn_id self.extra_headers = extra_headers or {} self.extra_options = extra_options or {} self.endpoint_prefix = sanitize_endpoint_prefix(endpoint_prefix) @@ -510,9 +509,9 @@ def __init__( extra_headers: dict[str, Any] | None = None, endpoint_prefix: str | None = None, ) -> None: - super().__init__() + super().__init__(http_conn_id=livy_conn_id) self.method = "POST" - self.http_conn_id = livy_conn_id + self.auth_type = self.default_auth_type self.extra_headers = extra_headers or {} self.extra_options = extra_options or {} self.endpoint_prefix = sanitize_endpoint_prefix(endpoint_prefix) From 9ab5bdd7f5349dce4a67a268183252b0f9d42316 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 16:36:22 +0200 Subject: [PATCH 234/286] refactor: Removed docstring for removed json parameter in run method of HttpAsyncHook --- providers/http/src/airflow/providers/http/hooks/http.py | 1 - 1 file changed, 1 deletion(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 9cf5d4983dbc5..932aeb31d29c0 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -397,7 +397,6 @@ async def run( :param endpoint: Endpoint to be called, i.e. ``resource/v1/query?``. :param data: Payload to be uploaded or request parameters. - :param json: Payload to be uploaded as JSON. :param headers: Additional headers to be passed through as a dict. :param extra_options: Additional kwargs to pass when creating a request. For example, ``run(json=obj)`` is passed as From 20f15415d0ad015d2b2bb9c10953add80d4d5b28 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 16:38:39 +0200 Subject: [PATCH 235/286] refactor: Aligned HttpTrigger with version from main branch --- .../http/src/airflow/providers/http/triggers/http.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/providers/http/src/airflow/providers/http/triggers/http.py b/providers/http/src/airflow/providers/http/triggers/http.py index d25d3a55cfb5b..ec9780bdeab49 100644 --- a/providers/http/src/airflow/providers/http/triggers/http.py +++ b/providers/http/src/airflow/providers/http/triggers/http.py @@ -73,7 +73,7 @@ def __init__( self.extra_options = extra_options def serialize(self) -> tuple[str, dict[str, Any]]: - """Serialize HttpTrigger arguments and classpath.""" + """Serializes HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpTrigger", { @@ -88,7 +88,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Make a series of asynchronous http calls via a http hook.""" + """Makes a series of asynchronous http calls via an http hook.""" hook = HttpAsyncHook( method=self.method, http_conn_id=self.http_conn_id, @@ -165,7 +165,7 @@ def __init__( self.poke_interval = poke_interval def serialize(self) -> tuple[str, dict[str, Any]]: - """Serialize HttpTrigger arguments and classpath.""" + """Serializes HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpSensorTrigger", { @@ -180,7 +180,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Make a series of asynchronous http calls via an http hook.""" + """Makes a series of asynchronous http calls via an http hook.""" hook = self._get_async_hook() while True: try: @@ -193,7 +193,6 @@ async def run(self) -> AsyncIterator[TriggerEvent]: extra_options=self.extra_options, ) yield TriggerEvent(True) - return except AirflowException as exc: if str(exc).startswith("404"): await asyncio.sleep(self.poke_interval) From 16b2f77d56ca9fd38c6328d8983f4af4b8584af2 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 17:03:29 +0200 Subject: [PATCH 236/286] refactor: Changed docstrings in HttpTrigger to imperative mode --- providers/http/src/airflow/providers/http/triggers/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/providers/http/src/airflow/providers/http/triggers/http.py b/providers/http/src/airflow/providers/http/triggers/http.py index ec9780bdeab49..5975389830f36 100644 --- a/providers/http/src/airflow/providers/http/triggers/http.py +++ b/providers/http/src/airflow/providers/http/triggers/http.py @@ -73,7 +73,7 @@ def __init__( self.extra_options = extra_options def serialize(self) -> tuple[str, dict[str, Any]]: - """Serializes HttpTrigger arguments and classpath.""" + """Serialize HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpTrigger", { @@ -88,7 +88,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Makes a series of asynchronous http calls via an http hook.""" + """Make a series of asynchronous http calls via a http hook.""" hook = HttpAsyncHook( method=self.method, http_conn_id=self.http_conn_id, From d852ccb11805398935782f5e075009a8034ba019 Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 11 Apr 2024 12:15:06 +0200 Subject: [PATCH 237/286] refactor: Updated docstrings of serialize and run method of HttpTrigger --- providers/http/src/airflow/providers/http/triggers/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/providers/http/src/airflow/providers/http/triggers/http.py b/providers/http/src/airflow/providers/http/triggers/http.py index 5975389830f36..d30f41990f5b0 100644 --- a/providers/http/src/airflow/providers/http/triggers/http.py +++ b/providers/http/src/airflow/providers/http/triggers/http.py @@ -165,7 +165,7 @@ def __init__( self.poke_interval = poke_interval def serialize(self) -> tuple[str, dict[str, Any]]: - """Serializes HttpTrigger arguments and classpath.""" + """Serialize HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpSensorTrigger", { @@ -180,7 +180,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Makes a series of asynchronous http calls via an http hook.""" + """Make a series of asynchronous http calls via a http hook.""" hook = self._get_async_hook() while True: try: From 1ea24db452eb18b78d66e4b7b009089dcc99d56b Mon Sep 17 00:00:00 2001 From: David Blain Date: Mon, 6 May 2024 13:09:59 +0200 Subject: [PATCH 238/286] refactor: Moved get_connection_form_widgets method from HttpHook to HttpHookMixin --- .../livy/src/airflow/providers/apache/livy/hooks/livy.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py index e9e1a902ae610..4f1e3934890bd 100644 --- a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py @@ -85,6 +85,10 @@ class LivyHook(HttpHook): conn_type = "livy" hook_name = "Apache Livy" + @classmethod + def get_connection_form_widgets(cls) -> dict[str, Any]: + return super().get_connection_form_widgets() + def __init__( self, livy_conn_id: str = default_conn_name, From c6e33c2a1ed602eaeeef59b3434be350f85129ff Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 7 May 2024 19:34:37 +0200 Subject: [PATCH 239/286] refactor: Enhanced extra_dejson property to allow load string escaped nested json structures --- airflow/models/connection.py | 2 +- .../livy/src/airflow/providers/apache/livy/hooks/livy.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/airflow/models/connection.py b/airflow/models/connection.py index 01df1626657da..19aeaf9e63089 100644 --- a/airflow/models/connection.py +++ b/airflow/models/connection.py @@ -432,7 +432,7 @@ def get_extra_dejson(self, nested: bool = False) -> dict: self.log.exception("Failed parsing the json for conn_id %s", self.conn_id) # Mask sensitive keys from this list - mask_secret(extra) + mask_secret(obj) return extra diff --git a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py index 4f1e3934890bd..c4684bc5b63a0 100644 --- a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py @@ -89,6 +89,10 @@ class LivyHook(HttpHook): def get_connection_form_widgets(cls) -> dict[str, Any]: return super().get_connection_form_widgets() + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + return super().get_ui_field_behaviour() + def __init__( self, livy_conn_id: str = default_conn_name, From 5498b3311238649ea20cbb5545d319be25c705a1 Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 8 May 2024 16:02:23 +0200 Subject: [PATCH 240/286] refactor: Changed conn_type to ftp in test_process_form_invalid_extra_removed as http as livy do now also have custom fields --- tests/www/views/test_views_connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/www/views/test_views_connection.py b/tests/www/views/test_views_connection.py index 1e21dc4856ed1..19a36c0ac6b39 100644 --- a/tests/www/views/test_views_connection.py +++ b/tests/www/views/test_views_connection.py @@ -462,9 +462,9 @@ def test_process_form_invalid_extra_removed(admin_client): Note: This can only be tested with a Hook which does not have any custom fields (otherwise the custom fields override the extra data when editing a Connection). Thus, this is currently - tested with livy. + tested with ftp. """ - conn_details = {"conn_id": "test_conn", "conn_type": "livy"} + conn_details = {"conn_id": "test_conn", "conn_type": "ftp"} conn = Connection(**conn_details, extra='{"foo": "bar"}') conn.id = 1 From 7e2168231dc59c2f1b03882660db35782587cf69 Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 18 Jul 2024 20:14:20 +0200 Subject: [PATCH 241/286] refactor: HttpHook now uses patched version of Connection + added test which checks when this patched class has to be removed so we don't forget --- .../src/airflow/providers/apache/druid/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py b/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py index 7585be9880dc1..d00c3c8da7757 100644 --- a/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py +++ b/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py @@ -37,3 +37,17 @@ raise RuntimeError( f"The package `apache-airflow-providers-apache-druid:{__version__}` needs Apache Airflow 2.9.0+" ) + + +def airflow_dependency_version(): + import re + import yaml + + from os.path import join, dirname + + with open(join(dirname(__file__), "provider.yaml"), encoding="utf-8") as file: + for dependency in yaml.safe_load(file)["dependencies"]: + if dependency.startswith('apache-airflow'): + match = re.search(r'>=([\d\.]+)', dependency) + if match: + return packaging.version.parse(match.group(1)) From 052ac7c1b062f1216d98e81aed10b31f147a8653 Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 18 Jul 2024 20:56:42 +0200 Subject: [PATCH 242/286] refactor: Fixed some static checks --- .../druid/src/airflow/providers/apache/druid/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py b/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py index d00c3c8da7757..18870506f868b 100644 --- a/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py +++ b/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py @@ -41,13 +41,13 @@ def airflow_dependency_version(): import re - import yaml + from os.path import dirname, join - from os.path import join, dirname + import yaml with open(join(dirname(__file__), "provider.yaml"), encoding="utf-8") as file: for dependency in yaml.safe_load(file)["dependencies"]: - if dependency.startswith('apache-airflow'): - match = re.search(r'>=([\d\.]+)', dependency) + if dependency.startswith("apache-airflow"): + match = re.search(r">=([\d\.]+)", dependency) if match: return packaging.version.parse(match.group(1)) From 8e73addb0dcdb32455fb685315669c33d2f65ee1 Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 5 Feb 2025 13:54:13 +0100 Subject: [PATCH 243/286] refactor: Removed wrong modifications from main --- airflow/www/forms.py | 25 +------------------ .../providers/apache/druid/__init__.py | 14 ----------- .../src/airflow/providers/http/hooks/http.py | 20 --------------- setup.cfg | 0 4 files changed, 1 insertion(+), 58 deletions(-) delete mode 100644 setup.cfg diff --git a/airflow/www/forms.py b/airflow/www/forms.py index b69184c6590c2..d19af7473f109 100644 --- a/airflow/www/forms.py +++ b/airflow/www/forms.py @@ -177,29 +177,6 @@ def populate_obj(self, item): field.populate_obj(item, name) -class BS3AccordionTextAreaFieldWidget(BS3TextAreaFieldWidget): - - @staticmethod - def _make_collapsable_panel(field: Field, content: Markup) -> str: - collapsable_id: str = f"collapsable_{field.id}" - return f""" -
-
-

- -

-
- -
- """ - - def __call__(self, field, **kwargs): - text_area = super(BS3TextAreaFieldWidget, self).__call__(field, **kwargs) - return self._make_collapsable_panel(field=field, content=text_area) - - @cache def create_connection_form_class() -> type[DynamicForm]: """ @@ -247,7 +224,7 @@ def process(self, formdata=None, obj=None, **kwargs): login = StringField(lazy_gettext("Login"), widget=BS3TextFieldWidget()) password = PasswordField(lazy_gettext("Password"), widget=BS3PasswordFieldWidget()) port = IntegerField(lazy_gettext("Port"), validators=[Optional()], widget=BS3TextFieldWidget()) - extra = TextAreaField(lazy_gettext("Extra"), widget=BS3AccordionTextAreaFieldWidget()) + extra = TextAreaField(lazy_gettext("Extra"), widget=BS3TextAreaFieldWidget()) for key, value in providers_manager.connection_form_widgets.items(): setattr(ConnectionForm, key, value.field) diff --git a/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py b/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py index 18870506f868b..7585be9880dc1 100644 --- a/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py +++ b/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py @@ -37,17 +37,3 @@ raise RuntimeError( f"The package `apache-airflow-providers-apache-druid:{__version__}` needs Apache Airflow 2.9.0+" ) - - -def airflow_dependency_version(): - import re - from os.path import dirname, join - - import yaml - - with open(join(dirname(__file__), "provider.yaml"), encoding="utf-8") as file: - for dependency in yaml.safe_load(file)["dependencies"]: - if dependency.startswith("apache-airflow"): - match = re.search(r">=([\d\.]+)", dependency) - if match: - return packaging.version.parse(match.group(1)) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 0a785930de79b..72ca10d796465 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -228,26 +228,6 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: ), } - @classmethod - def get_connection_form_widgets(cls) -> dict[str, Any]: - """Return connection widgets to add to connection form.""" - from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget, Select2Widget - from flask_babel import lazy_gettext - from wtforms.fields import SelectField, TextAreaField - - default_auth_type: str = "" - auth_types_choices = frozenset({default_auth_type}) | get_auth_types() - return { - "auth_type": SelectField( - lazy_gettext("Auth type"), - choices=[(clazz, clazz) for clazz in auth_types_choices], - widget=Select2Widget(), - default=default_auth_type - ), - "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), - } - # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index e69de29bb2d1d..0000000000000 From 60e0ca7f13c6e08819c2cb8081b70343affa1f45 Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 5 Feb 2025 15:14:10 +0100 Subject: [PATCH 244/286] refactor: Removed unused import from forms --- airflow/www/forms.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airflow/www/forms.py b/airflow/www/forms.py index d19af7473f109..7028e2026e449 100644 --- a/airflow/www/forms.py +++ b/airflow/www/forms.py @@ -33,7 +33,6 @@ from flask_appbuilder.forms import DynamicForm from flask_babel import lazy_gettext from flask_wtf import FlaskForm -from markupsafe import Markup from wtforms import widgets from wtforms.fields import Field, IntegerField, PasswordField, SelectField, StringField, TextAreaField from wtforms.validators import InputRequired, Optional From 36860e9314d8ce9713378b1806a957a64a0abba9 Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 5 Feb 2025 15:15:09 +0100 Subject: [PATCH 245/286] refactor: Removed white line from livy --- .../apache/livy/src/airflow/providers/apache/livy/hooks/livy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py index 6d154605440f8..a97c34a23f4f4 100644 --- a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py @@ -18,7 +18,6 @@ from __future__ import annotations - import json import re import warnings From dd2cf0038e4d64aacbfc251db740ac5f002f76bc Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 5 Feb 2025 15:16:24 +0100 Subject: [PATCH 246/286] refactor: Fixed get_extra_dejson in Connection --- airflow/models/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/models/connection.py b/airflow/models/connection.py index 85d3c7cb578c7..a8b9bb87985d8 100644 --- a/airflow/models/connection.py +++ b/airflow/models/connection.py @@ -432,7 +432,7 @@ def get_extra_dejson(self, nested: bool = False) -> dict: self.log.exception("Failed parsing the json for conn_id %s", self.conn_id) # Mask sensitive keys from this list - mask_secret(obj) + mask_secret(extra) return extra From 3fb5bc13e6d88decd1c74d49ff0b3124df056cf3 Mon Sep 17 00:00:00 2001 From: David Blain Date: Mon, 10 Feb 2025 14:41:31 +0100 Subject: [PATCH 247/286] refactor: Moved http connection images under http provider docs --- .../img/connection_auth_kwargs.png | Bin 9623 -> 0 bytes .../img/connection_auth_type.png | Bin 14199 -> 0 bytes .../img/connection_headers.png | Bin 5256 -> 0 bytes .../img/connection_username_password.png | Bin 4761 -> 0 bytes .../connections/img/connection_auth_kwargs.png | Bin 9623 -> 0 bytes .../connections/img/connection_auth_type.png | Bin 14199 -> 0 bytes .../docs/connections/img/connection_headers.png | Bin 5256 -> 0 bytes .../img/connection_username_password.png | Bin 4761 -> 0 bytes 8 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 docs/apache-airflow-providers-http/img/connection_auth_kwargs.png delete mode 100644 docs/apache-airflow-providers-http/img/connection_auth_type.png delete mode 100644 docs/apache-airflow-providers-http/img/connection_headers.png delete mode 100644 docs/apache-airflow-providers-http/img/connection_username_password.png diff --git a/docs/apache-airflow-providers-http/img/connection_auth_kwargs.png b/docs/apache-airflow-providers-http/img/connection_auth_kwargs.png deleted file mode 100644 index 7023c3a7a072f965f9dd053f77c5a5d64b396f47..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9623 zcmcI~2{_c>+jom(n~(ixI5hS2r&k-xWWM^15qC|U8&@yI zp4VbIoNz&&;S|%$vllM#zw{D3bLeCYFTFO4DogYQZ3Q#Q(MB#`@|PP{w$;0sjC|gQ zwZ00``u#3Nh5MrM%WwU6MOC!GEa0bjzF~#+9_k$RUt@1t_`%yIg(IZq`(>tL>=B>- z{$n_3=<@gXa@sCopHqhy8ve(KHqs61*aGb{L`~`|6OlkDb>1ArkdEJf(u%A*3sMN|@KU37UWK z`S!;+d!t~o=`Y)wx6bxn)?k%F?VSe!d^L0ZZ$r)AA%p$*$)o=^r})M@ajS#xngony z*Lg06eEks5LtY{{Vqn>ksZNFGX*0{b@+ z#0=Qog@Hj;9>jhwU?syRst>MH=7w=oH_sMKnFK}Mwee}-tenP+6yF*CgOf6#l9m~nV&Lm0k^g3ly*;YJZL)1?XSHiuScftr z#fh-AL>l<*5_9^a$4@YkHx|3Pz$brZ?SLRnt1aptkF^SCGF4So9T@qEx}W?S9Hhje zjy(hdWv_M_&Bzq|=dy@X#CzWgO-|?oQ*sLLA1p%)S2ocGlYcv~*k1^Hr`qWbrW`D1 ze39xdiiP$)c~TF*B%`J~(|0HvDL6}UXmC$AX=?Uck=2jS&$IIKgwo{Qg;dPIWDXvG zMpEM11PV^VaExN#`GO17fs{SQ=DebuB3m@@iB@cQBh_QnXofNT+W;eRTnE;5qA}H_ z{zc{+2E+#+k7QZLTNt<2>=yczPV0#hS6B%9kd3{-YcCyL!Ul_+k; zp0DzSG_^E_A8QO|+-h}b+8Fg8y{FmTOUkQv9CZ=kv~^}^Lc*}YvMAAFoY;}%=|t1SKU^tU@;Fdz77c@R*1UA%Eu_Y3tz$}<))Yv z-3pd<>}&eTERFRZ4--c09NoKpl|RsR_t%vKf8p&x_J~HJ$(xT zo$oS|$Z5$+e80CnXSF^*nB_j*ndQH~H}>(#(cCVJdOvxk)i2Uz0R!(`MoYbx$5G&7 zbo(K0GjYpF_4f8Q`5Qg(SozU`tNz$D_S>4h8^zdEijt6gb!toDK<=$u;A)fy!Y`h(--=f&9P7^^!9I4 zS6lByogoK?=((~MUPch>kn9^#u;HhK@Ta@&)yh^MbQOJm+xd0L-}4_>pNLW6Rf${5 zn7B7A{qPGNpZ|_4r;-v{iFBo2Xh>#+3G;|{E(gSV34$r{E%Y!9`-0-)9E>o*$tJE)bX!7Ho6BjelTH=PL$J?c1 ztcS}V=!m@Z84KnZ_Y8M`u0~g~R_#1AF0&rzJU`m7zjtE99k_*)p*%{1*Zcfkv+5{6 zR8O`>jGTCSrfnv=TK~22s0ZyRx?UYb5c6=Y2V+m39wWHzVWSoL=?cN?3qvg)87ltu zzh9h(5?Ru&?C-2D*OsZIjn3QJB{cg>n)^q~LN{gBOxrEsTCs>YYEnTcVzXZ93bGh0 zzrFi`%hISd+fZlbYVcBS(%oG>bOD1C|O-DZ0gNyeZ7| z;uZHP-86$>`0iguVGr-XX5S>WNe=s()OWpcM50@!i6`t8{wEP90XuQUY@7_G<` zMX#lM_TFlzVc))-lM#s5Zg<<^h7V_c#6Zqm$~pbCRB0|7%Gw{i@E|bt-6%$}x^(MT z-Qu_`f0Gqj1Z!{SyJoQ}*|2k|Qd9r z>FA|&f!~@IC8ok-O}>jCIHvPUqjzf_FCfwq>*MTI=x)nt{8ZFBRE`|3^>UW9s6CB; z+3O(2M`ZC5LsVYC9G_8Wt~q;JO}+n&`_z}P(KFTfvQsHJbB!C-1>(qCCFW<61a)mJ zfOn$ExQ(<4t2>BmWijT^^%ryB)2zikhe00HJII3zq&#Kg*d$5IyFLcNnc?t;1aLJx zoJ_6;UlJf$ak1T*Vm)Q#Qln0~QLgivXwu@@_BCrF#V znBPwJa3cfanoY|cE6bmaJ@LQff|QJNDw<>XEWazOvKK!+1>2;7y$rtLFc@1}@5Cq^ z5Oh90X3=-{nICF79SBuUIwF7??)EGxo70_>P)j;b9!5H#)Hl_1 zveiRRBuu#?H!9SwzaroBq4NfU|RlwHPbf$H1%Avp5&z!DSXF0SkUDbIQ zN5eK3l*mwE#zO9}ZBKBzYRHFZBD9^0n9#M-o()i^yOx{safWYyQzNa#c6BHxXF}Ja z=TzVIm+=V@q@7h6h!}-c`>^rC(p-m*cP7Kja zXnQ)RsZ+5HuBV_@p$s+68ylhGc+%wfh=gMoM5{ix-n#blmFo4>aIpk}F%GGZQlEpa z3wlgv9Xm2)uZ6;2Eq1EQT8`gz3s5uOC*K-(z)}<-x^bclCkeB(~dQ^KZsXG93)}Y@3u(u2tR+;ef__lr)x~c zs@fSnBczc+RPD}83uSqz>jXlZFN{@i>ojr-JW{l0sO6iiBS_stR=1xb>wi>xSK!1U zL2LdH!5&D=3-{dfOHz~^B!x^?4C$#}Wk7NbjRY>)20)`_Rah=>~yAj`Jqb1^m7l^l&k_AB-56!+#;7VCgf_!uFmkuic!7h?*gZt{)_2! z_c`Me!sS*?m7~g@XM9#~KLmRnseyl4WPAbV2*VohK#Q(PPtg}6O__^XUmF|M=ox4_ z=%bFpNYb{Qkrt(|FI1%aC8ceb?mr5CW!NDnU+Hzvvb;Ewb1u|6KlmS)o9_9`>qjTe z>2Sb~h^;ekty=tgZxaZOj%!mK5;{L2XR53y=c+y=nxBOWYq!@&;XMU942~iy{FPe6 zyH-AoE3SQaeEVrK6-tzPE!`imsL7?9x|XY91sBHJ&nU2JU$VP2Oh>`kp37-@+oHC zk8(4QJ%7dN;5ZA?yJa_tui}60sE-N^DUojZym`q_c#yXxfwn!* zgmwDa&!7ie?kNx?9j05>#_w!&(xPXh0=JQ?~35`m$S9)v?zSr?L@Jvwk6s^tZy$#4l(*9EbNxA;-XvBZZ zJM4>a9U6S-(>`mn37JGEW``@B@@_U>)9IcoaYH#25(MHuq%}@Bv>r`vc9*gF{Hzo_ ztrjf-1^GrHd~Esv&v8iJWujnQmEGS)rsXzTrWCmTT&M=K>$5pq^mZjKbwRcG_#i0{ zfE=s+z1^`uX62_(G^}}hy}enmDimbd2;eNfz$pLz)4-zJZ$tX;`ilQv992nE^em~| zsdDTeuea4{iRPJbZCL9w%c`!f9+@nk{Lvbh>j46?{%Ox(%zI{)9jQk@y9HHhY_?V= zmn%Y<`8vD8TfSzghk8XOtN3F~Yup}S-6pknm{tX~42Nu}xxeoOa&sZu%@1&;QipGa zr4Kt&cFsmQbnH4uJE2wwtYbbANF-d~*m4VBm-tpDZ^(?k zHMx6NMFkVJ4EgcPr!Y6`!COlszKg+~>58EXL+|?UM0ZK6SRzO3EfB(V#u>i9YB&iy z+jDO{rep=N-anpeGyg?7 z=xZ?-BKBfEud<()7eI@R0KDQUtRpZ6fVyUH}^g+ zA&J<5vq7al<_9Y_;67axeB6J>$cR-vbfP7u<>%lUw;YO}iBpPyexU4}#1II8f$xD* z$AX~q(iWkn%&HyJnf`ab-?)rD1Hk;|O1mWf`8kco9Id$V%C`-B3crMLH}0aRAIuQ4 ztolnVawp}4`j*=z8=5wP&H~yPNyaw}mLODimeiA#(LPLa1Ji&3C92VPegiJXgD9V{ zZP=Phl}l%rr8$E|jj4If7UYj_jSVQ1=PL-sX79Xsg>;iy_O`nGnp>UhN<&x(wq9ICsLozy*^cqcG&t(C$ zw(3eS6H>X((`ZL!e|Nv}%^m60idosmUy?UptWNLS35jbtq*4&9VH)fg4QGKFLxO$d zv0A1l;Rzf&M80I35<5}#UW z07qEE3+8>ihCzIrKDSjVSYl0kzd0V^s(hHis&i{~dP3QGuw3x>{GfETq3UI<0E++S zeScG4ZpkOQHS9cV=}930(7W#D6lmPP)D*^=r4hqRDFwI=mRYxi05;jOOES&ZOT9Jw z_}Uz2Vy+JR&;;ke@mjCl&50P-T^hT^dNMMt(_5HxoZA{70@?o>n`h3iK9?27D%{); z5l?FI{^1wEe#!dRu&)gzBq=$wj; z)iaW6`y2jzOc_i2*3r-dWV2@DpJ^Ya>d0TpbL3eNfRzC+LN9{H6b>TpMTfMz6PM$tC8tK0pe9(V zWZi{A+S51rlxIFrf+EDM06iHSw9ORnz`wauasS>?#Fa9pwquuw*@W?@sHY^sI_~}i zf}oipHjV&;k>wHRa<4Mz(Ie@Ke|7o!o9O;>34 zkKrMUHE3o}eR(nbFfi5OwW+4zR|;5y&C(Aij$p_OhEj1n*RbZOILHY*3;eWdQLLMF zIzIqXr-UP15p;Z4=lG5sJ|a+S{f`0U?8DygI7ok#>peObJ9_ox*Z{!*NTTR%I!8Jr zoohK=y(ypFaQtG1%>AaP-S--3_=Bu3aC~Or`dS>`r}LBijlc;X{AF#f5s)&^kDF(H zq63J2NT1$3#}L2nE4Gt|6DWm>PsYonhr1tA&mW0{Uv>`Pmp~YAh%l~BcSQ93e3!w* zCC9HnA;P8}!W8}1Xs~`a-4G?cF#=hbf{)M**Lgs$_&#EGnM7t(tl&Btk1>w>D)$sS zCJ`1#^!{;avbEK9q&!z=!WWsM;4#l~BO1HkuY-Inte?z+fBkC%xn#`vMJ>rS$9dpF}PO3F$M%_W;tTB|<`*t{}BY>T@%ppMeYVE~+I1h-1NxRg>A zg5`xDhwt((K#!h4BD{vHbfy*bUP2j>bZVPSi=GQU@XHJOIEX;Y=jZ3f73fqi3~Hbs zJ_v*!iFJDV`}gmk%UAppegu^@#0vx-|Fl)`jWBCcG+xhI$Wk&-tT`ukxAL_Nv&IR& z=tc7SPo5i0e7w`=s<^n|k9D0IGmaEgMs+O?Cok`{w#JRSN?4TVoIa!#J6o7X3c+NtLyj1m2J54Ku8NfltG`MtV znaa}wplZ*Z`o;I;)LyE!LlFt3=GB6X*25^3zua(;o(%7)7n>=Bz{7r9$kLJ*(zeYv zBYZg!s$ZR9A{?Jf#XI*wHYbwYy#1mR6_{AkouA{cF!s>rP(b)1q0|aLZrx|JApsah zY*OcV^+l(Pow|*r2WWdhp~nJYaEk%? zc(l&jd1Ip?ShzVxwAiKnL#p-ESB9yku(uPAh#?l5GUlDhSoN322%dY{mTZ{FZwcWRVk?*D1 zZD}EU3|?6}w%<>4@ooLK!~Clz7o(bcdGa_&gXAOOvnbNLKy5{fSDQSY$Khg08TCz9 zTi*ocTRh2E|2i8o%C?8av8LM5$JR2-vJ+)wxk^`8sE#>?=i+bHRzc42X}8u~ar43Eg8c3QY!gT3r0AERgA;z=Q0 z`NE*^n^T?j3fHmX3l|MKNnS^Dj`rBR*cdFc-_u1&qqi%&ZSQ(-P0B44ziabM(grE^ z!DiAeNi7D)na=C|qP5a!^1?e;uEXu+4>r`fk!P1@i=gO%o>vY;(}53Tu0M zToR*oBGz2u(Wj2Qa6;iNrxf}hh4;)P%5IHBU50lLPJNHfAr}G<^paXNj6c(#@6N7n zr$*>U*E|5%iz|Qmq418iOeb%$aEsr4k3D_16oWvrY!ecm-n0Gs>Ux>ZsfPBbV{pio z6YrISEJVD7dpGrPdg@skj-Ny1 z{7ovWvmTr*cA`t6Gw+`G`19@!xp>6y(>&spb^@U8{XIJT7(A5OpDQqQZ@OAYZm4KD3SsW($Znu zGtq6Y-#O7U$fsp!K@0bn!q&v7G!o7}XbhrvEw!%YbVb|I$Dz-Ae*L;Fsu=R@yjCHG zjS-pQi$pRBalV#}D6>jdUMnE)ezRxq^ldz&#(}6KbPGa>OuK0Q(FD=kO|CR;!RJ7N z!`7U$%OdyuG|C^}iGu#}&v5%a^IUEr{oSa9p3d-xYUg3D(U>#4UQC&EcLzKXp9#f< z#Cof#$TO}A;vNIHX$3oJ;n2(PPmN07yQG%}|FGx76v_FV!l_0)4-Y9 zbZ(yy2$hl0RkKsOEr@u@@})EoC@9erb-#b^YBISA>7}u#NA_lB@Mr5xrwnWd;ey+# z20lZ)kK|L4s8<2ycE2u6F4Fv`y26p~))|+6R=8Dr^*- z0)Ui`ajGj~xM}9rbq6JwpbIDxr z1LgG^!M34#v@h0uTAwQN9FW{JDl zwK;rq6m-JzND9E@Az(+%w_tn#IHZ%NLisRsJ^2RA7PXJ)!SoQFBne2v-nL^}5;Kwu z!LImtp6)0u6F^R$F#=ye1-}vOcS0%PJO>vawYnOtUl{A8dato{)<1Qb^%U#?-d(A^kiECYHjx0rcd9SJ7b{! z96pARQQ6}ZGRQC&#%4{n0d>qeBAz0K0tqb`SkvsiMas*105&;C05l&37UD#`U;plS zkz`u$=bdK2-wlW$YHw$20A>eHGz(~1xg|P4*m&JD{ty)8tw6bcSE_ zMU`0^_8k2*IdeY&@s&mUpaz!pDjj4V33WB}NdWpqm&*MT1praXlYM8j_^#2jHgpcjBpYOx2IWu@IqX04SL z9wn~mzqeyrKGm6S#(-oOGi5RRkt1Oq8HKNHKL)pdIVnRegA))!qI|@hXA!GQQdB1u zQ_Dp9vNZ6>>v*g*5QGXQnQ$E-e8(M|y;?CL8sP^!$YPvmu%YS}M1nI68ZIT}ipD_z z<6&R_HEiY+_&?mxSyv_W)t<$uuEafyjWcL~qm^d{zNFfORVpsdZ&4t2i|UnX^n zc@C8S32ILd(FYlJDXr?Jlkx;9RvFVAoLVAjYGtzh;(LJ{k}GNccJ<^JDh$}n6Fohq zwawFU91anWxNV7+{r$5?SXnQ;pGHwh8A>DMlzA9KA2ZWjW%u5XuA=^%L8%+hdNA+KUxJ+NQ^~; zTn9UHP(jy0^`zM4#$I$mj(&?!TSuq#wsw~CIzANk)`;z1NF&A3EomfU0tp(xzd z2ZcTVDiW>l`g~X*OFz&oTQJg7kzk+Sm8sM2upK1z0}1W_Bs2f_)jcE%Uk(~1PJpBb z;gB%@-1RSw06?<831r+ngG}CA#fXIes>*|VZUs2-Rx8-B;om}jF0P}m*bEr=Onkuk8g0E@GN8TH(>+$dxn|e`V?;qmYie-%1%QuKYo} zTHWJ?Mv(qiDu63;ZrM_cbD)O-WOK$3K5Gm~5*BR-9R~*&jIxmP5IZpcKlL$yg*J^v zz)AeA@PpaC=ZfDQxY6Vw(ZrJSs;e)5$+)5@cBlOOfi0X=TV&u^t3gWDPBoA=7(7ER zQ(FfL|KmeSgG(WWbN`Rs8w9{eDN`;Vv=|hht9ZJrw?>o5+so|!QQ4NeB${+x0%*88q;*~Y KT8Y}dfd2)j<)#Y& diff --git a/docs/apache-airflow-providers-http/img/connection_auth_type.png b/docs/apache-airflow-providers-http/img/connection_auth_type.png deleted file mode 100644 index 52eb584e5ccf6463273c9b0d35171944d451af9e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14199 zcmch8XIPWlwl2%HEh~tXqEt~qIwDQF282i_fCvGiAiYU%0u~|)Qlt}_l+cUx63P;! z6RNa8U?H85AP_(Zf%_%6_Bnf>bI)_{k9+^{0QtT-#~gFaG2eHLH}7?|)fhmmAUZla zhKCRpC>`BD5Ww%PXHEe>*-vEj)6q#QK2*7H;A2j}h4>qeWG!xxl`vB)-qyt4sieB$ zK!I^4{_6to9tVb=yr>fRj*oBX{%=hGK(XWx`L7FCMn$QE{n{j#%tES+jZ;z>wWBLy z)|Y&zJ5tO8c4S2zN-5sF;b`ymw>G~46VlOrV)=>wah3~pf%ezEbrt@9(0&^@tq1yz z&SF6ae1<#t0(k%VoqP=R-39QOm9E&Dc(-5w)ot1MK?$B)ipa)5&(OV0dve4x-+#J} zB|8atk~=s`G4{FqT4dbO+0(-W>d2h}*iREaZ1QOCh6GLh?;mc517iaZl3If2%-h1l zD4Cs1${fNWe1K1kTjTDRT-en3)V}zPm6+SKa(|U(%9$0!Mn{7)`&xTj;C)6W<=vi) zolM3Pbk9y2ote1PDHv>eF+1GCX)ZlyF|dG*IAL&+X6#VdTzezy?+)@swvKC{%slpb zrFbGljB3@g>)}~$r`{=%rzK~HPU*6J?cVp#x&9IDxvS}W-Ky%BiHEjpg<_xETUakU zI2MVy8Jw##>yf={`3_k4z}$f`*x4&8r*jp4zw*m)yAE56ToooKJ77@F?n<-w}U^m}xbX2m=)o+xvHvuLI6^WI#x3k`GC@K=GXzz0#)m5{mr@G%S7JkMK z>+txD-5Vq1_)jLoThg( z7wLwInsm|}hEu?K*|K}0JT(H*Y+nn#Sut@#s_yyOgQh~xw^=U9tmjg<8Q8ol9c+{K zMTKIe|7PSGi!!T5&3L#F`-{`mjZ)wp5#lDW5Lm!9g2}xZQ?lZ|-buazT4lagh8L;9 zbA~%W>ey@L?cIT5*=3dO-w3I~G}nveTmNDAMV`P_yz+T}?7Y)jZC2)BwUIoqYz1cL4D z?KR!qE9$)0IsdjnOH0cO*t*Sx_hsc?Jj2JM5F9he4m59bqhMIouVF&mt~f7Xu~Qm>K>&E8%#QO46*8mprfkZQUl<|REZ^47lDpg(M^^u zf6Y8J95}o`MmtpJZ9f#1mACh@$~l}ZE~{+*%L9f7(RNqG0X2ZD_s{uxSYj#L@>U|KplpF}}~g!L$O zDK#Zg?W3_^x|c-@>fGM*Vn(32k!f*+Vb^xB2&1$v5KeS-{I$?!3ya>OvWoVBr~KbW zFLu&4utgN%;qecr+5%z%_#`j{9o;>X62_s#2@$E(E6NT+#bS~`g1AhA)8;^&kVz$jkGX}Ji=?z^CT&{zT%^0!USH2>Qtg;DQEn9`?751^YuVZ5Pkes*W^W+@ zekpPiYrXMGGY$*{Om(ciqvKi8a*})`yUe9Dr|zB}!W`Z=SDCu@0U$>CB~~#XtG`~o z>JQ7)5f(dalOq?JcFZR$#uoIgRO#6C}*EwY6Dekdix!B-8D2XStOGu#Ak1 zmvyX8+jvs|Idj;iKdmYl-91^idA1#w1g+%2NcD_-c#zYh zTc_ql8GHuMvi^)VPPpfYOTs3#1MCkE&J4^{ zSohs_x9*qfSK2M|-9Ri6@XsX@;Zaxa;>*(g%A5C|kjRVJk;1_Rx|b0v-B3tqXy{rm z2a^cWDyv0pY0qHFRy8u~j;gq=M;aVtmlHwk2;!_4^`7qfTHTFVbv3w8K!K`d-2r{dE{w1Clu9ak=bM!;GTt()^PhF4HYISblQOftp!vINoYwDf)+a${@;ZIXgOtQK!!Pb^=J}}0 zv)HtS&P4MJyx!ZXv-OJdKNlX(U_IB)u(OFndp)T6jjj^z9S3eIFTW%1O*DmN+rmDD zyExFxkWo?H%uHg%zTcpbcp+m8ZmQB3|LU#aAB+6ik@}kaySD4fwN@^mBF6 zon2*ssI50D0OyIf={|}BJJ$$fx<7`oh+>BiP3}cPL0Tz4eLa%l@gh}A#jpfFmh#Vx z(K7DVlUpLDcU{i0kGmWl62{;JE23Caio8ty;g{b}uldICQZSr%9tY)i_ka0|jcz~@ zNH7bFi&(QFsNSng<*#90T*J=hirzjx&5T2=^81rYDUC1~L_6i~(|A!hrx~xwmfGat zPCU3Hf};Y@mVae``*7zW>7XT3ZA`-uv==0YEym0Q^rpYM>8F({bDN3_{T=}~jNf%6 z;xaRDx44cH69dwWf4-k{$oa8sX)&}@Gyxdey5GUcEW-d5b!tS`WwwQ@x5!wWK(ryk zpW-Yv4=tb`0X01nEHS)EuV0@a?+#>$CTk>!RKFWd0*P{NHrZ29SV&moB;@2c zrG;`ZX(Pku&k2_j9@JYvb~CQtm%W}y>=@ivm4%mf3BN7I+s-;{gk9Tx>{pyw^`0bu zfXLb2H$>*28QRej|3rcVi~XtrEVd((s{+;KPWyeXNpjUz2^tYp4f-Clt+)x0?{ zRiXGd=c7QKY$3$dpL=@h8J9aEYwp6)VU04I4B`jdV{72S2>-K821K|<&^~{iZMD}M zcXXs3hTG^lt|`@}FI8dkJ3Rb1I>J~SDk?8n?|p^Oh6Sv^wmRHv!wvD?wmMSoW@C5s zNTHG1(US}DV1ei4ju zH#pej(G+pW!x zXR877dLv8m!{6wZ?*dpDNT6HgnT(+^;<-ALn`&WdWGIsk-OG|V0H8ca>59?VP?S0V zMQ$BK5ol*mk9Zifh{T&y=PKznL)4HNj3-_a6W&81_-z_mzA!Wy148M_r78w=6E{&% zKGR(YfUQ1}5h&e8ILJ^knQcL9v4sMKpl^h*snRG3c;8Jzb@Z{ZrFD3!W~-m{db1?;dSA|vm6i1uo0R;c z11)k``}jXW?(lrc>d`AOOQ-NU7!~L(i|qmjxr>szEqc{69FtSuG!Z){f&5;e8WSY# zOA-SDithaI-y1pl*u>I4yqJIldmTgf_%gL~XgFxXtP@F0@`HgSlR~ko?zshD7$1G3 z5&=58;(>pA87IUYUMIPuji8Z>Oy%yQd|XE~S$<4fDhd0RS3ha|Q=H*gaa)_&(9~$bwnlupzK$? z;WNuh+8bEOIhp^7_fSK};X`p5Enppj5m7_W&gV}&1NPxETlpz{eV9vuy*&oky&8*( zk_yG%TU=O3lJh1e0@v?x9Qm0^j5d1W<2lGV0&gWAjV2mf+Jt9F#*ZeNSUQB4;e60l z7BoIs42=V01Q1Ao%y@3UXD~6dXnhnj{-+^I2tg|45eTk)`ppSP`E_cOGnM-tdJD@+ z+yBe}&VW@%A2qC;G%~&t4qWd5hwD?v9@`Bgkewwe+|1H4E8oTTc9o~+dj@!Meo=uG zQ(PRs1~*0Hp{Hk6qXMv#1Ic#1cA?g`lRf|eW z`V4MK`NhQbIuxmvH zO$>Y+FDMYu*VB6n@DwWDq8nI32_r5Yr=wp{+&TfK9V{Rhd!MFs4&b zUY-%ya3<{#=o}_mX@BYdR>e8*;7;5ps zMlmrlQQ=EWLV}H(yE_KJ9~j_{{@aQFOqtY88yz>XbPTVOT=bfA3DPVP^Gl{qoD^lB z6UE#_Zv$Gr>f0MBNBfC8-awBx?5qi@J9C*!Vs>7yj%r3E;JB_+uWQO3g~| zBmuL5tIVDHRGBK(ZaR;1)Tc&&y+`%TSJvZo%21v1IE&^<*$}{o7aV~90FGOt4rvii z+k<6xswax9${A$?4{95QfzJn6sCBRB)RT3AK1m4S=H{mP^yyO{YLpHL0)CkIPak?H zP&D34MPM+P**5CfrJs&7f&E!_>W+Q|^J;JpdTy8UVDH%<+5bA%Fp1HTL}^3rn=}hM zJ!lIddD8x(g&EzQFxu?L_F7XT$bv@h={kj|_6o_X^OHms4^ofM^rhfmPajv0o>9r` z%g3J>uOHXmDrjCgfx2*97D{5EdW@M5aeN(gInum&k6-2Z{d!I3y#38Yd5NgRf`bz_qLWC~Q0<*87VZJL?7Y zIX?<70QD`Mt&x5D*$Ls^DZhz>-Db+gG$-Py1X6(|OzSA$-$cG-IHh(yGgP)><(t#i zT8%$UzL~{lP5MaQu0y$BZaArOF=bEMQcB6=FvZDe&7HmGJ4rdADo$;$m+FEcGME}` zh%^Dj;euLoa`Tbxu=0|PhB@r{cuIev&+-AWvy08j`rG#`fTl8^B?u!9*AwJ0mT*G( zL}2y68rtAe>S0f;*`iYT3~MT(bFjOVJ>CR~W9p@2e+k@_Ldq3lH4J`+*f)e=B{Fi>M^c62Rh;ForEx zUk~y@S8wvAU!`uUADp&_1F5J1m=dRW9ZGUBS&KzWd01gR#t)Qt6!C1yF_9E< zTi%ELnHcZHfP_#&QPS!@FYLo|py8SP&$S6YI;0c{JgN%+wp4GBB8v6IFdBWE9LrzY zT0LkTTo8Du2=nEYrF@h2U&H_GpQ}OMFj^mP_Czz>+I0G&YdmH<{jh;}P(JUUJ;Wm< zX^Mq`_KdxS`k9qy_e2DuRQiXNW*U!>vvcXCEfDi0@6NQ*IqgZGjSRAx>)o-~N2AF> zwUEh(9utlYaNqbT9n59(G%((@m z+z7*tdvny!gKdR@x#h}buS3nx7nRt>ybD$_N?KFS8d-t*F%^LaV-wijH9aL-z>=D1 zk)I)_!Hu~v7-<#v+rDe5nU4Wo{uQ66?TTBaiqp@gO z_!1xD2U8k7TG;}aT2t4HpFI!#XLPhDEkWJ*-rQy$@AFu{oIjbbBL_Fn&s8Ji-Y+e5 zUnD3l^fRh^<8BMQFo5J~?t5F-j9)LvtQNys@dJr^SRn1d_!fZ7tew7>9pQrsUd)lx zrTqAQB~q~X=Dcw$6abfy+iCMyrQm;g5B!kP0 zE6FAv#;_Uj1-;|L{I_rXH8SQM&to>9FWFUhq!5fuOm&ogc%}t@bzoq{^_lIArN@XP zu==3<4>Dc^?J)A8*Dz{Xp6Ptg`RNZLWO5|eS|4*fc%_6*RUB$?s&@Fu3hcJ{Avs`Z z5Zc*j?oA25wMEcO{Nc(J2hQ|-)dDd}01LnZCxTkzEu0s%9Z*r3(Lc6Ex~|Frmozzd zsc=G$=$C%rC+o*m8HTiW+ryt|lyGRKlq}0-jpg-gBT~>)0p|JY+)O%1j%*8P?hM?m zrH^$eV8!FVSsWUGJwJE)(JR8>HGRE|r2F63ySI94{0Y85g-7BBKNF!Kr{^?A@kzG> z>zat-{uko|TV;z{;q-D4Elk7BeHQL7Xi5f`BOf$TEugu4rSDv4o% z+4$4d*1pSbn9;i)sBgqeq@QDGOn$O33;QTm6z){}MkjA%tlVBeo+P#Q6;^*9bgJsp(tD6<#s)d5T#sK3G-B@#SLY};&Yl1Z zR2$s1#oZ)GEkzVm?~w?$r(%dYZHcv$4tf@iys6sTX^4}p<-bci=4UilR57}RWE;)q z_CH>Kh($f@(@bo}E5~(JV=)DT1eu zFVp>c7s~T7J=D~R7L(79t=^pTFS)L+CgYASSaFpMd$3ga72=J1RdH^7DSN0DSDN;w z#D8s{11%S8-L?!6WA;ik*p3-Gi<5;pH*C*|`TN)0-JT*CIX=z&&Nt%E%zW0*$ud5@ zJjlj@e_pWA`SxTVo>ZW z2FEB*9~IR+*rS@OohGr~wB-eu_45hI)_uP7BH7z04vat`7Iz4fQgBV=v-U_fmQ)cw zzR&4(asfMQ#;|2}IWHmEQh){nIMgC8J|1gY=Qg34^0V&h{^kWT&NUJ}zm!f`;Njs} z972Uz_a;VS*lxTf>y%D9;_645I{YH`Q{=Ayt3d$loNW#1#g{hPQ#(^>fvXSQ30O)0 zCNGMw)pAdDuMe3}MS8x7_+t6rI-f zBW<|+N4jk-VbMMu>ET>9HW)quRgdQV*`sGifnJZoIE^aIP0-E)^+#=C%t-Dxrzbi_ zwqyeK4Ls`9ql;nHD!DGx4R&xfpgbY0L8@_vByy;uEBBH5U4?XyeXcsL_%hgTS0 z`q?+>l5CR}*xe_A6!)*Dq=f?a&ooOTXMn0`(~pKENrxPY2zfeaUBRES-VA@zguH`l zv+_H7dzN$RyL_WG{bIk)UZXBQ318ZSneT#+FDXcDC`>oIz{4hg2Z|Upno5r|2D)bN zK@rB{Hsjz2ky$5f2mXA;d+Sz*gtT<>jga$`Tw^86gF^8-9T8QC_g)v>Q13zS0koh1 zC-7GK?>hU>V9g#XVQXAwoJEh^HlM++ElZA$jv|x9tyLD@d8KdK1ZSh2A&CAz6Hbi} z-CNHHt;_^k3J3_S32`>U@uS7z&g(sIp^){Dp_h$TgHcgU zIs$r!Qmz&r=CJi7k;w+T7z(@O+%nF4=6&e3W=5p6OG)a<>`OWo8LHua!$Rf_e10iY zUmccbO49>Gw^RLByAG?rx(&sFF9nJkfbxsvT)(S)KT0~aHLRnAT}s7N`~^_yF)DTc z3I>v*z+Hpu%R7BQzmnQA+AgtL6Jk%zJ@F~1`PLdC<@eJ&HE2d?*^do1F4cstZYIIZ zt@cRrH=sQySKo^<`S_zF2)OSJ&gFSLCrz*TE-vtlO|01R&zqNuZ{tfbYsM9s!-{dN z^-b2Z!Gn81p?-y&X`#v!(IlNYR+!n3DIan=+$W-Qiq^h3sPCq&_6@v!jcLlr^({FRjDgaG4>(>3#VFungj6;q$)n?k}mkc>mPt(!u zbJDW7hc|z=F>BLoX|r{ZN{Aj6PzEMlmo9-vsz(ERiGhIDjRs5X-DX=(ZVkO;A!weD z6FMZS&qf}3j4{|PzAwqb;UvT#4$jr%7GzC8`MMjJpU(9GK~TN?3_M=3wLx|*PJ|yp z@)2-iOno%yw(-4f+0U;Lc|VJ+S7MG4qh~pM(!*kOwhF-a?jt3zeN7Mep?6ChxLy?3 z4xh*7u2))b{9K5hK9Avblhp^=c|+7H5=rbN+282y&CpUTf+&J$F-tZ$u^i$hKs;T2 zbmItU3$JKoY@b)>jyroOqMpAjY2V1nB<$yGGN#(Xbp$|rq?A*ESxq|=r{56}0MgD= z5EuxtZVW@|`UK<_0xdlM@1plOK;B&LBpG4YP8bDUysIe@0|`^XcUoZy8V&|^zi=M-s4A;sOX$*T1;1rsBG${FT<*_FLG(*32Fi^tP=L&u-1+kM0cR5J&DC$3AbA zb$lE)q*f7n@Oq9=qnb1$?fgMxjHvAjI6)EtE{+2uGqSRxAJ*?`uKU-@jlMp)MR+}V ziKw_JN-E0A$cIGhw0|fo9btMbVySq@hvm>oFJvy?v7=n4-c$n@Y4_an3KT-Gg%4>E z8MaK9^X`kkzSN&ylc1CMBWcfL?bc4&*47s0bkvsC7Ta7jy*3jNhp`(c3l6?$5k z=4VNUNo+k>&j$$>Cuidpr}Rd0H23Zxzhsm3#TEg~4KXQwn5*At^ORr0g=p#yC>zM$ z)$w3r6e-ktQ^>6RYYEGOb23q3czc^e#&$sWn@Ko-2Tq zA$-e&bQ?!(L2Om~3^>M6`*HLl4$^$MY2nJE78SH){#|tdF{7f)C;L+62s0BVpcOw6 zKQ7hgqcjE9)ZtP8YWI$F|;t2j#oW$&Oq3^0AN#jT~vUssBB; z0dO6)y)qH;Ki3ccPk|D3%KztR?EiieJ~W3?V6Qy6cX)Q9^ZE-_0OwKrHUQ?!_y%a3 zFn}ndwF2E%mZhc@7}!KXXCYeC6wt87!*{c#$xMQQ6+uczQx%IF9f#3(C}}=ZMKCl_ zeWVD%n|fmt5)?cRU$Ve&U*>&b?F?X^@2(X5@*ZUy=ZzW6Qp?1Xg5iP2_UZjVQ??}x zOX5qv2!KV({|wC_<+oRCX9uRCd7nQIOf>~eE`~jf>HXk7vNP#25&cDP<2G@)Y^VIkA9cy#;qZ$rlS`T)ceFnke z&wgAZ*ZL^}(vVOQqle|5J@LyVOtLfqpI1^MW!e71@o1Egqm^n;AHtFr)WW!UPpZgGLP%*k2QOYL6z{t2lUxYidIpQj^?iUYjo_aX z{qRV^cT>e9X*poT=6lNBz$rj31^q4=n|w&wPikm5EuNe$sPr!m<;F-1MlSsqh}L&O z#_i{4N*N)P6Jr6S?8juw18qAy(i>BR3JevMw(j1qXi1e5t;56(-kuAXaGYtvnIA69 zcU4~)bSNq=mIOpa@BX6z7}Xv8N|fQVRrItnlO7dg1~f*OJiid9eJB)51)`cqJ8aaB zmd(a8#iYjB$-^H`tN$6~)m<;D0-Enp!p55wl*u?-{8(wlZsGb|p0pY^6EdR|C(RW^ z%GjIy`s`x%?swF0qF6Vzu>f+hp7lZ~(@r@*Tw7{5M=P9DZEQru6}YG0^$p|AZU;3{ z$XkD0?#)D;2i;id?gzR!q`<#Z6+|g*yVaz|oA-s3B2xIfEptSl{>Ln{4<2$$f4-s+ zL=Xn;d5|CZcb_kXbM58>@+mKmBy%*-k_T`BpgR1#@G@&xyMGv?3LKWd{uQ2q>zmZ0 zZp$_U%Bui4aK0Th#H0&v0S(SNBDQK+F_$VX+F}+pe`Yg56?&DoMNi8ueSLp2?q{<~N}iH0>+K6G|4+q()-t?5Ww z0>uGfu%lK-X?6RJ?1xYg=Imwp3n6blX-o3|jbT zfKhMahF>F=qu)%;D$47R$$C!*4~{r+dIaUBZ1IGdUvmKRRTg>I=D;>V5trU9^|7h- z&1r^8)W^JMBn|#Jgv(7uHnkZ~Gt7WZG)aucV`VeJ%D*EW>zZgm_kSVYfv&{{kX!TRvA1pP#c1X0u5<26+ zC60q8_xoB9N-C%UToLdD$&T*70?eNenC#`;dDhxV_5!lq= zE)LqFDs>iQCH!*$yZLBnZI)=64-lCM3uZt^x1CE0cus#;6kgr?f;1Si(>O%559CIz zf7RF7so|L&*J)Q-qJpvM4ggkVMXOWmd+Xg?Ztqj_0LA0UW59NVRSM{W%{8ZEQ1}T{{VdS^2f@WOxT$~Ql_ET!Tq)Km$ zN6#}evjm9D&fWD(Yo_cMQsjnd_ZsZ@UK>V5Ex*$@SyB_1t85?O3X=bMvD8=@fU%v- zY9^;u(M&)=%-YA6io;Ccs^ZG35if4kQ}Xq59qT@kN6-5b9?df|jylGkSGwYq9@G6h z$3ibqE$DS-17HLJHxj|rvsZZo=9n)4SEc52eD_Z^ zzZ1~CL-I*RaiHU@M*9qC!8V3BRhX=I5X~2hrB4F)s~OEd_xOXSRndDZQM-rE?*KT1 zdr!+Au7UuN127v)JXls;fLh?fSJp5Nz5M`uN=Ii4^H=BQIH2}!2eJSK2lgIH5TUNF zP92f%LWDfE`@ZbD0w@bUA3z}?C~wq2mqXW$ zD7&nV57fdN$@8CIfG~MxLKUu1wF1V1LT_83oyCIL49K8=l+PK_V=8f7W?hvVH)s{k z>{3-ri|}ws)tyNB9I;B64xb@EBQux^dY7dDLH#Im@~C45P^{6{)qMi!!ZAQKtrtKN z7(g)HTjekY03stTZS4q5oS;#rY14~e-^v;P%GFc9QQTh7x@G?#-)S*5HFfOhp7jGC zUOyI4T#N^Y!a*o3q8Gyq3d;ac7TUETkWwEm`0k)Kod39eK9TaAy|Ap9ZONW0q5;0H z;<#r02MX(1q@c!mnab>I9vXtGs_BjI!XQVNppPH_)D_FFO;Qeg_^ZLrKG$AF{VI$z zAO$x#W2c$o9IsnoMw4wAa|7bib?~A!y&Y_xMcKK1fU5IQ-=w%0AW+483D4Pw^L@~L@K5TQ%5qoBkcfTgBV@d z8$1Db-em8ucP&ZgnD@H6MW10%5iF78Tzr9AdnM(4W>lAibruW97F-z$94=vZ z6Av+cIG@RI>(~jdVM;lkbEe_GW`G6H3&0S|rJdeOv{(6p-o)BhM>SVR!Oz;66iK}> zf#|S-o3FX?d<>IHq3v?;SK-6N(QtROi`lk9w66j^KQZUp1NBX{^;>p6jE~D`dr=X&N!8)Q|&Rw z-nmQl&ZSP#*HgWpo_pyQfT?22_wS@RiC&!@ki(wjZiBN{SwO2klxF<#}@Q|-ZX zGoR0{Iv(poI9Z%)4=jKc9lY+;sV%!nd+25#pyA^XegiCJ{^<8tXYX~N@jjOa96P#) M542TEz)xQPFTow~N&o-= diff --git a/docs/apache-airflow-providers-http/img/connection_headers.png b/docs/apache-airflow-providers-http/img/connection_headers.png deleted file mode 100644 index 413e9bbb38864faf0dd5664703b9089ff0b7bd5e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5256 zcmb7Ic{r5o`yXfn{-V})%5hC16>~W0*`i5pPc5~&!_Z*&@TQfZG(MG_BoO+`NXB8 z9cLtFwPB4YYQ)t9B8V#!9>!GTtzxn9z61NxOR26(i5#r%Yd8mX zpspJ8#$8$syuMdWkp$lPLfgu|3wA>n2RozP0kP;M(D-3DJV7{eSD?`7DFLe$svtVK zQXrI18VCt+$j?P)C&oNc2yK;mP)&FT>;wtK#y-+W5L@X#Lh$T(nAMSE;3&rnyAcEC z=B`9Y)31Gasqo@r&2Lc$Ksrplcbu4A)+;|xiiy6^5E1hE2n=@Smv23B#hBxd`M!T# z;oFxEnNSws+K+=nu|$kS1<~tp{^0_Dt>)tIEzT4=aOBWXNS{k_ety2;UKs4bA;@|O zqI_48U5@qP!-su9fX5X;*!ZA*_1&YXJa?|LQ*sBZ*nIV=EK&jnJF5asx!*D8 zqD|Tiyz1lgM;Zu9wrlLxa<^2*#&T(F&im)*@7WOyvr6k48XC^!Q6Dt=gTuhh;JSW! z(nzC#D9CHZ`xnj`gs`zBwwg({QRK?w@WqL7$}3N5ZJ|8t-809c6d6RJKWPi@wu&2x zcEr;P^YfK*^h2JGK+AEsXA3O%nX@t9lWTgD%@8mrgimB`Rxh%jl#)356_u5Rr}Hff z?kZUoZp_tbcLuA?4A$sUGj)_w4(KQjf$QcHa|a|64;?m;j&!dcTY@2ScBl@KMh*uy zaWRImoXw@-$oX-~>Tsk#A-+3jdAhG?1YE7D&CiQIA7B5B6iA|ZE=_j%Y;(4J-riS3 zzCK?Y?8CVhzotq4@DLf){qv5U(jL%R7dBit5tFhO#p1X{FQ4_;Y!)_YAF(n-l;GOZ z54)J*k0Tn12Z%>5n?=l&Q35k1hNBj#dRTh-(qK?$L7sVH;mw`Gp&QXVV0s4J)6kcP zj*}~sP)CV~x22ddX`Y|i<5PF7mTn3IloxEAW*u(6*i%(x&7R-&?tm0 zc^u>|u8Vn;W}COTY;i)gElS_{pk}(GerhFG)QC&e}LIYN^nK_x>Q`K6N@g{EVKwmRXK^9NZ>d z^HiGgx_1W4yW}9TxSH^Ygbw?&xJcU6R>z?bT%}2jLma$IRh2MOqZ71LnRK|p9M{JW zTX5+0%A?Yc`38ou2VZrDA~|&5x<%F%n~Y%OursJ-)Y7YY=%p)3vUlBA^97XL;cpAk zc(&GQ1>T3=j!M_wHJF$#QGbgZ7$D)Tv;p)1kgr^3(sXt&WWY)R!f?yFA<>ogRGAmHtO$}!F+-icSz0A;Zz(U~Nh3w5b9 z3lT|Ew2h5Tj9;7%s{yr*ZvrbxJI7E~#ci6CBa`P!UKp@s$;&)2y=T@5aP7IlEEP<4 zreFPNg5DMOrzkrGj>X()qMUWUWvZ=t;$AzPR&9RSvd^vQ8s=PG_iG+RVKD-8s+6=s zzmiAYm~Y6K7}Pf)>Uu6cDGy*Iv`Va0_-eZ_ZL}NiU+`>yQz}2aF^XKA7*0>vD|g>= zuY7vCrXSkPZG1xNMx#Rkt#I&Jg1qB310fI?_5Jkvn%G#*Mv1}xf(B5A4fY?lCT)^C zdw#zuKfx-nN&m!Jo+h5uHr$Sq!ygOsWGu_YtmPRtF06bv)$djH*vG+t4s@1wynw3; z`r)r5k<@I;;@T_c%1o&w&uSa?yVcpw5I1jH)u%~E9}UlqCF_C`zBH3n#&`N(LNLH>GgW*=?Jnd{y>_P zUfD%8If`a>6G9aex(h3Jp>gxKzSQKtBwCRGR=-mqnNfB|J#{3kS$nbjDHV@67Fq+# zja|k;FwrEzSQyLcVLu5#xa`IbM@&=ezkm89GeA7gw*9fxu|U*S-wj{Fl$e4&sAe}? zZStrst<#j1$2sQs$wcv3${l=1sl23{r6_%7FueHgPYRywmNI3VeazLgAor=W%-M@P zlk!}kaC+Pf?;d_8?x;dKvSG1fAUq@K(FodiIy$8+^RpO*J}|I&C(}v#9Viw{gB(Uhx^wD6z@c9;^~di z&VD}}jHPJVTg2isr^Fht`-aMn3>l99h`|eZbP$a2a$z4io4Br9$D6Vb(rxR?sorr0 zH+KrQr^I_ik&~_1YaZKG=&QIq^vpp?il5sNlhvLi&la6#)zG`it9-e!Uy#s)Jy=j_ z2ZG$PopwtK{sOuix%Gns#yG3_Umw^8p;7u_3zvif#+)?oCd}&7rOnI&1Iny#t z**6xC8=!a3^uJ;w%f@ln7NbX-;tk)sc#DV5MyL(3o2Lb(j~^10hwCAfje{riwU{uP z$(M~9@4>=8QWXD#YHJhVSS;43DQ=g~pS?CVVbh)vDwz3D1|h2@v9E2L%3@G@Zt&~~ zyt)2B!?Ez}vvUtHH7Yh4T4>VxyV7n5?8w#n-BUqZN3PDHK}WXhDZ*$AyA>jg&HX`n zM3fX1D4V1nnqKa6)6U_6E%X2va#y}WE5mQ0Ro_owJ5&8XmR1q6bv!H~oE&@E5Oky@SUzc)d}XKWJyRt_1;{R^`9_2p4PRVDHZO5d0FAFS3fT5Q*y zTI@98!ggX5`xK)dG)jX`_o)t9+~8e)p@W$Td!@Rap2@EK9{-p{d}~K?0G35(ogWJUxF=^gdUdFEF?O|X zWW2NbTK{jGS&Udp80iSWarKc(u-BNZukco-=@vl0Op55O@~SHEh|+X=c_@u?Cj%s> zD0XYwEr&=a1B^VJ=_B5m)UPb+-V#{67M-%7%$u?R0Pv@e9(^S)5DJ{?!I)4lzp2Cy zu*3MYMAKV>(qs+N)5sO%V6kFroqLcnHVR?^75TlzNjGI{9AH z?_c<|XX>QL%G^HTMM;&N>MeFa=B|(3)@Fa`%C`!7$p@M(mq&mM*ZfOB3F(dIfsgc;t6Edc<_QlVP9D>>+G z9(9a-`*sm8%$|EAAi~rBugTHw_wUnX__s%eV}BraTQF%V>uvWltjO2WGvA3dxIB`# z4G-^45R?i0p&MVq)CF}e*~W@6R$_#(sKai!0^@_q|>wch2;jFQ|)-Vb2mh+Ain#B!$aSK zgM%NxfB(MRZ>W|n^4EvyQ{Lj0enZ)Fq-_ox5-+~6@E)2cz{)=Tzg_tc)BZ1Z3{-_s zfV<`zR)@ctG;t4Od@A2E#Twi_4PhQ2LU4~K7~b^s?6YoAsJp>6Y=by@>0cbZva(Vy zbml|wSW6P**3uV71{~N}nJuoUsQ5;m_%9^@2pIJXO?#mJ`&B6TzIZWPx5g}3+T|0^ZBmz$t5%Ur1?3w22x^7u=(PV>RJdI`5- z^5^Wpb43lPll8hFUv~BI-zdBe2yHOv7u@*7&++0#R;vR_BVT|)XF5ik6MH~$bg;6r z;<`C&{WBWUsQwm}f+pItHnl}m?H-;4%917|w!CQhqGSC%M#99CRVu=2-ZvfdRh#nV zN?ooPuB$*F`~PZaU`p;UwSw5cL^-vBGu!X4KoNq$wxIO?Z{>})&HIxWyH+xol?#nJ NWnqUZF}w83zX2P`_&Wdq diff --git a/docs/apache-airflow-providers-http/img/connection_username_password.png b/docs/apache-airflow-providers-http/img/connection_username_password.png deleted file mode 100644 index 6e36e77dd4cb48f62107a3f654648587bddc58e7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4761 zcmchbcU03^o5!P|;P@gUqabyRBPa|Y0trZS6j4z?q$vUcqA-y*h?E3EU@S;qMnF0u z(g{VRlYmN5T9iFsdW&WcE+Gf+@j6Uopb@@XX=e3HCon2biA)vxaT*S!RJ3xgOLI`DzV z&2xKseS&`%p+IgvaCdjl&(1!j0f*;S(Sz>pj$4ZtC$~^wE!F+d(mk)c>gXq@hPy(@Cb59eAf4h1K$Ny zvvJ3q1Oy^s9O<$3lCe(nuy`l_}xUI_%)8AUWqzr*n+Y^pZitg6I zLeC6Vdf;g*nKB{9kSgh{rD%npDisdYg zw&r?`wq!pR5I#1!Ia65DJ<=ROmi8NpeRfDXd7>*VMgQd1EP~tCFV)(LekLrNK0(UJ z;64kL!JLjdt|rG;K@e?IRf<=N>*m|~;XX4xJwrpbb9|6jLlSbOdxrb-<@r)Hl|Lku zyO1VGn%Ip;hs?Xuv|qAGiK-`cS1CxX8)2p6e}u}Kw7*m#Z9=8Okk%#kX?ol(W)Ij@ zlH{JYR#{wXOxW4e(fJ+3JGGBc$+`i)#D>a>it$Cci%E-AKl-?<{o8xj(k&%2#4@|$ z&Y0hFb#>LdvNb;%qi-L$ooANSouuZnytG6=vz=grQ-eUp&667^+Ts;a&Eeyy+mplP z2)Z6>J#QV(@yaZ;E-`w%|Inw_`o#~ckGMri>-DNP@j*W-?5{1u(y{EEOQDCd z%aDe-g91V~+FvTQx9nnX%l#QJ+Y0W?mTJXJoTg+7bT^2j%$0sEm}s`W-f+_M>G&A(XNP6 zho85#oVVSEs>>*1(3=JGdmuMWvJbe)B$MM})-O$7P?K?$P{UZzf@bkm%iDH!_FaQn zhEX?C;GRb0924Wmp0aTPh@kax+&-9Ua5Tg5Qs|rG5W{YvgBwrWhnDlebNt z9hk5$ax&VOrDic^M?xu%yB!#9bM%BUKTL%7=;$PNdwnv3k%%U1^n}gMY*br3(fIgG zsC`o*#JFQM-+%GF069VFZaBj@1(vPlJu#LcZ<4x(=s#+iw1*#Zp(Y+$s)gOKpy^Mi zhh1JEh{`|oY+<3$n<1MnKkys2Z9Ux)@ z%BUKpVONr&Z|;;k8*i)*dNQ|f3*??JoyK&EP}~>~JsA>|8_DWt1a3eeJ-N5|VI}jl zGMR+Q*(I5YLM_6Y15OHtRX9Q~gnh||_Pq@YxK;hNF=yGk z|BgOMZ*$IA0Jii?4~WJH?1^JPu2w2Qf;Vo4H_G)Dd=BXh=iVR}^TRHM+Dgw^$;8e` zP$DG*Rzz~=_oft=tOs#9Y;VtPoe-f5S=C96a!oeMBG2Z&RVeC*f6spIFLmbFo?@NX z-$(3iy%C@2DEf0N_~e%_5<`V$yEaGhHzJNH86j{SAA4;j8F&G|99)9ZRFv$`dcdBi zu9uI*4r}Vk6o&h_d#IO&T@D>(j-DTBFx|o~vD0mqTH;;PV4m3X-3Q_Y2ELXyjh<)exI zAI=l-``$WPgJko zCOa?C2+evBNE_aoFXhY&d7nehVw^muQmF~`&aPISZY>J6*5|qedh&Tac5AJy9(QH( zm0IPMsCqSB_E1ax)UWj`*Btm3XMM|gi9PO4$$nVHgXxSCWr!5{^Jx6wl2xuh-mjxl z>sJSySG1r52UEhrsa5?&_IB9Q zkOnAEI3>)e{ID|3v6d+A3&aAo0YTqdcwwrOj>!D3vE=2>L>6#hcMp(G$SV=x=see* zezA@FgKI(7{j0FEvooJSAk=~-`Y5$jtM)OXL5uDODAYAE75gF&V31wiqv=~Re=1=G z$f=$(BhUK2el;TiLuQO3QAjt*f0xSq=BS^D*ykxWI+KMvgkV9i;VDWRx9}jNfL%DaGjA|W{vQ$NpS3m4=I-6QJZ}L}~wP_n}m)v?XSj1Nx_PDiK^uj29eT?=~Mn-f>J)|u$Y0Q?t#MSwG^ghl$AI*YQh zPGZ4#`&YAys;bE{_ed{f(V#TgRDd`-wlxn=(T4zRi8N6v&%LPE2D@vdn9hvyJOK)0 zo5bNQwNxHk*?5Vn)$syP6ln|k`s~Ko5m3|5!!&g$uh4iW@cxxT)+;#8UzfUEN6*nH zv?yuiGB};hJ>V9!_DASU zA8w_fYPQUpz`4E(xJHivGHtDklg(P_KGgdyG^%6DtOoA5MV+o&AQo=N*DjlLvwkSu zGld5EG;sN|JgWQ6HPoknI{`GAsz0FpaO|ptUZB5(wy)cH05C!;>Dbi)^kQzahB0rOS2)^^!pMBGqd>KLLc`E3qWiy%&Z<__TUz@VEwbOwXrxKJ5u3YvNnYXhWJ7bY-C*PQ~_Dzw@4q zcEWY^tbhD%ABytj1gg=ULuR4O$zh`M%fx*LCniZXf$I-xzzlPdW0oDL2W@J&Z{NOk z$17TQhaYX%CTXaz%nr&Mr=BHx4{sWI&DHu?bJMNu(Wi{(rNl`=i|iWidjZ+8*_hcn zTwe!ktI{1fRmN8>#Q2FIXJe)P=pUDckPlW`Vq_64XHt3^j*Y=&;I`3a(iH&Xk_p(& zRZbq87O-IFSJJoAH)Dyoi{Du~p#oD;jJFbo1WPg__Q4!Y$a6zDX6NJSxqW8aS7EDRj9^xD&NQP6Uc9bgIU$EzSuy7XkUDeWJ73?Nt>GqqxAGePKGm z5R9MD)q&%jw?d1gs+`u}vRGu(?95C154u4x6d?7k=NC@JpLJXuexIL}mF2ABTxfv- z*xT6+5O2abH*6Z%;@j@Ch9He$Y&03*q@cESMj*l(~?m8SC=FfD5o|Ix9xc zsD!xj0DRn^TkDCQ2pXG$oMrmnPw4oHL(KIfaA_uK=i|9Tm-f@spV3u+%1~@-kPTXM z-1b%NI^M9!GB+J35wTwy-(h+ShP3?`9cn=y1(?>As)tqJN~zsf7_rHMBRAgFb9$Z$ zp|w9G&miaP*bk;e`$`ZX9ZflcFmH-%&o|G_5jko22F%Ej4`_{%HO;ybVCW39*?lDqn1(u<{qD}X_YW2lcjDZtPyo^xHQn2o zV|6)CWJLAN{p;M+`8lb3EggwO?8X=;Mh~+xz~*eSC$A*D7JJ_@a`|AyqMX6$TUst1 z-FH@|vZjY>ywQlQSk){!=dO3DdTBA}DsKmgj0#@{qC13d(_6)PxV|>P53KaFMIeum z#4>N@@x$y~{zh2+mOga1(hc=BBI)8+aCBm&f};?~S92I1gGFyw-@CR!mgOcs{v9mw zSgXNV(bgh+f`Wij{0agP0++y&v?a~&`%-~MAxp`hbPV!2D#7{mX21p%0%^%#S95Ea&2 zqmQilODU|X*w>#nCFptQWHS?)&D*z2t6lTnPS;E>`|9+mBR5z>5m-R2$vMVp$@58? zC)nXvUdvIDk7z0aZ4iF6t&|#3ZEs`7Mj;%!@ot)dR zs}K|U;waI$QPms)nQDDfWZ3Y19^^!(0-7FonL^qnn`Ig1NNf9!4JdmBb|h<|m+$G} z>NBCyG2cr6D5(*fov^^Dmgl~HEB>&fF>K${(N2r99r$UmauCfl({Zv~mmEp-e^j_J z1lg+J(TCUF{4mwW6U5E=&e{GZ$=7-u7Wv&+Y%^LZMLr`C+K}Y*^~2NV!phd@<90tm zrJ#vn@XMEf6^%GjRyfl%wR+Q*2 zIy+?1^PStpdc&<^FC^2Ikm9U=TSukp{J%jB&r7>^DgT*=YK7eXx}-UiZLwfo1*9JQ2&Vsd b1GBE*^^2vH0QXn|ulCwy6NBQ54!`{!fAs15 diff --git a/providers/http/docs/connections/img/connection_auth_kwargs.png b/providers/http/docs/connections/img/connection_auth_kwargs.png index 7023c3a7a072f965f9dd053f77c5a5d64b396f47..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644 GIT binary patch literal 0 HcmV?d00001 literal 9623 zcmcI~2{_c>+jom(n~(ixI5hS2r&k-xWWM^15qC|U8&@yI zp4VbIoNz&&;S|%$vllM#zw{D3bLeCYFTFO4DogYQZ3Q#Q(MB#`@|PP{w$;0sjC|gQ zwZ00``u#3Nh5MrM%WwU6MOC!GEa0bjzF~#+9_k$RUt@1t_`%yIg(IZq`(>tL>=B>- z{$n_3=<@gXa@sCopHqhy8ve(KHqs61*aGb{L`~`|6OlkDb>1ArkdEJf(u%A*3sMN|@KU37UWK z`S!;+d!t~o=`Y)wx6bxn)?k%F?VSe!d^L0ZZ$r)AA%p$*$)o=^r})M@ajS#xngony z*Lg06eEks5LtY{{Vqn>ksZNFGX*0{b@+ z#0=Qog@Hj;9>jhwU?syRst>MH=7w=oH_sMKnFK}Mwee}-tenP+6yF*CgOf6#l9m~nV&Lm0k^g3ly*;YJZL)1?XSHiuScftr z#fh-AL>l<*5_9^a$4@YkHx|3Pz$brZ?SLRnt1aptkF^SCGF4So9T@qEx}W?S9Hhje zjy(hdWv_M_&Bzq|=dy@X#CzWgO-|?oQ*sLLA1p%)S2ocGlYcv~*k1^Hr`qWbrW`D1 ze39xdiiP$)c~TF*B%`J~(|0HvDL6}UXmC$AX=?Uck=2jS&$IIKgwo{Qg;dPIWDXvG zMpEM11PV^VaExN#`GO17fs{SQ=DebuB3m@@iB@cQBh_QnXofNT+W;eRTnE;5qA}H_ z{zc{+2E+#+k7QZLTNt<2>=yczPV0#hS6B%9kd3{-YcCyL!Ul_+k; zp0DzSG_^E_A8QO|+-h}b+8Fg8y{FmTOUkQv9CZ=kv~^}^Lc*}YvMAAFoY;}%=|t1SKU^tU@;Fdz77c@R*1UA%Eu_Y3tz$}<))Yv z-3pd<>}&eTERFRZ4--c09NoKpl|RsR_t%vKf8p&x_J~HJ$(xT zo$oS|$Z5$+e80CnXSF^*nB_j*ndQH~H}>(#(cCVJdOvxk)i2Uz0R!(`MoYbx$5G&7 zbo(K0GjYpF_4f8Q`5Qg(SozU`tNz$D_S>4h8^zdEijt6gb!toDK<=$u;A)fy!Y`h(--=f&9P7^^!9I4 zS6lByogoK?=((~MUPch>kn9^#u;HhK@Ta@&)yh^MbQOJm+xd0L-}4_>pNLW6Rf${5 zn7B7A{qPGNpZ|_4r;-v{iFBo2Xh>#+3G;|{E(gSV34$r{E%Y!9`-0-)9E>o*$tJE)bX!7Ho6BjelTH=PL$J?c1 ztcS}V=!m@Z84KnZ_Y8M`u0~g~R_#1AF0&rzJU`m7zjtE99k_*)p*%{1*Zcfkv+5{6 zR8O`>jGTCSrfnv=TK~22s0ZyRx?UYb5c6=Y2V+m39wWHzVWSoL=?cN?3qvg)87ltu zzh9h(5?Ru&?C-2D*OsZIjn3QJB{cg>n)^q~LN{gBOxrEsTCs>YYEnTcVzXZ93bGh0 zzrFi`%hISd+fZlbYVcBS(%oG>bOD1C|O-DZ0gNyeZ7| z;uZHP-86$>`0iguVGr-XX5S>WNe=s()OWpcM50@!i6`t8{wEP90XuQUY@7_G<` zMX#lM_TFlzVc))-lM#s5Zg<<^h7V_c#6Zqm$~pbCRB0|7%Gw{i@E|bt-6%$}x^(MT z-Qu_`f0Gqj1Z!{SyJoQ}*|2k|Qd9r z>FA|&f!~@IC8ok-O}>jCIHvPUqjzf_FCfwq>*MTI=x)nt{8ZFBRE`|3^>UW9s6CB; z+3O(2M`ZC5LsVYC9G_8Wt~q;JO}+n&`_z}P(KFTfvQsHJbB!C-1>(qCCFW<61a)mJ zfOn$ExQ(<4t2>BmWijT^^%ryB)2zikhe00HJII3zq&#Kg*d$5IyFLcNnc?t;1aLJx zoJ_6;UlJf$ak1T*Vm)Q#Qln0~QLgivXwu@@_BCrF#V znBPwJa3cfanoY|cE6bmaJ@LQff|QJNDw<>XEWazOvKK!+1>2;7y$rtLFc@1}@5Cq^ z5Oh90X3=-{nICF79SBuUIwF7??)EGxo70_>P)j;b9!5H#)Hl_1 zveiRRBuu#?H!9SwzaroBq4NfU|RlwHPbf$H1%Avp5&z!DSXF0SkUDbIQ zN5eK3l*mwE#zO9}ZBKBzYRHFZBD9^0n9#M-o()i^yOx{safWYyQzNa#c6BHxXF}Ja z=TzVIm+=V@q@7h6h!}-c`>^rC(p-m*cP7Kja zXnQ)RsZ+5HuBV_@p$s+68ylhGc+%wfh=gMoM5{ix-n#blmFo4>aIpk}F%GGZQlEpa z3wlgv9Xm2)uZ6;2Eq1EQT8`gz3s5uOC*K-(z)}<-x^bclCkeB(~dQ^KZsXG93)}Y@3u(u2tR+;ef__lr)x~c zs@fSnBczc+RPD}83uSqz>jXlZFN{@i>ojr-JW{l0sO6iiBS_stR=1xb>wi>xSK!1U zL2LdH!5&D=3-{dfOHz~^B!x^?4C$#}Wk7NbjRY>)20)`_Rah=>~yAj`Jqb1^m7l^l&k_AB-56!+#;7VCgf_!uFmkuic!7h?*gZt{)_2! z_c`Me!sS*?m7~g@XM9#~KLmRnseyl4WPAbV2*VohK#Q(PPtg}6O__^XUmF|M=ox4_ z=%bFpNYb{Qkrt(|FI1%aC8ceb?mr5CW!NDnU+Hzvvb;Ewb1u|6KlmS)o9_9`>qjTe z>2Sb~h^;ekty=tgZxaZOj%!mK5;{L2XR53y=c+y=nxBOWYq!@&;XMU942~iy{FPe6 zyH-AoE3SQaeEVrK6-tzPE!`imsL7?9x|XY91sBHJ&nU2JU$VP2Oh>`kp37-@+oHC zk8(4QJ%7dN;5ZA?yJa_tui}60sE-N^DUojZym`q_c#yXxfwn!* zgmwDa&!7ie?kNx?9j05>#_w!&(xPXh0=JQ?~35`m$S9)v?zSr?L@Jvwk6s^tZy$#4l(*9EbNxA;-XvBZZ zJM4>a9U6S-(>`mn37JGEW``@B@@_U>)9IcoaYH#25(MHuq%}@Bv>r`vc9*gF{Hzo_ ztrjf-1^GrHd~Esv&v8iJWujnQmEGS)rsXzTrWCmTT&M=K>$5pq^mZjKbwRcG_#i0{ zfE=s+z1^`uX62_(G^}}hy}enmDimbd2;eNfz$pLz)4-zJZ$tX;`ilQv992nE^em~| zsdDTeuea4{iRPJbZCL9w%c`!f9+@nk{Lvbh>j46?{%Ox(%zI{)9jQk@y9HHhY_?V= zmn%Y<`8vD8TfSzghk8XOtN3F~Yup}S-6pknm{tX~42Nu}xxeoOa&sZu%@1&;QipGa zr4Kt&cFsmQbnH4uJE2wwtYbbANF-d~*m4VBm-tpDZ^(?k zHMx6NMFkVJ4EgcPr!Y6`!COlszKg+~>58EXL+|?UM0ZK6SRzO3EfB(V#u>i9YB&iy z+jDO{rep=N-anpeGyg?7 z=xZ?-BKBfEud<()7eI@R0KDQUtRpZ6fVyUH}^g+ zA&J<5vq7al<_9Y_;67axeB6J>$cR-vbfP7u<>%lUw;YO}iBpPyexU4}#1II8f$xD* z$AX~q(iWkn%&HyJnf`ab-?)rD1Hk;|O1mWf`8kco9Id$V%C`-B3crMLH}0aRAIuQ4 ztolnVawp}4`j*=z8=5wP&H~yPNyaw}mLODimeiA#(LPLa1Ji&3C92VPegiJXgD9V{ zZP=Phl}l%rr8$E|jj4If7UYj_jSVQ1=PL-sX79Xsg>;iy_O`nGnp>UhN<&x(wq9ICsLozy*^cqcG&t(C$ zw(3eS6H>X((`ZL!e|Nv}%^m60idosmUy?UptWNLS35jbtq*4&9VH)fg4QGKFLxO$d zv0A1l;Rzf&M80I35<5}#UW z07qEE3+8>ihCzIrKDSjVSYl0kzd0V^s(hHis&i{~dP3QGuw3x>{GfETq3UI<0E++S zeScG4ZpkOQHS9cV=}930(7W#D6lmPP)D*^=r4hqRDFwI=mRYxi05;jOOES&ZOT9Jw z_}Uz2Vy+JR&;;ke@mjCl&50P-T^hT^dNMMt(_5HxoZA{70@?o>n`h3iK9?27D%{); z5l?FI{^1wEe#!dRu&)gzBq=$wj; z)iaW6`y2jzOc_i2*3r-dWV2@DpJ^Ya>d0TpbL3eNfRzC+LN9{H6b>TpMTfMz6PM$tC8tK0pe9(V zWZi{A+S51rlxIFrf+EDM06iHSw9ORnz`wauasS>?#Fa9pwquuw*@W?@sHY^sI_~}i zf}oipHjV&;k>wHRa<4Mz(Ie@Ke|7o!o9O;>34 zkKrMUHE3o}eR(nbFfi5OwW+4zR|;5y&C(Aij$p_OhEj1n*RbZOILHY*3;eWdQLLMF zIzIqXr-UP15p;Z4=lG5sJ|a+S{f`0U?8DygI7ok#>peObJ9_ox*Z{!*NTTR%I!8Jr zoohK=y(ypFaQtG1%>AaP-S--3_=Bu3aC~Or`dS>`r}LBijlc;X{AF#f5s)&^kDF(H zq63J2NT1$3#}L2nE4Gt|6DWm>PsYonhr1tA&mW0{Uv>`Pmp~YAh%l~BcSQ93e3!w* zCC9HnA;P8}!W8}1Xs~`a-4G?cF#=hbf{)M**Lgs$_&#EGnM7t(tl&Btk1>w>D)$sS zCJ`1#^!{;avbEK9q&!z=!WWsM;4#l~BO1HkuY-Inte?z+fBkC%xn#`vMJ>rS$9dpF}PO3F$M%_W;tTB|<`*t{}BY>T@%ppMeYVE~+I1h-1NxRg>A zg5`xDhwt((K#!h4BD{vHbfy*bUP2j>bZVPSi=GQU@XHJOIEX;Y=jZ3f73fqi3~Hbs zJ_v*!iFJDV`}gmk%UAppegu^@#0vx-|Fl)`jWBCcG+xhI$Wk&-tT`ukxAL_Nv&IR& z=tc7SPo5i0e7w`=s<^n|k9D0IGmaEgMs+O?Cok`{w#JRSN?4TVoIa!#J6o7X3c+NtLyj1m2J54Ku8NfltG`MtV znaa}wplZ*Z`o;I;)LyE!LlFt3=GB6X*25^3zua(;o(%7)7n>=Bz{7r9$kLJ*(zeYv zBYZg!s$ZR9A{?Jf#XI*wHYbwYy#1mR6_{AkouA{cF!s>rP(b)1q0|aLZrx|JApsah zY*OcV^+l(Pow|*r2WWdhp~nJYaEk%? zc(l&jd1Ip?ShzVxwAiKnL#p-ESB9yku(uPAh#?l5GUlDhSoN322%dY{mTZ{FZwcWRVk?*D1 zZD}EU3|?6}w%<>4@ooLK!~Clz7o(bcdGa_&gXAOOvnbNLKy5{fSDQSY$Khg08TCz9 zTi*ocTRh2E|2i8o%C?8av8LM5$JR2-vJ+)wxk^`8sE#>?=i+bHRzc42X}8u~ar43Eg8c3QY!gT3r0AERgA;z=Q0 z`NE*^n^T?j3fHmX3l|MKNnS^Dj`rBR*cdFc-_u1&qqi%&ZSQ(-P0B44ziabM(grE^ z!DiAeNi7D)na=C|qP5a!^1?e;uEXu+4>r`fk!P1@i=gO%o>vY;(}53Tu0M zToR*oBGz2u(Wj2Qa6;iNrxf}hh4;)P%5IHBU50lLPJNHfAr}G<^paXNj6c(#@6N7n zr$*>U*E|5%iz|Qmq418iOeb%$aEsr4k3D_16oWvrY!ecm-n0Gs>Ux>ZsfPBbV{pio z6YrISEJVD7dpGrPdg@skj-Ny1 z{7ovWvmTr*cA`t6Gw+`G`19@!xp>6y(>&spb^@U8{XIJT7(A5OpDQqQZ@OAYZm4KD3SsW($Znu zGtq6Y-#O7U$fsp!K@0bn!q&v7G!o7}XbhrvEw!%YbVb|I$Dz-Ae*L;Fsu=R@yjCHG zjS-pQi$pRBalV#}D6>jdUMnE)ezRxq^ldz&#(}6KbPGa>OuK0Q(FD=kO|CR;!RJ7N z!`7U$%OdyuG|C^}iGu#}&v5%a^IUEr{oSa9p3d-xYUg3D(U>#4UQC&EcLzKXp9#f< z#Cof#$TO}A;vNIHX$3oJ;n2(PPmN07yQG%}|FGx76v_FV!l_0)4-Y9 zbZ(yy2$hl0RkKsOEr@u@@})EoC@9erb-#b^YBISA>7}u#NA_lB@Mr5xrwnWd;ey+# z20lZ)kK|L4s8<2ycE2u6F4Fv`y26p~))|+6R=8Dr^*- z0)Ui`ajGj~xM}9rbq6JwpbIDxr z1LgG^!M34#v@h0uTAwQN9FW{JDl zwK;rq6m-JzND9E@Az(+%w_tn#IHZ%NLisRsJ^2RA7PXJ)!SoQFBne2v-nL^}5;Kwu z!LImtp6)0u6F^R$F#=ye1-}vOcS0%PJO>vawYnOtUl{A8dato{)<1Qb^%U#?-d(A^kiECYHjx0rcd9SJ7b{! z96pARQQ6}ZGRQC&#%4{n0d>qeBAz0K0tqb`SkvsiMas*105&;C05l&37UD#`U;plS zkz`u$=bdK2-wlW$YHw$20A>eHGz(~1xg|P4*m&JD{ty)8tw6bcSE_ zMU`0^_8k2*IdeY&@s&mUpaz!pDjj4V33WB}NdWpqm&*MT1praXlYM8j_^#2jHgpcjBpYOx2IWu@IqX04SL z9wn~mzqeyrKGm6S#(-oOGi5RRkt1Oq8HKNHKL)pdIVnRegA))!qI|@hXA!GQQdB1u zQ_Dp9vNZ6>>v*g*5QGXQnQ$E-e8(M|y;?CL8sP^!$YPvmu%YS}M1nI68ZIT}ipD_z z<6&R_HEiY+_&?mxSyv_W)t<$uuEafyjWcL~qm^d{zNFfORVpsdZ&4t2i|UnX^n zc@C8S32ILd(FYlJDXr?Jlkx;9RvFVAoLVAjYGtzh;(LJ{k}GNccJ<^JDh$}n6Fohq zwawFU91anWxNV7+{r$5?SXnQ;pGHwh8A>DMlzA9KA2ZWjW%u5XuA=^%L8%+hdNA+KUxJ+NQ^~; zTn9UHP(jy0^`zM4#$I$mj(&?!TSuq#wsw~CIzANk)`;z1NF&A3EomfU0tp(xzd z2ZcTVDiW>l`g~X*OFz&oTQJg7kzk+Sm8sM2upK1z0}1W_Bs2f_)jcE%Uk(~1PJpBb z;gB%@-1RSw06?<831r+ngG}CA#fXIes>*|VZUs2-Rx8-B;om}jF0P}m*bEr=Onkuk8g0E@GN8TH(>+$dxn|e`V?;qmYie-%1%QuKYo} zTHWJ?Mv(qiDu63;ZrM_cbD)O-WOK$3K5Gm~5*BR-9R~*&jIxmP5IZpcKlL$yg*J^v zz)AeA@PpaC=ZfDQxY6Vw(ZrJSs;e)5$+)5@cBlOOfi0X=TV&u^t3gWDPBoA=7(7ER zQ(FfL|KmeSgG(WWbN`Rs8w9{eDN`;Vv=|hht9ZJrw?>o5+so|!QQ4NeB${+x0%*88q;*~Y KT8Y}dfd2)j<)#Y& diff --git a/providers/http/docs/connections/img/connection_auth_type.png b/providers/http/docs/connections/img/connection_auth_type.png index 52eb584e5ccf6463273c9b0d35171944d451af9e..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644 GIT binary patch literal 0 HcmV?d00001 literal 14199 zcmch8XIPWlwl2%HEh~tXqEt~qIwDQF282i_fCvGiAiYU%0u~|)Qlt}_l+cUx63P;! z6RNa8U?H85AP_(Zf%_%6_Bnf>bI)_{k9+^{0QtT-#~gFaG2eHLH}7?|)fhmmAUZla zhKCRpC>`BD5Ww%PXHEe>*-vEj)6q#QK2*7H;A2j}h4>qeWG!xxl`vB)-qyt4sieB$ zK!I^4{_6to9tVb=yr>fRj*oBX{%=hGK(XWx`L7FCMn$QE{n{j#%tES+jZ;z>wWBLy z)|Y&zJ5tO8c4S2zN-5sF;b`ymw>G~46VlOrV)=>wah3~pf%ezEbrt@9(0&^@tq1yz z&SF6ae1<#t0(k%VoqP=R-39QOm9E&Dc(-5w)ot1MK?$B)ipa)5&(OV0dve4x-+#J} zB|8atk~=s`G4{FqT4dbO+0(-W>d2h}*iREaZ1QOCh6GLh?;mc517iaZl3If2%-h1l zD4Cs1${fNWe1K1kTjTDRT-en3)V}zPm6+SKa(|U(%9$0!Mn{7)`&xTj;C)6W<=vi) zolM3Pbk9y2ote1PDHv>eF+1GCX)ZlyF|dG*IAL&+X6#VdTzezy?+)@swvKC{%slpb zrFbGljB3@g>)}~$r`{=%rzK~HPU*6J?cVp#x&9IDxvS}W-Ky%BiHEjpg<_xETUakU zI2MVy8Jw##>yf={`3_k4z}$f`*x4&8r*jp4zw*m)yAE56ToooKJ77@F?n<-w}U^m}xbX2m=)o+xvHvuLI6^WI#x3k`GC@K=GXzz0#)m5{mr@G%S7JkMK z>+txD-5Vq1_)jLoThg( z7wLwInsm|}hEu?K*|K}0JT(H*Y+nn#Sut@#s_yyOgQh~xw^=U9tmjg<8Q8ol9c+{K zMTKIe|7PSGi!!T5&3L#F`-{`mjZ)wp5#lDW5Lm!9g2}xZQ?lZ|-buazT4lagh8L;9 zbA~%W>ey@L?cIT5*=3dO-w3I~G}nveTmNDAMV`P_yz+T}?7Y)jZC2)BwUIoqYz1cL4D z?KR!qE9$)0IsdjnOH0cO*t*Sx_hsc?Jj2JM5F9he4m59bqhMIouVF&mt~f7Xu~Qm>K>&E8%#QO46*8mprfkZQUl<|REZ^47lDpg(M^^u zf6Y8J95}o`MmtpJZ9f#1mACh@$~l}ZE~{+*%L9f7(RNqG0X2ZD_s{uxSYj#L@>U|KplpF}}~g!L$O zDK#Zg?W3_^x|c-@>fGM*Vn(32k!f*+Vb^xB2&1$v5KeS-{I$?!3ya>OvWoVBr~KbW zFLu&4utgN%;qecr+5%z%_#`j{9o;>X62_s#2@$E(E6NT+#bS~`g1AhA)8;^&kVz$jkGX}Ji=?z^CT&{zT%^0!USH2>Qtg;DQEn9`?751^YuVZ5Pkes*W^W+@ zekpPiYrXMGGY$*{Om(ciqvKi8a*})`yUe9Dr|zB}!W`Z=SDCu@0U$>CB~~#XtG`~o z>JQ7)5f(dalOq?JcFZR$#uoIgRO#6C}*EwY6Dekdix!B-8D2XStOGu#Ak1 zmvyX8+jvs|Idj;iKdmYl-91^idA1#w1g+%2NcD_-c#zYh zTc_ql8GHuMvi^)VPPpfYOTs3#1MCkE&J4^{ zSohs_x9*qfSK2M|-9Ri6@XsX@;Zaxa;>*(g%A5C|kjRVJk;1_Rx|b0v-B3tqXy{rm z2a^cWDyv0pY0qHFRy8u~j;gq=M;aVtmlHwk2;!_4^`7qfTHTFVbv3w8K!K`d-2r{dE{w1Clu9ak=bM!;GTt()^PhF4HYISblQOftp!vINoYwDf)+a${@;ZIXgOtQK!!Pb^=J}}0 zv)HtS&P4MJyx!ZXv-OJdKNlX(U_IB)u(OFndp)T6jjj^z9S3eIFTW%1O*DmN+rmDD zyExFxkWo?H%uHg%zTcpbcp+m8ZmQB3|LU#aAB+6ik@}kaySD4fwN@^mBF6 zon2*ssI50D0OyIf={|}BJJ$$fx<7`oh+>BiP3}cPL0Tz4eLa%l@gh}A#jpfFmh#Vx z(K7DVlUpLDcU{i0kGmWl62{;JE23Caio8ty;g{b}uldICQZSr%9tY)i_ka0|jcz~@ zNH7bFi&(QFsNSng<*#90T*J=hirzjx&5T2=^81rYDUC1~L_6i~(|A!hrx~xwmfGat zPCU3Hf};Y@mVae``*7zW>7XT3ZA`-uv==0YEym0Q^rpYM>8F({bDN3_{T=}~jNf%6 z;xaRDx44cH69dwWf4-k{$oa8sX)&}@Gyxdey5GUcEW-d5b!tS`WwwQ@x5!wWK(ryk zpW-Yv4=tb`0X01nEHS)EuV0@a?+#>$CTk>!RKFWd0*P{NHrZ29SV&moB;@2c zrG;`ZX(Pku&k2_j9@JYvb~CQtm%W}y>=@ivm4%mf3BN7I+s-;{gk9Tx>{pyw^`0bu zfXLb2H$>*28QRej|3rcVi~XtrEVd((s{+;KPWyeXNpjUz2^tYp4f-Clt+)x0?{ zRiXGd=c7QKY$3$dpL=@h8J9aEYwp6)VU04I4B`jdV{72S2>-K821K|<&^~{iZMD}M zcXXs3hTG^lt|`@}FI8dkJ3Rb1I>J~SDk?8n?|p^Oh6Sv^wmRHv!wvD?wmMSoW@C5s zNTHG1(US}DV1ei4ju zH#pej(G+pW!x zXR877dLv8m!{6wZ?*dpDNT6HgnT(+^;<-ALn`&WdWGIsk-OG|V0H8ca>59?VP?S0V zMQ$BK5ol*mk9Zifh{T&y=PKznL)4HNj3-_a6W&81_-z_mzA!Wy148M_r78w=6E{&% zKGR(YfUQ1}5h&e8ILJ^knQcL9v4sMKpl^h*snRG3c;8Jzb@Z{ZrFD3!W~-m{db1?;dSA|vm6i1uo0R;c z11)k``}jXW?(lrc>d`AOOQ-NU7!~L(i|qmjxr>szEqc{69FtSuG!Z){f&5;e8WSY# zOA-SDithaI-y1pl*u>I4yqJIldmTgf_%gL~XgFxXtP@F0@`HgSlR~ko?zshD7$1G3 z5&=58;(>pA87IUYUMIPuji8Z>Oy%yQd|XE~S$<4fDhd0RS3ha|Q=H*gaa)_&(9~$bwnlupzK$? z;WNuh+8bEOIhp^7_fSK};X`p5Enppj5m7_W&gV}&1NPxETlpz{eV9vuy*&oky&8*( zk_yG%TU=O3lJh1e0@v?x9Qm0^j5d1W<2lGV0&gWAjV2mf+Jt9F#*ZeNSUQB4;e60l z7BoIs42=V01Q1Ao%y@3UXD~6dXnhnj{-+^I2tg|45eTk)`ppSP`E_cOGnM-tdJD@+ z+yBe}&VW@%A2qC;G%~&t4qWd5hwD?v9@`Bgkewwe+|1H4E8oTTc9o~+dj@!Meo=uG zQ(PRs1~*0Hp{Hk6qXMv#1Ic#1cA?g`lRf|eW z`V4MK`NhQbIuxmvH zO$>Y+FDMYu*VB6n@DwWDq8nI32_r5Yr=wp{+&TfK9V{Rhd!MFs4&b zUY-%ya3<{#=o}_mX@BYdR>e8*;7;5ps zMlmrlQQ=EWLV}H(yE_KJ9~j_{{@aQFOqtY88yz>XbPTVOT=bfA3DPVP^Gl{qoD^lB z6UE#_Zv$Gr>f0MBNBfC8-awBx?5qi@J9C*!Vs>7yj%r3E;JB_+uWQO3g~| zBmuL5tIVDHRGBK(ZaR;1)Tc&&y+`%TSJvZo%21v1IE&^<*$}{o7aV~90FGOt4rvii z+k<6xswax9${A$?4{95QfzJn6sCBRB)RT3AK1m4S=H{mP^yyO{YLpHL0)CkIPak?H zP&D34MPM+P**5CfrJs&7f&E!_>W+Q|^J;JpdTy8UVDH%<+5bA%Fp1HTL}^3rn=}hM zJ!lIddD8x(g&EzQFxu?L_F7XT$bv@h={kj|_6o_X^OHms4^ofM^rhfmPajv0o>9r` z%g3J>uOHXmDrjCgfx2*97D{5EdW@M5aeN(gInum&k6-2Z{d!I3y#38Yd5NgRf`bz_qLWC~Q0<*87VZJL?7Y zIX?<70QD`Mt&x5D*$Ls^DZhz>-Db+gG$-Py1X6(|OzSA$-$cG-IHh(yGgP)><(t#i zT8%$UzL~{lP5MaQu0y$BZaArOF=bEMQcB6=FvZDe&7HmGJ4rdADo$;$m+FEcGME}` zh%^Dj;euLoa`Tbxu=0|PhB@r{cuIev&+-AWvy08j`rG#`fTl8^B?u!9*AwJ0mT*G( zL}2y68rtAe>S0f;*`iYT3~MT(bFjOVJ>CR~W9p@2e+k@_Ldq3lH4J`+*f)e=B{Fi>M^c62Rh;ForEx zUk~y@S8wvAU!`uUADp&_1F5J1m=dRW9ZGUBS&KzWd01gR#t)Qt6!C1yF_9E< zTi%ELnHcZHfP_#&QPS!@FYLo|py8SP&$S6YI;0c{JgN%+wp4GBB8v6IFdBWE9LrzY zT0LkTTo8Du2=nEYrF@h2U&H_GpQ}OMFj^mP_Czz>+I0G&YdmH<{jh;}P(JUUJ;Wm< zX^Mq`_KdxS`k9qy_e2DuRQiXNW*U!>vvcXCEfDi0@6NQ*IqgZGjSRAx>)o-~N2AF> zwUEh(9utlYaNqbT9n59(G%((@m z+z7*tdvny!gKdR@x#h}buS3nx7nRt>ybD$_N?KFS8d-t*F%^LaV-wijH9aL-z>=D1 zk)I)_!Hu~v7-<#v+rDe5nU4Wo{uQ66?TTBaiqp@gO z_!1xD2U8k7TG;}aT2t4HpFI!#XLPhDEkWJ*-rQy$@AFu{oIjbbBL_Fn&s8Ji-Y+e5 zUnD3l^fRh^<8BMQFo5J~?t5F-j9)LvtQNys@dJr^SRn1d_!fZ7tew7>9pQrsUd)lx zrTqAQB~q~X=Dcw$6abfy+iCMyrQm;g5B!kP0 zE6FAv#;_Uj1-;|L{I_rXH8SQM&to>9FWFUhq!5fuOm&ogc%}t@bzoq{^_lIArN@XP zu==3<4>Dc^?J)A8*Dz{Xp6Ptg`RNZLWO5|eS|4*fc%_6*RUB$?s&@Fu3hcJ{Avs`Z z5Zc*j?oA25wMEcO{Nc(J2hQ|-)dDd}01LnZCxTkzEu0s%9Z*r3(Lc6Ex~|Frmozzd zsc=G$=$C%rC+o*m8HTiW+ryt|lyGRKlq}0-jpg-gBT~>)0p|JY+)O%1j%*8P?hM?m zrH^$eV8!FVSsWUGJwJE)(JR8>HGRE|r2F63ySI94{0Y85g-7BBKNF!Kr{^?A@kzG> z>zat-{uko|TV;z{;q-D4Elk7BeHQL7Xi5f`BOf$TEugu4rSDv4o% z+4$4d*1pSbn9;i)sBgqeq@QDGOn$O33;QTm6z){}MkjA%tlVBeo+P#Q6;^*9bgJsp(tD6<#s)d5T#sK3G-B@#SLY};&Yl1Z zR2$s1#oZ)GEkzVm?~w?$r(%dYZHcv$4tf@iys6sTX^4}p<-bci=4UilR57}RWE;)q z_CH>Kh($f@(@bo}E5~(JV=)DT1eu zFVp>c7s~T7J=D~R7L(79t=^pTFS)L+CgYASSaFpMd$3ga72=J1RdH^7DSN0DSDN;w z#D8s{11%S8-L?!6WA;ik*p3-Gi<5;pH*C*|`TN)0-JT*CIX=z&&Nt%E%zW0*$ud5@ zJjlj@e_pWA`SxTVo>ZW z2FEB*9~IR+*rS@OohGr~wB-eu_45hI)_uP7BH7z04vat`7Iz4fQgBV=v-U_fmQ)cw zzR&4(asfMQ#;|2}IWHmEQh){nIMgC8J|1gY=Qg34^0V&h{^kWT&NUJ}zm!f`;Njs} z972Uz_a;VS*lxTf>y%D9;_645I{YH`Q{=Ayt3d$loNW#1#g{hPQ#(^>fvXSQ30O)0 zCNGMw)pAdDuMe3}MS8x7_+t6rI-f zBW<|+N4jk-VbMMu>ET>9HW)quRgdQV*`sGifnJZoIE^aIP0-E)^+#=C%t-Dxrzbi_ zwqyeK4Ls`9ql;nHD!DGx4R&xfpgbY0L8@_vByy;uEBBH5U4?XyeXcsL_%hgTS0 z`q?+>l5CR}*xe_A6!)*Dq=f?a&ooOTXMn0`(~pKENrxPY2zfeaUBRES-VA@zguH`l zv+_H7dzN$RyL_WG{bIk)UZXBQ318ZSneT#+FDXcDC`>oIz{4hg2Z|Upno5r|2D)bN zK@rB{Hsjz2ky$5f2mXA;d+Sz*gtT<>jga$`Tw^86gF^8-9T8QC_g)v>Q13zS0koh1 zC-7GK?>hU>V9g#XVQXAwoJEh^HlM++ElZA$jv|x9tyLD@d8KdK1ZSh2A&CAz6Hbi} z-CNHHt;_^k3J3_S32`>U@uS7z&g(sIp^){Dp_h$TgHcgU zIs$r!Qmz&r=CJi7k;w+T7z(@O+%nF4=6&e3W=5p6OG)a<>`OWo8LHua!$Rf_e10iY zUmccbO49>Gw^RLByAG?rx(&sFF9nJkfbxsvT)(S)KT0~aHLRnAT}s7N`~^_yF)DTc z3I>v*z+Hpu%R7BQzmnQA+AgtL6Jk%zJ@F~1`PLdC<@eJ&HE2d?*^do1F4cstZYIIZ zt@cRrH=sQySKo^<`S_zF2)OSJ&gFSLCrz*TE-vtlO|01R&zqNuZ{tfbYsM9s!-{dN z^-b2Z!Gn81p?-y&X`#v!(IlNYR+!n3DIan=+$W-Qiq^h3sPCq&_6@v!jcLlr^({FRjDgaG4>(>3#VFungj6;q$)n?k}mkc>mPt(!u zbJDW7hc|z=F>BLoX|r{ZN{Aj6PzEMlmo9-vsz(ERiGhIDjRs5X-DX=(ZVkO;A!weD z6FMZS&qf}3j4{|PzAwqb;UvT#4$jr%7GzC8`MMjJpU(9GK~TN?3_M=3wLx|*PJ|yp z@)2-iOno%yw(-4f+0U;Lc|VJ+S7MG4qh~pM(!*kOwhF-a?jt3zeN7Mep?6ChxLy?3 z4xh*7u2))b{9K5hK9Avblhp^=c|+7H5=rbN+282y&CpUTf+&J$F-tZ$u^i$hKs;T2 zbmItU3$JKoY@b)>jyroOqMpAjY2V1nB<$yGGN#(Xbp$|rq?A*ESxq|=r{56}0MgD= z5EuxtZVW@|`UK<_0xdlM@1plOK;B&LBpG4YP8bDUysIe@0|`^XcUoZy8V&|^zi=M-s4A;sOX$*T1;1rsBG${FT<*_FLG(*32Fi^tP=L&u-1+kM0cR5J&DC$3AbA zb$lE)q*f7n@Oq9=qnb1$?fgMxjHvAjI6)EtE{+2uGqSRxAJ*?`uKU-@jlMp)MR+}V ziKw_JN-E0A$cIGhw0|fo9btMbVySq@hvm>oFJvy?v7=n4-c$n@Y4_an3KT-Gg%4>E z8MaK9^X`kkzSN&ylc1CMBWcfL?bc4&*47s0bkvsC7Ta7jy*3jNhp`(c3l6?$5k z=4VNUNo+k>&j$$>Cuidpr}Rd0H23Zxzhsm3#TEg~4KXQwn5*At^ORr0g=p#yC>zM$ z)$w3r6e-ktQ^>6RYYEGOb23q3czc^e#&$sWn@Ko-2Tq zA$-e&bQ?!(L2Om~3^>M6`*HLl4$^$MY2nJE78SH){#|tdF{7f)C;L+62s0BVpcOw6 zKQ7hgqcjE9)ZtP8YWI$F|;t2j#oW$&Oq3^0AN#jT~vUssBB; z0dO6)y)qH;Ki3ccPk|D3%KztR?EiieJ~W3?V6Qy6cX)Q9^ZE-_0OwKrHUQ?!_y%a3 zFn}ndwF2E%mZhc@7}!KXXCYeC6wt87!*{c#$xMQQ6+uczQx%IF9f#3(C}}=ZMKCl_ zeWVD%n|fmt5)?cRU$Ve&U*>&b?F?X^@2(X5@*ZUy=ZzW6Qp?1Xg5iP2_UZjVQ??}x zOX5qv2!KV({|wC_<+oRCX9uRCd7nQIOf>~eE`~jf>HXk7vNP#25&cDP<2G@)Y^VIkA9cy#;qZ$rlS`T)ceFnke z&wgAZ*ZL^}(vVOQqle|5J@LyVOtLfqpI1^MW!e71@o1Egqm^n;AHtFr)WW!UPpZgGLP%*k2QOYL6z{t2lUxYidIpQj^?iUYjo_aX z{qRV^cT>e9X*poT=6lNBz$rj31^q4=n|w&wPikm5EuNe$sPr!m<;F-1MlSsqh}L&O z#_i{4N*N)P6Jr6S?8juw18qAy(i>BR3JevMw(j1qXi1e5t;56(-kuAXaGYtvnIA69 zcU4~)bSNq=mIOpa@BX6z7}Xv8N|fQVRrItnlO7dg1~f*OJiid9eJB)51)`cqJ8aaB zmd(a8#iYjB$-^H`tN$6~)m<;D0-Enp!p55wl*u?-{8(wlZsGb|p0pY^6EdR|C(RW^ z%GjIy`s`x%?swF0qF6Vzu>f+hp7lZ~(@r@*Tw7{5M=P9DZEQru6}YG0^$p|AZU;3{ z$XkD0?#)D;2i;id?gzR!q`<#Z6+|g*yVaz|oA-s3B2xIfEptSl{>Ln{4<2$$f4-s+ zL=Xn;d5|CZcb_kXbM58>@+mKmBy%*-k_T`BpgR1#@G@&xyMGv?3LKWd{uQ2q>zmZ0 zZp$_U%Bui4aK0Th#H0&v0S(SNBDQK+F_$VX+F}+pe`Yg56?&DoMNi8ueSLp2?q{<~N}iH0>+K6G|4+q()-t?5Ww z0>uGfu%lK-X?6RJ?1xYg=Imwp3n6blX-o3|jbT zfKhMahF>F=qu)%;D$47R$$C!*4~{r+dIaUBZ1IGdUvmKRRTg>I=D;>V5trU9^|7h- z&1r^8)W^JMBn|#Jgv(7uHnkZ~Gt7WZG)aucV`VeJ%D*EW>zZgm_kSVYfv&{{kX!TRvA1pP#c1X0u5<26+ zC60q8_xoB9N-C%UToLdD$&T*70?eNenC#`;dDhxV_5!lq= zE)LqFDs>iQCH!*$yZLBnZI)=64-lCM3uZt^x1CE0cus#;6kgr?f;1Si(>O%559CIz zf7RF7so|L&*J)Q-qJpvM4ggkVMXOWmd+Xg?Ztqj_0LA0UW59NVRSM{W%{8ZEQ1}T{{VdS^2f@WOxT$~Ql_ET!Tq)Km$ zN6#}evjm9D&fWD(Yo_cMQsjnd_ZsZ@UK>V5Ex*$@SyB_1t85?O3X=bMvD8=@fU%v- zY9^;u(M&)=%-YA6io;Ccs^ZG35if4kQ}Xq59qT@kN6-5b9?df|jylGkSGwYq9@G6h z$3ibqE$DS-17HLJHxj|rvsZZo=9n)4SEc52eD_Z^ zzZ1~CL-I*RaiHU@M*9qC!8V3BRhX=I5X~2hrB4F)s~OEd_xOXSRndDZQM-rE?*KT1 zdr!+Au7UuN127v)JXls;fLh?fSJp5Nz5M`uN=Ii4^H=BQIH2}!2eJSK2lgIH5TUNF zP92f%LWDfE`@ZbD0w@bUA3z}?C~wq2mqXW$ zD7&nV57fdN$@8CIfG~MxLKUu1wF1V1LT_83oyCIL49K8=l+PK_V=8f7W?hvVH)s{k z>{3-ri|}ws)tyNB9I;B64xb@EBQux^dY7dDLH#Im@~C45P^{6{)qMi!!ZAQKtrtKN z7(g)HTjekY03stTZS4q5oS;#rY14~e-^v;P%GFc9QQTh7x@G?#-)S*5HFfOhp7jGC zUOyI4T#N^Y!a*o3q8Gyq3d;ac7TUETkWwEm`0k)Kod39eK9TaAy|Ap9ZONW0q5;0H z;<#r02MX(1q@c!mnab>I9vXtGs_BjI!XQVNppPH_)D_FFO;Qeg_^ZLrKG$AF{VI$z zAO$x#W2c$o9IsnoMw4wAa|7bib?~A!y&Y_xMcKK1fU5IQ-=w%0AW+483D4Pw^L@~L@K5TQ%5qoBkcfTgBV@d z8$1Db-em8ucP&ZgnD@H6MW10%5iF78Tzr9AdnM(4W>lAibruW97F-z$94=vZ z6Av+cIG@RI>(~jdVM;lkbEe_GW`G6H3&0S|rJdeOv{(6p-o)BhM>SVR!Oz;66iK}> zf#|S-o3FX?d<>IHq3v?;SK-6N(QtROi`lk9w66j^KQZUp1NBX{^;>p6jE~D`dr=X&N!8)Q|&Rw z-nmQl&ZSP#*HgWpo_pyQfT?22_wS@RiC&!@ki(wjZiBN{SwO2klxF<#}@Q|-ZX zGoR0{Iv(poI9Z%)4=jKc9lY+;sV%!nd+25#pyA^XegiCJ{^<8tXYX~N@jjOa96P#) M542TEz)xQPFTow~N&o-= diff --git a/providers/http/docs/connections/img/connection_headers.png b/providers/http/docs/connections/img/connection_headers.png index 413e9bbb38864faf0dd5664703b9089ff0b7bd5e..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644 GIT binary patch literal 0 HcmV?d00001 literal 5256 zcmb7Ic{r5o`yXfn{-V})%5hC16>~W0*`i5pPc5~&!_Z*&@TQfZG(MG_BoO+`NXB8 z9cLtFwPB4YYQ)t9B8V#!9>!GTtzxn9z61NxOR26(i5#r%Yd8mX zpspJ8#$8$syuMdWkp$lPLfgu|3wA>n2RozP0kP;M(D-3DJV7{eSD?`7DFLe$svtVK zQXrI18VCt+$j?P)C&oNc2yK;mP)&FT>;wtK#y-+W5L@X#Lh$T(nAMSE;3&rnyAcEC z=B`9Y)31Gasqo@r&2Lc$Ksrplcbu4A)+;|xiiy6^5E1hE2n=@Smv23B#hBxd`M!T# z;oFxEnNSws+K+=nu|$kS1<~tp{^0_Dt>)tIEzT4=aOBWXNS{k_ety2;UKs4bA;@|O zqI_48U5@qP!-su9fX5X;*!ZA*_1&YXJa?|LQ*sBZ*nIV=EK&jnJF5asx!*D8 zqD|Tiyz1lgM;Zu9wrlLxa<^2*#&T(F&im)*@7WOyvr6k48XC^!Q6Dt=gTuhh;JSW! z(nzC#D9CHZ`xnj`gs`zBwwg({QRK?w@WqL7$}3N5ZJ|8t-809c6d6RJKWPi@wu&2x zcEr;P^YfK*^h2JGK+AEsXA3O%nX@t9lWTgD%@8mrgimB`Rxh%jl#)356_u5Rr}Hff z?kZUoZp_tbcLuA?4A$sUGj)_w4(KQjf$QcHa|a|64;?m;j&!dcTY@2ScBl@KMh*uy zaWRImoXw@-$oX-~>Tsk#A-+3jdAhG?1YE7D&CiQIA7B5B6iA|ZE=_j%Y;(4J-riS3 zzCK?Y?8CVhzotq4@DLf){qv5U(jL%R7dBit5tFhO#p1X{FQ4_;Y!)_YAF(n-l;GOZ z54)J*k0Tn12Z%>5n?=l&Q35k1hNBj#dRTh-(qK?$L7sVH;mw`Gp&QXVV0s4J)6kcP zj*}~sP)CV~x22ddX`Y|i<5PF7mTn3IloxEAW*u(6*i%(x&7R-&?tm0 zc^u>|u8Vn;W}COTY;i)gElS_{pk}(GerhFG)QC&e}LIYN^nK_x>Q`K6N@g{EVKwmRXK^9NZ>d z^HiGgx_1W4yW}9TxSH^Ygbw?&xJcU6R>z?bT%}2jLma$IRh2MOqZ71LnRK|p9M{JW zTX5+0%A?Yc`38ou2VZrDA~|&5x<%F%n~Y%OursJ-)Y7YY=%p)3vUlBA^97XL;cpAk zc(&GQ1>T3=j!M_wHJF$#QGbgZ7$D)Tv;p)1kgr^3(sXt&WWY)R!f?yFA<>ogRGAmHtO$}!F+-icSz0A;Zz(U~Nh3w5b9 z3lT|Ew2h5Tj9;7%s{yr*ZvrbxJI7E~#ci6CBa`P!UKp@s$;&)2y=T@5aP7IlEEP<4 zreFPNg5DMOrzkrGj>X()qMUWUWvZ=t;$AzPR&9RSvd^vQ8s=PG_iG+RVKD-8s+6=s zzmiAYm~Y6K7}Pf)>Uu6cDGy*Iv`Va0_-eZ_ZL}NiU+`>yQz}2aF^XKA7*0>vD|g>= zuY7vCrXSkPZG1xNMx#Rkt#I&Jg1qB310fI?_5Jkvn%G#*Mv1}xf(B5A4fY?lCT)^C zdw#zuKfx-nN&m!Jo+h5uHr$Sq!ygOsWGu_YtmPRtF06bv)$djH*vG+t4s@1wynw3; z`r)r5k<@I;;@T_c%1o&w&uSa?yVcpw5I1jH)u%~E9}UlqCF_C`zBH3n#&`N(LNLH>GgW*=?Jnd{y>_P zUfD%8If`a>6G9aex(h3Jp>gxKzSQKtBwCRGR=-mqnNfB|J#{3kS$nbjDHV@67Fq+# zja|k;FwrEzSQyLcVLu5#xa`IbM@&=ezkm89GeA7gw*9fxu|U*S-wj{Fl$e4&sAe}? zZStrst<#j1$2sQs$wcv3${l=1sl23{r6_%7FueHgPYRywmNI3VeazLgAor=W%-M@P zlk!}kaC+Pf?;d_8?x;dKvSG1fAUq@K(FodiIy$8+^RpO*J}|I&C(}v#9Viw{gB(Uhx^wD6z@c9;^~di z&VD}}jHPJVTg2isr^Fht`-aMn3>l99h`|eZbP$a2a$z4io4Br9$D6Vb(rxR?sorr0 zH+KrQr^I_ik&~_1YaZKG=&QIq^vpp?il5sNlhvLi&la6#)zG`it9-e!Uy#s)Jy=j_ z2ZG$PopwtK{sOuix%Gns#yG3_Umw^8p;7u_3zvif#+)?oCd}&7rOnI&1Iny#t z**6xC8=!a3^uJ;w%f@ln7NbX-;tk)sc#DV5MyL(3o2Lb(j~^10hwCAfje{riwU{uP z$(M~9@4>=8QWXD#YHJhVSS;43DQ=g~pS?CVVbh)vDwz3D1|h2@v9E2L%3@G@Zt&~~ zyt)2B!?Ez}vvUtHH7Yh4T4>VxyV7n5?8w#n-BUqZN3PDHK}WXhDZ*$AyA>jg&HX`n zM3fX1D4V1nnqKa6)6U_6E%X2va#y}WE5mQ0Ro_owJ5&8XmR1q6bv!H~oE&@E5Oky@SUzc)d}XKWJyRt_1;{R^`9_2p4PRVDHZO5d0FAFS3fT5Q*y zTI@98!ggX5`xK)dG)jX`_o)t9+~8e)p@W$Td!@Rap2@EK9{-p{d}~K?0G35(ogWJUxF=^gdUdFEF?O|X zWW2NbTK{jGS&Udp80iSWarKc(u-BNZukco-=@vl0Op55O@~SHEh|+X=c_@u?Cj%s> zD0XYwEr&=a1B^VJ=_B5m)UPb+-V#{67M-%7%$u?R0Pv@e9(^S)5DJ{?!I)4lzp2Cy zu*3MYMAKV>(qs+N)5sO%V6kFroqLcnHVR?^75TlzNjGI{9AH z?_c<|XX>QL%G^HTMM;&N>MeFa=B|(3)@Fa`%C`!7$p@M(mq&mM*ZfOB3F(dIfsgc;t6Edc<_QlVP9D>>+G z9(9a-`*sm8%$|EAAi~rBugTHw_wUnX__s%eV}BraTQF%V>uvWltjO2WGvA3dxIB`# z4G-^45R?i0p&MVq)CF}e*~W@6R$_#(sKai!0^@_q|>wch2;jFQ|)-Vb2mh+Ain#B!$aSK zgM%NxfB(MRZ>W|n^4EvyQ{Lj0enZ)Fq-_ox5-+~6@E)2cz{)=Tzg_tc)BZ1Z3{-_s zfV<`zR)@ctG;t4Od@A2E#Twi_4PhQ2LU4~K7~b^s?6YoAsJp>6Y=by@>0cbZva(Vy zbml|wSW6P**3uV71{~N}nJuoUsQ5;m_%9^@2pIJXO?#mJ`&B6TzIZWPx5g}3+T|0^ZBmz$t5%Ur1?3w22x^7u=(PV>RJdI`5- z^5^Wpb43lPll8hFUv~BI-zdBe2yHOv7u@*7&++0#R;vR_BVT|)XF5ik6MH~$bg;6r z;<`C&{WBWUsQwm}f+pItHnl}m?H-;4%917|w!CQhqGSC%M#99CRVu=2-ZvfdRh#nV zN?ooPuB$*F`~PZaU`p;UwSw5cL^-vBGu!X4KoNq$wxIO?Z{>})&HIxWyH+xol?#nJ NWnqUZF}w83zX2P`_&Wdq diff --git a/providers/http/docs/connections/img/connection_username_password.png b/providers/http/docs/connections/img/connection_username_password.png index 6e36e77dd4cb48f62107a3f654648587bddc58e7..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644 GIT binary patch literal 0 HcmV?d00001 literal 4761 zcmchbcU03^o5!P|;P@gUqabyRBPa|Y0trZS6j4z?q$vUcqA-y*h?E3EU@S;qMnF0u z(g{VRlYmN5T9iFsdW&WcE+Gf+@j6Uopb@@XX=e3HCon2biA)vxaT*S!RJ3xgOLI`DzV z&2xKseS&`%p+IgvaCdjl&(1!j0f*;S(Sz>pj$4ZtC$~^wE!F+d(mk)c>gXq@hPy(@Cb59eAf4h1K$Ny zvvJ3q1Oy^s9O<$3lCe(nuy`l_}xUI_%)8AUWqzr*n+Y^pZitg6I zLeC6Vdf;g*nKB{9kSgh{rD%npDisdYg zw&r?`wq!pR5I#1!Ia65DJ<=ROmi8NpeRfDXd7>*VMgQd1EP~tCFV)(LekLrNK0(UJ z;64kL!JLjdt|rG;K@e?IRf<=N>*m|~;XX4xJwrpbb9|6jLlSbOdxrb-<@r)Hl|Lku zyO1VGn%Ip;hs?Xuv|qAGiK-`cS1CxX8)2p6e}u}Kw7*m#Z9=8Okk%#kX?ol(W)Ij@ zlH{JYR#{wXOxW4e(fJ+3JGGBc$+`i)#D>a>it$Cci%E-AKl-?<{o8xj(k&%2#4@|$ z&Y0hFb#>LdvNb;%qi-L$ooANSouuZnytG6=vz=grQ-eUp&667^+Ts;a&Eeyy+mplP z2)Z6>J#QV(@yaZ;E-`w%|Inw_`o#~ckGMri>-DNP@j*W-?5{1u(y{EEOQDCd z%aDe-g91V~+FvTQx9nnX%l#QJ+Y0W?mTJXJoTg+7bT^2j%$0sEm}s`W-f+_M>G&A(XNP6 zho85#oVVSEs>>*1(3=JGdmuMWvJbe)B$MM})-O$7P?K?$P{UZzf@bkm%iDH!_FaQn zhEX?C;GRb0924Wmp0aTPh@kax+&-9Ua5Tg5Qs|rG5W{YvgBwrWhnDlebNt z9hk5$ax&VOrDic^M?xu%yB!#9bM%BUKTL%7=;$PNdwnv3k%%U1^n}gMY*br3(fIgG zsC`o*#JFQM-+%GF069VFZaBj@1(vPlJu#LcZ<4x(=s#+iw1*#Zp(Y+$s)gOKpy^Mi zhh1JEh{`|oY+<3$n<1MnKkys2Z9Ux)@ z%BUKpVONr&Z|;;k8*i)*dNQ|f3*??JoyK&EP}~>~JsA>|8_DWt1a3eeJ-N5|VI}jl zGMR+Q*(I5YLM_6Y15OHtRX9Q~gnh||_Pq@YxK;hNF=yGk z|BgOMZ*$IA0Jii?4~WJH?1^JPu2w2Qf;Vo4H_G)Dd=BXh=iVR}^TRHM+Dgw^$;8e` zP$DG*Rzz~=_oft=tOs#9Y;VtPoe-f5S=C96a!oeMBG2Z&RVeC*f6spIFLmbFo?@NX z-$(3iy%C@2DEf0N_~e%_5<`V$yEaGhHzJNH86j{SAA4;j8F&G|99)9ZRFv$`dcdBi zu9uI*4r}Vk6o&h_d#IO&T@D>(j-DTBFx|o~vD0mqTH;;PV4m3X-3Q_Y2ELXyjh<)exI zAI=l-``$WPgJko zCOa?C2+evBNE_aoFXhY&d7nehVw^muQmF~`&aPISZY>J6*5|qedh&Tac5AJy9(QH( zm0IPMsCqSB_E1ax)UWj`*Btm3XMM|gi9PO4$$nVHgXxSCWr!5{^Jx6wl2xuh-mjxl z>sJSySG1r52UEhrsa5?&_IB9Q zkOnAEI3>)e{ID|3v6d+A3&aAo0YTqdcwwrOj>!D3vE=2>L>6#hcMp(G$SV=x=see* zezA@FgKI(7{j0FEvooJSAk=~-`Y5$jtM)OXL5uDODAYAE75gF&V31wiqv=~Re=1=G z$f=$(BhUK2el;TiLuQO3QAjt*f0xSq=BS^D*ykxWI+KMvgkV9i;VDWRx9}jNfL%DaGjA|W{vQ$NpS3m4=I-6QJZ}L}~wP_n}m)v?XSj1Nx_PDiK^uj29eT?=~Mn-f>J)|u$Y0Q?t#MSwG^ghl$AI*YQh zPGZ4#`&YAys;bE{_ed{f(V#TgRDd`-wlxn=(T4zRi8N6v&%LPE2D@vdn9hvyJOK)0 zo5bNQwNxHk*?5Vn)$syP6ln|k`s~Ko5m3|5!!&g$uh4iW@cxxT)+;#8UzfUEN6*nH zv?yuiGB};hJ>V9!_DASU zA8w_fYPQUpz`4E(xJHivGHtDklg(P_KGgdyG^%6DtOoA5MV+o&AQo=N*DjlLvwkSu zGld5EG;sN|JgWQ6HPoknI{`GAsz0FpaO|ptUZB5(wy)cH05C!;>Dbi)^kQzahB0rOS2)^^!pMBGqd>KLLc`E3qWiy%&Z<__TUz@VEwbOwXrxKJ5u3YvNnYXhWJ7bY-C*PQ~_Dzw@4q zcEWY^tbhD%ABytj1gg=ULuR4O$zh`M%fx*LCniZXf$I-xzzlPdW0oDL2W@J&Z{NOk z$17TQhaYX%CTXaz%nr&Mr=BHx4{sWI&DHu?bJMNu(Wi{(rNl`=i|iWidjZ+8*_hcn zTwe!ktI{1fRmN8>#Q2FIXJe)P=pUDckPlW`Vq_64XHt3^j*Y=&;I`3a(iH&Xk_p(& zRZbq87O-IFSJJoAH)Dyoi{Du~p#oD;jJFbo1WPg__Q4!Y$a6zDX6NJSxqW8aS7EDRj9^xD&NQP6Uc9bgIU$EzSuy7XkUDeWJ73?Nt>GqqxAGePKGm z5R9MD)q&%jw?d1gs+`u}vRGu(?95C154u4x6d?7k=NC@JpLJXuexIL}mF2ABTxfv- z*xT6+5O2abH*6Z%;@j@Ch9He$Y&03*q@cESMj*l(~?m8SC=FfD5o|Ix9xc zsD!xj0DRn^TkDCQ2pXG$oMrmnPw4oHL(KIfaA_uK=i|9Tm-f@spV3u+%1~@-kPTXM z-1b%NI^M9!GB+J35wTwy-(h+ShP3?`9cn=y1(?>As)tqJN~zsf7_rHMBRAgFb9$Z$ zp|w9G&miaP*bk;e`$`ZX9ZflcFmH-%&o|G_5jko22F%Ej4`_{%HO;ybVCW39*?lDqn1(u<{qD}X_YW2lcjDZtPyo^xHQn2o zV|6)CWJLAN{p;M+`8lb3EggwO?8X=;Mh~+xz~*eSC$A*D7JJ_@a`|AyqMX6$TUst1 z-FH@|vZjY>ywQlQSk){!=dO3DdTBA}DsKmgj0#@{qC13d(_6)PxV|>P53KaFMIeum z#4>N@@x$y~{zh2+mOga1(hc=BBI)8+aCBm&f};?~S92I1gGFyw-@CR!mgOcs{v9mw zSgXNV(bgh+f`Wij{0agP0++y&v?a~&`%-~MAxp`hbPV!2D#7{mX21p%0%^%#S95Ea&2 zqmQilODU|X*w>#nCFptQWHS?)&D*z2t6lTnPS;E>`|9+mBR5z>5m-R2$vMVp$@58? zC)nXvUdvIDk7z0aZ4iF6t&|#3ZEs`7Mj;%!@ot)dR zs}K|U;waI$QPms)nQDDfWZ3Y19^^!(0-7FonL^qnn`Ig1NNf9!4JdmBb|h<|m+$G} z>NBCyG2cr6D5(*fov^^Dmgl~HEB>&fF>K${(N2r99r$UmauCfl({Zv~mmEp-e^j_J z1lg+J(TCUF{4mwW6U5E=&e{GZ$=7-u7Wv&+Y%^LZMLr`C+K}Y*^~2NV!phd@<90tm zrJ#vn@XMEf6^%GjRyfl%wR+Q*2 zIy+?1^PStpdc&<^FC^2Ikm9U=TSukp{J%jB&r7>^DgT*=YK7eXx}-UiZLwfo1*9JQ2&Vsd b1GBE*^^2vH0QXn|ulCwy6NBQ54!`{!fAs15 From 3b1d4f54c35b778b95145a9b33e6fd07a5c577f6 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 11 Feb 2025 12:45:48 +0100 Subject: [PATCH 248/286] refactor: Implemented async get_conn in HttpAsyncHook --- .../apache/livy/hooks/test_livy.py | 5 ++-- .../src/airflow/providers/http/hooks/http.py | 27 +++++++++++-------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/providers/apache/livy/tests/provider_tests/apache/livy/hooks/test_livy.py b/providers/apache/livy/tests/provider_tests/apache/livy/hooks/test_livy.py index 4bdb4ba4fe1fd..28fb08e1471e2 100644 --- a/providers/apache/livy/tests/provider_tests/apache/livy/hooks/test_livy.py +++ b/providers/apache/livy/tests/provider_tests/apache/livy/hooks/test_livy.py @@ -630,8 +630,9 @@ def set_conn(self): db.merge_conn(Connection(conn_id="missing_host", conn_type="http", port=1234)) db.merge_conn(Connection(conn_id="invalid_uri", uri="http://invalid_uri:4321")) + @pytest.mark.asyncio @pytest.mark.db_test - def test_build_get_hook(self): + async def test_build_get_hook(self): self.set_conn() connection_url_mapping = { # id, expected @@ -644,7 +645,7 @@ def test_build_get_hook(self): for conn_id, expected in connection_url_mapping.items(): hook = LivyAsyncHook(livy_conn_id=conn_id) - response_conn: Connection = hook.get_conn() + response_conn: Connection = await hook.get_conn() assert isinstance(response_conn, Connection) assert hook.base_url == expected diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 72ca10d796465..1a77271981ae5 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -492,6 +492,21 @@ def __init__( self.retry_limit = retry_limit self.retry_delay = retry_delay + async def get_conn(self, headers: dict[Any, Any] | None = None) -> Connection: + conn = await sync_to_async(self.get_connection)(self.http_conn_id) + + if conn.host and "://" in conn.host: + self.base_url = conn.host + else: + # schema defaults to HTTP + schema = conn.schema if conn.schema else "http" + host = conn.host if conn.host else "" + self.base_url = schema + "://" + host + + if conn.port: + self.base_url += f":{conn.port}" + return conn + async def run( self, session: aiohttp.ClientSession, @@ -519,18 +534,8 @@ async def run( auth = None if self.http_conn_id: - conn = await sync_to_async(self.get_connection)(self.http_conn_id) - - if conn.host and "://" in conn.host: - self.base_url = conn.host - else: - # schema defaults to HTTP - schema = conn.schema if conn.schema else "http" - host = conn.host if conn.host else "" - self.base_url = schema + "://" + host + conn = await self.get_conn() - if conn.port: - self.base_url += f":{conn.port}" if conn.login: auth = _extract_auth(conn, self.auth_type) if conn.extra: From 032c80e6468fe2ec7fd4b73318850c442b782dd5 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 11 Feb 2025 16:16:20 +0100 Subject: [PATCH 249/286] refactor: Fixed run method of HttpAsyncHook --- .../src/airflow/providers/http/hooks/http.py | 46 +++++++++---------- .../provider_tests/http/hooks/test_http.py | 2 +- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 1a77271981ae5..f9658217290e2 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -493,19 +493,20 @@ def __init__( self.retry_delay = retry_delay async def get_conn(self, headers: dict[Any, Any] | None = None) -> Connection: - conn = await sync_to_async(self.get_connection)(self.http_conn_id) + if self.http_conn_id: + conn = await sync_to_async(self.get_connection)(self.http_conn_id) - if conn.host and "://" in conn.host: - self.base_url = conn.host - else: - # schema defaults to HTTP - schema = conn.schema if conn.schema else "http" - host = conn.host if conn.host else "" - self.base_url = schema + "://" + host + if conn.host and "://" in conn.host: + self.base_url = conn.host + else: + # schema defaults to HTTP + schema = conn.schema if conn.schema else "http" + host = conn.host if conn.host else "" + self.base_url = schema + "://" + host - if conn.port: - self.base_url += f":{conn.port}" - return conn + if conn.port: + self.base_url += f":{conn.port}" + return conn async def run( self, @@ -533,18 +534,17 @@ async def run( _headers = {} auth = None - if self.http_conn_id: - conn = await self.get_conn() + conn = await self.get_conn() - if conn.login: - auth = _extract_auth(conn, self.auth_type) - if conn.extra: - extra = self._process_extra_options_from_connection(conn=conn, extra_options=extra_options) + if conn.login: + auth = _extract_auth(conn, self.auth_type) + if conn.extra: + extra = self._process_extra_options_from_connection(conn=conn, extra_options=extra_options) - try: - _headers.update(extra) - except TypeError: - self.log.warning("Connection to %s has invalid extra field.", conn.host) + try: + _headers.update(extra) + except TypeError: + self.log.warning("Connection to %s has invalid extra field.", conn.host) if headers: _headers.update(headers) @@ -577,8 +577,10 @@ async def run( auth=auth, **extra_options, ) + try: response.raise_for_status() + return response except ClientResponseError as e: self.log.warning( "[Try %d of %d] Request to %s failed.", @@ -594,8 +596,6 @@ async def run( await asyncio.sleep(self.retry_delay) - return response - raise NotImplementedError # should not reach this, but makes mypy happy @classmethod diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index 1296033d439cf..62a0fb101193f 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -810,7 +810,7 @@ async def test_async_request_uses_connection_extra(self): reason="OK", ) - with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): + with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection_with_extra(connection_extra)): hook = HttpAsyncHook() with mock.patch("aiohttp.ClientSession.post", new_callable=mock.AsyncMock) as mocked_function: async with aiohttp.ClientSession() as session: From 46d3b9053279670b01f0ca015ba68ccc389a1f82 Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 12 Feb 2025 08:26:00 +0100 Subject: [PATCH 250/286] refactor: reformatted test_async_request_uses_connection_extra of HttpAsyncHook --- providers/http/tests/provider_tests/http/hooks/test_http.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/providers/http/tests/provider_tests/http/hooks/test_http.py b/providers/http/tests/provider_tests/http/hooks/test_http.py index 62a0fb101193f..ad141d7a39891 100644 --- a/providers/http/tests/provider_tests/http/hooks/test_http.py +++ b/providers/http/tests/provider_tests/http/hooks/test_http.py @@ -810,7 +810,10 @@ async def test_async_request_uses_connection_extra(self): reason="OK", ) - with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection_with_extra(connection_extra)): + with mock.patch( + "airflow.hooks.base.BaseHook.get_connection", + side_effect=get_airflow_connection_with_extra(connection_extra), + ): hook = HttpAsyncHook() with mock.patch("aiohttp.ClientSession.post", new_callable=mock.AsyncMock) as mocked_function: async with aiohttp.ClientSession() as session: From 0b19aa3d51b07e138194877b4172da349a1555fc Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 12 Feb 2025 08:30:12 +0100 Subject: [PATCH 251/286] refactor: Refactored get_conn of HttpAsyncHook --- .../src/airflow/providers/http/hooks/http.py | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index f9658217290e2..49b42bf5ff050 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -34,7 +34,7 @@ from requests.models import DEFAULT_REDIRECT_LIMIT from requests_toolbelt.adapters.socket_options import TCPKeepAliveAdapter -from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning +from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning, AirflowConfigException from airflow.hooks.base import BaseHook from airflow.providers.http.exceptions import HttpErrorException, HttpMethodException from airflow.utils.module_loading import import_string @@ -493,20 +493,19 @@ def __init__( self.retry_delay = retry_delay async def get_conn(self, headers: dict[Any, Any] | None = None) -> Connection: - if self.http_conn_id: - conn = await sync_to_async(self.get_connection)(self.http_conn_id) + conn = await sync_to_async(self.get_connection)(self.http_conn_id) - if conn.host and "://" in conn.host: - self.base_url = conn.host - else: - # schema defaults to HTTP - schema = conn.schema if conn.schema else "http" - host = conn.host if conn.host else "" - self.base_url = schema + "://" + host - - if conn.port: - self.base_url += f":{conn.port}" - return conn + if conn.host and "://" in conn.host: + self.base_url = conn.host + else: + # schema defaults to HTTP + schema = conn.schema if conn.schema else "http" + host = conn.host if conn.host else "" + self.base_url = schema + "://" + host + + if conn.port: + self.base_url += f":{conn.port}" + return conn async def run( self, From 429e1b97dcc796a8e4eaf473a60801b8b158a040 Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 12 Feb 2025 08:47:50 +0100 Subject: [PATCH 252/286] refactor: Refactored _extract_auth of HttpHook --- .../src/airflow/providers/http/hooks/http.py | 52 +++++++++---------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 49b42bf5ff050..d3da590e3f355 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -42,7 +42,7 @@ if TYPE_CHECKING: from aiohttp.client_reqrep import ClientResponse from requests.adapters import HTTPAdapter - from requests.auth import AuthBase + from requests.auth import AuthBase, HTTPBasicAuth from airflow.models import Connection @@ -492,21 +492,6 @@ def __init__( self.retry_limit = retry_limit self.retry_delay = retry_delay - async def get_conn(self, headers: dict[Any, Any] | None = None) -> Connection: - conn = await sync_to_async(self.get_connection)(self.http_conn_id) - - if conn.host and "://" in conn.host: - self.base_url = conn.host - else: - # schema defaults to HTTP - schema = conn.schema if conn.schema else "http" - host = conn.host if conn.host else "" - self.base_url = schema + "://" + host - - if conn.port: - self.base_url += f":{conn.port}" - return conn - async def run( self, session: aiohttp.ClientSession, @@ -521,6 +506,7 @@ async def run( :param endpoint: Endpoint to be called, i.e. ``resource/v1/query?``. :param data: Payload to be uploaded or request parameters. + :param json: Payload to be uploaded as JSON. :param headers: Additional headers to be passed through as a dict. :param extra_options: Additional kwargs to pass when creating a request. For example, ``run(json=obj)`` is passed as @@ -533,17 +519,28 @@ async def run( _headers = {} auth = None - conn = await self.get_conn() - - if conn.login: - auth = _extract_auth(conn, self.auth_type) - if conn.extra: - extra = self._process_extra_options_from_connection(conn=conn, extra_options=extra_options) + if self.http_conn_id: + conn = await sync_to_async(self.get_connection)(self.http_conn_id) - try: - _headers.update(extra) - except TypeError: - self.log.warning("Connection to %s has invalid extra field.", conn.host) + if conn.host and "://" in conn.host: + self.base_url = conn.host + else: + # schema defaults to HTTP + schema = conn.schema if conn.schema else "http" + host = conn.host if conn.host else "" + self.base_url = schema + "://" + host + + if conn.port: + self.base_url += f":{conn.port}" + if conn.login: + auth = _extract_auth(conn, self.auth_type) + if conn.extra: + extra = self._process_extra_options_from_connection(conn=conn, extra_options=extra_options) + + try: + _headers.update(extra) + except TypeError: + self.log.warning("Connection to %s has invalid extra field.", conn.host) if headers: _headers.update(headers) @@ -579,7 +576,6 @@ async def run( try: response.raise_for_status() - return response except ClientResponseError as e: self.log.warning( "[Try %d of %d] Request to %s failed.", @@ -594,6 +590,8 @@ async def run( raise HttpErrorException(f"{e.status}:{e.message}") await asyncio.sleep(self.retry_delay) + else: + return response raise NotImplementedError # should not reach this, but makes mypy happy From bbcebc796208e411eb44f91f417140ed3536b9fb Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 12 Feb 2025 16:34:08 +0100 Subject: [PATCH 253/286] refactor: Removed unused imports in HttpHook --- providers/http/src/airflow/providers/http/hooks/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index d3da590e3f355..722c7ba0a13fb 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -34,7 +34,7 @@ from requests.models import DEFAULT_REDIRECT_LIMIT from requests_toolbelt.adapters.socket_options import TCPKeepAliveAdapter -from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning, AirflowConfigException +from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.hooks.base import BaseHook from airflow.providers.http.exceptions import HttpErrorException, HttpMethodException from airflow.utils.module_loading import import_string @@ -42,7 +42,7 @@ if TYPE_CHECKING: from aiohttp.client_reqrep import ClientResponse from requests.adapters import HTTPAdapter - from requests.auth import AuthBase, HTTPBasicAuth + from requests.auth import AuthBase from airflow.models import Connection From d7b2049b93685a3907530c8871564ac9681b1654 Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 14 Feb 2025 08:58:29 +0100 Subject: [PATCH 254/286] refactor: Updated test_build_get_hook --- .../tests/provider_tests/apache/livy/hooks/test_livy.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/providers/apache/livy/tests/provider_tests/apache/livy/hooks/test_livy.py b/providers/apache/livy/tests/provider_tests/apache/livy/hooks/test_livy.py index 28fb08e1471e2..71cee7bf613e1 100644 --- a/providers/apache/livy/tests/provider_tests/apache/livy/hooks/test_livy.py +++ b/providers/apache/livy/tests/provider_tests/apache/livy/hooks/test_livy.py @@ -630,9 +630,8 @@ def set_conn(self): db.merge_conn(Connection(conn_id="missing_host", conn_type="http", port=1234)) db.merge_conn(Connection(conn_id="invalid_uri", uri="http://invalid_uri:4321")) - @pytest.mark.asyncio @pytest.mark.db_test - async def test_build_get_hook(self): + def test_build_get_hook(self): self.set_conn() connection_url_mapping = { # id, expected @@ -645,9 +644,9 @@ async def test_build_get_hook(self): for conn_id, expected in connection_url_mapping.items(): hook = LivyAsyncHook(livy_conn_id=conn_id) - response_conn: Connection = await hook.get_conn() + response_conn: Connection = hook.get_connection(conn_id=conn_id) assert isinstance(response_conn, Connection) - assert hook.base_url == expected + assert hook._generate_base_url(response_conn) == expected def test_build_body(self): # minimal request From 28f4b6b2ca266f3be17b652e0b407227de6a0783 Mon Sep 17 00:00:00 2001 From: David Blain Date: Fri, 14 Feb 2025 09:37:08 +0100 Subject: [PATCH 255/286] refactor: Extracted _generate_base_url from HttpAsyncHook --- .../src/airflow/providers/http/hooks/http.py | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 722c7ba0a13fb..e60b6ee8b04f8 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -492,6 +492,18 @@ def __init__( self.retry_limit = retry_limit self.retry_delay = retry_delay + def _generate_base_url(self, conn: Connection) -> str: + if conn.host and "://" in conn.host: + base_url: str = conn.host + else: + # schema defaults to HTTP + schema = conn.schema if conn.schema else "http" + host = conn.host if conn.host else "" + base_url = f"{schema}://{host}" + if conn.port: + base_url = f"{base_url}:{conn.port}" + return base_url + async def run( self, session: aiohttp.ClientSession, @@ -521,17 +533,7 @@ async def run( if self.http_conn_id: conn = await sync_to_async(self.get_connection)(self.http_conn_id) - - if conn.host and "://" in conn.host: - self.base_url = conn.host - else: - # schema defaults to HTTP - schema = conn.schema if conn.schema else "http" - host = conn.host if conn.host else "" - self.base_url = schema + "://" + host - - if conn.port: - self.base_url += f":{conn.port}" + self.base_url = self._generate_base_url(conn=conn) if conn.login: auth = _extract_auth(conn, self.auth_type) if conn.extra: From 528f2f1509fd432fbdac3c085ca3dfdb0c236fec Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Sun, 12 Nov 2023 13:55:18 +0100 Subject: [PATCH 256/286] feat: Implement `auth_kwargs` parameter in Http Connection --- .../http/tests/unit/http/hooks/test_http.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/providers/http/tests/unit/http/hooks/test_http.py b/providers/http/tests/unit/http/hooks/test_http.py index 82a1ff9765156..e1a380e4588e5 100644 --- a/providers/http/tests/unit/http/hooks/test_http.py +++ b/providers/http/tests/unit/http/hooks/test_http.py @@ -67,6 +67,11 @@ def get_airflow_connection_with_login_and_password(conn_id: str = "http_default" return Connection(conn_id=conn_id, conn_type="http", host="test.com", login="username", password="pass") +class CustomAuthBase(HTTPBasicAuth): + def __init__(self, username: str, password: str, endpoint: str): + super().__init__(username, password) + + class TestHttpHook: """Test get, post and raise_for_status""" @@ -352,6 +357,25 @@ def test_connection_without_host(self, mock_get_connection): hook.get_conn({}) assert hook.base_url == "http://" + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_connection): + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + login="username", + password="pass", + extra='{"auth_kwargs": {"endpoint": "http://localhost"}}', + ) + mock_get_connection.return_value = conn + + hook = HttpHook(auth_type=CustomAuthBase) + session = hook.get_conn({}) + + auth.assert_called_once_with("username", "pass", endpoint="http://localhost") + assert "auth_kwargs" not in session.headers + @pytest.mark.parametrize("method", ["GET", "POST"]) def test_json_request(self, method, requests_mock): obj1 = {"a": 1, "b": "abc", "c": [1, 2, {"d": 10}]} From 0f4048a54f645c54b7d03590a9e42866f631b951 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 24 Nov 2023 00:25:53 +0100 Subject: [PATCH 257/286] fix: Correctly use auth_type from Connection --- .../http/tests/unit/http/hooks/test_http.py | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/providers/http/tests/unit/http/hooks/test_http.py b/providers/http/tests/unit/http/hooks/test_http.py index e1a380e4588e5..802dd0ef15c39 100644 --- a/providers/http/tests/unit/http/hooks/test_http.py +++ b/providers/http/tests/unit/http/hooks/test_http.py @@ -366,7 +366,7 @@ def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_conne conn_type="http", login="username", password="pass", - extra='{"auth_kwargs": {"endpoint": "http://localhost"}}', + extra='{"x-header": 0, "auth_kwargs": {"endpoint": "http://localhost"}}', ) mock_get_connection.return_value = conn @@ -375,6 +375,40 @@ def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_conne auth.assert_called_once_with("username", "pass", endpoint="http://localhost") assert "auth_kwargs" not in session.headers + assert "x-header" in session.headers + + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_header_and_auth_type(self, auth, mock_get_connection): + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + login="username", + password="pass", + extra='{"x-header": 0, "auth_type": "tests.providers.http.hooks.test_http.CustomAuthBase"}', + ) + mock_get_connection.return_value = conn + + session = HttpHook().get_conn({}) + auth.assert_called_once_with("username", "pass") + assert isinstance(session.auth, CustomAuthBase) + assert "auth_type" not in session.headers + assert "x-header" in session.headers + + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_extra_auth_type_and_no_credentials(self, auth, mock_get_connection): + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + extra='{"auth_type": "tests.providers.http.hooks.test_http.CustomAuthBase"}', + ) + mock_get_connection.return_value = conn + + HttpHook().get_conn({}) + auth.assert_called_once() @pytest.mark.parametrize("method", ["GET", "POST"]) def test_json_request(self, method, requests_mock): From 48286f96eb262aa45f14a5782e01fbf8997b4827 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 24 Nov 2023 01:55:44 +0100 Subject: [PATCH 258/286] feat: Add Connection documentation --- providers/http/src/airflow/providers/http/hooks/http.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index b22a01f8283db..eb63e27ee0925 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -51,6 +51,12 @@ class HttpHook(BaseHook): """ Interact with HTTP servers. + To configure the auth_type, in addition to the `auth_type` parameter, you can also: + * set the `auth_type` parameter in the Connection settings. + * define extra parameters used to instantiate the `auth_type` class, in the Connection settings. + + See :doc:`/connections/http` for full documentation. + :param method: the API method to be called :param http_conn_id: :ref:`http connection` that has the base API url i.e https://www.google.com/ and optional authentication credentials. Default From 1e03c1708c5d890686b5f1ecab0a884cdabbf8f7 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 5 Dec 2023 23:14:43 +0100 Subject: [PATCH 259/286] feat: Make available auth_types configurable from airflow config --- providers/http/docs/configurations-ref.rst | 0 providers/http/docs/index.rst | 1 + providers/http/provider.yaml | 15 ++++++++++ .../http/tests/unit/http/hooks/test_http.py | 28 ++++++++++++++++++- 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 providers/http/docs/configurations-ref.rst diff --git a/providers/http/docs/configurations-ref.rst b/providers/http/docs/configurations-ref.rst new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/providers/http/docs/index.rst b/providers/http/docs/index.rst index fd873092b915e..099c4213c4b10 100644 --- a/providers/http/docs/index.rst +++ b/providers/http/docs/index.rst @@ -42,6 +42,7 @@ :maxdepth: 1 :caption: References + Configuration Python API <_api/airflow/providers/http/index> .. toctree:: diff --git a/providers/http/provider.yaml b/providers/http/provider.yaml index dee0796c04891..7a991044ea801 100644 --- a/providers/http/provider.yaml +++ b/providers/http/provider.yaml @@ -93,3 +93,18 @@ triggers: connection-types: - hook-class-name: airflow.providers.http.hooks.http.HttpHook connection-type: http + +config: + http: + description: "Options for Http provider." + options: + extra_auth_types: + description: | + A comma separated list of auth_type classes, which can be used to + configure Http Connections in Airflow's UI. This list restricts which + classes can be arbitrary imported, and protects from dependency + injections. + type: string + version_added: 4.8.0 + example: "requests_kerberos.HTTPKerberosAuth,any.other.custom.HTTPAuth" + default: ~ diff --git a/providers/http/tests/unit/http/hooks/test_http.py b/providers/http/tests/unit/http/hooks/test_http.py index 802dd0ef15c39..fe6b8f882f2b7 100644 --- a/providers/http/tests/unit/http/hooks/test_http.py +++ b/providers/http/tests/unit/http/hooks/test_http.py @@ -36,7 +36,7 @@ from airflow.exceptions import AirflowException from airflow.models import Connection -from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook +from airflow.providers.http.hooks.http import HttpAsyncHook, HttpHook, get_auth_types @pytest.fixture @@ -72,6 +72,9 @@ def __init__(self, username: str, password: str, endpoint: str): super().__init__(username, password) +@mock.patch.dict( + "os.environ", AIRFLOW__HTTP__EXTRA_AUTH_TYPES="tests.providers.http.hooks.test_http.CustomAuthBase" +) class TestHttpHook: """Test get, post and raise_for_status""" @@ -377,6 +380,29 @@ def test_connection_with_extra_header_and_auth_kwargs(self, auth, mock_get_conne assert "auth_kwargs" not in session.headers assert "x-header" in session.headers + def test_available_connection_auth_types(self): + auth_types = get_auth_types() + assert auth_types == frozenset( + { + "request.auth.HTTPBasicAuth", + "request.auth.HTTPProxyAuth", + "request.auth.HTTPDigestAuth", + "tests.providers.http.hooks.test_http.CustomAuthBase", + } + ) + + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + def test_connection_with_invalid_auth_type_get_skipped(self, mock_get_connection, caplog): + auth_type: str = "auth_type.class.not.available.for.Import" + conn = Connection( + conn_id="http_default", + conn_type="http", + extra=f'{{"auth_type": "{auth_type}"}}', + ) + mock_get_connection.return_value = conn + HttpHook().get_conn({}) + assert f"Skipping import of auth_type '{auth_type}'." in caplog.text + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") def test_connection_with_extra_header_and_auth_type(self, auth, mock_get_connection): From 2608dfcb2873d5f3678a806f58eb58f96f8db1fb Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Wed, 6 Dec 2023 22:24:25 +0100 Subject: [PATCH 260/286] feat: Add fields for auth config and header config in Http Connection form --- .../src/airflow/providers/http/hooks/http.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index eb63e27ee0925..31a0bc5cfc8a5 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -113,6 +113,30 @@ def auth_type(self): def auth_type(self, v): self._auth_type = v + @classmethod + def get_connection_form_widgets(cls) -> dict[str, Any]: + """Return connection widgets to add to connection form.""" + from flask_babel import lazy_gettext + from wtforms.fields import SelectField, TextAreaField + + auth_types_choices = frozenset({""}) | get_auth_types() + return { + "auth_type": SelectField( + lazy_gettext("Auth type"), + choices=[(clazz, clazz) for clazz in auth_types_choices] + ), + "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs")), + "extra_headers": TextAreaField(lazy_gettext("Extra Headers")) + } + + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + """Return custom field behaviour.""" + return { + "hidden_fields": ["extra"], + "relabeling": {} + } + # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: From 10641e3307e2ac418835749f9ebd17b670c9d1aa Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Thu, 7 Dec 2023 08:48:01 +0100 Subject: [PATCH 261/286] fix: Correctly apply styling to extra fields --- .../http/src/airflow/providers/http/hooks/http.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 31a0bc5cfc8a5..538bc1e6e38cb 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -116,6 +116,7 @@ def auth_type(self, v): @classmethod def get_connection_form_widgets(cls) -> dict[str, Any]: """Return connection widgets to add to connection form.""" + from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget, Select2Widget from flask_babel import lazy_gettext from wtforms.fields import SelectField, TextAreaField @@ -123,19 +124,17 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: return { "auth_type": SelectField( lazy_gettext("Auth type"), - choices=[(clazz, clazz) for clazz in auth_types_choices] + choices=[(clazz, clazz) for clazz in auth_types_choices], + widget=Select2Widget(), ), - "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs")), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers")) + "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), + "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } @classmethod def get_ui_field_behaviour(cls) -> dict[str, Any]: """Return custom field behaviour.""" - return { - "hidden_fields": ["extra"], - "relabeling": {} - } + return {"hidden_fields": ["extra"], "relabeling": {}} # headers may be passed through directly or in the "extra" field in the connection # definition From cba90165731e435b4c846f8e20d9ce2e7f10f196 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 19 Dec 2023 01:05:47 +0100 Subject: [PATCH 262/286] feat: Implement simplistic collapsable textarea for "extra" --- airflow/www/forms.py | 26 ++++++++++++++++++- airflow/www/static/js/connection_form.js | 4 +-- .../src/airflow/providers/http/hooks/http.py | 9 +++---- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/airflow/www/forms.py b/airflow/www/forms.py index 7028e2026e449..b69184c6590c2 100644 --- a/airflow/www/forms.py +++ b/airflow/www/forms.py @@ -33,6 +33,7 @@ from flask_appbuilder.forms import DynamicForm from flask_babel import lazy_gettext from flask_wtf import FlaskForm +from markupsafe import Markup from wtforms import widgets from wtforms.fields import Field, IntegerField, PasswordField, SelectField, StringField, TextAreaField from wtforms.validators import InputRequired, Optional @@ -176,6 +177,29 @@ def populate_obj(self, item): field.populate_obj(item, name) +class BS3AccordionTextAreaFieldWidget(BS3TextAreaFieldWidget): + + @staticmethod + def _make_collapsable_panel(field: Field, content: Markup) -> str: + collapsable_id: str = f"collapsable_{field.id}" + return f""" +
+
+

+ +

+
+ +
+ """ + + def __call__(self, field, **kwargs): + text_area = super(BS3TextAreaFieldWidget, self).__call__(field, **kwargs) + return self._make_collapsable_panel(field=field, content=text_area) + + @cache def create_connection_form_class() -> type[DynamicForm]: """ @@ -223,7 +247,7 @@ def process(self, formdata=None, obj=None, **kwargs): login = StringField(lazy_gettext("Login"), widget=BS3TextFieldWidget()) password = PasswordField(lazy_gettext("Password"), widget=BS3PasswordFieldWidget()) port = IntegerField(lazy_gettext("Port"), validators=[Optional()], widget=BS3TextFieldWidget()) - extra = TextAreaField(lazy_gettext("Extra"), widget=BS3TextAreaFieldWidget()) + extra = TextAreaField(lazy_gettext("Extra"), widget=BS3AccordionTextAreaFieldWidget()) for key, value in providers_manager.connection_form_widgets.items(): setattr(ConnectionForm, key, value.field) diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index d039fc7275462..1c97e00803174 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -83,7 +83,7 @@ function restoreFieldBehaviours() { } ); - Array.from(document.querySelectorAll(".form-control")).forEach((elem) => { + Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach((elem) => { // eslint-disable-next-line no-param-reassign elem.placeholder = ""; elem.parentElement.parentElement.classList.remove("hide"); @@ -101,7 +101,7 @@ function applyFieldBehaviours(connection) { if (Array.isArray(connection.hidden_fields)) { connection.hidden_fields.forEach((field) => { document - .getElementById(field) + .querySelector(`label[for='${field}']`) .parentElement.parentElement.classList.add("hide"); }); } diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 538bc1e6e38cb..7268c13f8764d 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -25,6 +25,8 @@ import tenacity from aiohttp import ClientResponseError from asgiref.sync import sync_to_async +from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget +from markupsafe import Markup from requests.auth import HTTPBasicAuth from requests.models import DEFAULT_REDIRECT_LIMIT from requests_toolbelt.adapters.socket_options import TCPKeepAliveAdapter @@ -128,14 +130,9 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: widget=Select2Widget(), ), "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), + "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()) } - @classmethod - def get_ui_field_behaviour(cls) -> dict[str, Any]: - """Return custom field behaviour.""" - return {"hidden_fields": ["extra"], "relabeling": {}} - # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: From 68343338bfd0346aba1131cb423555d05e4de2ab Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Wed, 20 Dec 2023 16:21:08 +0100 Subject: [PATCH 263/286] fix: express clearly empty frozenset creation Goal is to have an empty default choice --- providers/http/src/airflow/providers/http/hooks/http.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 7268c13f8764d..0f6bea66bf624 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -122,7 +122,8 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: from flask_babel import lazy_gettext from wtforms.fields import SelectField, TextAreaField - auth_types_choices = frozenset({""}) | get_auth_types() + default_auth_type = frozenset({""}) + auth_types_choices = default_auth_type | get_auth_types() return { "auth_type": SelectField( lazy_gettext("Auth type"), From a5d2d4bccee35c86d70bcc11fb37ff7c3d9e86ec Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Wed, 20 Dec 2023 16:43:29 +0100 Subject: [PATCH 264/286] feat: Refactor Accordion TextArea to use wtform utils --- airflow/www/static/js/connection_form.js | 12 +++++++----- .../http/src/airflow/providers/http/hooks/http.py | 4 +--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index 1c97e00803174..119fe39daae54 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -83,11 +83,13 @@ function restoreFieldBehaviours() { } ); - Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach((elem) => { - // eslint-disable-next-line no-param-reassign - elem.placeholder = ""; - elem.parentElement.parentElement.classList.remove("hide"); - }); + Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach( + (elem) => { + // eslint-disable-next-line no-param-reassign + elem.placeholder = ""; + elem.parentElement.parentElement.classList.remove("hide"); + } + ); } /** diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 0f6bea66bf624..77563d2eb1a81 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -25,8 +25,6 @@ import tenacity from aiohttp import ClientResponseError from asgiref.sync import sync_to_async -from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget -from markupsafe import Markup from requests.auth import HTTPBasicAuth from requests.models import DEFAULT_REDIRECT_LIMIT from requests_toolbelt.adapters.socket_options import TCPKeepAliveAdapter @@ -131,7 +129,7 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: widget=Select2Widget(), ), "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()) + "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } # headers may be passed through directly or in the "extra" field in the connection From 04abb80f1d4a0304a621fe1a2b97abea53ee00a2 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Thu, 21 Dec 2023 13:04:19 +0100 Subject: [PATCH 265/286] feat: Implement 'collapse_extra' field behavior --- airflow/customized_form_field_behaviours.schema.json | 4 ++++ providers/http/src/airflow/providers/http/hooks/http.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/airflow/customized_form_field_behaviours.schema.json b/airflow/customized_form_field_behaviours.schema.json index 78791a87886c1..fa5ace958c5e8 100644 --- a/airflow/customized_form_field_behaviours.schema.json +++ b/airflow/customized_form_field_behaviours.schema.json @@ -22,6 +22,10 @@ "additionalProperties": { "type": "string" } + }, + "collapse_extra": { + "type": "boolean", + "description": "Collapse the 'Extra' field." } }, "additionalProperties": true, diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 77563d2eb1a81..2ddd9ca434efe 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -132,6 +132,10 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + return {"hidden_fields": [], "relabeling": {}, "collapse_extra": True} + # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: From 9cb0401bb467c2b3f5e1f727c2da4e8ce519ba96 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Sat, 30 Dec 2023 16:43:47 +0100 Subject: [PATCH 266/286] feat: Implement parameterizable behavior for collapsible field --- ...stomized_form_field_behaviours.schema.json | 19 +++++-- airflow/www/static/css/connection.css | 23 +++++++++ airflow/www/static/js/connection_form.js | 49 ++++++++++++++++--- .../www/templates/airflow/conn_create.html | 2 +- airflow/www/templates/airflow/conn_edit.html | 1 + airflow/www/webpack.config.js | 1 + .../src/airflow/providers/http/hooks/http.py | 2 +- 7 files changed, 85 insertions(+), 12 deletions(-) create mode 100644 airflow/www/static/css/connection.css diff --git a/airflow/customized_form_field_behaviours.schema.json b/airflow/customized_form_field_behaviours.schema.json index fa5ace958c5e8..8aa05945ebb01 100644 --- a/airflow/customized_form_field_behaviours.schema.json +++ b/airflow/customized_form_field_behaviours.schema.json @@ -23,9 +23,22 @@ "type": "string" } }, - "collapse_extra": { - "type": "boolean", - "description": "Collapse the 'Extra' field." + "collapsible_fields": { + "description": "List of collapsed fields for the hook, with their properties.", + "type": "object", + "patternProperties": { + "\"^.*$\"": { + "description": "Name of the field to enable collapsing.", + "type": "object", + "properties": { + "expanded": { + "description": "Set the default state of the field as expanded.", + "default": true, + "type": "boolean" + } + } + } + } } }, "additionalProperties": true, diff --git a/airflow/www/static/css/connection.css b/airflow/www/static/css/connection.css new file mode 100644 index 0000000000000..78edf0db5d4dc --- /dev/null +++ b/airflow/www/static/css/connection.css @@ -0,0 +1,23 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +.panel-invisible { + margin: 0; + border: 0; +} diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index 119fe39daae54..5c60638caf488 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -83,13 +83,28 @@ function restoreFieldBehaviours() { } ); - Array.from(document.querySelectorAll(".form-control, .form-panel")).forEach( - (elem) => { - // eslint-disable-next-line no-param-reassign - elem.placeholder = ""; - elem.parentElement.parentElement.classList.remove("hide"); - } - ); + Array.from(document.querySelectorAll(".form-control")).forEach((elem) => { + // eslint-disable-next-line no-param-reassign + elem.placeholder = ""; + elem.parentElement.parentElement.classList.remove("hide"); + }); + + Array.from(document.querySelectorAll(".form-panel")).forEach((elem) => { + elem.parentElement.parentElement.classList.remove("hide"); + + elem.classList.add("panel-invisible"); + const panelHeader = elem.children[0]; + panelHeader.classList.add("hidden"); + panelHeader.firstElementChild.firstElementChild.setAttribute( + "aria-expanded", + "true" + ); + + const collapsible = elem.children[1]; + collapsible.setAttribute("aria-expanded", "true"); + collapsible.classList.add("in"); + collapsible.style.height = null; + }); } /** @@ -122,6 +137,26 @@ function applyFieldBehaviours(connection) { document.getElementById(field).placeholder = placeholder; }); } + + if (connection.collapsible_fields) { + Object.entries(connection.collapsible_fields).forEach((entry) => { + const [field, properties] = entry; + + const collapsibleController = document.getElementById( + `control_collapsible_${field}` + ); + const panelHeader = collapsibleController.parentElement.parentElement; + panelHeader.classList.remove("hidden"); + panelHeader.parentElement.classList.remove("panel-invisible"); + + if (properties.expanded === false) { + const collapsible = document.getElementById(`collapsible_${field}`); + collapsible.classList.remove("in"); + collapsible.setAttribute("aria-expanded", "false"); + collapsibleController.setAttribute("aria-expanded", "false"); + } + }); + } } } diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index ac92b967f7e34..307450b05d16b 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -25,7 +25,7 @@ - + {# required for codemirror #} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 653b0dd3ce07f..11ebd6c4cb436 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -25,6 +25,7 @@ + {# required for codemirror #} diff --git a/airflow/www/webpack.config.js b/airflow/www/webpack.config.js index 9d5800f783f50..ad1a7098e0803 100644 --- a/airflow/www/webpack.config.js +++ b/airflow/www/webpack.config.js @@ -60,6 +60,7 @@ const config = { airflowDefaultTheme: `${CSS_DIR}/bootstrap-theme.css`, connectionForm: `${JS_DIR}/connection_form.js`, chart: [`${CSS_DIR}/chart.css`], + connection: [`${CSS_DIR}/connection.css`], dag: `${JS_DIR}/dag.js`, dagDependencies: `${JS_DIR}/dag_dependencies.js`, dags: [`${CSS_DIR}/dags.css`, `${JS_DIR}/dags.js`], diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 2ddd9ca434efe..192e57c1d70fc 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -134,7 +134,7 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: @classmethod def get_ui_field_behaviour(cls) -> dict[str, Any]: - return {"hidden_fields": [], "relabeling": {}, "collapse_extra": True} + return {"hidden_fields": [], "relabeling": {}, "collapsible_fields": {"extra": {"expanded": False}}} # headers may be passed through directly or in the "extra" field in the connection # definition From eec9547a741733bd4b72b26071b6f93a5bc374d9 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 5 Jan 2024 06:48:16 +0100 Subject: [PATCH 267/286] revert: Remove collapsible field --- ...stomized_form_field_behaviours.schema.json | 17 -------- airflow/www/static/css/connection.css | 23 ----------- airflow/www/static/js/connection_form.js | 39 +------------------ .../www/templates/airflow/conn_create.html | 1 - airflow/www/templates/airflow/conn_edit.html | 1 - airflow/www/webpack.config.js | 1 - .../src/airflow/providers/http/hooks/http.py | 4 -- 7 files changed, 1 insertion(+), 85 deletions(-) delete mode 100644 airflow/www/static/css/connection.css diff --git a/airflow/customized_form_field_behaviours.schema.json b/airflow/customized_form_field_behaviours.schema.json index 8aa05945ebb01..78791a87886c1 100644 --- a/airflow/customized_form_field_behaviours.schema.json +++ b/airflow/customized_form_field_behaviours.schema.json @@ -22,23 +22,6 @@ "additionalProperties": { "type": "string" } - }, - "collapsible_fields": { - "description": "List of collapsed fields for the hook, with their properties.", - "type": "object", - "patternProperties": { - "\"^.*$\"": { - "description": "Name of the field to enable collapsing.", - "type": "object", - "properties": { - "expanded": { - "description": "Set the default state of the field as expanded.", - "default": true, - "type": "boolean" - } - } - } - } } }, "additionalProperties": true, diff --git a/airflow/www/static/css/connection.css b/airflow/www/static/css/connection.css deleted file mode 100644 index 78edf0db5d4dc..0000000000000 --- a/airflow/www/static/css/connection.css +++ /dev/null @@ -1,23 +0,0 @@ -/*! - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -.panel-invisible { - margin: 0; - border: 0; -} diff --git a/airflow/www/static/js/connection_form.js b/airflow/www/static/js/connection_form.js index 5c60638caf488..d039fc7275462 100644 --- a/airflow/www/static/js/connection_form.js +++ b/airflow/www/static/js/connection_form.js @@ -88,23 +88,6 @@ function restoreFieldBehaviours() { elem.placeholder = ""; elem.parentElement.parentElement.classList.remove("hide"); }); - - Array.from(document.querySelectorAll(".form-panel")).forEach((elem) => { - elem.parentElement.parentElement.classList.remove("hide"); - - elem.classList.add("panel-invisible"); - const panelHeader = elem.children[0]; - panelHeader.classList.add("hidden"); - panelHeader.firstElementChild.firstElementChild.setAttribute( - "aria-expanded", - "true" - ); - - const collapsible = elem.children[1]; - collapsible.setAttribute("aria-expanded", "true"); - collapsible.classList.add("in"); - collapsible.style.height = null; - }); } /** @@ -118,7 +101,7 @@ function applyFieldBehaviours(connection) { if (Array.isArray(connection.hidden_fields)) { connection.hidden_fields.forEach((field) => { document - .querySelector(`label[for='${field}']`) + .getElementById(field) .parentElement.parentElement.classList.add("hide"); }); } @@ -137,26 +120,6 @@ function applyFieldBehaviours(connection) { document.getElementById(field).placeholder = placeholder; }); } - - if (connection.collapsible_fields) { - Object.entries(connection.collapsible_fields).forEach((entry) => { - const [field, properties] = entry; - - const collapsibleController = document.getElementById( - `control_collapsible_${field}` - ); - const panelHeader = collapsibleController.parentElement.parentElement; - panelHeader.classList.remove("hidden"); - panelHeader.parentElement.classList.remove("panel-invisible"); - - if (properties.expanded === false) { - const collapsible = document.getElementById(`collapsible_${field}`); - collapsible.classList.remove("in"); - collapsible.setAttribute("aria-expanded", "false"); - collapsibleController.setAttribute("aria-expanded", "false"); - } - }); - } } } diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index 307450b05d16b..fb3e188949b66 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -25,7 +25,6 @@ - {# required for codemirror #} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 11ebd6c4cb436..653b0dd3ce07f 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -25,7 +25,6 @@ - {# required for codemirror #} diff --git a/airflow/www/webpack.config.js b/airflow/www/webpack.config.js index ad1a7098e0803..9d5800f783f50 100644 --- a/airflow/www/webpack.config.js +++ b/airflow/www/webpack.config.js @@ -60,7 +60,6 @@ const config = { airflowDefaultTheme: `${CSS_DIR}/bootstrap-theme.css`, connectionForm: `${JS_DIR}/connection_form.js`, chart: [`${CSS_DIR}/chart.css`], - connection: [`${CSS_DIR}/connection.css`], dag: `${JS_DIR}/dag.js`, dagDependencies: `${JS_DIR}/dag_dependencies.js`, dags: [`${CSS_DIR}/dags.css`, `${JS_DIR}/dags.js`], diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 192e57c1d70fc..77563d2eb1a81 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -132,10 +132,6 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), } - @classmethod - def get_ui_field_behaviour(cls) -> dict[str, Any]: - return {"hidden_fields": [], "relabeling": {}, "collapsible_fields": {"extra": {"expanded": False}}} - # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: From c8f9223714a31464a53c41ca3edbf0b2855233dc Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 5 Jan 2024 08:47:09 +0100 Subject: [PATCH 268/286] fix: set the default value for "auth_type" as empty string SelectField expects a string as value. The default of select choice cannot be None. --- providers/http/src/airflow/providers/http/hooks/http.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 77563d2eb1a81..9cf5d4983dbc5 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -120,13 +120,14 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: from flask_babel import lazy_gettext from wtforms.fields import SelectField, TextAreaField - default_auth_type = frozenset({""}) - auth_types_choices = default_auth_type | get_auth_types() + default_auth_type: str = "" + auth_types_choices = frozenset({default_auth_type}) | get_auth_types() return { "auth_type": SelectField( lazy_gettext("Auth type"), choices=[(clazz, clazz) for clazz in auth_types_choices], widget=Select2Widget(), + default=default_auth_type ), "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), From 50c76135c4e0c5e6218883cac62da29393c86905 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Fri, 5 Jan 2024 08:48:11 +0100 Subject: [PATCH 269/286] fix: Use Livy hook to test invalid extra removal --- tests/www/views/test_views_connection.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/www/views/test_views_connection.py b/tests/www/views/test_views_connection.py index f9a4efd11c15b..1e21dc4856ed1 100644 --- a/tests/www/views/test_views_connection.py +++ b/tests/www/views/test_views_connection.py @@ -459,8 +459,12 @@ def test_process_form_invalid_extra_removed(admin_client): """ Test that when an invalid json `extra` is passed in the form, it is removed and _not_ saved over the existing extras. + + Note: This can only be tested with a Hook which does not have any custom fields (otherwise + the custom fields override the extra data when editing a Connection). Thus, this is currently + tested with livy. """ - conn_details = {"conn_id": "test_conn", "conn_type": "http"} + conn_details = {"conn_id": "test_conn", "conn_type": "livy"} conn = Connection(**conn_details, extra='{"foo": "bar"}') conn.id = 1 From 0b2a166296fe7f1f682365035288f2d7ff79fba7 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Mon, 8 Jan 2024 16:40:49 +0100 Subject: [PATCH 270/286] feat: Implement CodeMirrorField for providers --- airflow/config_templates/default_webserver_config.py | 7 +++++++ airflow/utils/json.py | 10 ++++++++++ airflow/www/app.py | 3 +++ airflow/www/templates/airflow/conn_create.html | 1 + airflow/www/templates/airflow/conn_edit.html | 1 + providers/http/provider.yaml | 3 +-- setup.cfg | 0 7 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 setup.cfg diff --git a/airflow/config_templates/default_webserver_config.py b/airflow/config_templates/default_webserver_config.py index 4ad8ee6743f39..85b9d4d2c8dbb 100644 --- a/airflow/config_templates/default_webserver_config.py +++ b/airflow/config_templates/default_webserver_config.py @@ -35,6 +35,13 @@ WTF_CSRF_ENABLED = True WTF_CSRF_TIME_LIMIT = None +# Flask CodeMirror config +CODEMIRROR_LANGUAGES = ["javascript"] +# CODEMIRROR_THEME = '3024-day' +# CODEMIRROR_ADDONS = ( +# ('ADDON_DIR','ADDON_NAME'), +# ) + # ---------------------------------------------------- # AUTHENTICATION CONFIG (specific to FAB auth manager) # ---------------------------------------------------- diff --git a/airflow/utils/json.py b/airflow/utils/json.py index a8846282899f3..9622f4c0a6b30 100644 --- a/airflow/utils/json.py +++ b/airflow/utils/json.py @@ -123,5 +123,15 @@ def orm_object_hook(dct: dict) -> object: return deserialize(dct, False) +def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> dict | None: + """Safely loads JSON. + + Returns None by default if the given object is None. + """ + if obj is not None: + return json.loads(obj) + return default + + # backwards compatibility AirflowJsonEncoder = WebEncoder diff --git a/airflow/www/app.py b/airflow/www/app.py index 06a0e14de040e..f82478e30d4ed 100644 --- a/airflow/www/app.py +++ b/airflow/www/app.py @@ -22,6 +22,7 @@ from flask import Flask from flask_appbuilder import SQLA +from flask_codemirror import CodeMirror from flask_wtf.csrf import CSRFProtect from markupsafe import Markup from sqlalchemy.engine.url import make_url @@ -129,6 +130,8 @@ def create_app(config=None, testing=False): csrf.init_app(flask_app) + CodeMirror(flask_app) + init_wsgi_middleware(flask_app) db = SQLA() diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index fb3e188949b66..8e3d8db0d5e00 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -26,6 +26,7 @@ {# required for codemirror #} +{{ codemirror.include_codemirror() }} {% endblock %} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 653b0dd3ce07f..174bfa164c4c4 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -26,6 +26,7 @@ {# required for codemirror #} +{{ codemirror.include_codemirror() }} {% endblock %} diff --git a/providers/http/provider.yaml b/providers/http/provider.yaml index 7a991044ea801..bb0214e7c6e66 100644 --- a/providers/http/provider.yaml +++ b/providers/http/provider.yaml @@ -102,8 +102,7 @@ config: description: | A comma separated list of auth_type classes, which can be used to configure Http Connections in Airflow's UI. This list restricts which - classes can be arbitrary imported, and protects from dependency - injections. + classes can be arbitrary imported to prevent dependency injections. type: string version_added: 4.8.0 example: "requests_kerberos.HTTPKerberosAuth,any.other.custom.HTTPAuth" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000000..e69de29bb2d1d From f27ce95b5d3dbb621a68d835956c07859689e673 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Mon, 8 Jan 2024 19:21:34 +0100 Subject: [PATCH 271/286] revert: Remove CodeMirror from providers --- airflow/config_templates/default_webserver_config.py | 7 ------- airflow/www/app.py | 3 --- airflow/www/templates/airflow/conn_create.html | 1 - airflow/www/templates/airflow/conn_edit.html | 1 - 4 files changed, 12 deletions(-) diff --git a/airflow/config_templates/default_webserver_config.py b/airflow/config_templates/default_webserver_config.py index 85b9d4d2c8dbb..4ad8ee6743f39 100644 --- a/airflow/config_templates/default_webserver_config.py +++ b/airflow/config_templates/default_webserver_config.py @@ -35,13 +35,6 @@ WTF_CSRF_ENABLED = True WTF_CSRF_TIME_LIMIT = None -# Flask CodeMirror config -CODEMIRROR_LANGUAGES = ["javascript"] -# CODEMIRROR_THEME = '3024-day' -# CODEMIRROR_ADDONS = ( -# ('ADDON_DIR','ADDON_NAME'), -# ) - # ---------------------------------------------------- # AUTHENTICATION CONFIG (specific to FAB auth manager) # ---------------------------------------------------- diff --git a/airflow/www/app.py b/airflow/www/app.py index f82478e30d4ed..06a0e14de040e 100644 --- a/airflow/www/app.py +++ b/airflow/www/app.py @@ -22,7 +22,6 @@ from flask import Flask from flask_appbuilder import SQLA -from flask_codemirror import CodeMirror from flask_wtf.csrf import CSRFProtect from markupsafe import Markup from sqlalchemy.engine.url import make_url @@ -130,8 +129,6 @@ def create_app(config=None, testing=False): csrf.init_app(flask_app) - CodeMirror(flask_app) - init_wsgi_middleware(flask_app) db = SQLA() diff --git a/airflow/www/templates/airflow/conn_create.html b/airflow/www/templates/airflow/conn_create.html index 8e3d8db0d5e00..fb3e188949b66 100644 --- a/airflow/www/templates/airflow/conn_create.html +++ b/airflow/www/templates/airflow/conn_create.html @@ -26,7 +26,6 @@ {# required for codemirror #} -{{ codemirror.include_codemirror() }} {% endblock %} diff --git a/airflow/www/templates/airflow/conn_edit.html b/airflow/www/templates/airflow/conn_edit.html index 174bfa164c4c4..653b0dd3ce07f 100644 --- a/airflow/www/templates/airflow/conn_edit.html +++ b/airflow/www/templates/airflow/conn_edit.html @@ -26,7 +26,6 @@ {# required for codemirror #} -{{ codemirror.include_codemirror() }} {% endblock %} From 24ed0405c176b8cc2159c4a7ad89cbaabe6eb0a5 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Mon, 8 Jan 2024 20:47:50 +0100 Subject: [PATCH 272/286] feat: Add documentation --- airflow/utils/json.py | 2 +- .../img/connection_auth_kwargs.png | Bin 0 -> 9623 bytes .../img/connection_auth_type.png | Bin 0 -> 14199 bytes .../img/connection_headers.png | Bin 0 -> 5256 bytes .../img/connection_username_password.png | Bin 0 -> 4761 bytes 5 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 docs/apache-airflow-providers-http/img/connection_auth_kwargs.png create mode 100644 docs/apache-airflow-providers-http/img/connection_auth_type.png create mode 100644 docs/apache-airflow-providers-http/img/connection_headers.png create mode 100644 docs/apache-airflow-providers-http/img/connection_username_password.png diff --git a/airflow/utils/json.py b/airflow/utils/json.py index 9622f4c0a6b30..eb3cd40941197 100644 --- a/airflow/utils/json.py +++ b/airflow/utils/json.py @@ -123,7 +123,7 @@ def orm_object_hook(dct: dict) -> object: return deserialize(dct, False) -def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> dict | None: +def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> Any: """Safely loads JSON. Returns None by default if the given object is None. diff --git a/docs/apache-airflow-providers-http/img/connection_auth_kwargs.png b/docs/apache-airflow-providers-http/img/connection_auth_kwargs.png new file mode 100644 index 0000000000000000000000000000000000000000..7023c3a7a072f965f9dd053f77c5a5d64b396f47 GIT binary patch literal 9623 zcmcI~2{_c>+jom(n~(ixI5hS2r&k-xWWM^15qC|U8&@yI zp4VbIoNz&&;S|%$vllM#zw{D3bLeCYFTFO4DogYQZ3Q#Q(MB#`@|PP{w$;0sjC|gQ zwZ00``u#3Nh5MrM%WwU6MOC!GEa0bjzF~#+9_k$RUt@1t_`%yIg(IZq`(>tL>=B>- z{$n_3=<@gXa@sCopHqhy8ve(KHqs61*aGb{L`~`|6OlkDb>1ArkdEJf(u%A*3sMN|@KU37UWK z`S!;+d!t~o=`Y)wx6bxn)?k%F?VSe!d^L0ZZ$r)AA%p$*$)o=^r})M@ajS#xngony z*Lg06eEks5LtY{{Vqn>ksZNFGX*0{b@+ z#0=Qog@Hj;9>jhwU?syRst>MH=7w=oH_sMKnFK}Mwee}-tenP+6yF*CgOf6#l9m~nV&Lm0k^g3ly*;YJZL)1?XSHiuScftr z#fh-AL>l<*5_9^a$4@YkHx|3Pz$brZ?SLRnt1aptkF^SCGF4So9T@qEx}W?S9Hhje zjy(hdWv_M_&Bzq|=dy@X#CzWgO-|?oQ*sLLA1p%)S2ocGlYcv~*k1^Hr`qWbrW`D1 ze39xdiiP$)c~TF*B%`J~(|0HvDL6}UXmC$AX=?Uck=2jS&$IIKgwo{Qg;dPIWDXvG zMpEM11PV^VaExN#`GO17fs{SQ=DebuB3m@@iB@cQBh_QnXofNT+W;eRTnE;5qA}H_ z{zc{+2E+#+k7QZLTNt<2>=yczPV0#hS6B%9kd3{-YcCyL!Ul_+k; zp0DzSG_^E_A8QO|+-h}b+8Fg8y{FmTOUkQv9CZ=kv~^}^Lc*}YvMAAFoY;}%=|t1SKU^tU@;Fdz77c@R*1UA%Eu_Y3tz$}<))Yv z-3pd<>}&eTERFRZ4--c09NoKpl|RsR_t%vKf8p&x_J~HJ$(xT zo$oS|$Z5$+e80CnXSF^*nB_j*ndQH~H}>(#(cCVJdOvxk)i2Uz0R!(`MoYbx$5G&7 zbo(K0GjYpF_4f8Q`5Qg(SozU`tNz$D_S>4h8^zdEijt6gb!toDK<=$u;A)fy!Y`h(--=f&9P7^^!9I4 zS6lByogoK?=((~MUPch>kn9^#u;HhK@Ta@&)yh^MbQOJm+xd0L-}4_>pNLW6Rf${5 zn7B7A{qPGNpZ|_4r;-v{iFBo2Xh>#+3G;|{E(gSV34$r{E%Y!9`-0-)9E>o*$tJE)bX!7Ho6BjelTH=PL$J?c1 ztcS}V=!m@Z84KnZ_Y8M`u0~g~R_#1AF0&rzJU`m7zjtE99k_*)p*%{1*Zcfkv+5{6 zR8O`>jGTCSrfnv=TK~22s0ZyRx?UYb5c6=Y2V+m39wWHzVWSoL=?cN?3qvg)87ltu zzh9h(5?Ru&?C-2D*OsZIjn3QJB{cg>n)^q~LN{gBOxrEsTCs>YYEnTcVzXZ93bGh0 zzrFi`%hISd+fZlbYVcBS(%oG>bOD1C|O-DZ0gNyeZ7| z;uZHP-86$>`0iguVGr-XX5S>WNe=s()OWpcM50@!i6`t8{wEP90XuQUY@7_G<` zMX#lM_TFlzVc))-lM#s5Zg<<^h7V_c#6Zqm$~pbCRB0|7%Gw{i@E|bt-6%$}x^(MT z-Qu_`f0Gqj1Z!{SyJoQ}*|2k|Qd9r z>FA|&f!~@IC8ok-O}>jCIHvPUqjzf_FCfwq>*MTI=x)nt{8ZFBRE`|3^>UW9s6CB; z+3O(2M`ZC5LsVYC9G_8Wt~q;JO}+n&`_z}P(KFTfvQsHJbB!C-1>(qCCFW<61a)mJ zfOn$ExQ(<4t2>BmWijT^^%ryB)2zikhe00HJII3zq&#Kg*d$5IyFLcNnc?t;1aLJx zoJ_6;UlJf$ak1T*Vm)Q#Qln0~QLgivXwu@@_BCrF#V znBPwJa3cfanoY|cE6bmaJ@LQff|QJNDw<>XEWazOvKK!+1>2;7y$rtLFc@1}@5Cq^ z5Oh90X3=-{nICF79SBuUIwF7??)EGxo70_>P)j;b9!5H#)Hl_1 zveiRRBuu#?H!9SwzaroBq4NfU|RlwHPbf$H1%Avp5&z!DSXF0SkUDbIQ zN5eK3l*mwE#zO9}ZBKBzYRHFZBD9^0n9#M-o()i^yOx{safWYyQzNa#c6BHxXF}Ja z=TzVIm+=V@q@7h6h!}-c`>^rC(p-m*cP7Kja zXnQ)RsZ+5HuBV_@p$s+68ylhGc+%wfh=gMoM5{ix-n#blmFo4>aIpk}F%GGZQlEpa z3wlgv9Xm2)uZ6;2Eq1EQT8`gz3s5uOC*K-(z)}<-x^bclCkeB(~dQ^KZsXG93)}Y@3u(u2tR+;ef__lr)x~c zs@fSnBczc+RPD}83uSqz>jXlZFN{@i>ojr-JW{l0sO6iiBS_stR=1xb>wi>xSK!1U zL2LdH!5&D=3-{dfOHz~^B!x^?4C$#}Wk7NbjRY>)20)`_Rah=>~yAj`Jqb1^m7l^l&k_AB-56!+#;7VCgf_!uFmkuic!7h?*gZt{)_2! z_c`Me!sS*?m7~g@XM9#~KLmRnseyl4WPAbV2*VohK#Q(PPtg}6O__^XUmF|M=ox4_ z=%bFpNYb{Qkrt(|FI1%aC8ceb?mr5CW!NDnU+Hzvvb;Ewb1u|6KlmS)o9_9`>qjTe z>2Sb~h^;ekty=tgZxaZOj%!mK5;{L2XR53y=c+y=nxBOWYq!@&;XMU942~iy{FPe6 zyH-AoE3SQaeEVrK6-tzPE!`imsL7?9x|XY91sBHJ&nU2JU$VP2Oh>`kp37-@+oHC zk8(4QJ%7dN;5ZA?yJa_tui}60sE-N^DUojZym`q_c#yXxfwn!* zgmwDa&!7ie?kNx?9j05>#_w!&(xPXh0=JQ?~35`m$S9)v?zSr?L@Jvwk6s^tZy$#4l(*9EbNxA;-XvBZZ zJM4>a9U6S-(>`mn37JGEW``@B@@_U>)9IcoaYH#25(MHuq%}@Bv>r`vc9*gF{Hzo_ ztrjf-1^GrHd~Esv&v8iJWujnQmEGS)rsXzTrWCmTT&M=K>$5pq^mZjKbwRcG_#i0{ zfE=s+z1^`uX62_(G^}}hy}enmDimbd2;eNfz$pLz)4-zJZ$tX;`ilQv992nE^em~| zsdDTeuea4{iRPJbZCL9w%c`!f9+@nk{Lvbh>j46?{%Ox(%zI{)9jQk@y9HHhY_?V= zmn%Y<`8vD8TfSzghk8XOtN3F~Yup}S-6pknm{tX~42Nu}xxeoOa&sZu%@1&;QipGa zr4Kt&cFsmQbnH4uJE2wwtYbbANF-d~*m4VBm-tpDZ^(?k zHMx6NMFkVJ4EgcPr!Y6`!COlszKg+~>58EXL+|?UM0ZK6SRzO3EfB(V#u>i9YB&iy z+jDO{rep=N-anpeGyg?7 z=xZ?-BKBfEud<()7eI@R0KDQUtRpZ6fVyUH}^g+ zA&J<5vq7al<_9Y_;67axeB6J>$cR-vbfP7u<>%lUw;YO}iBpPyexU4}#1II8f$xD* z$AX~q(iWkn%&HyJnf`ab-?)rD1Hk;|O1mWf`8kco9Id$V%C`-B3crMLH}0aRAIuQ4 ztolnVawp}4`j*=z8=5wP&H~yPNyaw}mLODimeiA#(LPLa1Ji&3C92VPegiJXgD9V{ zZP=Phl}l%rr8$E|jj4If7UYj_jSVQ1=PL-sX79Xsg>;iy_O`nGnp>UhN<&x(wq9ICsLozy*^cqcG&t(C$ zw(3eS6H>X((`ZL!e|Nv}%^m60idosmUy?UptWNLS35jbtq*4&9VH)fg4QGKFLxO$d zv0A1l;Rzf&M80I35<5}#UW z07qEE3+8>ihCzIrKDSjVSYl0kzd0V^s(hHis&i{~dP3QGuw3x>{GfETq3UI<0E++S zeScG4ZpkOQHS9cV=}930(7W#D6lmPP)D*^=r4hqRDFwI=mRYxi05;jOOES&ZOT9Jw z_}Uz2Vy+JR&;;ke@mjCl&50P-T^hT^dNMMt(_5HxoZA{70@?o>n`h3iK9?27D%{); z5l?FI{^1wEe#!dRu&)gzBq=$wj; z)iaW6`y2jzOc_i2*3r-dWV2@DpJ^Ya>d0TpbL3eNfRzC+LN9{H6b>TpMTfMz6PM$tC8tK0pe9(V zWZi{A+S51rlxIFrf+EDM06iHSw9ORnz`wauasS>?#Fa9pwquuw*@W?@sHY^sI_~}i zf}oipHjV&;k>wHRa<4Mz(Ie@Ke|7o!o9O;>34 zkKrMUHE3o}eR(nbFfi5OwW+4zR|;5y&C(Aij$p_OhEj1n*RbZOILHY*3;eWdQLLMF zIzIqXr-UP15p;Z4=lG5sJ|a+S{f`0U?8DygI7ok#>peObJ9_ox*Z{!*NTTR%I!8Jr zoohK=y(ypFaQtG1%>AaP-S--3_=Bu3aC~Or`dS>`r}LBijlc;X{AF#f5s)&^kDF(H zq63J2NT1$3#}L2nE4Gt|6DWm>PsYonhr1tA&mW0{Uv>`Pmp~YAh%l~BcSQ93e3!w* zCC9HnA;P8}!W8}1Xs~`a-4G?cF#=hbf{)M**Lgs$_&#EGnM7t(tl&Btk1>w>D)$sS zCJ`1#^!{;avbEK9q&!z=!WWsM;4#l~BO1HkuY-Inte?z+fBkC%xn#`vMJ>rS$9dpF}PO3F$M%_W;tTB|<`*t{}BY>T@%ppMeYVE~+I1h-1NxRg>A zg5`xDhwt((K#!h4BD{vHbfy*bUP2j>bZVPSi=GQU@XHJOIEX;Y=jZ3f73fqi3~Hbs zJ_v*!iFJDV`}gmk%UAppegu^@#0vx-|Fl)`jWBCcG+xhI$Wk&-tT`ukxAL_Nv&IR& z=tc7SPo5i0e7w`=s<^n|k9D0IGmaEgMs+O?Cok`{w#JRSN?4TVoIa!#J6o7X3c+NtLyj1m2J54Ku8NfltG`MtV znaa}wplZ*Z`o;I;)LyE!LlFt3=GB6X*25^3zua(;o(%7)7n>=Bz{7r9$kLJ*(zeYv zBYZg!s$ZR9A{?Jf#XI*wHYbwYy#1mR6_{AkouA{cF!s>rP(b)1q0|aLZrx|JApsah zY*OcV^+l(Pow|*r2WWdhp~nJYaEk%? zc(l&jd1Ip?ShzVxwAiKnL#p-ESB9yku(uPAh#?l5GUlDhSoN322%dY{mTZ{FZwcWRVk?*D1 zZD}EU3|?6}w%<>4@ooLK!~Clz7o(bcdGa_&gXAOOvnbNLKy5{fSDQSY$Khg08TCz9 zTi*ocTRh2E|2i8o%C?8av8LM5$JR2-vJ+)wxk^`8sE#>?=i+bHRzc42X}8u~ar43Eg8c3QY!gT3r0AERgA;z=Q0 z`NE*^n^T?j3fHmX3l|MKNnS^Dj`rBR*cdFc-_u1&qqi%&ZSQ(-P0B44ziabM(grE^ z!DiAeNi7D)na=C|qP5a!^1?e;uEXu+4>r`fk!P1@i=gO%o>vY;(}53Tu0M zToR*oBGz2u(Wj2Qa6;iNrxf}hh4;)P%5IHBU50lLPJNHfAr}G<^paXNj6c(#@6N7n zr$*>U*E|5%iz|Qmq418iOeb%$aEsr4k3D_16oWvrY!ecm-n0Gs>Ux>ZsfPBbV{pio z6YrISEJVD7dpGrPdg@skj-Ny1 z{7ovWvmTr*cA`t6Gw+`G`19@!xp>6y(>&spb^@U8{XIJT7(A5OpDQqQZ@OAYZm4KD3SsW($Znu zGtq6Y-#O7U$fsp!K@0bn!q&v7G!o7}XbhrvEw!%YbVb|I$Dz-Ae*L;Fsu=R@yjCHG zjS-pQi$pRBalV#}D6>jdUMnE)ezRxq^ldz&#(}6KbPGa>OuK0Q(FD=kO|CR;!RJ7N z!`7U$%OdyuG|C^}iGu#}&v5%a^IUEr{oSa9p3d-xYUg3D(U>#4UQC&EcLzKXp9#f< z#Cof#$TO}A;vNIHX$3oJ;n2(PPmN07yQG%}|FGx76v_FV!l_0)4-Y9 zbZ(yy2$hl0RkKsOEr@u@@})EoC@9erb-#b^YBISA>7}u#NA_lB@Mr5xrwnWd;ey+# z20lZ)kK|L4s8<2ycE2u6F4Fv`y26p~))|+6R=8Dr^*- z0)Ui`ajGj~xM}9rbq6JwpbIDxr z1LgG^!M34#v@h0uTAwQN9FW{JDl zwK;rq6m-JzND9E@Az(+%w_tn#IHZ%NLisRsJ^2RA7PXJ)!SoQFBne2v-nL^}5;Kwu z!LImtp6)0u6F^R$F#=ye1-}vOcS0%PJO>vawYnOtUl{A8dato{)<1Qb^%U#?-d(A^kiECYHjx0rcd9SJ7b{! z96pARQQ6}ZGRQC&#%4{n0d>qeBAz0K0tqb`SkvsiMas*105&;C05l&37UD#`U;plS zkz`u$=bdK2-wlW$YHw$20A>eHGz(~1xg|P4*m&JD{ty)8tw6bcSE_ zMU`0^_8k2*IdeY&@s&mUpaz!pDjj4V33WB}NdWpqm&*MT1praXlYM8j_^#2jHgpcjBpYOx2IWu@IqX04SL z9wn~mzqeyrKGm6S#(-oOGi5RRkt1Oq8HKNHKL)pdIVnRegA))!qI|@hXA!GQQdB1u zQ_Dp9vNZ6>>v*g*5QGXQnQ$E-e8(M|y;?CL8sP^!$YPvmu%YS}M1nI68ZIT}ipD_z z<6&R_HEiY+_&?mxSyv_W)t<$uuEafyjWcL~qm^d{zNFfORVpsdZ&4t2i|UnX^n zc@C8S32ILd(FYlJDXr?Jlkx;9RvFVAoLVAjYGtzh;(LJ{k}GNccJ<^JDh$}n6Fohq zwawFU91anWxNV7+{r$5?SXnQ;pGHwh8A>DMlzA9KA2ZWjW%u5XuA=^%L8%+hdNA+KUxJ+NQ^~; zTn9UHP(jy0^`zM4#$I$mj(&?!TSuq#wsw~CIzANk)`;z1NF&A3EomfU0tp(xzd z2ZcTVDiW>l`g~X*OFz&oTQJg7kzk+Sm8sM2upK1z0}1W_Bs2f_)jcE%Uk(~1PJpBb z;gB%@-1RSw06?<831r+ngG}CA#fXIes>*|VZUs2-Rx8-B;om}jF0P}m*bEr=Onkuk8g0E@GN8TH(>+$dxn|e`V?;qmYie-%1%QuKYo} zTHWJ?Mv(qiDu63;ZrM_cbD)O-WOK$3K5Gm~5*BR-9R~*&jIxmP5IZpcKlL$yg*J^v zz)AeA@PpaC=ZfDQxY6Vw(ZrJSs;e)5$+)5@cBlOOfi0X=TV&u^t3gWDPBoA=7(7ER zQ(FfL|KmeSgG(WWbN`Rs8w9{eDN`;Vv=|hht9ZJrw?>o5+so|!QQ4NeB${+x0%*88q;*~Y KT8Y}dfd2)j<)#Y& literal 0 HcmV?d00001 diff --git a/docs/apache-airflow-providers-http/img/connection_auth_type.png b/docs/apache-airflow-providers-http/img/connection_auth_type.png new file mode 100644 index 0000000000000000000000000000000000000000..52eb584e5ccf6463273c9b0d35171944d451af9e GIT binary patch literal 14199 zcmch8XIPWlwl2%HEh~tXqEt~qIwDQF282i_fCvGiAiYU%0u~|)Qlt}_l+cUx63P;! z6RNa8U?H85AP_(Zf%_%6_Bnf>bI)_{k9+^{0QtT-#~gFaG2eHLH}7?|)fhmmAUZla zhKCRpC>`BD5Ww%PXHEe>*-vEj)6q#QK2*7H;A2j}h4>qeWG!xxl`vB)-qyt4sieB$ zK!I^4{_6to9tVb=yr>fRj*oBX{%=hGK(XWx`L7FCMn$QE{n{j#%tES+jZ;z>wWBLy z)|Y&zJ5tO8c4S2zN-5sF;b`ymw>G~46VlOrV)=>wah3~pf%ezEbrt@9(0&^@tq1yz z&SF6ae1<#t0(k%VoqP=R-39QOm9E&Dc(-5w)ot1MK?$B)ipa)5&(OV0dve4x-+#J} zB|8atk~=s`G4{FqT4dbO+0(-W>d2h}*iREaZ1QOCh6GLh?;mc517iaZl3If2%-h1l zD4Cs1${fNWe1K1kTjTDRT-en3)V}zPm6+SKa(|U(%9$0!Mn{7)`&xTj;C)6W<=vi) zolM3Pbk9y2ote1PDHv>eF+1GCX)ZlyF|dG*IAL&+X6#VdTzezy?+)@swvKC{%slpb zrFbGljB3@g>)}~$r`{=%rzK~HPU*6J?cVp#x&9IDxvS}W-Ky%BiHEjpg<_xETUakU zI2MVy8Jw##>yf={`3_k4z}$f`*x4&8r*jp4zw*m)yAE56ToooKJ77@F?n<-w}U^m}xbX2m=)o+xvHvuLI6^WI#x3k`GC@K=GXzz0#)m5{mr@G%S7JkMK z>+txD-5Vq1_)jLoThg( z7wLwInsm|}hEu?K*|K}0JT(H*Y+nn#Sut@#s_yyOgQh~xw^=U9tmjg<8Q8ol9c+{K zMTKIe|7PSGi!!T5&3L#F`-{`mjZ)wp5#lDW5Lm!9g2}xZQ?lZ|-buazT4lagh8L;9 zbA~%W>ey@L?cIT5*=3dO-w3I~G}nveTmNDAMV`P_yz+T}?7Y)jZC2)BwUIoqYz1cL4D z?KR!qE9$)0IsdjnOH0cO*t*Sx_hsc?Jj2JM5F9he4m59bqhMIouVF&mt~f7Xu~Qm>K>&E8%#QO46*8mprfkZQUl<|REZ^47lDpg(M^^u zf6Y8J95}o`MmtpJZ9f#1mACh@$~l}ZE~{+*%L9f7(RNqG0X2ZD_s{uxSYj#L@>U|KplpF}}~g!L$O zDK#Zg?W3_^x|c-@>fGM*Vn(32k!f*+Vb^xB2&1$v5KeS-{I$?!3ya>OvWoVBr~KbW zFLu&4utgN%;qecr+5%z%_#`j{9o;>X62_s#2@$E(E6NT+#bS~`g1AhA)8;^&kVz$jkGX}Ji=?z^CT&{zT%^0!USH2>Qtg;DQEn9`?751^YuVZ5Pkes*W^W+@ zekpPiYrXMGGY$*{Om(ciqvKi8a*})`yUe9Dr|zB}!W`Z=SDCu@0U$>CB~~#XtG`~o z>JQ7)5f(dalOq?JcFZR$#uoIgRO#6C}*EwY6Dekdix!B-8D2XStOGu#Ak1 zmvyX8+jvs|Idj;iKdmYl-91^idA1#w1g+%2NcD_-c#zYh zTc_ql8GHuMvi^)VPPpfYOTs3#1MCkE&J4^{ zSohs_x9*qfSK2M|-9Ri6@XsX@;Zaxa;>*(g%A5C|kjRVJk;1_Rx|b0v-B3tqXy{rm z2a^cWDyv0pY0qHFRy8u~j;gq=M;aVtmlHwk2;!_4^`7qfTHTFVbv3w8K!K`d-2r{dE{w1Clu9ak=bM!;GTt()^PhF4HYISblQOftp!vINoYwDf)+a${@;ZIXgOtQK!!Pb^=J}}0 zv)HtS&P4MJyx!ZXv-OJdKNlX(U_IB)u(OFndp)T6jjj^z9S3eIFTW%1O*DmN+rmDD zyExFxkWo?H%uHg%zTcpbcp+m8ZmQB3|LU#aAB+6ik@}kaySD4fwN@^mBF6 zon2*ssI50D0OyIf={|}BJJ$$fx<7`oh+>BiP3}cPL0Tz4eLa%l@gh}A#jpfFmh#Vx z(K7DVlUpLDcU{i0kGmWl62{;JE23Caio8ty;g{b}uldICQZSr%9tY)i_ka0|jcz~@ zNH7bFi&(QFsNSng<*#90T*J=hirzjx&5T2=^81rYDUC1~L_6i~(|A!hrx~xwmfGat zPCU3Hf};Y@mVae``*7zW>7XT3ZA`-uv==0YEym0Q^rpYM>8F({bDN3_{T=}~jNf%6 z;xaRDx44cH69dwWf4-k{$oa8sX)&}@Gyxdey5GUcEW-d5b!tS`WwwQ@x5!wWK(ryk zpW-Yv4=tb`0X01nEHS)EuV0@a?+#>$CTk>!RKFWd0*P{NHrZ29SV&moB;@2c zrG;`ZX(Pku&k2_j9@JYvb~CQtm%W}y>=@ivm4%mf3BN7I+s-;{gk9Tx>{pyw^`0bu zfXLb2H$>*28QRej|3rcVi~XtrEVd((s{+;KPWyeXNpjUz2^tYp4f-Clt+)x0?{ zRiXGd=c7QKY$3$dpL=@h8J9aEYwp6)VU04I4B`jdV{72S2>-K821K|<&^~{iZMD}M zcXXs3hTG^lt|`@}FI8dkJ3Rb1I>J~SDk?8n?|p^Oh6Sv^wmRHv!wvD?wmMSoW@C5s zNTHG1(US}DV1ei4ju zH#pej(G+pW!x zXR877dLv8m!{6wZ?*dpDNT6HgnT(+^;<-ALn`&WdWGIsk-OG|V0H8ca>59?VP?S0V zMQ$BK5ol*mk9Zifh{T&y=PKznL)4HNj3-_a6W&81_-z_mzA!Wy148M_r78w=6E{&% zKGR(YfUQ1}5h&e8ILJ^knQcL9v4sMKpl^h*snRG3c;8Jzb@Z{ZrFD3!W~-m{db1?;dSA|vm6i1uo0R;c z11)k``}jXW?(lrc>d`AOOQ-NU7!~L(i|qmjxr>szEqc{69FtSuG!Z){f&5;e8WSY# zOA-SDithaI-y1pl*u>I4yqJIldmTgf_%gL~XgFxXtP@F0@`HgSlR~ko?zshD7$1G3 z5&=58;(>pA87IUYUMIPuji8Z>Oy%yQd|XE~S$<4fDhd0RS3ha|Q=H*gaa)_&(9~$bwnlupzK$? z;WNuh+8bEOIhp^7_fSK};X`p5Enppj5m7_W&gV}&1NPxETlpz{eV9vuy*&oky&8*( zk_yG%TU=O3lJh1e0@v?x9Qm0^j5d1W<2lGV0&gWAjV2mf+Jt9F#*ZeNSUQB4;e60l z7BoIs42=V01Q1Ao%y@3UXD~6dXnhnj{-+^I2tg|45eTk)`ppSP`E_cOGnM-tdJD@+ z+yBe}&VW@%A2qC;G%~&t4qWd5hwD?v9@`Bgkewwe+|1H4E8oTTc9o~+dj@!Meo=uG zQ(PRs1~*0Hp{Hk6qXMv#1Ic#1cA?g`lRf|eW z`V4MK`NhQbIuxmvH zO$>Y+FDMYu*VB6n@DwWDq8nI32_r5Yr=wp{+&TfK9V{Rhd!MFs4&b zUY-%ya3<{#=o}_mX@BYdR>e8*;7;5ps zMlmrlQQ=EWLV}H(yE_KJ9~j_{{@aQFOqtY88yz>XbPTVOT=bfA3DPVP^Gl{qoD^lB z6UE#_Zv$Gr>f0MBNBfC8-awBx?5qi@J9C*!Vs>7yj%r3E;JB_+uWQO3g~| zBmuL5tIVDHRGBK(ZaR;1)Tc&&y+`%TSJvZo%21v1IE&^<*$}{o7aV~90FGOt4rvii z+k<6xswax9${A$?4{95QfzJn6sCBRB)RT3AK1m4S=H{mP^yyO{YLpHL0)CkIPak?H zP&D34MPM+P**5CfrJs&7f&E!_>W+Q|^J;JpdTy8UVDH%<+5bA%Fp1HTL}^3rn=}hM zJ!lIddD8x(g&EzQFxu?L_F7XT$bv@h={kj|_6o_X^OHms4^ofM^rhfmPajv0o>9r` z%g3J>uOHXmDrjCgfx2*97D{5EdW@M5aeN(gInum&k6-2Z{d!I3y#38Yd5NgRf`bz_qLWC~Q0<*87VZJL?7Y zIX?<70QD`Mt&x5D*$Ls^DZhz>-Db+gG$-Py1X6(|OzSA$-$cG-IHh(yGgP)><(t#i zT8%$UzL~{lP5MaQu0y$BZaArOF=bEMQcB6=FvZDe&7HmGJ4rdADo$;$m+FEcGME}` zh%^Dj;euLoa`Tbxu=0|PhB@r{cuIev&+-AWvy08j`rG#`fTl8^B?u!9*AwJ0mT*G( zL}2y68rtAe>S0f;*`iYT3~MT(bFjOVJ>CR~W9p@2e+k@_Ldq3lH4J`+*f)e=B{Fi>M^c62Rh;ForEx zUk~y@S8wvAU!`uUADp&_1F5J1m=dRW9ZGUBS&KzWd01gR#t)Qt6!C1yF_9E< zTi%ELnHcZHfP_#&QPS!@FYLo|py8SP&$S6YI;0c{JgN%+wp4GBB8v6IFdBWE9LrzY zT0LkTTo8Du2=nEYrF@h2U&H_GpQ}OMFj^mP_Czz>+I0G&YdmH<{jh;}P(JUUJ;Wm< zX^Mq`_KdxS`k9qy_e2DuRQiXNW*U!>vvcXCEfDi0@6NQ*IqgZGjSRAx>)o-~N2AF> zwUEh(9utlYaNqbT9n59(G%((@m z+z7*tdvny!gKdR@x#h}buS3nx7nRt>ybD$_N?KFS8d-t*F%^LaV-wijH9aL-z>=D1 zk)I)_!Hu~v7-<#v+rDe5nU4Wo{uQ66?TTBaiqp@gO z_!1xD2U8k7TG;}aT2t4HpFI!#XLPhDEkWJ*-rQy$@AFu{oIjbbBL_Fn&s8Ji-Y+e5 zUnD3l^fRh^<8BMQFo5J~?t5F-j9)LvtQNys@dJr^SRn1d_!fZ7tew7>9pQrsUd)lx zrTqAQB~q~X=Dcw$6abfy+iCMyrQm;g5B!kP0 zE6FAv#;_Uj1-;|L{I_rXH8SQM&to>9FWFUhq!5fuOm&ogc%}t@bzoq{^_lIArN@XP zu==3<4>Dc^?J)A8*Dz{Xp6Ptg`RNZLWO5|eS|4*fc%_6*RUB$?s&@Fu3hcJ{Avs`Z z5Zc*j?oA25wMEcO{Nc(J2hQ|-)dDd}01LnZCxTkzEu0s%9Z*r3(Lc6Ex~|Frmozzd zsc=G$=$C%rC+o*m8HTiW+ryt|lyGRKlq}0-jpg-gBT~>)0p|JY+)O%1j%*8P?hM?m zrH^$eV8!FVSsWUGJwJE)(JR8>HGRE|r2F63ySI94{0Y85g-7BBKNF!Kr{^?A@kzG> z>zat-{uko|TV;z{;q-D4Elk7BeHQL7Xi5f`BOf$TEugu4rSDv4o% z+4$4d*1pSbn9;i)sBgqeq@QDGOn$O33;QTm6z){}MkjA%tlVBeo+P#Q6;^*9bgJsp(tD6<#s)d5T#sK3G-B@#SLY};&Yl1Z zR2$s1#oZ)GEkzVm?~w?$r(%dYZHcv$4tf@iys6sTX^4}p<-bci=4UilR57}RWE;)q z_CH>Kh($f@(@bo}E5~(JV=)DT1eu zFVp>c7s~T7J=D~R7L(79t=^pTFS)L+CgYASSaFpMd$3ga72=J1RdH^7DSN0DSDN;w z#D8s{11%S8-L?!6WA;ik*p3-Gi<5;pH*C*|`TN)0-JT*CIX=z&&Nt%E%zW0*$ud5@ zJjlj@e_pWA`SxTVo>ZW z2FEB*9~IR+*rS@OohGr~wB-eu_45hI)_uP7BH7z04vat`7Iz4fQgBV=v-U_fmQ)cw zzR&4(asfMQ#;|2}IWHmEQh){nIMgC8J|1gY=Qg34^0V&h{^kWT&NUJ}zm!f`;Njs} z972Uz_a;VS*lxTf>y%D9;_645I{YH`Q{=Ayt3d$loNW#1#g{hPQ#(^>fvXSQ30O)0 zCNGMw)pAdDuMe3}MS8x7_+t6rI-f zBW<|+N4jk-VbMMu>ET>9HW)quRgdQV*`sGifnJZoIE^aIP0-E)^+#=C%t-Dxrzbi_ zwqyeK4Ls`9ql;nHD!DGx4R&xfpgbY0L8@_vByy;uEBBH5U4?XyeXcsL_%hgTS0 z`q?+>l5CR}*xe_A6!)*Dq=f?a&ooOTXMn0`(~pKENrxPY2zfeaUBRES-VA@zguH`l zv+_H7dzN$RyL_WG{bIk)UZXBQ318ZSneT#+FDXcDC`>oIz{4hg2Z|Upno5r|2D)bN zK@rB{Hsjz2ky$5f2mXA;d+Sz*gtT<>jga$`Tw^86gF^8-9T8QC_g)v>Q13zS0koh1 zC-7GK?>hU>V9g#XVQXAwoJEh^HlM++ElZA$jv|x9tyLD@d8KdK1ZSh2A&CAz6Hbi} z-CNHHt;_^k3J3_S32`>U@uS7z&g(sIp^){Dp_h$TgHcgU zIs$r!Qmz&r=CJi7k;w+T7z(@O+%nF4=6&e3W=5p6OG)a<>`OWo8LHua!$Rf_e10iY zUmccbO49>Gw^RLByAG?rx(&sFF9nJkfbxsvT)(S)KT0~aHLRnAT}s7N`~^_yF)DTc z3I>v*z+Hpu%R7BQzmnQA+AgtL6Jk%zJ@F~1`PLdC<@eJ&HE2d?*^do1F4cstZYIIZ zt@cRrH=sQySKo^<`S_zF2)OSJ&gFSLCrz*TE-vtlO|01R&zqNuZ{tfbYsM9s!-{dN z^-b2Z!Gn81p?-y&X`#v!(IlNYR+!n3DIan=+$W-Qiq^h3sPCq&_6@v!jcLlr^({FRjDgaG4>(>3#VFungj6;q$)n?k}mkc>mPt(!u zbJDW7hc|z=F>BLoX|r{ZN{Aj6PzEMlmo9-vsz(ERiGhIDjRs5X-DX=(ZVkO;A!weD z6FMZS&qf}3j4{|PzAwqb;UvT#4$jr%7GzC8`MMjJpU(9GK~TN?3_M=3wLx|*PJ|yp z@)2-iOno%yw(-4f+0U;Lc|VJ+S7MG4qh~pM(!*kOwhF-a?jt3zeN7Mep?6ChxLy?3 z4xh*7u2))b{9K5hK9Avblhp^=c|+7H5=rbN+282y&CpUTf+&J$F-tZ$u^i$hKs;T2 zbmItU3$JKoY@b)>jyroOqMpAjY2V1nB<$yGGN#(Xbp$|rq?A*ESxq|=r{56}0MgD= z5EuxtZVW@|`UK<_0xdlM@1plOK;B&LBpG4YP8bDUysIe@0|`^XcUoZy8V&|^zi=M-s4A;sOX$*T1;1rsBG${FT<*_FLG(*32Fi^tP=L&u-1+kM0cR5J&DC$3AbA zb$lE)q*f7n@Oq9=qnb1$?fgMxjHvAjI6)EtE{+2uGqSRxAJ*?`uKU-@jlMp)MR+}V ziKw_JN-E0A$cIGhw0|fo9btMbVySq@hvm>oFJvy?v7=n4-c$n@Y4_an3KT-Gg%4>E z8MaK9^X`kkzSN&ylc1CMBWcfL?bc4&*47s0bkvsC7Ta7jy*3jNhp`(c3l6?$5k z=4VNUNo+k>&j$$>Cuidpr}Rd0H23Zxzhsm3#TEg~4KXQwn5*At^ORr0g=p#yC>zM$ z)$w3r6e-ktQ^>6RYYEGOb23q3czc^e#&$sWn@Ko-2Tq zA$-e&bQ?!(L2Om~3^>M6`*HLl4$^$MY2nJE78SH){#|tdF{7f)C;L+62s0BVpcOw6 zKQ7hgqcjE9)ZtP8YWI$F|;t2j#oW$&Oq3^0AN#jT~vUssBB; z0dO6)y)qH;Ki3ccPk|D3%KztR?EiieJ~W3?V6Qy6cX)Q9^ZE-_0OwKrHUQ?!_y%a3 zFn}ndwF2E%mZhc@7}!KXXCYeC6wt87!*{c#$xMQQ6+uczQx%IF9f#3(C}}=ZMKCl_ zeWVD%n|fmt5)?cRU$Ve&U*>&b?F?X^@2(X5@*ZUy=ZzW6Qp?1Xg5iP2_UZjVQ??}x zOX5qv2!KV({|wC_<+oRCX9uRCd7nQIOf>~eE`~jf>HXk7vNP#25&cDP<2G@)Y^VIkA9cy#;qZ$rlS`T)ceFnke z&wgAZ*ZL^}(vVOQqle|5J@LyVOtLfqpI1^MW!e71@o1Egqm^n;AHtFr)WW!UPpZgGLP%*k2QOYL6z{t2lUxYidIpQj^?iUYjo_aX z{qRV^cT>e9X*poT=6lNBz$rj31^q4=n|w&wPikm5EuNe$sPr!m<;F-1MlSsqh}L&O z#_i{4N*N)P6Jr6S?8juw18qAy(i>BR3JevMw(j1qXi1e5t;56(-kuAXaGYtvnIA69 zcU4~)bSNq=mIOpa@BX6z7}Xv8N|fQVRrItnlO7dg1~f*OJiid9eJB)51)`cqJ8aaB zmd(a8#iYjB$-^H`tN$6~)m<;D0-Enp!p55wl*u?-{8(wlZsGb|p0pY^6EdR|C(RW^ z%GjIy`s`x%?swF0qF6Vzu>f+hp7lZ~(@r@*Tw7{5M=P9DZEQru6}YG0^$p|AZU;3{ z$XkD0?#)D;2i;id?gzR!q`<#Z6+|g*yVaz|oA-s3B2xIfEptSl{>Ln{4<2$$f4-s+ zL=Xn;d5|CZcb_kXbM58>@+mKmBy%*-k_T`BpgR1#@G@&xyMGv?3LKWd{uQ2q>zmZ0 zZp$_U%Bui4aK0Th#H0&v0S(SNBDQK+F_$VX+F}+pe`Yg56?&DoMNi8ueSLp2?q{<~N}iH0>+K6G|4+q()-t?5Ww z0>uGfu%lK-X?6RJ?1xYg=Imwp3n6blX-o3|jbT zfKhMahF>F=qu)%;D$47R$$C!*4~{r+dIaUBZ1IGdUvmKRRTg>I=D;>V5trU9^|7h- z&1r^8)W^JMBn|#Jgv(7uHnkZ~Gt7WZG)aucV`VeJ%D*EW>zZgm_kSVYfv&{{kX!TRvA1pP#c1X0u5<26+ zC60q8_xoB9N-C%UToLdD$&T*70?eNenC#`;dDhxV_5!lq= zE)LqFDs>iQCH!*$yZLBnZI)=64-lCM3uZt^x1CE0cus#;6kgr?f;1Si(>O%559CIz zf7RF7so|L&*J)Q-qJpvM4ggkVMXOWmd+Xg?Ztqj_0LA0UW59NVRSM{W%{8ZEQ1}T{{VdS^2f@WOxT$~Ql_ET!Tq)Km$ zN6#}evjm9D&fWD(Yo_cMQsjnd_ZsZ@UK>V5Ex*$@SyB_1t85?O3X=bMvD8=@fU%v- zY9^;u(M&)=%-YA6io;Ccs^ZG35if4kQ}Xq59qT@kN6-5b9?df|jylGkSGwYq9@G6h z$3ibqE$DS-17HLJHxj|rvsZZo=9n)4SEc52eD_Z^ zzZ1~CL-I*RaiHU@M*9qC!8V3BRhX=I5X~2hrB4F)s~OEd_xOXSRndDZQM-rE?*KT1 zdr!+Au7UuN127v)JXls;fLh?fSJp5Nz5M`uN=Ii4^H=BQIH2}!2eJSK2lgIH5TUNF zP92f%LWDfE`@ZbD0w@bUA3z}?C~wq2mqXW$ zD7&nV57fdN$@8CIfG~MxLKUu1wF1V1LT_83oyCIL49K8=l+PK_V=8f7W?hvVH)s{k z>{3-ri|}ws)tyNB9I;B64xb@EBQux^dY7dDLH#Im@~C45P^{6{)qMi!!ZAQKtrtKN z7(g)HTjekY03stTZS4q5oS;#rY14~e-^v;P%GFc9QQTh7x@G?#-)S*5HFfOhp7jGC zUOyI4T#N^Y!a*o3q8Gyq3d;ac7TUETkWwEm`0k)Kod39eK9TaAy|Ap9ZONW0q5;0H z;<#r02MX(1q@c!mnab>I9vXtGs_BjI!XQVNppPH_)D_FFO;Qeg_^ZLrKG$AF{VI$z zAO$x#W2c$o9IsnoMw4wAa|7bib?~A!y&Y_xMcKK1fU5IQ-=w%0AW+483D4Pw^L@~L@K5TQ%5qoBkcfTgBV@d z8$1Db-em8ucP&ZgnD@H6MW10%5iF78Tzr9AdnM(4W>lAibruW97F-z$94=vZ z6Av+cIG@RI>(~jdVM;lkbEe_GW`G6H3&0S|rJdeOv{(6p-o)BhM>SVR!Oz;66iK}> zf#|S-o3FX?d<>IHq3v?;SK-6N(QtROi`lk9w66j^KQZUp1NBX{^;>p6jE~D`dr=X&N!8)Q|&Rw z-nmQl&ZSP#*HgWpo_pyQfT?22_wS@RiC&!@ki(wjZiBN{SwO2klxF<#}@Q|-ZX zGoR0{Iv(poI9Z%)4=jKc9lY+;sV%!nd+25#pyA^XegiCJ{^<8tXYX~N@jjOa96P#) M542TEz)xQPFTow~N&o-= literal 0 HcmV?d00001 diff --git a/docs/apache-airflow-providers-http/img/connection_headers.png b/docs/apache-airflow-providers-http/img/connection_headers.png new file mode 100644 index 0000000000000000000000000000000000000000..413e9bbb38864faf0dd5664703b9089ff0b7bd5e GIT binary patch literal 5256 zcmb7Ic{r5o`yXfn{-V})%5hC16>~W0*`i5pPc5~&!_Z*&@TQfZG(MG_BoO+`NXB8 z9cLtFwPB4YYQ)t9B8V#!9>!GTtzxn9z61NxOR26(i5#r%Yd8mX zpspJ8#$8$syuMdWkp$lPLfgu|3wA>n2RozP0kP;M(D-3DJV7{eSD?`7DFLe$svtVK zQXrI18VCt+$j?P)C&oNc2yK;mP)&FT>;wtK#y-+W5L@X#Lh$T(nAMSE;3&rnyAcEC z=B`9Y)31Gasqo@r&2Lc$Ksrplcbu4A)+;|xiiy6^5E1hE2n=@Smv23B#hBxd`M!T# z;oFxEnNSws+K+=nu|$kS1<~tp{^0_Dt>)tIEzT4=aOBWXNS{k_ety2;UKs4bA;@|O zqI_48U5@qP!-su9fX5X;*!ZA*_1&YXJa?|LQ*sBZ*nIV=EK&jnJF5asx!*D8 zqD|Tiyz1lgM;Zu9wrlLxa<^2*#&T(F&im)*@7WOyvr6k48XC^!Q6Dt=gTuhh;JSW! z(nzC#D9CHZ`xnj`gs`zBwwg({QRK?w@WqL7$}3N5ZJ|8t-809c6d6RJKWPi@wu&2x zcEr;P^YfK*^h2JGK+AEsXA3O%nX@t9lWTgD%@8mrgimB`Rxh%jl#)356_u5Rr}Hff z?kZUoZp_tbcLuA?4A$sUGj)_w4(KQjf$QcHa|a|64;?m;j&!dcTY@2ScBl@KMh*uy zaWRImoXw@-$oX-~>Tsk#A-+3jdAhG?1YE7D&CiQIA7B5B6iA|ZE=_j%Y;(4J-riS3 zzCK?Y?8CVhzotq4@DLf){qv5U(jL%R7dBit5tFhO#p1X{FQ4_;Y!)_YAF(n-l;GOZ z54)J*k0Tn12Z%>5n?=l&Q35k1hNBj#dRTh-(qK?$L7sVH;mw`Gp&QXVV0s4J)6kcP zj*}~sP)CV~x22ddX`Y|i<5PF7mTn3IloxEAW*u(6*i%(x&7R-&?tm0 zc^u>|u8Vn;W}COTY;i)gElS_{pk}(GerhFG)QC&e}LIYN^nK_x>Q`K6N@g{EVKwmRXK^9NZ>d z^HiGgx_1W4yW}9TxSH^Ygbw?&xJcU6R>z?bT%}2jLma$IRh2MOqZ71LnRK|p9M{JW zTX5+0%A?Yc`38ou2VZrDA~|&5x<%F%n~Y%OursJ-)Y7YY=%p)3vUlBA^97XL;cpAk zc(&GQ1>T3=j!M_wHJF$#QGbgZ7$D)Tv;p)1kgr^3(sXt&WWY)R!f?yFA<>ogRGAmHtO$}!F+-icSz0A;Zz(U~Nh3w5b9 z3lT|Ew2h5Tj9;7%s{yr*ZvrbxJI7E~#ci6CBa`P!UKp@s$;&)2y=T@5aP7IlEEP<4 zreFPNg5DMOrzkrGj>X()qMUWUWvZ=t;$AzPR&9RSvd^vQ8s=PG_iG+RVKD-8s+6=s zzmiAYm~Y6K7}Pf)>Uu6cDGy*Iv`Va0_-eZ_ZL}NiU+`>yQz}2aF^XKA7*0>vD|g>= zuY7vCrXSkPZG1xNMx#Rkt#I&Jg1qB310fI?_5Jkvn%G#*Mv1}xf(B5A4fY?lCT)^C zdw#zuKfx-nN&m!Jo+h5uHr$Sq!ygOsWGu_YtmPRtF06bv)$djH*vG+t4s@1wynw3; z`r)r5k<@I;;@T_c%1o&w&uSa?yVcpw5I1jH)u%~E9}UlqCF_C`zBH3n#&`N(LNLH>GgW*=?Jnd{y>_P zUfD%8If`a>6G9aex(h3Jp>gxKzSQKtBwCRGR=-mqnNfB|J#{3kS$nbjDHV@67Fq+# zja|k;FwrEzSQyLcVLu5#xa`IbM@&=ezkm89GeA7gw*9fxu|U*S-wj{Fl$e4&sAe}? zZStrst<#j1$2sQs$wcv3${l=1sl23{r6_%7FueHgPYRywmNI3VeazLgAor=W%-M@P zlk!}kaC+Pf?;d_8?x;dKvSG1fAUq@K(FodiIy$8+^RpO*J}|I&C(}v#9Viw{gB(Uhx^wD6z@c9;^~di z&VD}}jHPJVTg2isr^Fht`-aMn3>l99h`|eZbP$a2a$z4io4Br9$D6Vb(rxR?sorr0 zH+KrQr^I_ik&~_1YaZKG=&QIq^vpp?il5sNlhvLi&la6#)zG`it9-e!Uy#s)Jy=j_ z2ZG$PopwtK{sOuix%Gns#yG3_Umw^8p;7u_3zvif#+)?oCd}&7rOnI&1Iny#t z**6xC8=!a3^uJ;w%f@ln7NbX-;tk)sc#DV5MyL(3o2Lb(j~^10hwCAfje{riwU{uP z$(M~9@4>=8QWXD#YHJhVSS;43DQ=g~pS?CVVbh)vDwz3D1|h2@v9E2L%3@G@Zt&~~ zyt)2B!?Ez}vvUtHH7Yh4T4>VxyV7n5?8w#n-BUqZN3PDHK}WXhDZ*$AyA>jg&HX`n zM3fX1D4V1nnqKa6)6U_6E%X2va#y}WE5mQ0Ro_owJ5&8XmR1q6bv!H~oE&@E5Oky@SUzc)d}XKWJyRt_1;{R^`9_2p4PRVDHZO5d0FAFS3fT5Q*y zTI@98!ggX5`xK)dG)jX`_o)t9+~8e)p@W$Td!@Rap2@EK9{-p{d}~K?0G35(ogWJUxF=^gdUdFEF?O|X zWW2NbTK{jGS&Udp80iSWarKc(u-BNZukco-=@vl0Op55O@~SHEh|+X=c_@u?Cj%s> zD0XYwEr&=a1B^VJ=_B5m)UPb+-V#{67M-%7%$u?R0Pv@e9(^S)5DJ{?!I)4lzp2Cy zu*3MYMAKV>(qs+N)5sO%V6kFroqLcnHVR?^75TlzNjGI{9AH z?_c<|XX>QL%G^HTMM;&N>MeFa=B|(3)@Fa`%C`!7$p@M(mq&mM*ZfOB3F(dIfsgc;t6Edc<_QlVP9D>>+G z9(9a-`*sm8%$|EAAi~rBugTHw_wUnX__s%eV}BraTQF%V>uvWltjO2WGvA3dxIB`# z4G-^45R?i0p&MVq)CF}e*~W@6R$_#(sKai!0^@_q|>wch2;jFQ|)-Vb2mh+Ain#B!$aSK zgM%NxfB(MRZ>W|n^4EvyQ{Lj0enZ)Fq-_ox5-+~6@E)2cz{)=Tzg_tc)BZ1Z3{-_s zfV<`zR)@ctG;t4Od@A2E#Twi_4PhQ2LU4~K7~b^s?6YoAsJp>6Y=by@>0cbZva(Vy zbml|wSW6P**3uV71{~N}nJuoUsQ5;m_%9^@2pIJXO?#mJ`&B6TzIZWPx5g}3+T|0^ZBmz$t5%Ur1?3w22x^7u=(PV>RJdI`5- z^5^Wpb43lPll8hFUv~BI-zdBe2yHOv7u@*7&++0#R;vR_BVT|)XF5ik6MH~$bg;6r z;<`C&{WBWUsQwm}f+pItHnl}m?H-;4%917|w!CQhqGSC%M#99CRVu=2-ZvfdRh#nV zN?ooPuB$*F`~PZaU`p;UwSw5cL^-vBGu!X4KoNq$wxIO?Z{>})&HIxWyH+xol?#nJ NWnqUZF}w83zX2P`_&Wdq literal 0 HcmV?d00001 diff --git a/docs/apache-airflow-providers-http/img/connection_username_password.png b/docs/apache-airflow-providers-http/img/connection_username_password.png new file mode 100644 index 0000000000000000000000000000000000000000..6e36e77dd4cb48f62107a3f654648587bddc58e7 GIT binary patch literal 4761 zcmchbcU03^o5!P|;P@gUqabyRBPa|Y0trZS6j4z?q$vUcqA-y*h?E3EU@S;qMnF0u z(g{VRlYmN5T9iFsdW&WcE+Gf+@j6Uopb@@XX=e3HCon2biA)vxaT*S!RJ3xgOLI`DzV z&2xKseS&`%p+IgvaCdjl&(1!j0f*;S(Sz>pj$4ZtC$~^wE!F+d(mk)c>gXq@hPy(@Cb59eAf4h1K$Ny zvvJ3q1Oy^s9O<$3lCe(nuy`l_}xUI_%)8AUWqzr*n+Y^pZitg6I zLeC6Vdf;g*nKB{9kSgh{rD%npDisdYg zw&r?`wq!pR5I#1!Ia65DJ<=ROmi8NpeRfDXd7>*VMgQd1EP~tCFV)(LekLrNK0(UJ z;64kL!JLjdt|rG;K@e?IRf<=N>*m|~;XX4xJwrpbb9|6jLlSbOdxrb-<@r)Hl|Lku zyO1VGn%Ip;hs?Xuv|qAGiK-`cS1CxX8)2p6e}u}Kw7*m#Z9=8Okk%#kX?ol(W)Ij@ zlH{JYR#{wXOxW4e(fJ+3JGGBc$+`i)#D>a>it$Cci%E-AKl-?<{o8xj(k&%2#4@|$ z&Y0hFb#>LdvNb;%qi-L$ooANSouuZnytG6=vz=grQ-eUp&667^+Ts;a&Eeyy+mplP z2)Z6>J#QV(@yaZ;E-`w%|Inw_`o#~ckGMri>-DNP@j*W-?5{1u(y{EEOQDCd z%aDe-g91V~+FvTQx9nnX%l#QJ+Y0W?mTJXJoTg+7bT^2j%$0sEm}s`W-f+_M>G&A(XNP6 zho85#oVVSEs>>*1(3=JGdmuMWvJbe)B$MM})-O$7P?K?$P{UZzf@bkm%iDH!_FaQn zhEX?C;GRb0924Wmp0aTPh@kax+&-9Ua5Tg5Qs|rG5W{YvgBwrWhnDlebNt z9hk5$ax&VOrDic^M?xu%yB!#9bM%BUKTL%7=;$PNdwnv3k%%U1^n}gMY*br3(fIgG zsC`o*#JFQM-+%GF069VFZaBj@1(vPlJu#LcZ<4x(=s#+iw1*#Zp(Y+$s)gOKpy^Mi zhh1JEh{`|oY+<3$n<1MnKkys2Z9Ux)@ z%BUKpVONr&Z|;;k8*i)*dNQ|f3*??JoyK&EP}~>~JsA>|8_DWt1a3eeJ-N5|VI}jl zGMR+Q*(I5YLM_6Y15OHtRX9Q~gnh||_Pq@YxK;hNF=yGk z|BgOMZ*$IA0Jii?4~WJH?1^JPu2w2Qf;Vo4H_G)Dd=BXh=iVR}^TRHM+Dgw^$;8e` zP$DG*Rzz~=_oft=tOs#9Y;VtPoe-f5S=C96a!oeMBG2Z&RVeC*f6spIFLmbFo?@NX z-$(3iy%C@2DEf0N_~e%_5<`V$yEaGhHzJNH86j{SAA4;j8F&G|99)9ZRFv$`dcdBi zu9uI*4r}Vk6o&h_d#IO&T@D>(j-DTBFx|o~vD0mqTH;;PV4m3X-3Q_Y2ELXyjh<)exI zAI=l-``$WPgJko zCOa?C2+evBNE_aoFXhY&d7nehVw^muQmF~`&aPISZY>J6*5|qedh&Tac5AJy9(QH( zm0IPMsCqSB_E1ax)UWj`*Btm3XMM|gi9PO4$$nVHgXxSCWr!5{^Jx6wl2xuh-mjxl z>sJSySG1r52UEhrsa5?&_IB9Q zkOnAEI3>)e{ID|3v6d+A3&aAo0YTqdcwwrOj>!D3vE=2>L>6#hcMp(G$SV=x=see* zezA@FgKI(7{j0FEvooJSAk=~-`Y5$jtM)OXL5uDODAYAE75gF&V31wiqv=~Re=1=G z$f=$(BhUK2el;TiLuQO3QAjt*f0xSq=BS^D*ykxWI+KMvgkV9i;VDWRx9}jNfL%DaGjA|W{vQ$NpS3m4=I-6QJZ}L}~wP_n}m)v?XSj1Nx_PDiK^uj29eT?=~Mn-f>J)|u$Y0Q?t#MSwG^ghl$AI*YQh zPGZ4#`&YAys;bE{_ed{f(V#TgRDd`-wlxn=(T4zRi8N6v&%LPE2D@vdn9hvyJOK)0 zo5bNQwNxHk*?5Vn)$syP6ln|k`s~Ko5m3|5!!&g$uh4iW@cxxT)+;#8UzfUEN6*nH zv?yuiGB};hJ>V9!_DASU zA8w_fYPQUpz`4E(xJHivGHtDklg(P_KGgdyG^%6DtOoA5MV+o&AQo=N*DjlLvwkSu zGld5EG;sN|JgWQ6HPoknI{`GAsz0FpaO|ptUZB5(wy)cH05C!;>Dbi)^kQzahB0rOS2)^^!pMBGqd>KLLc`E3qWiy%&Z<__TUz@VEwbOwXrxKJ5u3YvNnYXhWJ7bY-C*PQ~_Dzw@4q zcEWY^tbhD%ABytj1gg=ULuR4O$zh`M%fx*LCniZXf$I-xzzlPdW0oDL2W@J&Z{NOk z$17TQhaYX%CTXaz%nr&Mr=BHx4{sWI&DHu?bJMNu(Wi{(rNl`=i|iWidjZ+8*_hcn zTwe!ktI{1fRmN8>#Q2FIXJe)P=pUDckPlW`Vq_64XHt3^j*Y=&;I`3a(iH&Xk_p(& zRZbq87O-IFSJJoAH)Dyoi{Du~p#oD;jJFbo1WPg__Q4!Y$a6zDX6NJSxqW8aS7EDRj9^xD&NQP6Uc9bgIU$EzSuy7XkUDeWJ73?Nt>GqqxAGePKGm z5R9MD)q&%jw?d1gs+`u}vRGu(?95C154u4x6d?7k=NC@JpLJXuexIL}mF2ABTxfv- z*xT6+5O2abH*6Z%;@j@Ch9He$Y&03*q@cESMj*l(~?m8SC=FfD5o|Ix9xc zsD!xj0DRn^TkDCQ2pXG$oMrmnPw4oHL(KIfaA_uK=i|9Tm-f@spV3u+%1~@-kPTXM z-1b%NI^M9!GB+J35wTwy-(h+ShP3?`9cn=y1(?>As)tqJN~zsf7_rHMBRAgFb9$Z$ zp|w9G&miaP*bk;e`$`ZX9ZflcFmH-%&o|G_5jko22F%Ej4`_{%HO;ybVCW39*?lDqn1(u<{qD}X_YW2lcjDZtPyo^xHQn2o zV|6)CWJLAN{p;M+`8lb3EggwO?8X=;Mh~+xz~*eSC$A*D7JJ_@a`|AyqMX6$TUst1 z-FH@|vZjY>ywQlQSk){!=dO3DdTBA}DsKmgj0#@{qC13d(_6)PxV|>P53KaFMIeum z#4>N@@x$y~{zh2+mOga1(hc=BBI)8+aCBm&f};?~S92I1gGFyw-@CR!mgOcs{v9mw zSgXNV(bgh+f`Wij{0agP0++y&v?a~&`%-~MAxp`hbPV!2D#7{mX21p%0%^%#S95Ea&2 zqmQilODU|X*w>#nCFptQWHS?)&D*z2t6lTnPS;E>`|9+mBR5z>5m-R2$vMVp$@58? zC)nXvUdvIDk7z0aZ4iF6t&|#3ZEs`7Mj;%!@ot)dR zs}K|U;waI$QPms)nQDDfWZ3Y19^^!(0-7FonL^qnn`Ig1NNf9!4JdmBb|h<|m+$G} z>NBCyG2cr6D5(*fov^^Dmgl~HEB>&fF>K${(N2r99r$UmauCfl({Zv~mmEp-e^j_J z1lg+J(TCUF{4mwW6U5E=&e{GZ$=7-u7Wv&+Y%^LZMLr`C+K}Y*^~2NV!phd@<90tm zrJ#vn@XMEf6^%GjRyfl%wR+Q*2 zIy+?1^PStpdc&<^FC^2Ikm9U=TSukp{J%jB&r7>^DgT*=YK7eXx}-UiZLwfo1*9JQ2&Vsd b1GBE*^^2vH0QXn|ulCwy6NBQ54!`{!fAs15 literal 0 HcmV?d00001 From dd38374664acca911e6782a6a699d6f0a0d091b6 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 08:18:23 +0100 Subject: [PATCH 273/286] feat: Implement auth_type and auth_kwargs in the AsyncHttpHook --- airflow/utils/json.py | 10 ---------- providers/http/tests/unit/http/hooks/test_http.py | 7 ++++--- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/airflow/utils/json.py b/airflow/utils/json.py index eb3cd40941197..a8846282899f3 100644 --- a/airflow/utils/json.py +++ b/airflow/utils/json.py @@ -123,15 +123,5 @@ def orm_object_hook(dct: dict) -> object: return deserialize(dct, False) -def none_safe_loads(obj: str | bytes | bytearray | None, default: Any = None) -> Any: - """Safely loads JSON. - - Returns None by default if the given object is None. - """ - if obj is not None: - return json.loads(obj) - return default - - # backwards compatibility AirflowJsonEncoder = WebEncoder diff --git a/providers/http/tests/unit/http/hooks/test_http.py b/providers/http/tests/unit/http/hooks/test_http.py index fe6b8f882f2b7..1e944dda1d090 100644 --- a/providers/http/tests/unit/http/hooks/test_http.py +++ b/providers/http/tests/unit/http/hooks/test_http.py @@ -384,9 +384,10 @@ def test_available_connection_auth_types(self): auth_types = get_auth_types() assert auth_types == frozenset( { - "request.auth.HTTPBasicAuth", - "request.auth.HTTPProxyAuth", - "request.auth.HTTPDigestAuth", + "requests.auth.HTTPBasicAuth", + "requests.auth.HTTPProxyAuth", + "requests.auth.HTTPDigestAuth", + "aiohttp.BasicAuth", "tests.providers.http.hooks.test_http.CustomAuthBase", } ) From 2c7e2e256519ab20e1020cbbe576a3f15599c73a Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 19:04:06 +0100 Subject: [PATCH 274/286] feat: Add tests --- .../http/tests/unit/http/hooks/test_http.py | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/providers/http/tests/unit/http/hooks/test_http.py b/providers/http/tests/unit/http/hooks/test_http.py index 1e944dda1d090..b6f43995b979d 100644 --- a/providers/http/tests/unit/http/hooks/test_http.py +++ b/providers/http/tests/unit/http/hooks/test_http.py @@ -437,6 +437,32 @@ def test_connection_with_extra_auth_type_and_no_credentials(self, auth, mock_get HttpHook().get_conn({}) auth.assert_called_once() + @mock.patch("airflow.providers.http.hooks.http.HttpHook.get_connection") + @mock.patch("tests.providers.http.hooks.test_http.CustomAuthBase.__init__") + def test_connection_with_string_headers_and_auth_kwargs(self, auth, mock_get_connection): + """When passed via the UI, the 'headers' and 'auth_kwargs' fields' data is + saved as string. + """ + auth.return_value = None + conn = Connection( + conn_id="http_default", + conn_type="http", + login="username", + password="pass", + extra=r""" + {"auth_kwargs": "{\r\n \"endpoint\": \"http://localhost\"\r\n}", + "headers": "{\r\n \"some\": \"headers\"\r\n}"} + """, + ) + mock_get_connection.return_value = conn + + hook = HttpHook(auth_type=CustomAuthBase) + session = hook.get_conn({}) + + auth.assert_called_once_with("username", "pass", endpoint="http://localhost") + assert "auth_kwargs" not in session.headers + assert "some" in session.headers + @pytest.mark.parametrize("method", ["GET", "POST"]) def test_json_request(self, method, requests_mock): obj1 = {"a": 1, "b": "abc", "c": [1, 2, {"d": 10}]} @@ -724,7 +750,7 @@ async def test_async_post_request_with_error_code(self): async def test_async_request_uses_connection_extra(self): """Test api call asynchronously with a connection that has extra field.""" - connection_extra = {"bearer": "test"} + connection_extra = {"bearer": "test", "some": "header"} with aioresponses() as m: m.post( From 5d37ee8e60bff519394875c338306e294cc204e3 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 21:04:22 +0100 Subject: [PATCH 275/286] fix: Add header and auth into FakeSession test object --- providers/http/tests/unit/http/sensors/test_http.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/providers/http/tests/unit/http/sensors/test_http.py b/providers/http/tests/unit/http/sensors/test_http.py index 78a11e15bb7c1..47af8f49c48cf 100644 --- a/providers/http/tests/unit/http/sensors/test_http.py +++ b/providers/http/tests/unit/http/sensors/test_http.py @@ -238,10 +238,14 @@ def resp_check(_): class FakeSession: + """Mock requests.Session object.""" + def __init__(self): self.response = requests.Response() self.response.status_code = 200 self.response._content = "apache/airflow".encode("ascii", "ignore") + self.headers = {} + self.auth = None def send(self, *args, **kwargs): return self.response From f66d52bedf150733468faec538661b407400cd12 Mon Sep 17 00:00:00 2001 From: Joffrey Bienvenu Date: Tue, 9 Jan 2024 21:14:54 +0100 Subject: [PATCH 276/286] fix: Use default BasicAuth in LivyAsyncHook --- docs/spelling_wordlist.txt | 1 + .../livy/src/airflow/providers/apache/livy/hooks/livy.py | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 2f7a408bb1a47..7be1e1f9f6db4 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -25,6 +25,7 @@ afterall AgentKey aio aiobotocore +aiohttp AioSession aiplatform Airbnb diff --git a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py index f185ccdfbe33b..e9e1a902ae610 100644 --- a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py @@ -93,9 +93,8 @@ def __init__( auth_type: Any | None = None, endpoint_prefix: str | None = None, ) -> None: - super().__init__() + super().__init__(http_conn_id=livy_conn_id) self.method = "POST" - self.http_conn_id = livy_conn_id self.extra_headers = extra_headers or {} self.extra_options = extra_options or {} self.endpoint_prefix = sanitize_endpoint_prefix(endpoint_prefix) @@ -510,9 +509,9 @@ def __init__( extra_headers: dict[str, Any] | None = None, endpoint_prefix: str | None = None, ) -> None: - super().__init__() + super().__init__(http_conn_id=livy_conn_id) self.method = "POST" - self.http_conn_id = livy_conn_id + self.auth_type = self.default_auth_type self.extra_headers = extra_headers or {} self.extra_options = extra_options or {} self.endpoint_prefix = sanitize_endpoint_prefix(endpoint_prefix) From b53c1ab27cec2f0995dbbad5bdbb4c8940fd0bec Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 16:36:22 +0200 Subject: [PATCH 277/286] refactor: Removed docstring for removed json parameter in run method of HttpAsyncHook --- providers/http/src/airflow/providers/http/hooks/http.py | 1 - 1 file changed, 1 deletion(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 9cf5d4983dbc5..932aeb31d29c0 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -397,7 +397,6 @@ async def run( :param endpoint: Endpoint to be called, i.e. ``resource/v1/query?``. :param data: Payload to be uploaded or request parameters. - :param json: Payload to be uploaded as JSON. :param headers: Additional headers to be passed through as a dict. :param extra_options: Additional kwargs to pass when creating a request. For example, ``run(json=obj)`` is passed as From 83129cb544c26d8612f7499716e74ae295300f85 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 16:38:39 +0200 Subject: [PATCH 278/286] refactor: Aligned HttpTrigger with version from main branch --- .../http/src/airflow/providers/http/triggers/http.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/providers/http/src/airflow/providers/http/triggers/http.py b/providers/http/src/airflow/providers/http/triggers/http.py index d25d3a55cfb5b..ec9780bdeab49 100644 --- a/providers/http/src/airflow/providers/http/triggers/http.py +++ b/providers/http/src/airflow/providers/http/triggers/http.py @@ -73,7 +73,7 @@ def __init__( self.extra_options = extra_options def serialize(self) -> tuple[str, dict[str, Any]]: - """Serialize HttpTrigger arguments and classpath.""" + """Serializes HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpTrigger", { @@ -88,7 +88,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Make a series of asynchronous http calls via a http hook.""" + """Makes a series of asynchronous http calls via an http hook.""" hook = HttpAsyncHook( method=self.method, http_conn_id=self.http_conn_id, @@ -165,7 +165,7 @@ def __init__( self.poke_interval = poke_interval def serialize(self) -> tuple[str, dict[str, Any]]: - """Serialize HttpTrigger arguments and classpath.""" + """Serializes HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpSensorTrigger", { @@ -180,7 +180,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Make a series of asynchronous http calls via an http hook.""" + """Makes a series of asynchronous http calls via an http hook.""" hook = self._get_async_hook() while True: try: @@ -193,7 +193,6 @@ async def run(self) -> AsyncIterator[TriggerEvent]: extra_options=self.extra_options, ) yield TriggerEvent(True) - return except AirflowException as exc: if str(exc).startswith("404"): await asyncio.sleep(self.poke_interval) From 203165ed03fc7194879797b22f1ae32e4ccff487 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 9 Apr 2024 17:03:29 +0200 Subject: [PATCH 279/286] refactor: Changed docstrings in HttpTrigger to imperative mode --- providers/http/src/airflow/providers/http/triggers/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/providers/http/src/airflow/providers/http/triggers/http.py b/providers/http/src/airflow/providers/http/triggers/http.py index ec9780bdeab49..5975389830f36 100644 --- a/providers/http/src/airflow/providers/http/triggers/http.py +++ b/providers/http/src/airflow/providers/http/triggers/http.py @@ -73,7 +73,7 @@ def __init__( self.extra_options = extra_options def serialize(self) -> tuple[str, dict[str, Any]]: - """Serializes HttpTrigger arguments and classpath.""" + """Serialize HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpTrigger", { @@ -88,7 +88,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Makes a series of asynchronous http calls via an http hook.""" + """Make a series of asynchronous http calls via a http hook.""" hook = HttpAsyncHook( method=self.method, http_conn_id=self.http_conn_id, From a2c82c1a258b45825ef8f3a4400bd86e6d2da062 Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 11 Apr 2024 12:15:06 +0200 Subject: [PATCH 280/286] refactor: Updated docstrings of serialize and run method of HttpTrigger --- providers/http/src/airflow/providers/http/triggers/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/providers/http/src/airflow/providers/http/triggers/http.py b/providers/http/src/airflow/providers/http/triggers/http.py index 5975389830f36..d30f41990f5b0 100644 --- a/providers/http/src/airflow/providers/http/triggers/http.py +++ b/providers/http/src/airflow/providers/http/triggers/http.py @@ -165,7 +165,7 @@ def __init__( self.poke_interval = poke_interval def serialize(self) -> tuple[str, dict[str, Any]]: - """Serializes HttpTrigger arguments and classpath.""" + """Serialize HttpTrigger arguments and classpath.""" return ( "airflow.providers.http.triggers.http.HttpSensorTrigger", { @@ -180,7 +180,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: ) async def run(self) -> AsyncIterator[TriggerEvent]: - """Makes a series of asynchronous http calls via an http hook.""" + """Make a series of asynchronous http calls via a http hook.""" hook = self._get_async_hook() while True: try: From 0d99f07534122e72848936e1c2cba794cfea9d6e Mon Sep 17 00:00:00 2001 From: David Blain Date: Mon, 6 May 2024 13:09:59 +0200 Subject: [PATCH 281/286] refactor: Moved get_connection_form_widgets method from HttpHook to HttpHookMixin --- .../livy/src/airflow/providers/apache/livy/hooks/livy.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py index e9e1a902ae610..4f1e3934890bd 100644 --- a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py @@ -85,6 +85,10 @@ class LivyHook(HttpHook): conn_type = "livy" hook_name = "Apache Livy" + @classmethod + def get_connection_form_widgets(cls) -> dict[str, Any]: + return super().get_connection_form_widgets() + def __init__( self, livy_conn_id: str = default_conn_name, From fb876b59246738476e818644db8e3e17bc683a01 Mon Sep 17 00:00:00 2001 From: David Blain Date: Tue, 7 May 2024 19:34:37 +0200 Subject: [PATCH 282/286] refactor: Enhanced extra_dejson property to allow load string escaped nested json structures --- airflow/models/connection.py | 2 +- .../livy/src/airflow/providers/apache/livy/hooks/livy.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/airflow/models/connection.py b/airflow/models/connection.py index a8b9bb87985d8..85d3c7cb578c7 100644 --- a/airflow/models/connection.py +++ b/airflow/models/connection.py @@ -432,7 +432,7 @@ def get_extra_dejson(self, nested: bool = False) -> dict: self.log.exception("Failed parsing the json for conn_id %s", self.conn_id) # Mask sensitive keys from this list - mask_secret(extra) + mask_secret(obj) return extra diff --git a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py index 4f1e3934890bd..c4684bc5b63a0 100644 --- a/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py +++ b/providers/apache/livy/src/airflow/providers/apache/livy/hooks/livy.py @@ -89,6 +89,10 @@ class LivyHook(HttpHook): def get_connection_form_widgets(cls) -> dict[str, Any]: return super().get_connection_form_widgets() + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + return super().get_ui_field_behaviour() + def __init__( self, livy_conn_id: str = default_conn_name, From d1bf09078ec67230bc98480e32eb35eb34329e6c Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 8 May 2024 16:02:23 +0200 Subject: [PATCH 283/286] refactor: Changed conn_type to ftp in test_process_form_invalid_extra_removed as http as livy do now also have custom fields --- tests/www/views/test_views_connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/www/views/test_views_connection.py b/tests/www/views/test_views_connection.py index 1e21dc4856ed1..19a36c0ac6b39 100644 --- a/tests/www/views/test_views_connection.py +++ b/tests/www/views/test_views_connection.py @@ -462,9 +462,9 @@ def test_process_form_invalid_extra_removed(admin_client): Note: This can only be tested with a Hook which does not have any custom fields (otherwise the custom fields override the extra data when editing a Connection). Thus, this is currently - tested with livy. + tested with ftp. """ - conn_details = {"conn_id": "test_conn", "conn_type": "livy"} + conn_details = {"conn_id": "test_conn", "conn_type": "ftp"} conn = Connection(**conn_details, extra='{"foo": "bar"}') conn.id = 1 From 7c4538ce22d5f6fdc67506d97857182acd828b5f Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 18 Jul 2024 20:14:20 +0200 Subject: [PATCH 284/286] refactor: HttpHook now uses patched version of Connection + added test which checks when this patched class has to be removed so we don't forget --- .../src/airflow/providers/apache/druid/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py b/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py index 7585be9880dc1..d00c3c8da7757 100644 --- a/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py +++ b/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py @@ -37,3 +37,17 @@ raise RuntimeError( f"The package `apache-airflow-providers-apache-druid:{__version__}` needs Apache Airflow 2.9.0+" ) + + +def airflow_dependency_version(): + import re + import yaml + + from os.path import join, dirname + + with open(join(dirname(__file__), "provider.yaml"), encoding="utf-8") as file: + for dependency in yaml.safe_load(file)["dependencies"]: + if dependency.startswith('apache-airflow'): + match = re.search(r'>=([\d\.]+)', dependency) + if match: + return packaging.version.parse(match.group(1)) From bc54208311b992b7bf54ac5cdaaddbb3c8999267 Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 18 Jul 2024 20:56:42 +0200 Subject: [PATCH 285/286] refactor: Fixed some static checks --- .../druid/src/airflow/providers/apache/druid/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py b/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py index d00c3c8da7757..18870506f868b 100644 --- a/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py +++ b/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py @@ -41,13 +41,13 @@ def airflow_dependency_version(): import re - import yaml + from os.path import dirname, join - from os.path import join, dirname + import yaml with open(join(dirname(__file__), "provider.yaml"), encoding="utf-8") as file: for dependency in yaml.safe_load(file)["dependencies"]: - if dependency.startswith('apache-airflow'): - match = re.search(r'>=([\d\.]+)', dependency) + if dependency.startswith("apache-airflow"): + match = re.search(r">=([\d\.]+)", dependency) if match: return packaging.version.parse(match.group(1)) From f25589d88a9d963df270db43ac8b084466c40ad8 Mon Sep 17 00:00:00 2001 From: David Blain Date: Wed, 5 Feb 2025 13:54:13 +0100 Subject: [PATCH 286/286] refactor: Removed wrong modifications from main --- airflow/models/connection.py | 2 +- airflow/www/forms.py | 26 +------------------ .../providers/apache/druid/__init__.py | 14 ---------- .../src/airflow/providers/http/hooks/http.py | 20 -------------- setup.cfg | 0 5 files changed, 2 insertions(+), 60 deletions(-) delete mode 100644 setup.cfg diff --git a/airflow/models/connection.py b/airflow/models/connection.py index 85d3c7cb578c7..a8b9bb87985d8 100644 --- a/airflow/models/connection.py +++ b/airflow/models/connection.py @@ -432,7 +432,7 @@ def get_extra_dejson(self, nested: bool = False) -> dict: self.log.exception("Failed parsing the json for conn_id %s", self.conn_id) # Mask sensitive keys from this list - mask_secret(obj) + mask_secret(extra) return extra diff --git a/airflow/www/forms.py b/airflow/www/forms.py index b69184c6590c2..7028e2026e449 100644 --- a/airflow/www/forms.py +++ b/airflow/www/forms.py @@ -33,7 +33,6 @@ from flask_appbuilder.forms import DynamicForm from flask_babel import lazy_gettext from flask_wtf import FlaskForm -from markupsafe import Markup from wtforms import widgets from wtforms.fields import Field, IntegerField, PasswordField, SelectField, StringField, TextAreaField from wtforms.validators import InputRequired, Optional @@ -177,29 +176,6 @@ def populate_obj(self, item): field.populate_obj(item, name) -class BS3AccordionTextAreaFieldWidget(BS3TextAreaFieldWidget): - - @staticmethod - def _make_collapsable_panel(field: Field, content: Markup) -> str: - collapsable_id: str = f"collapsable_{field.id}" - return f""" -
-
-

- -

-
- -
- """ - - def __call__(self, field, **kwargs): - text_area = super(BS3TextAreaFieldWidget, self).__call__(field, **kwargs) - return self._make_collapsable_panel(field=field, content=text_area) - - @cache def create_connection_form_class() -> type[DynamicForm]: """ @@ -247,7 +223,7 @@ def process(self, formdata=None, obj=None, **kwargs): login = StringField(lazy_gettext("Login"), widget=BS3TextFieldWidget()) password = PasswordField(lazy_gettext("Password"), widget=BS3PasswordFieldWidget()) port = IntegerField(lazy_gettext("Port"), validators=[Optional()], widget=BS3TextFieldWidget()) - extra = TextAreaField(lazy_gettext("Extra"), widget=BS3AccordionTextAreaFieldWidget()) + extra = TextAreaField(lazy_gettext("Extra"), widget=BS3TextAreaFieldWidget()) for key, value in providers_manager.connection_form_widgets.items(): setattr(ConnectionForm, key, value.field) diff --git a/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py b/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py index 18870506f868b..7585be9880dc1 100644 --- a/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py +++ b/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py @@ -37,17 +37,3 @@ raise RuntimeError( f"The package `apache-airflow-providers-apache-druid:{__version__}` needs Apache Airflow 2.9.0+" ) - - -def airflow_dependency_version(): - import re - from os.path import dirname, join - - import yaml - - with open(join(dirname(__file__), "provider.yaml"), encoding="utf-8") as file: - for dependency in yaml.safe_load(file)["dependencies"]: - if dependency.startswith("apache-airflow"): - match = re.search(r">=([\d\.]+)", dependency) - if match: - return packaging.version.parse(match.group(1)) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 5fa7cf142337e..80255f1842646 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -228,26 +228,6 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: ), } - @classmethod - def get_connection_form_widgets(cls) -> dict[str, Any]: - """Return connection widgets to add to connection form.""" - from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget, Select2Widget - from flask_babel import lazy_gettext - from wtforms.fields import SelectField, TextAreaField - - default_auth_type: str = "" - auth_types_choices = frozenset({default_auth_type}) | get_auth_types() - return { - "auth_type": SelectField( - lazy_gettext("Auth type"), - choices=[(clazz, clazz) for clazz in auth_types_choices], - widget=Select2Widget(), - default=default_auth_type - ), - "auth_kwargs": TextAreaField(lazy_gettext("Auth kwargs"), widget=BS3TextAreaFieldWidget()), - "extra_headers": TextAreaField(lazy_gettext("Extra Headers"), widget=BS3TextAreaFieldWidget()), - } - # headers may be passed through directly or in the "extra" field in the connection # definition def get_conn(self, headers: dict[Any, Any] | None = None) -> requests.Session: diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index e69de29bb2d1d..0000000000000