Fix few issues with constraints generation broken by moving providers (…

…apache#46139) As part of apache#46045 we are moving all providers to the new structure, each with it's own pyproject.toml. However we noticed that constraint generation is somewhat broken during the move: * providers moved to the new structure disappeared from pypi constraints * http provider in SOME PRs started to appear in "source constraints" Both should not happen: * pypi constraints should contain all providers (latest versions in PyPI) * source providers should contain no providers - only their dependencies There are two reasons for those two issues, and this PR fixes both: 1) The PyPI providers were removed because `uv pip install .` without editable flag is currently - pretty unexpectedly - installing local workspaced providers rather than install them from PyPI and there are no controls to disable it. Workaround for that is that during constraint generation we remove temporarily all providers from workspace definition in the pyproject.toml - this way providers are installed from PyPI. Feature request to handle it has been raised in the astral-sh/uv#10991 2) The http provider was added to the constraint in PRs where the http provider is a required dependency (for example in discord provider). Since the http provider is not yet migrated, discord installation in workspace will pull it from PyPI rather than from the local workspace. This has been fixed by simply migrating the http provider to the new structure,
niklasr22 · Feb 8, 2025 · a9e69b5 · a9e69b5
1 parent 1e2ae0d
commit a9e69b5
Show file tree

Hide file tree

Showing 46 changed files with 605 additions and 116 deletions.
diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml
@@ -232,10 +232,7 @@ labelPRBasedOnFilePath:
     - providers/hashicorp/**
 
   provider:http:
-    - providers/src/airflow/providers/http/**/*
-    - docs/apache-airflow-providers-http/**/*
-    - providers/tests/http/**/*
-    - providers/tests/system/http/**/*
+    - providers/http/**
 
   provider:imap:
     - providers/imap/**

diff --git a/dev/breeze/tests/test_run_test_args.py b/dev/breeze/tests/test_run_test_args.py
@@ -102,48 +102,16 @@ def test_irregular_provider_with_extra_ignore_should_be_valid_cmd(mock_run_comma
     assert match_pattern.search(arg_str), arg_str
 
 
-def test_primary_test_arg_is_excluded_by_extra_pytest_arg(mock_run_command):
-    test_provider = "http"  # "Providers[<id>]" scans the source tree so we need to use a real provider id
-    test_provider_not_skipped = "ftp"
-    _run_test(
-        shell_params=ShellParams(
-            test_group=GroupOfTests.PROVIDERS,
-            test_type=f"Providers[{test_provider},{test_provider_not_skipped}]",
-        ),
-        extra_pytest_args=(f"--ignore=providers/tests/{test_provider}",),
-        python_version="3.9",
-        output=None,
-        test_timeout=60,
-        skip_docker_compose_down=True,
-    )
-
-    assert mock_run_command.call_count > 1
-    run_cmd_call = mock_run_command.call_args_list[1]
-    arg_str = " ".join(run_cmd_call.args[0])
-
-    # The command pattern we look for is "<container id> --verbosity=0 \
-    # <*other args we don't care about*> --ignore=providers/tests/<provider name>"
-    # The providers/tests/http argument has been eliminated by the code that preps the args; this is a bug,
-    # bc without a directory or module arg, pytest tests everything (which we don't want!)
-    # We check "--verbosity=0" to ensure nothing is between the airflow container id and the verbosity arg,
-    # IOW that the primary test arg is removed
-    match_pattern = re.compile(
-        f"airflow providers/tests/{test_provider_not_skipped} --verbosity=0 .+ --ignore=providers/tests/{test_provider}"
-    )
-
-    assert match_pattern.search(arg_str)
-
-
 def test_test_is_skipped_if_all_are_ignored(mock_run_command):
     test_providers = [
         "http",
-        "ftp",
+        "standard",
     ]  # "Providers[<id>]" scans the source tree so we need to use a real provider id
     _run_test(
         shell_params=ShellParams(
             test_group=GroupOfTests.PROVIDERS, test_type=f"Providers[{','.join(test_providers)}]"
         ),
-        extra_pytest_args=tuple(f"--ignore=providers/tests/{provider}" for provider in test_providers),
+        extra_pytest_args=tuple(f"--ignore=providers/{provider}/tests" for provider in test_providers),
         python_version="3.9",
         output=None,
         test_timeout=60,

diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py
@@ -568,7 +568,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
                 (
                     "INTHEWILD.md",
                     "chart/aaaa.txt",
-                    "providers/tests/http/file.py",
+                    "providers/http/tests/file.py",
                 ),
                 {
                     "selected-providers-list-as-string": "amazon apache.livy "

diff --git a/docs/.gitignore b/docs/.gitignore
@@ -17,6 +17,7 @@ apache-airflow-providers-datadog
 apache-airflow-providers-edge
 apache-airflow-providers-exasol
 apache-airflow-providers-facebook
+apache-airflow-providers-http
 apache-airflow-providers-openlineage
 apache-airflow-providers-hashicorp
 apache-airflow-providers-imap

diff --git a/docs/apache-airflow-providers-http/changelog.rst b/docs/apache-airflow-providers-http/changelog.rst
diff --git a/providers/http/README.rst b/providers/http/README.rst
@@ -0,0 +1,65 @@
+
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+ .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
+
+ .. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+    `PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
+
+
+Package ``apache-airflow-providers-http``
+
+Release: ``5.0.0``
+
+
+`Hypertext Transfer Protocol (HTTP) <https://www.w3.org/Protocols/>`__
+
+
+Provider package
+----------------
+
+This is a provider package for ``http`` provider. All classes for this provider package
+are in ``airflow.providers.http`` python package.
+
+You can find package information and changelog for the provider
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-http/5.0.0/>`_.
+
+Installation
+------------
+
+You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below
+for the minimum Airflow version supported) via
+``pip install apache-airflow-providers-http``
+
+The package supports the following python versions: 3.9,3.10,3.11,3.12
+
+Requirements
+------------
+
+=====================  ====================
+PIP package            Version required
+=====================  ====================
+``apache-airflow``     ``>=2.9.0``
+``requests``           ``>=2.27.0,<3``
+``requests-toolbelt``  ``>=0.4.0``
+``aiohttp``            ``!=3.11.0,>=3.9.2``
+``asgiref``            ``>=2.3.0``
+=====================  ====================
+
+The changelog for the provider package can be found in the
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-http/5.0.0/changelog.html>`_.
diff --git a/...roviders/http/.latest-doc-only-change.txt → ...ers/http/docs/.latest-doc-only-change.txt b/...roviders/http/.latest-doc-only-change.txt → ...ers/http/docs/.latest-doc-only-change.txt
diff --git a/.../src/airflow/providers/http/CHANGELOG.rst → providers/http/docs/changelog.rst b/.../src/airflow/providers/http/CHANGELOG.rst → providers/http/docs/changelog.rst
@@ -277,7 +277,7 @@ Bug Fixes
 ~~~~~~~~~
 
 * ``Fix json data for async PUTs (#35405)``
-* ``Fix: Paginate on lastest Response (#35560)``
+* ``Fix: Paginate on latest Response (#35560)``
 
 .. Below changes are excluded from the changelog. Move them to
    appropriate section above if needed. Do not delete the lines(!):

diff --git a/...apache-airflow-providers-http/commits.rst → providers/http/docs/commits.rst b/...apache-airflow-providers-http/commits.rst → providers/http/docs/commits.rst
diff --git a/...rflow-providers-http/connections/http.rst → providers/http/docs/connections/http.rst b/...rflow-providers-http/connections/http.rst → providers/http/docs/connections/http.rst
diff --git a/docs/apache-airflow-providers-http/index.rst → providers/http/docs/index.rst b/docs/apache-airflow-providers-http/index.rst → providers/http/docs/index.rst
diff --git a/...ttp/installing-providers-from-sources.rst → ...ocs/installing-providers-from-sources.rst b/...ttp/installing-providers-from-sources.rst → ...ocs/installing-providers-from-sources.rst
diff --git a/docs/integration-logos/http/HTTP.png → ...ders/http/docs/integration-logos/HTTP.png b/docs/integration-logos/http/HTTP.png → ...ders/http/docs/integration-logos/HTTP.png
diff --git a/...ache-airflow-providers-http/operators.rst → providers/http/docs/operators.rst b/...ache-airflow-providers-http/operators.rst → providers/http/docs/operators.rst
@@ -32,14 +32,14 @@ to ``true``.
 
 Here we are poking until httpbin gives us a response text containing ``httpbin``.
 
-.. exampleinclude:: /../../providers/tests/system/http/example_http.py
+.. exampleinclude:: /../../providers/http/tests/system/http/example_http.py
     :language: python
     :start-after: [START howto_operator_http_http_sensor_check]
     :end-before: [END howto_operator_http_http_sensor_check]
 
 This sensor can also be used in deferrable mode
 
-.. exampleinclude:: /../../providers/tests/system/http/example_http.py
+.. exampleinclude:: /../../providers/http/tests/system/http/example_http.py
     :language: python
     :start-after: [START howto_operator_http_http_sensor_check_deferrable]
     :end-before: [END howto_operator_http_http_sensor_check_deferrable]
@@ -76,14 +76,14 @@ the response text back.
 In the first example we are calling a ``POST`` with json data and succeed when we get the same json data back
 otherwise the task will fail.
 
-.. exampleinclude:: /../../providers/tests/system/http/example_http.py
+.. exampleinclude:: /../../providers/http/tests/system/http/example_http.py
     :language: python
     :start-after: [START howto_operator_http_task_post_op]
     :end-before: [END howto_operator_http_task_post_op]
 
 Here we are calling a ``GET`` request and pass params to it. The task will succeed regardless of the response text.
 
-.. exampleinclude:: /../../providers/tests/system/http/example_http.py
+.. exampleinclude:: /../../providers/http/tests/system/http/example_http.py
     :language: python
     :start-after: [START howto_operator_http_task_get_op]
     :end-before: [END howto_operator_http_task_get_op]
@@ -98,30 +98,30 @@ it on the next task downstream use ``response_filter``. This is useful if:
 Below is an example of retrieving data from a REST API and only returning a nested property instead of the full
 response body.
 
-.. exampleinclude:: /../../providers/tests/system/http/example_http.py
+.. exampleinclude:: /../../providers/http/tests/system/http/example_http.py
     :language: python
     :start-after: [START howto_operator_http_task_get_op_response_filter]
     :end-before: [END howto_operator_http_task_get_op_response_filter]
 
 In the third example we are performing a ``PUT`` operation to put / set data according to the data that is being
 provided to the request.
 
-.. exampleinclude:: /../../providers/tests/system/http/example_http.py
+.. exampleinclude:: /../../providers/http/tests/system/http/example_http.py
     :language: python
     :start-after: [START howto_operator_http_task_put_op]
     :end-before: [END howto_operator_http_task_put_op]
 
 In this example we call a ``DELETE`` operation to the ``delete`` endpoint. This time we are passing form data to the
 request.
 
-.. exampleinclude:: /../../providers/tests/system/http/example_http.py
+.. exampleinclude:: /../../providers/http/tests/system/http/example_http.py
     :language: python
     :start-after: [START howto_operator_http_task_del_op]
     :end-before: [END howto_operator_http_task_del_op]
 
 Here we pass form data to a ``POST`` operation which is equal to a usual form submit.
 
-.. exampleinclude:: /../../providers/tests/system/http/example_http.py
+.. exampleinclude:: /../../providers/http/tests/system/http/example_http.py
     :language: python
     :start-after: [START howto_operator_http_task_post_op_formenc]
     :end-before: [END howto_operator_http_task_post_op_formenc]
@@ -140,7 +140,7 @@ You can write a ``pagination_function`` that will receive the raw ``request.Resp
 generate new request parameters (as ``dict``) based on this cursor. The HttpOperator will repeat calls to the
 API until the function stop returning anything.
 
-.. exampleinclude:: /../../providers/tests/system/http/example_http.py
+.. exampleinclude:: /../../providers/http/tests/system/http/example_http.py
     :language: python
     :start-after: [START howto_operator_http_pagination_function]
     :end-before: [END howto_operator_http_pagination_function]
diff --git a/...pache-airflow-providers-http/security.rst → providers/http/docs/security.rst b/...pache-airflow-providers-http/security.rst → providers/http/docs/security.rst
diff --git a/.../src/airflow/providers/http/provider.yaml → providers/http/provider.yaml b/.../src/airflow/providers/http/provider.yaml → providers/http/provider.yaml
@@ -63,22 +63,12 @@ versions:
   - 1.1.0
   - 1.0.0
 
-dependencies:
-  - apache-airflow>=2.9.0
-  # The 2.26.0 release of requests got rid of the chardet LGPL mandatory dependency, allowing us to
-  # release it as a requirement for airflow
-  - requests>=2.27.0,<3
-  - requests-toolbelt>=0.4.0
-  # aiohttp 3.11.0 had incompatible change: https://github.com/aio-libs/aiohttp/issues/9866
-  - aiohttp>=3.9.2,!=3.11.0
-  - asgiref>=2.3.0
-
 integrations:
   - integration-name: Hypertext Transfer Protocol (HTTP)
     external-doc-url: https://www.w3.org/Protocols/
     how-to-guide:
       - /docs/apache-airflow-providers-http/operators.rst
-    logo: /integration-logos/http/HTTP.png
+    logo: /docs/integration-logos/HTTP.png
     tags: [protocol]
 
 operators:

diff --git a/providers/http/pyproject.toml b/providers/http/pyproject.toml
@@ -0,0 +1,84 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
+
+# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE
+# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
+[build-system]
+requires = ["flit_core==3.10.1"]
+build-backend = "flit_core.buildapi"
+
+[project]
+name = "apache-airflow-providers-http"
+version = "5.0.0"
+description = "Provider package apache-airflow-providers-http for Apache Airflow"
+readme = "README.rst"
+authors = [
+    {name="Apache Software Foundation", email="[email protected]"},
+]
+maintainers = [
+    {name="Apache Software Foundation", email="[email protected]"},
+]
+keywords = [ "airflow-provider", "http", "airflow", "integration" ]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Environment :: Console",
+    "Environment :: Web Environment",
+    "Intended Audience :: Developers",
+    "Intended Audience :: System Administrators",
+    "Framework :: Apache Airflow",
+    "Framework :: Apache Airflow :: Provider",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: System :: Monitoring",
+]
+requires-python = "~=3.9"
+
+# The dependencies should be modified in place in the generated file
+# Any change in the dependencies is preserved when the file is regenerated
+dependencies = [
+    "apache-airflow>=2.9.0",
+    # The 2.26.0 release of requests got rid of the chardet LGPL mandatory dependency, allowing us to
+    # release it as a requirement for airflow
+    "requests>=2.27.0,<3",
+    "requests-toolbelt>=0.4.0",
+    # aiohttp 3.11.0 had incompatible change: https://github.com/aio-libs/aiohttp/issues/9866
+    "aiohttp>=3.9.2,!=3.11.0",
+    "asgiref>=2.3.0",
+]
+
+[project.urls]
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-http/5.0.0"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-http/5.0.0/changelog.html"
+"Bug Tracker" = "https://github.com/apache/airflow/issues"
+"Source Code" = "https://github.com/apache/airflow"
+"Slack Chat" = "https://s.apache.org/airflow-slack"
+"Twitter" = "https://x.com/ApacheAirflow"
+"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/"
+
+[project.entry-points."apache_airflow_provider"]
+provider_info = "airflow.providers.http.get_provider_info:get_provider_info"
+
+[tool.flit.module]
+name = "airflow.providers.http"
+
+[tool.pytest.ini_options]
+ignore = "tests/system/"