Migrate Amazon provider package

Fixes: apache#46045

.github/boring-cyborg.yml

@@ -25,10 +25,7 @@ labelPRBasedOnFilePath:
     - providers/alibaba/**
 
   provider:amazon:
-    - providers/src/airflow/providers/amazon/**/*
-    - providers/tests/amazon/aws/**/*
-    - docs/apache-airflow-providers-amazon/**/*
-    - providers/tests/system/amazon/aws/**/*
+    - providers/amazon/**
 
   provider:apache-beam:
     - providers/apache/beam/**

.pre-commit-config.yaml

@@ -625,8 +625,8 @@ repos:
           ^airflow/operators/__init__.py$|
           ^providers/common/sql/tests/provider_tests/common/sql/operators/test_sql_execute.py$|
           ^providers/fab/src/airflow/providers/fab/www/static/css/bootstrap-theme.css$|
-          ^providers/src/airflow/providers/amazon/aws/hooks/emr.py$|
-          ^providers/src/airflow/providers/amazon/aws/operators/emr.py$|
+          ^providers/amazon/src/airflow/providers/amazon/aws/hooks/emr.py$|
+          ^providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py$|
           ^providers/apache/cassandra/src/airflow/providers/apache/cassandra/hooks/cassandra.py$|
           ^providers/apache/hive/src/airflow/providers/apache/hive/operators/hive_stats.py$|
           ^providers/apache/hive/src/airflow/providers/apache/hive/transfers/vertica_to_hive.py$|

dev/breeze/tests/test_packages.py

@@ -37,7 +37,6 @@
     get_old_source_providers_package_path,
     get_pip_package_name,
     get_provider_info_dict,
-    get_provider_jinja_context,
     get_provider_requirements,
     get_removed_provider_ids,
     get_short_package_name,
@@ -379,38 +378,5 @@ def test_get_provider_info_dict():
     assert len(provider_info_dict["notifications"]) > 2
     assert len(provider_info_dict["secrets-backends"]) > 1
     assert len(provider_info_dict["logging"]) > 1
-    assert len(provider_info_dict["additional-extras"]) > 3
     assert len(provider_info_dict["config"].keys()) > 1
     assert len(provider_info_dict["executors"]) > 0
-
-
-# TODO(potiuk) - remove when all providers are new-style
-def test_old_provider_jinja_context():
-    provider_info = get_provider_info_dict("amazon")
-    version = provider_info["versions"][0]
-    context = get_provider_jinja_context(
-        provider_id="amazon", current_release_version=version, version_suffix="rc1"
-    )
-    expected = {
-        "PROVIDER_ID": "amazon",
-        "PACKAGE_PIP_NAME": "apache-airflow-providers-amazon",
-        "PACKAGE_DIST_PREFIX": "apache_airflow_providers_amazon",
-        "FULL_PACKAGE_NAME": "airflow.providers.amazon",
-        "RELEASE": version,
-        "RELEASE_NO_LEADING_ZEROS": version,
-        "VERSION_SUFFIX": ".rc1",
-        "PROVIDER_DESCRIPTION": "Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).\n",
-        "CHANGELOG_RELATIVE_PATH": "../../providers/src/airflow/providers/amazon",
-        "SUPPORTED_PYTHON_VERSIONS": ["3.9", "3.10", "3.11", "3.12"],
-        "PLUGINS": [],
-        "MIN_AIRFLOW_VERSION": "2.9.0",
-        "PROVIDER_REMOVED": False,
-        "PROVIDER_INFO": provider_info,
-    }
-
-    for key, value in expected.items():
-        assert context[key] == value
-    assert """"google" = [
-     "apache-airflow-providers-google",
-]""" in context["EXTRAS_REQUIREMENTS"]
-    assert len(context["PIP_REQUIREMENTS"]) > 10

dev/breeze/tests/test_pytest_args_for_test_types.py

@@ -88,7 +88,7 @@ def _all_new_providers() -> list[str]:
         (
             GroupOfTests.PROVIDERS,
             "Providers[amazon]",
-            ["providers/tests/amazon"],
+            ["providers/amazon/tests"],
         ),
         (
             GroupOfTests.PROVIDERS,
@@ -99,7 +99,7 @@ def _all_new_providers() -> list[str]:
             GroupOfTests.PROVIDERS,
             "Providers[amazon,google,apache.hive]",
             [
-                "providers/tests/amazon",
+                "providers/amazon/tests",
                 "providers/google/tests",
                 "providers/apache/hive/tests",
             ],
@@ -114,7 +114,6 @@ def _all_new_providers() -> list[str]:
                     if provider not in ["amazon", "google", "microsoft/azure"]
                 ],
                 "providers/tests",
-                "--ignore=providers/tests/amazon",
             ],
         ),
         (
@@ -245,14 +244,14 @@ def test_pytest_args_for_missing_provider():
             GroupOfTests.PROVIDERS,
             "Providers[amazon]",
             [
-                "providers/tests/amazon",
+                "providers/amazon/tests",
             ],
         ),
         (
             GroupOfTests.PROVIDERS,
             "Providers[amazon] Providers[google]",
             [
-                "providers/tests/amazon",
+                "providers/amazon/tests",
                 "providers/google/tests",
             ],
         ),
@@ -278,9 +277,7 @@ def test_pytest_args_for_missing_provider():
                     if provider not in ["amazon", "google"]
                 ],
                 "providers/tests",
-                *[
-                    "providers/google/tests"
-                ],  # Once amazon is migrated to the new structure, amazon needs to be added to the list here.
+                *["providers/amazon/tests", "providers/google/tests"],
             ],
         ),
         (

dev/breeze/tests/test_selective_checks.py

@@ -717,7 +717,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
             )
         ),
         pytest.param(
-            ("providers/src/airflow/providers/amazon/__init__.py",),
+            ("providers/amazon/src/airflow/providers/amazon/__init__.py",),
             {
                 "selected-providers-list-as-string": "amazon apache.hive cncf.kubernetes "
                 "common.compat common.sql exasol ftp google http imap microsoft.azure "
@@ -769,7 +769,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
             id="Providers tests run without amazon tests if no amazon file changed",
         ),
         pytest.param(
-            ("providers/src/airflow/providers/amazon/file.py",),
+            ("providers/amazon/src/airflow/providers/amazon/file.py",),
             {
                 "selected-providers-list-as-string": "amazon apache.hive cncf.kubernetes "
                 "common.compat common.sql exasol ftp google http imap microsoft.azure "

docs/.gitignore

@@ -2,6 +2,7 @@
 # Eventually when we swtich to individually build docs for each provider, we should remove this altogether
 apache-airflow-providers-airbyte
 apache-airflow-providers-alibaba
+apache-airflow-providers-amazon
 apache-airflow-providers-apache-beam
 apache-airflow-providers-apache-cassandra
 apache-airflow-providers-apache-drill

providers/amazon/README.rst

@@ -0,0 +1,106 @@
+
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+ .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
+
+ .. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+    `PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
+
+
+Package ``apache-airflow-providers-amazon``
+
+Release: ``9.2.0``
+
+
+Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
+
+
+Provider package
+----------------
+
+This is a provider package for ``amazon`` provider. All classes for this provider package
+are in ``airflow.providers.amazon`` python package.
+
+You can find package information and changelog for the provider
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.2.0/>`_.
+
+Installation
+------------
+
+You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below
+for the minimum Airflow version supported) via
+``pip install apache-airflow-providers-amazon``
+
+The package supports the following python versions: 3.9,3.10,3.11,3.12
+
+Requirements
+------------
+
+==========================================  ======================
+PIP package                                 Version required
+==========================================  ======================
+``apache-airflow``                          ``>=2.9.0``
+``apache-airflow-providers-common-compat``  ``>=1.3.0``
+``apache-airflow-providers-common-sql``     ``>=1.20.0``
+``apache-airflow-providers-http``
+``boto3``                                   ``>=1.34.90``
+``botocore``                                ``>=1.34.90``
+``inflection``                              ``>=0.5.1``
+``watchtower``                              ``>=3.0.0,!=3.3.0,<4``
+``jsonpath_ng``                             ``>=1.5.3``
+``redshift_connector``                      ``>=2.0.918``
+``asgiref``                                 ``>=2.3.0``
+``PyAthena``                                ``>=3.0.10``
+``jmespath``                                ``>=0.7.0``
+``python3-saml``                            ``>=1.16.0``
+==========================================  ======================
+
+Cross provider package dependencies
+-----------------------------------
+
+Those are dependencies that might be needed in order to use all the features of the package.
+You need to install the specified provider packages in order to use them.
+
+You can install such cross-provider dependencies when installing from PyPI. For example:
+
+.. code-block:: bash
+
+    pip install apache-airflow-providers-amazon[apache.hive]
+
+
+======================================================================================================================  ===================
+Dependent package                                                                                                       Extra
+======================================================================================================================  ===================
+`apache-airflow-providers-apache-hive <https://airflow.apache.org/docs/apache-airflow-providers-apache-hive>`_          ``apache.hive``
+`apache-airflow-providers-cncf-kubernetes <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes>`_  ``cncf.kubernetes``
+`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_      ``common.compat``
+`apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_            ``common.sql``
+`apache-airflow-providers-exasol <https://airflow.apache.org/docs/apache-airflow-providers-exasol>`_                    ``exasol``
+`apache-airflow-providers-ftp <https://airflow.apache.org/docs/apache-airflow-providers-ftp>`_                          ``ftp``
+`apache-airflow-providers-google <https://airflow.apache.org/docs/apache-airflow-providers-google>`_                    ``google``
+`apache-airflow-providers-http <https://airflow.apache.org/docs/apache-airflow-providers-http>`_                        ``http``
+`apache-airflow-providers-imap <https://airflow.apache.org/docs/apache-airflow-providers-imap>`_                        ``imap``
+`apache-airflow-providers-microsoft-azure <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure>`_  ``microsoft.azure``
+`apache-airflow-providers-mongo <https://airflow.apache.org/docs/apache-airflow-providers-mongo>`_                      ``mongo``
+`apache-airflow-providers-openlineage <https://airflow.apache.org/docs/apache-airflow-providers-openlineage>`_          ``openlineage``
+`apache-airflow-providers-salesforce <https://airflow.apache.org/docs/apache-airflow-providers-salesforce>`_            ``salesforce``
+`apache-airflow-providers-ssh <https://airflow.apache.org/docs/apache-airflow-providers-ssh>`_                          ``ssh``
+======================================================================================================================  ===================
+
+The changelog for the provider package can be found in the
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.2.0/changelog.html>`_.

docs/apache-airflow-providers-amazon/example-dags.rst → providers/amazon/docs/example-dags.rst

@@ -20,4 +20,4 @@ Example DAGs
 
 You can learn how to use Amazon AWS integrations by analyzing the source code of the example DAGs:
 
-* `Amazon AWS <https://github.com/apache/airflow/tree/providers-amazon/|version|/providers/tests/system/amazon/aws>`__
+* `Amazon AWS <https://github.com/apache/airflow/tree/providers-amazon/|version|/providers/amazon/tests/system/amazon/aws>`__

docs/apache-airflow-providers-amazon/operators/appflow.rst → providers/amazon/docs/operators/appflow.rst

@@ -49,7 +49,7 @@ Run Flow
 To run an AppFlow flow keeping as is, use:
 :class:`~airflow.providers.amazon.aws.operators.appflow.AppflowRunOperator`.
 
-.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_appflow_run.py
+.. exampleinclude:: /../../providers/amazon/tests/system/amazon/aws/example_appflow_run.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_appflow_run]
@@ -66,7 +66,7 @@ Run Flow Full
 To run an AppFlow flow removing all filters, use:
 :class:`~airflow.providers.amazon.aws.operators.appflow.AppflowRunFullOperator`.
 
-.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_appflow.py
+.. exampleinclude:: /../../providers/amazon/tests/system/amazon/aws/example_appflow.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_appflow_run_full]
@@ -83,7 +83,7 @@ Run Flow Daily
 To run an AppFlow flow filtering daily records, use:
 :class:`~airflow.providers.amazon.aws.operators.appflow.AppflowRunDailyOperator`.
 
-.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_appflow.py
+.. exampleinclude:: /../../providers/amazon/tests/system/amazon/aws/example_appflow.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_appflow_run_daily]
@@ -100,7 +100,7 @@ Run Flow Before
 To run an AppFlow flow filtering future records and selecting the past ones, use:
 :class:`~airflow.providers.amazon.aws.operators.appflow.AppflowRunBeforeOperator`.
 
-.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_appflow.py
+.. exampleinclude:: /../../providers/amazon/tests/system/amazon/aws/example_appflow.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_appflow_run_before]
@@ -117,7 +117,7 @@ Run Flow After
 To run an AppFlow flow filtering past records and selecting the future ones, use:
 :class:`~airflow.providers.amazon.aws.operators.appflow.AppflowRunAfterOperator`.
 
-.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_appflow.py
+.. exampleinclude:: /../../providers/amazon/tests/system/amazon/aws/example_appflow.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_appflow_run_after]
@@ -134,7 +134,7 @@ Skipping Tasks For Empty Runs
 To skip tasks when some AppFlow run return zero records, use:
 :class:`~airflow.providers.amazon.aws.operators.appflow.AppflowRecordsShortCircuitOperator`.
 
-.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_appflow.py
+.. exampleinclude:: /../../providers/amazon/tests/system/amazon/aws/example_appflow.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_appflow_shortcircuit]

docs/apache-airflow-providers-amazon/operators/athena/athena_boto.rst → providers/amazon/docs/operators/athena/athena_boto.rst

@@ -48,9 +48,9 @@ to run a query in Amazon Athena.
 
 In the following example, we query an existing Athena table and send the results to
 an existing Amazon S3 bucket.  For more examples of how to use this operator, please
-see the `Sample DAG <https://github.com/apache/airflow/blob/|version|/providers/tests/system/amazon/aws/example_athena.py>`__.
+see the `Sample DAG <https://github.com/apache/airflow/blob/|version|/providers/amazon/tests/system/amazon/aws/example_athena.py>`__.
 
-.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_athena.py
+.. exampleinclude:: /../../providers/amazon/tests/system/amazon/aws/example_athena.py
     :language: python
     :start-after: [START howto_operator_athena]
     :dedent: 4
@@ -67,7 +67,7 @@ Wait on Amazon Athena query results
 Use the :class:`~airflow.providers.amazon.aws.sensors.athena.AthenaSensor`
 to wait for the results of a query in Amazon Athena.
 
-.. exampleinclude:: /../../providers/tests/system/amazon/aws/example_athena.py
+.. exampleinclude:: /../../providers/amazon/tests/system/amazon/aws/example_athena.py
     :language: python
     :start-after: [START howto_sensor_athena]
     :dedent: 4