From 5b1343112ce099dd32334b8e805d280c9c030cac Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 23 Oct 2024 14:06:27 +0200 Subject: [PATCH] Fixes behaviour of example dag tests for main/other branches (#43273) (#43307) While the #43260 attempted to address the problem where example dag importability tests should skip provider tests on non-main, it did not actually solve the problem. While debugging it, it turned out that since #42505, the provider tests were not executed in main "at all" - the "providers" directory was not included in the list of places to check for the example dags (they were in "airflow" in v2-10-test") this is why it "looked like" the solution worked in "main". This PR fixes both problems: * brings back importability of provider's example_dags in main branch * properly excludes the providers examples in non-main branch --- tests/always/test_example_dags.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/always/test_example_dags.py b/tests/always/test_example_dags.py index 15fa6165cc582..765dadf953cc4 100644 --- a/tests/always/test_example_dags.py +++ b/tests/always/test_example_dags.py @@ -120,7 +120,14 @@ def get_python_excluded_providers_folders() -> list[str]: def example_not_excluded_dags(xfail_db_exception: bool = False): - example_dirs = ["airflow/**/example_dags/example_*.py", "tests/system/**/example_*.py"] + example_dirs = [ + "airflow/**/example_dags/example_*.py", + "tests/system/**/example_*.py", + "providers/**/example_*.py", + ] + + default_branch = os.environ.get("DEFAULT_BRANCH", "main") + include_providers = default_branch == "main" suspended_providers_folders = get_suspended_providers_folders() current_python_excluded_providers_folders = get_python_excluded_providers_folders() @@ -135,14 +142,7 @@ def example_not_excluded_dags(xfail_db_exception: bool = False): for provider in current_python_excluded_providers_folders ] providers_folders = tuple([AIRFLOW_SOURCES_ROOT.joinpath(pp).as_posix() for pp in PROVIDERS_PREFIXES]) - - default_branch = os.environ.get("DEFAULT_BRANCH", "main") - include_providers = default_branch == "main" - for example_dir in example_dirs: - if not include_providers and "providers/" in example_dir: - print(f"Skipping {example_dir} because providers are not included for {default_branch} branch.") - continue candidates = glob(f"{AIRFLOW_SOURCES_ROOT.as_posix()}/{example_dir}", recursive=True) for candidate in sorted(candidates): param_marks = [] @@ -167,6 +167,11 @@ def example_not_excluded_dags(xfail_db_exception: bool = False): param_marks.append(pytest.mark.xfail(reason="Expected DB call", strict=True)) if candidate.startswith(providers_folders): + if not include_providers: + print( + f"Skipping {candidate} because providers are not included for {default_branch} branch." + ) + continue # Do not raise an error for airflow.exceptions.RemovedInAirflow3Warning. # We should not rush to enforce new syntax updates in providers # because a version of Airflow that deprecates certain features may not yet be released.