From 7d6c6c0e9b501e22b36295293e5654645a1b171d Mon Sep 17 00:00:00 2001 From: haseebmalik18 Date: Sun, 26 Apr 2026 21:55:45 -0400 Subject: [PATCH] Add provider compat tests against v3-X-test branch tips #54454 --- .github/workflows/test-providers.yml | 11 ++-- .../src/airflow_breeze/global_constants.py | 11 ++++ .../install_airflow_and_providers.py | 64 ++++++++++++------- 3 files changed, 58 insertions(+), 28 deletions(-) diff --git a/.github/workflows/test-providers.yml b/.github/workflows/test-providers.yml index db1f31f2453aa..c98a4ba2f5d58 100644 --- a/.github/workflows/test-providers.yml +++ b/.github/workflows/test-providers.yml @@ -47,7 +47,8 @@ on: # yamllint disable-line rule:truthy providers-compatibility-tests-matrix: description: > JSON-formatted array of providers compatibility tests in the form of array of dicts - (airflow-version, python-versions, remove-providers, run-unit-tests) + (airflow-version, python-versions, remove-providers, run-unit-tests, + airflow-constraints-reference) required: true type: string providers-test-types-list-as-strings-in-json: @@ -231,6 +232,7 @@ jobs: - name: "Download airflow package: wheel" run: | pip download "apache-airflow==${{ matrix.compat.airflow-version }}" -d dist --no-deps + if: "!contains(matrix.compat.airflow-version, ':')" - name: > Install and verify all provider distributions and airflow on Airflow ${{ matrix.compat.airflow-version }}:Python ${{ matrix.compat.python-version }} @@ -238,13 +240,13 @@ jobs: # automatically if: matrix.compat.run-unit-tests != 'true' env: - AIRFLOW_VERSION: "${{ matrix.compat.airflow-version }}" + CONSTRAINTS_REF: "${{ matrix.compat.airflow-constraints-reference }}" run: > breeze release-management verify-provider-distributions --use-distributions-from-dist --distribution-format wheel --use-airflow-version wheel - --airflow-constraints-reference constraints-${AIRFLOW_VERSION} + --airflow-constraints-reference "${CONSTRAINTS_REF}" --providers-skip-constraints --install-airflow-with-constraints - name: Check amount of disk space available @@ -258,13 +260,14 @@ jobs: PROVIDERS_TEST_TYPES: "${{ matrix.test-types.test_types }}" AIRFLOW_VERSION: "${{ matrix.compat.airflow-version }}" REMOVE_PROVIDERS: "${{ matrix.compat.remove-providers }}" + CONSTRAINTS_REF: "${{ matrix.compat.airflow-constraints-reference }}" run: > breeze testing providers-tests --run-in-parallel --parallel-test-types "${PROVIDERS_TEST_TYPES}" --use-distributions-from-dist --distribution-format wheel --use-airflow-version "${AIRFLOW_VERSION}" - --airflow-constraints-reference constraints-${AIRFLOW_VERSION} + --airflow-constraints-reference "${CONSTRAINTS_REF}" --install-airflow-with-constraints --providers-skip-constraints --skip-providers "${REMOVE_PROVIDERS}" diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 0d29d334f12cb..c85809c1641f0 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -767,24 +767,35 @@ def get_airflow_extras(): { "python-version": "3.10", "airflow-version": "2.11.1", + "airflow-constraints-reference": "constraints-2.11.1", "remove-providers": "common.messaging edge3 fab git keycloak informatica common.ai opensearch", "run-unit-tests": "true", }, { "python-version": "3.10", "airflow-version": "3.0.6", + "airflow-constraints-reference": "constraints-3.0.6", "remove-providers": "", "run-unit-tests": "true", }, { "python-version": "3.10", "airflow-version": "3.1.8", + "airflow-constraints-reference": "constraints-3.1.8", "remove-providers": "", "run-unit-tests": "true", }, { "python-version": "3.10", "airflow-version": "3.2.1", + "airflow-constraints-reference": "constraints-3.2.1", + "remove-providers": "", + "run-unit-tests": "true", + }, + { + "python-version": "3.10", + "airflow-version": "apache/airflow:v3-2-test", + "airflow-constraints-reference": "constraints-3-2", "remove-providers": "", "run-unit-tests": "true", }, diff --git a/scripts/in_container/install_airflow_and_providers.py b/scripts/in_container/install_airflow_and_providers.py index c8223f3eeff10..7a9f4a1d5fce2 100755 --- a/scripts/in_container/install_airflow_and_providers.py +++ b/scripts/in_container/install_airflow_and_providers.py @@ -473,7 +473,7 @@ def find_installation_spec( github_repository=github_repository, python_version=python_version, ) - compile_ui_assets = True + compile_ui_assets = not use_distributions_from_dist console.print(f"\nInstalling airflow task-sdk from GitHub {resolved_version}\n") airflow_task_sdk_distribution = f"apache-airflow-task-sdk @ {vcs_url}#subdirectory=task-sdk" airflow_constraints_location = get_airflow_constraints_location( @@ -695,10 +695,8 @@ def compile_ui_assets( console.print( f"[bright_blue]Copying UI source from: {extracted_ui_directory} to: {source_ui_directory}" ) - if source_ui_directory.exists(): - shutil.rmtree(source_ui_directory) source_ui_directory.parent.mkdir(parents=True, exist_ok=True) - shutil.copytree(extracted_ui_directory, source_ui_directory) + shutil.copytree(extracted_ui_directory, source_ui_directory, dirs_exist_ok=True) else: console.print(f"[yellow]Main UI directory not found at: {extracted_ui_directory}") @@ -1024,7 +1022,9 @@ def install_airflow_and_providers( if installation_spec.airflow_ctl_distribution: _install_airflow_ctl_with_constraints(installation_spec, github_actions) if installation_spec.provider_distributions or not install_airflow_with_constraints: - _install_airflow_and_optionally_providers_together(installation_spec, github_actions) + _install_airflow_and_optionally_providers_together( + installation_spec, github_actions, airflow_already_installed=install_airflow_with_constraints + ) if mount_sources == "providers-and-tests": if ( use_airflow_version @@ -1114,11 +1114,17 @@ def install_airflow_and_providers( SIMPLE_AUTH_MANAGER_SOURCE_UI_DIRECTORY, SIMPLE_AUTH_MANAGER_UI_DIST_PREFIX, ) + airflow_path = get_airflow_installation_path() + for dist_prefix in (CORE_UI_DIST_PREFIX, SIMPLE_AUTH_MANAGER_UI_DIST_PREFIX): + dist_dir = airflow_path / dist_prefix + dist_dir.mkdir(parents=True, exist_ok=True) console.print("\n[green]Done!") def _install_airflow_and_optionally_providers_together( - installation_spec: InstallationSpec, github_actions: bool + installation_spec: InstallationSpec, + github_actions: bool, + airflow_already_installed: bool = False, ): console.print("[bright_blue]Installing airflow and optionally providers together") base_install_cmd = [ @@ -1129,26 +1135,36 @@ def _install_airflow_and_optionally_providers_together( if installation_spec.pre_release: console.print("[bright_blue]Allowing pre-release versions of airflow and providers") base_install_cmd.extend(["--pre"]) - if installation_spec.airflow_distribution: - console.print( - f"\n[bright_blue]Adding airflow distribution to installation: {installation_spec.airflow_distribution} " - ) - base_install_cmd.append(installation_spec.airflow_distribution) - if installation_spec.airflow_core_distribution: - console.print( - f"\n[bright_blue]Adding airflow core distribution to installation: {installation_spec.airflow_core_distribution}" - ) - base_install_cmd.append(installation_spec.airflow_core_distribution) - if installation_spec.airflow_task_sdk_distribution: - console.print( - f"\n[bright_blue]Adding task-sdk distribution to installation: {installation_spec.airflow_task_sdk_distribution}" - ) - base_install_cmd.append(installation_spec.airflow_task_sdk_distribution) - if installation_spec.airflow_ctl_distribution: + if airflow_already_installed: console.print( - f"\n[bright_blue]Adding airflow ctl distribution to installation: {installation_spec.airflow_ctl_distribution}" + "\n[bright_blue]Airflow already installed with constraints - " + "only installing provider distributions" ) - base_install_cmd.append(installation_spec.airflow_ctl_distribution) + else: + if installation_spec.airflow_distribution: + console.print( + f"\n[bright_blue]Adding airflow distribution to installation: " + f"{installation_spec.airflow_distribution} " + ) + base_install_cmd.append(installation_spec.airflow_distribution) + if installation_spec.airflow_core_distribution: + console.print( + f"\n[bright_blue]Adding airflow core distribution to installation: " + f"{installation_spec.airflow_core_distribution}" + ) + base_install_cmd.append(installation_spec.airflow_core_distribution) + if installation_spec.airflow_task_sdk_distribution: + console.print( + f"\n[bright_blue]Adding task-sdk distribution to installation: " + f"{installation_spec.airflow_task_sdk_distribution}" + ) + base_install_cmd.append(installation_spec.airflow_task_sdk_distribution) + if installation_spec.airflow_ctl_distribution: + console.print( + f"\n[bright_blue]Adding airflow ctl distribution to installation: " + f"{installation_spec.airflow_ctl_distribution}" + ) + base_install_cmd.append(installation_spec.airflow_ctl_distribution) console.print("\n[bright_blue]Adding provider distributions to installation:") for provider_package in sorted(installation_spec.provider_distributions): console.print(f" {provider_package}")