diff --git a/Dockerfile b/Dockerfile index eb3b1b2bd039e..5d500d4b76a5a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -890,6 +890,54 @@ function common::get_constraints_location() { fi } +function common::resolve_build_constraints() { + BUILD_CONSTRAINTS_INSTALL_FLAGS=() + if [[ -z ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION=} ]]; then + return + fi + + local target="${HOME}/build-constraints.txt" + if [[ ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} =~ ^https?:// ]]; then + echo + echo "${COLOR_BLUE}Downloading build constraints from ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} to ${target}${COLOR_RESET}" + echo + rm -f "${target}" + if ! curl -sSf -o "${target}" "${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}"; then + rm -f "${target}" + echo + echo "${COLOR_RED}Build constraints file not found at explicitly set ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}${COLOR_RESET}" + echo + exit 1 + fi + else + if [[ ! -f ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} || ! -s ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} ]]; then + echo + echo "${COLOR_RED}Build constraints must be a non-empty file: ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}${COLOR_RESET}" + echo + exit 1 + fi + echo + echo "${COLOR_BLUE}Copying build constraints from ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} to ${target}${COLOR_RESET}" + echo + if [[ ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} != "${target}" ]]; then + cp "${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}" "${target}" + fi + fi + + if [[ ! -s ${target} ]]; then + rm -f "${target}" + echo + echo "${COLOR_RED}Build constraints file is empty: ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}${COLOR_RESET}" + echo + exit 1 + fi + if [[ ${PACKAGING_TOOL} == "uv" ]]; then + BUILD_CONSTRAINTS_INSTALL_FLAGS=(--build-constraints "${target}") + else + BUILD_CONSTRAINTS_INSTALL_FLAGS=(--build-constraint "${target}") + fi +} + function common::show_packaging_tool_version_and_location() { echo "PATH=${PATH}" echo "Installed pip: $(pip --version): $(which pip)" @@ -1027,9 +1075,11 @@ function install_airflow_and_providers_from_docker_context_files(){ # This is needed to get distribution names for local context distributions if [[ -f "${HOME}/constraints.txt" ]]; then - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} --constraint ${HOME}/constraints.txt packaging + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + "${BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}" --constraint ${HOME}/constraints.txt packaging else - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} packaging + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + "${BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}" packaging fi if [[ -n ${AIRFLOW_EXTRAS=} ]]; then @@ -1108,6 +1158,7 @@ function install_airflow_and_providers_from_docker_context_files(){ set -x if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + "${BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}" \ "${flags[@]}" \ "${install_airflow_distribution[@]}" "${install_airflow_core_distribution[@]}" "${airflow_distributions[@]}"; then set +x @@ -1145,6 +1196,7 @@ function install_all_other_distributions_from_docker_context_files() { if [[ -n "${reinstalling_other_distributions}" ]]; then set -x ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + "${BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}" \ --force-reinstall --no-deps --no-index ${reinstalling_other_distributions} common::install_packaging_tools set +x @@ -1155,6 +1207,7 @@ common::get_colors common::get_packaging_tool common::get_airflow_version_specification common::get_constraints_location +common::resolve_build_constraints common::show_packaging_tool_version_and_location install_airflow_and_providers_from_docker_context_files @@ -1289,7 +1342,8 @@ function install_from_sources() { } function install_from_external_spec() { - local installation_command_flags + common::resolve_build_constraints + local installation_command_flags if [[ ${AIRFLOW_INSTALLATION_METHOD} == "apache-airflow" ]]; then installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" else @@ -1311,14 +1365,18 @@ function install_from_external_spec() { echo "${COLOR_BLUE}Installing all packages with highest resolutions. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}" echo set -x - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} \ + ${ADDITIONAL_PIP_INSTALL_FLAGS} "${BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}" \ + ${installation_command_flags} set +x else echo echo "${COLOR_BLUE}Installing all packages with constraints. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}" echo set -x - if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then + if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + "${BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}" ${installation_command_flags} \ + --constraint "${HOME}/constraints.txt"; then set +x if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then echo @@ -1391,6 +1449,7 @@ function install_additional_dependencies() { set -x ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + "${BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}" \ ${ADDITIONAL_PYTHON_DEPS} set +x common::install_packaging_tools @@ -1406,6 +1465,7 @@ function install_additional_dependencies() { set -x ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + "${BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}" \ ${ADDITIONAL_PYTHON_DEPS} set +x common::install_packaging_tools @@ -1421,6 +1481,7 @@ common::get_colors common::get_packaging_tool common::get_airflow_version_specification common::get_constraints_location +common::resolve_build_constraints common::show_packaging_tool_version_and_location install_additional_dependencies @@ -1936,6 +1997,7 @@ ARG CONSTRAINTS_GITHUB_REPOSITORY="apache/airflow" ARG AIRFLOW_CONSTRAINTS_MODE="constraints" ARG AIRFLOW_CONSTRAINTS_REFERENCE="" ARG AIRFLOW_CONSTRAINTS_LOCATION="" +ARG AIRFLOW_BUILD_CONSTRAINTS_LOCATION="" ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main" # By default do not fallback to installation without constraints because it can hide problems with constraints ARG AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION="false" @@ -1991,6 +2053,7 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ AIRFLOW_CONSTRAINTS_MODE=${AIRFLOW_CONSTRAINTS_MODE} \ AIRFLOW_CONSTRAINTS_REFERENCE=${AIRFLOW_CONSTRAINTS_REFERENCE} \ AIRFLOW_CONSTRAINTS_LOCATION=${AIRFLOW_CONSTRAINTS_LOCATION} \ + AIRFLOW_BUILD_CONSTRAINTS_LOCATION=${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} \ AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION=${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} \ DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH} \ PATH=${AIRFLOW_USER_HOME_DIR}/.local/bin:${PATH} \ diff --git a/Dockerfile.ci b/Dockerfile.ci index 42b2db30b6f95..dbf55d74c1b7c 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -830,6 +830,54 @@ function common::get_constraints_location() { fi } +function common::resolve_build_constraints() { + BUILD_CONSTRAINTS_INSTALL_FLAGS=() + if [[ -z ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION=} ]]; then + return + fi + + local target="${HOME}/build-constraints.txt" + if [[ ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} =~ ^https?:// ]]; then + echo + echo "${COLOR_BLUE}Downloading build constraints from ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} to ${target}${COLOR_RESET}" + echo + rm -f "${target}" + if ! curl -sSf -o "${target}" "${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}"; then + rm -f "${target}" + echo + echo "${COLOR_RED}Build constraints file not found at explicitly set ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}${COLOR_RESET}" + echo + exit 1 + fi + else + if [[ ! -f ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} || ! -s ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} ]]; then + echo + echo "${COLOR_RED}Build constraints must be a non-empty file: ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}${COLOR_RESET}" + echo + exit 1 + fi + echo + echo "${COLOR_BLUE}Copying build constraints from ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} to ${target}${COLOR_RESET}" + echo + if [[ ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} != "${target}" ]]; then + cp "${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}" "${target}" + fi + fi + + if [[ ! -s ${target} ]]; then + rm -f "${target}" + echo + echo "${COLOR_RED}Build constraints file is empty: ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}${COLOR_RESET}" + echo + exit 1 + fi + if [[ ${PACKAGING_TOOL} == "uv" ]]; then + BUILD_CONSTRAINTS_INSTALL_FLAGS=(--build-constraints "${target}") + else + BUILD_CONSTRAINTS_INSTALL_FLAGS=(--build-constraint "${target}") + fi +} + function common::show_packaging_tool_version_and_location() { echo "PATH=${PATH}" echo "Installed pip: $(pip --version): $(which pip)" @@ -991,7 +1039,8 @@ function install_from_sources() { } function install_from_external_spec() { - local installation_command_flags + common::resolve_build_constraints + local installation_command_flags if [[ ${AIRFLOW_INSTALLATION_METHOD} == "apache-airflow" ]]; then installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" else @@ -1013,14 +1062,18 @@ function install_from_external_spec() { echo "${COLOR_BLUE}Installing all packages with highest resolutions. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}" echo set -x - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} \ + ${ADDITIONAL_PIP_INSTALL_FLAGS} "${BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}" \ + ${installation_command_flags} set +x else echo echo "${COLOR_BLUE}Installing all packages with constraints. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}" echo set -x - if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then + if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + "${BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}" ${installation_command_flags} \ + --constraint "${HOME}/constraints.txt"; then set +x if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then echo @@ -1093,6 +1146,7 @@ function install_additional_dependencies() { set -x ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + "${BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}" \ ${ADDITIONAL_PYTHON_DEPS} set +x common::install_packaging_tools @@ -1108,6 +1162,7 @@ function install_additional_dependencies() { set -x ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + "${BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}" \ ${ADDITIONAL_PYTHON_DEPS} set +x common::install_packaging_tools @@ -1123,6 +1178,7 @@ common::get_colors common::get_packaging_tool common::get_airflow_version_specification common::get_constraints_location +common::resolve_build_constraints common::show_packaging_tool_version_and_location install_additional_dependencies @@ -1810,6 +1866,7 @@ ARG CONSTRAINTS_GITHUB_REPOSITORY="apache/airflow" ARG AIRFLOW_CONSTRAINTS_MODE="constraints-source-providers" ARG AIRFLOW_CONSTRAINTS_REFERENCE="" ARG AIRFLOW_CONSTRAINTS_LOCATION="" +ARG AIRFLOW_BUILD_CONSTRAINTS_LOCATION="" ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main" # By default fallback to installation without constraints because in CI image it should always be tried ARG AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION="true" @@ -1842,6 +1899,7 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\ AIRFLOW_CONSTRAINTS_MODE=${AIRFLOW_CONSTRAINTS_MODE} \ AIRFLOW_CONSTRAINTS_REFERENCE=${AIRFLOW_CONSTRAINTS_REFERENCE} \ AIRFLOW_CONSTRAINTS_LOCATION=${AIRFLOW_CONSTRAINTS_LOCATION} \ + AIRFLOW_BUILD_CONSTRAINTS_LOCATION=${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} \ AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION=${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} \ DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH} \ AIRFLOW_CI_BUILD_EPOCH=${AIRFLOW_CI_BUILD_EPOCH} \ diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh index d8ebb2e261f6d..b2597f30e2c83 100644 --- a/scripts/docker/common.sh +++ b/scripts/docker/common.sh @@ -134,6 +134,56 @@ function common::get_constraints_location() { fi } +# The resolved flags are consumed by scripts that source common.sh. +# shellcheck disable=SC2034 +function common::resolve_build_constraints() { + BUILD_CONSTRAINTS_INSTALL_FLAGS=() + if [[ -z ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION=} ]]; then + return + fi + + local target="${HOME}/build-constraints.txt" + if [[ ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} =~ ^https?:// ]]; then + echo + echo "${COLOR_BLUE}Downloading build constraints from ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} to ${target}${COLOR_RESET}" + echo + rm -f "${target}" + if ! curl -sSf -o "${target}" "${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}"; then + rm -f "${target}" + echo + echo "${COLOR_RED}Build constraints file not found at explicitly set ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}${COLOR_RESET}" + echo + exit 1 + fi + else + if [[ ! -f ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} || ! -s ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} ]]; then + echo + echo "${COLOR_RED}Build constraints must be a non-empty file: ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}${COLOR_RESET}" + echo + exit 1 + fi + echo + echo "${COLOR_BLUE}Copying build constraints from ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} to ${target}${COLOR_RESET}" + echo + if [[ ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION} != "${target}" ]]; then + cp "${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}" "${target}" + fi + fi + + if [[ ! -s ${target} ]]; then + rm -f "${target}" + echo + echo "${COLOR_RED}Build constraints file is empty: ${AIRFLOW_BUILD_CONSTRAINTS_LOCATION}${COLOR_RESET}" + echo + exit 1 + fi + if [[ ${PACKAGING_TOOL} == "uv" ]]; then + BUILD_CONSTRAINTS_INSTALL_FLAGS=(--build-constraints "${target}") + else + BUILD_CONSTRAINTS_INSTALL_FLAGS=(--build-constraint "${target}") + fi +} + function common::show_packaging_tool_version_and_location() { echo "PATH=${PATH}" echo "Installed pip: $(pip --version): $(which pip)" diff --git a/scripts/docker/install_additional_dependencies.sh b/scripts/docker/install_additional_dependencies.sh index 5b59e9ecb633e..fed3e0c1d6a48 100644 --- a/scripts/docker/install_additional_dependencies.sh +++ b/scripts/docker/install_additional_dependencies.sh @@ -32,6 +32,7 @@ function install_additional_dependencies() { set -x ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + "${BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}" \ ${ADDITIONAL_PYTHON_DEPS} set +x common::install_packaging_tools @@ -47,6 +48,7 @@ function install_additional_dependencies() { set -x ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + "${BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}" \ ${ADDITIONAL_PYTHON_DEPS} set +x common::install_packaging_tools @@ -62,6 +64,7 @@ common::get_colors common::get_packaging_tool common::get_airflow_version_specification common::get_constraints_location +common::resolve_build_constraints common::show_packaging_tool_version_and_location install_additional_dependencies diff --git a/scripts/docker/install_airflow_when_building_images.sh b/scripts/docker/install_airflow_when_building_images.sh index 772465678a8c4..3c8198691a5b5 100644 --- a/scripts/docker/install_airflow_when_building_images.sh +++ b/scripts/docker/install_airflow_when_building_images.sh @@ -95,7 +95,8 @@ function install_from_sources() { } function install_from_external_spec() { - local installation_command_flags + common::resolve_build_constraints + local installation_command_flags if [[ ${AIRFLOW_INSTALLATION_METHOD} == "apache-airflow" ]]; then installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" else @@ -117,14 +118,18 @@ function install_from_external_spec() { echo "${COLOR_BLUE}Installing all packages with highest resolutions. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}" echo set -x - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} \ + ${ADDITIONAL_PIP_INSTALL_FLAGS} "${BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}" \ + ${installation_command_flags} set +x else echo echo "${COLOR_BLUE}Installing all packages with constraints. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}" echo set -x - if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then + if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + "${BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}" ${installation_command_flags} \ + --constraint "${HOME}/constraints.txt"; then set +x if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then echo diff --git a/scripts/docker/install_from_docker_context_files.sh b/scripts/docker/install_from_docker_context_files.sh index 089b0f2d74bd4..50c8505a524ea 100644 --- a/scripts/docker/install_from_docker_context_files.sh +++ b/scripts/docker/install_from_docker_context_files.sh @@ -44,9 +44,11 @@ function install_airflow_and_providers_from_docker_context_files(){ # This is needed to get distribution names for local context distributions if [[ -f "${HOME}/constraints.txt" ]]; then - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} --constraint ${HOME}/constraints.txt packaging + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + "${BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}" --constraint ${HOME}/constraints.txt packaging else - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} packaging + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + "${BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}" packaging fi if [[ -n ${AIRFLOW_EXTRAS=} ]]; then @@ -125,6 +127,7 @@ function install_airflow_and_providers_from_docker_context_files(){ set -x if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + "${BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}" \ "${flags[@]}" \ "${install_airflow_distribution[@]}" "${install_airflow_core_distribution[@]}" "${airflow_distributions[@]}"; then set +x @@ -166,6 +169,7 @@ function install_all_other_distributions_from_docker_context_files() { if [[ -n "${reinstalling_other_distributions}" ]]; then set -x ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + "${BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}" \ --force-reinstall --no-deps --no-index ${reinstalling_other_distributions} common::install_packaging_tools set +x @@ -176,6 +180,7 @@ common::get_colors common::get_packaging_tool common::get_airflow_version_specification common::get_constraints_location +common::resolve_build_constraints common::show_packaging_tool_version_and_location install_airflow_and_providers_from_docker_context_files diff --git a/scripts/in_container/install_airflow_and_providers.py b/scripts/in_container/install_airflow_and_providers.py index 4f203e8c6c3f0..8e14f0984f5bc 100755 --- a/scripts/in_container/install_airflow_and_providers.py +++ b/scripts/in_container/install_airflow_and_providers.py @@ -23,6 +23,8 @@ import re import shutil import sys +import urllib.error +import urllib.request from functools import cache from pathlib import Path from typing import NamedTuple @@ -201,6 +203,48 @@ def get_airflow_constraints_location( ) +def _download_build_constraints(url: str, target: Path) -> bool: + try: + urllib.request.urlretrieve(url, target) + except (OSError, urllib.error.URLError): + return False + return True + + +def resolve_build_constraints_file(build_constraints_location: str | None) -> Path | None: + """Resolve an explicitly configured build constraints location.""" + if not build_constraints_location: + return None + + if build_constraints_location.startswith(("http://", "https://")): + target = Path(os.environ["HOME"]) / "build-constraints.txt" + target.unlink(missing_ok=True) + if not _download_build_constraints(build_constraints_location, target) or not ( + target.is_file() and target.stat().st_size > 0 + ): + target.unlink(missing_ok=True) + console.print( + f"[red]Build constraints file is unavailable or empty at explicitly set " + f"{build_constraints_location}" + ) + sys.exit(1) + return target + + local_path = Path(build_constraints_location).expanduser() + if not local_path.is_file() or local_path.stat().st_size == 0: + console.print( + f"[red]Build constraints must be an existing non-empty file: {build_constraints_location}" + ) + sys.exit(1) + return local_path + + +def _get_build_constraints_flags(build_constraints_file: Path | None) -> list[str]: + if build_constraints_file and build_constraints_file.is_file() and build_constraints_file.stat().st_size: + return ["--build-constraints", str(build_constraints_file)] + return [] + + def get_providers_constraints_location( default_constraints_branch: str, github_repository: str, @@ -257,6 +301,7 @@ class InstallationSpec(NamedTuple): airflow_task_sdk_distribution: str | None airflow_ctl_distribution: str | None airflow_ctl_constraints_location: str | None + build_constraints_file: Path | None compile_ui_assets: bool | None mount_ui_dist: bool provider_distributions: list[str] @@ -336,6 +381,7 @@ def find_installation_spec( airflow_constraints_location: str | None, airflow_constraints_reference: str, airflow_extras: str, + build_constraints_location: str | None, install_airflow_with_constraints: bool, default_constraints_branch: str, github_repository: str, @@ -562,6 +608,7 @@ def find_installation_spec( airflow_task_sdk_distribution=airflow_task_sdk_distribution, airflow_ctl_distribution=airflow_ctl_distribution, airflow_ctl_constraints_location=airflow_ctl_constraints_location, + build_constraints_file=resolve_build_constraints_file(build_constraints_location), compile_ui_assets=compile_ui_assets, mount_ui_dist=mount_ui_dist, provider_distributions=provider_distributions_list, @@ -849,6 +896,11 @@ def check_mounted_ui_dist(dist_prefix: str, host_source_prefix: str): envvar="AIRFLOW_CONSTRAINTS_REFERENCE", help="Airflow constraints reference constraints reference: constraints-(BRANCH or TAG)", ) +@click.option( + "--build-constraints-location", + envvar="AIRFLOW_BUILD_CONSTRAINTS_LOCATION", + help="Explicit build constraints URL or local non-empty file.", +) @click.option( "--airflow-extras", default="", @@ -971,6 +1023,7 @@ def install_airflow_and_providers( airflow_constraints_location: str, airflow_constraints_reference: str, airflow_extras: str, + build_constraints_location: str | None, default_constraints_branch: str, github_actions: bool, github_repository: str, @@ -1004,6 +1057,7 @@ def install_airflow_and_providers( airflow_constraints_location=airflow_constraints_location, airflow_constraints_reference=airflow_constraints_reference, airflow_extras=airflow_extras, + build_constraints_location=build_constraints_location, install_airflow_with_constraints=install_airflow_with_constraints, default_constraints_branch=default_constraints_branch, github_repository=github_repository, @@ -1156,6 +1210,7 @@ def _install_airflow_and_optionally_providers_together( for provider_package in installation_spec.provider_distributions: base_install_cmd.append(provider_package) install_providers_command = base_install_cmd.copy() + install_providers_command.extend(_get_build_constraints_flags(installation_spec.build_constraints_file)) if installation_spec.provider_constraints_location: console.print( f"[bright_blue]Installing with provider constraints: {installation_spec.provider_constraints_location}\n" @@ -1170,7 +1225,7 @@ def _install_airflow_and_optionally_providers_together( ) run_command(base_install_cmd, github_actions=github_actions, check=True) else: - run_command(base_install_cmd, github_actions=github_actions, check=True) + run_command(install_providers_command, github_actions=github_actions, check=True) def _install_airflow_ctl_with_constraints(installation_spec: InstallationSpec, github_actions: bool): @@ -1191,6 +1246,7 @@ def _install_airflow_ctl_with_constraints(installation_spec: InstallationSpec, g if installation_spec.airflow_distribution: base_install_airflow_ctl_cmd.append(installation_spec.airflow_distribution) install_airflow_ctl_cmd = base_install_airflow_ctl_cmd.copy() + install_airflow_ctl_cmd.extend(_get_build_constraints_flags(installation_spec.build_constraints_file)) if installation_spec.airflow_ctl_constraints_location: console.print(f"[bright_blue]Use constraints: {installation_spec.airflow_ctl_constraints_location}") install_airflow_ctl_cmd.extend(["--constraint", installation_spec.airflow_ctl_constraints_location]) @@ -1204,7 +1260,7 @@ def _install_airflow_ctl_with_constraints(installation_spec: InstallationSpec, g run_command(base_install_airflow_ctl_cmd, github_actions=github_actions, check=True) else: console.print() - run_command(base_install_airflow_ctl_cmd, github_actions=github_actions, check=True) + run_command(install_airflow_ctl_cmd, github_actions=github_actions, check=True) def _install_only_airflow_airflow_core_task_sdk_with_constraints( @@ -1256,6 +1312,7 @@ def _install_only_airflow_airflow_core_task_sdk_with_constraints( ) console.print() install_airflow_cmd = base_install_airflow_cmd.copy() + install_airflow_cmd.extend(_get_build_constraints_flags(installation_spec.build_constraints_file)) if installation_spec.airflow_constraints_location: console.print(f"[bright_blue]Use constraints: {installation_spec.airflow_constraints_location}") install_airflow_cmd.extend(["--constraint", installation_spec.airflow_constraints_location]) diff --git a/scripts/in_container/install_development_dependencies.py b/scripts/in_container/install_development_dependencies.py index cabd03ed2bc95..de61e4a419bb2 100755 --- a/scripts/in_container/install_development_dependencies.py +++ b/scripts/in_container/install_development_dependencies.py @@ -32,6 +32,7 @@ import sys from in_container_utils import AIRFLOW_ROOT_PATH, click, console, run_command +from install_airflow_and_providers import _get_build_constraints_flags, resolve_build_constraints_file from packaging.requirements import Requirement @@ -42,6 +43,11 @@ envvar="CONSTRAINT", help="Constraints file or url to use for installation", ) +@click.option( + "--build-constraints-location", + envvar="AIRFLOW_BUILD_CONSTRAINTS_LOCATION", + help="Explicit build constraints URL or local non-empty file.", +) @click.option( "--github-actions", is_flag=True, @@ -50,7 +56,9 @@ envvar="GITHUB_ACTIONS", help="Running in GitHub Actions", ) -def install_development_dependencies(constraint: str, github_actions: bool): +def install_development_dependencies( + constraint: str, build_constraints_location: str | None, github_actions: bool +): pyproject_toml_of_devel_commons = (AIRFLOW_ROOT_PATH / "devel-common" / "pyproject.toml").read_text() development_dependencies: list[str] = [] in_devel_common_dependencies = False @@ -88,7 +96,16 @@ def install_development_dependencies(constraint: str, github_actions: bool): path.parent.as_posix() for path in (AIRFLOW_ROOT_PATH / "shared").glob("*/pyproject.toml") ) development_dependencies.extend(shared_distributions) - command = ["uv", "pip", "install", *development_dependencies, "--constraints", constraint] + build_constraints_file = resolve_build_constraints_file(build_constraints_location) + command = [ + "uv", + "pip", + "install", + *development_dependencies, + *_get_build_constraints_flags(build_constraints_file), + "--constraints", + constraint, + ] result = run_command(command, check=False, github_actions=github_actions) if result.returncode != 0: console.print("[yellow]Failed to install development dependencies with constraints[/]\n") diff --git a/scripts/tests/in_container/test_build_constraints_installation.py b/scripts/tests/in_container/test_build_constraints_installation.py new file mode 100644 index 0000000000000..6ceb142c006c8 --- /dev/null +++ b/scripts/tests/in_container/test_build_constraints_installation.py @@ -0,0 +1,835 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import re +import subprocess +import sys +import zipfile +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +import install_development_dependencies as install_dev_deps +import pytest +from install_airflow_and_providers import ( + InstallationSpec, + _get_build_constraints_flags, + _install_airflow_and_optionally_providers_together, + _install_airflow_ctl_with_constraints, + _install_only_airflow_airflow_core_task_sdk_with_constraints, + find_installation_spec, + resolve_build_constraints_file, +) + +AIRFLOW_ROOT = Path(__file__).resolve().parents[3] +COMMON_SCRIPT = AIRFLOW_ROOT / "scripts/docker/common.sh" +INSTALL_AIRFLOW_SCRIPT = AIRFLOW_ROOT / "scripts/docker/install_airflow_when_building_images.sh" +INSTALL_ADDITIONAL_SCRIPT = AIRFLOW_ROOT / "scripts/docker/install_additional_dependencies.sh" + + +class TestResolveBuildConstraintsFile: + @mock.patch("install_airflow_and_providers._download_build_constraints", autospec=True) + def test_unset_location_is_noop(self, download): + assert resolve_build_constraints_file(None) is None + download.assert_not_called() + + def test_returns_explicit_local_file(self, tmp_path): + build_constraints = tmp_path / "build-constraints.txt" + build_constraints.write_text("setuptools==80.0.0\n") + + assert resolve_build_constraints_file(str(build_constraints)) == build_constraints + + @pytest.mark.parametrize("path_kind", ["missing", "empty", "directory"]) + def test_rejects_invalid_explicit_local_path(self, tmp_path, path_kind): + build_constraints = tmp_path / "build-constraints.txt" + if path_kind == "empty": + build_constraints.write_text("") + elif path_kind == "directory": + build_constraints.mkdir() + + with pytest.raises(SystemExit): + resolve_build_constraints_file(str(build_constraints)) + + @mock.patch("install_airflow_and_providers._download_build_constraints", autospec=True) + def test_downloads_explicit_url_to_deterministic_path(self, download, tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + + def write_build_constraints(url, target): + target.write_text("setuptools==80.0.0\n") + return True + + download.side_effect = write_build_constraints + + result = resolve_build_constraints_file("https://example.invalid/build-constraints.txt") + + assert result == tmp_path / "build-constraints.txt" + assert result.read_text() == "setuptools==80.0.0\n" + download.assert_called_once_with( + "https://example.invalid/build-constraints.txt", + tmp_path / "build-constraints.txt", + ) + + @pytest.mark.parametrize("downloaded_content", [None, ""]) + @mock.patch("install_airflow_and_providers._download_build_constraints", autospec=True) + def test_rejects_failed_or_empty_explicit_download( + self, download, downloaded_content, tmp_path, monkeypatch + ): + monkeypatch.setenv("HOME", str(tmp_path)) + target = tmp_path / "build-constraints.txt" + target.write_text("stale==1\n") + + def download_result(url, output): + if downloaded_content is not None: + output.write_text(downloaded_content) + return True + return False + + download.side_effect = download_result + + with pytest.raises(SystemExit): + resolve_build_constraints_file("https://example.invalid/build-constraints.txt") + + assert not target.exists() + + +@pytest.mark.parametrize("path_kind", ["none", "missing", "empty", "directory"]) +def test_get_build_constraints_flags_skips_invalid_files(tmp_path, path_kind): + build_constraints: Path | None = tmp_path / "build-constraints.txt" + if path_kind == "none": + build_constraints = None + elif path_kind == "empty": + build_constraints.write_text("") + elif path_kind == "directory": + build_constraints.mkdir() + + assert _get_build_constraints_flags(build_constraints) == [] + + +def test_get_build_constraints_flags_uses_uv_plural_flag(tmp_path): + build_constraints = tmp_path / "build-constraints.txt" + build_constraints.write_text("setuptools==80.0.0\n") + + assert _get_build_constraints_flags(build_constraints) == [ + "--build-constraints", + str(build_constraints), + ] + + +def _installation_spec( + build_constraints_file: Path | None, + *, + airflow_constraints_location: str | None = "runtime-constraints.txt", + airflow_ctl_constraints_location: str | None = "ctl-constraints.txt", + provider_constraints_location: str | None = "provider-constraints.txt", +) -> InstallationSpec: + return InstallationSpec( + airflow_distribution="apache-airflow==3.2.0", + airflow_core_distribution="apache-airflow-core==3.2.0", + airflow_constraints_location=airflow_constraints_location, + airflow_task_sdk_distribution="apache-airflow-task-sdk==1.2.0", + airflow_ctl_distribution="apache-airflow-ctl==1.2.0", + airflow_ctl_constraints_location=airflow_ctl_constraints_location, + build_constraints_file=build_constraints_file, + compile_ui_assets=False, + mount_ui_dist=False, + provider_distributions=["apache-airflow-providers-standard==1.0.0"], + provider_constraints_location=provider_constraints_location, + ) + + +@mock.patch("install_airflow_and_providers.resolve_build_constraints_file", autospec=True) +def test_find_installation_spec_resolves_only_explicit_location(resolve, tmp_path): + build_constraints = tmp_path / "build-constraints.txt" + resolve.return_value = build_constraints + + result = find_installation_spec( + airflow_constraints_mode="constraints", + airflow_constraints_location=None, + airflow_constraints_reference="", + airflow_extras="", + build_constraints_location="https://example.invalid/build-constraints.txt", + install_airflow_with_constraints=False, + default_constraints_branch="constraints-main", + github_repository="apache/airflow", + install_selected_providers="", + distribution_format="wheel", + providers_constraints_mode="constraints", + providers_constraints_location=None, + providers_constraints_reference="", + providers_skip_constraints=True, + python_version="3.12", + use_airflow_version="", + use_distributions_from_dist=False, + mount_ui_dist=False, + ) + + assert result.build_constraints_file == build_constraints + resolve.assert_called_once_with("https://example.invalid/build-constraints.txt") + + +class TestPythonInstallCommandAssembly: + @mock.patch("install_airflow_and_providers.run_command", autospec=True) + def test_provider_install_includes_build_constraints_without_runtime_constraints( + self, run_command, tmp_path + ): + build_constraints = tmp_path / "build-constraints.txt" + build_constraints.write_text("setuptools==80.0.0\n") + run_command.return_value = SimpleNamespace(returncode=0) + + _install_airflow_and_optionally_providers_together( + _installation_spec( + build_constraints, + provider_constraints_location=None, + ), + github_actions=False, + ) + + command = run_command.call_args.args[0] + assert command[command.index("--build-constraints") + 1] == str(build_constraints) + assert "--constraint" not in command + + @mock.patch("install_airflow_and_providers.run_command", autospec=True) + def test_provider_fallback_removes_both_constraints(self, run_command, tmp_path): + build_constraints = tmp_path / "build-constraints.txt" + build_constraints.write_text("setuptools==80.0.0\n") + run_command.side_effect = [ + SimpleNamespace(returncode=1), + SimpleNamespace(returncode=0), + ] + + _install_airflow_and_optionally_providers_together( + _installation_spec(build_constraints), + github_actions=False, + ) + + constrained, fallback = [call.args[0] for call in run_command.call_args_list] + assert "--constraint" in constrained + assert "--build-constraints" in constrained + assert "--constraint" not in fallback + assert "--build-constraints" not in fallback + + @mock.patch("install_airflow_and_providers.run_command", autospec=True) + def test_ctl_install_includes_build_constraints_without_runtime_constraints(self, run_command, tmp_path): + build_constraints = tmp_path / "build-constraints.txt" + build_constraints.write_text("setuptools==80.0.0\n") + + _install_airflow_ctl_with_constraints( + _installation_spec( + build_constraints, + airflow_ctl_constraints_location=None, + ), + github_actions=False, + ) + + command = run_command.call_args.args[0] + assert command[command.index("--build-constraints") + 1] == str(build_constraints) + assert "--constraint" not in command + + @mock.patch("install_airflow_and_providers.run_command", autospec=True) + def test_ctl_fallback_removes_both_constraints(self, run_command, tmp_path): + build_constraints = tmp_path / "build-constraints.txt" + build_constraints.write_text("setuptools==80.0.0\n") + run_command.side_effect = [ + SimpleNamespace(returncode=1), + SimpleNamespace(returncode=0), + ] + + _install_airflow_ctl_with_constraints( + _installation_spec(build_constraints), + github_actions=False, + ) + + constrained, fallback = [call.args[0] for call in run_command.call_args_list] + assert "--constraint" in constrained + assert "--build-constraints" in constrained + assert "--constraint" not in fallback + assert "--build-constraints" not in fallback + + @mock.patch("install_airflow_and_providers.run_command", autospec=True) + def test_separate_airflow_fallback_removes_both_constraints(self, run_command, tmp_path): + build_constraints = tmp_path / "build-constraints.txt" + build_constraints.write_text("setuptools==80.0.0\n") + run_command.side_effect = [ + SimpleNamespace(returncode=1), + SimpleNamespace(returncode=0), + ] + + _install_only_airflow_airflow_core_task_sdk_with_constraints( + _installation_spec(build_constraints), + github_actions=False, + ) + + constrained, fallback = [call.args[0] for call in run_command.call_args_list] + assert "--constraint" in constrained + assert "--build-constraints" in constrained + assert "--constraint" not in fallback + assert "--build-constraints" not in fallback + + +class TestInstallDevelopmentDependencies: + def _prepare_airflow_root(self, tmp_path): + airflow_root = tmp_path / "airflow" + (airflow_root / "devel-common").mkdir(parents=True) + (airflow_root / "generated").mkdir() + (airflow_root / "devel-common" / "pyproject.toml").write_text( + """[project] +dependencies = [ + "pendulum>=3", + "pytest>=8; python_version >= '3.0'", + "skip-me>=1; python_version < '1.0'", +] +""" + ) + (airflow_root / "generated" / "provider_dependencies.json").write_text( + json.dumps({"amazon": {"devel-deps": ["boto3>=1"]}}) + ) + return airflow_root + + def _run_install( + self, + monkeypatch, + airflow_root, + constraint, + build_constraints_location, + returncodes=(0,), + ): + calls = [] + returncode_iter = iter(returncodes) + + def run_command(command, **kwargs): + calls.append((command, kwargs)) + return SimpleNamespace(returncode=next(returncode_iter)) + + monkeypatch.setattr(install_dev_deps, "AIRFLOW_ROOT_PATH", airflow_root) + monkeypatch.setattr(install_dev_deps, "run_command", run_command) + callback = install_dev_deps.install_development_dependencies.callback + assert callback + callback( + constraint=str(constraint), + build_constraints_location=build_constraints_location, + github_actions=False, + ) + return calls + + def test_adds_explicit_build_constraints(self, tmp_path, monkeypatch): + airflow_root = self._prepare_airflow_root(tmp_path) + build_constraints = tmp_path / "build-constraints.txt" + build_constraints.write_text("setuptools==80.0.0\n") + + calls = self._run_install( + monkeypatch, + airflow_root, + tmp_path / "constraints.txt", + str(build_constraints), + ) + + command = calls[0][0] + assert "pendulum>=3" in command + assert "pytest>=8" in command + assert "skip-me>=1" not in command + assert "boto3>=1" in command + assert command[command.index("--build-constraints") + 1] == str(build_constraints) + + def test_unset_location_preserves_existing_command(self, tmp_path, monkeypatch): + airflow_root = self._prepare_airflow_root(tmp_path) + constraint = tmp_path / "constraints.txt" + + calls = self._run_install(monkeypatch, airflow_root, constraint, None) + + command = calls[0][0] + assert "--build-constraints" not in command + assert command[-2:] == ["--constraints", str(constraint)] + + def test_fallback_removes_both_constraints(self, tmp_path, monkeypatch): + airflow_root = self._prepare_airflow_root(tmp_path) + build_constraints = tmp_path / "build-constraints.txt" + build_constraints.write_text("setuptools==80.0.0\n") + + calls = self._run_install( + monkeypatch, + airflow_root, + tmp_path / "constraints.txt", + str(build_constraints), + returncodes=(1, 0), + ) + + constrained, fallback = [command for command, _ in calls] + assert "--constraints" in constrained + assert "--build-constraints" in constrained + assert "--constraints" not in fallback + assert "--build-constraints" not in fallback + + +def _run_bash(script: str, *, env: dict[str, str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["bash", "-c", script], + cwd=AIRFLOW_ROOT, + env={**os.environ, **env}, + text=True, + capture_output=True, + check=False, + ) + + +class TestShellBuildConstraints: + def test_unset_location_does_not_use_stale_file_or_network(self, tmp_path): + stale = tmp_path / "build-constraints.txt" + stale.write_text("stale==1\n") + curl_log = tmp_path / "curl.log" + script = f""" +source "{COMMON_SCRIPT}" +common::get_colors +PACKAGING_TOOL=uv +curl() {{ echo called >> "$CURL_LOG"; return 1; }} +unset AIRFLOW_BUILD_CONSTRAINTS_LOCATION +common::resolve_build_constraints +printf '%s\n' "${{#BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}}" +""" + + result = _run_bash( + script, + env={"HOME": str(tmp_path), "CURL_LOG": str(curl_log)}, + ) + + assert result.returncode == 0, result.stderr + assert result.stdout.rstrip().endswith("0") + assert not curl_log.exists() + assert stale.read_text() == "stale==1\n" + + @pytest.mark.parametrize( + ("packaging_tool", "expected_flag"), + [("uv", "--build-constraints"), ("pip", "--build-constraint")], + ) + def test_local_file_uses_tool_specific_flag(self, tmp_path, packaging_tool, expected_flag): + source_dir = tmp_path / "path with spaces" + source_dir.mkdir() + source = source_dir / "build-constraints.txt" + source.write_text("setuptools==80.0.0\n") + script = f""" +source "{COMMON_SCRIPT}" +common::get_colors +PACKAGING_TOOL="$PACKAGING_TOOL" +common::resolve_build_constraints +printf '<%s>\n' "${{BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}}" +""" + + result = _run_bash( + script, + env={ + "HOME": str(tmp_path), + "PACKAGING_TOOL": packaging_tool, + "AIRFLOW_BUILD_CONSTRAINTS_LOCATION": str(source), + }, + ) + + assert result.returncode == 0, result.stderr + assert f"<{expected_flag}>" in result.stdout + assert f"<{tmp_path / 'build-constraints.txt'}>" in result.stdout + + def test_explicit_url_download_is_used(self, tmp_path): + source = tmp_path / "download-source.txt" + source.write_text("setuptools==80.0.0\n") + script = f""" +source "{COMMON_SCRIPT}" +common::get_colors +PACKAGING_TOOL=uv +curl() {{ + local output + while [[ $# -gt 0 ]]; do + if [[ $1 == "-o" ]]; then output=$2; shift 2; else shift; fi + done + cp "$DOWNLOAD_SOURCE" "$output" +}} +common::resolve_build_constraints +printf '<%s>\n' "${{BUILD_CONSTRAINTS_INSTALL_FLAGS[@]}}" +""" + + result = _run_bash( + script, + env={ + "HOME": str(tmp_path), + "DOWNLOAD_SOURCE": str(source), + "AIRFLOW_BUILD_CONSTRAINTS_LOCATION": "https://example.invalid/build-constraints.txt", + }, + ) + + assert result.returncode == 0, result.stderr + assert "<--build-constraints>" in result.stdout + assert (tmp_path / "build-constraints.txt").read_text() == "setuptools==80.0.0\n" + + @pytest.mark.parametrize("path_kind", ["missing", "empty", "directory"]) + def test_invalid_explicit_local_path_fails(self, tmp_path, path_kind): + source = tmp_path / "build-constraints-source.txt" + if path_kind == "empty": + source.write_text("") + elif path_kind == "directory": + source.mkdir() + script = f""" +source "{COMMON_SCRIPT}" +common::get_colors +PACKAGING_TOOL=uv +common::resolve_build_constraints +""" + + result = _run_bash( + script, + env={ + "HOME": str(tmp_path), + "AIRFLOW_BUILD_CONSTRAINTS_LOCATION": str(source), + }, + ) + + assert result.returncode != 0 + assert "non-empty file" in result.stdout + + def test_failed_url_does_not_leave_stale_target(self, tmp_path): + target = tmp_path / "build-constraints.txt" + target.write_text("stale==1\n") + script = f""" +source "{COMMON_SCRIPT}" +common::get_colors +PACKAGING_TOOL=uv +curl() {{ return 22; }} +common::resolve_build_constraints +""" + + result = _run_bash( + script, + env={ + "HOME": str(tmp_path), + "AIRFLOW_BUILD_CONSTRAINTS_LOCATION": "https://example.invalid/build-constraints.txt", + }, + ) + + assert result.returncode != 0 + assert not target.exists() + + def test_runtime_constraints_fallback_removes_build_constraints(self, tmp_path): + build_constraints = tmp_path / "explicit-build-constraints.txt" + build_constraints.write_text("setuptools==80.0.0\n") + command_log = tmp_path / "commands.log" + failed_once = tmp_path / "failed" + script = f""" +source "{COMMON_SCRIPT}" +source <(sed -n '/^function install_from_external_spec()/,/^}}$/p' "{INSTALL_AIRFLOW_SCRIPT}") +common::get_colors +record_command() {{ + printf '%s\n' "$*" >> "$COMMAND_LOG" + if [[ ! -f "$FAILED_ONCE" ]]; then touch "$FAILED_ONCE"; return 1; fi + return 0 +}} +PACKAGING_TOOL=uv +PACKAGING_TOOL_CMD=record_command +EXTRA_INSTALL_FLAGS="" +UPGRADE_TO_HIGHEST_RESOLUTION="" +UPGRADE_IF_NEEDED="" +ADDITIONAL_PIP_INSTALL_FLAGS="" +EXTRA_UNINSTALL_FLAGS="" +AIRFLOW_INSTALLATION_METHOD=apache-airflow +AIRFLOW_EXTRAS="" +AIRFLOW_VERSION_SPECIFICATION="" +AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION=true +UPGRADE_RANDOM_INDICATOR_STRING="" +install_from_external_spec +""" + + result = _run_bash( + script, + env={ + "HOME": str(tmp_path), + "COMMAND_LOG": str(command_log), + "FAILED_ONCE": str(failed_once), + "AIRFLOW_BUILD_CONSTRAINTS_LOCATION": str(build_constraints), + }, + ) + + assert result.returncode == 0, result.stderr + constrained, fallback = command_log.read_text().splitlines() + assert "--constraint" in constrained + assert "--build-constraints" in constrained + assert "--constraint" not in fallback + assert "--build-constraints" not in fallback + + def test_source_install_does_not_receive_build_constraints(self, tmp_path): + build_constraints = tmp_path / "build-constraints.txt" + build_constraints.write_text("setuptools==80.0.0\n") + command_log = tmp_path / "commands.log" + script = f""" +source "{COMMON_SCRIPT}" +source <(sed -n '/^function install_from_sources()/,/^}}$/p' "{INSTALL_AIRFLOW_SCRIPT}") +common::get_colors +uv() {{ printf '%s\n' "$*" >> "$COMMAND_LOG"; }} +PACKAGING_TOOL_CMD=uv +VIRTUAL_ENV="" +UPGRADE_RANDOM_INDICATOR_STRING=upgrade +install_from_sources +""" + + result = _run_bash( + script, + env={ + "HOME": str(tmp_path), + "COMMAND_LOG": str(command_log), + "AIRFLOW_BUILD_CONSTRAINTS_LOCATION": str(build_constraints), + }, + ) + + assert result.returncode == 0, result.stderr + command = command_log.read_text() + assert "sync" in command + assert "--build-constraints" not in command + assert "--build-constraint" not in command + + def test_additional_dependencies_receive_explicit_build_constraints(self, tmp_path): + build_constraints = tmp_path / "explicit-build-constraints.txt" + build_constraints.write_text("setuptools==80.0.0\n") + command_log = tmp_path / "commands.log" + script = f""" +source "{COMMON_SCRIPT}" +source <(sed -n '/^function install_additional_dependencies()/,/^}}$/p' "{INSTALL_ADDITIONAL_SCRIPT}") +common::get_colors +record_command() {{ printf '%s\n' "$*" >> "$COMMAND_LOG"; }} +common::install_packaging_tools() {{ :; }} +pip() {{ :; }} +PACKAGING_TOOL=uv +PACKAGING_TOOL_CMD=record_command +EXTRA_INSTALL_FLAGS="" +UPGRADE_TO_HIGHEST_RESOLUTION="" +UPGRADE_IF_NEEDED="" +ADDITIONAL_PIP_INSTALL_FLAGS="" +ADDITIONAL_PYTHON_DEPS="example-package" +UPGRADE_RANDOM_INDICATOR_STRING="" +common::resolve_build_constraints +install_additional_dependencies +""" + + result = _run_bash( + script, + env={ + "HOME": str(tmp_path), + "COMMAND_LOG": str(command_log), + "AIRFLOW_BUILD_CONSTRAINTS_LOCATION": str(build_constraints), + }, + ) + + assert result.returncode == 0, result.stderr + command = command_log.read_text() + assert "--build-constraints" in command + assert "example-package" in command + + +_WHEEL_HELPER_SOURCE = r""" +from __future__ import annotations + +import base64 +import hashlib +import re +import zipfile +from pathlib import Path + + +def _record_line(arcname, payload): + digest = hashlib.sha256(payload).digest() + encoded = base64.urlsafe_b64encode(digest).rstrip(b"=").decode() + return f"{arcname},sha256={encoded},{len(payload)}" + + +def write_wheel(out_dir, *, distribution, version): + escaped = re.sub(r"[^A-Za-z0-9.]+", "_", distribution) + dist_info = f"{escaped}-{version}.dist-info" + wheel_path = Path(out_dir) / f"{escaped}-{version}-py3-none-any.whl" + members = [ + ( + f"{dist_info}/METADATA", + f"Metadata-Version: 2.1\nName: {distribution}\nVersion: {version}\n".encode(), + ), + ( + f"{dist_info}/WHEEL", + b"Wheel-Version: 1.0\nGenerator: airflow-test\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + ), + ] + record = [_record_line(name, payload) for name, payload in members] + record.append(f"{dist_info}/RECORD,,") + members.append((f"{dist_info}/RECORD", ("\n".join(record) + "\n").encode())) + with zipfile.ZipFile(wheel_path, "w", zipfile.ZIP_DEFLATED) as archive: + for name, payload in members: + archive.writestr(name, payload) + return wheel_path +""" + +_BACKEND_SOURCE = r""" +from __future__ import annotations + +import sys +from importlib.metadata import version +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) +from _wheel_helper import write_wheel + + +def _assert_marker_version(): + actual = version("airflow-build-constraint-marker") + if actual != "2.0.0": + raise RuntimeError(f"Expected marker 2.0.0, got {actual}") + + +def get_requires_for_build_wheel(config_settings=None): + return [] + + +def prepare_metadata_for_build_wheel(metadata_directory, config_settings=None): + _assert_marker_version() + target = Path(metadata_directory) / "airflow_build_constraint_subject-0.1.0.dist-info" + target.mkdir(parents=True, exist_ok=True) + (target / "METADATA").write_text( + "Metadata-Version: 2.1\nName: airflow-build-constraint-subject\nVersion: 0.1.0\n" + ) + (target / "WHEEL").write_text( + "Wheel-Version: 1.0\nGenerator: airflow-test\nRoot-Is-Purelib: true\nTag: py3-none-any\n" + ) + return target.name + + +def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): + _assert_marker_version() + return write_wheel( + wheel_directory, + distribution="airflow-build-constraint-subject", + version="0.1.0", + ).name +""" + +_PYPROJECT_TOML = """[build-system] +requires = ["airflow-build-constraint-marker>=2"] +build-backend = "backend" +backend-path = ["."] +""" + + +def _write_wheel(out_dir, *, distribution, version): + escaped = re.sub(r"[^A-Za-z0-9.]+", "_", distribution) + dist_info = f"{escaped}-{version}.dist-info" + wheel_path = Path(out_dir) / f"{escaped}-{version}-py3-none-any.whl" + members = [ + ( + f"{dist_info}/METADATA", + f"Metadata-Version: 2.1\nName: {distribution}\nVersion: {version}\n".encode(), + ), + ( + f"{dist_info}/WHEEL", + b"Wheel-Version: 1.0\nGenerator: airflow-test\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + ), + ] + record = [] + for name, payload in members: + digest = hashlib.sha256(payload).digest() + encoded = base64.urlsafe_b64encode(digest).rstrip(b"=").decode() + record.append(f"{name},sha256={encoded},{len(payload)}") + record.append(f"{dist_info}/RECORD,,") + members.append((f"{dist_info}/RECORD", ("\n".join(record) + "\n").encode())) + with zipfile.ZipFile(wheel_path, "w", zipfile.ZIP_DEFLATED) as archive: + for name, payload in members: + archive.writestr(name, payload) + return wheel_path + + +def _pip_supports_build_constraint(): + result = subprocess.run( + [sys.executable, "-m", "pip", "install", "--help"], + text=True, + capture_output=True, + check=False, + ) + return result.returncode == 0 and "--build-constraint" in result.stdout + + +@pytest.mark.skipif( + not _pip_supports_build_constraint(), + reason="pip does not support --build-constraint", +) +def test_pip_build_constraint_controls_build_isolation(tmp_path): + wheelhouse = tmp_path / "wheelhouse" + wheelhouse.mkdir() + _write_wheel( + wheelhouse, + distribution="airflow-build-constraint-marker", + version="1.0.0", + ) + _write_wheel( + wheelhouse, + distribution="airflow-build-constraint-marker", + version="2.0.0", + ) + subject = tmp_path / "subject" + subject.mkdir() + (subject / "_wheel_helper.py").write_text(_WHEEL_HELPER_SOURCE) + (subject / "backend.py").write_text(_BACKEND_SOURCE) + (subject / "pyproject.toml").write_text(_PYPROJECT_TOML) + good_constraints = tmp_path / "good.txt" + good_constraints.write_text("airflow-build-constraint-marker==2.0.0\n") + bad_constraints = tmp_path / "bad.txt" + bad_constraints.write_text("airflow-build-constraint-marker==1.0.0\n") + base_command = [ + sys.executable, + "-m", + "pip", + "install", + "--isolated", + "--disable-pip-version-check", + "--no-index", + "--find-links", + str(wheelhouse), + ] + + good = subprocess.run( + [ + *base_command, + "--target", + str(tmp_path / "target-good"), + "--build-constraint", + str(good_constraints), + str(subject), + ], + text=True, + capture_output=True, + timeout=180, + check=False, + ) + assert good.returncode == 0, f"stdout:\n{good.stdout}\nstderr:\n{good.stderr}" + + bad = subprocess.run( + [ + *base_command, + "--target", + str(tmp_path / "target-bad"), + "--build-constraint", + str(bad_constraints), + str(subject), + ], + text=True, + capture_output=True, + timeout=180, + check=False, + ) + assert bad.returncode != 0 + assert "airflow-build-constraint-marker" in (bad.stdout + bad.stderr).lower()