diff --git a/.github/workflows/generate-constraints.yml b/.github/workflows/generate-constraints.yml index 5b6a839a4f3d3..6191433d0adda 100644 --- a/.github/workflows/generate-constraints.yml +++ b/.github/workflows/generate-constraints.yml @@ -135,7 +135,9 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: constraints-${{ matrix.python-version }} - path: ./files/constraints-${{ matrix.python-version }}/constraints-*.txt + path: | + ./files/constraints-${{ matrix.python-version }}/constraints-*.txt + ./files/constraints-${{ matrix.python-version }}/build-constraints-*.txt retention-days: 7 if-no-files-found: error - name: "Dependency upgrade summary" diff --git a/dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md b/dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md index 68104fee02d33..0fc216d1e9393 100644 --- a/dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md +++ b/dev/MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md @@ -224,9 +224,9 @@ breeze release-management generate-constraints --airflow-constraints-mode constr AIRFLOW_SOURCES=$(pwd) ``` -The constraints will be generated in `files/constraints-PYTHON_VERSION/constraints-*.txt` files. You need to -check out the right 'constraints-' branch in a separate repository, and then you can copy, commit and push the -generated files. +The constraints will be generated in `files/constraints-PYTHON_VERSION/constraints-*.txt` and +`files/constraints-PYTHON_VERSION/build-constraints-*.txt` files. You need to check out the right +'constraints-' branch in a separate repository, and then you can copy, commit and push the generated files. You need to be a committer, and you have to be authenticated in the apache/airflow repository for your git commands to be able to push the new constraints @@ -234,7 +234,7 @@ git commands to be able to push the new constraints ```bash cd git pull -cp ${AIRFLOW_SOURCES}/files/constraints-*/constraints*.txt . +cp ${AIRFLOW_SOURCES}/files/constraints-*/{constraints,build-constraints}*.txt . git diff git add . git commit -m "Your commit message here" --no-verify diff --git a/dev/breeze/doc/09_release_management_tasks.rst b/dev/breeze/doc/09_release_management_tasks.rst index 63d691950875a..7e408255cff58 100644 --- a/dev/breeze/doc/09_release_management_tasks.rst +++ b/dev/breeze/doc/09_release_management_tasks.rst @@ -647,6 +647,12 @@ Generating constraints Whenever ``pyproject.toml`` gets modified, the CI main job will re-generate constraint files. Those constraint files are stored in separated orphan branches: ``constraints-main``, ``constraints-2-0``. +In addition to runtime constraints, the generation also produces **build constraints** files +(``build-constraints-PYTHON_VERSION.txt``). Build constraints pin the versions of build-time dependencies +(e.g. ``setuptools``, ``hatchling``, ``maturin``) for package compilation via build isolation. +These are published alongside runtime constraints on the same branches, but Airflow installation flows do +not consume them yet. + Those are constraint files as described in detail in the ``_ contributing documentation. diff --git a/scripts/ci/constraints/ci_commit_constraints.sh b/scripts/ci/constraints/ci_commit_constraints.sh index 46de3e4323448..e01831e001688 100755 --- a/scripts/ci/constraints/ci_commit_constraints.sh +++ b/scripts/ci/constraints/ci_commit_constraints.sh @@ -18,8 +18,9 @@ cd constraints || exit 1 git config --local user.email "dev@airflow.apache.org" git config --local user.name "Automated GitHub Actions commit" -git diff --color --exit-code --ignore-matching-lines="^#.*" || \ -git commit --all --message "Updating constraints. GitHub run id:${GITHUB_RUN_ID} +git add -- 'constraints-*.txt' 'build-constraints-*.txt' +git diff --cached --color --exit-code --ignore-matching-lines="^#.*" || \ +git commit --message "Updating constraints. GitHub run id:${GITHUB_RUN_ID} This update in constraints is automatically committed by the CI 'constraints-push' step based on '${GITHUB_REF}' in the '${GITHUB_REPOSITORY}' repository with commit sha ${GITHUB_SHA}. diff --git a/scripts/ci/constraints/ci_diff_constraints.sh b/scripts/ci/constraints/ci_diff_constraints.sh index 336b580b2d840..4d279f3d6a19a 100755 --- a/scripts/ci/constraints/ci_diff_constraints.sh +++ b/scripts/ci/constraints/ci_diff_constraints.sh @@ -15,14 +15,17 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -cp -v ./files/constraints-*/constraints*.txt constraints/ +cp -v ./files/constraints-*/{constraints,build-constraints}*.txt constraints/ cd constraints || exit 1 +git add -- 'constraints-*.txt' 'build-constraints-*.txt' set +e -git diff --color --exit-code --ignore-matching-lines="^#.*" +git diff --cached --color --exit-code --ignore-matching-lines="^#.*" diff_status=$? set -e +git reset HEAD -- . > /dev/null 2>&1 + if [[ ${diff_status} -eq 0 ]]; then echo "No changes in constraints" elif [[ ${diff_status} -eq 1 ]]; then diff --git a/scripts/in_container/run_generate_constraints.py b/scripts/in_container/run_generate_constraints.py index 3b3a839d1e5e5..16ed87f788f45 100755 --- a/scripts/in_container/run_generate_constraints.py +++ b/scripts/in_container/run_generate_constraints.py @@ -18,9 +18,19 @@ from __future__ import annotations import ast +import io import json import os +import re import sys +import tarfile +import tempfile +import time +import urllib.error +import urllib.request +import zipfile +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass from datetime import datetime from functools import cached_property @@ -128,6 +138,27 @@ def _read_dynamic_version_from_init(init_path: Path) -> str: # """ +BUILD_CONSTRAINTS_PREFIX = f""" +# +# This build constraints file was automatically generated on {now} +# for the "{DEFAULT_BRANCH}" branch of Airflow. +# +# Build constraints pin the versions of PEP 517 build-time dependencies +# (setuptools, hatchling, maturin, etc.) used during package installation +# from source distributions (sdists). +# +# Usage with uv: +# uv pip install apache-airflow \\ +# --constraint constraints-{PYTHON_VERSION}.txt \\ +# --build-constraints build-constraints-{PYTHON_VERSION}.txt +# +# Usage with pip (>= 25.3): +# pip install apache-airflow \\ +# --constraint constraints-{PYTHON_VERSION}.txt \\ +# --build-constraint build-constraints-{PYTHON_VERSION}.txt +# +""" + @dataclass class ConfigParams: @@ -469,6 +500,354 @@ def generate_constraints_no_providers(config_params: ConfigParams) -> None: diff_constraints(config_params) +@dataclass +class _LockPackage: + name: str + version: str + sdist_url: str | None + has_universal_wheel: bool + + +def _normalize_package_name(name: str) -> str: + """Normalize a package name according to PEP 503.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def _extract_package_name(requirement: str) -> str: + """Extract and normalize a package name from a PEP 508 requirement.""" + match = re.match(r"^([A-Za-z0-9][-A-Za-z0-9_.]*)", requirement.strip()) + if match: + return _normalize_package_name(match.group(1)) + return _normalize_package_name(requirement.strip()) + + +def _collect_workspace_build_reqs(workspace_root: Path) -> dict[str, set[str]]: + """Collect build-system.requires from authoritative workspace pyproject.toml files.""" + skip_dirs = { + ".git", + ".venv", + "node_modules", + "__pycache__", + ".tox", + ".mypy_cache", + ".ruff_cache", + ".build", + "dist", + "files", + } + build_reqs: dict[str, set[str]] = {} + for pyproject_path in sorted(workspace_root.glob("**/pyproject.toml")): + if any(part in skip_dirs for part in pyproject_path.parts): + continue + try: + with pyproject_path.open("rb") as file: + data = tomllib.load(file) + for requirement in data.get("build-system", {}).get("requires", []): + name = _extract_package_name(requirement) + build_reqs.setdefault(name, set()).add(requirement) + except Exception as error: + console.print(f"[yellow]Warning: failed to parse {pyproject_path}: {error}") + return build_reqs + + +def _parse_uv_lock(uv_lock_path: Path) -> list[_LockPackage]: + """Parse package metadata needed to decide which locked sdists to scan.""" + with uv_lock_path.open("rb") as file: + lock_data = tomllib.load(file) + + packages: list[_LockPackage] = [] + for package in lock_data.get("package", []): + sdist = package.get("sdist", {}) + wheels = package.get("wheels", []) + packages.append( + _LockPackage( + name=package.get("name", ""), + version=package.get("version", ""), + sdist_url=sdist.get("url") if isinstance(sdist, dict) else None, + has_universal_wheel=any("none-any" in wheel.get("url", "") for wheel in wheels), + ) + ) + return packages + + +def _extract_build_reqs_from_tar(sdist_url: str) -> list[str]: + """Read build requirements from the top-level pyproject.toml in a tar.gz sdist.""" + request = urllib.request.Request(sdist_url) + with urllib.request.urlopen(request, timeout=120) as response: + with tarfile.open(fileobj=response, mode="r|gz") as archive: + for index, member in enumerate(archive): + if index > 10000: + break + if member.name.count("/") == 1 and member.name.endswith("/pyproject.toml"): + extracted = archive.extractfile(member) + if extracted: + data = tomllib.loads(extracted.read().decode()) + return data.get("build-system", {}).get("requires", []) + return [] + + +def _extract_build_reqs_from_zip(sdist_url: str) -> list[str]: + """Read build requirements from the top-level pyproject.toml in a zip sdist.""" + request = urllib.request.Request(sdist_url) + with urllib.request.urlopen(request, timeout=120) as response: + with zipfile.ZipFile(io.BytesIO(response.read())) as archive: + for name in archive.namelist(): + if name.count("/") == 1 and name.endswith("/pyproject.toml"): + data = tomllib.loads(archive.read(name).decode()) + return data.get("build-system", {}).get("requires", []) + return [] + + +def _is_transient_download_error(error: BaseException) -> bool: + if isinstance(error, urllib.error.HTTPError): + return error.code in {408, 425, 429, 500, 502, 503, 504} + return isinstance(error, (ConnectionError, TimeoutError, urllib.error.URLError)) + + +def _stream_build_reqs_from_sdist( + sdist_url: str, + *, + attempts: int = 3, + sleep: Callable[[float], None] = time.sleep, +) -> list[str]: + """Download an sdist and extract its build requirements with bounded retry.""" + extract = _extract_build_reqs_from_zip if sdist_url.endswith(".zip") else _extract_build_reqs_from_tar + for attempt in range(1, attempts + 1): + try: + return extract(sdist_url) + except Exception as error: + if not _is_transient_download_error(error) or attempt == attempts: + raise + delay = 2 ** (attempt - 1) + console.print( + f"[yellow]Transient error downloading {sdist_url} " + f"(attempt {attempt}/{attempts}): {error}; retrying in {delay}s" + ) + sleep(delay) + raise RuntimeError(f"Failed to scan {sdist_url} after {attempts} attempts") + + +def _collect_upstream_build_reqs( + uv_lock_path: Path, + cache_path: Path, + max_workers: int = 10, +) -> dict[str, set[str]]: + """Scan locked packages without universal wheels and cache successful results.""" + cache: dict[str, list[str]] = {} + if cache_path.exists(): + try: + cache = json.loads(cache_path.read_text()) + except Exception as error: + console.print( + f"[yellow]Warning: build-constraints cache at {cache_path} is " + f"corrupted ({error}); discarding cache — this run will rescan all packages (slow)." + ) + + packages = _parse_uv_lock(uv_lock_path) + current_keys = { + f"{package.name}=={package.version}" + for package in packages + if not package.has_universal_wheel and package.sdist_url + } + targets = [ + package + for package in packages + if not package.has_universal_wheel + and package.sdist_url + and f"{package.name}=={package.version}" not in cache + ] + + cache_dirty = False + failed: list[str] = [] + if targets: + console.print( + f"[bright_blue]Scanning {len(targets)} package sdists for build dependencies " + f"({len(cache)} cached)..." + ) + with ThreadPoolExecutor(max_workers=max_workers) as pool: + futures = { + pool.submit(_stream_build_reqs_from_sdist, package.sdist_url): package + for package in targets + if package.sdist_url + } + for future in as_completed(futures): + package = futures[future] + key = f"{package.name}=={package.version}" + try: + requirements = future.result() + if requirements: + cache[key] = requirements + else: + console.print( + f"[yellow]Warning: no pyproject.toml in {key}, assuming setuptools and wheel" + ) + cache[key] = ["setuptools", "wheel"] + cache_dirty = True + except Exception as error: + console.print(f"[red]Error scanning {key} ({package.sdist_url}): {error}") + failed.append(f"{key}: {error}") + + if cache_dirty: + cache_path.write_text(json.dumps(cache, indent=2, sort_keys=True)) + console.print(f"[green]Build dependency cache updated: {len(cache)} entries") + + if failed: + raise RuntimeError( + f"Failed to scan build dependencies for {len(failed)} package(s):\n" + + "\n".join(f" - {message}" for message in failed) + ) + + all_reqs: dict[str, set[str]] = {} + for key in current_keys: + for requirement in cache.get(key, []): + name = _extract_package_name(requirement) + all_reqs.setdefault(name, set()).add(requirement) + return all_reqs + + +def _is_exact_pin(requirement: str) -> bool: + """Return whether a requirement is one exact version pin.""" + requirement_without_marker = requirement.split(";")[0].strip() + return bool( + re.match( + r"^[A-Za-z0-9][-A-Za-z0-9_.]*\s*==\s*[^\*!=,]+$", + requirement_without_marker, + ) + ) + + +def _find_conflicting_build_requirement(stderr: str, known_names: set[str]) -> str | None: + candidate_pattern = re.compile( + r"\b([A-Za-z0-9][A-Za-z0-9._-]*)\s*(?:\[[^\]]+\])?\s*(?:\{[^}]+\})?\s*[<>=!~]" + ) + candidates = [_normalize_package_name(name) for name in candidate_pattern.findall(stderr)] + for normalized_name in candidates: + if normalized_name in known_names and candidates.count(normalized_name) >= 2: + return normalized_name + return None + + +def _resolve_build_requirements( + build_reqs: dict[str, set[str]], + output_path: Path, + config_params: ConfigParams, +) -> None: + """Resolve ranged build requirements to pinned versions with uv pip compile.""" + lines_by_name: dict[str, set[str]] = {} + for name, requirements in build_reqs.items(): + for requirement in requirements: + if not _is_exact_pin(requirement): + lines_by_name.setdefault(name, set()).add(requirement) + + if not lines_by_name: + console.print("[yellow]Warning: no range-specifier build requirements to resolve") + output_path.write_text("") + return + + skipped: list[str] = [] + max_attempts = 10 + for attempt in range(1, max_attempts + 1): + all_lines = sorted( + {requirement for requirements in lines_by_name.values() for requirement in requirements} + ) + file_descriptor, tmp_reqs_path = tempfile.mkstemp(prefix="build-reqs-", suffix=".txt") + try: + Path(tmp_reqs_path).write_text("\n".join(all_lines) + "\n") + os.close(file_descriptor) + result = run_command( + [ + "uv", + "pip", + "compile", + tmp_reqs_path, + "--no-config", + "--python-version", + config_params.python, + "--resolution", + "highest", + "--upgrade", + "--no-python-downloads", + "--no-annotate", + "--no-header", + "-o", + output_path.as_posix(), + ], + github_actions=config_params.github_actions, + cwd=AIRFLOW_ROOT_PATH, + check=False, + text=True, + capture_output=True, + ) + finally: + Path(tmp_reqs_path).unlink(missing_ok=True) + + if result.returncode == 0: + break + + stderr = result.stderr or "" + known_names = set(lines_by_name) + conflict_name = _find_conflicting_build_requirement(stderr, known_names) + if conflict_name: + console.print( + f"[yellow]Warning: conflicting build requirements for '{conflict_name}' — " + "removing from build constraints; package build isolation will resolve it independently" + ) + del lines_by_name[conflict_name] + skipped.append(conflict_name) + continue + + extracted_names = sorted( + { + _normalize_package_name(name) + for name in re.findall( + r"\b([A-Za-z0-9][A-Za-z0-9._-]*)\s*(?:\[[^\]]+\])?\s*(?:\{[^}]+\})?\s*[<>=!~]", + stderr, + ) + } + ) + if extracted_names: + console.print( + f"[yellow]Warning: uv stderr referenced {extracted_names}, but none matched " + f"known build dependencies {sorted(known_names)}" + ) + else: + console.print( + "[yellow]Warning: uv stderr contained no classifiable build dependency; " + "the diagnostic format may have changed" + ) + console.print(f"[red]{stderr}") + raise RuntimeError(f"uv pip compile failed (attempt {attempt}): {stderr}") + else: + raise RuntimeError(f"uv pip compile failed after {max_attempts} attempts. Skipped: {sorted(skipped)}") + + if skipped: + console.print(f"[yellow]Skipped {len(skipped)} conflicting build deps: {sorted(skipped)}") + + +def generate_build_constraints(config_params: ConfigParams) -> None: + """Generate one pinned build constraints file for the selected Python version.""" + console.print("[bright_blue]Generating build constraints...") + workspace_reqs = _collect_workspace_build_reqs(AIRFLOW_ROOT_PATH) + console.print(f"[bright_blue]Workspace build deps ({len(workspace_reqs)}): {sorted(workspace_reqs)}") + upstream_reqs = _collect_upstream_build_reqs( + uv_lock_path=AIRFLOW_ROOT_PATH / "uv.lock", + cache_path=config_params.constraints_dir / "build-deps-cache.json", + ) + console.print(f"[bright_blue]Upstream build deps ({len(upstream_reqs)}): {sorted(upstream_reqs)}") + + all_reqs: dict[str, set[str]] = {} + for requirements in (workspace_reqs, upstream_reqs): + for name, specifiers in requirements.items(): + all_reqs.setdefault(name, set()).update(specifiers) + + console.print(f"[bright_blue]Total unique build deps to resolve: {len(all_reqs)}") + output_path = config_params.constraints_dir / f"build-constraints-{config_params.python}.txt" + _resolve_build_requirements(all_reqs, output_path, config_params) + pinned_content = output_path.read_text() + output_path.write_text(BUILD_CONSTRAINTS_PREFIX + pinned_content) + console.print(f"[green]Build constraints generated: {output_path}") + + ALLOWED_CONSTRAINTS_MODES = ["constraints", "constraints-source-providers", "constraints-no-providers"] @@ -538,6 +917,7 @@ def generate_constraints( else: console.print(f"[red]Unknown constraints mode: {airflow_constraints_mode}") sys.exit(1) + generate_build_constraints(config_params) console.print("[green]Generated constraints:") files = config_params.constraints_dir.rglob("*.txt") for file in files: diff --git a/scripts/tests/in_container/__init__.py b/scripts/tests/in_container/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/scripts/tests/in_container/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/scripts/tests/in_container/conftest.py b/scripts/tests/in_container/conftest.py new file mode 100644 index 0000000000000..6038b773c42e3 --- /dev/null +++ b/scripts/tests/in_container/conftest.py @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from pathlib import Path + +# This file is generated only during image builds, but the module reads it at import time. +_generated_dir = Path(__file__).resolve().parents[3] / "generated" +_provider_dependencies_file = _generated_dir / "provider_dependencies.json" +if not _provider_dependencies_file.exists(): + _generated_dir.mkdir(parents=True, exist_ok=True) + _provider_dependencies_file.write_text("{}") diff --git a/scripts/tests/in_container/test_generate_build_constraints.py b/scripts/tests/in_container/test_generate_build_constraints.py new file mode 100644 index 0000000000000..ae3a8862de169 --- /dev/null +++ b/scripts/tests/in_container/test_generate_build_constraints.py @@ -0,0 +1,949 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import io +import json +import os +import shutil +import subprocess +import tarfile +import urllib.error +import zipfile +from email.message import Message +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +import pytest +from run_generate_constraints import ( + BUILD_CONSTRAINTS_PREFIX, + ConfigParams, + _collect_upstream_build_reqs, + _collect_workspace_build_reqs, + _extract_build_reqs_from_tar, + _extract_build_reqs_from_zip, + _extract_package_name, + _is_exact_pin, + _normalize_package_name, + _parse_uv_lock, + _resolve_build_requirements, + _stream_build_reqs_from_sdist, + generate_build_constraints, + generate_constraints, +) + +AIRFLOW_ROOT = Path(__file__).resolve().parents[3] + + +def _config(python: str = "3.12") -> ConfigParams: + return ConfigParams( + airflow_constraints_mode="constraints-source-providers", + constraints_github_repository="apache/airflow", + default_constraints_branch="main", + github_actions=False, + python=python, + ) + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("setuptools", "setuptools"), + ("Cython", "cython"), + ("my_package", "my-package"), + ("my.package", "my-package"), + ("My--Package", "my-package"), + ], +) +def test_normalize_package_name(raw, expected): + assert _normalize_package_name(raw) == expected + + +@pytest.mark.parametrize( + ("requirement", "expected"), + [ + ("setuptools", "setuptools"), + ("maturin>=1.9.4,<2", "maturin"), + ("hatchling==1.29.0", "hatchling"), + ("Cython>=3.0", "cython"), + ("GitPython>=3.1.30", "gitpython"), + ], +) +def test_extract_package_name(requirement, expected): + assert _extract_package_name(requirement) == expected + + +@pytest.mark.parametrize( + ("requirement", "expected"), + [ + ("setuptools==80.9.0", True), + ("Cython == 3.1.1", True), + ("tomli==2.4.1; python_version < '3.11'", True), + ("setuptools>=70.0", False), + ("maturin>=1.9.4,<2", False), + ("setuptools!=74.0.0", False), + ("cython==3.*", False), + ("cffi~=1.17", False), + ("wheel", False), + ], +) +def test_is_exact_pin(requirement, expected): + assert _is_exact_pin(requirement) is expected + + +class TestCollectWorkspaceBuildReqs: + def test_collects_all_specifiers_for_each_package(self, tmp_path): + for directory, content in { + "pkg_a": '[build-system]\nrequires = ["hatchling==1.29.0", "setuptools>=70"]\n', + "pkg_b": '[build-system]\nrequires = ["hatchling>=1.20"]\n', + }.items(): + package_dir = tmp_path / directory + package_dir.mkdir() + (package_dir / "pyproject.toml").write_text(content) + + result = _collect_workspace_build_reqs(tmp_path) + + assert result == { + "hatchling": {"hatchling==1.29.0", "hatchling>=1.20"}, + "setuptools": {"setuptools>=70"}, + } + + @pytest.mark.parametrize( + "directory", + [ + ".git/worktree", + ".venv/lib", + ".build/airflow-source", + "dist/extracted", + "files/constraints-3.12", + "node_modules/package", + "__pycache__/package", + ".tox/package", + ".mypy_cache/package", + ".ruff_cache/package", + ], + ) + def test_skips_transient_and_generated_directories(self, tmp_path, directory): + authoritative = tmp_path / "package" + authoritative.mkdir() + (authoritative / "pyproject.toml").write_text('[build-system]\nrequires = ["hatchling==1.29.0"]\n') + transient = tmp_path / directory + transient.mkdir(parents=True) + (transient / "pyproject.toml").write_text('[build-system]\nrequires = ["pollution>=1"]\n') + + result = _collect_workspace_build_reqs(tmp_path) + + assert result == {"hatchling": {"hatchling==1.29.0"}} + + @mock.patch("run_generate_constraints.console", autospec=True) + def test_warns_and_continues_for_malformed_pyproject(self, console, tmp_path): + (tmp_path / "pyproject.toml").write_text("not valid toml {{{") + + assert _collect_workspace_build_reqs(tmp_path) == {} + assert "failed to parse" in console.print.call_args.args[0] + + +def _make_uv_lock(tmp_path: Path, packages: list[dict[str, object]]) -> Path: + lines: list[str] = [] + for package in packages: + lines.extend( + [ + "[[package]]", + f'name = "{package["name"]}"', + f'version = "{package["version"]}"', + ] + ) + if sdist_url := package.get("sdist_url"): + lines.extend(["", "[package.sdist]", f'url = "{sdist_url}"']) + if package.get("universal_wheel"): + lines.extend( + [ + "", + "[[package.wheels]]", + f'url = "https://example.invalid/{package["name"]}-py3-none-any.whl"', + ] + ) + elif package.get("platform_wheel"): + lines.extend( + [ + "", + "[[package.wheels]]", + f'url = "https://example.invalid/{package["name"]}-cp312-linux_x86_64.whl"', + ] + ) + lines.append("") + lock_path = tmp_path / "uv.lock" + lock_path.write_text("\n".join(lines)) + return lock_path + + +def test_parse_uv_lock_selects_sdist_and_wheel_metadata(tmp_path): + lock_path = _make_uv_lock( + tmp_path, + [ + { + "name": "pydantic-core", + "version": "2.33.2", + "sdist_url": "https://example.invalid/pydantic-core.tar.gz", + "platform_wheel": True, + }, + {"name": "requests", "version": "2.31.0", "universal_wheel": True}, + ], + ) + + packages = _parse_uv_lock(lock_path) + + assert [ + ( + package.name, + package.version, + package.sdist_url, + package.has_universal_wheel, + ) + for package in packages + ] == [ + ( + "pydantic-core", + "2.33.2", + "https://example.invalid/pydantic-core.tar.gz", + False, + ), + ("requests", "2.31.0", None, True), + ] + + +def _pyproject_toml(requirements: list[str]) -> bytes: + return ( + f'[build-system]\nrequires = {json.dumps(requirements)}\nbuild-backend = "setuptools.build_meta"\n' + ).encode() + + +def _make_tar_sdist( + tmp_path: Path, + requirements: list[str] | None, + *, + nested_requirements: list[str] | None = None, +) -> Path: + archive_path = tmp_path / "package-1.0.0.tar.gz" + with tarfile.open(archive_path, "w:gz") as archive: + if nested_requirements: + nested_data = _pyproject_toml(nested_requirements) + nested_info = tarfile.TarInfo("package-1.0.0/vendor/project/pyproject.toml") + nested_info.size = len(nested_data) + archive.addfile(nested_info, io.BytesIO(nested_data)) + if requirements is None: + setup_data = b"from setuptools import setup\nsetup()\n" + info = tarfile.TarInfo("package-1.0.0/setup.py") + info.size = len(setup_data) + archive.addfile(info, io.BytesIO(setup_data)) + else: + data = _pyproject_toml(requirements) + info = tarfile.TarInfo("package-1.0.0/pyproject.toml") + info.size = len(data) + archive.addfile(info, io.BytesIO(data)) + return archive_path + + +def _make_zip_sdist( + tmp_path: Path, + requirements: list[str] | None, + *, + nested_requirements: list[str] | None = None, +) -> Path: + archive_path = tmp_path / "package-1.0.0.zip" + with zipfile.ZipFile(archive_path, "w") as archive: + if nested_requirements: + archive.writestr( + "package-1.0.0/vendor/project/pyproject.toml", + _pyproject_toml(nested_requirements), + ) + if requirements is None: + archive.writestr("package-1.0.0/setup.py", "from setuptools import setup\nsetup()\n") + else: + archive.writestr("package-1.0.0/pyproject.toml", _pyproject_toml(requirements)) + return archive_path + + +@pytest.mark.parametrize( + ("factory", "extractor"), + [ + (_make_tar_sdist, _extract_build_reqs_from_tar), + (_make_zip_sdist, _extract_build_reqs_from_zip), + ], +) +def test_extracts_only_top_level_sdist_pyproject(tmp_path, factory, extractor): + archive_path = factory( + tmp_path, + ["right-backend>=2"], + nested_requirements=["wrong-backend>=1"], + ) + + assert extractor(archive_path.as_uri()) == ["right-backend>=2"] + + +@pytest.mark.parametrize( + ("factory", "extractor"), + [ + (_make_tar_sdist, _extract_build_reqs_from_tar), + (_make_zip_sdist, _extract_build_reqs_from_zip), + ], +) +def test_legacy_sdist_has_no_declared_build_requirements(tmp_path, factory, extractor): + assert extractor(factory(tmp_path, None).as_uri()) == [] + + +class TestSdistDownloadRetry: + @pytest.mark.parametrize( + "error", + [ + urllib.error.URLError("temporary DNS failure"), + TimeoutError("timed out"), + ConnectionError("connection reset"), + urllib.error.HTTPError( + "https://example.invalid/package.tar.gz", + 503, + "unavailable", + Message(), + None, + ), + ], + ) + @mock.patch("run_generate_constraints._extract_build_reqs_from_tar", autospec=True) + def test_retries_transient_errors_with_exponential_backoff(self, extract, error): + extract.side_effect = [error, error, ["setuptools>=70"]] + sleep = mock.create_autospec(lambda seconds: None) + + result = _stream_build_reqs_from_sdist( + "https://example.invalid/package.tar.gz", + sleep=sleep, + ) + + assert result == ["setuptools>=70"] + assert sleep.call_args_list == [mock.call(1), mock.call(2)] + + @mock.patch("run_generate_constraints._extract_build_reqs_from_tar", autospec=True) + def test_raises_after_retry_attempts_are_exhausted(self, extract): + error = urllib.error.URLError("temporary DNS failure") + extract.side_effect = error + sleep = mock.create_autospec(lambda seconds: None) + + with pytest.raises(urllib.error.URLError, match="temporary DNS failure"): + _stream_build_reqs_from_sdist( + "https://example.invalid/package.tar.gz", + sleep=sleep, + ) + + assert extract.call_count == 3 + assert sleep.call_args_list == [mock.call(1), mock.call(2)] + + @mock.patch("run_generate_constraints._extract_build_reqs_from_tar", autospec=True) + def test_does_not_retry_parse_errors(self, extract): + extract.side_effect = ValueError("invalid pyproject") + sleep = mock.create_autospec(lambda seconds: None) + + with pytest.raises(ValueError, match="invalid pyproject"): + _stream_build_reqs_from_sdist( + "https://example.invalid/package.tar.gz", + sleep=sleep, + ) + + extract.assert_called_once() + sleep.assert_not_called() + + @mock.patch( + "run_generate_constraints._extract_build_reqs_from_zip", + autospec=True, + return_value=["hatchling>=1"], + ) + def test_dispatches_zip_sdists(self, extract): + assert _stream_build_reqs_from_sdist("https://example.invalid/package.zip") == ["hatchling>=1"] + extract.assert_called_once() + + +class TestCollectUpstreamBuildReqs: + def test_uses_cache_and_skips_universal_wheels(self, tmp_path): + lock_path = _make_uv_lock( + tmp_path, + [ + { + "name": "platform-package", + "version": "1.0", + "sdist_url": "https://example.invalid/platform.tar.gz", + "platform_wheel": True, + }, + { + "name": "universal-package", + "version": "2.0", + "sdist_url": "https://example.invalid/universal.tar.gz", + "universal_wheel": True, + }, + {"name": "wheel-only", "version": "3.0", "platform_wheel": True}, + ], + ) + cache_path = tmp_path / "cache.json" + cache_path.write_text(json.dumps({"platform-package==1.0": ["maturin>=1.9,<2"]})) + + with mock.patch("run_generate_constraints._stream_build_reqs_from_sdist", autospec=True) as scan: + result = _collect_upstream_build_reqs(lock_path, cache_path) + + assert result == {"maturin": {"maturin>=1.9,<2"}} + scan.assert_not_called() + + def test_excludes_stale_cache_entries(self, tmp_path): + lock_path = _make_uv_lock( + tmp_path, + [ + { + "name": "current-package", + "version": "1.0", + "sdist_url": "https://example.invalid/current.tar.gz", + "platform_wheel": True, + } + ], + ) + cache_path = tmp_path / "cache.json" + cache_path.write_text( + json.dumps( + { + "current-package==1.0": ["setuptools>=70"], + "stale-package==0.1": ["flit-core>=3"], + } + ) + ) + + result = _collect_upstream_build_reqs(lock_path, cache_path) + + assert result == {"setuptools": {"setuptools>=70"}} + + @mock.patch( + "run_generate_constraints._stream_build_reqs_from_sdist", + autospec=True, + return_value=["setuptools>=70", "wheel"], + ) + def test_caches_successful_scans(self, scan, tmp_path): + lock_path = _make_uv_lock( + tmp_path, + [ + { + "name": "source-package", + "version": "1.0", + "sdist_url": "https://example.invalid/source.tar.gz", + "platform_wheel": True, + } + ], + ) + cache_path = tmp_path / "cache.json" + + result = _collect_upstream_build_reqs(lock_path, cache_path) + + assert result == { + "setuptools": {"setuptools>=70"}, + "wheel": {"wheel"}, + } + assert json.loads(cache_path.read_text()) == {"source-package==1.0": ["setuptools>=70", "wheel"]} + scan.assert_called_once() + + @mock.patch( + "run_generate_constraints._stream_build_reqs_from_sdist", + autospec=True, + return_value=[], + ) + def test_caches_legacy_sdists_as_setuptools_and_wheel(self, scan, tmp_path): + lock_path = _make_uv_lock( + tmp_path, + [ + { + "name": "legacy-package", + "version": "1.0", + "sdist_url": "https://example.invalid/legacy.tar.gz", + "platform_wheel": True, + } + ], + ) + cache_path = tmp_path / "cache.json" + + result = _collect_upstream_build_reqs(lock_path, cache_path) + + assert result == { + "setuptools": {"setuptools"}, + "wheel": {"wheel"}, + } + assert json.loads(cache_path.read_text()) == {"legacy-package==1.0": ["setuptools", "wheel"]} + scan.assert_called_once() + + @mock.patch( + "run_generate_constraints._stream_build_reqs_from_sdist", + autospec=True, + side_effect=ConnectionError("network failure"), + ) + def test_does_not_cache_failed_scans(self, scan, tmp_path): + lock_path = _make_uv_lock( + tmp_path, + [ + { + "name": "failed-package", + "version": "1.0", + "sdist_url": "https://example.invalid/failed.tar.gz", + "platform_wheel": True, + } + ], + ) + cache_path = tmp_path / "cache.json" + + with pytest.raises(RuntimeError, match="failed-package==1.0"): + _collect_upstream_build_reqs(lock_path, cache_path) + + assert not cache_path.exists() + scan.assert_called_once() + + def test_preserves_multiple_ranges_for_one_build_dependency(self, tmp_path): + lock_path = _make_uv_lock( + tmp_path, + [ + { + "name": "package-a", + "version": "1.0", + "sdist_url": "https://example.invalid/a.tar.gz", + "platform_wheel": True, + }, + { + "name": "package-b", + "version": "2.0", + "sdist_url": "https://example.invalid/b.tar.gz", + "platform_wheel": True, + }, + ], + ) + cache_path = tmp_path / "cache.json" + cache_path.write_text( + json.dumps( + { + "package-a==1.0": ["setuptools>=70"], + "package-b==2.0": ["setuptools>=68,<72"], + } + ) + ) + + result = _collect_upstream_build_reqs(lock_path, cache_path) + + assert result == {"setuptools": {"setuptools>=70", "setuptools>=68,<72"}} + + @mock.patch("run_generate_constraints.console", autospec=True) + def test_discards_corrupt_cache(self, console, tmp_path): + lock_path = _make_uv_lock(tmp_path, []) + cache_path = tmp_path / "cache.json" + cache_path.write_text("{broken") + + assert _collect_upstream_build_reqs(lock_path, cache_path) == {} + assert "discarding cache" in console.print.call_args.args[0] + + +def _success_result() -> SimpleNamespace: + return SimpleNamespace(returncode=0, stdout="", stderr="") + + +class TestResolveBuildRequirements: + @mock.patch("run_generate_constraints.run_command", autospec=True) + def test_omits_exact_pins_and_passes_all_ranges_to_uv(self, run_command, tmp_path): + output_path = tmp_path / "build-constraints-3.12.txt" + captured: list[str] = [] + + def compile_requirements(command, **kwargs): + captured.append(Path(command[3]).read_text()) + output_path.write_text("hatchling==1.30.1\nsetuptools==80.9.0\n") + return _success_result() + + run_command.side_effect = compile_requirements + _resolve_build_requirements( + { + "hatchling": {"hatchling==1.29.0", "hatchling>=1.20"}, + "setuptools": {"setuptools>=70"}, + "cython": {"cython==3.1.1", "cython>=3.1.2,<3.3"}, + }, + output_path, + _config(), + ) + + assert captured == ["cython>=3.1.2,<3.3\nhatchling>=1.20\nsetuptools>=70\n"] + command = run_command.call_args.args[0] + assert command[:3] == ["uv", "pip", "compile"] + assert command[4:] == [ + "--no-config", + "--python-version", + "3.12", + "--resolution", + "highest", + "--upgrade", + "--no-python-downloads", + "--no-annotate", + "--no-header", + "-o", + str(output_path), + ] + assert run_command.call_args.kwargs["cwd"] == AIRFLOW_ROOT + + @mock.patch("run_generate_constraints.run_command", autospec=True) + def test_writes_empty_file_when_every_requirement_is_exact(self, run_command, tmp_path): + output_path = tmp_path / "build-constraints.txt" + + _resolve_build_requirements( + {"hatchling": {"hatchling==1.30.1"}}, + output_path, + _config(), + ) + + assert output_path.read_text() == "" + run_command.assert_not_called() + + @pytest.mark.parametrize( + "stderr", + [ + "Because you require cython>=3.0,<3.1 and cython>=3.1.2,<3.3, requirements conflict.", + "The project depends on cython>=3.0,<3.1 and cython>=3.1.2,<3.3.", + ( + "Because you require cffi{platform_python_implementation != 'PyPy'}>=2.0 " + "and cffi{python_full_version < '3.14'}>=1.17,<2.dev0, requirements conflict." + ), + "Because you require pdm_backend>=2,<3 and pdm_backend>=3, requirements conflict.", + ], + ) + @mock.patch("run_generate_constraints.run_command", autospec=True) + def test_skips_identified_conflict_and_retries(self, run_command, stderr, tmp_path): + if "cffi" in stderr: + conflict_name = "cffi" + conflicting = { + "cffi>=2.0; platform_python_implementation != 'PyPy'", + "cffi>=1.17,<2.dev0; python_full_version < '3.14'", + } + elif "pdm_backend" in stderr: + conflict_name = "pdm-backend" + conflicting = {"pdm_backend>=2,<3", "pdm_backend>=3"} + else: + conflict_name = "cython" + conflicting = {"cython>=3.0,<3.1", "cython>=3.1.2,<3.3"} + + output_path = tmp_path / "build-constraints.txt" + calls = 0 + + def compile_requirements(command, **kwargs): + nonlocal calls + calls += 1 + if calls == 1: + return SimpleNamespace(returncode=1, stdout="", stderr=stderr) + retry_input = Path(command[3]).read_text() + assert conflict_name not in retry_input + output_path.write_text("setuptools==80.9.0\n") + return _success_result() + + run_command.side_effect = compile_requirements + _resolve_build_requirements( + { + conflict_name: conflicting, + "setuptools": {"setuptools>=70"}, + }, + output_path, + _config(), + ) + + assert calls == 2 + assert output_path.read_text() == "setuptools==80.9.0\n" + + @mock.patch("run_generate_constraints.run_command", autospec=True) + def test_uses_first_known_package_in_stderr_order(self, run_command, tmp_path): + output_path = tmp_path / "build-constraints.txt" + inputs: list[str] = [] + + def compile_requirements(command, **kwargs): + inputs.append(Path(command[3]).read_text()) + if len(inputs) == 1: + return SimpleNamespace( + returncode=1, + stdout="", + stderr=( + "cython>=3.0,<3.1 conflicts with cython>=3.1.2; " + "setuptools>=70 conflicts with setuptools<60" + ), + ) + output_path.write_text("setuptools==80.9.0\nhatchling==1.30.1\n") + return _success_result() + + run_command.side_effect = compile_requirements + _resolve_build_requirements( + { + "setuptools": {"setuptools>=70"}, + "hatchling": {"hatchling>=1.20"}, + "cython": {"cython>=3.0,<3.1", "cython>=3.1.2"}, + }, + output_path, + _config(), + ) + + assert "cython" not in inputs[1] + assert "setuptools>=70" in inputs[1] + + @pytest.mark.parametrize( + "stderr", + [ + "error: network unreachable for files.pythonhosted.org", + "package-not-in-input>=1 failed to download", + "failed to download setuptools>=70 from the package index", + ], + ) + @mock.patch("run_generate_constraints.run_command", autospec=True) + def test_fails_loudly_when_uv_output_cannot_be_classified(self, run_command, stderr, tmp_path): + run_command.return_value = SimpleNamespace(returncode=1, stdout="", stderr=stderr) + + with pytest.raises(RuntimeError, match="uv pip compile failed") as error: + _resolve_build_requirements( + {"setuptools": {"setuptools>=70"}}, + tmp_path / "build-constraints.txt", + _config(), + ) + + assert stderr in str(error.value) + + @mock.patch("run_generate_constraints.run_command", autospec=True) + def test_has_a_bounded_number_of_conflict_retries(self, run_command, tmp_path): + names = [f"backend-{index}" for index in range(10)] + build_reqs = {name: {f"{name}>=2", f"{name}<1"} for name in names} + calls = 0 + + def fail_with_next_conflict(command, **kwargs): + nonlocal calls + name = names[calls] + calls += 1 + return SimpleNamespace( + returncode=1, + stdout="", + stderr=f"{name}>=2 conflicts with {name}<1", + ) + + run_command.side_effect = fail_with_next_conflict + + with pytest.raises(RuntimeError, match="after 10 attempts"): + _resolve_build_requirements( + build_reqs, + tmp_path / "build-constraints.txt", + _config(), + ) + + assert calls == 10 + + +@pytest.mark.skipif( + shutil.which("uv") is None, + reason="uv is required for the real resolver smoke test", +) +def test_real_uv_conflict_diagnostic_can_be_classified(tmp_path): + output_path = tmp_path / "build-constraints.txt" + + _resolve_build_requirements( + { + "cython": {"cython>=3.0,<3.1", "cython>=3.1.2,<3.3"}, + "setuptools": {"setuptools>=70"}, + }, + output_path, + _config(), + ) + + content = output_path.read_text().lower() + assert "cython" not in content + assert "setuptools==" in content + + +@mock.patch("run_generate_constraints._resolve_build_requirements", autospec=True) +@mock.patch("run_generate_constraints._collect_upstream_build_reqs", autospec=True) +@mock.patch("run_generate_constraints._collect_workspace_build_reqs", autospec=True) +def test_generate_build_constraints_merges_requirements_and_adds_header( + collect_workspace, + collect_upstream, + resolve, + tmp_path, +): + collect_workspace.return_value = { + "setuptools": {"setuptools>=70"}, + "hatchling": {"hatchling>=1.20"}, + } + collect_upstream.return_value = { + "setuptools": {"setuptools>=68,<72"}, + "maturin": {"maturin>=1.9,<2"}, + } + config = _config() + + def write_pins(requirements, output_path, config_params): + assert requirements == { + "setuptools": {"setuptools>=70", "setuptools>=68,<72"}, + "hatchling": {"hatchling>=1.20"}, + "maturin": {"maturin>=1.9,<2"}, + } + output_path.write_text("hatchling==1.30.1\nmaturin==1.9.6\nsetuptools==71.1.0\n") + + resolve.side_effect = write_pins + with mock.patch.object(ConfigParams, "constraints_dir", tmp_path): + generate_build_constraints(config) + + output_path = tmp_path / "build-constraints-3.12.txt" + assert output_path.read_text() == ( + BUILD_CONSTRAINTS_PREFIX + "hatchling==1.30.1\nmaturin==1.9.6\nsetuptools==71.1.0\n" + ) + collect_upstream.assert_called_once_with( + uv_lock_path=AIRFLOW_ROOT / "uv.lock", + cache_path=tmp_path / "build-deps-cache.json", + ) + + +@pytest.mark.parametrize( + ("mode", "runtime_function"), + [ + ("constraints", "generate_constraints_pypi_providers"), + ("constraints-source-providers", "generate_constraints_source_providers"), + ("constraints-no-providers", "generate_constraints_no_providers"), + ], +) +def test_each_runtime_mode_also_generates_build_constraints_without_changing_runtime_output( + tmp_path, + mode, + runtime_function, +): + runtime_content = f"{mode}-runtime-output\n" + + def generate_runtime(config): + config.current_constraints_file.write_text(runtime_content) + + def generate_build(config): + assert config.current_constraints_file.read_text() == runtime_content + (config.constraints_dir / f"build-constraints-{config.python}.txt").write_text("setuptools==80.9.0\n") + + with ( + mock.patch.object(ConfigParams, "constraints_dir", tmp_path), + mock.patch( + f"run_generate_constraints.{runtime_function}", + autospec=True, + side_effect=generate_runtime, + ), + mock.patch( + "run_generate_constraints.generate_build_constraints", + autospec=True, + side_effect=generate_build, + ), + ): + generate_constraints.callback( + airflow_constraints_mode=mode, + constraints_github_repository="apache/airflow", + default_constraints_branch="main", + github_actions=False, + python="3.12", + use_uv=True, + ) + + assert (tmp_path / f"{mode}-3.12.txt").read_text() == runtime_content + assert (tmp_path / "build-constraints-3.12.txt").read_text() == "setuptools==80.9.0\n" + + +def _run(command: list[str], *, cwd: Path, env: dict[str, str] | None = None) -> subprocess.CompletedProcess: + return subprocess.run( + command, + cwd=cwd, + env=env, + text=True, + capture_output=True, + check=True, + ) + + +def _initialize_constraints_repository(tmp_path: Path) -> tuple[Path, Path]: + files_dir = tmp_path / "files" / "constraints-3.12" + files_dir.mkdir(parents=True) + constraints_dir = tmp_path / "constraints" + constraints_dir.mkdir() + _run(["git", "init"], cwd=constraints_dir) + _run(["git", "config", "user.email", "test@example.com"], cwd=constraints_dir) + _run(["git", "config", "user.name", "Test User"], cwd=constraints_dir) + (constraints_dir / "constraints-3.12.txt").write_text("# old\npackage==1\n") + (constraints_dir / "notes.txt").write_text("keep me out of publication commits\n") + _run(["git", "add", "."], cwd=constraints_dir) + _run(["git", "commit", "-m", "Initial constraints"], cwd=constraints_dir) + return files_dir, constraints_dir + + +def _publication_env() -> dict[str, str]: + return { + **os.environ, + "GITHUB_RUN_ID": "1234", + "GITHUB_REF": "refs/heads/main", + "GITHUB_REPOSITORY": "apache/airflow", + "GITHUB_SHA": "deadbeef", + } + + +class TestConstraintsPublicationScripts: + def test_stages_and_commits_new_build_constraints_only(self, tmp_path): + files_dir, constraints_dir = _initialize_constraints_repository(tmp_path) + (files_dir / "constraints-3.12.txt").write_text("# new\npackage==2\n") + (files_dir / "build-constraints-3.12.txt").write_text("# generated\nsetuptools==80.9.0\n") + (constraints_dir / "notes.txt").write_text("unrelated local edit\n") + + diff = _run( + ["bash", str(AIRFLOW_ROOT / "scripts/ci/constraints/ci_diff_constraints.sh")], + cwd=tmp_path, + ) + assert "Changes detected in constraints" in diff.stdout + assert _run(["git", "diff", "--cached", "--name-only"], cwd=constraints_dir).stdout == "" + + _run( + ["bash", str(AIRFLOW_ROOT / "scripts/ci/constraints/ci_commit_constraints.sh")], + cwd=tmp_path, + env=_publication_env(), + ) + + committed_files = set( + _run( + ["git", "show", "--pretty=format:", "--name-only", "HEAD"], + cwd=constraints_dir, + ).stdout.split() + ) + assert committed_files == { + "build-constraints-3.12.txt", + "constraints-3.12.txt", + } + assert _run(["git", "status", "--short"], cwd=constraints_dir).stdout == " M notes.txt\n" + + def test_comment_only_changes_do_not_create_publication_commit(self, tmp_path): + files_dir, constraints_dir = _initialize_constraints_repository(tmp_path) + (constraints_dir / "build-constraints-3.12.txt").write_text( + "# old generated time\nsetuptools==80.9.0\n" + ) + _run(["git", "add", "build-constraints-3.12.txt"], cwd=constraints_dir) + _run(["git", "commit", "-m", "Add build constraints"], cwd=constraints_dir) + before = _run(["git", "rev-parse", "HEAD"], cwd=constraints_dir).stdout + + (files_dir / "constraints-3.12.txt").write_text("# new generated time\npackage==1\n") + (files_dir / "build-constraints-3.12.txt").write_text("# new generated time\nsetuptools==80.9.0\n") + + diff = _run( + ["bash", str(AIRFLOW_ROOT / "scripts/ci/constraints/ci_diff_constraints.sh")], + cwd=tmp_path, + ) + assert "No changes in constraints" in diff.stdout + _run( + ["bash", str(AIRFLOW_ROOT / "scripts/ci/constraints/ci_commit_constraints.sh")], + cwd=tmp_path, + env=_publication_env(), + ) + + assert _run(["git", "rev-parse", "HEAD"], cwd=constraints_dir).stdout == before