From 803ccbd1406d9685534b1ced68c196513c20a2a5 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 3 Apr 2025 12:01:49 -0400 Subject: [PATCH 1/3] refactor: pull out dependency_constraint Signed-off-by: Henry Schreiner --- cibuildwheel/platforms/ios.py | 18 ++++++++---------- cibuildwheel/platforms/macos.py | 17 ++++++++--------- cibuildwheel/platforms/pyodide.py | 6 +++--- cibuildwheel/platforms/windows.py | 21 +++++++++++---------- cibuildwheel/venv.py | 22 ++++++++++++++-------- 5 files changed, 44 insertions(+), 40 deletions(-) diff --git a/cibuildwheel/platforms/ios.py b/cibuildwheel/platforms/ios.py index 06fce61fa..fcffe727f 100644 --- a/cibuildwheel/platforms/ios.py +++ b/cibuildwheel/platforms/ios.py @@ -24,7 +24,6 @@ from ..logger import log from ..options import Options from ..selector import BuildSelector -from ..typing import PathOrStr from ..util import resources from ..util.cmd import call, shell from ..util.file import ( @@ -39,7 +38,7 @@ find_compatible_wheel, get_pip_version, ) -from ..venv import virtualenv +from ..venv import constraint_flags, virtualenv from .macos import install_cpython as install_build_cpython @@ -151,7 +150,7 @@ def cross_virtualenv( multiarch: str, build_python: Path, venv_path: Path, - dependency_constraint_flags: Sequence[PathOrStr], + dependency_constraint: Path | None, xbuild_tools: Sequence[str] | None, ) -> dict[str, str]: """Create a cross-compilation virtual environment. @@ -188,7 +187,7 @@ def cross_virtualenv( py_version, build_python, venv_path, - dependency_constraint_flags, + dependency_constraint, use_uv=False, ) @@ -279,7 +278,7 @@ def setup_python( tmp: Path, *, python_configuration: PythonConfiguration, - dependency_constraint_flags: Sequence[PathOrStr], + dependency_constraint: Path | None, environment: ParsedEnvironment, build_frontend: BuildFrontendName, xbuild_tools: Sequence[str] | None, @@ -327,6 +326,8 @@ def setup_python( log.step("Creating cross build environment...") + dependency_constraint_flags = constraint_flags(dependency_constraint) + venv_path = tmp / "venv" env = cross_virtualenv( py_version=python_configuration.version, @@ -334,7 +335,7 @@ def setup_python( multiarch=python_configuration.multiarch, build_python=build_python, venv_path=venv_path, - dependency_constraint_flags=dependency_constraint_flags, + dependency_constraint=dependency_constraint, xbuild_tools=xbuild_tools, ) venv_bin_path = venv_path / "bin" @@ -453,14 +454,11 @@ def build(options: Options, tmp_path: Path) -> None: constraints_path = build_options.dependency_constraints.get_for_python_version( version=config.version, tmp_dir=identifier_tmp_dir ) - dependency_constraint_flags: Sequence[PathOrStr] = ( - ["-c", constraints_path] if constraints_path else [] - ) target_install_path, env = setup_python( identifier_tmp_dir / "build", python_configuration=config, - dependency_constraint_flags=dependency_constraint_flags, + dependency_constraint=constraints_path, environment=build_options.environment, build_frontend=build_frontend.name, xbuild_tools=build_options.xbuild_tools, diff --git a/cibuildwheel/platforms/macos.py b/cibuildwheel/platforms/macos.py index 4c75859f0..cea5cb3ab 100644 --- a/cibuildwheel/platforms/macos.py +++ b/cibuildwheel/platforms/macos.py @@ -7,7 +7,7 @@ import subprocess import sys import typing -from collections.abc import Sequence, Set +from collections.abc import Set from dataclasses import dataclass from pathlib import Path from typing import Literal, assert_never @@ -23,7 +23,6 @@ from ..logger import log from ..options import Options from ..selector import BuildSelector -from ..typing import PathOrStr from ..util import resources from ..util.cmd import call, shell from ..util.file import ( @@ -34,7 +33,7 @@ ) from ..util.helpers import prepare_command, unwrap from ..util.packaging import combine_constraints, find_compatible_wheel, get_pip_version -from ..venv import find_uv, virtualenv +from ..venv import constraint_flags, find_uv, virtualenv @functools.cache @@ -195,7 +194,7 @@ def install_pypy(tmp: Path, url: str) -> Path: def setup_python( tmp: Path, python_configuration: PythonConfiguration, - dependency_constraint_flags: Sequence[PathOrStr], + dependency_constraint: Path | None, environment: ParsedEnvironment, build_frontend: BuildFrontendName, ) -> tuple[Path, dict[str, str]]: @@ -220,13 +219,15 @@ def setup_python( f"{base_python.name} not found, has {list(base_python.parent.iterdir())}" ) + dependency_constraint_flags = constraint_flags(dependency_constraint) + log.step("Setting up build environment...") venv_path = tmp / "venv" env = virtualenv( python_configuration.version, base_python, venv_path, - dependency_constraint_flags, + dependency_constraint, use_uv=use_uv, ) venv_bin_path = venv_path / "bin" @@ -427,14 +428,12 @@ def build(options: Options, tmp_path: Path) -> None: constraints_path = build_options.dependency_constraints.get_for_python_version( version=config.version, tmp_dir=identifier_tmp_dir ) - dependency_constraint_flags: Sequence[PathOrStr] = ( - ["-c", constraints_path] if constraints_path else [] - ) + dependency_constraint_flags = constraint_flags(constraints_path) base_python, env = setup_python( identifier_tmp_dir / "build", config, - dependency_constraint_flags, + constraints_path, build_options.environment, build_frontend.name, ) diff --git a/cibuildwheel/platforms/pyodide.py b/cibuildwheel/platforms/pyodide.py index ca9edcb20..8b06f5b8c 100644 --- a/cibuildwheel/platforms/pyodide.py +++ b/cibuildwheel/platforms/pyodide.py @@ -152,7 +152,7 @@ def setup_python( log.step("Setting up build environment...") venv_path = tmp / "venv" - env = virtualenv(python_configuration.version, base_python, venv_path, [], use_uv=False) + env = virtualenv(python_configuration.version, base_python, venv_path, None, use_uv=False) venv_bin_path = venv_path / "bin" assert venv_bin_path.exists() env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" @@ -269,8 +269,8 @@ def build(options: Options, tmp_path: Path) -> None: constraints_path = build_options.dependency_constraints.get_for_python_version( version=config.version, variant="pyodide", tmp_dir=identifier_tmp_dir ) - dependency_constraint_flags: Sequence[PathOrStr] = ( - ["-c", constraints_path] if constraints_path else [] + dependency_constraint_flags = ( + ["-c", constraints_path.as_uri()] if constraints_path else [] ) env = setup_python( diff --git a/cibuildwheel/platforms/windows.py b/cibuildwheel/platforms/windows.py index 6d56236a2..74e2e9b85 100644 --- a/cibuildwheel/platforms/windows.py +++ b/cibuildwheel/platforms/windows.py @@ -3,7 +3,7 @@ import shutil import subprocess import textwrap -from collections.abc import MutableMapping, Sequence, Set +from collections.abc import MutableMapping, Set from dataclasses import dataclass from functools import cache from pathlib import Path @@ -18,13 +18,12 @@ from ..logger import log from ..options import Options from ..selector import BuildSelector -from ..typing import PathOrStr from ..util import resources from ..util.cmd import call, shell from ..util.file import CIBW_CACHE_PATH, copy_test_sources, download, extract_zip, move_file from ..util.helpers import prepare_command, unwrap from ..util.packaging import combine_constraints, find_compatible_wheel, get_pip_version -from ..venv import find_uv, virtualenv +from ..venv import constraint_flags, find_uv, virtualenv def get_nuget_args( @@ -219,7 +218,7 @@ def can_use_uv(python_configuration: PythonConfiguration) -> bool: def setup_python( tmp: Path, python_configuration: PythonConfiguration, - dependency_constraint_flags: Sequence[PathOrStr], + dependency_constraint: Path | None, environment: ParsedEnvironment, build_frontend: BuildFrontendName, ) -> tuple[Path, dict[str, str]]: @@ -248,7 +247,7 @@ def setup_python( if build_frontend == "build[uv]" and not can_use_uv(python_configuration): build_frontend = "build" - use_uv = build_frontend == "build[uv]" + use_uv = build_frontend in {"build[uv]", "uv"} uv_path = find_uv() log.step("Setting up build environment...") @@ -257,10 +256,12 @@ def setup_python( python_configuration.version, base_python, venv_path, - dependency_constraint_flags, + dependency_constraint, use_uv=use_uv, ) + dependency_constraint_flags = constraint_flags(dependency_constraint) + # set up environment variables for run_with_env env["PYTHON_VERSION"] = python_configuration.version env["PYTHON_ARCH"] = python_configuration.arch @@ -276,7 +277,7 @@ def setup_python( "install", "--upgrade", "pip", - *dependency_constraint_flags, + *constraint_flags(dependency_constraint), env=env, cwd=venv_path, ) @@ -370,15 +371,15 @@ def build(options: Options, tmp_path: Path) -> None: version=config.version, tmp_dir=identifier_tmp_dir, ) - dependency_constraint_flags: Sequence[PathOrStr] = ( - ["-c", constraints_path] if constraints_path else [] + dependency_constraint_flags = ( + ["-c", constraints_path.as_uri()] if constraints_path else [] ) # install Python base_python, env = setup_python( identifier_tmp_dir / "build", config, - dependency_constraint_flags, + constraints_path, build_options.environment, build_frontend.name, ) diff --git a/cibuildwheel/venv.py b/cibuildwheel/venv.py index 6f7ca400a..b8e9cb789 100644 --- a/cibuildwheel/venv.py +++ b/cibuildwheel/venv.py @@ -12,7 +12,6 @@ from packaging.requirements import InvalidRequirement, Requirement from packaging.version import Version -from .typing import PathOrStr from .util import resources from .util.cmd import call from .util.file import CIBW_CACHE_PATH, download @@ -36,8 +35,18 @@ def _ensure_virtualenv(version: str) -> Path: return path +def constraint_flags( + dependency_constraint: Path | None, +) -> Sequence[str]: + """ + Returns the flags to pass to pip for the given dependency constraint. + """ + + return ["-c", dependency_constraint.as_uri()] if dependency_constraint else [] + + def _parse_pip_constraint_for_virtualenv( - dependency_constraint_flags: Sequence[PathOrStr], + constraint_path: Path | None, ) -> str: """ Parses the constraints file referenced by `dependency_constraint_flags` and returns a dict where @@ -48,10 +57,7 @@ def _parse_pip_constraint_for_virtualenv( If it can't get an exact version, the real constraint will be handled by the {macos|windows}.setup_python function. """ - assert len(dependency_constraint_flags) in {0, 2} - if len(dependency_constraint_flags) == 2: - assert dependency_constraint_flags[0] == "-c" - constraint_path = Path(dependency_constraint_flags[1]) + if constraint_path: assert constraint_path.exists() with constraint_path.open(encoding="utf-8") as constraint_file: for line_ in constraint_file: @@ -84,7 +90,7 @@ def virtualenv( version: str, python: Path, venv_path: Path, - dependency_constraint_flags: Sequence[PathOrStr], + dependency_constraint: Path | None, *, use_uv: bool, ) -> dict[str, str]: @@ -103,7 +109,7 @@ def virtualenv( call("uv", "venv", venv_path, "--python", python) else: virtualenv_app = _ensure_virtualenv(version) - pip_constraint = _parse_pip_constraint_for_virtualenv(dependency_constraint_flags) + pip_constraint = _parse_pip_constraint_for_virtualenv(dependency_constraint) additional_flags = [f"--pip={pip_constraint}", "--no-setuptools", "--no-wheel"] # Using symlinks to pre-installed seed packages is really the fastest way to get a virtual From d6335189e4e242fb6d5d98985be0422f4667c460 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 8 Apr 2025 11:48:31 -0400 Subject: [PATCH 2/3] refactor: address feedback Signed-off-by: Henry Schreiner --- cibuildwheel/platforms/ios.py | 10 ++++------ cibuildwheel/platforms/macos.py | 19 +++++++++++-------- cibuildwheel/platforms/pyodide.py | 16 ++++++---------- cibuildwheel/platforms/windows.py | 15 ++++++--------- 4 files changed, 27 insertions(+), 33 deletions(-) diff --git a/cibuildwheel/platforms/ios.py b/cibuildwheel/platforms/ios.py index fcffe727f..3473ef740 100644 --- a/cibuildwheel/platforms/ios.py +++ b/cibuildwheel/platforms/ios.py @@ -177,8 +177,8 @@ def cross_virtualenv( :param build_python: The path to the python binary for the build platform :param venv_path: The path where the cross virtual environment should be created. - :param dependency_constraint_flags: Any flags that should be used when - constraining dependencies in the environment. + :param dependency_constraint: A path to a constraint file that should be + used when constraining dependencies in the environment. :param xbuild_tools: A list of executable names (without paths) that are on the path, but must be preserved in the cross environment. """ @@ -326,8 +326,6 @@ def setup_python( log.step("Creating cross build environment...") - dependency_constraint_flags = constraint_flags(dependency_constraint) - venv_path = tmp / "venv" env = cross_virtualenv( py_version=python_configuration.version, @@ -352,7 +350,7 @@ def setup_python( "install", "--upgrade", "pip", - *dependency_constraint_flags, + *constraint_flags(dependency_constraint), env=env, cwd=venv_path, ) @@ -398,7 +396,7 @@ def setup_python( "install", "--upgrade", "build[virtualenv]", - *dependency_constraint_flags, + *constraint_flags(dependency_constraint), env=env, ) else: diff --git a/cibuildwheel/platforms/macos.py b/cibuildwheel/platforms/macos.py index cea5cb3ab..428971263 100644 --- a/cibuildwheel/platforms/macos.py +++ b/cibuildwheel/platforms/macos.py @@ -219,8 +219,6 @@ def setup_python( f"{base_python.name} not found, has {list(base_python.parent.iterdir())}" ) - dependency_constraint_flags = constraint_flags(dependency_constraint) - log.step("Setting up build environment...") venv_path = tmp / "venv" env = virtualenv( @@ -257,7 +255,7 @@ def setup_python( "install", "--upgrade", "pip", - *dependency_constraint_flags, + *constraint_flags(dependency_constraint), env=env, cwd=venv_path, ) @@ -352,7 +350,7 @@ def setup_python( "install", "--upgrade", "delocate", - *dependency_constraint_flags, + *constraint_flags(dependency_constraint), env=env, ) elif build_frontend == "build": @@ -362,7 +360,7 @@ def setup_python( "--upgrade", "delocate", "build[virtualenv]", - *dependency_constraint_flags, + *constraint_flags(dependency_constraint), env=env, ) elif build_frontend == "build[uv]": @@ -374,7 +372,7 @@ def setup_python( "--upgrade", "delocate", "build[virtualenv, uv]", - *dependency_constraint_flags, + *constraint_flags(dependency_constraint), env=env, ) else: @@ -428,7 +426,6 @@ def build(options: Options, tmp_path: Path) -> None: constraints_path = build_options.dependency_constraints.get_for_python_version( version=config.version, tmp_dir=identifier_tmp_dir ) - dependency_constraint_flags = constraint_flags(constraints_path) base_python, env = setup_python( identifier_tmp_dir / "build", @@ -620,7 +617,13 @@ def build(options: Options, tmp_path: Path) -> None: # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. if not use_uv: - call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env) + call( + "pip", + "install", + "virtualenv", + *constraint_flags(constraints_path), + env=env, + ) venv_dir = identifier_tmp_dir / f"venv-test-{testing_arch}" diff --git a/cibuildwheel/platforms/pyodide.py b/cibuildwheel/platforms/pyodide.py index 8b06f5b8c..1c05dd0cd 100644 --- a/cibuildwheel/platforms/pyodide.py +++ b/cibuildwheel/platforms/pyodide.py @@ -3,7 +3,7 @@ import shutil import sys import tomllib -from collections.abc import Sequence, Set +from collections.abc import Set from dataclasses import dataclass from pathlib import Path from tempfile import TemporaryDirectory @@ -18,7 +18,6 @@ from ..logger import log from ..options import Options from ..selector import BuildSelector -from ..typing import PathOrStr from ..util import resources from ..util.cmd import call, shell from ..util.file import ( @@ -31,7 +30,7 @@ ) from ..util.helpers import prepare_command from ..util.packaging import combine_constraints, find_compatible_wheel, get_pip_version -from ..venv import virtualenv +from ..venv import constraint_flags, virtualenv IS_WIN: Final[bool] = sys.platform.startswith("win") @@ -145,7 +144,7 @@ def get_base_python(identifier: str) -> Path: def setup_python( tmp: Path, python_configuration: PythonConfiguration, - dependency_constraint_flags: Sequence[PathOrStr], + constraints_path: Path | None, environment: ParsedEnvironment, ) -> dict[str, str]: base_python = get_base_python(python_configuration.identifier) @@ -166,7 +165,7 @@ def setup_python( "install", "--upgrade", "pip", - *dependency_constraint_flags, + *constraint_flags(constraints_path), env=env, cwd=venv_path, ) @@ -198,7 +197,7 @@ def setup_python( "auditwheel-emscripten", "build[virtualenv]", "pyodide-build", - *dependency_constraint_flags, + *constraint_flags(constraints_path), env=env, ) @@ -269,14 +268,11 @@ def build(options: Options, tmp_path: Path) -> None: constraints_path = build_options.dependency_constraints.get_for_python_version( version=config.version, variant="pyodide", tmp_dir=identifier_tmp_dir ) - dependency_constraint_flags = ( - ["-c", constraints_path.as_uri()] if constraints_path else [] - ) env = setup_python( identifier_tmp_dir / "build", config, - dependency_constraint_flags, + constraints_path, build_options.environment, ) pip_version = get_pip_version(env) diff --git a/cibuildwheel/platforms/windows.py b/cibuildwheel/platforms/windows.py index 74e2e9b85..e429015b9 100644 --- a/cibuildwheel/platforms/windows.py +++ b/cibuildwheel/platforms/windows.py @@ -247,7 +247,7 @@ def setup_python( if build_frontend == "build[uv]" and not can_use_uv(python_configuration): build_frontend = "build" - use_uv = build_frontend in {"build[uv]", "uv"} + use_uv = build_frontend == "build[uv]" uv_path = find_uv() log.step("Setting up build environment...") @@ -260,8 +260,6 @@ def setup_python( use_uv=use_uv, ) - dependency_constraint_flags = constraint_flags(dependency_constraint) - # set up environment variables for run_with_env env["PYTHON_VERSION"] = python_configuration.version env["PYTHON_ARCH"] = python_configuration.arch @@ -311,7 +309,7 @@ def setup_python( "install", "--upgrade", "build[virtualenv]", - *dependency_constraint_flags, + *constraint_flags(dependency_constraint), env=env, ) elif build_frontend == "build[uv]": @@ -322,7 +320,7 @@ def setup_python( "install", "--upgrade", "build[virtualenv]", - *dependency_constraint_flags, + *constraint_flags(dependency_constraint), env=env, ) @@ -371,9 +369,6 @@ def build(options: Options, tmp_path: Path) -> None: version=config.version, tmp_dir=identifier_tmp_dir, ) - dependency_constraint_flags = ( - ["-c", constraints_path.as_uri()] if constraints_path else [] - ) # install Python base_python, env = setup_python( @@ -492,7 +487,9 @@ def build(options: Options, tmp_path: Path) -> None: # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. if not use_uv: - call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env) + call( + "pip", "install", "virtualenv", *constraint_flags(constraints_path), env=env + ) venv_dir = identifier_tmp_dir / "venv-test" From 6d7d94c55ddf2f6740731ab91debd3e01e8cf67d Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 8 Apr 2025 11:54:48 -0400 Subject: [PATCH 3/3] ci: better parallel Signed-off-by: Henry Schreiner --- azure-pipelines.yml | 4 ++-- bin/run_tests.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 76b1c6186..eaf93958e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,7 +7,7 @@ pr: jobs: - job: linux_311 timeoutInMinutes: 120 - pool: {vmImage: 'Ubuntu-20.04'} + pool: {vmImage: 'Ubuntu-22.04'} steps: - task: UsePythonVersion@0 inputs: @@ -27,7 +27,7 @@ jobs: - bash: | python -m pip install dependency-groups python -m dependency_groups test | xargs python -m pip install -e. - python ./bin/run_tests.py --num-processes 2 + python ./bin/run_tests.py - job: windows_311 pool: {vmImage: 'windows-2019'} diff --git a/bin/run_tests.py b/bin/run_tests.py index bbfbc25a5..b807938b1 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -8,7 +8,11 @@ from pathlib import Path if __name__ == "__main__": - default_cpu_count = os.cpu_count() or 2 + if sys.version_info < (3, 13): + default_cpu_count = os.cpu_count() or 2 + else: + default_cpu_count = os.process_cpu_count() or 2 + parser = argparse.ArgumentParser() parser.add_argument( "--run-podman", action="store_true", default=False, help="run podman tests (linux only)"