fix: enforce minimum version of docker/podman (#1961)

* fix: enforce minimum version of docker/podman

This allows to always pass `--platform` to the OCI engine
thus fixing issues with multiarch images.

* Allow older versions with warnings

* Upgrade docker on Travis CI

* fix: use `docker cp` instead of `tar`

* Enforce docker>=24.0

* move log to include container.copy_into

* fix: travis-ci, only update docker on aarch64

* skip test_multiarch_image on s390x / ppc64le

* skip flaky test

* chore: only install test deps on Travis CI

* fix: do not try to pull images tagged `cibw_local`

* use "--pull=never" for local images

* Use docker image inspect to check if an image needs to be pulled

Author: mayeut

.circleci/prepare.sh

@@ -4,6 +4,8 @@ set -o xtrace
 
 if [ "$(uname -s)" == "Darwin" ]; then
   sudo softwareupdate --install-rosetta --agree-to-license
+else
+  docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
 fi
 
 $PYTHON --version

.cirrus.yml

@@ -17,6 +17,7 @@ linux_x86_task:
     memory: 8G
 
   install_pre_requirements_script:
+    - docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
     - apt install -y python3-venv python-is-python3
   <<: *RUN_TESTS
 
@@ -30,6 +31,7 @@ linux_aarch64_task:
     memory: 4G
 
   install_pre_requirements_script:
+    - docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
     - apt install -y python3-venv python-is-python3
   <<: *RUN_TESTS
 

.github/workflows/test.yml

@@ -59,6 +59,11 @@ jobs:
         docker system prune -a -f
         df -h
 
+    # for oci_container unit tests
+    - name: Set up QEMU
+      if: runner.os == 'Linux'
+      uses: docker/setup-qemu-action@v3
+
     - name: Install dependencies
       run: |
         uv pip install --system ".[test]"
@@ -157,10 +162,7 @@ jobs:
       run: python -m pip install ".[test,uv]"
 
     - name: Set up QEMU
-      id: qemu
       uses: docker/setup-qemu-action@v3
-      with:
-        platforms: all
 
     - name: Run the emulation tests
       run: pytest --run-emulation ${{ matrix.arch }} test/test_emulation.py

.gitlab-ci.yml

@@ -15,6 +15,7 @@ linux:
     PYTEST_ADDOPTS: -k "unit_test or test_0_basic" --suppress-no-test-exit-code
   script:
     - curl -sSL https://get.docker.com/ | sh
+    - docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
     - python -m pip install -e ".[dev]" pytest-custom-exit-code
     - python ./bin/run_tests.py
 

.travis.yml

@@ -20,6 +20,14 @@ jobs:
       group: edge
       virt: vm
       env: PYTHON=python
+      # docker is outdated in the arm64-graviton2 vm focal image (19.x)
+      # we need to upgrade to get >= 24.0
+      addons:
+        apt:
+          sources:
+            - sourceline: 'deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable'
+          packages:
+            - docker-ce docker-ce-cli containerd.io
 
     - name: Linux | ppc64le | Python 3.9
       python: 3.9
@@ -48,8 +56,9 @@ jobs:
       env: PYTHON=python
 
 install:
+- if [ "${TRAVIS_OS_NAME}" == "linux" ]; then docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all; fi
 - $PYTHON -m pip install -U pip
-- $PYTHON -m pip install -e ".[dev]" pytest-custom-exit-code
+- $PYTHON -m pip install -e ".[test]" pytest-custom-exit-code
 
 script: |
   # travis_wait disable the output while waiting

appveyor.yml

@@ -17,6 +17,9 @@ init:
       if (-not ($BRANCH -eq 'main' -or $BRANCH.ToLower().StartsWith('appveyor-'))) {
         $env:PYTEST_ADDOPTS = '-k "unit_test or test_0_basic" --suppress-no-test-exit-code'
       }
+      if ($IsLinux) {
+        docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
+      }
 
 install:
   - python -m pip install -U pip

azure-pipelines.yml

@@ -13,6 +13,7 @@ jobs:
       inputs:
         versionSpec: '3.8'
     - bash: |
+        docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
         python -m pip install -e ".[dev]"
         python ./bin/run_tests.py
 

bin/run_tests.py

@@ -28,7 +28,7 @@
     # unit tests
     unit_test_args = [sys.executable, "-m", "pytest", "unit_test"]
 
-    if sys.platform.startswith("linux"):
+    if sys.platform.startswith("linux") and os.environ.get("CIBW_PLATFORM", "linux") == "linux":
         # run the docker unit tests only on Linux
         unit_test_args += ["--run-docker"]
 

cibuildwheel/architecture.py

@@ -64,7 +64,9 @@ def parse_config(config: str, platform: PlatformName) -> set[Architecture]:
             if arch_str == "auto":
                 result |= Architecture.auto_archs(platform=platform)
             elif arch_str == "native":
-                result.add(Architecture(platform_module.machine()))
+                native_arch = Architecture.native_arch(platform=platform)
+                if native_arch:
+                    result.add(native_arch)
             elif arch_str == "all":
                 result |= Architecture.all_archs(platform=platform)
             elif arch_str == "auto64":

cibuildwheel/errors.py

@@ -58,3 +58,7 @@ def __init__(self, wheel_name: str) -> None:
         )
         super().__init__(message)
         self.return_code = 6
+
+
+class OCIEngineTooOldError(FatalError):
+    return_code = 7

cibuildwheel/linux.py

@@ -14,7 +14,7 @@
 from ._compat.typing import assert_never
 from .architecture import Architecture
 from .logger import log
-from .oci_container import OCIContainer, OCIContainerEngineConfig
+from .oci_container import OCIContainer, OCIContainerEngineConfig, OCIPlatform
 from .options import BuildOptions, Options
 from .typing import PathOrStr
 from .util import (
@@ -29,6 +29,14 @@
     unwrap,
 )
 
+ARCHITECTURE_OCI_PLATFORM_MAP = {
+    Architecture.x86_64: OCIPlatform.AMD64,
+    Architecture.i686: OCIPlatform.i386,
+    Architecture.aarch64: OCIPlatform.ARM64,
+    Architecture.ppc64le: OCIPlatform.PPC64LE,
+    Architecture.s390x: OCIPlatform.S390X,
+}
+
 
 @dataclass(frozen=True)
 class PythonConfiguration:
@@ -196,6 +204,8 @@ def build_in_container(
 
         dependency_constraint_flags: list[PathOrStr] = []
 
+        log.step("Setting up build environment...")
+
         if build_options.dependency_constraints:
             constraints_file = build_options.dependency_constraints.get_for_python_version(
                 config.version
@@ -205,8 +215,6 @@ def build_in_container(
             container.copy_into(constraints_file, container_constraints_file)
             dependency_constraint_flags = ["-c", container_constraints_file]
 
-        log.step("Setting up build environment...")
-
         env = container.get_environment()
         env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
         env["PIP_ROOT_USER_ACTION"] = "ignore"
@@ -446,10 +454,11 @@ def build(options: Options, tmp_path: Path) -> None:  # noqa: ARG001
             log.step(f"Starting container image {build_step.container_image}...")
 
             print(f"info: This container will host the build for {', '.join(ids_to_build)}...")
+            architecture = Architecture(build_step.platform_tag.split("_", 1)[1])
 
             with OCIContainer(
                 image=build_step.container_image,
-                enforce_32_bit=build_step.platform_tag.endswith("i686"),
+                oci_platform=ARCHITECTURE_OCI_PLATFORM_MAP[architecture],
                 cwd=container_project_path,
                 engine=build_step.container_engine,
             ) as container: