From ac76f965e130b2c500e9c14f003e6f7d4bfed177 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 26 Jan 2026 17:43:16 +0100 Subject: [PATCH 01/13] WIP: [Python] Simplify Windows wheel build ecosystem --- ...eaded-wheel-windows-test-vs2022.dockerfile | 16 ++--- ...-wheel-windows-test-vs2022-base.dockerfile | 65 ------------------- ...ython-wheel-windows-test-vs2022.dockerfile | 12 ++-- ...ython-wheel-windows-vs2022-base.dockerfile | 15 +++++ .../python-wheel-windows-vs2022.dockerfile | 1 - compose.yaml | 12 +--- 6 files changed, 25 insertions(+), 96 deletions(-) delete mode 100644 ci/docker/python-wheel-windows-test-vs2022-base.dockerfile diff --git a/ci/docker/python-free-threaded-wheel-windows-test-vs2022.dockerfile b/ci/docker/python-free-threaded-wheel-windows-test-vs2022.dockerfile index ab257b271e58..b8940110f149 100644 --- a/ci/docker/python-free-threaded-wheel-windows-test-vs2022.dockerfile +++ b/ci/docker/python-free-threaded-wheel-windows-test-vs2022.dockerfile @@ -24,20 +24,12 @@ ARG base # hadolint ignore=DL3006 FROM ${base} -ARG python=3.13 +ARG python=3.10 -# hadolint ignore=SC1072 -RUN (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.1") & \ - (if "%python%"=="3.14" setx PYTHON_VERSION "3.14.0") +ARG python_variant_suffix="" +ENV PYTHON_VERSION=${python}${python_variant_suffix} -SHELL ["powershell", "-NoProfile", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] -RUN $version = $env:PYTHON_VERSION; \ - $filename = 'python-' + $version + '-amd64.exe'; \ - $url = 'https://www.python.org/ftp/python/' + $version + '/' + $filename; \ - Invoke-WebRequest -Uri $url -OutFile $filename; \ - Start-Process -FilePath $filename -ArgumentList '/quiet', 'Include_freethreaded=1' -Wait - -ENV PYTHON_CMD="py -${python}t" +RUN pymanager install %PYTHON_VERSION% SHELL ["cmd", "/S", "/C"] RUN %PYTHON_CMD% -m pip install -U pip setuptools & \ diff --git a/ci/docker/python-wheel-windows-test-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-test-vs2022-base.dockerfile deleted file mode 100644 index bd1da7b14b37..000000000000 --- a/ci/docker/python-wheel-windows-test-vs2022-base.dockerfile +++ /dev/null @@ -1,65 +0,0 @@ -# escape=` - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# NOTE: You must update PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION in .env -# when you update this file. - -FROM mcr.microsoft.com/windows/servercore:ltsc2022 - -# Ensure we in a command shell and not Powershell -SHELL ["cmd", "/S", "/C"] - -# Install MSVC BuildTools -# -# The set of components below (lines starting with --add) is the most minimal -# set we could find that would still compile Arrow C++. -RUN ` - curl -SL --output vs_buildtools.exe https://aka.ms/vs/17/release/vs_buildtools.exe ` - && (start /w vs_buildtools.exe --quiet --wait --norestart --nocache ` - --installPath "%ProgramFiles(x86)%\Microsoft Visual Studio\2022\BuildTools" ` - --add Microsoft.VisualStudio.Component.VC.CoreBuildTools ` - --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` - --add Microsoft.VisualStudio.Component.Windows11SDK.26100 ` - --add Microsoft.VisualStudio.Component.VC.CMake.Project ` - || IF "%ERRORLEVEL%"=="3010" EXIT 0) ` - && del /q vs_buildtools.exe - -# Install choco CLI -# -# We switch into Powershell just for this command and switch back to cmd -# See https://chocolatey.org/install#completely-offline-install -SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] -RUN ` - Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) -SHELL ["cmd", "/S", "/C"] - -# Install git, wget, minio -RUN choco install --no-progress -r -y git wget -RUN curl https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2025-01-20T14-49-07Z ` - --output "C:\Windows\Minio.exe" - -# Install the GCS testbench using a well-known Python version. -# NOTE: cannot use pipx's `--fetch-missing-python` because of -# https://github.com/pypa/pipx/issues/1521, therefore download Python ourselves. -RUN choco install -r -y --pre --no-progress python --version=3.11.9 -ENV PIPX_BIN_DIR=C:\\Windows\\ -ENV PIPX_PYTHON="C:\Python311\python.exe" -COPY ci/scripts/install_gcs_testbench.bat C:/arrow/ci/scripts/ -RUN call "C:\arrow\ci\scripts\install_gcs_testbench.bat" && ` - storage-testbench -h diff --git a/ci/docker/python-wheel-windows-test-vs2022.dockerfile b/ci/docker/python-wheel-windows-test-vs2022.dockerfile index 73a9e167fea6..6fc13c1d123f 100644 --- a/ci/docker/python-wheel-windows-test-vs2022.dockerfile +++ b/ci/docker/python-wheel-windows-test-vs2022.dockerfile @@ -26,16 +26,12 @@ FROM ${base} # hadolint shell=cmd.exe -# Define the full version number otherwise choco falls back to patch number 0 (3.10 => 3.10.0) ARG python=3.10 -RUN (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PYTHON_CMD "py -3.10") & \ - (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.9" && setx PYTHON_CMD "py -3.11") & \ - (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.10" && setx PYTHON_CMD "py -3.12") & \ - (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.9" && setx PYTHON_CMD "py -3.13") & \ - (if "%python%"=="3.14" setx PYTHON_VERSION "3.14.0" && setx PYTHON_CMD "py -3.14") -# hadolint ignore=DL3059 -RUN choco install -r -y --pre --no-progress --force python --version=%PYTHON_VERSION% +ARG python_variant_suffix="" +ENV PYTHON_VERSION=${python}${python_variant_suffix} + +RUN pymanager install %PYTHON_VERSION% # hadolint ignore=DL3059 RUN %PYTHON_CMD% -m pip install -U pip setuptools diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile index 426286ebe07d..1d53e863017b 100644 --- a/ci/docker/python-wheel-windows-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -104,6 +104,21 @@ RUN choco install --no-progress -r -y git gzip ninja wget # Add UNIX tools to PATH RUN setx path "%path%;C:\Program Files\Git\usr\bin" +# Install git, wget, minio +RUN choco install --no-progress -r -y git wget +RUN curl https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2025-01-20T14-49-07Z ` + --output "C:\Windows\Minio.exe" + +# Install the GCS testbench using a well-known Python version. +# NOTE: cannot use pipx's `--fetch-missing-python` because of +# https://github.com/pypa/pipx/issues/1521, therefore download Python ourselves. +RUN choco install -r -y --pre --no-progress python --version=3.11.9 +ENV PIPX_BIN_DIR=C:\\Windows\\ +ENV PIPX_PYTHON="C:\Python311\python.exe" +COPY ci/scripts/install_gcs_testbench.bat C:/arrow/ci/scripts/ +RUN call "C:\arrow\ci\scripts\install_gcs_testbench.bat" && ` + storage-testbench -h + # Install vcpkg # # Compiling vcpkg itself from a git tag doesn't work anymore since vcpkg has diff --git a/ci/docker/python-wheel-windows-vs2022.dockerfile b/ci/docker/python-wheel-windows-vs2022.dockerfile index e25ebef156c6..0ed5be4939ae 100644 --- a/ci/docker/python-wheel-windows-vs2022.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022.dockerfile @@ -21,7 +21,6 @@ ARG base FROM ${base} -# Define the full version number otherwise choco falls back to patch number 0 (3.10 => 3.10.0) ARG python=3.10 ARG python_variant_suffix="" diff --git a/compose.yaml b/compose.yaml index c99f19e35e0a..edbfa151e291 100644 --- a/compose.yaml +++ b/compose.yaml @@ -178,7 +178,6 @@ x-hierarchy: - python-wheel-windows-vs2022-base: - python-wheel-windows-vs2022 - python-free-threaded-wheel-windows-vs2022 - - python-wheel-windows-test-base: - python-wheel-windows-test - python-free-threaded-wheel-windows-test @@ -1416,18 +1415,11 @@ services: volumes: *python-wheel-windows-vs2022-volumes command: arrow\\ci\\scripts\\python_wheel_windows_build.bat - python-wheel-windows-test-base: - image: ${REPO}:python-wheel-windows-test-vs2022-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} - build: - context: . - dockerfile: ci/docker/python-wheel-windows-test-vs2022-base.dockerfile - volumes: *python-wheel-windows-vs2022-volumes - python-wheel-windows-test: image: ${REPO}:python-${PYTHON}-wheel-windows-test-vs2022-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} build: args: - base: ${REPO}:python-wheel-windows-test-vs2022-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} + base: ${REPO}:python-${PYTHON}-wheel-windows-vs2022-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} python: ${PYTHON} context: . dockerfile: ci/docker/python-wheel-windows-test-vs2022.dockerfile @@ -1438,7 +1430,7 @@ services: image: ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-test-vs2022-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} build: args: - base: ${REPO}:python-wheel-windows-test-vs2022-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} + base: ${REPO}:python-${PYTHON}-wheel-windows-vs2022-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} python: ${PYTHON} context: . dockerfile: ci/docker/python-free-threaded-wheel-windows-test-vs2022.dockerfile From 054b8dc16b93ea88b1c4c2fd7c8fd8d5ce58ece2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 27 Jan 2026 14:42:28 +0100 Subject: [PATCH 02/13] Add python_variant_suffix for testing free-threaded --- compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/compose.yaml b/compose.yaml index edbfa151e291..1f0e721440d8 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1432,6 +1432,7 @@ services: args: base: ${REPO}:python-${PYTHON}-wheel-windows-vs2022-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} python: ${PYTHON} + python_variant_suffix: t context: . dockerfile: ci/docker/python-free-threaded-wheel-windows-test-vs2022.dockerfile volumes: *python-wheel-windows-vs2022-volumes From a07afbf68f68c58f03744f5c6d5ed0efc130b985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 27 Jan 2026 19:47:07 +0100 Subject: [PATCH 03/13] Try upgrading wheel windows image revision --- .env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 6d64d2847807..6f830ca229b6 100644 --- a/.env +++ b/.env @@ -102,8 +102,8 @@ VCPKG="4334d8b4c8916018600212ab4dd4bbdc343065d1" # 2025.09.17 Release # ci/docker/python-*-windows-*.dockerfile or the vcpkg config. # This is a workaround for our CI problem that "archery docker build" doesn't # use pulled built images in dev/tasks/python-wheels/github.windows.yml. -PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2026-01-22 -PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=2026-01-22 +PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2026-01-27 +PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=2026-01-27 # Use conanio/${CONAN_BASE}:{CONAN_VERSION} for "docker compose run --rm conan". # See https://github.com/conan-io/conan-docker-tools#readme and From e1ab7da83dac4ab2001fbf900290f9951fc6fb6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 28 Jan 2026 09:34:08 +0100 Subject: [PATCH 04/13] Fix base image --- compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.yaml b/compose.yaml index 1f0e721440d8..029af9d6fa22 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1430,7 +1430,7 @@ services: image: ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-test-vs2022-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} build: args: - base: ${REPO}:python-${PYTHON}-wheel-windows-vs2022-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} + base: ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-vs2022-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} python: ${PYTHON} python_variant_suffix: t context: . From 66650984678b110f6242440f7ff16826fc78ee5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 28 Jan 2026 13:05:18 +0100 Subject: [PATCH 05/13] Remove unnecessary change --- .../python-free-threaded-wheel-windows-test-vs2022.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/python-free-threaded-wheel-windows-test-vs2022.dockerfile b/ci/docker/python-free-threaded-wheel-windows-test-vs2022.dockerfile index b8940110f149..2493dc27c86a 100644 --- a/ci/docker/python-free-threaded-wheel-windows-test-vs2022.dockerfile +++ b/ci/docker/python-free-threaded-wheel-windows-test-vs2022.dockerfile @@ -24,7 +24,7 @@ ARG base # hadolint ignore=DL3006 FROM ${base} -ARG python=3.10 +ARG python=3.13 ARG python_variant_suffix="" ENV PYTHON_VERSION=${python}${python_variant_suffix} From 2aa3a1f0c582c20e092d288bab55e2cd7ea48e3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 28 Jan 2026 13:24:02 +0100 Subject: [PATCH 06/13] Base for testing images should be the base image not the build image --- compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose.yaml b/compose.yaml index 029af9d6fa22..3474643fe6e7 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1419,7 +1419,7 @@ services: image: ${REPO}:python-${PYTHON}-wheel-windows-test-vs2022-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} build: args: - base: ${REPO}:python-${PYTHON}-wheel-windows-vs2022-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} + base: ${REPO}:python-wheel-windows-vs2022-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} python: ${PYTHON} context: . dockerfile: ci/docker/python-wheel-windows-test-vs2022.dockerfile @@ -1430,7 +1430,7 @@ services: image: ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-test-vs2022-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} build: args: - base: ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-vs2022-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} + base: ${REPO}:python-wheel-windows-vs2022-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} python: ${PYTHON} python_variant_suffix: t context: . From 1ca2938615416c49f204d489ee10fdcd54d236d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 28 Jan 2026 16:44:43 +0100 Subject: [PATCH 07/13] Dockerfiles for test should specify PYTHON_CMD --- .../python-free-threaded-wheel-windows-test-vs2022.dockerfile | 2 ++ ci/docker/python-wheel-windows-test-vs2022.dockerfile | 3 +++ 2 files changed, 5 insertions(+) diff --git a/ci/docker/python-free-threaded-wheel-windows-test-vs2022.dockerfile b/ci/docker/python-free-threaded-wheel-windows-test-vs2022.dockerfile index 2493dc27c86a..0ffb08edf8fa 100644 --- a/ci/docker/python-free-threaded-wheel-windows-test-vs2022.dockerfile +++ b/ci/docker/python-free-threaded-wheel-windows-test-vs2022.dockerfile @@ -31,6 +31,8 @@ ENV PYTHON_VERSION=${python}${python_variant_suffix} RUN pymanager install %PYTHON_VERSION% +ENV PYTHON_CMD="py -${python}${python_variant_suffix}" + SHELL ["cmd", "/S", "/C"] RUN %PYTHON_CMD% -m pip install -U pip setuptools & \ if "%python%"=="3.13" ( \ diff --git a/ci/docker/python-wheel-windows-test-vs2022.dockerfile b/ci/docker/python-wheel-windows-test-vs2022.dockerfile index 6fc13c1d123f..5a5bf1506a67 100644 --- a/ci/docker/python-wheel-windows-test-vs2022.dockerfile +++ b/ci/docker/python-wheel-windows-test-vs2022.dockerfile @@ -32,6 +32,9 @@ ARG python_variant_suffix="" ENV PYTHON_VERSION=${python}${python_variant_suffix} RUN pymanager install %PYTHON_VERSION% + +ENV PYTHON_CMD="py -${python}${python_variant_suffix}" + # hadolint ignore=DL3059 RUN %PYTHON_CMD% -m pip install -U pip setuptools From f4d33c277295f961a153eba47f3c308d31b2d9c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 29 Jan 2026 11:47:05 +0100 Subject: [PATCH 08/13] Use pymanager to install Python for GCS testbench --- ci/docker/python-wheel-windows-vs2022-base.dockerfile | 6 ++---- ci/scripts/install_gcs_testbench.bat | 9 +++------ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile index 1d53e863017b..8778b85b8297 100644 --- a/ci/docker/python-wheel-windows-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -110,11 +110,9 @@ RUN curl https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELE --output "C:\Windows\Minio.exe" # Install the GCS testbench using a well-known Python version. -# NOTE: cannot use pipx's `--fetch-missing-python` because of -# https://github.com/pypa/pipx/issues/1521, therefore download Python ourselves. -RUN choco install -r -y --pre --no-progress python --version=3.11.9 ENV PIPX_BIN_DIR=C:\\Windows\\ -ENV PIPX_PYTHON="C:\Python311\python.exe" +ENV PIPX_PYTHON_VERSION=3.11 +RUN pymanager install %PIPX_PYTHON_VERSION% COPY ci/scripts/install_gcs_testbench.bat C:/arrow/ci/scripts/ RUN call "C:\arrow\ci\scripts\install_gcs_testbench.bat" && ` storage-testbench -h diff --git a/ci/scripts/install_gcs_testbench.bat b/ci/scripts/install_gcs_testbench.bat index d0ceb7be2b69..8b30087064af 100644 --- a/ci/scripts/install_gcs_testbench.bat +++ b/ci/scripts/install_gcs_testbench.bat @@ -20,15 +20,12 @@ set GCS_TESTBENCH_VERSION="v0.55.0" set PIPX_FLAGS=--verbose -if NOT "%PIPX_PYTHON%"=="" ( - set PIPX_FLAGS=--python %PIPX_PYTHON% %PIPX_FLAGS% -) -python -m pip install -U pipx || exit /B 1 +py -%PIPX_PYTHON_VERSION% -m pip install -U pipx|| exit /B 1 @REM Install GCS testbench %GCS_TESTBENCH_VERSION% -pipx install %PIPX_FLAGS% ^ +py -%PIPX_PYTHON_VERSION% -m pipx install --python py %PIPX_FLAGS% ^ "https://github.com/googleapis/storage-testbench/archive/%GCS_TESTBENCH_VERSION%.tar.gz" ^ || exit /B 1 -pipx list --verbose +py -%PIPX_PYTHON_VERSION% -m pipx list --verbose From eac026f0fd19b9410d21bf2fd6299ea405476f3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 30 Jan 2026 10:52:41 +0100 Subject: [PATCH 09/13] Try removing python installation and see if there's a python on the base image --- ci/docker/python-wheel-windows-vs2022-base.dockerfile | 3 --- ci/scripts/install_gcs_testbench.bat | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile index 8778b85b8297..2973b769e6f7 100644 --- a/ci/docker/python-wheel-windows-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -110,9 +110,6 @@ RUN curl https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELE --output "C:\Windows\Minio.exe" # Install the GCS testbench using a well-known Python version. -ENV PIPX_BIN_DIR=C:\\Windows\\ -ENV PIPX_PYTHON_VERSION=3.11 -RUN pymanager install %PIPX_PYTHON_VERSION% COPY ci/scripts/install_gcs_testbench.bat C:/arrow/ci/scripts/ RUN call "C:\arrow\ci\scripts\install_gcs_testbench.bat" && ` storage-testbench -h diff --git a/ci/scripts/install_gcs_testbench.bat b/ci/scripts/install_gcs_testbench.bat index 8b30087064af..b4ed0d9f957a 100644 --- a/ci/scripts/install_gcs_testbench.bat +++ b/ci/scripts/install_gcs_testbench.bat @@ -21,11 +21,11 @@ set GCS_TESTBENCH_VERSION="v0.55.0" set PIPX_FLAGS=--verbose -py -%PIPX_PYTHON_VERSION% -m pip install -U pipx|| exit /B 1 +python -m pip install -U pipx|| exit /B 1 @REM Install GCS testbench %GCS_TESTBENCH_VERSION% -py -%PIPX_PYTHON_VERSION% -m pipx install --python py %PIPX_FLAGS% ^ +pipx install %PIPX_FLAGS% ^ "https://github.com/googleapis/storage-testbench/archive/%GCS_TESTBENCH_VERSION%.tar.gz" ^ || exit /B 1 -py -%PIPX_PYTHON_VERSION% -m pipx list --verbose +pipx list --verbose From 98410dd6ec394682f870d30527035d216059c7fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 30 Jan 2026 15:01:48 +0100 Subject: [PATCH 10/13] Use base image Python instead for gcs --- ci/scripts/install_gcs_testbench.bat | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/install_gcs_testbench.bat b/ci/scripts/install_gcs_testbench.bat index b4ed0d9f957a..aa8a1756aa09 100644 --- a/ci/scripts/install_gcs_testbench.bat +++ b/ci/scripts/install_gcs_testbench.bat @@ -24,8 +24,8 @@ set PIPX_FLAGS=--verbose python -m pip install -U pipx|| exit /B 1 @REM Install GCS testbench %GCS_TESTBENCH_VERSION% -pipx install %PIPX_FLAGS% ^ +python -m pipx install --fetch-missing-python --python 3.11 %PIPX_FLAGS% ^ "https://github.com/googleapis/storage-testbench/archive/%GCS_TESTBENCH_VERSION%.tar.gz" ^ || exit /B 1 -pipx list --verbose +python -m pipx list --verbose \ No newline at end of file From a1b065284d4adbee057e4cd5c6f2e1407716b1b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 3 Feb 2026 10:48:28 +0100 Subject: [PATCH 11/13] Revert "Use base image Python instead for gcs" This reverts commit 98410dd6ec394682f870d30527035d216059c7fc. --- ci/scripts/install_gcs_testbench.bat | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/install_gcs_testbench.bat b/ci/scripts/install_gcs_testbench.bat index aa8a1756aa09..b4ed0d9f957a 100644 --- a/ci/scripts/install_gcs_testbench.bat +++ b/ci/scripts/install_gcs_testbench.bat @@ -24,8 +24,8 @@ set PIPX_FLAGS=--verbose python -m pip install -U pipx|| exit /B 1 @REM Install GCS testbench %GCS_TESTBENCH_VERSION% -python -m pipx install --fetch-missing-python --python 3.11 %PIPX_FLAGS% ^ +pipx install %PIPX_FLAGS% ^ "https://github.com/googleapis/storage-testbench/archive/%GCS_TESTBENCH_VERSION%.tar.gz" ^ || exit /B 1 -python -m pipx list --verbose \ No newline at end of file +pipx list --verbose From e4058efb05ac697403f2addd16e38d31745cc26a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 3 Feb 2026 10:48:36 +0100 Subject: [PATCH 12/13] Revert "Try removing python installation and see if there's a python on the base image" This reverts commit eac026f0fd19b9410d21bf2fd6299ea405476f3f. --- ci/docker/python-wheel-windows-vs2022-base.dockerfile | 3 +++ ci/scripts/install_gcs_testbench.bat | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile index 2973b769e6f7..8778b85b8297 100644 --- a/ci/docker/python-wheel-windows-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -110,6 +110,9 @@ RUN curl https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELE --output "C:\Windows\Minio.exe" # Install the GCS testbench using a well-known Python version. +ENV PIPX_BIN_DIR=C:\\Windows\\ +ENV PIPX_PYTHON_VERSION=3.11 +RUN pymanager install %PIPX_PYTHON_VERSION% COPY ci/scripts/install_gcs_testbench.bat C:/arrow/ci/scripts/ RUN call "C:\arrow\ci\scripts\install_gcs_testbench.bat" && ` storage-testbench -h diff --git a/ci/scripts/install_gcs_testbench.bat b/ci/scripts/install_gcs_testbench.bat index b4ed0d9f957a..8b30087064af 100644 --- a/ci/scripts/install_gcs_testbench.bat +++ b/ci/scripts/install_gcs_testbench.bat @@ -21,11 +21,11 @@ set GCS_TESTBENCH_VERSION="v0.55.0" set PIPX_FLAGS=--verbose -python -m pip install -U pipx|| exit /B 1 +py -%PIPX_PYTHON_VERSION% -m pip install -U pipx|| exit /B 1 @REM Install GCS testbench %GCS_TESTBENCH_VERSION% -pipx install %PIPX_FLAGS% ^ +py -%PIPX_PYTHON_VERSION% -m pipx install --python py %PIPX_FLAGS% ^ "https://github.com/googleapis/storage-testbench/archive/%GCS_TESTBENCH_VERSION%.tar.gz" ^ || exit /B 1 -pipx list --verbose +py -%PIPX_PYTHON_VERSION% -m pipx list --verbose From 796f7c75e85f8743f8ceb4cd4e9926c218a7cb05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 3 Feb 2026 10:48:42 +0100 Subject: [PATCH 13/13] Revert "Use pymanager to install Python for GCS testbench" This reverts commit f4d33c277295f961a153eba47f3c308d31b2d9c4. --- ci/docker/python-wheel-windows-vs2022-base.dockerfile | 6 ++++-- ci/scripts/install_gcs_testbench.bat | 9 ++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile index 8778b85b8297..1d53e863017b 100644 --- a/ci/docker/python-wheel-windows-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -110,9 +110,11 @@ RUN curl https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELE --output "C:\Windows\Minio.exe" # Install the GCS testbench using a well-known Python version. +# NOTE: cannot use pipx's `--fetch-missing-python` because of +# https://github.com/pypa/pipx/issues/1521, therefore download Python ourselves. +RUN choco install -r -y --pre --no-progress python --version=3.11.9 ENV PIPX_BIN_DIR=C:\\Windows\\ -ENV PIPX_PYTHON_VERSION=3.11 -RUN pymanager install %PIPX_PYTHON_VERSION% +ENV PIPX_PYTHON="C:\Python311\python.exe" COPY ci/scripts/install_gcs_testbench.bat C:/arrow/ci/scripts/ RUN call "C:\arrow\ci\scripts\install_gcs_testbench.bat" && ` storage-testbench -h diff --git a/ci/scripts/install_gcs_testbench.bat b/ci/scripts/install_gcs_testbench.bat index 8b30087064af..d0ceb7be2b69 100644 --- a/ci/scripts/install_gcs_testbench.bat +++ b/ci/scripts/install_gcs_testbench.bat @@ -20,12 +20,15 @@ set GCS_TESTBENCH_VERSION="v0.55.0" set PIPX_FLAGS=--verbose +if NOT "%PIPX_PYTHON%"=="" ( + set PIPX_FLAGS=--python %PIPX_PYTHON% %PIPX_FLAGS% +) -py -%PIPX_PYTHON_VERSION% -m pip install -U pipx|| exit /B 1 +python -m pip install -U pipx || exit /B 1 @REM Install GCS testbench %GCS_TESTBENCH_VERSION% -py -%PIPX_PYTHON_VERSION% -m pipx install --python py %PIPX_FLAGS% ^ +pipx install %PIPX_FLAGS% ^ "https://github.com/googleapis/storage-testbench/archive/%GCS_TESTBENCH_VERSION%.tar.gz" ^ || exit /B 1 -py -%PIPX_PYTHON_VERSION% -m pipx list --verbose +pipx list --verbose