GH-47797: [CI][Python] Update Python installs for free-threaded wheel tasks

ci/docker/python-free-threaded-wheel-musllinux-test-imports.dockerfile

@@ -19,38 +19,32 @@ ARG base
 FROM ${base}
 
 ARG python_version=3.13
+ARG arch=aarch64
+ARG build_date
 
-RUN apk add --no-cache \
+RUN apk update && \
+    apk add --no-cache \
     bash \
-    build-base \
-    bzip2-dev \
-    g++ \
+    curl \
     git \
-    libffi-dev \
-    libnsl-dev \
-    libtirpc-dev \
-    linux-headers \
-    ncurses-dev \
-    openssl-dev \
-    pkgconf \
+    tar \
     tzdata \
-    zlib-dev
+    zstd
 
-# Install Python without GIL
+# Install Python with free-threading from python-build-standalone
+# See available releases at: https://github.com/astral-sh/python-build-standalone/releases
 RUN set -e; \
     case "${python_version}" in \
       3.13) python_patch_version="3.13.9";; \
       3.14) python_patch_version="3.14.0";; \
     esac && \
-    wget https://github.com/python/cpython/archive/refs/tags/v${python_patch_version}.tar.gz && \
-    tar -xzf v${python_patch_version}.tar.gz && \
-    rm v${python_patch_version}.tar.gz && \
-    cd cpython-${python_patch_version}/ && \
-    ./configure --disable-gil --with-ensurepip && \
-    make -j && \
-    make install && \
-    cd ../ && \
-    rm -rf cpython-${python_patch_version}/
+    curl -L -o python.tar.zst \
+    https://github.com/astral-sh/python-build-standalone/releases/download/${build_date}/cpython-${python_patch_version}+${build_date}-${arch}-unknown-linux-musl-freethreaded+lto-full.tar.zst && \
+    mkdir -p /opt/python && \
+    tar -xf python.tar.zst -C /opt/python --strip-components=1 && \
+    rm python.tar.zst
+
+ENV PATH="/opt/python/install/bin:${PATH}"
 
 ENV ARROW_PYTHON_VENV /arrow-dev
 RUN python${python_version}t -m venv ${ARROW_PYTHON_VENV}

ci/docker/python-free-threaded-wheel-musllinux-test-unittests.dockerfile

@@ -19,38 +19,34 @@ ARG base
 FROM ${base}
 
 ARG python_version=3.13
+ARG arch=aarch64
+ARG build_date
 
-RUN apk add --no-cache \
+RUN apk update && \
+    apk add --no-cache \
     bash \
     build-base \
-    bzip2-dev \
+    curl \
     g++ \
     git \
-    libffi-dev \
-    libnsl-dev \
-    libtirpc-dev \
-    linux-headers \
-    ncurses-dev \
-    openssl-dev \
-    pkgconf \
+    tar \
     tzdata \
-    zlib-dev
+    zstd
 
-# Install Python without GIL
+# Install Python with free-threading from python-build-standalone
+# See available releases at: https://github.com/astral-sh/python-build-standalone/releases
 RUN set -e; \
     case "${python_version}" in \
       3.13) python_patch_version="3.13.9";; \
       3.14) python_patch_version="3.14.0";; \
     esac && \
-    wget https://github.com/python/cpython/archive/refs/tags/v${python_patch_version}.tar.gz && \
-    tar -xzf v${python_patch_version}.tar.gz && \
-    rm v${python_patch_version}.tar.gz && \
-    cd cpython-${python_patch_version}/ && \
-    ./configure --disable-gil --with-ensurepip && \
-    make -j && \
-    make install && \
-    cd ../ && \
-    rm -rf cpython-${python_patch_version}/
+    curl -L -o python.tar.zst \
+    https://github.com/astral-sh/python-build-standalone/releases/download/${build_date}/cpython-${python_patch_version}+${build_date}-${arch}-unknown-linux-musl-freethreaded+lto-full.tar.zst && \
+    mkdir -p /opt/python && \
+    tar -xf python.tar.zst -C /opt/python --strip-components=1 && \
+    rm python.tar.zst
+
+ENV PATH="/opt/python/install/bin:${PATH}"
 
 ENV ARROW_PYTHON_VENV /arrow-dev
 RUN python${python_version}t -m venv ${ARROW_PYTHON_VENV}

ci/docker/python-wheel-windows-test-vs2022.dockerfile

@@ -42,4 +42,4 @@ RUN %PYTHON_CMD% -m pip install -U pip setuptools
 COPY python/requirements-wheel-test.txt C:/arrow/python/
 RUN %PYTHON_CMD% -m pip install -r C:/arrow/python/requirements-wheel-test.txt
 
-ENV PYTHON=$python
+ENV PYTHON=${python}

ci/docker/python-wheel-windows-vs2022-base.dockerfile

@@ -74,13 +74,25 @@ RUN `
 
 # Install choco CLI
 #
-# Switch into Powershell just for this command because choco only provides a
-# Powershell installation script. After, we switch back to cmd.
+# Switch into Powershell just for the following commands because choco and Python install manager (MSIX)
+# only provide a Powershell installation scripts. After, we switch back to cmd.
 #
 # See https://chocolatey.org/install#completely-offline-install
 SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
 RUN `
-  Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
+  Set-ExecutionPolicy Bypass -Scope Process -Force; `
+  [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; `
+  iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
+
+# Install the Python install manager
+#
+# See https://docs.python.org/dev/using/windows.html#python-install-manager and
+# https://www.python.org/ftp/python/pymanager/
+RUN `
+  $pymanager_url = 'https://www.python.org/ftp/python/pymanager/python-manager-25.0.msix'; `
+  Invoke-WebRequest -Uri $pymanager_url -OutFile 'C:\Windows\pymanager.msix'; `
+  Add-AppxPackage C:\Windows\pymanager.msix
+
 SHELL ["cmd", "/S", "/C"]
 
 # Install CMake and other tools

ci/docker/python-wheel-windows-vs2022.dockerfile

@@ -23,16 +23,16 @@ FROM ${base}
 
 # Define the full version number otherwise choco falls back to patch number 0 (3.10 => 3.10.0)
 ARG python=3.10
-RUN (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PYTHON_CMD "py -3.10") & \
-    (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.9" && setx PYTHON_CMD "py -3.11") & \
-    (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.10" && setx PYTHON_CMD "py -3.12") & \
-    (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.9" && setx PYTHON_CMD "py -3.13") & \
-    (if "%python%"=="3.14" setx PYTHON_VERSION "3.14.0" && setx PYTHON_CMD "py -3.14")
 
-RUN choco install -r -y --pre --no-progress python --version=%PYTHON_VERSION%
-RUN %PYTHON_CMD% -m pip install -U pip setuptools
+# Use python_abi_tag env var to select regular or free-threaded Python
+ARG python_variant=default
+ENV PYTHON_VERSION=${python}
+ENV PYTHON_VARIANT=${python_variant}
+RUN pymanager install --version %PYTHON_VERSION% --variant %PYTHON_VARIANT%
+
+RUN py -%PYTHON_VERSION% -m pip install -U pip setuptools
 
 COPY python/requirements-wheel-build.txt C:/arrow/python/
-RUN %PYTHON_CMD% -m pip install -r C:/arrow/python/requirements-wheel-build.txt
+RUN py -%PYTHON_VERSION% -m pip install -r C:/arrow/python/requirements-wheel-build.txt
 
 ENV PYTHON=${python}

compose.yaml

@@ -1248,6 +1248,8 @@ services:
       args:
         base: "${ARCH}/alpine:${ALPINE_LINUX}"
         python_version: ${PYTHON}
+        build_date: "20251014"  # python-build-standalone release date
+        arch: ${ARCH_ALIAS}
       context: .
       dockerfile: ci/docker/python-free-threaded-wheel-musllinux-test-imports.dockerfile
       cache_from:
@@ -1290,6 +1292,8 @@ services:
       args:
         base: "${ARCH}/alpine:${ALPINE_LINUX}"
         python_version: ${PYTHON}
+        build_date: "20251014"  # python-build-standalone release date
+        arch: ${ARCH_ALIAS}
       context: .
       dockerfile: ci/docker/python-free-threaded-wheel-musllinux-test-unittests.dockerfile
       cache_from:
@@ -1404,6 +1408,7 @@ services:
       args:
         base: ${REPO}:python-wheel-windows-vs2022-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
         python: ${PYTHON}
+        python_variant: default
       context: .
       dockerfile: ci/docker/python-wheel-windows-vs2022.dockerfile
       # This should make the pushed images reusable, but the image gets rebuilt.
@@ -1420,8 +1425,9 @@ services:
       args:
         base: ${REPO}:python-wheel-windows-vs2022-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
         python: ${PYTHON}
+        python_variant: freethreaded
       context: .
-      dockerfile: ci/docker/python-free-threaded-wheel-windows-vs2022.dockerfile
+      dockerfile: ci/docker/python-wheel-windows-vs2022.dockerfile
       # This should make the pushed images reusable, but the image gets rebuilt.
       # Uncomment if no local cache is available.
       # cache_from: