From 964ccb2fa947bc7b54bb889e6f910e3463cf9c3a Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Thu, 16 Nov 2023 01:14:38 -0800 Subject: [PATCH 1/2] [SPARK-45953][INFRA] Add Python 3.10 to Infra docker image --- dev/infra/Dockerfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile index 8d12f00a0348..3360508bece9 100644 --- a/dev/infra/Dockerfile +++ b/dev/infra/Dockerfile @@ -95,3 +95,15 @@ RUN python3.9 -m pip install 'torch<=2.0.1' torchvision --index-url https://down RUN python3.9 -m pip install torcheval # Add Deepspeed as a testing dependency for DeepspeedTorchDistributor RUN python3.9 -m pip install deepspeed + +# Install Python 3.10 at the last stage to avoid breaking Python 3.9 +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update && apt-get install -y \ + python3.10 \ + && rm -rf /var/lib/apt/lists/* +RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 +RUN python3.10 -m pip install numpy 'pyarrow>=14.0.0' 'pandas<=2.1.3' scipy unittest-xml-reporting plotly>=4.8 'mlflow>=2.3.1' coverage matplotlib openpyxl 'memory-profiler==0.60.0' 'scikit-learn==1.1.*' +RUN python3.10 -m pip install 'grpcio>=1.48,<1.57' 'grpcio-status>=1.48,<1.57' 'protobuf==3.20.3' 'googleapis-common-protos==1.56.4' +RUN python3.10 -m pip install 'torch<=2.0.1' torchvision --index-url https://download.pytorch.org/whl/cpu +RUN python3.10 -m pip install torcheval +RUN python3.10 -m pip install deepspeed From 07b32867500343c15037e408f3c0eaa210241064 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Thu, 16 Nov 2023 10:44:12 -0800 Subject: [PATCH 2/2] Add distutils --- dev/infra/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile index 3360508bece9..0231414eec6e 100644 --- a/dev/infra/Dockerfile +++ b/dev/infra/Dockerfile @@ -99,7 +99,7 @@ RUN python3.9 -m pip install deepspeed # Install Python 3.10 at the last stage to avoid breaking Python 3.9 RUN add-apt-repository ppa:deadsnakes/ppa RUN apt-get update && apt-get install -y \ - python3.10 \ + python3.10 python3.10-distutils \ && rm -rf /var/lib/apt/lists/* RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 RUN python3.10 -m pip install numpy 'pyarrow>=14.0.0' 'pandas<=2.1.3' scipy unittest-xml-reporting plotly>=4.8 'mlflow>=2.3.1' coverage matplotlib openpyxl 'memory-profiler==0.60.0' 'scikit-learn==1.1.*'