From f27a2f116102cb2c2a8be5c637f2383c36f0ef5f Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Fri, 17 Nov 2023 10:23:53 -0800 Subject: [PATCH] [SPARK-45979][INFRA] Add `Python 3.11` to Infra docker image --- dev/infra/Dockerfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile index c53cde35f70ba..a6e55200a55bc 100644 --- a/dev/infra/Dockerfile +++ b/dev/infra/Dockerfile @@ -111,3 +111,15 @@ RUN python3.10 -m pip install 'grpcio>=1.48,<1.57' 'grpcio-status>=1.48,<1.57' ' RUN python3.10 -m pip install 'torch<=2.0.1' torchvision --index-url https://download.pytorch.org/whl/cpu RUN python3.10 -m pip install torcheval RUN python3.10 -m pip install deepspeed + +# Install Python 3.11 at the last stage to avoid breaking the existing Python installations +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update && apt-get install -y \ + python3.11 python3.11-distutils \ + && rm -rf /var/lib/apt/lists/* +RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 +RUN python3.11 -m pip install numpy 'pyarrow>=14.0.0' 'pandas<=2.1.3' scipy unittest-xml-reporting plotly>=4.8 'mlflow>=2.3.1' coverage matplotlib openpyxl 'memory-profiler==0.60.0' 'scikit-learn==1.1.*' +RUN python3.11 -m pip install 'grpcio>=1.48,<1.57' 'grpcio-status>=1.48,<1.57' 'protobuf==3.20.3' 'googleapis-common-protos==1.56.4' +RUN python3.11 -m pip install 'torch<=2.0.1' torchvision --index-url https://download.pytorch.org/whl/cpu +RUN python3.11 -m pip install torcheval +RUN python3.11 -m pip install deepspeed