From 7a33e7f55ad597c0525f5c62f9399c72b6f031f9 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Tue, 16 Jan 2024 20:41:06 +0800 Subject: [PATCH 1/3] df -h --- dev/infra/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile index 78814ace9b2e0..1f3c24c511cce 100644 --- a/dev/infra/Dockerfile +++ b/dev/infra/Dockerfile @@ -137,3 +137,5 @@ RUN python3.12 -m pip install $BASIC_PIP_PKGS $CONNECT_PIP_PKGS lxml RUN python3.12 -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu RUN python3.12 -m pip install torchvision --index-url https://download.pytorch.org/whl/cpu RUN python3.12 -m pip install torcheval + +RUN df -h From a31969572a3766b6e7085121f8e318f2f4e3bca9 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Wed, 17 Jan 2024 08:20:42 +0800 Subject: [PATCH 2/3] test cache purge --- dev/infra/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile index 1f3c24c511cce..54f62bbc82023 100644 --- a/dev/infra/Dockerfile +++ b/dev/infra/Dockerfile @@ -19,7 +19,7 @@ # See also in https://hub.docker.com/_/ubuntu FROM ubuntu:focal-20221019 -ENV FULL_REFRESH_DATE 20231117 +ENV FULL_REFRESH_DATE 20240117 ENV DEBIAN_FRONTEND noninteractive ENV DEBCONF_NONINTERACTIVE_SEEN true @@ -104,6 +104,7 @@ RUN python3.9 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PIP # Add torch as a testing dependency for TorchDistributor and DeepspeedTorchDistributor RUN python3.9 -m pip install 'torch<=2.0.1' torchvision --index-url https://download.pytorch.org/whl/cpu RUN python3.9 -m pip install deepspeed torcheval +RUN python3.9 -m pip cache purge # Install Python 3.10 at the last stage to avoid breaking Python 3.9 RUN add-apt-repository ppa:deadsnakes/ppa @@ -114,6 +115,7 @@ RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 RUN python3.10 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PIP_PKGS RUN python3.10 -m pip install 'torch<=2.0.1' torchvision --index-url https://download.pytorch.org/whl/cpu RUN python3.10 -m pip install deepspeed torcheval +RUN python3.10 -m pip cache purge # Install Python 3.11 at the last stage to avoid breaking the existing Python installations RUN add-apt-repository ppa:deadsnakes/ppa @@ -124,6 +126,7 @@ RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 RUN python3.11 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PIP_PKGS RUN python3.11 -m pip install 'torch<=2.0.1' torchvision --index-url https://download.pytorch.org/whl/cpu RUN python3.11 -m pip install deepspeed torcheval +RUN python3.11 -m pip cache purge # Install Python 3.12 at the last stage to avoid breaking the existing Python installations RUN add-apt-repository ppa:deadsnakes/ppa @@ -137,5 +140,6 @@ RUN python3.12 -m pip install $BASIC_PIP_PKGS $CONNECT_PIP_PKGS lxml RUN python3.12 -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu RUN python3.12 -m pip install torchvision --index-url https://download.pytorch.org/whl/cpu RUN python3.12 -m pip install torcheval +RUN python3.12 -m pip cache purge RUN df -h From 1aa1db60466e2170fb07ee8b9240ea960d60e94c Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Wed, 17 Jan 2024 17:04:24 +0800 Subject: [PATCH 3/3] nit --- .github/workflows/build_and_test.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 40bcf734c6af9..74c12a9a9f5ae 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -417,10 +417,6 @@ jobs: - name: Free up disk space shell: 'script -q -e -c "bash {0}"' run: | - if [[ "$MODULES_TO_TEST" != *"pyspark-ml"* ]] && [[ "$BRANCH" != "branch-3.5" ]]; then - # uninstall libraries dedicated for ML testing - python3.9 -m pip uninstall -y torch torchvision torcheval torchtnt tensorboard mlflow deepspeed - fi if [ -f ./dev/free_disk_space_container ]; then ./dev/free_disk_space_container fi