From fa8eb8e454f31668ba675e4cf465ac4640cff6da Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Tue, 31 Jan 2023 11:58:44 -0800 Subject: [PATCH 01/11] Add Python 3.11 Linux CPU Unittesting --- .github/workflows/test-linux-cpu.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index 5dc7550d868..b16747b0a69 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -16,7 +16,7 @@ jobs: tests: strategy: matrix: - python_version: ["3.8", "3.9", "3.10"] + python_version: ["3.8", "3.9", "3.10", "3.11"] fail-fast: false uses: pytorch/test-infra/.github/workflows/linux_job.yml@main with: @@ -39,7 +39,12 @@ jobs: fi # Create Conda Env - conda create -yp ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy + if [[ "${PYTHON_VERSION}" = "3.11" ]]; then + conda create -yp ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy + else + conda create -yp ci_env -c conda-forge python="${PYTHON_VERSION}" numpy libpng jpeg scipy + fi + conda activate /work/ci_env # Install PyTorch, Torchvision, and testing libraries From 9fa765a25d4cf58588c36b7b070262679e55c5c3 Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Tue, 31 Jan 2023 13:01:59 -0800 Subject: [PATCH 02/11] scipy not compatible with py3.11, removing for now --- .github/workflows/test-linux-cpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index b16747b0a69..8233cdde5c6 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -40,7 +40,7 @@ jobs: # Create Conda Env if [[ "${PYTHON_VERSION}" = "3.11" ]]; then - conda create -yp ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy + conda create -yp ci_env python="${PYTHON_VERSION}" numpy libpng jpeg else conda create -yp ci_env -c conda-forge python="${PYTHON_VERSION}" numpy libpng jpeg scipy fi From b93584fc0d56a55352c1ee0d1bc36a05a0033c18 Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Tue, 31 Jan 2023 13:12:03 -0800 Subject: [PATCH 03/11] install numpy from pypi not conda --- .github/workflows/test-linux-cpu.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index 8233cdde5c6..4524112cc8b 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -40,7 +40,8 @@ jobs: # Create Conda Env if [[ "${PYTHON_VERSION}" = "3.11" ]]; then - conda create -yp ci_env python="${PYTHON_VERSION}" numpy libpng jpeg + conda create -yp ci_env python="${PYTHON_VERSION}" libpng jpeg + python3 -m pip install numpy else conda create -yp ci_env -c conda-forge python="${PYTHON_VERSION}" numpy libpng jpeg scipy fi From 37d216301ae1bdd36cba7ec6a092cc9e1a88f2ff Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Tue, 31 Jan 2023 13:53:04 -0800 Subject: [PATCH 04/11] use pytorch nightly wheel instead of conda --- .github/workflows/test-linux-cpu.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index 4524112cc8b..13d2e4fb64b 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -50,11 +50,16 @@ jobs: # Install PyTorch, Torchvision, and testing libraries set -ex - conda install \ - --yes \ - -c "pytorch-${CHANNEL}" \ - -c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${VERSION}*"] \ - "${CUDATOOLKIT}" + if [[ "${PYTHON_VERSION}" = "3.11" ]]; then + conda install \ + --yes \ + -c "pytorch-${CHANNEL}" \ + -c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${VERSION}*"] \ + "${CUDATOOLKIT}" + else + python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + fi + python3 setup.py develop python3 -m pip install pytest pytest-mock 'av<10' From 9501358f5a4b7b93b066125bf00adb7fbabb3446 Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Thu, 2 Feb 2023 11:09:32 -0800 Subject: [PATCH 05/11] use numpy from pytorch-nightly channel --- .github/workflows/test-linux-cpu.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index 13d2e4fb64b..cd65c0bb98e 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -40,10 +40,10 @@ jobs: # Create Conda Env if [[ "${PYTHON_VERSION}" = "3.11" ]]; then - conda create -yp ci_env python="${PYTHON_VERSION}" libpng jpeg - python3 -m pip install numpy + conda create -yp ci_env -c conda-forge python="${PYTHON_VERSION}" libpng jpeg + conda install -c pytorch-nightly numpy else - conda create -yp ci_env -c conda-forge python="${PYTHON_VERSION}" numpy libpng jpeg scipy + conda create -yp ci_env python="${PYTHON_VERSION}" numpy libpng jpeg fi conda activate /work/ci_env From dfa40c1dfabb8f5107c88a1893f0bf50f239c782 Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Thu, 2 Feb 2023 12:36:04 -0800 Subject: [PATCH 06/11] bash set --- .github/workflows/test-linux-cpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index cd65c0bb98e..5d9b9044c32 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -24,6 +24,7 @@ jobs: repository: pytorch/vision script: | # Mark Build Directory Safe + set -euxo pipefail git config --global --add safe.directory /__w/vision/vision # Set up Environment Variables @@ -49,7 +50,6 @@ jobs: conda activate /work/ci_env # Install PyTorch, Torchvision, and testing libraries - set -ex if [[ "${PYTHON_VERSION}" = "3.11" ]]; then conda install \ --yes \ From d99f4e51c72035c791e58d53a46f4944289af6d3 Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Thu, 2 Feb 2023 13:47:05 -0800 Subject: [PATCH 07/11] remove pip dep --- .circleci/unittest/linux/scripts/environment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/unittest/linux/scripts/environment.yml b/.circleci/unittest/linux/scripts/environment.yml index fae96c5f93c..d4dfce5fcce 100644 --- a/.circleci/unittest/linux/scripts/environment.yml +++ b/.circleci/unittest/linux/scripts/environment.yml @@ -5,7 +5,6 @@ dependencies: - pytest - pytest-cov - pytest-mock - - pip - libpng - jpeg - ca-certificates From f40008654cdf472c18638e3b73ab523035cc3b0b Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Thu, 2 Feb 2023 13:48:30 -0800 Subject: [PATCH 08/11] numpy from nightly conflicting with conda-build --- .github/workflows/test-linux-cpu.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index 5d9b9044c32..d0872e0b01e 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -42,7 +42,8 @@ jobs: # Create Conda Env if [[ "${PYTHON_VERSION}" = "3.11" ]]; then conda create -yp ci_env -c conda-forge python="${PYTHON_VERSION}" libpng jpeg - conda install -c pytorch-nightly numpy + # conda install -c pytorch-nightly numpy + python3 -m pip install numpy else conda create -yp ci_env python="${PYTHON_VERSION}" numpy libpng jpeg fi From f86437528d40b815a3323c56aa1633323280a363 Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Thu, 2 Feb 2023 13:54:59 -0800 Subject: [PATCH 09/11] remove av libraries from test deps --- .github/workflows/test-linux-cpu.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index d0872e0b01e..dfff261571f 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -62,7 +62,12 @@ jobs: fi python3 setup.py develop - python3 -m pip install pytest pytest-mock 'av<10' + if [[ "${PYTHON_VERSION}" = "3.11" ]]; then + # Looks like the av libraries are not available for 3.11 + python3 -m pip install pytest pytest-mock + else + python3 -m pip install pytest pytest-mock 'av<10' + fi # Run Tests python3 -m torch.utils.collect_env From ad6c3e74a7fd59fa7343b440a7ee80dcccb462dd Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Thu, 2 Feb 2023 14:26:32 -0800 Subject: [PATCH 10/11] try to get av tests to work --- .circleci/unittest/linux/scripts/environment.yml | 1 + .github/workflows/test-linux-cpu.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.circleci/unittest/linux/scripts/environment.yml b/.circleci/unittest/linux/scripts/environment.yml index d4dfce5fcce..fae96c5f93c 100644 --- a/.circleci/unittest/linux/scripts/environment.yml +++ b/.circleci/unittest/linux/scripts/environment.yml @@ -5,6 +5,7 @@ dependencies: - pytest - pytest-cov - pytest-mock + - pip - libpng - jpeg - ca-certificates diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index dfff261571f..f3438df5077 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -65,6 +65,7 @@ jobs: if [[ "${PYTHON_VERSION}" = "3.11" ]]; then # Looks like the av libraries are not available for 3.11 python3 -m pip install pytest pytest-mock + python3 -m pip install av else python3 -m pip install pytest pytest-mock 'av<10' fi From a7c07b5812dbe18aec28880b99f4875825f5dc7f Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Mon, 6 Feb 2023 11:50:52 -0800 Subject: [PATCH 11/11] remove av for 3.11 since compatible binaries are not available --- .github/workflows/test-linux-cpu.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index f3438df5077..dfff261571f 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -65,7 +65,6 @@ jobs: if [[ "${PYTHON_VERSION}" = "3.11" ]]; then # Looks like the av libraries are not available for 3.11 python3 -m pip install pytest pytest-mock - python3 -m pip install av else python3 -m pip install pytest pytest-mock 'av<10' fi