From 4636bb75f5f54b0b043b8dd30a81638d3e626d27 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 30 Jan 2023 10:16:47 +0100 Subject: [PATCH 01/47] [DEBUG] macos GHA workflow --- .github/workflows/test-macos-cpu.yml | 57 ++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 .github/workflows/test-macos-cpu.yml diff --git a/.github/workflows/test-macos-cpu.yml b/.github/workflows/test-macos-cpu.yml new file mode 100644 index 00000000000..9fd36ac8e6f --- /dev/null +++ b/.github/workflows/test-macos-cpu.yml @@ -0,0 +1,57 @@ +name: Unit-tests on macOS CPU + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +env: + CHANNEL: "nightly" + +jobs: + tests: + strategy: + matrix: + python_version: ["3.8", "3.9", "3.10"] + fail-fast: false + uses: pytorch/test-infra/.github/workflows/macos_job.yml@main + with: + runner: macos-12 + repository: pytorch/vision + script: | + # Mark Build Directory Safe + git config --global --add safe.directory /__w/vision/vision + + # Set up Environment Variables + export PYTHON_VERSION="${{ matrix.python_version }}" + export VERSION="cpu" + export CUDATOOLKIT="cpuonly" + + # Set CHANNEL + if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then + export CHANNEL=test + else + export CHANNEL=nightly + fi + + # Create Conda Env + conda create -y -n ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy + conda activate ci_env + + # Install PyTorch, Torchvision, and testing libraries + set -ex + conda install \ + --yes \ + -c "pytorch-${CHANNEL}" \ + -c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${VERSION}*"] \ + "${CUDATOOLKIT}" + python3 setup.py develop + python3 -m pip install pytest pytest-mock 'av<10' + + # Run Tests + python3 -m torch.utils.collect_env + python3 -m pytest --junitxml=test-results/junit.xml -v --durations 20 From 90de366df29251e0c2a4b6639f86d68e46e9c8e0 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 30 Jan 2023 10:26:25 +0100 Subject: [PATCH 02/47] try fix pytorch install --- .github/workflows/test-macos-cpu.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-macos-cpu.yml b/.github/workflows/test-macos-cpu.yml index 9fd36ac8e6f..cf1339846ce 100644 --- a/.github/workflows/test-macos-cpu.yml +++ b/.github/workflows/test-macos-cpu.yml @@ -47,7 +47,8 @@ jobs: conda install \ --yes \ -c "pytorch-${CHANNEL}" \ - -c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${VERSION}*"] \ + -c nvidia \ + pytorch \ "${CUDATOOLKIT}" python3 setup.py develop python3 -m pip install pytest pytest-mock 'av<10' From 30f087fa58751f0467c7c8e970c63702298fad04 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 30 Jan 2023 10:51:38 +0100 Subject: [PATCH 03/47] add more requirments --- .github/workflows/test-macos-cpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-macos-cpu.yml b/.github/workflows/test-macos-cpu.yml index cf1339846ce..fc07a13016b 100644 --- a/.github/workflows/test-macos-cpu.yml +++ b/.github/workflows/test-macos-cpu.yml @@ -39,7 +39,7 @@ jobs: fi # Create Conda Env - conda create -y -n ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy + conda create -y -n ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy pillow requests conda activate ci_env # Install PyTorch, Torchvision, and testing libraries From 094057ffa93f992b6431840c9dc5ebca8ed5a1c6 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 30 Jan 2023 12:26:05 +0100 Subject: [PATCH 04/47] try using name activation for Linux build as well --- .github/workflows/test-linux-cpu.yml | 4 ++-- .github/workflows/test-macos-cpu.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index 5dc7550d868..139baafa962 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -39,8 +39,8 @@ jobs: fi # Create Conda Env - conda create -yp ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy - conda activate /work/ci_env + conda create -y -n ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy + conda activate ci_env # Install PyTorch, Torchvision, and testing libraries set -ex diff --git a/.github/workflows/test-macos-cpu.yml b/.github/workflows/test-macos-cpu.yml index fc07a13016b..984df7ab616 100644 --- a/.github/workflows/test-macos-cpu.yml +++ b/.github/workflows/test-macos-cpu.yml @@ -39,7 +39,7 @@ jobs: fi # Create Conda Env - conda create -y -n ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy pillow requests + conda create -y -n ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy ninja conda activate ci_env # Install PyTorch, Torchvision, and testing libraries From 5434d8f8a86a8027e7d820566fb097801f825255 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 30 Jan 2023 15:24:59 +0100 Subject: [PATCH 05/47] try install setuptools explicitly --- .github/workflows/test-macos-cpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-macos-cpu.yml b/.github/workflows/test-macos-cpu.yml index 984df7ab616..09e79b7159d 100644 --- a/.github/workflows/test-macos-cpu.yml +++ b/.github/workflows/test-macos-cpu.yml @@ -39,7 +39,7 @@ jobs: fi # Create Conda Env - conda create -y -n ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy ninja + conda create -y -n ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy ninja setuptools conda activate ci_env # Install PyTorch, Torchvision, and testing libraries From 7e3cac7f3d86b2acab0e32ba756ea84bff8752b6 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 30 Jan 2023 15:40:05 +0100 Subject: [PATCH 06/47] try explicitly installing clang --- .github/workflows/test-macos-cpu.yml | 71 ++++++++++++++++++---------- 1 file changed, 45 insertions(+), 26 deletions(-) diff --git a/.github/workflows/test-macos-cpu.yml b/.github/workflows/test-macos-cpu.yml index 09e79b7159d..a269a8b16f8 100644 --- a/.github/workflows/test-macos-cpu.yml +++ b/.github/workflows/test-macos-cpu.yml @@ -23,36 +23,55 @@ jobs: runner: macos-12 repository: pytorch/vision script: | - # Mark Build Directory Safe - git config --global --add safe.directory /__w/vision/vision - - # Set up Environment Variables - export PYTHON_VERSION="${{ matrix.python_version }}" - export VERSION="cpu" - export CUDATOOLKIT="cpuonly" - - # Set CHANNEL + echo '::group::Set PyTorch conda channel' + # TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then - export CHANNEL=test + POSTFIX=test else - export CHANNEL=nightly + POSTFIX=nightly fi - - # Create Conda Env - conda create -y -n ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy ninja setuptools - conda activate ci_env + PYTORCH_CHANNEL=pytorch-"${POSTFIX}" + echo "${PYTORCH_CHANNEL}" + echo '::endgroup::' - # Install PyTorch, Torchvision, and testing libraries - set -ex + echo '::group::Set PyTorch conda mutex' + PYTORCH_MUTEX=cpuonly + echo "${PYTORCH_MUTEX}" + echo '::endgroup::' + + echo '::group::Create conda environment' + conda create \ + --name ci + --quiet --yes \ + python=${{ matrix.python-version }} pip \ + clang_osx-64 clangxx_osx-64 \ + setuptools ninja \ + libpng jpeg + conda activate ci + pip install 'av<10' + echo '::endgroup::' + + echo '::group::Install PyTorch' conda install \ - --yes \ - -c "pytorch-${CHANNEL}" \ + --quiet --yes \ + -c "${PYTORCH_CHANNEL}" \ -c nvidia \ pytorch \ - "${CUDATOOLKIT}" - python3 setup.py develop - python3 -m pip install pytest pytest-mock 'av<10' - - # Run Tests - python3 -m torch.utils.collect_env - python3 -m pytest --junitxml=test-results/junit.xml -v --durations 20 + "${PYTORCH_MUTEX}" + echo '::endgroup::' + + echo '::group::Install TorchVision' + python setup.py develop + echo '::endgroup::' + + echo '::group::Collect PyTorch environment information' + python -m torch.utils.collect_env + echo '::endgroup::' + + echo '::group::Install testing utilities' + pip install --progress-bar=off pytest pytest-mock pytest-cov + echo '::endgroup::' + + echo '::group::Run tests' + pytest --durations=25 + echo '::endgroup::' From 59544c9cb984fbbb3c2ef66a7af5ed830ae31c78 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 30 Jan 2023 15:43:48 +0100 Subject: [PATCH 07/47] fix multiline --- .github/workflows/test-macos-cpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-macos-cpu.yml b/.github/workflows/test-macos-cpu.yml index a269a8b16f8..037ee5efab3 100644 --- a/.github/workflows/test-macos-cpu.yml +++ b/.github/workflows/test-macos-cpu.yml @@ -41,7 +41,7 @@ jobs: echo '::group::Create conda environment' conda create \ - --name ci + --name ci \ --quiet --yes \ python=${{ matrix.python-version }} pip \ clang_osx-64 clangxx_osx-64 \ From 8810582ddae72682b2e3b48f812ef6cebc7db7e7 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 30 Jan 2023 15:57:59 +0100 Subject: [PATCH 08/47] try search for gcc on all platforms --- .github/workflows/test-linux-cpu.yml | 65 ++++++++------- .github/workflows/test-macos-cpu.yml | 111 ++++++++++++------------- .github/workflows/test-windows-cpu.yml | 75 +++++++++++++++++ 3 files changed, 162 insertions(+), 89 deletions(-) create mode 100644 .github/workflows/test-windows-cpu.yml diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index 139baafa962..c55fd45b152 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -23,35 +23,36 @@ jobs: runner: linux.12xlarge repository: pytorch/vision script: | - # Mark Build Directory Safe - git config --global --add safe.directory /__w/vision/vision - - # Set up Environment Variables - export PYTHON_VERSION="${{ matrix.python_version }}" - export VERSION="cpu" - export CUDATOOLKIT="cpuonly" - - # Set CHANNEL - if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then - export CHANNEL=test - else - export CHANNEL=nightly - fi - - # Create Conda Env - conda create -y -n ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy - conda activate ci_env - - # Install PyTorch, Torchvision, and testing libraries - set -ex - conda install \ - --yes \ - -c "pytorch-${CHANNEL}" \ - -c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${VERSION}*"] \ - "${CUDATOOLKIT}" - python3 setup.py develop - python3 -m pip install pytest pytest-mock 'av<10' - - # Run Tests - python3 -m torch.utils.collect_env - python3 -m pytest --junitxml=test-results/junit.xml -v --durations 20 + conda search gcc +# # Mark Build Directory Safe +# git config --global --add safe.directory /__w/vision/vision +# +# # Set up Environment Variables +# export PYTHON_VERSION="${{ matrix.python_version }}" +# export VERSION="cpu" +# export CUDATOOLKIT="cpuonly" +# +# # Set CHANNEL +# if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then +# export CHANNEL=test +# else +# export CHANNEL=nightly +# fi +# +# # Create Conda Env +# conda create -y -n ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy +# conda activate ci_env +# +# # Install PyTorch, Torchvision, and testing libraries +# set -ex +# conda install \ +# --yes \ +# -c "pytorch-${CHANNEL}" \ +# -c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${VERSION}*"] \ +# "${CUDATOOLKIT}" +# python3 setup.py develop +# python3 -m pip install pytest pytest-mock 'av<10' +# +# # Run Tests +# python3 -m torch.utils.collect_env +# python3 -m pytest --junitxml=test-results/junit.xml -v --durations 20 diff --git a/.github/workflows/test-macos-cpu.yml b/.github/workflows/test-macos-cpu.yml index 037ee5efab3..ccc5cdd7224 100644 --- a/.github/workflows/test-macos-cpu.yml +++ b/.github/workflows/test-macos-cpu.yml @@ -9,69 +9,66 @@ on: - release/* workflow_dispatch: -env: - CHANNEL: "nightly" - jobs: tests: strategy: matrix: python_version: ["3.8", "3.9", "3.10"] fail-fast: false - uses: pytorch/test-infra/.github/workflows/macos_job.yml@main + uses: pytorch/test-infra/.github/workflows/windows_job.yml@main with: - runner: macos-12 repository: pytorch/vision script: | - echo '::group::Set PyTorch conda channel' - # TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. - if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then - POSTFIX=test - else - POSTFIX=nightly - fi - PYTORCH_CHANNEL=pytorch-"${POSTFIX}" - echo "${PYTORCH_CHANNEL}" - echo '::endgroup::' - - echo '::group::Set PyTorch conda mutex' - PYTORCH_MUTEX=cpuonly - echo "${PYTORCH_MUTEX}" - echo '::endgroup::' - - echo '::group::Create conda environment' - conda create \ - --name ci \ - --quiet --yes \ - python=${{ matrix.python-version }} pip \ - clang_osx-64 clangxx_osx-64 \ - setuptools ninja \ - libpng jpeg - conda activate ci - pip install 'av<10' - echo '::endgroup::' - - echo '::group::Install PyTorch' - conda install \ - --quiet --yes \ - -c "${PYTORCH_CHANNEL}" \ - -c nvidia \ - pytorch \ - "${PYTORCH_MUTEX}" - echo '::endgroup::' - - echo '::group::Install TorchVision' - python setup.py develop - echo '::endgroup::' - - echo '::group::Collect PyTorch environment information' - python -m torch.utils.collect_env - echo '::endgroup::' - - echo '::group::Install testing utilities' - pip install --progress-bar=off pytest pytest-mock pytest-cov - echo '::endgroup::' - - echo '::group::Run tests' - pytest --durations=25 - echo '::endgroup::' + conda search gcc +# echo '::group::Set PyTorch conda channel' +# # TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. +# if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then +# POSTFIX=test +# else +# POSTFIX=nightly +# fi +# PYTORCH_CHANNEL=pytorch-"${POSTFIX}" +# echo "${PYTORCH_CHANNEL}" +# echo '::endgroup::' +# +# echo '::group::Set PyTorch conda mutex' +# PYTORCH_MUTEX=cpuonly +# echo "${PYTORCH_MUTEX}" +# echo '::endgroup::' +# +# echo '::group::Create conda environment' +# conda create \ +# --name ci \ +# --quiet --yes \ +# python=${{ matrix.python-version }} pip \ +# clang_osx-64 clangxx_osx-64 \ +# setuptools ninja \ +# libpng jpeg +# conda activate ci +# pip install 'av<10' +# echo '::endgroup::' +# +# echo '::group::Install PyTorch' +# conda install \ +# --quiet --yes \ +# -c "${PYTORCH_CHANNEL}" \ +# -c nvidia \ +# pytorch \ +# "${PYTORCH_MUTEX}" +# echo '::endgroup::' +# +# echo '::group::Install TorchVision' +# python setup.py develop +# echo '::endgroup::' +# +# echo '::group::Collect PyTorch environment information' +# python -m torch.utils.collect_env +# echo '::endgroup::' +# +# echo '::group::Install testing utilities' +# pip install --progress-bar=off pytest pytest-mock pytest-cov +# echo '::endgroup::' +# +# echo '::group::Run tests' +# pytest --durations=25 +# echo '::endgroup::' diff --git a/.github/workflows/test-windows-cpu.yml b/.github/workflows/test-windows-cpu.yml new file mode 100644 index 00000000000..9b44ea6cf9f --- /dev/null +++ b/.github/workflows/test-windows-cpu.yml @@ -0,0 +1,75 @@ +name: Unit-tests on macOS CPU + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +jobs: + tests: + strategy: + matrix: + python_version: ["3.8", "3.9", "3.10"] + fail-fast: false + uses: pytorch/test-infra/.github/workflows/macos_job.yml@main + with: + runner: macos-12 + repository: pytorch/vision + script: | + conda search gcc +# echo '::group::Set PyTorch conda channel' +# # TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. +# if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then +# POSTFIX=test +# else +# POSTFIX=nightly +# fi +# PYTORCH_CHANNEL=pytorch-"${POSTFIX}" +# echo "${PYTORCH_CHANNEL}" +# echo '::endgroup::' +# +# echo '::group::Set PyTorch conda mutex' +# PYTORCH_MUTEX=cpuonly +# echo "${PYTORCH_MUTEX}" +# echo '::endgroup::' +# +# echo '::group::Create conda environment' +# conda create \ +# --name ci \ +# --quiet --yes \ +# python=${{ matrix.python-version }} pip \ +# clang_osx-64 clangxx_osx-64 \ +# setuptools ninja \ +# libpng jpeg +# conda activate ci +# pip install 'av<10' +# echo '::endgroup::' +# +# echo '::group::Install PyTorch' +# conda install \ +# --quiet --yes \ +# -c "${PYTORCH_CHANNEL}" \ +# -c nvidia \ +# pytorch \ +# "${PYTORCH_MUTEX}" +# echo '::endgroup::' +# +# echo '::group::Install TorchVision' +# python setup.py develop +# echo '::endgroup::' +# +# echo '::group::Collect PyTorch environment information' +# python -m torch.utils.collect_env +# echo '::endgroup::' +# +# echo '::group::Install testing utilities' +# pip install --progress-bar=off pytest pytest-mock pytest-cov +# echo '::endgroup::' +# +# echo '::group::Run tests' +# pytest --durations=25 +# echo '::endgroup::' From 9b83493bb123a821e7c3b98e292ac88d8404fc8b Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 30 Jan 2023 15:58:36 +0100 Subject: [PATCH 09/47] fix windows runner --- .github/workflows/test-windows-cpu.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-windows-cpu.yml b/.github/workflows/test-windows-cpu.yml index 9b44ea6cf9f..486d58fd5df 100644 --- a/.github/workflows/test-windows-cpu.yml +++ b/.github/workflows/test-windows-cpu.yml @@ -17,7 +17,6 @@ jobs: fail-fast: false uses: pytorch/test-infra/.github/workflows/macos_job.yml@main with: - runner: macos-12 repository: pytorch/vision script: | conda search gcc From 971db921d674f2a19e28e2c81c1d7efa47b1f39b Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 30 Jan 2023 16:03:37 +0100 Subject: [PATCH 10/47] also search for clang --- .github/workflows/test-linux-cpu.yml | 1 + .github/workflows/test-macos-cpu.yml | 3 ++- .github/workflows/test-windows-cpu.yml | 5 +++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index c55fd45b152..5429fae350c 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -24,6 +24,7 @@ jobs: repository: pytorch/vision script: | conda search gcc + conda search clang # # Mark Build Directory Safe # git config --global --add safe.directory /__w/vision/vision # diff --git a/.github/workflows/test-macos-cpu.yml b/.github/workflows/test-macos-cpu.yml index ccc5cdd7224..563817cd2dd 100644 --- a/.github/workflows/test-macos-cpu.yml +++ b/.github/workflows/test-macos-cpu.yml @@ -15,11 +15,12 @@ jobs: matrix: python_version: ["3.8", "3.9", "3.10"] fail-fast: false - uses: pytorch/test-infra/.github/workflows/windows_job.yml@main + uses: pytorch/test-infra/.github/workflows/macos_job.yml@main with: repository: pytorch/vision script: | conda search gcc + conda search clang # echo '::group::Set PyTorch conda channel' # # TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. # if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then diff --git a/.github/workflows/test-windows-cpu.yml b/.github/workflows/test-windows-cpu.yml index 486d58fd5df..ef50ee74ca2 100644 --- a/.github/workflows/test-windows-cpu.yml +++ b/.github/workflows/test-windows-cpu.yml @@ -1,4 +1,4 @@ -name: Unit-tests on macOS CPU +name: Unit-tests on Windows CPU on: pull_request: @@ -15,11 +15,12 @@ jobs: matrix: python_version: ["3.8", "3.9", "3.10"] fail-fast: false - uses: pytorch/test-infra/.github/workflows/macos_job.yml@main + uses: pytorch/test-infra/.github/workflows/windows_job.yml@main with: repository: pytorch/vision script: | conda search gcc + conda search clang # echo '::group::Set PyTorch conda channel' # # TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. # if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then From 186f4e7e06add809bc4767a8c5fef1c5a535d089 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 30 Jan 2023 16:14:52 +0100 Subject: [PATCH 11/47] reinstate mac --- .github/workflows/test-windows-cpu.yml | 106 ++++++++++++------------- 1 file changed, 52 insertions(+), 54 deletions(-) diff --git a/.github/workflows/test-windows-cpu.yml b/.github/workflows/test-windows-cpu.yml index ef50ee74ca2..8e581e2be2d 100644 --- a/.github/workflows/test-windows-cpu.yml +++ b/.github/workflows/test-windows-cpu.yml @@ -19,57 +19,55 @@ jobs: with: repository: pytorch/vision script: | - conda search gcc - conda search clang -# echo '::group::Set PyTorch conda channel' -# # TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. -# if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then -# POSTFIX=test -# else -# POSTFIX=nightly -# fi -# PYTORCH_CHANNEL=pytorch-"${POSTFIX}" -# echo "${PYTORCH_CHANNEL}" -# echo '::endgroup::' -# -# echo '::group::Set PyTorch conda mutex' -# PYTORCH_MUTEX=cpuonly -# echo "${PYTORCH_MUTEX}" -# echo '::endgroup::' -# -# echo '::group::Create conda environment' -# conda create \ -# --name ci \ -# --quiet --yes \ -# python=${{ matrix.python-version }} pip \ -# clang_osx-64 clangxx_osx-64 \ -# setuptools ninja \ -# libpng jpeg -# conda activate ci -# pip install 'av<10' -# echo '::endgroup::' -# -# echo '::group::Install PyTorch' -# conda install \ -# --quiet --yes \ -# -c "${PYTORCH_CHANNEL}" \ -# -c nvidia \ -# pytorch \ -# "${PYTORCH_MUTEX}" -# echo '::endgroup::' -# -# echo '::group::Install TorchVision' -# python setup.py develop -# echo '::endgroup::' -# -# echo '::group::Collect PyTorch environment information' -# python -m torch.utils.collect_env -# echo '::endgroup::' -# -# echo '::group::Install testing utilities' -# pip install --progress-bar=off pytest pytest-mock pytest-cov -# echo '::endgroup::' -# -# echo '::group::Run tests' -# pytest --durations=25 -# echo '::endgroup::' + echo '::group::Set PyTorch conda channel' + # TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. + if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then + POSTFIX=test + else + POSTFIX=nightly + fi + PYTORCH_CHANNEL=pytorch-"${POSTFIX}" + echo "${PYTORCH_CHANNEL}" + echo '::endgroup::' + + echo '::group::Set PyTorch conda mutex' + PYTORCH_MUTEX=cpuonly + echo "${PYTORCH_MUTEX}" + echo '::endgroup::' + + echo '::group::Create conda environment' + conda create \ + --name ci \ + --quiet --yes \ + python=${{ matrix.python-version }} pip \ + clang_osx-64 clangxx_osx-64 \ + setuptools ninja \ + libpng jpeg + conda activate ci + pip install 'av<10' + echo '::endgroup::' + + echo '::group::Install PyTorch' + conda install \ + --quiet --yes \ + -c "${PYTORCH_CHANNEL}" \ + -c nvidia \ + pytorch \ + "${PYTORCH_MUTEX}" + echo '::endgroup::' + + echo '::group::Install TorchVision' + python setup.py develop + echo '::endgroup::' + + echo '::group::Collect PyTorch environment information' + python -m torch.utils.collect_env + echo '::endgroup::' + + echo '::group::Install testing utilities' + pip install --progress-bar=off pytest pytest-mock pytest-cov + echo '::endgroup::' + + echo '::group::Run tests' + pytest --durations=25 + echo '::endgroup::' From a1024ff75cc49398a39b42090d76ab38ceede037 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 30 Jan 2023 16:20:14 +0100 Subject: [PATCH 12/47] try default on linux as well --- .github/workflows/test-linux-cpu.yml | 89 ++++++++++++--------- .github/workflows/test-macos-cpu.yml | 106 ++++++++++++------------- .github/workflows/test-windows-cpu.yml | 2 +- 3 files changed, 105 insertions(+), 92 deletions(-) diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index 5429fae350c..f3db11e03a3 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -9,9 +9,6 @@ on: - release/* workflow_dispatch: -env: - CHANNEL: "nightly" - jobs: tests: strategy: @@ -23,37 +20,55 @@ jobs: runner: linux.12xlarge repository: pytorch/vision script: | - conda search gcc - conda search clang -# # Mark Build Directory Safe -# git config --global --add safe.directory /__w/vision/vision -# -# # Set up Environment Variables -# export PYTHON_VERSION="${{ matrix.python_version }}" -# export VERSION="cpu" -# export CUDATOOLKIT="cpuonly" -# -# # Set CHANNEL -# if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then -# export CHANNEL=test -# else -# export CHANNEL=nightly -# fi -# -# # Create Conda Env -# conda create -y -n ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy -# conda activate ci_env -# -# # Install PyTorch, Torchvision, and testing libraries -# set -ex -# conda install \ -# --yes \ -# -c "pytorch-${CHANNEL}" \ -# -c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${VERSION}*"] \ -# "${CUDATOOLKIT}" -# python3 setup.py develop -# python3 -m pip install pytest pytest-mock 'av<10' -# -# # Run Tests -# python3 -m torch.utils.collect_env -# python3 -m pytest --junitxml=test-results/junit.xml -v --durations 20 + echo '::group::Set PyTorch conda channel' + # TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. + if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then + POSTFIX=test + else + POSTFIX=nightly + fi + PYTORCH_CHANNEL=pytorch-"${POSTFIX}" + echo "${PYTORCH_CHANNEL}" + echo '::endgroup::' + + echo '::group::Set PyTorch conda mutex' + PYTORCH_MUTEX=cpuonly + echo "${PYTORCH_MUTEX}" + echo '::endgroup::' + + echo '::group::Create conda environment' + conda create \ + --name ci \ + --quiet --yes \ + python=${{ matrix.python-version }} pip \ + gcc_linux-64 gxx_linux-64 \ + setuptools ninja \ + libpng jpeg + conda activate ci + pip install 'av<10' + echo '::endgroup::' + + echo '::group::Install PyTorch' + conda install \ + --quiet --yes \ + -c "${PYTORCH_CHANNEL}" \ + -c nvidia \ + pytorch \ + "${PYTORCH_MUTEX}" + echo '::endgroup::' + + echo '::group::Install TorchVision' + python setup.py develop + echo '::endgroup::' + + echo '::group::Collect PyTorch environment information' + python -m torch.utils.collect_env + echo '::endgroup::' + + echo '::group::Install testing utilities' + pip install --progress-bar=off pytest pytest-mock pytest-cov + echo '::endgroup::' + + echo '::group::Run tests' + pytest --durations=25 + echo '::endgroup::' diff --git a/.github/workflows/test-macos-cpu.yml b/.github/workflows/test-macos-cpu.yml index 563817cd2dd..63fd11505e2 100644 --- a/.github/workflows/test-macos-cpu.yml +++ b/.github/workflows/test-macos-cpu.yml @@ -19,57 +19,55 @@ jobs: with: repository: pytorch/vision script: | - conda search gcc - conda search clang -# echo '::group::Set PyTorch conda channel' -# # TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. -# if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then -# POSTFIX=test -# else -# POSTFIX=nightly -# fi -# PYTORCH_CHANNEL=pytorch-"${POSTFIX}" -# echo "${PYTORCH_CHANNEL}" -# echo '::endgroup::' -# -# echo '::group::Set PyTorch conda mutex' -# PYTORCH_MUTEX=cpuonly -# echo "${PYTORCH_MUTEX}" -# echo '::endgroup::' -# -# echo '::group::Create conda environment' -# conda create \ -# --name ci \ -# --quiet --yes \ -# python=${{ matrix.python-version }} pip \ -# clang_osx-64 clangxx_osx-64 \ -# setuptools ninja \ -# libpng jpeg -# conda activate ci -# pip install 'av<10' -# echo '::endgroup::' -# -# echo '::group::Install PyTorch' -# conda install \ -# --quiet --yes \ -# -c "${PYTORCH_CHANNEL}" \ -# -c nvidia \ -# pytorch \ -# "${PYTORCH_MUTEX}" -# echo '::endgroup::' -# -# echo '::group::Install TorchVision' -# python setup.py develop -# echo '::endgroup::' -# -# echo '::group::Collect PyTorch environment information' -# python -m torch.utils.collect_env -# echo '::endgroup::' -# -# echo '::group::Install testing utilities' -# pip install --progress-bar=off pytest pytest-mock pytest-cov -# echo '::endgroup::' -# -# echo '::group::Run tests' -# pytest --durations=25 -# echo '::endgroup::' + echo '::group::Set PyTorch conda channel' + # TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. + if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then + POSTFIX=test + else + POSTFIX=nightly + fi + PYTORCH_CHANNEL=pytorch-"${POSTFIX}" + echo "${PYTORCH_CHANNEL}" + echo '::endgroup::' + + echo '::group::Set PyTorch conda mutex' + PYTORCH_MUTEX=cpuonly + echo "${PYTORCH_MUTEX}" + echo '::endgroup::' + + echo '::group::Create conda environment' + conda create \ + --name ci \ + --quiet --yes \ + python=${{ matrix.python-version }} pip \ + clang_osx-64 clangxx_osx-64 \ + setuptools ninja \ + libpng jpeg + conda activate ci + pip install 'av<10' + echo '::endgroup::' + + echo '::group::Install PyTorch' + conda install \ + --quiet --yes \ + -c "${PYTORCH_CHANNEL}" \ + -c nvidia \ + pytorch \ + "${PYTORCH_MUTEX}" + echo '::endgroup::' + + echo '::group::Install TorchVision' + python setup.py develop + echo '::endgroup::' + + echo '::group::Collect PyTorch environment information' + python -m torch.utils.collect_env + echo '::endgroup::' + + echo '::group::Install testing utilities' + pip install --progress-bar=off pytest pytest-mock pytest-cov + echo '::endgroup::' + + echo '::group::Run tests' + pytest --durations=25 + echo '::endgroup::' diff --git a/.github/workflows/test-windows-cpu.yml b/.github/workflows/test-windows-cpu.yml index 8e581e2be2d..91c9e813568 100644 --- a/.github/workflows/test-windows-cpu.yml +++ b/.github/workflows/test-windows-cpu.yml @@ -40,7 +40,7 @@ jobs: --name ci \ --quiet --yes \ python=${{ matrix.python-version }} pip \ - clang_osx-64 clangxx_osx-64 \ + m2w64-gcc m2w64-gxx \ setuptools ninja \ libpng jpeg conda activate ci From 585e19ee184350a80c3f724cb156d747cacb4051 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 30 Jan 2023 16:34:45 +0100 Subject: [PATCH 13/47] set +-u --- .github/workflows/test-linux-cpu.yml | 2 ++ .github/workflows/test-macos-cpu.yml | 2 ++ .github/workflows/test-windows-cpu.yml | 4 +++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index f3db11e03a3..c6d8f021630 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -44,7 +44,9 @@ jobs: gcc_linux-64 gxx_linux-64 \ setuptools ninja \ libpng jpeg + set +u conda activate ci + set -u pip install 'av<10' echo '::endgroup::' diff --git a/.github/workflows/test-macos-cpu.yml b/.github/workflows/test-macos-cpu.yml index 63fd11505e2..8de303ba891 100644 --- a/.github/workflows/test-macos-cpu.yml +++ b/.github/workflows/test-macos-cpu.yml @@ -43,7 +43,9 @@ jobs: clang_osx-64 clangxx_osx-64 \ setuptools ninja \ libpng jpeg + set +u conda activate ci + set -u pip install 'av<10' echo '::endgroup::' diff --git a/.github/workflows/test-windows-cpu.yml b/.github/workflows/test-windows-cpu.yml index 91c9e813568..010b3bc8aba 100644 --- a/.github/workflows/test-windows-cpu.yml +++ b/.github/workflows/test-windows-cpu.yml @@ -40,10 +40,12 @@ jobs: --name ci \ --quiet --yes \ python=${{ matrix.python-version }} pip \ - m2w64-gcc m2w64-gxx \ + m2w64-gcc \ setuptools ninja \ libpng jpeg + set +u conda activate ci + set -u pip install 'av<10' echo '::endgroup::' From c25e375a37865d560edbd2a92deb10d8125329c2 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 30 Jan 2023 16:39:14 +0100 Subject: [PATCH 14/47] never re-enable --- .github/workflows/test-linux-cpu.yml | 1 - .github/workflows/test-macos-cpu.yml | 1 - .github/workflows/test-windows-cpu.yml | 1 - 3 files changed, 3 deletions(-) diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index c6d8f021630..112c02ab5b9 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -46,7 +46,6 @@ jobs: libpng jpeg set +u conda activate ci - set -u pip install 'av<10' echo '::endgroup::' diff --git a/.github/workflows/test-macos-cpu.yml b/.github/workflows/test-macos-cpu.yml index 8de303ba891..47b8ebf19c5 100644 --- a/.github/workflows/test-macos-cpu.yml +++ b/.github/workflows/test-macos-cpu.yml @@ -45,7 +45,6 @@ jobs: libpng jpeg set +u conda activate ci - set -u pip install 'av<10' echo '::endgroup::' diff --git a/.github/workflows/test-windows-cpu.yml b/.github/workflows/test-windows-cpu.yml index 010b3bc8aba..489663e9f0b 100644 --- a/.github/workflows/test-windows-cpu.yml +++ b/.github/workflows/test-windows-cpu.yml @@ -45,7 +45,6 @@ jobs: libpng jpeg set +u conda activate ci - set -u pip install 'av<10' echo '::endgroup::' From 82a6226ce5a63468747096901793795be87d754d Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 31 Jan 2023 11:07:10 +0100 Subject: [PATCH 15/47] try remove compilers once again --- .github/workflows/test-linux-cpu.yml | 2 -- .github/workflows/test-macos-cpu.yml | 2 -- .github/workflows/test-windows-cpu.yml | 2 -- 3 files changed, 6 deletions(-) diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index 112c02ab5b9..958a68ac41a 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -41,10 +41,8 @@ jobs: --name ci \ --quiet --yes \ python=${{ matrix.python-version }} pip \ - gcc_linux-64 gxx_linux-64 \ setuptools ninja \ libpng jpeg - set +u conda activate ci pip install 'av<10' echo '::endgroup::' diff --git a/.github/workflows/test-macos-cpu.yml b/.github/workflows/test-macos-cpu.yml index 47b8ebf19c5..2d299510f1a 100644 --- a/.github/workflows/test-macos-cpu.yml +++ b/.github/workflows/test-macos-cpu.yml @@ -40,10 +40,8 @@ jobs: --name ci \ --quiet --yes \ python=${{ matrix.python-version }} pip \ - clang_osx-64 clangxx_osx-64 \ setuptools ninja \ libpng jpeg - set +u conda activate ci pip install 'av<10' echo '::endgroup::' diff --git a/.github/workflows/test-windows-cpu.yml b/.github/workflows/test-windows-cpu.yml index 489663e9f0b..41bfd8e90ac 100644 --- a/.github/workflows/test-windows-cpu.yml +++ b/.github/workflows/test-windows-cpu.yml @@ -40,10 +40,8 @@ jobs: --name ci \ --quiet --yes \ python=${{ matrix.python-version }} pip \ - m2w64-gcc \ setuptools ninja \ libpng jpeg - set +u conda activate ci pip install 'av<10' echo '::endgroup::' From a59949786a14e7c481f17f34944ebb9b11938f4c Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 31 Jan 2023 11:19:59 +0100 Subject: [PATCH 16/47] fix matrix variable --- .github/workflows/test-linux-cpu.yml | 2 +- .github/workflows/test-macos-cpu.yml | 2 +- .github/workflows/test-windows-cpu.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index 958a68ac41a..4e62deb0dfb 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -13,7 +13,7 @@ jobs: tests: strategy: matrix: - python_version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10"] fail-fast: false uses: pytorch/test-infra/.github/workflows/linux_job.yml@main with: diff --git a/.github/workflows/test-macos-cpu.yml b/.github/workflows/test-macos-cpu.yml index 2d299510f1a..c19ef0ebf95 100644 --- a/.github/workflows/test-macos-cpu.yml +++ b/.github/workflows/test-macos-cpu.yml @@ -13,7 +13,7 @@ jobs: tests: strategy: matrix: - python_version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10"] fail-fast: false uses: pytorch/test-infra/.github/workflows/macos_job.yml@main with: diff --git a/.github/workflows/test-windows-cpu.yml b/.github/workflows/test-windows-cpu.yml index 41bfd8e90ac..26a45a20b84 100644 --- a/.github/workflows/test-windows-cpu.yml +++ b/.github/workflows/test-windows-cpu.yml @@ -13,7 +13,7 @@ jobs: tests: strategy: matrix: - python_version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10"] fail-fast: false uses: pytorch/test-infra/.github/workflows/windows_job.yml@main with: From 2607d81c4fc89af029fde005947a445c92863b95 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 31 Jan 2023 12:22:19 +0100 Subject: [PATCH 17/47] add CONDA_BUILD env var debug --- .github/workflows/test-macos-cpu.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test-macos-cpu.yml b/.github/workflows/test-macos-cpu.yml index c19ef0ebf95..ae5777955c6 100644 --- a/.github/workflows/test-macos-cpu.yml +++ b/.github/workflows/test-macos-cpu.yml @@ -19,6 +19,8 @@ jobs: with: repository: pytorch/vision script: | + echo CONDA_BUILD=$CONDA_BUILD + echo '::group::Set PyTorch conda channel' # TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then @@ -43,8 +45,11 @@ jobs: setuptools ninja \ libpng jpeg conda activate ci + echo CONDA_BUILD=$CONDA_BUILD pip install 'av<10' echo '::endgroup::' + + echo CONDA_BUILD=$CONDA_BUILD echo '::group::Install PyTorch' conda install \ @@ -54,6 +59,8 @@ jobs: pytorch \ "${PYTORCH_MUTEX}" echo '::endgroup::' + + echo CONDA_BUILD=$CONDA_BUILD echo '::group::Install TorchVision' python setup.py develop From c6537dc76afdc942131cff7e7d72ccc8bb5fba2f Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 31 Jan 2023 13:44:53 +0100 Subject: [PATCH 18/47] unset CONDA_BUILD --- .github/workflows/test-linux-cpu.yml | 6 ++++++ .github/workflows/test-macos-cpu.yml | 13 ++++++------- .github/workflows/test-windows-cpu.yml | 6 ++++++ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index 4e62deb0dfb..81cba17e41d 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -57,6 +57,12 @@ jobs: echo '::endgroup::' echo '::group::Install TorchVision' + # The `setuptools` package installed through `conda` includes a patch that errors if something is installed + # through `setuptools` while the `CONDA_BUILD` environment variable is set. + # https://github.com/AnacondaRecipes/setuptools-feedstock/blob/f5d8d256810ce28fc0cf34170bc34e06d3754041/recipe/patches/0002-disable-downloads-inside-conda-build.patch + # (Although we are not using the `-c conda-forge` channel, the patch is equivalent but not public for + # `setuptools` from the `-c defaults` channel + unset CONDA_BUILD python setup.py develop echo '::endgroup::' diff --git a/.github/workflows/test-macos-cpu.yml b/.github/workflows/test-macos-cpu.yml index ae5777955c6..af5bab8ed37 100644 --- a/.github/workflows/test-macos-cpu.yml +++ b/.github/workflows/test-macos-cpu.yml @@ -19,8 +19,6 @@ jobs: with: repository: pytorch/vision script: | - echo CONDA_BUILD=$CONDA_BUILD - echo '::group::Set PyTorch conda channel' # TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then @@ -45,11 +43,8 @@ jobs: setuptools ninja \ libpng jpeg conda activate ci - echo CONDA_BUILD=$CONDA_BUILD pip install 'av<10' echo '::endgroup::' - - echo CONDA_BUILD=$CONDA_BUILD echo '::group::Install PyTorch' conda install \ @@ -59,10 +54,14 @@ jobs: pytorch \ "${PYTORCH_MUTEX}" echo '::endgroup::' - - echo CONDA_BUILD=$CONDA_BUILD echo '::group::Install TorchVision' + # The `setuptools` package installed through `conda` includes a patch that errors if something is installed + # through `setuptools` while the `CONDA_BUILD` environment variable is set. + # https://github.com/AnacondaRecipes/setuptools-feedstock/blob/f5d8d256810ce28fc0cf34170bc34e06d3754041/recipe/patches/0002-disable-downloads-inside-conda-build.patch + # (Although we are not using the `-c conda-forge` channel, the patch is equivalent but not public for + # `setuptools` from the `-c defaults` channel + unset CONDA_BUILD python setup.py develop echo '::endgroup::' diff --git a/.github/workflows/test-windows-cpu.yml b/.github/workflows/test-windows-cpu.yml index 26a45a20b84..13e7aeeb48d 100644 --- a/.github/workflows/test-windows-cpu.yml +++ b/.github/workflows/test-windows-cpu.yml @@ -56,6 +56,12 @@ jobs: echo '::endgroup::' echo '::group::Install TorchVision' + # the `setuptools` package installed through `conda` includes a patch that errors if something is installed + # through `setuptools` while the `CONDA_BUILD` environment variable is set. + # https://github.com/AnacondaRecipes/setuptools-feedstock/blob/f5d8d256810ce28fc0cf34170bc34e06d3754041/recipe/patches/0002-disable-downloads-inside-conda-build.patch + # (Although we are not using the `-c conda-forge` channel, the patch is equivalent but not public for + # `setuptools` from the `-c defaults` channel + unset CONDA_BUILD python setup.py develop echo '::endgroup::' From 8f42b3995f2e3a8a86c95091b4bd9863ab2585d2 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 31 Jan 2023 13:57:16 +0100 Subject: [PATCH 19/47] debug certifi --- .github/workflows/test-linux-cpu.yml | 4 ++++ .github/workflows/test-macos-cpu.yml | 4 ++++ .github/workflows/test-windows-cpu.yml | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index 81cba17e41d..6d405d4d8a7 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -55,6 +55,10 @@ jobs: pytorch \ "${PYTORCH_MUTEX}" echo '::endgroup::' + + python -c "import certifi; print(certifi.where())" + python -c "import pip._vendor.certifi as certifi; print(certifi.where())" + echo PIP_CERT=$PIP_CERT echo '::group::Install TorchVision' # The `setuptools` package installed through `conda` includes a patch that errors if something is installed diff --git a/.github/workflows/test-macos-cpu.yml b/.github/workflows/test-macos-cpu.yml index af5bab8ed37..96c8152b02d 100644 --- a/.github/workflows/test-macos-cpu.yml +++ b/.github/workflows/test-macos-cpu.yml @@ -54,6 +54,10 @@ jobs: pytorch \ "${PYTORCH_MUTEX}" echo '::endgroup::' + + python -c "import certifi; print(certifi.where())" + python -c "import pip._vendor.certifi as certifi; print(certifi.where())" + echo PIP_CERT=$PIP_CERT echo '::group::Install TorchVision' # The `setuptools` package installed through `conda` includes a patch that errors if something is installed diff --git a/.github/workflows/test-windows-cpu.yml b/.github/workflows/test-windows-cpu.yml index 13e7aeeb48d..4d4c9272f56 100644 --- a/.github/workflows/test-windows-cpu.yml +++ b/.github/workflows/test-windows-cpu.yml @@ -54,6 +54,10 @@ jobs: pytorch \ "${PYTORCH_MUTEX}" echo '::endgroup::' + + python -c "import certifi; print(certifi.where())" + python -c "import pip._vendor.certifi as certifi; print(certifi.where())" + echo PIP_CERT=$PIP_CERT echo '::group::Install TorchVision' # the `setuptools` package installed through `conda` includes a patch that errors if something is installed From a6c62e3cba212414ea5952fca10aed5f192d4471 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 31 Jan 2023 14:06:44 +0100 Subject: [PATCH 20/47] try matrix over jobs --- .github/workflows/test-linux-cpu.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index 6d405d4d8a7..664766e5d01 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -13,11 +13,15 @@ jobs: tests: strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + include: + - os: linux + runner: linux.12xlarge + - os: windows + runner: windows.4xlarge fail-fast: false - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/${{ matrix.os }}_job.yml@main with: - runner: linux.12xlarge + runner: ${{ matrix.runner }} repository: pytorch/vision script: | echo '::group::Set PyTorch conda channel' @@ -40,7 +44,7 @@ jobs: conda create \ --name ci \ --quiet --yes \ - python=${{ matrix.python-version }} pip \ + python pip \ setuptools ninja \ libpng jpeg conda activate ci From dded7ef62d149edc0c68c89d85c18325807f1cff Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 31 Jan 2023 14:19:02 +0100 Subject: [PATCH 21/47] preinstall dependencies --- .github/workflows/test-linux-cpu.yml | 7 ++----- .github/workflows/test-macos-cpu.yml | 7 ++----- .github/workflows/test-windows-cpu.yml | 7 ++----- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index 664766e5d01..e1c0b7cac1c 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -46,7 +46,8 @@ jobs: --quiet --yes \ python pip \ setuptools ninja \ - libpng jpeg + libpng jpeg \ + Pillow numpy requests conda activate ci pip install 'av<10' echo '::endgroup::' @@ -59,10 +60,6 @@ jobs: pytorch \ "${PYTORCH_MUTEX}" echo '::endgroup::' - - python -c "import certifi; print(certifi.where())" - python -c "import pip._vendor.certifi as certifi; print(certifi.where())" - echo PIP_CERT=$PIP_CERT echo '::group::Install TorchVision' # The `setuptools` package installed through `conda` includes a patch that errors if something is installed diff --git a/.github/workflows/test-macos-cpu.yml b/.github/workflows/test-macos-cpu.yml index 96c8152b02d..573e9bd31df 100644 --- a/.github/workflows/test-macos-cpu.yml +++ b/.github/workflows/test-macos-cpu.yml @@ -41,7 +41,8 @@ jobs: --quiet --yes \ python=${{ matrix.python-version }} pip \ setuptools ninja \ - libpng jpeg + libpng jpeg \ + Pillow numpy requests conda activate ci pip install 'av<10' echo '::endgroup::' @@ -54,10 +55,6 @@ jobs: pytorch \ "${PYTORCH_MUTEX}" echo '::endgroup::' - - python -c "import certifi; print(certifi.where())" - python -c "import pip._vendor.certifi as certifi; print(certifi.where())" - echo PIP_CERT=$PIP_CERT echo '::group::Install TorchVision' # The `setuptools` package installed through `conda` includes a patch that errors if something is installed diff --git a/.github/workflows/test-windows-cpu.yml b/.github/workflows/test-windows-cpu.yml index 4d4c9272f56..e1340922632 100644 --- a/.github/workflows/test-windows-cpu.yml +++ b/.github/workflows/test-windows-cpu.yml @@ -41,7 +41,8 @@ jobs: --quiet --yes \ python=${{ matrix.python-version }} pip \ setuptools ninja \ - libpng jpeg + libpng jpeg \ + Pillow numpy requests conda activate ci pip install 'av<10' echo '::endgroup::' @@ -54,10 +55,6 @@ jobs: pytorch \ "${PYTORCH_MUTEX}" echo '::endgroup::' - - python -c "import certifi; print(certifi.where())" - python -c "import pip._vendor.certifi as certifi; print(certifi.where())" - echo PIP_CERT=$PIP_CERT echo '::group::Install TorchVision' # the `setuptools` package installed through `conda` includes a patch that errors if something is installed From 21374efd0521f656b40c96322766fc7d9e8ffb15 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 31 Jan 2023 14:06:44 +0100 Subject: [PATCH 22/47] Revert "try matrix over jobs" This reverts commit a6c62e3cba212414ea5952fca10aed5f192d4471. --- .github/workflows/test-linux-cpu.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index e1c0b7cac1c..3e0888e520d 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -13,15 +13,11 @@ jobs: tests: strategy: matrix: - include: - - os: linux - runner: linux.12xlarge - - os: windows - runner: windows.4xlarge + python-version: ["3.8", "3.9", "3.10"] fail-fast: false - uses: pytorch/test-infra/.github/workflows/${{ matrix.os }}_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main with: - runner: ${{ matrix.runner }} + runner: linux.12xlarge repository: pytorch/vision script: | echo '::group::Set PyTorch conda channel' @@ -44,7 +40,7 @@ jobs: conda create \ --name ci \ --quiet --yes \ - python pip \ + python=${{ matrix.python-version }} pip \ setuptools ninja \ libpng jpeg \ Pillow numpy requests From c258895c2bc295a07aee72a1aff8faf4ff834715 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 1 Feb 2023 09:44:37 +0100 Subject: [PATCH 23/47] remove Windows workflow --- .github/workflows/test-windows-cpu.yml | 79 -------------------------- 1 file changed, 79 deletions(-) delete mode 100644 .github/workflows/test-windows-cpu.yml diff --git a/.github/workflows/test-windows-cpu.yml b/.github/workflows/test-windows-cpu.yml deleted file mode 100644 index e1340922632..00000000000 --- a/.github/workflows/test-windows-cpu.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: Unit-tests on Windows CPU - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: - -jobs: - tests: - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10"] - fail-fast: false - uses: pytorch/test-infra/.github/workflows/windows_job.yml@main - with: - repository: pytorch/vision - script: | - echo '::group::Set PyTorch conda channel' - # TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. - if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then - POSTFIX=test - else - POSTFIX=nightly - fi - PYTORCH_CHANNEL=pytorch-"${POSTFIX}" - echo "${PYTORCH_CHANNEL}" - echo '::endgroup::' - - echo '::group::Set PyTorch conda mutex' - PYTORCH_MUTEX=cpuonly - echo "${PYTORCH_MUTEX}" - echo '::endgroup::' - - echo '::group::Create conda environment' - conda create \ - --name ci \ - --quiet --yes \ - python=${{ matrix.python-version }} pip \ - setuptools ninja \ - libpng jpeg \ - Pillow numpy requests - conda activate ci - pip install 'av<10' - echo '::endgroup::' - - echo '::group::Install PyTorch' - conda install \ - --quiet --yes \ - -c "${PYTORCH_CHANNEL}" \ - -c nvidia \ - pytorch \ - "${PYTORCH_MUTEX}" - echo '::endgroup::' - - echo '::group::Install TorchVision' - # the `setuptools` package installed through `conda` includes a patch that errors if something is installed - # through `setuptools` while the `CONDA_BUILD` environment variable is set. - # https://github.com/AnacondaRecipes/setuptools-feedstock/blob/f5d8d256810ce28fc0cf34170bc34e06d3754041/recipe/patches/0002-disable-downloads-inside-conda-build.patch - # (Although we are not using the `-c conda-forge` channel, the patch is equivalent but not public for - # `setuptools` from the `-c defaults` channel - unset CONDA_BUILD - python setup.py develop - echo '::endgroup::' - - echo '::group::Collect PyTorch environment information' - python -m torch.utils.collect_env - echo '::endgroup::' - - echo '::group::Install testing utilities' - pip install --progress-bar=off pytest pytest-mock pytest-cov - echo '::endgroup::' - - echo '::group::Run tests' - pytest --durations=25 - echo '::endgroup::' From ff45d05cbbf12bb975f5337e0fdf248f66daaaa0 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 1 Feb 2023 10:01:27 +0100 Subject: [PATCH 24/47] try unify linux and mac --- .github/script_unittest.sh | 61 ++++++++++++++++++++++++++ .github/workflows/test-linux-cpu.yml | 65 +++------------------------- .github/workflows/test-macos-cpu.yml | 64 +++------------------------ 3 files changed, 73 insertions(+), 117 deletions(-) create mode 100755 .github/script_unittest.sh diff --git a/.github/script_unittest.sh b/.github/script_unittest.sh new file mode 100755 index 00000000000..6a86474afda --- /dev/null +++ b/.github/script_unittest.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +echo '::group::Set PyTorch conda channel' +# TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. +if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then + POSTFIX=test +else + POSTFIX=nightly +fi +PYTORCH_CHANNEL=pytorch-"${POSTFIX}" +echo "${PYTORCH_CHANNEL}" +echo '::endgroup::' + +echo '::group::Set PyTorch conda mutex' +PYTORCH_MUTEX=cpuonly +echo "${PYTORCH_MUTEX}" +echo '::endgroup::' + +echo '::group::Create conda environment' +conda create \ + --name ci \ + --quiet --yes \ + python="${PYTHON_VERSION}" pip \ + setuptools ninja \ + libpng jpeg \ + Pillow numpy requests +conda activate ci +pip install 'av<10' +echo '::endgroup::' + +echo '::group::Install PyTorch' +conda install \ + --quiet --yes \ + -c "${PYTORCH_CHANNEL}" \ + -c nvidia \ + pytorch \ + "${PYTORCH_MUTEX}" +echo '::endgroup::' + +echo '::group::Install TorchVision' +# The `setuptools` package installed through `conda` includes a patch that errors if something is installed +# through `setuptools` while the `CONDA_BUILD` environment variable is set. +# https://github.com/AnacondaRecipes/setuptools-feedstock/blob/f5d8d256810ce28fc0cf34170bc34e06d3754041/recipe/patches/0002-disable-downloads-inside-conda-build.patch +# (Although we are not using the `-c conda-forge` channel, the patch is equivalent but not public for +# `setuptools` from the `-c defaults` channel +# TODO: investigate where `CONDA_BUILD` is set and maybe fix it there +unset CONDA_BUILD +python setup.py develop +echo '::endgroup::' + +echo '::group::Collect PyTorch environment information' +python -m torch.utils.collect_env +echo '::endgroup::' + +echo '::group::Install testing utilities' +pip install --progress-bar=off pytest pytest-mock pytest-cov +echo '::endgroup::' + +echo '::group::Run tests' +pytest --durations=25 +echo '::endgroup::' \ No newline at end of file diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index 3e0888e520d..36e6f4a8889 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -13,68 +13,15 @@ jobs: tests: strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: + - "3.8" + - "3.9" + - "3.10" fail-fast: false uses: pytorch/test-infra/.github/workflows/linux_job.yml@main with: - runner: linux.12xlarge repository: pytorch/vision script: | - echo '::group::Set PyTorch conda channel' - # TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. - if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then - POSTFIX=test - else - POSTFIX=nightly - fi - PYTORCH_CHANNEL=pytorch-"${POSTFIX}" - echo "${PYTORCH_CHANNEL}" - echo '::endgroup::' + export PYTHON_VERSION=${{ matrix.python-version }} - echo '::group::Set PyTorch conda mutex' - PYTORCH_MUTEX=cpuonly - echo "${PYTORCH_MUTEX}" - echo '::endgroup::' - - echo '::group::Create conda environment' - conda create \ - --name ci \ - --quiet --yes \ - python=${{ matrix.python-version }} pip \ - setuptools ninja \ - libpng jpeg \ - Pillow numpy requests - conda activate ci - pip install 'av<10' - echo '::endgroup::' - - echo '::group::Install PyTorch' - conda install \ - --quiet --yes \ - -c "${PYTORCH_CHANNEL}" \ - -c nvidia \ - pytorch \ - "${PYTORCH_MUTEX}" - echo '::endgroup::' - - echo '::group::Install TorchVision' - # The `setuptools` package installed through `conda` includes a patch that errors if something is installed - # through `setuptools` while the `CONDA_BUILD` environment variable is set. - # https://github.com/AnacondaRecipes/setuptools-feedstock/blob/f5d8d256810ce28fc0cf34170bc34e06d3754041/recipe/patches/0002-disable-downloads-inside-conda-build.patch - # (Although we are not using the `-c conda-forge` channel, the patch is equivalent but not public for - # `setuptools` from the `-c defaults` channel - unset CONDA_BUILD - python setup.py develop - echo '::endgroup::' - - echo '::group::Collect PyTorch environment information' - python -m torch.utils.collect_env - echo '::endgroup::' - - echo '::group::Install testing utilities' - pip install --progress-bar=off pytest pytest-mock pytest-cov - echo '::endgroup::' - - echo '::group::Run tests' - pytest --durations=25 - echo '::endgroup::' + ./.github/script_unittest.sh diff --git a/.github/workflows/test-macos-cpu.yml b/.github/workflows/test-macos-cpu.yml index 573e9bd31df..65a62a87604 100644 --- a/.github/workflows/test-macos-cpu.yml +++ b/.github/workflows/test-macos-cpu.yml @@ -13,67 +13,15 @@ jobs: tests: strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: + - "3.8" + - "3.9" + - "3.10" fail-fast: false uses: pytorch/test-infra/.github/workflows/macos_job.yml@main with: repository: pytorch/vision script: | - echo '::group::Set PyTorch conda channel' - # TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. - if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then - POSTFIX=test - else - POSTFIX=nightly - fi - PYTORCH_CHANNEL=pytorch-"${POSTFIX}" - echo "${PYTORCH_CHANNEL}" - echo '::endgroup::' + export PYTHON_VERSION=${{ matrix.python-version }} - echo '::group::Set PyTorch conda mutex' - PYTORCH_MUTEX=cpuonly - echo "${PYTORCH_MUTEX}" - echo '::endgroup::' - - echo '::group::Create conda environment' - conda create \ - --name ci \ - --quiet --yes \ - python=${{ matrix.python-version }} pip \ - setuptools ninja \ - libpng jpeg \ - Pillow numpy requests - conda activate ci - pip install 'av<10' - echo '::endgroup::' - - echo '::group::Install PyTorch' - conda install \ - --quiet --yes \ - -c "${PYTORCH_CHANNEL}" \ - -c nvidia \ - pytorch \ - "${PYTORCH_MUTEX}" - echo '::endgroup::' - - echo '::group::Install TorchVision' - # The `setuptools` package installed through `conda` includes a patch that errors if something is installed - # through `setuptools` while the `CONDA_BUILD` environment variable is set. - # https://github.com/AnacondaRecipes/setuptools-feedstock/blob/f5d8d256810ce28fc0cf34170bc34e06d3754041/recipe/patches/0002-disable-downloads-inside-conda-build.patch - # (Although we are not using the `-c conda-forge` channel, the patch is equivalent but not public for - # `setuptools` from the `-c defaults` channel - unset CONDA_BUILD - python setup.py develop - echo '::endgroup::' - - echo '::group::Collect PyTorch environment information' - python -m torch.utils.collect_env - echo '::endgroup::' - - echo '::group::Install testing utilities' - pip install --progress-bar=off pytest pytest-mock pytest-cov - echo '::endgroup::' - - echo '::group::Run tests' - pytest --durations=25 - echo '::endgroup::' + ./.github/script_unittest.sh From 7a60180eda3d373da7485b9d7801e011687bc400 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 1 Feb 2023 10:19:24 +0100 Subject: [PATCH 25/47] use proper linux runner --- .github/script_unittest.sh | 2 +- .github/workflows/test-linux-cpu.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/script_unittest.sh b/.github/script_unittest.sh index 6a86474afda..029e0f4a6bb 100755 --- a/.github/script_unittest.sh +++ b/.github/script_unittest.sh @@ -58,4 +58,4 @@ echo '::endgroup::' echo '::group::Run tests' pytest --durations=25 -echo '::endgroup::' \ No newline at end of file +echo '::endgroup::' diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index 36e6f4a8889..4844399bae0 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -21,6 +21,7 @@ jobs: uses: pytorch/test-infra/.github/workflows/linux_job.yml@main with: repository: pytorch/vision + runner: linux.12xlarge script: | export PYTHON_VERSION=${{ matrix.python-version }} From 29c14146c02a40c6cb60d87314d81463ea277541 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 1 Feb 2023 10:59:55 +0100 Subject: [PATCH 26/47] consolidate CPU and GPU workflows --- .github/script_unittest.sh | 17 +++++- .github/workflows/test-linux-cpu.yml | 28 --------- .github/workflows/test-linux-gpu.yml | 61 ------------------- .github/workflows/test-linux.yml | 40 ++++++++++++ .../{test-macos-cpu.yml => test-macos.yml} | 4 +- 5 files changed, 59 insertions(+), 91 deletions(-) delete mode 100644 .github/workflows/test-linux-cpu.yml delete mode 100644 .github/workflows/test-linux-gpu.yml create mode 100644 .github/workflows/test-linux.yml rename .github/workflows/{test-macos-cpu.yml => test-macos.yml} (80%) diff --git a/.github/script_unittest.sh b/.github/script_unittest.sh index 029e0f4a6bb..ad7aed38b47 100755 --- a/.github/script_unittest.sh +++ b/.github/script_unittest.sh @@ -12,7 +12,18 @@ echo "${PYTORCH_CHANNEL}" echo '::endgroup::' echo '::group::Set PyTorch conda mutex' -PYTORCH_MUTEX=cpuonly +case $GPU_ARCH_TYPE in + cpu) + PYTORCH_MUTEX=cpuonly + ;; + cuda) + PYTORCH_MUTEX="pytorch-cuda=${GPU_ARCH_VERSION}" + ;; + *) + echo "Unknown GPU_ARCH_TYPE=${GPU_ARCH_TYPE}" + exit 1 + ;; +esac echo "${PYTORCH_MUTEX}" echo '::endgroup::' @@ -35,6 +46,10 @@ conda install \ -c nvidia \ pytorch \ "${PYTORCH_MUTEX}" + +if [[ $GPU_ARCH_TYPE = 'cuda' ]]; then + python3 -c "import torch; exit(not torch.cuda.is_available())" +fi echo '::endgroup::' echo '::group::Install TorchVision' diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml deleted file mode 100644 index 4844399bae0..00000000000 --- a/.github/workflows/test-linux-cpu.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Unit-tests on Linux CPU - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: - -jobs: - tests: - strategy: - matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - fail-fast: false - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - runner: linux.12xlarge - script: | - export PYTHON_VERSION=${{ matrix.python-version }} - - ./.github/script_unittest.sh diff --git a/.github/workflows/test-linux-gpu.yml b/.github/workflows/test-linux-gpu.yml deleted file mode 100644 index a4d938f23ed..00000000000 --- a/.github/workflows/test-linux-gpu.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Unit-tests on Linux GPU - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: - -env: - CHANNEL: "nightly" - -jobs: - tests: - strategy: - matrix: - python_version: ["3.8"] - cuda_arch_version: ["11.6"] - fail-fast: false - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - runner: linux.g5.4xlarge.nvidia.gpu - repository: pytorch/vision - gpu-arch-type: cuda - gpu-arch-version: ${{ matrix.cuda_arch_version }} - timeout: 120 - script: | - # Mark Build Directory Safe - git config --global --add safe.directory /__w/vision/vision - - # Set up Environment Variables - export PYTHON_VERSION="${{ matrix.python_version }}" - export VERSION="${{ matrix.cuda_arch_version }}" - export CUDATOOLKIT="pytorch-cuda=${VERSION}" - - # Set CHANNEL - if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then - export CHANNEL=test - else - export CHANNEL=nightly - fi - - # Create Conda Env - conda create -yp ci_env python="${PYTHON_VERSION}" numpy libpng jpeg scipy - conda activate /work/ci_env - - # Install PyTorch, Torchvision, and testing libraries - set -ex - conda install \ - --yes \ - -c "pytorch-${CHANNEL}" \ - -c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${VERSION}*"] \ - "${CUDATOOLKIT}" - python3 setup.py develop - python3 -m pip install pytest pytest-mock 'av<10' - - # Run Tests - python3 -m torch.utils.collect_env - python3 -m pytest --junitxml=test-results/junit.xml -v --durations 20 diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml new file mode 100644 index 00000000000..f1417c94bb1 --- /dev/null +++ b/.github/workflows/test-linux.yml @@ -0,0 +1,40 @@ +name: Unit-tests on Linux + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +jobs: + tests: + strategy: + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10" + runner: ["linux.12xlarge"] + timeout: 30 + gpu-arch-type: ["cpu"] + include: + - python-version: 3.8 + runner: linux.g5.4xlarge.nvidia.gpu + timeout: 120 + gpu-arch-type: cuda + gpu-arch-version: "11.6" + fail-fast: false + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + runner: ${{ matrix.runner }} + timeout: ${{ matrix.timeout }} + script: | + export PYTHON_VERSION=${{ matrix.python-version }} + export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} + export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} + + ./.github/script_unittest.sh diff --git a/.github/workflows/test-macos-cpu.yml b/.github/workflows/test-macos.yml similarity index 80% rename from .github/workflows/test-macos-cpu.yml rename to .github/workflows/test-macos.yml index 65a62a87604..4af5940745f 100644 --- a/.github/workflows/test-macos-cpu.yml +++ b/.github/workflows/test-macos.yml @@ -1,4 +1,4 @@ -name: Unit-tests on macOS CPU +name: Unit-tests on macOS on: pull_request: @@ -17,11 +17,13 @@ jobs: - "3.8" - "3.9" - "3.10" + gpu-arch-type: ["cpu"] fail-fast: false uses: pytorch/test-infra/.github/workflows/macos_job.yml@main with: repository: pytorch/vision script: | export PYTHON_VERSION=${{ matrix.python-version }} + export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} ./.github/script_unittest.sh From a7213d0b62328ec978af45af27dfb37ae65c68fe Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 1 Feb 2023 20:14:01 +0100 Subject: [PATCH 27/47] increase timeout on macos --- .github/workflows/test-macos.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 4af5940745f..af8028704ad 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -22,6 +22,7 @@ jobs: uses: pytorch/test-infra/.github/workflows/macos_job.yml@main with: repository: pytorch/vision + timeout: 60 script: | export PYTHON_VERSION=${{ matrix.python-version }} export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} From 21bb9583477bf91cf3939b1f101db119a65a876f Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 1 Feb 2023 20:15:00 +0100 Subject: [PATCH 28/47] fix indent --- .github/workflows/test-linux.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index f1417c94bb1..c1a920bd21d 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -20,12 +20,12 @@ jobs: runner: ["linux.12xlarge"] timeout: 30 gpu-arch-type: ["cpu"] - include: - - python-version: 3.8 - runner: linux.g5.4xlarge.nvidia.gpu - timeout: 120 - gpu-arch-type: cuda - gpu-arch-version: "11.6" + include: + - python-version: 3.8 + runner: linux.g5.4xlarge.nvidia.gpu + timeout: 120 + gpu-arch-type: cuda + gpu-arch-version: "11.6" fail-fast: false uses: pytorch/test-infra/.github/workflows/linux_job.yml@main with: From d08fa2f69447ae3b4acce4efd837f05716a5eed5 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 1 Feb 2023 20:41:23 +0100 Subject: [PATCH 29/47] fix timeout for linux as well --- .github/workflows/test-linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index c1a920bd21d..8800ef6ea68 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -18,7 +18,7 @@ jobs: - "3.9" - "3.10" runner: ["linux.12xlarge"] - timeout: 30 + timeout: [60] gpu-arch-type: ["cpu"] include: - python-version: 3.8 From fe4fd868199473b4daf0d979e4eb12fbba1509fe Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 2 Feb 2023 08:56:11 +0100 Subject: [PATCH 30/47] add explicit exit from script --- .github/script_unittest.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/script_unittest.sh b/.github/script_unittest.sh index ad7aed38b47..fd2d472b42c 100755 --- a/.github/script_unittest.sh +++ b/.github/script_unittest.sh @@ -74,3 +74,7 @@ echo '::endgroup::' echo '::group::Run tests' pytest --durations=25 echo '::endgroup::' + +CODE=$? +echo $CODE +exit $CODE From f670df7830b8f0aa10df6e1b09a00591ac319a49 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 2 Feb 2023 08:56:38 +0100 Subject: [PATCH 31/47] increase timeout for mac workflows --- .github/workflows/test-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index af8028704ad..d96f33503c6 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -22,7 +22,7 @@ jobs: uses: pytorch/test-infra/.github/workflows/macos_job.yml@main with: repository: pytorch/vision - timeout: 60 + timeout: 120 script: | export PYTHON_VERSION=${{ matrix.python-version }} export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} From b46ecbb55e3c9bc30ac1106f86fba68c41ef698b Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 2 Feb 2023 08:56:58 +0100 Subject: [PATCH 32/47] align timeout for all workflows --- .github/workflows/test-linux.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 8800ef6ea68..2a197448dcf 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -18,12 +18,10 @@ jobs: - "3.9" - "3.10" runner: ["linux.12xlarge"] - timeout: [60] gpu-arch-type: ["cpu"] include: - python-version: 3.8 runner: linux.g5.4xlarge.nvidia.gpu - timeout: 120 gpu-arch-type: cuda gpu-arch-version: "11.6" fail-fast: false @@ -31,7 +29,7 @@ jobs: with: repository: pytorch/vision runner: ${{ matrix.runner }} - timeout: ${{ matrix.timeout }} + timeout: 120 script: | export PYTHON_VERSION=${{ matrix.python-version }} export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} From 349288f78210713ef6660dce2478d39425a63046 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 2 Feb 2023 09:02:35 +0100 Subject: [PATCH 33/47] expand macos runners --- .github/workflows/test-macos.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index d96f33503c6..6ccee9167b0 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -17,12 +17,16 @@ jobs: - "3.8" - "3.9" - "3.10" + runner: + - macos-12 + - macos-m1-12 gpu-arch-type: ["cpu"] fail-fast: false uses: pytorch/test-infra/.github/workflows/macos_job.yml@main with: repository: pytorch/vision timeout: 120 + runner: ${{ matrix.runner }} script: | export PYTHON_VERSION=${{ matrix.python-version }} export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} From fe27b6ea9e4015cf17e618c6f4e88d546361c598 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 2 Feb 2023 11:23:42 +0100 Subject: [PATCH 34/47] use strict mode --- .github/script_unittest.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/script_unittest.sh b/.github/script_unittest.sh index fd2d472b42c..750a0048171 100755 --- a/.github/script_unittest.sh +++ b/.github/script_unittest.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -eu + echo '::group::Set PyTorch conda channel' # TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then @@ -74,7 +76,3 @@ echo '::endgroup::' echo '::group::Run tests' pytest --durations=25 echo '::endgroup::' - -CODE=$? -echo $CODE -exit $CODE From 95d95acb7fbb9608b6cdd805413cdcb870f2ae47 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 2 Feb 2023 16:24:25 +0100 Subject: [PATCH 35/47] try using conda env from path --- .github/script_unittest.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/script_unittest.sh b/.github/script_unittest.sh index 750a0048171..cee23a74dbc 100755 --- a/.github/script_unittest.sh +++ b/.github/script_unittest.sh @@ -30,14 +30,15 @@ echo "${PYTORCH_MUTEX}" echo '::endgroup::' echo '::group::Create conda environment' +ENV_PATH="${PWD}/ci-env" conda create \ - --name ci \ + --prefix $ENV_PATH \ --quiet --yes \ python="${PYTHON_VERSION}" pip \ setuptools ninja \ libpng jpeg \ Pillow numpy requests -conda activate ci +conda activate $ENV_PATH pip install 'av<10' echo '::endgroup::' From 7fd032bd8c826b87bc85924a2f5f2ac8aa9ff696 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 2 Feb 2023 16:29:18 +0100 Subject: [PATCH 36/47] Revert "try using conda env from path" This reverts commit 95d95acb7fbb9608b6cdd805413cdcb870f2ae47. --- .github/script_unittest.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/script_unittest.sh b/.github/script_unittest.sh index cee23a74dbc..750a0048171 100755 --- a/.github/script_unittest.sh +++ b/.github/script_unittest.sh @@ -30,15 +30,14 @@ echo "${PYTORCH_MUTEX}" echo '::endgroup::' echo '::group::Create conda environment' -ENV_PATH="${PWD}/ci-env" conda create \ - --prefix $ENV_PATH \ + --name ci \ --quiet --yes \ python="${PYTHON_VERSION}" pip \ setuptools ninja \ libpng jpeg \ Pillow numpy requests -conda activate $ENV_PATH +conda activate ci pip install 'av<10' echo '::endgroup::' From 0b52f38a21b3a3f40079aab5aaf164c358ba52c2 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 2 Feb 2023 16:31:34 +0100 Subject: [PATCH 37/47] try conda init --- .github/script_unittest.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/script_unittest.sh b/.github/script_unittest.sh index 750a0048171..e82b295d36c 100755 --- a/.github/script_unittest.sh +++ b/.github/script_unittest.sh @@ -29,7 +29,7 @@ esac echo "${PYTORCH_MUTEX}" echo '::endgroup::' -echo '::group::Create conda environment' +echo '::group::Create build environment' conda create \ --name ci \ --quiet --yes \ @@ -37,6 +37,8 @@ conda create \ setuptools ninja \ libpng jpeg \ Pillow numpy requests +conda init bash +source "${HOME}/.bashrc" conda activate ci pip install 'av<10' echo '::endgroup::' From 1010f82f2d6ca9ea3a783429694707530f64703a Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 2 Feb 2023 17:01:10 +0100 Subject: [PATCH 38/47] source correct bash config --- .github/script_unittest.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/script_unittest.sh b/.github/script_unittest.sh index e82b295d36c..ccbb1650754 100755 --- a/.github/script_unittest.sh +++ b/.github/script_unittest.sh @@ -2,6 +2,11 @@ set -eu +echo '::group::Prepare conda' +BASH_CONFIG=$(conda init bash | grep modified | tr -s ' ' | cut -d ' ' -f2) +source "${BASH_CONFIG}" +echo '::endgroup::' + echo '::group::Set PyTorch conda channel' # TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`. if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then @@ -37,8 +42,6 @@ conda create \ setuptools ninja \ libpng jpeg \ Pillow numpy requests -conda init bash -source "${HOME}/.bashrc" conda activate ci pip install 'av<10' echo '::endgroup::' From eb26cb478150a71e08a4ef0537b698d9e2b6d1f7 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 2 Feb 2023 17:07:02 +0100 Subject: [PATCH 39/47] try allowing unset variables during activation --- .github/script_unittest.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/script_unittest.sh b/.github/script_unittest.sh index ccbb1650754..52735133f4d 100755 --- a/.github/script_unittest.sh +++ b/.github/script_unittest.sh @@ -4,7 +4,9 @@ set -eu echo '::group::Prepare conda' BASH_CONFIG=$(conda init bash | grep modified | tr -s ' ' | cut -d ' ' -f2) +set +u source "${BASH_CONFIG}" +set -u echo '::endgroup::' echo '::group::Set PyTorch conda channel' From 46111b4d1c9f31904a09b4da7d324e133e3ca1cf Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 2 Feb 2023 17:12:33 +0100 Subject: [PATCH 40/47] more debug --- .github/script_unittest.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/script_unittest.sh b/.github/script_unittest.sh index 52735133f4d..04f4a0457b9 100755 --- a/.github/script_unittest.sh +++ b/.github/script_unittest.sh @@ -3,7 +3,9 @@ set -eu echo '::group::Prepare conda' -BASH_CONFIG=$(conda init bash | grep modified | tr -s ' ' | cut -d ' ' -f2) +BASH_CONFIG=$(conda init bash | tee log | grep modified | tr -s ' ' | cut -d ' ' -f2) +cat log +cat "${BASH_CONFIG}" set +u source "${BASH_CONFIG}" set -u From 71c8add6497b1fd70d2bc6684ca3587e5b1061d0 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 2 Feb 2023 17:34:08 +0100 Subject: [PATCH 41/47] add sleep for SSH access --- .github/workflows/test-linux.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index c11b7a61a3d..97ca25e453f 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -34,5 +34,7 @@ jobs: export PYTHON_VERSION=${{ matrix.python-version }} export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} + + sleep 3600 ./.github/script_unittest.sh From 2fcceea8b2d960a5f6ca6784afefdb6e22c32f68 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 2 Feb 2023 17:56:44 +0100 Subject: [PATCH 42/47] try invoking hook manually --- .github/script_unittest.sh | 9 +++------ .github/workflows/test-linux.yml | 4 ++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/script_unittest.sh b/.github/script_unittest.sh index 04f4a0457b9..ade86d66777 100755 --- a/.github/script_unittest.sh +++ b/.github/script_unittest.sh @@ -1,14 +1,11 @@ #!/usr/bin/env bash +# FIXME: readd -e set -eu echo '::group::Prepare conda' -BASH_CONFIG=$(conda init bash | tee log | grep modified | tr -s ' ' | cut -d ' ' -f2) -cat log -cat "${BASH_CONFIG}" -set +u -source "${BASH_CONFIG}" -set -u +CONDA_PATH=$(which conda) +eval "$(${CONDA_PATH} shell.bash hook)" echo '::endgroup::' echo '::group::Set PyTorch conda channel' diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 97ca25e453f..57e9dac1233 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -34,7 +34,7 @@ jobs: export PYTHON_VERSION=${{ matrix.python-version }} export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} - - sleep 3600 ./.github/script_unittest.sh + + sleep 3600 From bd94cc8aa71c0bda97c55dfe3b24b40cc7630084 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 2 Feb 2023 17:57:03 +0100 Subject: [PATCH 43/47] remove set -e for now --- .github/script_unittest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/script_unittest.sh b/.github/script_unittest.sh index ade86d66777..83f5d2fa589 100755 --- a/.github/script_unittest.sh +++ b/.github/script_unittest.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # FIXME: readd -e -set -eu +set -u echo '::group::Prepare conda' CONDA_PATH=$(which conda) From e39510b2607952e0f15872532247438d4771bb8b Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 2 Feb 2023 18:16:19 +0100 Subject: [PATCH 44/47] cleanup --- .github/script_unittest.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/script_unittest.sh b/.github/script_unittest.sh index 83f5d2fa589..eed892a6240 100755 --- a/.github/script_unittest.sh +++ b/.github/script_unittest.sh @@ -1,11 +1,18 @@ #!/usr/bin/env bash -# FIXME: readd -e -set -u +set -eu echo '::group::Prepare conda' CONDA_PATH=$(which conda) eval "$(${CONDA_PATH} shell.bash hook)" +# The `setuptools` package installed through `conda` includes a patch that errors if something is installed +# through `setuptools` while the `CONDA_BUILD` environment variable is set. +# https://github.com/AnacondaRecipes/setuptools-feedstock/blob/f5d8d256810ce28fc0cf34170bc34e06d3754041/recipe/patches/0002-disable-downloads-inside-conda-build.patch +# (Although we are not using the `-c conda-forge` channel, the patch is equivalent but not public for +# `setuptools` from the `-c defaults` channel) +# Since we aren't using `conda build` here, we unset it to avoid installation problems later +# TODO: investigate where `CONDA_BUILD` is set and maybe fix unset it there +unset CONDA_BUILD echo '::endgroup::' echo '::group::Set PyTorch conda channel' @@ -19,7 +26,7 @@ PYTORCH_CHANNEL=pytorch-"${POSTFIX}" echo "${PYTORCH_CHANNEL}" echo '::endgroup::' -echo '::group::Set PyTorch conda mutex' +echo '::group::Set PyTorch GPU mutex' case $GPU_ARCH_TYPE in cpu) PYTORCH_MUTEX=cpuonly @@ -61,13 +68,6 @@ fi echo '::endgroup::' echo '::group::Install TorchVision' -# The `setuptools` package installed through `conda` includes a patch that errors if something is installed -# through `setuptools` while the `CONDA_BUILD` environment variable is set. -# https://github.com/AnacondaRecipes/setuptools-feedstock/blob/f5d8d256810ce28fc0cf34170bc34e06d3754041/recipe/patches/0002-disable-downloads-inside-conda-build.patch -# (Although we are not using the `-c conda-forge` channel, the patch is equivalent but not public for -# `setuptools` from the `-c defaults` channel -# TODO: investigate where `CONDA_BUILD` is set and maybe fix it there -unset CONDA_BUILD python setup.py develop echo '::endgroup::' From 90199a8e8da9ac23acde4a92207b356fc42c28f4 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 2 Feb 2023 18:29:06 +0100 Subject: [PATCH 45/47] remove obsolete m1 workflow --- .github/workflows/test-m1.yml | 50 ----------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 .github/workflows/test-m1.yml diff --git a/.github/workflows/test-m1.yml b/.github/workflows/test-m1.yml deleted file mode 100644 index c03fa9f76e4..00000000000 --- a/.github/workflows/test-m1.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Unit-tests on M1 -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: -env: - CHANNEL: "nightly" -jobs: - tests: - name: "Unit-tests on M1" - runs-on: macos-m1-12 - strategy: - matrix: - py_vers: [ "3.8"] - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Set Release CHANNEL (for release) - if: ${{ (github.event_name == 'pull_request' && startsWith(github.base_ref, 'release')) || startsWith(github.ref, 'refs/heads/release') }} - run: | - echo "CHANNEL=test" >> "$GITHUB_ENV" - - name: Install TorchVision - shell: arch -arch arm64 bash {0} - env: - ENV_NAME: conda-env-${{ github.run_id }} - PY_VERS: ${{ matrix.py_vers }} - run: | - . ~/miniconda3/etc/profile.d/conda.sh - # Needed for JPEG library detection as setup.py detects conda presence by running `shutil.which('conda')` - export PATH=~/miniconda3/bin:$PATH - set -ex - conda create -yp ${ENV_NAME} python=${PY_VERS} numpy libpng jpeg scipy - conda run -p ${ENV_NAME} python3 -mpip install --pre torch --extra-index-url=https://download.pytorch.org/whl/${CHANNEL} - conda run -p ${ENV_NAME} python3 setup.py develop - conda run -p ${ENV_NAME} python3 -mpip install pytest pytest-mock 'av<10' - - name: Run tests - shell: arch -arch arm64 bash {0} - env: - ENV_NAME: conda-env-${{ github.run_id }} - PY_VERS: ${{ matrix.py_vers }} - run: | - . ~/miniconda3/etc/profile.d/conda.sh - set -ex - conda run -p ${ENV_NAME} --no-capture-output python3 -u -mpytest -v --tb=long --durations 20 - conda env remove -p ${ENV_NAME} From afe9abf98b0f70b08e9c8fecac56383cb0c87c92 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 3 Feb 2023 14:40:12 +0100 Subject: [PATCH 46/47] try renable windows workflows --- .github/workflows/test-linux.yml | 2 -- .github/workflows/test-windows.yml | 38 ++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/test-windows.yml diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 57e9dac1233..c11b7a61a3d 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -36,5 +36,3 @@ jobs: export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} ./.github/script_unittest.sh - - sleep 3600 diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml new file mode 100644 index 00000000000..3e44e297651 --- /dev/null +++ b/.github/workflows/test-windows.yml @@ -0,0 +1,38 @@ +name: Unit-tests on Windows + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +jobs: + tests: + strategy: + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10" + runner: ["windows.4xlarge"] + gpu-arch-type: ["cpu"] + include: + - python-version: 3.8 + runner: windows.8xlarge.nvidia.gpu + gpu-arch-type: cuda + gpu-arch-version: "11.7" + fail-fast: false + uses: pytorch/test-infra/.github/workflows/windows_job.yml@main + with: + repository: pytorch/vision + runner: ${{ matrix.runner }} + timeout: 120 + script: | + export PYTHON_VERSION=${{ matrix.python-version }} + export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} + export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} + + ./.github/script_unittest.sh From 9c84c48eac754775955ffd2069b8f9ee09154923 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 3 Feb 2023 14:43:20 +0100 Subject: [PATCH 47/47] make bash mode more strict --- .github/script_unittest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/script_unittest.sh b/.github/script_unittest.sh index eed892a6240..21406ce9985 100755 --- a/.github/script_unittest.sh +++ b/.github/script_unittest.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -eu +set -euxo pipefail echo '::group::Prepare conda' CONDA_PATH=$(which conda)