From bf024cd303c561ec62f8cc9926d70407d1c7cad8 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Apr 2022 13:51:41 +0200 Subject: [PATCH 01/17] add Windows and macOS CI jobs for prototype tests --- .circleci/config.yml | 14 --------- .circleci/config.yml.in | 14 --------- .github/workflows/prototype-tests.yml | 42 +++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/prototype-tests.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index da40b942b0e..e12eea224ed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -357,20 +357,6 @@ jobs: - run_tests_selective: file_or_dir: test/test_onnx.py - unittest_prototype: - docker: - - image: cimg/python:3.7 - resource_class: xlarge - steps: - - checkout - - install_torchvision - - install_prototype_dependencies - - pip_install: - args: scipy pycocotools h5py - descr: Install optional dependencies - - run_tests_selective: - file_or_dir: test/test_prototype_*.py - unittest_extended: docker: - image: cimg/python:3.7 diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index a348faaa866..744e7de7ab4 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -357,20 +357,6 @@ jobs: - run_tests_selective: file_or_dir: test/test_onnx.py - unittest_prototype: - docker: - - image: cimg/python:3.7 - resource_class: xlarge - steps: - - checkout - - install_torchvision - - install_prototype_dependencies - - pip_install: - args: scipy pycocotools h5py - descr: Install optional dependencies - - run_tests_selective: - file_or_dir: test/test_prototype_*.py - unittest_extended: docker: - image: cimg/python:3.7 diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml new file mode 100644 index 00000000000..1c8695c8f06 --- /dev/null +++ b/.github/workflows/prototype-tests.yml @@ -0,0 +1,42 @@ +name: tests + +on: + pull_request: + +jobs: + prototype: + strategy: + matrix: + os: + - ubuntu-latest + - windows-latest + - macos-latest + + runs-on: ${{ matrix.os }} + + steps: + - name: Set up python + uses: actions/setup-python@v3 + with: + python-version: 3.7 + + - name: Upgrade system packages + run: python -m pip install --upgrade pip setuptools wheel + + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install PyTorch nightly builds + run: pip install --progress-bar=off --pre torch torchdata --extra-index-url https://download.pytorch.org/whl/nightly/cpu/ + + - name: Install torchvision + run: pip install --progress-bar=off --no-build-isolation --editable . + + - name: Install other prototype dependencies + run: pip install --progress-bar=off scipy pycocotools h5py iopath + + - name: Install test requirements + run: pip install --progress-bar=off pytest pytest-mock + + - name: Run prototype tests + run: pytest --junitxml=test-results/junit.xml -v --durations 20 test/test_prototype_*.py From b61e66fa67cd51137b5e27621fa49af0bbd3fe8c Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Apr 2022 13:53:17 +0200 Subject: [PATCH 02/17] fix CircleCI config --- .circleci/config.yml | 1 - .circleci/config.yml.in | 1 - 2 files changed, 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e12eea224ed..ece9d1a7b97 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1620,7 +1620,6 @@ workflows: jobs: - unittest_torchhub - unittest_onnx - - unittest_prototype - unittest_extended - unittest_linux_cpu: cu_version: cpu diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 744e7de7ab4..63a41c3d39a 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -1106,7 +1106,6 @@ workflows: jobs: - unittest_torchhub - unittest_onnx - - unittest_prototype - unittest_extended {{ unittest_workflows() }} From 01e98acf6b96651b515444fe977ead6eae34077b Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Apr 2022 14:01:28 +0200 Subject: [PATCH 03/17] cleanup --- .github/workflows/prototype-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index 1c8695c8f06..77513de2a62 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -39,4 +39,4 @@ jobs: run: pip install --progress-bar=off pytest pytest-mock - name: Run prototype tests - run: pytest --junitxml=test-results/junit.xml -v --durations 20 test/test_prototype_*.py + run: pytest -v --durations 20 test/test_prototype_*.py From 9aa270bdb6f03b0b55956bb4c690c9e061d1ff06 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Apr 2022 14:01:44 +0200 Subject: [PATCH 04/17] use 3.8 as base to surface errors --- .github/workflows/prototype-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index 77513de2a62..7780eaecf84 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -18,7 +18,7 @@ jobs: - name: Set up python uses: actions/setup-python@v3 with: - python-version: 3.7 + python-version: 3.8 - name: Upgrade system packages run: python -m pip install --upgrade pip setuptools wheel From 6b06774688db2e33c529e22dda946548a1e27cad Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Apr 2022 14:03:29 +0200 Subject: [PATCH 05/17] try using bash explicitly --- .github/workflows/prototype-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index 7780eaecf84..22768c22518 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -39,4 +39,5 @@ jobs: run: pip install --progress-bar=off pytest pytest-mock - name: Run prototype tests + shell: bash run: pytest -v --durations 20 test/test_prototype_*.py From c50745de7b3a83e01aaf23be10caa88897b5457f Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Apr 2022 14:12:04 +0200 Subject: [PATCH 06/17] cleanup --- .github/workflows/prototype-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index 22768c22518..2dae0b9be5b 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -6,6 +6,7 @@ on: jobs: prototype: strategy: + fail-fast: false matrix: os: - ubuntu-latest @@ -40,4 +41,4 @@ jobs: - name: Run prototype tests shell: bash - run: pytest -v --durations 20 test/test_prototype_*.py + run: pytest --durations 20 test/test_prototype_*.py From 8421129eb7095117d0ebc3d609dea248e6807cf3 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Apr 2022 14:18:40 +0200 Subject: [PATCH 07/17] try test reports --- .github/workflows/prototype-tests.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index 2dae0b9be5b..7a036cecf52 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -41,4 +41,13 @@ jobs: - name: Run prototype tests shell: bash - run: pytest --durations 20 test/test_prototype_*.py + run: pytest --junitxml=test-results/junit.xml --durations 20 test/test_prototype_*.py + + - name: Report test results + uses: dorny/test-reporter@v1.5.0 + if: always() + with: + name: TODO change this name + path: test-results/junit.xml + reporter: java-junit + fail-on-error: false From a6c911f654419e1f8a840d26830768db2d9b5148 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Apr 2022 14:35:28 +0200 Subject: [PATCH 08/17] debug --- .github/workflows/prototype-tests.yml | 26 +++++++++++++++----------- test_foo.py | 24 ++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 11 deletions(-) create mode 100644 test_foo.py diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index 7a036cecf52..51aaea8d3d6 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -27,27 +27,31 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - - name: Install PyTorch nightly builds - run: pip install --progress-bar=off --pre torch torchdata --extra-index-url https://download.pytorch.org/whl/nightly/cpu/ - - - name: Install torchvision - run: pip install --progress-bar=off --no-build-isolation --editable . - - - name: Install other prototype dependencies - run: pip install --progress-bar=off scipy pycocotools h5py iopath +# - name: Install PyTorch nightly builds +# run: pip install --progress-bar=off --pre torch torchdata --extra-index-url https://download.pytorch.org/whl/nightly/cpu/ +# +# - name: Install torchvision +# run: pip install --progress-bar=off --no-build-isolation --editable . +# +# - name: Install other prototype dependencies +# run: pip install --progress-bar=off scipy pycocotools h5py iopath - name: Install test requirements run: pip install --progress-bar=off pytest pytest-mock +# - name: Run prototype tests +# shell: bash +# run: pytest --junitxml=test-results/junit.xml --durations 20 test/test_prototype_*.py + - name: Run prototype tests shell: bash - run: pytest --junitxml=test-results/junit.xml --durations 20 test/test_prototype_*.py + run: pytest --junitxml=test-results/junit.xml --durations 20 test_foo.py - name: Report test results uses: dorny/test-reporter@v1.5.0 if: always() with: - name: TODO change this name - path: test-results/junit.xml + name: prototype tests + path: ./test-results/junit.xml reporter: java-junit fail-on-error: false diff --git a/test_foo.py b/test_foo.py new file mode 100644 index 00000000000..93d4988c83b --- /dev/null +++ b/test_foo.py @@ -0,0 +1,24 @@ +import pytest + + +def test_success(): + assert True + + +def test_failure(): + assert False + + +@pytest.mark.skip +def test_skip(): + pass + + +@pytest.mark.xfail +def test_xfail(): + assert False + + +@pytest.mark.xfail +def test_pass(): + assert True From 55996a152d8e2a0d1153cf672cec7ed26373060e Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Apr 2022 14:36:53 +0200 Subject: [PATCH 09/17] disable CircleCI --- .circleci/config.yml | 3303 ------------------------------------------ 1 file changed, 3303 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index ece9d1a7b97..00000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,3303 +0,0 @@ -version: 2.1 - -# How to test the Linux jobs: -# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ -# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7 -# - Replace binary_linux_wheel_py3.7 with the name of the job you want to test. -# Job names are 'name:' key. - -executors: - windows-cpu: - machine: - resource_class: windows.xlarge - image: windows-server-2019-vs2019:stable - shell: bash.exe - - windows-gpu: - machine: - resource_class: windows.gpu.nvidia.medium - image: windows-server-2019-nvidia:stable - shell: bash.exe - -commands: - checkout_merge: - description: "checkout merge branch" - steps: - - checkout -# - run: -# name: Checkout merge branch -# command: | -# set -ex -# BRANCH=$(git rev-parse --abbrev-ref HEAD) -# if [[ "$BRANCH" != "main" ]]; then -# git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH} -# git checkout "merged/$CIRCLE_BRANCH" -# fi - designate_upload_channel: - description: "inserts the correct upload channel into ${BASH_ENV}" - steps: - - run: - name: adding UPLOAD_CHANNEL to BASH_ENV - command: | - our_upload_channel=nightly - # On tags upload to test instead - if [[ -n "${CIRCLE_TAG}" ]]; then - our_upload_channel=test - fi - echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} - - brew_update: - description: "Update Homebrew and install base formulae" - steps: - - run: - name: Update Homebrew - no_output_timeout: "10m" - command: | - set -ex - - # Update repositories manually. - # Running `brew update` produces a comparison between the - # current checkout and the updated checkout, which takes a - # very long time because the existing checkout is 2y old. - for path in $(find /usr/local/Homebrew -type d -name .git) - do - cd $path/.. - git fetch --depth=1 origin - git reset --hard origin/master - done - - export HOMEBREW_NO_AUTO_UPDATE=1 - - # Install expect and moreutils so that we can call `unbuffer` and `ts`. - # moreutils installs a `parallel` executable by default, which conflicts - # with the executable from the GNU `parallel`, so we must unlink GNU - # `parallel` first, and relink it afterwards. - brew install coreutils - brew unlink parallel - brew install moreutils - brew link parallel --overwrite - brew install expect - - brew_install: - description: "Install Homebrew formulae" - parameters: - formulae: - type: string - default: "" - steps: - - run: - name: Install << parameters.formulae >> - no_output_timeout: "10m" - command: | - set -ex - export HOMEBREW_NO_AUTO_UPDATE=1 - brew install << parameters.formulae >> - - run_brew_for_ios_build: - steps: - - brew_update - - brew_install: - formulae: libtool - - apt_install: - parameters: - args: - type: string - descr: - type: string - default: "" - update: - type: boolean - default: true - steps: - - run: - name: > - <<^ parameters.descr >> apt install << parameters.args >> <> - <<# parameters.descr >> << parameters.descr >> <> - command: | - <<# parameters.update >> sudo apt update -qy <> - sudo apt install << parameters.args >> - - pip_install: - parameters: - args: - type: string - descr: - type: string - default: "" - user: - type: boolean - default: true - steps: - - run: - name: > - <<^ parameters.descr >> pip install << parameters.args >> <> - <<# parameters.descr >> << parameters.descr >> <> - command: > - pip install - <<# parameters.user >> --user <> - --progress-bar=off - << parameters.args >> - - install_torchvision: - parameters: - editable: - type: boolean - default: true - steps: - - pip_install: - args: --pre torch --extra-index-url https://download.pytorch.org/whl/nightly/cpu - descr: Install PyTorch from nightly releases - - pip_install: - args: --no-build-isolation <<# parameters.editable >> --editable <> . - descr: Install torchvision <<# parameters.editable >> in editable mode <> - - install_prototype_dependencies: - steps: - - pip_install: - args: iopath - descr: Install third-party dependencies - - pip_install: - args: --pre torchdata --extra-index-url https://download.pytorch.org/whl/nightly/cpu - descr: Install torchdata from nightly releases - - # Most of the test suite is handled by the `unittest` jobs, with completely different workflow and setup. - # This command can be used if only a selection of tests need to be run, for ad-hoc files. - run_tests_selective: - parameters: - file_or_dir: - type: string - steps: - - run: - name: Install test utilities - command: pip install --progress-bar=off pytest pytest-mock - - run: - name: Run tests - command: pytest --junitxml=test-results/junit.xml -v --durations 20 <> - - store_test_results: - path: test-results - - download_model_weights: - parameters: - extract_roots: - type: string - default: "torchvision/models" - background: - type: boolean - default: true - steps: - - apt_install: - args: parallel wget - descr: Install download utilitites - - run: - name: Download model weights - background: << parameters.background >> - command: | - mkdir -p ~/.cache/torch/hub/checkpoints - python scripts/collect_model_urls.py << parameters.extract_roots >> \ - | parallel -j0 'wget --no-verbose -O ~/.cache/torch/hub/checkpoints/`basename {}` {}\?source=ci' - -binary_common: &binary_common - parameters: - # Edit these defaults to do a release - build_version: - description: "version number of release binary; by default, build a nightly" - type: string - default: "" - pytorch_version: - description: "PyTorch version to build against; by default, use a nightly" - type: string - default: "" - # Don't edit these - python_version: - description: "Python version to build against (e.g., 3.7)" - type: string - cu_version: - description: "CUDA version to build against, in CU format (e.g., cpu or cu100)" - type: string - default: "cpu" - unicode_abi: - description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" - type: string - default: "" - wheel_docker_image: - description: "Wheel only: what docker image to use" - type: string - default: "pytorch/manylinux-cuda102" - conda_docker_image: - description: "Conda only: what docker image to use" - type: string - default: "pytorch/conda-builder:cpu" - environment: - PYTHON_VERSION: << parameters.python_version >> - PYTORCH_VERSION: << parameters.pytorch_version >> - UNICODE_ABI: << parameters.unicode_abi >> - CU_VERSION: << parameters.cu_version >> - -torchvision_ios_params: &torchvision_ios_params - parameters: - build_environment: - type: string - default: "" - ios_arch: - type: string - default: "" - ios_platform: - type: string - default: "" - environment: - BUILD_ENVIRONMENT: << parameters.build_environment >> - IOS_ARCH: << parameters.ios_arch >> - IOS_PLATFORM: << parameters.ios_platform >> - -torchvision_android_params: &torchvision_android_params - parameters: - build_environment: - type: string - default: "" - environment: - BUILD_ENVIRONMENT: << parameters.build_environment >> - -smoke_test_common: &smoke_test_common - <<: *binary_common - docker: - - image: torchvision/smoke_test:latest - -jobs: - circleci_consistency: - docker: - - image: cimg/python:3.7 - steps: - - checkout - - pip_install: - args: jinja2 pyyaml - - run: - name: Check CircleCI config consistency - command: | - python .circleci/regenerate.py - git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) - - lint_python_and_config: - docker: - - image: cimg/python:3.7 - steps: - - checkout - - pip_install: - args: pre-commit - descr: Install lint utilities - - run: - name: Install pre-commit hooks - command: pre-commit install-hooks - - run: - name: Lint Python code and config files - command: pre-commit run --all-files - - run: - name: Required lint modifications - when: on_fail - command: git --no-pager diff - - lint_c: - docker: - - image: cimg/python:3.7 - steps: - - apt_install: - args: libtinfo5 - descr: Install additional system libraries - - checkout - - run: - name: Install lint utilities - command: | - curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o clang-format - chmod +x clang-format - sudo mv clang-format /opt/clang-format - - run: - name: Lint C code - command: ./.circleci/unittest/linux/scripts/run-clang-format.py -r torchvision/csrc --clang-format-executable /opt/clang-format - - run: - name: Required lint modifications - when: on_fail - command: git --no-pager diff - - type_check_python: - docker: - - image: cimg/python:3.7 - steps: - - apt_install: - args: libturbojpeg-dev - descr: Install additional system libraries - - checkout - - install_torchvision: - editable: true - - install_prototype_dependencies - - pip_install: - args: mypy - descr: Install Python type check utilities - - run: - name: Check Python types statically - command: mypy --install-types --non-interactive --config-file mypy.ini - - unittest_torchhub: - docker: - - image: cimg/python:3.7 - steps: - - checkout - - install_torchvision - - run_tests_selective: - file_or_dir: test/test_hub.py - - unittest_onnx: - docker: - - image: cimg/python:3.7 - steps: - - checkout - - install_torchvision - - pip_install: - args: onnx onnxruntime - descr: Install ONNX - - run_tests_selective: - file_or_dir: test/test_onnx.py - - unittest_extended: - docker: - - image: cimg/python:3.7 - resource_class: xlarge - steps: - - checkout - - download_model_weights - - install_torchvision - - run: - name: Enable extended tests - command: echo 'export PYTORCH_TEST_WITH_EXTENDED=1' >> $BASH_ENV - - run_tests_selective: - file_or_dir: test/test_extended_*.py - - binary_linux_wheel: - <<: *binary_common - docker: - - image: << parameters.wheel_docker_image >> - resource_class: 2xlarge+ - steps: - - checkout_merge - - designate_upload_channel - - run: packaging/build_wheel.sh - - store_artifacts: - path: dist - - persist_to_workspace: - root: dist - paths: - - "*" - - binary_linux_conda: - <<: *binary_common - docker: - - image: "<< parameters.conda_docker_image >>" - resource_class: 2xlarge+ - steps: - - checkout_merge - - designate_upload_channel - - run: packaging/build_conda.sh - - store_artifacts: - path: /opt/conda/conda-bld/linux-64 - - persist_to_workspace: - root: /opt/conda/conda-bld/linux-64 - paths: - - "*" - - store_test_results: - path: build_results/ - - binary_win_conda: - <<: *binary_common - executor: windows-cpu - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Build conda packages - no_output_timeout: 20m - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - packaging/windows/internal/cuda_install.bat - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate base - conda install -yq conda-build "conda-package-handling!=1.5.0" - # cudatoolkit >= 11 isn't available for windows in the nvidia channel - if [[ "${CU_VERSION}" =~ cu11.* ]]; then - export CONDA_CHANNEL_FLAGS="-c conda-forge" - fi - packaging/build_conda.sh - rm /C/tools/miniconda3/conda-bld/win-64/vs${VC_YEAR}*.tar.bz2 - - store_artifacts: - path: C:/tools/miniconda3/conda-bld/win-64 - - persist_to_workspace: - root: C:/tools/miniconda3/conda-bld/win-64 - paths: - - "*" - - store_test_results: - path: build_results/ - - binary_win_wheel: - <<: *binary_common - executor: windows-cpu - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Build wheel packages - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - packaging/windows/internal/cuda_install.bat - packaging/build_wheel.sh - - store_artifacts: - path: dist - - persist_to_workspace: - root: dist - paths: - - "*" - - store_test_results: - path: build_results/ - - binary_macos_wheel: - <<: *binary_common - macos: - xcode: "12.0" - steps: - - checkout_merge - - designate_upload_channel - - run: - # Cannot easily deduplicate this as source'ing activate - # will set environment variables which we need to propagate - # to build_wheel.sh - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - packaging/build_wheel.sh - - store_artifacts: - path: dist - - persist_to_workspace: - root: dist - paths: - - "*" - - binary_ios_build: - <<: *torchvision_ios_params - macos: - xcode: "12.0" - steps: - - attach_workspace: - at: ~/workspace - - checkout - - run_brew_for_ios_build - - run: - name: Build - no_output_timeout: "1h" - command: | - script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_build.sh" - cat "$script" - source "$script" - - persist_to_workspace: - root: /Users/distiller/workspace/ - paths: ios - - binary_ios_upload: - <<: *torchvision_ios_params - macos: - xcode: "12.0" - steps: - - attach_workspace: - at: ~/workspace - - checkout - - run_brew_for_ios_build - - run: - name: Upload - no_output_timeout: "1h" - command: | - script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_upload.sh" - cat "$script" - source "$script" - - binary_android_build: - <<: *torchvision_android_params - docker: - - image: cimg/android:2021.08-ndk - resource_class: xlarge - steps: - - attach_workspace: - at: ~/workspace - - checkout - - run: - name: Build - no_output_timeout: "1h" - command: | - script="/home/circleci/project/.circleci/unittest/android/scripts/binary_android_build.sh" - cat "$script" - source "$script" - - store_artifacts: - path: ~/workspace/artifacts - - binary_android_upload: - <<: *torchvision_android_params - docker: - - image: cimg/android:2021.08-ndk - resource_class: xlarge - steps: - - attach_workspace: - at: ~/workspace - - checkout - - run: - name: Upload - no_output_timeout: "1h" - command: | - script="/home/circleci/project/.circleci/unittest/android/scripts/binary_android_upload.sh" - cat "$script" - source "$script" - - binary_macos_conda: - <<: *binary_common - macos: - xcode: "12.0" - steps: - - checkout_merge - - designate_upload_channel - - run: - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - conda install -yq conda-build - packaging/build_conda.sh - - store_artifacts: - path: /Users/distiller/miniconda3/conda-bld/osx-64 - - persist_to_workspace: - root: /Users/distiller/miniconda3/conda-bld/osx-64 - paths: - - "*" - - store_test_results: - path: build_results/ - - # Requires org-member context - binary_conda_upload: - docker: - - image: continuumio/miniconda - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - run: - command: | - # Prevent credential from leaking - conda install -yq anaconda-client - set -x - anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force - - # Requires org-member context - binary_wheel_upload: - parameters: - subfolder: - description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)" - type: string - docker: - - image: cimg/python:3.7 - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - checkout - - pip_install: - args: awscli - - run: - command: | - export PATH="$HOME/.local/bin:$PATH" - # Prevent credential from leaking - set +x - export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" - export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" - set -x - for pkg in ~/workspace/*.whl; do - aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read - done - - smoke_test_linux_conda: - <<: *smoke_test_common - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - run: - name: install binaries - command: | - set -x - source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - conda install -v -y -c pytorch-nightly pytorch - conda install -v -y $(ls ~/workspace/torchvision*.tar.bz2) - - run: - name: smoke test - command: | - source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - python -c "import torchvision" - - smoke_test_linux_pip: - <<: *smoke_test_common - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - run: - name: install binaries - command: | - set -x - source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - - pip_install: - args: $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html - - run: - name: smoke test - command: | - source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - python -c "import torchvision" - - smoke_test_docker_image_build: - machine: - image: ubuntu-2004:202104-01 - resource_class: large - environment: - image_name: torchvision/smoke_test - steps: - - checkout - - designate_upload_channel - - run: - name: Build and push Docker image - no_output_timeout: "1h" - command: | - set +x - echo "${DOCKER_HUB_TOKEN}" | docker login --username "${DOCKER_HUB_USERNAME}" --password-stdin - set -x - cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID} - docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} ${image_name}:latest - docker push ${image_name}:${CIRCLE_WORKFLOW_ID} - docker push ${image_name}:latest - - smoke_test_win_conda: - <<: *binary_common - executor: - name: windows-cpu - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - run: - name: install binaries - command: | - set -x - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda env remove -n python${PYTHON_VERSION} || true - conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} - conda activate python${PYTHON_VERSION} - conda install "Pillow>=5.3.0,!=8.3.*" - conda install -v -y -c pytorch-nightly pytorch - conda install -v -y $(ls ~/workspace/torchvision*.tar.bz2) - - run: - name: smoke test - command: | - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate python${PYTHON_VERSION} - python -c "import torchvision" - - smoke_test_win_pip: - <<: *binary_common - executor: - name: windows-cpu - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - run: - name: install binaries - command: | - set -x - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} - conda activate python${PYTHON_VERSION} - - pip_install: - args: $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html - - run: - name: smoke test - command: | - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate python${PYTHON_VERSION} - python -c "import torchvision" - - unittest_linux_cpu: - <<: *binary_common - docker: - - image: "pytorch/manylinux-cuda102" - resource_class: 2xlarge+ - steps: - - checkout - - designate_upload_channel - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - restore_cache: - - keys: - - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - - run: - name: Setup - command: .circleci/unittest/linux/scripts/setup_env.sh - - save_cache: - - key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - paths: - - conda - - env - - run: - name: Install torchvision - command: .circleci/unittest/linux/scripts/install.sh - - run: - name: Run tests - command: .circleci/unittest/linux/scripts/run_test.sh - - run: - name: Post process - command: .circleci/unittest/linux/scripts/post_process.sh - - store_test_results: - path: test-results - - unittest_linux_gpu: - <<: *binary_common - machine: - image: ubuntu-2004-cuda-11.4:202110-01 - resource_class: gpu.nvidia.medium - environment: - image_name: "pytorch/manylinux-cuda102" - PYTHON_VERSION: << parameters.python_version >> - steps: - - checkout - - designate_upload_channel - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - restore_cache: - - keys: - - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - - run: - name: Setup - command: docker run -e PYTHON_VERSION -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh - - save_cache: - - key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - paths: - - conda - - env - - run: - # Here we create an envlist file that contains some env variables that we want the docker container to be aware of. - # Normally, the CIRCLECI variable is set and available on all CI workflows: https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables. - # They're avaiable in all the other workflows (OSX and Windows). - # But here, we're running the unittest_linux_gpu workflows in a docker container, where those variables aren't accessible. - # So instead we dump the variables we need in env.list and we pass that file when invoking "docker run". - name: export CIRCLECI env var - command: echo "CIRCLECI=true" >> ./env.list - - run: - name: Install torchvision - command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL -e CU_VERSION "${image_name}" .circleci/unittest/linux/scripts/install.sh - - run: - name: Run tests - command: docker run --env-file ./env.list -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh - - run: - name: Post Process - command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/post_process.sh - - store_test_results: - path: test-results - - unittest_windows_cpu: - <<: *binary_common - executor: - name: windows-cpu - steps: - - checkout - - designate_upload_channel - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - restore_cache: - - keys: - - env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - - run: - name: Setup - command: .circleci/unittest/windows/scripts/setup_env.sh - - save_cache: - - key: env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - paths: - - conda - - env - - run: - name: Install torchvision - command: .circleci/unittest/windows/scripts/install.sh - - run: - name: Run tests - command: .circleci/unittest/windows/scripts/run_test.sh - - run: - name: Post process - command: .circleci/unittest/windows/scripts/post_process.sh - - store_test_results: - path: test-results - - unittest_windows_gpu: - <<: *binary_common - executor: - name: windows-gpu - environment: - CUDA_VERSION: "11.3" - PYTHON_VERSION: << parameters.python_version >> - steps: - - checkout - - designate_upload_channel - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - restore_cache: - - keys: - - env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - - run: - name: Setup - command: .circleci/unittest/windows/scripts/setup_env.sh - - save_cache: - - key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - paths: - - conda - - env - - run: - name: Install CUDA - command: packaging/windows/internal/cuda_install.bat - - run: - name: Update CUDA driver - command: packaging/windows/internal/driver_update.bat - - run: - name: Install torchvision - command: .circleci/unittest/windows/scripts/install.sh - - run: - name: Run tests - command: .circleci/unittest/windows/scripts/run_test.sh - - run: - name: Post process - command: .circleci/unittest/windows/scripts/post_process.sh - - store_test_results: - path: test-results - - unittest_macos_cpu: - <<: *binary_common - macos: - xcode: "12.0" - resource_class: large - steps: - - checkout - - designate_upload_channel - - run: - name: Install wget - command: HOMEBREW_NO_AUTO_UPDATE=1 brew install wget - # Disable brew auto update which is very slow - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - restore_cache: - - keys: - - env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - - run: - name: Setup - command: .circleci/unittest/linux/scripts/setup_env.sh - - save_cache: - - key: env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - paths: - - conda - - env - - run: - name: Install torchvision - command: .circleci/unittest/linux/scripts/install.sh - - run: - name: Run tests - command: .circleci/unittest/linux/scripts/run_test.sh - - run: - name: Post process - command: .circleci/unittest/linux/scripts/post_process.sh - - store_test_results: - path: test-results - - cmake_linux_cpu: - <<: *binary_common - docker: - - image: "pytorch/manylinux-cuda102" - resource_class: 2xlarge+ - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Setup conda - command: .circleci/unittest/linux/scripts/setup_env.sh - - run: packaging/build_cmake.sh - - cmake_linux_gpu: - <<: *binary_common - machine: - image: ubuntu-2004-cuda-11.4:202110-01 - resource_class: gpu.nvidia.small - environment: - PYTHON_VERSION: << parameters.python_version >> - PYTORCH_VERSION: << parameters.pytorch_version >> - UNICODE_ABI: << parameters.unicode_abi >> - CU_VERSION: << parameters.cu_version >> - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Setup conda - command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh - - run: - name: Build torchvision C++ distribution and test - command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh - - cmake_macos_cpu: - <<: *binary_common - macos: - xcode: "12.0" - steps: - - checkout_merge - - designate_upload_channel - - run: - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - conda install -yq conda-build cmake - packaging/build_cmake.sh - - cmake_windows_cpu: - <<: *binary_common - executor: - name: windows-cpu - steps: - - checkout_merge - - designate_upload_channel - - run: - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - packaging/build_cmake.sh - - cmake_windows_gpu: - <<: *binary_common - executor: - name: windows-gpu - steps: - - checkout_merge - - designate_upload_channel - - run: - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - packaging/windows/internal/cuda_install.bat - packaging/build_cmake.sh - - build_docs: - <<: *binary_common - docker: - - image: cimg/python:3.7 - resource_class: 2xlarge+ - steps: - - attach_workspace: - at: ~/workspace - - checkout - - download_model_weights - - run: - name: Setup - command: .circleci/unittest/linux/scripts/setup_env.sh - - designate_upload_channel - - run: - name: Install torchvision - command: .circleci/unittest/linux/scripts/install.sh - - run: - name: Build docs - command: | - set -ex - # turn v1.12.0rc3 into 1.12.0 - tag=$(echo $CIRCLE_TAG | sed -e 's/v*\([0-9.]*\).*/\1/') - VERSION=${tag:-main} - eval "$(./conda/bin/conda shell.bash hook)" - conda activate ./env - pushd docs - pip install --progress-bar=off -r requirements.txt - make html - popd - - persist_to_workspace: - root: ./ - paths: - - "*" - - store_artifacts: - path: ./docs/build/html - destination: docs - - upload_docs: - <<: *binary_common - docker: - - image: "pytorch/manylinux-cuda100" - resource_class: 2xlarge+ - steps: - - attach_workspace: - at: ~/workspace - - run: - name: Generate netrc - command: | - # set credentials for https pushing - # requires the org-member context - cat > ~/.netrc \< Date: Thu, 28 Apr 2022 14:39:43 +0200 Subject: [PATCH 10/17] add write permissions for write --- .github/workflows/prototype-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index 51aaea8d3d6..06c55a98524 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -3,6 +3,9 @@ name: tests on: pull_request: +permissions: + checks: write + jobs: prototype: strategy: From 290b910693f6d7d76793b195296760df73da6e57 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Apr 2022 14:41:53 +0200 Subject: [PATCH 11/17] expand permissions --- .github/workflows/prototype-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index 06c55a98524..f740d11f8d3 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -5,6 +5,8 @@ on: permissions: checks: write + id-token: write + contents: read jobs: prototype: From d93a3420e78898e9698d571fa1c3baf8825f3fa4 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Apr 2022 14:45:45 +0200 Subject: [PATCH 12/17] try move permissions on job rather than global --- .github/workflows/prototype-tests.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index f740d11f8d3..d9ac11172b7 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -3,13 +3,11 @@ name: tests on: pull_request: -permissions: - checks: write - id-token: write - contents: read - jobs: prototype: + permissions: + checks: write + strategy: fail-fast: false matrix: From 01dd40c2db92810a0c8494f7ccd49c0bab4256ac Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Apr 2022 14:47:22 +0200 Subject: [PATCH 13/17] debug --- .github/workflows/prototype-tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index d9ac11172b7..fc619a317cb 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -50,6 +50,10 @@ jobs: shell: bash run: pytest --junitxml=test-results/junit.xml --durations 20 test_foo.py + - name: Debug + shell: bash + run: cat ./test-results/junit.xml + - name: Report test results uses: dorny/test-reporter@v1.5.0 if: always() From bcf529a6832d353eb0cb73ba66e81937756157df Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Apr 2022 14:48:26 +0200 Subject: [PATCH 14/17] always debug --- .github/workflows/prototype-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index fc619a317cb..4c89eca0d0f 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -51,6 +51,7 @@ jobs: run: pytest --junitxml=test-results/junit.xml --durations 20 test_foo.py - name: Debug + if: always() shell: bash run: cat ./test-results/junit.xml From f7d673559f5fe0dbfbde6f1b59181d7b713a296d Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Apr 2022 14:51:16 +0200 Subject: [PATCH 15/17] maximum permissions --- .github/workflows/prototype-tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index 4c89eca0d0f..cca87bcb2b9 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -5,8 +5,7 @@ on: jobs: prototype: - permissions: - checks: write + permissions: write-all strategy: fail-fast: false From 5a508f19d6eb2622ce52f3906c63cb0823bed1d3 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Apr 2022 14:58:31 +0200 Subject: [PATCH 16/17] cleanup --- .github/workflows/prototype-tests.yml | 40 +++++++-------------------- 1 file changed, 10 insertions(+), 30 deletions(-) diff --git a/.github/workflows/prototype-tests.yml b/.github/workflows/prototype-tests.yml index cca87bcb2b9..53e66b18eaa 100644 --- a/.github/workflows/prototype-tests.yml +++ b/.github/workflows/prototype-tests.yml @@ -5,15 +5,13 @@ on: jobs: prototype: - permissions: write-all - strategy: - fail-fast: false matrix: os: - ubuntu-latest - windows-latest - macos-latest + fail-fast: false runs-on: ${{ matrix.os }} @@ -29,36 +27,18 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 -# - name: Install PyTorch nightly builds -# run: pip install --progress-bar=off --pre torch torchdata --extra-index-url https://download.pytorch.org/whl/nightly/cpu/ -# -# - name: Install torchvision -# run: pip install --progress-bar=off --no-build-isolation --editable . -# -# - name: Install other prototype dependencies -# run: pip install --progress-bar=off scipy pycocotools h5py iopath + - name: Install PyTorch nightly builds + run: pip install --progress-bar=off --pre torch torchdata --extra-index-url https://download.pytorch.org/whl/nightly/cpu/ + + - name: Install torchvision + run: pip install --progress-bar=off --no-build-isolation --editable . + + - name: Install other prototype dependencies + run: pip install --progress-bar=off scipy pycocotools h5py iopath - name: Install test requirements run: pip install --progress-bar=off pytest pytest-mock -# - name: Run prototype tests -# shell: bash -# run: pytest --junitxml=test-results/junit.xml --durations 20 test/test_prototype_*.py - - name: Run prototype tests shell: bash - run: pytest --junitxml=test-results/junit.xml --durations 20 test_foo.py - - - name: Debug - if: always() - shell: bash - run: cat ./test-results/junit.xml - - - name: Report test results - uses: dorny/test-reporter@v1.5.0 - if: always() - with: - name: prototype tests - path: ./test-results/junit.xml - reporter: java-junit - fail-on-error: false + run: pytest --junitxml=test-results/junit.xml --durations 20 test/test_prototype_*.py From a5213c6d74fba4ed4478822c1a04bba186e323d1 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 28 Apr 2022 15:13:11 +0200 Subject: [PATCH 17/17] cleanup --- .circleci/config.yml | 3303 +++++++++++++++++++++++++ .github/workflows/prototype-tests.yml | 4 +- test_foo.py | 24 - 3 files changed, 3305 insertions(+), 26 deletions(-) create mode 100644 .circleci/config.yml delete mode 100644 test_foo.py diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000000..ece9d1a7b97 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,3303 @@ +version: 2.1 + +# How to test the Linux jobs: +# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ +# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7 +# - Replace binary_linux_wheel_py3.7 with the name of the job you want to test. +# Job names are 'name:' key. + +executors: + windows-cpu: + machine: + resource_class: windows.xlarge + image: windows-server-2019-vs2019:stable + shell: bash.exe + + windows-gpu: + machine: + resource_class: windows.gpu.nvidia.medium + image: windows-server-2019-nvidia:stable + shell: bash.exe + +commands: + checkout_merge: + description: "checkout merge branch" + steps: + - checkout +# - run: +# name: Checkout merge branch +# command: | +# set -ex +# BRANCH=$(git rev-parse --abbrev-ref HEAD) +# if [[ "$BRANCH" != "main" ]]; then +# git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH} +# git checkout "merged/$CIRCLE_BRANCH" +# fi + designate_upload_channel: + description: "inserts the correct upload channel into ${BASH_ENV}" + steps: + - run: + name: adding UPLOAD_CHANNEL to BASH_ENV + command: | + our_upload_channel=nightly + # On tags upload to test instead + if [[ -n "${CIRCLE_TAG}" ]]; then + our_upload_channel=test + fi + echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} + + brew_update: + description: "Update Homebrew and install base formulae" + steps: + - run: + name: Update Homebrew + no_output_timeout: "10m" + command: | + set -ex + + # Update repositories manually. + # Running `brew update` produces a comparison between the + # current checkout and the updated checkout, which takes a + # very long time because the existing checkout is 2y old. + for path in $(find /usr/local/Homebrew -type d -name .git) + do + cd $path/.. + git fetch --depth=1 origin + git reset --hard origin/master + done + + export HOMEBREW_NO_AUTO_UPDATE=1 + + # Install expect and moreutils so that we can call `unbuffer` and `ts`. + # moreutils installs a `parallel` executable by default, which conflicts + # with the executable from the GNU `parallel`, so we must unlink GNU + # `parallel` first, and relink it afterwards. + brew install coreutils + brew unlink parallel + brew install moreutils + brew link parallel --overwrite + brew install expect + + brew_install: + description: "Install Homebrew formulae" + parameters: + formulae: + type: string + default: "" + steps: + - run: + name: Install << parameters.formulae >> + no_output_timeout: "10m" + command: | + set -ex + export HOMEBREW_NO_AUTO_UPDATE=1 + brew install << parameters.formulae >> + + run_brew_for_ios_build: + steps: + - brew_update + - brew_install: + formulae: libtool + + apt_install: + parameters: + args: + type: string + descr: + type: string + default: "" + update: + type: boolean + default: true + steps: + - run: + name: > + <<^ parameters.descr >> apt install << parameters.args >> <> + <<# parameters.descr >> << parameters.descr >> <> + command: | + <<# parameters.update >> sudo apt update -qy <> + sudo apt install << parameters.args >> + + pip_install: + parameters: + args: + type: string + descr: + type: string + default: "" + user: + type: boolean + default: true + steps: + - run: + name: > + <<^ parameters.descr >> pip install << parameters.args >> <> + <<# parameters.descr >> << parameters.descr >> <> + command: > + pip install + <<# parameters.user >> --user <> + --progress-bar=off + << parameters.args >> + + install_torchvision: + parameters: + editable: + type: boolean + default: true + steps: + - pip_install: + args: --pre torch --extra-index-url https://download.pytorch.org/whl/nightly/cpu + descr: Install PyTorch from nightly releases + - pip_install: + args: --no-build-isolation <<# parameters.editable >> --editable <> . + descr: Install torchvision <<# parameters.editable >> in editable mode <> + + install_prototype_dependencies: + steps: + - pip_install: + args: iopath + descr: Install third-party dependencies + - pip_install: + args: --pre torchdata --extra-index-url https://download.pytorch.org/whl/nightly/cpu + descr: Install torchdata from nightly releases + + # Most of the test suite is handled by the `unittest` jobs, with completely different workflow and setup. + # This command can be used if only a selection of tests need to be run, for ad-hoc files. + run_tests_selective: + parameters: + file_or_dir: + type: string + steps: + - run: + name: Install test utilities + command: pip install --progress-bar=off pytest pytest-mock + - run: + name: Run tests + command: pytest --junitxml=test-results/junit.xml -v --durations 20 <> + - store_test_results: + path: test-results + + download_model_weights: + parameters: + extract_roots: + type: string + default: "torchvision/models" + background: + type: boolean + default: true + steps: + - apt_install: + args: parallel wget + descr: Install download utilitites + - run: + name: Download model weights + background: << parameters.background >> + command: | + mkdir -p ~/.cache/torch/hub/checkpoints + python scripts/collect_model_urls.py << parameters.extract_roots >> \ + | parallel -j0 'wget --no-verbose -O ~/.cache/torch/hub/checkpoints/`basename {}` {}\?source=ci' + +binary_common: &binary_common + parameters: + # Edit these defaults to do a release + build_version: + description: "version number of release binary; by default, build a nightly" + type: string + default: "" + pytorch_version: + description: "PyTorch version to build against; by default, use a nightly" + type: string + default: "" + # Don't edit these + python_version: + description: "Python version to build against (e.g., 3.7)" + type: string + cu_version: + description: "CUDA version to build against, in CU format (e.g., cpu or cu100)" + type: string + default: "cpu" + unicode_abi: + description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" + type: string + default: "" + wheel_docker_image: + description: "Wheel only: what docker image to use" + type: string + default: "pytorch/manylinux-cuda102" + conda_docker_image: + description: "Conda only: what docker image to use" + type: string + default: "pytorch/conda-builder:cpu" + environment: + PYTHON_VERSION: << parameters.python_version >> + PYTORCH_VERSION: << parameters.pytorch_version >> + UNICODE_ABI: << parameters.unicode_abi >> + CU_VERSION: << parameters.cu_version >> + +torchvision_ios_params: &torchvision_ios_params + parameters: + build_environment: + type: string + default: "" + ios_arch: + type: string + default: "" + ios_platform: + type: string + default: "" + environment: + BUILD_ENVIRONMENT: << parameters.build_environment >> + IOS_ARCH: << parameters.ios_arch >> + IOS_PLATFORM: << parameters.ios_platform >> + +torchvision_android_params: &torchvision_android_params + parameters: + build_environment: + type: string + default: "" + environment: + BUILD_ENVIRONMENT: << parameters.build_environment >> + +smoke_test_common: &smoke_test_common + <<: *binary_common + docker: + - image: torchvision/smoke_test:latest + +jobs: + circleci_consistency: + docker: + - image: cimg/python:3.7 + steps: + - checkout + - pip_install: + args: jinja2 pyyaml + - run: + name: Check CircleCI config consistency + command: | + python .circleci/regenerate.py + git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) + + lint_python_and_config: + docker: + - image: cimg/python:3.7 + steps: + - checkout + - pip_install: + args: pre-commit + descr: Install lint utilities + - run: + name: Install pre-commit hooks + command: pre-commit install-hooks + - run: + name: Lint Python code and config files + command: pre-commit run --all-files + - run: + name: Required lint modifications + when: on_fail + command: git --no-pager diff + + lint_c: + docker: + - image: cimg/python:3.7 + steps: + - apt_install: + args: libtinfo5 + descr: Install additional system libraries + - checkout + - run: + name: Install lint utilities + command: | + curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o clang-format + chmod +x clang-format + sudo mv clang-format /opt/clang-format + - run: + name: Lint C code + command: ./.circleci/unittest/linux/scripts/run-clang-format.py -r torchvision/csrc --clang-format-executable /opt/clang-format + - run: + name: Required lint modifications + when: on_fail + command: git --no-pager diff + + type_check_python: + docker: + - image: cimg/python:3.7 + steps: + - apt_install: + args: libturbojpeg-dev + descr: Install additional system libraries + - checkout + - install_torchvision: + editable: true + - install_prototype_dependencies + - pip_install: + args: mypy + descr: Install Python type check utilities + - run: + name: Check Python types statically + command: mypy --install-types --non-interactive --config-file mypy.ini + + unittest_torchhub: + docker: + - image: cimg/python:3.7 + steps: + - checkout + - install_torchvision + - run_tests_selective: + file_or_dir: test/test_hub.py + + unittest_onnx: + docker: + - image: cimg/python:3.7 + steps: + - checkout + - install_torchvision + - pip_install: + args: onnx onnxruntime + descr: Install ONNX + - run_tests_selective: + file_or_dir: test/test_onnx.py + + unittest_extended: + docker: + - image: cimg/python:3.7 + resource_class: xlarge + steps: + - checkout + - download_model_weights + - install_torchvision + - run: + name: Enable extended tests + command: echo 'export PYTORCH_TEST_WITH_EXTENDED=1' >> $BASH_ENV + - run_tests_selective: + file_or_dir: test/test_extended_*.py + + binary_linux_wheel: + <<: *binary_common + docker: + - image: << parameters.wheel_docker_image >> + resource_class: 2xlarge+ + steps: + - checkout_merge + - designate_upload_channel + - run: packaging/build_wheel.sh + - store_artifacts: + path: dist + - persist_to_workspace: + root: dist + paths: + - "*" + + binary_linux_conda: + <<: *binary_common + docker: + - image: "<< parameters.conda_docker_image >>" + resource_class: 2xlarge+ + steps: + - checkout_merge + - designate_upload_channel + - run: packaging/build_conda.sh + - store_artifacts: + path: /opt/conda/conda-bld/linux-64 + - persist_to_workspace: + root: /opt/conda/conda-bld/linux-64 + paths: + - "*" + - store_test_results: + path: build_results/ + + binary_win_conda: + <<: *binary_common + executor: windows-cpu + steps: + - checkout_merge + - designate_upload_channel + - run: + name: Build conda packages + no_output_timeout: 20m + command: | + set -ex + source packaging/windows/internal/vc_install_helper.sh + packaging/windows/internal/cuda_install.bat + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" + conda activate base + conda install -yq conda-build "conda-package-handling!=1.5.0" + # cudatoolkit >= 11 isn't available for windows in the nvidia channel + if [[ "${CU_VERSION}" =~ cu11.* ]]; then + export CONDA_CHANNEL_FLAGS="-c conda-forge" + fi + packaging/build_conda.sh + rm /C/tools/miniconda3/conda-bld/win-64/vs${VC_YEAR}*.tar.bz2 + - store_artifacts: + path: C:/tools/miniconda3/conda-bld/win-64 + - persist_to_workspace: + root: C:/tools/miniconda3/conda-bld/win-64 + paths: + - "*" + - store_test_results: + path: build_results/ + + binary_win_wheel: + <<: *binary_common + executor: windows-cpu + steps: + - checkout_merge + - designate_upload_channel + - run: + name: Build wheel packages + command: | + set -ex + source packaging/windows/internal/vc_install_helper.sh + packaging/windows/internal/cuda_install.bat + packaging/build_wheel.sh + - store_artifacts: + path: dist + - persist_to_workspace: + root: dist + paths: + - "*" + - store_test_results: + path: build_results/ + + binary_macos_wheel: + <<: *binary_common + macos: + xcode: "12.0" + steps: + - checkout_merge + - designate_upload_channel + - run: + # Cannot easily deduplicate this as source'ing activate + # will set environment variables which we need to propagate + # to build_wheel.sh + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + packaging/build_wheel.sh + - store_artifacts: + path: dist + - persist_to_workspace: + root: dist + paths: + - "*" + + binary_ios_build: + <<: *torchvision_ios_params + macos: + xcode: "12.0" + steps: + - attach_workspace: + at: ~/workspace + - checkout + - run_brew_for_ios_build + - run: + name: Build + no_output_timeout: "1h" + command: | + script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_build.sh" + cat "$script" + source "$script" + - persist_to_workspace: + root: /Users/distiller/workspace/ + paths: ios + + binary_ios_upload: + <<: *torchvision_ios_params + macos: + xcode: "12.0" + steps: + - attach_workspace: + at: ~/workspace + - checkout + - run_brew_for_ios_build + - run: + name: Upload + no_output_timeout: "1h" + command: | + script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_upload.sh" + cat "$script" + source "$script" + + binary_android_build: + <<: *torchvision_android_params + docker: + - image: cimg/android:2021.08-ndk + resource_class: xlarge + steps: + - attach_workspace: + at: ~/workspace + - checkout + - run: + name: Build + no_output_timeout: "1h" + command: | + script="/home/circleci/project/.circleci/unittest/android/scripts/binary_android_build.sh" + cat "$script" + source "$script" + - store_artifacts: + path: ~/workspace/artifacts + + binary_android_upload: + <<: *torchvision_android_params + docker: + - image: cimg/android:2021.08-ndk + resource_class: xlarge + steps: + - attach_workspace: + at: ~/workspace + - checkout + - run: + name: Upload + no_output_timeout: "1h" + command: | + script="/home/circleci/project/.circleci/unittest/android/scripts/binary_android_upload.sh" + cat "$script" + source "$script" + + binary_macos_conda: + <<: *binary_common + macos: + xcode: "12.0" + steps: + - checkout_merge + - designate_upload_channel + - run: + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + conda install -yq conda-build + packaging/build_conda.sh + - store_artifacts: + path: /Users/distiller/miniconda3/conda-bld/osx-64 + - persist_to_workspace: + root: /Users/distiller/miniconda3/conda-bld/osx-64 + paths: + - "*" + - store_test_results: + path: build_results/ + + # Requires org-member context + binary_conda_upload: + docker: + - image: continuumio/miniconda + steps: + - attach_workspace: + at: ~/workspace + - designate_upload_channel + - run: + command: | + # Prevent credential from leaking + conda install -yq anaconda-client + set -x + anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force + + # Requires org-member context + binary_wheel_upload: + parameters: + subfolder: + description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)" + type: string + docker: + - image: cimg/python:3.7 + steps: + - attach_workspace: + at: ~/workspace + - designate_upload_channel + - checkout + - pip_install: + args: awscli + - run: + command: | + export PATH="$HOME/.local/bin:$PATH" + # Prevent credential from leaking + set +x + export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" + export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" + set -x + for pkg in ~/workspace/*.whl; do + aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read + done + + smoke_test_linux_conda: + <<: *smoke_test_common + steps: + - attach_workspace: + at: ~/workspace + - designate_upload_channel + - run: + name: install binaries + command: | + set -x + source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} + conda install -v -y -c pytorch-nightly pytorch + conda install -v -y $(ls ~/workspace/torchvision*.tar.bz2) + - run: + name: smoke test + command: | + source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} + python -c "import torchvision" + + smoke_test_linux_pip: + <<: *smoke_test_common + steps: + - attach_workspace: + at: ~/workspace + - designate_upload_channel + - run: + name: install binaries + command: | + set -x + source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} + - pip_install: + args: $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html + - run: + name: smoke test + command: | + source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} + python -c "import torchvision" + + smoke_test_docker_image_build: + machine: + image: ubuntu-2004:202104-01 + resource_class: large + environment: + image_name: torchvision/smoke_test + steps: + - checkout + - designate_upload_channel + - run: + name: Build and push Docker image + no_output_timeout: "1h" + command: | + set +x + echo "${DOCKER_HUB_TOKEN}" | docker login --username "${DOCKER_HUB_USERNAME}" --password-stdin + set -x + cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID} + docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} ${image_name}:latest + docker push ${image_name}:${CIRCLE_WORKFLOW_ID} + docker push ${image_name}:latest + + smoke_test_win_conda: + <<: *binary_common + executor: + name: windows-cpu + steps: + - attach_workspace: + at: ~/workspace + - designate_upload_channel + - run: + name: install binaries + command: | + set -x + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" + conda env remove -n python${PYTHON_VERSION} || true + conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} + conda activate python${PYTHON_VERSION} + conda install "Pillow>=5.3.0,!=8.3.*" + conda install -v -y -c pytorch-nightly pytorch + conda install -v -y $(ls ~/workspace/torchvision*.tar.bz2) + - run: + name: smoke test + command: | + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" + conda activate python${PYTHON_VERSION} + python -c "import torchvision" + + smoke_test_win_pip: + <<: *binary_common + executor: + name: windows-cpu + steps: + - attach_workspace: + at: ~/workspace + - designate_upload_channel + - run: + name: install binaries + command: | + set -x + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" + conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} + conda activate python${PYTHON_VERSION} + - pip_install: + args: $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html + - run: + name: smoke test + command: | + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" + conda activate python${PYTHON_VERSION} + python -c "import torchvision" + + unittest_linux_cpu: + <<: *binary_common + docker: + - image: "pytorch/manylinux-cuda102" + resource_class: 2xlarge+ + steps: + - checkout + - designate_upload_channel + - run: + name: Generate cache key + # This will refresh cache on Sundays, nightly build should generate new cache. + command: echo "$(date +"%Y-%U")" > .circleci-weekly + - restore_cache: + + keys: + - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + + - run: + name: Setup + command: .circleci/unittest/linux/scripts/setup_env.sh + - save_cache: + + key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + + paths: + - conda + - env + - run: + name: Install torchvision + command: .circleci/unittest/linux/scripts/install.sh + - run: + name: Run tests + command: .circleci/unittest/linux/scripts/run_test.sh + - run: + name: Post process + command: .circleci/unittest/linux/scripts/post_process.sh + - store_test_results: + path: test-results + + unittest_linux_gpu: + <<: *binary_common + machine: + image: ubuntu-2004-cuda-11.4:202110-01 + resource_class: gpu.nvidia.medium + environment: + image_name: "pytorch/manylinux-cuda102" + PYTHON_VERSION: << parameters.python_version >> + steps: + - checkout + - designate_upload_channel + - run: + name: Generate cache key + # This will refresh cache on Sundays, nightly build should generate new cache. + command: echo "$(date +"%Y-%U")" > .circleci-weekly + - restore_cache: + + keys: + - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + + - run: + name: Setup + command: docker run -e PYTHON_VERSION -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh + - save_cache: + + key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + + paths: + - conda + - env + - run: + # Here we create an envlist file that contains some env variables that we want the docker container to be aware of. + # Normally, the CIRCLECI variable is set and available on all CI workflows: https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables. + # They're avaiable in all the other workflows (OSX and Windows). + # But here, we're running the unittest_linux_gpu workflows in a docker container, where those variables aren't accessible. + # So instead we dump the variables we need in env.list and we pass that file when invoking "docker run". + name: export CIRCLECI env var + command: echo "CIRCLECI=true" >> ./env.list + - run: + name: Install torchvision + command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL -e CU_VERSION "${image_name}" .circleci/unittest/linux/scripts/install.sh + - run: + name: Run tests + command: docker run --env-file ./env.list -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh + - run: + name: Post Process + command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/post_process.sh + - store_test_results: + path: test-results + + unittest_windows_cpu: + <<: *binary_common + executor: + name: windows-cpu + steps: + - checkout + - designate_upload_channel + - run: + name: Generate cache key + # This will refresh cache on Sundays, nightly build should generate new cache. + command: echo "$(date +"%Y-%U")" > .circleci-weekly + - restore_cache: + + keys: + - env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + + - run: + name: Setup + command: .circleci/unittest/windows/scripts/setup_env.sh + - save_cache: + + key: env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + + paths: + - conda + - env + - run: + name: Install torchvision + command: .circleci/unittest/windows/scripts/install.sh + - run: + name: Run tests + command: .circleci/unittest/windows/scripts/run_test.sh + - run: + name: Post process + command: .circleci/unittest/windows/scripts/post_process.sh + - store_test_results: + path: test-results + + unittest_windows_gpu: + <<: *binary_common + executor: + name: windows-gpu + environment: + CUDA_VERSION: "11.3" + PYTHON_VERSION: << parameters.python_version >> + steps: + - checkout + - designate_upload_channel + - run: + name: Generate cache key + # This will refresh cache on Sundays, nightly build should generate new cache. + command: echo "$(date +"%Y-%U")" > .circleci-weekly + - restore_cache: + + keys: + - env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + + - run: + name: Setup + command: .circleci/unittest/windows/scripts/setup_env.sh + - save_cache: + + key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + + paths: + - conda + - env + - run: + name: Install CUDA + command: packaging/windows/internal/cuda_install.bat + - run: + name: Update CUDA driver + command: packaging/windows/internal/driver_update.bat + - run: + name: Install torchvision + command: .circleci/unittest/windows/scripts/install.sh + - run: + name: Run tests + command: .circleci/unittest/windows/scripts/run_test.sh + - run: + name: Post process + command: .circleci/unittest/windows/scripts/post_process.sh + - store_test_results: + path: test-results + + unittest_macos_cpu: + <<: *binary_common + macos: + xcode: "12.0" + resource_class: large + steps: + - checkout + - designate_upload_channel + - run: + name: Install wget + command: HOMEBREW_NO_AUTO_UPDATE=1 brew install wget + # Disable brew auto update which is very slow + - run: + name: Generate cache key + # This will refresh cache on Sundays, nightly build should generate new cache. + command: echo "$(date +"%Y-%U")" > .circleci-weekly + - restore_cache: + + keys: + - env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + + - run: + name: Setup + command: .circleci/unittest/linux/scripts/setup_env.sh + - save_cache: + + key: env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + + paths: + - conda + - env + - run: + name: Install torchvision + command: .circleci/unittest/linux/scripts/install.sh + - run: + name: Run tests + command: .circleci/unittest/linux/scripts/run_test.sh + - run: + name: Post process + command: .circleci/unittest/linux/scripts/post_process.sh + - store_test_results: + path: test-results + + cmake_linux_cpu: + <<: *binary_common + docker: + - image: "pytorch/manylinux-cuda102" + resource_class: 2xlarge+ + steps: + - checkout_merge + - designate_upload_channel + - run: + name: Setup conda + command: .circleci/unittest/linux/scripts/setup_env.sh + - run: packaging/build_cmake.sh + + cmake_linux_gpu: + <<: *binary_common + machine: + image: ubuntu-2004-cuda-11.4:202110-01 + resource_class: gpu.nvidia.small + environment: + PYTHON_VERSION: << parameters.python_version >> + PYTORCH_VERSION: << parameters.pytorch_version >> + UNICODE_ABI: << parameters.unicode_abi >> + CU_VERSION: << parameters.cu_version >> + steps: + - checkout_merge + - designate_upload_channel + - run: + name: Setup conda + command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh + - run: + name: Build torchvision C++ distribution and test + command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh + + cmake_macos_cpu: + <<: *binary_common + macos: + xcode: "12.0" + steps: + - checkout_merge + - designate_upload_channel + - run: + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + conda install -yq conda-build cmake + packaging/build_cmake.sh + + cmake_windows_cpu: + <<: *binary_common + executor: + name: windows-cpu + steps: + - checkout_merge + - designate_upload_channel + - run: + command: | + set -ex + source packaging/windows/internal/vc_install_helper.sh + packaging/build_cmake.sh + + cmake_windows_gpu: + <<: *binary_common + executor: + name: windows-gpu + steps: + - checkout_merge + - designate_upload_channel + - run: + command: | + set -ex + source packaging/windows/internal/vc_install_helper.sh + packaging/windows/internal/cuda_install.bat + packaging/build_cmake.sh + + build_docs: + <<: *binary_common + docker: + - image: cimg/python:3.7 + resource_class: 2xlarge+ + steps: + - attach_workspace: + at: ~/workspace + - checkout + - download_model_weights + - run: + name: Setup + command: .circleci/unittest/linux/scripts/setup_env.sh + - designate_upload_channel + - run: + name: Install torchvision + command: .circleci/unittest/linux/scripts/install.sh + - run: + name: Build docs + command: | + set -ex + # turn v1.12.0rc3 into 1.12.0 + tag=$(echo $CIRCLE_TAG | sed -e 's/v*\([0-9.]*\).*/\1/') + VERSION=${tag:-main} + eval "$(./conda/bin/conda shell.bash hook)" + conda activate ./env + pushd docs + pip install --progress-bar=off -r requirements.txt + make html + popd + - persist_to_workspace: + root: ./ + paths: + - "*" + - store_artifacts: + path: ./docs/build/html + destination: docs + + upload_docs: + <<: *binary_common + docker: + - image: "pytorch/manylinux-cuda100" + resource_class: 2xlarge+ + steps: + - attach_workspace: + at: ~/workspace + - run: + name: Generate netrc + command: | + # set credentials for https pushing + # requires the org-member context + cat > ~/.netrc \<