From 8c5b87778889acd8e8d37c30fd4a6bb3ce7d91e2 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 May 2023 11:58:32 +0200 Subject: [PATCH 1/7] [DEBUG] try revert linpng uninstallation on macos-12 --- .github/scripts/setup-env.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/scripts/setup-env.sh b/.github/scripts/setup-env.sh index a7ba4a6bd12..9a1d5e00a17 100755 --- a/.github/scripts/setup-env.sh +++ b/.github/scripts/setup-env.sh @@ -24,12 +24,12 @@ esac if [[ "${OS_TYPE}" == "macos" && $(uname -m) == x86_64 ]]; then echo '::group::Uninstall system JPEG libraries on macOS' - # The x86 macOS runners, e.g. the GitHub Actions native "macos-12" runner, has some JPEG and PNG libraries - # installed by default that interfere with our build. We uninstall them here and use the one from conda below. - IMAGE_LIBS=$(brew list | grep -E "jpeg|png") - echo "${IMAGE_LIBS}" - for lib in "${IMAGE_LIBS}"; do - brew uninstall --ignore-dependencies --force "${lib}" || true + # The x86 macOS runners, e.g. the GitHub Actions native "macos-12" runner, has some JPEG libraries installed by + # default that interfere with our build. We uninstall them here and use the one from conda below. + JPEG_LIBS=$(brew list | grep jpeg) + echo $JPEG_LIBS + for lib in $JPEG_LIBS; do + brew uninstall --ignore-dependencies --force $lib || true done echo '::endgroup::' fi From f30e4136a63985cf9afc5fca028e246ce5d6ebd4 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 May 2023 11:59:57 +0200 Subject: [PATCH 2/7] [REVERTME] disable unrelated CI --- .circleci/.gitignore | 1 - .circleci/build_docs/commit_docs.sh | 35 -- .circleci/config.yml | 271 -------------- .circleci/config.yml.in | 251 ------------- .circleci/regenerate.py | 58 --- .circleci/smoke_test/docker/Dockerfile | 34 -- .../android/scripts/binary_android_build.sh | 27 -- .../android/scripts/binary_android_upload.sh | 34 -- .../android/scripts/install_gradle.sh | 19 - .../unittest/ios/scripts/binary_ios_build.sh | 47 --- .../unittest/ios/scripts/binary_ios_upload.sh | 42 --- .../unittest/linux/scripts/environment.yml | 16 - .circleci/unittest/linux/scripts/install.sh | 45 --- .../unittest/linux/scripts/post_process.sh | 6 - .../linux/scripts/run-clang-format.py | 331 ------------------ .circleci/unittest/linux/scripts/run_test.sh | 22 -- .circleci/unittest/linux/scripts/setup_env.sh | 47 --- .../unittest/windows/scripts/environment.yml | 19 - .circleci/unittest/windows/scripts/install.sh | 53 --- .../windows/scripts/install_conda.bat | 1 - .../unittest/windows/scripts/post_process.sh | 6 - .../unittest/windows/scripts/run_test.sh | 12 - .../unittest/windows/scripts/set_cuda_envs.sh | 48 --- .../unittest/windows/scripts/setup_env.sh | 45 --- .../windows/scripts/vc_env_helper.bat | 39 --- .github/scripts/unittest.sh | 3 +- .github/workflows/build-conda-linux.yml | 51 --- .github/workflows/build-conda-m1.yml | 52 --- .github/workflows/build-conda-macos.yml | 52 --- .github/workflows/build-conda-windows.yml | 51 --- .github/workflows/build-wheels-linux.yml | 50 --- .github/workflows/build-wheels-m1.yml | 51 --- .github/workflows/build-wheels-macos.yml | 51 --- .github/workflows/build-wheels-windows.yml | 52 --- .github/workflows/docs.yml | 115 ------ .github/workflows/lint.yml | 107 ------ .github/workflows/pr-labels.yml | 35 -- .../workflows/prototype-tests-linux-gpu.yml | 54 --- .github/workflows/tests-schedule.yml | 57 --- .github/workflows/update-viablestrict.yml | 23 -- 40 files changed, 2 insertions(+), 2311 deletions(-) delete mode 100644 .circleci/.gitignore delete mode 100755 .circleci/build_docs/commit_docs.sh delete mode 100644 .circleci/config.yml delete mode 100644 .circleci/config.yml.in delete mode 100755 .circleci/regenerate.py delete mode 100644 .circleci/smoke_test/docker/Dockerfile delete mode 100644 .circleci/unittest/android/scripts/binary_android_build.sh delete mode 100644 .circleci/unittest/android/scripts/binary_android_upload.sh delete mode 100755 .circleci/unittest/android/scripts/install_gradle.sh delete mode 100755 .circleci/unittest/ios/scripts/binary_ios_build.sh delete mode 100644 .circleci/unittest/ios/scripts/binary_ios_upload.sh delete mode 100644 .circleci/unittest/linux/scripts/environment.yml delete mode 100755 .circleci/unittest/linux/scripts/install.sh delete mode 100755 .circleci/unittest/linux/scripts/post_process.sh delete mode 100755 .circleci/unittest/linux/scripts/run-clang-format.py delete mode 100755 .circleci/unittest/linux/scripts/run_test.sh delete mode 100755 .circleci/unittest/linux/scripts/setup_env.sh delete mode 100644 .circleci/unittest/windows/scripts/environment.yml delete mode 100644 .circleci/unittest/windows/scripts/install.sh delete mode 100644 .circleci/unittest/windows/scripts/install_conda.bat delete mode 100644 .circleci/unittest/windows/scripts/post_process.sh delete mode 100644 .circleci/unittest/windows/scripts/run_test.sh delete mode 100644 .circleci/unittest/windows/scripts/set_cuda_envs.sh delete mode 100644 .circleci/unittest/windows/scripts/setup_env.sh delete mode 100644 .circleci/unittest/windows/scripts/vc_env_helper.bat delete mode 100644 .github/workflows/build-conda-linux.yml delete mode 100644 .github/workflows/build-conda-m1.yml delete mode 100644 .github/workflows/build-conda-macos.yml delete mode 100644 .github/workflows/build-conda-windows.yml delete mode 100644 .github/workflows/build-wheels-linux.yml delete mode 100644 .github/workflows/build-wheels-m1.yml delete mode 100644 .github/workflows/build-wheels-macos.yml delete mode 100644 .github/workflows/build-wheels-windows.yml delete mode 100644 .github/workflows/docs.yml delete mode 100644 .github/workflows/lint.yml delete mode 100644 .github/workflows/pr-labels.yml delete mode 100644 .github/workflows/prototype-tests-linux-gpu.yml delete mode 100644 .github/workflows/tests-schedule.yml delete mode 100644 .github/workflows/update-viablestrict.yml diff --git a/.circleci/.gitignore b/.circleci/.gitignore deleted file mode 100644 index 485dee64bcf..00000000000 --- a/.circleci/.gitignore +++ /dev/null @@ -1 +0,0 @@ -.idea diff --git a/.circleci/build_docs/commit_docs.sh b/.circleci/build_docs/commit_docs.sh deleted file mode 100755 index 04e3538fefc..00000000000 --- a/.circleci/build_docs/commit_docs.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -set -ex - - -if [ "$2" == "" ]; then - echo call as "$0" "" "" - echo where src is the root of the built documentation git checkout and - echo branch should be "main" or "1.7" or so - exit 1 -fi - -src=$1 -target=$2 - -echo "committing docs from ${src} to ${target}" - -pushd "${src}" -git checkout gh-pages -mkdir -p ./"${target}" -rm -rf ./"${target}"/* -cp -r "${src}/docs/build/html/"* ./"$target" -if [ "${target}" == "main" ]; then - mkdir -p ./_static - rm -rf ./_static/* - cp -r "${src}/docs/build/html/_static/"* ./_static - git add --all ./_static || true -fi -git add --all ./"${target}" || true -git config user.email "soumith+bot@pytorch.org" -git config user.name "pytorchbot" -# If there aren't changes, don't make a commit; push is no-op -git commit -m "auto-generating sphinx docs" || true -git remote add https https://github.com/pytorch/vision.git -git push -u https gh-pages diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 0463007af46..00000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,271 +0,0 @@ -version: 2.1 - -# How to test the Linux jobs: -# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ -# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_win_wheel_py3.8 -# - Replace binary_win_wheel_py3.8 with the name of the job you want to test. -# Job names are 'name:' key. - -executors: - windows-cpu: - machine: - resource_class: windows.xlarge - image: windows-server-2019-vs2019:stable - shell: bash.exe - - windows-gpu: - machine: - resource_class: windows.gpu.nvidia.medium - image: windows-server-2019-nvidia:stable - shell: bash.exe - -commands: - checkout_merge: - description: "checkout merge branch" - steps: - - checkout -# - run: -# name: Checkout merge branch -# command: | -# set -ex -# BRANCH=$(git rev-parse --abbrev-ref HEAD) -# if [[ "$BRANCH" != "main" ]]; then -# git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH} -# git checkout "merged/$CIRCLE_BRANCH" -# fi - designate_upload_channel: - description: "inserts the correct upload channel into ${BASH_ENV}" - steps: - - run: - name: adding UPLOAD_CHANNEL to BASH_ENV - command: | - our_upload_channel=nightly - # On tags upload to test instead - if [[ -n "${CIRCLE_TAG}" ]]; then - our_upload_channel=test - fi - echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} - - pip_install: - parameters: - args: - type: string - descr: - type: string - default: "" - user: - type: boolean - default: true - steps: - - run: - name: > - <<^ parameters.descr >> pip install << parameters.args >> <> - <<# parameters.descr >> << parameters.descr >> <> - command: > - pip install - <<# parameters.user >> --user <> - --progress-bar=off - << parameters.args >> - -binary_common: &binary_common - parameters: - # Edit these defaults to do a release - build_version: - description: "version number of release binary; by default, build a nightly" - type: string - default: "" - pytorch_version: - description: "PyTorch version to build against; by default, use a nightly" - type: string - default: "" - # Don't edit these - python_version: - description: "Python version to build against (e.g., 3.8)" - type: string - cu_version: - description: "CUDA version to build against, in CU format (e.g., cpu or cu100)" - type: string - default: "cpu" - unicode_abi: - description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" - type: string - default: "" - wheel_docker_image: - description: "Wheel only: what docker image to use" - type: string - default: "" - conda_docker_image: - description: "Conda only: what docker image to use" - type: string - default: "pytorch/conda-builder:cpu" - environment: - PYTHON_VERSION: << parameters.python_version >> - PYTORCH_VERSION: << parameters.pytorch_version >> - UNICODE_ABI: << parameters.unicode_abi >> - CU_VERSION: << parameters.cu_version >> - MACOSX_DEPLOYMENT_TARGET: 10.9 - -smoke_test_common: &smoke_test_common - <<: *binary_common - docker: - - image: torchvision/smoke_test:latest - -jobs: - circleci_consistency: - docker: - - image: cimg/python:3.8 - steps: - - checkout - - pip_install: - args: jinja2 pyyaml - - run: - name: Check CircleCI config consistency - command: | - python .circleci/regenerate.py - git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) - - smoke_test_docker_image_build: - machine: - image: ubuntu-2004:202104-01 - resource_class: large - environment: - image_name: torchvision/smoke_test - steps: - - checkout - - designate_upload_channel - - run: - name: Build and push Docker image - no_output_timeout: "1h" - command: | - set +x - echo "${DOCKER_HUB_TOKEN}" | docker login --username "${DOCKER_HUB_USERNAME}" --password-stdin - set -x - cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID} - docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} ${image_name}:latest - docker push ${image_name}:${CIRCLE_WORKFLOW_ID} - docker push ${image_name}:latest - - cmake_linux_cpu: - <<: *binary_common - docker: - - image: "pytorch/manylinux-cpu" - resource_class: 2xlarge+ - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Setup conda - command: .circleci/unittest/linux/scripts/setup_env.sh - - run: packaging/build_cmake.sh - - cmake_linux_gpu: - <<: *binary_common - machine: - image: ubuntu-2004-cuda-11.4:202110-01 - resource_class: gpu.nvidia.small - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Setup conda - command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh - - run: - name: Build torchvision C++ distribution and test - no_output_timeout: 30m - command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh - - cmake_macos_cpu: - <<: *binary_common - macos: - xcode: "14.0" - steps: - - checkout_merge - - designate_upload_channel - - run: - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - conda install -yq conda-build cmake python=<< parameters.python_version >> - packaging/build_cmake.sh - - cmake_windows_cpu: - <<: *binary_common - executor: - name: windows-cpu - steps: - - checkout_merge - - designate_upload_channel - - run: - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate base - conda create -yn python39 python=3.9 - conda activate python39 - packaging/build_cmake.sh - - cmake_windows_gpu: - <<: *binary_common - executor: - name: windows-gpu - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Update CUDA driver - command: packaging/windows/internal/driver_update.bat - - run: - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - packaging/windows/internal/cuda_install.bat - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate - conda update -y conda - conda create -yn python39 python=3.9 - conda activate python39 - packaging/build_cmake.sh - - -workflows: - lint: - jobs: - - circleci_consistency - - cmake: - jobs: - - cmake_linux_cpu: - cu_version: cpu - name: cmake_linux_cpu - python_version: '3.8' - - cmake_linux_gpu: - cu_version: cu117 - name: cmake_linux_gpu - python_version: '3.8' - wheel_docker_image: pytorch/manylinux-cuda117 - - cmake_windows_cpu: - cu_version: cpu - name: cmake_windows_cpu - python_version: '3.8' - - cmake_windows_gpu: - cu_version: cu117 - name: cmake_windows_gpu - python_version: '3.8' - - cmake_macos_cpu: - cu_version: cpu - name: cmake_macos_cpu - python_version: '3.8' - - docker_build: - triggers: - - schedule: - cron: "0 10 * * 0" - filters: - branches: - only: - - main - jobs: - - smoke_test_docker_image_build: - context: org-member diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in deleted file mode 100644 index f3a88f2d987..00000000000 --- a/.circleci/config.yml.in +++ /dev/null @@ -1,251 +0,0 @@ -version: 2.1 - -# How to test the Linux jobs: -# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ -# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_win_wheel_py3.8 -# - Replace binary_win_wheel_py3.8 with the name of the job you want to test. -# Job names are 'name:' key. - -executors: - windows-cpu: - machine: - resource_class: windows.xlarge - image: windows-server-2019-vs2019:stable - shell: bash.exe - - windows-gpu: - machine: - resource_class: windows.gpu.nvidia.medium - image: windows-server-2019-nvidia:stable - shell: bash.exe - -commands: - checkout_merge: - description: "checkout merge branch" - steps: - - checkout -# - run: -# name: Checkout merge branch -# command: | -# set -ex -# BRANCH=$(git rev-parse --abbrev-ref HEAD) -# if [[ "$BRANCH" != "main" ]]; then -# git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH} -# git checkout "merged/$CIRCLE_BRANCH" -# fi - designate_upload_channel: - description: "inserts the correct upload channel into ${BASH_ENV}" - steps: - - run: - name: adding UPLOAD_CHANNEL to BASH_ENV - command: | - our_upload_channel=nightly - # On tags upload to test instead - if [[ -n "${CIRCLE_TAG}" ]]; then - our_upload_channel=test - fi - echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} - - pip_install: - parameters: - args: - type: string - descr: - type: string - default: "" - user: - type: boolean - default: true - steps: - - run: - name: > - <<^ parameters.descr >> pip install << parameters.args >> <> - <<# parameters.descr >> << parameters.descr >> <> - command: > - pip install - <<# parameters.user >> --user <> - --progress-bar=off - << parameters.args >> - -binary_common: &binary_common - parameters: - # Edit these defaults to do a release - build_version: - description: "version number of release binary; by default, build a nightly" - type: string - default: "" - pytorch_version: - description: "PyTorch version to build against; by default, use a nightly" - type: string - default: "" - # Don't edit these - python_version: - description: "Python version to build against (e.g., 3.8)" - type: string - cu_version: - description: "CUDA version to build against, in CU format (e.g., cpu or cu100)" - type: string - default: "cpu" - unicode_abi: - description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" - type: string - default: "" - wheel_docker_image: - description: "Wheel only: what docker image to use" - type: string - default: "" - conda_docker_image: - description: "Conda only: what docker image to use" - type: string - default: "pytorch/conda-builder:cpu" - environment: - PYTHON_VERSION: << parameters.python_version >> - PYTORCH_VERSION: << parameters.pytorch_version >> - UNICODE_ABI: << parameters.unicode_abi >> - CU_VERSION: << parameters.cu_version >> - MACOSX_DEPLOYMENT_TARGET: 10.9 - -smoke_test_common: &smoke_test_common - <<: *binary_common - docker: - - image: torchvision/smoke_test:latest - -jobs: - circleci_consistency: - docker: - - image: cimg/python:3.8 - steps: - - checkout - - pip_install: - args: jinja2 pyyaml - - run: - name: Check CircleCI config consistency - command: | - python .circleci/regenerate.py - git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) - - smoke_test_docker_image_build: - machine: - image: ubuntu-2004:202104-01 - resource_class: large - environment: - image_name: torchvision/smoke_test - steps: - - checkout - - designate_upload_channel - - run: - name: Build and push Docker image - no_output_timeout: "1h" - command: | - set +x - echo "${DOCKER_HUB_TOKEN}" | docker login --username "${DOCKER_HUB_USERNAME}" --password-stdin - set -x - cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID} - docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} ${image_name}:latest - docker push ${image_name}:${CIRCLE_WORKFLOW_ID} - docker push ${image_name}:latest - - cmake_linux_cpu: - <<: *binary_common - docker: - - image: "pytorch/manylinux-cpu" - resource_class: 2xlarge+ - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Setup conda - command: .circleci/unittest/linux/scripts/setup_env.sh - - run: packaging/build_cmake.sh - - cmake_linux_gpu: - <<: *binary_common - machine: - image: ubuntu-2004-cuda-11.4:202110-01 - resource_class: gpu.nvidia.small - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Setup conda - command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh - - run: - name: Build torchvision C++ distribution and test - no_output_timeout: 30m - command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh - - cmake_macos_cpu: - <<: *binary_common - macos: - xcode: "14.0" - steps: - - checkout_merge - - designate_upload_channel - - run: - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - conda install -yq conda-build cmake python=<< parameters.python_version >> - packaging/build_cmake.sh - - cmake_windows_cpu: - <<: *binary_common - executor: - name: windows-cpu - steps: - - checkout_merge - - designate_upload_channel - - run: - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate base - conda create -yn python39 python=3.9 - conda activate python39 - packaging/build_cmake.sh - - cmake_windows_gpu: - <<: *binary_common - executor: - name: windows-gpu - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Update CUDA driver - command: packaging/windows/internal/driver_update.bat - - run: - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - packaging/windows/internal/cuda_install.bat - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate - conda update -y conda - conda create -yn python39 python=3.9 - conda activate python39 - packaging/build_cmake.sh - - -workflows: - lint: - jobs: - - circleci_consistency - - cmake: - jobs: - {{ cmake_workflows() }} - - docker_build: - triggers: - - schedule: - cron: "0 10 * * 0" - filters: - branches: - only: - - main - jobs: - - smoke_test_docker_image_build: - context: org-member diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py deleted file mode 100755 index e7b8db3d56e..00000000000 --- a/.circleci/regenerate.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 - -""" -This script should use a very simple, functional programming style. -Avoid Jinja macros in favor of native Python functions. - -Don't go overboard on code generation; use Python only to generate -content that can't be easily declared statically using CircleCI's YAML API. - -Data declarations (e.g. the nested loops for defining the configuration matrix) -should be at the top of the file for easy updating. - -See this comment for design rationale: -https://github.com/pytorch/vision/pull/1321#issuecomment-531033978 -""" - -import os.path - -import jinja2 -import yaml -from jinja2 import select_autoescape - - -def indent(indentation, data_list): - return ("\n" + " " * indentation).join(yaml.dump(data_list, default_flow_style=False).splitlines()) - - -def cmake_workflows(indentation=6): - jobs = [] - python_version = "3.8" - for os_type in ["linux", "windows", "macos"]: - # Skip OSX CUDA - device_types = ["cpu", "gpu"] if os_type != "macos" else ["cpu"] - for device in device_types: - job = {"name": f"cmake_{os_type}_{device}", "python_version": python_version} - - job["cu_version"] = "cu117" if device == "gpu" else "cpu" - if device == "gpu" and os_type == "linux": - job["wheel_docker_image"] = "pytorch/manylinux-cuda117" - jobs.append({f"cmake_{os_type}_{device}": job}) - return indent(indentation, jobs) - - -if __name__ == "__main__": - d = os.path.dirname(__file__) - env = jinja2.Environment( - loader=jinja2.FileSystemLoader(d), - lstrip_blocks=True, - autoescape=select_autoescape(enabled_extensions=("html", "xml")), - keep_trailing_newline=True, - ) - - with open(os.path.join(d, "config.yml"), "w") as f: - f.write( - env.get_template("config.yml.in").render( - cmake_workflows=cmake_workflows, - ) - ) diff --git a/.circleci/smoke_test/docker/Dockerfile b/.circleci/smoke_test/docker/Dockerfile deleted file mode 100644 index 34bdcda1053..00000000000 --- a/.circleci/smoke_test/docker/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# this Dockerfile is for torchvision smoke test, it will be created periodically via CI system -# if you need to do it locally, follow below steps once you have Docker installed -# assuming you're within the directory where this Dockerfile located -# $ docker build . -t torchvision/smoketest - -# if you want to push to aws ecr, make sure you have the rights to write to ECR, then run -# $ eval $(aws ecr get-login --region us-east-1 --no-include-email) -# $ export MYTAG=localbuild ## you can choose whatever tag you like -# $ docker tag torchvision/smoketest 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchvision/smoke_test:${MYTAG} -# $ docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchvision/smoke_test:${MYTAG} - -FROM ubuntu:latest - -RUN apt-get -qq update && apt-get -qq -y install curl bzip2 libsox-fmt-all \ - && curl -sSL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh \ - && bash /tmp/miniconda.sh -bfp /usr/local \ - && rm -rf /tmp/miniconda.sh \ - && conda install -y python=3 \ - && conda update conda \ - && apt-get -qq -y remove curl bzip2 \ - && apt-get -qq -y autoremove \ - && apt-get autoclean \ - && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log \ - && conda clean --all --yes - -ENV PATH /opt/conda/bin:$PATH - -RUN conda create -y --name python3.7 python=3.7 -RUN conda create -y --name python3.8 python=3.8 -RUN conda create -y --name python3.9 python=3.9 -RUN conda create -y --name python3.10 python=3.10 -SHELL [ "/bin/bash", "-c" ] -RUN echo "source /usr/local/etc/profile.d/conda.sh" >> ~/.bashrc -CMD [ "/bin/bash"] diff --git a/.circleci/unittest/android/scripts/binary_android_build.sh b/.circleci/unittest/android/scripts/binary_android_build.sh deleted file mode 100644 index 0d8c0d47d8a..00000000000 --- a/.circleci/unittest/android/scripts/binary_android_build.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -set -ex -o pipefail - -echo "DIR: $(pwd)" -echo "ANDROID_HOME=${ANDROID_HOME}" -echo "ANDROID_NDK_HOME=${ANDROID_NDK_HOME}" -echo "JAVA_HOME=${JAVA_HOME}" - -WORKSPACE=/home/circleci/workspace -VISION_ANDROID=/home/circleci/project/android - -. /home/circleci/project/.circleci/unittest/android/scripts/install_gradle.sh - -GRADLE_LOCAL_PROPERTIES=${VISION_ANDROID}/local.properties -rm -f $GRADLE_LOCAL_PROPERTIES - -echo "sdk.dir=${ANDROID_HOME}" >> $GRADLE_LOCAL_PROPERTIES -echo "ndk.dir=${ANDROID_NDK_HOME}" >> $GRADLE_LOCAL_PROPERTIES - -echo "GRADLE_PATH $GRADLE_PATH" -echo "GRADLE_HOME $GRADLE_HOME" - -${GRADLE_PATH} --scan --stacktrace --debug --no-daemon -p ${VISION_ANDROID} assemble || true - -mkdir -p ~/workspace/artifacts -find . -type f -name *aar -print | xargs tar cfvz ~/workspace/artifacts/artifacts-aars.tgz -find . -type f -name *apk -print | xargs tar cfvz ~/workspace/artifacts/artifacts-apks.tgz diff --git a/.circleci/unittest/android/scripts/binary_android_upload.sh b/.circleci/unittest/android/scripts/binary_android_upload.sh deleted file mode 100644 index 1472a877d90..00000000000 --- a/.circleci/unittest/android/scripts/binary_android_upload.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -set -ex -o pipefail - -echo "DIR: $(pwd)" -echo "ANDROID_HOME=${ANDROID_HOME}" -echo "ANDROID_NDK_HOME=${ANDROID_NDK_HOME}" -echo "JAVA_HOME=${JAVA_HOME}" - -WORKSPACE=/home/circleci/workspace -VISION_ANDROID=/home/circleci/project/android - -. /home/circleci/project/.circleci/unittest/android/scripts/install_gradle.sh - -GRADLE_LOCAL_PROPERTIES=${VISION_ANDROID}/local.properties -rm -f $GRADLE_LOCAL_PROPERTIES -GRADLE_PROPERTIES=/home/circleci/project/android/gradle.properties - -echo "sdk.dir=${ANDROID_HOME}" >> $GRADLE_LOCAL_PROPERTIES -echo "ndk.dir=${ANDROID_NDK_HOME}" >> $GRADLE_LOCAL_PROPERTIES - -echo "SONATYPE_NEXUS_USERNAME=${SONATYPE_NEXUS_USERNAME}" >> $GRADLE_PROPERTIES -echo "mavenCentralRepositoryUsername=${SONATYPE_NEXUS_USERNAME}" >> $GRADLE_PROPERTIES -echo "SONATYPE_NEXUS_PASSWORD=${SONATYPE_NEXUS_PASSWORD}" >> $GRADLE_PROPERTIES -echo "mavenCentralRepositoryPassword=${SONATYPE_NEXUS_PASSWORD}" >> $GRADLE_PROPERTIES - -echo "signing.keyId=${ANDROID_SIGN_KEY}" >> $GRADLE_PROPERTIES -echo "signing.password=${ANDROID_SIGN_PASS}" >> $GRADLE_PROPERTIES - -cat /home/circleci/project/android/gradle.properties | grep VERSION - -${GRADLE_PATH} --scan --stacktrace --debug --no-daemon -p ${VISION_ANDROID} ops:uploadArchives - -mkdir -p ~/workspace/artifacts -find . -type f -name *aar -print | xargs tar cfvz ~/workspace/artifacts/artifacts-aars.tgz diff --git a/.circleci/unittest/android/scripts/install_gradle.sh b/.circleci/unittest/android/scripts/install_gradle.sh deleted file mode 100755 index ff90c657e65..00000000000 --- a/.circleci/unittest/android/scripts/install_gradle.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -set -ex - -_https_amazon_aws=https://downloads.gradle-dn.com/distributions -GRADLE_VERSION=6.8.3 - -_gradle_home=/opt/gradle -sudo rm -rf $gradle_home -sudo mkdir -p $_gradle_home - -curl --silent --output /tmp/gradle.zip --retry 3 $_https_amazon_aws/gradle-${GRADLE_VERSION}-bin.zip - -sudo unzip -q /tmp/gradle.zip -d $_gradle_home -rm /tmp/gradle.zip - -sudo chmod -R 777 $_gradle_home - -export GRADLE_HOME=$_gradle_home/gradle-$GRADLE_VERSION -export GRADLE_PATH=${GRADLE_HOME}/bin/gradle diff --git a/.circleci/unittest/ios/scripts/binary_ios_build.sh b/.circleci/unittest/ios/scripts/binary_ios_build.sh deleted file mode 100755 index 1f117481f2d..00000000000 --- a/.circleci/unittest/ios/scripts/binary_ios_build.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -ex -o pipefail - -echo "" -echo "DIR: $(pwd)" -WORKSPACE=/Users/distiller/workspace -PROJ_ROOT_IOS=/Users/distiller/project/ios -PYTORCH_IOS_NIGHTLY_NAME=libtorch_ios_nightly_build.zip -export TCLLIBPATH="/usr/local/lib" - -# install conda -curl --retry 3 -o ~/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -chmod +x ~/conda.sh -/bin/bash ~/conda.sh -b -p ~/anaconda -export PATH="~/anaconda/bin:${PATH}" -source ~/anaconda/bin/activate - -# install dependencies -conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi requests wget --yes -conda install -c conda-forge valgrind --yes -export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} - -# sync submodules -cd ${PROJ_ROOT_IOS} -git submodule sync -git submodule update --init --recursive - -# download pytorch-iOS nightly build and unzip it -mkdir -p ${PROJ_ROOT_IOS}/lib -mkdir -p ${PROJ_ROOT_IOS}/build -mkdir -p ${PROJ_ROOT_IOS}/pytorch -TORCH_ROOT="${PROJ_ROOT_IOS}/pytorch" - -cd ${TORCH_ROOT} -wget https://ossci-ios-build.s3.amazonaws.com/${PYTORCH_IOS_NIGHTLY_NAME} -mkdir -p ./build_ios -unzip -d ./build_ios ./${PYTORCH_IOS_NIGHTLY_NAME} - -LIBTORCH_HEADER_ROOT="${TORCH_ROOT}/build_ios/install/include" -cd ${PROJ_ROOT_IOS} -IOS_ARCH=${IOS_ARCH} LIBTORCH_HEADER_ROOT=${LIBTORCH_HEADER_ROOT} ./build_ios.sh -rm -rf ${TORCH_ROOT} - -# store the binary -DEST_DIR=${WORKSPACE}/ios/${IOS_ARCH} -mkdir -p ${DEST_DIR} -cp ${PROJ_ROOT_IOS}/lib/*.a ${DEST_DIR} diff --git a/.circleci/unittest/ios/scripts/binary_ios_upload.sh b/.circleci/unittest/ios/scripts/binary_ios_upload.sh deleted file mode 100644 index ce56388e5da..00000000000 --- a/.circleci/unittest/ios/scripts/binary_ios_upload.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -set -ex -o pipefail - -echo "" -echo "DIR: $(pwd)" - -WORKSPACE=/Users/distiller/workspace -PROJ_ROOT=/Users/distiller/project -ARTIFACTS_DIR=${WORKSPACE}/ios -ls ${ARTIFACTS_DIR} -ZIP_DIR=${WORKSPACE}/zip -mkdir -p ${ZIP_DIR}/install/lib - -# build a FAT bianry -cd ${ZIP_DIR}/install/lib -libs=("${ARTIFACTS_DIR}/x86_64/libtorchvision_ops.a" "${ARTIFACTS_DIR}/arm64/libtorchvision_ops.a") -lipo -create "${libs[@]}" -o ${ZIP_DIR}/install/lib/libtorchvision_ops.a -lipo -i ${ZIP_DIR}/install/lib/*.a - -# copy the license -cp ${PROJ_ROOT}/LICENSE ${ZIP_DIR}/ -# zip the library -ZIPFILE=libtorchvision_ops_ios_nightly_build.zip -cd ${ZIP_DIR} -#for testing -touch version.txt -echo $(date +%s) > version.txt -zip -r ${ZIPFILE} install version.txt LICENSE - -# upload to aws -# Install conda then 'conda install' awscli -curl --retry 3 -o ~/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -chmod +x ~/conda.sh -/bin/bash ~/conda.sh -b -p ~/anaconda -export PATH="~/anaconda/bin:${PATH}" -source ~/anaconda/bin/activate -conda install -c conda-forge awscli --yes -set +x -export AWS_ACCESS_KEY_ID=${AWS_S3_ACCESS_KEY_FOR_PYTORCH_BINARY_UPLOAD} -export AWS_SECRET_ACCESS_KEY=${AWS_S3_ACCESS_SECRET_FOR_PYTORCH_BINARY_UPLOAD} -set -x -aws s3 cp ${ZIPFILE} s3://ossci-ios-build/ --acl public-read diff --git a/.circleci/unittest/linux/scripts/environment.yml b/.circleci/unittest/linux/scripts/environment.yml deleted file mode 100644 index fae96c5f93c..00000000000 --- a/.circleci/unittest/linux/scripts/environment.yml +++ /dev/null @@ -1,16 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pytest - - pytest-cov - - pytest-mock - - pip - - libpng - - jpeg - - ca-certificates - - h5py - - pip: - - future - - scipy - - av < 10 diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh deleted file mode 100755 index 6375e191072..00000000000 --- a/.circleci/unittest/linux/scripts/install.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -unset PYTORCH_VERSION -# For unittest, nightly PyTorch is used as the following section, -# so no need to set PYTORCH_VERSION. -# In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config. - -set -ex - -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env - -if [ "${CU_VERSION:-}" == cpu ] ; then - cudatoolkit="cpuonly" - version="cpu" -else - if [[ ${#CU_VERSION} -eq 4 ]]; then - CUDA_VERSION="${CU_VERSION:2:1}.${CU_VERSION:3:1}" - elif [[ ${#CU_VERSION} -eq 5 ]]; then - CUDA_VERSION="${CU_VERSION:2:2}.${CU_VERSION:4:1}" - fi - echo "Using CUDA $CUDA_VERSION as determined by CU_VERSION: ${CU_VERSION} " - version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")" - cudatoolkit="pytorch-cuda=${version}" - - # make sure local cuda is set to required cuda version and not CUDA version by default - rm -f /usr/local/cuda - ln -s /usr/local/cuda-${version} /usr/local/cuda -fi - -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac - -printf "Installing PyTorch with %s\n" "${cudatoolkit}" -if [ "${os}" == "MacOSX" ]; then - conda install -y -c "pytorch-${UPLOAD_CHANNEL}" "pytorch-${UPLOAD_CHANNEL}"::pytorch "${cudatoolkit}" -else - conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c nvidia "pytorch-${UPLOAD_CHANNEL}"::pytorch[build="*${version}*"] "${cudatoolkit}" -fi - - -printf "* Installing torchvision\n" -python setup.py develop diff --git a/.circleci/unittest/linux/scripts/post_process.sh b/.circleci/unittest/linux/scripts/post_process.sh deleted file mode 100755 index e97bf2a7b1b..00000000000 --- a/.circleci/unittest/linux/scripts/post_process.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -e - -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env diff --git a/.circleci/unittest/linux/scripts/run-clang-format.py b/.circleci/unittest/linux/scripts/run-clang-format.py deleted file mode 100755 index 5c61b2519e0..00000000000 --- a/.circleci/unittest/linux/scripts/run-clang-format.py +++ /dev/null @@ -1,331 +0,0 @@ -#!/usr/bin/env python -""" -MIT License - -Copyright (c) 2017 Guillaume Papin - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -A wrapper script around clang-format, suitable for linting multiple files -and to use for continuous integration. - -This is an alternative API for the clang-format command line. -It runs over multiple files and directories in parallel. -A diff output is produced and a sensible exit code is returned. - -""" - -import argparse -import difflib -import fnmatch -import multiprocessing -import os -import signal -import subprocess -import sys -import traceback -from functools import partial - -try: - from subprocess import DEVNULL # py3k -except ImportError: - DEVNULL = open(os.devnull, "wb") - - -DEFAULT_EXTENSIONS = "c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx,cu" - - -class ExitStatus: - SUCCESS = 0 - DIFF = 1 - TROUBLE = 2 - - -def list_files(files, recursive=False, extensions=None, exclude=None): - if extensions is None: - extensions = [] - if exclude is None: - exclude = [] - - out = [] - for file in files: - if recursive and os.path.isdir(file): - for dirpath, dnames, fnames in os.walk(file): - fpaths = [os.path.join(dirpath, fname) for fname in fnames] - for pattern in exclude: - # os.walk() supports trimming down the dnames list - # by modifying it in-place, - # to avoid unnecessary directory listings. - dnames[:] = [x for x in dnames if not fnmatch.fnmatch(os.path.join(dirpath, x), pattern)] - fpaths = [x for x in fpaths if not fnmatch.fnmatch(x, pattern)] - for f in fpaths: - ext = os.path.splitext(f)[1][1:] - if ext in extensions: - out.append(f) - else: - out.append(file) - return out - - -def make_diff(file, original, reformatted): - return list( - difflib.unified_diff( - original, reformatted, fromfile=f"{file}\t(original)", tofile=f"{file}\t(reformatted)", n=3 - ) - ) - - -class DiffError(Exception): - def __init__(self, message, errs=None): - super().__init__(message) - self.errs = errs or [] - - -class UnexpectedError(Exception): - def __init__(self, message, exc=None): - super().__init__(message) - self.formatted_traceback = traceback.format_exc() - self.exc = exc - - -def run_clang_format_diff_wrapper(args, file): - try: - ret = run_clang_format_diff(args, file) - return ret - except DiffError: - raise - except Exception as e: - raise UnexpectedError(f"{file}: {e.__class__.__name__}: {e}", e) - - -def run_clang_format_diff(args, file): - try: - with open(file, encoding="utf-8") as f: - original = f.readlines() - except OSError as exc: - raise DiffError(str(exc)) - invocation = [args.clang_format_executable, file] - - # Use of utf-8 to decode the process output. - # - # Hopefully, this is the correct thing to do. - # - # It's done due to the following assumptions (which may be incorrect): - # - clang-format will returns the bytes read from the files as-is, - # without conversion, and it is already assumed that the files use utf-8. - # - if the diagnostics were internationalized, they would use utf-8: - # > Adding Translations to Clang - # > - # > Not possible yet! - # > Diagnostic strings should be written in UTF-8, - # > the client can translate to the relevant code page if needed. - # > Each translation completely replaces the format string - # > for the diagnostic. - # > -- http://clang.llvm.org/docs/InternalsManual.html#internals-diag-translation - - try: - proc = subprocess.Popen( - invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, encoding="utf-8" - ) - except OSError as exc: - raise DiffError(f"Command '{subprocess.list2cmdline(invocation)}' failed to start: {exc}") - proc_stdout = proc.stdout - proc_stderr = proc.stderr - - # hopefully the stderr pipe won't get full and block the process - outs = list(proc_stdout.readlines()) - errs = list(proc_stderr.readlines()) - proc.wait() - if proc.returncode: - raise DiffError( - "Command '{}' returned non-zero exit status {}".format( - subprocess.list2cmdline(invocation), proc.returncode - ), - errs, - ) - return make_diff(file, original, outs), errs - - -def bold_red(s): - return "\x1b[1m\x1b[31m" + s + "\x1b[0m" - - -def colorize(diff_lines): - def bold(s): - return "\x1b[1m" + s + "\x1b[0m" - - def cyan(s): - return "\x1b[36m" + s + "\x1b[0m" - - def green(s): - return "\x1b[32m" + s + "\x1b[0m" - - def red(s): - return "\x1b[31m" + s + "\x1b[0m" - - for line in diff_lines: - if line[:4] in ["--- ", "+++ "]: - yield bold(line) - elif line.startswith("@@ "): - yield cyan(line) - elif line.startswith("+"): - yield green(line) - elif line.startswith("-"): - yield red(line) - else: - yield line - - -def print_diff(diff_lines, use_color): - if use_color: - diff_lines = colorize(diff_lines) - sys.stdout.writelines(diff_lines) - - -def print_trouble(prog, message, use_colors): - error_text = "error:" - if use_colors: - error_text = bold_red(error_text) - print(f"{prog}: {error_text} {message}", file=sys.stderr) - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--clang-format-executable", - metavar="EXECUTABLE", - help="path to the clang-format executable", - default="clang-format", - ) - parser.add_argument( - "--extensions", - help=f"comma separated list of file extensions (default: {DEFAULT_EXTENSIONS})", - default=DEFAULT_EXTENSIONS, - ) - parser.add_argument("-r", "--recursive", action="store_true", help="run recursively over directories") - parser.add_argument("files", metavar="file", nargs="+") - parser.add_argument("-q", "--quiet", action="store_true") - parser.add_argument( - "-j", - metavar="N", - type=int, - default=0, - help="run N clang-format jobs in parallel (default number of cpus + 1)", - ) - parser.add_argument( - "--color", default="auto", choices=["auto", "always", "never"], help="show colored diff (default: auto)" - ) - parser.add_argument( - "-e", - "--exclude", - metavar="PATTERN", - action="append", - default=[], - help="exclude paths matching the given glob-like pattern(s) from recursive search", - ) - - args = parser.parse_args() - - # use default signal handling, like diff return SIGINT value on ^C - # https://bugs.python.org/issue14229#msg156446 - signal.signal(signal.SIGINT, signal.SIG_DFL) - try: - signal.SIGPIPE - except AttributeError: - # compatibility, SIGPIPE does not exist on Windows - pass - else: - signal.signal(signal.SIGPIPE, signal.SIG_DFL) - - colored_stdout = False - colored_stderr = False - if args.color == "always": - colored_stdout = True - colored_stderr = True - elif args.color == "auto": - colored_stdout = sys.stdout.isatty() - colored_stderr = sys.stderr.isatty() - - version_invocation = [args.clang_format_executable, "--version"] - try: - subprocess.check_call(version_invocation, stdout=DEVNULL) - except subprocess.CalledProcessError as e: - print_trouble(parser.prog, str(e), use_colors=colored_stderr) - return ExitStatus.TROUBLE - except OSError as e: - print_trouble( - parser.prog, - f"Command '{subprocess.list2cmdline(version_invocation)}' failed to start: {e}", - use_colors=colored_stderr, - ) - return ExitStatus.TROUBLE - - retcode = ExitStatus.SUCCESS - files = list_files( - args.files, recursive=args.recursive, exclude=args.exclude, extensions=args.extensions.split(",") - ) - - if not files: - return - - njobs = args.j - if njobs == 0: - njobs = multiprocessing.cpu_count() + 1 - njobs = min(len(files), njobs) - - if njobs == 1: - # execute directly instead of in a pool, - # less overhead, simpler stacktraces - it = (run_clang_format_diff_wrapper(args, file) for file in files) - pool = None - else: - pool = multiprocessing.Pool(njobs) - it = pool.imap_unordered(partial(run_clang_format_diff_wrapper, args), files) - while True: - try: - outs, errs = next(it) - except StopIteration: - break - except DiffError as e: - print_trouble(parser.prog, str(e), use_colors=colored_stderr) - retcode = ExitStatus.TROUBLE - sys.stderr.writelines(e.errs) - except UnexpectedError as e: - print_trouble(parser.prog, str(e), use_colors=colored_stderr) - sys.stderr.write(e.formatted_traceback) - retcode = ExitStatus.TROUBLE - # stop at the first unexpected error, - # something could be very wrong, - # don't process all files unnecessarily - if pool: - pool.terminate() - break - else: - sys.stderr.writelines(errs) - if outs == []: - continue - if not args.quiet: - print_diff(outs, use_color=colored_stdout) - if retcode == ExitStatus.SUCCESS: - retcode = ExitStatus.DIFF - return retcode - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.circleci/unittest/linux/scripts/run_test.sh b/.circleci/unittest/linux/scripts/run_test.sh deleted file mode 100755 index 5348baa71dd..00000000000 --- a/.circleci/unittest/linux/scripts/run_test.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -set -e - -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env - -python -m torch.utils.collect_env - -case "$(uname -s)" in - Darwin*) - # The largest macOS runner is not able to handle the regular test suite plus the transforms v2 tests at the same - # time due to insufficient resources. Thus, we ignore the transforms v2 tests at first and run them in a separate - # step afterwards. - GLOB='test/test_transforms_v2*' - pytest --junitxml=test-results/junit.xml -v --durations 20 --ignore-glob="${GLOB}" - eval "pytest --junitxml=test-results/junit-transforms-v2.xml -v --durations 20 ${GLOB}" - ;; - *) - pytest --junitxml=test-results/junit.xml -v --durations 20 - ;; -esac diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh deleted file mode 100755 index 8a8a78f1fb2..00000000000 --- a/.circleci/unittest/linux/scripts/setup_env.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -# This script is for setting up environment in which unit test is ran. -# To speed up the CI time, the resulting environment is cached. -# -# Do not install PyTorch and torchvision here, otherwise they also get cached. - -set -ex - -this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -# Avoid error: "fatal: unsafe repository" -git config --global --add safe.directory '*' -root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" - -cd "${root_dir}" - -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac - -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" - -# 2. Create test environment at ./env -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" - -# 3. Install Conda dependencies -printf "* Installing dependencies (except PyTorch)\n" -FFMPEG_PIN="=4.2" -if [[ "${PYTHON_VERSION}" == "3.9" ]]; then - FFMPEG_PIN=">=4.2" -fi - -conda install -y -c pytorch "ffmpeg${FFMPEG_PIN}" -conda env update --file "${this_dir}/environment.yml" --prune diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml deleted file mode 100644 index d229aafb41a..00000000000 --- a/.circleci/unittest/windows/scripts/environment.yml +++ /dev/null @@ -1,19 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pytest - - pytest-cov - - pytest-mock - - pip - - libpng - - jpeg - - ca-certificates - - hdf5 - - setuptools - - pip: - - future - - scipy - - av !=9.1.1, <10 - - dataclasses - - h5py diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh deleted file mode 100644 index 7c55c8144d9..00000000000 --- a/.circleci/unittest/windows/scripts/install.sh +++ /dev/null @@ -1,53 +0,0 @@ - -#!/usr/bin/env bash - -unset PYTORCH_VERSION -# For unittest, nightly PyTorch is used as the following section, -# so no need to set PYTORCH_VERSION. -# In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config. - -set -ex - -this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" -conda activate ./env - -# TODO, refactor the below logic to make it easy to understand how to get correct cuda_version. -if [ "${CU_VERSION:-}" == cpu ] ; then - cudatoolkit="cpuonly" - version="cpu" -else - if [[ ${#CU_VERSION} -eq 4 ]]; then - CUDA_VERSION="${CU_VERSION:2:1}.${CU_VERSION:3:1}" - elif [[ ${#CU_VERSION} -eq 5 ]]; then - CUDA_VERSION="${CU_VERSION:2:2}.${CU_VERSION:4:1}" - fi - - cuda_toolkit_pckg="cudatoolkit" - if [[ $CUDA_VERSION == 11.6 || $CUDA_VERSION == 11.7 || $CUDA_VERSION == 11.8 || $CUDA_VERSION == 12.1 ]]; then - cuda_toolkit_pckg="pytorch-cuda" - fi - - echo "Using CUDA $CUDA_VERSION as determined by CU_VERSION" - version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")" - cudatoolkit="${cuda_toolkit_pckg}=${version}" -fi - -printf "Installing PyTorch with %s\n" "${cudatoolkit}" -conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c nvidia "pytorch-${UPLOAD_CHANNEL}"::pytorch[build="*${version}*"] "${cudatoolkit}" - -torch_cuda=$(python -c "import torch; print(torch.cuda.is_available())") -echo torch.cuda.is_available is $torch_cuda - -if [ ! -z "${CUDA_VERSION:-}" ] ; then - if [ "$torch_cuda" == "False" ]; then - echo "torch with cuda installed but torch.cuda.is_available() is False" - exit 1 - fi -fi - -source "$this_dir/set_cuda_envs.sh" - -printf "* Installing torchvision\n" -"$this_dir/vc_env_helper.bat" python setup.py develop diff --git a/.circleci/unittest/windows/scripts/install_conda.bat b/.circleci/unittest/windows/scripts/install_conda.bat deleted file mode 100644 index 6052ad08b10..00000000000 --- a/.circleci/unittest/windows/scripts/install_conda.bat +++ /dev/null @@ -1 +0,0 @@ -start /wait "" "%miniconda_exe%" /S /InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /D=%tmp_conda% diff --git a/.circleci/unittest/windows/scripts/post_process.sh b/.circleci/unittest/windows/scripts/post_process.sh deleted file mode 100644 index 5c5cbb758a9..00000000000 --- a/.circleci/unittest/windows/scripts/post_process.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -e - -eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" -conda activate ./env diff --git a/.circleci/unittest/windows/scripts/run_test.sh b/.circleci/unittest/windows/scripts/run_test.sh deleted file mode 100644 index 802ad37f511..00000000000 --- a/.circleci/unittest/windows/scripts/run_test.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -e - -eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" -conda activate ./env - -this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source "$this_dir/set_cuda_envs.sh" - -python -m torch.utils.collect_env -pytest --junitxml=test-results/junit.xml -v --durations 20 diff --git a/.circleci/unittest/windows/scripts/set_cuda_envs.sh b/.circleci/unittest/windows/scripts/set_cuda_envs.sh deleted file mode 100644 index 7db3137b594..00000000000 --- a/.circleci/unittest/windows/scripts/set_cuda_envs.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -set -ex - -echo CU_VERSION is "${CU_VERSION}" -echo CUDA_VERSION is "${CUDA_VERSION}" - -# Currenly, CU_VERSION and CUDA_VERSION are not consistent. -# to understand this code, see https://github.com/pytorch/vision/issues/4443 -version="cpu" -if [[ ! -z "${CUDA_VERSION}" ]] ; then - version="$CUDA_VERSION" -else - if [[ ${#CU_VERSION} -eq 5 ]]; then - version="${CU_VERSION:2:2}.${CU_VERSION:4:1}" - fi -fi - -# Don't use if [[ "$version" == "cpu" ]]; then exit 0 fi. -# It would exit the shell. One result is cpu tests would not run if the shell exit. -# Unless there's an error, Don't exit. -if [[ "$version" != "cpu" ]]; then - # set cuda envs - export PATH="/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${version}/bin:/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${version}/libnvvp:$PATH" - export CUDA_PATH_V${version/./_}="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${version}" - export CUDA_PATH="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${version}" - - if [ ! -d "$CUDA_PATH" ]; then - echo "$CUDA_PATH" does not exist - exit 1 - fi - - if [ ! -f "${CUDA_PATH}\include\nvjpeg.h" ]; then - echo "nvjpeg does not exist" - exit 1 - fi - - # check cuda driver version - for path in '/c/Program Files/NVIDIA Corporation/NVSMI/nvidia-smi.exe' /c/Windows/System32/nvidia-smi.exe; do - if [[ -x "$path" ]]; then - "$path" || echo "true"; - break - fi - done - - which nvcc - nvcc --version - env | grep CUDA -fi diff --git a/.circleci/unittest/windows/scripts/setup_env.sh b/.circleci/unittest/windows/scripts/setup_env.sh deleted file mode 100644 index 84697875979..00000000000 --- a/.circleci/unittest/windows/scripts/setup_env.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -# This script is for setting up environment in which unit test is ran. -# To speed up the CI time, the resulting environment is cached. -# -# Do not install PyTorch and torchvision here, otherwise they also get cached. - -set -ex - -this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" - -cd "${root_dir}" - -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - export tmp_conda="$(echo $conda_dir | tr '/' '\\')" - export miniconda_exe="$(echo $root_dir | tr '/' '\\')\\miniconda.exe" - curl --output miniconda.exe https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -O - "$this_dir/install_conda.bat" - unset tmp_conda - unset miniconda_exe -fi - -eval "$(${conda_dir}/Scripts/conda.exe 'shell.bash' 'hook')" - -# 2. Create test environment at ./env -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" - -# 3. Install Conda dependencies -printf "* Installing dependencies (except PyTorch)\n" -conda env update --file "${this_dir}/environment.yml" --prune - -# 4. Downgrade setuptools on Python 3.7. -# See https://github.com/pytorch/vision/pull/5868 -if [[ "${PYTHON_VERSION}" == '3.7' ]]; then - pip install --upgrade setuptools==58.0.4 -fi diff --git a/.circleci/unittest/windows/scripts/vc_env_helper.bat b/.circleci/unittest/windows/scripts/vc_env_helper.bat deleted file mode 100644 index 9410135677a..00000000000 --- a/.circleci/unittest/windows/scripts/vc_env_helper.bat +++ /dev/null @@ -1,39 +0,0 @@ -@echo on - -set VC_VERSION_LOWER=16 -set VC_VERSION_UPPER=17 - -for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [%VC_VERSION_LOWER%^,%VC_VERSION_UPPER%^) -property installationPath`) do ( - if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( - set "VS15INSTALLDIR=%%i" - set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat" - goto vswhere - ) -) - -:vswhere -if "%VSDEVCMD_ARGS%" == "" ( - call "%VS15VCVARSALL%" x64 || exit /b 1 -) else ( - call "%VS15VCVARSALL%" x64 %VSDEVCMD_ARGS% || exit /b 1 -) - -@echo on - -set DISTUTILS_USE_SDK=1 - -set args=%1 -shift -:start -if [%1] == [] goto done -set args=%args% %1 -shift -goto start - -:done -if "%args%" == "" ( - echo Usage: vc_env_helper.bat [command] [args] - echo e.g. vc_env_helper.bat cl /c test.cpp -) - -%args% || exit /b 1 diff --git a/.github/scripts/unittest.sh b/.github/scripts/unittest.sh index 2a0b7154200..267edfedcbc 100755 --- a/.github/scripts/unittest.sh +++ b/.github/scripts/unittest.sh @@ -11,4 +11,5 @@ echo '::group::Install testing utilities' pip install --progress-bar=off pytest pytest-mock pytest-cov echo '::endgroup::' -pytest --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" -v --durations=25 +python test/smoke_test.py +#pytest --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" -v --durations=25 diff --git a/.github/workflows/build-conda-linux.yml b/.github/workflows/build-conda-linux.yml deleted file mode 100644 index e44bb877ed6..00000000000 --- a/.github/workflows/build-conda-linux.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Build Linux Conda - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: conda - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: "" - post-script: "" - conda-package-directory: packaging/torchvision - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_conda_linux.yml@main - with: - conda-package-directory: ${{ matrix.conda-package-directory }} - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-m1.yml b/.github/workflows/build-conda-m1.yml deleted file mode 100644 index 7006b6714aa..00000000000 --- a/.github/workflows/build-conda-m1.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Build M1 Conda - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: conda - os: macos-arm64 - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: "" - post-script: "" - conda-package-directory: packaging/torchvision - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_conda_macos.yml@main - with: - conda-package-directory: ${{ matrix.conda-package-directory }} - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - runner-type: macos-m1-12 - trigger-event: ${{ github.event_name }} - secrets: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-macos.yml b/.github/workflows/build-conda-macos.yml deleted file mode 100644 index 39d00534b3a..00000000000 --- a/.github/workflows/build-conda-macos.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Build Macos Conda - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: conda - os: macos - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: "" - post-script: "" - conda-package-directory: packaging/torchvision - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_conda_macos.yml@main - with: - conda-package-directory: ${{ matrix.conda-package-directory }} - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - runner-type: macos-12 - trigger-event: ${{ github.event_name }} - secrets: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-windows.yml b/.github/workflows/build-conda-windows.yml deleted file mode 100644 index 954d09d39ec..00000000000 --- a/.github/workflows/build-conda-windows.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Build Windows Conda - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: conda - os: windows - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: packaging/pre_build_script.sh - env-script: packaging/windows/internal/vc_env_helper.bat - post-script: "" - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_conda_windows.yml@main - with: - conda-package-directory: ${{ matrix.conda-package-directory }} - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-wheels-linux.yml b/.github/workflows/build-wheels-linux.yml deleted file mode 100644 index e997d648ec2..00000000000 --- a/.github/workflows/build-wheels-linux.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Build Linux Wheels - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-m1.yml b/.github/workflows/build-wheels-m1.yml deleted file mode 100644 index 66c7687acc9..00000000000 --- a/.github/workflows/build-wheels-m1.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Build M1 Wheels - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: macos-arm64 - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - runner-type: macos-m1-12 - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-macos.yml b/.github/workflows/build-wheels-macos.yml deleted file mode 100644 index 6c5ebc0fc37..00000000000 --- a/.github/workflows/build-wheels-macos.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Build Macos Wheels - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: macos - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - runner-type: macos-12 - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-windows.yml b/.github/workflows/build-wheels-windows.yml deleted file mode 100644 index 3d818ece538..00000000000 --- a/.github/workflows/build-wheels-windows.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Build Windows Wheels - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: windows - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: packaging/pre_build_script.sh - env-script: packaging/windows/internal/vc_env_helper.bat - post-script: "python packaging/wheel/relocate.py" - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_wheels_windows.yml@main - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-script: ${{ matrix.env-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index f4cc76db05e..00000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,115 +0,0 @@ -name: Docs - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - - v[0-9]+.[0-9]+.[0-9] - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - build: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - upload-artifact: docs - script: | - set -euo pipefail - - export PYTHON_VERSION=3.8 - export GPU_ARCH_TYPE=cpu - export GPU_ARCH_VERSION='' - ./.github/scripts/setup-env.sh - - # Prepare conda - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - conda activate ci - # FIXME: not sure why we need this. `ldd torchvision/video_reader.so` shows that it - # already links against the one pulled from conda. However, at runtime it pulls from - # /lib64 - # Should we maybe always do this in `./.github/scripts/setup-env.sh` so that we don't - # have to pay attention in all other workflows? - export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH}" - - cd docs - - echo '::group::Install doc requirements' - pip install --progress-bar=off -r requirements.txt - echo '::endgroup::' - - if [[ ${{ github.event_name }} == push && (${{ github.ref_type }} == tag || (${{ github.ref_type }} == branch && ${{ github.ref_name }} == release/*)) ]]; then - echo '::group::Enable version string sanitization' - # This environment variable just has to exist and must not be empty. The actual value is arbitrary. - # See docs/source/conf.py for details - export TORCHVISION_SANITIZE_VERSION_STR_IN_DOCS=1 - echo '::endgroup::' - fi - - # The runner does not have sufficient memory to run with as many processes as there are - # cores (`-j auto`). Thus, we limit to a single process (`-j 1`) here. - sed -i -e 's/-j auto/-j 1/' Makefile - make html - - cp -r build/html "${RUNNER_ARTIFACT_DIR}" - - # On PRs we also want to upload the docs into our S3 bucket for preview. - if [[ ${{ github.event_name == 'pull_request' }} ]]; then - cp -r build/html/* "${RUNNER_DOCS_DIR}" - fi - - upload: - needs: build - if: github.repository == 'pytorch/vision' && github.event_name == 'push' && - ((github.ref_type == 'branch' && github.ref_name == 'main') || github.ref_type == 'tag') - permissions: - contents: write - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - download-artifact: docs - ref: gh-pages - script: | - set -euo pipefail - - REF_TYPE=${{ github.ref_type }} - REF_NAME=${{ github.ref_name }} - - if [[ "${REF_TYPE}" == branch ]]; then - TARGET_FOLDER="${REF_NAME}" - elif [[ "${REF_TYPE}" == tag ]]; then - case "${REF_NAME}" in - *-rc*) - echo "Aborting upload since this is an RC tag: ${REF_NAME}" - exit 0 - ;; - *) - # Strip the leading "v" as well as the trailing patch version. For example: - # 'v0.15.2' -> '0.15' - TARGET_FOLDER=$(echo "${REF_NAME}" | sed 's/v\([0-9]\+\)\.\([0-9]\+\)\.[0-9]\+/\1.\2/') - ;; - esac - fi - echo "Target Folder: ${TARGET_FOLDER}" - - mkdir -p "${TARGET_FOLDER}" - rm -rf "${TARGET_FOLDER}"/* - mv "${RUNNER_ARTIFACT_DIR}"/html/* "${TARGET_FOLDER}" - git add "${TARGET_FOLDER}" || true - - if [[ "${TARGET_FOLDER}" == main ]]; then - mkdir -p _static - rm -rf _static/* - cp -r "${TARGET_FOLDER}"/_static/* _static - git add _static || true - fi - - git config user.name 'pytorchbot' - git config user.email 'soumith+bot@pytorch.org' - git commit -m "auto-generating sphinx docs" || true - git push diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index ec8d285c9f3..00000000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,107 +0,0 @@ -name: Lint - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: - -jobs: - python-source-and-configs: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - script: | - set -euo pipefail - - echo '::group::Setup environment' - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - conda create --name ci --quiet --yes python=3.8 pip - conda activate ci - echo '::endgroup::' - - echo '::group::Install lint tools' - pip install --progress-bar=off pre-commit - echo '::endgroup::' - - set +e - pre-commit run --all-files - - if [ $? -ne 0 ]; then - git --no-pager diff - exit 1 - fi - - c-source: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - script: | - set -euo pipefail - - echo '::group::Setup environment' - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - # clang-format needs some shared libraries that conflict with the system ones. Thus, we install them from conda - # and prepend the libraries to linker path to prioritize them. `ncurses=5` is only available on the conda-forge - # channel. Since we are not building or testing here, this is fine. - conda create --name ci --quiet --yes -c conda-forge python=3.8 ncurses=5 libgcc - conda activate ci - export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH}" - echo '::endgroup::' - - echo '::group::Install lint tools' - curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o ./clang-format - chmod +x ./clang-format - echo '::endgroup::' - - echo '::group::Lint C source' - set +e - ./.circleci/unittest/linux/scripts/run-clang-format.py -r torchvision/csrc --clang-format-executable ./clang-format - - if [ $? -ne 0 ]; then - git --no-pager diff - exit 1 - fi - echo '::endgroup::' - - - python-types: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - script: | - set -euo pipefail - - export PYTHON_VERSION=3.8 - export GPU_ARCH_TYPE=cpu - export GPU_ARCH_VERSION='' - - ./.github/scripts/setup-env.sh - - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - conda activate ci - - echo '::group::Install lint tools' - pip install --progress-bar=off mypy - echo '::endgroup::' - - echo '::group::Lint Python types' - mypy --install-types --non-interactive --config-file mypy.ini - echo '::endgroup::' - - bc: - if: github.event.pull_request - runs-on: ubuntu-latest - steps: - - name: Run BC Lint Action - uses: pytorch/test-infra/.github/actions/bc-lint@main - with: - repo: ${{ github.event.pull_request.head.repo.full_name }} - base_sha: ${{ github.event.pull_request.base.sha }} - head_sha: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/pr-labels.yml b/.github/workflows/pr-labels.yml deleted file mode 100644 index 20c37e4fd88..00000000000 --- a/.github/workflows/pr-labels.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: pr-labels - -on: - push: - branches: - - main - -jobs: - is-properly-labeled: - runs-on: ubuntu-latest - - steps: - - name: Set up python - uses: actions/setup-python@v2 - - - name: Install requests - run: pip install requests - - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Process commit and find merger responsible for labeling - id: commit - run: echo "::set-output name=merger::$(python .github/process_commit.py ${{ github.sha }})" - - - name: Ping merger responsible for labeling if necessary - if: ${{ steps.commit.outputs.merger != '' }} - uses: mshick/add-pr-comment@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - message: | - Hey ${{ steps.commit.outputs.merger }}! - - You merged this PR, but no labels were added. The list of valid labels is available at https://github.com/pytorch/vision/blob/main/.github/process_commit.py diff --git a/.github/workflows/prototype-tests-linux-gpu.yml b/.github/workflows/prototype-tests-linux-gpu.yml deleted file mode 100644 index dee425054d5..00000000000 --- a/.github/workflows/prototype-tests-linux-gpu.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Prototype tests on Linux - -on: - pull_request: - -jobs: - unittests-prototype: - strategy: - matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.11" - runner: ["linux.12xlarge"] - gpu-arch-type: ["cpu"] - include: - - python-version: "3.8" - runner: linux.g5.4xlarge.nvidia.gpu - gpu-arch-type: cuda - gpu-arch-version: "11.7" - fail-fast: false - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - runner: ${{ matrix.runner }} - gpu-arch-type: ${{ matrix.gpu-arch-type }} - gpu-arch-version: ${{ matrix.gpu-arch-version }} - timeout: 120 - script: | - set -euo pipefail - - export PYTHON_VERSION=${{ matrix.python-version }} - export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} - export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} - ./.github/scripts/setup-env.sh - - # Prepare conda - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - conda activate ci - - echo '::group::Install testing utilities' - pip install --progress-bar=off pytest pytest-mock pytest-cov - echo '::endgroup::' - - # We don't want to run the prototype datasets tests. Since the positional glob into `pytest`, i.e. - # `test/test_prototype*.py` takes the highest priority, neither `--ignore` nor `--ignore-glob` can help us here. - rm test/test_prototype_datasets*.py - pytest \ - -v --durations=25 \ - --cov=torchvision/prototype --cov-report=term-missing \ - --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" \ - test/test_prototype_*.py diff --git a/.github/workflows/tests-schedule.yml b/.github/workflows/tests-schedule.yml deleted file mode 100644 index 5426fdc997a..00000000000 --- a/.github/workflows/tests-schedule.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: tests - -on: - pull_request: - paths: - - "test/test_datasets_download.py" - - ".github/failed_schedule_issue_template.md" - - ".github/workflows/tests-schedule.yml" - - schedule: - - cron: "0 9 * * *" - -jobs: - download: - runs-on: ubuntu-latest - - steps: - - name: Set up python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Upgrade system packages - run: python -m pip install --upgrade pip setuptools wheel - - - name: SSL - run: python -c 'import ssl; print(ssl.OPENSSL_VERSION)' - - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Install torch nightly build - run: pip install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - - - name: Install torchvision - run: pip install --no-build-isolation --editable . - - - name: Install all optional dataset requirements - run: pip install scipy pycocotools lmdb requests - - - name: Install tests requirements - run: pip install pytest - - - name: Run tests - run: pytest -ra -v test/test_datasets_download.py - - - uses: JasonEtco/create-an-issue@v2.4.0 - name: Create issue if download tests failed - if: failure() && github.event_name == 'schedule' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} - WORKFLOW: ${{ github.workflow }} - JOB: ${{ github.job }} - ID: ${{ github.run_id }} - with: - filename: .github/failed_schedule_issue_template.md diff --git a/.github/workflows/update-viablestrict.yml b/.github/workflows/update-viablestrict.yml deleted file mode 100644 index 2d9c2265676..00000000000 --- a/.github/workflows/update-viablestrict.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Update viable/strict - -on: - pull_request: - paths: - - .github/workflows/update-viablestrict.yml - schedule: - - cron: 10,40 * * * * - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }} - cancel-in-progress: false - -jobs: - do_update_viablestrict: - uses: pytorch/test-infra/.github/workflows/update-viablestrict.yml@main - with: - repository: pytorch/vision - required_checks: "Build Linux,Build M1,Build Macos,Tests on Linux,Tests on macOS,Docs,Lint" - secrets: - ROCKSET_API_KEY: ${{ secrets.ROCKSET_API_KEY }} - GITHUB_DEPLOY_KEY : ${{ secrets.VISION_GITHUB_DEPLOY_KEY }} From 09295b3ea6996f03dda773e240c9f505e404cfe6 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 May 2023 11:59:57 +0200 Subject: [PATCH 3/7] Revert "[REVERTME] disable unrelated CI" This reverts commit f30e4136a63985cf9afc5fca028e246ce5d6ebd4. --- .circleci/.gitignore | 1 + .circleci/build_docs/commit_docs.sh | 35 ++ .circleci/config.yml | 271 ++++++++++++++ .circleci/config.yml.in | 251 +++++++++++++ .circleci/regenerate.py | 58 +++ .circleci/smoke_test/docker/Dockerfile | 34 ++ .../android/scripts/binary_android_build.sh | 27 ++ .../android/scripts/binary_android_upload.sh | 34 ++ .../android/scripts/install_gradle.sh | 19 + .../unittest/ios/scripts/binary_ios_build.sh | 47 +++ .../unittest/ios/scripts/binary_ios_upload.sh | 42 +++ .../unittest/linux/scripts/environment.yml | 16 + .circleci/unittest/linux/scripts/install.sh | 45 +++ .../unittest/linux/scripts/post_process.sh | 6 + .../linux/scripts/run-clang-format.py | 331 ++++++++++++++++++ .circleci/unittest/linux/scripts/run_test.sh | 22 ++ .circleci/unittest/linux/scripts/setup_env.sh | 47 +++ .../unittest/windows/scripts/environment.yml | 19 + .circleci/unittest/windows/scripts/install.sh | 53 +++ .../windows/scripts/install_conda.bat | 1 + .../unittest/windows/scripts/post_process.sh | 6 + .../unittest/windows/scripts/run_test.sh | 12 + .../unittest/windows/scripts/set_cuda_envs.sh | 48 +++ .../unittest/windows/scripts/setup_env.sh | 45 +++ .../windows/scripts/vc_env_helper.bat | 39 +++ .github/scripts/unittest.sh | 3 +- .github/workflows/build-conda-linux.yml | 51 +++ .github/workflows/build-conda-m1.yml | 52 +++ .github/workflows/build-conda-macos.yml | 52 +++ .github/workflows/build-conda-windows.yml | 51 +++ .github/workflows/build-wheels-linux.yml | 50 +++ .github/workflows/build-wheels-m1.yml | 51 +++ .github/workflows/build-wheels-macos.yml | 51 +++ .github/workflows/build-wheels-windows.yml | 52 +++ .github/workflows/docs.yml | 115 ++++++ .github/workflows/lint.yml | 107 ++++++ .github/workflows/pr-labels.yml | 35 ++ .../workflows/prototype-tests-linux-gpu.yml | 54 +++ .github/workflows/tests-schedule.yml | 57 +++ .github/workflows/update-viablestrict.yml | 23 ++ 40 files changed, 2311 insertions(+), 2 deletions(-) create mode 100644 .circleci/.gitignore create mode 100755 .circleci/build_docs/commit_docs.sh create mode 100644 .circleci/config.yml create mode 100644 .circleci/config.yml.in create mode 100755 .circleci/regenerate.py create mode 100644 .circleci/smoke_test/docker/Dockerfile create mode 100644 .circleci/unittest/android/scripts/binary_android_build.sh create mode 100644 .circleci/unittest/android/scripts/binary_android_upload.sh create mode 100755 .circleci/unittest/android/scripts/install_gradle.sh create mode 100755 .circleci/unittest/ios/scripts/binary_ios_build.sh create mode 100644 .circleci/unittest/ios/scripts/binary_ios_upload.sh create mode 100644 .circleci/unittest/linux/scripts/environment.yml create mode 100755 .circleci/unittest/linux/scripts/install.sh create mode 100755 .circleci/unittest/linux/scripts/post_process.sh create mode 100755 .circleci/unittest/linux/scripts/run-clang-format.py create mode 100755 .circleci/unittest/linux/scripts/run_test.sh create mode 100755 .circleci/unittest/linux/scripts/setup_env.sh create mode 100644 .circleci/unittest/windows/scripts/environment.yml create mode 100644 .circleci/unittest/windows/scripts/install.sh create mode 100644 .circleci/unittest/windows/scripts/install_conda.bat create mode 100644 .circleci/unittest/windows/scripts/post_process.sh create mode 100644 .circleci/unittest/windows/scripts/run_test.sh create mode 100644 .circleci/unittest/windows/scripts/set_cuda_envs.sh create mode 100644 .circleci/unittest/windows/scripts/setup_env.sh create mode 100644 .circleci/unittest/windows/scripts/vc_env_helper.bat create mode 100644 .github/workflows/build-conda-linux.yml create mode 100644 .github/workflows/build-conda-m1.yml create mode 100644 .github/workflows/build-conda-macos.yml create mode 100644 .github/workflows/build-conda-windows.yml create mode 100644 .github/workflows/build-wheels-linux.yml create mode 100644 .github/workflows/build-wheels-m1.yml create mode 100644 .github/workflows/build-wheels-macos.yml create mode 100644 .github/workflows/build-wheels-windows.yml create mode 100644 .github/workflows/docs.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/pr-labels.yml create mode 100644 .github/workflows/prototype-tests-linux-gpu.yml create mode 100644 .github/workflows/tests-schedule.yml create mode 100644 .github/workflows/update-viablestrict.yml diff --git a/.circleci/.gitignore b/.circleci/.gitignore new file mode 100644 index 00000000000..485dee64bcf --- /dev/null +++ b/.circleci/.gitignore @@ -0,0 +1 @@ +.idea diff --git a/.circleci/build_docs/commit_docs.sh b/.circleci/build_docs/commit_docs.sh new file mode 100755 index 00000000000..04e3538fefc --- /dev/null +++ b/.circleci/build_docs/commit_docs.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -ex + + +if [ "$2" == "" ]; then + echo call as "$0" "" "" + echo where src is the root of the built documentation git checkout and + echo branch should be "main" or "1.7" or so + exit 1 +fi + +src=$1 +target=$2 + +echo "committing docs from ${src} to ${target}" + +pushd "${src}" +git checkout gh-pages +mkdir -p ./"${target}" +rm -rf ./"${target}"/* +cp -r "${src}/docs/build/html/"* ./"$target" +if [ "${target}" == "main" ]; then + mkdir -p ./_static + rm -rf ./_static/* + cp -r "${src}/docs/build/html/_static/"* ./_static + git add --all ./_static || true +fi +git add --all ./"${target}" || true +git config user.email "soumith+bot@pytorch.org" +git config user.name "pytorchbot" +# If there aren't changes, don't make a commit; push is no-op +git commit -m "auto-generating sphinx docs" || true +git remote add https https://github.com/pytorch/vision.git +git push -u https gh-pages diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000000..0463007af46 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,271 @@ +version: 2.1 + +# How to test the Linux jobs: +# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ +# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_win_wheel_py3.8 +# - Replace binary_win_wheel_py3.8 with the name of the job you want to test. +# Job names are 'name:' key. + +executors: + windows-cpu: + machine: + resource_class: windows.xlarge + image: windows-server-2019-vs2019:stable + shell: bash.exe + + windows-gpu: + machine: + resource_class: windows.gpu.nvidia.medium + image: windows-server-2019-nvidia:stable + shell: bash.exe + +commands: + checkout_merge: + description: "checkout merge branch" + steps: + - checkout +# - run: +# name: Checkout merge branch +# command: | +# set -ex +# BRANCH=$(git rev-parse --abbrev-ref HEAD) +# if [[ "$BRANCH" != "main" ]]; then +# git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH} +# git checkout "merged/$CIRCLE_BRANCH" +# fi + designate_upload_channel: + description: "inserts the correct upload channel into ${BASH_ENV}" + steps: + - run: + name: adding UPLOAD_CHANNEL to BASH_ENV + command: | + our_upload_channel=nightly + # On tags upload to test instead + if [[ -n "${CIRCLE_TAG}" ]]; then + our_upload_channel=test + fi + echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} + + pip_install: + parameters: + args: + type: string + descr: + type: string + default: "" + user: + type: boolean + default: true + steps: + - run: + name: > + <<^ parameters.descr >> pip install << parameters.args >> <> + <<# parameters.descr >> << parameters.descr >> <> + command: > + pip install + <<# parameters.user >> --user <> + --progress-bar=off + << parameters.args >> + +binary_common: &binary_common + parameters: + # Edit these defaults to do a release + build_version: + description: "version number of release binary; by default, build a nightly" + type: string + default: "" + pytorch_version: + description: "PyTorch version to build against; by default, use a nightly" + type: string + default: "" + # Don't edit these + python_version: + description: "Python version to build against (e.g., 3.8)" + type: string + cu_version: + description: "CUDA version to build against, in CU format (e.g., cpu or cu100)" + type: string + default: "cpu" + unicode_abi: + description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" + type: string + default: "" + wheel_docker_image: + description: "Wheel only: what docker image to use" + type: string + default: "" + conda_docker_image: + description: "Conda only: what docker image to use" + type: string + default: "pytorch/conda-builder:cpu" + environment: + PYTHON_VERSION: << parameters.python_version >> + PYTORCH_VERSION: << parameters.pytorch_version >> + UNICODE_ABI: << parameters.unicode_abi >> + CU_VERSION: << parameters.cu_version >> + MACOSX_DEPLOYMENT_TARGET: 10.9 + +smoke_test_common: &smoke_test_common + <<: *binary_common + docker: + - image: torchvision/smoke_test:latest + +jobs: + circleci_consistency: + docker: + - image: cimg/python:3.8 + steps: + - checkout + - pip_install: + args: jinja2 pyyaml + - run: + name: Check CircleCI config consistency + command: | + python .circleci/regenerate.py + git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) + + smoke_test_docker_image_build: + machine: + image: ubuntu-2004:202104-01 + resource_class: large + environment: + image_name: torchvision/smoke_test + steps: + - checkout + - designate_upload_channel + - run: + name: Build and push Docker image + no_output_timeout: "1h" + command: | + set +x + echo "${DOCKER_HUB_TOKEN}" | docker login --username "${DOCKER_HUB_USERNAME}" --password-stdin + set -x + cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID} + docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} ${image_name}:latest + docker push ${image_name}:${CIRCLE_WORKFLOW_ID} + docker push ${image_name}:latest + + cmake_linux_cpu: + <<: *binary_common + docker: + - image: "pytorch/manylinux-cpu" + resource_class: 2xlarge+ + steps: + - checkout_merge + - designate_upload_channel + - run: + name: Setup conda + command: .circleci/unittest/linux/scripts/setup_env.sh + - run: packaging/build_cmake.sh + + cmake_linux_gpu: + <<: *binary_common + machine: + image: ubuntu-2004-cuda-11.4:202110-01 + resource_class: gpu.nvidia.small + steps: + - checkout_merge + - designate_upload_channel + - run: + name: Setup conda + command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh + - run: + name: Build torchvision C++ distribution and test + no_output_timeout: 30m + command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh + + cmake_macos_cpu: + <<: *binary_common + macos: + xcode: "14.0" + steps: + - checkout_merge + - designate_upload_channel + - run: + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + conda install -yq conda-build cmake python=<< parameters.python_version >> + packaging/build_cmake.sh + + cmake_windows_cpu: + <<: *binary_common + executor: + name: windows-cpu + steps: + - checkout_merge + - designate_upload_channel + - run: + command: | + set -ex + source packaging/windows/internal/vc_install_helper.sh + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" + conda activate base + conda create -yn python39 python=3.9 + conda activate python39 + packaging/build_cmake.sh + + cmake_windows_gpu: + <<: *binary_common + executor: + name: windows-gpu + steps: + - checkout_merge + - designate_upload_channel + - run: + name: Update CUDA driver + command: packaging/windows/internal/driver_update.bat + - run: + command: | + set -ex + source packaging/windows/internal/vc_install_helper.sh + packaging/windows/internal/cuda_install.bat + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" + conda activate + conda update -y conda + conda create -yn python39 python=3.9 + conda activate python39 + packaging/build_cmake.sh + + +workflows: + lint: + jobs: + - circleci_consistency + + cmake: + jobs: + - cmake_linux_cpu: + cu_version: cpu + name: cmake_linux_cpu + python_version: '3.8' + - cmake_linux_gpu: + cu_version: cu117 + name: cmake_linux_gpu + python_version: '3.8' + wheel_docker_image: pytorch/manylinux-cuda117 + - cmake_windows_cpu: + cu_version: cpu + name: cmake_windows_cpu + python_version: '3.8' + - cmake_windows_gpu: + cu_version: cu117 + name: cmake_windows_gpu + python_version: '3.8' + - cmake_macos_cpu: + cu_version: cpu + name: cmake_macos_cpu + python_version: '3.8' + + docker_build: + triggers: + - schedule: + cron: "0 10 * * 0" + filters: + branches: + only: + - main + jobs: + - smoke_test_docker_image_build: + context: org-member diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in new file mode 100644 index 00000000000..f3a88f2d987 --- /dev/null +++ b/.circleci/config.yml.in @@ -0,0 +1,251 @@ +version: 2.1 + +# How to test the Linux jobs: +# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ +# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_win_wheel_py3.8 +# - Replace binary_win_wheel_py3.8 with the name of the job you want to test. +# Job names are 'name:' key. + +executors: + windows-cpu: + machine: + resource_class: windows.xlarge + image: windows-server-2019-vs2019:stable + shell: bash.exe + + windows-gpu: + machine: + resource_class: windows.gpu.nvidia.medium + image: windows-server-2019-nvidia:stable + shell: bash.exe + +commands: + checkout_merge: + description: "checkout merge branch" + steps: + - checkout +# - run: +# name: Checkout merge branch +# command: | +# set -ex +# BRANCH=$(git rev-parse --abbrev-ref HEAD) +# if [[ "$BRANCH" != "main" ]]; then +# git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH} +# git checkout "merged/$CIRCLE_BRANCH" +# fi + designate_upload_channel: + description: "inserts the correct upload channel into ${BASH_ENV}" + steps: + - run: + name: adding UPLOAD_CHANNEL to BASH_ENV + command: | + our_upload_channel=nightly + # On tags upload to test instead + if [[ -n "${CIRCLE_TAG}" ]]; then + our_upload_channel=test + fi + echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} + + pip_install: + parameters: + args: + type: string + descr: + type: string + default: "" + user: + type: boolean + default: true + steps: + - run: + name: > + <<^ parameters.descr >> pip install << parameters.args >> <> + <<# parameters.descr >> << parameters.descr >> <> + command: > + pip install + <<# parameters.user >> --user <> + --progress-bar=off + << parameters.args >> + +binary_common: &binary_common + parameters: + # Edit these defaults to do a release + build_version: + description: "version number of release binary; by default, build a nightly" + type: string + default: "" + pytorch_version: + description: "PyTorch version to build against; by default, use a nightly" + type: string + default: "" + # Don't edit these + python_version: + description: "Python version to build against (e.g., 3.8)" + type: string + cu_version: + description: "CUDA version to build against, in CU format (e.g., cpu or cu100)" + type: string + default: "cpu" + unicode_abi: + description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" + type: string + default: "" + wheel_docker_image: + description: "Wheel only: what docker image to use" + type: string + default: "" + conda_docker_image: + description: "Conda only: what docker image to use" + type: string + default: "pytorch/conda-builder:cpu" + environment: + PYTHON_VERSION: << parameters.python_version >> + PYTORCH_VERSION: << parameters.pytorch_version >> + UNICODE_ABI: << parameters.unicode_abi >> + CU_VERSION: << parameters.cu_version >> + MACOSX_DEPLOYMENT_TARGET: 10.9 + +smoke_test_common: &smoke_test_common + <<: *binary_common + docker: + - image: torchvision/smoke_test:latest + +jobs: + circleci_consistency: + docker: + - image: cimg/python:3.8 + steps: + - checkout + - pip_install: + args: jinja2 pyyaml + - run: + name: Check CircleCI config consistency + command: | + python .circleci/regenerate.py + git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) + + smoke_test_docker_image_build: + machine: + image: ubuntu-2004:202104-01 + resource_class: large + environment: + image_name: torchvision/smoke_test + steps: + - checkout + - designate_upload_channel + - run: + name: Build and push Docker image + no_output_timeout: "1h" + command: | + set +x + echo "${DOCKER_HUB_TOKEN}" | docker login --username "${DOCKER_HUB_USERNAME}" --password-stdin + set -x + cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID} + docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} ${image_name}:latest + docker push ${image_name}:${CIRCLE_WORKFLOW_ID} + docker push ${image_name}:latest + + cmake_linux_cpu: + <<: *binary_common + docker: + - image: "pytorch/manylinux-cpu" + resource_class: 2xlarge+ + steps: + - checkout_merge + - designate_upload_channel + - run: + name: Setup conda + command: .circleci/unittest/linux/scripts/setup_env.sh + - run: packaging/build_cmake.sh + + cmake_linux_gpu: + <<: *binary_common + machine: + image: ubuntu-2004-cuda-11.4:202110-01 + resource_class: gpu.nvidia.small + steps: + - checkout_merge + - designate_upload_channel + - run: + name: Setup conda + command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh + - run: + name: Build torchvision C++ distribution and test + no_output_timeout: 30m + command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh + + cmake_macos_cpu: + <<: *binary_common + macos: + xcode: "14.0" + steps: + - checkout_merge + - designate_upload_channel + - run: + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + conda install -yq conda-build cmake python=<< parameters.python_version >> + packaging/build_cmake.sh + + cmake_windows_cpu: + <<: *binary_common + executor: + name: windows-cpu + steps: + - checkout_merge + - designate_upload_channel + - run: + command: | + set -ex + source packaging/windows/internal/vc_install_helper.sh + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" + conda activate base + conda create -yn python39 python=3.9 + conda activate python39 + packaging/build_cmake.sh + + cmake_windows_gpu: + <<: *binary_common + executor: + name: windows-gpu + steps: + - checkout_merge + - designate_upload_channel + - run: + name: Update CUDA driver + command: packaging/windows/internal/driver_update.bat + - run: + command: | + set -ex + source packaging/windows/internal/vc_install_helper.sh + packaging/windows/internal/cuda_install.bat + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" + conda activate + conda update -y conda + conda create -yn python39 python=3.9 + conda activate python39 + packaging/build_cmake.sh + + +workflows: + lint: + jobs: + - circleci_consistency + + cmake: + jobs: + {{ cmake_workflows() }} + + docker_build: + triggers: + - schedule: + cron: "0 10 * * 0" + filters: + branches: + only: + - main + jobs: + - smoke_test_docker_image_build: + context: org-member diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py new file mode 100755 index 00000000000..e7b8db3d56e --- /dev/null +++ b/.circleci/regenerate.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +""" +This script should use a very simple, functional programming style. +Avoid Jinja macros in favor of native Python functions. + +Don't go overboard on code generation; use Python only to generate +content that can't be easily declared statically using CircleCI's YAML API. + +Data declarations (e.g. the nested loops for defining the configuration matrix) +should be at the top of the file for easy updating. + +See this comment for design rationale: +https://github.com/pytorch/vision/pull/1321#issuecomment-531033978 +""" + +import os.path + +import jinja2 +import yaml +from jinja2 import select_autoescape + + +def indent(indentation, data_list): + return ("\n" + " " * indentation).join(yaml.dump(data_list, default_flow_style=False).splitlines()) + + +def cmake_workflows(indentation=6): + jobs = [] + python_version = "3.8" + for os_type in ["linux", "windows", "macos"]: + # Skip OSX CUDA + device_types = ["cpu", "gpu"] if os_type != "macos" else ["cpu"] + for device in device_types: + job = {"name": f"cmake_{os_type}_{device}", "python_version": python_version} + + job["cu_version"] = "cu117" if device == "gpu" else "cpu" + if device == "gpu" and os_type == "linux": + job["wheel_docker_image"] = "pytorch/manylinux-cuda117" + jobs.append({f"cmake_{os_type}_{device}": job}) + return indent(indentation, jobs) + + +if __name__ == "__main__": + d = os.path.dirname(__file__) + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(d), + lstrip_blocks=True, + autoescape=select_autoescape(enabled_extensions=("html", "xml")), + keep_trailing_newline=True, + ) + + with open(os.path.join(d, "config.yml"), "w") as f: + f.write( + env.get_template("config.yml.in").render( + cmake_workflows=cmake_workflows, + ) + ) diff --git a/.circleci/smoke_test/docker/Dockerfile b/.circleci/smoke_test/docker/Dockerfile new file mode 100644 index 00000000000..34bdcda1053 --- /dev/null +++ b/.circleci/smoke_test/docker/Dockerfile @@ -0,0 +1,34 @@ +# this Dockerfile is for torchvision smoke test, it will be created periodically via CI system +# if you need to do it locally, follow below steps once you have Docker installed +# assuming you're within the directory where this Dockerfile located +# $ docker build . -t torchvision/smoketest + +# if you want to push to aws ecr, make sure you have the rights to write to ECR, then run +# $ eval $(aws ecr get-login --region us-east-1 --no-include-email) +# $ export MYTAG=localbuild ## you can choose whatever tag you like +# $ docker tag torchvision/smoketest 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchvision/smoke_test:${MYTAG} +# $ docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchvision/smoke_test:${MYTAG} + +FROM ubuntu:latest + +RUN apt-get -qq update && apt-get -qq -y install curl bzip2 libsox-fmt-all \ + && curl -sSL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh \ + && bash /tmp/miniconda.sh -bfp /usr/local \ + && rm -rf /tmp/miniconda.sh \ + && conda install -y python=3 \ + && conda update conda \ + && apt-get -qq -y remove curl bzip2 \ + && apt-get -qq -y autoremove \ + && apt-get autoclean \ + && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log \ + && conda clean --all --yes + +ENV PATH /opt/conda/bin:$PATH + +RUN conda create -y --name python3.7 python=3.7 +RUN conda create -y --name python3.8 python=3.8 +RUN conda create -y --name python3.9 python=3.9 +RUN conda create -y --name python3.10 python=3.10 +SHELL [ "/bin/bash", "-c" ] +RUN echo "source /usr/local/etc/profile.d/conda.sh" >> ~/.bashrc +CMD [ "/bin/bash"] diff --git a/.circleci/unittest/android/scripts/binary_android_build.sh b/.circleci/unittest/android/scripts/binary_android_build.sh new file mode 100644 index 00000000000..0d8c0d47d8a --- /dev/null +++ b/.circleci/unittest/android/scripts/binary_android_build.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -ex -o pipefail + +echo "DIR: $(pwd)" +echo "ANDROID_HOME=${ANDROID_HOME}" +echo "ANDROID_NDK_HOME=${ANDROID_NDK_HOME}" +echo "JAVA_HOME=${JAVA_HOME}" + +WORKSPACE=/home/circleci/workspace +VISION_ANDROID=/home/circleci/project/android + +. /home/circleci/project/.circleci/unittest/android/scripts/install_gradle.sh + +GRADLE_LOCAL_PROPERTIES=${VISION_ANDROID}/local.properties +rm -f $GRADLE_LOCAL_PROPERTIES + +echo "sdk.dir=${ANDROID_HOME}" >> $GRADLE_LOCAL_PROPERTIES +echo "ndk.dir=${ANDROID_NDK_HOME}" >> $GRADLE_LOCAL_PROPERTIES + +echo "GRADLE_PATH $GRADLE_PATH" +echo "GRADLE_HOME $GRADLE_HOME" + +${GRADLE_PATH} --scan --stacktrace --debug --no-daemon -p ${VISION_ANDROID} assemble || true + +mkdir -p ~/workspace/artifacts +find . -type f -name *aar -print | xargs tar cfvz ~/workspace/artifacts/artifacts-aars.tgz +find . -type f -name *apk -print | xargs tar cfvz ~/workspace/artifacts/artifacts-apks.tgz diff --git a/.circleci/unittest/android/scripts/binary_android_upload.sh b/.circleci/unittest/android/scripts/binary_android_upload.sh new file mode 100644 index 00000000000..1472a877d90 --- /dev/null +++ b/.circleci/unittest/android/scripts/binary_android_upload.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -ex -o pipefail + +echo "DIR: $(pwd)" +echo "ANDROID_HOME=${ANDROID_HOME}" +echo "ANDROID_NDK_HOME=${ANDROID_NDK_HOME}" +echo "JAVA_HOME=${JAVA_HOME}" + +WORKSPACE=/home/circleci/workspace +VISION_ANDROID=/home/circleci/project/android + +. /home/circleci/project/.circleci/unittest/android/scripts/install_gradle.sh + +GRADLE_LOCAL_PROPERTIES=${VISION_ANDROID}/local.properties +rm -f $GRADLE_LOCAL_PROPERTIES +GRADLE_PROPERTIES=/home/circleci/project/android/gradle.properties + +echo "sdk.dir=${ANDROID_HOME}" >> $GRADLE_LOCAL_PROPERTIES +echo "ndk.dir=${ANDROID_NDK_HOME}" >> $GRADLE_LOCAL_PROPERTIES + +echo "SONATYPE_NEXUS_USERNAME=${SONATYPE_NEXUS_USERNAME}" >> $GRADLE_PROPERTIES +echo "mavenCentralRepositoryUsername=${SONATYPE_NEXUS_USERNAME}" >> $GRADLE_PROPERTIES +echo "SONATYPE_NEXUS_PASSWORD=${SONATYPE_NEXUS_PASSWORD}" >> $GRADLE_PROPERTIES +echo "mavenCentralRepositoryPassword=${SONATYPE_NEXUS_PASSWORD}" >> $GRADLE_PROPERTIES + +echo "signing.keyId=${ANDROID_SIGN_KEY}" >> $GRADLE_PROPERTIES +echo "signing.password=${ANDROID_SIGN_PASS}" >> $GRADLE_PROPERTIES + +cat /home/circleci/project/android/gradle.properties | grep VERSION + +${GRADLE_PATH} --scan --stacktrace --debug --no-daemon -p ${VISION_ANDROID} ops:uploadArchives + +mkdir -p ~/workspace/artifacts +find . -type f -name *aar -print | xargs tar cfvz ~/workspace/artifacts/artifacts-aars.tgz diff --git a/.circleci/unittest/android/scripts/install_gradle.sh b/.circleci/unittest/android/scripts/install_gradle.sh new file mode 100755 index 00000000000..ff90c657e65 --- /dev/null +++ b/.circleci/unittest/android/scripts/install_gradle.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -ex + +_https_amazon_aws=https://downloads.gradle-dn.com/distributions +GRADLE_VERSION=6.8.3 + +_gradle_home=/opt/gradle +sudo rm -rf $gradle_home +sudo mkdir -p $_gradle_home + +curl --silent --output /tmp/gradle.zip --retry 3 $_https_amazon_aws/gradle-${GRADLE_VERSION}-bin.zip + +sudo unzip -q /tmp/gradle.zip -d $_gradle_home +rm /tmp/gradle.zip + +sudo chmod -R 777 $_gradle_home + +export GRADLE_HOME=$_gradle_home/gradle-$GRADLE_VERSION +export GRADLE_PATH=${GRADLE_HOME}/bin/gradle diff --git a/.circleci/unittest/ios/scripts/binary_ios_build.sh b/.circleci/unittest/ios/scripts/binary_ios_build.sh new file mode 100755 index 00000000000..1f117481f2d --- /dev/null +++ b/.circleci/unittest/ios/scripts/binary_ios_build.sh @@ -0,0 +1,47 @@ +#!/bin/bash +set -ex -o pipefail + +echo "" +echo "DIR: $(pwd)" +WORKSPACE=/Users/distiller/workspace +PROJ_ROOT_IOS=/Users/distiller/project/ios +PYTORCH_IOS_NIGHTLY_NAME=libtorch_ios_nightly_build.zip +export TCLLIBPATH="/usr/local/lib" + +# install conda +curl --retry 3 -o ~/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh +chmod +x ~/conda.sh +/bin/bash ~/conda.sh -b -p ~/anaconda +export PATH="~/anaconda/bin:${PATH}" +source ~/anaconda/bin/activate + +# install dependencies +conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi requests wget --yes +conda install -c conda-forge valgrind --yes +export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} + +# sync submodules +cd ${PROJ_ROOT_IOS} +git submodule sync +git submodule update --init --recursive + +# download pytorch-iOS nightly build and unzip it +mkdir -p ${PROJ_ROOT_IOS}/lib +mkdir -p ${PROJ_ROOT_IOS}/build +mkdir -p ${PROJ_ROOT_IOS}/pytorch +TORCH_ROOT="${PROJ_ROOT_IOS}/pytorch" + +cd ${TORCH_ROOT} +wget https://ossci-ios-build.s3.amazonaws.com/${PYTORCH_IOS_NIGHTLY_NAME} +mkdir -p ./build_ios +unzip -d ./build_ios ./${PYTORCH_IOS_NIGHTLY_NAME} + +LIBTORCH_HEADER_ROOT="${TORCH_ROOT}/build_ios/install/include" +cd ${PROJ_ROOT_IOS} +IOS_ARCH=${IOS_ARCH} LIBTORCH_HEADER_ROOT=${LIBTORCH_HEADER_ROOT} ./build_ios.sh +rm -rf ${TORCH_ROOT} + +# store the binary +DEST_DIR=${WORKSPACE}/ios/${IOS_ARCH} +mkdir -p ${DEST_DIR} +cp ${PROJ_ROOT_IOS}/lib/*.a ${DEST_DIR} diff --git a/.circleci/unittest/ios/scripts/binary_ios_upload.sh b/.circleci/unittest/ios/scripts/binary_ios_upload.sh new file mode 100644 index 00000000000..ce56388e5da --- /dev/null +++ b/.circleci/unittest/ios/scripts/binary_ios_upload.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -ex -o pipefail + +echo "" +echo "DIR: $(pwd)" + +WORKSPACE=/Users/distiller/workspace +PROJ_ROOT=/Users/distiller/project +ARTIFACTS_DIR=${WORKSPACE}/ios +ls ${ARTIFACTS_DIR} +ZIP_DIR=${WORKSPACE}/zip +mkdir -p ${ZIP_DIR}/install/lib + +# build a FAT bianry +cd ${ZIP_DIR}/install/lib +libs=("${ARTIFACTS_DIR}/x86_64/libtorchvision_ops.a" "${ARTIFACTS_DIR}/arm64/libtorchvision_ops.a") +lipo -create "${libs[@]}" -o ${ZIP_DIR}/install/lib/libtorchvision_ops.a +lipo -i ${ZIP_DIR}/install/lib/*.a + +# copy the license +cp ${PROJ_ROOT}/LICENSE ${ZIP_DIR}/ +# zip the library +ZIPFILE=libtorchvision_ops_ios_nightly_build.zip +cd ${ZIP_DIR} +#for testing +touch version.txt +echo $(date +%s) > version.txt +zip -r ${ZIPFILE} install version.txt LICENSE + +# upload to aws +# Install conda then 'conda install' awscli +curl --retry 3 -o ~/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh +chmod +x ~/conda.sh +/bin/bash ~/conda.sh -b -p ~/anaconda +export PATH="~/anaconda/bin:${PATH}" +source ~/anaconda/bin/activate +conda install -c conda-forge awscli --yes +set +x +export AWS_ACCESS_KEY_ID=${AWS_S3_ACCESS_KEY_FOR_PYTORCH_BINARY_UPLOAD} +export AWS_SECRET_ACCESS_KEY=${AWS_S3_ACCESS_SECRET_FOR_PYTORCH_BINARY_UPLOAD} +set -x +aws s3 cp ${ZIPFILE} s3://ossci-ios-build/ --acl public-read diff --git a/.circleci/unittest/linux/scripts/environment.yml b/.circleci/unittest/linux/scripts/environment.yml new file mode 100644 index 00000000000..fae96c5f93c --- /dev/null +++ b/.circleci/unittest/linux/scripts/environment.yml @@ -0,0 +1,16 @@ +channels: + - pytorch + - defaults +dependencies: + - pytest + - pytest-cov + - pytest-mock + - pip + - libpng + - jpeg + - ca-certificates + - h5py + - pip: + - future + - scipy + - av < 10 diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh new file mode 100755 index 00000000000..6375e191072 --- /dev/null +++ b/.circleci/unittest/linux/scripts/install.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +unset PYTORCH_VERSION +# For unittest, nightly PyTorch is used as the following section, +# so no need to set PYTORCH_VERSION. +# In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config. + +set -ex + +eval "$(./conda/bin/conda shell.bash hook)" +conda activate ./env + +if [ "${CU_VERSION:-}" == cpu ] ; then + cudatoolkit="cpuonly" + version="cpu" +else + if [[ ${#CU_VERSION} -eq 4 ]]; then + CUDA_VERSION="${CU_VERSION:2:1}.${CU_VERSION:3:1}" + elif [[ ${#CU_VERSION} -eq 5 ]]; then + CUDA_VERSION="${CU_VERSION:2:2}.${CU_VERSION:4:1}" + fi + echo "Using CUDA $CUDA_VERSION as determined by CU_VERSION: ${CU_VERSION} " + version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")" + cudatoolkit="pytorch-cuda=${version}" + + # make sure local cuda is set to required cuda version and not CUDA version by default + rm -f /usr/local/cuda + ln -s /usr/local/cuda-${version} /usr/local/cuda +fi + +case "$(uname -s)" in + Darwin*) os=MacOSX;; + *) os=Linux +esac + +printf "Installing PyTorch with %s\n" "${cudatoolkit}" +if [ "${os}" == "MacOSX" ]; then + conda install -y -c "pytorch-${UPLOAD_CHANNEL}" "pytorch-${UPLOAD_CHANNEL}"::pytorch "${cudatoolkit}" +else + conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c nvidia "pytorch-${UPLOAD_CHANNEL}"::pytorch[build="*${version}*"] "${cudatoolkit}" +fi + + +printf "* Installing torchvision\n" +python setup.py develop diff --git a/.circleci/unittest/linux/scripts/post_process.sh b/.circleci/unittest/linux/scripts/post_process.sh new file mode 100755 index 00000000000..e97bf2a7b1b --- /dev/null +++ b/.circleci/unittest/linux/scripts/post_process.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -e + +eval "$(./conda/bin/conda shell.bash hook)" +conda activate ./env diff --git a/.circleci/unittest/linux/scripts/run-clang-format.py b/.circleci/unittest/linux/scripts/run-clang-format.py new file mode 100755 index 00000000000..5c61b2519e0 --- /dev/null +++ b/.circleci/unittest/linux/scripts/run-clang-format.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python +""" +MIT License + +Copyright (c) 2017 Guillaume Papin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +A wrapper script around clang-format, suitable for linting multiple files +and to use for continuous integration. + +This is an alternative API for the clang-format command line. +It runs over multiple files and directories in parallel. +A diff output is produced and a sensible exit code is returned. + +""" + +import argparse +import difflib +import fnmatch +import multiprocessing +import os +import signal +import subprocess +import sys +import traceback +from functools import partial + +try: + from subprocess import DEVNULL # py3k +except ImportError: + DEVNULL = open(os.devnull, "wb") + + +DEFAULT_EXTENSIONS = "c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx,cu" + + +class ExitStatus: + SUCCESS = 0 + DIFF = 1 + TROUBLE = 2 + + +def list_files(files, recursive=False, extensions=None, exclude=None): + if extensions is None: + extensions = [] + if exclude is None: + exclude = [] + + out = [] + for file in files: + if recursive and os.path.isdir(file): + for dirpath, dnames, fnames in os.walk(file): + fpaths = [os.path.join(dirpath, fname) for fname in fnames] + for pattern in exclude: + # os.walk() supports trimming down the dnames list + # by modifying it in-place, + # to avoid unnecessary directory listings. + dnames[:] = [x for x in dnames if not fnmatch.fnmatch(os.path.join(dirpath, x), pattern)] + fpaths = [x for x in fpaths if not fnmatch.fnmatch(x, pattern)] + for f in fpaths: + ext = os.path.splitext(f)[1][1:] + if ext in extensions: + out.append(f) + else: + out.append(file) + return out + + +def make_diff(file, original, reformatted): + return list( + difflib.unified_diff( + original, reformatted, fromfile=f"{file}\t(original)", tofile=f"{file}\t(reformatted)", n=3 + ) + ) + + +class DiffError(Exception): + def __init__(self, message, errs=None): + super().__init__(message) + self.errs = errs or [] + + +class UnexpectedError(Exception): + def __init__(self, message, exc=None): + super().__init__(message) + self.formatted_traceback = traceback.format_exc() + self.exc = exc + + +def run_clang_format_diff_wrapper(args, file): + try: + ret = run_clang_format_diff(args, file) + return ret + except DiffError: + raise + except Exception as e: + raise UnexpectedError(f"{file}: {e.__class__.__name__}: {e}", e) + + +def run_clang_format_diff(args, file): + try: + with open(file, encoding="utf-8") as f: + original = f.readlines() + except OSError as exc: + raise DiffError(str(exc)) + invocation = [args.clang_format_executable, file] + + # Use of utf-8 to decode the process output. + # + # Hopefully, this is the correct thing to do. + # + # It's done due to the following assumptions (which may be incorrect): + # - clang-format will returns the bytes read from the files as-is, + # without conversion, and it is already assumed that the files use utf-8. + # - if the diagnostics were internationalized, they would use utf-8: + # > Adding Translations to Clang + # > + # > Not possible yet! + # > Diagnostic strings should be written in UTF-8, + # > the client can translate to the relevant code page if needed. + # > Each translation completely replaces the format string + # > for the diagnostic. + # > -- http://clang.llvm.org/docs/InternalsManual.html#internals-diag-translation + + try: + proc = subprocess.Popen( + invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, encoding="utf-8" + ) + except OSError as exc: + raise DiffError(f"Command '{subprocess.list2cmdline(invocation)}' failed to start: {exc}") + proc_stdout = proc.stdout + proc_stderr = proc.stderr + + # hopefully the stderr pipe won't get full and block the process + outs = list(proc_stdout.readlines()) + errs = list(proc_stderr.readlines()) + proc.wait() + if proc.returncode: + raise DiffError( + "Command '{}' returned non-zero exit status {}".format( + subprocess.list2cmdline(invocation), proc.returncode + ), + errs, + ) + return make_diff(file, original, outs), errs + + +def bold_red(s): + return "\x1b[1m\x1b[31m" + s + "\x1b[0m" + + +def colorize(diff_lines): + def bold(s): + return "\x1b[1m" + s + "\x1b[0m" + + def cyan(s): + return "\x1b[36m" + s + "\x1b[0m" + + def green(s): + return "\x1b[32m" + s + "\x1b[0m" + + def red(s): + return "\x1b[31m" + s + "\x1b[0m" + + for line in diff_lines: + if line[:4] in ["--- ", "+++ "]: + yield bold(line) + elif line.startswith("@@ "): + yield cyan(line) + elif line.startswith("+"): + yield green(line) + elif line.startswith("-"): + yield red(line) + else: + yield line + + +def print_diff(diff_lines, use_color): + if use_color: + diff_lines = colorize(diff_lines) + sys.stdout.writelines(diff_lines) + + +def print_trouble(prog, message, use_colors): + error_text = "error:" + if use_colors: + error_text = bold_red(error_text) + print(f"{prog}: {error_text} {message}", file=sys.stderr) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--clang-format-executable", + metavar="EXECUTABLE", + help="path to the clang-format executable", + default="clang-format", + ) + parser.add_argument( + "--extensions", + help=f"comma separated list of file extensions (default: {DEFAULT_EXTENSIONS})", + default=DEFAULT_EXTENSIONS, + ) + parser.add_argument("-r", "--recursive", action="store_true", help="run recursively over directories") + parser.add_argument("files", metavar="file", nargs="+") + parser.add_argument("-q", "--quiet", action="store_true") + parser.add_argument( + "-j", + metavar="N", + type=int, + default=0, + help="run N clang-format jobs in parallel (default number of cpus + 1)", + ) + parser.add_argument( + "--color", default="auto", choices=["auto", "always", "never"], help="show colored diff (default: auto)" + ) + parser.add_argument( + "-e", + "--exclude", + metavar="PATTERN", + action="append", + default=[], + help="exclude paths matching the given glob-like pattern(s) from recursive search", + ) + + args = parser.parse_args() + + # use default signal handling, like diff return SIGINT value on ^C + # https://bugs.python.org/issue14229#msg156446 + signal.signal(signal.SIGINT, signal.SIG_DFL) + try: + signal.SIGPIPE + except AttributeError: + # compatibility, SIGPIPE does not exist on Windows + pass + else: + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + colored_stdout = False + colored_stderr = False + if args.color == "always": + colored_stdout = True + colored_stderr = True + elif args.color == "auto": + colored_stdout = sys.stdout.isatty() + colored_stderr = sys.stderr.isatty() + + version_invocation = [args.clang_format_executable, "--version"] + try: + subprocess.check_call(version_invocation, stdout=DEVNULL) + except subprocess.CalledProcessError as e: + print_trouble(parser.prog, str(e), use_colors=colored_stderr) + return ExitStatus.TROUBLE + except OSError as e: + print_trouble( + parser.prog, + f"Command '{subprocess.list2cmdline(version_invocation)}' failed to start: {e}", + use_colors=colored_stderr, + ) + return ExitStatus.TROUBLE + + retcode = ExitStatus.SUCCESS + files = list_files( + args.files, recursive=args.recursive, exclude=args.exclude, extensions=args.extensions.split(",") + ) + + if not files: + return + + njobs = args.j + if njobs == 0: + njobs = multiprocessing.cpu_count() + 1 + njobs = min(len(files), njobs) + + if njobs == 1: + # execute directly instead of in a pool, + # less overhead, simpler stacktraces + it = (run_clang_format_diff_wrapper(args, file) for file in files) + pool = None + else: + pool = multiprocessing.Pool(njobs) + it = pool.imap_unordered(partial(run_clang_format_diff_wrapper, args), files) + while True: + try: + outs, errs = next(it) + except StopIteration: + break + except DiffError as e: + print_trouble(parser.prog, str(e), use_colors=colored_stderr) + retcode = ExitStatus.TROUBLE + sys.stderr.writelines(e.errs) + except UnexpectedError as e: + print_trouble(parser.prog, str(e), use_colors=colored_stderr) + sys.stderr.write(e.formatted_traceback) + retcode = ExitStatus.TROUBLE + # stop at the first unexpected error, + # something could be very wrong, + # don't process all files unnecessarily + if pool: + pool.terminate() + break + else: + sys.stderr.writelines(errs) + if outs == []: + continue + if not args.quiet: + print_diff(outs, use_color=colored_stdout) + if retcode == ExitStatus.SUCCESS: + retcode = ExitStatus.DIFF + return retcode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.circleci/unittest/linux/scripts/run_test.sh b/.circleci/unittest/linux/scripts/run_test.sh new file mode 100755 index 00000000000..5348baa71dd --- /dev/null +++ b/.circleci/unittest/linux/scripts/run_test.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -e + +eval "$(./conda/bin/conda shell.bash hook)" +conda activate ./env + +python -m torch.utils.collect_env + +case "$(uname -s)" in + Darwin*) + # The largest macOS runner is not able to handle the regular test suite plus the transforms v2 tests at the same + # time due to insufficient resources. Thus, we ignore the transforms v2 tests at first and run them in a separate + # step afterwards. + GLOB='test/test_transforms_v2*' + pytest --junitxml=test-results/junit.xml -v --durations 20 --ignore-glob="${GLOB}" + eval "pytest --junitxml=test-results/junit-transforms-v2.xml -v --durations 20 ${GLOB}" + ;; + *) + pytest --junitxml=test-results/junit.xml -v --durations 20 + ;; +esac diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh new file mode 100755 index 00000000000..8a8a78f1fb2 --- /dev/null +++ b/.circleci/unittest/linux/scripts/setup_env.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# This script is for setting up environment in which unit test is ran. +# To speed up the CI time, the resulting environment is cached. +# +# Do not install PyTorch and torchvision here, otherwise they also get cached. + +set -ex + +this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +# Avoid error: "fatal: unsafe repository" +git config --global --add safe.directory '*' +root_dir="$(git rev-parse --show-toplevel)" +conda_dir="${root_dir}/conda" +env_dir="${root_dir}/env" + +cd "${root_dir}" + +case "$(uname -s)" in + Darwin*) os=MacOSX;; + *) os=Linux +esac + +# 1. Install conda at ./conda +if [ ! -d "${conda_dir}" ]; then + printf "* Installing conda\n" + wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" + bash ./miniconda.sh -b -f -p "${conda_dir}" +fi +eval "$(${conda_dir}/bin/conda shell.bash hook)" + +# 2. Create test environment at ./env +if [ ! -d "${env_dir}" ]; then + printf "* Creating a test environment\n" + conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" +fi +conda activate "${env_dir}" + +# 3. Install Conda dependencies +printf "* Installing dependencies (except PyTorch)\n" +FFMPEG_PIN="=4.2" +if [[ "${PYTHON_VERSION}" == "3.9" ]]; then + FFMPEG_PIN=">=4.2" +fi + +conda install -y -c pytorch "ffmpeg${FFMPEG_PIN}" +conda env update --file "${this_dir}/environment.yml" --prune diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml new file mode 100644 index 00000000000..d229aafb41a --- /dev/null +++ b/.circleci/unittest/windows/scripts/environment.yml @@ -0,0 +1,19 @@ +channels: + - pytorch + - defaults +dependencies: + - pytest + - pytest-cov + - pytest-mock + - pip + - libpng + - jpeg + - ca-certificates + - hdf5 + - setuptools + - pip: + - future + - scipy + - av !=9.1.1, <10 + - dataclasses + - h5py diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh new file mode 100644 index 00000000000..7c55c8144d9 --- /dev/null +++ b/.circleci/unittest/windows/scripts/install.sh @@ -0,0 +1,53 @@ + +#!/usr/bin/env bash + +unset PYTORCH_VERSION +# For unittest, nightly PyTorch is used as the following section, +# so no need to set PYTORCH_VERSION. +# In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config. + +set -ex + +this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" +conda activate ./env + +# TODO, refactor the below logic to make it easy to understand how to get correct cuda_version. +if [ "${CU_VERSION:-}" == cpu ] ; then + cudatoolkit="cpuonly" + version="cpu" +else + if [[ ${#CU_VERSION} -eq 4 ]]; then + CUDA_VERSION="${CU_VERSION:2:1}.${CU_VERSION:3:1}" + elif [[ ${#CU_VERSION} -eq 5 ]]; then + CUDA_VERSION="${CU_VERSION:2:2}.${CU_VERSION:4:1}" + fi + + cuda_toolkit_pckg="cudatoolkit" + if [[ $CUDA_VERSION == 11.6 || $CUDA_VERSION == 11.7 || $CUDA_VERSION == 11.8 || $CUDA_VERSION == 12.1 ]]; then + cuda_toolkit_pckg="pytorch-cuda" + fi + + echo "Using CUDA $CUDA_VERSION as determined by CU_VERSION" + version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")" + cudatoolkit="${cuda_toolkit_pckg}=${version}" +fi + +printf "Installing PyTorch with %s\n" "${cudatoolkit}" +conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c nvidia "pytorch-${UPLOAD_CHANNEL}"::pytorch[build="*${version}*"] "${cudatoolkit}" + +torch_cuda=$(python -c "import torch; print(torch.cuda.is_available())") +echo torch.cuda.is_available is $torch_cuda + +if [ ! -z "${CUDA_VERSION:-}" ] ; then + if [ "$torch_cuda" == "False" ]; then + echo "torch with cuda installed but torch.cuda.is_available() is False" + exit 1 + fi +fi + +source "$this_dir/set_cuda_envs.sh" + +printf "* Installing torchvision\n" +"$this_dir/vc_env_helper.bat" python setup.py develop diff --git a/.circleci/unittest/windows/scripts/install_conda.bat b/.circleci/unittest/windows/scripts/install_conda.bat new file mode 100644 index 00000000000..6052ad08b10 --- /dev/null +++ b/.circleci/unittest/windows/scripts/install_conda.bat @@ -0,0 +1 @@ +start /wait "" "%miniconda_exe%" /S /InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /D=%tmp_conda% diff --git a/.circleci/unittest/windows/scripts/post_process.sh b/.circleci/unittest/windows/scripts/post_process.sh new file mode 100644 index 00000000000..5c5cbb758a9 --- /dev/null +++ b/.circleci/unittest/windows/scripts/post_process.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -e + +eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" +conda activate ./env diff --git a/.circleci/unittest/windows/scripts/run_test.sh b/.circleci/unittest/windows/scripts/run_test.sh new file mode 100644 index 00000000000..802ad37f511 --- /dev/null +++ b/.circleci/unittest/windows/scripts/run_test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e + +eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" +conda activate ./env + +this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source "$this_dir/set_cuda_envs.sh" + +python -m torch.utils.collect_env +pytest --junitxml=test-results/junit.xml -v --durations 20 diff --git a/.circleci/unittest/windows/scripts/set_cuda_envs.sh b/.circleci/unittest/windows/scripts/set_cuda_envs.sh new file mode 100644 index 00000000000..7db3137b594 --- /dev/null +++ b/.circleci/unittest/windows/scripts/set_cuda_envs.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -ex + +echo CU_VERSION is "${CU_VERSION}" +echo CUDA_VERSION is "${CUDA_VERSION}" + +# Currenly, CU_VERSION and CUDA_VERSION are not consistent. +# to understand this code, see https://github.com/pytorch/vision/issues/4443 +version="cpu" +if [[ ! -z "${CUDA_VERSION}" ]] ; then + version="$CUDA_VERSION" +else + if [[ ${#CU_VERSION} -eq 5 ]]; then + version="${CU_VERSION:2:2}.${CU_VERSION:4:1}" + fi +fi + +# Don't use if [[ "$version" == "cpu" ]]; then exit 0 fi. +# It would exit the shell. One result is cpu tests would not run if the shell exit. +# Unless there's an error, Don't exit. +if [[ "$version" != "cpu" ]]; then + # set cuda envs + export PATH="/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${version}/bin:/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${version}/libnvvp:$PATH" + export CUDA_PATH_V${version/./_}="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${version}" + export CUDA_PATH="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${version}" + + if [ ! -d "$CUDA_PATH" ]; then + echo "$CUDA_PATH" does not exist + exit 1 + fi + + if [ ! -f "${CUDA_PATH}\include\nvjpeg.h" ]; then + echo "nvjpeg does not exist" + exit 1 + fi + + # check cuda driver version + for path in '/c/Program Files/NVIDIA Corporation/NVSMI/nvidia-smi.exe' /c/Windows/System32/nvidia-smi.exe; do + if [[ -x "$path" ]]; then + "$path" || echo "true"; + break + fi + done + + which nvcc + nvcc --version + env | grep CUDA +fi diff --git a/.circleci/unittest/windows/scripts/setup_env.sh b/.circleci/unittest/windows/scripts/setup_env.sh new file mode 100644 index 00000000000..84697875979 --- /dev/null +++ b/.circleci/unittest/windows/scripts/setup_env.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# This script is for setting up environment in which unit test is ran. +# To speed up the CI time, the resulting environment is cached. +# +# Do not install PyTorch and torchvision here, otherwise they also get cached. + +set -ex + +this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +root_dir="$(git rev-parse --show-toplevel)" +conda_dir="${root_dir}/conda" +env_dir="${root_dir}/env" + +cd "${root_dir}" + +# 1. Install conda at ./conda +if [ ! -d "${conda_dir}" ]; then + printf "* Installing conda\n" + export tmp_conda="$(echo $conda_dir | tr '/' '\\')" + export miniconda_exe="$(echo $root_dir | tr '/' '\\')\\miniconda.exe" + curl --output miniconda.exe https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -O + "$this_dir/install_conda.bat" + unset tmp_conda + unset miniconda_exe +fi + +eval "$(${conda_dir}/Scripts/conda.exe 'shell.bash' 'hook')" + +# 2. Create test environment at ./env +if [ ! -d "${env_dir}" ]; then + printf "* Creating a test environment\n" + conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" +fi +conda activate "${env_dir}" + +# 3. Install Conda dependencies +printf "* Installing dependencies (except PyTorch)\n" +conda env update --file "${this_dir}/environment.yml" --prune + +# 4. Downgrade setuptools on Python 3.7. +# See https://github.com/pytorch/vision/pull/5868 +if [[ "${PYTHON_VERSION}" == '3.7' ]]; then + pip install --upgrade setuptools==58.0.4 +fi diff --git a/.circleci/unittest/windows/scripts/vc_env_helper.bat b/.circleci/unittest/windows/scripts/vc_env_helper.bat new file mode 100644 index 00000000000..9410135677a --- /dev/null +++ b/.circleci/unittest/windows/scripts/vc_env_helper.bat @@ -0,0 +1,39 @@ +@echo on + +set VC_VERSION_LOWER=16 +set VC_VERSION_UPPER=17 + +for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [%VC_VERSION_LOWER%^,%VC_VERSION_UPPER%^) -property installationPath`) do ( + if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( + set "VS15INSTALLDIR=%%i" + set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat" + goto vswhere + ) +) + +:vswhere +if "%VSDEVCMD_ARGS%" == "" ( + call "%VS15VCVARSALL%" x64 || exit /b 1 +) else ( + call "%VS15VCVARSALL%" x64 %VSDEVCMD_ARGS% || exit /b 1 +) + +@echo on + +set DISTUTILS_USE_SDK=1 + +set args=%1 +shift +:start +if [%1] == [] goto done +set args=%args% %1 +shift +goto start + +:done +if "%args%" == "" ( + echo Usage: vc_env_helper.bat [command] [args] + echo e.g. vc_env_helper.bat cl /c test.cpp +) + +%args% || exit /b 1 diff --git a/.github/scripts/unittest.sh b/.github/scripts/unittest.sh index 267edfedcbc..2a0b7154200 100755 --- a/.github/scripts/unittest.sh +++ b/.github/scripts/unittest.sh @@ -11,5 +11,4 @@ echo '::group::Install testing utilities' pip install --progress-bar=off pytest pytest-mock pytest-cov echo '::endgroup::' -python test/smoke_test.py -#pytest --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" -v --durations=25 +pytest --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" -v --durations=25 diff --git a/.github/workflows/build-conda-linux.yml b/.github/workflows/build-conda-linux.yml new file mode 100644 index 00000000000..e44bb877ed6 --- /dev/null +++ b/.github/workflows/build-conda-linux.yml @@ -0,0 +1,51 @@ +name: Build Linux Conda + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: conda + os: linux + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: "" + post-script: "" + conda-package-directory: packaging/torchvision + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_conda_linux.yml@main + with: + conda-package-directory: ${{ matrix.conda-package-directory }} + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + trigger-event: ${{ github.event_name }} + secrets: + CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-m1.yml b/.github/workflows/build-conda-m1.yml new file mode 100644 index 00000000000..7006b6714aa --- /dev/null +++ b/.github/workflows/build-conda-m1.yml @@ -0,0 +1,52 @@ +name: Build M1 Conda + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: conda + os: macos-arm64 + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: "" + post-script: "" + conda-package-directory: packaging/torchvision + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_conda_macos.yml@main + with: + conda-package-directory: ${{ matrix.conda-package-directory }} + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + runner-type: macos-m1-12 + trigger-event: ${{ github.event_name }} + secrets: + CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-macos.yml b/.github/workflows/build-conda-macos.yml new file mode 100644 index 00000000000..39d00534b3a --- /dev/null +++ b/.github/workflows/build-conda-macos.yml @@ -0,0 +1,52 @@ +name: Build Macos Conda + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: conda + os: macos + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: "" + post-script: "" + conda-package-directory: packaging/torchvision + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_conda_macos.yml@main + with: + conda-package-directory: ${{ matrix.conda-package-directory }} + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + runner-type: macos-12 + trigger-event: ${{ github.event_name }} + secrets: + CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-windows.yml b/.github/workflows/build-conda-windows.yml new file mode 100644 index 00000000000..954d09d39ec --- /dev/null +++ b/.github/workflows/build-conda-windows.yml @@ -0,0 +1,51 @@ +name: Build Windows Conda + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: conda + os: windows + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: packaging/pre_build_script.sh + env-script: packaging/windows/internal/vc_env_helper.bat + post-script: "" + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_conda_windows.yml@main + with: + conda-package-directory: ${{ matrix.conda-package-directory }} + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + trigger-event: ${{ github.event_name }} + secrets: + CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-wheels-linux.yml b/.github/workflows/build-wheels-linux.yml new file mode 100644 index 00000000000..e997d648ec2 --- /dev/null +++ b/.github/workflows/build-wheels-linux.yml @@ -0,0 +1,50 @@ +name: Build Linux Wheels + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: linux + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: packaging/pre_build_script.sh + post-script: packaging/post_build_script.sh + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + trigger-event: ${{ github.event_name }} + secrets: + AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-m1.yml b/.github/workflows/build-wheels-m1.yml new file mode 100644 index 00000000000..66c7687acc9 --- /dev/null +++ b/.github/workflows/build-wheels-m1.yml @@ -0,0 +1,51 @@ +name: Build M1 Wheels + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: macos-arm64 + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: packaging/pre_build_script.sh + post-script: packaging/post_build_script.sh + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + runner-type: macos-m1-12 + smoke-test-script: ${{ matrix.smoke-test-script }} + trigger-event: ${{ github.event_name }} + secrets: + AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-macos.yml b/.github/workflows/build-wheels-macos.yml new file mode 100644 index 00000000000..6c5ebc0fc37 --- /dev/null +++ b/.github/workflows/build-wheels-macos.yml @@ -0,0 +1,51 @@ +name: Build Macos Wheels + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: macos + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: packaging/pre_build_script.sh + post-script: packaging/post_build_script.sh + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + runner-type: macos-12 + smoke-test-script: ${{ matrix.smoke-test-script }} + trigger-event: ${{ github.event_name }} + secrets: + AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-windows.yml b/.github/workflows/build-wheels-windows.yml new file mode 100644 index 00000000000..3d818ece538 --- /dev/null +++ b/.github/workflows/build-wheels-windows.yml @@ -0,0 +1,52 @@ +name: Build Windows Wheels + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: windows + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: packaging/pre_build_script.sh + env-script: packaging/windows/internal/vc_env_helper.bat + post-script: "python packaging/wheel/relocate.py" + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_wheels_windows.yml@main + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + env-script: ${{ matrix.env-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + trigger-event: ${{ github.event_name }} + secrets: + AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000000..f4cc76db05e --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,115 @@ +name: Docs + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + - v[0-9]+.[0-9]+.[0-9] + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + build: + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + upload-artifact: docs + script: | + set -euo pipefail + + export PYTHON_VERSION=3.8 + export GPU_ARCH_TYPE=cpu + export GPU_ARCH_VERSION='' + ./.github/scripts/setup-env.sh + + # Prepare conda + CONDA_PATH=$(which conda) + eval "$(${CONDA_PATH} shell.bash hook)" + conda activate ci + # FIXME: not sure why we need this. `ldd torchvision/video_reader.so` shows that it + # already links against the one pulled from conda. However, at runtime it pulls from + # /lib64 + # Should we maybe always do this in `./.github/scripts/setup-env.sh` so that we don't + # have to pay attention in all other workflows? + export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH}" + + cd docs + + echo '::group::Install doc requirements' + pip install --progress-bar=off -r requirements.txt + echo '::endgroup::' + + if [[ ${{ github.event_name }} == push && (${{ github.ref_type }} == tag || (${{ github.ref_type }} == branch && ${{ github.ref_name }} == release/*)) ]]; then + echo '::group::Enable version string sanitization' + # This environment variable just has to exist and must not be empty. The actual value is arbitrary. + # See docs/source/conf.py for details + export TORCHVISION_SANITIZE_VERSION_STR_IN_DOCS=1 + echo '::endgroup::' + fi + + # The runner does not have sufficient memory to run with as many processes as there are + # cores (`-j auto`). Thus, we limit to a single process (`-j 1`) here. + sed -i -e 's/-j auto/-j 1/' Makefile + make html + + cp -r build/html "${RUNNER_ARTIFACT_DIR}" + + # On PRs we also want to upload the docs into our S3 bucket for preview. + if [[ ${{ github.event_name == 'pull_request' }} ]]; then + cp -r build/html/* "${RUNNER_DOCS_DIR}" + fi + + upload: + needs: build + if: github.repository == 'pytorch/vision' && github.event_name == 'push' && + ((github.ref_type == 'branch' && github.ref_name == 'main') || github.ref_type == 'tag') + permissions: + contents: write + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + download-artifact: docs + ref: gh-pages + script: | + set -euo pipefail + + REF_TYPE=${{ github.ref_type }} + REF_NAME=${{ github.ref_name }} + + if [[ "${REF_TYPE}" == branch ]]; then + TARGET_FOLDER="${REF_NAME}" + elif [[ "${REF_TYPE}" == tag ]]; then + case "${REF_NAME}" in + *-rc*) + echo "Aborting upload since this is an RC tag: ${REF_NAME}" + exit 0 + ;; + *) + # Strip the leading "v" as well as the trailing patch version. For example: + # 'v0.15.2' -> '0.15' + TARGET_FOLDER=$(echo "${REF_NAME}" | sed 's/v\([0-9]\+\)\.\([0-9]\+\)\.[0-9]\+/\1.\2/') + ;; + esac + fi + echo "Target Folder: ${TARGET_FOLDER}" + + mkdir -p "${TARGET_FOLDER}" + rm -rf "${TARGET_FOLDER}"/* + mv "${RUNNER_ARTIFACT_DIR}"/html/* "${TARGET_FOLDER}" + git add "${TARGET_FOLDER}" || true + + if [[ "${TARGET_FOLDER}" == main ]]; then + mkdir -p _static + rm -rf _static/* + cp -r "${TARGET_FOLDER}"/_static/* _static + git add _static || true + fi + + git config user.name 'pytorchbot' + git config user.email 'soumith+bot@pytorch.org' + git commit -m "auto-generating sphinx docs" || true + git push diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000000..ec8d285c9f3 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,107 @@ +name: Lint + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +jobs: + python-source-and-configs: + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + script: | + set -euo pipefail + + echo '::group::Setup environment' + CONDA_PATH=$(which conda) + eval "$(${CONDA_PATH} shell.bash hook)" + conda create --name ci --quiet --yes python=3.8 pip + conda activate ci + echo '::endgroup::' + + echo '::group::Install lint tools' + pip install --progress-bar=off pre-commit + echo '::endgroup::' + + set +e + pre-commit run --all-files + + if [ $? -ne 0 ]; then + git --no-pager diff + exit 1 + fi + + c-source: + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + script: | + set -euo pipefail + + echo '::group::Setup environment' + CONDA_PATH=$(which conda) + eval "$(${CONDA_PATH} shell.bash hook)" + # clang-format needs some shared libraries that conflict with the system ones. Thus, we install them from conda + # and prepend the libraries to linker path to prioritize them. `ncurses=5` is only available on the conda-forge + # channel. Since we are not building or testing here, this is fine. + conda create --name ci --quiet --yes -c conda-forge python=3.8 ncurses=5 libgcc + conda activate ci + export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH}" + echo '::endgroup::' + + echo '::group::Install lint tools' + curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o ./clang-format + chmod +x ./clang-format + echo '::endgroup::' + + echo '::group::Lint C source' + set +e + ./.circleci/unittest/linux/scripts/run-clang-format.py -r torchvision/csrc --clang-format-executable ./clang-format + + if [ $? -ne 0 ]; then + git --no-pager diff + exit 1 + fi + echo '::endgroup::' + + + python-types: + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + script: | + set -euo pipefail + + export PYTHON_VERSION=3.8 + export GPU_ARCH_TYPE=cpu + export GPU_ARCH_VERSION='' + + ./.github/scripts/setup-env.sh + + CONDA_PATH=$(which conda) + eval "$(${CONDA_PATH} shell.bash hook)" + conda activate ci + + echo '::group::Install lint tools' + pip install --progress-bar=off mypy + echo '::endgroup::' + + echo '::group::Lint Python types' + mypy --install-types --non-interactive --config-file mypy.ini + echo '::endgroup::' + + bc: + if: github.event.pull_request + runs-on: ubuntu-latest + steps: + - name: Run BC Lint Action + uses: pytorch/test-infra/.github/actions/bc-lint@main + with: + repo: ${{ github.event.pull_request.head.repo.full_name }} + base_sha: ${{ github.event.pull_request.base.sha }} + head_sha: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/pr-labels.yml b/.github/workflows/pr-labels.yml new file mode 100644 index 00000000000..20c37e4fd88 --- /dev/null +++ b/.github/workflows/pr-labels.yml @@ -0,0 +1,35 @@ +name: pr-labels + +on: + push: + branches: + - main + +jobs: + is-properly-labeled: + runs-on: ubuntu-latest + + steps: + - name: Set up python + uses: actions/setup-python@v2 + + - name: Install requests + run: pip install requests + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Process commit and find merger responsible for labeling + id: commit + run: echo "::set-output name=merger::$(python .github/process_commit.py ${{ github.sha }})" + + - name: Ping merger responsible for labeling if necessary + if: ${{ steps.commit.outputs.merger != '' }} + uses: mshick/add-pr-comment@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + message: | + Hey ${{ steps.commit.outputs.merger }}! + + You merged this PR, but no labels were added. The list of valid labels is available at https://github.com/pytorch/vision/blob/main/.github/process_commit.py diff --git a/.github/workflows/prototype-tests-linux-gpu.yml b/.github/workflows/prototype-tests-linux-gpu.yml new file mode 100644 index 00000000000..dee425054d5 --- /dev/null +++ b/.github/workflows/prototype-tests-linux-gpu.yml @@ -0,0 +1,54 @@ +name: Prototype tests on Linux + +on: + pull_request: + +jobs: + unittests-prototype: + strategy: + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + runner: ["linux.12xlarge"] + gpu-arch-type: ["cpu"] + include: + - python-version: "3.8" + runner: linux.g5.4xlarge.nvidia.gpu + gpu-arch-type: cuda + gpu-arch-version: "11.7" + fail-fast: false + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + runner: ${{ matrix.runner }} + gpu-arch-type: ${{ matrix.gpu-arch-type }} + gpu-arch-version: ${{ matrix.gpu-arch-version }} + timeout: 120 + script: | + set -euo pipefail + + export PYTHON_VERSION=${{ matrix.python-version }} + export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} + export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} + ./.github/scripts/setup-env.sh + + # Prepare conda + CONDA_PATH=$(which conda) + eval "$(${CONDA_PATH} shell.bash hook)" + conda activate ci + + echo '::group::Install testing utilities' + pip install --progress-bar=off pytest pytest-mock pytest-cov + echo '::endgroup::' + + # We don't want to run the prototype datasets tests. Since the positional glob into `pytest`, i.e. + # `test/test_prototype*.py` takes the highest priority, neither `--ignore` nor `--ignore-glob` can help us here. + rm test/test_prototype_datasets*.py + pytest \ + -v --durations=25 \ + --cov=torchvision/prototype --cov-report=term-missing \ + --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" \ + test/test_prototype_*.py diff --git a/.github/workflows/tests-schedule.yml b/.github/workflows/tests-schedule.yml new file mode 100644 index 00000000000..5426fdc997a --- /dev/null +++ b/.github/workflows/tests-schedule.yml @@ -0,0 +1,57 @@ +name: tests + +on: + pull_request: + paths: + - "test/test_datasets_download.py" + - ".github/failed_schedule_issue_template.md" + - ".github/workflows/tests-schedule.yml" + + schedule: + - cron: "0 9 * * *" + +jobs: + download: + runs-on: ubuntu-latest + + steps: + - name: Set up python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Upgrade system packages + run: python -m pip install --upgrade pip setuptools wheel + + - name: SSL + run: python -c 'import ssl; print(ssl.OPENSSL_VERSION)' + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install torch nightly build + run: pip install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + + - name: Install torchvision + run: pip install --no-build-isolation --editable . + + - name: Install all optional dataset requirements + run: pip install scipy pycocotools lmdb requests + + - name: Install tests requirements + run: pip install pytest + + - name: Run tests + run: pytest -ra -v test/test_datasets_download.py + + - uses: JasonEtco/create-an-issue@v2.4.0 + name: Create issue if download tests failed + if: failure() && github.event_name == 'schedule' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + WORKFLOW: ${{ github.workflow }} + JOB: ${{ github.job }} + ID: ${{ github.run_id }} + with: + filename: .github/failed_schedule_issue_template.md diff --git a/.github/workflows/update-viablestrict.yml b/.github/workflows/update-viablestrict.yml new file mode 100644 index 00000000000..2d9c2265676 --- /dev/null +++ b/.github/workflows/update-viablestrict.yml @@ -0,0 +1,23 @@ +name: Update viable/strict + +on: + pull_request: + paths: + - .github/workflows/update-viablestrict.yml + schedule: + - cron: 10,40 * * * * + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + do_update_viablestrict: + uses: pytorch/test-infra/.github/workflows/update-viablestrict.yml@main + with: + repository: pytorch/vision + required_checks: "Build Linux,Build M1,Build Macos,Tests on Linux,Tests on macOS,Docs,Lint" + secrets: + ROCKSET_API_KEY: ${{ secrets.ROCKSET_API_KEY }} + GITHUB_DEPLOY_KEY : ${{ secrets.VISION_GITHUB_DEPLOY_KEY }} From f033fac57306322ad9112f28fc8da9e029b615a6 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 May 2023 12:48:33 +0200 Subject: [PATCH 4/7] Revert "[DEBUG] try revert linpng uninstallation on macos-12" This reverts commit 8c5b87778889acd8e8d37c30fd4a6bb3ce7d91e2. --- .github/scripts/setup-env.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/scripts/setup-env.sh b/.github/scripts/setup-env.sh index 9a1d5e00a17..a7ba4a6bd12 100755 --- a/.github/scripts/setup-env.sh +++ b/.github/scripts/setup-env.sh @@ -24,12 +24,12 @@ esac if [[ "${OS_TYPE}" == "macos" && $(uname -m) == x86_64 ]]; then echo '::group::Uninstall system JPEG libraries on macOS' - # The x86 macOS runners, e.g. the GitHub Actions native "macos-12" runner, has some JPEG libraries installed by - # default that interfere with our build. We uninstall them here and use the one from conda below. - JPEG_LIBS=$(brew list | grep jpeg) - echo $JPEG_LIBS - for lib in $JPEG_LIBS; do - brew uninstall --ignore-dependencies --force $lib || true + # The x86 macOS runners, e.g. the GitHub Actions native "macos-12" runner, has some JPEG and PNG libraries + # installed by default that interfere with our build. We uninstall them here and use the one from conda below. + IMAGE_LIBS=$(brew list | grep -E "jpeg|png") + echo "${IMAGE_LIBS}" + for lib in "${IMAGE_LIBS}"; do + brew uninstall --ignore-dependencies --force "${lib}" || true done echo '::endgroup::' fi From 334b4e59fb919c8e7f703e82ed91164213ae096b Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 May 2023 12:56:54 +0200 Subject: [PATCH 5/7] fix bash loop --- .github/scripts/setup-env.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/scripts/setup-env.sh b/.github/scripts/setup-env.sh index a7ba4a6bd12..c86e5bf9f04 100755 --- a/.github/scripts/setup-env.sh +++ b/.github/scripts/setup-env.sh @@ -27,9 +27,8 @@ if [[ "${OS_TYPE}" == "macos" && $(uname -m) == x86_64 ]]; then # The x86 macOS runners, e.g. the GitHub Actions native "macos-12" runner, has some JPEG and PNG libraries # installed by default that interfere with our build. We uninstall them here and use the one from conda below. IMAGE_LIBS=$(brew list | grep -E "jpeg|png") - echo "${IMAGE_LIBS}" - for lib in "${IMAGE_LIBS}"; do - brew uninstall --ignore-dependencies --force "${lib}" || true + for lib in $IMAGE_LIBS; do + brew uninstall --ignore-dependencies --force "${lib}" done echo '::endgroup::' fi From 48b25c2436739c0a60c3b6edfab6f7c352dbeec0 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 May 2023 12:57:50 +0200 Subject: [PATCH 6/7] [REVERTME] disable unrelated CI --- .circleci/.gitignore | 1 - .circleci/build_docs/commit_docs.sh | 35 -- .circleci/config.yml | 271 -------------- .circleci/config.yml.in | 251 ------------- .circleci/regenerate.py | 58 --- .circleci/smoke_test/docker/Dockerfile | 34 -- .../android/scripts/binary_android_build.sh | 27 -- .../android/scripts/binary_android_upload.sh | 34 -- .../android/scripts/install_gradle.sh | 19 - .../unittest/ios/scripts/binary_ios_build.sh | 47 --- .../unittest/ios/scripts/binary_ios_upload.sh | 42 --- .../unittest/linux/scripts/environment.yml | 16 - .circleci/unittest/linux/scripts/install.sh | 45 --- .../unittest/linux/scripts/post_process.sh | 6 - .../linux/scripts/run-clang-format.py | 331 ------------------ .circleci/unittest/linux/scripts/run_test.sh | 22 -- .circleci/unittest/linux/scripts/setup_env.sh | 47 --- .../unittest/windows/scripts/environment.yml | 19 - .circleci/unittest/windows/scripts/install.sh | 53 --- .../windows/scripts/install_conda.bat | 1 - .../unittest/windows/scripts/post_process.sh | 6 - .../unittest/windows/scripts/run_test.sh | 12 - .../unittest/windows/scripts/set_cuda_envs.sh | 48 --- .../unittest/windows/scripts/setup_env.sh | 45 --- .../windows/scripts/vc_env_helper.bat | 39 --- .github/scripts/unittest.sh | 3 +- .github/workflows/build-cmake.yml | 92 ----- .github/workflows/build-conda-linux.yml | 51 --- .github/workflows/build-conda-m1.yml | 52 --- .github/workflows/build-conda-macos.yml | 52 --- .github/workflows/build-conda-windows.yml | 51 --- .github/workflows/build-wheels-linux.yml | 50 --- .github/workflows/build-wheels-m1.yml | 51 --- .github/workflows/build-wheels-macos.yml | 51 --- .github/workflows/build-wheels-windows.yml | 52 --- .github/workflows/docs.yml | 115 ------ .github/workflows/lint.yml | 107 ------ .github/workflows/pr-labels.yml | 35 -- .../workflows/prototype-tests-linux-gpu.yml | 54 --- .github/workflows/tests-schedule.yml | 57 --- .github/workflows/update-viablestrict.yml | 23 -- 41 files changed, 2 insertions(+), 2403 deletions(-) delete mode 100644 .circleci/.gitignore delete mode 100755 .circleci/build_docs/commit_docs.sh delete mode 100644 .circleci/config.yml delete mode 100644 .circleci/config.yml.in delete mode 100755 .circleci/regenerate.py delete mode 100644 .circleci/smoke_test/docker/Dockerfile delete mode 100644 .circleci/unittest/android/scripts/binary_android_build.sh delete mode 100644 .circleci/unittest/android/scripts/binary_android_upload.sh delete mode 100755 .circleci/unittest/android/scripts/install_gradle.sh delete mode 100755 .circleci/unittest/ios/scripts/binary_ios_build.sh delete mode 100644 .circleci/unittest/ios/scripts/binary_ios_upload.sh delete mode 100644 .circleci/unittest/linux/scripts/environment.yml delete mode 100755 .circleci/unittest/linux/scripts/install.sh delete mode 100755 .circleci/unittest/linux/scripts/post_process.sh delete mode 100755 .circleci/unittest/linux/scripts/run-clang-format.py delete mode 100755 .circleci/unittest/linux/scripts/run_test.sh delete mode 100755 .circleci/unittest/linux/scripts/setup_env.sh delete mode 100644 .circleci/unittest/windows/scripts/environment.yml delete mode 100644 .circleci/unittest/windows/scripts/install.sh delete mode 100644 .circleci/unittest/windows/scripts/install_conda.bat delete mode 100644 .circleci/unittest/windows/scripts/post_process.sh delete mode 100644 .circleci/unittest/windows/scripts/run_test.sh delete mode 100644 .circleci/unittest/windows/scripts/set_cuda_envs.sh delete mode 100644 .circleci/unittest/windows/scripts/setup_env.sh delete mode 100644 .circleci/unittest/windows/scripts/vc_env_helper.bat delete mode 100644 .github/workflows/build-cmake.yml delete mode 100644 .github/workflows/build-conda-linux.yml delete mode 100644 .github/workflows/build-conda-m1.yml delete mode 100644 .github/workflows/build-conda-macos.yml delete mode 100644 .github/workflows/build-conda-windows.yml delete mode 100644 .github/workflows/build-wheels-linux.yml delete mode 100644 .github/workflows/build-wheels-m1.yml delete mode 100644 .github/workflows/build-wheels-macos.yml delete mode 100644 .github/workflows/build-wheels-windows.yml delete mode 100644 .github/workflows/docs.yml delete mode 100644 .github/workflows/lint.yml delete mode 100644 .github/workflows/pr-labels.yml delete mode 100644 .github/workflows/prototype-tests-linux-gpu.yml delete mode 100644 .github/workflows/tests-schedule.yml delete mode 100644 .github/workflows/update-viablestrict.yml diff --git a/.circleci/.gitignore b/.circleci/.gitignore deleted file mode 100644 index 485dee64bcf..00000000000 --- a/.circleci/.gitignore +++ /dev/null @@ -1 +0,0 @@ -.idea diff --git a/.circleci/build_docs/commit_docs.sh b/.circleci/build_docs/commit_docs.sh deleted file mode 100755 index 04e3538fefc..00000000000 --- a/.circleci/build_docs/commit_docs.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -set -ex - - -if [ "$2" == "" ]; then - echo call as "$0" "" "" - echo where src is the root of the built documentation git checkout and - echo branch should be "main" or "1.7" or so - exit 1 -fi - -src=$1 -target=$2 - -echo "committing docs from ${src} to ${target}" - -pushd "${src}" -git checkout gh-pages -mkdir -p ./"${target}" -rm -rf ./"${target}"/* -cp -r "${src}/docs/build/html/"* ./"$target" -if [ "${target}" == "main" ]; then - mkdir -p ./_static - rm -rf ./_static/* - cp -r "${src}/docs/build/html/_static/"* ./_static - git add --all ./_static || true -fi -git add --all ./"${target}" || true -git config user.email "soumith+bot@pytorch.org" -git config user.name "pytorchbot" -# If there aren't changes, don't make a commit; push is no-op -git commit -m "auto-generating sphinx docs" || true -git remote add https https://github.com/pytorch/vision.git -git push -u https gh-pages diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 0463007af46..00000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,271 +0,0 @@ -version: 2.1 - -# How to test the Linux jobs: -# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ -# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_win_wheel_py3.8 -# - Replace binary_win_wheel_py3.8 with the name of the job you want to test. -# Job names are 'name:' key. - -executors: - windows-cpu: - machine: - resource_class: windows.xlarge - image: windows-server-2019-vs2019:stable - shell: bash.exe - - windows-gpu: - machine: - resource_class: windows.gpu.nvidia.medium - image: windows-server-2019-nvidia:stable - shell: bash.exe - -commands: - checkout_merge: - description: "checkout merge branch" - steps: - - checkout -# - run: -# name: Checkout merge branch -# command: | -# set -ex -# BRANCH=$(git rev-parse --abbrev-ref HEAD) -# if [[ "$BRANCH" != "main" ]]; then -# git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH} -# git checkout "merged/$CIRCLE_BRANCH" -# fi - designate_upload_channel: - description: "inserts the correct upload channel into ${BASH_ENV}" - steps: - - run: - name: adding UPLOAD_CHANNEL to BASH_ENV - command: | - our_upload_channel=nightly - # On tags upload to test instead - if [[ -n "${CIRCLE_TAG}" ]]; then - our_upload_channel=test - fi - echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} - - pip_install: - parameters: - args: - type: string - descr: - type: string - default: "" - user: - type: boolean - default: true - steps: - - run: - name: > - <<^ parameters.descr >> pip install << parameters.args >> <> - <<# parameters.descr >> << parameters.descr >> <> - command: > - pip install - <<# parameters.user >> --user <> - --progress-bar=off - << parameters.args >> - -binary_common: &binary_common - parameters: - # Edit these defaults to do a release - build_version: - description: "version number of release binary; by default, build a nightly" - type: string - default: "" - pytorch_version: - description: "PyTorch version to build against; by default, use a nightly" - type: string - default: "" - # Don't edit these - python_version: - description: "Python version to build against (e.g., 3.8)" - type: string - cu_version: - description: "CUDA version to build against, in CU format (e.g., cpu or cu100)" - type: string - default: "cpu" - unicode_abi: - description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" - type: string - default: "" - wheel_docker_image: - description: "Wheel only: what docker image to use" - type: string - default: "" - conda_docker_image: - description: "Conda only: what docker image to use" - type: string - default: "pytorch/conda-builder:cpu" - environment: - PYTHON_VERSION: << parameters.python_version >> - PYTORCH_VERSION: << parameters.pytorch_version >> - UNICODE_ABI: << parameters.unicode_abi >> - CU_VERSION: << parameters.cu_version >> - MACOSX_DEPLOYMENT_TARGET: 10.9 - -smoke_test_common: &smoke_test_common - <<: *binary_common - docker: - - image: torchvision/smoke_test:latest - -jobs: - circleci_consistency: - docker: - - image: cimg/python:3.8 - steps: - - checkout - - pip_install: - args: jinja2 pyyaml - - run: - name: Check CircleCI config consistency - command: | - python .circleci/regenerate.py - git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) - - smoke_test_docker_image_build: - machine: - image: ubuntu-2004:202104-01 - resource_class: large - environment: - image_name: torchvision/smoke_test - steps: - - checkout - - designate_upload_channel - - run: - name: Build and push Docker image - no_output_timeout: "1h" - command: | - set +x - echo "${DOCKER_HUB_TOKEN}" | docker login --username "${DOCKER_HUB_USERNAME}" --password-stdin - set -x - cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID} - docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} ${image_name}:latest - docker push ${image_name}:${CIRCLE_WORKFLOW_ID} - docker push ${image_name}:latest - - cmake_linux_cpu: - <<: *binary_common - docker: - - image: "pytorch/manylinux-cpu" - resource_class: 2xlarge+ - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Setup conda - command: .circleci/unittest/linux/scripts/setup_env.sh - - run: packaging/build_cmake.sh - - cmake_linux_gpu: - <<: *binary_common - machine: - image: ubuntu-2004-cuda-11.4:202110-01 - resource_class: gpu.nvidia.small - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Setup conda - command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh - - run: - name: Build torchvision C++ distribution and test - no_output_timeout: 30m - command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh - - cmake_macos_cpu: - <<: *binary_common - macos: - xcode: "14.0" - steps: - - checkout_merge - - designate_upload_channel - - run: - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - conda install -yq conda-build cmake python=<< parameters.python_version >> - packaging/build_cmake.sh - - cmake_windows_cpu: - <<: *binary_common - executor: - name: windows-cpu - steps: - - checkout_merge - - designate_upload_channel - - run: - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate base - conda create -yn python39 python=3.9 - conda activate python39 - packaging/build_cmake.sh - - cmake_windows_gpu: - <<: *binary_common - executor: - name: windows-gpu - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Update CUDA driver - command: packaging/windows/internal/driver_update.bat - - run: - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - packaging/windows/internal/cuda_install.bat - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate - conda update -y conda - conda create -yn python39 python=3.9 - conda activate python39 - packaging/build_cmake.sh - - -workflows: - lint: - jobs: - - circleci_consistency - - cmake: - jobs: - - cmake_linux_cpu: - cu_version: cpu - name: cmake_linux_cpu - python_version: '3.8' - - cmake_linux_gpu: - cu_version: cu117 - name: cmake_linux_gpu - python_version: '3.8' - wheel_docker_image: pytorch/manylinux-cuda117 - - cmake_windows_cpu: - cu_version: cpu - name: cmake_windows_cpu - python_version: '3.8' - - cmake_windows_gpu: - cu_version: cu117 - name: cmake_windows_gpu - python_version: '3.8' - - cmake_macos_cpu: - cu_version: cpu - name: cmake_macos_cpu - python_version: '3.8' - - docker_build: - triggers: - - schedule: - cron: "0 10 * * 0" - filters: - branches: - only: - - main - jobs: - - smoke_test_docker_image_build: - context: org-member diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in deleted file mode 100644 index f3a88f2d987..00000000000 --- a/.circleci/config.yml.in +++ /dev/null @@ -1,251 +0,0 @@ -version: 2.1 - -# How to test the Linux jobs: -# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ -# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_win_wheel_py3.8 -# - Replace binary_win_wheel_py3.8 with the name of the job you want to test. -# Job names are 'name:' key. - -executors: - windows-cpu: - machine: - resource_class: windows.xlarge - image: windows-server-2019-vs2019:stable - shell: bash.exe - - windows-gpu: - machine: - resource_class: windows.gpu.nvidia.medium - image: windows-server-2019-nvidia:stable - shell: bash.exe - -commands: - checkout_merge: - description: "checkout merge branch" - steps: - - checkout -# - run: -# name: Checkout merge branch -# command: | -# set -ex -# BRANCH=$(git rev-parse --abbrev-ref HEAD) -# if [[ "$BRANCH" != "main" ]]; then -# git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH} -# git checkout "merged/$CIRCLE_BRANCH" -# fi - designate_upload_channel: - description: "inserts the correct upload channel into ${BASH_ENV}" - steps: - - run: - name: adding UPLOAD_CHANNEL to BASH_ENV - command: | - our_upload_channel=nightly - # On tags upload to test instead - if [[ -n "${CIRCLE_TAG}" ]]; then - our_upload_channel=test - fi - echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} - - pip_install: - parameters: - args: - type: string - descr: - type: string - default: "" - user: - type: boolean - default: true - steps: - - run: - name: > - <<^ parameters.descr >> pip install << parameters.args >> <> - <<# parameters.descr >> << parameters.descr >> <> - command: > - pip install - <<# parameters.user >> --user <> - --progress-bar=off - << parameters.args >> - -binary_common: &binary_common - parameters: - # Edit these defaults to do a release - build_version: - description: "version number of release binary; by default, build a nightly" - type: string - default: "" - pytorch_version: - description: "PyTorch version to build against; by default, use a nightly" - type: string - default: "" - # Don't edit these - python_version: - description: "Python version to build against (e.g., 3.8)" - type: string - cu_version: - description: "CUDA version to build against, in CU format (e.g., cpu or cu100)" - type: string - default: "cpu" - unicode_abi: - description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" - type: string - default: "" - wheel_docker_image: - description: "Wheel only: what docker image to use" - type: string - default: "" - conda_docker_image: - description: "Conda only: what docker image to use" - type: string - default: "pytorch/conda-builder:cpu" - environment: - PYTHON_VERSION: << parameters.python_version >> - PYTORCH_VERSION: << parameters.pytorch_version >> - UNICODE_ABI: << parameters.unicode_abi >> - CU_VERSION: << parameters.cu_version >> - MACOSX_DEPLOYMENT_TARGET: 10.9 - -smoke_test_common: &smoke_test_common - <<: *binary_common - docker: - - image: torchvision/smoke_test:latest - -jobs: - circleci_consistency: - docker: - - image: cimg/python:3.8 - steps: - - checkout - - pip_install: - args: jinja2 pyyaml - - run: - name: Check CircleCI config consistency - command: | - python .circleci/regenerate.py - git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) - - smoke_test_docker_image_build: - machine: - image: ubuntu-2004:202104-01 - resource_class: large - environment: - image_name: torchvision/smoke_test - steps: - - checkout - - designate_upload_channel - - run: - name: Build and push Docker image - no_output_timeout: "1h" - command: | - set +x - echo "${DOCKER_HUB_TOKEN}" | docker login --username "${DOCKER_HUB_USERNAME}" --password-stdin - set -x - cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID} - docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} ${image_name}:latest - docker push ${image_name}:${CIRCLE_WORKFLOW_ID} - docker push ${image_name}:latest - - cmake_linux_cpu: - <<: *binary_common - docker: - - image: "pytorch/manylinux-cpu" - resource_class: 2xlarge+ - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Setup conda - command: .circleci/unittest/linux/scripts/setup_env.sh - - run: packaging/build_cmake.sh - - cmake_linux_gpu: - <<: *binary_common - machine: - image: ubuntu-2004-cuda-11.4:202110-01 - resource_class: gpu.nvidia.small - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Setup conda - command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh - - run: - name: Build torchvision C++ distribution and test - no_output_timeout: 30m - command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh - - cmake_macos_cpu: - <<: *binary_common - macos: - xcode: "14.0" - steps: - - checkout_merge - - designate_upload_channel - - run: - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - conda install -yq conda-build cmake python=<< parameters.python_version >> - packaging/build_cmake.sh - - cmake_windows_cpu: - <<: *binary_common - executor: - name: windows-cpu - steps: - - checkout_merge - - designate_upload_channel - - run: - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate base - conda create -yn python39 python=3.9 - conda activate python39 - packaging/build_cmake.sh - - cmake_windows_gpu: - <<: *binary_common - executor: - name: windows-gpu - steps: - - checkout_merge - - designate_upload_channel - - run: - name: Update CUDA driver - command: packaging/windows/internal/driver_update.bat - - run: - command: | - set -ex - source packaging/windows/internal/vc_install_helper.sh - packaging/windows/internal/cuda_install.bat - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate - conda update -y conda - conda create -yn python39 python=3.9 - conda activate python39 - packaging/build_cmake.sh - - -workflows: - lint: - jobs: - - circleci_consistency - - cmake: - jobs: - {{ cmake_workflows() }} - - docker_build: - triggers: - - schedule: - cron: "0 10 * * 0" - filters: - branches: - only: - - main - jobs: - - smoke_test_docker_image_build: - context: org-member diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py deleted file mode 100755 index e7b8db3d56e..00000000000 --- a/.circleci/regenerate.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 - -""" -This script should use a very simple, functional programming style. -Avoid Jinja macros in favor of native Python functions. - -Don't go overboard on code generation; use Python only to generate -content that can't be easily declared statically using CircleCI's YAML API. - -Data declarations (e.g. the nested loops for defining the configuration matrix) -should be at the top of the file for easy updating. - -See this comment for design rationale: -https://github.com/pytorch/vision/pull/1321#issuecomment-531033978 -""" - -import os.path - -import jinja2 -import yaml -from jinja2 import select_autoescape - - -def indent(indentation, data_list): - return ("\n" + " " * indentation).join(yaml.dump(data_list, default_flow_style=False).splitlines()) - - -def cmake_workflows(indentation=6): - jobs = [] - python_version = "3.8" - for os_type in ["linux", "windows", "macos"]: - # Skip OSX CUDA - device_types = ["cpu", "gpu"] if os_type != "macos" else ["cpu"] - for device in device_types: - job = {"name": f"cmake_{os_type}_{device}", "python_version": python_version} - - job["cu_version"] = "cu117" if device == "gpu" else "cpu" - if device == "gpu" and os_type == "linux": - job["wheel_docker_image"] = "pytorch/manylinux-cuda117" - jobs.append({f"cmake_{os_type}_{device}": job}) - return indent(indentation, jobs) - - -if __name__ == "__main__": - d = os.path.dirname(__file__) - env = jinja2.Environment( - loader=jinja2.FileSystemLoader(d), - lstrip_blocks=True, - autoescape=select_autoescape(enabled_extensions=("html", "xml")), - keep_trailing_newline=True, - ) - - with open(os.path.join(d, "config.yml"), "w") as f: - f.write( - env.get_template("config.yml.in").render( - cmake_workflows=cmake_workflows, - ) - ) diff --git a/.circleci/smoke_test/docker/Dockerfile b/.circleci/smoke_test/docker/Dockerfile deleted file mode 100644 index 34bdcda1053..00000000000 --- a/.circleci/smoke_test/docker/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# this Dockerfile is for torchvision smoke test, it will be created periodically via CI system -# if you need to do it locally, follow below steps once you have Docker installed -# assuming you're within the directory where this Dockerfile located -# $ docker build . -t torchvision/smoketest - -# if you want to push to aws ecr, make sure you have the rights to write to ECR, then run -# $ eval $(aws ecr get-login --region us-east-1 --no-include-email) -# $ export MYTAG=localbuild ## you can choose whatever tag you like -# $ docker tag torchvision/smoketest 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchvision/smoke_test:${MYTAG} -# $ docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchvision/smoke_test:${MYTAG} - -FROM ubuntu:latest - -RUN apt-get -qq update && apt-get -qq -y install curl bzip2 libsox-fmt-all \ - && curl -sSL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh \ - && bash /tmp/miniconda.sh -bfp /usr/local \ - && rm -rf /tmp/miniconda.sh \ - && conda install -y python=3 \ - && conda update conda \ - && apt-get -qq -y remove curl bzip2 \ - && apt-get -qq -y autoremove \ - && apt-get autoclean \ - && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log \ - && conda clean --all --yes - -ENV PATH /opt/conda/bin:$PATH - -RUN conda create -y --name python3.7 python=3.7 -RUN conda create -y --name python3.8 python=3.8 -RUN conda create -y --name python3.9 python=3.9 -RUN conda create -y --name python3.10 python=3.10 -SHELL [ "/bin/bash", "-c" ] -RUN echo "source /usr/local/etc/profile.d/conda.sh" >> ~/.bashrc -CMD [ "/bin/bash"] diff --git a/.circleci/unittest/android/scripts/binary_android_build.sh b/.circleci/unittest/android/scripts/binary_android_build.sh deleted file mode 100644 index 0d8c0d47d8a..00000000000 --- a/.circleci/unittest/android/scripts/binary_android_build.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -set -ex -o pipefail - -echo "DIR: $(pwd)" -echo "ANDROID_HOME=${ANDROID_HOME}" -echo "ANDROID_NDK_HOME=${ANDROID_NDK_HOME}" -echo "JAVA_HOME=${JAVA_HOME}" - -WORKSPACE=/home/circleci/workspace -VISION_ANDROID=/home/circleci/project/android - -. /home/circleci/project/.circleci/unittest/android/scripts/install_gradle.sh - -GRADLE_LOCAL_PROPERTIES=${VISION_ANDROID}/local.properties -rm -f $GRADLE_LOCAL_PROPERTIES - -echo "sdk.dir=${ANDROID_HOME}" >> $GRADLE_LOCAL_PROPERTIES -echo "ndk.dir=${ANDROID_NDK_HOME}" >> $GRADLE_LOCAL_PROPERTIES - -echo "GRADLE_PATH $GRADLE_PATH" -echo "GRADLE_HOME $GRADLE_HOME" - -${GRADLE_PATH} --scan --stacktrace --debug --no-daemon -p ${VISION_ANDROID} assemble || true - -mkdir -p ~/workspace/artifacts -find . -type f -name *aar -print | xargs tar cfvz ~/workspace/artifacts/artifacts-aars.tgz -find . -type f -name *apk -print | xargs tar cfvz ~/workspace/artifacts/artifacts-apks.tgz diff --git a/.circleci/unittest/android/scripts/binary_android_upload.sh b/.circleci/unittest/android/scripts/binary_android_upload.sh deleted file mode 100644 index 1472a877d90..00000000000 --- a/.circleci/unittest/android/scripts/binary_android_upload.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -set -ex -o pipefail - -echo "DIR: $(pwd)" -echo "ANDROID_HOME=${ANDROID_HOME}" -echo "ANDROID_NDK_HOME=${ANDROID_NDK_HOME}" -echo "JAVA_HOME=${JAVA_HOME}" - -WORKSPACE=/home/circleci/workspace -VISION_ANDROID=/home/circleci/project/android - -. /home/circleci/project/.circleci/unittest/android/scripts/install_gradle.sh - -GRADLE_LOCAL_PROPERTIES=${VISION_ANDROID}/local.properties -rm -f $GRADLE_LOCAL_PROPERTIES -GRADLE_PROPERTIES=/home/circleci/project/android/gradle.properties - -echo "sdk.dir=${ANDROID_HOME}" >> $GRADLE_LOCAL_PROPERTIES -echo "ndk.dir=${ANDROID_NDK_HOME}" >> $GRADLE_LOCAL_PROPERTIES - -echo "SONATYPE_NEXUS_USERNAME=${SONATYPE_NEXUS_USERNAME}" >> $GRADLE_PROPERTIES -echo "mavenCentralRepositoryUsername=${SONATYPE_NEXUS_USERNAME}" >> $GRADLE_PROPERTIES -echo "SONATYPE_NEXUS_PASSWORD=${SONATYPE_NEXUS_PASSWORD}" >> $GRADLE_PROPERTIES -echo "mavenCentralRepositoryPassword=${SONATYPE_NEXUS_PASSWORD}" >> $GRADLE_PROPERTIES - -echo "signing.keyId=${ANDROID_SIGN_KEY}" >> $GRADLE_PROPERTIES -echo "signing.password=${ANDROID_SIGN_PASS}" >> $GRADLE_PROPERTIES - -cat /home/circleci/project/android/gradle.properties | grep VERSION - -${GRADLE_PATH} --scan --stacktrace --debug --no-daemon -p ${VISION_ANDROID} ops:uploadArchives - -mkdir -p ~/workspace/artifacts -find . -type f -name *aar -print | xargs tar cfvz ~/workspace/artifacts/artifacts-aars.tgz diff --git a/.circleci/unittest/android/scripts/install_gradle.sh b/.circleci/unittest/android/scripts/install_gradle.sh deleted file mode 100755 index ff90c657e65..00000000000 --- a/.circleci/unittest/android/scripts/install_gradle.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -set -ex - -_https_amazon_aws=https://downloads.gradle-dn.com/distributions -GRADLE_VERSION=6.8.3 - -_gradle_home=/opt/gradle -sudo rm -rf $gradle_home -sudo mkdir -p $_gradle_home - -curl --silent --output /tmp/gradle.zip --retry 3 $_https_amazon_aws/gradle-${GRADLE_VERSION}-bin.zip - -sudo unzip -q /tmp/gradle.zip -d $_gradle_home -rm /tmp/gradle.zip - -sudo chmod -R 777 $_gradle_home - -export GRADLE_HOME=$_gradle_home/gradle-$GRADLE_VERSION -export GRADLE_PATH=${GRADLE_HOME}/bin/gradle diff --git a/.circleci/unittest/ios/scripts/binary_ios_build.sh b/.circleci/unittest/ios/scripts/binary_ios_build.sh deleted file mode 100755 index 1f117481f2d..00000000000 --- a/.circleci/unittest/ios/scripts/binary_ios_build.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -ex -o pipefail - -echo "" -echo "DIR: $(pwd)" -WORKSPACE=/Users/distiller/workspace -PROJ_ROOT_IOS=/Users/distiller/project/ios -PYTORCH_IOS_NIGHTLY_NAME=libtorch_ios_nightly_build.zip -export TCLLIBPATH="/usr/local/lib" - -# install conda -curl --retry 3 -o ~/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -chmod +x ~/conda.sh -/bin/bash ~/conda.sh -b -p ~/anaconda -export PATH="~/anaconda/bin:${PATH}" -source ~/anaconda/bin/activate - -# install dependencies -conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi requests wget --yes -conda install -c conda-forge valgrind --yes -export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} - -# sync submodules -cd ${PROJ_ROOT_IOS} -git submodule sync -git submodule update --init --recursive - -# download pytorch-iOS nightly build and unzip it -mkdir -p ${PROJ_ROOT_IOS}/lib -mkdir -p ${PROJ_ROOT_IOS}/build -mkdir -p ${PROJ_ROOT_IOS}/pytorch -TORCH_ROOT="${PROJ_ROOT_IOS}/pytorch" - -cd ${TORCH_ROOT} -wget https://ossci-ios-build.s3.amazonaws.com/${PYTORCH_IOS_NIGHTLY_NAME} -mkdir -p ./build_ios -unzip -d ./build_ios ./${PYTORCH_IOS_NIGHTLY_NAME} - -LIBTORCH_HEADER_ROOT="${TORCH_ROOT}/build_ios/install/include" -cd ${PROJ_ROOT_IOS} -IOS_ARCH=${IOS_ARCH} LIBTORCH_HEADER_ROOT=${LIBTORCH_HEADER_ROOT} ./build_ios.sh -rm -rf ${TORCH_ROOT} - -# store the binary -DEST_DIR=${WORKSPACE}/ios/${IOS_ARCH} -mkdir -p ${DEST_DIR} -cp ${PROJ_ROOT_IOS}/lib/*.a ${DEST_DIR} diff --git a/.circleci/unittest/ios/scripts/binary_ios_upload.sh b/.circleci/unittest/ios/scripts/binary_ios_upload.sh deleted file mode 100644 index ce56388e5da..00000000000 --- a/.circleci/unittest/ios/scripts/binary_ios_upload.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -set -ex -o pipefail - -echo "" -echo "DIR: $(pwd)" - -WORKSPACE=/Users/distiller/workspace -PROJ_ROOT=/Users/distiller/project -ARTIFACTS_DIR=${WORKSPACE}/ios -ls ${ARTIFACTS_DIR} -ZIP_DIR=${WORKSPACE}/zip -mkdir -p ${ZIP_DIR}/install/lib - -# build a FAT bianry -cd ${ZIP_DIR}/install/lib -libs=("${ARTIFACTS_DIR}/x86_64/libtorchvision_ops.a" "${ARTIFACTS_DIR}/arm64/libtorchvision_ops.a") -lipo -create "${libs[@]}" -o ${ZIP_DIR}/install/lib/libtorchvision_ops.a -lipo -i ${ZIP_DIR}/install/lib/*.a - -# copy the license -cp ${PROJ_ROOT}/LICENSE ${ZIP_DIR}/ -# zip the library -ZIPFILE=libtorchvision_ops_ios_nightly_build.zip -cd ${ZIP_DIR} -#for testing -touch version.txt -echo $(date +%s) > version.txt -zip -r ${ZIPFILE} install version.txt LICENSE - -# upload to aws -# Install conda then 'conda install' awscli -curl --retry 3 -o ~/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -chmod +x ~/conda.sh -/bin/bash ~/conda.sh -b -p ~/anaconda -export PATH="~/anaconda/bin:${PATH}" -source ~/anaconda/bin/activate -conda install -c conda-forge awscli --yes -set +x -export AWS_ACCESS_KEY_ID=${AWS_S3_ACCESS_KEY_FOR_PYTORCH_BINARY_UPLOAD} -export AWS_SECRET_ACCESS_KEY=${AWS_S3_ACCESS_SECRET_FOR_PYTORCH_BINARY_UPLOAD} -set -x -aws s3 cp ${ZIPFILE} s3://ossci-ios-build/ --acl public-read diff --git a/.circleci/unittest/linux/scripts/environment.yml b/.circleci/unittest/linux/scripts/environment.yml deleted file mode 100644 index fae96c5f93c..00000000000 --- a/.circleci/unittest/linux/scripts/environment.yml +++ /dev/null @@ -1,16 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pytest - - pytest-cov - - pytest-mock - - pip - - libpng - - jpeg - - ca-certificates - - h5py - - pip: - - future - - scipy - - av < 10 diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh deleted file mode 100755 index 6375e191072..00000000000 --- a/.circleci/unittest/linux/scripts/install.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -unset PYTORCH_VERSION -# For unittest, nightly PyTorch is used as the following section, -# so no need to set PYTORCH_VERSION. -# In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config. - -set -ex - -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env - -if [ "${CU_VERSION:-}" == cpu ] ; then - cudatoolkit="cpuonly" - version="cpu" -else - if [[ ${#CU_VERSION} -eq 4 ]]; then - CUDA_VERSION="${CU_VERSION:2:1}.${CU_VERSION:3:1}" - elif [[ ${#CU_VERSION} -eq 5 ]]; then - CUDA_VERSION="${CU_VERSION:2:2}.${CU_VERSION:4:1}" - fi - echo "Using CUDA $CUDA_VERSION as determined by CU_VERSION: ${CU_VERSION} " - version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")" - cudatoolkit="pytorch-cuda=${version}" - - # make sure local cuda is set to required cuda version and not CUDA version by default - rm -f /usr/local/cuda - ln -s /usr/local/cuda-${version} /usr/local/cuda -fi - -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac - -printf "Installing PyTorch with %s\n" "${cudatoolkit}" -if [ "${os}" == "MacOSX" ]; then - conda install -y -c "pytorch-${UPLOAD_CHANNEL}" "pytorch-${UPLOAD_CHANNEL}"::pytorch "${cudatoolkit}" -else - conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c nvidia "pytorch-${UPLOAD_CHANNEL}"::pytorch[build="*${version}*"] "${cudatoolkit}" -fi - - -printf "* Installing torchvision\n" -python setup.py develop diff --git a/.circleci/unittest/linux/scripts/post_process.sh b/.circleci/unittest/linux/scripts/post_process.sh deleted file mode 100755 index e97bf2a7b1b..00000000000 --- a/.circleci/unittest/linux/scripts/post_process.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -e - -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env diff --git a/.circleci/unittest/linux/scripts/run-clang-format.py b/.circleci/unittest/linux/scripts/run-clang-format.py deleted file mode 100755 index 5c61b2519e0..00000000000 --- a/.circleci/unittest/linux/scripts/run-clang-format.py +++ /dev/null @@ -1,331 +0,0 @@ -#!/usr/bin/env python -""" -MIT License - -Copyright (c) 2017 Guillaume Papin - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -A wrapper script around clang-format, suitable for linting multiple files -and to use for continuous integration. - -This is an alternative API for the clang-format command line. -It runs over multiple files and directories in parallel. -A diff output is produced and a sensible exit code is returned. - -""" - -import argparse -import difflib -import fnmatch -import multiprocessing -import os -import signal -import subprocess -import sys -import traceback -from functools import partial - -try: - from subprocess import DEVNULL # py3k -except ImportError: - DEVNULL = open(os.devnull, "wb") - - -DEFAULT_EXTENSIONS = "c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx,cu" - - -class ExitStatus: - SUCCESS = 0 - DIFF = 1 - TROUBLE = 2 - - -def list_files(files, recursive=False, extensions=None, exclude=None): - if extensions is None: - extensions = [] - if exclude is None: - exclude = [] - - out = [] - for file in files: - if recursive and os.path.isdir(file): - for dirpath, dnames, fnames in os.walk(file): - fpaths = [os.path.join(dirpath, fname) for fname in fnames] - for pattern in exclude: - # os.walk() supports trimming down the dnames list - # by modifying it in-place, - # to avoid unnecessary directory listings. - dnames[:] = [x for x in dnames if not fnmatch.fnmatch(os.path.join(dirpath, x), pattern)] - fpaths = [x for x in fpaths if not fnmatch.fnmatch(x, pattern)] - for f in fpaths: - ext = os.path.splitext(f)[1][1:] - if ext in extensions: - out.append(f) - else: - out.append(file) - return out - - -def make_diff(file, original, reformatted): - return list( - difflib.unified_diff( - original, reformatted, fromfile=f"{file}\t(original)", tofile=f"{file}\t(reformatted)", n=3 - ) - ) - - -class DiffError(Exception): - def __init__(self, message, errs=None): - super().__init__(message) - self.errs = errs or [] - - -class UnexpectedError(Exception): - def __init__(self, message, exc=None): - super().__init__(message) - self.formatted_traceback = traceback.format_exc() - self.exc = exc - - -def run_clang_format_diff_wrapper(args, file): - try: - ret = run_clang_format_diff(args, file) - return ret - except DiffError: - raise - except Exception as e: - raise UnexpectedError(f"{file}: {e.__class__.__name__}: {e}", e) - - -def run_clang_format_diff(args, file): - try: - with open(file, encoding="utf-8") as f: - original = f.readlines() - except OSError as exc: - raise DiffError(str(exc)) - invocation = [args.clang_format_executable, file] - - # Use of utf-8 to decode the process output. - # - # Hopefully, this is the correct thing to do. - # - # It's done due to the following assumptions (which may be incorrect): - # - clang-format will returns the bytes read from the files as-is, - # without conversion, and it is already assumed that the files use utf-8. - # - if the diagnostics were internationalized, they would use utf-8: - # > Adding Translations to Clang - # > - # > Not possible yet! - # > Diagnostic strings should be written in UTF-8, - # > the client can translate to the relevant code page if needed. - # > Each translation completely replaces the format string - # > for the diagnostic. - # > -- http://clang.llvm.org/docs/InternalsManual.html#internals-diag-translation - - try: - proc = subprocess.Popen( - invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, encoding="utf-8" - ) - except OSError as exc: - raise DiffError(f"Command '{subprocess.list2cmdline(invocation)}' failed to start: {exc}") - proc_stdout = proc.stdout - proc_stderr = proc.stderr - - # hopefully the stderr pipe won't get full and block the process - outs = list(proc_stdout.readlines()) - errs = list(proc_stderr.readlines()) - proc.wait() - if proc.returncode: - raise DiffError( - "Command '{}' returned non-zero exit status {}".format( - subprocess.list2cmdline(invocation), proc.returncode - ), - errs, - ) - return make_diff(file, original, outs), errs - - -def bold_red(s): - return "\x1b[1m\x1b[31m" + s + "\x1b[0m" - - -def colorize(diff_lines): - def bold(s): - return "\x1b[1m" + s + "\x1b[0m" - - def cyan(s): - return "\x1b[36m" + s + "\x1b[0m" - - def green(s): - return "\x1b[32m" + s + "\x1b[0m" - - def red(s): - return "\x1b[31m" + s + "\x1b[0m" - - for line in diff_lines: - if line[:4] in ["--- ", "+++ "]: - yield bold(line) - elif line.startswith("@@ "): - yield cyan(line) - elif line.startswith("+"): - yield green(line) - elif line.startswith("-"): - yield red(line) - else: - yield line - - -def print_diff(diff_lines, use_color): - if use_color: - diff_lines = colorize(diff_lines) - sys.stdout.writelines(diff_lines) - - -def print_trouble(prog, message, use_colors): - error_text = "error:" - if use_colors: - error_text = bold_red(error_text) - print(f"{prog}: {error_text} {message}", file=sys.stderr) - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--clang-format-executable", - metavar="EXECUTABLE", - help="path to the clang-format executable", - default="clang-format", - ) - parser.add_argument( - "--extensions", - help=f"comma separated list of file extensions (default: {DEFAULT_EXTENSIONS})", - default=DEFAULT_EXTENSIONS, - ) - parser.add_argument("-r", "--recursive", action="store_true", help="run recursively over directories") - parser.add_argument("files", metavar="file", nargs="+") - parser.add_argument("-q", "--quiet", action="store_true") - parser.add_argument( - "-j", - metavar="N", - type=int, - default=0, - help="run N clang-format jobs in parallel (default number of cpus + 1)", - ) - parser.add_argument( - "--color", default="auto", choices=["auto", "always", "never"], help="show colored diff (default: auto)" - ) - parser.add_argument( - "-e", - "--exclude", - metavar="PATTERN", - action="append", - default=[], - help="exclude paths matching the given glob-like pattern(s) from recursive search", - ) - - args = parser.parse_args() - - # use default signal handling, like diff return SIGINT value on ^C - # https://bugs.python.org/issue14229#msg156446 - signal.signal(signal.SIGINT, signal.SIG_DFL) - try: - signal.SIGPIPE - except AttributeError: - # compatibility, SIGPIPE does not exist on Windows - pass - else: - signal.signal(signal.SIGPIPE, signal.SIG_DFL) - - colored_stdout = False - colored_stderr = False - if args.color == "always": - colored_stdout = True - colored_stderr = True - elif args.color == "auto": - colored_stdout = sys.stdout.isatty() - colored_stderr = sys.stderr.isatty() - - version_invocation = [args.clang_format_executable, "--version"] - try: - subprocess.check_call(version_invocation, stdout=DEVNULL) - except subprocess.CalledProcessError as e: - print_trouble(parser.prog, str(e), use_colors=colored_stderr) - return ExitStatus.TROUBLE - except OSError as e: - print_trouble( - parser.prog, - f"Command '{subprocess.list2cmdline(version_invocation)}' failed to start: {e}", - use_colors=colored_stderr, - ) - return ExitStatus.TROUBLE - - retcode = ExitStatus.SUCCESS - files = list_files( - args.files, recursive=args.recursive, exclude=args.exclude, extensions=args.extensions.split(",") - ) - - if not files: - return - - njobs = args.j - if njobs == 0: - njobs = multiprocessing.cpu_count() + 1 - njobs = min(len(files), njobs) - - if njobs == 1: - # execute directly instead of in a pool, - # less overhead, simpler stacktraces - it = (run_clang_format_diff_wrapper(args, file) for file in files) - pool = None - else: - pool = multiprocessing.Pool(njobs) - it = pool.imap_unordered(partial(run_clang_format_diff_wrapper, args), files) - while True: - try: - outs, errs = next(it) - except StopIteration: - break - except DiffError as e: - print_trouble(parser.prog, str(e), use_colors=colored_stderr) - retcode = ExitStatus.TROUBLE - sys.stderr.writelines(e.errs) - except UnexpectedError as e: - print_trouble(parser.prog, str(e), use_colors=colored_stderr) - sys.stderr.write(e.formatted_traceback) - retcode = ExitStatus.TROUBLE - # stop at the first unexpected error, - # something could be very wrong, - # don't process all files unnecessarily - if pool: - pool.terminate() - break - else: - sys.stderr.writelines(errs) - if outs == []: - continue - if not args.quiet: - print_diff(outs, use_color=colored_stdout) - if retcode == ExitStatus.SUCCESS: - retcode = ExitStatus.DIFF - return retcode - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.circleci/unittest/linux/scripts/run_test.sh b/.circleci/unittest/linux/scripts/run_test.sh deleted file mode 100755 index 5348baa71dd..00000000000 --- a/.circleci/unittest/linux/scripts/run_test.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -set -e - -eval "$(./conda/bin/conda shell.bash hook)" -conda activate ./env - -python -m torch.utils.collect_env - -case "$(uname -s)" in - Darwin*) - # The largest macOS runner is not able to handle the regular test suite plus the transforms v2 tests at the same - # time due to insufficient resources. Thus, we ignore the transforms v2 tests at first and run them in a separate - # step afterwards. - GLOB='test/test_transforms_v2*' - pytest --junitxml=test-results/junit.xml -v --durations 20 --ignore-glob="${GLOB}" - eval "pytest --junitxml=test-results/junit-transforms-v2.xml -v --durations 20 ${GLOB}" - ;; - *) - pytest --junitxml=test-results/junit.xml -v --durations 20 - ;; -esac diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh deleted file mode 100755 index 8a8a78f1fb2..00000000000 --- a/.circleci/unittest/linux/scripts/setup_env.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -# This script is for setting up environment in which unit test is ran. -# To speed up the CI time, the resulting environment is cached. -# -# Do not install PyTorch and torchvision here, otherwise they also get cached. - -set -ex - -this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -# Avoid error: "fatal: unsafe repository" -git config --global --add safe.directory '*' -root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" - -cd "${root_dir}" - -case "$(uname -s)" in - Darwin*) os=MacOSX;; - *) os=Linux -esac - -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" - bash ./miniconda.sh -b -f -p "${conda_dir}" -fi -eval "$(${conda_dir}/bin/conda shell.bash hook)" - -# 2. Create test environment at ./env -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" - -# 3. Install Conda dependencies -printf "* Installing dependencies (except PyTorch)\n" -FFMPEG_PIN="=4.2" -if [[ "${PYTHON_VERSION}" == "3.9" ]]; then - FFMPEG_PIN=">=4.2" -fi - -conda install -y -c pytorch "ffmpeg${FFMPEG_PIN}" -conda env update --file "${this_dir}/environment.yml" --prune diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml deleted file mode 100644 index d229aafb41a..00000000000 --- a/.circleci/unittest/windows/scripts/environment.yml +++ /dev/null @@ -1,19 +0,0 @@ -channels: - - pytorch - - defaults -dependencies: - - pytest - - pytest-cov - - pytest-mock - - pip - - libpng - - jpeg - - ca-certificates - - hdf5 - - setuptools - - pip: - - future - - scipy - - av !=9.1.1, <10 - - dataclasses - - h5py diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh deleted file mode 100644 index 7c55c8144d9..00000000000 --- a/.circleci/unittest/windows/scripts/install.sh +++ /dev/null @@ -1,53 +0,0 @@ - -#!/usr/bin/env bash - -unset PYTORCH_VERSION -# For unittest, nightly PyTorch is used as the following section, -# so no need to set PYTORCH_VERSION. -# In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config. - -set -ex - -this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" -conda activate ./env - -# TODO, refactor the below logic to make it easy to understand how to get correct cuda_version. -if [ "${CU_VERSION:-}" == cpu ] ; then - cudatoolkit="cpuonly" - version="cpu" -else - if [[ ${#CU_VERSION} -eq 4 ]]; then - CUDA_VERSION="${CU_VERSION:2:1}.${CU_VERSION:3:1}" - elif [[ ${#CU_VERSION} -eq 5 ]]; then - CUDA_VERSION="${CU_VERSION:2:2}.${CU_VERSION:4:1}" - fi - - cuda_toolkit_pckg="cudatoolkit" - if [[ $CUDA_VERSION == 11.6 || $CUDA_VERSION == 11.7 || $CUDA_VERSION == 11.8 || $CUDA_VERSION == 12.1 ]]; then - cuda_toolkit_pckg="pytorch-cuda" - fi - - echo "Using CUDA $CUDA_VERSION as determined by CU_VERSION" - version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")" - cudatoolkit="${cuda_toolkit_pckg}=${version}" -fi - -printf "Installing PyTorch with %s\n" "${cudatoolkit}" -conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c nvidia "pytorch-${UPLOAD_CHANNEL}"::pytorch[build="*${version}*"] "${cudatoolkit}" - -torch_cuda=$(python -c "import torch; print(torch.cuda.is_available())") -echo torch.cuda.is_available is $torch_cuda - -if [ ! -z "${CUDA_VERSION:-}" ] ; then - if [ "$torch_cuda" == "False" ]; then - echo "torch with cuda installed but torch.cuda.is_available() is False" - exit 1 - fi -fi - -source "$this_dir/set_cuda_envs.sh" - -printf "* Installing torchvision\n" -"$this_dir/vc_env_helper.bat" python setup.py develop diff --git a/.circleci/unittest/windows/scripts/install_conda.bat b/.circleci/unittest/windows/scripts/install_conda.bat deleted file mode 100644 index 6052ad08b10..00000000000 --- a/.circleci/unittest/windows/scripts/install_conda.bat +++ /dev/null @@ -1 +0,0 @@ -start /wait "" "%miniconda_exe%" /S /InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /D=%tmp_conda% diff --git a/.circleci/unittest/windows/scripts/post_process.sh b/.circleci/unittest/windows/scripts/post_process.sh deleted file mode 100644 index 5c5cbb758a9..00000000000 --- a/.circleci/unittest/windows/scripts/post_process.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -e - -eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" -conda activate ./env diff --git a/.circleci/unittest/windows/scripts/run_test.sh b/.circleci/unittest/windows/scripts/run_test.sh deleted file mode 100644 index 802ad37f511..00000000000 --- a/.circleci/unittest/windows/scripts/run_test.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -e - -eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" -conda activate ./env - -this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source "$this_dir/set_cuda_envs.sh" - -python -m torch.utils.collect_env -pytest --junitxml=test-results/junit.xml -v --durations 20 diff --git a/.circleci/unittest/windows/scripts/set_cuda_envs.sh b/.circleci/unittest/windows/scripts/set_cuda_envs.sh deleted file mode 100644 index 7db3137b594..00000000000 --- a/.circleci/unittest/windows/scripts/set_cuda_envs.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -set -ex - -echo CU_VERSION is "${CU_VERSION}" -echo CUDA_VERSION is "${CUDA_VERSION}" - -# Currenly, CU_VERSION and CUDA_VERSION are not consistent. -# to understand this code, see https://github.com/pytorch/vision/issues/4443 -version="cpu" -if [[ ! -z "${CUDA_VERSION}" ]] ; then - version="$CUDA_VERSION" -else - if [[ ${#CU_VERSION} -eq 5 ]]; then - version="${CU_VERSION:2:2}.${CU_VERSION:4:1}" - fi -fi - -# Don't use if [[ "$version" == "cpu" ]]; then exit 0 fi. -# It would exit the shell. One result is cpu tests would not run if the shell exit. -# Unless there's an error, Don't exit. -if [[ "$version" != "cpu" ]]; then - # set cuda envs - export PATH="/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${version}/bin:/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${version}/libnvvp:$PATH" - export CUDA_PATH_V${version/./_}="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${version}" - export CUDA_PATH="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${version}" - - if [ ! -d "$CUDA_PATH" ]; then - echo "$CUDA_PATH" does not exist - exit 1 - fi - - if [ ! -f "${CUDA_PATH}\include\nvjpeg.h" ]; then - echo "nvjpeg does not exist" - exit 1 - fi - - # check cuda driver version - for path in '/c/Program Files/NVIDIA Corporation/NVSMI/nvidia-smi.exe' /c/Windows/System32/nvidia-smi.exe; do - if [[ -x "$path" ]]; then - "$path" || echo "true"; - break - fi - done - - which nvcc - nvcc --version - env | grep CUDA -fi diff --git a/.circleci/unittest/windows/scripts/setup_env.sh b/.circleci/unittest/windows/scripts/setup_env.sh deleted file mode 100644 index 84697875979..00000000000 --- a/.circleci/unittest/windows/scripts/setup_env.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -# This script is for setting up environment in which unit test is ran. -# To speed up the CI time, the resulting environment is cached. -# -# Do not install PyTorch and torchvision here, otherwise they also get cached. - -set -ex - -this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -root_dir="$(git rev-parse --show-toplevel)" -conda_dir="${root_dir}/conda" -env_dir="${root_dir}/env" - -cd "${root_dir}" - -# 1. Install conda at ./conda -if [ ! -d "${conda_dir}" ]; then - printf "* Installing conda\n" - export tmp_conda="$(echo $conda_dir | tr '/' '\\')" - export miniconda_exe="$(echo $root_dir | tr '/' '\\')\\miniconda.exe" - curl --output miniconda.exe https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -O - "$this_dir/install_conda.bat" - unset tmp_conda - unset miniconda_exe -fi - -eval "$(${conda_dir}/Scripts/conda.exe 'shell.bash' 'hook')" - -# 2. Create test environment at ./env -if [ ! -d "${env_dir}" ]; then - printf "* Creating a test environment\n" - conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" -fi -conda activate "${env_dir}" - -# 3. Install Conda dependencies -printf "* Installing dependencies (except PyTorch)\n" -conda env update --file "${this_dir}/environment.yml" --prune - -# 4. Downgrade setuptools on Python 3.7. -# See https://github.com/pytorch/vision/pull/5868 -if [[ "${PYTHON_VERSION}" == '3.7' ]]; then - pip install --upgrade setuptools==58.0.4 -fi diff --git a/.circleci/unittest/windows/scripts/vc_env_helper.bat b/.circleci/unittest/windows/scripts/vc_env_helper.bat deleted file mode 100644 index 9410135677a..00000000000 --- a/.circleci/unittest/windows/scripts/vc_env_helper.bat +++ /dev/null @@ -1,39 +0,0 @@ -@echo on - -set VC_VERSION_LOWER=16 -set VC_VERSION_UPPER=17 - -for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [%VC_VERSION_LOWER%^,%VC_VERSION_UPPER%^) -property installationPath`) do ( - if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( - set "VS15INSTALLDIR=%%i" - set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat" - goto vswhere - ) -) - -:vswhere -if "%VSDEVCMD_ARGS%" == "" ( - call "%VS15VCVARSALL%" x64 || exit /b 1 -) else ( - call "%VS15VCVARSALL%" x64 %VSDEVCMD_ARGS% || exit /b 1 -) - -@echo on - -set DISTUTILS_USE_SDK=1 - -set args=%1 -shift -:start -if [%1] == [] goto done -set args=%args% %1 -shift -goto start - -:done -if "%args%" == "" ( - echo Usage: vc_env_helper.bat [command] [args] - echo e.g. vc_env_helper.bat cl /c test.cpp -) - -%args% || exit /b 1 diff --git a/.github/scripts/unittest.sh b/.github/scripts/unittest.sh index 2a0b7154200..663181517fb 100755 --- a/.github/scripts/unittest.sh +++ b/.github/scripts/unittest.sh @@ -11,4 +11,5 @@ echo '::group::Install testing utilities' pip install --progress-bar=off pytest pytest-mock pytest-cov echo '::endgroup::' -pytest --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" -v --durations=25 +#pytest --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" -v --durations=25 +python test/smoke_test.py diff --git a/.github/workflows/build-cmake.yml b/.github/workflows/build-cmake.yml deleted file mode 100644 index 8fdf99c84d1..00000000000 --- a/.github/workflows/build-cmake.yml +++ /dev/null @@ -1,92 +0,0 @@ -name: CMake - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: - -jobs: - linux: - strategy: - matrix: - include: - - runner: linux.12xlarge - gpu-arch-type: cpu - - runner: linux.g5.4xlarge.nvidia.gpu - gpu-arch-type: cuda - gpu-arch-version: "11.8" - fail-fast: false - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - runner: ${{ matrix.runner }} - gpu-arch-type: ${{ matrix.gpu-arch-type }} - gpu-arch-version: ${{ matrix.gpu-arch-version }} - script: | - set -euo pipefail - - export PYTHON_VERSION=3.8 - export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} - export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} - - ./.github/scripts/cmake.sh - - macos: - strategy: - matrix: - include: - - runner: macos-12 - - runner: macos-m1-12 - fail-fast: false - uses: pytorch/test-infra/.github/workflows/macos_job.yml@main - with: - repository: pytorch/vision - runner: ${{ matrix.runner }} - script: | - set -euo pipefail - - export PYTHON_VERSION=3.8 - export GPU_ARCH_TYPE=cpu - export GPU_ARCH_VERSION='' - - ./.github/scripts/cmake.sh - - windows: - strategy: - matrix: - include: - - runner: windows.4xlarge - gpu-arch-type: cpu - - runner: windows.g5.4xlarge.nvidia.gpu - gpu-arch-type: cuda - gpu-arch-version: "11.8" - fail-fast: false - uses: pytorch/test-infra/.github/workflows/windows_job.yml@main - with: - repository: pytorch/vision - runner: ${{ matrix.runner }} - gpu-arch-type: ${{ matrix.gpu-arch-type }} - gpu-arch-version: ${{ matrix.gpu-arch-version }} - script: | - set -euo pipefail - - source packaging/windows/internal/vc_install_helper.sh - - # FIXME: Basically, we are reinstalling CUDA here. We only need this, because we need to copy some files that - # can be extracted from the CUDA installer, but are not available on our Windows AMI. - # See https://github.com/pytorch/test-infra/pull/4189 - if [[ ${{ matrix.gpu-arch-type }} == cuda ]]; then - export CU_VERSION=cu$(echo ${{ matrix.gpu-arch-version }} | sed 's/\.//') - echo CU_VERSION="${CU_VERSION}" - packaging/windows/internal/cuda_install.bat - fi - - export PYTHON_VERSION=3.8 - export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} - export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} - - ./.github/scripts/cmake.sh diff --git a/.github/workflows/build-conda-linux.yml b/.github/workflows/build-conda-linux.yml deleted file mode 100644 index e44bb877ed6..00000000000 --- a/.github/workflows/build-conda-linux.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Build Linux Conda - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: conda - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: "" - post-script: "" - conda-package-directory: packaging/torchvision - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_conda_linux.yml@main - with: - conda-package-directory: ${{ matrix.conda-package-directory }} - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-m1.yml b/.github/workflows/build-conda-m1.yml deleted file mode 100644 index 7006b6714aa..00000000000 --- a/.github/workflows/build-conda-m1.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Build M1 Conda - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: conda - os: macos-arm64 - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: "" - post-script: "" - conda-package-directory: packaging/torchvision - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_conda_macos.yml@main - with: - conda-package-directory: ${{ matrix.conda-package-directory }} - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - runner-type: macos-m1-12 - trigger-event: ${{ github.event_name }} - secrets: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-macos.yml b/.github/workflows/build-conda-macos.yml deleted file mode 100644 index 39d00534b3a..00000000000 --- a/.github/workflows/build-conda-macos.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Build Macos Conda - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: conda - os: macos - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: "" - post-script: "" - conda-package-directory: packaging/torchvision - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_conda_macos.yml@main - with: - conda-package-directory: ${{ matrix.conda-package-directory }} - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - runner-type: macos-12 - trigger-event: ${{ github.event_name }} - secrets: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-windows.yml b/.github/workflows/build-conda-windows.yml deleted file mode 100644 index 954d09d39ec..00000000000 --- a/.github/workflows/build-conda-windows.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Build Windows Conda - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: conda - os: windows - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: packaging/pre_build_script.sh - env-script: packaging/windows/internal/vc_env_helper.bat - post-script: "" - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_conda_windows.yml@main - with: - conda-package-directory: ${{ matrix.conda-package-directory }} - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-wheels-linux.yml b/.github/workflows/build-wheels-linux.yml deleted file mode 100644 index e997d648ec2..00000000000 --- a/.github/workflows/build-wheels-linux.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Build Linux Wheels - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-m1.yml b/.github/workflows/build-wheels-m1.yml deleted file mode 100644 index 66c7687acc9..00000000000 --- a/.github/workflows/build-wheels-m1.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Build M1 Wheels - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: macos-arm64 - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - runner-type: macos-m1-12 - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-macos.yml b/.github/workflows/build-wheels-macos.yml deleted file mode 100644 index 6c5ebc0fc37..00000000000 --- a/.github/workflows/build-wheels-macos.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Build Macos Wheels - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: macos - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - runner-type: macos-12 - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-windows.yml b/.github/workflows/build-wheels-windows.yml deleted file mode 100644 index 3d818ece538..00000000000 --- a/.github/workflows/build-wheels-windows.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Build Windows Wheels - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: windows - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: packaging/pre_build_script.sh - env-script: packaging/windows/internal/vc_env_helper.bat - post-script: "python packaging/wheel/relocate.py" - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_wheels_windows.yml@main - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-script: ${{ matrix.env-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index f4cc76db05e..00000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,115 +0,0 @@ -name: Docs - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - - v[0-9]+.[0-9]+.[0-9] - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - build: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - upload-artifact: docs - script: | - set -euo pipefail - - export PYTHON_VERSION=3.8 - export GPU_ARCH_TYPE=cpu - export GPU_ARCH_VERSION='' - ./.github/scripts/setup-env.sh - - # Prepare conda - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - conda activate ci - # FIXME: not sure why we need this. `ldd torchvision/video_reader.so` shows that it - # already links against the one pulled from conda. However, at runtime it pulls from - # /lib64 - # Should we maybe always do this in `./.github/scripts/setup-env.sh` so that we don't - # have to pay attention in all other workflows? - export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH}" - - cd docs - - echo '::group::Install doc requirements' - pip install --progress-bar=off -r requirements.txt - echo '::endgroup::' - - if [[ ${{ github.event_name }} == push && (${{ github.ref_type }} == tag || (${{ github.ref_type }} == branch && ${{ github.ref_name }} == release/*)) ]]; then - echo '::group::Enable version string sanitization' - # This environment variable just has to exist and must not be empty. The actual value is arbitrary. - # See docs/source/conf.py for details - export TORCHVISION_SANITIZE_VERSION_STR_IN_DOCS=1 - echo '::endgroup::' - fi - - # The runner does not have sufficient memory to run with as many processes as there are - # cores (`-j auto`). Thus, we limit to a single process (`-j 1`) here. - sed -i -e 's/-j auto/-j 1/' Makefile - make html - - cp -r build/html "${RUNNER_ARTIFACT_DIR}" - - # On PRs we also want to upload the docs into our S3 bucket for preview. - if [[ ${{ github.event_name == 'pull_request' }} ]]; then - cp -r build/html/* "${RUNNER_DOCS_DIR}" - fi - - upload: - needs: build - if: github.repository == 'pytorch/vision' && github.event_name == 'push' && - ((github.ref_type == 'branch' && github.ref_name == 'main') || github.ref_type == 'tag') - permissions: - contents: write - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - download-artifact: docs - ref: gh-pages - script: | - set -euo pipefail - - REF_TYPE=${{ github.ref_type }} - REF_NAME=${{ github.ref_name }} - - if [[ "${REF_TYPE}" == branch ]]; then - TARGET_FOLDER="${REF_NAME}" - elif [[ "${REF_TYPE}" == tag ]]; then - case "${REF_NAME}" in - *-rc*) - echo "Aborting upload since this is an RC tag: ${REF_NAME}" - exit 0 - ;; - *) - # Strip the leading "v" as well as the trailing patch version. For example: - # 'v0.15.2' -> '0.15' - TARGET_FOLDER=$(echo "${REF_NAME}" | sed 's/v\([0-9]\+\)\.\([0-9]\+\)\.[0-9]\+/\1.\2/') - ;; - esac - fi - echo "Target Folder: ${TARGET_FOLDER}" - - mkdir -p "${TARGET_FOLDER}" - rm -rf "${TARGET_FOLDER}"/* - mv "${RUNNER_ARTIFACT_DIR}"/html/* "${TARGET_FOLDER}" - git add "${TARGET_FOLDER}" || true - - if [[ "${TARGET_FOLDER}" == main ]]; then - mkdir -p _static - rm -rf _static/* - cp -r "${TARGET_FOLDER}"/_static/* _static - git add _static || true - fi - - git config user.name 'pytorchbot' - git config user.email 'soumith+bot@pytorch.org' - git commit -m "auto-generating sphinx docs" || true - git push diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index ec8d285c9f3..00000000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,107 +0,0 @@ -name: Lint - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: - -jobs: - python-source-and-configs: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - script: | - set -euo pipefail - - echo '::group::Setup environment' - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - conda create --name ci --quiet --yes python=3.8 pip - conda activate ci - echo '::endgroup::' - - echo '::group::Install lint tools' - pip install --progress-bar=off pre-commit - echo '::endgroup::' - - set +e - pre-commit run --all-files - - if [ $? -ne 0 ]; then - git --no-pager diff - exit 1 - fi - - c-source: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - script: | - set -euo pipefail - - echo '::group::Setup environment' - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - # clang-format needs some shared libraries that conflict with the system ones. Thus, we install them from conda - # and prepend the libraries to linker path to prioritize them. `ncurses=5` is only available on the conda-forge - # channel. Since we are not building or testing here, this is fine. - conda create --name ci --quiet --yes -c conda-forge python=3.8 ncurses=5 libgcc - conda activate ci - export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH}" - echo '::endgroup::' - - echo '::group::Install lint tools' - curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o ./clang-format - chmod +x ./clang-format - echo '::endgroup::' - - echo '::group::Lint C source' - set +e - ./.circleci/unittest/linux/scripts/run-clang-format.py -r torchvision/csrc --clang-format-executable ./clang-format - - if [ $? -ne 0 ]; then - git --no-pager diff - exit 1 - fi - echo '::endgroup::' - - - python-types: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - script: | - set -euo pipefail - - export PYTHON_VERSION=3.8 - export GPU_ARCH_TYPE=cpu - export GPU_ARCH_VERSION='' - - ./.github/scripts/setup-env.sh - - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - conda activate ci - - echo '::group::Install lint tools' - pip install --progress-bar=off mypy - echo '::endgroup::' - - echo '::group::Lint Python types' - mypy --install-types --non-interactive --config-file mypy.ini - echo '::endgroup::' - - bc: - if: github.event.pull_request - runs-on: ubuntu-latest - steps: - - name: Run BC Lint Action - uses: pytorch/test-infra/.github/actions/bc-lint@main - with: - repo: ${{ github.event.pull_request.head.repo.full_name }} - base_sha: ${{ github.event.pull_request.base.sha }} - head_sha: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/pr-labels.yml b/.github/workflows/pr-labels.yml deleted file mode 100644 index 20c37e4fd88..00000000000 --- a/.github/workflows/pr-labels.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: pr-labels - -on: - push: - branches: - - main - -jobs: - is-properly-labeled: - runs-on: ubuntu-latest - - steps: - - name: Set up python - uses: actions/setup-python@v2 - - - name: Install requests - run: pip install requests - - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Process commit and find merger responsible for labeling - id: commit - run: echo "::set-output name=merger::$(python .github/process_commit.py ${{ github.sha }})" - - - name: Ping merger responsible for labeling if necessary - if: ${{ steps.commit.outputs.merger != '' }} - uses: mshick/add-pr-comment@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - message: | - Hey ${{ steps.commit.outputs.merger }}! - - You merged this PR, but no labels were added. The list of valid labels is available at https://github.com/pytorch/vision/blob/main/.github/process_commit.py diff --git a/.github/workflows/prototype-tests-linux-gpu.yml b/.github/workflows/prototype-tests-linux-gpu.yml deleted file mode 100644 index dee425054d5..00000000000 --- a/.github/workflows/prototype-tests-linux-gpu.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Prototype tests on Linux - -on: - pull_request: - -jobs: - unittests-prototype: - strategy: - matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.11" - runner: ["linux.12xlarge"] - gpu-arch-type: ["cpu"] - include: - - python-version: "3.8" - runner: linux.g5.4xlarge.nvidia.gpu - gpu-arch-type: cuda - gpu-arch-version: "11.7" - fail-fast: false - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - runner: ${{ matrix.runner }} - gpu-arch-type: ${{ matrix.gpu-arch-type }} - gpu-arch-version: ${{ matrix.gpu-arch-version }} - timeout: 120 - script: | - set -euo pipefail - - export PYTHON_VERSION=${{ matrix.python-version }} - export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} - export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} - ./.github/scripts/setup-env.sh - - # Prepare conda - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - conda activate ci - - echo '::group::Install testing utilities' - pip install --progress-bar=off pytest pytest-mock pytest-cov - echo '::endgroup::' - - # We don't want to run the prototype datasets tests. Since the positional glob into `pytest`, i.e. - # `test/test_prototype*.py` takes the highest priority, neither `--ignore` nor `--ignore-glob` can help us here. - rm test/test_prototype_datasets*.py - pytest \ - -v --durations=25 \ - --cov=torchvision/prototype --cov-report=term-missing \ - --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" \ - test/test_prototype_*.py diff --git a/.github/workflows/tests-schedule.yml b/.github/workflows/tests-schedule.yml deleted file mode 100644 index 5426fdc997a..00000000000 --- a/.github/workflows/tests-schedule.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: tests - -on: - pull_request: - paths: - - "test/test_datasets_download.py" - - ".github/failed_schedule_issue_template.md" - - ".github/workflows/tests-schedule.yml" - - schedule: - - cron: "0 9 * * *" - -jobs: - download: - runs-on: ubuntu-latest - - steps: - - name: Set up python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Upgrade system packages - run: python -m pip install --upgrade pip setuptools wheel - - - name: SSL - run: python -c 'import ssl; print(ssl.OPENSSL_VERSION)' - - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Install torch nightly build - run: pip install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - - - name: Install torchvision - run: pip install --no-build-isolation --editable . - - - name: Install all optional dataset requirements - run: pip install scipy pycocotools lmdb requests - - - name: Install tests requirements - run: pip install pytest - - - name: Run tests - run: pytest -ra -v test/test_datasets_download.py - - - uses: JasonEtco/create-an-issue@v2.4.0 - name: Create issue if download tests failed - if: failure() && github.event_name == 'schedule' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} - WORKFLOW: ${{ github.workflow }} - JOB: ${{ github.job }} - ID: ${{ github.run_id }} - with: - filename: .github/failed_schedule_issue_template.md diff --git a/.github/workflows/update-viablestrict.yml b/.github/workflows/update-viablestrict.yml deleted file mode 100644 index 2d9c2265676..00000000000 --- a/.github/workflows/update-viablestrict.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Update viable/strict - -on: - pull_request: - paths: - - .github/workflows/update-viablestrict.yml - schedule: - - cron: 10,40 * * * * - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }} - cancel-in-progress: false - -jobs: - do_update_viablestrict: - uses: pytorch/test-infra/.github/workflows/update-viablestrict.yml@main - with: - repository: pytorch/vision - required_checks: "Build Linux,Build M1,Build Macos,Tests on Linux,Tests on macOS,Docs,Lint" - secrets: - ROCKSET_API_KEY: ${{ secrets.ROCKSET_API_KEY }} - GITHUB_DEPLOY_KEY : ${{ secrets.VISION_GITHUB_DEPLOY_KEY }} From d7371d4353dd4b9385fa4daa21fd4a485f3a6850 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 24 May 2023 13:13:31 +0200 Subject: [PATCH 7/7] Revert "[REVERTME] disable unrelated CI" This reverts commit 48b25c2436739c0a60c3b6edfab6f7c352dbeec0. --- .circleci/.gitignore | 1 + .circleci/build_docs/commit_docs.sh | 35 ++ .circleci/config.yml | 271 ++++++++++++++ .circleci/config.yml.in | 251 +++++++++++++ .circleci/regenerate.py | 58 +++ .circleci/smoke_test/docker/Dockerfile | 34 ++ .../android/scripts/binary_android_build.sh | 27 ++ .../android/scripts/binary_android_upload.sh | 34 ++ .../android/scripts/install_gradle.sh | 19 + .../unittest/ios/scripts/binary_ios_build.sh | 47 +++ .../unittest/ios/scripts/binary_ios_upload.sh | 42 +++ .../unittest/linux/scripts/environment.yml | 16 + .circleci/unittest/linux/scripts/install.sh | 45 +++ .../unittest/linux/scripts/post_process.sh | 6 + .../linux/scripts/run-clang-format.py | 331 ++++++++++++++++++ .circleci/unittest/linux/scripts/run_test.sh | 22 ++ .circleci/unittest/linux/scripts/setup_env.sh | 47 +++ .../unittest/windows/scripts/environment.yml | 19 + .circleci/unittest/windows/scripts/install.sh | 53 +++ .../windows/scripts/install_conda.bat | 1 + .../unittest/windows/scripts/post_process.sh | 6 + .../unittest/windows/scripts/run_test.sh | 12 + .../unittest/windows/scripts/set_cuda_envs.sh | 48 +++ .../unittest/windows/scripts/setup_env.sh | 45 +++ .../windows/scripts/vc_env_helper.bat | 39 +++ .github/scripts/unittest.sh | 3 +- .github/workflows/build-cmake.yml | 92 +++++ .github/workflows/build-conda-linux.yml | 51 +++ .github/workflows/build-conda-m1.yml | 52 +++ .github/workflows/build-conda-macos.yml | 52 +++ .github/workflows/build-conda-windows.yml | 51 +++ .github/workflows/build-wheels-linux.yml | 50 +++ .github/workflows/build-wheels-m1.yml | 51 +++ .github/workflows/build-wheels-macos.yml | 51 +++ .github/workflows/build-wheels-windows.yml | 52 +++ .github/workflows/docs.yml | 115 ++++++ .github/workflows/lint.yml | 107 ++++++ .github/workflows/pr-labels.yml | 35 ++ .../workflows/prototype-tests-linux-gpu.yml | 54 +++ .github/workflows/tests-schedule.yml | 57 +++ .github/workflows/update-viablestrict.yml | 23 ++ 41 files changed, 2403 insertions(+), 2 deletions(-) create mode 100644 .circleci/.gitignore create mode 100755 .circleci/build_docs/commit_docs.sh create mode 100644 .circleci/config.yml create mode 100644 .circleci/config.yml.in create mode 100755 .circleci/regenerate.py create mode 100644 .circleci/smoke_test/docker/Dockerfile create mode 100644 .circleci/unittest/android/scripts/binary_android_build.sh create mode 100644 .circleci/unittest/android/scripts/binary_android_upload.sh create mode 100755 .circleci/unittest/android/scripts/install_gradle.sh create mode 100755 .circleci/unittest/ios/scripts/binary_ios_build.sh create mode 100644 .circleci/unittest/ios/scripts/binary_ios_upload.sh create mode 100644 .circleci/unittest/linux/scripts/environment.yml create mode 100755 .circleci/unittest/linux/scripts/install.sh create mode 100755 .circleci/unittest/linux/scripts/post_process.sh create mode 100755 .circleci/unittest/linux/scripts/run-clang-format.py create mode 100755 .circleci/unittest/linux/scripts/run_test.sh create mode 100755 .circleci/unittest/linux/scripts/setup_env.sh create mode 100644 .circleci/unittest/windows/scripts/environment.yml create mode 100644 .circleci/unittest/windows/scripts/install.sh create mode 100644 .circleci/unittest/windows/scripts/install_conda.bat create mode 100644 .circleci/unittest/windows/scripts/post_process.sh create mode 100644 .circleci/unittest/windows/scripts/run_test.sh create mode 100644 .circleci/unittest/windows/scripts/set_cuda_envs.sh create mode 100644 .circleci/unittest/windows/scripts/setup_env.sh create mode 100644 .circleci/unittest/windows/scripts/vc_env_helper.bat create mode 100644 .github/workflows/build-cmake.yml create mode 100644 .github/workflows/build-conda-linux.yml create mode 100644 .github/workflows/build-conda-m1.yml create mode 100644 .github/workflows/build-conda-macos.yml create mode 100644 .github/workflows/build-conda-windows.yml create mode 100644 .github/workflows/build-wheels-linux.yml create mode 100644 .github/workflows/build-wheels-m1.yml create mode 100644 .github/workflows/build-wheels-macos.yml create mode 100644 .github/workflows/build-wheels-windows.yml create mode 100644 .github/workflows/docs.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/pr-labels.yml create mode 100644 .github/workflows/prototype-tests-linux-gpu.yml create mode 100644 .github/workflows/tests-schedule.yml create mode 100644 .github/workflows/update-viablestrict.yml diff --git a/.circleci/.gitignore b/.circleci/.gitignore new file mode 100644 index 00000000000..485dee64bcf --- /dev/null +++ b/.circleci/.gitignore @@ -0,0 +1 @@ +.idea diff --git a/.circleci/build_docs/commit_docs.sh b/.circleci/build_docs/commit_docs.sh new file mode 100755 index 00000000000..04e3538fefc --- /dev/null +++ b/.circleci/build_docs/commit_docs.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -ex + + +if [ "$2" == "" ]; then + echo call as "$0" "" "" + echo where src is the root of the built documentation git checkout and + echo branch should be "main" or "1.7" or so + exit 1 +fi + +src=$1 +target=$2 + +echo "committing docs from ${src} to ${target}" + +pushd "${src}" +git checkout gh-pages +mkdir -p ./"${target}" +rm -rf ./"${target}"/* +cp -r "${src}/docs/build/html/"* ./"$target" +if [ "${target}" == "main" ]; then + mkdir -p ./_static + rm -rf ./_static/* + cp -r "${src}/docs/build/html/_static/"* ./_static + git add --all ./_static || true +fi +git add --all ./"${target}" || true +git config user.email "soumith+bot@pytorch.org" +git config user.name "pytorchbot" +# If there aren't changes, don't make a commit; push is no-op +git commit -m "auto-generating sphinx docs" || true +git remote add https https://github.com/pytorch/vision.git +git push -u https gh-pages diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000000..0463007af46 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,271 @@ +version: 2.1 + +# How to test the Linux jobs: +# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ +# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_win_wheel_py3.8 +# - Replace binary_win_wheel_py3.8 with the name of the job you want to test. +# Job names are 'name:' key. + +executors: + windows-cpu: + machine: + resource_class: windows.xlarge + image: windows-server-2019-vs2019:stable + shell: bash.exe + + windows-gpu: + machine: + resource_class: windows.gpu.nvidia.medium + image: windows-server-2019-nvidia:stable + shell: bash.exe + +commands: + checkout_merge: + description: "checkout merge branch" + steps: + - checkout +# - run: +# name: Checkout merge branch +# command: | +# set -ex +# BRANCH=$(git rev-parse --abbrev-ref HEAD) +# if [[ "$BRANCH" != "main" ]]; then +# git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH} +# git checkout "merged/$CIRCLE_BRANCH" +# fi + designate_upload_channel: + description: "inserts the correct upload channel into ${BASH_ENV}" + steps: + - run: + name: adding UPLOAD_CHANNEL to BASH_ENV + command: | + our_upload_channel=nightly + # On tags upload to test instead + if [[ -n "${CIRCLE_TAG}" ]]; then + our_upload_channel=test + fi + echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} + + pip_install: + parameters: + args: + type: string + descr: + type: string + default: "" + user: + type: boolean + default: true + steps: + - run: + name: > + <<^ parameters.descr >> pip install << parameters.args >> <> + <<# parameters.descr >> << parameters.descr >> <> + command: > + pip install + <<# parameters.user >> --user <> + --progress-bar=off + << parameters.args >> + +binary_common: &binary_common + parameters: + # Edit these defaults to do a release + build_version: + description: "version number of release binary; by default, build a nightly" + type: string + default: "" + pytorch_version: + description: "PyTorch version to build against; by default, use a nightly" + type: string + default: "" + # Don't edit these + python_version: + description: "Python version to build against (e.g., 3.8)" + type: string + cu_version: + description: "CUDA version to build against, in CU format (e.g., cpu or cu100)" + type: string + default: "cpu" + unicode_abi: + description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" + type: string + default: "" + wheel_docker_image: + description: "Wheel only: what docker image to use" + type: string + default: "" + conda_docker_image: + description: "Conda only: what docker image to use" + type: string + default: "pytorch/conda-builder:cpu" + environment: + PYTHON_VERSION: << parameters.python_version >> + PYTORCH_VERSION: << parameters.pytorch_version >> + UNICODE_ABI: << parameters.unicode_abi >> + CU_VERSION: << parameters.cu_version >> + MACOSX_DEPLOYMENT_TARGET: 10.9 + +smoke_test_common: &smoke_test_common + <<: *binary_common + docker: + - image: torchvision/smoke_test:latest + +jobs: + circleci_consistency: + docker: + - image: cimg/python:3.8 + steps: + - checkout + - pip_install: + args: jinja2 pyyaml + - run: + name: Check CircleCI config consistency + command: | + python .circleci/regenerate.py + git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) + + smoke_test_docker_image_build: + machine: + image: ubuntu-2004:202104-01 + resource_class: large + environment: + image_name: torchvision/smoke_test + steps: + - checkout + - designate_upload_channel + - run: + name: Build and push Docker image + no_output_timeout: "1h" + command: | + set +x + echo "${DOCKER_HUB_TOKEN}" | docker login --username "${DOCKER_HUB_USERNAME}" --password-stdin + set -x + cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID} + docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} ${image_name}:latest + docker push ${image_name}:${CIRCLE_WORKFLOW_ID} + docker push ${image_name}:latest + + cmake_linux_cpu: + <<: *binary_common + docker: + - image: "pytorch/manylinux-cpu" + resource_class: 2xlarge+ + steps: + - checkout_merge + - designate_upload_channel + - run: + name: Setup conda + command: .circleci/unittest/linux/scripts/setup_env.sh + - run: packaging/build_cmake.sh + + cmake_linux_gpu: + <<: *binary_common + machine: + image: ubuntu-2004-cuda-11.4:202110-01 + resource_class: gpu.nvidia.small + steps: + - checkout_merge + - designate_upload_channel + - run: + name: Setup conda + command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh + - run: + name: Build torchvision C++ distribution and test + no_output_timeout: 30m + command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh + + cmake_macos_cpu: + <<: *binary_common + macos: + xcode: "14.0" + steps: + - checkout_merge + - designate_upload_channel + - run: + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + conda install -yq conda-build cmake python=<< parameters.python_version >> + packaging/build_cmake.sh + + cmake_windows_cpu: + <<: *binary_common + executor: + name: windows-cpu + steps: + - checkout_merge + - designate_upload_channel + - run: + command: | + set -ex + source packaging/windows/internal/vc_install_helper.sh + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" + conda activate base + conda create -yn python39 python=3.9 + conda activate python39 + packaging/build_cmake.sh + + cmake_windows_gpu: + <<: *binary_common + executor: + name: windows-gpu + steps: + - checkout_merge + - designate_upload_channel + - run: + name: Update CUDA driver + command: packaging/windows/internal/driver_update.bat + - run: + command: | + set -ex + source packaging/windows/internal/vc_install_helper.sh + packaging/windows/internal/cuda_install.bat + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" + conda activate + conda update -y conda + conda create -yn python39 python=3.9 + conda activate python39 + packaging/build_cmake.sh + + +workflows: + lint: + jobs: + - circleci_consistency + + cmake: + jobs: + - cmake_linux_cpu: + cu_version: cpu + name: cmake_linux_cpu + python_version: '3.8' + - cmake_linux_gpu: + cu_version: cu117 + name: cmake_linux_gpu + python_version: '3.8' + wheel_docker_image: pytorch/manylinux-cuda117 + - cmake_windows_cpu: + cu_version: cpu + name: cmake_windows_cpu + python_version: '3.8' + - cmake_windows_gpu: + cu_version: cu117 + name: cmake_windows_gpu + python_version: '3.8' + - cmake_macos_cpu: + cu_version: cpu + name: cmake_macos_cpu + python_version: '3.8' + + docker_build: + triggers: + - schedule: + cron: "0 10 * * 0" + filters: + branches: + only: + - main + jobs: + - smoke_test_docker_image_build: + context: org-member diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in new file mode 100644 index 00000000000..f3a88f2d987 --- /dev/null +++ b/.circleci/config.yml.in @@ -0,0 +1,251 @@ +version: 2.1 + +# How to test the Linux jobs: +# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ +# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_win_wheel_py3.8 +# - Replace binary_win_wheel_py3.8 with the name of the job you want to test. +# Job names are 'name:' key. + +executors: + windows-cpu: + machine: + resource_class: windows.xlarge + image: windows-server-2019-vs2019:stable + shell: bash.exe + + windows-gpu: + machine: + resource_class: windows.gpu.nvidia.medium + image: windows-server-2019-nvidia:stable + shell: bash.exe + +commands: + checkout_merge: + description: "checkout merge branch" + steps: + - checkout +# - run: +# name: Checkout merge branch +# command: | +# set -ex +# BRANCH=$(git rev-parse --abbrev-ref HEAD) +# if [[ "$BRANCH" != "main" ]]; then +# git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH} +# git checkout "merged/$CIRCLE_BRANCH" +# fi + designate_upload_channel: + description: "inserts the correct upload channel into ${BASH_ENV}" + steps: + - run: + name: adding UPLOAD_CHANNEL to BASH_ENV + command: | + our_upload_channel=nightly + # On tags upload to test instead + if [[ -n "${CIRCLE_TAG}" ]]; then + our_upload_channel=test + fi + echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} + + pip_install: + parameters: + args: + type: string + descr: + type: string + default: "" + user: + type: boolean + default: true + steps: + - run: + name: > + <<^ parameters.descr >> pip install << parameters.args >> <> + <<# parameters.descr >> << parameters.descr >> <> + command: > + pip install + <<# parameters.user >> --user <> + --progress-bar=off + << parameters.args >> + +binary_common: &binary_common + parameters: + # Edit these defaults to do a release + build_version: + description: "version number of release binary; by default, build a nightly" + type: string + default: "" + pytorch_version: + description: "PyTorch version to build against; by default, use a nightly" + type: string + default: "" + # Don't edit these + python_version: + description: "Python version to build against (e.g., 3.8)" + type: string + cu_version: + description: "CUDA version to build against, in CU format (e.g., cpu or cu100)" + type: string + default: "cpu" + unicode_abi: + description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" + type: string + default: "" + wheel_docker_image: + description: "Wheel only: what docker image to use" + type: string + default: "" + conda_docker_image: + description: "Conda only: what docker image to use" + type: string + default: "pytorch/conda-builder:cpu" + environment: + PYTHON_VERSION: << parameters.python_version >> + PYTORCH_VERSION: << parameters.pytorch_version >> + UNICODE_ABI: << parameters.unicode_abi >> + CU_VERSION: << parameters.cu_version >> + MACOSX_DEPLOYMENT_TARGET: 10.9 + +smoke_test_common: &smoke_test_common + <<: *binary_common + docker: + - image: torchvision/smoke_test:latest + +jobs: + circleci_consistency: + docker: + - image: cimg/python:3.8 + steps: + - checkout + - pip_install: + args: jinja2 pyyaml + - run: + name: Check CircleCI config consistency + command: | + python .circleci/regenerate.py + git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) + + smoke_test_docker_image_build: + machine: + image: ubuntu-2004:202104-01 + resource_class: large + environment: + image_name: torchvision/smoke_test + steps: + - checkout + - designate_upload_channel + - run: + name: Build and push Docker image + no_output_timeout: "1h" + command: | + set +x + echo "${DOCKER_HUB_TOKEN}" | docker login --username "${DOCKER_HUB_USERNAME}" --password-stdin + set -x + cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID} + docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} ${image_name}:latest + docker push ${image_name}:${CIRCLE_WORKFLOW_ID} + docker push ${image_name}:latest + + cmake_linux_cpu: + <<: *binary_common + docker: + - image: "pytorch/manylinux-cpu" + resource_class: 2xlarge+ + steps: + - checkout_merge + - designate_upload_channel + - run: + name: Setup conda + command: .circleci/unittest/linux/scripts/setup_env.sh + - run: packaging/build_cmake.sh + + cmake_linux_gpu: + <<: *binary_common + machine: + image: ubuntu-2004-cuda-11.4:202110-01 + resource_class: gpu.nvidia.small + steps: + - checkout_merge + - designate_upload_channel + - run: + name: Setup conda + command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh + - run: + name: Build torchvision C++ distribution and test + no_output_timeout: 30m + command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh + + cmake_macos_cpu: + <<: *binary_common + macos: + xcode: "14.0" + steps: + - checkout_merge + - designate_upload_channel + - run: + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + conda install -yq conda-build cmake python=<< parameters.python_version >> + packaging/build_cmake.sh + + cmake_windows_cpu: + <<: *binary_common + executor: + name: windows-cpu + steps: + - checkout_merge + - designate_upload_channel + - run: + command: | + set -ex + source packaging/windows/internal/vc_install_helper.sh + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" + conda activate base + conda create -yn python39 python=3.9 + conda activate python39 + packaging/build_cmake.sh + + cmake_windows_gpu: + <<: *binary_common + executor: + name: windows-gpu + steps: + - checkout_merge + - designate_upload_channel + - run: + name: Update CUDA driver + command: packaging/windows/internal/driver_update.bat + - run: + command: | + set -ex + source packaging/windows/internal/vc_install_helper.sh + packaging/windows/internal/cuda_install.bat + eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" + conda activate + conda update -y conda + conda create -yn python39 python=3.9 + conda activate python39 + packaging/build_cmake.sh + + +workflows: + lint: + jobs: + - circleci_consistency + + cmake: + jobs: + {{ cmake_workflows() }} + + docker_build: + triggers: + - schedule: + cron: "0 10 * * 0" + filters: + branches: + only: + - main + jobs: + - smoke_test_docker_image_build: + context: org-member diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py new file mode 100755 index 00000000000..e7b8db3d56e --- /dev/null +++ b/.circleci/regenerate.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +""" +This script should use a very simple, functional programming style. +Avoid Jinja macros in favor of native Python functions. + +Don't go overboard on code generation; use Python only to generate +content that can't be easily declared statically using CircleCI's YAML API. + +Data declarations (e.g. the nested loops for defining the configuration matrix) +should be at the top of the file for easy updating. + +See this comment for design rationale: +https://github.com/pytorch/vision/pull/1321#issuecomment-531033978 +""" + +import os.path + +import jinja2 +import yaml +from jinja2 import select_autoescape + + +def indent(indentation, data_list): + return ("\n" + " " * indentation).join(yaml.dump(data_list, default_flow_style=False).splitlines()) + + +def cmake_workflows(indentation=6): + jobs = [] + python_version = "3.8" + for os_type in ["linux", "windows", "macos"]: + # Skip OSX CUDA + device_types = ["cpu", "gpu"] if os_type != "macos" else ["cpu"] + for device in device_types: + job = {"name": f"cmake_{os_type}_{device}", "python_version": python_version} + + job["cu_version"] = "cu117" if device == "gpu" else "cpu" + if device == "gpu" and os_type == "linux": + job["wheel_docker_image"] = "pytorch/manylinux-cuda117" + jobs.append({f"cmake_{os_type}_{device}": job}) + return indent(indentation, jobs) + + +if __name__ == "__main__": + d = os.path.dirname(__file__) + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(d), + lstrip_blocks=True, + autoescape=select_autoescape(enabled_extensions=("html", "xml")), + keep_trailing_newline=True, + ) + + with open(os.path.join(d, "config.yml"), "w") as f: + f.write( + env.get_template("config.yml.in").render( + cmake_workflows=cmake_workflows, + ) + ) diff --git a/.circleci/smoke_test/docker/Dockerfile b/.circleci/smoke_test/docker/Dockerfile new file mode 100644 index 00000000000..34bdcda1053 --- /dev/null +++ b/.circleci/smoke_test/docker/Dockerfile @@ -0,0 +1,34 @@ +# this Dockerfile is for torchvision smoke test, it will be created periodically via CI system +# if you need to do it locally, follow below steps once you have Docker installed +# assuming you're within the directory where this Dockerfile located +# $ docker build . -t torchvision/smoketest + +# if you want to push to aws ecr, make sure you have the rights to write to ECR, then run +# $ eval $(aws ecr get-login --region us-east-1 --no-include-email) +# $ export MYTAG=localbuild ## you can choose whatever tag you like +# $ docker tag torchvision/smoketest 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchvision/smoke_test:${MYTAG} +# $ docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchvision/smoke_test:${MYTAG} + +FROM ubuntu:latest + +RUN apt-get -qq update && apt-get -qq -y install curl bzip2 libsox-fmt-all \ + && curl -sSL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh \ + && bash /tmp/miniconda.sh -bfp /usr/local \ + && rm -rf /tmp/miniconda.sh \ + && conda install -y python=3 \ + && conda update conda \ + && apt-get -qq -y remove curl bzip2 \ + && apt-get -qq -y autoremove \ + && apt-get autoclean \ + && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log \ + && conda clean --all --yes + +ENV PATH /opt/conda/bin:$PATH + +RUN conda create -y --name python3.7 python=3.7 +RUN conda create -y --name python3.8 python=3.8 +RUN conda create -y --name python3.9 python=3.9 +RUN conda create -y --name python3.10 python=3.10 +SHELL [ "/bin/bash", "-c" ] +RUN echo "source /usr/local/etc/profile.d/conda.sh" >> ~/.bashrc +CMD [ "/bin/bash"] diff --git a/.circleci/unittest/android/scripts/binary_android_build.sh b/.circleci/unittest/android/scripts/binary_android_build.sh new file mode 100644 index 00000000000..0d8c0d47d8a --- /dev/null +++ b/.circleci/unittest/android/scripts/binary_android_build.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -ex -o pipefail + +echo "DIR: $(pwd)" +echo "ANDROID_HOME=${ANDROID_HOME}" +echo "ANDROID_NDK_HOME=${ANDROID_NDK_HOME}" +echo "JAVA_HOME=${JAVA_HOME}" + +WORKSPACE=/home/circleci/workspace +VISION_ANDROID=/home/circleci/project/android + +. /home/circleci/project/.circleci/unittest/android/scripts/install_gradle.sh + +GRADLE_LOCAL_PROPERTIES=${VISION_ANDROID}/local.properties +rm -f $GRADLE_LOCAL_PROPERTIES + +echo "sdk.dir=${ANDROID_HOME}" >> $GRADLE_LOCAL_PROPERTIES +echo "ndk.dir=${ANDROID_NDK_HOME}" >> $GRADLE_LOCAL_PROPERTIES + +echo "GRADLE_PATH $GRADLE_PATH" +echo "GRADLE_HOME $GRADLE_HOME" + +${GRADLE_PATH} --scan --stacktrace --debug --no-daemon -p ${VISION_ANDROID} assemble || true + +mkdir -p ~/workspace/artifacts +find . -type f -name *aar -print | xargs tar cfvz ~/workspace/artifacts/artifacts-aars.tgz +find . -type f -name *apk -print | xargs tar cfvz ~/workspace/artifacts/artifacts-apks.tgz diff --git a/.circleci/unittest/android/scripts/binary_android_upload.sh b/.circleci/unittest/android/scripts/binary_android_upload.sh new file mode 100644 index 00000000000..1472a877d90 --- /dev/null +++ b/.circleci/unittest/android/scripts/binary_android_upload.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -ex -o pipefail + +echo "DIR: $(pwd)" +echo "ANDROID_HOME=${ANDROID_HOME}" +echo "ANDROID_NDK_HOME=${ANDROID_NDK_HOME}" +echo "JAVA_HOME=${JAVA_HOME}" + +WORKSPACE=/home/circleci/workspace +VISION_ANDROID=/home/circleci/project/android + +. /home/circleci/project/.circleci/unittest/android/scripts/install_gradle.sh + +GRADLE_LOCAL_PROPERTIES=${VISION_ANDROID}/local.properties +rm -f $GRADLE_LOCAL_PROPERTIES +GRADLE_PROPERTIES=/home/circleci/project/android/gradle.properties + +echo "sdk.dir=${ANDROID_HOME}" >> $GRADLE_LOCAL_PROPERTIES +echo "ndk.dir=${ANDROID_NDK_HOME}" >> $GRADLE_LOCAL_PROPERTIES + +echo "SONATYPE_NEXUS_USERNAME=${SONATYPE_NEXUS_USERNAME}" >> $GRADLE_PROPERTIES +echo "mavenCentralRepositoryUsername=${SONATYPE_NEXUS_USERNAME}" >> $GRADLE_PROPERTIES +echo "SONATYPE_NEXUS_PASSWORD=${SONATYPE_NEXUS_PASSWORD}" >> $GRADLE_PROPERTIES +echo "mavenCentralRepositoryPassword=${SONATYPE_NEXUS_PASSWORD}" >> $GRADLE_PROPERTIES + +echo "signing.keyId=${ANDROID_SIGN_KEY}" >> $GRADLE_PROPERTIES +echo "signing.password=${ANDROID_SIGN_PASS}" >> $GRADLE_PROPERTIES + +cat /home/circleci/project/android/gradle.properties | grep VERSION + +${GRADLE_PATH} --scan --stacktrace --debug --no-daemon -p ${VISION_ANDROID} ops:uploadArchives + +mkdir -p ~/workspace/artifacts +find . -type f -name *aar -print | xargs tar cfvz ~/workspace/artifacts/artifacts-aars.tgz diff --git a/.circleci/unittest/android/scripts/install_gradle.sh b/.circleci/unittest/android/scripts/install_gradle.sh new file mode 100755 index 00000000000..ff90c657e65 --- /dev/null +++ b/.circleci/unittest/android/scripts/install_gradle.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -ex + +_https_amazon_aws=https://downloads.gradle-dn.com/distributions +GRADLE_VERSION=6.8.3 + +_gradle_home=/opt/gradle +sudo rm -rf $gradle_home +sudo mkdir -p $_gradle_home + +curl --silent --output /tmp/gradle.zip --retry 3 $_https_amazon_aws/gradle-${GRADLE_VERSION}-bin.zip + +sudo unzip -q /tmp/gradle.zip -d $_gradle_home +rm /tmp/gradle.zip + +sudo chmod -R 777 $_gradle_home + +export GRADLE_HOME=$_gradle_home/gradle-$GRADLE_VERSION +export GRADLE_PATH=${GRADLE_HOME}/bin/gradle diff --git a/.circleci/unittest/ios/scripts/binary_ios_build.sh b/.circleci/unittest/ios/scripts/binary_ios_build.sh new file mode 100755 index 00000000000..1f117481f2d --- /dev/null +++ b/.circleci/unittest/ios/scripts/binary_ios_build.sh @@ -0,0 +1,47 @@ +#!/bin/bash +set -ex -o pipefail + +echo "" +echo "DIR: $(pwd)" +WORKSPACE=/Users/distiller/workspace +PROJ_ROOT_IOS=/Users/distiller/project/ios +PYTORCH_IOS_NIGHTLY_NAME=libtorch_ios_nightly_build.zip +export TCLLIBPATH="/usr/local/lib" + +# install conda +curl --retry 3 -o ~/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh +chmod +x ~/conda.sh +/bin/bash ~/conda.sh -b -p ~/anaconda +export PATH="~/anaconda/bin:${PATH}" +source ~/anaconda/bin/activate + +# install dependencies +conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi requests wget --yes +conda install -c conda-forge valgrind --yes +export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} + +# sync submodules +cd ${PROJ_ROOT_IOS} +git submodule sync +git submodule update --init --recursive + +# download pytorch-iOS nightly build and unzip it +mkdir -p ${PROJ_ROOT_IOS}/lib +mkdir -p ${PROJ_ROOT_IOS}/build +mkdir -p ${PROJ_ROOT_IOS}/pytorch +TORCH_ROOT="${PROJ_ROOT_IOS}/pytorch" + +cd ${TORCH_ROOT} +wget https://ossci-ios-build.s3.amazonaws.com/${PYTORCH_IOS_NIGHTLY_NAME} +mkdir -p ./build_ios +unzip -d ./build_ios ./${PYTORCH_IOS_NIGHTLY_NAME} + +LIBTORCH_HEADER_ROOT="${TORCH_ROOT}/build_ios/install/include" +cd ${PROJ_ROOT_IOS} +IOS_ARCH=${IOS_ARCH} LIBTORCH_HEADER_ROOT=${LIBTORCH_HEADER_ROOT} ./build_ios.sh +rm -rf ${TORCH_ROOT} + +# store the binary +DEST_DIR=${WORKSPACE}/ios/${IOS_ARCH} +mkdir -p ${DEST_DIR} +cp ${PROJ_ROOT_IOS}/lib/*.a ${DEST_DIR} diff --git a/.circleci/unittest/ios/scripts/binary_ios_upload.sh b/.circleci/unittest/ios/scripts/binary_ios_upload.sh new file mode 100644 index 00000000000..ce56388e5da --- /dev/null +++ b/.circleci/unittest/ios/scripts/binary_ios_upload.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -ex -o pipefail + +echo "" +echo "DIR: $(pwd)" + +WORKSPACE=/Users/distiller/workspace +PROJ_ROOT=/Users/distiller/project +ARTIFACTS_DIR=${WORKSPACE}/ios +ls ${ARTIFACTS_DIR} +ZIP_DIR=${WORKSPACE}/zip +mkdir -p ${ZIP_DIR}/install/lib + +# build a FAT bianry +cd ${ZIP_DIR}/install/lib +libs=("${ARTIFACTS_DIR}/x86_64/libtorchvision_ops.a" "${ARTIFACTS_DIR}/arm64/libtorchvision_ops.a") +lipo -create "${libs[@]}" -o ${ZIP_DIR}/install/lib/libtorchvision_ops.a +lipo -i ${ZIP_DIR}/install/lib/*.a + +# copy the license +cp ${PROJ_ROOT}/LICENSE ${ZIP_DIR}/ +# zip the library +ZIPFILE=libtorchvision_ops_ios_nightly_build.zip +cd ${ZIP_DIR} +#for testing +touch version.txt +echo $(date +%s) > version.txt +zip -r ${ZIPFILE} install version.txt LICENSE + +# upload to aws +# Install conda then 'conda install' awscli +curl --retry 3 -o ~/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh +chmod +x ~/conda.sh +/bin/bash ~/conda.sh -b -p ~/anaconda +export PATH="~/anaconda/bin:${PATH}" +source ~/anaconda/bin/activate +conda install -c conda-forge awscli --yes +set +x +export AWS_ACCESS_KEY_ID=${AWS_S3_ACCESS_KEY_FOR_PYTORCH_BINARY_UPLOAD} +export AWS_SECRET_ACCESS_KEY=${AWS_S3_ACCESS_SECRET_FOR_PYTORCH_BINARY_UPLOAD} +set -x +aws s3 cp ${ZIPFILE} s3://ossci-ios-build/ --acl public-read diff --git a/.circleci/unittest/linux/scripts/environment.yml b/.circleci/unittest/linux/scripts/environment.yml new file mode 100644 index 00000000000..fae96c5f93c --- /dev/null +++ b/.circleci/unittest/linux/scripts/environment.yml @@ -0,0 +1,16 @@ +channels: + - pytorch + - defaults +dependencies: + - pytest + - pytest-cov + - pytest-mock + - pip + - libpng + - jpeg + - ca-certificates + - h5py + - pip: + - future + - scipy + - av < 10 diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh new file mode 100755 index 00000000000..6375e191072 --- /dev/null +++ b/.circleci/unittest/linux/scripts/install.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +unset PYTORCH_VERSION +# For unittest, nightly PyTorch is used as the following section, +# so no need to set PYTORCH_VERSION. +# In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config. + +set -ex + +eval "$(./conda/bin/conda shell.bash hook)" +conda activate ./env + +if [ "${CU_VERSION:-}" == cpu ] ; then + cudatoolkit="cpuonly" + version="cpu" +else + if [[ ${#CU_VERSION} -eq 4 ]]; then + CUDA_VERSION="${CU_VERSION:2:1}.${CU_VERSION:3:1}" + elif [[ ${#CU_VERSION} -eq 5 ]]; then + CUDA_VERSION="${CU_VERSION:2:2}.${CU_VERSION:4:1}" + fi + echo "Using CUDA $CUDA_VERSION as determined by CU_VERSION: ${CU_VERSION} " + version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")" + cudatoolkit="pytorch-cuda=${version}" + + # make sure local cuda is set to required cuda version and not CUDA version by default + rm -f /usr/local/cuda + ln -s /usr/local/cuda-${version} /usr/local/cuda +fi + +case "$(uname -s)" in + Darwin*) os=MacOSX;; + *) os=Linux +esac + +printf "Installing PyTorch with %s\n" "${cudatoolkit}" +if [ "${os}" == "MacOSX" ]; then + conda install -y -c "pytorch-${UPLOAD_CHANNEL}" "pytorch-${UPLOAD_CHANNEL}"::pytorch "${cudatoolkit}" +else + conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c nvidia "pytorch-${UPLOAD_CHANNEL}"::pytorch[build="*${version}*"] "${cudatoolkit}" +fi + + +printf "* Installing torchvision\n" +python setup.py develop diff --git a/.circleci/unittest/linux/scripts/post_process.sh b/.circleci/unittest/linux/scripts/post_process.sh new file mode 100755 index 00000000000..e97bf2a7b1b --- /dev/null +++ b/.circleci/unittest/linux/scripts/post_process.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -e + +eval "$(./conda/bin/conda shell.bash hook)" +conda activate ./env diff --git a/.circleci/unittest/linux/scripts/run-clang-format.py b/.circleci/unittest/linux/scripts/run-clang-format.py new file mode 100755 index 00000000000..5c61b2519e0 --- /dev/null +++ b/.circleci/unittest/linux/scripts/run-clang-format.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python +""" +MIT License + +Copyright (c) 2017 Guillaume Papin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +A wrapper script around clang-format, suitable for linting multiple files +and to use for continuous integration. + +This is an alternative API for the clang-format command line. +It runs over multiple files and directories in parallel. +A diff output is produced and a sensible exit code is returned. + +""" + +import argparse +import difflib +import fnmatch +import multiprocessing +import os +import signal +import subprocess +import sys +import traceback +from functools import partial + +try: + from subprocess import DEVNULL # py3k +except ImportError: + DEVNULL = open(os.devnull, "wb") + + +DEFAULT_EXTENSIONS = "c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx,cu" + + +class ExitStatus: + SUCCESS = 0 + DIFF = 1 + TROUBLE = 2 + + +def list_files(files, recursive=False, extensions=None, exclude=None): + if extensions is None: + extensions = [] + if exclude is None: + exclude = [] + + out = [] + for file in files: + if recursive and os.path.isdir(file): + for dirpath, dnames, fnames in os.walk(file): + fpaths = [os.path.join(dirpath, fname) for fname in fnames] + for pattern in exclude: + # os.walk() supports trimming down the dnames list + # by modifying it in-place, + # to avoid unnecessary directory listings. + dnames[:] = [x for x in dnames if not fnmatch.fnmatch(os.path.join(dirpath, x), pattern)] + fpaths = [x for x in fpaths if not fnmatch.fnmatch(x, pattern)] + for f in fpaths: + ext = os.path.splitext(f)[1][1:] + if ext in extensions: + out.append(f) + else: + out.append(file) + return out + + +def make_diff(file, original, reformatted): + return list( + difflib.unified_diff( + original, reformatted, fromfile=f"{file}\t(original)", tofile=f"{file}\t(reformatted)", n=3 + ) + ) + + +class DiffError(Exception): + def __init__(self, message, errs=None): + super().__init__(message) + self.errs = errs or [] + + +class UnexpectedError(Exception): + def __init__(self, message, exc=None): + super().__init__(message) + self.formatted_traceback = traceback.format_exc() + self.exc = exc + + +def run_clang_format_diff_wrapper(args, file): + try: + ret = run_clang_format_diff(args, file) + return ret + except DiffError: + raise + except Exception as e: + raise UnexpectedError(f"{file}: {e.__class__.__name__}: {e}", e) + + +def run_clang_format_diff(args, file): + try: + with open(file, encoding="utf-8") as f: + original = f.readlines() + except OSError as exc: + raise DiffError(str(exc)) + invocation = [args.clang_format_executable, file] + + # Use of utf-8 to decode the process output. + # + # Hopefully, this is the correct thing to do. + # + # It's done due to the following assumptions (which may be incorrect): + # - clang-format will returns the bytes read from the files as-is, + # without conversion, and it is already assumed that the files use utf-8. + # - if the diagnostics were internationalized, they would use utf-8: + # > Adding Translations to Clang + # > + # > Not possible yet! + # > Diagnostic strings should be written in UTF-8, + # > the client can translate to the relevant code page if needed. + # > Each translation completely replaces the format string + # > for the diagnostic. + # > -- http://clang.llvm.org/docs/InternalsManual.html#internals-diag-translation + + try: + proc = subprocess.Popen( + invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, encoding="utf-8" + ) + except OSError as exc: + raise DiffError(f"Command '{subprocess.list2cmdline(invocation)}' failed to start: {exc}") + proc_stdout = proc.stdout + proc_stderr = proc.stderr + + # hopefully the stderr pipe won't get full and block the process + outs = list(proc_stdout.readlines()) + errs = list(proc_stderr.readlines()) + proc.wait() + if proc.returncode: + raise DiffError( + "Command '{}' returned non-zero exit status {}".format( + subprocess.list2cmdline(invocation), proc.returncode + ), + errs, + ) + return make_diff(file, original, outs), errs + + +def bold_red(s): + return "\x1b[1m\x1b[31m" + s + "\x1b[0m" + + +def colorize(diff_lines): + def bold(s): + return "\x1b[1m" + s + "\x1b[0m" + + def cyan(s): + return "\x1b[36m" + s + "\x1b[0m" + + def green(s): + return "\x1b[32m" + s + "\x1b[0m" + + def red(s): + return "\x1b[31m" + s + "\x1b[0m" + + for line in diff_lines: + if line[:4] in ["--- ", "+++ "]: + yield bold(line) + elif line.startswith("@@ "): + yield cyan(line) + elif line.startswith("+"): + yield green(line) + elif line.startswith("-"): + yield red(line) + else: + yield line + + +def print_diff(diff_lines, use_color): + if use_color: + diff_lines = colorize(diff_lines) + sys.stdout.writelines(diff_lines) + + +def print_trouble(prog, message, use_colors): + error_text = "error:" + if use_colors: + error_text = bold_red(error_text) + print(f"{prog}: {error_text} {message}", file=sys.stderr) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--clang-format-executable", + metavar="EXECUTABLE", + help="path to the clang-format executable", + default="clang-format", + ) + parser.add_argument( + "--extensions", + help=f"comma separated list of file extensions (default: {DEFAULT_EXTENSIONS})", + default=DEFAULT_EXTENSIONS, + ) + parser.add_argument("-r", "--recursive", action="store_true", help="run recursively over directories") + parser.add_argument("files", metavar="file", nargs="+") + parser.add_argument("-q", "--quiet", action="store_true") + parser.add_argument( + "-j", + metavar="N", + type=int, + default=0, + help="run N clang-format jobs in parallel (default number of cpus + 1)", + ) + parser.add_argument( + "--color", default="auto", choices=["auto", "always", "never"], help="show colored diff (default: auto)" + ) + parser.add_argument( + "-e", + "--exclude", + metavar="PATTERN", + action="append", + default=[], + help="exclude paths matching the given glob-like pattern(s) from recursive search", + ) + + args = parser.parse_args() + + # use default signal handling, like diff return SIGINT value on ^C + # https://bugs.python.org/issue14229#msg156446 + signal.signal(signal.SIGINT, signal.SIG_DFL) + try: + signal.SIGPIPE + except AttributeError: + # compatibility, SIGPIPE does not exist on Windows + pass + else: + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + colored_stdout = False + colored_stderr = False + if args.color == "always": + colored_stdout = True + colored_stderr = True + elif args.color == "auto": + colored_stdout = sys.stdout.isatty() + colored_stderr = sys.stderr.isatty() + + version_invocation = [args.clang_format_executable, "--version"] + try: + subprocess.check_call(version_invocation, stdout=DEVNULL) + except subprocess.CalledProcessError as e: + print_trouble(parser.prog, str(e), use_colors=colored_stderr) + return ExitStatus.TROUBLE + except OSError as e: + print_trouble( + parser.prog, + f"Command '{subprocess.list2cmdline(version_invocation)}' failed to start: {e}", + use_colors=colored_stderr, + ) + return ExitStatus.TROUBLE + + retcode = ExitStatus.SUCCESS + files = list_files( + args.files, recursive=args.recursive, exclude=args.exclude, extensions=args.extensions.split(",") + ) + + if not files: + return + + njobs = args.j + if njobs == 0: + njobs = multiprocessing.cpu_count() + 1 + njobs = min(len(files), njobs) + + if njobs == 1: + # execute directly instead of in a pool, + # less overhead, simpler stacktraces + it = (run_clang_format_diff_wrapper(args, file) for file in files) + pool = None + else: + pool = multiprocessing.Pool(njobs) + it = pool.imap_unordered(partial(run_clang_format_diff_wrapper, args), files) + while True: + try: + outs, errs = next(it) + except StopIteration: + break + except DiffError as e: + print_trouble(parser.prog, str(e), use_colors=colored_stderr) + retcode = ExitStatus.TROUBLE + sys.stderr.writelines(e.errs) + except UnexpectedError as e: + print_trouble(parser.prog, str(e), use_colors=colored_stderr) + sys.stderr.write(e.formatted_traceback) + retcode = ExitStatus.TROUBLE + # stop at the first unexpected error, + # something could be very wrong, + # don't process all files unnecessarily + if pool: + pool.terminate() + break + else: + sys.stderr.writelines(errs) + if outs == []: + continue + if not args.quiet: + print_diff(outs, use_color=colored_stdout) + if retcode == ExitStatus.SUCCESS: + retcode = ExitStatus.DIFF + return retcode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.circleci/unittest/linux/scripts/run_test.sh b/.circleci/unittest/linux/scripts/run_test.sh new file mode 100755 index 00000000000..5348baa71dd --- /dev/null +++ b/.circleci/unittest/linux/scripts/run_test.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -e + +eval "$(./conda/bin/conda shell.bash hook)" +conda activate ./env + +python -m torch.utils.collect_env + +case "$(uname -s)" in + Darwin*) + # The largest macOS runner is not able to handle the regular test suite plus the transforms v2 tests at the same + # time due to insufficient resources. Thus, we ignore the transforms v2 tests at first and run them in a separate + # step afterwards. + GLOB='test/test_transforms_v2*' + pytest --junitxml=test-results/junit.xml -v --durations 20 --ignore-glob="${GLOB}" + eval "pytest --junitxml=test-results/junit-transforms-v2.xml -v --durations 20 ${GLOB}" + ;; + *) + pytest --junitxml=test-results/junit.xml -v --durations 20 + ;; +esac diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh new file mode 100755 index 00000000000..8a8a78f1fb2 --- /dev/null +++ b/.circleci/unittest/linux/scripts/setup_env.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# This script is for setting up environment in which unit test is ran. +# To speed up the CI time, the resulting environment is cached. +# +# Do not install PyTorch and torchvision here, otherwise they also get cached. + +set -ex + +this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +# Avoid error: "fatal: unsafe repository" +git config --global --add safe.directory '*' +root_dir="$(git rev-parse --show-toplevel)" +conda_dir="${root_dir}/conda" +env_dir="${root_dir}/env" + +cd "${root_dir}" + +case "$(uname -s)" in + Darwin*) os=MacOSX;; + *) os=Linux +esac + +# 1. Install conda at ./conda +if [ ! -d "${conda_dir}" ]; then + printf "* Installing conda\n" + wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" + bash ./miniconda.sh -b -f -p "${conda_dir}" +fi +eval "$(${conda_dir}/bin/conda shell.bash hook)" + +# 2. Create test environment at ./env +if [ ! -d "${env_dir}" ]; then + printf "* Creating a test environment\n" + conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" +fi +conda activate "${env_dir}" + +# 3. Install Conda dependencies +printf "* Installing dependencies (except PyTorch)\n" +FFMPEG_PIN="=4.2" +if [[ "${PYTHON_VERSION}" == "3.9" ]]; then + FFMPEG_PIN=">=4.2" +fi + +conda install -y -c pytorch "ffmpeg${FFMPEG_PIN}" +conda env update --file "${this_dir}/environment.yml" --prune diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml new file mode 100644 index 00000000000..d229aafb41a --- /dev/null +++ b/.circleci/unittest/windows/scripts/environment.yml @@ -0,0 +1,19 @@ +channels: + - pytorch + - defaults +dependencies: + - pytest + - pytest-cov + - pytest-mock + - pip + - libpng + - jpeg + - ca-certificates + - hdf5 + - setuptools + - pip: + - future + - scipy + - av !=9.1.1, <10 + - dataclasses + - h5py diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh new file mode 100644 index 00000000000..7c55c8144d9 --- /dev/null +++ b/.circleci/unittest/windows/scripts/install.sh @@ -0,0 +1,53 @@ + +#!/usr/bin/env bash + +unset PYTORCH_VERSION +# For unittest, nightly PyTorch is used as the following section, +# so no need to set PYTORCH_VERSION. +# In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config. + +set -ex + +this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" +conda activate ./env + +# TODO, refactor the below logic to make it easy to understand how to get correct cuda_version. +if [ "${CU_VERSION:-}" == cpu ] ; then + cudatoolkit="cpuonly" + version="cpu" +else + if [[ ${#CU_VERSION} -eq 4 ]]; then + CUDA_VERSION="${CU_VERSION:2:1}.${CU_VERSION:3:1}" + elif [[ ${#CU_VERSION} -eq 5 ]]; then + CUDA_VERSION="${CU_VERSION:2:2}.${CU_VERSION:4:1}" + fi + + cuda_toolkit_pckg="cudatoolkit" + if [[ $CUDA_VERSION == 11.6 || $CUDA_VERSION == 11.7 || $CUDA_VERSION == 11.8 || $CUDA_VERSION == 12.1 ]]; then + cuda_toolkit_pckg="pytorch-cuda" + fi + + echo "Using CUDA $CUDA_VERSION as determined by CU_VERSION" + version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")" + cudatoolkit="${cuda_toolkit_pckg}=${version}" +fi + +printf "Installing PyTorch with %s\n" "${cudatoolkit}" +conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c nvidia "pytorch-${UPLOAD_CHANNEL}"::pytorch[build="*${version}*"] "${cudatoolkit}" + +torch_cuda=$(python -c "import torch; print(torch.cuda.is_available())") +echo torch.cuda.is_available is $torch_cuda + +if [ ! -z "${CUDA_VERSION:-}" ] ; then + if [ "$torch_cuda" == "False" ]; then + echo "torch with cuda installed but torch.cuda.is_available() is False" + exit 1 + fi +fi + +source "$this_dir/set_cuda_envs.sh" + +printf "* Installing torchvision\n" +"$this_dir/vc_env_helper.bat" python setup.py develop diff --git a/.circleci/unittest/windows/scripts/install_conda.bat b/.circleci/unittest/windows/scripts/install_conda.bat new file mode 100644 index 00000000000..6052ad08b10 --- /dev/null +++ b/.circleci/unittest/windows/scripts/install_conda.bat @@ -0,0 +1 @@ +start /wait "" "%miniconda_exe%" /S /InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /D=%tmp_conda% diff --git a/.circleci/unittest/windows/scripts/post_process.sh b/.circleci/unittest/windows/scripts/post_process.sh new file mode 100644 index 00000000000..5c5cbb758a9 --- /dev/null +++ b/.circleci/unittest/windows/scripts/post_process.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -e + +eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" +conda activate ./env diff --git a/.circleci/unittest/windows/scripts/run_test.sh b/.circleci/unittest/windows/scripts/run_test.sh new file mode 100644 index 00000000000..802ad37f511 --- /dev/null +++ b/.circleci/unittest/windows/scripts/run_test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e + +eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" +conda activate ./env + +this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source "$this_dir/set_cuda_envs.sh" + +python -m torch.utils.collect_env +pytest --junitxml=test-results/junit.xml -v --durations 20 diff --git a/.circleci/unittest/windows/scripts/set_cuda_envs.sh b/.circleci/unittest/windows/scripts/set_cuda_envs.sh new file mode 100644 index 00000000000..7db3137b594 --- /dev/null +++ b/.circleci/unittest/windows/scripts/set_cuda_envs.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -ex + +echo CU_VERSION is "${CU_VERSION}" +echo CUDA_VERSION is "${CUDA_VERSION}" + +# Currenly, CU_VERSION and CUDA_VERSION are not consistent. +# to understand this code, see https://github.com/pytorch/vision/issues/4443 +version="cpu" +if [[ ! -z "${CUDA_VERSION}" ]] ; then + version="$CUDA_VERSION" +else + if [[ ${#CU_VERSION} -eq 5 ]]; then + version="${CU_VERSION:2:2}.${CU_VERSION:4:1}" + fi +fi + +# Don't use if [[ "$version" == "cpu" ]]; then exit 0 fi. +# It would exit the shell. One result is cpu tests would not run if the shell exit. +# Unless there's an error, Don't exit. +if [[ "$version" != "cpu" ]]; then + # set cuda envs + export PATH="/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${version}/bin:/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${version}/libnvvp:$PATH" + export CUDA_PATH_V${version/./_}="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${version}" + export CUDA_PATH="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${version}" + + if [ ! -d "$CUDA_PATH" ]; then + echo "$CUDA_PATH" does not exist + exit 1 + fi + + if [ ! -f "${CUDA_PATH}\include\nvjpeg.h" ]; then + echo "nvjpeg does not exist" + exit 1 + fi + + # check cuda driver version + for path in '/c/Program Files/NVIDIA Corporation/NVSMI/nvidia-smi.exe' /c/Windows/System32/nvidia-smi.exe; do + if [[ -x "$path" ]]; then + "$path" || echo "true"; + break + fi + done + + which nvcc + nvcc --version + env | grep CUDA +fi diff --git a/.circleci/unittest/windows/scripts/setup_env.sh b/.circleci/unittest/windows/scripts/setup_env.sh new file mode 100644 index 00000000000..84697875979 --- /dev/null +++ b/.circleci/unittest/windows/scripts/setup_env.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# This script is for setting up environment in which unit test is ran. +# To speed up the CI time, the resulting environment is cached. +# +# Do not install PyTorch and torchvision here, otherwise they also get cached. + +set -ex + +this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +root_dir="$(git rev-parse --show-toplevel)" +conda_dir="${root_dir}/conda" +env_dir="${root_dir}/env" + +cd "${root_dir}" + +# 1. Install conda at ./conda +if [ ! -d "${conda_dir}" ]; then + printf "* Installing conda\n" + export tmp_conda="$(echo $conda_dir | tr '/' '\\')" + export miniconda_exe="$(echo $root_dir | tr '/' '\\')\\miniconda.exe" + curl --output miniconda.exe https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -O + "$this_dir/install_conda.bat" + unset tmp_conda + unset miniconda_exe +fi + +eval "$(${conda_dir}/Scripts/conda.exe 'shell.bash' 'hook')" + +# 2. Create test environment at ./env +if [ ! -d "${env_dir}" ]; then + printf "* Creating a test environment\n" + conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" +fi +conda activate "${env_dir}" + +# 3. Install Conda dependencies +printf "* Installing dependencies (except PyTorch)\n" +conda env update --file "${this_dir}/environment.yml" --prune + +# 4. Downgrade setuptools on Python 3.7. +# See https://github.com/pytorch/vision/pull/5868 +if [[ "${PYTHON_VERSION}" == '3.7' ]]; then + pip install --upgrade setuptools==58.0.4 +fi diff --git a/.circleci/unittest/windows/scripts/vc_env_helper.bat b/.circleci/unittest/windows/scripts/vc_env_helper.bat new file mode 100644 index 00000000000..9410135677a --- /dev/null +++ b/.circleci/unittest/windows/scripts/vc_env_helper.bat @@ -0,0 +1,39 @@ +@echo on + +set VC_VERSION_LOWER=16 +set VC_VERSION_UPPER=17 + +for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [%VC_VERSION_LOWER%^,%VC_VERSION_UPPER%^) -property installationPath`) do ( + if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( + set "VS15INSTALLDIR=%%i" + set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat" + goto vswhere + ) +) + +:vswhere +if "%VSDEVCMD_ARGS%" == "" ( + call "%VS15VCVARSALL%" x64 || exit /b 1 +) else ( + call "%VS15VCVARSALL%" x64 %VSDEVCMD_ARGS% || exit /b 1 +) + +@echo on + +set DISTUTILS_USE_SDK=1 + +set args=%1 +shift +:start +if [%1] == [] goto done +set args=%args% %1 +shift +goto start + +:done +if "%args%" == "" ( + echo Usage: vc_env_helper.bat [command] [args] + echo e.g. vc_env_helper.bat cl /c test.cpp +) + +%args% || exit /b 1 diff --git a/.github/scripts/unittest.sh b/.github/scripts/unittest.sh index 663181517fb..2a0b7154200 100755 --- a/.github/scripts/unittest.sh +++ b/.github/scripts/unittest.sh @@ -11,5 +11,4 @@ echo '::group::Install testing utilities' pip install --progress-bar=off pytest pytest-mock pytest-cov echo '::endgroup::' -#pytest --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" -v --durations=25 -python test/smoke_test.py +pytest --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" -v --durations=25 diff --git a/.github/workflows/build-cmake.yml b/.github/workflows/build-cmake.yml new file mode 100644 index 00000000000..8fdf99c84d1 --- /dev/null +++ b/.github/workflows/build-cmake.yml @@ -0,0 +1,92 @@ +name: CMake + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +jobs: + linux: + strategy: + matrix: + include: + - runner: linux.12xlarge + gpu-arch-type: cpu + - runner: linux.g5.4xlarge.nvidia.gpu + gpu-arch-type: cuda + gpu-arch-version: "11.8" + fail-fast: false + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + runner: ${{ matrix.runner }} + gpu-arch-type: ${{ matrix.gpu-arch-type }} + gpu-arch-version: ${{ matrix.gpu-arch-version }} + script: | + set -euo pipefail + + export PYTHON_VERSION=3.8 + export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} + export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} + + ./.github/scripts/cmake.sh + + macos: + strategy: + matrix: + include: + - runner: macos-12 + - runner: macos-m1-12 + fail-fast: false + uses: pytorch/test-infra/.github/workflows/macos_job.yml@main + with: + repository: pytorch/vision + runner: ${{ matrix.runner }} + script: | + set -euo pipefail + + export PYTHON_VERSION=3.8 + export GPU_ARCH_TYPE=cpu + export GPU_ARCH_VERSION='' + + ./.github/scripts/cmake.sh + + windows: + strategy: + matrix: + include: + - runner: windows.4xlarge + gpu-arch-type: cpu + - runner: windows.g5.4xlarge.nvidia.gpu + gpu-arch-type: cuda + gpu-arch-version: "11.8" + fail-fast: false + uses: pytorch/test-infra/.github/workflows/windows_job.yml@main + with: + repository: pytorch/vision + runner: ${{ matrix.runner }} + gpu-arch-type: ${{ matrix.gpu-arch-type }} + gpu-arch-version: ${{ matrix.gpu-arch-version }} + script: | + set -euo pipefail + + source packaging/windows/internal/vc_install_helper.sh + + # FIXME: Basically, we are reinstalling CUDA here. We only need this, because we need to copy some files that + # can be extracted from the CUDA installer, but are not available on our Windows AMI. + # See https://github.com/pytorch/test-infra/pull/4189 + if [[ ${{ matrix.gpu-arch-type }} == cuda ]]; then + export CU_VERSION=cu$(echo ${{ matrix.gpu-arch-version }} | sed 's/\.//') + echo CU_VERSION="${CU_VERSION}" + packaging/windows/internal/cuda_install.bat + fi + + export PYTHON_VERSION=3.8 + export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} + export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} + + ./.github/scripts/cmake.sh diff --git a/.github/workflows/build-conda-linux.yml b/.github/workflows/build-conda-linux.yml new file mode 100644 index 00000000000..e44bb877ed6 --- /dev/null +++ b/.github/workflows/build-conda-linux.yml @@ -0,0 +1,51 @@ +name: Build Linux Conda + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: conda + os: linux + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: "" + post-script: "" + conda-package-directory: packaging/torchvision + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_conda_linux.yml@main + with: + conda-package-directory: ${{ matrix.conda-package-directory }} + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + trigger-event: ${{ github.event_name }} + secrets: + CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-m1.yml b/.github/workflows/build-conda-m1.yml new file mode 100644 index 00000000000..7006b6714aa --- /dev/null +++ b/.github/workflows/build-conda-m1.yml @@ -0,0 +1,52 @@ +name: Build M1 Conda + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: conda + os: macos-arm64 + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: "" + post-script: "" + conda-package-directory: packaging/torchvision + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_conda_macos.yml@main + with: + conda-package-directory: ${{ matrix.conda-package-directory }} + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + runner-type: macos-m1-12 + trigger-event: ${{ github.event_name }} + secrets: + CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-macos.yml b/.github/workflows/build-conda-macos.yml new file mode 100644 index 00000000000..39d00534b3a --- /dev/null +++ b/.github/workflows/build-conda-macos.yml @@ -0,0 +1,52 @@ +name: Build Macos Conda + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: conda + os: macos + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: "" + post-script: "" + conda-package-directory: packaging/torchvision + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_conda_macos.yml@main + with: + conda-package-directory: ${{ matrix.conda-package-directory }} + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + runner-type: macos-12 + trigger-event: ${{ github.event_name }} + secrets: + CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-windows.yml b/.github/workflows/build-conda-windows.yml new file mode 100644 index 00000000000..954d09d39ec --- /dev/null +++ b/.github/workflows/build-conda-windows.yml @@ -0,0 +1,51 @@ +name: Build Windows Conda + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: conda + os: windows + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: packaging/pre_build_script.sh + env-script: packaging/windows/internal/vc_env_helper.bat + post-script: "" + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_conda_windows.yml@main + with: + conda-package-directory: ${{ matrix.conda-package-directory }} + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + trigger-event: ${{ github.event_name }} + secrets: + CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-wheels-linux.yml b/.github/workflows/build-wheels-linux.yml new file mode 100644 index 00000000000..e997d648ec2 --- /dev/null +++ b/.github/workflows/build-wheels-linux.yml @@ -0,0 +1,50 @@ +name: Build Linux Wheels + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: linux + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: packaging/pre_build_script.sh + post-script: packaging/post_build_script.sh + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + trigger-event: ${{ github.event_name }} + secrets: + AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-m1.yml b/.github/workflows/build-wheels-m1.yml new file mode 100644 index 00000000000..66c7687acc9 --- /dev/null +++ b/.github/workflows/build-wheels-m1.yml @@ -0,0 +1,51 @@ +name: Build M1 Wheels + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: macos-arm64 + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: packaging/pre_build_script.sh + post-script: packaging/post_build_script.sh + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + runner-type: macos-m1-12 + smoke-test-script: ${{ matrix.smoke-test-script }} + trigger-event: ${{ github.event_name }} + secrets: + AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-macos.yml b/.github/workflows/build-wheels-macos.yml new file mode 100644 index 00000000000..6c5ebc0fc37 --- /dev/null +++ b/.github/workflows/build-wheels-macos.yml @@ -0,0 +1,51 @@ +name: Build Macos Wheels + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: macos + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: packaging/pre_build_script.sh + post-script: packaging/post_build_script.sh + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + runner-type: macos-12 + smoke-test-script: ${{ matrix.smoke-test-script }} + trigger-event: ${{ github.event_name }} + secrets: + AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-windows.yml b/.github/workflows/build-wheels-windows.yml new file mode 100644 index 00000000000..3d818ece538 --- /dev/null +++ b/.github/workflows/build-wheels-windows.yml @@ -0,0 +1,52 @@ +name: Build Windows Wheels + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: windows + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: packaging/pre_build_script.sh + env-script: packaging/windows/internal/vc_env_helper.bat + post-script: "python packaging/wheel/relocate.py" + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_wheels_windows.yml@main + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + env-script: ${{ matrix.env-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + trigger-event: ${{ github.event_name }} + secrets: + AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000000..f4cc76db05e --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,115 @@ +name: Docs + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + - v[0-9]+.[0-9]+.[0-9] + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + build: + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + upload-artifact: docs + script: | + set -euo pipefail + + export PYTHON_VERSION=3.8 + export GPU_ARCH_TYPE=cpu + export GPU_ARCH_VERSION='' + ./.github/scripts/setup-env.sh + + # Prepare conda + CONDA_PATH=$(which conda) + eval "$(${CONDA_PATH} shell.bash hook)" + conda activate ci + # FIXME: not sure why we need this. `ldd torchvision/video_reader.so` shows that it + # already links against the one pulled from conda. However, at runtime it pulls from + # /lib64 + # Should we maybe always do this in `./.github/scripts/setup-env.sh` so that we don't + # have to pay attention in all other workflows? + export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH}" + + cd docs + + echo '::group::Install doc requirements' + pip install --progress-bar=off -r requirements.txt + echo '::endgroup::' + + if [[ ${{ github.event_name }} == push && (${{ github.ref_type }} == tag || (${{ github.ref_type }} == branch && ${{ github.ref_name }} == release/*)) ]]; then + echo '::group::Enable version string sanitization' + # This environment variable just has to exist and must not be empty. The actual value is arbitrary. + # See docs/source/conf.py for details + export TORCHVISION_SANITIZE_VERSION_STR_IN_DOCS=1 + echo '::endgroup::' + fi + + # The runner does not have sufficient memory to run with as many processes as there are + # cores (`-j auto`). Thus, we limit to a single process (`-j 1`) here. + sed -i -e 's/-j auto/-j 1/' Makefile + make html + + cp -r build/html "${RUNNER_ARTIFACT_DIR}" + + # On PRs we also want to upload the docs into our S3 bucket for preview. + if [[ ${{ github.event_name == 'pull_request' }} ]]; then + cp -r build/html/* "${RUNNER_DOCS_DIR}" + fi + + upload: + needs: build + if: github.repository == 'pytorch/vision' && github.event_name == 'push' && + ((github.ref_type == 'branch' && github.ref_name == 'main') || github.ref_type == 'tag') + permissions: + contents: write + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + download-artifact: docs + ref: gh-pages + script: | + set -euo pipefail + + REF_TYPE=${{ github.ref_type }} + REF_NAME=${{ github.ref_name }} + + if [[ "${REF_TYPE}" == branch ]]; then + TARGET_FOLDER="${REF_NAME}" + elif [[ "${REF_TYPE}" == tag ]]; then + case "${REF_NAME}" in + *-rc*) + echo "Aborting upload since this is an RC tag: ${REF_NAME}" + exit 0 + ;; + *) + # Strip the leading "v" as well as the trailing patch version. For example: + # 'v0.15.2' -> '0.15' + TARGET_FOLDER=$(echo "${REF_NAME}" | sed 's/v\([0-9]\+\)\.\([0-9]\+\)\.[0-9]\+/\1.\2/') + ;; + esac + fi + echo "Target Folder: ${TARGET_FOLDER}" + + mkdir -p "${TARGET_FOLDER}" + rm -rf "${TARGET_FOLDER}"/* + mv "${RUNNER_ARTIFACT_DIR}"/html/* "${TARGET_FOLDER}" + git add "${TARGET_FOLDER}" || true + + if [[ "${TARGET_FOLDER}" == main ]]; then + mkdir -p _static + rm -rf _static/* + cp -r "${TARGET_FOLDER}"/_static/* _static + git add _static || true + fi + + git config user.name 'pytorchbot' + git config user.email 'soumith+bot@pytorch.org' + git commit -m "auto-generating sphinx docs" || true + git push diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000000..ec8d285c9f3 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,107 @@ +name: Lint + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +jobs: + python-source-and-configs: + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + script: | + set -euo pipefail + + echo '::group::Setup environment' + CONDA_PATH=$(which conda) + eval "$(${CONDA_PATH} shell.bash hook)" + conda create --name ci --quiet --yes python=3.8 pip + conda activate ci + echo '::endgroup::' + + echo '::group::Install lint tools' + pip install --progress-bar=off pre-commit + echo '::endgroup::' + + set +e + pre-commit run --all-files + + if [ $? -ne 0 ]; then + git --no-pager diff + exit 1 + fi + + c-source: + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + script: | + set -euo pipefail + + echo '::group::Setup environment' + CONDA_PATH=$(which conda) + eval "$(${CONDA_PATH} shell.bash hook)" + # clang-format needs some shared libraries that conflict with the system ones. Thus, we install them from conda + # and prepend the libraries to linker path to prioritize them. `ncurses=5` is only available on the conda-forge + # channel. Since we are not building or testing here, this is fine. + conda create --name ci --quiet --yes -c conda-forge python=3.8 ncurses=5 libgcc + conda activate ci + export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH}" + echo '::endgroup::' + + echo '::group::Install lint tools' + curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o ./clang-format + chmod +x ./clang-format + echo '::endgroup::' + + echo '::group::Lint C source' + set +e + ./.circleci/unittest/linux/scripts/run-clang-format.py -r torchvision/csrc --clang-format-executable ./clang-format + + if [ $? -ne 0 ]; then + git --no-pager diff + exit 1 + fi + echo '::endgroup::' + + + python-types: + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + script: | + set -euo pipefail + + export PYTHON_VERSION=3.8 + export GPU_ARCH_TYPE=cpu + export GPU_ARCH_VERSION='' + + ./.github/scripts/setup-env.sh + + CONDA_PATH=$(which conda) + eval "$(${CONDA_PATH} shell.bash hook)" + conda activate ci + + echo '::group::Install lint tools' + pip install --progress-bar=off mypy + echo '::endgroup::' + + echo '::group::Lint Python types' + mypy --install-types --non-interactive --config-file mypy.ini + echo '::endgroup::' + + bc: + if: github.event.pull_request + runs-on: ubuntu-latest + steps: + - name: Run BC Lint Action + uses: pytorch/test-infra/.github/actions/bc-lint@main + with: + repo: ${{ github.event.pull_request.head.repo.full_name }} + base_sha: ${{ github.event.pull_request.base.sha }} + head_sha: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/pr-labels.yml b/.github/workflows/pr-labels.yml new file mode 100644 index 00000000000..20c37e4fd88 --- /dev/null +++ b/.github/workflows/pr-labels.yml @@ -0,0 +1,35 @@ +name: pr-labels + +on: + push: + branches: + - main + +jobs: + is-properly-labeled: + runs-on: ubuntu-latest + + steps: + - name: Set up python + uses: actions/setup-python@v2 + + - name: Install requests + run: pip install requests + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Process commit and find merger responsible for labeling + id: commit + run: echo "::set-output name=merger::$(python .github/process_commit.py ${{ github.sha }})" + + - name: Ping merger responsible for labeling if necessary + if: ${{ steps.commit.outputs.merger != '' }} + uses: mshick/add-pr-comment@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + message: | + Hey ${{ steps.commit.outputs.merger }}! + + You merged this PR, but no labels were added. The list of valid labels is available at https://github.com/pytorch/vision/blob/main/.github/process_commit.py diff --git a/.github/workflows/prototype-tests-linux-gpu.yml b/.github/workflows/prototype-tests-linux-gpu.yml new file mode 100644 index 00000000000..dee425054d5 --- /dev/null +++ b/.github/workflows/prototype-tests-linux-gpu.yml @@ -0,0 +1,54 @@ +name: Prototype tests on Linux + +on: + pull_request: + +jobs: + unittests-prototype: + strategy: + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + runner: ["linux.12xlarge"] + gpu-arch-type: ["cpu"] + include: + - python-version: "3.8" + runner: linux.g5.4xlarge.nvidia.gpu + gpu-arch-type: cuda + gpu-arch-version: "11.7" + fail-fast: false + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + runner: ${{ matrix.runner }} + gpu-arch-type: ${{ matrix.gpu-arch-type }} + gpu-arch-version: ${{ matrix.gpu-arch-version }} + timeout: 120 + script: | + set -euo pipefail + + export PYTHON_VERSION=${{ matrix.python-version }} + export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} + export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} + ./.github/scripts/setup-env.sh + + # Prepare conda + CONDA_PATH=$(which conda) + eval "$(${CONDA_PATH} shell.bash hook)" + conda activate ci + + echo '::group::Install testing utilities' + pip install --progress-bar=off pytest pytest-mock pytest-cov + echo '::endgroup::' + + # We don't want to run the prototype datasets tests. Since the positional glob into `pytest`, i.e. + # `test/test_prototype*.py` takes the highest priority, neither `--ignore` nor `--ignore-glob` can help us here. + rm test/test_prototype_datasets*.py + pytest \ + -v --durations=25 \ + --cov=torchvision/prototype --cov-report=term-missing \ + --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" \ + test/test_prototype_*.py diff --git a/.github/workflows/tests-schedule.yml b/.github/workflows/tests-schedule.yml new file mode 100644 index 00000000000..5426fdc997a --- /dev/null +++ b/.github/workflows/tests-schedule.yml @@ -0,0 +1,57 @@ +name: tests + +on: + pull_request: + paths: + - "test/test_datasets_download.py" + - ".github/failed_schedule_issue_template.md" + - ".github/workflows/tests-schedule.yml" + + schedule: + - cron: "0 9 * * *" + +jobs: + download: + runs-on: ubuntu-latest + + steps: + - name: Set up python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Upgrade system packages + run: python -m pip install --upgrade pip setuptools wheel + + - name: SSL + run: python -c 'import ssl; print(ssl.OPENSSL_VERSION)' + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install torch nightly build + run: pip install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + + - name: Install torchvision + run: pip install --no-build-isolation --editable . + + - name: Install all optional dataset requirements + run: pip install scipy pycocotools lmdb requests + + - name: Install tests requirements + run: pip install pytest + + - name: Run tests + run: pytest -ra -v test/test_datasets_download.py + + - uses: JasonEtco/create-an-issue@v2.4.0 + name: Create issue if download tests failed + if: failure() && github.event_name == 'schedule' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + WORKFLOW: ${{ github.workflow }} + JOB: ${{ github.job }} + ID: ${{ github.run_id }} + with: + filename: .github/failed_schedule_issue_template.md diff --git a/.github/workflows/update-viablestrict.yml b/.github/workflows/update-viablestrict.yml new file mode 100644 index 00000000000..2d9c2265676 --- /dev/null +++ b/.github/workflows/update-viablestrict.yml @@ -0,0 +1,23 @@ +name: Update viable/strict + +on: + pull_request: + paths: + - .github/workflows/update-viablestrict.yml + schedule: + - cron: 10,40 * * * * + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + do_update_viablestrict: + uses: pytorch/test-infra/.github/workflows/update-viablestrict.yml@main + with: + repository: pytorch/vision + required_checks: "Build Linux,Build M1,Build Macos,Tests on Linux,Tests on macOS,Docs,Lint" + secrets: + ROCKSET_API_KEY: ${{ secrets.ROCKSET_API_KEY }} + GITHUB_DEPLOY_KEY : ${{ secrets.VISION_GITHUB_DEPLOY_KEY }}