Merge branch 'master' into updt_utls

Author: fmassa

.circleci/config.yml

@@ -45,6 +45,67 @@ commands:
               our_upload_channel=test
             fi
             echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
+  install_cuda_compatible_cmath:
+    description: "Install CUDA compatible cmath"
+    steps:
+      - run:
+          name: _HACK_ Install CUDA compatible cmath
+          no_output_timeout: 1m
+          command: |
+              powershell .circleci/scripts/vs_install_cmath.ps1
+
+  brew_update:
+    description: "Update Homebrew and install base formulae"
+    steps:
+      - run:
+          name: Update Homebrew
+          no_output_timeout: "10m"
+          command: |
+            set -ex
+
+            # Update repositories manually.
+            # Running `brew update` produces a comparison between the
+            # current checkout and the updated checkout, which takes a
+            # very long time because the existing checkout is 2y old.
+            for path in $(find /usr/local/Homebrew -type d -name .git)
+            do
+            cd $path/..
+            git fetch --depth=1 origin
+            git reset --hard origin/master
+            done
+
+            export HOMEBREW_NO_AUTO_UPDATE=1
+
+            # Install expect and moreutils so that we can call `unbuffer` and `ts`.
+            # moreutils installs a `parallel` executable by default, which conflicts
+            # with the executable from the GNU `parallel`, so we must unlink GNU
+            # `parallel` first, and relink it afterwards.
+            brew install coreutils
+            brew unlink parallel
+            brew install moreutils
+            brew link parallel --overwrite
+            brew install expect
+
+  brew_install:
+    description: "Install Homebrew formulae"
+    parameters:
+      formulae:
+        type: string
+        default: ""
+    steps:
+      - run:
+          name: Install << parameters.formulae >>
+          no_output_timeout: "10m"
+          command: |
+            set -ex
+            export HOMEBREW_NO_AUTO_UPDATE=1
+            brew install << parameters.formulae >>
+
+  run_brew_for_ios_build:
+    steps:
+      - brew_update
+      - brew_install:
+          formulae: libtool
 
 binary_common: &binary_common
   parameters:
@@ -83,6 +144,22 @@ binary_common: &binary_common
     UNICODE_ABI: << parameters.unicode_abi >>
     CU_VERSION: << parameters.cu_version >>
 
+torchvision_ios_params: &torchvision_ios_params
+  parameters:
+    build_environment:
+      type: string
+      default: ""
+    ios_arch:
+      type: string
+      default: ""
+    ios_platform:
+      type: string
+      default: ""
+  environment:
+    BUILD_ENVIRONMENT: << parameters.build_environment >>
+    IOS_ARCH: << parameters.ios_arch >>
+    IOS_PLATFORM: << parameters.ios_platform >>
+
 smoke_test_common: &smoke_test_common
   <<: *binary_common
   docker:
@@ -212,6 +289,7 @@ jobs:
     steps:
       - checkout_merge
       - designate_upload_channel
+      - install_cuda_compatible_cmath
       - run:
           name: Build conda packages
           no_output_timeout: 20m
@@ -239,6 +317,7 @@ jobs:
     steps:
       - checkout_merge
       - designate_upload_channel
+      - install_cuda_compatible_cmath
       - run:
           name: Build wheel packages
           command: |
@@ -278,6 +357,43 @@ jobs:
           paths:
             - "*"
 
+  binary_ios_build:
+    <<: *torchvision_ios_params
+    macos:
+      xcode: "12.0"
+    steps:
+    - attach_workspace:
+        at: ~/workspace
+    - checkout
+    - run_brew_for_ios_build
+    - run:
+        name: Build
+        no_output_timeout: "1h"
+        command: |
+          script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_build.sh"
+          cat "$script"
+          source "$script"
+    - persist_to_workspace:
+        root: /Users/distiller/workspace/
+        paths: ios
+
+  binary_ios_upload:
+    <<: *torchvision_ios_params
+    macos:
+      xcode: "12.0"
+    steps:
+    - attach_workspace:
+        at: ~/workspace
+    - checkout
+    - run_brew_for_ios_build
+    - run:
+        name: Upload
+        no_output_timeout: "1h"
+        command: |
+          script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_upload.sh"
+          cat "$script"
+          source "$script"
+
   binary_macos_conda:
     <<: *binary_common
     macos:
@@ -546,6 +662,7 @@ jobs:
     steps:
       - checkout
       - designate_upload_channel
+      - install_cuda_compatible_cmath
       - run:
           name: Generate cache key
           # This will refresh cache on Sundays, nightly build should generate new cache.
@@ -587,6 +704,7 @@ jobs:
     steps:
       - checkout
       - designate_upload_channel
+      - install_cuda_compatible_cmath
       - run:
           name: Generate cache key
           # This will refresh cache on Sundays, nightly build should generate new cache.
@@ -595,7 +713,7 @@ jobs:
           {% raw %}
           keys:
             - env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
-          {% endraw %} 
+          {% endraw %}
       - run:
           name: Setup
           command: .circleci/unittest/windows/scripts/setup_env.sh
@@ -716,6 +834,7 @@ jobs:
     steps:
       - checkout_merge
       - designate_upload_channel
+      - install_cuda_compatible_cmath
       - run:
           command: |
             set -ex
@@ -729,6 +848,7 @@ jobs:
     steps:
       - checkout_merge
       - designate_upload_channel
+      - install_cuda_compatible_cmath
       - run:
           command: |
             set -ex
@@ -813,6 +933,7 @@ workflows:
       - clang_format
       - torchhub_test
       - torch_onnx_test
+      {{ ios_workflows() }}
 
   unittest:
     jobs:
@@ -832,6 +953,7 @@ workflows:
       - clang_format
       - torchhub_test
       - torch_onnx_test
+      {{ ios_workflows(nightly=True) }}
       {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
   docker_build:
     triggers:

.circleci/config.yml.in

@@ -29,7 +29,7 @@ def build_workflows(prefix='', filter_branch=None, upload=False, indentation=6,
     for btype in ["wheel", "conda"]:
         for os_type in ["linux", "macos", "win"]:
             python_versions = PYTHON_VERSIONS
-            cu_versions_dict = {"linux": ["cpu", "cu101", "cu102", "cu111", "rocm4.0.1"],
+            cu_versions_dict = {"linux": ["cpu", "cu101", "cu102", "cu111", "rocm4.0.1", "rocm4.1"],
                                 "win": ["cpu", "cu101", "cu102", "cu111"],
                                 "macos": ["cpu"]}
             cu_versions = cu_versions_dict[os_type]
@@ -261,6 +261,36 @@ def cmake_workflows(indentation=6):
     return indent(indentation, jobs)
 
 
+def ios_workflows(indentation=6, nightly=False):
+    jobs = []
+    build_job_names = []
+    name_prefix = "nightly_" if nightly else ""
+    env_prefix = "nightly-" if nightly else ""
+    for arch, platform in [('x86_64', 'SIMULATOR'), ('arm64', 'OS')]:
+        name = f'{name_prefix}binary_libtorchvision_ops_ios_12.0.0_{arch}'
+        build_job_names.append(name)
+        build_job = {
+            'build_environment': f'{env_prefix}binary-libtorchvision_ops-ios-12.0.0-{arch}',
+            'context': 'org-member',
+            'ios_arch': arch,
+            'ios_platform': platform,
+            'name': name,
+        }
+        if nightly:
+            build_job['filters'] = gen_filter_branch_tree('nightly')
+        jobs.append({'binary_ios_build': build_job})
+
+    if nightly:
+        upload_job = {
+            'build_environment': f'{env_prefix}binary-libtorchvision_ops-ios-12.0.0-upload',
+            'context': 'org-member',
+            'filters': gen_filter_branch_tree('nightly'),
+            'requires': build_job_names,
+        }
+        jobs.append({'binary_ios_upload': upload_job})
+    return indent(indentation, jobs)
+
+
 if __name__ == "__main__":
     d = os.path.dirname(__file__)
     env = jinja2.Environment(
@@ -275,4 +305,5 @@ def cmake_workflows(indentation=6):
             build_workflows=build_workflows,
             unittest_workflows=unittest_workflows,
             cmake_workflows=cmake_workflows,
+            ios_workflows=ios_workflows,
         ))

.circleci/regenerate.py

@@ -0,0 +1,5 @@
+$CMATH_DOWNLOAD_LINK = "https://raw.githubusercontent.com/microsoft/STL/12c684bba78f9b032050526abdebf14f58ca26a3/stl/inc/cmath"
+$VC14_28_INSTALL_PATH="C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.28.29910\include"
+
+curl.exe --retry 3 -kL $CMATH_DOWNLOAD_LINK --output "$home\cmath"
+Move-Item -Path "$home\cmath" -Destination "$VC14_28_INSTALL_PATH" -Force

.circleci/scripts/vs_install_cmath.ps1

@@ -0,0 +1,47 @@
+#!/bin/bash
+set -ex -o pipefail
+
+echo ""
+echo "DIR: $(pwd)"
+WORKSPACE=/Users/distiller/workspace
+PROJ_ROOT_IOS=/Users/distiller/project/ios
+PYTORCH_IOS_NIGHTLY_NAME=libtorch_ios_nightly_build.zip
+export TCLLIBPATH="/usr/local/lib"
+
+# install conda
+curl --retry 3 -o ~/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+chmod +x ~/conda.sh
+/bin/bash ~/conda.sh -b -p ~/anaconda
+export PATH="~/anaconda/bin:${PATH}"
+source ~/anaconda/bin/activate
+
+# install dependencies
+conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi requests typing_extensions wget --yes
+conda install -c conda-forge valgrind --yes
+export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
+
+# sync submodules
+cd ${PROJ_ROOT_IOS}
+git submodule sync
+git submodule update --init --recursive
+
+# download pytorch-iOS nightly build and unzip it
+mkdir -p ${PROJ_ROOT_IOS}/lib
+mkdir -p ${PROJ_ROOT_IOS}/build
+mkdir -p ${PROJ_ROOT_IOS}/pytorch
+TORCH_ROOT="${PROJ_ROOT_IOS}/pytorch"
+
+cd ${TORCH_ROOT}
+wget https://ossci-ios-build.s3.amazonaws.com/${PYTORCH_IOS_NIGHTLY_NAME}
+mkdir -p ./build_ios
+unzip -d ./build_ios ./${PYTORCH_IOS_NIGHTLY_NAME}
+
+LIBTORCH_HEADER_ROOT="${TORCH_ROOT}/build_ios/install/include"
+cd ${PROJ_ROOT_IOS}
+IOS_ARCH=${IOS_ARCH} LIBTORCH_HEADER_ROOT=${LIBTORCH_HEADER_ROOT} ./build_ios.sh
+rm -rf ${TORCH_ROOT}
+
+# store the binary
+DEST_DIR=${WORKSPACE}/ios/${IOS_ARCH}
+mkdir -p ${DEST_DIR}
+cp ${PROJ_ROOT_IOS}/lib/*.a ${DEST_DIR}

.circleci/unittest/ios/scripts/binary_ios_build.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+set -ex -o pipefail
+
+echo ""
+echo "DIR: $(pwd)"
+
+WORKSPACE=/Users/distiller/workspace
+PROJ_ROOT=/Users/distiller/project
+ARTIFACTS_DIR=${WORKSPACE}/ios
+ls ${ARTIFACTS_DIR}
+ZIP_DIR=${WORKSPACE}/zip
+mkdir -p ${ZIP_DIR}/install/lib
+
+# build a FAT bianry
+cd ${ZIP_DIR}/install/lib
+libs=("${ARTIFACTS_DIR}/x86_64/libtorchvision_ops.a" "${ARTIFACTS_DIR}/arm64/libtorchvision_ops.a")
+lipo -create "${libs[@]}" -o ${ZIP_DIR}/install/lib/libtorchvision_ops.a
+lipo -i ${ZIP_DIR}/install/lib/*.a
+
+# copy the license
+cp ${PROJ_ROOT}/LICENSE ${ZIP_DIR}/
+# zip the library
+ZIPFILE=libtorchvision_ops_ios_nightly_build.zip
+cd ${ZIP_DIR}
+#for testing
+touch version.txt
+echo $(date +%s) > version.txt
+zip -r ${ZIPFILE} install version.txt LICENSE
+
+# upload to aws
+# Install conda then 'conda install' awscli
+curl --retry 3 -o ~/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+chmod +x ~/conda.sh
+/bin/bash ~/conda.sh -b -p ~/anaconda
+export PATH="~/anaconda/bin:${PATH}"
+source ~/anaconda/bin/activate
+conda install -c conda-forge awscli --yes
+set +x
+export AWS_ACCESS_KEY_ID=${AWS_S3_ACCESS_KEY_FOR_PYTORCH_BINARY_UPLOAD}
+export AWS_SECRET_ACCESS_KEY=${AWS_S3_ACCESS_SECRET_FOR_PYTORCH_BINARY_UPLOAD}
+set -x
+aws s3 cp ${ZIPFILE} s3://ossci-ios-build/ --acl public-read