pytorch · mthrok · Aug 28, 2020 · Nov 30, 2020 · Nov 30, 2020 · Nov 30, 2020
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -42,8 +42,8 @@ commands:
     description: "installs tools required to build torchaudio"
     steps:
       - run:
-          name: Install cmake and pkg-config
-          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake pkg-config wget
+          name: Install pkg-config
+          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config wget
           # Disable brew auto update which is very slow
 
 binary_common: &binary_common
@@ -399,14 +399,14 @@ jobs:
       - restore_cache:
 
           keys:
-            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-cpu-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
 
       - run:
           name: Setup
           command: .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
 
-          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-cpu-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
 
           paths:
             - conda
@@ -433,7 +433,7 @@ jobs:
     resource_class: gpu.small
     environment:
       <<: *environment
-      image_name: "pytorch/torchaudio_unittest_base:manylinux-cuda10.1"
+      image_name: pytorch/torchaudio_unittest_base:manylinux-cuda10.1-cudnn7-20201203
     steps:
       - checkout
       - attach_workspace:
@@ -443,7 +443,7 @@ jobs:
       - restore_cache:
 
           keys:
-            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-gpu-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
 
       - run:
           name: Pull Docker image
@@ -453,7 +453,7 @@ jobs:
           command: docker run -t --gpus all -e PYTHON_VERSION -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
 
-          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-gpu-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
 
           paths:
             - conda
@@ -463,7 +463,7 @@ jobs:
             - third_party/src
       - run:
           name: Install torchaudio
-          command: docker run -t --gpus all -e UPLOAD_CHANNEL -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
+          command: docker run -t --gpus all -e UPLOAD_CHANNEL -e CUDA_HOME -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
       - run:
           name: Run tests
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh

diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in
@@ -42,8 +42,8 @@ commands:
     description: "installs tools required to build torchaudio"
     steps:
       - run:
-          name: Install cmake and pkg-config
-          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake pkg-config wget
+          name: Install pkg-config
+          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config wget
           # Disable brew auto update which is very slow
 
 binary_common: &binary_common
@@ -399,14 +399,14 @@ jobs:
       - restore_cache:
           {% raw %}
           keys:
-            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-cpu-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
           {% endraw %}
       - run:
           name: Setup
           command: .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
           {% raw %}
-          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-cpu-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
           {% endraw %}
           paths:
             - conda
@@ -433,7 +433,7 @@ jobs:
     resource_class: gpu.small
     environment:
       <<: *environment
-      image_name: "pytorch/torchaudio_unittest_base:manylinux-cuda10.1"
+      image_name: pytorch/torchaudio_unittest_base:manylinux-cuda10.1-cudnn7-20201203
     steps:
       - checkout
       - attach_workspace:
@@ -443,7 +443,7 @@ jobs:
       - restore_cache:
           {% raw %}
           keys:
-            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-gpu-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
           {% endraw %}
       - run:
           name: Pull Docker image
@@ -453,7 +453,7 @@ jobs:
           command: docker run -t --gpus all -e PYTHON_VERSION -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
           {% raw %}
-          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-gpu-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
           {% endraw %}
           paths:
             - conda
@@ -463,7 +463,7 @@ jobs:
             - third_party/src
       - run:
           name: Install torchaudio
-          command: docker run -t --gpus all -e UPLOAD_CHANNEL -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
+          command: docker run -t --gpus all -e UPLOAD_CHANNEL -e CUDA_HOME -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
       - run:
           name: Run tests
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh

diff --git a/.circleci/torchscript_bc_test/environment.yml b/.circleci/torchscript_bc_test/environment.yml
@@ -7,7 +7,8 @@ dependencies:
   - pytest
   - pytest-cov
   - codecov
-  - librosa
+  - cmake
+  - librosa>=0.8.0
   - llvmlite==0.31  # See https://github.com/pytorch/audio/pull/766
   - pip
   - pip:

diff --git a/.circleci/unittest/linux/docker/build_and_push.sh b/.circleci/unittest/linux/docker/build_and_push.sh
@@ -7,15 +7,14 @@ if [ $# -ne 1 ]; then
     exit 1
 fi
 
+datestr="$(date "+%Y%m%d")"
 if [ "$1" = "cpu" ]; then
     base_image="ubuntu:18.04"
-    image="pytorch/torchaudio_unittest_base:manylinux"
-elif [[ "$1" =~ ^(9.2|10.1)$ ]]; then
-    base_image="nvidia/cuda:$1-runtime-ubuntu18.04"
-    image="pytorch/torchaudio_unittest_base:manylinux-cuda$1"
+    image="pytorch/torchaudio_unittest_base:manylinux-${datestr}"
 else
-    printf "Unexpected <CUDA_VERSION> string: %s" "$1"
-    exit 1;
+    base_image="nvidia/cuda:$1-devel-ubuntu18.04"
+    docker pull "${base_image}"
+    image="pytorch/torchaudio_unittest_base:manylinux-cuda$1-${datestr}"
 fi
 
 cd "$( dirname "${BASH_SOURCE[0]}" )"

diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh
@@ -5,7 +5,7 @@ unset PYTORCH_VERSION
 # so no need to set PYTORCH_VERSION.
 # In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config.
 
-set -e
+set -ex
 
 root_dir="$(git rev-parse --show-toplevel)"
 conda_dir="${root_dir}/conda"
@@ -38,6 +38,8 @@ conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch ${cudatoolkit}
 
 # 2. Install torchaudio
 printf "* Installing torchaudio\n"
+python -m torch.utils.collect_env
+echo "CUDA_HOME: ${CUDA_HOME}"
 BUILD_SOX=1 python setup.py install
 
 # 3. Install Test tools

diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh
@@ -5,7 +5,7 @@
 #
 # Do not install PyTorch and torchaudio here, otherwise they also get cached.
 
-set -e
+set -ex
 
 root_dir="$(git rev-parse --show-toplevel)"
 conda_dir="${root_dir}/conda"
@@ -41,11 +41,3 @@ conda activate "${env_dir}"
 
 # 3. Install minimal build tools
 pip --quiet install cmake ninja
-
-# 4. Buld codecs
-mkdir -p third_party/build
-(
-    cd third_party/build
-    cmake -GNinja ..
-    cmake --build .
-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -0,0 +1,110 @@
+cmake_minimum_required(VERSION 3.14)
+
+project(torchaudio)
+
+option(BUILD_SOX "Build SoX and bind statically" OFF)
+option(BUILD_LIBTORCHAUDIO "Build C++ Library" ON)
+option(BUILD_PYTHON_EXTENSION "Build Python extension" OFF)
+
+set(CMAKE_CXX_STANDARD 14 CACHE STRING "The C++ standard whose features are requested to build this target.")
+
+find_package(Torch REQUIRED)
+# Set -D_GLIBCXX_USE_CXX11_ABI for third party builds
+if (DEFINED _GLIBCXX_USE_CXX11_ABI)
+  set(CXXFLAGS "${CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=${_GLIBCXX_USE_CXX11_ABI}")
+endif()
+
+add_subdirectory(third_party)
+
+if (APPLE)
+  SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
+endif()
+
+################################################################################
+# Beginning of torchaudio extension configuration
+################################################################################
+
+set(TORCHAUDIO_CSRC "${CMAKE_CURRENT_SOURCE_DIR}/torchaudio/csrc")
+
+set(
+  LIBTORCHAUDIO_SOURCES
+  "${TORCHAUDIO_CSRC}/sox_io.cpp"
+  "${TORCHAUDIO_CSRC}/sox_utils.cpp"
+  "${TORCHAUDIO_CSRC}/sox_effects.cpp"
+  "${TORCHAUDIO_CSRC}/sox_effects_chain.cpp"
+  "${TORCHAUDIO_CSRC}/register.cpp"
+  )
+
+################################################################################
+# libtorchaudio.so
+################################################################################
+if(BUILD_LIBTORCHAUDIO)
+  add_library(
+    libtorchaudio
+    SHARED
+    ${LIBTORCHAUDIO_SOURCES}
+    )
+  set_target_properties(libtorchaudio PROPERTIES PREFIX "")
+
+  target_include_directories(
+    libtorchaudio
+    PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    )
+
+  target_link_libraries(
+    libtorchaudio
+    "${TORCHAUDIO_THIRD_PARTIES}"
+    "${TORCH_LIBRARIES}"
+    )
+
+  install(
+    TARGETS
+    libtorchaudio
+    )
+
+  set(TORCHAUDIO_LIBRARY -Wl,--no-as-needed libtorchaudio -Wl,--as-needed CACHE INTERNAL "")
+endif()
+
+################################################################################
+# _torchaudio.so
+################################################################################
+if (BUILD_PYTHON_EXTENSION)
+  find_package(Python REQUIRED Interpreter Development)
+
+  add_library(
+    _torchaudio
+    SHARED
+    ${TORCHAUDIO_CSRC}/sox.cpp
+    ${LIBTORCHAUDIO_SOURCES}
+    )
+
+  set_target_properties(_torchaudio PROPERTIES PREFIX "")
+
+  if (APPLE)
+    # https://github.com/facebookarchive/caffe2/issues/854#issuecomment-364538485
+    # https://github.com/pytorch/pytorch/commit/73f6715f4725a0723d8171d3131e09ac7abf0666
+    set_target_properties(_torchaudio PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+  endif()
+
+  target_include_directories(
+    _torchaudio
+    PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    ${Python_INCLUDE_DIRS}
+    )
+
+  # See https://github.com/pytorch/pytorch/issues/38122
+  find_library(TORCH_PYTHON_LIBRARY torch_python PATHS "${TORCH_INSTALL_PREFIX}/lib")
+
+  target_link_libraries(
+    _torchaudio
+    ${TORCHAUDIO_THIRD_PARTIES}
+    ${TORCH_LIBRARIES}
+    ${TORCH_PYTHON_LIBRARY}
+    )
+
+  # We do not define install for _torchaudio.so
+  # The location of installation is controlled by "CMAKE_LIBRARY_OUTPUT_DIRECTORY" and the
+  # resulting file is handled by setuptools.
+endif()
-Original file line number
+Diff line change
@@ Expand Up / @@ -7,7 +7,8 @@ dependencies: @@
       - pytest
       - pytest-cov
       - codecov
-      - librosa
+      - cmake
+      - librosa>=0.8.0
       - llvmlite==0.31  # See https://github.com/pytorch/audio/pull/766
       - pip
       - pip:
@@ Expand Down @@