diff --git a/.circleci/config.yml b/.circleci/config.yml
index f0ee6764b1..83a084dbbb 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -42,8 +42,8 @@ commands:
     description: "installs tools required to build torchaudio"
     steps:
       - run:
-          name: Install cmake and pkg-config
-          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake pkg-config wget
+          name: Install pkg-config
+          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config wget
           # Disable brew auto update which is very slow
 
 binary_common: &binary_common
@@ -399,14 +399,14 @@ jobs:
       - restore_cache:
 
           keys:
-            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-cpu-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
 
       - run:
           name: Setup
           command: .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
 
-          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-cpu-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
 
           paths:
             - conda
@@ -433,7 +433,7 @@ jobs:
     resource_class: gpu.small
     environment:
       <<: *environment
-      image_name: "pytorch/torchaudio_unittest_base:manylinux-cuda10.1"
+      image_name: pytorch/torchaudio_unittest_base:manylinux-cuda10.1-cudnn7-20201203
     steps:
       - checkout
       - attach_workspace:
@@ -443,7 +443,7 @@ jobs:
       - restore_cache:
 
           keys:
-            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-gpu-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
 
       - run:
           name: Pull Docker image
@@ -453,7 +453,7 @@ jobs:
           command: docker run -t --gpus all -e PYTHON_VERSION -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
 
-          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-gpu-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
 
           paths:
             - conda
@@ -463,7 +463,7 @@ jobs:
             - third_party/src
       - run:
           name: Install torchaudio
-          command: docker run -t --gpus all -e UPLOAD_CHANNEL -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
+          command: docker run -t --gpus all -e UPLOAD_CHANNEL -e CUDA_HOME -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
       - run:
           name: Run tests
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in
index bdbc3c774d..b841ac1d58 100644
--- a/.circleci/config.yml.in
+++ b/.circleci/config.yml.in
@@ -42,8 +42,8 @@ commands:
     description: "installs tools required to build torchaudio"
     steps:
       - run:
-          name: Install cmake and pkg-config
-          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake pkg-config wget
+          name: Install pkg-config
+          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config wget
           # Disable brew auto update which is very slow
 
 binary_common: &binary_common
@@ -399,14 +399,14 @@ jobs:
       - restore_cache:
           {% raw %}
           keys:
-            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-cpu-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
           {% endraw %}
       - run:
           name: Setup
           command: .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
           {% raw %}
-          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-cpu-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
           {% endraw %}
           paths:
             - conda
@@ -433,7 +433,7 @@ jobs:
     resource_class: gpu.small
     environment:
       <<: *environment
-      image_name: "pytorch/torchaudio_unittest_base:manylinux-cuda10.1"
+      image_name: pytorch/torchaudio_unittest_base:manylinux-cuda10.1-cudnn7-20201203
     steps:
       - checkout
       - attach_workspace:
@@ -443,7 +443,7 @@ jobs:
       - restore_cache:
           {% raw %}
           keys:
-            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-gpu-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
           {% endraw %}
       - run:
           name: Pull Docker image
@@ -453,7 +453,7 @@ jobs:
           command: docker run -t --gpus all -e PYTHON_VERSION -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
           {% raw %}
-          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-gpu-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
           {% endraw %}
           paths:
             - conda
@@ -463,7 +463,7 @@ jobs:
             - third_party/src
       - run:
           name: Install torchaudio
-          command: docker run -t --gpus all -e UPLOAD_CHANNEL -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
+          command: docker run -t --gpus all -e UPLOAD_CHANNEL -e CUDA_HOME -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
       - run:
           name: Run tests
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
diff --git a/.circleci/torchscript_bc_test/environment.yml b/.circleci/torchscript_bc_test/environment.yml
index 108d97d193..3b8d239fd9 100644
--- a/.circleci/torchscript_bc_test/environment.yml
+++ b/.circleci/torchscript_bc_test/environment.yml
@@ -7,7 +7,8 @@ dependencies:
   - pytest
   - pytest-cov
   - codecov
-  - librosa
+  - cmake
+  - librosa>=0.8.0
   - llvmlite==0.31  # See https://github.com/pytorch/audio/pull/766
   - pip
   - pip:
diff --git a/.circleci/unittest/linux/docker/build_and_push.sh b/.circleci/unittest/linux/docker/build_and_push.sh
index 8bf33eaaa3..e7ced13ad3 100755
--- a/.circleci/unittest/linux/docker/build_and_push.sh
+++ b/.circleci/unittest/linux/docker/build_and_push.sh
@@ -7,15 +7,14 @@ if [ $# -ne 1 ]; then
     exit 1
 fi
 
+datestr="$(date "+%Y%m%d")"
 if [ "$1" = "cpu" ]; then
     base_image="ubuntu:18.04"
-    image="pytorch/torchaudio_unittest_base:manylinux"
-elif [[ "$1" =~ ^(9.2|10.1)$ ]]; then
-    base_image="nvidia/cuda:$1-runtime-ubuntu18.04"
-    image="pytorch/torchaudio_unittest_base:manylinux-cuda$1"
+    image="pytorch/torchaudio_unittest_base:manylinux-${datestr}"
 else
-    printf "Unexpected <CUDA_VERSION> string: %s" "$1"
-    exit 1;
+    base_image="nvidia/cuda:$1-devel-ubuntu18.04"
+    docker pull "${base_image}"
+    image="pytorch/torchaudio_unittest_base:manylinux-cuda$1-${datestr}"
 fi
 
 cd "$( dirname "${BASH_SOURCE[0]}" )"
diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh
index 27dec79251..309fbc34d6 100755
--- a/.circleci/unittest/linux/scripts/install.sh
+++ b/.circleci/unittest/linux/scripts/install.sh
@@ -5,7 +5,7 @@ unset PYTORCH_VERSION
 # so no need to set PYTORCH_VERSION.
 # In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config.
 
-set -e
+set -ex
 
 root_dir="$(git rev-parse --show-toplevel)"
 conda_dir="${root_dir}/conda"
@@ -38,6 +38,8 @@ conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch ${cudatoolkit}
 
 # 2. Install torchaudio
 printf "* Installing torchaudio\n"
+python -m torch.utils.collect_env
+echo "CUDA_HOME: ${CUDA_HOME}"
 BUILD_SOX=1 python setup.py install
 
 # 3. Install Test tools
diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh
index 26292a00d5..085dc61a5b 100755
--- a/.circleci/unittest/linux/scripts/setup_env.sh
+++ b/.circleci/unittest/linux/scripts/setup_env.sh
@@ -5,7 +5,7 @@
 #
 # Do not install PyTorch and torchaudio here, otherwise they also get cached.
 
-set -e
+set -ex
 
 root_dir="$(git rev-parse --show-toplevel)"
 conda_dir="${root_dir}/conda"
@@ -41,11 +41,3 @@ conda activate "${env_dir}"
 
 # 3. Install minimal build tools
 pip --quiet install cmake ninja
-
-# 4. Buld codecs
-mkdir -p third_party/build
-(
-    cd third_party/build
-    cmake -GNinja ..
-    cmake --build .
-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
new file mode 100644
index 0000000000..2654cf0265
--- /dev/null
+++ b/CMakeLists.txt
@@ -0,0 +1,110 @@
+cmake_minimum_required(VERSION 3.14)
+
+project(torchaudio)
+
+option(BUILD_SOX "Build SoX and bind statically" OFF)
+option(BUILD_LIBTORCHAUDIO "Build C++ Library" ON)
+option(BUILD_PYTHON_EXTENSION "Build Python extension" OFF)
+
+set(CMAKE_CXX_STANDARD 14 CACHE STRING "The C++ standard whose features are requested to build this target.")
+
+find_package(Torch REQUIRED)
+# Set -D_GLIBCXX_USE_CXX11_ABI for third party builds
+if (DEFINED _GLIBCXX_USE_CXX11_ABI)
+  set(CXXFLAGS "${CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=${_GLIBCXX_USE_CXX11_ABI}")
+endif()
+
+add_subdirectory(third_party)
+
+if (APPLE)
+  SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
+endif()
+
+################################################################################
+# Beginning of torchaudio extension configuration
+################################################################################
+
+set(TORCHAUDIO_CSRC "${CMAKE_CURRENT_SOURCE_DIR}/torchaudio/csrc")
+
+set(
+  LIBTORCHAUDIO_SOURCES
+  "${TORCHAUDIO_CSRC}/sox_io.cpp"
+  "${TORCHAUDIO_CSRC}/sox_utils.cpp"
+  "${TORCHAUDIO_CSRC}/sox_effects.cpp"
+  "${TORCHAUDIO_CSRC}/sox_effects_chain.cpp"
+  "${TORCHAUDIO_CSRC}/register.cpp"
+  )
+
+################################################################################
+# libtorchaudio.so
+################################################################################
+if(BUILD_LIBTORCHAUDIO)
+  add_library(
+    libtorchaudio
+    SHARED
+    ${LIBTORCHAUDIO_SOURCES}
+    )
+  set_target_properties(libtorchaudio PROPERTIES PREFIX "")
+
+  target_include_directories(
+    libtorchaudio
+    PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    )
+
+  target_link_libraries(
+    libtorchaudio
+    "${TORCHAUDIO_THIRD_PARTIES}"
+    "${TORCH_LIBRARIES}"
+    )
+
+  install(
+    TARGETS
+    libtorchaudio
+    )
+
+  set(TORCHAUDIO_LIBRARY -Wl,--no-as-needed libtorchaudio -Wl,--as-needed CACHE INTERNAL "")
+endif()
+
+################################################################################
+# _torchaudio.so
+################################################################################
+if (BUILD_PYTHON_EXTENSION)
+  find_package(Python REQUIRED Interpreter Development)
+
+  add_library(
+    _torchaudio
+    SHARED
+    ${TORCHAUDIO_CSRC}/sox.cpp
+    ${LIBTORCHAUDIO_SOURCES}
+    )
+
+  set_target_properties(_torchaudio PROPERTIES PREFIX "")
+
+  if (APPLE)
+    # https://github.com/facebookarchive/caffe2/issues/854#issuecomment-364538485
+    # https://github.com/pytorch/pytorch/commit/73f6715f4725a0723d8171d3131e09ac7abf0666
+    set_target_properties(_torchaudio PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+  endif()
+
+  target_include_directories(
+    _torchaudio
+    PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    ${Python_INCLUDE_DIRS}
+    )
+
+  # See https://github.com/pytorch/pytorch/issues/38122
+  find_library(TORCH_PYTHON_LIBRARY torch_python PATHS "${TORCH_INSTALL_PREFIX}/lib")
+
+  target_link_libraries(
+    _torchaudio
+    ${TORCHAUDIO_THIRD_PARTIES}
+    ${TORCH_LIBRARIES}
+    ${TORCH_PYTHON_LIBRARY}
+    )
+
+  # We do not define install for _torchaudio.so
+  # The location of installation is controlled by "CMAKE_LIBRARY_OUTPUT_DIRECTORY" and the
+  # resulting file is handled by setuptools.
+endif()
diff --git a/build_tools/setup_helpers/extension.py b/build_tools/setup_helpers/extension.py
index b9fb30e114..ce98900a2f 100644
--- a/build_tools/setup_helpers/extension.py
+++ b/build_tools/setup_helpers/extension.py
@@ -1,23 +1,23 @@
 import os
+import sys
 import platform
 import subprocess
+import sysconfig
 from pathlib import Path
 
-from torch.utils.cpp_extension import (
-    CppExtension,
-    BuildExtension as TorchBuildExtension
-)
+import distutils.sysconfig
+from setuptools import Extension
+from setuptools.command.build_ext import build_ext
+import torch
 
 __all__ = [
     'get_ext_modules',
-    'BuildExtension',
+    'CMakeBuild',
 ]
 
 _THIS_DIR = Path(__file__).parent.resolve()
 _ROOT_DIR = _THIS_DIR.parent.parent.resolve()
-_CSRC_DIR = _ROOT_DIR / 'torchaudio' / 'csrc'
-_TP_BASE_DIR = _ROOT_DIR / 'third_party'
-_TP_INSTALL_DIR = _TP_BASE_DIR / 'install'
+_TORCHAUDIO_DIR = _ROOT_DIR / 'torchaudio'
 
 
 def _get_build_sox():
@@ -33,108 +33,98 @@ def _get_build_sox():
     return False
 
 
-_BUILD_SOX = _get_build_sox()
-
-
-def _get_eca(debug):
-    eca = []
-    if debug:
-        eca += ["-O0", "-g"]
-    else:
-        eca += ["-O3"]
-    return eca
-
-
-def _get_ela(debug):
-    ela = []
-    if debug:
-        if platform.system() == "Windows":
-            ela += ["/DEBUG:FULL"]
-        else:
-            ela += ["-O0", "-g"]
-    else:
-        ela += ["-O3"]
-    return ela
-
-
-def _get_srcs():
-    return [str(p) for p in _CSRC_DIR.glob('**/*.cpp')]
-
-
-def _get_include_dirs():
-    dirs = [
-        str(_ROOT_DIR),
-    ]
-    if _BUILD_SOX:
-        dirs.append(str(_TP_INSTALL_DIR / 'include'))
-    return dirs
-
-
-def _get_extra_objects():
-    objs = []
-    if _BUILD_SOX:
-        # NOTE: The order of the library listed bellow matters.
-        #
-        # (the most important thing is that dependencies come after a library
-        # e.g., sox comes first, flac/vorbis comes before ogg, and
-        # vorbisenc/vorbisfile comes before vorbis
-        libs = [
-            'libsox.a',
-            'libmad.a',
-            'libFLAC.a',
-            'libmp3lame.a',
-            'libopusfile.a',
-            'libopus.a',
-            'libvorbisenc.a',
-            'libvorbisfile.a',
-            'libvorbis.a',
-            'libogg.a',
-        ]
-        for lib in libs:
-            objs.append(str(_TP_INSTALL_DIR / 'lib' / lib))
-    return objs
+def _get_cxx11_abi():
+    try:
+        return int(torch._C._GLIBCXX_USE_CXX11_ABI)
+    except ImportError:
+        return 0
 
 
-def _get_libraries():
-    return [] if _BUILD_SOX else ['sox']
+def get_ext_modules():
+    if platform.system() == 'Windows':
+        return None
+    return [Extension(name='torchaudio._torchaudio', sources=[])]
 
 
-def _build_third_party():
-    build_dir = str(_TP_BASE_DIR / 'build')
-    os.makedirs(build_dir, exist_ok=True)
-    subprocess.run(
-        args=['cmake', '..'],
-        cwd=build_dir,
-        check=True,
-    )
-    subprocess.run(
-        args=['cmake', '--build', '.'],
-        cwd=build_dir,
-        check=True,
-    )
+def _get_python_include_dir():
+    # https://github.com/pytorch/pytorch/blob/7f869dca70606c42994d822ba11362a353411a1c/cmake/Dependencies.cmake#L904-L940
+    dir_ = distutils.sysconfig.get_python_inc()
+    if os.path.exists(dir_):
+        return dir_
+    dir_ = sysconfig.get_paths()['include']
+    if os.path.exists(dir_):
+        return dir_
+    raise RuntimeError('Cannot find Python development include directory.')
 
 
-_EXT_NAME = 'torchaudio._torchaudio'
+def _get_python_library():
+    lib = sysconfig.get_paths()['stdlib']
+    if os.path.exists(lib):
+        return lib
+    raise RuntimeError(f'Cannot find Python library. {lib}')
 
 
-def get_ext_modules(debug=False):
-    if platform.system() == 'Windows':
-        return None
-    return [
-        CppExtension(
-            _EXT_NAME,
-            _get_srcs(),
-            libraries=_get_libraries(),
-            include_dirs=_get_include_dirs(),
-            extra_compile_args=_get_eca(debug),
-            extra_objects=_get_extra_objects(),
-            extra_link_args=_get_ela(debug),
-        ),
-    ]
-
-
-class BuildExtension(TorchBuildExtension):
+# Based off of
+# https://github.com/pybind/cmake_example/blob/580c5fd29d4651db99d8874714b07c0c49a53f8a/setup.py
+class CMakeBuild(build_ext):
+    def run(self):
+        try:
+            subprocess.check_output(['cmake', '--version'])
+        except OSError:
+            raise RuntimeError("CMake is not available.")
+        super().run()
+
     def build_extension(self, ext):
-        if ext.name == _EXT_NAME and _BUILD_SOX:
-            _build_third_party()
-        super().build_extension(ext)
+        extdir = os.path.abspath(
+            os.path.dirname(self.get_ext_fullpath(ext.name)))
+
+        # required for auto-detection of auxiliary "native" libs
+        if not extdir.endswith(os.path.sep):
+            extdir += os.path.sep
+
+        cfg = "Debug" if self.debug else "Release"
+
+        cmake_args = [
+            f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}",
+            f"-DCMAKE_BUILD_TYPE={cfg}",
+            f"-DCMAKE_PREFIX_PATH={torch.utils.cmake_prefix_path}",
+            f"-DBUILD_SOX:BOOL={_get_build_sox()}",
+            f"-DPython_ROOT_DIR={Path(sys.executable).parent.parent.resolve()}",
+            f"-D_GLIBCXX_USE_CXX11_ABI={_get_cxx11_abi()}",
+            "-DBUILD_PYTHON_EXTENSION:BOOL=ON",
+            "-DBUILD_LIBTORCHAUDIO:BOOL=OFF",
+        ]
+        build_args = [
+            "--verbose",
+        ]
+
+        if 'CMAKE_CXX_FLAGS' in os.environ:
+            cmake_args += [f"-DCMAKE_CXX_FLAGS={os.environ['CMAKE_CXX_FLAGS']}"]
+
+        # Default to Ninja
+        if 'CMAKE_GENERATOR' not in os.environ:
+            cmake_args += ["-GNinja"]
+
+        # Set CMAKE_BUILD_PARALLEL_LEVEL to control the parallel build level
+        # across all generators.
+        if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ:
+            # self.parallel is a Python 3 only way to set parallel jobs by hand
+            # using -j in the build_ext call, not supported by pip or PyPA-build.
+            if hasattr(self, "parallel") and self.parallel:
+                # CMake 3.12+ only.
+                build_args += ["-j{}".format(self.parallel)]
+
+        if not os.path.exists(self.build_temp):
+            os.makedirs(self.build_temp)
+
+        subprocess.check_call(
+            ["cmake", str(_ROOT_DIR)] + cmake_args, cwd=self.build_temp)
+        subprocess.check_call(
+            ["cmake", "--build", "."] + build_args, cwd=self.build_temp)
+
+    def get_ext_filename(self, fullname):
+        ext_filename = super().get_ext_filename(fullname)
+        ext_filename_parts = ext_filename.split('.')
+        without_abi = ext_filename_parts[:-2] + ext_filename_parts[-1:]
+        ext_filename = '.'.join(without_abi)
+        return ext_filename
diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh
index ad45a08d59..738ab41322 100755
--- a/packaging/build_wheel.sh
+++ b/packaging/build_wheel.sh
@@ -8,7 +8,7 @@ export BUILD_TYPE="wheel"
 export NO_CUDA_PACKAGE=1
 setup_env 0.8.0
 setup_wheel_python
-pip_install numpy future
+pip_install numpy future cmake ninja
 setup_pip_pytorch_version
 python setup.py clean
 if [[ "$OSTYPE" == "msys" ]]; then
diff --git a/packaging/torchaudio/meta.yaml b/packaging/torchaudio/meta.yaml
index bd4f004a1b..ad12821d0e 100644
--- a/packaging/torchaudio/meta.yaml
+++ b/packaging/torchaudio/meta.yaml
@@ -14,11 +14,12 @@ requirements:
     - python
     - setuptools
     - cpuonly
+    - cmake
+    - ninja
     {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT') }}
 
   run:
     - python
-    - typing # [py2k]
     {{ environ.get('CONDA_PYTORCH_CONSTRAINT') }}
 
 build:
diff --git a/setup.py b/setup.py
index 4ced5fc8fd..3fa08aa241 100644
--- a/setup.py
+++ b/setup.py
@@ -82,9 +82,7 @@ def run(self):
     ],
     packages=find_packages(exclude=["build*", "test*", "torchaudio.csrc*", "third_party*", "build_tools*"]),
     ext_modules=setup_helpers.get_ext_modules(),
-    cmdclass={
-        'build_ext': setup_helpers.BuildExtension.with_options(no_python_abi_suffix=True)
-    },
+    cmdclass={'build_ext': setup_helpers.CMakeBuild},
     install_requires=[pytorch_package_dep],
     zip_safe=False,
 )
diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt
index da068be8f3..3754454888 100644
--- a/third_party/CMakeLists.txt
+++ b/third_party/CMakeLists.txt
@@ -1,82 +1,167 @@
-cmake_minimum_required(VERSION 3.1)
-
-project(torchaudio_third_parties)
-include(ExternalProject)
-
-set(INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/install)
-set(ARCHIVE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/archives)
-set(COMMON_ARGS --quiet --disable-shared --enable-static --prefix=${INSTALL_DIR} --with-pic --disable-dependency-tracking --disable-debug --disable-examples --disable-doc)
-
-ExternalProject_Add(libmad
-  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz
-  URL_HASH SHA256=bbfac3ed6bfbc2823d3775ebb931087371e142bb0e9bb1bee51a76a6e0078690
-  PATCH_COMMAND patch < ${CMAKE_CURRENT_SOURCE_DIR}/patch/libmad.patch
-  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libmad/configure ${COMMON_ARGS}
-)
-
-ExternalProject_Add(libmp3lame
-  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz
-  URL_HASH SHA256=24346b4158e4af3bd9f2e194bb23eb473c75fb7377011523353196b19b9a23ff
-  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libmp3lame/configure ${COMMON_ARGS} --enable-nasm
-)
-
-ExternalProject_Add(libogg
-  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://ftp.osuosl.org/pub/xiph/releases/ogg/libogg-1.3.3.tar.gz
-  URL_HASH SHA256=c2e8a485110b97550f453226ec644ebac6cb29d1caef2902c007edab4308d985
-  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libogg/configure ${COMMON_ARGS}
-)
-
-ExternalProject_Add(libflac
-  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
-  DEPENDS libogg
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://ftp.osuosl.org/pub/xiph/releases/flac/flac-1.3.2.tar.xz
-  URL_HASH SHA256=91cfc3ed61dc40f47f050a109b08610667d73477af6ef36dcad31c31a4a8d53f
-  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_codec_helper.sh ${CMAKE_CURRENT_SOURCE_DIR}/src/libflac/configure ${COMMON_ARGS} --with-ogg
-)
-
-ExternalProject_Add(libvorbis
-  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
-  DEPENDS libogg
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://ftp.osuosl.org/pub/xiph/releases/vorbis/libvorbis-1.3.6.tar.gz
-  URL_HASH SHA256=6ed40e0241089a42c48604dc00e362beee00036af2d8b3f46338031c9e0351cb
-  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_codec_helper.sh ${CMAKE_CURRENT_SOURCE_DIR}/src/libvorbis/configure ${COMMON_ARGS} --with-ogg
-)
-
-ExternalProject_Add(libopus
-  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
-  DEPENDS libogg
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://ftp.osuosl.org/pub/xiph/releases/opus/opus-1.3.1.tar.gz
-  URL_HASH SHA256=65b58e1e25b2a114157014736a3d9dfeaad8d41be1c8179866f144a2fb44ff9d
-  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_codec_helper.sh ${CMAKE_CURRENT_SOURCE_DIR}/src/libopus/configure ${COMMON_ARGS} --with-ogg
-)
-
-ExternalProject_Add(opusfile
-  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
-  DEPENDS libopus
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  STAMP_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/opusfile-stamp
-  SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/opusfile
-  URL https://ftp.osuosl.org/pub/xiph/releases/opus/opusfile-0.12.tar.gz
-  URL_HASH SHA256=118d8601c12dd6a44f52423e68ca9083cc9f2bfe72da7a8c1acb22a80ae3550b
-  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_codec_helper.sh ${CMAKE_CURRENT_SOURCE_DIR}/src/opusfile/configure ${COMMON_ARGS} --disable-http
-)
-
-ExternalProject_Add(libsox
-  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
-  DEPENDS libogg libflac libvorbis opusfile libmp3lame libmad
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2
-  URL_HASH SHA256=81a6956d4330e75b5827316e44ae381e6f1e8928003c6aa45896da9041ea149c
-  # OpenMP is by default compiled against GNU OpenMP, which conflicts with the version of OpenMP that PyTorch uses.
-  # See https://github.com/pytorch/audio/pull/1026
-  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_codec_helper.sh ${CMAKE_CURRENT_SOURCE_DIR}/src/libsox/configure ${COMMON_ARGS} --with-lame --with-flac --with-mad --with-oggvorbis --without-alsa --without-coreaudio --without-png --without-oss --without-sndfile --with-opus --disable-openmp
-)
+cmake_minimum_required(VERSION 3.14)
+
+if (BUILD_SOX)
+
+  include(ExternalProject)
+
+  set(INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/install)
+  set(ARCHIVE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/archives)
+  set(COMMON_ARGS --quiet --disable-silent-rules --disable-shared --enable-static --prefix=${INSTALL_DIR} --with-pic --disable-dependency-tracking --disable-debug --disable-examples --disable-doc)
+
+  # We need to use `${CMAKE_COMMAND} -E env ${envs} <COMMANAD>` pattern, to propagate custom environment variables
+  # to ExternalProject_Add. We constrcut the custom environment variables here
+  set(envs "PKG_CONFIG_PATH=${INSTALL_DIR}/lib/pkgconfig" "LDFLAGS=-L${INSTALL_DIR}/lib $ENV{LDFLAGS}" "CPPFLAGS=-I${INSTALL_DIR}/include $ENV{CPPFLAGS}")
+
+  ExternalProject_Add(mad_
+    PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
+    DOWNLOAD_DIR ${ARCHIVE_DIR}
+    URL https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz
+    URL_HASH SHA256=bbfac3ed6bfbc2823d3775ebb931087371e142bb0e9bb1bee51a76a6e0078690
+    PATCH_COMMAND patch < ${CMAKE_CURRENT_SOURCE_DIR}/patch/libmad.patch
+    CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_SOURCE_DIR}/src/mad_/configure ${COMMON_ARGS}
+    BUILD_BYPRODUCTS ${INSTALL_DIR}/lib/libmad.a
+    )
+
+  add_library(mad INTERFACE)
+  add_dependencies(mad mad_)
+  target_link_libraries(mad INTERFACE ${INSTALL_DIR}/lib/libmad.a)
+  target_include_directories(mad INTERFACE ${INSTALL_DIR}/include)
+
+  ExternalProject_Add(mp3lame_
+    PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
+    DOWNLOAD_DIR ${ARCHIVE_DIR}
+    URL https://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz
+    URL_HASH SHA256=24346b4158e4af3bd9f2e194bb23eb473c75fb7377011523353196b19b9a23ff
+    CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_SOURCE_DIR}/src/mp3lame_/configure ${COMMON_ARGS} --enable-nasm
+    BUILD_BYPRODUCTS ${INSTALL_DIR}/lib/libmp3lame.a
+    )
+
+  add_library(mp3lame INTERFACE)
+  add_dependencies(mp3lame mp3lame_)
+  target_link_libraries(mp3lame INTERFACE ${INSTALL_DIR}/lib/libmp3lame.a)
+  target_include_directories(mp3lame INTERFACE ${INSTALL_DIR}/include)
+
+  ExternalProject_Add(ogg_
+    PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
+    DOWNLOAD_DIR ${ARCHIVE_DIR}
+    URL https://ftp.osuosl.org/pub/xiph/releases/ogg/libogg-1.3.3.tar.gz
+    URL_HASH SHA256=c2e8a485110b97550f453226ec644ebac6cb29d1caef2902c007edab4308d985
+    CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_SOURCE_DIR}/src/ogg_/configure ${COMMON_ARGS}
+    BUILD_BYPRODUCTS ${INSTALL_DIR}/lib/libogg.a
+    )
+
+  add_library(ogg INTERFACE)
+  add_dependencies(ogg ogg_)
+  target_link_libraries(ogg INTERFACE ${INSTALL_DIR}/lib/libogg.a)
+  target_include_directories(ogg INTERFACE ${INSTALL_DIR}/include)
+
+  ExternalProject_Add(flac_
+    PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
+    DEPENDS ogg_
+    DOWNLOAD_DIR ${ARCHIVE_DIR}
+    URL https://ftp.osuosl.org/pub/xiph/releases/flac/flac-1.3.2.tar.xz
+    URL_HASH SHA256=91cfc3ed61dc40f47f050a109b08610667d73477af6ef36dcad31c31a4a8d53f
+    CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_SOURCE_DIR}/src/flac_/configure ${COMMON_ARGS} --with-ogg --disable-cpplibs
+    BUILD_BYPRODUCTS
+    ${INSTALL_DIR}/lib/libFLAC.a
+    )
+
+  add_library(flac INTERFACE)
+  add_dependencies(flac flac_)
+  target_link_libraries(
+    flac INTERFACE
+    ${INSTALL_DIR}/lib/libFLAC.a
+    )
+  target_include_directories(flac INTERFACE ${INSTALL_DIR}/include)
+
+  ExternalProject_Add(vorbis_
+    PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
+    DEPENDS ogg_
+    DOWNLOAD_DIR ${ARCHIVE_DIR}
+    URL https://ftp.osuosl.org/pub/xiph/releases/vorbis/libvorbis-1.3.6.tar.gz
+    URL_HASH SHA256=6ed40e0241089a42c48604dc00e362beee00036af2d8b3f46338031c9e0351cb
+    CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_SOURCE_DIR}/src/vorbis_/configure ${COMMON_ARGS} --with-ogg
+    BUILD_BYPRODUCTS
+    ${INSTALL_DIR}/lib/libvorbisenc.a
+    ${INSTALL_DIR}/lib/libvorbisfile.a
+    ${INSTALL_DIR}/lib/libvorbis.a
+    )
+
+  add_library(vorbis INTERFACE)
+  add_dependencies(vorbis vorbis_)
+  target_link_libraries(
+    vorbis INTERFACE
+    ${INSTALL_DIR}/lib/libvorbisenc.a
+    ${INSTALL_DIR}/lib/libvorbisfile.a
+    ${INSTALL_DIR}/lib/libvorbis.a
+    )
+  target_include_directories(vorbis INTERFACE ${INSTALL_DIR}/include)
+
+  ExternalProject_Add(opus_
+    PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
+    DEPENDS ogg_
+    DOWNLOAD_DIR ${ARCHIVE_DIR}
+    URL https://ftp.osuosl.org/pub/xiph/releases/opus/opus-1.3.1.tar.gz
+    URL_HASH SHA256=65b58e1e25b2a114157014736a3d9dfeaad8d41be1c8179866f144a2fb44ff9d
+    CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_SOURCE_DIR}/src/opus_/configure ${COMMON_ARGS} --with-ogg
+    BUILD_BYPRODUCTS ${INSTALL_DIR}/lib/libopus.a
+    )
+
+  add_library(opus INTERFACE)
+  add_dependencies(opus opus_)
+  target_link_libraries(opus INTERFACE ${INSTALL_DIR}/lib/libopus.a)
+  target_include_directories(opus INTERFACE ${INSTALL_DIR}/include)
+
+  ExternalProject_Add(opusfile_
+    PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
+    DEPENDS opus_
+    DOWNLOAD_DIR ${ARCHIVE_DIR}
+    STAMP_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/opusfile-stamp
+    SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/opusfile
+    URL https://ftp.osuosl.org/pub/xiph/releases/opus/opusfile-0.12.tar.gz
+    URL_HASH SHA256=118d8601c12dd6a44f52423e68ca9083cc9f2bfe72da7a8c1acb22a80ae3550b
+    CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_SOURCE_DIR}/src/opusfile/configure ${COMMON_ARGS} --disable-http
+    BUILD_BYPRODUCTS ${INSTALL_DIR}/lib/libopusfile.a
+    )
+
+  add_library(opusfile INTERFACE)
+  add_dependencies(opusfile opusfile_)
+  target_link_libraries(opusfile INTERFACE ${INSTALL_DIR}/lib/libopusfile.a)
+  target_include_directories(opusfile INTERFACE ${INSTALL_DIR}/include)
+
+  ExternalProject_Add(sox_
+    PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
+    DEPENDS ogg_ flac_ vorbis_ opusfile_ mp3lame_ mad_
+    DOWNLOAD_DIR ${ARCHIVE_DIR}
+    URL https://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2
+    URL_HASH SHA256=81a6956d4330e75b5827316e44ae381e6f1e8928003c6aa45896da9041ea149c
+    # OpenMP is by default compiled against GNU OpenMP, which conflicts with the version of OpenMP that PyTorch uses.
+    # See https://github.com/pytorch/audio/pull/1026
+    CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_SOURCE_DIR}/src/sox_/configure ${COMMON_ARGS} --with-lame --with-flac --with-mad --with-oggvorbis --without-alsa --without-coreaudio --without-png --without-oss --without-sndfile --with-opus --without-amrnb --without-amrwb --disable-openmp
+    BUILD_BYPRODUCTS ${INSTALL_DIR}/lib/libsox.a
+    )
+
+  add_library(sox INTERFACE)
+  add_dependencies(sox sox_)
+  target_link_libraries(sox INTERFACE ${INSTALL_DIR}/lib/libsox.a)
+  target_include_directories(sox INTERFACE ${INSTALL_DIR}/include)
+
+  set(
+    TORCHAUDIO_THIRD_PARTIES
+    sox
+    mad
+    flac
+    mp3lame
+    opusfile
+    opus
+    vorbis
+    ogg
+    PARENT_SCOPE
+    )
+else()
+  set(
+    TORCHAUDIO_THIRD_PARTIES
+    sox
+    PARENT_SCOPE
+    )
+endif()
diff --git a/third_party/build_codec_helper.sh b/third_party/build_codec_helper.sh
deleted file mode 100755
index e7f2614781..0000000000
--- a/third_party/build_codec_helper.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-# Helper script for building codecs depending on libogg, such as libopus and opus.
-# It is difficult to set environment variable inside of ExternalProject_Add,
-# so this script sets necessary environment variables before running the given command
-
-this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-install_dir="${this_dir}/install"
-
-export PKG_CONFIG_PATH="${install_dir}/lib/pkgconfig"
-export LDFLAGS="-L${install_dir}/lib ${LDFLAGS}"
-export CPPFLAGS="-I${install_dir}/include ${CPPFLAGS}"
-
-$@