diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh
index 6ece517335..131c9e43bf 100755
--- a/.circleci/unittest/linux/scripts/install.sh
+++ b/.circleci/unittest/linux/scripts/install.sh
@@ -20,4 +20,4 @@ printf "Installing PyTorch with %s\n" "${cudatoolkit}"
 conda install -y -c pytorch-nightly pytorch "${cudatoolkit}"
 
 printf "* Installing torchaudio\n"
-python setup.py develop
+BUILD_SOX=1 python setup.py develop
diff --git a/.circleci/unittest/linux/scripts/run_test.sh b/.circleci/unittest/linux/scripts/run_test.sh
index 700eb7ad64..fa2bfa6c3c 100755
--- a/.circleci/unittest/linux/scripts/run_test.sh
+++ b/.circleci/unittest/linux/scripts/run_test.sh
@@ -6,4 +6,5 @@ eval "$(./conda/bin/conda shell.bash hook)"
 conda activate ./env
 
 python -m torch.utils.collect_env
+export PATH="${PWD}/third_party/build/bin/:${PATH}"
 pytest --cov=torchaudio --junitxml=test-results/junit.xml -v --durations 20 test
diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh
index 07271b6789..1fd5268e85 100755
--- a/.circleci/unittest/linux/scripts/setup_env.sh
+++ b/.circleci/unittest/linux/scripts/setup_env.sh
@@ -34,4 +34,4 @@ printf "* Installing dependencies (except PyTorch)\n"
 conda env update --file "${this_dir}/environment.yml" --prune
 
 # 4. Build codecs
-# build_tools/setup_helpers/build_third_party.sh
+build_tools/setup_helpers/build_third_party.sh
diff --git a/build_tools/setup_helpers/build_third_party.sh b/build_tools/setup_helpers/build_third_party.sh
index 9577776cc9..b75365791e 100755
--- a/build_tools/setup_helpers/build_third_party.sh
+++ b/build_tools/setup_helpers/build_third_party.sh
@@ -21,6 +21,20 @@ mkdir -p "${tmp_dir}" "${build_dir}"
 
 . "${this_dir}/build_third_party_helper.sh"
 
+if ! found_ogg "${build_dir}" ; then
+    get_ogg "${tmp_dir}"
+    if [ "${download_only}" = "false" ]; then
+        build_ogg "${tmp_dir}" "${build_dir}"
+    fi
+fi
+
+if ! found_vorbis "${build_dir}" ; then
+    get_vorbis "${tmp_dir}"
+    if [ "${download_only}" = "false" ]; then
+        build_vorbis "${tmp_dir}" "${build_dir}"
+    fi
+fi
+
 if ! found_lame "${build_dir}" ; then
     get_lame "${tmp_dir}"
     if [ "${download_only}" = "false" ]; then
diff --git a/build_tools/setup_helpers/build_third_party_helper.sh b/build_tools/setup_helpers/build_third_party_helper.sh
index 7cca812409..a7d662c9d4 100644
--- a/build_tools/setup_helpers/build_third_party_helper.sh
+++ b/build_tools/setup_helpers/build_third_party_helper.sh
@@ -24,6 +24,18 @@ all_found() {
     done
 }
 
+found_ogg() {
+    all_found "$1" 'include/ogg/ogg.h' 'lib/libogg.a'
+}
+
+found_vorbis() {
+    all_found "$1" \
+              'include/vorbis/vorbisenc.h' \
+              'include/vorbis/vorbisfile.h' \
+              'lib/libvorbis.a' \
+              'lib/libvorbisenc.a' \
+              'lib/libvorbisfile.a'
+}
 
 found_lame() {
     all_found "$1" 'include/lame/lame.h' 'lib/libmp3lame.a'
@@ -57,6 +69,82 @@ found_sox() {
     all_found "$1" 'include/sox.h' 'lib/libsox.a'
 }
 
+# libogg 1.3.4 has bug on mac OS.
+# https://trac.macports.org/ticket/58924
+OGG="libogg-1.3.3"
+OGG_ARCHIVE="${OGG}.tar.gz"
+
+get_ogg() {
+    work_dir="$1"
+    url="https://ftp.osuosl.org/pub/xiph/releases/ogg/${OGG_ARCHIVE}"
+    (
+        cd "${work_dir}"
+        if [ ! -d "${OGG}" ]; then
+            if [ ! -f "${OGG_ARCHIVE}" ]; then
+                printf "Fetching libogg from %s\n" "${url}"
+                curl $CURL_OPTS -O "${url}"
+            fi
+        fi
+    )
+}
+
+build_ogg() {
+    work_dir="$1"
+    install_dir="$2"
+    (
+        cd "${work_dir}"
+        if [ ! -d "${OGG}" ]; then
+            tar xfp "${OGG_ARCHIVE}"
+        fi
+        cd "${OGG}"
+        printf "Building libogg\n"
+        if [ ! -f Makefile ]; then
+            ./configure ${CONFIG_OPTS} \
+                        --disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \
+                        --with-pic --disable-dependency-tracking
+        fi
+        make ${MAKE_OPTS} > make.log 2>&1
+        make install
+    )
+}
+
+VORBIS="libvorbis-1.3.6"
+VORBIS_ARCHIVE="${VORBIS}.tar.gz"
+
+get_vorbis() {
+    work_dir="$1"
+    url="https://ftp.osuosl.org/pub/xiph/releases/vorbis/${VORBIS_ARCHIVE}"
+    (
+        cd "${work_dir}"
+        if [ ! -d "${VORBIS}" ]; then
+            if [ ! -f "${VORBIS_ARCHIVE}" ]; then
+                printf "Fetching libvorbis from %s\n" "${url}"
+                curl $CURL_OPTS -O "${url}"
+            fi
+        fi
+    )
+}
+
+build_vorbis() {
+    work_dir="$1"
+    install_dir="$2"
+    (
+        cd "${work_dir}"
+        if [ ! -d "${VORBIS}" ]; then
+            tar xfp "${VORBIS_ARCHIVE}"
+        fi
+        cd "${VORBIS}"
+        printf "Building libvorbis\n"
+        if [ ! -f Makefile ]; then
+            ./configure ${CONFIG_OPTS} \
+                        --disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \
+                        --with-pic --disable-dependency-tracking
+        fi
+        make ${MAKE_OPTS} > make.log 2>&1
+        make install
+    )
+}
+
 LAME="lame-3.99.5"
 LAME_ARCHIVE="${LAME}.tar.gz"
 
@@ -126,7 +214,7 @@ build_flac() {
         if [ ! -f Makefile ]; then
             ./configure ${CONFIG_OPTS} \
                         --disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \
-                        --with-pic --disable-debug --disable-dependency-tracking
+                        --with-pic --with-ogg="${install_dir}" --disable-debug --disable-dependency-tracking
         fi
         make ${MAKE_OPTS} > make.log 2>&1
         make ${MAKE_OPTS} install
@@ -207,8 +295,8 @@ build_sox() {
             # it statically if we do.
             ./configure ${CONFIG_OPTS} --disable-shared --enable-static --prefix="${install_dir}" \
                         LDFLAGS="-L${install_dir}/lib" CPPFLAGS="-I${install_dir}/include" \
-                        --with-lame --with-flac --with-mad --without-alsa --without-coreaudio \
-                        --without-png --without-oggvorbis --without-oss --without-sndfile \
+                        --with-lame --with-flac --with-mad --with-oggvorbis --without-alsa --without-coreaudio \
+                        --without-png --without-oss --without-sndfile \
                         CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking
         fi
         make ${MAKE_OPTS} > make.log 2>&1
diff --git a/build_tools/setup_helpers/extension.py b/build_tools/setup_helpers/extension.py
index dcf61ab259..92a5260129 100644
--- a/build_tools/setup_helpers/extension.py
+++ b/build_tools/setup_helpers/extension.py
@@ -76,8 +76,18 @@ def _get_extra_objects():
         # NOTE: The order of the library listed bellow matters.
         #
         # (the most important thing is that dependencies come after a library
-        # e.g., sox comes first)
-        libs = ['libsox.a', 'libmad.a', 'libFLAC.a', 'libmp3lame.a']
+        # e.g., sox comes first, flac/vorbis comes before ogg, and
+        # vorbisenc/vorbisfile comes before vorbis
+        libs = [
+            'libsox.a',
+            'libmad.a',
+            'libFLAC.a',
+            'libmp3lame.a',
+            'libvorbisenc.a',
+            'libvorbisfile.a',
+            'libvorbis.a',
+            'libogg.a',
+        ]
         for lib in libs:
             objs.append(str(_TP_INSTALL_DIR / 'lib' / lib))
     return objs