From aeff943ed04c5243d953df2cd142929182ca7880 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Fri, 26 Jun 2020 13:19:23 -0400 Subject: [PATCH 1/2] Add vorbis to binary build (#750) --- .circleci/unittest/linux/scripts/install.sh | 2 +- .circleci/unittest/linux/scripts/run_test.sh | 1 + .circleci/unittest/linux/scripts/setup_env.sh | 2 +- .../setup_helpers/build_third_party.sh | 14 +++ .../setup_helpers/build_third_party_helper.sh | 94 ++++++++++++++++++- build_tools/setup_helpers/extension.py | 14 ++- 6 files changed, 120 insertions(+), 7 deletions(-) diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index 6ece517335..131c9e43bf 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -20,4 +20,4 @@ printf "Installing PyTorch with %s\n" "${cudatoolkit}" conda install -y -c pytorch-nightly pytorch "${cudatoolkit}" printf "* Installing torchaudio\n" -python setup.py develop +BUILD_SOX=1 python setup.py develop diff --git a/.circleci/unittest/linux/scripts/run_test.sh b/.circleci/unittest/linux/scripts/run_test.sh index 700eb7ad64..fa2bfa6c3c 100755 --- a/.circleci/unittest/linux/scripts/run_test.sh +++ b/.circleci/unittest/linux/scripts/run_test.sh @@ -6,4 +6,5 @@ eval "$(./conda/bin/conda shell.bash hook)" conda activate ./env python -m torch.utils.collect_env +export PATH="${PWD}/third_party/build/bin/:${PATH}" pytest --cov=torchaudio --junitxml=test-results/junit.xml -v --durations 20 test diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh index 07271b6789..1fd5268e85 100755 --- a/.circleci/unittest/linux/scripts/setup_env.sh +++ b/.circleci/unittest/linux/scripts/setup_env.sh @@ -34,4 +34,4 @@ printf "* Installing dependencies (except PyTorch)\n" conda env update --file "${this_dir}/environment.yml" --prune # 4. Build codecs -# build_tools/setup_helpers/build_third_party.sh +build_tools/setup_helpers/build_third_party.sh diff --git a/build_tools/setup_helpers/build_third_party.sh b/build_tools/setup_helpers/build_third_party.sh index 9577776cc9..b75365791e 100755 --- a/build_tools/setup_helpers/build_third_party.sh +++ b/build_tools/setup_helpers/build_third_party.sh @@ -21,6 +21,20 @@ mkdir -p "${tmp_dir}" "${build_dir}" . "${this_dir}/build_third_party_helper.sh" +if ! found_ogg "${build_dir}" ; then + get_ogg "${tmp_dir}" + if [ "${download_only}" = "false" ]; then + build_ogg "${tmp_dir}" "${build_dir}" + fi +fi + +if ! found_vorbis "${build_dir}" ; then + get_vorbis "${tmp_dir}" + if [ "${download_only}" = "false" ]; then + build_vorbis "${tmp_dir}" "${build_dir}" + fi +fi + if ! found_lame "${build_dir}" ; then get_lame "${tmp_dir}" if [ "${download_only}" = "false" ]; then diff --git a/build_tools/setup_helpers/build_third_party_helper.sh b/build_tools/setup_helpers/build_third_party_helper.sh index 7cca812409..a7d662c9d4 100644 --- a/build_tools/setup_helpers/build_third_party_helper.sh +++ b/build_tools/setup_helpers/build_third_party_helper.sh @@ -24,6 +24,18 @@ all_found() { done } +found_ogg() { + all_found "$1" 'include/ogg/ogg.h' 'lib/libogg.a' +} + +found_vorbis() { + all_found "$1" \ + 'include/vorbis/vorbisenc.h' \ + 'include/vorbis/vorbisfile.h' \ + 'lib/libvorbis.a' \ + 'lib/libvorbisenc.a' \ + 'lib/libvorbisfile.a' +} found_lame() { all_found "$1" 'include/lame/lame.h' 'lib/libmp3lame.a' @@ -57,6 +69,82 @@ found_sox() { all_found "$1" 'include/sox.h' 'lib/libsox.a' } +# libogg 1.3.4 has bug on mac OS. +# https://trac.macports.org/ticket/58924 +OGG="libogg-1.3.3" +OGG_ARCHIVE="${OGG}.tar.gz" + +get_ogg() { + work_dir="$1" + url="https://ftp.osuosl.org/pub/xiph/releases/ogg/${OGG_ARCHIVE}" + ( + cd "${work_dir}" + if [ ! -d "${OGG}" ]; then + if [ ! -f "${OGG_ARCHIVE}" ]; then + printf "Fetching libogg from %s\n" "${url}" + curl $CURL_OPTS -O "${url}" + fi + fi + ) +} + +build_ogg() { + work_dir="$1" + install_dir="$2" + ( + cd "${work_dir}" + if [ ! -d "${OGG}" ]; then + tar xfp "${OGG_ARCHIVE}" + fi + cd "${OGG}" + printf "Building libogg\n" + if [ ! -f Makefile ]; then + ./configure ${CONFIG_OPTS} \ + --disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \ + --with-pic --disable-dependency-tracking + fi + make ${MAKE_OPTS} > make.log 2>&1 + make install + ) +} + +VORBIS="libvorbis-1.3.6" +VORBIS_ARCHIVE="${VORBIS}.tar.gz" + +get_vorbis() { + work_dir="$1" + url="https://ftp.osuosl.org/pub/xiph/releases/vorbis/${VORBIS_ARCHIVE}" + ( + cd "${work_dir}" + if [ ! -d "${VORBIS}" ]; then + if [ ! -f "${VORBIS_ARCHIVE}" ]; then + printf "Fetching libvorbis from %s\n" "${url}" + curl $CURL_OPTS -O "${url}" + fi + fi + ) +} + +build_vorbis() { + work_dir="$1" + install_dir="$2" + ( + cd "${work_dir}" + if [ ! -d "${VORBIS}" ]; then + tar xfp "${VORBIS_ARCHIVE}" + fi + cd "${VORBIS}" + printf "Building libvorbis\n" + if [ ! -f Makefile ]; then + ./configure ${CONFIG_OPTS} \ + --disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \ + --with-pic --disable-dependency-tracking + fi + make ${MAKE_OPTS} > make.log 2>&1 + make install + ) +} + LAME="lame-3.99.5" LAME_ARCHIVE="${LAME}.tar.gz" @@ -126,7 +214,7 @@ build_flac() { if [ ! -f Makefile ]; then ./configure ${CONFIG_OPTS} \ --disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \ - --with-pic --disable-debug --disable-dependency-tracking + --with-pic --with-ogg="${install_dir}" --disable-debug --disable-dependency-tracking fi make ${MAKE_OPTS} > make.log 2>&1 make ${MAKE_OPTS} install @@ -207,8 +295,8 @@ build_sox() { # it statically if we do. ./configure ${CONFIG_OPTS} --disable-shared --enable-static --prefix="${install_dir}" \ LDFLAGS="-L${install_dir}/lib" CPPFLAGS="-I${install_dir}/include" \ - --with-lame --with-flac --with-mad --without-alsa --without-coreaudio \ - --without-png --without-oggvorbis --without-oss --without-sndfile \ + --with-lame --with-flac --with-mad --with-oggvorbis --without-alsa --without-coreaudio \ + --without-png --without-oss --without-sndfile \ CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking fi make ${MAKE_OPTS} > make.log 2>&1 diff --git a/build_tools/setup_helpers/extension.py b/build_tools/setup_helpers/extension.py index dcf61ab259..92a5260129 100644 --- a/build_tools/setup_helpers/extension.py +++ b/build_tools/setup_helpers/extension.py @@ -76,8 +76,18 @@ def _get_extra_objects(): # NOTE: The order of the library listed bellow matters. # # (the most important thing is that dependencies come after a library - # e.g., sox comes first) - libs = ['libsox.a', 'libmad.a', 'libFLAC.a', 'libmp3lame.a'] + # e.g., sox comes first, flac/vorbis comes before ogg, and + # vorbisenc/vorbisfile comes before vorbis + libs = [ + 'libsox.a', + 'libmad.a', + 'libFLAC.a', + 'libmp3lame.a', + 'libvorbisenc.a', + 'libvorbisfile.a', + 'libvorbis.a', + 'libogg.a', + ] for lib in libs: objs.append(str(_TP_INSTALL_DIR / 'lib' / lib)) return objs From d36c15269fb6b9ab907a1dd293028dc35bb50db2 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Mon, 29 Jun 2020 18:54:51 +0000 Subject: [PATCH 2/2] Fix lint error --- torchaudio/csrc/sox.cpp | 12 +++++------- torchaudio/csrc/sox.h | 3 ++- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/torchaudio/csrc/sox.cpp b/torchaudio/csrc/sox.cpp index 3ae81bef19..c76bce232b 100644 --- a/torchaudio/csrc/sox.cpp +++ b/torchaudio/csrc/sox.cpp @@ -83,11 +83,11 @@ std::tuple get_info( } std::vector get_effect_names() { - sox_effect_fn_t const * fns = sox_get_effect_fns(); + sox_effect_fn_t const* fns = sox_get_effect_fns(); std::vector sv; - for(int i = 0; fns[i]; ++i) { - const sox_effect_handler_t *eh = fns[i] (); - if(eh && eh->name) + for (int i = 0; fns[i]; ++i) { + const sox_effect_handler_t* eh = fns[i](); + if (eh && eh->name) sv.push_back(eh->name); } return sv; @@ -502,7 +502,5 @@ PYBIND11_MODULE(_torchaudio, m) { &torch::audio::initialize_sox, "initialize sox for effects"); m.def( - "shutdown_sox", - &torch::audio::shutdown_sox, - "shutdown sox for effects"); + "shutdown_sox", &torch::audio::shutdown_sox, "shutdown sox for effects"); } diff --git a/torchaudio/csrc/sox.h b/torchaudio/csrc/sox.h index 8d851c9b21..f5741379e6 100644 --- a/torchaudio/csrc/sox.h +++ b/torchaudio/csrc/sox.h @@ -48,7 +48,8 @@ std::tuple get_info( // get names of all sox effects std::vector get_effect_names(); -// Initialize and Shutdown SoX effects chain. These functions should only be run once. +// Initialize and Shutdown SoX effects chain. These functions should only be +// run once. int initialize_sox(); int shutdown_sox();