diff --git a/.circleci/config.yml b/.circleci/config.yml index 718327e74f..413a19c7bd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,6 +38,17 @@ commands: our_upload_channel=test fi echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} + install_cmake_macos: + description: "installs cmake on macOS. Use binary distribution as brew is slow" + steps: + - run: + name: Install cmake + command: | + curl -L -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.16.5/cmake-3.16.5-Darwin-x86_64.tar.gz + mkdir cmake + tar -xf cmake.tar.gz --strip 3 -C cmake + rm cmake.tar.gz + echo 'export PATH='"${PWD}/cmake/bin"':${PATH}' >> ${BASH_ENV} binary_common: &binary_common parameters: @@ -82,7 +93,7 @@ jobs: download_third_parties_nix: docker: - - image: "centos:8" + - image: "pytorch/torchaudio_unittest_base:manylinux" resource_class: small steps: - checkout @@ -90,20 +101,22 @@ jobs: - restore_cache: keys: - - tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }} + - tp-nix-v2-{{ checksum ".cachekey" }} - run: - command: ./build_tools/setup_helpers/build_third_party.sh $PWD --download-only + command: | + mkdir -p third_party/archives/ + wget --no-clobber --directory-prefix=third_party/archives/ $(awk '/URL /{print $2}' third_party/CMakeLists.txt) - save_cache: - key: tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }} + key: tp-nix-v2-{{ checksum ".cachekey" }} paths: - - third_party/tmp + - third_party/archives - persist_to_workspace: root: third_party paths: - - tmp + - archives binary_linux_wheel: <<: *binary_common @@ -145,6 +158,7 @@ jobs: xcode: "9.0" steps: - checkout + - install_cmake_macos - attach_workspace: at: third_party - run: @@ -169,6 +183,7 @@ jobs: xcode: "9.0" steps: - checkout + - install_cmake_macos - attach_workspace: at: third_party - run: @@ -383,19 +398,21 @@ jobs: - restore_cache: keys: - - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }} + - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }} - run: name: Setup command: .circleci/unittest/linux/scripts/setup_env.sh - save_cache: - key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }} + key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }} paths: - conda - env - third_party/build + - third_party/install + - third_party/src - run: name: Install torchaudio command: .circleci/unittest/linux/scripts/install.sh @@ -423,19 +440,21 @@ jobs: - restore_cache: keys: - - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }} + - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }} - run: name: Setup command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh - save_cache: - key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }} + key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }} paths: - conda - env - third_party/build + - third_party/install + - third_party/src - run: name: Install torchaudio command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh @@ -542,6 +561,8 @@ jobs: - conda - env - third_party/build + - third_party/install + - third_party/src - run: name: Run style check command: .circleci/unittest/linux/scripts/run_style_checks.sh diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 9d9e348cc8..398577c79a 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -38,6 +38,17 @@ commands: our_upload_channel=test fi echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} + install_cmake_macos: + description: "installs cmake on macOS. Use binary distribution as brew is slow" + steps: + - run: + name: Install cmake + command: | + curl -L -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.16.5/cmake-3.16.5-Darwin-x86_64.tar.gz + mkdir cmake + tar -xf cmake.tar.gz --strip 3 -C cmake + rm cmake.tar.gz + echo 'export PATH='"${PWD}/cmake/bin"':${PATH}' >> ${BASH_ENV} binary_common: &binary_common parameters: @@ -82,7 +93,7 @@ jobs: download_third_parties_nix: docker: - - image: "centos:8" + - image: "pytorch/torchaudio_unittest_base:manylinux" resource_class: small steps: - checkout @@ -90,20 +101,22 @@ jobs: - restore_cache: {% raw %} keys: - - tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }} + - tp-nix-v2-{{ checksum ".cachekey" }} {% endraw %} - run: - command: ./build_tools/setup_helpers/build_third_party.sh $PWD --download-only + command: | + mkdir -p third_party/archives/ + wget --no-clobber --directory-prefix=third_party/archives/ $(awk '/URL /{print $2}' third_party/CMakeLists.txt) - save_cache: {% raw %} - key: tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }} + key: tp-nix-v2-{{ checksum ".cachekey" }} {% endraw %} paths: - - third_party/tmp + - third_party/archives - persist_to_workspace: root: third_party paths: - - tmp + - archives binary_linux_wheel: <<: *binary_common @@ -145,6 +158,7 @@ jobs: xcode: "9.0" steps: - checkout + - install_cmake_macos - attach_workspace: at: third_party - run: @@ -169,6 +183,7 @@ jobs: xcode: "9.0" steps: - checkout + - install_cmake_macos - attach_workspace: at: third_party - run: @@ -383,19 +398,21 @@ jobs: - restore_cache: {% raw %} keys: - - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }} + - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }} {% endraw %} - run: name: Setup command: .circleci/unittest/linux/scripts/setup_env.sh - save_cache: {% raw %} - key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }} + key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }} {% endraw %} paths: - conda - env - third_party/build + - third_party/install + - third_party/src - run: name: Install torchaudio command: .circleci/unittest/linux/scripts/install.sh @@ -423,19 +440,21 @@ jobs: - restore_cache: {% raw %} keys: - - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }} + - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }} {% endraw %} - run: name: Setup command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh - save_cache: {% raw %} - key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }} + key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }} {% endraw %} paths: - conda - env - third_party/build + - third_party/install + - third_party/src - run: name: Install torchaudio command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh @@ -542,6 +561,8 @@ jobs: - conda - env - third_party/build + - third_party/install + - third_party/src - run: name: Run style check command: .circleci/unittest/linux/scripts/run_style_checks.sh diff --git a/.circleci/unittest/linux/docker/Dockerfile b/.circleci/unittest/linux/docker/Dockerfile index dc9b623591..d1de89688e 100644 --- a/.circleci/unittest/linux/docker/Dockerfile +++ b/.circleci/unittest/linux/docker/Dockerfile @@ -58,6 +58,7 @@ RUN apt update && apt install -y \ sox \ libsox-dev \ libsox-fmt-all \ + cmake \ && rm -rf /var/lib/apt/lists/* COPY --from=builder /kaldi /kaldi COPY --from=builder /third_party /third_party diff --git a/.circleci/unittest/linux/scripts/run_test.sh b/.circleci/unittest/linux/scripts/run_test.sh index fa2bfa6c3c..9ea942851e 100755 --- a/.circleci/unittest/linux/scripts/run_test.sh +++ b/.circleci/unittest/linux/scripts/run_test.sh @@ -6,5 +6,5 @@ eval "$(./conda/bin/conda shell.bash hook)" conda activate ./env python -m torch.utils.collect_env -export PATH="${PWD}/third_party/build/bin/:${PATH}" +export PATH="${PWD}/third_party/install/bin/:${PATH}" pytest --cov=torchaudio --junitxml=test-results/junit.xml -v --durations 20 test diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh index 1fd5268e85..489094a3b9 100755 --- a/.circleci/unittest/linux/scripts/setup_env.sh +++ b/.circleci/unittest/linux/scripts/setup_env.sh @@ -33,5 +33,10 @@ conda activate "${env_dir}" printf "* Installing dependencies (except PyTorch)\n" conda env update --file "${this_dir}/environment.yml" --prune -# 4. Build codecs -build_tools/setup_helpers/build_third_party.sh +# 4. Buld codecs +mkdir -p third_party/build +( + cd third_party/build + cmake .. + cmake --build . +) diff --git a/.gitignore b/.gitignore index 41a434acb2..e8a9d4c409 100644 --- a/.gitignore +++ b/.gitignore @@ -121,3 +121,9 @@ gen.yml examples/interactive_asr/data/*.txt examples/interactive_asr/data/*.model examples/interactive_asr/data/*.pt + +# third parties +third_party/archives/ +third_party/install/ +third_party/src/ +third_party/tmp/ diff --git a/build_tools/setup_helpers/build_third_party.sh b/build_tools/setup_helpers/build_third_party.sh deleted file mode 100755 index b75365791e..0000000000 --- a/build_tools/setup_helpers/build_third_party.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -# Build third party libraries (SoX, lame, libmad, and flac) -# Usage: ./build_thid_parth.sh [prefix] [download_only?=false] - -set -e - -this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -root_dir="${this_dir}/../.." - -prefix="${1:-}" -if [ -z "${prefix}" ]; then - prefix="${root_dir}" -fi -download_only="${2:-false}" - -tp_dir="${prefix}/third_party" -tmp_dir="${tp_dir}/tmp" -build_dir="${tp_dir}/build" - -mkdir -p "${tmp_dir}" "${build_dir}" - -. "${this_dir}/build_third_party_helper.sh" - -if ! found_ogg "${build_dir}" ; then - get_ogg "${tmp_dir}" - if [ "${download_only}" = "false" ]; then - build_ogg "${tmp_dir}" "${build_dir}" - fi -fi - -if ! found_vorbis "${build_dir}" ; then - get_vorbis "${tmp_dir}" - if [ "${download_only}" = "false" ]; then - build_vorbis "${tmp_dir}" "${build_dir}" - fi -fi - -if ! found_lame "${build_dir}" ; then - get_lame "${tmp_dir}" - if [ "${download_only}" = "false" ]; then - build_lame "${tmp_dir}" "${build_dir}" - fi -fi - -if ! found_flac "${build_dir}" ; then - get_flac "${tmp_dir}" - if [ "${download_only}" = "false" ]; then - build_flac "${tmp_dir}" "${build_dir}" - fi -fi - -if ! found_mad "${build_dir}" ; then - get_mad "${tmp_dir}" - if [ "${download_only}" = "false" ]; then - build_mad "${tmp_dir}" "${build_dir}" - fi -fi - -if ! found_sox "${build_dir}" ; then - get_sox "${tmp_dir}" - if [ "${download_only}" = "false" ]; then - build_sox "${tmp_dir}" "${build_dir}" - fi -fi diff --git a/build_tools/setup_helpers/build_third_party_helper.sh b/build_tools/setup_helpers/build_third_party_helper.sh deleted file mode 100644 index a7d662c9d4..0000000000 --- a/build_tools/setup_helpers/build_third_party_helper.sh +++ /dev/null @@ -1,305 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -# Global options -CURL_OPTS="-L --retry 10 --connect-timeout 5 --max-time 180" -MAKE_OPTS="-j" -CONFIG_OPTS="" - -if [ -z ${DEBUG+x} ]; then - CURL_OPTS="${CURL_OPTS} --silent --show-error" - MAKE_OPTS="${MAKE_OPTS} --quiet" - CONFIG_OPTS="${CONFIG_OPTS} --quiet" -fi - -all_found() { - dir="$1" - shift - while [ "$#" -gt 0 ]; do - if [ ! -f "${dir}/$1" ]; then - return 1 - fi - shift - done -} - -found_ogg() { - all_found "$1" 'include/ogg/ogg.h' 'lib/libogg.a' -} - -found_vorbis() { - all_found "$1" \ - 'include/vorbis/vorbisenc.h' \ - 'include/vorbis/vorbisfile.h' \ - 'lib/libvorbis.a' \ - 'lib/libvorbisenc.a' \ - 'lib/libvorbisfile.a' -} - -found_lame() { - all_found "$1" 'include/lame/lame.h' 'lib/libmp3lame.a' -} - -found_flac() { - all_found "$1" \ - 'include/FLAC/format.h' \ - 'include/FLAC/stream_decoder.h' \ - 'include/FLAC/export.h' \ - 'include/FLAC/ordinals.h' \ - 'include/FLAC/all.h' \ - 'include/FLAC/assert.h' \ - 'include/FLAC/callback.h' \ - 'include/FLAC/metadata.h' \ - 'include/FLAC/stream_encoder.h' \ - 'include/FLAC++/export.h' \ - 'include/FLAC++/decoder.h' \ - 'include/FLAC++/all.h' \ - 'include/FLAC++/metadata.h' \ - 'include/FLAC++/encoder.h' \ - 'lib/libFLAC++.a' \ - 'lib/libFLAC.a' -} - -found_mad() { - all_found "$1" 'include/mad.h' 'lib/libmad.a' -} - -found_sox() { - all_found "$1" 'include/sox.h' 'lib/libsox.a' -} - -# libogg 1.3.4 has bug on mac OS. -# https://trac.macports.org/ticket/58924 -OGG="libogg-1.3.3" -OGG_ARCHIVE="${OGG}.tar.gz" - -get_ogg() { - work_dir="$1" - url="https://ftp.osuosl.org/pub/xiph/releases/ogg/${OGG_ARCHIVE}" - ( - cd "${work_dir}" - if [ ! -d "${OGG}" ]; then - if [ ! -f "${OGG_ARCHIVE}" ]; then - printf "Fetching libogg from %s\n" "${url}" - curl $CURL_OPTS -O "${url}" - fi - fi - ) -} - -build_ogg() { - work_dir="$1" - install_dir="$2" - ( - cd "${work_dir}" - if [ ! -d "${OGG}" ]; then - tar xfp "${OGG_ARCHIVE}" - fi - cd "${OGG}" - printf "Building libogg\n" - if [ ! -f Makefile ]; then - ./configure ${CONFIG_OPTS} \ - --disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \ - --with-pic --disable-dependency-tracking - fi - make ${MAKE_OPTS} > make.log 2>&1 - make install - ) -} - -VORBIS="libvorbis-1.3.6" -VORBIS_ARCHIVE="${VORBIS}.tar.gz" - -get_vorbis() { - work_dir="$1" - url="https://ftp.osuosl.org/pub/xiph/releases/vorbis/${VORBIS_ARCHIVE}" - ( - cd "${work_dir}" - if [ ! -d "${VORBIS}" ]; then - if [ ! -f "${VORBIS_ARCHIVE}" ]; then - printf "Fetching libvorbis from %s\n" "${url}" - curl $CURL_OPTS -O "${url}" - fi - fi - ) -} - -build_vorbis() { - work_dir="$1" - install_dir="$2" - ( - cd "${work_dir}" - if [ ! -d "${VORBIS}" ]; then - tar xfp "${VORBIS_ARCHIVE}" - fi - cd "${VORBIS}" - printf "Building libvorbis\n" - if [ ! -f Makefile ]; then - ./configure ${CONFIG_OPTS} \ - --disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \ - --with-pic --disable-dependency-tracking - fi - make ${MAKE_OPTS} > make.log 2>&1 - make install - ) -} - -LAME="lame-3.99.5" -LAME_ARCHIVE="${LAME}.tar.gz" - -get_lame() { - work_dir="$1" - url="https://downloads.sourceforge.net/project/lame/lame/3.99/${LAME_ARCHIVE}" - ( - cd "${work_dir}" - if [ ! -d "${LAME}" ]; then - if [ ! -f "${LAME_ARCHIVE}" ]; then - printf "Fetching liblame from %s\n" "${url}" - curl $CURL_OPTS -O "${url}" - fi - fi - ) -} - -build_lame() { - work_dir="$1" - install_dir="$2" - ( - cd "${work_dir}" - if [ ! -d "${LAME}" ]; then - tar xfp "${LAME_ARCHIVE}" - fi - cd "${LAME}" - # build statically - printf "Building liblame\n" - if [ ! -f Makefile ]; then - ./configure ${CONFIG_OPTS} \ - --disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \ - --with-pic --disable-debug --disable-dependency-tracking --enable-nasm - fi - make ${MAKE_OPTS} > make.log 2>&1 - make ${MAKE_OPTS} install - ) -} - -FLAC="flac-1.3.2" -FLAC_ARCHIVE="${FLAC}.tar.xz" - -get_flac() { - work_dir="$1" - url="https://downloads.sourceforge.net/project/flac/flac-src/${FLAC_ARCHIVE}" - ( - cd "${work_dir}" - if [ ! -d "${FLAC}" ]; then - if [ ! -f "${FLAC_ARCHIVE}" ]; then - printf "Fetching flac from %s\n" "${url}" - curl $CURL_OPTS -O "${url}" - fi - fi - ) -} - -build_flac() { - work_dir="$1" - install_dir="$2" - ( - cd "${work_dir}" - if [ ! -d "${FLAC}" ]; then - tar xfp "${FLAC_ARCHIVE}" - fi - cd "${FLAC}" - # build statically - printf "Building flac\n" - if [ ! -f Makefile ]; then - ./configure ${CONFIG_OPTS} \ - --disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \ - --with-pic --with-ogg="${install_dir}" --disable-debug --disable-dependency-tracking - fi - make ${MAKE_OPTS} > make.log 2>&1 - make ${MAKE_OPTS} install - ) -} - -LIBMAD="libmad-0.15.1b" -LIBMAD_ARCHIVE="${LIBMAD}.tar.gz" - -get_mad() { - work_dir="$1" - url="https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/${LIBMAD_ARCHIVE}" - ( - cd "${work_dir}" - if [ ! -d "${LIBMAD}" ]; then - if [ ! -f "${LIBMAD_ARCHIVE}" ]; then - printf "Fetching mad from %s\n" "${url}" - curl $CURL_OPTS -O "${url}" - fi - fi - ) -} - -build_mad() { - work_dir="$1" - install_dir="$2" - ( - cd "${work_dir}" - if [ ! -d "${LIBMAD}" ]; then - tar xfp "${LIBMAD_ARCHIVE}" - fi - cd "${LIBMAD}" - # build statically - printf "Building mad\n" - if [ ! -f Makefile ]; then - # See https://stackoverflow.com/a/12864879/23845 - sed -i.bak 's/-march=i486//' configure - ./configure ${CONFIG_OPTS} \ - --disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \ - --with-pic --disable-debug --disable-dependency-tracking - fi - make ${MAKE_OPTS} > make.log 2>&1 - make ${MAKE_OPTS} install - ) -} - -SOX="sox-14.4.2" -SOX_ARCHIVE="${SOX}.tar.bz2" - -get_sox() { - work_dir="$1" - url="https://downloads.sourceforge.net/project/sox/sox/14.4.2/${SOX_ARCHIVE}" - ( - cd "${work_dir}" - if [ ! -d "${SOX}" ]; then - if [ ! -f "${SOX_ARCHIVE}" ]; then - printf "Fetching SoX from %s\n" "${url}" - curl $CURL_OPTS -O "${url}" - fi - fi - ) -} - -build_sox() { - work_dir="$1" - install_dir="$2" - ( - cd "${work_dir}" - if [ ! -d "${SOX}" ]; then - tar xfp "${SOX_ARCHIVE}" - fi - cd "${SOX}" - # build statically - printf "Building SoX\n" - if [ ! -f Makefile ]; then - # --without-png makes OS X build less hazardous; somehow the build - # finds png and enables it. We don't want it; we'd need to package - # it statically if we do. - ./configure ${CONFIG_OPTS} --disable-shared --enable-static --prefix="${install_dir}" \ - LDFLAGS="-L${install_dir}/lib" CPPFLAGS="-I${install_dir}/include" \ - --with-lame --with-flac --with-mad --with-oggvorbis --without-alsa --without-coreaudio \ - --without-png --without-oss --without-sndfile \ - CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking - fi - make ${MAKE_OPTS} > make.log 2>&1 - make ${MAKE_OPTS} install - ) -} diff --git a/build_tools/setup_helpers/extension.py b/build_tools/setup_helpers/extension.py index 92a5260129..ab10014a46 100644 --- a/build_tools/setup_helpers/extension.py +++ b/build_tools/setup_helpers/extension.py @@ -17,7 +17,7 @@ _ROOT_DIR = _THIS_DIR.parent.parent.resolve() _CSRC_DIR = _ROOT_DIR / 'torchaudio' / 'csrc' _TP_BASE_DIR = _ROOT_DIR / 'third_party' -_TP_INSTALL_DIR = _TP_BASE_DIR / 'build' +_TP_INSTALL_DIR = _TP_BASE_DIR / 'install' def _get_build_sox(): @@ -97,15 +97,19 @@ def _get_libraries(): return [] if _BUILD_SOX else ['sox'] -def _build_codecs(): +def _build_third_party(): + build_dir = str(_TP_BASE_DIR / 'build') + os.makedirs(build_dir, exist_ok=True) subprocess.run( - args=[str(_THIS_DIR / 'build_third_party.sh')], + args=['cmake', '..'], + cwd=build_dir, + check=True, + ) + subprocess.run( + args=['cmake', '--build', '.'], + cwd=build_dir, check=True, ) - - -def _configure_third_party(): - _build_codecs() _EXT_NAME = 'torchaudio._torchaudio' @@ -130,5 +134,5 @@ def get_ext_modules(debug=False): class BuildExtension(TorchBuildExtension): def build_extension(self, ext): if ext.name == _EXT_NAME and _BUILD_SOX: - _configure_third_party() + _build_third_party() super().build_extension(ext) diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt new file mode 100644 index 0000000000..57b070adbb --- /dev/null +++ b/third_party/CMakeLists.txt @@ -0,0 +1,60 @@ +cmake_minimum_required(VERSION 3.1) + +project(torchaudio_third_parties) +include(ExternalProject) + +set(INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/install) +set(ARCHIVE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/archives) +set(COMMON_ARGS --quiet --disable-shared --enable-static --prefix=${INSTALL_DIR} --with-pic --disable-dependency-tracking --disable-debug --disable-examples --disable-doc) + +ExternalProject_Add(libmad + PREFIX ${CMAKE_CURRENT_SOURCE_DIR} + DOWNLOAD_DIR ${ARCHIVE_DIR} + URL https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz + URL_HASH SHA256=bbfac3ed6bfbc2823d3775ebb931087371e142bb0e9bb1bee51a76a6e0078690 + PATCH_COMMAND patch < ${CMAKE_CURRENT_SOURCE_DIR}/patch/libmad.patch + CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libmad/configure ${COMMON_ARGS} +) + +ExternalProject_Add(libmp3lame + PREFIX ${CMAKE_CURRENT_SOURCE_DIR} + DOWNLOAD_DIR ${ARCHIVE_DIR} + URL https://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz + URL_HASH SHA256=24346b4158e4af3bd9f2e194bb23eb473c75fb7377011523353196b19b9a23ff + CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libmp3lame/configure ${COMMON_ARGS} --enable-nasm +) + +ExternalProject_Add(libogg + PREFIX ${CMAKE_CURRENT_SOURCE_DIR} + DOWNLOAD_DIR ${ARCHIVE_DIR} + URL https://ftp.osuosl.org/pub/xiph/releases/ogg/libogg-1.3.3.tar.gz + URL_HASH SHA256=c2e8a485110b97550f453226ec644ebac6cb29d1caef2902c007edab4308d985 + CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libogg/configure ${COMMON_ARGS} +) + +ExternalProject_Add(libflac + PREFIX ${CMAKE_CURRENT_SOURCE_DIR} + DEPENDS libogg + DOWNLOAD_DIR ${ARCHIVE_DIR} + URL https://ftp.osuosl.org/pub/xiph/releases/flac/flac-1.3.2.tar.xz + URL_HASH SHA256=91cfc3ed61dc40f47f050a109b08610667d73477af6ef36dcad31c31a4a8d53f + CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libflac/configure ${COMMON_ARGS} --with-ogg=${INSTALL_DIR} +) + +ExternalProject_Add(libvorbis + PREFIX ${CMAKE_CURRENT_SOURCE_DIR} + DEPENDS libogg + DOWNLOAD_DIR ${ARCHIVE_DIR} + URL https://ftp.osuosl.org/pub/xiph/releases/vorbis/libvorbis-1.3.6.tar.gz + URL_HASH SHA256=6ed40e0241089a42c48604dc00e362beee00036af2d8b3f46338031c9e0351cb + CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libvorbis/configure ${COMMON_ARGS} --with-ogg=${INSTALL_DIR} +) + +ExternalProject_Add(libsox + PREFIX ${CMAKE_CURRENT_SOURCE_DIR} + DEPENDS libogg libflac libvorbis libmp3lame libmad + DOWNLOAD_DIR ${ARCHIVE_DIR} + URL https://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2 + URL_HASH SHA256=81a6956d4330e75b5827316e44ae381e6f1e8928003c6aa45896da9041ea149c + CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libsox/configure ${COMMON_ARGS} LDFLAGS=-L${INSTALL_DIR}/lib CPPFLAGS=-I${INSTALL_DIR}/include --with-lame --with-flac --with-mad --with-oggvorbis --without-alsa --without-coreaudio --without-png --without-oss --without-sndfile +) diff --git a/third_party/patch/libmad.patch b/third_party/patch/libmad.patch new file mode 100644 index 0000000000..a805787831 --- /dev/null +++ b/third_party/patch/libmad.patch @@ -0,0 +1,86 @@ +See the followings for the origin of this patch +http://www.linuxfromscratch.org/blfs/view/svn/multimedia/libmad.html +http://www.linuxfromscratch.org/patches/blfs/svn/libmad-0.15.1b-fixes-1.patch +--- src/libmad/configure 2004-02-05 09:34:07.000000000 +0000 ++++ src/libmad/configure.new 2020-06-30 21:10:28.528018931 +0000 +@@ -19083,71 +19083,7 @@ + + if test "$GCC" = yes + then +- if test -z "$arch" +- then +- case "$host" in +- i386-*) ;; +- i?86-*) arch="-march=i486" ;; +- arm*-empeg-*) arch="-march=armv4 -mtune=strongarm1100" ;; +- armv4*-*) arch="-march=armv4 -mtune=strongarm" ;; +- powerpc-*) ;; +- mips*-agenda-*) arch="-mcpu=vr4100" ;; +- mips*-luxsonor-*) arch="-mips1 -mcpu=r3000 -Wa,-m4010" ;; +- esac +- fi +- +- case "$optimize" in +- -O|"-O "*) +- optimize="-O" +- optimize="$optimize -fforce-mem" +- optimize="$optimize -fforce-addr" +- : #x optimize="$optimize -finline-functions" +- : #- optimize="$optimize -fstrength-reduce" +- optimize="$optimize -fthread-jumps" +- optimize="$optimize -fcse-follow-jumps" +- optimize="$optimize -fcse-skip-blocks" +- : #x optimize="$optimize -frerun-cse-after-loop" +- : #x optimize="$optimize -frerun-loop-opt" +- : #x optimize="$optimize -fgcse" +- optimize="$optimize -fexpensive-optimizations" +- optimize="$optimize -fregmove" +- : #* optimize="$optimize -fdelayed-branch" +- : #x optimize="$optimize -fschedule-insns" +- optimize="$optimize -fschedule-insns2" +- : #? optimize="$optimize -ffunction-sections" +- : #? optimize="$optimize -fcaller-saves" +- : #> optimize="$optimize -funroll-loops" +- : #> optimize="$optimize -funroll-all-loops" +- : #x optimize="$optimize -fmove-all-movables" +- : #x optimize="$optimize -freduce-all-givs" +- : #? optimize="$optimize -fstrict-aliasing" +- : #* optimize="$optimize -fstructure-noalias" +- +- case "$host" in +- arm*-*) +- optimize="$optimize -fstrength-reduce" +- ;; +- mips*-*) +- optimize="$optimize -fstrength-reduce" +- optimize="$optimize -finline-functions" +- ;; +- i?86-*) +- optimize="$optimize -fstrength-reduce" +- ;; +- powerpc-apple-*) +- # this triggers an internal compiler error with gcc2 +- : #optimize="$optimize -fstrength-reduce" +- +- # this is really only beneficial with gcc3 +- : #optimize="$optimize -finline-functions" +- ;; +- *) +- # this sometimes provokes bugs in gcc 2.95.2 +- : #optimize="$optimize -fstrength-reduce" +- ;; +- esac +- ;; +- esac ++ optimize="-O2" + fi + + case "$host" in +@@ -21497,6 +21433,7 @@ + then + case "$host" in + i?86-*) FPM="INTEL" ;; ++ x86_64*) FPM="64BIT" ;; + arm*-*) FPM="ARM" ;; + mips*-*) FPM="MIPS" ;; + sparc*-*) FPM="SPARC" ;;