Use cmake for third party

Author: mthrok

.circleci/config.yml

@@ -38,6 +38,17 @@ commands:
               our_upload_channel=test
             fi
             echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
+  install_cmake_macos:
+    description: "installs cmake on macOS. Use binary distribution as brew is slow"
+    steps:
+      - run:
+          name: Install cmake
+          command: |
+            curl -L -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.16.5/cmake-3.16.5-Darwin-x86_64.tar.gz
+            mkdir cmake
+            tar -xf cmake.tar.gz --strip 3 -C cmake
+            rm cmake.tar.gz
+            echo 'export PATH='"${PWD}/cmake/bin"':${PATH}' >> ${BASH_ENV}
 
 binary_common: &binary_common
   parameters:
@@ -82,28 +93,30 @@ jobs:
 
   download_third_parties_nix:
     docker:
-      - image: "centos:8"
+      - image: "pytorch/torchaudio_unittest_base:manylinux"
     resource_class: small
     steps:
       - checkout
       - generate_cache_key
       - restore_cache:
 
           keys:
-            - tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
+            - tp-nix-v2-{{ checksum ".cachekey" }}
 
       - run:
-          command: ./build_tools/setup_helpers/build_third_party.sh $PWD --download-only
+          command: |
+              mkdir -p third_party/archives/
+              wget --no-clobber --directory-prefix=third_party/archives/ $(awk '/URL /{print $2}' third_party/CMakeLists.txt)
       - save_cache:
 
-          key: tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
+          key: tp-nix-v2-{{ checksum ".cachekey" }}
 
           paths:
-            - third_party/tmp
+            - third_party/archives
       - persist_to_workspace:
           root: third_party
           paths:
-            - tmp
+            - archives
 
   binary_linux_wheel:
     <<: *binary_common
@@ -145,6 +158,7 @@ jobs:
       xcode: "9.0"
     steps:
       - checkout
+      - install_cmake_macos
       - attach_workspace:
           at: third_party
       - run:
@@ -169,6 +183,7 @@ jobs:
       xcode: "9.0"
     steps:
       - checkout
+      - install_cmake_macos
       - attach_workspace:
           at: third_party
       - run:
@@ -383,19 +398,21 @@ jobs:
       - restore_cache:
 
           keys:
-            - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
 
       - run:
           name: Setup
           command: .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
 
-          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
 
           paths:
             - conda
             - env
             - third_party/build
+            - third_party/install
+            - third_party/src
       - run:
           name: Install torchaudio
           command: .circleci/unittest/linux/scripts/install.sh
@@ -423,19 +440,21 @@ jobs:
       - restore_cache:
 
           keys:
-            - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
 
       - run:
           name: Setup
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
 
-          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
 
           paths:
             - conda
             - env
             - third_party/build
+            - third_party/install
+            - third_party/src
       - run:
           name: Install torchaudio
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
@@ -542,6 +561,8 @@ jobs:
             - conda
             - env
             - third_party/build
+            - third_party/install
+            - third_party/src
       - run:
           name: Run style check
           command: .circleci/unittest/linux/scripts/run_style_checks.sh

.circleci/config.yml.in

@@ -38,6 +38,17 @@ commands:
               our_upload_channel=test
             fi
             echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
+  install_cmake_macos:
+    description: "installs cmake on macOS. Use binary distribution as brew is slow"
+    steps:
+      - run:
+          name: Install cmake
+          command: |
+            curl -L -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.16.5/cmake-3.16.5-Darwin-x86_64.tar.gz
+            mkdir cmake
+            tar -xf cmake.tar.gz --strip 3 -C cmake
+            rm cmake.tar.gz
+            echo 'export PATH='"${PWD}/cmake/bin"':${PATH}' >> ${BASH_ENV}
 
 binary_common: &binary_common
   parameters:
@@ -82,28 +93,30 @@ jobs:
 
   download_third_parties_nix:
     docker:
-      - image: "centos:8"
+      - image: "pytorch/torchaudio_unittest_base:manylinux"
     resource_class: small
     steps:
       - checkout
       - generate_cache_key
       - restore_cache:
           {% raw %}
           keys:
-            - tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
+            - tp-nix-v2-{{ checksum ".cachekey" }}
           {% endraw %}
       - run:
-          command: ./build_tools/setup_helpers/build_third_party.sh $PWD --download-only
+          command: |
+              mkdir -p third_party/archives/
+              wget --no-clobber --directory-prefix=third_party/archives/ $(awk '/URL /{print $2}' third_party/CMakeLists.txt)
       - save_cache:
           {% raw %}
-          key: tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
+          key: tp-nix-v2-{{ checksum ".cachekey" }}
           {% endraw %}
           paths:
-            - third_party/tmp
+            - third_party/archives
       - persist_to_workspace:
           root: third_party
           paths:
-            - tmp
+            - archives
 
   binary_linux_wheel:
     <<: *binary_common
@@ -145,6 +158,7 @@ jobs:
       xcode: "9.0"
     steps:
       - checkout
+      - install_cmake_macos
       - attach_workspace:
           at: third_party
       - run:
@@ -169,6 +183,7 @@ jobs:
       xcode: "9.0"
     steps:
       - checkout
+      - install_cmake_macos
       - attach_workspace:
           at: third_party
       - run:
@@ -383,19 +398,21 @@ jobs:
       - restore_cache:
           {% raw %}
           keys:
-            - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
           {% endraw %}
       - run:
           name: Setup
           command: .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
           {% raw %}
-          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
           {% endraw %}
           paths:
             - conda
             - env
             - third_party/build
+            - third_party/install
+            - third_party/src
       - run:
           name: Install torchaudio
           command: .circleci/unittest/linux/scripts/install.sh
@@ -423,19 +440,21 @@ jobs:
       - restore_cache:
           {% raw %}
           keys:
-            - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
           {% endraw %}
       - run:
           name: Setup
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
           {% raw %}
-          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
           {% endraw %}
           paths:
             - conda
             - env
             - third_party/build
+            - third_party/install
+            - third_party/src
       - run:
           name: Install torchaudio
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
@@ -542,6 +561,8 @@ jobs:
             - conda
             - env
             - third_party/build
+            - third_party/install
+            - third_party/src
       - run:
           name: Run style check
           command: .circleci/unittest/linux/scripts/run_style_checks.sh

.circleci/unittest/linux/docker/Dockerfile

@@ -58,6 +58,8 @@ RUN apt update && apt install -y \
         sox \
         libsox-dev \
         libsox-fmt-all \
+        cmake \
+        ninja-build \
     && rm -rf /var/lib/apt/lists/*
 COPY --from=builder /kaldi /kaldi
 COPY --from=builder /third_party /third_party

.circleci/unittest/linux/scripts/run_test.sh

@@ -6,5 +6,5 @@ eval "$(./conda/bin/conda shell.bash hook)"
 conda activate ./env
 
 python -m torch.utils.collect_env
-export PATH="${PWD}/third_party/build/bin/:${PATH}"
+export PATH="${PWD}/third_party/install/bin/:${PATH}"
 pytest --cov=torchaudio --junitxml=test-results/junit.xml -v --durations 20 test

.circleci/unittest/linux/scripts/setup_env.sh

@@ -33,5 +33,10 @@ conda activate "${env_dir}"
 printf "* Installing dependencies (except PyTorch)\n"
 conda env update --file "${this_dir}/environment.yml" --prune
 
-# 4. Build codecs
-build_tools/setup_helpers/build_third_party.sh
+# 4. Buld codecs
+mkdir -p third_party/build
+(
+    cd third_party/build
+    cmake ..
+    cmake --build .
+)

.gitignore

@@ -121,3 +121,8 @@ gen.yml
 examples/interactive_asr/data/*.txt
 examples/interactive_asr/data/*.model
 examples/interactive_asr/data/*.pt
+
+# third parties
+third_party/archives/
+third_party/install/
+third_party/src/