Import torchaudio 20200723

Summary: Import torchaudio 20200723 #814

Reviewed By: fmassa

Differential Revision: D22666393

fbshipit-source-id: 50df07b5c158fe4e95ada7ea54381b2e26f6aecd

Author: mthrok

.circleci/config.yml

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+
+"""
+This script should use a very simple, functional programming style.
+Avoid Jinja macros in favor of native Python functions.
+
+Don't go overboard on code generation; use Python only to generate
+content that can't be easily declared statically using CircleCI's YAML API.
+
+Data declarations (e.g. the nested loops for defining the configuration matrix)
+should be at the top of the file for easy updating.
+
+See this comment for design rationale:
+https://github.com/pytorch/vision/pull/1321#issuecomment-531033978
+"""
+
+import jinja2
+import yaml
+import os.path
+
+
+PYTHON_VERSIONS = ["3.6", "3.7", "3.8"]
+
+
+def build_workflows(prefix='', upload=False, filter_branch=None, indentation=6):
+    w = []
+    w += build_download_job(filter_branch)
+    for btype in ["wheel", "conda"]:
+        for os_type in ["linux", "macos", "windows"]:
+            for python_version in PYTHON_VERSIONS:
+                w += build_workflow_pair(btype, os_type, python_version, filter_branch, prefix, upload)
+
+    return indent(indentation, w)
+
+
+def build_download_job(filter_branch):
+    job = {
+        "name": "download_third_parties_nix",
+    }
+
+    if filter_branch:
+        job["filters"] = gen_filter_branch_tree(filter_branch)
+    return [{"download_third_parties_nix": job}]
+
+
+def build_workflow_pair(btype, os_type, python_version, filter_branch, prefix='', upload=False):
+
+    w = []
+    base_workflow_name = "{prefix}binary_{os_type}_{btype}_py{python_version}".format(
+        prefix=prefix,
+        os_type=os_type,
+        btype=btype,
+        python_version=python_version,
+    )
+
+    w.append(generate_base_workflow(base_workflow_name, python_version, filter_branch, os_type, btype))
+
+    if upload:
+
+        is_py3_linux = os_type in ['linux', "windows"] and not python_version.startswith("2.")
+
+        w.append(generate_upload_workflow(base_workflow_name, filter_branch, btype))
+
+        if filter_branch == 'nightly' and is_py3_linux:
+            pydistro = 'pip' if btype == 'wheel' else 'conda'
+            w.append(generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type))
+
+    return w
+
+
+def generate_base_workflow(base_workflow_name, python_version, filter_branch, os_type, btype):
+
+    d = {
+        "name": base_workflow_name,
+        "python_version": python_version,
+    }
+
+    if os_type in ['linux', 'macos']:
+        d['requires'] = ['download_third_parties_nix']
+
+    if filter_branch:
+        d["filters"] = gen_filter_branch_tree(filter_branch)
+
+    return {"binary_{os_type}_{btype}".format(os_type=os_type, btype=btype): d}
+
+
+def gen_filter_branch_tree(*branches):
+    return {
+        "branches": {
+            "only": list(branches),
+        },
+        "tags": {
+            # Using a raw string here to avoid having to escape
+            # anything
+            "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
+        }
+    }
+
+
+def generate_upload_workflow(base_workflow_name, filter_branch, btype):
+    d = {
+        "name": "{base_workflow_name}_upload".format(base_workflow_name=base_workflow_name),
+        "context": "org-member",
+        "requires": [base_workflow_name],
+    }
+
+    if filter_branch:
+        d["filters"] = gen_filter_branch_tree(filter_branch)
+
+    return {"binary_{btype}_upload".format(btype=btype): d}
+
+
+def generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type):
+
+    required_build_suffix = "_upload"
+    required_build_name = base_workflow_name + required_build_suffix
+
+    smoke_suffix = "smoke_test_{pydistro}".format(pydistro=pydistro)
+    d = {
+        "name": "{base_workflow_name}_{smoke_suffix}".format(
+            base_workflow_name=base_workflow_name, smoke_suffix=smoke_suffix),
+        "requires": [required_build_name],
+        "python_version": python_version,
+    }
+
+    if filter_branch:
+        d["filters"] = gen_filter_branch_tree(filter_branch)
+
+    return {"smoke_test_{os_type}_{pydistro}".format(os_type=os_type, pydistro=pydistro): d}
+
+
+def indent(indentation, data_list):
+    return ("\n" + " " * indentation).join(yaml.dump(data_list).splitlines())
+
+
+def unittest_workflows(indentation=6):
+    jobs = []
+    jobs += build_download_job(None)
+    for os_type in ["linux", "windows", "macos"]:
+        for device_type in ["cpu", "gpu"]:
+            if os_type == "macos" and device_type == "gpu":
+                continue
+
+            for i, python_version in enumerate(PYTHON_VERSIONS):
+                job = {
+                    "name": f"unittest_{os_type}_{device_type}_py{python_version}",
+                    "python_version": python_version,
+                }
+
+                if device_type == 'gpu':
+                    job['filters'] = gen_filter_branch_tree('master', 'nightly')
+
+                if os_type != "windows":
+                    job['requires'] = ['download_third_parties_nix']
+
+                jobs.append({f"unittest_{os_type}_{device_type}": job})
+
+                if i == 0 and os_type == "linux" and device_type == "cpu":
+                    jobs.append({
+                        f"stylecheck": {
+                            "name": f"stylecheck_py{python_version}",
+                            "python_version": python_version,
+                        }
+                    })
+    return indent(indentation, jobs)
+
+
+if __name__ == "__main__":
+    d = os.path.dirname(__file__)
+    env = jinja2.Environment(
+        loader=jinja2.FileSystemLoader(d),
+        lstrip_blocks=True,
+        autoescape=False,
+    )
+
+    with open(os.path.join(d, 'config.yml'), 'w') as f:
+        f.write(env.get_template('config.yml.in').render(
+            build_workflows=build_workflows,
+            unittest_workflows=unittest_workflows,
+        ))

.circleci/config.yml.in

@@ -0,0 +1,36 @@
+# this Dockerfile is for torchaudio smoke test, it will be created periodically via CI system
+# if you need to do it locally, follow below steps once you have Docker installed
+# assuming you're within the directory where this Dockerfile located
+#  $ docker build . -t torchaudio/smoketest
+
+# if you want to push to aws ecr, make sure you have the rights to write to ECR, then run
+# $ eval $(aws ecr get-login --region us-east-1 --no-include-email)
+# $ export MYTAG=localbuild  ## you can choose whatever tag you like
+# $ docker tag torchaudio/smoketest 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchaudio/smoke_test:${MYTAG}
+# $ docker push  308535385114.dkr.ecr.us-east-1.amazonaws.com/torchaudio/smoke_test:${MYTAG}
+
+FROM ubuntu:latest
+
+RUN apt-get -qq update && apt-get -qq -y install curl bzip2 sox libsox-dev libsox-fmt-all \
+    && curl -sSL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh \
+    && bash /tmp/miniconda.sh -bfp /usr/local \
+    && rm -rf /tmp/miniconda.sh \
+    && conda install -y python=3 \
+    && conda update conda \
+    && apt-get -qq -y remove curl bzip2 \
+    && apt-get -qq -y autoremove \
+    && apt-get autoclean \
+    && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log \
+    && conda clean --all --yes
+
+ENV PATH /opt/conda/bin:$PATH
+
+RUN conda create -y --name python3.6 python=3.6
+RUN conda create -y --name python3.7 python=3.7
+RUN conda create -y --name python3.8 python=3.8
+SHELL [ "/bin/bash", "-c" ]
+RUN echo "source /usr/local/etc/profile.d/conda.sh" >> ~/.bashrc
+RUN source /usr/local/etc/profile.d/conda.sh && conda activate python3.6 && conda install -y -c conda-forge sox && conda install -y numpy
+RUN source /usr/local/etc/profile.d/conda.sh && conda activate python3.7 && conda install -y -c conda-forge sox && conda install -y numpy
+RUN source /usr/local/etc/profile.d/conda.sh && conda activate python3.8 && conda install -y -c conda-forge sox && conda install -y numpy
+CMD [ "/bin/bash"]

.circleci/regenerate.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+
+"""
+To compare new version with previous:
+
+    ./regenerate.sh
+    meld <(git show HEAD:./config.yml | ./sort-yaml.py) <(cat config.yml | ./sort-yaml.py)
+"""
+
+
+import sys
+import yaml
+
+sys.stdout.write(yaml.dump(yaml.load(sys.stdin, Loader=yaml.FullLoader), sort_keys=True))

.circleci/smoke_test/docker/Dockerfile

@@ -0,0 +1,6 @@
+This directory contains;
+
+ - docker
+   Docker image definition and scripts to build and update Docker image for unittest.
+ - scripts
+   Scripts used by CircleCI to run unit tests.

.circleci/test/test_sort_yaml.py

@@ -0,0 +1,2 @@
+*
+!scripts

.circleci/unittest/linux/README.md

@@ -0,0 +1,2 @@
+scripts/build_third_parties.sh
+Dockerfile.tmp

.circleci/unittest/linux/docker/.dockerignore

@@ -0,0 +1,59 @@
+FROM ubuntu:18.04 as builder
+
+RUN apt update -q
+
+################################################################################
+# Build Kaldi
+################################################################################
+RUN apt install -q -y \
+        autoconf \
+        automake \
+        bzip2 \
+        g++ \
+        gfortran \
+        git \
+        libatlas-base-dev \
+        libtool \
+        make \
+        python2.7 \
+        python3 \
+        sox \
+        subversion \
+        unzip \
+        wget \
+        zlib1g-dev
+
+# KALDI uses MKL as a default math library, but we are going to copy featbin binaries and dependent
+# shared libraries to the final image, so we use ATLAS, which is easy to reinstall in the final image.
+RUN git clone --depth 1 https://github.com/kaldi-asr/kaldi.git /opt/kaldi && \
+        cd /opt/kaldi/tools && \
+        make -j $(nproc) && \
+        cd /opt/kaldi/src && \
+        ./configure --shared --mathlib=ATLAS --use-cuda=no && \
+        make featbin -j $(nproc)
+
+# Copy featbins and dependent libraries
+ADD ./scripts /scripts
+RUN bash /scripts/copy_kaldi_executables.sh /opt/kaldi /kaldi
+
+################################################################################
+# Build the final image
+################################################################################
+FROM BASE_IMAGE
+RUN apt update && apt install -y \
+        g++ \
+        gfortran \
+        git \
+        libatlas3-base \
+        wget \
+        curl \
+        make \
+        file \
+        sox \
+        libsox-dev \
+        libsox-fmt-all \
+        cmake \
+        pkg-config \
+    && rm -rf /var/lib/apt/lists/*
+COPY --from=builder /kaldi /kaldi
+ENV PATH="${PATH}:/kaldi/bin" LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/kaldi/lib"

.circleci/unittest/linux/docker/.gitignore

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if [ $# -ne 1 ]; then
+    printf "Usage %s <CUDA_VERSION>\n\n" "$0"
+    exit 1
+fi
+
+if [ "$1" = "cpu" ]; then
+    base_image="ubuntu:18.04"
+    image="pytorch/torchaudio_unittest_base:manylinux"
+elif [[ "$1" =~ ^(9.2|10.1)$ ]]; then
+    base_image="nvidia/cuda:$1-runtime-ubuntu18.04"
+    image="pytorch/torchaudio_unittest_base:manylinux-cuda$1"
+else
+    printf "Unexpected <CUDA_VERSION> string: %s" "$1"
+    exit 1;
+fi
+
+cd "$( dirname "${BASH_SOURCE[0]}" )"
+
+# docker build also accepts reading from STDIN
+# but in that case, no context (other files) can be passed, so we write out Dockerfile
+sed "s|BASE_IMAGE|${base_image}|g" Dockerfile > Dockerfile.tmp
+docker build -t "${image}" -f Dockerfile.tmp .
+docker push "${image}"