Packaging improvements for nightlies (#198)

* Packaging improvements for nightlies

- Instructions for how to install stable and master binaries,
  without hardcoding wheel filenames
- Turn on Python 2 build (numba is only needed for testing)
- Make builds a little more hermetic
- Prefer building against nightly CPU-only torch build
- Don't install all of requirements.txt, it's not necessary
- Updates for OS X too, including portability issues

Signed-off-by: Edward Z. Yang <[email protected]>

Author: ezyang

README.md

@@ -49,20 +49,22 @@ Installation
 
 ### Binaries
 
-To install pip wheels of 0.2.0, select the appropriate pip wheel for
-your version of Python:
+To install the latest pip wheels, run:
 
 ```
-# Wheels for Python 2 are NOT supported
+pip install torchaudio -f https://download.pytorch.org/whl/torch_stable.html
+```
 
-# Python 3.5
-pip3 install http://download.pytorch.org/whl/torchaudio-0.2-cp35-cp35m-linux_x86_64.whl
+(If you do not have torch already installed, this will default to installing
+torch from PyPI. If you need a different torch configuration, preinstall torch
+before running this command.)
 
-# Python 3.6
-pip3 install http://download.pytorch.org/whl/torchaudio-0.2-cp36-cp36m-linux_x86_64.whl
+At the moment, there is no automated nightly build process, but we occasionally
+build nightlies based on PyTorch nightlies by hand following the instructions in
+[build_tools/packaging](build_tools/packaging).  To install the latest nightly, run:
 
-# Python 3.7
-pip3 install http://download.pytorch.org/whl/torchaudio-0.2-cp37-cp37m-linux_x86_64.whl
+```
+pip install torchaudio_nightly -f https://download.pytorch.org/whl/nightly/torch_nightly.html
 ```
 
 ### From Source

build_tools/packaging/wheel/build_from_source.sh

@@ -28,16 +28,21 @@ popd
 
 # build mad, statically
 pushd libmad-0.15.1b
+# See https://stackoverflow.com/a/12864879/23845
+sed -i.bak 's/-march=i486//' configure
 ./configure --disable-shared --enable-static --prefix="$PREFIX/audio/third_party/mad" CFLAGS=-fPIC CXXFLAGS=-fPIC \
     --with-pic --disable-debug --disable-dependency-tracking
 make -s -j && make install
 popd
 
 # build sox, statically
+# --without-png makes OS X build less hazardous; somehow the build
+# finds png and enables it.  We don't want it; we'd need to package
+# it statically if we do.
 pushd sox-14.4.2
 ./configure --disable-shared --enable-static --prefix="$PREFIX/audio/third_party/sox" \
     LDFLAGS="-L$PREFIX/audio/third_party/lame/lib -L$PREFIX/audio/third_party/flac/lib -L$PREFIX/audio/third_party/mad/lib" \
     CPPFLAGS="-I$PREFIX/audio/third_party/lame/include -I$PREFIX/audio/third_party/flac/include -I$PREFIX/audio/third_party/mad/include" \
-    --with-lame --with-flac --with-mad --without-oggvorbis --without-oss --without-sndfile CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking
+    --with-lame --with-flac --with-mad --without-png --without-oggvorbis --without-oss --without-sndfile CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking
 make -s -j && make install
 popd

build_tools/packaging/wheel/linux_manywheel.sh

@@ -25,21 +25,26 @@ export PREFIX="/tmp"
 cd /tmp/audio
 
 for PYDIR in "${python_installations[@]}"; do
-    # wheels for numba does not work with python 2.7
-    if [[ "$PYDIR" == "/opt/python/cp27-cp27m/" || "$PYDIR" == "/opt/python/cp27-cp27mu/" ]]; then
-      continue;
-    fi
     export PATH=$PYDIR/bin:$OLD_PATH
     pip install --upgrade pip
 
+    # For true hermetic builds, you ought to be constructing the docker
+    # from scratch each time.  But this makes things marginally safer if
+    # you aren't doing this.
+    pip uninstall -y torch || true
+    pip uninstall -y torch_nightly || true
+
     export TORCHAUDIO_PYTORCH_DEPENDENCY_NAME=torch_nightly
-    pip install torch_nightly -f https://download.pytorch.org/whl/nightly/torch_nightly.html
+    pip install torch_nightly -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
     # CPU/CUDA variants of PyTorch have ABI compatible PyTorch.  Therefore, we
-    # strip off the local package qualifier
+    # strip off the local package qualifier.  Also, we choose to build against
+    # the CPU build, because it takes less time to download.
     export TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION="$(pip show torch_nightly | grep ^Version: | sed 's/Version: \+//' | sed 's/+.\+//')"
     echo "Building against ${TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION}"
 
-    pip install -r requirements.txt
+    # NB: do not actually install requirements.txt; that is only needed for
+    # testing
+    pip install numpy future
     IS_WHEEL=1 python setup.py clean
     IS_WHEEL=1 python setup.py bdist_wheel
     mkdir -p $OUT_DIR

build_tools/packaging/wheel/osx_wheel.sh

@@ -44,14 +44,13 @@ do
     conda activate $env_name
 
     export TORCHAUDIO_PYTORCH_DEPENDENCY_NAME=torch_nightly
-    pip install torch_nightly -f https://download.pytorch.org/whl/nightly/torch_nightly.html
+    pip install torch_nightly -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
     # NB: OS X builds don't have local package qualifiers
-    export TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION="$(pip show torch_nightly | grep ^Version: | sed 's/Version: \+//')"
+    # NB: Don't use \+ here, it's not portable
+    export TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION="$(pip show torch_nightly | grep ^Version: | sed 's/Version:  *//')"
     echo "Building against ${TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION}"
 
-     # install torchaudio dependencies
-    pip install -r requirements.txt
-
+    pip install numpy future
     IS_WHEEL=1 python setup.py clean
     IS_WHEEL=1 python setup.py bdist_wheel
     mkdir -p $OUT_DIR