From 3a85b79c11afccee01e2f25b2ba30ccb774b14ba Mon Sep 17 00:00:00 2001 From: Eli Uriegas <1700823+seemethere@users.noreply.github.com> Date: Thu, 3 Dec 2020 11:37:44 -0800 Subject: [PATCH 1/7] Add Python 3.9 support (cherry picked from commit 1b7ee833f5b43337dfc27763aa2bd1fb93f8a34c) Signed-off-by: Eli Uriegas --- .circleci/config.yml | 249 +++++++++++++++++- .circleci/config.yml.in | 30 ++- .circleci/regenerate.py | 2 +- .circleci/unittest/linux/scripts/install.sh | 20 +- .circleci/unittest/linux/scripts/setup_env.sh | 6 + .../unittest/windows/scripts/environment.yml | 16 ++ .circleci/unittest/windows/scripts/install.sh | 8 +- .../unittest/windows/scripts/setup_env.sh | 6 + packaging/pkg_helpers.bash | 10 +- packaging/torchaudio/meta.yaml | 2 + setup.py | 1 + 11 files changed, 334 insertions(+), 16 deletions(-) create mode 100644 .circleci/unittest/windows/scripts/environment.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 72fa72cc3e..a6f00b6715 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,6 +45,16 @@ commands: name: Install pkg-config command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config wget # Disable brew auto update which is very slow + load_conda_channel_flags: + description: "Determines whether we need extra conda channels" + steps: + - run: + name: Adding CONDA_CHANNEL_FLAGS to BASH_ENV + command: | + CONDA_CHANNEL_FLAGS="" + if [[ "${PYTHON_VERSION}" = *3.9* ]]; then + echo "export CONDA_CHANNEL_FLAGS=-c=conda-forge" >> ${BASH_ENV} + fi binary_common: &binary_common parameters: @@ -117,7 +127,7 @@ jobs: binary_linux_wheel: <<: *binary_common docker: - - image: "pytorch/manylinux-cuda100" + - image: "pytorch/manylinux-cuda102" resource_class: 2xlarge+ steps: - checkout @@ -138,6 +148,7 @@ jobs: resource_class: 2xlarge+ steps: - checkout + - load_conda_channel_flags - attach_workspace: at: third_party - run: packaging/build_conda.sh @@ -155,6 +166,7 @@ jobs: steps: - checkout - install_build_tools_macos + - load_conda_channel_flags - attach_workspace: at: third_party - run: @@ -180,6 +192,7 @@ jobs: steps: - checkout - install_build_tools_macos + - load_conda_channel_flags - attach_workspace: at: third_party - run: @@ -202,6 +215,7 @@ jobs: name: windows-cpu steps: - checkout + - load_conda_channel_flags - run: name: build command: | @@ -221,6 +235,7 @@ jobs: name: windows-cpu steps: - checkout + - load_conda_channel_flags - run: name: build command: | @@ -278,6 +293,7 @@ jobs: - attach_workspace: at: ~/workspace - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -297,6 +313,7 @@ jobs: - attach_workspace: at: ~/workspace - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -317,6 +334,7 @@ jobs: - attach_workspace: at: ~/workspace - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -342,6 +360,7 @@ jobs: - attach_workspace: at: ~/workspace - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -395,6 +414,7 @@ jobs: - attach_workspace: at: third_party - designate_upload_channel + - load_conda_channel_flags - run: name: Setup command: .circleci/unittest/linux/scripts/setup_env.sh @@ -423,6 +443,7 @@ jobs: - attach_workspace: at: third_party - designate_upload_channel + - load_conda_channel_flags - run: name: Pull Docker image command: docker pull --quiet "${image_name}" @@ -431,7 +452,7 @@ jobs: command: docker run -t --gpus all -e PYTHON_VERSION -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh - run: name: Install torchaudio - command: docker run -t --gpus all -e UPLOAD_CHANNEL -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh + command: docker run -t --gpus all -e UPLOAD_CHANNEL -e CONDA_CHANNEL_FLAGS -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh - run: name: Run tests command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh @@ -448,6 +469,7 @@ jobs: steps: - checkout - designate_upload_channel + - load_conda_channel_flags - run: name: Setup command: .circleci/unittest/windows/scripts/setup_env.sh @@ -473,6 +495,7 @@ jobs: steps: - checkout - designate_upload_channel + - load_conda_channel_flags - run: name: Setup command: .circleci/unittest/windows/scripts/setup_env.sh @@ -496,6 +519,7 @@ jobs: steps: - checkout - install_build_tools_macos + - load_conda_channel_flags - attach_workspace: at: third_party - designate_upload_channel @@ -522,6 +546,7 @@ jobs: steps: - checkout - designate_upload_channel + - load_conda_channel_flags - run: name: Setup command: .circleci/unittest/linux/scripts/setup_env.sh @@ -538,6 +563,7 @@ jobs: - attach_workspace: at: ~/workspace - checkout + - load_conda_channel_flags - run: name: Install pytorch-audio command: .circleci/build_docs/install_wheels.sh @@ -610,6 +636,11 @@ workflows: python_version: '3.8' requires: - download_third_parties_nix + - binary_linux_wheel: + name: binary_linux_wheel_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix - binary_macos_wheel: name: binary_macos_wheel_py3.6 python_version: '3.6' @@ -625,6 +656,11 @@ workflows: python_version: '3.8' requires: - download_third_parties_nix + - binary_macos_wheel: + name: binary_macos_wheel_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix - binary_windows_wheel: name: binary_windows_wheel_py3.6 python_version: '3.6' @@ -634,6 +670,9 @@ workflows: - binary_windows_wheel: name: binary_windows_wheel_py3.8 python_version: '3.8' + - binary_windows_wheel: + name: binary_windows_wheel_py3.9 + python_version: '3.9' - binary_linux_conda: name: binary_linux_conda_py3.6 python_version: '3.6' @@ -649,6 +688,11 @@ workflows: python_version: '3.8' requires: - download_third_parties_nix + - binary_linux_conda: + name: binary_linux_conda_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix - binary_macos_conda: name: binary_macos_conda_py3.6 python_version: '3.6' @@ -664,6 +708,11 @@ workflows: python_version: '3.8' requires: - download_third_parties_nix + - binary_macos_conda: + name: binary_macos_conda_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix - binary_windows_conda: name: binary_windows_conda_py3.6 python_version: '3.6' @@ -673,6 +722,9 @@ workflows: - binary_windows_conda: name: binary_windows_conda_py3.8 python_version: '3.8' + - binary_windows_conda: + name: binary_windows_conda_py3.9 + python_version: '3.9' - build_docs: name: build_docs python_version: '3.8' @@ -717,6 +769,11 @@ workflows: python_version: '3.8' requires: - download_third_parties_nix + - unittest_linux_cpu: + name: unittest_linux_cpu_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix - unittest_linux_gpu: name: unittest_linux_gpu_py3.6 python_version: '3.6' @@ -732,6 +789,11 @@ workflows: python_version: '3.8' requires: - download_third_parties_nix + - unittest_linux_gpu: + name: unittest_linux_gpu_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix - unittest_windows_cpu: name: unittest_windows_cpu_py3.6 python_version: '3.6' @@ -741,6 +803,9 @@ workflows: - unittest_windows_cpu: name: unittest_windows_cpu_py3.8 python_version: '3.8' + - unittest_windows_cpu: + name: unittest_windows_cpu_py3.9 + python_version: '3.9' - unittest_windows_gpu: name: unittest_windows_gpu_py3.6 python_version: '3.6' @@ -750,6 +815,9 @@ workflows: - unittest_windows_gpu: name: unittest_windows_gpu_py3.8 python_version: '3.8' + - unittest_windows_gpu: + name: unittest_windows_gpu_py3.9 + python_version: '3.9' - unittest_macos_cpu: name: unittest_macos_cpu_py3.6 python_version: '3.6' @@ -765,6 +833,11 @@ workflows: python_version: '3.8' requires: - download_third_parties_nix + - unittest_macos_cpu: + name: unittest_macos_cpu_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix nightly: jobs: - circleci_consistency: @@ -878,6 +951,39 @@ workflows: python_version: '3.8' requires: - nightly_binary_linux_wheel_py3.8_upload + - binary_linux_wheel: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_wheel_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix + - binary_wheel_upload: + context: org-member + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_wheel_py3.9_upload + requires: + - nightly_binary_linux_wheel_py3.9 + - smoke_test_linux_pip: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_wheel_py3.9_smoke_test_pip + python_version: '3.9' + requires: + - nightly_binary_linux_wheel_py3.9_upload - binary_macos_wheel: filters: branches: @@ -944,6 +1050,28 @@ workflows: name: nightly_binary_macos_wheel_py3.8_upload requires: - nightly_binary_macos_wheel_py3.8 + - binary_macos_wheel: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_macos_wheel_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix + - binary_wheel_upload: + context: org-member + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_macos_wheel_py3.9_upload + requires: + - nightly_binary_macos_wheel_py3.9 - binary_windows_wheel: filters: branches: @@ -1037,6 +1165,37 @@ workflows: python_version: '3.8' requires: - nightly_binary_windows_wheel_py3.8_upload + - binary_windows_wheel: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_wheel_py3.9 + python_version: '3.9' + - binary_wheel_upload: + context: org-member + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_wheel_py3.9_upload + requires: + - nightly_binary_windows_wheel_py3.9 + - smoke_test_windows_pip: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_wheel_py3.9_smoke_test_pip + python_version: '3.9' + requires: + - nightly_binary_windows_wheel_py3.9_upload - binary_linux_conda: filters: branches: @@ -1136,6 +1295,39 @@ workflows: python_version: '3.8' requires: - nightly_binary_linux_conda_py3.8_upload + - binary_linux_conda: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_conda_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix + - binary_conda_upload: + context: org-member + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_conda_py3.9_upload + requires: + - nightly_binary_linux_conda_py3.9 + - smoke_test_linux_conda: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_conda_py3.9_smoke_test_conda + python_version: '3.9' + requires: + - nightly_binary_linux_conda_py3.9_upload - binary_macos_conda: filters: branches: @@ -1202,6 +1394,28 @@ workflows: name: nightly_binary_macos_conda_py3.8_upload requires: - nightly_binary_macos_conda_py3.8 + - binary_macos_conda: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_macos_conda_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix + - binary_conda_upload: + context: org-member + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_macos_conda_py3.9_upload + requires: + - nightly_binary_macos_conda_py3.9 - binary_windows_conda: filters: branches: @@ -1295,6 +1509,37 @@ workflows: python_version: '3.8' requires: - nightly_binary_windows_conda_py3.8_upload + - binary_windows_conda: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_conda_py3.9 + python_version: '3.9' + - binary_conda_upload: + context: org-member + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_conda_py3.9_upload + requires: + - nightly_binary_windows_conda_py3.9 + - smoke_test_windows_conda: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_conda_py3.9_smoke_test_conda + python_version: '3.9' + requires: + - nightly_binary_windows_conda_py3.9_upload docker_build: triggers: - schedule: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index a0199d9ff8..d5734c6d65 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -45,6 +45,16 @@ commands: name: Install pkg-config command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config wget # Disable brew auto update which is very slow + load_conda_channel_flags: + description: "Determines whether we need extra conda channels" + steps: + - run: + name: Adding CONDA_CHANNEL_FLAGS to BASH_ENV + command: | + CONDA_CHANNEL_FLAGS="" + if [[ "${PYTHON_VERSION}" = *3.9* ]]; then + echo "export CONDA_CHANNEL_FLAGS=-c=conda-forge" >> ${BASH_ENV} + fi binary_common: &binary_common parameters: @@ -117,7 +127,7 @@ jobs: binary_linux_wheel: <<: *binary_common docker: - - image: "pytorch/manylinux-cuda100" + - image: "pytorch/manylinux-cuda102" resource_class: 2xlarge+ steps: - checkout @@ -138,6 +148,7 @@ jobs: resource_class: 2xlarge+ steps: - checkout + - load_conda_channel_flags - attach_workspace: at: third_party - run: packaging/build_conda.sh @@ -155,6 +166,7 @@ jobs: steps: - checkout - install_build_tools_macos + - load_conda_channel_flags - attach_workspace: at: third_party - run: @@ -180,6 +192,7 @@ jobs: steps: - checkout - install_build_tools_macos + - load_conda_channel_flags - attach_workspace: at: third_party - run: @@ -202,6 +215,7 @@ jobs: name: windows-cpu steps: - checkout + - load_conda_channel_flags - run: name: build command: | @@ -221,6 +235,7 @@ jobs: name: windows-cpu steps: - checkout + - load_conda_channel_flags - run: name: build command: | @@ -278,6 +293,7 @@ jobs: - attach_workspace: at: ~/workspace - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -297,6 +313,7 @@ jobs: - attach_workspace: at: ~/workspace - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -317,6 +334,7 @@ jobs: - attach_workspace: at: ~/workspace - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -342,6 +360,7 @@ jobs: - attach_workspace: at: ~/workspace - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -395,6 +414,7 @@ jobs: - attach_workspace: at: third_party - designate_upload_channel + - load_conda_channel_flags - run: name: Setup command: .circleci/unittest/linux/scripts/setup_env.sh @@ -423,6 +443,7 @@ jobs: - attach_workspace: at: third_party - designate_upload_channel + - load_conda_channel_flags - run: name: Pull Docker image command: docker pull --quiet "${image_name}" @@ -431,7 +452,7 @@ jobs: command: docker run -t --gpus all -e PYTHON_VERSION -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh - run: name: Install torchaudio - command: docker run -t --gpus all -e UPLOAD_CHANNEL -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh + command: docker run -t --gpus all -e UPLOAD_CHANNEL -e CONDA_CHANNEL_FLAGS -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh - run: name: Run tests command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh @@ -448,6 +469,7 @@ jobs: steps: - checkout - designate_upload_channel + - load_conda_channel_flags - run: name: Setup command: .circleci/unittest/windows/scripts/setup_env.sh @@ -473,6 +495,7 @@ jobs: steps: - checkout - designate_upload_channel + - load_conda_channel_flags - run: name: Setup command: .circleci/unittest/windows/scripts/setup_env.sh @@ -496,6 +519,7 @@ jobs: steps: - checkout - install_build_tools_macos + - load_conda_channel_flags - attach_workspace: at: third_party - designate_upload_channel @@ -522,6 +546,7 @@ jobs: steps: - checkout - designate_upload_channel + - load_conda_channel_flags - run: name: Setup command: .circleci/unittest/linux/scripts/setup_env.sh @@ -538,6 +563,7 @@ jobs: - attach_workspace: at: ~/workspace - checkout + - load_conda_channel_flags - run: name: Install pytorch-audio command: .circleci/build_docs/install_wheels.sh diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 588eba54be..0985c1f9f6 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -19,7 +19,7 @@ import os.path -PYTHON_VERSIONS = ["3.6", "3.7", "3.8"] +PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] DOC_VERSION = ('linux', '3.8') diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index 209c86be71..9b4d6bfaa6 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -34,7 +34,10 @@ else cudatoolkit="cudatoolkit=${version}" fi printf "Installing PyTorch with %s\n" "${cudatoolkit}" -conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch ${cudatoolkit} +( + set -x + conda install ${CONDA_CHANNEL_FLAGS:-} -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch ${cudatoolkit} +) # 2. Install torchaudio printf "* Installing torchaudio\n" @@ -43,11 +46,12 @@ BUILD_TRANSDUCER=1 BUILD_SOX=1 python setup.py install # 3. Install Test tools printf "* Installing test tools\n" -if [ "${os}" == Linux ] ; then - conda install -y -c conda-forge codecov pytest pytest-cov - pip install kaldi-io 'librosa>=0.8.0' parameterized SoundFile scipy 'requests>=2.20' -else - # Note: installing librosa via pip fail because it will try to compile numba. - conda install -y -c conda-forge codecov pytest pytest-cov 'librosa>=0.8.0' parameterized scipy - pip install kaldi-io SoundFile 'requests>=2.20' +NUMBA_DEV_CHANNEL="" +if [[ "${PYTORCH_VERSION}" = 3.9 ]]; then + # Numba isn't available for Python 3.9 except on the numba dev channel and building from source fails + # See https://github.com/librosa/librosa/issues/1270#issuecomment-759065048 + NUMBA_DEV_CHANNEL="-c numba/label/dev" fi +# Note: installing librosa via pip fail because it will try to compile numba. +conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} codecov pytest pytest-cov 'librosa>=0.8.0' parameterized scipy +pip install kaldi-io SoundFile 'requests>=2.20' diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh index 085dc61a5b..175ca2f795 100755 --- a/.circleci/unittest/linux/scripts/setup_env.sh +++ b/.circleci/unittest/linux/scripts/setup_env.sh @@ -38,6 +38,12 @@ if [ ! -d "${env_dir}" ]; then conda create --prefix "${env_dir}" -y python="${PYTHON_VERSION}" fi conda activate "${env_dir}" +NUMPY_PIN="1.11" +if [[ "${PYTHON_VERSION}" = "3.9" ]]; then + NUMPY_PIN="1.20" +fi +printf "* Installing numpy>=%s\n" "${NUMPY_PIN}\n" +conda install -y -c conda-forge "numpy>=${NUMPY_PIN}" # 3. Install minimal build tools pip --quiet install cmake ninja diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml new file mode 100644 index 0000000000..27322ccc01 --- /dev/null +++ b/.circleci/unittest/windows/scripts/environment.yml @@ -0,0 +1,16 @@ +channels: + - conda-forge + - defaults +dependencies: + - flake8 + - pytest + - pytest-cov + - codecov + - scipy >= 1.4.1 + - pip + - pip: + - kaldi-io + - PySoundFile + - future + - parameterized + - dataclasses diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh index 901cb40450..75ebb067a9 100644 --- a/.circleci/unittest/windows/scripts/install.sh +++ b/.circleci/unittest/windows/scripts/install.sh @@ -25,7 +25,13 @@ else cudatoolkit="cudatoolkit=${version}" fi printf "Installing PyTorch with %s\n" "${cudatoolkit}" -conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch "${cudatoolkit}" +conda install ${CONDA_CHANNEL_FLAGS:-} -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch "${cudatoolkit}" + +# TODO: Remove this after packages become available +# Currently there's no librosa package available for Python 3.9, so lets just skip the dependency for now +if [[ $(python --version) != *3.9* ]]; then + pip install 'librosa>=0.8.0' +fi # 2. Install torchaudio printf "* Installing torchaudio\n" diff --git a/.circleci/unittest/windows/scripts/setup_env.sh b/.circleci/unittest/windows/scripts/setup_env.sh index 929a98cb8f..15062cacd7 100644 --- a/.circleci/unittest/windows/scripts/setup_env.sh +++ b/.circleci/unittest/windows/scripts/setup_env.sh @@ -37,3 +37,9 @@ if [ ! -d "${env_dir}" ]; then conda create --prefix "${env_dir}" -y python="${PYTHON_VERSION}" fi conda activate "${env_dir}" +NUMPY_PIN="1.11" +if [[ "${PYTHON_VERSION}" = "3.9" ]]; then + NUMPY_PIN="1.20" +fi +printf "* Installing numpy>=%s\n" "${NUMPY_PIN}\n" +conda install -y -c conda-forge numpy>="${NUMPY_PIN}" diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 36760b9484..209e145282 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -141,6 +141,7 @@ setup_wheel_python() { 3.6) python_abi=cp36-cp36m ;; 3.7) python_abi=cp37-cp37m ;; 3.8) python_abi=cp38-cp38 ;; + 3.9) python_abi=cp39-cp39 ;; *) echo "Unrecognized PYTHON_VERSION=$PYTHON_VERSION" exit 1 @@ -176,11 +177,16 @@ setup_pip_pytorch_version() { # # You MUST have populated PYTORCH_VERSION_SUFFIX before hand. setup_conda_pytorch_constraint() { + CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS}" if [[ -z "$PYTORCH_VERSION" ]]; then - export CONDA_CHANNEL_FLAGS="-c pytorch-nightly" + export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c pytorch-nightly" export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | python -c "import sys, json, re; print(re.sub(r'\\+.*$', '', json.load(sys.stdin)['pytorch'][-1]['version']))")" else - export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-${UPLOAD_CHANNEL}" + export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c pytorch -c pytorch-test -c pytorch-nightly" + fi + # Some dependencies for Python 3.9 are only on conda-forge + if [[ "${PYTHON_VERSION}" = "3.9" ]]; then + export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c conda-forge" fi if [[ "$CU_VERSION" == cpu ]]; then export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${PYTORCH_VERSION_SUFFIX}" diff --git a/packaging/torchaudio/meta.yaml b/packaging/torchaudio/meta.yaml index ad12821d0e..5f9e485eba 100644 --- a/packaging/torchaudio/meta.yaml +++ b/packaging/torchaudio/meta.yaml @@ -16,10 +16,12 @@ requirements: - cpuonly - cmake - ninja + - defaults::numpy >=1.11 {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT') }} run: - python + - defaults::numpy >=1.11 {{ environ.get('CONDA_PYTORCH_CONSTRAINT') }} build: diff --git a/setup.py b/setup.py index 1ee56dfcb7..459ff5e19e 100644 --- a/setup.py +++ b/setup.py @@ -76,6 +76,7 @@ def run(self): "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Multimedia :: Sound/Audio", "Topic :: Scientific/Engineering :: Artificial Intelligence" From 07452d6030c4ef2912b9a8c1d30b14844edc43ab Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Thu, 18 Feb 2021 17:40:56 -0800 Subject: [PATCH 2/7] remove numpy pin, install defaults numpy Signed-off-by: Eli Uriegas --- .circleci/unittest/linux/scripts/install.sh | 5 +++-- .circleci/unittest/linux/scripts/setup_env.sh | 6 ------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index 9b4d6bfaa6..b2f8417fcc 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -36,6 +36,7 @@ fi printf "Installing PyTorch with %s\n" "${cudatoolkit}" ( set -x + conda install ${CONDA_CHANNEL_FLAGS:-} -y defaults::numpy conda install ${CONDA_CHANNEL_FLAGS:-} -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch ${cudatoolkit} ) @@ -53,5 +54,5 @@ if [[ "${PYTORCH_VERSION}" = 3.9 ]]; then NUMBA_DEV_CHANNEL="-c numba/label/dev" fi # Note: installing librosa via pip fail because it will try to compile numba. -conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} codecov pytest pytest-cov 'librosa>=0.8.0' parameterized scipy -pip install kaldi-io SoundFile 'requests>=2.20' +conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} 'librosa>=0.8.0' parameterized scipy +pip install kaldi-io SoundFile 'requests>=2.20' codecov pytest pytest-cov diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh index 175ca2f795..085dc61a5b 100755 --- a/.circleci/unittest/linux/scripts/setup_env.sh +++ b/.circleci/unittest/linux/scripts/setup_env.sh @@ -38,12 +38,6 @@ if [ ! -d "${env_dir}" ]; then conda create --prefix "${env_dir}" -y python="${PYTHON_VERSION}" fi conda activate "${env_dir}" -NUMPY_PIN="1.11" -if [[ "${PYTHON_VERSION}" = "3.9" ]]; then - NUMPY_PIN="1.20" -fi -printf "* Installing numpy>=%s\n" "${NUMPY_PIN}\n" -conda install -y -c conda-forge "numpy>=${NUMPY_PIN}" # 3. Install minimal build tools pip --quiet install cmake ninja From 905d023ed26dcf120670421f59b9457cce0533c2 Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Thu, 18 Feb 2021 17:46:01 -0800 Subject: [PATCH 3/7] ensure we're using the right env variable Signed-off-by: Eli Uriegas --- .circleci/unittest/linux/scripts/install.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index b2f8417fcc..24636c4235 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -48,11 +48,14 @@ BUILD_TRANSDUCER=1 BUILD_SOX=1 python setup.py install # 3. Install Test tools printf "* Installing test tools\n" NUMBA_DEV_CHANNEL="" -if [[ "${PYTORCH_VERSION}" = 3.9 ]]; then +if [[ "${PYTHON_VERSION}" = 3.9 ]]; then # Numba isn't available for Python 3.9 except on the numba dev channel and building from source fails # See https://github.com/librosa/librosa/issues/1270#issuecomment-759065048 NUMBA_DEV_CHANNEL="-c numba/label/dev" fi # Note: installing librosa via pip fail because it will try to compile numba. -conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} 'librosa>=0.8.0' parameterized scipy -pip install kaldi-io SoundFile 'requests>=2.20' codecov pytest pytest-cov +( + set -x + conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} 'librosa>=0.8.0' parameterized 'requests>=2.20' + pip install kaldi-io SoundFile codecov pytest pytest-cov scipy +) From 7c1e79d2b2e1428550b904f580744aa937b532be Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Thu, 18 Feb 2021 17:50:26 -0800 Subject: [PATCH 4/7] do the same thing for windows Signed-off-by: Eli Uriegas --- .circleci/unittest/windows/scripts/install.sh | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh index 75ebb067a9..775b055363 100644 --- a/.circleci/unittest/windows/scripts/install.sh +++ b/.circleci/unittest/windows/scripts/install.sh @@ -27,17 +27,21 @@ fi printf "Installing PyTorch with %s\n" "${cudatoolkit}" conda install ${CONDA_CHANNEL_FLAGS:-} -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch "${cudatoolkit}" -# TODO: Remove this after packages become available -# Currently there's no librosa package available for Python 3.9, so lets just skip the dependency for now -if [[ $(python --version) != *3.9* ]]; then - pip install 'librosa>=0.8.0' -fi - # 2. Install torchaudio printf "* Installing torchaudio\n" python setup.py install # 3. Install Test tools printf "* Installing test tools\n" -conda install -y -c conda-forge codecov pytest pytest-cov -pip install kaldi-io 'librosa>=0.8.0' parameterized PySoundFile scipy +NUMBA_DEV_CHANNEL="" +if [[ "${PYTHON_VERSION}" = 3.9 ]]; then + # Numba isn't available for Python 3.9 except on the numba dev channel and building from source fails + # See https://github.com/librosa/librosa/issues/1270#issuecomment-759065048 + NUMBA_DEV_CHANNEL="-c numba/label/dev" +fi +# Note: installing librosa via pip fail because it will try to compile numba. +( + set -x + conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} 'librosa>=0.8.0' parameterized 'requests>=2.20' + pip install kaldi-io SoundFile codecov pytest pytest-cov scipy +) From c8e7cdf028d8155980127cdce005cdd2024e5957 Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Fri, 19 Feb 2021 10:17:20 -0800 Subject: [PATCH 5/7] derive python version from actual python Signed-off-by: Eli Uriegas --- .circleci/unittest/linux/scripts/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index 24636c4235..f2455d3c70 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -48,7 +48,7 @@ BUILD_TRANSDUCER=1 BUILD_SOX=1 python setup.py install # 3. Install Test tools printf "* Installing test tools\n" NUMBA_DEV_CHANNEL="" -if [[ "${PYTHON_VERSION}" = 3.9 ]]; then +if [[ "$(python --version)" =~ *3.9* ]]; then # Numba isn't available for Python 3.9 except on the numba dev channel and building from source fails # See https://github.com/librosa/librosa/issues/1270#issuecomment-759065048 NUMBA_DEV_CHANNEL="-c numba/label/dev" From 4f105ed0174ebca5e3df3f89ada93117b380daa5 Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Fri, 19 Feb 2021 10:28:30 -0800 Subject: [PATCH 6/7] do python check with python, just regular comparison Signed-off-by: Eli Uriegas --- .circleci/unittest/linux/scripts/install.sh | 2 +- .circleci/unittest/windows/scripts/install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index f2455d3c70..0ca69a7844 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -48,7 +48,7 @@ BUILD_TRANSDUCER=1 BUILD_SOX=1 python setup.py install # 3. Install Test tools printf "* Installing test tools\n" NUMBA_DEV_CHANNEL="" -if [[ "$(python --version)" =~ *3.9* ]]; then +if [[ "$(python --version)" = *3.9* ]]; then # Numba isn't available for Python 3.9 except on the numba dev channel and building from source fails # See https://github.com/librosa/librosa/issues/1270#issuecomment-759065048 NUMBA_DEV_CHANNEL="-c numba/label/dev" diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh index 775b055363..cf531b47e5 100644 --- a/.circleci/unittest/windows/scripts/install.sh +++ b/.circleci/unittest/windows/scripts/install.sh @@ -34,7 +34,7 @@ python setup.py install # 3. Install Test tools printf "* Installing test tools\n" NUMBA_DEV_CHANNEL="" -if [[ "${PYTHON_VERSION}" = 3.9 ]]; then +if [[ "$(python --version)" = *3.9* ]]; then # Numba isn't available for Python 3.9 except on the numba dev channel and building from source fails # See https://github.com/librosa/librosa/issues/1270#issuecomment-759065048 NUMBA_DEV_CHANNEL="-c numba/label/dev" From a3c4e704622f19d3750c6bb952a52675495cbfc3 Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Fri, 19 Feb 2021 11:01:52 -0800 Subject: [PATCH 7/7] ensure we're installing pytorch from the correct channel conda channel priorities strike again, they're so unpredictable Signed-off-by: Eli Uriegas --- .circleci/unittest/linux/scripts/install.sh | 2 +- .circleci/unittest/windows/scripts/install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index 0ca69a7844..3dc735e8d8 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -37,7 +37,7 @@ printf "Installing PyTorch with %s\n" "${cudatoolkit}" ( set -x conda install ${CONDA_CHANNEL_FLAGS:-} -y defaults::numpy - conda install ${CONDA_CHANNEL_FLAGS:-} -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch ${cudatoolkit} + conda install ${CONDA_CHANNEL_FLAGS:-} -y -c "pytorch-${UPLOAD_CHANNEL}" "pytorch-${UPLOAD_CHANNEL}::pytorch" ${cudatoolkit} ) # 2. Install torchaudio diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh index cf531b47e5..4be877b730 100644 --- a/.circleci/unittest/windows/scripts/install.sh +++ b/.circleci/unittest/windows/scripts/install.sh @@ -25,7 +25,7 @@ else cudatoolkit="cudatoolkit=${version}" fi printf "Installing PyTorch with %s\n" "${cudatoolkit}" -conda install ${CONDA_CHANNEL_FLAGS:-} -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch "${cudatoolkit}" +conda install ${CONDA_CHANNEL_FLAGS:-} -y -c "pytorch-${UPLOAD_CHANNEL}" "pytorch-${UPLOAD_CHANNEL}::pytorch" ${cudatoolkit} # 2. Install torchaudio printf "* Installing torchaudio\n"