From e5f3c22e174a864b14d3e7ff379af6d71f8e92e6 Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Tue, 1 Dec 2020 14:55:47 -0800 Subject: [PATCH 01/12] .circleci: Add Python 3.9 support Signed-off-by: Eli Uriegas --- .circleci/config.yml | 233 ++++++++++++++++++++++++++++++++++++++++ .circleci/regenerate.py | 2 +- 2 files changed, 234 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fefff2523a..8a82712fb4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -655,6 +655,11 @@ workflows: python_version: '3.8' requires: - download_third_parties_nix + - binary_linux_wheel: + name: binary_linux_wheel_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix - binary_macos_wheel: name: binary_macos_wheel_py3.6 python_version: '3.6' @@ -670,6 +675,11 @@ workflows: python_version: '3.8' requires: - download_third_parties_nix + - binary_macos_wheel: + name: binary_macos_wheel_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix - binary_windows_wheel: name: binary_windows_wheel_py3.6 python_version: '3.6' @@ -679,6 +689,9 @@ workflows: - binary_windows_wheel: name: binary_windows_wheel_py3.8 python_version: '3.8' + - binary_windows_wheel: + name: binary_windows_wheel_py3.9 + python_version: '3.9' - binary_linux_conda: name: binary_linux_conda_py3.6 python_version: '3.6' @@ -694,6 +707,11 @@ workflows: python_version: '3.8' requires: - download_third_parties_nix + - binary_linux_conda: + name: binary_linux_conda_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix - binary_macos_conda: name: binary_macos_conda_py3.6 python_version: '3.6' @@ -709,6 +727,11 @@ workflows: python_version: '3.8' requires: - download_third_parties_nix + - binary_macos_conda: + name: binary_macos_conda_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix - binary_windows_conda: name: binary_windows_conda_py3.6 python_version: '3.6' @@ -718,6 +741,9 @@ workflows: - binary_windows_conda: name: binary_windows_conda_py3.8 python_version: '3.8' + - binary_windows_conda: + name: binary_windows_conda_py3.9 + python_version: '3.9' unittest: jobs: - torchscript_bc_test: @@ -743,6 +769,11 @@ workflows: python_version: '3.8' requires: - download_third_parties_nix + - unittest_linux_cpu: + name: unittest_linux_cpu_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix - unittest_linux_gpu: filters: branches: @@ -779,6 +810,18 @@ workflows: python_version: '3.8' requires: - download_third_parties_nix + - unittest_linux_gpu: + filters: + branches: + only: + - master + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: unittest_linux_gpu_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix - unittest_windows_cpu: name: unittest_windows_cpu_py3.6 python_version: '3.6' @@ -788,6 +831,9 @@ workflows: - unittest_windows_cpu: name: unittest_windows_cpu_py3.8 python_version: '3.8' + - unittest_windows_cpu: + name: unittest_windows_cpu_py3.9 + python_version: '3.9' - unittest_windows_gpu: filters: branches: @@ -818,6 +864,16 @@ workflows: only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ name: unittest_windows_gpu_py3.8 python_version: '3.8' + - unittest_windows_gpu: + filters: + branches: + only: + - master + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: unittest_windows_gpu_py3.9 + python_version: '3.9' - unittest_macos_cpu: name: unittest_macos_cpu_py3.6 python_version: '3.6' @@ -833,6 +889,11 @@ workflows: python_version: '3.8' requires: - download_third_parties_nix + - unittest_macos_cpu: + name: unittest_macos_cpu_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix nightly: jobs: - circleci_consistency: @@ -946,6 +1007,39 @@ workflows: python_version: '3.8' requires: - nightly_binary_linux_wheel_py3.8_upload + - binary_linux_wheel: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_wheel_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix + - binary_wheel_upload: + context: org-member + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_wheel_py3.9_upload + requires: + - nightly_binary_linux_wheel_py3.9 + - smoke_test_linux_pip: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_wheel_py3.9_smoke_test_pip + python_version: '3.9' + requires: + - nightly_binary_linux_wheel_py3.9_upload - binary_macos_wheel: filters: branches: @@ -1012,6 +1106,28 @@ workflows: name: nightly_binary_macos_wheel_py3.8_upload requires: - nightly_binary_macos_wheel_py3.8 + - binary_macos_wheel: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_macos_wheel_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix + - binary_wheel_upload: + context: org-member + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_macos_wheel_py3.9_upload + requires: + - nightly_binary_macos_wheel_py3.9 - binary_windows_wheel: filters: branches: @@ -1105,6 +1221,37 @@ workflows: python_version: '3.8' requires: - nightly_binary_windows_wheel_py3.8_upload + - binary_windows_wheel: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_wheel_py3.9 + python_version: '3.9' + - binary_wheel_upload: + context: org-member + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_wheel_py3.9_upload + requires: + - nightly_binary_windows_wheel_py3.9 + - smoke_test_windows_pip: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_wheel_py3.9_smoke_test_pip + python_version: '3.9' + requires: + - nightly_binary_windows_wheel_py3.9_upload - binary_linux_conda: filters: branches: @@ -1204,6 +1351,39 @@ workflows: python_version: '3.8' requires: - nightly_binary_linux_conda_py3.8_upload + - binary_linux_conda: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_conda_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix + - binary_conda_upload: + context: org-member + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_conda_py3.9_upload + requires: + - nightly_binary_linux_conda_py3.9 + - smoke_test_linux_conda: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_conda_py3.9_smoke_test_conda + python_version: '3.9' + requires: + - nightly_binary_linux_conda_py3.9_upload - binary_macos_conda: filters: branches: @@ -1270,6 +1450,28 @@ workflows: name: nightly_binary_macos_conda_py3.8_upload requires: - nightly_binary_macos_conda_py3.8 + - binary_macos_conda: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_macos_conda_py3.9 + python_version: '3.9' + requires: + - download_third_parties_nix + - binary_conda_upload: + context: org-member + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_macos_conda_py3.9_upload + requires: + - nightly_binary_macos_conda_py3.9 - binary_windows_conda: filters: branches: @@ -1363,6 +1565,37 @@ workflows: python_version: '3.8' requires: - nightly_binary_windows_conda_py3.8_upload + - binary_windows_conda: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_conda_py3.9 + python_version: '3.9' + - binary_conda_upload: + context: org-member + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_conda_py3.9_upload + requires: + - nightly_binary_windows_conda_py3.9 + - smoke_test_windows_conda: + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_conda_py3.9_smoke_test_conda + python_version: '3.9' + requires: + - nightly_binary_windows_conda_py3.9_upload docker_build: triggers: - schedule: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 2c9fe7ee4f..85b01e6b72 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -19,7 +19,7 @@ import os.path -PYTHON_VERSIONS = ["3.6", "3.7", "3.8"] +PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] def build_workflows(prefix='', upload=False, filter_branch=None, indentation=6): From 6d2b3b3f0b7d6e214d9b0e0c9af4b07b4320fab2 Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Tue, 1 Dec 2020 14:57:14 -0800 Subject: [PATCH 02/12] bump versions Signed-off-by: Eli Uriegas --- .circleci/config.yml.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 6aa6d9738f..12b942156b 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -52,11 +52,11 @@ binary_common: &binary_common build_version: description: "version number of release binary; by default, build a nightly" type: string - default: "0.7.0" + default: "0.7.1" pytorch_version: description: "PyTorch version to build against; by default, use a nightly" type: string - default: "1.7.0" + default: "1.7.1" # Don't edit these python_version: description: "Python version to build against (e.g., 3.8)" From d6ad9805f6a51a524df23121e1816b13bb1e7842 Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Tue, 1 Dec 2020 14:59:59 -0800 Subject: [PATCH 03/12] add python 3.9 Signed-off-by: Eli Uriegas --- packaging/pkg_helpers.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 57b23b93d3..cd1455cb19 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -140,6 +140,7 @@ setup_wheel_python() { 3.6) python_abi=cp36-cp36m ;; 3.7) python_abi=cp37-cp37m ;; 3.8) python_abi=cp38-cp38 ;; + 3.8) python_abi=cp39-cp39 ;; *) echo "Unrecognized PYTHON_VERSION=$PYTHON_VERSION" exit 1 From e4438b099cb37bf0eb100464e2336708b5622d3f Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Tue, 1 Dec 2020 15:03:41 -0800 Subject: [PATCH 04/12] regenerate config.yml Signed-off-by: Eli Uriegas --- .circleci/config.yml | 6 +++--- .circleci/config.yml.in | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8a82712fb4..12c685012e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,11 +52,11 @@ binary_common: &binary_common build_version: description: "version number of release binary; by default, build a nightly" type: string - default: "0.7.0" + default: "0.7.1" pytorch_version: description: "PyTorch version to build against; by default, use a nightly" type: string - default: "1.7.0" + default: "1.7.1" # Don't edit these python_version: description: "Python version to build against (e.g., 3.8)" @@ -117,7 +117,7 @@ jobs: binary_linux_wheel: <<: *binary_common docker: - - image: "pytorch/manylinux-cuda100" + - image: "pytorch/manylinux-cuda102" resource_class: 2xlarge+ steps: - checkout diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 12b942156b..53812328e1 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -117,7 +117,7 @@ jobs: binary_linux_wheel: <<: *binary_common docker: - - image: "pytorch/manylinux-cuda100" + - image: "pytorch/manylinux-cuda102" resource_class: 2xlarge+ steps: - checkout From 7b0fb14e8c45026cf8e7653dced0454ab6924b1c Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Tue, 1 Dec 2020 15:09:47 -0800 Subject: [PATCH 05/12] add CONDA_CHANNEL_FLAGS command Signed-off-by: Eli Uriegas --- .circleci/config.yml | 26 ++++++++++++++++++++++++++ .circleci/config.yml.in | 26 ++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 12c685012e..de7154fe38 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,6 +45,16 @@ commands: name: Install cmake and pkg-config command: HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake pkg-config wget # Disable brew auto update which is very slow + load_conda_channel_flags: + description: "Determines whether we need extra conda channels" + steps: + - run: + name: Adding CONDA_CHANNEL_FLAGS to BASH_ENV + command: | + CONDA_CHANNEL_FLAGS="" + if [[ "${PYTHON_VERSION}" = *3.9* ]]; then + echo "export CONDA_CHANNEL_FLAGS=-c=conda-forge" >> ${BASH_ENV} + fi binary_common: &binary_common parameters: @@ -138,6 +148,7 @@ jobs: resource_class: 2xlarge+ steps: - checkout + - load_conda_channel_flags - attach_workspace: at: third_party - run: packaging/build_conda.sh @@ -155,6 +166,7 @@ jobs: steps: - checkout - install_build_tools_macos + - load_conda_channel_flags - attach_workspace: at: third_party - run: @@ -180,6 +192,7 @@ jobs: steps: - checkout - install_build_tools_macos + - load_conda_channel_flags - attach_workspace: at: third_party - run: @@ -202,6 +215,7 @@ jobs: name: windows-cpu steps: - checkout + - load_conda_channel_flags - run: name: build command: | @@ -221,6 +235,7 @@ jobs: name: windows-cpu steps: - checkout + - load_conda_channel_flags - run: name: build command: | @@ -278,6 +293,7 @@ jobs: - attach_workspace: at: ~/workspace - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -297,6 +313,7 @@ jobs: - attach_workspace: at: ~/workspace - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -317,6 +334,7 @@ jobs: - attach_workspace: at: ~/workspace - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -342,6 +360,7 @@ jobs: - attach_workspace: at: ~/workspace - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -395,6 +414,7 @@ jobs: - attach_workspace: at: third_party - designate_upload_channel + - load_conda_channel_flags - generate_cache_key - restore_cache: @@ -438,6 +458,7 @@ jobs: - attach_workspace: at: third_party - designate_upload_channel + - load_conda_channel_flags - generate_cache_key - restore_cache: @@ -476,6 +497,7 @@ jobs: steps: - checkout - designate_upload_channel + - load_conda_channel_flags - generate_cache_key - restore_cache: @@ -513,6 +535,7 @@ jobs: steps: - checkout - designate_upload_channel + - load_conda_channel_flags - generate_cache_key - restore_cache: @@ -549,6 +572,7 @@ jobs: steps: - checkout - install_build_tools_macos + - load_conda_channel_flags - attach_workspace: at: third_party - designate_upload_channel @@ -575,6 +599,7 @@ jobs: steps: - checkout - designate_upload_channel + - load_conda_channel_flags - generate_cache_key - restore_cache: @@ -605,6 +630,7 @@ jobs: steps: - checkout - generate_cache_key + - load_conda_channel_flags - restore_cache: keys: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 53812328e1..d188fe4477 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -45,6 +45,16 @@ commands: name: Install cmake and pkg-config command: HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake pkg-config wget # Disable brew auto update which is very slow + load_conda_channel_flags: + description: "Determines whether we need extra conda channels" + steps: + - run: + name: Adding CONDA_CHANNEL_FLAGS to BASH_ENV + command: | + CONDA_CHANNEL_FLAGS="" + if [[ "${PYTHON_VERSION}" = *3.9* ]]; then + echo "export CONDA_CHANNEL_FLAGS=-c=conda-forge" >> ${BASH_ENV} + fi binary_common: &binary_common parameters: @@ -138,6 +148,7 @@ jobs: resource_class: 2xlarge+ steps: - checkout + - load_conda_channel_flags - attach_workspace: at: third_party - run: packaging/build_conda.sh @@ -155,6 +166,7 @@ jobs: steps: - checkout - install_build_tools_macos + - load_conda_channel_flags - attach_workspace: at: third_party - run: @@ -180,6 +192,7 @@ jobs: steps: - checkout - install_build_tools_macos + - load_conda_channel_flags - attach_workspace: at: third_party - run: @@ -202,6 +215,7 @@ jobs: name: windows-cpu steps: - checkout + - load_conda_channel_flags - run: name: build command: | @@ -221,6 +235,7 @@ jobs: name: windows-cpu steps: - checkout + - load_conda_channel_flags - run: name: build command: | @@ -278,6 +293,7 @@ jobs: - attach_workspace: at: ~/workspace - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -297,6 +313,7 @@ jobs: - attach_workspace: at: ~/workspace - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -317,6 +334,7 @@ jobs: - attach_workspace: at: ~/workspace - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -342,6 +360,7 @@ jobs: - attach_workspace: at: ~/workspace - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -395,6 +414,7 @@ jobs: - attach_workspace: at: third_party - designate_upload_channel + - load_conda_channel_flags - generate_cache_key - restore_cache: {% raw %} @@ -438,6 +458,7 @@ jobs: - attach_workspace: at: third_party - designate_upload_channel + - load_conda_channel_flags - generate_cache_key - restore_cache: {% raw %} @@ -476,6 +497,7 @@ jobs: steps: - checkout - designate_upload_channel + - load_conda_channel_flags - generate_cache_key - restore_cache: {% raw %} @@ -513,6 +535,7 @@ jobs: steps: - checkout - designate_upload_channel + - load_conda_channel_flags - generate_cache_key - restore_cache: {% raw %} @@ -549,6 +572,7 @@ jobs: steps: - checkout - install_build_tools_macos + - load_conda_channel_flags - attach_workspace: at: third_party - designate_upload_channel @@ -575,6 +599,7 @@ jobs: steps: - checkout - designate_upload_channel + - load_conda_channel_flags - generate_cache_key - restore_cache: {% raw %} @@ -605,6 +630,7 @@ jobs: steps: - checkout - generate_cache_key + - load_conda_channel_flags - restore_cache: {% raw %} keys: From 070517ce59cada18f76c575d019f005d1e2df5f4 Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Tue, 1 Dec 2020 15:12:13 -0800 Subject: [PATCH 06/12] re-do python 3.9 Signed-off-by: Eli Uriegas --- packaging/pkg_helpers.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index cd1455cb19..80ea2d4933 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -140,7 +140,7 @@ setup_wheel_python() { 3.6) python_abi=cp36-cp36m ;; 3.7) python_abi=cp37-cp37m ;; 3.8) python_abi=cp38-cp38 ;; - 3.8) python_abi=cp39-cp39 ;; + 3.9) python_abi=cp39-cp39 ;; *) echo "Unrecognized PYTHON_VERSION=$PYTHON_VERSION" exit 1 From 40c0d30f7ea3062906a853a7c1b592e9df07a038 Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Tue, 1 Dec 2020 15:17:03 -0800 Subject: [PATCH 07/12] make sure we don't overwrite conda channel flags Signed-off-by: Eli Uriegas --- packaging/pkg_helpers.bash | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 80ea2d4933..24887f5806 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -182,11 +182,12 @@ setup_pip_pytorch_version() { # # You MUST have populated PYTORCH_VERSION_SUFFIX before hand. setup_conda_pytorch_constraint() { + CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS}" if [[ -z "$PYTORCH_VERSION" ]]; then - export CONDA_CHANNEL_FLAGS="-c pytorch-nightly" + export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c pytorch-nightly" export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | python -c "import sys, json, re; print(re.sub(r'\\+.*$', '', json.load(sys.stdin)['pytorch'][-1]['version']))")" else - export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-test -c pytorch-nightly" + export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c pytorch -c pytorch-test -c pytorch-nightly" fi if [[ "$CU_VERSION" == cpu ]]; then export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${PYTORCH_VERSION_SUFFIX}" From e5732a32bf59d592f30cec2c45793403cf4193ac Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Tue, 1 Dec 2020 15:19:49 -0800 Subject: [PATCH 08/12] make sure to use CONDA_CHANNEL_FLAGS Signed-off-by: Eli Uriegas --- .circleci/unittest/linux/scripts/install.sh | 2 +- .circleci/unittest/windows/scripts/install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index bde82ff9ee..a7f2e4d4f1 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -20,7 +20,7 @@ else cudatoolkit="cudatoolkit=${version}" fi printf "Installing PyTorch with %s\n" "${cudatoolkit}" -conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch ${cudatoolkit} +conda install ${CONDA_CHANNEL_FLAGS:-} -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch ${cudatoolkit} printf "* Installing dependencies for test\n" conda install -y -c conda-forge pytest pytest-cov codecov 'librosa>=0.8.0' scipy parameterized diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh index c294dc74a3..310442bf77 100644 --- a/.circleci/unittest/windows/scripts/install.sh +++ b/.circleci/unittest/windows/scripts/install.sh @@ -17,7 +17,7 @@ else cudatoolkit="cudatoolkit=${version}" fi printf "Installing PyTorch with %s\n" "${cudatoolkit}" -conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch "${cudatoolkit}" +conda install ${CONDA_CHANNEL_FLAGS:-} -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch "${cudatoolkit}" printf "* Installing torchaudio\n" python setup.py install From a84d905f6e63b79d63b98d34febaee7ad32024bb Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Tue, 1 Dec 2020 15:34:20 -0800 Subject: [PATCH 09/12] move some dependencies around Signed-off-by: Eli Uriegas --- .circleci/unittest/linux/scripts/install.sh | 5 +++-- .circleci/unittest/windows/scripts/environment.yml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index a7f2e4d4f1..9d6ab37898 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -23,8 +23,9 @@ printf "Installing PyTorch with %s\n" "${cudatoolkit}" conda install ${CONDA_CHANNEL_FLAGS:-} -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch ${cudatoolkit} printf "* Installing dependencies for test\n" -conda install -y -c conda-forge pytest pytest-cov codecov 'librosa>=0.8.0' scipy parameterized -pip install kaldi-io +conda install -y -c conda-forge pytest pytest-cov codecov scipy parameterized +# librosa doesn't have conda packages for python 3.9+ +pip install kaldi-io 'librosa>=0.8.0' printf "* Building codecs\n" mkdir -p third_party/build diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml index 4a0b71d653..2e7d720886 100644 --- a/.circleci/unittest/windows/scripts/environment.yml +++ b/.circleci/unittest/windows/scripts/environment.yml @@ -7,9 +7,9 @@ dependencies: - pytest - pytest-cov - codecov + - scipy >= 1.4.1 - pip - pip: - - scipy == 1.4.1 - kaldi-io - PySoundFile - librosa >= 0.8.0 From 5e70b3e7a88574c3ced78e2672f6432bbebecc12 Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Wed, 2 Dec 2020 10:08:55 -0800 Subject: [PATCH 10/12] only install librosa from pip for python 3.9 Signed-off-by: Eli Uriegas --- .circleci/unittest/linux/scripts/install.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index 9d6ab37898..2633e7e520 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -23,9 +23,15 @@ printf "Installing PyTorch with %s\n" "${cudatoolkit}" conda install ${CONDA_CHANNEL_FLAGS:-} -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch ${cudatoolkit} printf "* Installing dependencies for test\n" -conda install -y -c conda-forge pytest pytest-cov codecov scipy parameterized -# librosa doesn't have conda packages for python 3.9+ -pip install kaldi-io 'librosa>=0.8.0' +CONDA_PKGS="librosa>=0.8.0" +PIP_PKGS="" +if [[ $(python --version) = *3.9* ]]; then + CONDA_PKGS="" + # librosa doesn't have conda packages for python 3.9+ + PIP_PKGS="librosa>=0.8.0" +fi +conda install -y -c conda-forge pytest pytest-cov codecov scipy parameterized ${CONDA_PKGS} +pip install kaldi-io ${PIP_PKGS} printf "* Building codecs\n" mkdir -p third_party/build From c2ca3db918ec2f8719d9458be178ed08ea64b457 Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Wed, 2 Dec 2020 11:33:03 -0800 Subject: [PATCH 11/12] Don't install librosa for py3.9 Signed-off-by: Eli Uriegas --- .circleci/unittest/linux/scripts/install.sh | 7 +++---- .circleci/unittest/windows/scripts/environment.yml | 1 - .circleci/unittest/windows/scripts/install.sh | 6 ++++++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index 2633e7e520..cb6a3e947f 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -24,14 +24,13 @@ conda install ${CONDA_CHANNEL_FLAGS:-} -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch printf "* Installing dependencies for test\n" CONDA_PKGS="librosa>=0.8.0" -PIP_PKGS="" +# TODO: Remove this after packages become available +# Currently there's no librosa package available for Python 3.9, so lets just skip the dependency for now if [[ $(python --version) = *3.9* ]]; then CONDA_PKGS="" - # librosa doesn't have conda packages for python 3.9+ - PIP_PKGS="librosa>=0.8.0" fi conda install -y -c conda-forge pytest pytest-cov codecov scipy parameterized ${CONDA_PKGS} -pip install kaldi-io ${PIP_PKGS} +pip install kaldi-io printf "* Building codecs\n" mkdir -p third_party/build diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml index 2e7d720886..4fe8134d04 100644 --- a/.circleci/unittest/windows/scripts/environment.yml +++ b/.circleci/unittest/windows/scripts/environment.yml @@ -12,7 +12,6 @@ dependencies: - pip: - kaldi-io - PySoundFile - - librosa >= 0.8.0 - future - parameterized - dataclasses diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh index 310442bf77..7b3b6b1ed9 100644 --- a/.circleci/unittest/windows/scripts/install.sh +++ b/.circleci/unittest/windows/scripts/install.sh @@ -19,5 +19,11 @@ fi printf "Installing PyTorch with %s\n" "${cudatoolkit}" conda install ${CONDA_CHANNEL_FLAGS:-} -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch "${cudatoolkit}" +# TODO: Remove this after packages become available +# Currently there's no librosa package available for Python 3.9, so lets just skip the dependency for now +if [[ $(python --version) != *3.9* ]]; then + pip install 'librosa>=0.8.0' +fi + printf "* Installing torchaudio\n" python setup.py install From 878ad258ca4aa66ff3932cc78f02d8da4f2ef32f Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Thu, 3 Dec 2020 10:14:42 -0800 Subject: [PATCH 12/12] install pysoundfile Signed-off-by: Eli Uriegas --- .circleci/unittest/linux/scripts/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index cb6a3e947f..75314fe4a7 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -27,7 +27,7 @@ CONDA_PKGS="librosa>=0.8.0" # TODO: Remove this after packages become available # Currently there's no librosa package available for Python 3.9, so lets just skip the dependency for now if [[ $(python --version) = *3.9* ]]; then - CONDA_PKGS="" + CONDA_PKGS="pysoundfile" fi conda install -y -c conda-forge pytest pytest-cov codecov scipy parameterized ${CONDA_PKGS} pip install kaldi-io