From 58f1a3cef570a591761a481fb71f3c3db4d12392 Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Thu, 17 Jun 2021 12:10:53 -0700 Subject: [PATCH 1/5] ci: Remove mentions of conda-forge We run into a lot of build / runtime issues when relying on things from conda-forge. Now that Python 3.9 is more thoroughly supported on the default anaconda channel lets remove the hooks for conda-forge now. Signed-off-by: Eli Uriegas --- .circleci/config.yml | 11 +---------- .circleci/config.yml.in | 11 +---------- .circleci/unittest/linux/scripts/environment.yml | 2 -- .circleci/unittest/linux/scripts/install.sh | 2 +- .circleci/unittest/windows/scripts/environment.yml | 2 -- .circleci/unittest/windows/scripts/install.sh | 2 +- CONTRIBUTING.md | 2 +- packaging/build_conda.sh | 2 +- packaging/pkg_helpers.bash | 8 +------- 9 files changed, 7 insertions(+), 35 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 09b8b57a98b..285f274ac4f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -577,11 +577,7 @@ jobs: set -x eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda env remove -n python${PYTHON_VERSION} || true - CONDA_CHANNEL_FLAGS="" - if [[ "${PYTHON_VERSION}" = 3.9 ]]; then - CONDA_CHANNEL_FLAGS="-c=conda-forge" - fi - conda create ${CONDA_CHANNEL_FLAGS} -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} + conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda activate python${PYTHON_VERSION} conda install Pillow>=5.3.0 conda install -v -y -c pytorch-nightly pytorch @@ -606,11 +602,6 @@ jobs: command: | set -x eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - CONDA_CHANNEL_FLAGS="" - if [[ "${PYTHON_VERSION}" = 3.9 ]]; then - CONDA_CHANNEL_FLAGS="-c=conda-forge" - fi - conda create ${CONDA_CHANNEL_FLAGS} -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda activate python${PYTHON_VERSION} pip install $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index f2d2e472a58..43b2e2d7faf 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -577,11 +577,7 @@ jobs: set -x eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda env remove -n python${PYTHON_VERSION} || true - CONDA_CHANNEL_FLAGS="" - if [[ "${PYTHON_VERSION}" = 3.9 ]]; then - CONDA_CHANNEL_FLAGS="-c=conda-forge" - fi - conda create ${CONDA_CHANNEL_FLAGS} -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} + conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda activate python${PYTHON_VERSION} conda install Pillow>=5.3.0 conda install -v -y -c pytorch-nightly pytorch @@ -606,11 +602,6 @@ jobs: command: | set -x eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - CONDA_CHANNEL_FLAGS="" - if [[ "${PYTHON_VERSION}" = 3.9 ]]; then - CONDA_CHANNEL_FLAGS="-c=conda-forge" - fi - conda create ${CONDA_CHANNEL_FLAGS} -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda activate python${PYTHON_VERSION} pip install $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html diff --git a/.circleci/unittest/linux/scripts/environment.yml b/.circleci/unittest/linux/scripts/environment.yml index c0d36f95a43..67ceb836a10 100644 --- a/.circleci/unittest/linux/scripts/environment.yml +++ b/.circleci/unittest/linux/scripts/environment.yml @@ -1,8 +1,6 @@ channels: - pytorch - defaults - # using conda-forge for python v3.9 - - conda-forge dependencies: - pytest - pytest-cov diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index 7058e4d7095..e9b9e80c077 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -24,7 +24,7 @@ else fi printf "Installing PyTorch with %s\n" "${cudatoolkit}" -conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c conda-forge "pytorch-${UPLOAD_CHANNEL}"::pytorch "${cudatoolkit}" pytest +conda install -y -c "pytorch-${UPLOAD_CHANNEL}" "pytorch-${UPLOAD_CHANNEL}"::pytorch "${cudatoolkit}" pytest if [ $PYTHON_VERSION == "3.6" ]; then printf "Installing minimal PILLOW version\n" diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml index 11de8e089e8..fc4d2f89a0e 100644 --- a/.circleci/unittest/windows/scripts/environment.yml +++ b/.circleci/unittest/windows/scripts/environment.yml @@ -1,8 +1,6 @@ channels: - pytorch - defaults - # use conda-forge for python v3.9+ - - conda-forge dependencies: - pytest - pytest-cov diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh index a90e88a71d4..45e381c775b 100644 --- a/.circleci/unittest/windows/scripts/install.sh +++ b/.circleci/unittest/windows/scripts/install.sh @@ -26,7 +26,7 @@ else fi printf "Installing PyTorch with %s\n" "${cudatoolkit}" -conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c conda-forge "pytorch-${UPLOAD_CHANNEL}"::pytorch "${cudatoolkit}" pytest +conda install -y -c "pytorch-${UPLOAD_CHANNEL}" "pytorch-${UPLOAD_CHANNEL}"::pytorch "${cudatoolkit}" pytest if [ $PYTHON_VERSION == "3.6" ]; then printf "Installing minimal PILLOW version\n" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3fa41efec26..75405c94a83 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,7 +33,7 @@ clear and has sufficient instructions to be able to reproduce the issue. ### Install PyTorch Nightly ```bash -conda install pytorch -c pytorch-nightly -c conda-forge +conda install pytorch -c pytorch-nightly # or with pip (see https://pytorch.org/get-started/locally/) # pip install numpy # pip install --pre torch -f https://download.pytorch.org/whl/nightly/cu102/torch_nightly.html diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh index 85c7d88273f..1c98c139da0 100755 --- a/packaging/build_conda.sh +++ b/packaging/build_conda.sh @@ -11,4 +11,4 @@ setup_conda_pytorch_constraint setup_conda_cudatoolkit_constraint setup_visual_studio_constraint setup_junit_results_folder -conda build $CONDA_CHANNEL_FLAGS -c defaults -c conda-forge --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchvision +conda build $CONDA_CHANNEL_FLAGS -c defaults --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchvision diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 48f7588594a..88cb536f8e8 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -180,13 +180,10 @@ setup_wheel_python() { if [[ "$(uname)" == Darwin || "$OSTYPE" == "msys" ]]; then eval "$(conda shell.bash hook)" conda env remove -n "env$PYTHON_VERSION" || true - if [[ "$PYTHON_VERSION" == 3.9 ]]; then - export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c=conda-forge" - fi conda create ${CONDA_CHANNEL_FLAGS} -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION" conda activate "env$PYTHON_VERSION" # Install libpng from Anaconda (defaults) - conda install ${CONDA_CHANNEL_FLAGS} -c conda-forge libpng "jpeg<=9b" -y + conda install ${CONDA_CHANNEL_FLAGS} libpng "jpeg<=9b" -y else # Install native CentOS libJPEG, LAME, freetype and GnuTLS yum install -y libjpeg-turbo-devel lame freetype gnutls @@ -277,9 +274,6 @@ setup_conda_pytorch_constraint() { if [[ "$OSTYPE" == msys && "$CU_VERSION" == cu92 ]]; then export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c defaults -c numba/label/dev" fi - if [[ "$PYTHON_VERSION" == 3.9 ]]; then - export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c=conda-forge" - fi } # Translate CUDA_VERSION into CUDA_CUDATOOLKIT_CONSTRAINT From 5ba192e2fce573a5b6d3467e4cbddf95ed8ff4ef Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Thu, 17 Jun 2021 12:23:58 -0700 Subject: [PATCH 2/5] remove av from test requires Signed-off-by: Eli Uriegas --- packaging/torchvision/meta.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/packaging/torchvision/meta.yaml b/packaging/torchvision/meta.yaml index c9b6d04fdc7..222071ede93 100644 --- a/packaging/torchvision/meta.yaml +++ b/packaging/torchvision/meta.yaml @@ -52,7 +52,6 @@ test: requires: - pytest - scipy - - av # NOTE: Pinned to fix issues with size_t on Windows - jpeg <=9b - ca-certificates From c333ae05bfeb7d4310c328ad019ad00f6eba3803 Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Thu, 17 Jun 2021 12:28:49 -0700 Subject: [PATCH 3/5] also include nvidia channel for cudatoolkit Signed-off-by: Eli Uriegas --- packaging/build_conda.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh index 1c98c139da0..a28fd9381f4 100755 --- a/packaging/build_conda.sh +++ b/packaging/build_conda.sh @@ -11,4 +11,5 @@ setup_conda_pytorch_constraint setup_conda_cudatoolkit_constraint setup_visual_studio_constraint setup_junit_results_folder -conda build $CONDA_CHANNEL_FLAGS -c defaults --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchvision +# nvidia channel included for cudatoolkit >= 11 +conda build $CONDA_CHANNEL_FLAGS -c defaults -c nvidia --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchvision From 55d3cbe5fa7ebd24dbe312c582c83fe2edb595c4 Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Thu, 17 Jun 2021 14:02:11 -0700 Subject: [PATCH 4/5] enable conda forge for windows cuda 11 Signed-off-by: Eli Uriegas --- .circleci/config.yml | 4 ++++ .circleci/config.yml.in | 4 ++++ packaging/build_conda.sh | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 285f274ac4f..f090ff539c4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -311,6 +311,10 @@ jobs: eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" + # cudatoolkit >= 11 isn't available for windows in the nvidia channel + if [[ "${CU_VERSION}" =~ "cu11*" ]]; then + export CONDA_CHANNEL_FLAGS="-c conda-forge" + fi packaging/build_conda.sh rm /C/tools/miniconda3/conda-bld/win-64/vs${VC_YEAR}*.tar.bz2 - store_artifacts: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 43b2e2d7faf..18a23c2e639 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -311,6 +311,10 @@ jobs: eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" + # cudatoolkit >= 11 isn't available for windows in the nvidia channel + if [[ "${CU_VERSION}" =~ "cu11*" ]]; then + export CONDA_CHANNEL_FLAGS="-c conda-forge" + fi packaging/build_conda.sh rm /C/tools/miniconda3/conda-bld/win-64/vs${VC_YEAR}*.tar.bz2 - store_artifacts: diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh index a28fd9381f4..9b14c288724 100755 --- a/packaging/build_conda.sh +++ b/packaging/build_conda.sh @@ -12,4 +12,4 @@ setup_conda_cudatoolkit_constraint setup_visual_studio_constraint setup_junit_results_folder # nvidia channel included for cudatoolkit >= 11 -conda build $CONDA_CHANNEL_FLAGS -c defaults -c nvidia --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchvision +conda build -c defaults -c nvidia $CONDA_CHANNEL_FLAGS --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchvision From fb796cb13e9d4791db90a551421e5e7334b88e24 Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Thu, 17 Jun 2021 14:25:32 -0700 Subject: [PATCH 5/5] don't overwrite CONDA_CHANNEL_FLAGS, just append Signed-off-by: Eli Uriegas --- .circleci/config.yml | 2 +- .circleci/config.yml.in | 2 +- packaging/pkg_helpers.bash | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f090ff539c4..83e7ebd0f49 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -312,7 +312,7 @@ jobs: conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" # cudatoolkit >= 11 isn't available for windows in the nvidia channel - if [[ "${CU_VERSION}" =~ "cu11*" ]]; then + if [[ "${CU_VERSION}" =~ cu11.* ]]; then export CONDA_CHANNEL_FLAGS="-c conda-forge" fi packaging/build_conda.sh diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 18a23c2e639..19539785e9b 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -312,7 +312,7 @@ jobs: conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" # cudatoolkit >= 11 isn't available for windows in the nvidia channel - if [[ "${CU_VERSION}" =~ "cu11*" ]]; then + if [[ "${CU_VERSION}" =~ cu11.* ]]; then export CONDA_CHANNEL_FLAGS="-c conda-forge" fi packaging/build_conda.sh diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 88cb536f8e8..dfde11b707a 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -247,7 +247,7 @@ setup_pip_pytorch_version() { # You MUST have populated PYTORCH_VERSION_SUFFIX before hand. setup_conda_pytorch_constraint() { if [[ -z "$PYTORCH_VERSION" ]]; then - export CONDA_CHANNEL_FLAGS="-c pytorch-nightly -c pytorch" + export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c pytorch-nightly -c pytorch" export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | \ python -c "import os, sys, json, re; cuver = os.environ.get('CU_VERSION'); \ cuver_1 = cuver.replace('cu', 'cuda') if cuver != 'cpu' else cuver; \ @@ -262,7 +262,7 @@ setup_conda_pytorch_constraint() { exit 1 fi else - export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-${UPLOAD_CHANNEL}" + export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c pytorch -c pytorch-${UPLOAD_CHANNEL}" fi if [[ "$CU_VERSION" == cpu ]]; then export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${PYTORCH_VERSION_SUFFIX}"