From 9bd858d9b808dfefb14d3d23f81c4c5ee575f176 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Wed, 14 Oct 2020 17:21:36 +0000 Subject: [PATCH 1/3] Parameterize UPLOAD_CHANNEL --- .circleci/config.yml | 24 ++++++++++++++++++------ .circleci/config.yml.in | 24 ++++++++++++++++++------ 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ba994b9166..8e1e219e09 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,7 +25,7 @@ commands: command: | our_upload_channel=nightly # On tags upload to test instead - if [[ -n "${CIRCLE_TAG}" ]]; then + if [[ -n "${CIRCLE_TAG}" ]] || [[ ${CIRCLE_BRANCH} =~ release/* ]]; then our_upload_channel=test fi echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} @@ -75,6 +75,7 @@ jobs: resource_class: 2xlarge+ steps: - checkout + - designate_upload_channel - run: packaging/build_wheel.sh - store_artifacts: path: dist @@ -90,6 +91,7 @@ jobs: resource_class: 2xlarge+ steps: - checkout + - designate_upload_channel - run: packaging/build_conda.sh - store_artifacts: path: /opt/conda/conda-bld/linux-64 @@ -104,6 +106,7 @@ jobs: name: windows-cpu steps: - checkout + - designate_upload_channel - run: name: build command: | @@ -123,6 +126,7 @@ jobs: name: windows-cpu steps: - checkout + - designate_upload_channel - run: name: build command: | @@ -144,6 +148,7 @@ jobs: xcode: "9.4.1" steps: - checkout + - designate_upload_channel - run: # Installing cmake with `brew install cmake` takes 30 mins, so we use binary distribution. command: | @@ -169,6 +174,7 @@ jobs: xcode: "9.4.1" steps: - checkout + - designate_upload_channel - run: command: | curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh @@ -196,7 +202,7 @@ jobs: # Prevent credential from leaking conda install -yq anaconda-client set -x - anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/conda-bld/*/*.tar.bz2 -u pytorch-nightly --label main --no-progress --force + anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/conda-bld/*/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force # Requires org-member context binary_wheel_upload: @@ -217,7 +223,7 @@ jobs: export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" set -x for pkg in ~/workspace/*.whl; do - aws s3 cp "$pkg" "s3://pytorch/whl/nightly/" --acl public-read + aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/" --acl public-read done smoke_test_linux_conda: @@ -244,12 +250,13 @@ jobs: steps: - attach_workspace: at: ~/workspace + - designate_upload_channel - run: name: install binaries command: | set -x source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - pip install $(ls ~/workspace/torchtext*.whl) --pre -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + pip install $(ls ~/workspace/torchtext*.whl) --pre -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/cpu/torch_${UPLOAD_CHANNEL}.html" - run: name: smoke test command: | @@ -283,6 +290,7 @@ jobs: steps: - attach_workspace: at: ~/workspace + - designate_upload_channel - run: name: install binaries command: | @@ -291,7 +299,7 @@ jobs: conda env remove -n python${PYTHON_VERSION} || true conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda activate python${PYTHON_VERSION} - conda install -v -y -c pytorch-nightly pytorch + conda install -v -y -c pytorch-"${UPLOAD_CHANNEL}" pytorch conda install -v -y -c ~/workspace/conda-bld torchtext - run: name: smoke test @@ -307,6 +315,7 @@ jobs: steps: - attach_workspace: at: ~/workspace + - designate_upload_channel - run: name: install binaries command: | @@ -315,7 +324,7 @@ jobs: conda env remove -n python${PYTHON_VERSION} || true conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda activate python${PYTHON_VERSION} - pip install $(ls ~/workspace/torchtext*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html + pip install $(ls ~/workspace/torchtext*.whl) --pre -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/torch_${UPLOAD_CHANNEL}.html" - run: name: smoke test command: | @@ -330,6 +339,7 @@ jobs: resource_class: 2xlarge+ steps: - checkout + - designate_upload_channel - run: name: Generate cache key # This will refresh cache on Sundays, nightly build should generate new cache. @@ -381,6 +391,7 @@ jobs: name: windows-cpu steps: - checkout + - designate_upload_channel - run: name: Generate cache key # This will refresh cache on Sundays, nightly build should generate new cache. @@ -433,6 +444,7 @@ jobs: resource_class: medium steps: - checkout + - designate_upload_channel - run: name: Generate cache key # This will refresh cache on Sundays, nightly build should generate new cache. diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 5693bc2bb6..770393a0f3 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -25,7 +25,7 @@ commands: command: | our_upload_channel=nightly # On tags upload to test instead - if [[ -n "${CIRCLE_TAG}" ]]; then + if [[ -n "${CIRCLE_TAG}" ]] || [[ ${CIRCLE_BRANCH} =~ release/* ]]; then our_upload_channel=test fi echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} @@ -75,6 +75,7 @@ jobs: resource_class: 2xlarge+ steps: - checkout + - designate_upload_channel - run: packaging/build_wheel.sh - store_artifacts: path: dist @@ -90,6 +91,7 @@ jobs: resource_class: 2xlarge+ steps: - checkout + - designate_upload_channel - run: packaging/build_conda.sh - store_artifacts: path: /opt/conda/conda-bld/linux-64 @@ -104,6 +106,7 @@ jobs: name: windows-cpu steps: - checkout + - designate_upload_channel - run: name: build command: | @@ -123,6 +126,7 @@ jobs: name: windows-cpu steps: - checkout + - designate_upload_channel - run: name: build command: | @@ -144,6 +148,7 @@ jobs: xcode: "9.4.1" steps: - checkout + - designate_upload_channel - run: # Installing cmake with `brew install cmake` takes 30 mins, so we use binary distribution. command: | @@ -169,6 +174,7 @@ jobs: xcode: "9.4.1" steps: - checkout + - designate_upload_channel - run: command: | curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh @@ -196,7 +202,7 @@ jobs: # Prevent credential from leaking conda install -yq anaconda-client set -x - anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/conda-bld/*/*.tar.bz2 -u pytorch-nightly --label main --no-progress --force + anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/conda-bld/*/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force # Requires org-member context binary_wheel_upload: @@ -217,7 +223,7 @@ jobs: export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" set -x for pkg in ~/workspace/*.whl; do - aws s3 cp "$pkg" "s3://pytorch/whl/nightly/" --acl public-read + aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/" --acl public-read done smoke_test_linux_conda: @@ -244,12 +250,13 @@ jobs: steps: - attach_workspace: at: ~/workspace + - designate_upload_channel - run: name: install binaries command: | set -x source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - pip install $(ls ~/workspace/torchtext*.whl) --pre -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + pip install $(ls ~/workspace/torchtext*.whl) --pre -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/cpu/torch_${UPLOAD_CHANNEL}.html" - run: name: smoke test command: | @@ -283,6 +290,7 @@ jobs: steps: - attach_workspace: at: ~/workspace + - designate_upload_channel - run: name: install binaries command: | @@ -291,7 +299,7 @@ jobs: conda env remove -n python${PYTHON_VERSION} || true conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda activate python${PYTHON_VERSION} - conda install -v -y -c pytorch-nightly pytorch + conda install -v -y -c pytorch-"${UPLOAD_CHANNEL}" pytorch conda install -v -y -c ~/workspace/conda-bld torchtext - run: name: smoke test @@ -307,6 +315,7 @@ jobs: steps: - attach_workspace: at: ~/workspace + - designate_upload_channel - run: name: install binaries command: | @@ -315,7 +324,7 @@ jobs: conda env remove -n python${PYTHON_VERSION} || true conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda activate python${PYTHON_VERSION} - pip install $(ls ~/workspace/torchtext*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html + pip install $(ls ~/workspace/torchtext*.whl) --pre -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/torch_${UPLOAD_CHANNEL}.html" - run: name: smoke test command: | @@ -330,6 +339,7 @@ jobs: resource_class: 2xlarge+ steps: - checkout + - designate_upload_channel - run: name: Generate cache key # This will refresh cache on Sundays, nightly build should generate new cache. @@ -381,6 +391,7 @@ jobs: name: windows-cpu steps: - checkout + - designate_upload_channel - run: name: Generate cache key # This will refresh cache on Sundays, nightly build should generate new cache. @@ -433,6 +444,7 @@ jobs: resource_class: medium steps: - checkout + - designate_upload_channel - run: name: Generate cache key # This will refresh cache on Sundays, nightly build should generate new cache. From 2465452c39d30500cbf66e441f601848f04e3360 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Wed, 14 Oct 2020 17:38:19 +0000 Subject: [PATCH 2/3] Fix pkg_helpers.bash --- packaging/pkg_helpers.bash | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index b7254c19c3..ce4bda723d 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -173,6 +173,7 @@ setup_pip_pytorch_version() { else pip_install "torch==$PYTORCH_VERSION$PYTORCH_VERSION_SUFFIX" \ -f https://download.pytorch.org/whl/torch_stable.html \ + -f https://download.pytorch.org/whl/test/torch_test.html \ -f https://download.pytorch.org/whl/nightly/torch_nightly.html fi } @@ -186,7 +187,7 @@ setup_conda_pytorch_constraint() { export CONDA_CHANNEL_FLAGS="-c pytorch-nightly" export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | python -c "import sys, json, re; print(re.sub(r'\\+.*$', '', json.load(sys.stdin)['pytorch'][-1]['version']))")" else - export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-nightly" + export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-test -c pytorch-nightly" fi if [[ "$CU_VERSION" == cpu ]]; then export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${PYTORCH_VERSION_SUFFIX}" From 9b96a9d45954b41c50746acb4f678732b0d76800 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Thu, 15 Oct 2020 19:38:48 +0000 Subject: [PATCH 3/3] Update pkg_helpers.bash --- packaging/pkg_helpers.bash | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index ce4bda723d..7ce802ed14 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -173,8 +173,7 @@ setup_pip_pytorch_version() { else pip_install "torch==$PYTORCH_VERSION$PYTORCH_VERSION_SUFFIX" \ -f https://download.pytorch.org/whl/torch_stable.html \ - -f https://download.pytorch.org/whl/test/torch_test.html \ - -f https://download.pytorch.org/whl/nightly/torch_nightly.html + -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/torch_${UPLOAD_CHANNEL}.html" fi } @@ -187,7 +186,7 @@ setup_conda_pytorch_constraint() { export CONDA_CHANNEL_FLAGS="-c pytorch-nightly" export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | python -c "import sys, json, re; print(re.sub(r'\\+.*$', '', json.load(sys.stdin)['pytorch'][-1]['version']))")" else - export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-test -c pytorch-nightly" + export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-${UPLOAD_CHANNEL}" fi if [[ "$CU_VERSION" == cpu ]]; then export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${PYTORCH_VERSION_SUFFIX}"