Skip to content

Commit 6713f03

Browse files
mthrokfmassa
mthrok
and
fmassa
authored
Update unittest and packaging to use UPLOAD_CHANNEL (#2846) (#2848)
Co-authored-by: Francisco Massa <[email protected]>
1 parent 34708e4 commit 6713f03

File tree

6 files changed

+53
-12
lines changed

6 files changed

+53
-12
lines changed

‎.circleci/config.yml‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ jobs:
138138
resource_class: 2xlarge+
139139
steps:
140140
- checkout_merge
141+
- designate_upload_channel
141142
- run: packaging/build_wheel.sh
142143
- store_artifacts:
143144
path: dist
@@ -153,6 +154,7 @@ jobs:
153154
resource_class: 2xlarge+
154155
steps:
155156
- checkout_merge
157+
- designate_upload_channel
156158
- run: packaging/build_conda.sh
157159
- store_artifacts:
158160
path: /opt/conda/conda-bld/linux-64
@@ -168,6 +170,7 @@ jobs:
168170
executor: windows-cpu
169171
steps:
170172
- checkout_merge
173+
- designate_upload_channel
171174
- run:
172175
name: Build conda packages
173176
command: |
@@ -193,6 +196,7 @@ jobs:
193196
executor: windows-cpu
194197
steps:
195198
- checkout_merge
199+
- designate_upload_channel
196200
- run:
197201
name: Build wheel packages
198202
command: |
@@ -215,6 +219,7 @@ jobs:
215219
xcode: "9.4.1"
216220
steps:
217221
- checkout_merge
222+
- designate_upload_channel
218223
- run:
219224
# Cannot easily deduplicate this as source'ing activate
220225
# will set environment variables which we need to propagate
@@ -237,6 +242,7 @@ jobs:
237242
xcode: "9.4.1"
238243
steps:
239244
- checkout_merge
245+
- designate_upload_channel
240246
- run:
241247
command: |
242248
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
@@ -299,6 +305,7 @@ jobs:
299305
steps:
300306
- attach_workspace:
301307
at: ~/workspace
308+
- designate_upload_channel
302309
- run:
303310
name: install binaries
304311
command: |
@@ -317,6 +324,7 @@ jobs:
317324
steps:
318325
- attach_workspace:
319326
at: ~/workspace
327+
- designate_upload_channel
320328
- run:
321329
name: install binaries
322330
command: |
@@ -337,6 +345,7 @@ jobs:
337345
image_name: torchvision/smoke_test
338346
steps:
339347
- checkout
348+
- designate_upload_channel
340349
- run:
341350
name: Build and push Docker image
342351
no_output_timeout: "1h"
@@ -356,6 +365,7 @@ jobs:
356365
steps:
357366
- attach_workspace:
358367
at: ~/workspace
368+
- designate_upload_channel
359369
- run:
360370
name: install binaries
361371
command: |
@@ -381,6 +391,7 @@ jobs:
381391
steps:
382392
- attach_workspace:
383393
at: ~/workspace
394+
- designate_upload_channel
384395
- run:
385396
name: install binaries
386397
command: |
@@ -404,6 +415,7 @@ jobs:
404415
resource_class: 2xlarge+
405416
steps:
406417
- checkout
418+
- designate_upload_channel
407419
- run:
408420
name: Generate cache key
409421
# This will refresh cache on Sundays, nightly build should generate new cache.
@@ -444,6 +456,7 @@ jobs:
444456
image_name: "pytorch/manylinux-cuda101"
445457
steps:
446458
- checkout
459+
- designate_upload_channel
447460
- run:
448461
name: Generate cache key
449462
# This will refresh cache on Sundays, nightly build should generate new cache.
@@ -465,7 +478,7 @@ jobs:
465478
- env
466479
- run:
467480
name: Install torchvision
468-
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
481+
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL "${image_name}" .circleci/unittest/linux/scripts/install.sh
469482
- run:
470483
name: Run tests
471484
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
@@ -481,6 +494,7 @@ jobs:
481494
name: windows-cpu
482495
steps:
483496
- checkout
497+
- designate_upload_channel
484498
- run:
485499
name: Generate cache key
486500
# This will refresh cache on Sundays, nightly build should generate new cache.
@@ -520,6 +534,7 @@ jobs:
520534
CUDA_VERSION: "10.1"
521535
steps:
522536
- checkout
537+
- designate_upload_channel
523538
- run:
524539
name: Generate cache key
525540
# This will refresh cache on Sundays, nightly build should generate new cache.
@@ -558,6 +573,7 @@ jobs:
558573
resource_class: large
559574
steps:
560575
- checkout
576+
- designate_upload_channel
561577
- run:
562578
name: Install wget
563579
command: HOMEBREW_NO_AUTO_UPDATE=1 brew install wget
@@ -600,6 +616,7 @@ jobs:
600616
resource_class: 2xlarge+
601617
steps:
602618
- checkout_merge
619+
- designate_upload_channel
603620
- run:
604621
name: Setup conda
605622
command: .circleci/unittest/linux/scripts/setup_env.sh
@@ -617,19 +634,21 @@ jobs:
617634
CU_VERSION: << parameters.cu_version >>
618635
steps:
619636
- checkout_merge
637+
- designate_upload_channel
620638
- run:
621639
name: Setup conda
622640
command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh
623641
- run:
624642
name: Build torchvision C++ distribution and test
625-
command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh
643+
command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh
626644

627645
cmake_macos_cpu:
628646
<<: *binary_common
629647
macos:
630648
xcode: "9.0"
631649
steps:
632650
- checkout_merge
651+
- designate_upload_channel
633652
- run:
634653
command: |
635654
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
@@ -644,6 +663,7 @@ jobs:
644663
name: windows-cpu
645664
steps:
646665
- checkout_merge
666+
- designate_upload_channel
647667
- run:
648668
command: |
649669
set -ex
@@ -656,6 +676,7 @@ jobs:
656676
name: windows-gpu
657677
steps:
658678
- checkout_merge
679+
- designate_upload_channel
659680
- run:
660681
command: |
661682
set -ex

‎.circleci/config.yml.in‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ jobs:
138138
resource_class: 2xlarge+
139139
steps:
140140
- checkout_merge
141+
- designate_upload_channel
141142
- run: packaging/build_wheel.sh
142143
- store_artifacts:
143144
path: dist
@@ -153,6 +154,7 @@ jobs:
153154
resource_class: 2xlarge+
154155
steps:
155156
- checkout_merge
157+
- designate_upload_channel
156158
- run: packaging/build_conda.sh
157159
- store_artifacts:
158160
path: /opt/conda/conda-bld/linux-64
@@ -168,6 +170,7 @@ jobs:
168170
executor: windows-cpu
169171
steps:
170172
- checkout_merge
173+
- designate_upload_channel
171174
- run:
172175
name: Build conda packages
173176
command: |
@@ -193,6 +196,7 @@ jobs:
193196
executor: windows-cpu
194197
steps:
195198
- checkout_merge
199+
- designate_upload_channel
196200
- run:
197201
name: Build wheel packages
198202
command: |
@@ -215,6 +219,7 @@ jobs:
215219
xcode: "9.4.1"
216220
steps:
217221
- checkout_merge
222+
- designate_upload_channel
218223
- run:
219224
# Cannot easily deduplicate this as source'ing activate
220225
# will set environment variables which we need to propagate
@@ -237,6 +242,7 @@ jobs:
237242
xcode: "9.4.1"
238243
steps:
239244
- checkout_merge
245+
- designate_upload_channel
240246
- run:
241247
command: |
242248
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
@@ -299,6 +305,7 @@ jobs:
299305
steps:
300306
- attach_workspace:
301307
at: ~/workspace
308+
- designate_upload_channel
302309
- run:
303310
name: install binaries
304311
command: |
@@ -317,6 +324,7 @@ jobs:
317324
steps:
318325
- attach_workspace:
319326
at: ~/workspace
327+
- designate_upload_channel
320328
- run:
321329
name: install binaries
322330
command: |
@@ -337,6 +345,7 @@ jobs:
337345
image_name: torchvision/smoke_test
338346
steps:
339347
- checkout
348+
- designate_upload_channel
340349
- run:
341350
name: Build and push Docker image
342351
no_output_timeout: "1h"
@@ -356,6 +365,7 @@ jobs:
356365
steps:
357366
- attach_workspace:
358367
at: ~/workspace
368+
- designate_upload_channel
359369
- run:
360370
name: install binaries
361371
command: |
@@ -381,6 +391,7 @@ jobs:
381391
steps:
382392
- attach_workspace:
383393
at: ~/workspace
394+
- designate_upload_channel
384395
- run:
385396
name: install binaries
386397
command: |
@@ -404,6 +415,7 @@ jobs:
404415
resource_class: 2xlarge+
405416
steps:
406417
- checkout
418+
- designate_upload_channel
407419
- run:
408420
name: Generate cache key
409421
# This will refresh cache on Sundays, nightly build should generate new cache.
@@ -444,6 +456,7 @@ jobs:
444456
image_name: "pytorch/manylinux-cuda101"
445457
steps:
446458
- checkout
459+
- designate_upload_channel
447460
- run:
448461
name: Generate cache key
449462
# This will refresh cache on Sundays, nightly build should generate new cache.
@@ -465,7 +478,7 @@ jobs:
465478
- env
466479
- run:
467480
name: Install torchvision
468-
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
481+
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL "${image_name}" .circleci/unittest/linux/scripts/install.sh
469482
- run:
470483
name: Run tests
471484
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
@@ -481,6 +494,7 @@ jobs:
481494
name: windows-cpu
482495
steps:
483496
- checkout
497+
- designate_upload_channel
484498
- run:
485499
name: Generate cache key
486500
# This will refresh cache on Sundays, nightly build should generate new cache.
@@ -520,6 +534,7 @@ jobs:
520534
CUDA_VERSION: "10.1"
521535
steps:
522536
- checkout
537+
- designate_upload_channel
523538
- run:
524539
name: Generate cache key
525540
# This will refresh cache on Sundays, nightly build should generate new cache.
@@ -558,6 +573,7 @@ jobs:
558573
resource_class: large
559574
steps:
560575
- checkout
576+
- designate_upload_channel
561577
- run:
562578
name: Install wget
563579
command: HOMEBREW_NO_AUTO_UPDATE=1 brew install wget
@@ -600,6 +616,7 @@ jobs:
600616
resource_class: 2xlarge+
601617
steps:
602618
- checkout_merge
619+
- designate_upload_channel
603620
- run:
604621
name: Setup conda
605622
command: .circleci/unittest/linux/scripts/setup_env.sh
@@ -617,19 +634,21 @@ jobs:
617634
CU_VERSION: << parameters.cu_version >>
618635
steps:
619636
- checkout_merge
637+
- designate_upload_channel
620638
- run:
621639
name: Setup conda
622640
command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh
623641
- run:
624642
name: Build torchvision C++ distribution and test
625-
command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh
643+
command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh
626644

627645
cmake_macos_cpu:
628646
<<: *binary_common
629647
macos:
630648
xcode: "9.0"
631649
steps:
632650
- checkout_merge
651+
- designate_upload_channel
633652
- run:
634653
command: |
635654
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
@@ -644,6 +663,7 @@ jobs:
644663
name: windows-cpu
645664
steps:
646665
- checkout_merge
666+
- designate_upload_channel
647667
- run:
648668
command: |
649669
set -ex
@@ -656,6 +676,7 @@ jobs:
656676
name: windows-gpu
657677
steps:
658678
- checkout_merge
679+
- designate_upload_channel
659680
- run:
660681
command: |
661682
set -ex

‎.circleci/unittest/linux/scripts/install.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ else
2323
cudatoolkit="cudatoolkit=${version}"
2424
fi
2525
printf "Installing PyTorch with %s\n" "${cudatoolkit}"
26-
conda install -y -c pytorch-nightly pytorch "${cudatoolkit}"
26+
conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch "${cudatoolkit}"
2727

2828
printf "* Installing torchvision\n"
29-
python setup.py develop
29+
python setup.py develop

‎.circleci/unittest/windows/scripts/install.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ else
2525
cudatoolkit="cudatoolkit=${version}"
2626
fi
2727
printf "Installing PyTorch with %s\n" "${cudatoolkit}"
28-
conda install -y -c pytorch-nightly pytorch "${cudatoolkit}"
28+
conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch "${cudatoolkit}"
2929

3030
printf "* Installing torchvision\n"
31-
"$this_dir/vc_env_helper.bat" python setup.py develop
31+
"$this_dir/vc_env_helper.bat" python setup.py develop

‎packaging/build_cmake.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ fi
2323
setup_visual_studio_constraint
2424
setup_junit_results_folder
2525

26-
conda install -yq pytorch=$PYTORCH_VERSION $CONDA_CUDATOOLKIT_CONSTRAINT $CONDA_CPUONLY_FEATURE -c pytorch-nightly
26+
conda install -yq pytorch=$PYTORCH_VERSION $CONDA_CUDATOOLKIT_CONSTRAINT $CONDA_CPUONLY_FEATURE -c "pytorch-${UPLOAD_CHANNEL}"
2727
TORCH_PATH=$(dirname $(python -c "import torch; print(torch.__file__)"))
2828

2929
if [[ "$(uname)" == Darwin || "$OSTYPE" == "msys" ]]; then

0 commit comments

Comments
 (0)