Skip to content

Commit 1e1a7a4

Browse files
ezyangezyang
authored
Add uploading support (#220)
Also bugfixes and updates to the packaging scripts.
1 parent 636e149 commit 1e1a7a4

File tree

5 files changed

+479
-100
lines changed

5 files changed

+479
-100
lines changed

‎.circleci/config.yml‎

Lines changed: 220 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,13 @@ version: 2.1
22

33
# How to test the Linux jobs:
44
# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
5-
# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel
6-
# - Replace binary_linux_wheel with the name of the job you want to test
5+
# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7
6+
# - Replace binary_linux_wheel_py3.7 with the name of the job you want to test.
7+
# Job names are 'name:' key.
78

89
binary_common: &binary_common
910
parameters:
10-
# Edit these defaults to do a release`
11+
# Edit these defaults to do a release
1112
build_version:
1213
description: "version number of release binary; by default, build a nightly"
1314
type: string
@@ -29,7 +30,7 @@ binary_common: &binary_common
2930
BUILD_VERSION: << parameters.build_version >>
3031
PYTORCH_VERSION: << parameters.pytorch_version >>
3132
UNICODE_ABI: << parameters.unicode_abi >>
32-
CUDA_VERSION: cpu
33+
CU_VERSION: cpu
3334

3435
jobs:
3536
circleci_consistency:
@@ -53,6 +54,10 @@ jobs:
5354
- run: packaging/build_wheel.sh
5455
- store_artifacts:
5556
path: dist
57+
- persist_to_workspace:
58+
root: dist
59+
paths:
60+
- "*"
5661

5762
binary_linux_conda:
5863
<<: *binary_common
@@ -64,6 +69,10 @@ jobs:
6469
- run: packaging/build_conda.sh
6570
- store_artifacts:
6671
path: /opt/conda/conda-bld/linux-64
72+
- persist_to_workspace:
73+
root: /opt/conda/conda-bld/linux-64
74+
paths:
75+
- "*"
6776

6877
binary_macos_wheel:
6978
<<: *binary_common
@@ -82,6 +91,10 @@ jobs:
8291
packaging/build_wheel.sh
8392
- store_artifacts:
8493
path: dist
94+
- persist_to_workspace:
95+
root: dist
96+
paths:
97+
- "*"
8598

8699
binary_macos_conda:
87100
<<: *binary_common
@@ -98,11 +111,55 @@ jobs:
98111
packaging/build_conda.sh
99112
- store_artifacts:
100113
path: /Users/distiller/miniconda3/conda-bld/osx-64
114+
- persist_to_workspace:
115+
root: /Users/distiller/miniconda3/conda-bld/osx-64
116+
paths:
117+
- "*"
118+
119+
# Requires org-member context
120+
binary_conda_upload:
121+
docker:
122+
- image: continuumio/miniconda
123+
steps:
124+
- attach_workspace:
125+
at: ~/workspace
126+
- run:
127+
command: |
128+
# Prevent credential from leaking
129+
conda install -yq anaconda-client
130+
set +x
131+
anaconda login \
132+
--username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \
133+
--password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD"
134+
set -x
135+
anaconda upload ~/workspace/*.tar.bz2 -u pytorch-nightly --label main --no-progress --force
136+
137+
# Requires org-member context
138+
binary_wheel_upload:
139+
docker:
140+
- image: circleci/python:3.7
141+
steps:
142+
- attach_workspace:
143+
at: ~/workspace
144+
- checkout
145+
- run:
146+
command: |
147+
pip install --user awscli
148+
export PATH="$HOME/.local/bin:$PATH"
149+
# Prevent credential from leaking
150+
set +x
151+
export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
152+
export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
153+
set -x
154+
for pkg in ~/workspace/*.whl; do
155+
aws s3 cp "$pkg" "s3://pytorch/whl/nightly/" --acl public-read
156+
done
101157
102158
workflows:
103159
build:
104160
jobs:
105161
- circleci_consistency
162+
106163
- binary_linux_wheel:
107164
name: binary_linux_wheel_py2.7
108165
python_version: "2.7"
@@ -158,4 +215,162 @@ workflows:
158215
python_version: "3.6"
159216
- binary_macos_conda:
160217
name: binary_macos_conda_py3.7
161-
python_version: "3.7"
218+
python_version: "3.7"
219+
220+
nightly:
221+
triggers:
222+
- schedule:
223+
cron: "0 9 * * *"
224+
filters:
225+
branches:
226+
only:
227+
- master
228+
jobs:
229+
- circleci_consistency
230+
231+
- binary_linux_wheel:
232+
name: nightly_binary_linux_wheel_py2.7
233+
python_version: "2.7"
234+
- binary_wheel_upload:
235+
name: nightly_binary_linux_wheel_py2.7_upload
236+
context: org-member
237+
requires:
238+
- nightly_binary_linux_wheel_py2.7
239+
- binary_linux_wheel:
240+
name: nightly_binary_linux_wheel_py2.7_unicode
241+
python_version: "2.7"
242+
unicode_abi: "1"
243+
- binary_wheel_upload:
244+
name: nightly_binary_linux_wheel_py2.7_unicode_upload
245+
context: org-member
246+
requires:
247+
- nightly_binary_linux_wheel_py2.7_unicode
248+
- binary_linux_wheel:
249+
name: nightly_binary_linux_wheel_py3.5
250+
python_version: "3.5"
251+
- binary_wheel_upload:
252+
name: nightly_binary_linux_wheel_py3.5_upload
253+
context: org-member
254+
requires:
255+
- nightly_binary_linux_wheel_py3.5
256+
- binary_linux_wheel:
257+
name: nightly_binary_linux_wheel_py3.6
258+
python_version: "3.6"
259+
- binary_wheel_upload:
260+
name: nightly_binary_linux_wheel_py3.6_upload
261+
context: org-member
262+
requires:
263+
- nightly_binary_linux_wheel_py3.6
264+
- binary_linux_wheel:
265+
name: nightly_binary_linux_wheel_py3.7
266+
python_version: "3.7"
267+
- binary_wheel_upload:
268+
name: nightly_binary_linux_wheel_py3.7_upload
269+
context: org-member
270+
requires:
271+
- nightly_binary_linux_wheel_py3.7
272+
- binary_macos_wheel:
273+
name: nightly_binary_macos_wheel_py2.7
274+
python_version: "2.7"
275+
- binary_wheel_upload:
276+
name: nightly_binary_macos_wheel_py2.7_upload
277+
context: org-member
278+
requires:
279+
- nightly_binary_macos_wheel_py2.7
280+
- binary_macos_wheel:
281+
name: nightly_binary_macos_wheel_py2.7_unicode
282+
python_version: "2.7"
283+
unicode_abi: "1"
284+
- binary_wheel_upload:
285+
name: nightly_binary_macos_wheel_py2.7_unicode_upload
286+
context: org-member
287+
requires:
288+
- nightly_binary_macos_wheel_py2.7_unicode
289+
- binary_macos_wheel:
290+
name: nightly_binary_macos_wheel_py3.5
291+
python_version: "3.5"
292+
- binary_wheel_upload:
293+
name: nightly_binary_macos_wheel_py3.5_upload
294+
context: org-member
295+
requires:
296+
- nightly_binary_macos_wheel_py3.5
297+
- binary_macos_wheel:
298+
name: nightly_binary_macos_wheel_py3.6
299+
python_version: "3.6"
300+
- binary_wheel_upload:
301+
name: nightly_binary_macos_wheel_py3.6_upload
302+
context: org-member
303+
requires:
304+
- nightly_binary_macos_wheel_py3.6
305+
- binary_macos_wheel:
306+
name: nightly_binary_macos_wheel_py3.7
307+
python_version: "3.7"
308+
- binary_wheel_upload:
309+
name: nightly_binary_macos_wheel_py3.7_upload
310+
context: org-member
311+
requires:
312+
- nightly_binary_macos_wheel_py3.7
313+
- binary_linux_conda:
314+
name: nightly_binary_linux_conda_py2.7
315+
python_version: "2.7"
316+
- binary_conda_upload:
317+
name: nightly_binary_linux_conda_py2.7_upload
318+
context: org-member
319+
requires:
320+
- nightly_binary_linux_conda_py2.7
321+
- binary_linux_conda:
322+
name: nightly_binary_linux_conda_py3.5
323+
python_version: "3.5"
324+
- binary_conda_upload:
325+
name: nightly_binary_linux_conda_py3.5_upload
326+
context: org-member
327+
requires:
328+
- nightly_binary_linux_conda_py3.5
329+
- binary_linux_conda:
330+
name: nightly_binary_linux_conda_py3.6
331+
python_version: "3.6"
332+
- binary_conda_upload:
333+
name: nightly_binary_linux_conda_py3.6_upload
334+
context: org-member
335+
requires:
336+
- nightly_binary_linux_conda_py3.6
337+
- binary_linux_conda:
338+
name: nightly_binary_linux_conda_py3.7
339+
python_version: "3.7"
340+
- binary_conda_upload:
341+
name: nightly_binary_linux_conda_py3.7_upload
342+
context: org-member
343+
requires:
344+
- nightly_binary_linux_conda_py3.7
345+
- binary_macos_conda:
346+
name: nightly_binary_macos_conda_py2.7
347+
python_version: "2.7"
348+
- binary_conda_upload:
349+
name: nightly_binary_macos_conda_py2.7_upload
350+
context: org-member
351+
requires:
352+
- nightly_binary_macos_conda_py2.7
353+
- binary_macos_conda:
354+
name: nightly_binary_macos_conda_py3.5
355+
python_version: "3.5"
356+
- binary_conda_upload:
357+
name: nightly_binary_macos_conda_py3.5_upload
358+
context: org-member
359+
requires:
360+
- nightly_binary_macos_conda_py3.5
361+
- binary_macos_conda:
362+
name: nightly_binary_macos_conda_py3.6
363+
python_version: "3.6"
364+
- binary_conda_upload:
365+
name: nightly_binary_macos_conda_py3.6_upload
366+
context: org-member
367+
requires:
368+
- nightly_binary_macos_conda_py3.6
369+
- binary_macos_conda:
370+
name: nightly_binary_macos_conda_py3.7
371+
python_version: "3.7"
372+
- binary_conda_upload:
373+
name: nightly_binary_macos_conda_py3.7_upload
374+
context: org-member
375+
requires:
376+
- nightly_binary_macos_conda_py3.7

0 commit comments

Comments
 (0)