Skip to content

Commit 45d1da5

Browse files
ezyangezyang
committed
Rewrite our packaging from scratch (#217)
* New entry points are packaging/build_wheel.sh and packaging/build_conda.sh. The only mandatory environment variable you have to set is PYTHON_VERSION * CircleCI configuration uses 2.1-style parametrized builds to let you toggle python version, etc. as you do builds. We create a separate job per build configuration for maximum parallelism * build_tools/packaging got moved to packaging, to be in-line with directory structure in torchvision * The build_conda.sh and build_wheel.sh delegate most of the heavy lifting to pkg_helpers.bash, which defines a number of bash functions for performing common operations. The intent is that I'll copy-paste this file between other domain API projects. * TORCHAUDIO_ prefix removed from envvars, so that I can more easily share packaging scripts between projects * BUILD_VERSION is completely gone; just change the version number if you need to rebuild * No more logic for cloning and checking out a fresh copy of torchaudio
1 parent b2c73b6 commit 45d1da5

File tree

14 files changed

+492
-181
lines changed

14 files changed

+492
-181
lines changed

‎.circleci/config.yml‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
version: 2.1
2+
3+
# How to test the Linux jobs:
4+
# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
5+
# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel
6+
# - Replace binary_linux_wheel with the name of the job you want to test
7+
8+
binary_common: &binary_common
9+
parameters:
10+
# Edit these defaults to do a release`
11+
build_version:
12+
description: "version number of release binary; by default, build a nightly"
13+
type: string
14+
default: ""
15+
pytorch_version:
16+
description: "PyTorch version to build against; by default, use a nightly"
17+
type: string
18+
default: ""
19+
# Don't edit these
20+
python_version:
21+
description: "Python version to build against (e.g., 3.7)"
22+
type: string
23+
unicode_abi:
24+
description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)"
25+
type: string
26+
default: ""
27+
environment:
28+
PYTHON_VERSION: << parameters.python_version >>
29+
BUILD_VERSION: << parameters.build_version >>
30+
PYTORCH_VERSION: << parameters.pytorch_version >>
31+
UNICODE_ABI: << parameters.unicode_abi >>
32+
CUDA_VERSION: cpu
33+
34+
jobs:
35+
circleci_consistency:
36+
docker:
37+
- image: circleci/python:3.7
38+
steps:
39+
- checkout
40+
- run:
41+
command: |
42+
pip install --user --progress-bar off jinja2
43+
python .circleci/regenerate.py
44+
git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)
45+
46+
binary_linux_wheel:
47+
<<: *binary_common
48+
docker:
49+
- image: "soumith/manylinux-cuda100"
50+
resource_class: 2xlarge+
51+
steps:
52+
- checkout
53+
- run: packaging/build_wheel.sh
54+
- store_artifacts:
55+
path: dist
56+
57+
binary_linux_conda:
58+
<<: *binary_common
59+
docker:
60+
- image: "soumith/conda-cuda"
61+
resource_class: 2xlarge+
62+
steps:
63+
- checkout
64+
- run: packaging/build_conda.sh
65+
- store_artifacts:
66+
path: /opt/conda/conda-bld/linux-64
67+
68+
binary_macos_wheel:
69+
<<: *binary_common
70+
macos:
71+
xcode: "9.0"
72+
steps:
73+
- checkout
74+
- run:
75+
# Cannot easily deduplicate this as source'ing activate
76+
# will set environment variables which we need to propagate
77+
# to build_wheel.sh
78+
command: |
79+
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
80+
sh conda.sh -b
81+
source $HOME/miniconda3/bin/activate
82+
packaging/build_wheel.sh
83+
- store_artifacts:
84+
path: dist
85+
86+
binary_macos_conda:
87+
<<: *binary_common
88+
macos:
89+
xcode: "9.0"
90+
steps:
91+
- checkout
92+
- run:
93+
command: |
94+
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
95+
sh conda.sh -b
96+
source $HOME/miniconda3/bin/activate
97+
conda install -yq conda-build
98+
packaging/build_conda.sh
99+
- store_artifacts:
100+
path: /Users/distiller/miniconda3/conda-bld/osx-64
101+
102+
workflows:
103+
build:
104+
jobs:
105+
- circleci_consistency
106+
- binary_linux_wheel:
107+
name: binary_linux_wheel_py2.7
108+
python_version: "2.7"
109+
- binary_linux_wheel:
110+
name: binary_linux_wheel_py2.7_unicode
111+
python_version: "2.7"
112+
unicode_abi: "1"
113+
- binary_linux_wheel:
114+
name: binary_linux_wheel_py3.5
115+
python_version: "3.5"
116+
- binary_linux_wheel:
117+
name: binary_linux_wheel_py3.6
118+
python_version: "3.6"
119+
- binary_linux_wheel:
120+
name: binary_linux_wheel_py3.7
121+
python_version: "3.7"
122+
- binary_macos_wheel:
123+
name: binary_macos_wheel_py2.7
124+
python_version: "2.7"
125+
- binary_macos_wheel:
126+
name: binary_macos_wheel_py2.7_unicode
127+
python_version: "2.7"
128+
unicode_abi: "1"
129+
- binary_macos_wheel:
130+
name: binary_macos_wheel_py3.5
131+
python_version: "3.5"
132+
- binary_macos_wheel:
133+
name: binary_macos_wheel_py3.6
134+
python_version: "3.6"
135+
- binary_macos_wheel:
136+
name: binary_macos_wheel_py3.7
137+
python_version: "3.7"
138+
- binary_linux_conda:
139+
name: binary_linux_conda_py2.7
140+
python_version: "2.7"
141+
- binary_linux_conda:
142+
name: binary_linux_conda_py3.5
143+
python_version: "3.5"
144+
- binary_linux_conda:
145+
name: binary_linux_conda_py3.6
146+
python_version: "3.6"
147+
- binary_linux_conda:
148+
name: binary_linux_conda_py3.7
149+
python_version: "3.7"
150+
- binary_macos_conda:
151+
name: binary_macos_conda_py2.7
152+
python_version: "2.7"
153+
- binary_macos_conda:
154+
name: binary_macos_conda_py3.5
155+
python_version: "3.5"
156+
- binary_macos_conda:
157+
name: binary_macos_conda_py3.6
158+
python_version: "3.6"
159+
- binary_macos_conda:
160+
name: binary_macos_conda_py3.7
161+
python_version: "3.7"

‎.circleci/config.yml.in‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
version: 2.1
2+
3+
# How to test the Linux jobs:
4+
# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
5+
# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel
6+
# - Replace binary_linux_wheel with the name of the job you want to test
7+
8+
binary_common: &binary_common
9+
parameters:
10+
# Edit these defaults to do a release`
11+
build_version:
12+
description: "version number of release binary; by default, build a nightly"
13+
type: string
14+
default: ""
15+
pytorch_version:
16+
description: "PyTorch version to build against; by default, use a nightly"
17+
type: string
18+
default: ""
19+
# Don't edit these
20+
python_version:
21+
description: "Python version to build against (e.g., 3.7)"
22+
type: string
23+
unicode_abi:
24+
description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)"
25+
type: string
26+
default: ""
27+
environment:
28+
PYTHON_VERSION: << parameters.python_version >>
29+
BUILD_VERSION: << parameters.build_version >>
30+
PYTORCH_VERSION: << parameters.pytorch_version >>
31+
UNICODE_ABI: << parameters.unicode_abi >>
32+
CUDA_VERSION: cpu
33+
34+
jobs:
35+
circleci_consistency:
36+
docker:
37+
- image: circleci/python:3.7
38+
steps:
39+
- checkout
40+
- run:
41+
command: |
42+
pip install --user --progress-bar off jinja2
43+
python .circleci/regenerate.py
44+
git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)
45+
46+
binary_linux_wheel:
47+
<<: *binary_common
48+
docker:
49+
- image: "soumith/manylinux-cuda100"
50+
resource_class: 2xlarge+
51+
steps:
52+
- checkout
53+
- run: packaging/build_wheel.sh
54+
- store_artifacts:
55+
path: dist
56+
57+
binary_linux_conda:
58+
<<: *binary_common
59+
docker:
60+
- image: "soumith/conda-cuda"
61+
resource_class: 2xlarge+
62+
steps:
63+
- checkout
64+
- run: packaging/build_conda.sh
65+
- store_artifacts:
66+
path: /opt/conda/conda-bld/linux-64
67+
68+
binary_macos_wheel:
69+
<<: *binary_common
70+
macos:
71+
xcode: "9.0"
72+
steps:
73+
- checkout
74+
- run:
75+
# Cannot easily deduplicate this as source'ing activate
76+
# will set environment variables which we need to propagate
77+
# to build_wheel.sh
78+
command: |
79+
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
80+
sh conda.sh -b
81+
source $HOME/miniconda3/bin/activate
82+
packaging/build_wheel.sh
83+
- store_artifacts:
84+
path: dist
85+
86+
binary_macos_conda:
87+
<<: *binary_common
88+
macos:
89+
xcode: "9.0"
90+
steps:
91+
- checkout
92+
- run:
93+
command: |
94+
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
95+
sh conda.sh -b
96+
source $HOME/miniconda3/bin/activate
97+
conda install -yq conda-build
98+
packaging/build_conda.sh
99+
- store_artifacts:
100+
path: /Users/distiller/miniconda3/conda-bld/osx-64
101+
102+
workflows:
103+
build:
104+
jobs:
105+
- circleci_consistency
106+
{%- for btype in ["wheel", "conda"] -%}
107+
{%- for os in ["linux", "macos"] -%}
108+
{%- for python_version in ["2.7", "3.5", "3.6", "3.7"] %}
109+
- binary_{{os}}_{{btype}}:
110+
name: binary_{{os}}_{{btype}}_py{{python_version}}
111+
python_version: "{{python_version}}"
112+
{%- if btype == "wheel" and python_version == "2.7" %}
113+
- binary_{{os}}_{{btype}}:
114+
name: binary_{{os}}_{{btype}}_py{{python_version}}_unicode
115+
python_version: "{{python_version}}"
116+
unicode_abi: "1"
117+
{%- endif -%}
118+
{%- endfor -%}
119+
{%- endfor -%}
120+
{%- endfor -%}

‎.circleci/regenerate.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/usr/bin/env python
2+
3+
import jinja2
4+
import os.path
5+
6+
d = os.path.dirname(__file__)
7+
env = jinja2.Environment(
8+
loader=jinja2.FileSystemLoader(d),
9+
lstrip_blocks=True,
10+
autoescape=False,
11+
)
12+
with open(os.path.join(d, 'config.yml'), 'w') as f:
13+
f.write(env.get_template('config.yml.in').render())

‎build_tools/packaging/conda/build_audio.sh‎

Lines changed: 0 additions & 48 deletions
This file was deleted.

‎build_tools/packaging/wheel/linux_manywheel.sh‎

Lines changed: 0 additions & 46 deletions
This file was deleted.

0 commit comments

Comments
 (0)