Skip to content

Commit 2b1a3be

Browse files
holly1238holly1238
authored
Merge branch 'master' into master
2 parents 6a01dad + fa83e33 commit 2b1a3be

File tree

291 files changed

+29282
-3635
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

291 files changed

+29282
-3635
lines changed

‎.circleci/config.yml‎

Lines changed: 408 additions & 49 deletions
Large diffs are not rendered by default.

‎.circleci/scripts/build_for_windows.sh‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
#!/bin/bash
2+
set -eux -o pipefail
3+
4+
retry () {
5+
$* || (sleep 1 && $*) || (sleep 2 && $*)
6+
}
7+
8+
SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
9+
PROJECT_DIR="${SOURCE_DIR}/../.."
10+
pushd $SOURCE_DIR
11+
12+
#install wget and make
13+
curl -k https://ymu.dl.osdn.jp/mingw/68260/mingw-get-0.6.3-mingw32-pre-20170905-1-bin.zip -o mingw32.zip
14+
unzip mingw32.zip -d mingw32
15+
mingw32/bin/mingw-get.exe install mingw32-make
16+
mingw32/bin/mingw-get.exe install msys-findutils
17+
mv mingw32/bin/mingw32-make.exe mingw32/bin/make.exe
18+
curl -k https://eternallybored.org/misc/wget/1.20.3/64/wget.exe -o mingw32/bin/wget.exe
19+
export PATH="${SOURCE_DIR}/mingw32/bin:${SOURCE_DIR}/mingw32/msys/1.0/bin:$PATH"
20+
21+
#install anaconda3
22+
export CONDA_HOME="${SOURCE_DIR}/conda"
23+
export tmp_conda="${SOURCE_DIR}/conda"
24+
export miniconda_exe="${SOURCE_DIR}/miniconda.exe"
25+
rm -rf conda miniconda.exe
26+
curl -k https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -o miniconda.exe
27+
./install_conda.bat
28+
export PATH="${tmp_conda}:${tmp_conda}/Library/usr/bin:${tmp_conda}/Library/bin:${tmp_conda}/Scripts:${tmp_conda}/bin:$PATH"
29+
30+
eval "$(conda shell.bash hook)"
31+
conda create -qyn testenv python=3.7
32+
conda activate testenv
33+
34+
REQUIREMENTS="$(grep -v '^ *#\|^torch\|^torchaudio\|^torchvision|^torchtext' $PROJECT_DIR/requirements.txt | grep .)"
35+
echo $REQUIREMENTS > requirements.txt
36+
pip install -r requirements.txt
37+
pip install pySoundFile
38+
# Force uninstall torch & related packages, we'll install them using conda later.
39+
pip uninstall -y torch torchvision torchtext
40+
conda install -yq -c pytorch "cudatoolkit=10.1" pytorch torchvision torchtext torchaudio
41+
python -m spacy download de
42+
python -m spacy download en
43+
pushd ${PROJECT_DIR}
44+
DIR=.jenkins
45+
export NUM_WORKERS=4
46+
47+
if [[ "${CIRCLE_JOB}" == *worker_* ]]; then
48+
python $DIR/remove_runnable_code.py intermediate_source/model_parallel_tutorial.py intermediate_source/model_parallel_tutorial.py || true
49+
python $DIR/remove_runnable_code.py advanced_source/static_quantization_tutorial.py advanced_source/static_quantization_tutorial.py || true
50+
python $DIR/remove_runnable_code.py beginner_source/hyperparameter_tuning_tutorial.py beginner_source/hyperparameter_tuning_tutorial.py || true
51+
python $DIR/remove_runnable_code.py beginner_source/audio_preprocessing_tutorial.py beginner_source/audio_preprocessing_tutorial.py || true
52+
# Temp remove for mnist download issue.
53+
python $DIR/remove_runnable_code.py beginner_source/fgsm_tutorial.py beginner_source/fgsm_tutorial.py || true
54+
55+
export WORKER_ID=$(echo "${CIRCLE_JOB}" | tr -dc '0-9')
56+
count=0
57+
FILES_TO_RUN=()
58+
for work_directory in beginner_source intermediate_source advanced_source recipes_source prototype_source; do
59+
for filename in $(find $work_directory -name '\*.py' -not -path '\*/data/\*'); do
60+
if [ $(($count % $NUM_WORKERS)) != $WORKER_ID ]; then
61+
echo "Removing runnable code from "$filename
62+
python $DIR/remove_runnable_code.py $filename $filename
63+
else
64+
echo "Keeping "$filename
65+
FILES_TO_RUN+=($(basename $filename .py))
66+
fi
67+
count=$((count+1))
68+
done
69+
done
70+
echo "FILES_TO_RUN: " ${FILES_TO_RUN[@]}
71+
fi
72+
73+
if [[ ! -d advanced_source/data || ! -d beginner_source/data || ! -d intermediate_source/data || ! -d prototype_source/data ]];then
74+
make download
75+
fi
76+
77+
make html

‎.circleci/scripts/install_conda.bat‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
start /wait "" "%miniconda_exe%" /S /InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /D=%tmp_conda%

‎.circleci/scripts/windows_cuda_install.sh‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/bin/bash
2+
set -eux -o pipefail
3+
4+
curl --retry 3 -kLO https://ossci-windows.s3.amazonaws.com/cuda_10.1.243_426.00_win10.exe
5+
7z x cuda_10.1.243_426.00_win10.exe -ocuda_10.1.243_426.00_win10
6+
cd cuda_10.1.243_426.00_win10
7+
mkdir cuda_install_logs
8+
9+
set +e
10+
11+
./setup.exe -s nvcc_10.1 cuobjdump_10.1 nvprune_10.1 cupti_10.1 cublas_10.1 cublas_dev_10.1 cudart_10.1 cufft_10.1 cufft_dev_10.1 curand_10.1 curand_dev_10.1 cusolver_10.1 cusolver_dev_10.1 cusparse_10.1 cusparse_dev_10.1 nvgraph_10.1 nvgraph_dev_10.1 npp_10.1 npp_dev_10.1 nvrtc_10.1 nvrtc_dev_10.1 nvml_dev_10.1 -loglevel:6 -log:"$(pwd -W)/cuda_install_logs"
12+
13+
set -e
14+
15+
curl --retry 3 -kLO https://ossci-windows.s3.amazonaws.com/NvToolsExt.7z
16+
7z x NvToolsExt.7z -oNvToolsExt
17+
mkdir -p "C:/Program Files/NVIDIA Corporation/NvToolsExt"
18+
cp -r NvToolsExt/* "C:/Program Files/NVIDIA Corporation/NvToolsExt/"
19+
export NVTOOLSEXT_PATH="C:\\Program Files\\NVIDIA Corporation\\NvToolsExt\\"
20+
21+
if ! ls "/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/bin/nvcc.exe"
22+
then
23+
echo "CUDA installation failed"
24+
mkdir -p /c/w/build-results
25+
7z a "c:\\w\\build-results\\cuda_install_logs.7z" cuda_install_logs
26+
exit 1
27+
fi
28+
29+
cd ..
30+
rm -rf ./cuda_10.1.243_426.00_win10
31+
rm -f ./cuda_10.1.243_426.00_win10.exe

‎.devcontainer/Dockerfile‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
FROM mcr.microsoft.com/vscode/devcontainers/python:3.8
2+
3+
COPY requirements.txt /tmp/pip-tmp/
4+
5+
RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
6+
&& apt-get install git gcc unzip make -y \
7+
&& pip3 install --disable-pip-version-check --no-cache-dir -r /tmp/pip-tmp/requirements.txt \
8+
&& rm -rf /tmp/pip-tmp

‎.devcontainer/devcontainer.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"name": "PyTorch Tutorials",
3+
"build": {
4+
"context": "..",
5+
"dockerfile": "Dockerfile",
6+
"args": {}
7+
},
8+
"settings": {
9+
"terminal.integrated.shell.linux": "/bin/bash",
10+
"workbench.startupEditor": "none",
11+
"files.autoSave": "afterDelay",
12+
"python.dataScience.enabled": true,
13+
"python.dataScience.alwaysTrustNotebooks": true,
14+
"python.insidersChannel": "weekly",
15+
"python.showStartPage": false
16+
},
17+
"extensions": ["ms-python.python", "lextudio.restructuredtext"]
18+
}

‎.devcontainer/requirements.txt‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Refer to ./jenkins/build.sh for tutorial build instructions
2+
3+
sphinx==1.8.2
4+
sphinx-gallery==0.3.1
5+
tqdm
6+
numpy
7+
matplotlib
8+
torch
9+
torchvision
10+
torchtext
11+
torchaudio
12+
PyHamcrest
13+
bs4
14+
awscli==1.16.35
15+
flask
16+
spacy
17+
ray[tune]
18+
19+
# PyTorch Theme
20+
-e git+git://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme
21+
22+
ipython
23+
24+
# to run examples
25+
pandas
26+
scikit-image
27+
pillow==8.1.1
28+
wget
29+
30+
# for codespaces env
31+
pylint

‎.gitignore‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,22 @@ beginner
33
intermediate
44
advanced
55
pytorch_basics
6+
recipes
7+
prototype
68

79
#data things
810
_data/
9-
beginner_source/hymenoptera_data
10-
beginner_source/blitz/data
11-
beginner_source/faces
12-
beginner_source/hybrid_frontend/data
13-
intermediate_source/data/
1411
advanced_source/images/
15-
*data.zip
16-
faces.zip
12+
advanced_source/data/
13+
beginner_source/.data/
14+
beginner_source/data/
15+
beginner_source/blitz/data/
16+
beginner_source/faces/
17+
beginner_source/hybrid_frontend/data/
18+
beginner_source/hymenoptera_data/
19+
intermediate_source/data/
20+
*.zip
21+
MNIST/
1722

1823
#builds
1924
_build/
@@ -28,6 +33,7 @@ __pycache__/
2833
*.so
2934

3035
# Distribution / packaging
36+
src/
3137
.Python
3238
env/
3339
build/
@@ -94,7 +100,7 @@ target/
94100
.python-version
95101

96102
# celery beat schedule file
97-
celerybeat-schedule
103+
celerybeat-schedule
98104
# dotenv
99105
.env
100106

@@ -112,3 +118,6 @@ ENV/
112118
.DS_Store
113119
cleanup.sh
114120
*.swp
121+
122+
# PyTorch things
123+
*.pt

‎.jenkins/build.sh‎

Lines changed: 42 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@ else
66
export BUCKET_NAME=pytorch-tutorial-build-pull-request
77
fi
88

9+
# set locale for click dependency in spacy
10+
export LC_ALL=C.UTF-8
11+
export LANG=C.UTF-8
12+
913
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
1014

1115
sudo apt-get update
@@ -15,8 +19,14 @@ export PATH=/opt/conda/bin:$PATH
1519
rm -rf src
1620
pip install -r $DIR/../requirements.txt
1721

18-
export PATH=/opt/conda/bin:$PATH
19-
pip install sphinx==1.8.2 pandas
22+
# export PATH=/opt/conda/bin:$PATH
23+
# pip install sphinx==1.8.2 pandas
24+
25+
#Install PyTorch Nightly for test.
26+
# Nightly - pip install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/nightly/cu102/torch_nightly.html
27+
# RC Link
28+
# pip uninstall -y torch torchvision torchaudio torchtext
29+
# pip install -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext
2030

2131
# For Tensorboard. Until 1.14 moves to the release channel.
2232
pip install tb-nightly
@@ -28,9 +38,7 @@ python -m spacy download de
2838
# PyTorch Theme
2939
rm -rf src
3040
pip install -e git+git://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme
31-
# pillow >= 4.2 will throw error when trying to write mode RGBA as JPEG,
32-
# this is a workaround to the issue.
33-
pip install sphinx-gallery==0.3.1 tqdm matplotlib ipython pillow==4.1.1
41+
pip install sphinx-gallery==0.3.1 tqdm matplotlib ipython pillow==8.1.0
3442

3543
aws configure set default.s3.multipart_threshold 5120MB
3644

@@ -39,6 +47,11 @@ export NUM_WORKERS=20
3947
if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then
4048
# Step 1: Remove runnable code from tutorials that are not supposed to be run
4149
python $DIR/remove_runnable_code.py beginner_source/aws_distributed_training_tutorial.py beginner_source/aws_distributed_training_tutorial.py || true
50+
# python $DIR/remove_runnable_code.py advanced_source/ddp_pipeline_tutorial.py advanced_source/ddp_pipeline_tutorial.py || true
51+
# Temp remove for mnist download issue. (Re-enabled for 1.8.1)
52+
# python $DIR/remove_runnable_code.py beginner_source/fgsm_tutorial.py beginner_source/fgsm_tutorial.py || true
53+
# python $DIR/remove_runnable_code.py intermediate_source/spatial_transformer_tutorial.py intermediate_source/spatial_transformer_tutorial.py || true
54+
4255
# TODO: Fix bugs in these tutorials to make them runnable again
4356
# python $DIR/remove_runnable_code.py beginner_source/audio_classifier_tutorial.py beginner_source/audio_classifier_tutorial.py || true
4457

@@ -76,6 +89,26 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then
7689
FILES_TO_RUN+=($(basename $filename .py))
7790
fi
7891
count=$((count+1))
92+
done
93+
for filename in $(find recipes_source/ -name '*.py' -not -path '*/data/*'); do
94+
if [ $(($count % $NUM_WORKERS)) != $WORKER_ID ]; then
95+
echo "Removing runnable code from "$filename
96+
python $DIR/remove_runnable_code.py $filename $filename
97+
else
98+
echo "Keeping "$filename
99+
FILES_TO_RUN+=($(basename $filename .py))
100+
fi
101+
count=$((count+1))
102+
done
103+
for filename in $(find prototype_source/ -name '*.py' -not -path '*/data/*'); do
104+
if [ $(($count % $NUM_WORKERS)) != $WORKER_ID ]; then
105+
echo "Removing runnable code from "$filename
106+
python $DIR/remove_runnable_code.py $filename $filename
107+
else
108+
echo "Keeping "$filename
109+
FILES_TO_RUN+=($(basename $filename .py))
110+
fi
111+
count=$((count+1))
79112
done
80113
echo "FILES_TO_RUN: " ${FILES_TO_RUN[@]}
81114

@@ -84,13 +117,13 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then
84117

85118
# Step 4: If any of the generated files are not related the tutorial files we want to run,
86119
# then we remove them
87-
for filename in $(find docs/beginner docs/intermediate docs/advanced -name '*.html'); do
120+
for filename in $(find docs/beginner docs/intermediate docs/advanced docs/recipes docs/prototype -name '*.html'); do
88121
file_basename=$(basename $filename .html)
89122
if [[ ! " ${FILES_TO_RUN[@]} " =~ " ${file_basename} " ]]; then
90123
rm $filename
91124
fi
92125
done
93-
for filename in $(find docs/beginner docs/intermediate docs/advanced -name '*.rst'); do
126+
for filename in $(find docs/beginner docs/intermediate docs/advanced docs/recipes docs/prototype -name '*.rst'); do
94127
file_basename=$(basename $filename .rst)
95128
if [[ ! " ${FILES_TO_RUN[@]} " =~ " ${file_basename} " ]]; then
96129
rm $filename
@@ -108,13 +141,13 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then
108141
rm $filename
109142
fi
110143
done
111-
for filename in $(find docs/_sources/beginner docs/_sources/intermediate docs/_sources/advanced -name '*.rst.txt'); do
144+
for filename in $(find docs/_sources/beginner docs/_sources/intermediate docs/_sources/advanced docs/_sources/recipes -name '*.rst.txt'); do
112145
file_basename=$(basename $filename .rst.txt)
113146
if [[ ! " ${FILES_TO_RUN[@]} " =~ " ${file_basename} " ]]; then
114147
rm $filename
115148
fi
116149
done
117-
for filename in $(find docs/.doctrees/beginner docs/.doctrees/intermediate docs/.doctrees/advanced -name '*.doctree'); do
150+
for filename in $(find docs/.doctrees/beginner docs/.doctrees/intermediate docs/.doctrees/advanced docs/.doctrees/recipes docs/.doctrees/prototype -name '*.doctree'); do
118151
file_basename=$(basename $filename .doctree)
119152
if [[ ! " ${FILES_TO_RUN[@]} " =~ " ${file_basename} " ]]; then
120153
rm $filename

‎.jenkins/remove_runnable_code.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,17 @@
1616
if line.startswith('#'):
1717
ret_lines.append(line)
1818
state = STATE_NORMAL
19+
elif ((line.startswith('"""') or line.startswith('r"""')) and
20+
line.endswith('"""')):
21+
ret_lines.append(line)
22+
state = STATE_NORMAL
1923
elif line.startswith('"""') or line.startswith('r"""'):
2024
ret_lines.append(line)
2125
state = STATE_IN_MULTILINE_COMMENT_BLOCK_DOUBLE_QUOTE
26+
elif ((line.startswith("'''") or line.startswith("r'''")) and
27+
line.endswith("'''")):
28+
ret_lines.append(line)
29+
state = STATE_NORMAL
2230
elif line.startswith("'''") or line.startswith("r'''"):
2331
ret_lines.append(line)
2432
state = STATE_IN_MULTILINE_COMMENT_BLOCK_SINGLE_QUOTE

0 commit comments

Comments
 (0)