Merge branch 'master' into master

Author: holly1238

.circleci/config.yml

@@ -0,0 +1,79 @@
+#!/bin/bash
+set -eux -o pipefail
+
+retry () {
+  $*  || (sleep 1 && $*) || (sleep 2 && $*)
+}
+
+SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+PROJECT_DIR="${SOURCE_DIR}/../.."
+pushd $SOURCE_DIR
+
+#install wget and make
+curl -k https://ymu.dl.osdn.jp/mingw/68260/mingw-get-0.6.3-mingw32-pre-20170905-1-bin.zip -o mingw32.zip
+unzip mingw32.zip -d mingw32
+mingw32/bin/mingw-get.exe install mingw32-make
+mingw32/bin/mingw-get.exe install msys-findutils
+mv mingw32/bin/mingw32-make.exe mingw32/bin/make.exe
+curl -k https://eternallybored.org/misc/wget/1.20.3/64/wget.exe -o mingw32/bin/wget.exe
+export PATH="${SOURCE_DIR}/mingw32/bin:${SOURCE_DIR}/mingw32/msys/1.0/bin:$PATH"
+
+#install anaconda3
+export CONDA_HOME="${SOURCE_DIR}/conda"
+export tmp_conda="${SOURCE_DIR}/conda"
+export miniconda_exe="${SOURCE_DIR}/miniconda.exe"
+rm -rf conda miniconda.exe
+curl -k https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -o miniconda.exe
+./install_conda.bat
+export PATH="${tmp_conda}:${tmp_conda}/Library/usr/bin:${tmp_conda}/Library/bin:${tmp_conda}/Scripts:${tmp_conda}/bin:$PATH"
+
+eval "$(conda shell.bash hook)"
+conda create -qyn testenv python=3.7
+conda activate testenv
+
+REQUIREMENTS="$(grep -v '^ *#\|^torch\|^torchaudio\|^torchvision|^torchtext' $PROJECT_DIR/requirements.txt  | grep .)"
+echo $REQUIREMENTS > requirements.txt
+pip install -r requirements.txt
+pip install pySoundFile
+# Force uninstall torch & related packages, we'll install them using conda later.
+pip uninstall -y torch torchvision torchtext
+conda install -yq -c pytorch "cudatoolkit=10.1" pytorch torchvision torchtext torchaudio
+python -m spacy download de
+python -m spacy download en
+pushd ${PROJECT_DIR}
+DIR=.jenkins
+export NUM_WORKERS=4
+
+if [[ "${CIRCLE_JOB}" == *worker_* ]]; then
+  python $DIR/remove_runnable_code.py intermediate_source/model_parallel_tutorial.py intermediate_source/model_parallel_tutorial.py || true
+  python $DIR/remove_runnable_code.py advanced_source/static_quantization_tutorial.py advanced_source/static_quantization_tutorial.py || true
+  python $DIR/remove_runnable_code.py beginner_source/hyperparameter_tuning_tutorial.py beginner_source/hyperparameter_tuning_tutorial.py || true
+  python $DIR/remove_runnable_code.py beginner_source/audio_preprocessing_tutorial.py  beginner_source/audio_preprocessing_tutorial.py || true
+  python $DIR/remove_runnable_code.py beginner_source/dcgan_faces_tutorial.py  beginner_source/dcgan_faces_tutorial.py || true
+  python $DIR/remove_runnable_code.py intermediate_source/tensorboard_profiler_tutorial.py intermediate_source/tensorboard_profiler_tutorial.py || true
+  # Temp remove for mnist download issue. (Re-enabled for 1.8.1)
+  # python $DIR/remove_runnable_code.py beginner_source/fgsm_tutorial.py  beginner_source/fgsm_tutorial.py || true
+
+  export WORKER_ID=$(echo "${CIRCLE_JOB}" | tr -dc '0-9')
+  count=0
+  FILES_TO_RUN=()
+  for work_directory  in beginner_source intermediate_source advanced_source recipes_source prototype_source; do
+    for filename in $(find $work_directory -name '\*.py' -not -path '\*/data/\*'); do
+      if [ $(($count % $NUM_WORKERS)) != $WORKER_ID ]; then
+        echo "Removing runnable code from "$filename
+        python $DIR/remove_runnable_code.py $filename $filename
+      else
+        echo "Keeping "$filename
+        FILES_TO_RUN+=($(basename $filename .py))
+      fi
+      count=$((count+1))
+    done
+  done
+  echo "FILES_TO_RUN: " ${FILES_TO_RUN[@]}
+fi
+
+if [[ ! -d advanced_source/data || ! -d beginner_source/data || ! -d intermediate_source/data || ! -d prototype_source/data ]];then
+    make download
+fi
+
+make html

.circleci/scripts/build_for_windows.sh

@@ -0,0 +1 @@
+start /wait "" "%miniconda_exe%" /S /InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /D=%tmp_conda%

.circleci/scripts/install_conda.bat

@@ -0,0 +1,31 @@
+#!/bin/bash
+set -eux -o pipefail
+
+curl --retry 3 -kLO https://ossci-windows.s3.amazonaws.com/cuda_10.1.243_426.00_win10.exe
+7z x cuda_10.1.243_426.00_win10.exe -ocuda_10.1.243_426.00_win10
+cd cuda_10.1.243_426.00_win10
+mkdir cuda_install_logs
+
+set +e
+
+./setup.exe -s nvcc_10.1 cuobjdump_10.1 nvprune_10.1 cupti_10.1 cublas_10.1 cublas_dev_10.1 cudart_10.1 cufft_10.1 cufft_dev_10.1 curand_10.1 curand_dev_10.1 cusolver_10.1 cusolver_dev_10.1 cusparse_10.1 cusparse_dev_10.1 nvgraph_10.1 nvgraph_dev_10.1 npp_10.1 npp_dev_10.1 nvrtc_10.1 nvrtc_dev_10.1 nvml_dev_10.1 -loglevel:6 -log:"$(pwd -W)/cuda_install_logs"
+
+set -e
+
+curl --retry 3 -kLO https://ossci-windows.s3.amazonaws.com/NvToolsExt.7z
+7z x NvToolsExt.7z -oNvToolsExt
+mkdir -p "C:/Program Files/NVIDIA Corporation/NvToolsExt"
+cp -r NvToolsExt/* "C:/Program Files/NVIDIA Corporation/NvToolsExt/"
+export NVTOOLSEXT_PATH="C:\\Program Files\\NVIDIA Corporation\\NvToolsExt\\"
+
+if ! ls "/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/bin/nvcc.exe"
+then
+    echo "CUDA installation failed"
+    mkdir -p /c/w/build-results
+    7z a "c:\\w\\build-results\\cuda_install_logs.7z" cuda_install_logs
+    exit 1
+fi
+
+cd ..
+rm -rf ./cuda_10.1.243_426.00_win10
+rm -f ./cuda_10.1.243_426.00_win10.exe

.circleci/scripts/windows_cuda_install.sh

@@ -0,0 +1,8 @@
+FROM mcr.microsoft.com/vscode/devcontainers/python:3.8
+
+COPY requirements.txt /tmp/pip-tmp/
+
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    && apt-get install git gcc unzip make -y \
+    && pip3 install --disable-pip-version-check --no-cache-dir -r /tmp/pip-tmp/requirements.txt \
+    && rm -rf /tmp/pip-tmp

.devcontainer/Dockerfile

@@ -0,0 +1,18 @@
+{
+  "name": "PyTorch Tutorials",
+  "build": {
+    "context": "..",
+    "dockerfile": "Dockerfile",
+    "args": {}
+  },
+  "settings": {
+    "terminal.integrated.shell.linux": "/bin/bash",
+    "workbench.startupEditor": "none",
+    "files.autoSave": "afterDelay",
+    "python.dataScience.enabled": true,
+    "python.dataScience.alwaysTrustNotebooks": true,
+    "python.insidersChannel": "weekly",
+    "python.showStartPage": false
+  },
+  "extensions": ["ms-python.python", "lextudio.restructuredtext"]
+}

.devcontainer/devcontainer.json

@@ -0,0 +1,31 @@
+# Refer to ./jenkins/build.sh for tutorial build instructions
+
+sphinx==1.8.2
+sphinx-gallery==0.3.1
+tqdm
+numpy
+matplotlib
+torch
+torchvision
+torchtext
+torchaudio
+PyHamcrest
+bs4
+awscli==1.16.35
+flask
+spacy
+ray[tune]
+
+# PyTorch Theme
+-e git+git://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme
+
+ipython
+
+# to run examples
+pandas
+scikit-image
+pillow==8.1.1
+wget
+
+# for codespaces env
+pylint

.devcontainer/requirements.txt

@@ -3,16 +3,22 @@ beginner
 intermediate
 advanced
 pytorch_basics
+recipes
+prototype
 
 #data things
-beginner_source/hymenoptera_data
-beginner_source/blitz/data
-beginner_source/faces
-beginner_source/hybrid_frontend/data
-intermediate_source/data/
+_data/
 advanced_source/images/
-*data.zip
-faces.zip
+advanced_source/data/
+beginner_source/.data/
+beginner_source/data/
+beginner_source/blitz/data/
+beginner_source/faces/
+beginner_source/hybrid_frontend/data/
+beginner_source/hymenoptera_data/
+intermediate_source/data/
+*.zip
+MNIST/
 
 #builds
 _build/
@@ -27,6 +33,7 @@ __pycache__/
 *.so
 
 # Distribution / packaging
+src/
 .Python
 env/
 build/
@@ -93,7 +100,7 @@ target/
 .python-version
 
 # celery beat schedule file
-celerybeat-schedule 
+celerybeat-schedule
 # dotenv
 .env
 
@@ -111,3 +118,6 @@ ENV/
 .DS_Store
 cleanup.sh
 *.swp
+
+# PyTorch things
+*.pt

.gitignore

@@ -6,6 +6,10 @@ else
   export BUCKET_NAME=pytorch-tutorial-build-pull-request
 fi
 
+# set locale for click dependency in spacy
+export LC_ALL=C.UTF-8
+export LANG=C.UTF-8
+
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 
 sudo apt-get update
@@ -14,50 +18,42 @@ sudo apt-get install -y --no-install-recommends unzip p7zip-full sox libsox-dev
 export PATH=/opt/conda/bin:$PATH
 rm -rf src
 pip install -r $DIR/../requirements.txt
-pip uninstall -y torchvision || true
-
-# Install a nightly build of pytorch
 
-# GPU, requires CUDA version 8.0
-pip install cython torch_nightly -f https://download.pytorch.org/whl/nightly/cu80/torch_nightly.html
+# export PATH=/opt/conda/bin:$PATH
+# pip install sphinx==1.8.2 pandas
 
-# GPU, requires CUDA version 9.0
-# pip install cython torch_nightly -f https://download.pytorch.org/whl/nightly/cu90/torch_nightly.html
+#Install PyTorch Nightly for test. 
+# Nightly - pip install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/nightly/cu102/torch_nightly.html
+# RC Link
+# pip uninstall -y torch torchvision torchaudio torchtext
+# pip install -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext
 
-# GPU, requires CUDA version 9.2
-# pip install cython torch_nightly -f https://download.pytorch.org/whl/nightly/cu92/torch_nightly.html
+# For Tensorboard. Until 1.14 moves to the release channel.
+pip install tb-nightly
 
-# CPU
-# pip install cython torch_nightly -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
+# Install two language tokenizers for Translation with TorchText tutorial
+python -m spacy download en
+python -m spacy download de
 
-export PATH=/opt/conda/bin:$PATH
-conda install -y sphinx==1.8.2 pandas
 # PyTorch Theme
 rm -rf src
 pip install -e git+git://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme
-# pillow >= 4.2 will throw error when trying to write mode RGBA as JPEG,
-# this is a workaround to the issue.
-pip install sphinx-gallery tqdm matplotlib ipython pillow==4.1.1
-
-# Install torchvision from source
-git clone https://github.com/pytorch/vision --quiet
-pushd vision
-pip install . --no-deps  # We don't want it to install the stock PyTorch version from pip
-popd
-
-# Install torchaudio from source
-git clone https://github.com/pytorch/audio --quiet
-pushd audio
-python setup.py install
-popd
+pip install sphinx-gallery==0.3.1 tqdm matplotlib ipython pillow==8.1.0
 
 aws configure set default.s3.multipart_threshold 5120MB
 
 # Decide whether to parallelize tutorial builds, based on $JOB_BASE_NAME
 export NUM_WORKERS=20
 if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then
   # Step 1: Remove runnable code from tutorials that are not supposed to be run
-  python $DIR/remove_runnable_code.py beginner_source/aws_distributed_training_tutorial.py beginner_source/aws_distributed_training_tutorial.py
+  python $DIR/remove_runnable_code.py beginner_source/aws_distributed_training_tutorial.py beginner_source/aws_distributed_training_tutorial.py || true
+  # python $DIR/remove_runnable_code.py advanced_source/ddp_pipeline_tutorial.py advanced_source/ddp_pipeline_tutorial.py || true
+  # Temp remove for mnist download issue. (Re-enabled for 1.8.1)
+  # python $DIR/remove_runnable_code.py beginner_source/fgsm_tutorial.py beginner_source/fgsm_tutorial.py || true
+  # python $DIR/remove_runnable_code.py intermediate_source/spatial_transformer_tutorial.py intermediate_source/spatial_transformer_tutorial.py || true
+
+  # TODO: Fix bugs in these tutorials to make them runnable again
+  # python $DIR/remove_runnable_code.py beginner_source/audio_classifier_tutorial.py beginner_source/audio_classifier_tutorial.py || true
 
   # Step 2: Keep certain tutorials based on file count, and remove runnable code in all other tutorials
   # IMPORTANT NOTE: We assume that each tutorial has a UNIQUE filename.
@@ -93,6 +89,26 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then
       FILES_TO_RUN+=($(basename $filename .py))
     fi
     count=$((count+1))
+   done
+   for filename in $(find recipes_source/ -name '*.py' -not -path '*/data/*'); do
+    if [ $(($count % $NUM_WORKERS)) != $WORKER_ID ]; then
+      echo "Removing runnable code from "$filename
+      python $DIR/remove_runnable_code.py $filename $filename
+    else
+      echo "Keeping "$filename
+      FILES_TO_RUN+=($(basename $filename .py))
+    fi
+    count=$((count+1))
+   done
+   for filename in $(find prototype_source/ -name '*.py' -not -path '*/data/*'); do
+    if [ $(($count % $NUM_WORKERS)) != $WORKER_ID ]; then
+      echo "Removing runnable code from "$filename
+      python $DIR/remove_runnable_code.py $filename $filename
+    else
+      echo "Keeping "$filename
+      FILES_TO_RUN+=($(basename $filename .py))
+    fi
+    count=$((count+1))
   done
   echo "FILES_TO_RUN: " ${FILES_TO_RUN[@]}
 
@@ -101,13 +117,13 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then
 
   # Step 4: If any of the generated files are not related the tutorial files we want to run,
   # then we remove them
-  for filename in $(find docs/beginner docs/intermediate docs/advanced -name '*.html'); do
+  for filename in $(find docs/beginner docs/intermediate docs/advanced docs/recipes docs/prototype -name '*.html'); do
     file_basename=$(basename $filename .html)
     if [[ ! " ${FILES_TO_RUN[@]} " =~ " ${file_basename} " ]]; then
       rm $filename
     fi
   done
-  for filename in $(find docs/beginner docs/intermediate docs/advanced -name '*.rst'); do
+  for filename in $(find docs/beginner docs/intermediate docs/advanced docs/recipes docs/prototype -name '*.rst'); do
     file_basename=$(basename $filename .rst)
     if [[ ! " ${FILES_TO_RUN[@]} " =~ " ${file_basename} " ]]; then
       rm $filename
@@ -125,13 +141,13 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then
       rm $filename
     fi
   done
-  for filename in $(find docs/_sources/beginner docs/_sources/intermediate docs/_sources/advanced -name '*.rst.txt'); do
+  for filename in $(find docs/_sources/beginner docs/_sources/intermediate docs/_sources/advanced docs/_sources/recipes -name '*.rst.txt'); do
     file_basename=$(basename $filename .rst.txt)
     if [[ ! " ${FILES_TO_RUN[@]} " =~ " ${file_basename} " ]]; then
       rm $filename
     fi
   done
-  for filename in $(find docs/.doctrees/beginner docs/.doctrees/intermediate docs/.doctrees/advanced -name '*.doctree'); do
+  for filename in $(find docs/.doctrees/beginner docs/.doctrees/intermediate docs/.doctrees/advanced docs/.doctrees/recipes docs/.doctrees/prototype -name '*.doctree'); do
     file_basename=$(basename $filename .doctree)
     if [[ ! " ${FILES_TO_RUN[@]} " =~ " ${file_basename} " ]]; then
       rm $filename
@@ -197,6 +213,3 @@ elif [[ "${JOB_BASE_NAME}" == *manager ]]; then
 else
   make docs
 fi
-
-rm -rf vision
-rm -rf audio

.jenkins/build.sh

@@ -8,7 +8,7 @@
     html = html_file.read()
 html_soup = BeautifulSoup(html, 'html.parser')
 
-elems = html_soup.find_all("div", {"class": "highlight-python"})
+elems = html_soup.find_all("div", {"class": "highlight-default"})
 for elem in elems:
     if "%%%%%%INVISIBLE_CODE_BLOCK%%%%%%" in str(elem):
         elem.decompose()