Merge branch 'master' into cache_sphinx

Author: brianjo

.circleci/config.yml

@@ -208,7 +208,7 @@ pytorch_tutorial_build_manager_defaults: &pytorch_tutorial_build_manager_default
   resource_class: medium
   <<: *pytorch_tutorial_build_defaults
 
-pytorch_windows_build: &pytorch_windows_build
+pytorch_windows_build_worker: &pytorch_windows_build_worker
   executor: windows-with-nvidia-gpu
   steps:
     - checkout
@@ -217,10 +217,25 @@ pytorch_windows_build: &pytorch_windows_build
         no_output_timeout: 30m
         command: |
           .circleci/scripts/windows_cuda_install.sh
+    - run:
+        name: Generate cache key
+        # This will refresh cache on Sundays, build should generate new cache.
+        command: echo "$(date +"%Y-%U")" > .circleci-weekly
+    - restore_cache:
+        keys:
+          - data-{{ checksum "Makefile" }}-{{ checksum ".circleci-weekly" }}
     - run:
         name: test
+        no_output_timeout: "1h"
         command: |
           .circleci/scripts/build_for_windows.sh
+    - save_cache:
+        key: data-{{ checksum "Makefile" }}-{{ checksum ".circleci-weekly" }}
+        paths:
+          - advanced_source/data
+          - beginner_source/data
+          - intermediate_source/data
+          - prototype_source/data
 
 jobs:
   pytorch_tutorial_pr_build_worker_0:
@@ -349,8 +364,29 @@ jobs:
   pytorch_tutorial_master_build_manager:
     <<: *pytorch_tutorial_build_manager_defaults
 
-  pytorch_windows_build_worker:
-    <<: *pytorch_windows_build
+  pytorch_tutorial_windows_pr_build_worker_0:
+    <<: *pytorch_windows_build_worker
+
+  pytorch_tutorial_windows_pr_build_worker_1:
+    <<: *pytorch_windows_build_worker
+
+  pytorch_tutorial_windows_pr_build_worker_2:
+    <<: *pytorch_windows_build_worker
+
+  pytorch_tutorial_windows_pr_build_worker_3:
+    <<: *pytorch_windows_build_worker
+
+  pytorch_tutorial_windows_master_build_worker_0:
+    <<: *pytorch_windows_build_worker
+
+  pytorch_tutorial_windows_master_build_worker_1:
+    <<: *pytorch_windows_build_worker
+
+  pytorch_tutorial_windows_master_build_worker_2:
+    <<: *pytorch_windows_build_worker
+
+  pytorch_tutorial_windows_master_build_worker_3:
+    <<: *pytorch_windows_build_worker
 
 workflows:
   build:
@@ -588,11 +624,48 @@ workflows:
             branches:
               only:
                 - master
-#      - pytorch_windows_build_worker:
-#          name: win_test_worker
-#          type: approval
-#          filters:
-#            branches:
-#              only:
-#                - master
+      - pytorch_tutorial_windows_pr_build_worker_0:
+          filters:
+            branches:
+              ignore:
+                - master
+      - pytorch_tutorial_windows_pr_build_worker_1:
+          filters:
+            branches:
+              ignore:
+                - master
+      - pytorch_tutorial_windows_pr_build_worker_2:
+          filters:
+            branches:
+              ignore:
+                - master
+      - pytorch_tutorial_windows_pr_build_worker_3:
+          filters:
+            branches:
+              ignore:
+                - master
+      - pytorch_tutorial_windows_master_build_worker_0:
+          context: org-member
+          filters:
+            branches:
+              only:
+                - master
+      - pytorch_tutorial_windows_master_build_worker_1:
+          context: org-member
+          filters:
+            branches:
+              only:
+                - master
+      - pytorch_tutorial_windows_master_build_worker_2:
+          context: org-member
+          filters:
+            branches:
+              only:
+                - master
+      - pytorch_tutorial_windows_master_build_worker_3:
+          context: org-member
+          filters:
+            branches:
+              only:
+                - master
 

.circleci/scripts/build_for_windows.sh

@@ -16,7 +16,7 @@ mingw32/bin/mingw-get.exe install mingw32-make
 mingw32/bin/mingw-get.exe install msys-findutils
 mv mingw32/bin/mingw32-make.exe mingw32/bin/make.exe
 curl -k https://eternallybored.org/misc/wget/1.20.3/64/wget.exe -o mingw32/bin/wget.exe
-export PATH="${SOURCE_DIR}/mingw32/bin:$PATH"
+export PATH="${SOURCE_DIR}/mingw32/bin:${SOURCE_DIR}/mingw32/msys/1.0/bin:$PATH"
 
 #install anaconda3
 export CONDA_HOME="${SOURCE_DIR}/conda"
@@ -32,18 +32,42 @@ conda create -qyn testenv python=3.7
 conda activate testenv
 
 conda install sphinx
-pip install sphinx_gallery==0.3.1 flask pandas spacy ipython scipy pySoundFile scikit-image
+pip install sphinx_gallery==0.3.1 sphinx-copybutton flask pandas spacy ipython scipy pySoundFile scikit-image
 pip install -e git+git://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme
+pip install ray[tune] tabulate
 conda install -yq -c pytorch "cudatoolkit=10.1" pytorch torchvision torchtext
 conda install torchaudio -c pytorch-test
 python -m spacy download de
 python -m spacy download en
 pushd ${PROJECT_DIR}
 DIR=.jenkins
-python $DIR/remove_runnable_code.py beginner_source/aws_distributed_training_tutorial.py beginner_source/aws_distributed_training_tutorial.py || true
-python $DIR/remove_runnable_code.py beginner_source/data_loading_tutorial.py beginner_source/data_loading_tutorial.py || true
-python $DIR/remove_runnable_code.py beginner_source/dcgan_faces_tutorial.py beginner_source/dcgan_faces_tutorial.py || true
-python $DIR/remove_runnable_code.py intermediate_source/model_parallel_tutorial.py intermediate_source/model_parallel_tutorial.py || true
-python $DIR/remove_runnable_code.py intermediate_source/memory_format_tutorial.py intermediate_source/memory_format_tutorial.py || true
+export NUM_WORKERS=4
 
-make docs
+if [[ "${CIRCLE_JOB}" == *worker_* ]]; then
+  python $DIR/remove_runnable_code.py intermediate_source/model_parallel_tutorial.py intermediate_source/model_parallel_tutorial.py || true
+  python $DIR/remove_runnable_code.py advanced_source/static_quantization_tutorial.py advanced_source/static_quantization_tutorial.py || true
+  python $DIR/remove_runnable_code.py beginner_source/hyperparameter_tuning_tutorial.py beginner_source/hyperparameter_tuning_tutorial.py || true
+
+  export WORKER_ID=$(echo "${CIRCLE_JOB}" | tr -dc '0-9')
+  count=0
+  FILES_TO_RUN=()
+  for work_directory  in beginner_source intermediate_source advanced_source recipes_source prototype_source; do
+    for filename in $(find $work_directory -name '\*.py' -not -path '\*/data/\*'); do
+      if [ $(($count % $NUM_WORKERS)) != $WORKER_ID ]; then
+        echo "Removing runnable code from "$filename
+        python $DIR/remove_runnable_code.py $filename $filename
+      else
+        echo "Keeping "$filename
+        FILES_TO_RUN+=($(basename $filename .py))
+      fi
+      count=$((count+1))
+    done
+  done
+  echo "FILES_TO_RUN: " ${FILES_TO_RUN[@]}
+fi
+
+if [[ ! -d advanced_source/data || ! -d beginner_source/data || ! -d intermediate_source/data || ! -d prototype_source/data ]];then
+    make download
+fi
+
+make html

README.md

@@ -29,3 +29,10 @@ In case you prefer to write your tutorial in jupyter, you can use [this script](
 - You can skip the computationally intensive graph generation by running `make html-noplot` to build basic html documentation to `_build/html`. This way, you can quickly preview your tutorial.
 
 > If you get **ModuleNotFoundError: No module named 'pytorch_sphinx_theme' make: *** [html-noplot] Error 2**, from /tutorials/src/pytorch-sphinx-theme run `python setup.py install`. 
+
+
+## About contributing to PyTorch Documentation and Tutorials
+* You can find information about contributing to PyTorch documentation in the 
+PyTorch Repo [README.md](https://github.com/pytorch/pytorch/blob/master/README.md) file. 
+* Additional information can be found in [PyTorch CONTRIBUTING.md](https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md).
+

_static/img/deeplabv3_android.png

@@ -2,17 +2,6 @@
 
 {% block footer %}
 {{ super() }}
-<script>
-  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
-  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
-  m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
-  })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
-
-  ga('create', 'UA-90545585-2', 'auto');
-  ga('send', 'pageview');
-
-</script>
-
 <script async src="https://www.googletagmanager.com/gtag/js?id=UA-117752657-2"></script>
 
 <script>
@@ -45,28 +34,34 @@
       tutorialLink: window.location.href,
       downloadTitle: $(this).attr("data-response")
     });
-    ga('send', {
-      hitType: 'event',
-      eventCategory: 'Download',
-      eventAction: 'click',
-      eventLabel: $(this).attr("data-response")
+
+    gtag('event', 'click', {
+      'event_category': 'Download',
+      'event_label': $(this).attr("data-response")
+    });
+
+    gtag('event', 'click', {
+      'event_category': $(this).attr("data-response"),
+      'event_label': $("h1").first().text(),
+      'tutorial_link': window.location.href
     });
    });
 
    $("[data-behavior='was-this-helpful-event']").on('click', function(){
     $(".helpful-question").hide();
     $(".was-helpful-thank-you").show();
+
     fbq('trackCustom', "Was this Helpful?", {
       tutorialLink: window.location.href,
       tutorialTitle: $('h1:first').text(),
       helpful: $(this).attr("data-response")
     });
-    ga('send', {
-      hitType: 'event',
-      eventCategory: 'Was this Helpful?',
-      eventAction: 'click',
-      eventLabel: $(this).attr("data-response")
+
+    gtag('event', $(this).attr("data-response"), {
+      'event_category': 'Was this Helpful?',
+      'event_label': $(this).attr("data-response")
     });
+
     gtag('event', $(this).attr("data-response"), {
       'event_category': 'Was this Helpful?',
       'event_label': $("h1").first().text()

_static/img/deeplabv3_android2.png

@@ -213,7 +213,7 @@ def batchify(data, bsz):
 def get_batch(source, i):
     seq_len = min(bptt, len(source) - 1 - i)
     data = source[i:i+seq_len]
-    target = source[i+1:i+1+seq_len].view(-1)
+    target = source[i+1:i+1+seq_len].reshape(-1)
     return data, target
 
 def repackage_hidden(h):