Merge branch 'master' into bugfix/dali-iterator

Author: carmocca

.azure/hpu-tests.yml

@@ -14,61 +14,59 @@ pr:
   - "master"
   - "release/*"
 
-variables:
-  - name: continue
-    value: '1'
-
 jobs:
   - job: testing
     # how long to run the job before automatically cancelling
     timeoutInMinutes: "10"
     # how much time to give 'run always even if cancelled tasks' before stopping them
     cancelTimeoutInMinutes: "2"
-    pool: intel-hpus
+    pool: habana-gaudi-hpus
+    container:
+      image: "vault.habana.ai/gaudi-docker/1.5.0/ubuntu20.04/habanalabs/pytorch-installer-1.11.0:latest"
+      options: "--runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host --shm-size=4g --name cd-container -v /usr/bin/docker:/tmp/docker:ro"
     workspace:
       clean: all
 
     steps:
-    - bash: |
-        CHANGED_FILES=$(git diff --name-status origin/master -- . | awk  '{print $2}')
-        FILTER='src/pytorch_lightning|requirements/pytorch|tests/tests_pytorch|examples/pl_*'
-        echo $CHANGED_FILES > changed_files.txt
-        MATCHES=$(cat changed_files.txt | grep -E $FILTER)
-        echo $MATCHES
-        if [ -z "$MATCHES" ]; then
-            echo "Skip"
-            echo "##vso[task.setvariable variable=continue]0"
-        else
-            echo "Continue"
-            echo "##vso[task.setvariable variable=continue]1"
-        fi
-      displayName: Skipper
+    - script: |
+        /tmp/docker exec -t -u 0 cd-container \
+        sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
+      displayName: 'Install Sudo in container (thanks Microsoft!)'
 
     - bash: |
-        apt-get install -y hwinfo
+        sudo apt-get install -y hwinfo
         hwinfo --short
+        python --version
+        sudo pip install pip -U
       displayName: 'Instance HW info'
-      condition: eq(variables['continue'], '1')
 
     - bash: |
-        pip install -e .[extra] -r requirements/pytorch/test.txt
+        set -e
+        pip --version
+        sudo pip uninstall -y lightning pytorch-lightning
+        pip install fire
+        python .actions/assistant.py requirements-prune-pkgs torch,torchvision,torchtext
+        pip install ".[extra,test]"
+        pip list
       env:
         PACKAGE_NAME: pytorch
         FREEZE_REQUIREMENTS: 1
       displayName: 'Install dependencies'
-      condition: eq(variables['continue'], '1')
+
+    - bash: |
+        hl-smi -L
+        lsmod | grep habanalabs
+      displayName: 'Check the driver status'
 
     - bash: |
         python -m pytest -sv accelerators/test_hpu.py --forked --junitxml=hpu1_test-results.xml
       workingDirectory: tests/tests_pytorch
       displayName: 'Single card HPU test'
-      condition: eq(variables['continue'], '1')
 
     - bash: |
         python -m pytest -sv accelerators/test_hpu.py --forked --hpus 8 --junitxml=hpu8_test-results.xml
       workingDirectory: tests/tests_pytorch
       displayName: 'Multi card(8) HPU test'
-      condition: eq(variables['continue'], '1')
 
     - bash: |
         python -m pytest -sv plugins/precision/hpu/test_hpu.py --hmp-bf16 \
@@ -77,18 +75,16 @@ jobs:
           --junitxml=hpu1_precision_test-results.xml
       workingDirectory: tests/tests_pytorch
       displayName: 'HPU precision test'
-      condition: eq(variables['continue'], '1')
 
     - bash: |
         export PYTHONPATH="${PYTHONPATH}:$(pwd)"
         python "pl_hpu/mnist_sample.py"
       workingDirectory: examples
       displayName: 'Testing: HPU examples'
-      condition: eq(variables['continue'], '1')
 
     - task: PublishTestResults@2
       inputs:
         testResultsFiles: 'hpu*_test-results.xml'
         testRunTitle: '$(Agent.OS) - $(Build.DefinitionName) - Python $(python.version)'
-      condition: and(succeededOrFailed(), eq(variables['continue'], '1'))
+      condition: succeededOrFailed()
       displayName: 'Publish test results'

.azure/ipu-tests.yml

@@ -15,8 +15,6 @@ pr:
 variables:
   - name: poplar_sdk
     value: "poplar_sdk-ubuntu_20_04-2.3.1+793-89796d462d"
-  - name: continue
-    value: '1'
 
 jobs:
   - job: testing
@@ -27,31 +25,14 @@ jobs:
       clean: all
 
     steps:
-    - bash: |
-        CHANGED_FILES=$(git diff --name-status origin/master -- . | awk  '{print $2}')
-        FILTER='src/pytorch_lightning|requirements/pytorch|tests/tests_pytorch|examples/pl_*'
-        echo $CHANGED_FILES > changed_files.txt
-        MATCHES=$(cat changed_files.txt | grep -E $FILTER)
-        echo $MATCHES
-        if [ -z "$MATCHES" ]; then
-            echo "Skip"
-            echo "##vso[task.setvariable variable=continue]0"
-        else
-            echo "Continue"
-            echo "##vso[task.setvariable variable=continue]1"
-        fi
-      displayName: Skipper
-
     - script: tar -xvzf /opt/poplar/${{ variables.poplar_sdk }}.tar.gz
       displayName: "Extract Poplar SDK"
-      condition: eq(variables['continue'], '1')
 
     - script: |
         set -eux
         pip debug --verbose
         pip install ${{ variables.poplar_sdk }}/poptorch-*ubuntu*.whl
       displayName: "Install poptorch"
-      condition: eq(variables['continue'], '1')
 
     - script: |
         set -eux
@@ -68,7 +49,6 @@ jobs:
             i=$((i + 1))
         done
       displayName: "Reset IPU devices"
-      condition: eq(variables['continue'], '1')
 
     - bash: |
         export GIT_TERMINAL_PROMPT=1
@@ -80,21 +60,18 @@ jobs:
         PACKAGE_NAME: pytorch
         FREEZE_REQUIREMENTS: 1
       displayName: 'Install dependencies'
-      condition: eq(variables['continue'], '1')
 
     - bash: |
         python requirements/collect_env_details.py
         python -c "import torch"
       displayName: 'Env details'
-      condition: eq(variables['continue'], '1')
 
     - script: |
         set -eux
         source ${{ variables.poplar_sdk }}/poplar-ubuntu*/enable.sh
         source ${{ variables.poplar_sdk }}/popart-ubuntu*/enable.sh
         python -c "import poptorch; print(poptorch.__version__)"
       displayName: "Check poptorch installation"
-      condition: eq(variables['continue'], '1')
 
     - bash: |
         source ${{ variables.poplar_sdk }}/poplar-ubuntu*/enable.sh
@@ -105,4 +82,3 @@ jobs:
         POPTORCH_WAIT_FOR_IPU: 1
         PL_RUN_IPU_TESTS: 1
       displayName: 'Testing: PyTorch standard'
-      condition: eq(variables['continue'], '1')

.github/CODEOWNERS

@@ -13,17 +13,17 @@
 *.yml           @borda @carmocca @akihironitta @tchaton
 
 # Docs
-/docs/                                      @edenlightning @tchaton @borda @awaelchli @RobertLaurella
+/docs/                                      @edenlightning @tchaton @borda @awaelchli @RobertLaurella @Felonious-Spellfire
 /.github/*.md                               @edenlightning @williamfalcon @borda
 /.github/ISSUE_TEMPLATE/                    @edenlightning @borda @tchaton
 /docs/source-pytorch/conf.py                @borda @awaelchli @carmocca
 /docs/source-pytorch/index.rst              @williamfalcon
 /docs/source-pytorch/levels                 @williamfalcon @RobertLaurella
 /docs/source-pytorch/expertise_levels       @williamfalcon @RobertLaurella
-/docs/source-app/                           @williamfalcon @RobertLaurella @tchaton @awaelchli
+/docs/source-app/                           @williamfalcon @Felonious-Spellfire @RobertLaurella @tchaton @awaelchli
 /docs/source-app/conf.py                    @borda @awaelchli @carmocca
 /docs/source-app/index.rst                  @williamfalcon
-/docs/source-app/expertise_levels           @williamfalcon @RobertLaurella
+/docs/source-app/expertise_levels           @williamfalcon @Felonious-Spellfire @RobertLaurella
 
 # Packages
 /src/pytorch_lightning/accelerators         @williamfalcon @tchaton @SeanNaren @awaelchli @justusschock @kaushikb11

.github/labeler.yml

@@ -1,7 +1,27 @@
 app:
-- any: ['src/lightning*/**/*', 'tests/tests_app*/**/*', 'docs/source-app/**/*', 'requirements/app/*']
+- any: ['src/lightning/**/*']
+  all: ['!src/pytorch_lightning/**/*', '!tests/tests_pytorch/**/*', '!docs/source-pytorch/**/*', '!requirements/pytorch/*']
+- any: ['src/lightning_app/**/*']
+  all: ['!src/pytorch_lightning/**/*', '!tests/tests_pytorch/**/*', '!docs/source-pytorch/**/*', '!requirements/pytorch/*']
+- any: ['tests/tests_app/**/*']
+  all: ['!src/pytorch_lightning/**/*', '!tests/tests_pytorch/**/*', '!docs/source-pytorch/**/*', '!requirements/pytorch/*']
+- any: ['tests/tests_app_examples/**/*']
+  all: ['!src/pytorch_lightning/**/*', '!tests/tests_pytorch/**/*', '!docs/source-pytorch/**/*', '!requirements/pytorch/*']
+- any: ['examples/app_*/**/*']
+  all: ['!src/pytorch_lightning/**/*', '!tests/tests_pytorch/**/*', '!docs/source-pytorch/**/*', '!requirements/pytorch/*']
+- any: ['docs/source-app/**/*']
+  all: ['!src/pytorch_lightning/**/*', '!tests/tests_pytorch/**/*', '!docs/source-pytorch/**/*', '!requirements/pytorch/*']
+- any: ['requirements/app/*']
   all: ['!src/pytorch_lightning/**/*', '!tests/tests_pytorch/**/*', '!docs/source-pytorch/**/*', '!requirements/pytorch/*']
 
 pl:
-- any: ['src/pytorch_lightning/**/*', 'tests/tests_pytorch/**/*', 'docs/source-pytorch/**/*', 'requirements/pytorch/*']
-  all: ['!src/lightning*/**/*', '!tests/tests_app*/**/*', '!docs/source-app/**/*', '!requirements/app/*']
+- any: ['src/pytorch_lightning/**/*']
+  all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_app_examples/**/*', '!docs/source-app/**/*', '!requirements/app/*']
+- any: ['examples/*pl_*/**/*']
+  all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_app_examples/**/*', '!docs/source-app/**/*', '!requirements/app/*']
+- any: ['tests/tests_pytorch/**/*']
+  all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_app_examples/**/*', '!docs/source-app/**/*', '!requirements/app/*']
+- any: ['docs/source-pytorch/**/*']
+  all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_app_examples/**/*', '!docs/source-app/**/*', '!requirements/app/*']
+- any: ['requirements/pytorch/*']
+  all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_app_examples/**/*', '!docs/source-app/**/*', '!requirements/app/*']

.github/workflows/ci-app_cloud_e2e_test.yml

@@ -4,9 +4,9 @@ name: cloud-testing
 
 on: # Trigger the workflow on push or pull request, but only for the master branch
   push:
-    branches:
-      - "master"
+    branches: [master, "release/*"]
   pull_request:
+    branches: [master, "release/*"]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}

.github/workflows/ci-app_examples.yml

@@ -3,14 +3,9 @@ name: Test App - examples
 # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
 on: # Trigger the workflow on push or pull request, but only for the master branch
   push:
-    branches:
-      - "master"
+    branches: [master, "release/*"]
   pull_request:
-    paths:
-      - "src/lightning_app/**"
-      - "tests/tests_app_examples/**"
-      - "requirements/app/**"
-      - "setup.py"
+    branches: [master, "release/*"]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}

.github/workflows/ci-app_tests.yml

@@ -3,8 +3,7 @@ name: Test App
 # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
 on: # Trigger the workflow on push or pull request, but only for the master branch
   push:
-    branches:
-      - "master"
+    branches: [master, "release/*"]
   pull_request:
     paths:
       - "src/lightning_app/**"

.github/workflows/ci_pkg-install.yml

@@ -6,8 +6,6 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
     branches: [master, "release/*"]
   pull_request:
     branches: [master, "release/*"]
-    paths-ignore:
-      - "src/lightning_app/**"  # todo: implement job skip
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}

.github/workflows/ci_pr-gatekeeper.yml

@@ -25,8 +25,8 @@ jobs:
         id: touched
         run: |
           patterns = ('docs/source-${{ matrix.pkg }}', 'src/lightning_${{ matrix.pkg }}', 'tests/tests_${{ matrix.pkg }}')
-          changed = any(p in "${{steps.changed-files.outputs.all_changed_and_modified_files}}" for p in patterns):
-          print('::set-output name=files::' + int(changed))
+          changed = any(p in "${{steps.changed-files.outputs.all_changed_and_modified_files}}" for p in patterns)
+          print(f'::set-output name=files::{int(changed)}')
         shell: python
       - uses: octodemo/pr-gatekeeper@main
         if: steps.touched.outputs.files == 1

.github/workflows/cicd-pytorch_dockers.yml

@@ -8,11 +8,10 @@ on:
     paths:
       - "dockers/**"
       - "!dockers/README.md"
-      - "requirements/*"
+      - "requirements/**"
       - "requirements.txt"
       - "environment.yml"
       - ".github/workflows/*docker*.yml"
-      - ".github/workflows/events-nightly.yml"
       - "setup.py"
   schedule:
     - cron: "0 0 * * *"  # at the end of every day
@@ -226,7 +225,7 @@ jobs:
           build-args: |
             DIST=latest
             GAUDI_VERSION=${{ matrix.gaudi_version }}
-            PYTORCH_VERSION=${{ matrix.pytorch_version }}
+            PYTORCH_INSTALLER_VERSION=${{ matrix.pytorch_version }}
           file: dockers/ci-runner-hpu/Dockerfile
           push: ${{ env.PUSH_TO_HUB }}
           tags: pytorchlightning/pytorch_lightning:hpu-ci-runner-gaudi${{ matrix.gaudi_version }}