Merge branch 'master' into sync_dist

Author: rohitgr7

.azure/hpu-tests.yml

@@ -20,23 +20,44 @@ jobs:
     timeoutInMinutes: "10"
     # how much time to give 'run always even if cancelled tasks' before stopping them
     cancelTimeoutInMinutes: "2"
-    pool: intel-hpus
+    pool: habana-gaudi-hpus
+    container:
+      image: "vault.habana.ai/gaudi-docker/1.5.0/ubuntu20.04/habanalabs/pytorch-installer-1.11.0:latest"
+      options: "--runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host --shm-size=4g --name cd-container -v /usr/bin/docker:/tmp/docker:ro"
     workspace:
       clean: all
 
     steps:
+    - script: |
+        /tmp/docker exec -t -u 0 cd-container \
+        sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
+      displayName: 'Install Sudo in container (thanks Microsoft!)'
+
     - bash: |
-        apt-get install -y hwinfo
+        sudo apt-get install -y hwinfo
         hwinfo --short
+        python --version
+        sudo pip install pip -U
       displayName: 'Instance HW info'
 
     - bash: |
-        pip install -e .[extra] -r requirements/pytorch/test.txt
+        set -e
+        pip --version
+        sudo pip uninstall -y lightning pytorch-lightning
+        pip install fire
+        python .actions/assistant.py requirements-prune-pkgs torch,torchvision,torchtext
+        pip install ".[extra,test]"
+        pip list
       env:
         PACKAGE_NAME: pytorch
         FREEZE_REQUIREMENTS: 1
       displayName: 'Install dependencies'
 
+    - bash: |
+        hl-smi -L
+        lsmod | grep habanalabs
+      displayName: 'Check the driver status'
+
     - bash: |
         python -m pytest -sv accelerators/test_hpu.py --forked --junitxml=hpu1_test-results.xml
       workingDirectory: tests/tests_pytorch

.github/gatekeeper-config_app.yml

@@ -1,8 +1,7 @@
 approvals:
-  # check will fail if there is no approval
-  minimum: 1
   groups:
     - name: 'Lightning Apps'
+      minimum: 1
       from:
         - alecmerdler
         - awaelchli

.github/gatekeeper-config_pytorch.yml

@@ -1,8 +1,7 @@
 approvals:
-  # check will fail if there is no approval
-  minimum: 1
   groups:
     - name: 'PyTorch Lightning'
+      minimum: 1
       from:
         - awaelchli
         - Borda

.github/labeler.yml

@@ -1,7 +1,27 @@
 app:
-- any: ['src/lightning*/**/*', 'tests/tests_app*/**/*', 'docs/source-app/**/*', 'requirements/app/*']
+- any: ['src/lightning/**/*']
+  all: ['!src/pytorch_lightning/**/*', '!tests/tests_pytorch/**/*', '!docs/source-pytorch/**/*', '!requirements/pytorch/*']
+- any: ['src/lightning_app/**/*']
+  all: ['!src/pytorch_lightning/**/*', '!tests/tests_pytorch/**/*', '!docs/source-pytorch/**/*', '!requirements/pytorch/*']
+- any: ['tests/tests_app/**/*']
+  all: ['!src/pytorch_lightning/**/*', '!tests/tests_pytorch/**/*', '!docs/source-pytorch/**/*', '!requirements/pytorch/*']
+- any: ['tests/tests_app_examples/**/*']
+  all: ['!src/pytorch_lightning/**/*', '!tests/tests_pytorch/**/*', '!docs/source-pytorch/**/*', '!requirements/pytorch/*']
+- any: ['examples/app_*/**/*']
+  all: ['!src/pytorch_lightning/**/*', '!tests/tests_pytorch/**/*', '!docs/source-pytorch/**/*', '!requirements/pytorch/*']
+- any: ['docs/source-app/**/*']
+  all: ['!src/pytorch_lightning/**/*', '!tests/tests_pytorch/**/*', '!docs/source-pytorch/**/*', '!requirements/pytorch/*']
+- any: ['requirements/app/*']
   all: ['!src/pytorch_lightning/**/*', '!tests/tests_pytorch/**/*', '!docs/source-pytorch/**/*', '!requirements/pytorch/*']
 
 pl:
-- any: ['src/pytorch_lightning/**/*', 'tests/tests_pytorch/**/*', 'docs/source-pytorch/**/*', 'requirements/pytorch/*']
-  all: ['!src/lightning*/**/*', '!tests/tests_app*/**/*', '!docs/source-app/**/*', '!requirements/app/*']
+- any: ['src/pytorch_lightning/**/*']
+  all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_app_examples/**/*', '!docs/source-app/**/*', '!requirements/app/*']
+- any: ['examples/*pl_*/**/*']
+  all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_app_examples/**/*', '!docs/source-app/**/*', '!requirements/app/*']
+- any: ['tests/tests_pytorch/**/*']
+  all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_app_examples/**/*', '!docs/source-app/**/*', '!requirements/app/*']
+- any: ['docs/source-pytorch/**/*']
+  all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_app_examples/**/*', '!docs/source-app/**/*', '!requirements/app/*']
+- any: ['requirements/pytorch/*']
+  all: ['!src/lightning/**/*', '!src/lightning_app/**/*', '!tests/tests_app/**/*', '!tests/tests_app_examples/**/*', '!docs/source-app/**/*', '!requirements/app/*']

.github/workflows/ci-app_cloud_e2e_test.yml

@@ -4,9 +4,9 @@ name: cloud-testing
 
 on: # Trigger the workflow on push or pull request, but only for the master branch
   push:
-    branches:
-      - "master"
+    branches: [master, "release/*"]
   pull_request:
+    branches: [master, "release/*"]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}

.github/workflows/ci-app_examples.yml

@@ -3,14 +3,9 @@ name: Test App - examples
 # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
 on: # Trigger the workflow on push or pull request, but only for the master branch
   push:
-    branches:
-      - "master"
+    branches: [master, "release/*"]
   pull_request:
-    paths:
-      - "src/lightning_app/**"
-      - "tests/tests_app_examples/**"
-      - "requirements/app/**"
-      - "setup.py"
+    branches: [master, "release/*"]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}

.github/workflows/ci-app_tests.yml

@@ -3,8 +3,7 @@ name: Test App
 # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
 on: # Trigger the workflow on push or pull request, but only for the master branch
   push:
-    branches:
-      - "master"
+    branches: [master, "release/*"]
   pull_request:
     paths:
       - "src/lightning_app/**"

.github/workflows/ci_pkg-install.yml

@@ -6,8 +6,6 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
     branches: [master, "release/*"]
   pull_request:
     branches: [master, "release/*"]
-    paths-ignore:
-      - "src/lightning_app/**"  # todo: implement job skip
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}

.github/workflows/ci_pr-gatekeeper.yml

@@ -26,7 +26,7 @@ jobs:
         run: |
           patterns = ('docs/source-${{ matrix.pkg }}', 'src/lightning_${{ matrix.pkg }}', 'tests/tests_${{ matrix.pkg }}')
           changed = any(p in "${{steps.changed-files.outputs.all_changed_and_modified_files}}" for p in patterns)
-          print('::set-output name=files::' + int(changed))
+          print(f'::set-output name=files::{int(changed)}')
         shell: python
       - uses: octodemo/pr-gatekeeper@main
         if: steps.touched.outputs.files == 1

.github/workflows/cicd-pytorch_dockers.yml

@@ -8,11 +8,10 @@ on:
     paths:
       - "dockers/**"
       - "!dockers/README.md"
-      - "requirements/*"
+      - "requirements/**"
       - "requirements.txt"
       - "environment.yml"
       - ".github/workflows/*docker*.yml"
-      - ".github/workflows/events-nightly.yml"
       - "setup.py"
   schedule:
     - cron: "0 0 * * *"  # at the end of every day
@@ -226,7 +225,7 @@ jobs:
           build-args: |
             DIST=latest
             GAUDI_VERSION=${{ matrix.gaudi_version }}
-            PYTORCH_VERSION=${{ matrix.pytorch_version }}
+            PYTORCH_INSTALLER_VERSION=${{ matrix.pytorch_version }}
           file: dockers/ci-runner-hpu/Dockerfile
           push: ${{ env.PUSH_TO_HUB }}
           tags: pytorchlightning/pytorch_lightning:hpu-ci-runner-gaudi${{ matrix.gaudi_version }}