Merge branch 'master' into bug/prof_missing_hooks

Author: awaelchli

.actions/setup_tools.py

@@ -94,11 +94,10 @@ def load_readme_description(path_dir: str, homepage: str, version: str) -> str:
     text = text.replace("pytorch-lightning.readthedocs.io/en/stable/", f"pytorch-lightning.readthedocs.io/en/{version}")
     # codecov badge
     text = text.replace("/branch/master/graph/badge.svg", f"/release/{version}/graph/badge.svg")
-    # replace github badges for release ones
+    # github actions badge
     text = text.replace("badge.svg?branch=master&event=push", f"badge.svg?tag={version}")
-    # Azure...
+    # azure pipelines badge
     text = text.replace("?branchName=master", f"?branchName=refs%2Ftags%2F{version}")
-    text = re.sub(r"\?definitionId=\d+&branchName=master", f"?definitionId=2&branchName=refs%2Ftags%2F{version}", text)
 
     skip_begin = r"<!-- following section will be skipped from PyPI description -->"
     skip_end = r"<!-- end skipping PyPI description -->"

.azure/gpu-benchmark.yml

@@ -28,7 +28,7 @@ jobs:
     cancelTimeoutInMinutes: "2"
     pool: azure-jirka-spot
     container:
-      image: "pytorchlightning/pytorch_lightning:base-cuda-py3.9-torch1.12"
+      image: "pytorchlightning/pytorch_lightning:base-cuda-py3.9-torch1.12-cuda11.3.1"
       options: "--runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=all --shm-size=32g"
     workspace:
       clean: all

.azure/gpu-tests.yml

@@ -26,7 +26,7 @@ jobs:
     strategy:
       matrix:
         'PyTorch - stable':
-          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.9-torch1.12"
+          image: "pytorchlightning/pytorch_lightning:base-cuda-py3.9-torch1.12-cuda11.3.1"
     # how long to run the job before automatically cancelling
     timeoutInMinutes: "80"
     # how much time to give 'run always even if cancelled tasks' before stopping them
@@ -44,7 +44,7 @@ jobs:
 
     - bash: |
         CHANGED_FILES=$(git diff --name-status origin/master -- . | awk  '{print $2}')
-        FILTER='src/pytorch_lightning|requirements/pytorch|tests/tests_pytorch|examples/pl_*|.azure/*'
+        FILTER='src/pytorch_lightning|requirements/pytorch|tests/tests_pytorch|examples/pl_*|.azure/gpu-tests.yml'
         echo $CHANGED_FILES > changed_files.txt
         MATCHES=$(cat changed_files.txt | grep -E $FILTER)
         echo $MATCHES
@@ -75,7 +75,7 @@ jobs:
         CUDA_VERSION_MM=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda.split('.')[:2])))")
         pip install "bagua-cuda$CUDA_VERSION_MM>=0.9.0"
         pip install -e .[strategies]
-        pip install deepspeed>0.6.4  # TODO: remove when docker images are upgraded
+        pip install -U deepspeed  # TODO: remove when docker images are upgraded
         pip install --requirement requirements/pytorch/devel.txt
         pip list
       env:

.github/BECOMING_A_CORE_CONTRIBUTOR.md

@@ -62,4 +62,4 @@ We are on the lookout for new people to join, however, if you feel like you meet
 
 ## Employment
 
-You can also become a [Grid.ai](https://www.grid.ai) employee or intern and work on Lightning. To get started, you can email `careers@grid.ai` with your resume or check out our [open job postings](https://boards.greenhouse.io/gridai).
+You can also become a [Lightning AI](https://lightning.ai/) employee or intern and work on Lightning. To get started, you can email `careers@lightning.ai` with your resume or check out our [open job postings](https://boards.greenhouse.io/lightningai).

.github/CODEOWNERS

@@ -5,7 +5,7 @@
 # the repo. Unless a later match takes precedence,
 # @global-owner1 and @global-owner2 will be requested for
 # review when someone opens a pull request.
-* @williamfalcon @borda @tchaton @SeanNaren @carmocca @awaelchli @justusschock @kaushikb11 @rohitgr7
+* @williamfalcon @borda @tchaton @awaelchli @kaushikb11 @rohitgr7
 
 # CI/CD and configs
 /.github/       @borda @carmocca @akihironitta @tchaton
@@ -26,50 +26,52 @@
 /docs/source-app/expertise_levels           @williamfalcon @Felonious-Spellfire @RobertLaurella
 
 # Packages
+/src/pytorch_lightning                      @carmocca @justusschock
 /src/pytorch_lightning/accelerators         @williamfalcon @tchaton @SeanNaren @awaelchli @justusschock @kaushikb11
 /src/pytorch_lightning/callbacks            @williamfalcon @tchaton @carmocca @borda @kaushikb11
-/src/pytorch_lightning/core                 @tchaton @SeanNaren @borda @carmocca @justusschock @kaushikb11
+/src/pytorch_lightning/core                 @tchaton @borda @carmocca @justusschock @kaushikb11
 /src/pytorch_lightning/distributed          @williamfalcon @tchaton @awaelchli @kaushikb11
 /src/pytorch_lightning/lite                 @tchaton @awaelchli @carmocca
 /src/pytorch_lightning/loggers              @tchaton @awaelchli @borda
-/src/pytorch_lightning/loggers/wandb.py     @borisdayma
+/src/pytorch_lightning/loggers/wandb.py     @borisdayma @borda
 /src/pytorch_lightning/loggers/neptune.py   @shnela @HubertJaworski @pkasprzyk @pitercl @Raalsky @aniezurawski @kamil-kaczmarek
 /src/pytorch_lightning/loops                @tchaton @awaelchli @justusschock @carmocca
-/src/pytorch_lightning/overrides            @tchaton @SeanNaren @borda
-/src/pytorch_lightning/plugins              @tchaton @SeanNaren @awaelchli @justusschock
+/src/pytorch_lightning/overrides            @tchaton @borda
+/src/pytorch_lightning/plugins              @tchaton @awaelchli @justusschock
 /src/pytorch_lightning/profilers            @williamfalcon @tchaton @borda @carmocca
 /src/pytorch_lightning/profilers/pytorch.py @nbcsm @guotuofeng
 /src/pytorch_lightning/strategies           @tchaton @SeanNaren @awaelchli @justusschock @kaushikb11
-/src/pytorch_lightning/trainer              @williamfalcon @borda @tchaton @SeanNaren @carmocca @awaelchli @justusschock @kaushikb11
-/src/pytorch_lightning/trainer/connectors   @tchaton @SeanNaren @carmocca @borda
+/src/pytorch_lightning/trainer              @williamfalcon @borda @tchaton @carmocca @awaelchli @justusschock @kaushikb11
+/src/pytorch_lightning/trainer/connectors   @tchaton @carmocca @borda
 /src/pytorch_lightning/tuner                @SkafteNicki @borda @awaelchli
-/src/pytorch_lightning/utilities            @borda @tchaton @SeanNaren @carmocca
+/src/pytorch_lightning/utilities            @borda @tchaton @carmocca
 
-/src/lightning_app                           @tchaton @awaelchli @manskx @hhsecond
+/src/lightning_app                          @tchaton @manskx
+/src/lightning_app/cli/pl-app-template      @tchaton @awaelchli @Borda
+/src/lightning_app/core                     @tchaton @awaelchli @manskx
+/src/lightning_app/core/queues.py           @tchaton @hhsecond @manskx
+/src/lightning_app/runners/cloud.py         @tchaton @hhsecond
+/src/lightning_app/testing                  @tchaton @manskx
+/src/lightning_app/__about__.py             @nohalon @edenlightning @lantiga
 
 # Examples
-/examples/app_*                            @tchaton @awaelchli @manskx @hhsecond
+/examples/app_*                             @tchaton @awaelchli @manskx @hhsecond
 # App tests
-/tests/tests_app                          @tchaton @awaelchli @manskx @hhsecond
-/tests/tests_app_examples                 @tchaton @awaelchli @manskx @hhsecond
+/tests/tests_app                            @tchaton @awaelchli @manskx @hhsecond
+/tests/tests_app_examples                   @tchaton @awaelchli @manskx @hhsecond
 
 # Specifics
-/src/pytorch_lightning/trainer/connectors/logger_connector      @tchaton @carmocca
-/src/pytorch_lightning/trainer/progress.py                      @tchaton @awaelchli @carmocca
-
+/src/pytorch_lightning/trainer/connectors/logger_connector @tchaton @carmocca
+/src/pytorch_lightning/trainer/progress.py                 @tchaton @awaelchli @carmocca
 # API
-/src/pytorch_lightning/callbacks/base.py    @williamfalcon @awaelchli @ananthsub @carmocca
-/src/pytorch_lightning/core/datamodule.py   @williamFalcon @awaelchli @ananthsub @carmocca
-/src/pytorch_lightning/trainer/trainer.py   @williamfalcon @tchaton @awaelchli
-/src/pytorch_lightning/core/hooks.py        @williamfalcon @tchaton @awaelchli @ananthsub @carmocca
-/src/pytorch_lightning/core/lightning.py    @williamfalcon @tchaton @awaelchli
-
-# Testing
-/tests/helpers/boring_model.py          @williamfalcon @tchaton @borda
+/src/pytorch_lightning/callbacks/callback.py @williamfalcon @awaelchli @ananthsub @carmocca
+/src/pytorch_lightning/core/datamodule.py    @williamFalcon @awaelchli @ananthsub @carmocca
+/src/pytorch_lightning/trainer/trainer.py    @williamfalcon @tchaton @awaelchli
+/src/pytorch_lightning/core/hooks.py         @williamfalcon @tchaton @awaelchli @ananthsub @carmocca
+/src/pytorch_lightning/core/module.py        @williamfalcon @tchaton @awaelchli
 
-/.github/CODEOWNERS              @williamfalcon
-/.github/approve_config.yml      @williamfalcon
-/SECURITY.md                     @williamfalcon
-/README.md                       @williamfalcon @edenlightning @borda
-/setup.py                        @williamfalcon @borda @carmocca
+/.github/CODEOWNERS                  @williamfalcon
+/SECURITY.md                         @williamfalcon
+/README.md                           @williamfalcon @edenlightning @borda
+/setup.py                            @williamfalcon @borda @carmocca
 /src/pytorch_lightning/__about__.py  @williamfalcon @borda @carmocca

.github/ISSUE_TEMPLATE/bug_report.md

@@ -41,8 +41,16 @@ You can get the script and run it with:
 ```bash
 wget https://raw.githubusercontent.com/Lightning-AI/lightning/master/requirements/collect_env_details.py
 python collect_env_details.py
+
 ```
 
+
+<details>
+  <summary>Details</summary>
+    Paste the output here and move this toggle outside of the comment block.
+</details>
+
+
 You can also fill out the list below manually.
 -->
 

.github/checkgroup.yml

@@ -0,0 +1,165 @@
+custom_service_name: "Lightning CI required checker"
+subprojects:
+  - id: "CI: CircleCI"
+    paths:
+      - ".circleci/**"
+    checks:
+      - "test-on-tpus"
+
+  - id: "CI: Azure"
+    paths:
+      - ".azure/**"
+    checks:
+      - "pytorch-lightning (GPUs)"
+      - "pytorch-lightning (GPUs) (testing PyTorch - stable)"
+      - "pytorch-lightning (HPUs)"
+      - "pytorch-lightning (IPUs)"
+
+  - id: "pytorch_lightning"
+    paths:
+      # all examples don't need to be added because they aren't used in CI, but these are
+      - "examples/run_ddp_examples.sh"
+      - "examples/convert_from_pt_to_pl/**"
+      - "examples/run_pl_examples.sh"
+      - "examples/pl_basics/backbone_image_classifier.py"
+      - "examples/pl_basics/autoencoder.py"
+      - "examples/pl_loops/mnist_lite.py"
+      - "examples/pl_fault_tolerant/automatic.py"
+      - "examples/test_pl_examples.py"
+      - "examples/pl_integrations/dali_image_classifier.py"
+      - "requirements/pytorch/**"
+      - "src/pytorch_lightning/**"
+      - "tests/tests_pytorch/**"
+      - "setup.cfg"  # includes pytest config
+      - ".github/workflows/ci-pytorch*.yml"
+      - ".github/workflows/docs-*.yml"
+    checks:
+      - "conda (3.8, 1.10)"
+      - "conda (3.8, 1.9)"
+      - "conda (3.9, 1.11)"
+      - "conda (3.9, 1.12)"
+      - "cpu (macOS-11, 3.10, latest, stable)"
+      - "cpu (macOS-11, 3.7, latest, stable)"
+      - "cpu (macOS-11, 3.7, oldest, stable)"
+      - "cpu (ubuntu-20.04, 3.10, latest, stable)"
+      - "cpu (ubuntu-20.04, 3.7, latest, stable)"
+      - "cpu (ubuntu-20.04, 3.7, oldest, stable)"
+      - "cpu (windows-2022, 3.10, latest, stable)"
+      - "cpu (windows-2022, 3.7, latest, stable)"
+      - "cpu (windows-2022, 3.7, oldest, stable)"
+      - "doctest (pytorch)"
+      - "make-docs (pytorch)"
+      - "mypy"
+      - "PR Gatekeeper (pytorch)"
+      - "pytorch-lightning (GPUs)"
+      - "pytorch-lightning (GPUs) (testing PyTorch - stable)"
+      - "pytorch-lightning (HPUs)"
+      - "pytorch-lightning (IPUs)"
+      - "slow (macOS-11, 3.7, 1.11)"
+      - "slow (ubuntu-20.04, 3.7, 1.11)"
+      - "slow (windows-2022, 3.7, 1.11)"
+      - "test-on-tpus"
+
+  - id: "pytorch_lightning: Docs"
+    paths:
+      - "docs/source-pytorch/**"
+      - ".github/workflows/docs-*.yml"
+      - "requirements/pytorch/**"
+    checks:
+      - "doctest (pytorch)"
+      - "make-docs (pytorch)"
+
+  - id: "pytorch_lightning: Docker"
+    paths:
+      - "dockers/**"
+    checks:
+      - "build-conda (3.8, 1.10)"
+      - "build-conda (3.8, 1.9)"
+      - "build-conda (3.9, 1.11)"
+      - "build-conda (3.9, 1.12)"
+      - "build-cuda (3.8, 1.9, 11.1.1)"
+      - "build-cuda (3.9, 1.10, 11.3.1)"
+      - "build-cuda (3.9, 1.11, 11.3.1)"
+      - "build-cuda (3.9, 1.12, 11.3.1)"
+      - "build-cuda (3.9, 1.9, 11.1.1)"
+      - "build-hpu (1.5.0, 1.11.0)"
+      - "build-ipu (3.9, 1.9)"
+      - "build-NGC"
+      - "build-pl (3.9, 1.10, 11.3.1)"
+      - "build-pl (3.9, 1.11, 11.3.1)"
+      - "build-pl (3.9, 1.12, 11.3.1)"
+      - "build-pl (3.9, 1.9, 11.1.1)"
+      - "build-xla (3.7, 1.12)"
+
+  - id: "pytorch_lightning: mypy"
+    paths:
+      - ".github/workflows/code-checks.yml"
+      - "pyproject.toml"  # includes mypy config
+    checks:
+      - "mypy"
+
+  - id: "lightning_app"
+    paths:
+      - ".github/workflows/ci-app*.yml"
+      - "examples/app_**"
+      - "requirements/app/**"
+      - "src/lightning_app/**"
+      - "tests/tests_app/**"
+      - "tests/tests_app_examples/**"
+      - "tests/tests_clusters/**"
+      # the examples are used in the app CI
+      - "examples/app_*"
+    checks:
+      - "Cloud Test (boring_app)"
+      - "Cloud Test (collect_failures)"
+      - "Cloud Test (commands_and_api)"
+      - "Cloud Test (custom_work_dependencies)"
+      - "Cloud Test (drive)"
+      - "Cloud Test (idle_timeout)"
+      - "Cloud Test (payload)"
+      - "Cloud Test (template_jupyterlab)"
+      - "Cloud Test (template_react_ui)"
+      - "Cloud Test (template_streamlit_ui)"
+      - "Cloud Test (v0_app)"
+      - "doctest (app)"
+      - "make-docs (app)"
+      - "pytest (macOS-11, 3.8, latest)"
+      - "pytest (macOS-11, 3.8, oldest)"
+      - "pytest (ubuntu-20.04, 3.8, latest)"
+      - "pytest (ubuntu-20.04, 3.8, oldest)"
+      - "pytest (windows-2022, 3.8, latest)"
+      - "pytest (windows-2022, 3.8, oldest)"
+
+  - id: "lightning_app: Docs"
+    paths:
+      - "docs/source-app/**"
+      - ".github/workflows/docs-*.yml"
+      - "requirements/app/**"
+    checks:
+      - "doctest (app)"
+      - "make-docs (app)"
+
+  - id: "install"
+    paths:
+      - ".actions/setup_tools.py"
+      - ".github/workflows/ci-pkg-install.yml"
+      - "setup.py"
+      - "src/lightning/**"
+      # all __about__, __version__, __setup__
+      - "src/*/__*.py"
+    checks:
+      - "install-meta-pypi (macOS-11, 3.8)"
+      - "install-meta-pypi (ubuntu-20.04, 3.8)"
+      - "install-meta-pypi (windows-2022, 3.8)"
+      - "install-meta-src (macOS-11, 3.8)"
+      - "install-meta-src (macOS-11, lightning, 3.8)"
+      - "install-meta-src (ubuntu-20.04, 3.8)"
+      - "install-meta-src (ubuntu-20.04, lightning, 3.8)"
+      - "install-meta-src (windows-2022, 3.8)"
+      - "install-meta-src (windows-2022, lightning, 3.8)"
+      - "install-standalone (macOS-11, app, 3.8)"
+      - "install-standalone (macOS-11, pytorch, 3.8)"
+      - "install-standalone (ubuntu-20.04, app, 3.8)"
+      - "install-standalone (ubuntu-20.04, pytorch, 3.8)"
+      - "install-standalone (windows-2022, app, 3.8)"
+      - "install-standalone (windows-2022, pytorch, 3.8)"