Merge branch 'master' into bugfix/5007

Author: carmocca

.github/workflows/docs-link.yml

@@ -0,0 +1,14 @@
+name: Docs Link
+
+on: [status]
+
+jobs:
+  circleci_artifacts_redirector_job:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: larsoner/circleci-artifacts-redirector-action@master
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        artifact-path: 0/html/index.html
+        circleci-jobs: build-Docs
+        job-title: Check the rendered docs here!

CHANGELOG.md

@@ -412,6 +412,16 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 - Removed `Strategy.optimizer_zero_grad` ([#11246](https://github.com/PyTorchLightning/pytorch-lightning/pull/11246))
 
+
+- Removed `Strategy.on_gpu` ([#11537](https://github.com/PyTorchLightning/pytorch-lightning/pull/11537))
+
+
+- Removed `Strategy.on_tpu` property ([#11536](https://github.com/PyTorchLightning/pytorch-lightning/pull/11536))
+
+
+ - Removed access to `_short_id` in `NeptuneLogger` ([#11517](https://github.com/PyTorchLightning/pytorch-lightning/pull/11517))
+
+
 ### Fixed
 
 - Fixed security vulnerabilities CVE-2020-1747 and CVE-2020-14343 caused by the `PyYAML` dependency ([#11099](https://github.com/PyTorchLightning/pytorch-lightning/pull/11099))
@@ -432,12 +442,18 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Fixed wrong typehint for `Trainer.lightning_optimizers` ([#11155](https://github.com/PyTorchLightning/pytorch-lightning/pull/11155))
 
 
+- Fixed the format of the configuration saved automatically by the CLI's `SaveConfigCallback` ([#11532](https://github.com/PyTorchLightning/pytorch-lightning/pull/11532))
+
+
 - Fixed type promotion when tensors of higher category than float are logged ([#11401](https://github.com/PyTorchLightning/pytorch-lightning/pull/11401))
 
 
 - Fixed the lr-scheduler state not being dumped to checkpoint when using the deepspeed strategy ([#11307](https://github.com/PyTorchLightning/pytorch-lightning/pull/11307))
 
 
+- Fixed `SimpleProfiler` summary ([#11414](https://github.com/PyTorchLightning/pytorch-lightning/pull/11414))
+
+
 - Disbled sampler replacement when using `IterableDataset` ([#11507](https://github.com/PyTorchLightning/pytorch-lightning/pull/11507))
 
 

docs/source/accelerators/gpu.rst

@@ -36,6 +36,11 @@ Lightning supports multiple ways of doing distributed training.
 
 |
 
+Model Parallel Training
+-----------------------
+
+Check out the :ref:`Model Parallel Guide <model_parallel>` documentation.
+
 ----------
 
 Preparing your code

docs/source/advanced/advanced_gpu.rst

@@ -212,7 +212,10 @@ The :class:`~pytorch_lightning.loggers.WandbLogger` is available anywhere except
     class MyModule(LightningModule):
         def any_lightning_module_function_or_hook(self):
             some_img = fake_image()
-            self.log({"generated_images": [wandb.Image(some_img, caption="...")]})
+            # Option 1
+            self.logger.experiment.log({"generated_images": [wandb.Image(some_img, caption="...")]})
+            # Option 2 for specifically logging images
+            self.logger.log_image(key="generated_images", images=[some_img])
 
 .. seealso::
     - :class:`~pytorch_lightning.loggers.WandbLogger` docs.

docs/source/common/loggers.rst

@@ -293,9 +293,10 @@ def __init__(
     def _retrieve_run_data(self):
         try:
             self._run_instance.wait()
-            self._run_short_id = self.run._short_id  # skipcq: PYL-W0212
+            self._run_short_id = self._run_instance["sys/id"].fetch()
             self._run_name = self._run_instance["sys/name"].fetch()
         except NeptuneOfflineModeFetchException:
+            self._run_short_id = "OFFLINE"
             self._run_name = "offline-name"
 
     @property

pytorch_lightning/loggers/neptune.py

@@ -45,6 +45,9 @@ def __init__(
             filename: If present, filename where the profiler results will be saved instead of printing to stdout.
                 The ``.txt`` extension will be used automatically.
 
+            extended: If ``True``, adds extra columns representing number of calls and percentage of total time spent on
+                respective action.
+
         Raises:
             ValueError:
                 If you attempt to start an action which has already started, or
@@ -73,7 +76,8 @@ def _make_report(self) -> Tuple[list, float]:
         total_duration = time.monotonic() - self.start_time
         report = [[a, d, 100.0 * np.sum(d) / total_duration] for a, d in self.recorded_durations.items()]
         report.sort(key=lambda x: x[2], reverse=True)
-        return report, total_duration
+        total_calls = sum(len(x[1]) for x in report)
+        return report, total_calls, total_duration
 
     def summary(self) -> str:
         sep = os.linesep
@@ -88,16 +92,17 @@ def summary(self) -> str:
                 max_key = max(len(k) for k in self.recorded_durations.keys())
 
                 def log_row(action, mean, num_calls, total, per):
-                    row = f"{sep}{action:<{max_key}s}\t|  {mean:<15}\t|"
-                    row += f"{num_calls:<15}\t|  {total:<15}\t|  {per:<15}\t|"
+                    row = f"{sep}|  {action:<{max_key}s}\t|  {mean:<15}\t|"
+                    row += f"  {num_calls:<15}\t|  {total:<15}\t|  {per:<15}\t|"
                     return row
 
-                output_string += log_row("Action", "Mean duration (s)", "Num calls", "Total time (s)", "Percentage %")
-                output_string_len = len(output_string)
-                output_string += f"{sep}{'-' * output_string_len}"
-                report, total_duration = self._make_report()
-                output_string += log_row("Total", "-", "_", f"{total_duration:.5}", "100 %")
-                output_string += f"{sep}{'-' * output_string_len}"
+                header_string = log_row("Action", "Mean duration (s)", "Num calls", "Total time (s)", "Percentage %")
+                output_string_len = len(header_string.expandtabs())
+                sep_lines = f"{sep}{'-' * output_string_len}"
+                output_string += sep_lines + header_string + sep_lines
+                report, total_calls, total_duration = self._make_report()
+                output_string += log_row("Total", "-", f"{total_calls:}", f"{total_duration:.5}", "100 %")
+                output_string += sep_lines
                 for action, durations, duration_per in report:
                     output_string += log_row(
                         action,
@@ -106,15 +111,20 @@ def log_row(action, mean, num_calls, total, per):
                         f"{np.sum(durations):.5}",
                         f"{duration_per:.5}",
                     )
+                output_string += sep_lines
         else:
+            max_key = max(len(k) for k in self.recorded_durations)
 
             def log_row(action, mean, total):
-                return f"{sep}{action:<20s}\t|  {mean:<15}\t|  {total:<15}"
+                return f"{sep}|  {action:<{max_key}s}\t|  {mean:<15}\t|  {total:<15}\t|"
 
-            output_string += log_row("Action", "Mean duration (s)", "Total time (s)")
-            output_string += f"{sep}{'-' * 65}"
+            header_string = log_row("Action", "Mean duration (s)", "Total time (s)")
+            output_string_len = len(header_string.expandtabs())
+            sep_lines = f"{sep}{'-' * output_string_len}"
+            output_string += sep_lines + header_string + sep_lines
 
             for action, durations in self.recorded_durations.items():
                 output_string += log_row(action, f"{np.mean(durations):.5}", f"{np.sum(durations):.5}")
+            output_string += sep_lines
         output_string += sep
         return output_string

pytorch_lightning/profiler/simple.py

@@ -296,7 +296,7 @@ def _register_ddp_hooks(self) -> None:
         # In 1.8, DDP communication hooks only work with NCCL backend and SPSD (single process single device) mode
         # Since 1.9, DDP communication hooks can work on all backends.
         if _TORCH_GREATER_EQUAL_1_9 or (
-            _TORCH_GREATER_EQUAL_1_8 and self.on_gpu and self._is_single_process_single_device
+            _TORCH_GREATER_EQUAL_1_8 and self.root_device.type == "cuda" and self._is_single_process_single_device
         ):
             register_ddp_comm_hook(
                 model=self.model,
@@ -514,7 +514,7 @@ def teardown(self) -> None:
         if self.sync_batchnorm:
             self.model = _revert_sync_batchnorm(self.model)
 
-        if self.on_gpu:
+        if self.root_device.type == "cuda":
             # GPU teardown
             log.detail(f"{self.__class__.__name__}: moving model to CPU")
             self.lightning_module.cpu()

pytorch_lightning/strategies/ddp.py

@@ -200,7 +200,7 @@ def pre_configure_ddp(self):
     def _register_ddp_hooks(self) -> None:
         # currently, DDP communication hooks only work with NCCL backend and SPSD (single process single device) mode
         # https://github.com/pytorch/pytorch/blob/v1.8.0/torch/nn/parallel/distributed.py#L1080-L1084
-        if _TORCH_GREATER_EQUAL_1_8 and self.on_gpu and self._is_single_process_single_device:
+        if _TORCH_GREATER_EQUAL_1_8 and self.root_device.type == "cuda" and self._is_single_process_single_device:
             register_ddp_comm_hook(
                 model=self.model,
                 ddp_comm_state=self._ddp_comm_state,
@@ -378,7 +378,7 @@ def teardown(self) -> None:
         if self.sync_batchnorm:
             self.model = _revert_sync_batchnorm(self.model)
 
-        if self.on_gpu:
+        if self.root_device.type == "cuda":
             # GPU teardown
             self.lightning_module.cpu()
             # clean up memory

pytorch_lightning/strategies/ddp_spawn.py

@@ -153,7 +153,7 @@ def test_step_end(self, output):
 
     def teardown(self) -> None:
         super().teardown()
-        if self.on_gpu:
+        if self.root_device.type == "cuda":
             # GPU teardown
             self.lightning_module.cpu()
             # clean up memory