Merge branch 'master' into refactor/rename-lr_scheduler_config

Author: akihironitta

CHANGELOG.md

@@ -216,6 +216,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - `Trainer.logged_metrics` now always contains scalar tensors, even when a Python scalar was logged ([#11270](https://github.com/PyTorchLightning/pytorch-lightning/pull/11270))
 
 
+- The tuner now uses the checkpoint connector to copy and restore its state ([#11518](https://github.com/PyTorchLightning/pytorch-lightning/pull/11518))
+
+
 - Changed `MisconfigurationException` to `ModuleNotFoundError` when `rich` isn't available ([#11360](https://github.com/PyTorchLightning/pytorch-lightning/pull/11360))
 
 
@@ -416,6 +419,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Removed `Strategy.on_tpu` property ([#11536](https://github.com/PyTorchLightning/pytorch-lightning/pull/11536))
 
 
+ - Removed access to `_short_id` in `NeptuneLogger` ([#11517](https://github.com/PyTorchLightning/pytorch-lightning/pull/11517))
+
+
 ### Fixed
 
 - Fixed security vulnerabilities CVE-2020-1747 and CVE-2020-14343 caused by the `PyYAML` dependency ([#11099](https://github.com/PyTorchLightning/pytorch-lightning/pull/11099))
@@ -436,6 +442,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Fixed wrong typehint for `Trainer.lightning_optimizers` ([#11155](https://github.com/PyTorchLightning/pytorch-lightning/pull/11155))
 
 
+- Fixed the format of the configuration saved automatically by the CLI's `SaveConfigCallback` ([#11532](https://github.com/PyTorchLightning/pytorch-lightning/pull/11532))
+
+
 - Fixed type promotion when tensors of higher category than float are logged ([#11401](https://github.com/PyTorchLightning/pytorch-lightning/pull/11401))
 
 

docs/source/common/loggers.rst

@@ -212,7 +212,10 @@ The :class:`~pytorch_lightning.loggers.WandbLogger` is available anywhere except
     class MyModule(LightningModule):
         def any_lightning_module_function_or_hook(self):
             some_img = fake_image()
-            self.log({"generated_images": [wandb.Image(some_img, caption="...")]})
+            # Option 1
+            self.logger.experiment.log({"generated_images": [wandb.Image(some_img, caption="...")]})
+            # Option 2 for specifically logging images
+            self.logger.log_image(key="generated_images", images=[some_img])
 
 .. seealso::
     - :class:`~pytorch_lightning.loggers.WandbLogger` docs.

pytorch_lightning/accelerators/accelerator.py

@@ -11,15 +11,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from abc import abstractmethod
+from abc import ABC, abstractmethod
 from typing import Any, Dict, Union
 
 import torch
 
 import pytorch_lightning as pl
 
 
-class Accelerator:
+class Accelerator(ABC):
     """The Accelerator Base Class. An Accelerator is meant to deal with one type of Hardware.
 
     Currently there are accelerators for:
@@ -45,7 +45,7 @@ def setup(self, trainer: "pl.Trainer") -> None:
         """
 
     def get_device_stats(self, device: Union[str, torch.device]) -> Dict[str, Any]:
-        """Gets stats for a given device.
+        """Get stats for a given device.
 
         Args:
             device: device for which to get stats
@@ -58,4 +58,4 @@ def get_device_stats(self, device: Union[str, torch.device]) -> Dict[str, Any]:
     @staticmethod
     @abstractmethod
     def auto_device_count() -> int:
-        """Get the devices when set to auto."""
+        """Get the device count when set to auto."""

pytorch_lightning/callbacks/base.py

@@ -12,11 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 r"""
-Abstract base class used to build new callbacks.
+Base class used to build new callbacks.
 
 """
 
-import abc
 from typing import Any, Dict, List, Optional, Type
 
 import torch
@@ -26,7 +25,7 @@
 from pytorch_lightning.utilities.types import STEP_OUTPUT
 
 
-class Callback(abc.ABC):
+class Callback:
     r"""
     Abstract base class used to build new callbacks.
 
@@ -62,15 +61,12 @@ def on_configure_sharded_model(self, trainer: "pl.Trainer", pl_module: "pl.Light
 
     def on_before_accelerator_backend_setup(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called before accelerator is being setup."""
-        pass
 
     def setup(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule", stage: Optional[str] = None) -> None:
         """Called when fit, validate, test, predict, or tune begins."""
-        pass
 
     def teardown(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule", stage: Optional[str] = None) -> None:
         """Called when fit, validate, test, predict, or tune ends."""
-        pass
 
     def on_init_start(self, trainer: "pl.Trainer") -> None:
         r"""
@@ -79,7 +75,6 @@ def on_init_start(self, trainer: "pl.Trainer") -> None:
 
         Called when the trainer initialization begins, model has not yet been set.
         """
-        pass
 
     def on_init_end(self, trainer: "pl.Trainer") -> None:
         r"""
@@ -88,23 +83,18 @@ def on_init_end(self, trainer: "pl.Trainer") -> None:
 
         Called when the trainer initialization ends, model has not yet been set.
         """
-        pass
 
     def on_fit_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when fit begins."""
-        pass
 
     def on_fit_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when fit ends."""
-        pass
 
     def on_sanity_check_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when the validation sanity check starts."""
-        pass
 
     def on_sanity_check_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when the validation sanity check ends."""
-        pass
 
     def on_train_batch_start(
         self,
@@ -115,7 +105,6 @@ def on_train_batch_start(
         unused: int = 0,
     ) -> None:
         """Called when the train batch begins."""
-        pass
 
     def on_train_batch_end(
         self,
@@ -127,11 +116,9 @@ def on_train_batch_end(
         unused: int = 0,
     ) -> None:
         """Called when the train batch ends."""
-        pass
 
     def on_train_epoch_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when the train epoch begins."""
-        pass
 
     def on_train_epoch_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when the train epoch ends.
@@ -141,53 +128,41 @@ def on_train_epoch_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModu
         1. Implement `training_epoch_end` in the `LightningModule` and access outputs via the module OR
         2. Cache data across train batch hooks inside the callback implementation to post-process in this hook.
         """
-        pass
 
     def on_validation_epoch_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when the val epoch begins."""
-        pass
 
     def on_validation_epoch_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when the val epoch ends."""
-        pass
 
     def on_test_epoch_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when the test epoch begins."""
-        pass
 
     def on_test_epoch_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when the test epoch ends."""
-        pass
 
     def on_predict_epoch_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when the predict epoch begins."""
-        pass
 
     def on_predict_epoch_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule", outputs: List[Any]) -> None:
         """Called when the predict epoch ends."""
-        pass
 
     def on_epoch_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when either of train/val/test epoch begins."""
-        pass
 
     def on_epoch_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when either of train/val/test epoch ends."""
-        pass
 
     def on_batch_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when the training batch begins."""
-        pass
 
     def on_batch_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when the training batch ends."""
-        pass
 
     def on_validation_batch_start(
         self, trainer: "pl.Trainer", pl_module: "pl.LightningModule", batch: Any, batch_idx: int, dataloader_idx: int
     ) -> None:
         """Called when the validation batch begins."""
-        pass
 
     def on_validation_batch_end(
         self,
@@ -199,13 +174,11 @@ def on_validation_batch_end(
         dataloader_idx: int,
     ) -> None:
         """Called when the validation batch ends."""
-        pass
 
     def on_test_batch_start(
         self, trainer: "pl.Trainer", pl_module: "pl.LightningModule", batch: Any, batch_idx: int, dataloader_idx: int
     ) -> None:
         """Called when the test batch begins."""
-        pass
 
     def on_test_batch_end(
         self,
@@ -217,13 +190,11 @@ def on_test_batch_end(
         dataloader_idx: int,
     ) -> None:
         """Called when the test batch ends."""
-        pass
 
     def on_predict_batch_start(
         self, trainer: "pl.Trainer", pl_module: "pl.LightningModule", batch: Any, batch_idx: int, dataloader_idx: int
     ) -> None:
         """Called when the predict batch begins."""
-        pass
 
     def on_predict_batch_end(
         self,
@@ -235,47 +206,36 @@ def on_predict_batch_end(
         dataloader_idx: int,
     ) -> None:
         """Called when the predict batch ends."""
-        pass
 
     def on_train_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when the train begins."""
-        pass
 
     def on_train_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when the train ends."""
-        pass
 
     def on_pretrain_routine_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when the pretrain routine begins."""
-        pass
 
     def on_pretrain_routine_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when the pretrain routine ends."""
-        pass
 
     def on_validation_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when the validation loop begins."""
-        pass
 
     def on_validation_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when the validation loop ends."""
-        pass
 
     def on_test_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when the test begins."""
-        pass
 
     def on_test_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when the test ends."""
-        pass
 
     def on_predict_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when the predict begins."""
-        pass
 
     def on_predict_end(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called when predict ends."""
-        pass
 
     def on_keyboard_interrupt(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         r"""
@@ -284,11 +244,9 @@ def on_keyboard_interrupt(self, trainer: "pl.Trainer", pl_module: "pl.LightningM
 
         Called when any trainer execution is interrupted by KeyboardInterrupt.
         """
-        pass
 
     def on_exception(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule", exception: BaseException) -> None:
         """Called when any trainer execution is interrupted by an exception."""
-        pass
 
     def on_save_checkpoint(
         self, trainer: "pl.Trainer", pl_module: "pl.LightningModule", checkpoint: Dict[str, Any]
@@ -303,7 +261,6 @@ def on_save_checkpoint(
         Returns:
             The callback state.
         """
-        pass
 
     def on_load_checkpoint(
         self, trainer: "pl.Trainer", pl_module: "pl.LightningModule", callback_state: Dict[str, Any]
@@ -320,22 +277,17 @@ def on_load_checkpoint(
             If your ``on_load_checkpoint`` hook behavior doesn't rely on a state,
             you will still need to override ``on_save_checkpoint`` to return a ``dummy state``.
         """
-        pass
 
     def on_before_backward(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule", loss: torch.Tensor) -> None:
         """Called before ``loss.backward()``."""
-        pass
 
     def on_after_backward(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:
         """Called after ``loss.backward()`` and before optimizers are stepped."""
-        pass
 
     def on_before_optimizer_step(
         self, trainer: "pl.Trainer", pl_module: "pl.LightningModule", optimizer: Optimizer, opt_idx: int
     ) -> None:
         """Called before ``optimizer.step()``."""
-        pass
 
     def on_before_zero_grad(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule", optimizer: Optimizer) -> None:
         """Called before ``optimizer.zero_grad()``."""
-        pass

pytorch_lightning/loggers/neptune.py

@@ -293,9 +293,10 @@ def __init__(
     def _retrieve_run_data(self):
         try:
             self._run_instance.wait()
-            self._run_short_id = self.run._short_id  # skipcq: PYL-W0212
+            self._run_short_id = self._run_instance["sys/id"].fetch()
             self._run_name = self._run_instance["sys/name"].fetch()
         except NeptuneOfflineModeFetchException:
+            self._run_short_id = "OFFLINE"
             self._run_name = "offline-name"
 
     @property

pytorch_lightning/loops/dataloader/evaluation_loop.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from collections import OrderedDict
 from typing import Any, List, Sequence, Union
 
 import torch
@@ -115,9 +116,10 @@ def advance(self, *args: Any, **kwargs: Any) -> None:
         )
         dl_max_batches = self._max_batches[dataloader_idx]
 
-        dl_outputs = self.epoch_loop.run(
-            dataloader, dataloader_idx if self.num_dataloaders > 1 else None, dl_max_batches
-        )
+        kwargs = OrderedDict()
+        if self.num_dataloaders > 1:
+            kwargs["dataloader_idx"] = dataloader_idx
+        dl_outputs = self.epoch_loop.run(dataloader, dl_max_batches, kwargs)
 
         # store batch level output per dataloader
         self._outputs.append(dl_outputs)