Merge branch 'master' into bugfix/ep_end_ckpt

Author: rohitgr7

docs/source/metrics.rst

@@ -137,6 +137,56 @@ This metrics API is independent of PyTorch Lightning. Metrics can directly be us
     To change this, after initializing the metric, the method ``.persistent(mode)`` can
     be used to enable (``mode=True``) or disable (``mode=False``) this behaviour.
 
+*******************
+Metrics and devices
+*******************
+
+Metrics are simple subclasses of :class:`~torch.nn.Module` and their metric states behave
+similar to buffers and parameters of modules. This means that metrics states should
+be moved to the same device as the input of the metric:
+
+.. code-block:: python
+
+    import torch
+    from pytorch_lightning.metrics import Accuracy
+
+    target = torch.tensor([1, 1, 0, 0], device=torch.device("cuda", 0))
+    preds = torch.tensor([0, 1, 0, 0], device=torch.device("cuda", 0))
+
+    # Metric states are always initialized on cpu, and needs to be moved to
+    # the correct device
+    confmat = Accuracy(num_classes=2).to(torch.device("cuda", 0))
+    out = confmat(preds, target)
+    print(out.device) # cuda:0
+
+However, when **properly defined** inside a :class:`~pytorch_lightning.core.lightning.LightningModule`
+, Lightning will automatically move the metrics to the same device as the data. Being
+**properly defined** means that the metric is correctly identified as a child module of the
+model (check ``.children()`` attribute of the model). Therefore, metrics cannot be placed
+in native python ``list`` and ``dict``, as they will not be correctly identified
+as child modules. Instead of ``list`` use :class:`~torch.nn.ModuleList` and instead of
+``dict`` use :class:`~torch.nn.ModuleDict`.
+
+.. testcode::
+
+    class MyModule(LightningModule):
+        def __init__(self):
+            ...
+            # valid ways metrics will be identified as child modules
+            self.metric1 = pl.metrics.Accuracy()
+            self.metric2 = torch.nn.ModuleList(pl.metrics.Accuracy())
+            self.metric3 = torch.nn.ModuleDict({'accuracy': Accuracy()})
+
+        def training_step(self, batch, batch_idx):
+            # all metrics will be on the same device as the input batch
+            data, target = batch
+            preds = self(data)
+            ...
+            val1 = self.metric1(preds, target)
+            val2 = self.metric2[0](preds, target)
+            val3 = self.metric3['accuracy'](preds, target)
+
+
 *********************
 Implementing a Metric
 *********************

docs/source/multi_gpu.rst

@@ -239,23 +239,6 @@ Note in particular the difference between `gpus=0`, `gpus=[0]` and `gpus="0"`.
     to be in "exclusive mode", such that only one process at a time can access them.
     For more details see the :ref:`Trainer guide <trainer>`.
 
-
-Remove CUDA flags
-^^^^^^^^^^^^^^^^^
-
-CUDA flags make certain GPUs visible to your script.
-Lightning sets these for you automatically, there's NO NEED to do this yourself.
-
-.. testcode::
-
-    # lightning will set according to what you give the trainer
-    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-    os.environ["CUDA_VISIBLE_DEVICES"] = "0"
-
-However, when using a cluster, Lightning will NOT set these flags (and you should not either).
-SLURM will set these for you.
-For more details see the :ref:`SLURM cluster guide <slurm>`.
-
 ----------
 
 Distributed modes

pl_examples/basic_examples/autoencoder.py

@@ -15,17 +15,16 @@
 from argparse import ArgumentParser
 
 import torch
-import torch.nn.functional as F
 from torch import nn
-from torch.utils.data import DataLoader
-from torch.utils.data import random_split
+import torch.nn.functional as F
+from torch.utils.data import DataLoader, random_split
 
+from pl_examples import cli_lightning_logo, TORCHVISION_AVAILABLE
 import pytorch_lightning as pl
-from pl_examples import TORCHVISION_AVAILABLE, cli_lightning_logo
 
 if TORCHVISION_AVAILABLE:
-    from torchvision.datasets.mnist import MNIST
     from torchvision import transforms
+    from torchvision.datasets.mnist import MNIST
 else:
     from tests.base.datasets import MNIST
 

pl_examples/basic_examples/backbone_image_classifier.py

@@ -18,12 +18,12 @@
 from torch.nn import functional as F
 from torch.utils.data import DataLoader, random_split
 
+from pl_examples import cli_lightning_logo, DATASETS_PATH, TORCHVISION_AVAILABLE
 import pytorch_lightning as pl
-from pl_examples import DATASETS_PATH, TORCHVISION_AVAILABLE, cli_lightning_logo
 
 if TORCHVISION_AVAILABLE:
-    from torchvision.datasets.mnist import MNIST
     from torchvision import transforms
+    from torchvision.datasets.mnist import MNIST
 else:
     from tests.base.datasets import MNIST
 

pl_examples/basic_examples/conv_sequential_example.py

@@ -20,16 +20,16 @@
 To run:
 python conv_model_sequential_example.py --accelerator ddp --gpus 4 --max_epochs 1  --batch_size 256 --use_ddp_sequential
 """
-import math
 from argparse import ArgumentParser
+import math
 
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
 import torchvision
 
-import pytorch_lightning as pl
 from pl_examples import cli_lightning_logo
+import pytorch_lightning as pl
 from pytorch_lightning import Trainer
 from pytorch_lightning.metrics.functional import accuracy
 from pytorch_lightning.plugins.ddp_sequential_plugin import DDPSequentialPlugin

pl_examples/basic_examples/dali_image_classifier.py

@@ -13,29 +13,29 @@
 # limitations under the License.
 from abc import ABC
 from argparse import ArgumentParser
+from distutils.version import LooseVersion
 from random import shuffle
 from warnings import warn
-from distutils.version import LooseVersion
 
 import numpy as np
 import torch
 from torch.nn import functional as F
 from torch.utils.data import random_split
 
+from pl_examples import cli_lightning_logo, DALI_AVAILABLE, TORCHVISION_AVAILABLE
 import pytorch_lightning as pl
-from pl_examples import TORCHVISION_AVAILABLE, DALI_AVAILABLE, cli_lightning_logo
 
 if TORCHVISION_AVAILABLE:
-    from torchvision.datasets.mnist import MNIST
     from torchvision import transforms
+    from torchvision.datasets.mnist import MNIST
 else:
     from tests.base.datasets import MNIST
 
 if DALI_AVAILABLE:
+    from nvidia.dali import __version__ as dali_version
     from nvidia.dali import ops
     from nvidia.dali.pipeline import Pipeline
     from nvidia.dali.plugin.pytorch import DALIClassificationIterator
-    from nvidia.dali import __version__ as dali_version
 
     NEW_DALI_API = LooseVersion(dali_version) >= LooseVersion('0.28.0')
     if NEW_DALI_API:

pl_examples/basic_examples/simple_image_classifier.py

@@ -18,9 +18,9 @@
 import torch
 from torch.nn import functional as F
 
-import pytorch_lightning as pl
 from pl_examples import cli_lightning_logo
 from pl_examples.basic_examples.mnist_datamodule import MNISTDataModule
+import pytorch_lightning as pl
 
 
 class LitClassifier(pl.LightningModule):

pl_examples/bug_report_model.py

@@ -20,11 +20,12 @@
 # --------------------------------------------
 # --------------------------------------------
 import os
+
 import torch
 from torch.utils.data import Dataset
 
 from pl_examples import cli_lightning_logo
-from pytorch_lightning import Trainer, LightningModule
+from pytorch_lightning import LightningModule, Trainer
 
 
 class RandomDataset(Dataset):

pl_examples/domain_templates/computer_vision_fine_tuning.py

@@ -38,22 +38,21 @@
 from collections import OrderedDict
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Optional, Generator, Union
+from typing import Generator, Optional, Union
 
 import torch
-import torch.nn.functional as F
 from torch import optim
 from torch.nn import Module
+import torch.nn.functional as F
 from torch.optim.lr_scheduler import MultiStepLR
 from torch.optim.optimizer import Optimizer
 from torch.utils.data import DataLoader
-from torchvision import models
-from torchvision import transforms
+from torchvision import models, transforms
 from torchvision.datasets import ImageFolder
 from torchvision.datasets.utils import download_and_extract_archive
 
-import pytorch_lightning as pl
 from pl_examples import cli_lightning_logo
+import pytorch_lightning as pl
 from pytorch_lightning import _logger as log
 
 BN_TYPES = (torch.nn.BatchNorm1d, torch.nn.BatchNorm2d, torch.nn.BatchNorm3d)

pl_examples/domain_templates/generative_adversarial_net.py

@@ -19,20 +19,20 @@
 
 tensorboard --logdir default
 """
-import os
 from argparse import ArgumentParser, Namespace
+import os
 
 import numpy as np
 import torch
 import torch.nn as nn
 import torch.nn.functional as F  # noqa
-import torchvision
-import torchvision.transforms as transforms
 from torch.utils.data import DataLoader
+import torchvision
 from torchvision.datasets import MNIST
+import torchvision.transforms as transforms
 
 from pl_examples import cli_lightning_logo
-from pytorch_lightning.core import LightningModule, LightningDataModule
+from pytorch_lightning.core import LightningDataModule, LightningModule
 from pytorch_lightning.trainer import Trainer