Merge branch 'release/1.2-dev' into refactor/legacy-accel-plug

Author: mergify[bot]

pytorch_lightning/core/step_result.py

@@ -700,232 +700,6 @@ def collate_tensors(items: Union[List, Tuple]) -> Union[Tensor, List, Tuple]:
     return items
 
 
-class EvalResult(Result):
-    def __init__(
-        self,
-        early_stop_on: Optional[Tensor] = None,
-        checkpoint_on: Optional[Tensor] = None,
-        hiddens: Optional[Tensor] = None,
-    ):
-        """
-        Used in val/train loop to auto-log to a logger or progress bar without needing to define
-        a _step_end or _epoch_end method
-
-        Example::
-
-            def validation_step(self, batch, batch_idx):
-                loss = ...
-                result = EvalResult()
-                result.log('val_loss', loss)
-                return result
-
-            def test_step(self, batch, batch_idx):
-                loss = ...
-                result = EvalResult()
-                result.log('val_loss', loss)
-                return result
-
-        Args:
-            early_stop_on: Metric to early stop on.
-                Should be a one element tensor if combined with default
-                :class:`~pytorch_lightning.callbacks.early_stopping.EarlyStopping`.
-                If this result is returned by
-                :meth:`~pytorch_lightning.core.lightning.LightningModule.validation_step`,
-                the specified value will be averaged across all steps.
-            checkpoint_on: Metric to checkpoint on.
-                Should be a one element tensor if combined with default checkpoint callback.
-                If this result is returned by
-                :meth:`~pytorch_lightning.core.lightning.LightningModule.validation_step`,
-                the specified value will be averaged across all steps.
-            hiddens:
-        """
-
-        super().__init__(None, early_stop_on, checkpoint_on, hiddens)
-
-    def log(
-        self,
-        name,
-        value,
-        prog_bar: bool = False,
-        logger: bool = True,
-        on_step: bool = False,
-        on_epoch: bool = True,
-        reduce_fx: Callable = torch.mean,
-        tbptt_reduce_fx: Callable = torch.mean,
-        tbptt_pad_token: int = 0,
-        enable_graph: bool = False,
-        sync_dist: bool = False,
-        sync_dist_op: Union[Any, str] = 'mean',
-        sync_dist_group: Optional[Any] = None,
-    ):
-        """
-        Log a key, value
-
-        Example::
-
-            result.log('val_loss', loss)
-
-            # defaults used
-            result.log(
-                name,
-                value,
-                on_step=False,
-                on_epoch=True,
-                logger=True,
-                prog_bar=False,
-                reduce_fx=torch.mean
-            )
-
-
-        Args:
-            name: key name
-            value: value name
-            prog_bar: if True logs to the progress base
-            logger: if True logs to the logger
-            on_step: if True logs the output of validation_step or test_step
-            on_epoch: if True, logs the output of the training loop aggregated
-            reduce_fx: Torch.mean by default
-            tbptt_reduce_fx: function to reduce on truncated back prop
-            tbptt_pad_token: token to use for padding
-            enable_graph: if True, will not auto detach the graph
-            sync_dist: if True, reduces the metric across GPUs/TPUs
-            sync_dist_op: the op to sync across
-            sync_dist_group: the ddp group
-        """
-        super().log(
-            name=name,
-            value=value,
-            prog_bar=prog_bar,
-            logger=logger,
-            on_step=on_step,
-            on_epoch=on_epoch,
-            reduce_fx=reduce_fx,
-            enable_graph=enable_graph,
-            sync_dist=sync_dist,
-            sync_dist_group=sync_dist_group,
-            sync_dist_op=sync_dist_op,
-            tbptt_pad_token=tbptt_pad_token,
-            tbptt_reduce_fx=tbptt_reduce_fx,
-        )
-
-    def log_dict(
-        self,
-        dictionary: dict,
-        prog_bar: bool = False,
-        logger: bool = True,
-        on_step: bool = False,
-        on_epoch: bool = True,
-        reduce_fx: Callable = torch.mean,
-        tbptt_reduce_fx: Callable = torch.mean,
-        tbptt_pad_token: int = 0,
-        enable_graph: bool = False,
-        sync_dist: bool = False,
-        sync_dist_op: Union[Any, str] = 'mean',
-        sync_dist_group: Optional[Any] = None,
-    ):
-        """
-        Log a dictonary of values at once
-
-        Example::
-
-            values = {'loss': loss, 'acc': acc, ..., 'metric_n': metric_n}
-            result.log_dict(values)
-
-        Args:
-            dictionary: key value pairs (str, tensors)
-            prog_bar: if True logs to the progress base
-            logger: if True logs to the logger
-            on_step: if True logs the output of validation_step or test_step
-            on_epoch: if True, logs the output of the training loop aggregated
-            reduce_fx: Torch.mean by default
-            tbptt_reduce_fx: function to reduce on truncated back prop
-            tbptt_pad_token: token to use for padding
-            enable_graph: if True, will not auto detach the graph
-            sync_dist: if True, reduces the metric across GPUs/TPUs
-            sync_dist_op: the op to sync across
-            sync_dist_group: the ddp group
-        """
-        for k, v in dictionary.items():
-            self.log(
-                name=k,
-                value=v,
-                prog_bar=prog_bar,
-                logger=logger,
-                on_step=on_step,
-                on_epoch=on_epoch,
-                reduce_fx=reduce_fx,
-                enable_graph=enable_graph,
-                sync_dist=sync_dist,
-                sync_dist_group=sync_dist_group,
-                sync_dist_op=sync_dist_op,
-                tbptt_pad_token=tbptt_pad_token,
-                tbptt_reduce_fx=tbptt_reduce_fx,
-            )
-
-    def get_callback_metrics(self) -> dict:
-        result = {}
-        if self.early_stop_on:
-            result['early_stop_on'] = self.early_stop_on
-        if self.checkpoint_on:
-            result['checkpoint_on'] = self.checkpoint_on
-        return result
-
-    def write(self, name: str, values: Union[Tensor, list], filename: str = 'predictions.pt'):
-        """Add feature name and value pair to collection of predictions that will be written to disk on
-        `validation_end` or `test_end`. If running on multiple GPUs, you will get separate `n_gpu`
-        prediction files with the rank prepended onto filename.
-
-        Example::
-
-            result = pl.EvalResult()
-            result.write('ids', [0, 1, 2])
-            result.write('preds', ['cat', 'dog', 'dog'])
-
-        Args:
-            name: Feature name that will turn into column header of predictions file
-            values: Flat tensor or list of row values for given feature column 'name'.
-            filename: Filepath where your predictions will be saved. Defaults to 'predictions.pt'.
-        """
-        # Type check the incoming arguments
-        if not isinstance(name, str):
-            raise ValueError(f"Expected str for 'name' but got {type(name)}")
-        if not isinstance(filename, str):
-            raise ValueError(f"Expected str for 'filename' but got {type(name)}")
-
-        if isinstance(values, Tensor):
-            values = values.detach()
-
-        preds = getattr(self, 'predictions', None)
-        if preds is None:
-            self.predictions = {filename: {name: values}}
-        elif filename not in preds:
-            preds[filename] = {name: values}
-        elif name not in preds[filename]:
-            preds[filename][name] = values
-        elif isinstance(values, Tensor):
-            preds[filename][name] = torch.cat((preds[filename][name], values))
-        elif isinstance(values, list):
-            preds[filename][name].extend(values)
-
-    def write_dict(self, predictions_dict, filename='predictions.pt'):
-        """Calls EvalResult.write() for each key-value pair in predictions_dict.
-
-        It is recommended that you use this function call instead of .write if you need to
-        store more than one column of predictions in your output file.
-
-        Example::
-
-            predictions_to_write = {'preds': ['cat', 'dog'], 'ids': tensor([0, 1])}
-            result.write_dict(predictions_to_write)
-
-        Args:
-            predictions_dict ([type]): Dict of predictions to store and then write to filename at eval end.
-            filename (str, optional): File where your predictions will be stored. Defaults to './predictions.pt'.
-        """
-        for k, v in predictions_dict.items():
-            self.write(k, v, filename)
-
-
 def weighted_mean(result, weights):
 
     if isinstance(result, dict):

pytorch_lightning/trainer/connectors/logger_connector/logger_connector.py

@@ -19,7 +19,7 @@
 import torch
 
 from pytorch_lightning.core import memory
-from pytorch_lightning.core.step_result import EvalResult, Result
+from pytorch_lightning.core.step_result import Result
 from pytorch_lightning.loggers import LoggerCollection, TensorBoardLogger
 from pytorch_lightning.trainer.connectors.logger_connector.callback_hook_validator import CallbackHookNameValidator
 from pytorch_lightning.trainer.connectors.logger_connector.epoch_result_store import EpochResultStore, LoggerStages
@@ -259,8 +259,8 @@ def add_progress_bar_metrics(self, metrics):
 
         self.trainer.dev_debugger.track_pbar_metrics_history(metrics)
 
-    def track_metrics_deprecated(self, deprecated_eval_results, using_eval_result, test_mode):
-        self._track_callback_metrics(deprecated_eval_results, using_eval_result)
+    def track_metrics_deprecated(self, deprecated_eval_results, test_mode):
+        self._track_callback_metrics(deprecated_eval_results)
         self.__process_eval_epoch_end_results_and_log_legacy(deprecated_eval_results, test_mode)
 
     def evaluation_epoch_end(self, testing):
@@ -314,53 +314,41 @@ def get_evaluate_epoch_results(self, test_mode):
         self.eval_loop_results = []
         return results
 
-    def _track_callback_metrics(self, eval_results, using_eval_result):
+    def _track_callback_metrics(self, eval_results):
         if len(eval_results) > 0 and (eval_results[0] is None or not isinstance(eval_results[0], Result)):
             return
 
-        if using_eval_result:
-            if isinstance(eval_results, list):
-                for eval_result in eval_results:
-                    self.trainer.logger_connector.callback_metrics.update(eval_result.callback_metrics)
-                    if self.trainer.testing:
-                        self.trainer.logger_connector.evaluation_callback_metrics.update(
-                            eval_result.callback_metrics)
-            else:
-                self.trainer.logger_connector.callback_metrics.update(eval_results.callback_metrics)
-                if self.trainer.testing:
-                    self.trainer.logger_connector.evaluation_callback_metrics.update(eval_results.callback_metrics)
-        else:
-            flat = {}
-            if isinstance(eval_results, list):
-                for eval_result in eval_results:
-                    # with a scalar return, auto set it to "val_loss" for callbacks
-                    if isinstance(eval_result, torch.Tensor):
-                        flat = {'val_loss': eval_result}
-                    elif isinstance(eval_result, dict):
-                        flat = flatten_dict(eval_result)
-
-                    # removing val_loss magic word to map to checkpoint + ES callback
-                    if 'val_loss' in flat:
-                        flat['checkpoint_on'] = flat['val_loss']
-                        flat['early_stop_on'] = flat['val_loss']
-                    self.trainer.logger_connector.callback_metrics.update(flat)
-                    if self.trainer.testing:
-                        self.trainer.logger_connector.evaluation_callback_metrics.update(flat)
-            else:
+        flat = {}
+        if isinstance(eval_results, list):
+            for eval_result in eval_results:
                 # with a scalar return, auto set it to "val_loss" for callbacks
-                if isinstance(eval_results, torch.Tensor):
-                    flat = {'val_loss': eval_results}
-                else:
-                    flat = flatten_dict(eval_results)
+                if isinstance(eval_result, torch.Tensor):
+                    flat = {'val_loss': eval_result}
+                elif isinstance(eval_result, dict):
+                    flat = flatten_dict(eval_result)
 
                 # removing val_loss magic word to map to checkpoint + ES callback
                 if 'val_loss' in flat:
                     flat['checkpoint_on'] = flat['val_loss']
                     flat['early_stop_on'] = flat['val_loss']
-
                 self.trainer.logger_connector.callback_metrics.update(flat)
                 if self.trainer.testing:
                     self.trainer.logger_connector.evaluation_callback_metrics.update(flat)
+        else:
+            # with a scalar return, auto set it to "val_loss" for callbacks
+            if isinstance(eval_results, torch.Tensor):
+                flat = {'val_loss': eval_results}
+            else:
+                flat = flatten_dict(eval_results)
+
+            # removing val_loss magic word to map to checkpoint + ES callback
+            if 'val_loss' in flat:
+                flat['checkpoint_on'] = flat['val_loss']
+                flat['early_stop_on'] = flat['val_loss']
+
+            self.trainer.logger_connector.callback_metrics.update(flat)
+            if self.trainer.testing:
+                self.trainer.logger_connector.evaluation_callback_metrics.update(flat)
 
     def __process_eval_epoch_end_results_and_log_legacy_update(self, prog_bar_metrics, log_metrics, callback_metrics):
         # eval loop returns all metrics
@@ -397,16 +385,7 @@ def __process_eval_epoch_end_results_and_log_legacy(self, eval_results, test_mod
             prog_bar_metrics, log_metrics, callback_metrics = {}, {}, {}
 
             for result_idx, result in enumerate(eval_results):
-                if isinstance(result, EvalResult):
-                    prog_bar_metrics = result.epoch_pbar_metrics
-                    log_metrics = result.epoch_log_metrics
-                    callback_metrics = result.callback_metrics
-
-                    # in testing we don't need the callback metrics
-                    if test_mode:
-                        callback_metrics = {}
-                else:
-                    _, prog_bar_metrics, log_metrics, callback_metrics, _ = self.trainer.process_dict_result(result)
+                _, prog_bar_metrics, log_metrics, callback_metrics, _ = self.trainer.process_dict_result(result)
 
                 if num_loaders > 1:
                     self.__process_eval_epoch_end_results_and_log_legacy_update(