From d45fc92aa4e580d0762bdd7284fbaabf7784d2bf Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Thu, 1 Jul 2021 19:54:07 +0200 Subject: [PATCH 1/7] Move result teardown to loops --- pytorch_lightning/loops/dataloader/evaluation_loop.py | 4 ++++ pytorch_lightning/loops/epoch/training_epoch_loop.py | 5 ++++- pytorch_lightning/loops/fit_loop.py | 3 +++ .../trainer/connectors/logger_connector/logger_connector.py | 6 ------ pytorch_lightning/trainer/trainer.py | 1 - 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pytorch_lightning/loops/dataloader/evaluation_loop.py b/pytorch_lightning/loops/dataloader/evaluation_loop.py index bdee54a174891..ca670a2f22986 100644 --- a/pytorch_lightning/loops/dataloader/evaluation_loop.py +++ b/pytorch_lightning/loops/dataloader/evaluation_loop.py @@ -266,3 +266,7 @@ def on_evaluation_epoch_end(self) -> None: self.trainer.call_hook(hook_name) self.trainer.call_hook("on_epoch_end") self.trainer.logger_connector.on_epoch_end() + + def teardown(self) -> None: + self._results.cpu() + self.epoch_loop.teardown() diff --git a/pytorch_lightning/loops/epoch/training_epoch_loop.py b/pytorch_lightning/loops/epoch/training_epoch_loop.py index 89891c0d6148a..26bb57e6486e5 100644 --- a/pytorch_lightning/loops/epoch/training_epoch_loop.py +++ b/pytorch_lightning/loops/epoch/training_epoch_loop.py @@ -222,7 +222,10 @@ def on_run_end(self) -> List[List[STEP_OUTPUT]]: def teardown(self) -> None: """Frees memory of tracked epoch outputs.""" - self.epoch_output = None + self._epoch_output = None + self._results.cpu() + self.batch_loop.teardown() + self.val_loop.teardown() def _run_validation(self): # reload dataloaders diff --git a/pytorch_lightning/loops/fit_loop.py b/pytorch_lightning/loops/fit_loop.py index bf42663fd5c9e..e6d14b3dd3905 100644 --- a/pytorch_lightning/loops/fit_loop.py +++ b/pytorch_lightning/loops/fit_loop.py @@ -286,3 +286,6 @@ def state_dict(self) -> Dict: def load_state_dict(self, state_dict: Dict) -> None: self.epoch_loop.load_state_dict(state_dict["epoch_loop"]) + + def teardown(self) -> None: + self.epoch_loop.teardown() diff --git a/pytorch_lightning/trainer/connectors/logger_connector/logger_connector.py b/pytorch_lightning/trainer/connectors/logger_connector/logger_connector.py index 5a950d40f8f54..48bd095418820 100644 --- a/pytorch_lightning/trainer/connectors/logger_connector/logger_connector.py +++ b/pytorch_lightning/trainer/connectors/logger_connector/logger_connector.py @@ -312,9 +312,3 @@ def progress_bar_metrics(self) -> Dict[str, float]: metrics = self.metrics[MetricSource.PBAR] self._progress_bar_metrics.update(metrics) return self._progress_bar_metrics - - def teardown(self): - self.trainer.fit_loop.epoch_loop._results.cpu() - self.trainer.fit_loop.epoch_loop.val_loop._results.cpu() - self.trainer.validate_loop._results.cpu() - self.trainer.test_loop._results.cpu() diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index 008cdb2239df7..d1e3fdb0ee9db 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -901,7 +901,6 @@ def _pre_dispatch(self): def _post_dispatch(self): self.accelerator.post_dispatch(self) self.accelerator.teardown() - self.logger_connector.teardown() def _dispatch(self): if self.evaluating: From 2519407be2c3eca0737e900a4a3e8589da3471d1 Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Thu, 1 Jul 2021 19:58:07 +0200 Subject: [PATCH 2/7] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cb387581a1a9..afdb1c96cf5ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -170,6 +170,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). * Refactored trainer `_run_*` functions and separate evaluation loops ([#8065](https://github.com/PyTorchLightning/pytorch-lightning/pull/8065)) * Refactored prediction loop interface; added new classes `PredictionLoop`, `PredictionEpochLoop` ([#7700](https://github.com/PyTorchLightning/pytorch-lightning/pull/7700), [#8077](https://github.com/PyTorchLightning/pytorch-lightning/pull/8077)) * Removed `pytorch_lightning/trainer/predict_loop.py` ([#8094](https://github.com/PyTorchLightning/pytorch-lightning/pull/8094)) + * Moved result teardown to the loops ([#8245](https://github.com/PyTorchLightning/pytorch-lightning/pull/8245)) - Refactored logging From ccaa23230801a44433a055088b6c8107516e9b58 Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Fri, 2 Jul 2021 01:03:37 +0200 Subject: [PATCH 3/7] Remove teardown from run --- pytorch_lightning/loops/base.py | 3 +-- pytorch_lightning/trainer/trainer.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/loops/base.py b/pytorch_lightning/loops/base.py index 1edc997e715ce..e57078fd3950d 100644 --- a/pytorch_lightning/loops/base.py +++ b/pytorch_lightning/loops/base.py @@ -100,7 +100,6 @@ def run(self, *args: Any, **kwargs: Any) -> Optional[Any]: break output = self.on_run_end() - self.teardown() return output @abstractmethod @@ -132,7 +131,7 @@ def on_run_end(self) -> Any: """Hook to be called at the end of the run. Its return argument is returned from :attr:`run`.""" def teardown(self) -> None: - """The very last method called inside :meth:`run`. Use to release memory etc.""" + """Use to release memory etc.""" def load_state_dict(self, state_dict: Dict) -> None: """Restore the loop state from the provided state_dict.""" diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index d1e3fdb0ee9db..eda12321b6340 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -901,6 +901,7 @@ def _pre_dispatch(self): def _post_dispatch(self): self.accelerator.post_dispatch(self) self.accelerator.teardown() + self._active_loop.teardown() def _dispatch(self): if self.evaluating: From a81b4df61dc43ffa700943bbd5a3320fdb81572d Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Fri, 2 Jul 2021 01:14:29 +0200 Subject: [PATCH 4/7] Move previous teardown to on_run_end --- pytorch_lightning/loops/epoch/evaluation_epoch_loop.py | 7 +++---- pytorch_lightning/loops/epoch/prediction_epoch_loop.py | 8 ++++---- pytorch_lightning/loops/epoch/training_epoch_loop.py | 8 +++++--- pytorch_lightning/loops/fit_loop.py | 5 +---- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/pytorch_lightning/loops/epoch/evaluation_epoch_loop.py b/pytorch_lightning/loops/epoch/evaluation_epoch_loop.py index d42a8941630a1..7f8ef06d7687f 100644 --- a/pytorch_lightning/loops/epoch/evaluation_epoch_loop.py +++ b/pytorch_lightning/loops/epoch/evaluation_epoch_loop.py @@ -119,11 +119,10 @@ def advance( def on_run_end(self) -> List[STEP_OUTPUT]: """Returns the outputs of the whole run""" - return self.outputs - - def teardown(self) -> None: - """Frees memory of tracked outputs""" + outputs = self.outputs + # free memory self.outputs = [] + return outputs def evaluation_step(self, batch: Any, batch_idx: int, dataloader_idx: int) -> Optional[STEP_OUTPUT]: """The evaluation step (validation_step or test_step depending on the trainer's state). diff --git a/pytorch_lightning/loops/epoch/prediction_epoch_loop.py b/pytorch_lightning/loops/epoch/prediction_epoch_loop.py index 258a81648a3e0..29a76793b4648 100644 --- a/pytorch_lightning/loops/epoch/prediction_epoch_loop.py +++ b/pytorch_lightning/loops/epoch/prediction_epoch_loop.py @@ -88,12 +88,12 @@ def advance( def on_run_end(self) -> Tuple[Any, Any]: """Returns the predictions and the corresponding batch indices""" - return self.predictions, self._all_batch_indices - - def teardown(self) -> None: - """Frees memory of collected predictions.""" + predictions = self.predictions + all_batch_indices = self._all_batch_indices + # free memory self.predictions = [] self._all_batch_indices = [] + return predictions, all_batch_indices def _predict_step(self, batch: Any, batch_idx: int, dataloader_idx: int) -> None: """Runs the actual predict step together with all the diff --git a/pytorch_lightning/loops/epoch/training_epoch_loop.py b/pytorch_lightning/loops/epoch/training_epoch_loop.py index 26bb57e6486e5..4780235ac8d9a 100644 --- a/pytorch_lightning/loops/epoch/training_epoch_loop.py +++ b/pytorch_lightning/loops/epoch/training_epoch_loop.py @@ -218,11 +218,13 @@ def on_run_end(self) -> List[List[STEP_OUTPUT]]: self._on_train_epoch_end_hook(processed_outputs) self.trainer.call_hook('on_epoch_end') self.trainer.logger_connector.on_epoch_end() - return self._epoch_output - def teardown(self) -> None: - """Frees memory of tracked epoch outputs.""" + epoch_output = self._epoch_output + # free memory self._epoch_output = None + return epoch_output + + def teardown(self) -> None: self._results.cpu() self.batch_loop.teardown() self.val_loop.teardown() diff --git a/pytorch_lightning/loops/fit_loop.py b/pytorch_lightning/loops/fit_loop.py index e6d14b3dd3905..7ab315036a814 100644 --- a/pytorch_lightning/loops/fit_loop.py +++ b/pytorch_lightning/loops/fit_loop.py @@ -231,7 +231,7 @@ def on_advance_end(self) -> None: self.global_step += 1 def on_run_end(self) -> None: - """Runs teardown logic and calls the ``on_train_end`` hook""" + """Calls the ``on_train_end`` hook""" # NOTE: the iteration_count/current_epoch is already incremented # Lightning today does not increment the current epoch at the last epoch run in Trainer.fit # To simulate that current behavior, we decrement here. @@ -260,9 +260,6 @@ def on_run_end(self) -> None: # give accelerators a chance to finish self.trainer.accelerator.on_train_end() - # reset bookkeeping - self.trainer._running_stage = None - def should_accumulate(self) -> bool: """Whether the gradients should be accumulated""" return self.epoch_loop.batch_loop.should_accumulate() From 9ea207847025ec026713a285c99afd38d0a6223c Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Fri, 2 Jul 2021 01:20:48 +0200 Subject: [PATCH 5/7] Add comment --- pytorch_lightning/trainer/trainer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index eda12321b6340..76aa0e465b6d0 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -900,6 +900,8 @@ def _pre_dispatch(self): def _post_dispatch(self): self.accelerator.post_dispatch(self) + # these `teardown` calls are here instead of in `_call_teardown_hook` since they are internal teardowns + # which need to happen before. self.accelerator.teardown() self._active_loop.teardown() From 9030119cb9b0b6b1acff65352259c65eb9404ae4 Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Fri, 2 Jul 2021 01:43:30 +0200 Subject: [PATCH 6/7] Merge 8250 --- pytorch_lightning/core/lightning.py | 2 +- .../loops/batch/training_batch_loop.py | 8 ++--- .../loops/dataloader/evaluation_loop.py | 13 ++++---- .../loops/dataloader/prediction_loop.py | 5 +++- .../loops/epoch/training_epoch_loop.py | 12 +------- pytorch_lightning/loops/fit_loop.py | 17 +++++++---- .../logger_connector/logger_connector.py | 4 +-- pytorch_lightning/trainer/properties.py | 30 +++++++++++-------- pytorch_lightning/trainer/trainer.py | 8 ++--- tests/callbacks/test_callback_hook_outputs.py | 2 +- 10 files changed, 50 insertions(+), 51 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 42db2e67a6ceb..2478f698e659b 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -543,7 +543,7 @@ def write_prediction( ' and will be removed in v1.5.' ) - self.trainer.evaluation_loop.predictions._add_prediction(name, value, filename) + self.trainer._evaluation_loop.predictions._add_prediction(name, value, filename) def write_prediction_dict(self, predictions_dict: Dict[str, Any], filename: str = 'predictions.pt'): """ diff --git a/pytorch_lightning/loops/batch/training_batch_loop.py b/pytorch_lightning/loops/batch/training_batch_loop.py index e261a61a366fb..bd73dcfc1239a 100644 --- a/pytorch_lightning/loops/batch/training_batch_loop.py +++ b/pytorch_lightning/loops/batch/training_batch_loop.py @@ -47,7 +47,7 @@ def __init__(self) -> None: self.running_loss: TensorRunningAccum = TensorRunningAccum(window_length=20) self.batch_idx: int = 0 self.split_idx: Optional[int] = None - self.warning_cache: WarningCache = WarningCache() + self._warning_cache: WarningCache = WarningCache() self._hiddens: Optional[Tensor] = None self._optimizer_freq_cumsum: Optional[int] = None @@ -75,7 +75,7 @@ def run(self, batch: Any, batch_idx: int, dataloader_idx: int) -> AttributeDict: dataloader_idx: the index of the dataloader producing the current batch """ if batch is None: - self.warning_cache.warn("train_dataloader yielded None. If this was on purpose, ignore this warning...") + self._warning_cache.warn("train_dataloader yielded None. If this was on purpose, ignore this warning...") return AttributeDict(signal=0, training_step_output=[[]]) # hook @@ -542,7 +542,7 @@ def training_step_and_backward( self._check_finite(result.loss) else: - self.warning_cache.warn( + self._warning_cache.warn( "training_step returned None. If this was on purpose, ignore this warning..." ) @@ -644,7 +644,7 @@ def _build_kwargs(self, batch: Any, batch_idx: int, opt_idx: int, hiddens: Optio has_opt_idx_in_train_step = is_param_in_hook_signature(training_step_fx, "optimizer_idx") if has_opt_idx_in_train_step: if not lightning_module.automatic_optimization: - self.warning_cache.deprecation( + self._warning_cache.deprecation( "`training_step` hook signature has changed in v1.3." " `optimizer_idx` argument has been removed in case of manual optimization. Support for" " the old signature will be removed in v1.5" diff --git a/pytorch_lightning/loops/dataloader/evaluation_loop.py b/pytorch_lightning/loops/dataloader/evaluation_loop.py index ca670a2f22986..02d802fb3fc15 100644 --- a/pytorch_lightning/loops/dataloader/evaluation_loop.py +++ b/pytorch_lightning/loops/dataloader/evaluation_loop.py @@ -33,9 +33,11 @@ def __init__(self): super().__init__() self._max_batches: Optional[Union[int, Sequence[int]]] = None self.outputs = [] + self.epoch_loop = EvaluationEpochLoop() - self._has_run: bool = False + self._results = ResultCollection(training=False) + self._has_run: bool = False @property def num_dataloaders(self) -> int: @@ -57,11 +59,6 @@ def dataloaders(self) -> Sequence[DataLoader]: return self.trainer.test_dataloaders return self.trainer.val_dataloaders - @property - def results(self) -> ResultCollection: - """Returns the current results""" - return self._results - @property def predictions(self): """Returns the predictions from all dataloaders""" @@ -184,8 +181,8 @@ def on_evaluation_start(self, *args: Any, **kwargs: Any) -> None: """Runs ``on_{validation/test}_start`` hooks""" self.should_track_batch_outputs_for_epoch_end: bool = self._should_track_batch_outputs_for_epoch_end() - assert self.results is not None - self.results.to(device=self.trainer.lightning_module.device) + assert self._results is not None + self._results.to(device=self.trainer.lightning_module.device) if self.trainer.testing: self.trainer.call_hook("on_test_start", *args, **kwargs) diff --git a/pytorch_lightning/loops/dataloader/prediction_loop.py b/pytorch_lightning/loops/dataloader/prediction_loop.py index 542f94fdb087e..37b4b83a25ebe 100644 --- a/pytorch_lightning/loops/dataloader/prediction_loop.py +++ b/pytorch_lightning/loops/dataloader/prediction_loop.py @@ -16,9 +16,12 @@ class PredictionLoop(DataLoaderLoop): def __init__(self): super().__init__() - self.epoch_loop: PredictionEpochLoop = PredictionEpochLoop() self.predictions: Optional[List[List[Any]]] = None self.epoch_batch_indices: Optional[List[List[int]]] = None + + self.epoch_loop: PredictionEpochLoop = PredictionEpochLoop() + + self._results = None # for `trainer._results` access self._return_predictions: bool = False @property diff --git a/pytorch_lightning/loops/epoch/training_epoch_loop.py b/pytorch_lightning/loops/epoch/training_epoch_loop.py index 4780235ac8d9a..3d6fbc8dcdc1c 100644 --- a/pytorch_lightning/loops/epoch/training_epoch_loop.py +++ b/pytorch_lightning/loops/epoch/training_epoch_loop.py @@ -34,9 +34,7 @@ def __init__(self, min_steps: int, max_steps: int): super().__init__() self.min_steps: int = min_steps self.max_steps: int = max_steps - self.global_step: int = 0 - # the total batch index across all epochs self.total_batch_idx: int = 0 # the current batch index in the loop that runs over the dataloader(s) @@ -50,18 +48,10 @@ def __init__(self, min_steps: int, max_steps: int): self.batch_loop = TrainingBatchLoop() self.val_loop = loops.EvaluationLoop() + self._results = ResultCollection(training=True) self._dataloader_idx: Optional[int] = None self._warning_cache: WarningCache = WarningCache() self._epoch_output: Optional[List[List[STEP_OUTPUT]]] = None - self._results = ResultCollection(training=True) - - @property - def results(self) -> ResultCollection: - if self.trainer.training: - return self._results - elif self.trainer.validating: - return self.val_loop.results - raise RuntimeError("`FitLoop.results` property isn't defined. Accessed outside of scope") @property def batch_idx(self) -> int: diff --git a/pytorch_lightning/loops/fit_loop.py b/pytorch_lightning/loops/fit_loop.py index 7ab315036a814..a7699eaec812c 100644 --- a/pytorch_lightning/loops/fit_loop.py +++ b/pytorch_lightning/loops/fit_loop.py @@ -50,11 +50,8 @@ def __init__( super().__init__() self.max_epochs = 1000 if (max_epochs is None and max_steps is None) else max_epochs self.min_epochs = 1 if (min_epochs is None and min_steps is None) else min_epochs - self.epoch_loop = TrainingEpochLoop(min_steps, max_steps) - @property - def results(self) -> ResultCollection: - return self.epoch_loop.results + self.epoch_loop = TrainingEpochLoop(min_steps, max_steps) @property def current_epoch(self) -> int: @@ -129,6 +126,14 @@ def _skip_backward(self, value: bool) -> None: """ Determines whether the loop will skip backward during automatic optimization. """ self.epoch_loop.batch_loop._skip_backward = value + @property + def _results(self) -> ResultCollection: + if self.trainer.training: + return self.epoch_loop._results + if self.trainer.validating: + return self.epoch_loop.val_loop._results + raise RuntimeError("`FitLoop._results` property isn't defined. Accessed outside of scope") + @property def done(self) -> bool: """Evaluates when to leave the loop. @@ -172,7 +177,7 @@ def reset(self) -> None: def on_run_start(self) -> None: """Calls the ``on_train_start`` hook.""" - self.results.to(device=self.trainer.lightning_module.device) + self._results.to(device=self.trainer.lightning_module.device) self.trainer.call_hook("on_train_start") def on_advance_start(self) -> None: @@ -224,7 +229,7 @@ def on_advance_end(self) -> None: self.epoch_loop.update_lr_schedulers('epoch', update_plateau_schedulers=True) - did_train_only = self.trainer.disable_validation or self.trainer.evaluation_loop.skip + did_train_only = self.trainer.disable_validation or self.epoch_loop.val_loop.skip if did_train_only: self.global_step -= 1 self._check_checkpoint_callback(True) diff --git a/pytorch_lightning/trainer/connectors/logger_connector/logger_connector.py b/pytorch_lightning/trainer/connectors/logger_connector/logger_connector.py index 48bd095418820..e248b5ff8cf13 100644 --- a/pytorch_lightning/trainer/connectors/logger_connector/logger_connector.py +++ b/pytorch_lightning/trainer/connectors/logger_connector/logger_connector.py @@ -154,9 +154,9 @@ def _prepare_eval_loop_results(self, metrics: Mapping[str, _METRIC]) -> None: if self.trainer.sanity_checking: return - num_dataloaders = self.trainer.evaluation_loop.num_dataloaders + num_dataloaders = self.trainer._evaluation_loop.num_dataloaders has_been_initialized = len(self.eval_loop_results) == num_dataloaders - for dl_idx in range(self.trainer.evaluation_loop.num_dataloaders): + for dl_idx in range(self.trainer._evaluation_loop.num_dataloaders): # remove callback metrics that don't belong to this dataloader callback_metrics = { k: v diff --git a/pytorch_lightning/trainer/properties.py b/pytorch_lightning/trainer/properties.py index b59066cb03b17..a9cf6e3bfbe67 100644 --- a/pytorch_lightning/trainer/properties.py +++ b/pytorch_lightning/trainer/properties.py @@ -29,6 +29,7 @@ from pytorch_lightning.core.optimizer import LightningOptimizer from pytorch_lightning.loggers import LightningLoggerBase from pytorch_lightning.loggers.tensorboard import TensorBoardLogger +from pytorch_lightning.loops import PredictionLoop from pytorch_lightning.loops.dataloader.evaluation_loop import EvaluationLoop from pytorch_lightning.loops.fit_loop import FitLoop from pytorch_lightning.plugins import ParallelPlugin, PrecisionPlugin, TrainingTypePlugin @@ -65,6 +66,7 @@ class TrainerProperties(ABC): fit_loop: FitLoop validate_loop: EvaluationLoop test_loop: EvaluationLoop + predict_loop: PredictionLoop """ Accelerator properties """ @@ -488,16 +490,6 @@ def sanity_checking(self, val: bool) -> None: Loop properties """ - @property - def evaluation_loop(self) -> EvaluationLoop: - if self.state.fn in (TrainerFn.FITTING, TrainerFn.TUNING): - return self.fit_loop.epoch_loop.val_loop - elif self.state.fn == TrainerFn.VALIDATING: - return self.validate_loop - if self.state.fn == TrainerFn.TESTING: - return self.test_loop - raise RuntimeError("The `Trainer.evaluation_loop` property isn't defined. Accessed outside of scope") - @property def global_step(self) -> int: return self.fit_loop.global_step @@ -527,11 +519,23 @@ def is_last_batch(self) -> bool: return self.fit_loop.epoch_loop.is_last_batch @property - def _active_loop(self) -> Optional[Union[FitLoop, EvaluationLoop]]: + def _evaluation_loop(self) -> EvaluationLoop: + if self.state.fn in (TrainerFn.FITTING, TrainerFn.TUNING): + return self.fit_loop.epoch_loop.val_loop + if self.state.fn == TrainerFn.VALIDATING: + return self.validate_loop + if self.state.fn == TrainerFn.TESTING: + return self.test_loop + raise RuntimeError("The `Trainer._evaluation_loop` property isn't defined. Accessed outside of scope") + + @property + def _active_loop(self) -> Optional[Union[FitLoop, EvaluationLoop, PredictionLoop]]: if self.training: return self.fit_loop if self.sanity_checking or self.evaluating: - return self.evaluation_loop + return self._evaluation_loop + if self.predicting: + return self.predict_loop """ Logging properties @@ -553,7 +557,7 @@ def progress_bar_metrics(self) -> dict: def _results(self) -> Optional[ResultCollection]: active_loop = self._active_loop if active_loop is not None: - return active_loop.results + return active_loop._results """ Other diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index 76aa0e465b6d0..fead17157dedc 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -998,10 +998,10 @@ def _run_evaluate(self) -> _EVALUATE_OUTPUT: assert self.evaluating # reload dataloaders - self.evaluation_loop.reload_evaluation_dataloaders() + self._evaluation_loop.reload_evaluation_dataloaders() with self.profiler.profile(f"run_{self.state.stage}_evaluation"), torch.no_grad(): - eval_loop_results = self.evaluation_loop.run() + eval_loop_results = self._evaluation_loop.run() # remove the tensors from the eval results for i, result in enumerate(eval_loop_results): @@ -1031,11 +1031,11 @@ def _run_sanity_check(self, ref_model): self.on_sanity_check_start() # reload dataloaders - self.evaluation_loop.reload_evaluation_dataloaders() + self._evaluation_loop.reload_evaluation_dataloaders() # run eval step with torch.no_grad(): - self.evaluation_loop.run() + self._evaluation_loop.run() self.on_sanity_check_end() diff --git a/tests/callbacks/test_callback_hook_outputs.py b/tests/callbacks/test_callback_hook_outputs.py index 36322482c5eba..eac95e9bf18c6 100644 --- a/tests/callbacks/test_callback_hook_outputs.py +++ b/tests/callbacks/test_callback_hook_outputs.py @@ -70,7 +70,7 @@ def test_free_memory_on_eval_outputs(tmpdir): class CB(Callback): def on_epoch_end(self, trainer, pl_module): - assert len(trainer.evaluation_loop.outputs) == 0 + assert len(trainer._evaluation_loop.outputs) == 0 model = BoringModel() From d75b9e2eb94a9a2836553eef10bb85ed097df0d0 Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Fri, 2 Jul 2021 02:18:08 +0200 Subject: [PATCH 7/7] Remove stage set to None where it shouldnt --- pytorch_lightning/trainer/trainer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index fead17157dedc..c5a95e45bbf66 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -979,7 +979,6 @@ def _run_train(self) -> None: self.on_keyboard_interrupt() # same treatment as below self.accelerator.on_train_end() - self.state.stage = None except BaseException: self.state.status = TrainerStatus.INTERRUPTED if distributed_available() and self.world_size > 1: