Remove outputs from evaluation epoch end hooks

Author: ananthsub

pytorch_lightning/callbacks/base.py

@@ -108,17 +108,15 @@ def on_validation_epoch_start(self, trainer: 'pl.Trainer', pl_module: 'pl.Lightn
         """Called when the val epoch begins."""
         pass
 
-    def on_validation_epoch_end(
-        self, trainer: 'pl.Trainer', pl_module: 'pl.LightningModule', outputs: EPOCH_OUTPUT
-    ) -> None:
+    def on_validation_epoch_end(self, trainer: 'pl.Trainer', pl_module: 'pl.LightningModule') -> None:
         """Called when the val epoch ends."""
         pass
 
     def on_test_epoch_start(self, trainer: 'pl.Trainer', pl_module: 'pl.LightningModule') -> None:
         """Called when the test epoch begins."""
         pass
 
-    def on_test_epoch_end(self, trainer: 'pl.Trainer', pl_module: 'pl.LightningModule', outputs: EPOCH_OUTPUT) -> None:
+    def on_test_epoch_end(self, trainer: 'pl.Trainer', pl_module: 'pl.LightningModule') -> None:
         """Called when the test epoch ends."""
         pass
 

pytorch_lightning/core/hooks.py

@@ -245,7 +245,7 @@ def on_validation_epoch_start(self) -> None:
         Called in the validation loop at the very beginning of the epoch.
         """
 
-    def on_validation_epoch_end(self, outputs: EPOCH_OUTPUT) -> None:
+    def on_validation_epoch_end(self) -> None:
         """
         Called in the validation loop at the very end of the epoch.
         """
@@ -255,7 +255,7 @@ def on_test_epoch_start(self) -> None:
         Called in the test loop at the very beginning of the epoch.
         """
 
-    def on_test_epoch_end(self, outputs: EPOCH_OUTPUT) -> None:
+    def on_test_epoch_end(self) -> None:
         """
         Called in the test loop at the very end of the epoch.
         """

pytorch_lightning/trainer/callback_hook.py

@@ -111,44 +111,20 @@ def on_validation_epoch_start(self):
         for callback in self.callbacks:
             callback.on_validation_epoch_start(self, self.lightning_module)
 
-    def on_validation_epoch_end(self, outputs: EPOCH_OUTPUT):
-        """Called when the epoch ends.
-
-        Args:
-            outputs: List of outputs on each ``validation`` epoch
-        """
+    def on_validation_epoch_end(self):
+        """Called when the validation epoch ends."""
         for callback in self.callbacks:
-            if is_param_in_hook_signature(callback.on_validation_epoch_end, "outputs"):
-                callback.on_validation_epoch_end(self, self.lightning_module, outputs)
-            else:
-                warning_cache.warn(
-                    "`Callback.on_validation_epoch_end` signature has changed in v1.3."
-                    " `outputs` parameter has been added."
-                    " Support for the old signature will be removed in v1.5", DeprecationWarning
-                )
-                callback.on_validation_epoch_end(self, self.lightning_module)
+            callback.on_validation_epoch_end(self, self.lightning_module)
 
     def on_test_epoch_start(self):
         """Called when the epoch begins."""
         for callback in self.callbacks:
             callback.on_test_epoch_start(self, self.lightning_module)
 
-    def on_test_epoch_end(self, outputs: EPOCH_OUTPUT):
-        """Called when the epoch ends.
-
-        Args:
-            outputs: List of outputs on each ``test`` epoch
-        """
+    def on_test_epoch_end(self):
+        """Called when the test epoch ends."""
         for callback in self.callbacks:
-            if is_param_in_hook_signature(callback.on_test_epoch_end, "outputs"):
-                callback.on_test_epoch_end(self, self.lightning_module, outputs)
-            else:
-                warning_cache.warn(
-                    "`Callback.on_test_epoch_end` signature has changed in v1.3."
-                    " `outputs` parameter has been added."
-                    " Support for the old signature will be removed in v1.5", DeprecationWarning
-                )
-                callback.on_test_epoch_end(self, self.lightning_module)
+            callback.on_test_epoch_end(self, self.lightning_module)
 
     def on_predict_epoch_start(self) -> None:
         """Called when the epoch begins."""

pytorch_lightning/trainer/evaluation_loop.py

@@ -255,14 +255,7 @@ def on_evaluation_epoch_end(self, outputs: Union[List[List[Dict]], List[Dict]])
 
             if is_overridden(hook_name, model_ref):
                 model_hook_fx = getattr(model_ref, hook_name)
-                if is_param_in_hook_signature(model_hook_fx, "outputs"):
-                    model_hook_fx(outputs)
-                else:
-                    self.warning_cache.warn(
-                        f"`ModelHooks.{hook_name}` signature has changed in v1.3. `outputs` parameter has been added."
-                        " Support for the old signature will be removed in v1.5", DeprecationWarning
-                    )
-                    model_hook_fx()
+                model_hook_fx()
 
         self.trainer._cache_logged_metrics()
 

tests/callbacks/test_callback_hook_outputs.py

@@ -68,48 +68,6 @@ def on_train_epoch_end(self, outputs) -> None:
     trainer.fit(model)
 
 
-def test_on_val_epoch_end_outputs(tmpdir):
-
-    class CB(Callback):
-
-        def on_validation_epoch_end(self, trainer, pl_module, outputs):
-            if trainer.running_sanity_check:
-                assert len(outputs) == trainer.num_sanity_val_batches[0]
-            else:
-                assert len(outputs) == trainer.num_val_batches[0]
-
-    model = BoringModel()
-
-    trainer = Trainer(
-        callbacks=CB(),
-        default_root_dir=tmpdir,
-        limit_train_batches=2,
-        limit_val_batches=2,
-        max_epochs=1,
-        weights_summary=None,
-    )
-
-    trainer.fit(model)
-
-
-def test_on_test_epoch_end_outputs(tmpdir):
-
-    class CB(Callback):
-
-        def on_test_epoch_end(self, trainer, pl_module, outputs):
-            assert len(outputs) == trainer.num_test_batches[0]
-
-    model = BoringModel()
-
-    trainer = Trainer(
-        callbacks=CB(),
-        default_root_dir=tmpdir,
-        weights_summary=None,
-    )
-
-    trainer.test(model)
-
-
 def test_free_memory_on_eval_outputs(tmpdir):
 
     class CB(Callback):

tests/callbacks/test_callbacks.py

@@ -58,7 +58,7 @@ def test_trainer_callback_hook_system_fit(_, tmpdir):
         call.on_validation_epoch_start(trainer, model),
         call.on_validation_batch_start(trainer, model, ANY, 0, 0),
         call.on_validation_batch_end(trainer, model, ANY, ANY, 0, 0),
-        call.on_validation_epoch_end(trainer, model, ANY),
+        call.on_validation_epoch_end(trainer, model),
         call.on_epoch_end(trainer, model),
         call.on_validation_end(trainer, model),
         call.on_sanity_check_end(trainer, model),
@@ -90,7 +90,7 @@ def test_trainer_callback_hook_system_fit(_, tmpdir):
         call.on_validation_epoch_start(trainer, model),
         call.on_validation_batch_start(trainer, model, ANY, 0, 0),
         call.on_validation_batch_end(trainer, model, ANY, ANY, 0, 0),
-        call.on_validation_epoch_end(trainer, model, ANY),
+        call.on_validation_epoch_end(trainer, model),
         call.on_epoch_end(trainer, model),
         call.on_validation_end(trainer, model),
         call.on_save_checkpoint(trainer, model),  # should take ANY but we are inspecting signature for BC
@@ -128,7 +128,7 @@ def test_trainer_callback_hook_system_test(tmpdir):
         call.on_test_batch_end(trainer, model, ANY, ANY, 0, 0),
         call.on_test_batch_start(trainer, model, ANY, 1, 0),
         call.on_test_batch_end(trainer, model, ANY, ANY, 1, 0),
-        call.on_test_epoch_end(trainer, model, ANY),
+        call.on_test_epoch_end(trainer, model),
         call.on_epoch_end(trainer, model),
         call.on_test_end(trainer, model),
         call.teardown(trainer, model, 'test'),
@@ -163,7 +163,7 @@ def test_trainer_callback_hook_system_validate(tmpdir):
         call.on_validation_batch_end(trainer, model, ANY, ANY, 0, 0),
         call.on_validation_batch_start(trainer, model, ANY, 1, 0),
         call.on_validation_batch_end(trainer, model, ANY, ANY, 1, 0),
-        call.on_validation_epoch_end(trainer, model, ANY),
+        call.on_validation_epoch_end(trainer, model),
         call.on_epoch_end(trainer, model),
         call.on_validation_end(trainer, model),
         call.teardown(trainer, model, 'validate'),

tests/core/test_hooks.py

@@ -27,7 +27,7 @@
 from pytorch_lightning.profiler import AdvancedProfiler, BaseProfiler, PyTorchProfiler, SimpleProfiler
 from pytorch_lightning.trainer.callback_hook import warning_cache as callback_warning_cache
 from tests.deprecated_api import no_deprecated_call
-from tests.helpers import BoringModel, BoringDataModule
+from tests.helpers import BoringDataModule, BoringModel
 from tests.helpers.utils import no_warning_call
 
 
@@ -213,96 +213,6 @@ def test_v1_5_0_model_checkpoint_period(tmpdir):
         ModelCheckpoint(dirpath=tmpdir, period=1)
 
 
-def test_v1_5_0_old_on_validation_epoch_end(tmpdir):
-    callback_warning_cache.clear()
-
-    class OldSignature(Callback):
-
-        def on_validation_epoch_end(self, trainer, pl_module):  # noqa
-            ...
-
-    model = BoringModel()
-    trainer = Trainer(default_root_dir=tmpdir, max_epochs=1, callbacks=OldSignature())
-
-    with pytest.deprecated_call(match="old signature will be removed in v1.5"):
-        trainer.fit(model)
-
-    class OldSignatureModel(BoringModel):
-
-        def on_validation_epoch_end(self):  # noqa
-            ...
-
-    model = OldSignatureModel()
-
-    with pytest.deprecated_call(match="old signature will be removed in v1.5"):
-        trainer.fit(model)
-
-    callback_warning_cache.clear()
-
-    class NewSignature(Callback):
-
-        def on_validation_epoch_end(self, trainer, pl_module, outputs):
-            ...
-
-    trainer.callbacks = [NewSignature()]
-    with no_deprecated_call(match="`Callback.on_validation_epoch_end` signature has changed in v1.3."):
-        trainer.fit(model)
-
-    class NewSignatureModel(BoringModel):
-
-        def on_validation_epoch_end(self, outputs):
-            ...
-
-    model = NewSignatureModel()
-    with no_deprecated_call(match="`ModelHooks.on_validation_epoch_end` signature has changed in v1.3."):
-        trainer.fit(model)
-
-
-def test_v1_5_0_old_on_test_epoch_end(tmpdir):
-    callback_warning_cache.clear()
-
-    class OldSignature(Callback):
-
-        def on_test_epoch_end(self, trainer, pl_module):  # noqa
-            ...
-
-    model = BoringModel()
-    trainer = Trainer(default_root_dir=tmpdir, max_epochs=1, callbacks=OldSignature())
-
-    with pytest.deprecated_call(match="old signature will be removed in v1.5"):
-        trainer.test(model)
-
-    class OldSignatureModel(BoringModel):
-
-        def on_test_epoch_end(self):  # noqa
-            ...
-
-    model = OldSignatureModel()
-
-    with pytest.deprecated_call(match="old signature will be removed in v1.5"):
-        trainer.test(model)
-
-    callback_warning_cache.clear()
-
-    class NewSignature(Callback):
-
-        def on_test_epoch_end(self, trainer, pl_module, outputs):
-            ...
-
-    trainer.callbacks = [NewSignature()]
-    with no_deprecated_call(match="`Callback.on_test_epoch_end` signature has changed in v1.3."):
-        trainer.test(model)
-
-    class NewSignatureModel(BoringModel):
-
-        def on_test_epoch_end(self, outputs):
-            ...
-
-    model = NewSignatureModel()
-    with no_deprecated_call(match="`ModelHooks.on_test_epoch_end` signature has changed in v1.3."):
-        trainer.test(model)
-
-
 @pytest.mark.parametrize("cls", (BaseProfiler, SimpleProfiler, AdvancedProfiler, PyTorchProfiler))
 def test_v1_5_0_profiler_output_filename(tmpdir, cls):
     filepath = str(tmpdir / "test.txt")

tests/deprecated_api/test_remove_1-5.py

@@ -681,10 +681,10 @@ def _assert_epoch_end(self, stage):
         def on_train_epoch_end(self, outputs):
             self._assert_epoch_end('train')
 
-        def on_validation_epoch_end(self, outputs):
+        def on_validation_epoch_end(self):
             self._assert_epoch_end('val')
 
-        def on_test_epoch_end(self, outputs):
+        def on_test_epoch_end(self):
             self._assert_epoch_end('test')
 
     def _assert_called(model, stage):