add test

Author: ashleve

pytorch_lightning/trainer/trainer.py

@@ -948,12 +948,6 @@ def test(
         #  Attach dataloaders (if given)
         self.data_connector.attach_dataloaders(model, test_dataloaders=test_dataloaders)
 
-        if not model_provided and ckpt_path == 'best' and self.fast_dev_run:
-            raise MisconfigurationException(
-                'You cannot execute testing when the model is not provided and `fast_dev_run=True`. '
-                'Provide model with `trainer.test(model=...)` or `trainer.test(ckpt_path=...)`'
-            )
-
         if not model_provided:
             self.tested_ckpt_path = self.__load_ckpt_weights(model, ckpt_path=ckpt_path)
 
@@ -971,10 +965,17 @@ def __load_ckpt_weights(
         ckpt_path: Optional[str] = None,
     ) -> Optional[str]:
         # if user requests the best checkpoint but we don't have it, error
-        if ckpt_path == 'best' and not self.checkpoint_callback.best_model_path:
-            raise MisconfigurationException(
-                'ckpt_path is "best", but `ModelCheckpoint` is not configured to save the best model.'
-            )
+        if ckpt_path == 'best':
+            if not self.checkpoint_callback.best_model_path and self.fast_dev_run:
+                raise MisconfigurationException(
+                    'You cannot execute `trainer.test()` or trainer.validate()`'
+                    ' when `fast_dev_run=True`.'
+                )
+
+            if not self.checkpoint_callback.best_model_path:
+                raise MisconfigurationException(
+                    'ckpt_path is "best", but `ModelCheckpoint` is not configured to save the best model.'
+                )
 
         # load best weights
         if ckpt_path is not None:

tests/trainer/test_trainer.py

@@ -1451,6 +1451,7 @@ def predict_step(self, batch, batch_idx, dataloader_idx=None):
 
 
 def test_trainer_predict_grad(tmpdir):
+
     class CustomBoringModel(BoringModel):
 
         def predict_step(self, batch, batch_idx, dataloader_idx=None):
@@ -1776,3 +1777,19 @@ def on_fit_start(self, trainer, pl_module: LightningModule) -> None:
 
     trainer = Trainer(default_root_dir=tmpdir, max_epochs=1, callbacks=[TestCallback()])
     trainer.fit(model, datamodule=dm)
+
+
+@pytest.mark.parametrize("fast_dev_run", [True, False])
+def test_exception_when_testing_or_validating_with_fast_dev_run(tmpdir, fast_dev_run):
+    model = BoringModel()
+
+    trainer = Trainer(default_root_dir=tmpdir, max_epochs=1, fast_dev_run=fast_dev_run)
+
+    trainer.fit(model)
+
+    if fast_dev_run:
+        with pytest.raises(MisconfigurationException, match=".*when `fast_dev_run=True`*"):
+            trainer.validate()
+
+        with pytest.raises(MisconfigurationException, match=".*when `fast_dev_run=True`*"):
+            trainer.test()