diff --git a/CHANGELOG.md b/CHANGELOG.md index bd552c8c42469..53949d80d4bba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -134,6 +134,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Changed +- Make `benchmark` flag optional and set its value based on the deterministic flag ([#11944](https://github.com/PyTorchLightning/pytorch-lightning/pull/11944)) + + - Implemented a new native and rich format in `_print_results` method of the `EvaluationLoop` ([#11332](https://github.com/PyTorchLightning/pytorch-lightning/pull/11332)) diff --git a/docs/source/common/trainer.rst b/docs/source/common/trainer.rst index 7f9658cb7677e..ed5ec193db603 100644 --- a/docs/source/common/trainer.rst +++ b/docs/source/common/trainer.rst @@ -416,18 +416,20 @@ benchmark | -If true enables cudnn.benchmark. -This flag is likely to increase the speed of your system if your -input sizes don't change. However, if it does, then it will likely -make your system slower. +Defaults to ``True`` if :paramref:`~pytorch_lightning.trainer.Trainer.deterministic` is not set. +This flag sets the ``torch.backends.cudnn.deterministic`` flag. You can read more about its impact +`here `__ -The speedup comes from allowing the cudnn auto-tuner to find the best -algorithm for the hardware `[see discussion here] -`_. +This is likely to increase the speed of your system if your input sizes don't change. However, if they do, then it +might make your system slower. The CUDNN auto-tuner will try to find the best algorithm for the hardware when a new +input size is encountered. Read more about it `here `__. Example:: - # default used by the Trainer + # defaults to True if not deterministic (which is False by default) + trainer = Trainer() + + # you can overwrite the value trainer = Trainer(benchmark=False) deterministic diff --git a/pytorch_lightning/trainer/connectors/accelerator_connector.py b/pytorch_lightning/trainer/connectors/accelerator_connector.py index f2d27a249f6f2..98802e224737c 100644 --- a/pytorch_lightning/trainer/connectors/accelerator_connector.py +++ b/pytorch_lightning/trainer/connectors/accelerator_connector.py @@ -99,7 +99,7 @@ def __init__( amp_type: str = "native", amp_level: Optional[str] = None, sync_batchnorm: bool = False, - benchmark: bool = False, + benchmark: Optional[bool] = None, replace_sampler_ddp: bool = True, deterministic: bool = False, num_processes: Optional[int] = None, # deprecated @@ -142,8 +142,14 @@ def __init__( B. Strategy > Accelerator/precision/plugins C. TODO When multiple flag set to the same thing """ + if benchmark and deterministic: + rank_zero_warn( + "You passed `deterministic=True` and `benchmark=True`. Note that PyTorch ignores" + " torch.backends.cudnn.deterministic=True when torch.backends.cudnn.benchmark=True.", + ) + self.benchmark = not deterministic if benchmark is None else benchmark # TODO: move to gpu accelerator - torch.backends.cudnn.benchmark = benchmark + torch.backends.cudnn.benchmark = self.benchmark self.replace_sampler_ddp = replace_sampler_ddp self.sync_batchnorm = sync_batchnorm self._init_deterministic(deterministic) diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index 620d892193a1e..64988e3a580e1 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -172,7 +172,7 @@ def __init__( num_sanity_val_steps: int = 2, resume_from_checkpoint: Optional[Union[Path, str]] = None, profiler: Optional[Union[BaseProfiler, str]] = None, - benchmark: bool = False, + benchmark: Optional[bool] = None, deterministic: bool = False, reload_dataloaders_every_n_epochs: int = 0, auto_lr_find: Union[bool, str] = False, @@ -228,8 +228,9 @@ def __init__( that only one process at a time can access them. Default: ``False``. - benchmark: If ``True``, enables cudnn.benchmark. - Default: ``False``. + benchmark: Sets ``torch.backends.cudnn.benchmark``. + Defaults to ``True`` if :paramref:`~pytorch_lightning.trainer.trainer.Trainer.deterministic` + is ``False``. Overwrite to manually set a different value. Default: ``None``. callbacks: Add a callback or list of callbacks. Default: ``None``. diff --git a/tests/trainer/test_trainer.py b/tests/trainer/test_trainer.py index 2a02075e79bd9..194ddd458e5ab 100644 --- a/tests/trainer/test_trainer.py +++ b/tests/trainer/test_trainer.py @@ -638,23 +638,31 @@ def test_trainer_max_steps_accumulate_batches(tmpdir): assert trainer.global_step == trainer.max_steps, "Model did not stop at max_steps" -def test_benchmark_option(tmpdir): +@pytest.mark.parametrize( + ["benchmark_", "deterministic", "expected"], + [ + (None, False, True), + (None, True, False), + (True, False, True), + (True, True, True), + (False, True, False), + (False, False, False), + ], +) +def test_benchmark_option(benchmark_, deterministic, expected): """Verify benchmark option.""" - model = BoringModel() - - # verify torch.backends.cudnn.benchmark is not turned on - assert not torch.backends.cudnn.benchmark + original_val = torch.backends.cudnn.benchmark - # fit model - trainer = Trainer(default_root_dir=tmpdir, max_epochs=1, benchmark=True) - trainer.fit(model) - - # verify training completed - assert trainer.state.finished, f"Training failed with {trainer.state}" + if benchmark_ and deterministic: + with pytest.warns(UserWarning, match="You passed `deterministic=True` and `benchmark=True`"): + trainer = Trainer(benchmark=benchmark_, deterministic=deterministic) + else: + trainer = Trainer(benchmark=benchmark_, deterministic=deterministic) + assert torch.backends.cudnn.benchmark == expected + assert trainer._accelerator_connector.benchmark == expected - # verify torch.backends.cudnn.benchmark is not turned off - assert torch.backends.cudnn.benchmark + torch.backends.cudnn.benchmark = original_val @pytest.mark.parametrize("ckpt_path", (None, "best", "specific"))