From eafb0470d99eedb26f116e34b3bd57ab32b88a83 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 8 Dec 2020 01:37:08 +0100 Subject: [PATCH 1/3] drop deprecated usage automatic_optimization --- pytorch_lightning/core/lightning.py | 6 +- .../trainer/configuration_validator.py | 9 ++- pytorch_lightning/trainer/trainer.py | 2 +- tests/core/test_lightning_module.py | 2 - tests/core/test_lightning_optimizer.py | 17 +++-- .../dynamic_args/test_multiple_optimizers.py | 5 +- .../optimization/test_manual_optimization.py | 75 ++++++++++++++----- 7 files changed, 82 insertions(+), 34 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index f1a0c725e2b12..b75d9ceb4a807 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1398,8 +1398,10 @@ def get_progress_bar_dict(self): def _verify_is_manual_optimization(self, fn_name): if self.trainer.train_loop.automatic_optimization: - m = f'to use {fn_name}, please disable automatic optimization: Trainer(automatic_optimization=False)' - raise MisconfigurationException(m) + raise MisconfigurationException( + f'to use {fn_name}, please disable automatic optimization:' + ' set model property `automatic_optimization` as False' + ) @classmethod def _auto_collect_arguments(cls, frame=None) -> Tuple[Dict, Dict]: diff --git a/pytorch_lightning/trainer/configuration_validator.py b/pytorch_lightning/trainer/configuration_validator.py index 974bd69229a73..af45b38d64848 100644 --- a/pytorch_lightning/trainer/configuration_validator.py +++ b/pytorch_lightning/trainer/configuration_validator.py @@ -79,7 +79,7 @@ def __verify_train_loop_configuration(self, model): if trainer.overriden_optimizer_step and not enable_pl_optimizer and automatic_optimization: rank_zero_warn( "When overriding `LightningModule` optimizer_step with" - " `Trainer(..., enable_pl_optimizer=False, automatic_optimization=True, ...)`," + " `Trainer(..., enable_pl_optimizer=False, ...)`," " we won't be calling `.zero_grad` we can't assume when you call your `optimizer.step()`." " For Lightning to take care of it, please use `Trainer(enable_pl_optimizer=True)`." ) @@ -90,14 +90,15 @@ def __verify_train_loop_configuration(self, model): if (has_overriden_optimization_functions) and going_to_accumulate_grad_batches and automatic_optimization: raise MisconfigurationException( 'When overriding `LightningModule` optimizer_step or optimizer_zero_grad with ' - '`Trainer(automatic_optimization=True, ...)`, `accumulate_grad_batches` should to be 1.' + '`Trainer(...)`, `accumulate_grad_batches` should to be 1.' ' It ensures optimizer_step or optimizer_zero_grad are called on every batch.' ) if (enable_pl_optimizer) and trainer.overriden_optimizer_zero_grad and not automatic_optimization: raise MisconfigurationException( - 'When overriding `LightningModule` optimizer_zero_grad with ' - '`Trainer(automatic_optimization=False, enable_pl_optimizer=True, ...) is not supported' + 'When overriding `LightningModule` optimizer_zero_grad' + ' and preserving model property `automatic_optimization` as True with' + ' `Trainer(enable_pl_optimizer=True, ...) is not supported' ) def __verify_eval_loop_configuration(self, model, eval_loop_name): diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index 6740406c5fe9a..0f3eab8b623d0 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -355,7 +355,7 @@ def __init__( ) # init train loop related flags - # TODO: deprecate in 1.2.0 + # TODO: deprecate in 1.3.0 if automatic_optimization is None: automatic_optimization = True else: diff --git a/tests/core/test_lightning_module.py b/tests/core/test_lightning_module.py index 0c71259373d1b..3e2e6d040f44c 100644 --- a/tests/core/test_lightning_module.py +++ b/tests/core/test_lightning_module.py @@ -38,7 +38,6 @@ def optimizer_step(self, *_, **__): default_root_dir=tmpdir, limit_train_batches=2, accumulate_grad_batches=2, - automatic_optimization=True ) trainer.fit(model) @@ -90,7 +89,6 @@ def optimizer_step(self, epoch, batch_idx, optimizer, optimizer_idx, default_root_dir=tmpdir, limit_train_batches=8, accumulate_grad_batches=1, - automatic_optimization=True, enable_pl_optimizer=enable_pl_optimizer ) diff --git a/tests/core/test_lightning_optimizer.py b/tests/core/test_lightning_optimizer.py index bd19c26784bc2..e6ec59ec4f5aa 100644 --- a/tests/core/test_lightning_optimizer.py +++ b/tests/core/test_lightning_optimizer.py @@ -112,6 +112,10 @@ def configure_optimizers(self): lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer_1, step_size=1) return [optimizer_1, optimizer_2], [lr_scheduler] + @property + def automatic_optimization(self) -> bool: + return False + model = TestModel() model.training_step_end = None model.training_epoch_end = None @@ -121,8 +125,8 @@ def configure_optimizers(self): limit_val_batches=1, max_epochs=1, weights_summary=None, - automatic_optimization=False, - enable_pl_optimizer=True) + enable_pl_optimizer=True, + ) trainer.fit(model) assert len(mock_sgd_step.mock_calls) == 2 @@ -161,6 +165,10 @@ def configure_optimizers(self): lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer_1, step_size=1) return [optimizer_1, optimizer_2], [lr_scheduler] + @property + def automatic_optimization(self) -> bool: + return False + model = TestModel() model.training_step_end = None model.training_epoch_end = None @@ -170,7 +178,6 @@ def configure_optimizers(self): limit_val_batches=1, max_epochs=1, weights_summary=None, - automatic_optimization=False, accumulate_grad_batches=2, enable_pl_optimizer=True, ) @@ -237,7 +244,6 @@ def configure_optimizers(self): max_epochs=1, weights_summary=None, enable_pl_optimizer=True, - automatic_optimization=True ) trainer.fit(model) @@ -291,7 +297,6 @@ def configure_optimizers(self): max_epochs=1, weights_summary=None, enable_pl_optimizer=True, - automatic_optimization=True ) trainer.fit(model) @@ -352,7 +357,6 @@ def configure_optimizers(self): max_epochs=1, weights_summary=None, enable_pl_optimizer=True, - automatic_optimization=True ) trainer.fit(model) @@ -406,7 +410,6 @@ def configure_optimizers(self): max_epochs=1, weights_summary=None, enable_pl_optimizer=True, - automatic_optimization=True, ) trainer.fit(model) diff --git a/tests/trainer/dynamic_args/test_multiple_optimizers.py b/tests/trainer/dynamic_args/test_multiple_optimizers.py index a01ef8e0b2a7e..48b1bf6ab7ac9 100644 --- a/tests/trainer/dynamic_args/test_multiple_optimizers.py +++ b/tests/trainer/dynamic_args/test_multiple_optimizers.py @@ -97,11 +97,14 @@ def configure_optimizers(self): optimizer_2 = torch.optim.SGD(self.layer.parameters(), lr=0.1) return optimizer, optimizer_2 + @property + def automatic_optimization(self) -> bool: + return False + model = TestModel() model.val_dataloader = None trainer = Trainer( - automatic_optimization=False, default_root_dir=tmpdir, limit_train_batches=2, limit_val_batches=2, diff --git a/tests/trainer/optimization/test_manual_optimization.py b/tests/trainer/optimization/test_manual_optimization.py index 2b2881cb928b4..5e341e9c66f63 100644 --- a/tests/trainer/optimization/test_manual_optimization.py +++ b/tests/trainer/optimization/test_manual_optimization.py @@ -69,12 +69,15 @@ def configure_optimizers(self): optimizer_2 = torch.optim.SGD(self.layer.parameters(), lr=0.1) return optimizer, optimizer_2 + @property + def automatic_optimization(self) -> bool: + return False + model = TestModel() model.val_dataloader = None limit_train_batches = 2 trainer = Trainer( - automatic_optimization=False, default_root_dir=tmpdir, limit_train_batches=limit_train_batches, limit_val_batches=2, @@ -133,12 +136,15 @@ def configure_optimizers(self): optimizer_2 = torch.optim.SGD(self.layer.parameters(), lr=0.1) return optimizer, optimizer_2 + @property + def automatic_optimization(self) -> bool: + return False + model = TestModel() model.val_dataloader = None limit_train_batches = 2 trainer = Trainer( - automatic_optimization=False, default_root_dir=tmpdir, limit_train_batches=limit_train_batches, limit_val_batches=2, @@ -198,12 +204,15 @@ def configure_optimizers(self): optimizer_2 = torch.optim.SGD(self.layer.parameters(), lr=0.1) return optimizer, optimizer_2 + @property + def automatic_optimization(self) -> bool: + return False + model = TestModel() model.val_dataloader = None limit_train_batches = 2 trainer = Trainer( - automatic_optimization=False, default_root_dir=tmpdir, limit_train_batches=limit_train_batches, limit_val_batches=2, @@ -265,12 +274,15 @@ def configure_optimizers(self): optimizer_2 = torch.optim.SGD(self.layer.parameters(), lr=0.1) return optimizer, optimizer_2 + @property + def automatic_optimization(self) -> bool: + return False + model = TestModel() model.val_dataloader = None limit_train_batches = 2 trainer = Trainer( - automatic_optimization=False, default_root_dir=tmpdir, limit_train_batches=limit_train_batches, limit_val_batches=2, @@ -278,7 +290,7 @@ def configure_optimizers(self): log_every_n_steps=1, weights_summary=None, precision=16, - gpus=1 + gpus=1, ) trainer.fit(model) @@ -335,12 +347,15 @@ def configure_optimizers(self): optimizer_2 = torch.optim.SGD(self.layer.parameters(), lr=0.1) return optimizer, optimizer_2 + @property + def automatic_optimization(self) -> bool: + return False + model = TestModel() model.val_dataloader = None limit_train_batches = 2 trainer = Trainer( - automatic_optimization=False, default_root_dir=tmpdir, limit_train_batches=limit_train_batches, limit_val_batches=2, @@ -412,6 +427,10 @@ def on_train_end(self): assert self.called["on_train_batch_start"] == 10 assert self.called["on_train_batch_end"] == 10 + @property + def automatic_optimization(self) -> bool: + return False + @pytest.mark.skipif(not torch.cuda.is_available(), reason="test requires GPU machine") @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") @@ -431,7 +450,6 @@ def test_manual_optimization_and_return_tensor(tmpdir): limit_train_batches=10, limit_test_batches=0, limit_val_batches=0, - automatic_optimization=False, precision=16, amp_backend='native', accelerator="ddp_spawn", @@ -461,7 +479,6 @@ def test_manual_optimization_and_return_detached_tensor(tmpdir): limit_train_batches=10, limit_test_batches=0, limit_val_batches=0, - automatic_optimization=False, precision=16, amp_backend='native', accelerator="ddp_spawn", @@ -538,6 +555,10 @@ def on_train_end(self): assert self.called["on_train_batch_start"] == 20 assert self.called["on_train_batch_end"] == 20 + @property + def automatic_optimization(self) -> bool: + return False + model = ExtendedModel() model.training_step_end = None model.training_epoch_end = None @@ -548,7 +569,6 @@ def on_train_end(self): limit_train_batches=20, limit_test_batches=0, limit_val_batches=0, - automatic_optimization=False, precision=16, amp_backend='native', accumulate_grad_batches=4, @@ -610,12 +630,15 @@ def configure_optimizers(self): optimizer_2 = torch.optim.SGD(self.layer.parameters(), lr=0.1) return optimizer, optimizer_2 + @property + def automatic_optimization(self) -> bool: + return False + model = TestModel() model.val_dataloader = None limit_train_batches = 2 trainer = Trainer( - automatic_optimization=False, default_root_dir=tmpdir, limit_train_batches=limit_train_batches, limit_val_batches=2, @@ -692,13 +715,16 @@ def configure_optimizers(self): optimizer = torch.optim.SGD(self.layer.parameters(), lr=0.1) return optimizer + @property + def automatic_optimization(self) -> bool: + return False + model = TestModel() model.val_dataloader = None model.training_epoch_end = None limit_train_batches = 2 trainer = Trainer( - automatic_optimization=False, default_root_dir=tmpdir, limit_train_batches=limit_train_batches, limit_val_batches=2, @@ -753,13 +779,16 @@ def configure_optimizers(self): optimizer = torch.optim.SGD(self.layer.parameters(), lr=0.1) return optimizer + @property + def automatic_optimization(self) -> bool: + return False + model = TestModel() model.val_dataloader = None model.training_epoch_end = None limit_train_batches = 4 trainer = Trainer( - automatic_optimization=False, default_root_dir=tmpdir, limit_train_batches=limit_train_batches, limit_val_batches=2, @@ -806,13 +835,16 @@ def configure_optimizers(self): optimizer = torch.optim.SGD(self.layer.parameters(), lr=0.1) return optimizer + @property + def automatic_optimization(self) -> bool: + return False + model = TestModel() model.val_dataloader = None model.training_epoch_end = None limit_train_batches = 4 trainer = Trainer( - automatic_optimization=False, default_root_dir=tmpdir, limit_train_batches=limit_train_batches, limit_val_batches=2, @@ -881,13 +913,16 @@ def configure_optimizers(self): optimizer_dis = torch.optim.Adam(self.layer.parameters(), lr=0.001) return [optimizer_gen, optimizer_dis] + @property + def automatic_optimization(self) -> bool: + return False + model = TestModel() model.val_dataloader = None model.training_epoch_end = None limit_train_batches = 8 trainer = Trainer( - automatic_optimization=False, default_root_dir=tmpdir, limit_train_batches=limit_train_batches, limit_val_batches=2, @@ -985,6 +1020,10 @@ def configure_optimizers(self): optimizer_dis = torch.optim.Adam(self.layer.parameters(), lr=0.001) return [optimizer_gen, optimizer_dis] + @property + def automatic_optimization(self) -> bool: + return False + seed_everything(42) model = TestModel() @@ -993,7 +1032,6 @@ def configure_optimizers(self): limit_train_batches = 8 trainer = Trainer( - automatic_optimization=False, default_root_dir=tmpdir, limit_train_batches=limit_train_batches, limit_val_batches=2, @@ -1023,13 +1061,16 @@ class TestModel(BoringModel): def optimizer_zero_grad(self, *_): pass + @property + def automatic_optimization(self) -> bool: + return False + model = TestModel() model.val_dataloader = None model.training_epoch_end = None limit_train_batches = 8 trainer = Trainer( - automatic_optimization=False, default_root_dir=tmpdir, limit_train_batches=limit_train_batches, limit_val_batches=2, @@ -1039,4 +1080,4 @@ def optimizer_zero_grad(self, *_): enable_pl_optimizer=True, ) except MisconfigurationException as e: - assert "`Trainer(automatic_optimization=False, enable_pl_optimizer=True, ...) is not supported" in str(e) + assert "`Trainer(enable_pl_optimizer=True, ...) is not supported" in str(e) From 79d0dd2ec551470cd223ef74caedf0c09ae715d5 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 8 Dec 2020 12:30:29 +0100 Subject: [PATCH 2/3] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrian Wälchli --- pytorch_lightning/trainer/configuration_validator.py | 2 +- pytorch_lightning/trainer/trainer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/trainer/configuration_validator.py b/pytorch_lightning/trainer/configuration_validator.py index af45b38d64848..22de3cadcd400 100644 --- a/pytorch_lightning/trainer/configuration_validator.py +++ b/pytorch_lightning/trainer/configuration_validator.py @@ -90,7 +90,7 @@ def __verify_train_loop_configuration(self, model): if (has_overriden_optimization_functions) and going_to_accumulate_grad_batches and automatic_optimization: raise MisconfigurationException( 'When overriding `LightningModule` optimizer_step or optimizer_zero_grad with ' - '`Trainer(...)`, `accumulate_grad_batches` should to be 1.' + ' `Trainer(...)`, `accumulate_grad_batches` should to be 1.' ' It ensures optimizer_step or optimizer_zero_grad are called on every batch.' ) diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index 0f3eab8b623d0..ddc934297ab1d 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -355,7 +355,7 @@ def __init__( ) # init train loop related flags - # TODO: deprecate in 1.3.0 + # TODO: remove in 1.3.0 if automatic_optimization is None: automatic_optimization = True else: From c4100e6fda89b5271413192f14d53d1c91b64258 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Wed, 9 Dec 2020 09:17:50 +0100 Subject: [PATCH 3/3] Apply suggestions from code review Co-authored-by: Rohit Gupta --- pytorch_lightning/trainer/configuration_validator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/trainer/configuration_validator.py b/pytorch_lightning/trainer/configuration_validator.py index 22de3cadcd400..21d6af043df02 100644 --- a/pytorch_lightning/trainer/configuration_validator.py +++ b/pytorch_lightning/trainer/configuration_validator.py @@ -89,8 +89,8 @@ def __verify_train_loop_configuration(self, model): has_overriden_optimization_functions = trainer.overriden_optimizer_step or trainer.overriden_optimizer_zero_grad if (has_overriden_optimization_functions) and going_to_accumulate_grad_batches and automatic_optimization: raise MisconfigurationException( - 'When overriding `LightningModule` optimizer_step or optimizer_zero_grad with ' - ' `Trainer(...)`, `accumulate_grad_batches` should to be 1.' + 'When overriding `LightningModule` optimizer_step or optimizer_zero_grad' + ' , `accumulate_grad_batches` in `Trainer` should to be 1.' ' It ensures optimizer_step or optimizer_zero_grad are called on every batch.' )