From 34e4ee03c533d5b4ba4183f7357404f90c852a56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 31 May 2021 13:27:08 +0200 Subject: [PATCH 1/5] call time_elapsed --- pytorch_lightning/callbacks/timer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pytorch_lightning/callbacks/timer.py b/pytorch_lightning/callbacks/timer.py index 9b93499c82ea1..ad930cbc7488f 100644 --- a/pytorch_lightning/callbacks/timer.py +++ b/pytorch_lightning/callbacks/timer.py @@ -170,4 +170,6 @@ def _check_time_remaining(self, trainer: 'pl.Trainer') -> None: should_stop = trainer.accelerator.broadcast(should_stop) trainer.should_stop = trainer.should_stop or should_stop if should_stop and self._verbose: - rank_zero_info(f"Time limit reached. Elapsed time is {self.time_elapsed}. Signaling Trainer to stop.") + rank_zero_info( + f"Time limit reached. Elapsed time is {self.time_elapsed(RunningStage.TRAINING)}." + f" Signaling Trainer to stop.") From d5ce796c34f5208d404cc51c819354cb0b87c5dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 31 May 2021 13:34:53 +0200 Subject: [PATCH 2/5] elapsed formatting --- pytorch_lightning/callbacks/timer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/callbacks/timer.py b/pytorch_lightning/callbacks/timer.py index ad930cbc7488f..48f53c41e34fa 100644 --- a/pytorch_lightning/callbacks/timer.py +++ b/pytorch_lightning/callbacks/timer.py @@ -170,6 +170,6 @@ def _check_time_remaining(self, trainer: 'pl.Trainer') -> None: should_stop = trainer.accelerator.broadcast(should_stop) trainer.should_stop = trainer.should_stop or should_stop if should_stop and self._verbose: + elapsed = timedelta(seconds=int(self.time_elapsed(RunningStage.TRAINING))) rank_zero_info( - f"Time limit reached. Elapsed time is {self.time_elapsed(RunningStage.TRAINING)}." - f" Signaling Trainer to stop.") + f"Time limit reached. Elapsed time is {elapsed}. Signaling Trainer to stop.") From 79d014469a2d3ff8bcbf584bc0d95f3c308209cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 31 May 2021 13:36:07 +0200 Subject: [PATCH 3/5] format --- pytorch_lightning/callbacks/timer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pytorch_lightning/callbacks/timer.py b/pytorch_lightning/callbacks/timer.py index 48f53c41e34fa..ba42419141253 100644 --- a/pytorch_lightning/callbacks/timer.py +++ b/pytorch_lightning/callbacks/timer.py @@ -171,5 +171,4 @@ def _check_time_remaining(self, trainer: 'pl.Trainer') -> None: trainer.should_stop = trainer.should_stop or should_stop if should_stop and self._verbose: elapsed = timedelta(seconds=int(self.time_elapsed(RunningStage.TRAINING))) - rank_zero_info( - f"Time limit reached. Elapsed time is {elapsed}. Signaling Trainer to stop.") + rank_zero_info(f"Time limit reached. Elapsed time is {elapsed}. Signaling Trainer to stop.") From 4c09914e9228a0691fe334b203f930db2cdf8a7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 31 May 2021 13:41:07 +0200 Subject: [PATCH 4/5] update test --- tests/callbacks/test_timer.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/callbacks/test_timer.py b/tests/callbacks/test_timer.py index c27eebbeb7805..16e01a6adcaf4 100644 --- a/tests/callbacks/test_timer.py +++ b/tests/callbacks/test_timer.py @@ -95,7 +95,7 @@ def test_timer_time_remaining(time_mock): assert round(timer.time_elapsed()) == 3 -def test_timer_stops_training(tmpdir): +def test_timer_stops_training(tmpdir, caplog): """ Test that the timer stops training before reaching max_epochs """ model = BoringModel() duration = timedelta(milliseconds=100) @@ -106,9 +106,12 @@ def test_timer_stops_training(tmpdir): max_epochs=1000, callbacks=[timer], ) - trainer.fit(model) + with caplog.at_level(logging.INFO): + trainer.fit(model) assert trainer.global_step > 1 assert trainer.current_epoch < 999 + assert "Time limit reached." in caplog.text + assert "Signaling Trainer to stop." in caplog.text @pytest.mark.parametrize("interval", ["step", "epoch"]) From 0304d6502c36844c43e6fa968d4f21101f85fe04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 31 May 2021 13:42:36 +0200 Subject: [PATCH 5/5] changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b75e439f1662..ce9b9fc07b3b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -164,6 +164,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Fixed print errors in `ProgressBar` when `trainer.fit` is not called ([#7674](https://github.com/PyTorchLightning/pytorch-lightning/pull/7674)) +- Fixed formatting of info message when max training time reached ([#7780](https://github.com/PyTorchLightning/pytorch-lightning/pull/7780)) + + ## [1.3.2] - 2021-05-18 ### Changed