Skip to content

Commit fc95a81

Browse files
awaelchlilexierule
authored andcommitted
fix info message when max training time reached (#7780)
* call time_elapsed * elapsed formatting * format * update test * changelog
1 parent f1753f3 commit fc95a81

File tree

2 files changed

+7
-3
lines changed

2 files changed

+7
-3
lines changed

pytorch_lightning/callbacks/timer.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,4 +170,5 @@ def _check_time_remaining(self, trainer: 'pl.Trainer') -> None:
170170
should_stop = trainer.accelerator.broadcast(should_stop)
171171
trainer.should_stop = trainer.should_stop or should_stop
172172
if should_stop and self._verbose:
173-
rank_zero_info(f"Time limit reached. Elapsed time is {self.time_elapsed}. Signaling Trainer to stop.")
173+
elapsed = timedelta(seconds=int(self.time_elapsed(RunningStage.TRAINING)))
174+
rank_zero_info(f"Time limit reached. Elapsed time is {elapsed}. Signaling Trainer to stop.")

tests/callbacks/test_timer.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ def test_timer_time_remaining(time_mock):
9595
assert round(timer.time_elapsed()) == 3
9696

9797

98-
def test_timer_stops_training(tmpdir):
98+
def test_timer_stops_training(tmpdir, caplog):
9999
""" Test that the timer stops training before reaching max_epochs """
100100
model = BoringModel()
101101
duration = timedelta(milliseconds=100)
@@ -106,9 +106,12 @@ def test_timer_stops_training(tmpdir):
106106
max_epochs=1000,
107107
callbacks=[timer],
108108
)
109-
trainer.fit(model)
109+
with caplog.at_level(logging.INFO):
110+
trainer.fit(model)
110111
assert trainer.global_step > 1
111112
assert trainer.current_epoch < 999
113+
assert "Time limit reached." in caplog.text
114+
assert "Signaling Trainer to stop." in caplog.text
112115

113116

114117
@pytest.mark.parametrize("interval", ["step", "epoch"])

0 commit comments

Comments
 (0)