From 3bca9a037aa431755dcb20754ef8c190c758f798 Mon Sep 17 00:00:00 2001 From: tchaton Date: Mon, 14 Dec 2020 11:11:35 +0100 Subject: [PATCH 1/3] remove nan loss whe missing --- pytorch_lightning/core/lightning.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 8019d865c0ca0..80804b902dfd8 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1392,12 +1392,15 @@ def get_progress_bar_dict(self): """ # call .item() only once but store elements without graphs running_train_loss = self.trainer.train_loop.running_loss.mean() - avg_training_loss = ( - running_train_loss.cpu().item() - if running_train_loss is not None - else float("NaN") - ) - tqdm_dict = {"loss": "{:.3g}".format(avg_training_loss)} + if self.trainer.train_loop.automatic_optimization: + avg_training_loss = ( + running_train_loss.cpu().item() + if running_train_loss is not None + else float("NaN") + ) + tqdm_dict = {"loss": "{:.3g}".format(avg_training_loss)} + else: + tqdm_dict = {} if self.trainer.truncated_bptt_steps is not None: tqdm_dict["split_idx"] = self.trainer.split_idx From a98d3f798c8d2ba5cb00949bd95d71c8c2628f96 Mon Sep 17 00:00:00 2001 From: chaton Date: Tue, 15 Dec 2020 09:47:17 +0100 Subject: [PATCH 2/3] Update pytorch_lightning/core/lightning.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Carlos Mocholí --- pytorch_lightning/core/lightning.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 80804b902dfd8..335702c165430 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1392,15 +1392,15 @@ def get_progress_bar_dict(self): """ # call .item() only once but store elements without graphs running_train_loss = self.trainer.train_loop.running_loss.mean() - if self.trainer.train_loop.automatic_optimization: - avg_training_loss = ( - running_train_loss.cpu().item() - if running_train_loss is not None - else float("NaN") - ) - tqdm_dict = {"loss": "{:.3g}".format(avg_training_loss)} - else: - tqdm_dict = {} + avg_training_loss = None + if running_train_loss is not None: + avg_training_loss = running_train_loss.cpu().item() + elif self.trainer.train_loop.automatic_optimization: + avg_training_loss = float('NaN) + + tqdm_dict = {} + if avg_training_loss is not None: + tqdm_dict["loss"] = f"{avg_training_loss:.3g}" if self.trainer.truncated_bptt_steps is not None: tqdm_dict["split_idx"] = self.trainer.split_idx From 368870914e3ff9f20bcd6d2ca46838cb3b479d57 Mon Sep 17 00:00:00 2001 From: Rohit Gupta Date: Wed, 16 Dec 2020 00:07:11 +0530 Subject: [PATCH 3/3] Apply suggestions from code review --- pytorch_lightning/core/lightning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 335702c165430..9ac6b33641a5a 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1396,7 +1396,7 @@ def get_progress_bar_dict(self): if running_train_loss is not None: avg_training_loss = running_train_loss.cpu().item() elif self.trainer.train_loop.automatic_optimization: - avg_training_loss = float('NaN) + avg_training_loss = float('NaN') tqdm_dict = {} if avg_training_loss is not None: