From 0afdd4519d3ca22f3a31bcb5766680441ef1cc0d Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sat, 7 Nov 2020 16:58:13 -0500 Subject: [PATCH 1/9] added log_dir shortcut to trainer properties for writing logs --- docs/source/trainer.rst | 3 + pytorch_lightning/trainer/properties.py | 13 +++ tests/trainer/properties/__init__.py | 0 tests/trainer/properties/log_dir.py | 125 ++++++++++++++++++++++++ tests/trainer/test_trainer.py | 1 + 5 files changed, 142 insertions(+) create mode 100644 tests/trainer/properties/__init__.py create mode 100644 tests/trainer/properties/log_dir.py diff --git a/docs/source/trainer.rst b/docs/source/trainer.rst index 0076c84e96a39..9a86512ea07c0 100644 --- a/docs/source/trainer.rst +++ b/docs/source/trainer.rst @@ -1666,6 +1666,9 @@ The metrics sent to the logger (visualizer). logged_metrics = trainer.logged_metrics assert logged_metrics['a_val'] == 2 +log_dir +******* +The directory is_global_zero ************** diff --git a/pytorch_lightning/trainer/properties.py b/pytorch_lightning/trainer/properties.py index 42b5f7a36641c..d55f817037cbc 100644 --- a/pytorch_lightning/trainer/properties.py +++ b/pytorch_lightning/trainer/properties.py @@ -44,10 +44,23 @@ class TrainerProperties(ABC): limit_val_batches: int _default_root_dir: str _weights_save_path: str + default_root_path: str model_connector: ModelConnector checkpoint_connector: CheckpointConnector callbacks: List[Callback] + @property + def log_dir(self): + if self.checkpoint_callback is not None: + dir = self.checkpoint_callback.dirpath + dir = os.path.split(dir)[0] + return dir + elif self.logger is not None: + return self.logger.log_dir + else: + dir = self._default_root_dir + return dir + @property def use_amp(self) -> bool: return self.precision == 16 diff --git a/tests/trainer/properties/__init__.py b/tests/trainer/properties/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/trainer/properties/log_dir.py b/tests/trainer/properties/log_dir.py new file mode 100644 index 0000000000000..021bb04a7c917 --- /dev/null +++ b/tests/trainer/properties/log_dir.py @@ -0,0 +1,125 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import torch +import pytest +from tests.base.boring_model import BoringModel, RandomDataset +from pytorch_lightning import Trainer +from pytorch_lightning.utilities import APEX_AVAILABLE +from pytorch_lightning.utilities.exceptions import MisconfigurationException + + +def test_logdir(tmpdir): + """ + Tests that the path is correct when checkpoint and loggers are used + """ + class TestModel(BoringModel): + def training_step(self, batch, batch_idx): + output = self.layer(batch) + loss = self.loss(batch, output) + + expected = os.path.join(self.trainer.default_root_dir, 'lightning_logs', 'version_0') + assert self.trainer.log_dir == expected + return {"loss": loss} + + model = TestModel() + + limit_train_batches = 2 + trainer = Trainer( + default_root_dir=tmpdir, + limit_train_batches=limit_train_batches, + limit_val_batches=2, + max_epochs=1, + ) + + trainer.fit(model) + + +def test_logdir_no_checkpoint_cb(tmpdir): + """ + Tests that the path is correct with no checkpoint + """ + class TestModel(BoringModel): + def training_step(self, batch, batch_idx): + output = self.layer(batch) + loss = self.loss(batch, output) + expected = os.path.join(self.trainer.default_root_dir, 'lightning_logs', 'version_0') + assert self.trainer.log_dir == expected + return {"loss": loss} + + model = TestModel() + + limit_train_batches = 2 + trainer = Trainer( + default_root_dir=tmpdir, + limit_train_batches=limit_train_batches, + limit_val_batches=2, + max_epochs=1, + checkpoint_callback=False + ) + + trainer.fit(model) + + +def test_logdir_no_logger(tmpdir): + """ + Tests that the path is correct even when there is no logger + """ + class TestModel(BoringModel): + def training_step(self, batch, batch_idx): + output = self.layer(batch) + loss = self.loss(batch, output) + expected = os.path.join(self.trainer.default_root_dir) + assert self.trainer.log_dir == expected + return {"loss": loss} + + model = TestModel() + + limit_train_batches = 2 + trainer = Trainer( + default_root_dir=tmpdir, + limit_train_batches=limit_train_batches, + limit_val_batches=2, + max_epochs=1, + logger=False, + ) + + trainer.fit(model) + + +def test_logdir_no_logger_no_checkpoint(tmpdir): + """ + Tests that the path is correct even when there is no logger + """ + class TestModel(BoringModel): + def training_step(self, batch, batch_idx): + output = self.layer(batch) + loss = self.loss(batch, output) + expected = os.path.join(self.trainer.default_root_dir) + assert self.trainer.log_dir == expected + return {"loss": loss} + + model = TestModel() + + limit_train_batches = 2 + trainer = Trainer( + default_root_dir=tmpdir, + limit_train_batches=limit_train_batches, + limit_val_batches=2, + max_epochs=1, + logger=False, + checkpoint_callback=False + ) + + trainer.fit(model) diff --git a/tests/trainer/test_trainer.py b/tests/trainer/test_trainer.py index 801e3df73068c..c0c9453cb772d 100644 --- a/tests/trainer/test_trainer.py +++ b/tests/trainer/test_trainer.py @@ -1499,3 +1499,4 @@ def test_trainer_profiler_incorrect_arg_type(profiler): match=r"Only None, bool, str and subclasses of `BaseProfiler` " r"are valid values for `Trainer`'s `profiler` parameter. *"): Trainer(profiler=profiler) + From 48de413c3a1899095205bb3f8b76d0a39fb6816f Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sat, 7 Nov 2020 17:07:31 -0500 Subject: [PATCH 2/9] added log_dir shortcut --- pytorch_lightning/trainer/properties.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pytorch_lightning/trainer/properties.py b/pytorch_lightning/trainer/properties.py index d55f817037cbc..d28fad943df56 100644 --- a/pytorch_lightning/trainer/properties.py +++ b/pytorch_lightning/trainer/properties.py @@ -26,6 +26,8 @@ from pytorch_lightning.utilities import argparse_utils from pytorch_lightning.utilities.cloud_io import get_filesystem from pytorch_lightning.utilities.model_utils import is_overridden +from pytorch_lightning.accelerators.accelerator import Accelerator +from pytorch_lightning.loggers.base import LightningLoggerBase class TrainerProperties(ABC): @@ -45,6 +47,8 @@ class TrainerProperties(ABC): _default_root_dir: str _weights_save_path: str default_root_path: str + accelerator: Accelerator + logger: LightningLoggerBase model_connector: ModelConnector checkpoint_connector: CheckpointConnector callbacks: List[Callback] @@ -54,12 +58,14 @@ def log_dir(self): if self.checkpoint_callback is not None: dir = self.checkpoint_callback.dirpath dir = os.path.split(dir)[0] - return dir elif self.logger is not None: - return self.logger.log_dir + dir = self.logger.log_dir else: dir = self._default_root_dir - return dir + + if self.accelerator_backend is not None: + dir = self.accelerator_backend.broadcast(dir) + return dir @property def use_amp(self) -> bool: From c270df88587af10ff36ef2506addbf328920da4f Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sat, 7 Nov 2020 17:09:24 -0500 Subject: [PATCH 3/9] added log_dir shortcut --- docs/source/loggers.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/source/loggers.rst b/docs/source/loggers.rst index 1c97fa8e2cc3c..a3b85450e6233 100644 --- a/docs/source/loggers.rst +++ b/docs/source/loggers.rst @@ -19,6 +19,15 @@ but you can pass to the :class:`~pytorch_lightning.trainer.trainer.Trainer` any Read more about :ref:`logging` options. +To log arbitrary artifacts like images or audio samples use the `trainer.log_dir` property to resolve +the path. + +.. code-block:: python + + def training_step(self, batch, batch_idx): + img = ... + log_image(img, self.trainer.log_dir) + Comet.ml ======== From 96f6de0d0bb0019b37b818df05b690dd0841b712 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sat, 7 Nov 2020 17:10:21 -0500 Subject: [PATCH 4/9] added log_dir shortcut --- tests/trainer/test_trainer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/trainer/test_trainer.py b/tests/trainer/test_trainer.py index c0c9453cb772d..801e3df73068c 100644 --- a/tests/trainer/test_trainer.py +++ b/tests/trainer/test_trainer.py @@ -1499,4 +1499,3 @@ def test_trainer_profiler_incorrect_arg_type(profiler): match=r"Only None, bool, str and subclasses of `BaseProfiler` " r"are valid values for `Trainer`'s `profiler` parameter. *"): Trainer(profiler=profiler) - From b1488b900cc5b9d8cf0de39f056beab4fccb90f3 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sat, 7 Nov 2020 17:10:50 -0500 Subject: [PATCH 5/9] added log_dir shortcut --- docs/source/trainer.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/source/trainer.rst b/docs/source/trainer.rst index 9a86512ea07c0..e82b0871ef85b 100644 --- a/docs/source/trainer.rst +++ b/docs/source/trainer.rst @@ -1668,7 +1668,15 @@ The metrics sent to the logger (visualizer). log_dir ******* -The directory +The directory for the current experiment. Use this to save images to, etc... + +.. code-block:: python + + def training_step(self, batch, batch_idx): + img = ... + save_img(img, self.trainer.log_dir) + + is_global_zero ************** From 59059e0d8127964197f0761f9377c6652bc8f05e Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sat, 7 Nov 2020 17:11:21 -0500 Subject: [PATCH 6/9] added log_dir shortcut --- pytorch_lightning/trainer/properties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/trainer/properties.py b/pytorch_lightning/trainer/properties.py index d28fad943df56..5687bf5907f63 100644 --- a/pytorch_lightning/trainer/properties.py +++ b/pytorch_lightning/trainer/properties.py @@ -47,7 +47,7 @@ class TrainerProperties(ABC): _default_root_dir: str _weights_save_path: str default_root_path: str - accelerator: Accelerator + accelerator_backend: Accelerator logger: LightningLoggerBase model_connector: ModelConnector checkpoint_connector: CheckpointConnector From 1c29455f8f599662dc1d76cc4c02fa3901a645a6 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sat, 7 Nov 2020 17:39:13 -0500 Subject: [PATCH 7/9] added log_dir shortcut --- pytorch_lightning/trainer/properties.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pytorch_lightning/trainer/properties.py b/pytorch_lightning/trainer/properties.py index 5687bf5907f63..af06b1bbc1352 100644 --- a/pytorch_lightning/trainer/properties.py +++ b/pytorch_lightning/trainer/properties.py @@ -28,6 +28,7 @@ from pytorch_lightning.utilities.model_utils import is_overridden from pytorch_lightning.accelerators.accelerator import Accelerator from pytorch_lightning.loggers.base import LightningLoggerBase +from pytorch_lightning.loggers.tensorboard import TensorBoardLogger class TrainerProperties(ABC): @@ -59,7 +60,10 @@ def log_dir(self): dir = self.checkpoint_callback.dirpath dir = os.path.split(dir)[0] elif self.logger is not None: - dir = self.logger.log_dir + if isinstance(self.logger, TensorBoardLogger): + dir = self.logger.log_dir + else: + dir = self.logger.save_dir else: dir = self._default_root_dir From 7f97a46fc58e579582ef54c213f665908328c68e Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sun, 8 Nov 2020 07:01:28 -0500 Subject: [PATCH 8/9] added log_dir shortcut --- pytorch_lightning/callbacks/model_checkpoint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/callbacks/model_checkpoint.py b/pytorch_lightning/callbacks/model_checkpoint.py index 413e96a39afcd..0d8d4cec81518 100644 --- a/pytorch_lightning/callbacks/model_checkpoint.py +++ b/pytorch_lightning/callbacks/model_checkpoint.py @@ -291,7 +291,7 @@ def __init_ckpt_dir(self, filepath, dirpath, filename, save_top_k): if dirpath and self._fs.protocol == 'file': dirpath = os.path.realpath(dirpath) - self.dirpath = dirpath or None + self.dirpath: Union[str, None] = dirpath or None self.filename = filename or None def __init_monitor_mode(self, monitor, mode): From 38a86350759c28d496ce594c746fba157d360f2e Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sun, 8 Nov 2020 11:30:21 -0500 Subject: [PATCH 9/9] added log_dir shortcut --- .pyrightconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.pyrightconfig.json b/.pyrightconfig.json index 3f00d9a3e4454..416b33f6ad6d2 100644 --- a/.pyrightconfig.json +++ b/.pyrightconfig.json @@ -30,6 +30,7 @@ "pytorch_lightning/trainer/training_tricks.py", "pytorch_lightning/trainer/batch_size_scaling.py", "pytorch_lightning/trainer/distrib_data_parallel.py", + "pytorch_lightning/trainer/properties.py", "pytorch_lightning/trainer/lr_scheduler_connector.py", "pytorch_lightning/trainer/training_loop_temp.py", "pytorch_lightning/trainer/connectors/checkpoint_connector.py",