From 21e9dc382db191d198f14604068d61960b7ef977 Mon Sep 17 00:00:00 2001 From: Leo Lu Date: Tue, 10 Jan 2023 22:13:54 +0800 Subject: [PATCH 1/3] Document how to use TensorBoardLogger with fsspec --- docs/source-pytorch/common/remote_fs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source-pytorch/common/remote_fs.rst b/docs/source-pytorch/common/remote_fs.rst index 199cba4ea2bf3..35aa88c663463 100644 --- a/docs/source-pytorch/common/remote_fs.rst +++ b/docs/source-pytorch/common/remote_fs.rst @@ -17,7 +17,7 @@ Working with different filesystems can be accomplished by appending a protocol l trainer = Trainer(default_root_dir="s3://my_bucket/data/") trainer.fit(model) -You could pass custom paths to loggers for logging data. +As for ``TensorBoardLogger``, by default, we use ``tensorboardX``. To use it with ``fsspec`` support, make sure you have ``tensorboard`` installed AND NO ``tensorflow`` installed in the current environment, as ``tensorboard`` will use ``tensorflow``'s GFile when detected, and it only uses fsspec on absence of ``tensorflow``. .. code-block:: python From 26d618982f2a49fb4940771ebec79cbae51bdd3e Mon Sep 17 00:00:00 2001 From: Leo Lu Date: Wed, 11 Jan 2023 14:23:32 +0800 Subject: [PATCH 2/3] Link to loggers api reference --- docs/source-pytorch/api_references.rst | 2 ++ docs/source-pytorch/common/remote_fs.rst | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/source-pytorch/api_references.rst b/docs/source-pytorch/api_references.rst index e380161d03ab6..2eff8c27cbe33 100644 --- a/docs/source-pytorch/api_references.rst +++ b/docs/source-pytorch/api_references.rst @@ -82,6 +82,8 @@ core ~optimizer.LightningOptimizer +.. _loggers-api-references: + loggers ------- diff --git a/docs/source-pytorch/common/remote_fs.rst b/docs/source-pytorch/common/remote_fs.rst index 35aa88c663463..1b842ca610077 100644 --- a/docs/source-pytorch/common/remote_fs.rst +++ b/docs/source-pytorch/common/remote_fs.rst @@ -17,7 +17,8 @@ Working with different filesystems can be accomplished by appending a protocol l trainer = Trainer(default_root_dir="s3://my_bucket/data/") trainer.fit(model) -As for ``TensorBoardLogger``, by default, we use ``tensorboardX``. To use it with ``fsspec`` support, make sure you have ``tensorboard`` installed AND NO ``tensorflow`` installed in the current environment, as ``tensorboard`` will use ``tensorflow``'s GFile when detected, and it only uses fsspec on absence of ``tensorflow``. + +For logging, remote filesystem support depends on the particular logger integration being used. Consult :ref:`the documentation of the individual logger ` for more details. .. code-block:: python From ce0d9ada203111f015582444bdc0e8414c39221a Mon Sep 17 00:00:00 2001 From: Leo Lu Date: Fri, 24 Mar 2023 13:08:19 +0800 Subject: [PATCH 3/3] add documents for tensorboard and csv logger --- src/lightning/pytorch/loggers/csv_logs.py | 2 ++ src/lightning/pytorch/loggers/tensorboard.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/lightning/pytorch/loggers/csv_logs.py b/src/lightning/pytorch/loggers/csv_logs.py index 5de0e27247faf..ee9267607d2c7 100644 --- a/src/lightning/pytorch/loggers/csv_logs.py +++ b/src/lightning/pytorch/loggers/csv_logs.py @@ -42,6 +42,8 @@ class ExperimentWriter(_FabricExperimentWriter): Currently, supports to log hyperparameters and metrics in YAML and CSV format, respectively. + This logger supports logging to remote filesystems via ``fsspec``. Make sure you have it installed. + Args: log_dir: Directory for the experiment logs """ diff --git a/src/lightning/pytorch/loggers/tensorboard.py b/src/lightning/pytorch/loggers/tensorboard.py index bcced33e5b9c3..4c033a2bc1a90 100644 --- a/src/lightning/pytorch/loggers/tensorboard.py +++ b/src/lightning/pytorch/loggers/tensorboard.py @@ -46,12 +46,15 @@ class TensorBoardLogger(Logger, FabricTensorBoardLogger): r""" - Log to local file system in `TensorBoard `_ format. + Log to local or remote file system in `TensorBoard `_ format. Implemented using :class:`~tensorboardX.SummaryWriter`. Logs are saved to ``os.path.join(save_dir, name, version)``. This is the default logger in Lightning, it comes preinstalled. + This logger supports logging to remote filesystems via ``fsspec``. Make sure you have it installed + and you don't have tensorflow (otherwise it will use tf.io.gfile instead of fsspec). + Example: .. testcode::