Merge branch 'master' into fix/horovod-allgather-plugin

Author: awaelchli

CHANGELOG.md

@@ -29,6 +29,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
     * Reset `current` progress counters when restarting an epoch loop that had already finished ([#9371](https://github.com/PyTorchLightning/pytorch-lightning/pull/9371))
     * Call `reset_on_restart` in the loop's `reset` hook instead of when loading a checkpoint ([#9561](https://github.com/PyTorchLightning/pytorch-lightning/pull/9561))
     * Use `completed` over `processed` in `reset_on_restart` ([#9656](https://github.com/PyTorchLightning/pytorch-lightning/pull/9656))
+    * Rename `reset_on_epoch` to `reset_on_run` ([#9658](https://github.com/PyTorchLightning/pytorch-lightning/pull/9658))
 
 
 - Added `batch_size` and `rank_zero_only` arguments for `log_dict` to match `log` ([#8628](https://github.com/PyTorchLightning/pytorch-lightning/pull/8628))
@@ -152,6 +153,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Added `RichModelSummary` callback ([#9546](https://github.com/PyTorchLightning/pytorch-lightning/pull/9546))
 
 
+- Added `enable_progress_bar` to Trainer constructor ([#9664](https://github.com/PyTorchLightning/pytorch-lightning/pull/9664))
+
+
 - Added `pl_legacy_patch` load utility for loading old checkpoints that have pickled legacy Lightning attributes ([#9166](https://github.com/PyTorchLightning/pytorch-lightning/pull/9166))
 
 
@@ -226,9 +230,13 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - `seed_everything` now fails when an invalid seed value is passed instead of selecting a random seed ([#8787](https://github.com/PyTorchLightning/pytorch-lightning/pull/8787))
 
 
+- Use a unique filename to save temp ckpt in tuner ([#96827](https://github.com/PyTorchLightning/pytorch-lightning/pull/9682))
+
+
 - Changed `HorovodPlugin.all_gather` to return a `torch.Tensor` instead of a list ([#9696](https://github.com/PyTorchLightning/pytorch-lightning/pull/9696))
 
 
+
 ### Deprecated
 
 - Deprecated `LightningModule.summarize()` in favor of `pytorch_lightning.utilities.model_summary.summarize()`
@@ -267,7 +275,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Deprecated `LightningLoggerBase.close`, `LoggerCollection.close` in favor of `LightningLoggerBase.finalize`, `LoggerCollection.finalize` ([#9422](https://github.com/PyTorchLightning/pytorch-lightning/pull/9422))
 
 
-- Deprecated passing `progress_bar_refresh_rate` to the `Trainer` constructor in favor of adding the `ProgressBar` callback with `refresh_rate` directly to the list of callbacks ([#9616](https://github.com/PyTorchLightning/pytorch-lightning/pull/9616))
+- Deprecated passing `progress_bar_refresh_rate` to the `Trainer` constructor in favor of adding the `ProgressBar` callback with `refresh_rate` directly to the list of callbacks, or passing `enable_progress_bar=False` to disable the progress bar ([#9616](https://github.com/PyTorchLightning/pytorch-lightning/pull/9616))
 
 
 ### Removed

benchmarks/test_basic_parity.py

@@ -157,7 +157,7 @@ def lightning_loop(cls_model, idx, device_type: str = "cuda", num_epochs=10):
     trainer = Trainer(
         # as the first run is skipped, no need to run it long
         max_epochs=num_epochs if idx > 0 else 1,
-        progress_bar_refresh_rate=0,
+        enable_progress_bar=False,
         weights_summary=None,
         gpus=1 if device_type == "cuda" else 0,
         checkpoint_callback=False,

docs/source/common/trainer.rst

@@ -1281,6 +1281,10 @@ See the :doc:`profiler documentation <../advanced/profiler>`. for more details.
 
 progress_bar_refresh_rate
 ^^^^^^^^^^^^^^^^^^^^^^^^^
+``progress_bar_refresh_rate`` has been deprecated in v1.5 and will be removed in v1.7.
+Please pass :class:`~pytorch_lightning.callbacks.progress.ProgressBar` with ``refresh_rate``
+directly to the Trainer's ``callbacks`` argument instead. To disable the progress bar,
+pass ``enable_progress_bar = False`` to the Trainer.
 
 .. raw:: html
 
@@ -1305,6 +1309,19 @@ Note:
       Lightning will set it to 20 in these environments if the user does not provide a value.
     - This argument is ignored if a custom callback is passed to :paramref:`~Trainer.callbacks`.
 
+enable_progress_bar
+^^^^^^^^^^^^^^^^^^^
+
+Whether to enable or disable the progress bar. Defaults to True.
+
+.. testcode::
+
+    # default used by the Trainer
+    trainer = Trainer(enable_progress_bar=True)
+
+    # disable progress bar
+    trainer = Trainer(enable_progress_bar=False)
+
 reload_dataloaders_every_n_epochs
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

pl_examples/domain_templates/computer_vision_fine_tuning.py

@@ -273,7 +273,6 @@ def add_arguments_to_parser(self, parser):
             {
                 "trainer.max_epochs": 15,
                 "trainer.weights_summary": None,
-                "trainer.progress_bar_refresh_rate": 1,
                 "trainer.num_sanity_val_steps": 0,
             }
         )

pytorch_lightning/callbacks/finetuning.py

@@ -285,7 +285,10 @@ def _store(
 
     def on_train_epoch_start(self, trainer, pl_module):
         """Called when the epoch begins."""
-        for opt_idx, optimizer in trainer.fit_loop.epoch_loop.batch_loop.get_active_optimizers():
+        # import is here to avoid circular imports
+        from pytorch_lightning.loops.utilities import _get_active_optimizers
+
+        for opt_idx, optimizer in _get_active_optimizers(trainer.optimizers, trainer.optimizer_frequencies):
             num_param_groups = len(optimizer.param_groups)
             self.finetune_function(pl_module, trainer.current_epoch, optimizer, opt_idx)
             current_param_groups = optimizer.param_groups

pytorch_lightning/loops/batch/training_batch_loop.py

@@ -11,16 +11,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Any, List, Optional, Tuple
+from typing import Any, List, Optional
 
-import numpy as np
 from deprecate import void
 from torch import Tensor
-from torch.optim import Optimizer
 
 from pytorch_lightning.loops.base import Loop
 from pytorch_lightning.loops.optimization.manual_loop import ManualOptimization
 from pytorch_lightning.loops.optimization.optimizer_loop import OptimizerLoop
+from pytorch_lightning.loops.utilities import _get_active_optimizers
 from pytorch_lightning.trainer.supporters import TensorRunningAccum
 from pytorch_lightning.utilities import AttributeDict
 from pytorch_lightning.utilities.types import STEP_OUTPUT
@@ -41,21 +40,13 @@ def __init__(self) -> None:
         self.manual_loop = ManualOptimization()
 
         self._warning_cache: WarningCache = WarningCache()
-        self._optimizer_freq_cumsum: Optional[int] = None
         self._remaining_splits: Optional[List[Any]] = None
 
     @property
     def done(self) -> bool:
         """Returns if all batch splits have been processed already."""
         return len(self._remaining_splits) == 0
 
-    @property
-    def optimizer_freq_cumsum(self) -> int:
-        """Returns the cumulated sum of optimizer frequencies."""
-        if self._optimizer_freq_cumsum is None:
-            self._optimizer_freq_cumsum = np.cumsum(self.trainer.optimizer_frequencies)
-        return self._optimizer_freq_cumsum
-
     def connect(
         self, optimizer_loop: Optional["Loop"] = None, manual_loop: Optional[ManualOptimization] = None
     ) -> None:
@@ -123,7 +114,8 @@ def advance(self, batch, batch_idx):
 
         if self.trainer.lightning_module.automatic_optimization:
             # in automatic optimization, hand over execution to the OptimizerLoop
-            batch_outputs = self.optimizer_loop.run(split_batch, self.get_active_optimizers(batch_idx), batch_idx)
+            optimizers = _get_active_optimizers(self.trainer.optimizers, self.trainer.optimizer_frequencies, batch_idx)
+            batch_outputs = self.optimizer_loop.run(split_batch, optimizers, batch_idx)
             # combine outputs from each optimizer
             for k in range(len(batch_outputs)):
                 self.batch_outputs[k].extend(batch_outputs[k])
@@ -142,10 +134,6 @@ def teardown(self) -> None:
         # release memory
         self._remaining_splits = None
 
-    def num_active_optimizers(self, batch_idx: Optional[int] = None) -> int:
-        """Gets the number of active optimizers based on their frequency."""
-        return len(self.get_active_optimizers(batch_idx))
-
     def _tbptt_split_batch(self, batch: Any) -> List[Any]:
         """Splits a single batch into a list of sequence steps for tbptt.
 
@@ -175,21 +163,3 @@ def _update_running_loss(self, current_loss: Tensor) -> None:
 
         # reset for next set of accumulated grads
         self.accumulated_loss.reset()
-
-    def get_active_optimizers(self, batch_idx: Optional[int] = None) -> List[Tuple[int, Optimizer]]:
-        """Returns the currently active optimizers. When multiple optimizers are used with different frequencies,
-        only one of the optimizers is active at a time.
-
-        Returns:
-            A list of tuples (opt_idx, optimizer) of currently active optimizers.
-        """
-        if not self.trainer.optimizer_frequencies:
-            # call training_step once per optimizer
-            return list(enumerate(self.trainer.optimizers))
-
-        optimizers_loop_length = self.optimizer_freq_cumsum[-1]
-        current_place_in_loop = batch_idx % optimizers_loop_length
-
-        # find optimzier index by looking for the first {item > current_place} in the cumsum list
-        opt_idx = np.searchsorted(self.optimizer_freq_cumsum, current_place_in_loop, side="right")
-        return [(opt_idx, self.trainer.optimizers[opt_idx])]

pytorch_lightning/loops/dataloader/dataloader_loop.py

@@ -56,9 +56,9 @@ def done(self) -> bool:
     def reset(self) -> None:
         """Resets the internal state."""
         if not self.restarting:
-            self.dataloader_progress.current.reset()
+            self.dataloader_progress.reset_on_run()
         else:
-            self.dataloader_progress.current.reset_on_restart()
+            self.dataloader_progress.reset_on_restart()
 
     def on_advance_start(self, *args: Any, **kwargs: Any) -> None:
         self.dataloader_progress.increment_ready()

pytorch_lightning/loops/epoch/evaluation_epoch_loop.py

@@ -58,9 +58,9 @@ def reset(self) -> None:
         self.outputs = []
 
         if not self.restarting:
-            self.batch_progress.current.reset()
+            self.batch_progress.reset_on_run()
         else:
-            self.batch_progress.current.reset_on_restart()
+            self.batch_progress.reset_on_restart()
 
     def on_run_start(
         self, data_fetcher: AbstractDataFetcher, dataloader_idx: int, dl_max_batches: int, num_dataloaders: int

pytorch_lightning/loops/epoch/prediction_epoch_loop.py

@@ -46,7 +46,7 @@ def reset(self) -> None:
         """Resets the loops internal state."""
         self._all_batch_indices: List[int] = []
         self.predictions: List[Any] = []
-        self.batch_progress.current.reset()
+        self.batch_progress.reset_on_run()
 
     def on_run_start(
         self,

pytorch_lightning/loops/epoch/training_epoch_loop.py

@@ -18,7 +18,7 @@
 from pytorch_lightning import loops  # import as loops to avoid circular imports
 from pytorch_lightning.loops.batch import TrainingBatchLoop
 from pytorch_lightning.loops.optimization.closure import OutputResult
-from pytorch_lightning.loops.utilities import _prepare_dataloader_iter
+from pytorch_lightning.loops.utilities import _get_active_optimizers, _prepare_dataloader_iter
 from pytorch_lightning.trainer.connectors.logger_connector.result import ResultCollection
 from pytorch_lightning.trainer.progress import BatchProgress, SchedulerProgress
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
@@ -96,15 +96,15 @@ def reset(self) -> None:
         assert self.batch_loop.optimizer_loop is not None
         if self.restarting:
             self.batch_progress.reset_on_restart()
-            self.scheduler_progress.current.reset_on_restart()
+            self.scheduler_progress.reset_on_restart()
             self.batch_loop.optimizer_loop.optim_progress.reset_on_restart()
         else:
-            self.batch_progress.reset_on_epoch()
-            self.scheduler_progress.reset_on_epoch()
-            self.batch_loop.optimizer_loop.optim_progress.reset_on_epoch()
+            self.batch_progress.reset_on_run()
+            self.scheduler_progress.reset_on_run()
+            self.batch_loop.optimizer_loop.optim_progress.reset_on_run()
 
         # track epoch output
-        self._epoch_output = [[] for _ in range(self.batch_loop.num_active_optimizers(self.total_batch_idx))]
+        self._epoch_output = [[] for _ in range(self._num_active_optimizers(self.total_batch_idx))]
 
     def on_run_start(self, dataloader_iter: Iterator, **kwargs: Any) -> None:
         # hook
@@ -340,10 +340,13 @@ def update_lr_schedulers(self, interval: str, update_plateau_schedulers: bool) -
         """updates the lr schedulers based on the given interval."""
         if interval == "step" and self._should_accumulate():
             return
+        active_optimizers = _get_active_optimizers(
+            self.trainer.optimizers, self.trainer.optimizer_frequencies, self.total_batch_idx
+        )
         self.trainer.optimizer_connector.update_learning_rates(
             interval=interval,
             update_plateau_schedulers=update_plateau_schedulers,
-            opt_indices=[opt_idx for opt_idx, _ in self.batch_loop.get_active_optimizers(self.total_batch_idx)],
+            opt_indices=[opt_idx for opt_idx, _ in active_optimizers],
         )
 
     def _should_check_val_fx(self, batch_idx: int, is_last_batch: bool) -> bool:
@@ -377,3 +380,7 @@ def _save_loggers_on_train_batch_end(self) -> None:
         should_flush_logs = self.trainer.logger_connector.should_flush_logs
         if should_flush_logs and self.trainer.is_global_zero and self.trainer.logger is not None:
             self.trainer.logger.save()
+
+    def _num_active_optimizers(self, batch_idx: Optional[int] = None) -> int:
+        """Gets the number of active optimizers based on their frequency."""
+        return len(_get_active_optimizers(self.trainer.optimizers, self.trainer.optimizer_frequencies, batch_idx))