Remove deprecated ClustertEnvironment methods (#13458)

* Remove deprecated ClustertEnvironment methods

* update changelog

* ignore typing error

Co-authored-by: Akihiro Nitta <[email protected]>

Author: awaelchli

src/pytorch_lightning/CHANGELOG.md

@@ -281,6 +281,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Removed support for the `DDP2Strategy` ([#12705](https://github.com/PyTorchLightning/pytorch-lightning/pull/12705))
 
 
+- Removed deprecated ClusterEnvironment properties `master_address` and `master_port` in favor of `main_address` and `main_port` ([#13458](https://github.com/PyTorchLightning/pytorch-lightning/pull/13458))
+
+
+- Removed deprecated ClusterEnvironment methods `KubeflowEnvironment.is_using_kubelfow()`, `LSFEnvironment.is_using_lsf()` and `TorchElasticEnvironment.is_using_torchelastic()` in favor of the `detect()` method ([#13458](https://github.com/PyTorchLightning/pytorch-lightning/pull/13458))
+
+
 - Removed deprecated `Callback.on_keyboard_interrupt` ([#13438](https://github.com/Lightning-AI/lightning/pull/13438))
 
 

src/pytorch_lightning/plugins/environments/cluster_environment.py

@@ -12,19 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from abc import ABC, abstractmethod
-from typing import Any, Type
-
-from pytorch_lightning.utilities import rank_zero_deprecation
 
 
 class ClusterEnvironment(ABC):
     """Specification of a cluster environment."""
 
-    def __new__(cls, *args: Any, **kwargs: Any) -> "ClusterEnvironment":
-        # TODO: remove in 1.7
-        _check_for_deprecated_methods(cls)
-        return super().__new__(cls)
-
     @property
     @abstractmethod
     def creates_processes_externally(self) -> bool:
@@ -72,16 +64,3 @@ def node_rank(self) -> int:
     def teardown(self) -> None:
         """Clean up any state set after execution finishes."""
         pass
-
-
-def _check_for_deprecated_methods(cls: Type[ClusterEnvironment]) -> None:
-    if hasattr(cls, "master_address") and callable(cls.master_address):
-        rank_zero_deprecation(
-            f"`{cls.__name__}.master_address` has been deprecated in v1.6 and will be removed in v1.7."
-            " Implement the property `main_address` instead (do not forget to add the `@property` decorator)."
-        )
-    if hasattr(cls, "master_port") and callable(cls.master_port):
-        rank_zero_deprecation(
-            f"`{cls.__name__}.master_port` has been deprecated in v1.6 and will be removed in v1.7."
-            " Implement the property `main_port` instead (do not forget to add the `@property` decorator)."
-        )

src/pytorch_lightning/plugins/environments/kubeflow_environment.py

@@ -16,7 +16,6 @@
 import os
 
 from pytorch_lightning.plugins.environments.cluster_environment import ClusterEnvironment
-from pytorch_lightning.utilities import rank_zero_deprecation
 
 log = logging.getLogger(__name__)
 
@@ -28,16 +27,6 @@ class KubeflowEnvironment(ClusterEnvironment):
     .. _Kubeflow: https://www.kubeflow.org
     """
 
-    def __init__(self) -> None:
-        super().__init__()
-        # TODO: remove in 1.7
-        if hasattr(self, "is_using_kubeflow") and callable(self.is_using_kubeflow):
-            rank_zero_deprecation(
-                f"`{self.__class__.__name__}.is_using_kubeflow` has been deprecated in v1.6 and will be removed in"
-                f" v1.7. Implement the static method `detect()` instead (do not forget to add the `@staticmethod`"
-                f" decorator)."
-            )
-
     @property
     def creates_processes_externally(self) -> bool:
         return True

src/pytorch_lightning/plugins/environments/lsf_environment.py

@@ -18,7 +18,6 @@
 
 from pytorch_lightning import _logger as log
 from pytorch_lightning.plugins.environments import ClusterEnvironment
-from pytorch_lightning.utilities import rank_zero_deprecation
 from pytorch_lightning.utilities.cloud_io import get_filesystem
 
 
@@ -48,12 +47,6 @@ class LSFEnvironment(ClusterEnvironment):
 
     def __init__(self) -> None:
         super().__init__()
-        # TODO: remove in 1.7
-        if hasattr(self, "is_using_lsf") and callable(self.is_using_lsf):
-            rank_zero_deprecation(
-                f"`{self.__class__.__name__}.is_using_lsf` has been deprecated in v1.6 and will be removed in v1.7."
-                " Implement the static method `detect()` instead (do not forget to add the `@staticmethod` decorator)."
-            )
         self._main_address = self._get_main_address()
         self._main_port = self._get_main_port()
         self._node_rank = self._get_node_rank()

src/pytorch_lightning/plugins/environments/torchelastic_environment.py

@@ -19,24 +19,14 @@
 
 from pytorch_lightning.plugins.environments.cluster_environment import ClusterEnvironment
 from pytorch_lightning.utilities.imports import _TORCH_GREATER_EQUAL_1_9_1
-from pytorch_lightning.utilities.rank_zero import rank_zero_deprecation, rank_zero_warn
+from pytorch_lightning.utilities.rank_zero import rank_zero_warn
 
 log = logging.getLogger(__name__)
 
 
 class TorchElasticEnvironment(ClusterEnvironment):
     """Environment for fault-tolerant and elastic training with `torchelastic <https://pytorch.org/elastic/>`_"""
 
-    def __init__(self) -> None:
-        super().__init__()
-        # TODO: remove in 1.7
-        if hasattr(self, "is_using_torchelastic") and callable(self.is_using_torchelastic):
-            rank_zero_deprecation(
-                f"`{self.__class__.__name__}.is_using_torchelastic` has been deprecated in v1.6 and will be removed in"
-                " v1.7. Implement the static method `detect()` instead (do not forget to add the `@staticmethod`"
-                " decorator)."
-            )
-
     @property
     def creates_processes_externally(self) -> bool:
         return True

src/pytorch_lightning/trainer/connectors/accelerator_connector.py

@@ -546,7 +546,8 @@ def _choose_and_init_cluster_environment(self) -> ClusterEnvironment:
             return SLURMEnvironment()
         for env_type in (BaguaEnvironment, TorchElasticEnvironment, KubeflowEnvironment, LSFEnvironment):
             if env_type.detect():
-                return env_type()
+                # Ignore type error because it is a false positive: https://github.com/python/mypy/issues/13044
+                return env_type()  # type: ignore[abstract]
         return LightningEnvironment()
 
     def _is_slurm_managing_tasks(self) -> bool:

tests/tests_pytorch/deprecated_api/test_remove_1-7.py

@@ -12,23 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Test deprecated functionality which will be removed in v1.7.0."""
-import os
 from re import escape
-from unittest import mock
 
 import pytest
 import torch
 
 from pytorch_lightning import Trainer
-from pytorch_lightning.plugins.environments import (
-    KubeflowEnvironment,
-    LightningEnvironment,
-    LSFEnvironment,
-    SLURMEnvironment,
-    TorchElasticEnvironment,
-)
 from pytorch_lightning.strategies import SingleDeviceStrategy
-from tests_pytorch.plugins.environments.test_lsf_environment import _make_rankfile
 
 
 def test_v1_7_0_deprecate_lightning_distributed(tmpdir):
@@ -47,83 +37,6 @@ def test_v1_7_0_deprecated_max_steps_none(tmpdir):
         trainer.fit_loop.max_steps = None
 
 
-@pytest.mark.parametrize(
-    "cls",
-    [
-        KubeflowEnvironment,
-        LightningEnvironment,
-        SLURMEnvironment,
-        TorchElasticEnvironment,
-    ],
-)
-def test_v1_7_0_cluster_environment_master_address(cls):
-    class MyClusterEnvironment(cls):
-        def master_address(self):
-            pass
-
-    with pytest.deprecated_call(
-        match="MyClusterEnvironment.master_address` has been deprecated in v1.6 and will be removed in v1.7"
-    ):
-        MyClusterEnvironment()
-
-
-@pytest.mark.parametrize(
-    "cls",
-    [
-        KubeflowEnvironment,
-        LightningEnvironment,
-        SLURMEnvironment,
-        TorchElasticEnvironment,
-    ],
-)
-def test_v1_7_0_cluster_environment_master_port(cls):
-    class MyClusterEnvironment(cls):
-        def master_port(self):
-            pass
-
-    with pytest.deprecated_call(
-        match="MyClusterEnvironment.master_port` has been deprecated in v1.6 and will be removed in v1.7"
-    ):
-        MyClusterEnvironment()
-
-
-@pytest.mark.parametrize(
-    "cls,method_name",
-    [
-        (KubeflowEnvironment, "is_using_kubeflow"),
-        (LSFEnvironment, "is_using_lsf"),
-        (TorchElasticEnvironment, "is_using_torchelastic"),
-    ],
-)
-def test_v1_7_0_cluster_environment_detection(cls, method_name, tmp_path):
-    class MyClusterEnvironment(cls):
-        @staticmethod
-        def is_using_kubeflow():
-            pass
-
-        @staticmethod
-        def is_using_lsf():
-            pass
-
-        @staticmethod
-        def is_using_torchelastic():
-            pass
-
-    environ = {
-        "LSB_DJOB_RANKFILE": _make_rankfile(tmp_path),
-        "LSB_JOBID": "1234",
-        "JSM_NAMESPACE_SIZE": "4",
-        "JSM_NAMESPACE_RANK": "3",
-        "JSM_NAMESPACE_LOCAL_RANK": "1",
-    }
-    with mock.patch.dict(os.environ, environ):
-        with mock.patch("socket.gethostname", return_value="10.10.10.2"):
-            with pytest.deprecated_call(
-                match=f"MyClusterEnvironment.{method_name}` has been deprecated in v1.6 and will be removed in v1.7"
-            ):
-                MyClusterEnvironment()
-
-
 def test_v1_7_0_post_dispatch_hook():
     class CustomPlugin(SingleDeviceStrategy):
         def post_dispatch(self, trainer):

tests/tests_pytorch/plugins/environments/test_lightning_environment.py

@@ -66,7 +66,7 @@ def test_node_rank_from_group_rank():
 
 
 @mock.patch.dict(os.environ, {}, clear=True)
-def test_random_master_port():
+def test_random_main_port():
     """Test randomly chosen main port when no main port was given by user."""
     env = LightningEnvironment()
     port = env.main_port

tests/tests_pytorch/plugins/environments/test_slurm_environment.py

@@ -83,7 +83,7 @@ def test_attributes_from_environment_variables(caplog):
     "slurm_node_list,expected",
     [("alpha,beta,gamma", "alpha"), ("alpha beta gamma", "alpha"), ("1.2.3.[100-110]", "1.2.3.100")],
 )
-def test_master_address_from_slurm_node_list(slurm_node_list, expected):
+def test_main_address_from_slurm_node_list(slurm_node_list, expected):
     """Test extracting the main node from different formats for the SLURM_NODELIST."""
     with mock.patch.dict(os.environ, {"SLURM_NODELIST": slurm_node_list}):
         env = SLURMEnvironment()