move SPECIAL_TOKENS_ATTRIBUTES to utils

Author: reachsumit

torchtext/transforms.py

@@ -14,7 +14,7 @@
 )
 from torchtext._torchtext import RegexTokenizer as RegexTokenizerPybind
 from torchtext.data.functional import load_sp_model
-from torchtext.utils import get_asset_local_path
+from torchtext.utils import get_asset_local_path, SPECIAL_TOKENS_ATTRIBUTES
 from torchtext.vocab import Vocab
 
 from . import functional as F
@@ -294,16 +294,6 @@ class GPT2BPETokenizer(Module):
     def __init__(self, encoder_json_path: str, vocab_bpe_path: str, return_tokens: bool = False) -> None:
         super().__init__()
         self._seperator = "\u0001"
-        self.SPECIAL_TOKENS_ATTRIBUTES = [
-            "bos_token",
-            "eos_token",
-            "unk_token",
-            "sep_token",
-            "pad_token",
-            "cls_token",
-            "mask_token",
-            "additional_special_tokens",
-        ]
         # load bpe encoder and bpe decoder
         with open(get_asset_local_path(encoder_json_path), "r", encoding="utf-8") as f:
             bpe_encoder = json.load(f)
@@ -371,8 +361,8 @@ def add_special_tokens(self, special_tokens_dict: Mapping[str, Union[str, Sequen
         """
         for key in special_tokens_dict.keys():
             assert (
-                key in self.SPECIAL_TOKENS_ATTRIBUTES
-            ), f"Key '{key}' is not in the special token list: {self.SPECIAL_TOKENS_ATTRIBUTES}"
+                key in SPECIAL_TOKENS_ATTRIBUTES
+            ), f"Key '{key}' is not in the special token list: {SPECIAL_TOKENS_ATTRIBUTES}"
 
         return self.bpe.add_special_tokens(
             {k: v for k, v in special_tokens_dict.items() if k != "additional_special_tokens"},

torchtext/utils.py

@@ -13,6 +13,17 @@
 
 logger = logging.getLogger(__name__)
 
+SPECIAL_TOKENS_ATTRIBUTES = [
+    "bos_token",
+    "eos_token",
+    "unk_token",
+    "sep_token",
+    "pad_token",
+    "cls_token",
+    "mask_token",
+    "additional_special_tokens",
+]
+
 
 def reporthook(t):
     """