Fix CI (#1366)

Current deepseek-v3 branch has a CI job broken

Author: H-Huang

torchtitan/components/tokenizer.py

@@ -408,14 +408,11 @@ def build_hf_tokenizer(
 ) -> Union[HuggingFaceTokenizer, BaseTokenizer]:
     """
     Builds a HuggingFaceTokenizer from the specified path.
-
     This function creates a HuggingFaceTokenizer instance that handles BOS/EOS token
     inference and intelligent encoding. The tokenizer automatically detects and loads
     from various file formats and infers special token behavior.
-
     Args:
         JobConfig: A JobConfig object containing the path to the tokenizer directory.
-
     Returns:
         tokenizer (HuggingFaceTokenizer): Loaded tokenizer instance with intelligent BOS/EOS handling
     """

torchtitan/train.py

@@ -128,7 +128,7 @@ def __init__(self, job_config: JobConfig):
 
         # build dataloader
         tokenizer = (
-            self.train_spec.build_tokenizer_fn(job_config.model.tokenizer_path)
+            self.train_spec.build_tokenizer_fn(job_config)
             if self.train_spec.build_tokenizer_fn is not None
             else None
         )