From 0afd6c0c6378c022ea8d38f43b51ef735f4df151 Mon Sep 17 00:00:00 2001 From: Lunwen He Date: Tue, 29 Oct 2024 13:03:05 -0700 Subject: [PATCH] upgrade lm_eval to 0.4.5 Pull Request resolved: https://github.com/pytorch/executorch/pull/6533 We have been using a pretty old `lm_eval` version. This is blocking us from upgrading other libraries like `transformers` and blocking some others work. For example, https://github.com/pytorch/executorch/pull/6489. In newer versions `lm_eval`, `pretrainedModel` becomes a required parameter. In 0.4.2, it defaults to `gpt2` if not provided. This PR upgrades our `lm_eval` version to the latest version 0.4.5 and set `pretrainedModel` to its original default value `gpt2`. Differential Revision: [D65079913](https://our.internmc.facebook.com/intern/diff/D65079913/) ghstack-source-id: 250754584 --- examples/models/llama/evaluate/eager_eval.py | 6 +++++- examples/models/llama/install_requirements.sh | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/models/llama/evaluate/eager_eval.py b/examples/models/llama/evaluate/eager_eval.py index e931c60c184..b3f04ef3bb5 100644 --- a/examples/models/llama/evaluate/eager_eval.py +++ b/examples/models/llama/evaluate/eager_eval.py @@ -31,7 +31,7 @@ def __init__( use_kv_cache: bool = False, ): device = "cuda" if torch.cuda.is_available() else "cpu" - super().__init__(device=device) + super().__init__(device=device, pretrained="gpt2") self._model = model self._tokenizer = tokenizer self._device = torch.device(device) @@ -47,6 +47,10 @@ def eot_token_id(self): return self._tokenizer.eot_id return self._tokenizer.eos_id + @property + def prefix_token_id(self): + return self.eot_token_id + @property def max_length(self): return self._max_seq_length diff --git a/examples/models/llama/install_requirements.sh b/examples/models/llama/install_requirements.sh index 470e804c2d4..3103daeb7d2 100755 --- a/examples/models/llama/install_requirements.sh +++ b/examples/models/llama/install_requirements.sh @@ -15,7 +15,7 @@ pip install --no-use-pep517 "git+https://github.com/pytorch/ao.git@${TORCHAO_VER # Install lm-eval for Model Evaluation with lm-evalution-harness # Install tiktoken for tokenizer -pip install lm_eval==0.4.2 +pip install lm_eval==0.4.5 pip install tiktoken blobfile # Call the install helper for further setup