From b3f2f58e2641cb7356ef387a39e68715a6ee9036 Mon Sep 17 00:00:00 2001 From: fduwjj Date: Mon, 5 Aug 2024 12:47:17 -0700 Subject: [PATCH] [EZ][405B] Use scientific notation for 405B model lr --- train_configs/llama3_405b.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train_configs/llama3_405b.toml b/train_configs/llama3_405b.toml index fb250642e8..b7f78dc250 100644 --- a/train_configs/llama3_405b.toml +++ b/train_configs/llama3_405b.toml @@ -23,7 +23,7 @@ tokenizer_path = "./torchtitan/datasets/tokenizer/original/tokenizer.model" [optimizer] name = "AdamW" -lr = 0.8e-4 +lr = 8e-5 [training] batch_size = 2