From ccb864bbeafce6274456a3266f45ae6de64fa3cf Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 25 Apr 2024 17:37:17 -0700 Subject: [PATCH] fix(factory.py): correctly apply bos token for llama3 instruct prompt template --- litellm/llms/prompt_templates/factory.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index ed719f600db..bda3846a8c1 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -1350,11 +1350,8 @@ def prompt_factory( return llama_2_chat_pt(messages=messages) elif "llama3" in model and "instruct" in model: return hf_chat_template( - model=model, + model="meta-llama/Meta-Llama-3-8B-Instruct", messages=messages, - chat_template=known_tokenizer_config[ # type: ignore - "meta-llama/Meta-Llama-3-8B-Instruct" - ]["tokenizer"]["chat_template"], ) elif custom_llm_provider == "perplexity": for message in messages: @@ -1369,11 +1366,8 @@ def prompt_factory( "meta-llama/llama-3" in model or "meta-llama-3" in model ) and "instruct" in model: return hf_chat_template( - model=model, + model="meta-llama/Meta-Llama-3-8B-Instruct", messages=messages, - chat_template=known_tokenizer_config[ # type: ignore - "meta-llama/Meta-Llama-3-8B-Instruct" - ]["tokenizer"]["chat_template"], ) elif ( "tiiuae/falcon" in model