scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-19 00:01:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 122

(fix) fix ibm_granite_pt and use hf_chat_template for llama-3 instruct prompts

Browse source
This commit is contained in:
Simon Sanchez Viloria 2024-05-06 17:06:46 +02:00
parent 3c248e308e
commit 62b3f25398
1 changed files with 7 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
litellm/llms/prompt_templates/factory.py
View file

@ -434,7 +434,7 @@ def format_prompt_togetherai(messages, prompt_format, chat_template):
def ibm_granite_pt(messages: list):
"""
IBM's Granite models uses the template:
IBM's Granite chat models uses the template:
<|system|> {system_message} <|user|> {user_message} <|assistant|> {assistant_message}
See: https://www.ibm.com/docs/en/watsonx-as-a-service?topic=solutions-supported-foundation-models
@ -453,9 +453,10 @@ def ibm_granite_pt(messages: list):
'assistant': {
'pre_message': '<|assistant|>\n',
'post_message': '\n',
}
}
).strip()
},
},
final_prompt_value='<|assistant|>\n',
)
### ANTHROPIC ###
@ -1400,15 +1401,9 @@ def prompt_factory(
return mistral_instruct_pt(messages=messages)
elif "meta-llama/llama-3" in model and "instruct" in model:
# https://llama.meta.com/docs/model-cards-and-prompt-formats/meta-llama-3/
return custom_prompt(
role_dict={
"system": {"pre_message": "<|start_header_id|>system<|end_header_id|>\n", "post_message": "<|eot_id|>"},
"user": {"pre_message": "<|start_header_id|>user<|end_header_id|>\n", "post_message": "<|eot_id|>"},
"assistant": {"pre_message": "<|start_header_id|>assistant<|end_header_id|>\n", "post_message": "<|eot_id|>"},
},
return hf_chat_template(
model="meta-llama/Meta-Llama-3-8B-Instruct",
messages=messages,
initial_prompt_value="<|begin_of_text|>",
final_prompt_value="<|start_header_id|>assistant<|end_header_id|>\n",
)
try:
if "meta-llama/llama-2" in model and "chat" in model: