scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-05 02:41:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 49

fix: add supports_assistant_prefill to Claude 3 Bedrock models (#30963)

Browse source 
The AWS Bedrock Anthropic Claude 3 entries in the model cost maps were
missing supports_assistant_prefill entirely. Because the key was absent
rather than true, callers that gate on its truthiness treated these
models as not supporting trailing-assistant prefill, even though every
Claude 3 model supports it; the direct anthropic API entries for
claude-3-haiku and claude-3-opus already carry true

This sets supports_assistant_prefill to true for all 20 affected Claude 3
Bedrock entries across regions (us, eu, apac, us-gov) and routes (invoke),
in both the primary price map and the litellm backup, and adds a
regression test covering the JSON maps and get_model_info

Fixes #30863
This commit is contained in:
Atharva Jaiswal 2026-06-23 18:34:54 +05:30 • committed by Sameer Kankute
parent 0e59c3bb17
commit 3bf41e0acf
No known key found for this signature in database
3 changed files with 139 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
litellm/model_prices_and_context_window_backup.json
View file

@ -778,6 +778,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -865,6 +866,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.25e-06,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -881,6 +883,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 7.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true,
@ -896,6 +899,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -2103,6 +2107,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -2138,6 +2143,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.25e-06,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -2176,6 +2182,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -8865,6 +8872,7 @@
},
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true,
@ -9327,6 +9335,7 @@
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.8e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -9343,6 +9352,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-06,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -9502,6 +9512,7 @@
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.8e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -9518,6 +9529,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-06,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -14380,6 +14392,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -14435,6 +14448,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.25e-06,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -14451,6 +14465,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 7.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true,
@ -14466,6 +14481,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -32555,6 +32571,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -32610,6 +32627,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.25e-06,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -32626,6 +32644,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 7.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true,
@ -32641,6 +32660,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,

20
model_prices_and_context_window.json
View file

@ -778,6 +778,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -865,6 +866,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.25e-06,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -881,6 +883,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 7.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true,
@ -896,6 +899,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -2103,6 +2107,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -2138,6 +2143,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.25e-06,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -2176,6 +2182,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -8865,6 +8872,7 @@
},
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true,
@ -9327,6 +9335,7 @@
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.8e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -9343,6 +9352,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-06,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -9502,6 +9512,7 @@
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.8e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -9518,6 +9529,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-06,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -14380,6 +14392,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -14435,6 +14448,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.25e-06,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -14451,6 +14465,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 7.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true,
@ -14466,6 +14481,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -32732,6 +32748,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -32787,6 +32804,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.25e-06,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
@ -32803,6 +32821,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 7.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true,
@ -32818,6 +32837,7 @@
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,

99
tests/test_litellm/test_claude_3_bedrock_assistant_prefill.py Normal file
View file

@ -0,0 +1,99 @@
"""
Regression test: the AWS Bedrock Anthropic Claude 3 family entries in the
model-cost maps were missing ``supports_assistant_prefill`` entirely. Because
the key was absent rather than ``true``, callers that gate on its truthiness
silently treated these models as not supporting trailing-assistant prefill,
even though every Claude 3 model supports it (the direct ``anthropic`` API
entries for ``claude-3-haiku``/``claude-3-opus`` already carry ``true``).
These tests pin ``supports_assistant_prefill == true`` for the affected entries
in both the primary price map and the ``litellm/`` backup, and verify
``get_model_info`` surfaces it, so the field cannot silently drop back to absent.
"""
import json
import os
import sys
sys.path.insert(0, os.path.abspath("../..")) # Adds the parent directory to the system path
import litellm
CLAUDE_3_BEDROCK_MODELS = (
"anthropic.claude-3-haiku-20240307-v1:0",
"anthropic.claude-3-sonnet-20240229-v1:0",
"anthropic.claude-3-opus-20240229-v1:0",
"anthropic.claude-3-5-sonnet-20240620-v1:0",
"us.anthropic.claude-3-haiku-20240307-v1:0",
"us.anthropic.claude-3-sonnet-20240229-v1:0",
"us.anthropic.claude-3-opus-20240229-v1:0",
"us.anthropic.claude-3-5-sonnet-20240620-v1:0",
"eu.anthropic.claude-3-haiku-20240307-v1:0",
"eu.anthropic.claude-3-sonnet-20240229-v1:0",
"eu.anthropic.claude-3-opus-20240229-v1:0",
"eu.anthropic.claude-3-5-sonnet-20240620-v1:0",
"apac.anthropic.claude-3-haiku-20240307-v1:0",
"apac.anthropic.claude-3-sonnet-20240229-v1:0",
"apac.anthropic.claude-3-5-sonnet-20240620-v1:0",
"bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0",
"bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0",
"bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0",
"bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0",
"bedrock/us-gov-west-1/anthropic.claude-3-5-sonnet-20240620-v1:0",
)
def _load_json(path: str) -> dict:
with open(path, encoding="utf-8") as f:
return json.load(f)
def _backup_path() -> str:
return os.path.join(
os.path.dirname(litellm.__file__),
"model_prices_and_context_window_backup.json",
)
def _main_path() -> str:
# This test lives at ``tests/test_litellm/``; the primary price map sits at
# the repo root, two directories up. Resolve it relative to this file so the
# test works regardless of where ``litellm`` itself is installed.
return os.path.join(
os.path.dirname(__file__),
"..",
"..",
"model_prices_and_context_window.json",
)
class TestClaude3BedrockAssistantPrefillData:
"""Both JSON price maps must mark every Claude 3 Bedrock entry as supporting
assistant prefill."""
def test_backup_supports_assistant_prefill(self):
price_map = _load_json(_backup_path())
for model in CLAUDE_3_BEDROCK_MODELS:
assert price_map[model]["supports_assistant_prefill"] is True, model
def test_main_supports_assistant_prefill(self):
price_map = _load_json(_main_path())
for model in CLAUDE_3_BEDROCK_MODELS:
assert price_map[model]["supports_assistant_prefill"] is True, model
class TestClaude3BedrockAssistantPrefillModelInfo:
"""``get_model_info`` must report ``supports_assistant_prefill`` for these
models."""
def test_get_model_info_supports_assistant_prefill(self):
# Patch litellm.model_cost with the local backup so the assertion does
# not depend on the remote fetch hitting a not-yet-merged main branch.
original = litellm.model_cost
try:
litellm.model_cost = _load_json(_backup_path())
for model in CLAUDE_3_BEDROCK_MODELS:
info = litellm.get_model_info(model)
assert info["supports_assistant_prefill"] is True, model
finally:
litellm.model_cost = original