scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-29 06:33:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 200

fix(anthropic): translate prompt_cache_breakpoint for Claude on Bedrock Mantle chat

Browse source
This commit is contained in:
mateo-berri 2026-10-15 16:31:35 +00:00
parent 6fa49e3f6e
commit 51af6c112e
2 changed files with 31 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
litellm/main.py
View file

@ -119,6 +119,7 @@ from litellm.llms.bedrock.common_utils import (
without_bedrock_route_prefix,
)
from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantle_chat_config
from litellm.llms.bedrock_mantle.common_utils import is_mantle_claude_model
from litellm.llms.cohere.common_utils import CohereModelInfo
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
@ -1092,6 +1093,8 @@ def _is_claude_tool_target(custom_llm_provider: str | None, model: str) -> bool:
def _is_claude_prompt_cache_target(custom_llm_provider: str | None, model: str, base_model: str | None) -> bool:
if custom_llm_provider == "bedrock_mantle":
return is_mantle_claude_model(model)
if custom_llm_provider in ("anthropic", "bedrock") and "claude" not in model.lower():
return base_model is not None and "claude" in base_model.lower()
return _is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model)

28
tests/unit/test_main.py
View file

@ -731,6 +731,34 @@ def test_breakpoints_on_one_converse_tool_result_take_one_slot_under_drop_params
assert _converse_cache_points(json.loads(route.calls.last.request.content)) == ("system", *("message",) * 3)
def test_a_breakpoint_reaches_bedrock_mantle_claude_as_cache_control(respx_mock: respx.MockRouter):
route: Final = respx_mock.post("https://bedrock-mantle.us-east-1.api.aws/anthropic/v1/messages").respond(
json={
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "anthropic.claude-opus-5-5",
"content": [{"type": "text", "text": "hi"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 1, "output_tokens": 1},
}
)
litellm.completion(
model="bedrock_mantle/anthropic.claude-opus-5-5",
messages=[
{"role": "system", "content": [{"type": "text", "text": "sys", "prompt_cache_breakpoint": _BREAKPOINT}]},
{"role": "user", "content": "Say hi."},
],
api_key="fake",
aws_region_name="us-east-1",
)
assert json.loads(route.calls.last.request.content)["system"] == [
{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}
]
def test_completion_keeps_a_function_call_whose_id_only_looks_like_a_claude_server_tool(
respx_mock: respx.MockRouter, openai_api_response
):