mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-06 02:48:13 +00:00
fix(caching): only skip injection on message or content marks
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
8f05215cee
commit
9f08d8aef8
2 changed files with 44 additions and 4 deletions
|
|
@ -518,22 +518,26 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
||||||
return []
|
return []
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _count_cache_control_blocks(message: object) -> int:
|
def _count_message_and_content_breakpoints(message: object) -> int:
|
||||||
message_count: Final = 1 if _carries_cache_breakpoint(message) else 0
|
message_count: Final = 1 if _carries_cache_breakpoint(message) else 0
|
||||||
content: Final = _as_object_list(_attribute_or_key(message, "content"))
|
content: Final = _as_object_list(_attribute_or_key(message, "content"))
|
||||||
content_count: Final = sum(1 for block in content if _carries_cache_breakpoint(block)) if content else 0
|
content_count: Final = sum(1 for block in content if _carries_cache_breakpoint(block)) if content else 0
|
||||||
|
return message_count + content_count
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _count_cache_control_blocks(message: object) -> int:
|
||||||
tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls"))
|
tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls"))
|
||||||
tool_call_count: Final = (
|
tool_call_count: Final = (
|
||||||
sum(1 for tool_call in tool_calls if _tool_call_cache_control_is_forwarded(tool_call, message))
|
sum(1 for tool_call in tool_calls if _tool_call_cache_control_is_forwarded(tool_call, message))
|
||||||
if tool_calls
|
if tool_calls
|
||||||
else 0
|
else 0
|
||||||
)
|
)
|
||||||
return message_count + content_count + tool_call_count
|
return AnthropicCacheControlHook._count_message_and_content_breakpoints(message) + tool_call_count
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _message_has_cache_control(message: AllMessageValues) -> bool:
|
def _message_has_cache_control(message: AllMessageValues) -> bool:
|
||||||
"""Return True if the message already carries any cache_control."""
|
"""Return True if message-level or content-block cache_control is present."""
|
||||||
return AnthropicCacheControlHook._count_cache_control_blocks(message) > 0
|
return AnthropicCacheControlHook._count_message_and_content_breakpoints(message) > 0
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _safe_insert_cache_control_in_message(
|
def _safe_insert_cache_control_in_message(
|
||||||
|
|
|
||||||
|
|
@ -1202,6 +1202,42 @@ def test_cache_control_hook_counts_pydantic_message_tool_call_marks():
|
||||||
assert AnthropicCacheControlHook.count_request_cache_breakpoints(cast(list[AllMessageValues], [message])) == 1
|
assert AnthropicCacheControlHook.count_request_cache_breakpoints(cast(list[AllMessageValues], [message])) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_injection_adds_message_mark_without_overwriting_tool_call_ttl():
|
||||||
|
hook: Final = AnthropicCacheControlHook()
|
||||||
|
tool_call_ttl: Final = {"type": "ephemeral", "ttl": "1h"}
|
||||||
|
messages: Final = [
|
||||||
|
{
|
||||||
|
"role": "assistant",
|
||||||
|
"content": "ok",
|
||||||
|
"tool_calls": [
|
||||||
|
{
|
||||||
|
"id": "call_1",
|
||||||
|
"type": "function",
|
||||||
|
"function": {"name": "lookup", "arguments": "{}"},
|
||||||
|
"cache_control": tool_call_ttl,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
_, processed, _ = hook.get_chat_completion_prompt(
|
||||||
|
model="bedrock/us.anthropic.claude-opus-4-6-v1:0",
|
||||||
|
messages=messages,
|
||||||
|
non_default_params={"cache_control_injection_points": [{"location": "message", "index": -1}]},
|
||||||
|
prompt_id=None,
|
||||||
|
prompt_variables=None,
|
||||||
|
dynamic_callback_params={},
|
||||||
|
)
|
||||||
|
|
||||||
|
assistant_message: Final = processed[0]
|
||||||
|
assistant_tool_calls: Final = assistant_message.get("tool_calls")
|
||||||
|
assert assistant_message.get("cache_control") == {"type": "ephemeral"}
|
||||||
|
assert isinstance(assistant_tool_calls, list)
|
||||||
|
tool_call: Final = assistant_tool_calls[0]
|
||||||
|
assert tool_call.get("cache_control") == tool_call_ttl
|
||||||
|
assert AnthropicCacheControlHook.count_request_cache_breakpoints(processed) == 2
|
||||||
|
|
||||||
|
|
||||||
def test_cache_control_hook_caps_at_four_blocks_with_client_cache_control():
|
def test_cache_control_hook_caps_at_four_blocks_with_client_cache_control():
|
||||||
"""Regression for LIT-3667 / Anthropic 'A maximum of 4 blocks ... Found 5'.
|
"""Regression for LIT-3667 / Anthropic 'A maximum of 4 blocks ... Found 5'.
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue