mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-12 03:48:56 +00:00
fix(responses): don't overwrite existing content-block cache_control; add list-content test
Per Greptile review on the PR: - The message-level cache_control was unconditionally overwriting the last content block's cache_control when that block already carried its own (preserved by _transform_responses_api_content_to_chat_completion_content). Now only propagates onto the last block when it doesn't already have one. - Added the missing test coverage for the content-block propagation path with list content, which no existing test exercised (all used plain string content, so the isinstance(content, list) branch was never hit).
This commit is contained in:
parent
4a1e4ada22
commit
3aacde577f
2 changed files with 56 additions and 2 deletions
|
|
@ -1430,10 +1430,11 @@ class LiteLLMCompletionResponsesConfig:
|
|||
if input_item.get("cache_control"):
|
||||
msg["cache_control"] = input_item.get("cache_control")
|
||||
# Also propagate to the last content element for providers
|
||||
# that read cache_control from content blocks (Anthropic, Bedrock)
|
||||
# that read cache_control from content blocks (Anthropic, Bedrock),
|
||||
# unless that block already carries its own cache_control.
|
||||
if isinstance(msg["content"], list) and len(msg["content"]) > 0:
|
||||
last_element = msg["content"][-1]
|
||||
if isinstance(last_element, dict):
|
||||
if isinstance(last_element, dict) and "cache_control" not in last_element:
|
||||
last_element["cache_control"] = input_item["cache_control"]
|
||||
return [msg]
|
||||
|
||||
|
|
|
|||
|
|
@ -5302,3 +5302,56 @@ class TestCacheControlPreservationOnMessages:
|
|||
assert "cache_control" not in result[0]
|
||||
assert result[1].get("cache_control") == {"type": "ephemeral"}
|
||||
assert result[2].get("cache_control") == {"type": "ephemeral"}
|
||||
|
||||
def test_cache_control_propagated_to_last_content_block_for_list_content(self):
|
||||
"""
|
||||
For Anthropic/Bedrock, cache_control must also be set on the last
|
||||
content block (not just the message level) to actually activate
|
||||
prompt caching when content is a list of blocks.
|
||||
"""
|
||||
input_item = {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "first block"},
|
||||
{"type": "input_text", "text": "last block"},
|
||||
],
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
|
||||
result = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message(
|
||||
input_item=input_item
|
||||
)
|
||||
|
||||
assert len(result) == 1
|
||||
msg = result[0]
|
||||
assert msg.get("cache_control") == {"type": "ephemeral"}
|
||||
assert isinstance(msg["content"], list)
|
||||
assert "cache_control" not in msg["content"][0]
|
||||
assert msg["content"][-1]["cache_control"] == {"type": "ephemeral"}
|
||||
|
||||
def test_cache_control_does_not_overwrite_existing_block_level_cache_control(self):
|
||||
"""
|
||||
If the last content block already carries its own cache_control
|
||||
(preserved by _transform_responses_api_content_to_chat_completion_content),
|
||||
the message-level cache_control must not silently overwrite it.
|
||||
"""
|
||||
input_item = {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": "last block",
|
||||
"cache_control": {"type": "ephemeral", "ttl": "1h"},
|
||||
},
|
||||
],
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
|
||||
result = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message(
|
||||
input_item=input_item
|
||||
)
|
||||
|
||||
assert len(result) == 1
|
||||
msg = result[0]
|
||||
assert msg.get("cache_control") == {"type": "ephemeral"}
|
||||
assert msg["content"][-1]["cache_control"] == {"type": "ephemeral", "ttl": "1h"}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue