diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 03408043f8e..a4b00af8da0 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -43687,6 +43687,9 @@ }, "ChatCompletionToolMessage": { "properties": { + "cache_control": { + "$ref": "#/components/schemas/ChatCompletionCachedContent" + }, "content": { "anyOf": [ { diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 3a27d21d0ce..66a63579698 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -47,6 +47,7 @@ from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantMessage, + ChatCompletionCachedContent, ChatCompletionImageObject, ChatCompletionImageUrlObject, ChatCompletionRedactedThinkingBlock, @@ -1452,14 +1453,23 @@ class LiteLLMCompletionResponsesConfig: # Since guardrails skip None content anyway, we return empty list to exclude it from structured messages if content is None: return [] - return [ - GenericChatCompletionMessage( - role=_input_item_role(input_item), - content=LiteLLMCompletionResponsesConfig._transform_responses_api_content_to_chat_completion_content( - content - ), + msg = GenericChatCompletionMessage( + role=input_item.get("role") or "user", + content=LiteLLMCompletionResponsesConfig._transform_responses_api_content_to_chat_completion_content( + content + ), + ) + input_item_cache_control = input_item.get("cache_control") + if input_item_cache_control: + msg["cache_control"] = cast( # cast-ok: Responses input uses the shared cache-control shape + "ChatCompletionCachedContent", input_item_cache_control ) - ] + # Also set on the last content block for providers (Anthropic, Bedrock) that read cache_control there. + if isinstance(msg["content"], list) and len(msg["content"]) > 0: + last_element = msg["content"][-1] + if isinstance(last_element, dict) and "cache_control" not in last_element: + last_element["cache_control"] = input_item_cache_control + return [msg] @staticmethod def _reasoning_text_from_content(input_item: Mapping[str, object]) -> str | None: @@ -1654,6 +1664,11 @@ class LiteLLMCompletionResponsesConfig: content=_normalize_function_call_output_to_tool_content(tool_call_output.get("output")), tool_call_id=str(call_id), ) + output_cache_control = tool_call_output.get("cache_control") + if output_cache_control: + tool_output_message["cache_control"] = cast( # cast-ok: Responses input uses the shared cache-control shape + "ChatCompletionCachedContent", output_cache_control + ) _tool_use_definition: Final = TOOL_CALLS_CACHE.get_cache( key=tool_call_output.get("call_id") or "", @@ -1750,6 +1765,13 @@ class LiteLLMCompletionResponsesConfig: role="assistant", content=None, # Function calls don't have content ) + function_call_cache_control = function_call.get("cache_control") + if function_call_cache_control: + chat_completion_response_message["cache_control"] = ( + cast( # cast-ok: Responses input uses the shared cache-control shape + "ChatCompletionCachedContent", function_call_cache_control + ) + ) return [chat_completion_response_message] diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index f241c4a7a27..3b623b1dd88 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -875,6 +875,7 @@ class ChatCompletionToolMessage(TypedDict): role: Literal["tool"] content: str | Iterable[ToolMessageContentPart] # writable-ok: Pydantic warns on ReadOnly TypedDict fields tool_call_id: str + cache_control: NotRequired[ChatCompletionCachedContent] class ChatCompletionFunctionMessage(TypedDict): @@ -907,6 +908,7 @@ class ChatCompletionDeveloperMessage(OpenAIChatCompletionDeveloperMessage, total class GenericChatCompletionMessage(TypedDict, total=False): role: Required[str] content: Required[str | list] + cache_control: ChatCompletionCachedContent ValidUserMessageContentTypes = [ @@ -1097,6 +1099,7 @@ class ChatCompletionResponseMessage(TypedDict, total=False): provider_specific_fields: dict | None reasoning_content: str | None thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] | None + cache_control: ChatCompletionCachedContent class ChatCompletionUsageBlock(TypedDict, total=False): diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index c13054f1cee..8f141fb26b6 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -5311,3 +5311,186 @@ async def test_bridge_rejects_untranslatable_tool_choice_with_a_400(stream: bool ) assert exc_info.value.status_code == 400 assert "tool_choice={'type': 'file_search'}" in str(exc_info.value) +class TestCacheControlPreservationOnMessages: + """Tests that cache_control is preserved when transforming Responses API input items + to Chat Completion messages. This is critical for providers like Bedrock/Anthropic + that use cache_control for prompt caching.""" + + def test_cache_control_on_user_message(self): + """cache_control on a user message should be passed through to GenericChatCompletionMessage""" + input_item = { + "role": "user", + "content": "Hello, world!", + "cache_control": {"type": "ephemeral"}, + } + + result = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( + input_item=input_item + ) + + assert len(result) == 1 + msg = result[0] + assert msg["role"] == "user" + assert msg.get("cache_control") == {"type": "ephemeral"} + + def test_cache_control_on_assistant_message(self): + """cache_control on an assistant message should be passed through""" + input_item = { + "role": "assistant", + "content": "I can help with that.", + "cache_control": {"type": "ephemeral"}, + } + + result = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( + input_item=input_item + ) + + assert len(result) == 1 + msg = result[0] + assert msg["role"] == "assistant" + assert msg.get("cache_control") == {"type": "ephemeral"} + + def test_no_cache_control_when_absent(self): + """Messages without cache_control should not have the field set""" + input_item = { + "role": "user", + "content": "Hello", + } + + result = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( + input_item=input_item + ) + + assert len(result) == 1 + msg = result[0] + assert "cache_control" not in msg + + def test_cache_control_on_function_call_output(self): + """cache_control on a function_call_output should be passed through to tool message""" + TOOL_CALLS_CACHE.set_cache( + key="call_123", + value={ + "id": "call_123", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "NYC"}'}, + }, + ) + + input_item = { + "type": "function_call_output", + "call_id": "call_123", + "output": '{"temp": 72}', + "cache_control": {"type": "ephemeral"}, + } + + result = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( + input_item=input_item + ) + + # Should have assistant message + tool message + assert len(result) == 2 + tool_msg = result[1] + assert tool_msg["role"] == "tool" + assert tool_msg.get("cache_control") == {"type": "ephemeral"} + + def test_cache_control_on_function_call(self): + """cache_control on a function_call input should be passed through to assistant message""" + input_item = { + "type": "function_call", + "call_id": "call_456", + "name": "get_weather", + "arguments": '{"city": "NYC"}', + "cache_control": {"type": "ephemeral"}, + } + + result = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( + input_item=input_item + ) + + assert len(result) == 1 + msg = result[0] + assert msg["role"] == "assistant" + assert msg.get("cache_control") == {"type": "ephemeral"} + + def test_cache_control_preserved_in_full_input_transform(self): + """cache_control should survive the full transform_responses_api_input_to_messages path""" + input_items = [ + { + "role": "user", + "content": "What is the weather?", + }, + { + "role": "assistant", + "content": "Let me check.", + "cache_control": {"type": "ephemeral"}, + }, + { + "role": "user", + "content": "Thanks!", + "cache_control": {"type": "ephemeral"}, + }, + ] + + result = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=input_items, + responses_api_request={}, + ) + + # 3 messages, no system + assert len(result) == 3 + assert "cache_control" not in result[0] + assert result[1].get("cache_control") == {"type": "ephemeral"} + assert result[2].get("cache_control") == {"type": "ephemeral"} + + def test_cache_control_propagated_to_last_content_block_for_list_content(self): + """ + For Anthropic/Bedrock, cache_control must also be set on the last + content block (not just the message level) to actually activate + prompt caching when content is a list of blocks. + """ + input_item = { + "role": "user", + "content": [ + {"type": "input_text", "text": "first block"}, + {"type": "input_text", "text": "last block"}, + ], + "cache_control": {"type": "ephemeral"}, + } + + result = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( + input_item=input_item + ) + + assert len(result) == 1 + msg = result[0] + assert msg.get("cache_control") == {"type": "ephemeral"} + assert isinstance(msg["content"], list) + assert "cache_control" not in msg["content"][0] + assert msg["content"][-1]["cache_control"] == {"type": "ephemeral"} + + def test_cache_control_does_not_overwrite_existing_block_level_cache_control(self): + """ + If the last content block already carries its own cache_control + (preserved by _transform_responses_api_content_to_chat_completion_content), + the message-level cache_control must not silently overwrite it. + """ + input_item = { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "last block", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + }, + ], + "cache_control": {"type": "ephemeral"}, + } + + result = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( + input_item=input_item + ) + + assert len(result) == 1 + msg = result[0] + assert msg.get("cache_control") == {"type": "ephemeral"} + assert msg["content"][-1]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index e4d9111ab7a..c7d1ecf4cf6 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -28744,6 +28744,7 @@ export interface components { }; /** ChatCompletionToolMessage */ ChatCompletionToolMessage: { + cache_control?: components["schemas"]["ChatCompletionCachedContent"]; /** Content */ content: string | (components["schemas"]["ChatCompletionTextObject"] | components["schemas"]["ChatCompletionImageObject"] | components["schemas"]["ChatCompletionToolReferenceObject"])[]; /**