From 2fd4bd2e09895043255a02d055a4dcd024bf8151 Mon Sep 17 00:00:00 2001 From: TobiMayr Date: Wed, 8 Jul 2026 16:54:59 +0200 Subject: [PATCH 1/8] fix(responses): propagate cache_control through responses->chat completion bridge User/assistant messages, tool call outputs, and function calls built from Responses API input items were dropping cache_control, breaking prompt caching for Bedrock/Anthropic when routed through the litellm_completion transform. Also propagate cache_control onto the last content element of user/assistant messages, since Anthropic/Bedrock read it from the content block rather than (or in addition to) the message level. Adds the missing cache_control field to ChatCompletionToolMessage and ChatCompletionResponseMessage TypedDicts to keep the assignments type-safe. --- .../transformation.py | 29 ++-- litellm/types/llms/openai.py | 3 + .../test_litellm_completion_responses.py | 130 ++++++++++++++++++ 3 files changed, 154 insertions(+), 8 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index bd239922fd3..3c5a69dc5e0 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1421,14 +1421,21 @@ class LiteLLMCompletionResponsesConfig: # Since guardrails skip None content anyway, we return empty list to exclude it from structured messages if content is None: return [] - return [ - GenericChatCompletionMessage( - role=_input_item_role(input_item), - content=LiteLLMCompletionResponsesConfig._transform_responses_api_content_to_chat_completion_content( - content - ), - ) - ] + msg = GenericChatCompletionMessage( + role=input_item.get("role") or "user", + content=LiteLLMCompletionResponsesConfig._transform_responses_api_content_to_chat_completion_content( + content + ), + ) + if input_item.get("cache_control"): + msg["cache_control"] = input_item.get("cache_control") + # Also propagate to the last content element for providers + # that read cache_control from content blocks (Anthropic, Bedrock) + if isinstance(msg["content"], list) and len(msg["content"]) > 0: + last_element = msg["content"][-1] + if isinstance(last_element, dict): + last_element["cache_control"] = input_item["cache_control"] + return [msg] @staticmethod def _reasoning_text_from_content(input_item: Mapping[str, object]) -> str | None: @@ -1646,6 +1653,9 @@ class LiteLLMCompletionResponsesConfig: content=_normalize_function_call_output_to_tool_content(tool_call_output.get("output")), tool_call_id=str(call_id), ) + output_cache_control = tool_call_output.get("cache_control") + if output_cache_control: + tool_output_message["cache_control"] = output_cache_control _tool_use_definition: Final = TOOL_CALLS_CACHE.get_cache( key=tool_call_output.get("call_id") or "", @@ -1742,6 +1752,9 @@ class LiteLLMCompletionResponsesConfig: role="assistant", content=None, # Function calls don't have content ) + function_call_cache_control = function_call.get("cache_control") + if function_call_cache_control: + chat_completion_response_message["cache_control"] = function_call_cache_control return [chat_completion_response_message] diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 99ab5920c4f..b8607f68e69 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -878,6 +878,7 @@ class ChatCompletionToolMessage(TypedDict): role: Literal["tool"] content: str | Iterable[ToolMessageContentPart] # writable-ok: Pydantic warns on ReadOnly TypedDict fields tool_call_id: str + cache_control: NotRequired[ChatCompletionCachedContent] class ChatCompletionFunctionMessage(TypedDict): @@ -910,6 +911,7 @@ class ChatCompletionDeveloperMessage(OpenAIChatCompletionDeveloperMessage, total class GenericChatCompletionMessage(TypedDict, total=False): role: Required[str] content: Required[str | list] + cache_control: ChatCompletionCachedContent ValidUserMessageContentTypes = [ @@ -1100,6 +1102,7 @@ class ChatCompletionResponseMessage(TypedDict, total=False): provider_specific_fields: dict | None reasoning_content: str | None thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] | None + cache_control: ChatCompletionCachedContent class ChatCompletionUsageBlock(TypedDict, total=False): diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 7b9de4644b4..ab1de8f4946 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -5172,3 +5172,133 @@ async def test_bridge_rejects_untranslatable_tool_choice_with_a_400(stream: bool ) assert exc_info.value.status_code == 400 assert "tool_choice={'type': 'file_search'}" in str(exc_info.value) +class TestCacheControlPreservationOnMessages: + """Tests that cache_control is preserved when transforming Responses API input items + to Chat Completion messages. This is critical for providers like Bedrock/Anthropic + that use cache_control for prompt caching.""" + + def test_cache_control_on_user_message(self): + """cache_control on a user message should be passed through to GenericChatCompletionMessage""" + input_item = { + "role": "user", + "content": "Hello, world!", + "cache_control": {"type": "ephemeral"}, + } + + result = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( + input_item=input_item + ) + + assert len(result) == 1 + msg = result[0] + assert msg["role"] == "user" + assert msg.get("cache_control") == {"type": "ephemeral"} + + def test_cache_control_on_assistant_message(self): + """cache_control on an assistant message should be passed through""" + input_item = { + "role": "assistant", + "content": "I can help with that.", + "cache_control": {"type": "ephemeral"}, + } + + result = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( + input_item=input_item + ) + + assert len(result) == 1 + msg = result[0] + assert msg["role"] == "assistant" + assert msg.get("cache_control") == {"type": "ephemeral"} + + def test_no_cache_control_when_absent(self): + """Messages without cache_control should not have the field set""" + input_item = { + "role": "user", + "content": "Hello", + } + + result = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( + input_item=input_item + ) + + assert len(result) == 1 + msg = result[0] + assert "cache_control" not in msg + + def test_cache_control_on_function_call_output(self): + """cache_control on a function_call_output should be passed through to tool message""" + TOOL_CALLS_CACHE.set_cache( + key="call_123", + value={ + "id": "call_123", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "NYC"}'}, + }, + ) + + input_item = { + "type": "function_call_output", + "call_id": "call_123", + "output": '{"temp": 72}', + "cache_control": {"type": "ephemeral"}, + } + + result = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( + input_item=input_item + ) + + # Should have assistant message + tool message + assert len(result) == 2 + tool_msg = result[1] + assert tool_msg["role"] == "tool" + assert tool_msg.get("cache_control") == {"type": "ephemeral"} + + def test_cache_control_on_function_call(self): + """cache_control on a function_call input should be passed through to assistant message""" + input_item = { + "type": "function_call", + "call_id": "call_456", + "name": "get_weather", + "arguments": '{"city": "NYC"}', + "cache_control": {"type": "ephemeral"}, + } + + result = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( + input_item=input_item + ) + + assert len(result) == 1 + msg = result[0] + assert msg["role"] == "assistant" + assert msg.get("cache_control") == {"type": "ephemeral"} + + def test_cache_control_preserved_in_full_input_transform(self): + """cache_control should survive the full transform_responses_api_input_to_messages path""" + input_items = [ + { + "role": "user", + "content": "What is the weather?", + }, + { + "role": "assistant", + "content": "Let me check.", + "cache_control": {"type": "ephemeral"}, + }, + { + "role": "user", + "content": "Thanks!", + "cache_control": {"type": "ephemeral"}, + }, + ] + + result = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=input_items, + responses_api_request={}, + ) + + # 3 messages, no system + assert len(result) == 3 + assert "cache_control" not in result[0] + assert result[1].get("cache_control") == {"type": "ephemeral"} + assert result[2].get("cache_control") == {"type": "ephemeral"} From 4a1e4ada22223dc57d6e25e9a94539fe24f7eca7 Mon Sep 17 00:00:00 2001 From: TobiMayr Date: Wed, 8 Jul 2026 19:32:47 +0200 Subject: [PATCH 2/8] chore(ui): regenerate schema.d.ts for ChatCompletionToolMessage.cache_control Adding cache_control to the ChatCompletionToolMessage TypedDict changed the proxy's OpenAPI spec; regenerated via `npm run gen:api`. --- ui/litellm-dashboard/src/lib/http/schema.d.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index ecfabf33e87..7011060b2f6 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -27269,6 +27269,7 @@ export interface components { }; /** ChatCompletionToolMessage */ ChatCompletionToolMessage: { + cache_control?: components["schemas"]["ChatCompletionCachedContent"]; /** Content */ content: string | (components["schemas"]["ChatCompletionTextObject"] | components["schemas"]["ChatCompletionImageObject"] | components["schemas"]["ChatCompletionToolReferenceObject"])[]; /** From 3aacde577fdf3ad88b21fc21f84dca9d995f1ea3 Mon Sep 17 00:00:00 2001 From: TobiMayr Date: Thu, 9 Jul 2026 10:59:30 +0200 Subject: [PATCH 3/8] fix(responses): don't overwrite existing content-block cache_control; add list-content test Per Greptile review on the PR: - The message-level cache_control was unconditionally overwriting the last content block's cache_control when that block already carried its own (preserved by _transform_responses_api_content_to_chat_completion_content). Now only propagates onto the last block when it doesn't already have one. - Added the missing test coverage for the content-block propagation path with list content, which no existing test exercised (all used plain string content, so the isinstance(content, list) branch was never hit). --- .../transformation.py | 5 +- .../test_litellm_completion_responses.py | 53 +++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 3c5a69dc5e0..70a4bd12ca7 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1430,10 +1430,11 @@ class LiteLLMCompletionResponsesConfig: if input_item.get("cache_control"): msg["cache_control"] = input_item.get("cache_control") # Also propagate to the last content element for providers - # that read cache_control from content blocks (Anthropic, Bedrock) + # that read cache_control from content blocks (Anthropic, Bedrock), + # unless that block already carries its own cache_control. if isinstance(msg["content"], list) and len(msg["content"]) > 0: last_element = msg["content"][-1] - if isinstance(last_element, dict): + if isinstance(last_element, dict) and "cache_control" not in last_element: last_element["cache_control"] = input_item["cache_control"] return [msg] diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index ab1de8f4946..f494ec84e79 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -5302,3 +5302,56 @@ class TestCacheControlPreservationOnMessages: assert "cache_control" not in result[0] assert result[1].get("cache_control") == {"type": "ephemeral"} assert result[2].get("cache_control") == {"type": "ephemeral"} + + def test_cache_control_propagated_to_last_content_block_for_list_content(self): + """ + For Anthropic/Bedrock, cache_control must also be set on the last + content block (not just the message level) to actually activate + prompt caching when content is a list of blocks. + """ + input_item = { + "role": "user", + "content": [ + {"type": "input_text", "text": "first block"}, + {"type": "input_text", "text": "last block"}, + ], + "cache_control": {"type": "ephemeral"}, + } + + result = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( + input_item=input_item + ) + + assert len(result) == 1 + msg = result[0] + assert msg.get("cache_control") == {"type": "ephemeral"} + assert isinstance(msg["content"], list) + assert "cache_control" not in msg["content"][0] + assert msg["content"][-1]["cache_control"] == {"type": "ephemeral"} + + def test_cache_control_does_not_overwrite_existing_block_level_cache_control(self): + """ + If the last content block already carries its own cache_control + (preserved by _transform_responses_api_content_to_chat_completion_content), + the message-level cache_control must not silently overwrite it. + """ + input_item = { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "last block", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + }, + ], + "cache_control": {"type": "ephemeral"}, + } + + result = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( + input_item=input_item + ) + + assert len(result) == 1 + msg = result[0] + assert msg.get("cache_control") == {"type": "ephemeral"} + assert msg["content"][-1]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} From 77aa69cb2b6f9ac7d3346bea90bab4ef4175ac00 Mon Sep 17 00:00:00 2001 From: TobiMayr Date: Thu, 9 Jul 2026 16:40:32 +0200 Subject: [PATCH 4/8] style(responses): extract cache_control into a named variable, trim comment Per style review: the message-building branch read cache_control 3 different ways in 5 lines (.get() twice, then [] indexing once), unlike the sibling tool-call-output/function-call functions this PR touches, which each extract it into a named variable up front (output_cache_control, function_call_cache_control). Matched that convention here too. Also trimmed the 3-line inline comment to one line, matching the file's usual comment length for similarly subtle behavior. --- .../transformation.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 70a4bd12ca7..65cd1e6d7c6 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1427,15 +1427,14 @@ class LiteLLMCompletionResponsesConfig: content ), ) - if input_item.get("cache_control"): - msg["cache_control"] = input_item.get("cache_control") - # Also propagate to the last content element for providers - # that read cache_control from content blocks (Anthropic, Bedrock), - # unless that block already carries its own cache_control. + input_item_cache_control = input_item.get("cache_control") + if input_item_cache_control: + msg["cache_control"] = input_item_cache_control + # Also set on the last content block for providers (Anthropic, Bedrock) that read cache_control there. if isinstance(msg["content"], list) and len(msg["content"]) > 0: last_element = msg["content"][-1] if isinstance(last_element, dict) and "cache_control" not in last_element: - last_element["cache_control"] = input_item["cache_control"] + last_element["cache_control"] = input_item_cache_control return [msg] @staticmethod From 3e1104c59262315454ca8f016edb4cce629475a0 Mon Sep 17 00:00:00 2001 From: TobiMayr Date: Wed, 30 Sep 2026 11:40:09 +0200 Subject: [PATCH 5/8] chore: refresh lazy OpenAPI snapshot --- litellm/proxy/_lazy_openapi_snapshot.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index bb063f8f77a..2c70c2697ed 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -41852,6 +41852,9 @@ }, "ChatCompletionToolMessage": { "properties": { + "cache_control": { + "$ref": "#/components/schemas/ChatCompletionCachedContent" + }, "content": { "anyOf": [ { From 5d8c44c6c73902eb2f25d82f840f56252564a34f Mon Sep 17 00:00:00 2001 From: TobiMayr Date: Wed, 30 Sep 2026 11:54:05 +0200 Subject: [PATCH 6/8] fix: type cache control propagation --- .../transformation.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 65cd1e6d7c6..a53e7f433a6 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -47,6 +47,7 @@ from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantMessage, + ChatCompletionCachedContent, ChatCompletionImageObject, ChatCompletionImageUrlObject, ChatCompletionRedactedThinkingBlock, @@ -1429,7 +1430,9 @@ class LiteLLMCompletionResponsesConfig: ) input_item_cache_control = input_item.get("cache_control") if input_item_cache_control: - msg["cache_control"] = input_item_cache_control + msg["cache_control"] = cast( + "ChatCompletionCachedContent", input_item_cache_control + ) # cast-ok: Responses input uses the shared cache-control shape # Also set on the last content block for providers (Anthropic, Bedrock) that read cache_control there. if isinstance(msg["content"], list) and len(msg["content"]) > 0: last_element = msg["content"][-1] @@ -1655,7 +1658,9 @@ class LiteLLMCompletionResponsesConfig: ) output_cache_control = tool_call_output.get("cache_control") if output_cache_control: - tool_output_message["cache_control"] = output_cache_control + tool_output_message["cache_control"] = cast( + "ChatCompletionCachedContent", output_cache_control + ) # cast-ok: Responses input uses the shared cache-control shape _tool_use_definition: Final = TOOL_CALLS_CACHE.get_cache( key=tool_call_output.get("call_id") or "", @@ -1754,7 +1759,9 @@ class LiteLLMCompletionResponsesConfig: ) function_call_cache_control = function_call.get("cache_control") if function_call_cache_control: - chat_completion_response_message["cache_control"] = function_call_cache_control + chat_completion_response_message["cache_control"] = cast( + "ChatCompletionCachedContent", function_call_cache_control + ) # cast-ok: Responses input uses the shared cache-control shape return [chat_completion_response_message] From 930168e1af78e9bbfcba2fabb7ecfd7003675970 Mon Sep 17 00:00:00 2001 From: TobiMayr Date: Wed, 30 Sep 2026 12:01:36 +0200 Subject: [PATCH 7/8] fix: place cache control type suppressions --- .../transformation.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index a53e7f433a6..84795d3fed6 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1430,9 +1430,9 @@ class LiteLLMCompletionResponsesConfig: ) input_item_cache_control = input_item.get("cache_control") if input_item_cache_control: - msg["cache_control"] = cast( + msg["cache_control"] = cast( # cast-ok: Responses input uses the shared cache-control shape "ChatCompletionCachedContent", input_item_cache_control - ) # cast-ok: Responses input uses the shared cache-control shape + ) # Also set on the last content block for providers (Anthropic, Bedrock) that read cache_control there. if isinstance(msg["content"], list) and len(msg["content"]) > 0: last_element = msg["content"][-1] @@ -1658,9 +1658,9 @@ class LiteLLMCompletionResponsesConfig: ) output_cache_control = tool_call_output.get("cache_control") if output_cache_control: - tool_output_message["cache_control"] = cast( + tool_output_message["cache_control"] = cast( # cast-ok: Responses input uses the shared cache-control shape "ChatCompletionCachedContent", output_cache_control - ) # cast-ok: Responses input uses the shared cache-control shape + ) _tool_use_definition: Final = TOOL_CALLS_CACHE.get_cache( key=tool_call_output.get("call_id") or "", @@ -1759,9 +1759,9 @@ class LiteLLMCompletionResponsesConfig: ) function_call_cache_control = function_call.get("cache_control") if function_call_cache_control: - chat_completion_response_message["cache_control"] = cast( + chat_completion_response_message["cache_control"] = cast( # cast-ok: Responses input uses the shared cache-control shape "ChatCompletionCachedContent", function_call_cache_control - ) # cast-ok: Responses input uses the shared cache-control shape + ) return [chat_completion_response_message] From 0b9065b6d1b7aa85b1db09cc8a86f2dae166386e Mon Sep 17 00:00:00 2001 From: TobiMayr Date: Wed, 30 Sep 2026 12:07:56 +0200 Subject: [PATCH 8/8] style: format cache control type conversion --- .../litellm_completion_transformation/transformation.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 84795d3fed6..9336c8f305a 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1759,8 +1759,10 @@ class LiteLLMCompletionResponsesConfig: ) function_call_cache_control = function_call.get("cache_control") if function_call_cache_control: - chat_completion_response_message["cache_control"] = cast( # cast-ok: Responses input uses the shared cache-control shape - "ChatCompletionCachedContent", function_call_cache_control + chat_completion_response_message["cache_control"] = ( + cast( # cast-ok: Responses input uses the shared cache-control shape + "ChatCompletionCachedContent", function_call_cache_control + ) ) return [chat_completion_response_message]