From 833f4393ce31c0d5cc7d2e43d88db80272f00284 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 6 Sep 2026 01:19:58 -0700 Subject: [PATCH] fix(responses): keep cache breakpoints on requests that reach the Responses API The chat-completions bridge rebuilt every content block without the prompt_cache_breakpoint marker the cache-control hook had just placed on it, so the request went out with prompt_cache_options set to explicit mode and nothing actually marked. Explicit mode caches only what is marked, so those deployments lost the implicit caching they were getting before the injection point was added The native surface had a second miss. The bridging check looked the provider config up with the still-prefixed model while the config keys on the bare one, so every bedrock_mantle model read as having no native Responses support, and role-targeted injection points were deferred to a chat-completions pass that never runs for them --- .../transformation.py | 24 ++++-- litellm/responses/main.py | 15 +++- ...responses_transformation_transformation.py | 63 ++++++++++++++ .../test_responses_prompt_management.py | 83 +++++++++++++++++++ 4 files changed, 177 insertions(+), 8 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index d70f947469a..42dac6167e0 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -23,6 +23,7 @@ from litellm import ModelResponse from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( responses_reasoning_item_from_thinking_blocks, + with_prompt_cache_breakpoint, ) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.bridges.completion_transformation import ( @@ -1003,16 +1004,22 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): # Handle multimodal content original_type = item.get("type") if original_type == "text": - converted = self._convert_content_str_to_input_text(item.get("text", ""), role) + converted = with_prompt_cache_breakpoint( + self._convert_content_str_to_input_text(item.get("text", ""), role), + item.get("prompt_cache_breakpoint"), + ) result.append(converted) verbose_logger.debug("Chat provider: text -> %s", converted) elif original_type == "image_url": # Map to responses API image format - converted = cast( - dict, - self._convert_content_to_responses_format_image( - cast(ChatCompletionImageObject, item), role + converted = with_prompt_cache_breakpoint( + cast( + dict, + self._convert_content_to_responses_format_image( + cast(ChatCompletionImageObject, item), role + ), ), + item.get("prompt_cache_breakpoint"), ) result.append(converted) verbose_logger.debug("Chat provider: image_url -> %s", converted) @@ -1024,8 +1031,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): result.append(converted) verbose_logger.debug("Chat provider: image -> %s", converted) elif item_type == "file": - converted = _input_file_from_file_value( - cast("ChatCompletionFileObject", item).get("file"), # cast-ok: type tag checked + converted = with_prompt_cache_breakpoint( + _input_file_from_file_value( + cast("ChatCompletionFileObject", item).get("file"), # cast-ok: type tag checked + ), + item.get("prompt_cache_breakpoint"), ) result.append(converted) verbose_logger.debug("Chat provider: file -> %s", converted) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 5e74b7324b4..24f7abd13b4 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -437,6 +437,15 @@ def _resolve_responses_api_provider_config( return OpenAILikeResponsesConfig() +def _responses_config_lookup_model(model: str, custom_llm_provider: str) -> str: + """The model id the dispatch hands the config lookup, with the routing prefixes off. + + A config keyed on the model, such as bedrock_mantle reading its cost-map entry, misses on + the prefixed id, so every one of those models reads as having no native Responses support. + """ + return _strip_responses_routing_prefix(model.removeprefix(f"{custom_llm_provider}/")) + + def _will_bridge_to_chat_completions( model: str, custom_llm_provider: str | None, use_chat_completions_api: bool, model_info: object ) -> bool: @@ -452,7 +461,11 @@ def _will_bridge_to_chat_completions( if custom_llm_provider is None: return True return _bridges_to_chat_completions( - _resolve_responses_api_provider_config(normalized_model[0], custom_llm_provider, model_info), + _resolve_responses_api_provider_config( + _responses_config_lookup_model(normalized_model[0], custom_llm_provider), + custom_llm_provider, + model_info, + ), use_chat_completions_api or normalized_model[1], ) diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 6590718878d..eeb951ce2da 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -3952,3 +3952,66 @@ def test_convert_chat_completion_messages_to_responses_api_tool_result_with_tool function_call_output = next(item for item in response if item.get("type") == "function_call_output") assert function_call_output["output"] == [{"type": "input_text", "text": "1 tool found"}] + + +def test_convert_chat_completion_messages_to_responses_api_keeps_prompt_cache_breakpoint(): + """The OpenAI-dialect cache marker has to survive the rebuild into Responses input blocks. + + The cache-control hook marks the block it wants cached before the bridge runs, so dropping + the marker here sends `prompt_cache_options: {"mode": "explicit"}` with nothing marked, which + turns off the implicit caching the request would otherwise have gotten. + """ + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + handler = LiteLLMResponsesTransformationHandler() + breakpoint_marker = {"mode": "explicit"} + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "cache up to here", "prompt_cache_breakpoint": breakpoint_marker}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/image.png"}, + "prompt_cache_breakpoint": breakpoint_marker, + }, + { + "type": "file", + "file": {"file_id": "file-abc123", "filename": "manual.pdf"}, + "prompt_cache_breakpoint": breakpoint_marker, + }, + ], + }, + ] + + response, _ = handler.convert_chat_completion_messages_to_responses_api(messages) + + content = response[0]["content"] + assert [block["type"] for block in content] == ["input_text", "input_image", "input_file"] + assert [block.get("prompt_cache_breakpoint") for block in content] == [breakpoint_marker] * 3 + + +def test_convert_chat_completion_messages_to_responses_api_omits_absent_prompt_cache_breakpoint(): + """Unmarked blocks must not grow the key, or every request would look explicitly cached.""" + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + handler = LiteLLMResponsesTransformationHandler() + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "no caching here"}, + {"type": "image_url", "image_url": {"url": "https://example.com/image.png"}}, + ], + }, + ] + + response, _ = handler.convert_chat_completion_messages_to_responses_api(messages) + + assert all("prompt_cache_breakpoint" not in block for block in response[0]["content"]) diff --git a/tests/test_litellm/responses/test_responses_prompt_management.py b/tests/test_litellm/responses/test_responses_prompt_management.py index 530afbd856b..6f8faa050dd 100644 --- a/tests/test_litellm/responses/test_responses_prompt_management.py +++ b/tests/test_litellm/responses/test_responses_prompt_management.py @@ -82,6 +82,24 @@ def _patch_responses_dispatch(): ] +def _make_hook_backed_logging_obj() -> MagicMock: + """A logging object that runs the real cache-control hook, the way a configured proxy's does.""" + hook = AnthropicCacheControlHook() + logging_obj = MagicMock() + logging_obj.__class__ = LiteLLMLoggingObj + logging_obj.should_run_prompt_management_hooks.return_value = True + logging_obj.get_chat_completion_prompt.side_effect = lambda **call: hook.get_chat_completion_prompt( + model=call["model"], + messages=call["messages"], + non_default_params=call["non_default_params"], + prompt_id=call["prompt_id"], + prompt_variables=call["prompt_variables"], + dynamic_callback_params={}, + ) + logging_obj.model_call_details = {} + return logging_obj + + def _make_cache_control_case() -> tuple[ ResponseInputParam, list[AllMessageValues], @@ -346,6 +364,71 @@ class TestResponsesAPIPromptManagement: assert sent_input[1] == reasoning_item assert sent_input[2]["id"] == "msg_1" + def test_native_responses_provider_places_role_targeted_cache_point(self): + """A provider serving Responses natively gets no second pass, so the point lands on this one. + + Handing a role-targeted point forward here sends it to the chat-completions bridge, which + never runs for this request, and the call goes upstream with nothing marked for caching. + """ + system_message = cast(AllMessageValues, {"role": "system", "content": "Analyze the request"}) + user_message = cast(AllMessageValues, {"role": "user", "content": "Check for security issues"}) + + with ( + patch.object( + import_module("litellm.responses.mcp.litellm_proxy_mcp_handler").LiteLLM_Proxy_MCP_Handler, + "_should_use_litellm_mcp_gateway", + return_value=False, + ), + patch.object( + import_module("litellm.responses.main").base_llm_http_handler, + "response_api_handler", + return_value=MagicMock(), + ) as mock_handler, + ): + import litellm + + litellm.responses( + input=cast(ResponseInputParam, [system_message, user_message]), + model="bedrock_mantle/openai.gpt-5.6-sol", + litellm_logging_obj=_make_hook_backed_logging_obj(), + cache_control_injection_points=[{"location": "message", "role": "system"}], + aws_region_name="us-east-1", + ) + + sent_input = mock_handler.call_args.kwargs["input"] + assert sent_input[0]["cache_control"] == {"type": "ephemeral"} + + def test_bridged_responses_request_still_defers_role_targeted_cache_point(self): + """The bridge builds its own system message, so its own pass is the one that marks it.""" + system_message = cast(AllMessageValues, {"role": "system", "content": "Analyze the request"}) + user_message = cast(AllMessageValues, {"role": "user", "content": "Check for security issues"}) + + with ( + patch.object( + import_module("litellm.responses.mcp.litellm_proxy_mcp_handler").LiteLLM_Proxy_MCP_Handler, + "_should_use_litellm_mcp_gateway", + return_value=False, + ), + patch.object( + import_module("litellm.responses.main").litellm_completion_transformation_handler, + "response_api_handler", + return_value=MagicMock(), + ) as mock_handler, + ): + import litellm + + litellm.responses( + input=cast(ResponseInputParam, [system_message, user_message]), + model="openai/chat_completions/gpt-5.6", + litellm_logging_obj=_make_hook_backed_logging_obj(), + cache_control_injection_points=[{"location": "message", "role": "system"}], + api_key="fake-key", + ) + + sent_input = mock_handler.call_args.kwargs["input"] + assert "cache_control" not in sent_input[0] + + def test_all_non_message_input_items_remain_unchanged(self): reasoning_item = { "type": "reasoning",