diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index d70f947469a..42dac6167e0 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -23,6 +23,7 @@ from litellm import ModelResponse from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( responses_reasoning_item_from_thinking_blocks, + with_prompt_cache_breakpoint, ) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.bridges.completion_transformation import ( @@ -1003,16 +1004,22 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): # Handle multimodal content original_type = item.get("type") if original_type == "text": - converted = self._convert_content_str_to_input_text(item.get("text", ""), role) + converted = with_prompt_cache_breakpoint( + self._convert_content_str_to_input_text(item.get("text", ""), role), + item.get("prompt_cache_breakpoint"), + ) result.append(converted) verbose_logger.debug("Chat provider: text -> %s", converted) elif original_type == "image_url": # Map to responses API image format - converted = cast( - dict, - self._convert_content_to_responses_format_image( - cast(ChatCompletionImageObject, item), role + converted = with_prompt_cache_breakpoint( + cast( + dict, + self._convert_content_to_responses_format_image( + cast(ChatCompletionImageObject, item), role + ), ), + item.get("prompt_cache_breakpoint"), ) result.append(converted) verbose_logger.debug("Chat provider: image_url -> %s", converted) @@ -1024,8 +1031,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): result.append(converted) verbose_logger.debug("Chat provider: image -> %s", converted) elif item_type == "file": - converted = _input_file_from_file_value( - cast("ChatCompletionFileObject", item).get("file"), # cast-ok: type tag checked + converted = with_prompt_cache_breakpoint( + _input_file_from_file_value( + cast("ChatCompletionFileObject", item).get("file"), # cast-ok: type tag checked + ), + item.get("prompt_cache_breakpoint"), ) result.append(converted) verbose_logger.debug("Chat provider: file -> %s", converted) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 5e74b7324b4..24f7abd13b4 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -437,6 +437,15 @@ def _resolve_responses_api_provider_config( return OpenAILikeResponsesConfig() +def _responses_config_lookup_model(model: str, custom_llm_provider: str) -> str: + """The model id the dispatch hands the config lookup, with the routing prefixes off. + + A config keyed on the model, such as bedrock_mantle reading its cost-map entry, misses on + the prefixed id, so every one of those models reads as having no native Responses support. + """ + return _strip_responses_routing_prefix(model.removeprefix(f"{custom_llm_provider}/")) + + def _will_bridge_to_chat_completions( model: str, custom_llm_provider: str | None, use_chat_completions_api: bool, model_info: object ) -> bool: @@ -452,7 +461,11 @@ def _will_bridge_to_chat_completions( if custom_llm_provider is None: return True return _bridges_to_chat_completions( - _resolve_responses_api_provider_config(normalized_model[0], custom_llm_provider, model_info), + _resolve_responses_api_provider_config( + _responses_config_lookup_model(normalized_model[0], custom_llm_provider), + custom_llm_provider, + model_info, + ), use_chat_completions_api or normalized_model[1], ) diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 6590718878d..eeb951ce2da 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -3952,3 +3952,66 @@ def test_convert_chat_completion_messages_to_responses_api_tool_result_with_tool function_call_output = next(item for item in response if item.get("type") == "function_call_output") assert function_call_output["output"] == [{"type": "input_text", "text": "1 tool found"}] + + +def test_convert_chat_completion_messages_to_responses_api_keeps_prompt_cache_breakpoint(): + """The OpenAI-dialect cache marker has to survive the rebuild into Responses input blocks. + + The cache-control hook marks the block it wants cached before the bridge runs, so dropping + the marker here sends `prompt_cache_options: {"mode": "explicit"}` with nothing marked, which + turns off the implicit caching the request would otherwise have gotten. + """ + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + handler = LiteLLMResponsesTransformationHandler() + breakpoint_marker = {"mode": "explicit"} + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "cache up to here", "prompt_cache_breakpoint": breakpoint_marker}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/image.png"}, + "prompt_cache_breakpoint": breakpoint_marker, + }, + { + "type": "file", + "file": {"file_id": "file-abc123", "filename": "manual.pdf"}, + "prompt_cache_breakpoint": breakpoint_marker, + }, + ], + }, + ] + + response, _ = handler.convert_chat_completion_messages_to_responses_api(messages) + + content = response[0]["content"] + assert [block["type"] for block in content] == ["input_text", "input_image", "input_file"] + assert [block.get("prompt_cache_breakpoint") for block in content] == [breakpoint_marker] * 3 + + +def test_convert_chat_completion_messages_to_responses_api_omits_absent_prompt_cache_breakpoint(): + """Unmarked blocks must not grow the key, or every request would look explicitly cached.""" + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + handler = LiteLLMResponsesTransformationHandler() + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "no caching here"}, + {"type": "image_url", "image_url": {"url": "https://example.com/image.png"}}, + ], + }, + ] + + response, _ = handler.convert_chat_completion_messages_to_responses_api(messages) + + assert all("prompt_cache_breakpoint" not in block for block in response[0]["content"]) diff --git a/tests/test_litellm/responses/test_responses_prompt_management.py b/tests/test_litellm/responses/test_responses_prompt_management.py index 530afbd856b..6f8faa050dd 100644 --- a/tests/test_litellm/responses/test_responses_prompt_management.py +++ b/tests/test_litellm/responses/test_responses_prompt_management.py @@ -82,6 +82,24 @@ def _patch_responses_dispatch(): ] +def _make_hook_backed_logging_obj() -> MagicMock: + """A logging object that runs the real cache-control hook, the way a configured proxy's does.""" + hook = AnthropicCacheControlHook() + logging_obj = MagicMock() + logging_obj.__class__ = LiteLLMLoggingObj + logging_obj.should_run_prompt_management_hooks.return_value = True + logging_obj.get_chat_completion_prompt.side_effect = lambda **call: hook.get_chat_completion_prompt( + model=call["model"], + messages=call["messages"], + non_default_params=call["non_default_params"], + prompt_id=call["prompt_id"], + prompt_variables=call["prompt_variables"], + dynamic_callback_params={}, + ) + logging_obj.model_call_details = {} + return logging_obj + + def _make_cache_control_case() -> tuple[ ResponseInputParam, list[AllMessageValues], @@ -346,6 +364,71 @@ class TestResponsesAPIPromptManagement: assert sent_input[1] == reasoning_item assert sent_input[2]["id"] == "msg_1" + def test_native_responses_provider_places_role_targeted_cache_point(self): + """A provider serving Responses natively gets no second pass, so the point lands on this one. + + Handing a role-targeted point forward here sends it to the chat-completions bridge, which + never runs for this request, and the call goes upstream with nothing marked for caching. + """ + system_message = cast(AllMessageValues, {"role": "system", "content": "Analyze the request"}) + user_message = cast(AllMessageValues, {"role": "user", "content": "Check for security issues"}) + + with ( + patch.object( + import_module("litellm.responses.mcp.litellm_proxy_mcp_handler").LiteLLM_Proxy_MCP_Handler, + "_should_use_litellm_mcp_gateway", + return_value=False, + ), + patch.object( + import_module("litellm.responses.main").base_llm_http_handler, + "response_api_handler", + return_value=MagicMock(), + ) as mock_handler, + ): + import litellm + + litellm.responses( + input=cast(ResponseInputParam, [system_message, user_message]), + model="bedrock_mantle/openai.gpt-5.6-sol", + litellm_logging_obj=_make_hook_backed_logging_obj(), + cache_control_injection_points=[{"location": "message", "role": "system"}], + aws_region_name="us-east-1", + ) + + sent_input = mock_handler.call_args.kwargs["input"] + assert sent_input[0]["cache_control"] == {"type": "ephemeral"} + + def test_bridged_responses_request_still_defers_role_targeted_cache_point(self): + """The bridge builds its own system message, so its own pass is the one that marks it.""" + system_message = cast(AllMessageValues, {"role": "system", "content": "Analyze the request"}) + user_message = cast(AllMessageValues, {"role": "user", "content": "Check for security issues"}) + + with ( + patch.object( + import_module("litellm.responses.mcp.litellm_proxy_mcp_handler").LiteLLM_Proxy_MCP_Handler, + "_should_use_litellm_mcp_gateway", + return_value=False, + ), + patch.object( + import_module("litellm.responses.main").litellm_completion_transformation_handler, + "response_api_handler", + return_value=MagicMock(), + ) as mock_handler, + ): + import litellm + + litellm.responses( + input=cast(ResponseInputParam, [system_message, user_message]), + model="openai/chat_completions/gpt-5.6", + litellm_logging_obj=_make_hook_backed_logging_obj(), + cache_control_injection_points=[{"location": "message", "role": "system"}], + api_key="fake-key", + ) + + sent_input = mock_handler.call_args.kwargs["input"] + assert "cache_control" not in sent_input[0] + + def test_all_non_message_input_items_remain_unchanged(self): reasoning_item = { "type": "reasoning",