scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-15 09:12:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 311

fix(responses): keep cache breakpoints on requests that reach the Responses API

Browse source 
The chat-completions bridge rebuilt every content block without the
prompt_cache_breakpoint marker the cache-control hook had just placed on it, so
the request went out with prompt_cache_options set to explicit mode and nothing
actually marked. Explicit mode caches only what is marked, so those deployments
lost the implicit caching they were getting before the injection point was added

The native surface had a second miss. The bridging check looked the provider
config up with the still-prefixed model while the config keys on the bare one,
so every bedrock_mantle model read as having no native Responses support, and
role-targeted injection points were deferred to a chat-completions pass that
never runs for them
This commit is contained in:
mateo-berri 2026-09-06 01:19:58 -07:00
parent b9f5cd6036
commit 833f4393ce
4 changed files with 177 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -23,6 +23,7 @@ from litellm import ModelResponse
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.common_utils import (
responses_reasoning_item_from_thinking_blocks,
with_prompt_cache_breakpoint,
)
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.bridges.completion_transformation import (
@ -1003,16 +1004,22 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
# Handle multimodal content
original_type = item.get("type")
if original_type == "text":
converted = self._convert_content_str_to_input_text(item.get("text", ""), role)
converted = with_prompt_cache_breakpoint(
self._convert_content_str_to_input_text(item.get("text", ""), role),
item.get("prompt_cache_breakpoint"),
)
result.append(converted)
verbose_logger.debug("Chat provider: text -> %s", converted)
elif original_type == "image_url":
# Map to responses API image format
converted = cast(
dict,
self._convert_content_to_responses_format_image(
cast(ChatCompletionImageObject, item), role
converted = with_prompt_cache_breakpoint(
cast(
dict,
self._convert_content_to_responses_format_image(
cast(ChatCompletionImageObject, item), role
),
),
item.get("prompt_cache_breakpoint"),
)
result.append(converted)
verbose_logger.debug("Chat provider: image_url -> %s", converted)
@ -1024,8 +1031,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
result.append(converted)
verbose_logger.debug("Chat provider: image -> %s", converted)
elif item_type == "file":
converted = _input_file_from_file_value(
cast("ChatCompletionFileObject", item).get("file"), # cast-ok: type tag checked
converted = with_prompt_cache_breakpoint(
_input_file_from_file_value(
cast("ChatCompletionFileObject", item).get("file"), # cast-ok: type tag checked
),
item.get("prompt_cache_breakpoint"),
)
result.append(converted)
verbose_logger.debug("Chat provider: file -> %s", converted)

15
litellm/responses/main.py
View file

@ -437,6 +437,15 @@ def _resolve_responses_api_provider_config(
return OpenAILikeResponsesConfig()
def _responses_config_lookup_model(model: str, custom_llm_provider: str) -> str:
"""The model id the dispatch hands the config lookup, with the routing prefixes off.
A config keyed on the model, such as bedrock_mantle reading its cost-map entry, misses on
the prefixed id, so every one of those models reads as having no native Responses support.
"""
return _strip_responses_routing_prefix(model.removeprefix(f"{custom_llm_provider}/"))
def _will_bridge_to_chat_completions(
model: str, custom_llm_provider: str | None, use_chat_completions_api: bool, model_info: object
) -> bool:
@ -452,7 +461,11 @@ def _will_bridge_to_chat_completions(
if custom_llm_provider is None:
return True
return _bridges_to_chat_completions(
_resolve_responses_api_provider_config(normalized_model[0], custom_llm_provider, model_info),
_resolve_responses_api_provider_config(
_responses_config_lookup_model(normalized_model[0], custom_llm_provider),
custom_llm_provider,
model_info,
),
use_chat_completions_api or normalized_model[1],
)

63
tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
View file

@ -3952,3 +3952,66 @@ def test_convert_chat_completion_messages_to_responses_api_tool_result_with_tool
function_call_output = next(item for item in response if item.get("type") == "function_call_output")
assert function_call_output["output"] == [{"type": "input_text", "text": "1 tool found"}]
def test_convert_chat_completion_messages_to_responses_api_keeps_prompt_cache_breakpoint():
"""The OpenAI-dialect cache marker has to survive the rebuild into Responses input blocks.
The cache-control hook marks the block it wants cached before the bridge runs, so dropping
the marker here sends `prompt_cache_options: {"mode": "explicit"}` with nothing marked, which
turns off the implicit caching the request would otherwise have gotten.
"""
from litellm.completion_extras.litellm_responses_transformation.transformation import (
LiteLLMResponsesTransformationHandler,
)
handler = LiteLLMResponsesTransformationHandler()
breakpoint_marker = {"mode": "explicit"}
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "cache up to here", "prompt_cache_breakpoint": breakpoint_marker},
{
"type": "image_url",
"image_url": {"url": "https://example.com/image.png"},
"prompt_cache_breakpoint": breakpoint_marker,
},
{
"type": "file",
"file": {"file_id": "file-abc123", "filename": "manual.pdf"},
"prompt_cache_breakpoint": breakpoint_marker,
},
],
},
]
response, _ = handler.convert_chat_completion_messages_to_responses_api(messages)
content = response[0]["content"]
assert [block["type"] for block in content] == ["input_text", "input_image", "input_file"]
assert [block.get("prompt_cache_breakpoint") for block in content] == [breakpoint_marker] * 3
def test_convert_chat_completion_messages_to_responses_api_omits_absent_prompt_cache_breakpoint():
"""Unmarked blocks must not grow the key, or every request would look explicitly cached."""
from litellm.completion_extras.litellm_responses_transformation.transformation import (
LiteLLMResponsesTransformationHandler,
)
handler = LiteLLMResponsesTransformationHandler()
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "no caching here"},
{"type": "image_url", "image_url": {"url": "https://example.com/image.png"}},
],
},
]
response, _ = handler.convert_chat_completion_messages_to_responses_api(messages)
assert all("prompt_cache_breakpoint" not in block for block in response[0]["content"])

83
tests/test_litellm/responses/test_responses_prompt_management.py
View file

@ -82,6 +82,24 @@ def _patch_responses_dispatch():
]
def _make_hook_backed_logging_obj() -> MagicMock:
"""A logging object that runs the real cache-control hook, the way a configured proxy's does."""
hook = AnthropicCacheControlHook()
logging_obj = MagicMock()
logging_obj.__class__ = LiteLLMLoggingObj
logging_obj.should_run_prompt_management_hooks.return_value = True
logging_obj.get_chat_completion_prompt.side_effect = lambda **call: hook.get_chat_completion_prompt(
model=call["model"],
messages=call["messages"],
non_default_params=call["non_default_params"],
prompt_id=call["prompt_id"],
prompt_variables=call["prompt_variables"],
dynamic_callback_params={},
)
logging_obj.model_call_details = {}
return logging_obj
def _make_cache_control_case() -> tuple[
ResponseInputParam,
list[AllMessageValues],
@ -346,6 +364,71 @@ class TestResponsesAPIPromptManagement:
assert sent_input[1] == reasoning_item
assert sent_input[2]["id"] == "msg_1"
def test_native_responses_provider_places_role_targeted_cache_point(self):
"""A provider serving Responses natively gets no second pass, so the point lands on this one.
Handing a role-targeted point forward here sends it to the chat-completions bridge, which
never runs for this request, and the call goes upstream with nothing marked for caching.
"""
system_message = cast(AllMessageValues, {"role": "system", "content": "Analyze the request"})
user_message = cast(AllMessageValues, {"role": "user", "content": "Check for security issues"})
with (
patch.object(
import_module("litellm.responses.mcp.litellm_proxy_mcp_handler").LiteLLM_Proxy_MCP_Handler,
"_should_use_litellm_mcp_gateway",
return_value=False,
),
patch.object(
import_module("litellm.responses.main").base_llm_http_handler,
"response_api_handler",
return_value=MagicMock(),
) as mock_handler,
):
import litellm
litellm.responses(
input=cast(ResponseInputParam, [system_message, user_message]),
model="bedrock_mantle/openai.gpt-5.6-sol",
litellm_logging_obj=_make_hook_backed_logging_obj(),
cache_control_injection_points=[{"location": "message", "role": "system"}],
aws_region_name="us-east-1",
)
sent_input = mock_handler.call_args.kwargs["input"]
assert sent_input[0]["cache_control"] == {"type": "ephemeral"}
def test_bridged_responses_request_still_defers_role_targeted_cache_point(self):
"""The bridge builds its own system message, so its own pass is the one that marks it."""
system_message = cast(AllMessageValues, {"role": "system", "content": "Analyze the request"})
user_message = cast(AllMessageValues, {"role": "user", "content": "Check for security issues"})
with (
patch.object(
import_module("litellm.responses.mcp.litellm_proxy_mcp_handler").LiteLLM_Proxy_MCP_Handler,
"_should_use_litellm_mcp_gateway",
return_value=False,
),
patch.object(
import_module("litellm.responses.main").litellm_completion_transformation_handler,
"response_api_handler",
return_value=MagicMock(),
) as mock_handler,
):
import litellm
litellm.responses(
input=cast(ResponseInputParam, [system_message, user_message]),
model="openai/chat_completions/gpt-5.6",
litellm_logging_obj=_make_hook_backed_logging_obj(),
cache_control_injection_points=[{"location": "message", "role": "system"}],
api_key="fake-key",
)
sent_input = mock_handler.call_args.kwargs["input"]
assert "cache_control" not in sent_input[0]
def test_all_non_message_input_items_remain_unchanged(self):
reasoning_item = {
"type": "reasoning",