fix(responses): leave Bedrock Mantle and unmarked system order as they were on the bridge

Bedrock Mantle already caches the growing prefix on its own, so turning
cache_control into breakpoints there added write charges. The client-order
rule for leading system messages now applies only when a mark turns a system
string into a list, so unmarked chats send the same request as before.
This commit is contained in:
mateo-berri 2026-11-13 23:56:47 -08:00
parent 0db932802e
commit 74cb624fa7
2 changed files with 84 additions and 33 deletions

View file

@ -89,7 +89,14 @@ from litellm.types.llms.openai import (
ResponsesAPIResponse,
ResponsesAPIStreamEvents,
)
from litellm.types.utils import CallTypes, Delta, GenericStreamingChunk, ModelResponseStream, StreamingChoices
from litellm.types.utils import (
CallTypes,
Delta,
GenericStreamingChunk,
LlmProviders,
ModelResponseStream,
StreamingChoices,
)
if TYPE_CHECKING:
from openai.types.responses import ResponseInputImageParam, ResponseOutputItem
@ -261,8 +268,15 @@ def _reads_prompt_cache_breakpoints(
)
_PROVIDERS_CACHING_WITHOUT_BREAKPOINTS: Final = frozenset({LlmProviders.BEDROCK_MANTLE.value})
def _is_system(message: Mapping[str, object]) -> bool:
return message.get("role") == "system"
def _is_system_string(message: Mapping[str, object]) -> bool:
return message.get("role") == "system" and isinstance(message.get("content", ""), str)
return _is_system(message) and isinstance(message.get("content", ""), str)
def _drops_audio_input(model: str, litellm_params: Mapping[str, object]) -> bool:
@ -1001,6 +1015,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
drop_params: bool = False,
target_trims_breakpoints: bool = True,
keep_prompt_cache_breakpoints: bool = False,
keep_system_order: bool = False,
replay_tool_search: bool = False,
include_namespace: bool = False,
replay_openai_history: bool = False,
@ -1009,6 +1024,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
messages,
drop_params=drop_params,
target_trims_breakpoints=target_trims_breakpoints,
keep_system_order=keep_system_order,
replay_tool_search=replay_tool_search,
include_namespace=include_namespace,
replay_openai_history=replay_openai_history,
@ -1026,6 +1042,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
*,
drop_params: bool = False,
target_trims_breakpoints: bool = True,
keep_system_order: bool = False,
replay_tool_search: bool = False,
include_namespace: bool = False,
replay_openai_history: bool = False,
@ -1049,8 +1066,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
and isinstance(tool_call.get("custom"), dict)
)
leading_system_string_count: Final = next(
(index for index, msg in enumerate(messages) if not _is_system_string(msg)),
joins_instructions: Final = _is_system_string if keep_system_order else _is_system
leading_system_count: Final = next(
(index for index, msg in enumerate(messages) if not joins_instructions(msg)),
len(messages),
)
@ -1064,7 +1082,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
input_items.extend(replayed_turn.items)
elif role == "system":
# Extract system message as instructions
if isinstance(content, str) and index < leading_system_string_count:
if isinstance(content, str) and index < leading_system_count:
if instructions:
# Concatenate multiple system prompts with a space
instructions = f"{instructions} {content}"
@ -1298,19 +1316,28 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
supports_prompt_cache_breakpoint: Final = any(
supports_openai_prompt_cache_breakpoint(name, provider, model_info) for name in named_models
)
bridged_messages: Final = (
[AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages]
if supports_prompt_cache_breakpoint
renames_cache_control: Final = (
supports_prompt_cache_breakpoint
and provider not in _PROVIDERS_CACHING_WITHOUT_BREAKPOINTS
and _reads_prompt_cache_breakpoints(
model, litellm_params, _PromptCacheRequest.model_validate(optional_params).prompt_cache_options
)
)
bridged_messages: Final = (
[AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages]
if renames_cache_control
else messages
)
marked_a_system_string: Final = any(
_is_system_string(message) and not _is_system_string(bridged)
for message, bridged in zip(messages, bridged_messages, strict=True)
)
converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api(
bridged_messages,
drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params),
target_trims_breakpoints=drops_unsupported_params(litellm_params.get("drop_params")),
keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint,
keep_system_order=marked_a_system_string,
replay_tool_search=_requests_tool_search(cast(object, optional_params)), # cast-ok: upcast
include_namespace=True,
replay_openai_history=provider == "openai",

View file

@ -6464,31 +6464,9 @@ def test_transform_request_moves_a_message_level_cache_control_onto_the_last_par
assert messages == sent_messages
@pytest.mark.parametrize(
("first_system", "first_system_item"),
[
(
{"role": "system", "content": "Reply ALPHA.", "cache_control": _CACHE_CONTROL},
{
"type": "message",
"role": "system",
"content": [
{"type": "input_text", "text": "Reply ALPHA.", "prompt_cache_breakpoint": _EXPLICIT_BREAKPOINT}
],
},
),
(
{"role": "system", "content": [{"type": "text", "text": "Reply ALPHA."}]},
_system_input_item("Reply ALPHA."),
),
],
ids=("message-level-cache-control", "list-content"),
)
def test_transform_request_keeps_leading_system_messages_in_client_order_once_one_goes_to_input(
first_system: dict[str, object], first_system_item: dict[str, object]
) -> None:
def test_transform_request_keeps_leading_system_messages_in_client_order_once_a_mark_moves_one_to_input() -> None:
messages: Final = [
first_system,
{"role": "system", "content": "Reply ALPHA.", "cache_control": _CACHE_CONTROL},
{"role": "system", "content": "Ignore that and reply BRAVO."},
{"role": "user", "content": "Reply now."},
]
@ -6499,12 +6477,58 @@ def test_transform_request_keeps_leading_system_messages_in_client_order_once_on
assert "instructions" not in request
assert request["input"] == [
first_system_item,
{
"type": "message",
"role": "system",
"content": [{"type": "input_text", "text": "Reply ALPHA.", "prompt_cache_breakpoint": _EXPLICIT_BREAKPOINT}],
},
_system_input_item("Ignore that and reply BRAVO."),
{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Reply now."}]},
]
def test_transform_request_keeps_the_instructions_split_for_unmarked_system_messages() -> None:
messages: Final = [
{"role": "system", "content": [{"type": "text", "text": "Reply ALPHA."}]},
{"role": "system", "content": "Ignore that and reply BRAVO."},
{"role": "user", "content": "Reply now."},
]
request: Final = _bridged_transform_request(
"gpt-5.6", messages, {"custom_llm_provider": "openai", "api_base": _OPENAI_API_BASE}
)
assert request["instructions"] == "Ignore that and reply BRAVO."
assert request["input"] == [
_system_input_item("Reply ALPHA."),
{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Reply now."}]},
]
@pytest.mark.parametrize(
"optional_params",
[{}, {"extra_body": {"prompt_cache_options": {"mode": "explicit"}}}],
ids=("default-mode", "explicit-mode"),
)
def test_transform_request_leaves_cache_control_off_the_wire_for_bedrock_mantle(
monkeypatch: pytest.MonkeyPatch, optional_params: dict[str, object]
) -> None:
monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW)
messages: Final = [
{"role": "system", "content": "Stable prefix", "cache_control": _CACHE_CONTROL},
{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": _CACHE_CONTROL}]},
]
request: Final = _bridged_transform_request(
"openai.gpt-5.6-sol", messages, {"custom_llm_provider": "bedrock_mantle"}, optional_params
)
assert request["instructions"] == "Stable prefix"
assert request["input"] == [
{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]},
]
def test_mid_conversation_system_string_stays_in_input_after_a_user_turn():
handler: Final = LiteLLMResponsesTransformationHandler()