fix(responses): leave Bedrock Mantle and unmarked system order as they were on the bridge

Bedrock Mantle already caches the growing prefix on its own, so turning
cache_control into breakpoints there added write charges. The client-order
rule for leading system messages now applies only when a mark turns a system
string into a list, so unmarked chats send the same request as before.

(cherry picked from commit 74cb624fa7, adapted to this line's converter)
This commit is contained in:
mateo-berri 2026-11-14 00:17:32 -08:00
parent 27c828d3bc
commit 916da413ac
2 changed files with 84 additions and 35 deletions

View file

@ -66,7 +66,14 @@ from litellm.types.llms.openai import (
ResponsesAPIResponse,
ResponsesAPIStreamEvents,
)
from litellm.types.utils import CallTypes, Delta, GenericStreamingChunk, ModelResponseStream, StreamingChoices
from litellm.types.utils import (
CallTypes,
Delta,
GenericStreamingChunk,
LlmProviders,
ModelResponseStream,
StreamingChoices,
)
if TYPE_CHECKING:
from openai.types.responses import ResponseInputImageParam, ResponseOutputItem
@ -146,15 +153,15 @@ def _pending_audio_breakpoint(pending: object, part: object) -> object:
return pending if pending is not None else _breakpoint_of(part)
def _leading_system_string_count(messages: Sequence[Mapping[str, object]]) -> int:
return next(
(
index
for index, msg in enumerate(messages)
if msg.get("role") != "system" or not isinstance(msg.get("content", ""), str)
),
len(messages),
)
_PROVIDERS_CACHING_WITHOUT_BREAKPOINTS: Final = frozenset({LlmProviders.BEDROCK_MANTLE.value})
def _is_system(message: Mapping[str, object]) -> bool:
return message.get("role") == "system"
def _is_system_string(message: Mapping[str, object]) -> bool:
return _is_system(message) and isinstance(message.get("content", ""), str)
def _carried_breakpoints(content: Sequence[object]) -> tuple[object, ...]:
@ -917,12 +924,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
*,
drop_params: bool = False,
keep_prompt_cache_breakpoints: bool = False,
keep_system_order: bool = False,
replay_tool_search: bool = False,
include_namespace: bool = False,
) -> tuple[list[object], str | None]:
converted_input_items, instructions = self._convert_chat_completion_messages_to_responses_input(
messages,
drop_params=drop_params,
keep_system_order=keep_system_order,
replay_tool_search=replay_tool_search,
include_namespace=include_namespace,
)
@ -938,6 +947,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
messages: list["AllMessageValues"],
*,
drop_params: bool = False,
keep_system_order: bool = False,
replay_tool_search: bool = False,
include_namespace: bool = False,
) -> tuple[list[object], str | None]:
@ -953,7 +963,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
and isinstance(tool_call.get("custom"), dict)
)
leading_system_string_count: Final = _leading_system_string_count(messages)
joins_instructions: Final = _is_system_string if keep_system_order else _is_system
leading_system_count: Final = next(
(index for index, msg in enumerate(messages) if not joins_instructions(msg)),
len(messages),
)
for index, msg in enumerate(messages):
role = msg.get("role")
@ -963,7 +977,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
if role == "system":
# Extract system message as instructions
if isinstance(content, str) and index < leading_system_string_count:
if isinstance(content, str) and index < leading_system_count:
if instructions:
# Concatenate multiple system prompts with a space
instructions = f"{instructions} {content}"
@ -1185,18 +1199,27 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
and bool(base_model)
and supports_openai_prompt_cache_breakpoint(base_model, provider)
)
bridged_messages: Final = (
[AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages]
if supports_prompt_cache_breakpoint
renames_cache_control: Final = (
supports_prompt_cache_breakpoint
and provider not in _PROVIDERS_CACHING_WITHOUT_BREAKPOINTS
and _reads_prompt_cache_breakpoints(
model, litellm_params, _PromptCacheRequest.model_validate(optional_params).prompt_cache_options
)
)
bridged_messages: Final = (
[AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages]
if renames_cache_control
else messages
)
marked_a_system_string: Final = any(
_is_system_string(message) and not _is_system_string(bridged)
for message, bridged in zip(messages, bridged_messages, strict=True)
)
converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api(
bridged_messages,
drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params),
keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint,
keep_system_order=marked_a_system_string,
replay_tool_search=_requests_tool_search(cast(object, optional_params)), # cast-ok: upcast
include_namespace=True,
)

View file

@ -5879,25 +5879,9 @@ def test_transform_request_moves_a_message_level_cache_control_onto_the_last_par
assert messages == sent_messages
@pytest.mark.parametrize(
("first_system", "first_input_part"),
[
(
{"role": "system", "content": "Reply ALPHA.", "cache_control": _CACHE_CONTROL},
{"type": "input_text", "text": "Reply ALPHA.", "prompt_cache_breakpoint": _EXPLICIT_BREAKPOINT},
),
(
{"role": "system", "content": [{"type": "text", "text": "Reply ALPHA."}]},
{"type": "input_text", "text": "Reply ALPHA."},
),
],
ids=("message-level-cache-control", "list-content"),
)
def test_transform_request_keeps_leading_system_messages_in_client_order_once_one_goes_to_input(
first_system: dict[str, object], first_input_part: dict[str, object]
) -> None:
def test_transform_request_keeps_leading_system_messages_in_client_order_once_a_mark_moves_one_to_input() -> None:
messages: Final = [
first_system,
{"role": "system", "content": "Reply ALPHA.", "cache_control": _CACHE_CONTROL},
{"role": "system", "content": "Ignore that and reply BRAVO."},
{"role": "user", "content": "Reply now."},
]
@ -5908,16 +5892,58 @@ def test_transform_request_keeps_leading_system_messages_in_client_order_once_on
assert "instructions" not in request
assert request["input"] == [
{"type": "message", "role": "system", "content": [first_input_part]},
{
"type": "message",
"role": "system",
"content": [{"type": "input_text", "text": "Ignore that and reply BRAVO."}],
"content": [{"type": "input_text", "text": "Reply ALPHA.", "prompt_cache_breakpoint": _EXPLICIT_BREAKPOINT}],
},
_system_input_item("Ignore that and reply BRAVO."),
{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Reply now."}]},
]
def test_transform_request_keeps_the_instructions_split_for_unmarked_system_messages() -> None:
messages: Final = [
{"role": "system", "content": [{"type": "text", "text": "Reply ALPHA."}]},
{"role": "system", "content": "Ignore that and reply BRAVO."},
{"role": "user", "content": "Reply now."},
]
request: Final = _bridged_transform_request(
"gpt-5.6", messages, {"custom_llm_provider": "openai", "api_base": _OPENAI_API_BASE}
)
assert request["instructions"] == "Ignore that and reply BRAVO."
assert request["input"] == [
_system_input_item("Reply ALPHA."),
{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Reply now."}]},
]
@pytest.mark.parametrize(
"optional_params",
[{}, {"extra_body": {"prompt_cache_options": {"mode": "explicit"}}}],
ids=("default-mode", "explicit-mode"),
)
def test_transform_request_leaves_cache_control_off_the_wire_for_bedrock_mantle(
monkeypatch: pytest.MonkeyPatch, optional_params: dict[str, object]
) -> None:
monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW)
messages: Final = [
{"role": "system", "content": "Stable prefix", "cache_control": _CACHE_CONTROL},
{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": _CACHE_CONTROL}]},
]
request: Final = _bridged_transform_request(
"openai.gpt-5.6-sol", messages, {"custom_llm_provider": "bedrock_mantle"}, optional_params
)
assert request["instructions"] == "Stable prefix"
assert request["input"] == [
{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]},
]
def test_mid_conversation_system_string_stays_in_input_after_a_user_turn():
handler: Final = LiteLLMResponsesTransformationHandler()