From 74cb624fa79aa3a84577f68d7d65d164435055dc Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:56:47 -0800 Subject: [PATCH] fix(responses): leave Bedrock Mantle and unmarked system order as they were on the bridge Bedrock Mantle already caches the growing prefix on its own, so turning cache_control into breakpoints there added write charges. The client-order rule for leading system messages now applies only when a mark turns a system string into a list, so unmarked chats send the same request as before. --- .../transformation.py | 43 +++++++++-- ...responses_transformation_transformation.py | 74 ++++++++++++------- 2 files changed, 84 insertions(+), 33 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index c37a666071b..c491f493670 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -89,7 +89,14 @@ from litellm.types.llms.openai import ( ResponsesAPIResponse, ResponsesAPIStreamEvents, ) -from litellm.types.utils import CallTypes, Delta, GenericStreamingChunk, ModelResponseStream, StreamingChoices +from litellm.types.utils import ( + CallTypes, + Delta, + GenericStreamingChunk, + LlmProviders, + ModelResponseStream, + StreamingChoices, +) if TYPE_CHECKING: from openai.types.responses import ResponseInputImageParam, ResponseOutputItem @@ -261,8 +268,15 @@ def _reads_prompt_cache_breakpoints( ) +_PROVIDERS_CACHING_WITHOUT_BREAKPOINTS: Final = frozenset({LlmProviders.BEDROCK_MANTLE.value}) + + +def _is_system(message: Mapping[str, object]) -> bool: + return message.get("role") == "system" + + def _is_system_string(message: Mapping[str, object]) -> bool: - return message.get("role") == "system" and isinstance(message.get("content", ""), str) + return _is_system(message) and isinstance(message.get("content", ""), str) def _drops_audio_input(model: str, litellm_params: Mapping[str, object]) -> bool: @@ -1001,6 +1015,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): drop_params: bool = False, target_trims_breakpoints: bool = True, keep_prompt_cache_breakpoints: bool = False, + keep_system_order: bool = False, replay_tool_search: bool = False, include_namespace: bool = False, replay_openai_history: bool = False, @@ -1009,6 +1024,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): messages, drop_params=drop_params, target_trims_breakpoints=target_trims_breakpoints, + keep_system_order=keep_system_order, replay_tool_search=replay_tool_search, include_namespace=include_namespace, replay_openai_history=replay_openai_history, @@ -1026,6 +1042,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): *, drop_params: bool = False, target_trims_breakpoints: bool = True, + keep_system_order: bool = False, replay_tool_search: bool = False, include_namespace: bool = False, replay_openai_history: bool = False, @@ -1049,8 +1066,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): and isinstance(tool_call.get("custom"), dict) ) - leading_system_string_count: Final = next( - (index for index, msg in enumerate(messages) if not _is_system_string(msg)), + joins_instructions: Final = _is_system_string if keep_system_order else _is_system + leading_system_count: Final = next( + (index for index, msg in enumerate(messages) if not joins_instructions(msg)), len(messages), ) @@ -1064,7 +1082,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): input_items.extend(replayed_turn.items) elif role == "system": # Extract system message as instructions - if isinstance(content, str) and index < leading_system_string_count: + if isinstance(content, str) and index < leading_system_count: if instructions: # Concatenate multiple system prompts with a space instructions = f"{instructions} {content}" @@ -1298,19 +1316,28 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): supports_prompt_cache_breakpoint: Final = any( supports_openai_prompt_cache_breakpoint(name, provider, model_info) for name in named_models ) - bridged_messages: Final = ( - [AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages] - if supports_prompt_cache_breakpoint + renames_cache_control: Final = ( + supports_prompt_cache_breakpoint + and provider not in _PROVIDERS_CACHING_WITHOUT_BREAKPOINTS and _reads_prompt_cache_breakpoints( model, litellm_params, _PromptCacheRequest.model_validate(optional_params).prompt_cache_options ) + ) + bridged_messages: Final = ( + [AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages] + if renames_cache_control else messages ) + marked_a_system_string: Final = any( + _is_system_string(message) and not _is_system_string(bridged) + for message, bridged in zip(messages, bridged_messages, strict=True) + ) converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api( bridged_messages, drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params), target_trims_breakpoints=drops_unsupported_params(litellm_params.get("drop_params")), keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint, + keep_system_order=marked_a_system_string, replay_tool_search=_requests_tool_search(cast(object, optional_params)), # cast-ok: upcast include_namespace=True, replay_openai_history=provider == "openai", diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 0cba58aa8e4..8f29465a2f6 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -6464,31 +6464,9 @@ def test_transform_request_moves_a_message_level_cache_control_onto_the_last_par assert messages == sent_messages -@pytest.mark.parametrize( - ("first_system", "first_system_item"), - [ - ( - {"role": "system", "content": "Reply ALPHA.", "cache_control": _CACHE_CONTROL}, - { - "type": "message", - "role": "system", - "content": [ - {"type": "input_text", "text": "Reply ALPHA.", "prompt_cache_breakpoint": _EXPLICIT_BREAKPOINT} - ], - }, - ), - ( - {"role": "system", "content": [{"type": "text", "text": "Reply ALPHA."}]}, - _system_input_item("Reply ALPHA."), - ), - ], - ids=("message-level-cache-control", "list-content"), -) -def test_transform_request_keeps_leading_system_messages_in_client_order_once_one_goes_to_input( - first_system: dict[str, object], first_system_item: dict[str, object] -) -> None: +def test_transform_request_keeps_leading_system_messages_in_client_order_once_a_mark_moves_one_to_input() -> None: messages: Final = [ - first_system, + {"role": "system", "content": "Reply ALPHA.", "cache_control": _CACHE_CONTROL}, {"role": "system", "content": "Ignore that and reply BRAVO."}, {"role": "user", "content": "Reply now."}, ] @@ -6499,12 +6477,58 @@ def test_transform_request_keeps_leading_system_messages_in_client_order_once_on assert "instructions" not in request assert request["input"] == [ - first_system_item, + { + "type": "message", + "role": "system", + "content": [{"type": "input_text", "text": "Reply ALPHA.", "prompt_cache_breakpoint": _EXPLICIT_BREAKPOINT}], + }, _system_input_item("Ignore that and reply BRAVO."), {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Reply now."}]}, ] +def test_transform_request_keeps_the_instructions_split_for_unmarked_system_messages() -> None: + messages: Final = [ + {"role": "system", "content": [{"type": "text", "text": "Reply ALPHA."}]}, + {"role": "system", "content": "Ignore that and reply BRAVO."}, + {"role": "user", "content": "Reply now."}, + ] + + request: Final = _bridged_transform_request( + "gpt-5.6", messages, {"custom_llm_provider": "openai", "api_base": _OPENAI_API_BASE} + ) + + assert request["instructions"] == "Ignore that and reply BRAVO." + assert request["input"] == [ + _system_input_item("Reply ALPHA."), + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Reply now."}]}, + ] + + +@pytest.mark.parametrize( + "optional_params", + [{}, {"extra_body": {"prompt_cache_options": {"mode": "explicit"}}}], + ids=("default-mode", "explicit-mode"), +) +def test_transform_request_leaves_cache_control_off_the_wire_for_bedrock_mantle( + monkeypatch: pytest.MonkeyPatch, optional_params: dict[str, object] +) -> None: + monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) + messages: Final = [ + {"role": "system", "content": "Stable prefix", "cache_control": _CACHE_CONTROL}, + {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": _CACHE_CONTROL}]}, + ] + + request: Final = _bridged_transform_request( + "openai.gpt-5.6-sol", messages, {"custom_llm_provider": "bedrock_mantle"}, optional_params + ) + + assert request["instructions"] == "Stable prefix" + assert request["input"] == [ + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + ] + + def test_mid_conversation_system_string_stays_in_input_after_a_user_turn(): handler: Final = LiteLLMResponsesTransformationHandler()