From 23ce902dcf779049ccc666c7d5bd7de875f1179e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 01:37:26 -0700 Subject: [PATCH 1/9] fix(responses bridge): carry client cache_control to prompt_cache_breakpoint for targets that read breakpoints Chat requests to a mode: responses model went through the chat-to-Responses bridge, which dropped a client's Anthropic cache_control marks. Mantle GPT deployments with explicit prompt caching got no prompt_cache_breakpoint, so nothing was cached. The bridge now uses the same gate and the same rename helper as the OpenAI chat route. A marked string system message becomes a one-part list, so it stays in input in client order instead of folding into instructions. Targets that do not read breakpoints are unchanged. The shared rename helper now skips parts with no cache_control, so a part with non-string keys no longer fails validation. --- .../transformation.py | 32 ++- .../anthropic_cache_control_hook.py | 2 +- ...mantle_gpt_prompt_cache_breakpoint_wire.py | 53 +++++ ...responses_transformation_transformation.py | 213 +++++++++++++++++- 4 files changed, 293 insertions(+), 7 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index ee3639ac387..516f2ab8cd4 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -24,7 +24,10 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError import litellm from litellm import ModelResponse from litellm._logging import verbose_logger -from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint +from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + supports_openai_prompt_cache_breakpoint, +) from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( responses_reasoning_items_from_thinking_blocks, @@ -211,6 +214,22 @@ def _drops_audio_input(model: str, litellm_params: Mapping[str, object]) -> bool return not _supports_audio_input(model, litellm_params) +def _messages_in_cache_mark_dialect( + model: str, + messages: Sequence["AllMessageValues"], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], +) -> Sequence["AllMessageValues"]: + if not AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( + model, + _served_provider(litellm_params), + litellm_params.get("api_base"), + optional_params.get("prompt_cache_options"), + ): + return messages + return tuple(AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages) + + def responses_provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]: return MappingProxyType( { @@ -798,7 +817,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): def convert_chat_completion_messages_to_responses_api( self, - messages: list["AllMessageValues"], + messages: Sequence["AllMessageValues"], *, drop_params: bool = False, keep_prompt_cache_breakpoints: bool = False, @@ -820,7 +839,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): def _convert_chat_completion_messages_to_responses_input( self, - messages: list["AllMessageValues"], + messages: Sequence["AllMessageValues"], *, drop_params: bool = False, replay_tool_search: bool = False, @@ -1076,7 +1095,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): and supports_openai_prompt_cache_breakpoint(base_model, provider) ) converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api( - messages, + _messages_in_cache_mark_dialect( + model, + messages, + cast(Mapping[str, object], optional_params), # cast-ok: upcast of the untyped params dict + litellm_params, + ), drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params), keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint, replay_tool_search=_requests_tool_search(cast(object, optional_params)), # cast-ok: upcast diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index b0deec24c1a..92ffbae799b 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -100,7 +100,7 @@ def _validated_object_list(value: object) -> list[object] | None: def _part_with_cache_control_renamed(part: object) -> object: - if not _accepts_prompt_cache_breakpoint(part): + if not _accepts_prompt_cache_breakpoint(part) or _attribute_or_key(part, "cache_control") is None: return part return cache_control_as_prompt_cache_breakpoint(_CONTENT_PART_ADAPTER.validate_python(part)) diff --git a/tests/integration/providers/test_bedrock_mantle_gpt_prompt_cache_breakpoint_wire.py b/tests/integration/providers/test_bedrock_mantle_gpt_prompt_cache_breakpoint_wire.py index bc46cf5cde1..6a0cf4b5148 100644 --- a/tests/integration/providers/test_bedrock_mantle_gpt_prompt_cache_breakpoint_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_gpt_prompt_cache_breakpoint_wire.py @@ -64,6 +64,7 @@ from integration.providers._mantle_gpt_prompt_cache_support import ( spend_rows, success_row, system_item, + user_item, ) from pydantic import JsonValue @@ -540,3 +541,55 @@ def test_f4_a_slow_mantle_stream_during_a_burst_completes_with_every_id_landing_ name: Final = mantle_deployment(gateway, scenario, wire) wire.drain() assert_burst_landed(wire, name, burst(gateway, name, 12), marked=True) + + +def client_marked_chat_body(name: str, prompt: str, *, stream: bool) -> dict[str, JsonValue]: + return request_body( + "chat", + name, + prompt, + stream=stream, + messages=[ + {"role": "system", "content": SYSTEM, "cache_control": EPHEMERAL}, + {"role": "user", "content": prompt}, + ], + ) + + +@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"]) +def test_g1_a_clients_cache_control_on_a_string_system_message_reaches_mantle_as_its_breakpoint( + gateway: Gateway, stream: bool +) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment(gateway, scenario, wire, points=[], prompt_cache_options=EXPLICIT) + outcome, received = observe( + gateway, wire, "chat", client_marked_chat_body(name, prompt_text(marker), stream=stream) + ) + assert_answered(outcome, marker) + assert_wire( + received, + expected_wire(GPT, prompt_text(marker), endpoint="chat", marked=True, options=EXPLICIT), + streaming=stream, + ) + success_row(name, marker) + + +def test_g2_a_clients_cache_control_sends_no_mark_to_a_mantle_row_that_does_not_read_breakpoints( + gateway: Gateway, +) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment( + gateway, scenario, wire, GPT_FLAGGED_ROW, model_info={"supports_prompt_cache_breakpoint": False}, points=[] + ) + outcome, received = observe( + gateway, wire, "chat", client_marked_chat_body(name, prompt_text(marker), stream=False) + ) + assert_answered(outcome, marker) + body: Final = body_of(received) + assert body["instructions"] == SYSTEM, received.body + assert body["input"] == [user_item(prompt_text(marker), "chat")], received.body + assert "prompt_cache_breakpoint" not in received.body.decode(), received.body + assert "cache_control" not in received.body.decode(), received.body + success_row(name, marker) diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 46f1dbf2e7a..e81ae2a607d 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -19,6 +19,7 @@ from openai.types.responses.response_reasoning_item import ( ResponseReasoningItem, Summary, ) +from pydantic import TypeAdapter import litellm from litellm.completion_extras.litellm_responses_transformation.transformation import ( @@ -5373,6 +5374,211 @@ def test_transform_request_strips_the_breakpoint_for_a_served_provider_without_a ] +_OPENAI_GPT_ROW: Final = {"litellm_provider": "openai", "mode": "chat", "supports_prompt_cache_breakpoint": True} +_EPHEMERAL: Final = {"type": "ephemeral"} +_CLIENT_MARKED_MESSAGES: Final = ( + {"role": "system", "content": "Rules"}, + {"role": "system", "content": "Stable prefix", "cache_control": _EPHEMERAL}, + {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": _EPHEMERAL}]}, +) +_PINNED_EXPLICIT: Final = {"mode": "explicit"} +_MESSAGE: Final = TypeAdapter(Mapping[str, object]) +_PARTS: Final = TypeAdapter(list[Mapping[str, object]]) + + +def _client_marked_messages() -> list[AllMessageValues]: + return cast(List[AllMessageValues], copy.deepcopy(list(_CLIENT_MARKED_MESSAGES))) # cast-ok: chat message dicts + + +def _marked_parts(role: object, content: object) -> frozenset[tuple[object, object, str]]: + if not isinstance(content, list): + return frozenset() + return frozenset( + (role, part.get("text"), json.dumps(part["prompt_cache_breakpoint"], sort_keys=True)) + for part in _PARTS.validate_python(content) + if "prompt_cache_breakpoint" in part + ) + + +def _marked_parts_of(items: object) -> frozenset[tuple[object, object, str]]: + messages: Final = (_MESSAGE.validate_python(item) for item in cast(list[object], items)) # cast-ok: wire list + return frozenset().union(*(_marked_parts(message.get("role"), message.get("content")) for message in messages)) + + +@pytest.mark.parametrize( + ("row_key", "row", "model", "custom_llm_provider", "api_base", "prompt_cache_options", "reads_breakpoints"), + [ + pytest.param( + "bedrock_mantle/openai.gpt-5.6-sol", + _MANTLE_GPT_ROW, + "openai.gpt-5.6-sol", + "bedrock_mantle", + None, + None, + True, + id="flagged-hosted-row", + ), + pytest.param( + "bedrock_mantle/openai.gpt-5.6-sol", + _MANTLE_GPT_ROW, + "openai.gpt-5.6-sol", + "bedrock_mantle", + None, + _PINNED_EXPLICIT, + True, + id="flagged-hosted-row-pinned-explicit", + ), + pytest.param( + "bedrock_mantle/gpt-6-luna", + {**_MANTLE_GPT_ROW, "supports_prompt_cache_breakpoint": False}, + "gpt-6-luna", + "bedrock_mantle", + None, + _PINNED_EXPLICIT, + False, + id="hosted-row-opted-out", + ), + pytest.param( + "bedrock_mantle/anthropic.claude-sonnet-test", + {"litellm_provider": "bedrock_mantle", "mode": "chat"}, + "anthropic.claude-sonnet-test", + "bedrock_mantle", + None, + _PINNED_EXPLICIT, + False, + id="claude-on-the-same-host", + ), + pytest.param( + "gpt-6-luna", + _OPENAI_GPT_ROW, + "gpt-6-luna", + "openai", + "https://api.openai.com/v1", + None, + True, + id="openai-api", + ), + pytest.param( + "gpt-6-luna", + _OPENAI_GPT_ROW, + "gpt-6-luna", + "openai", + "https://gateway.example.com/v1", + None, + False, + id="third-party-host", + ), + pytest.param( + "gpt-6-luna", + _OPENAI_GPT_ROW, + "gpt-6-luna", + "openai", + "https://gateway.example.com/v1", + _PINNED_EXPLICIT, + True, + id="third-party-host-opted-in", + ), + ], +) +def test_transform_request_marks_the_parts_the_chat_route_marks_for_the_same_deployment( + monkeypatch: pytest.MonkeyPatch, + row_key: str, + row: Mapping[str, object], + model: str, + custom_llm_provider: str, + api_base: str | None, + prompt_cache_options: Mapping[str, object] | None, + reads_breakpoints: bool, +) -> None: + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + monkeypatch.setitem(litellm.model_cost, row_key, row) + litellm_params: Final[dict[str, object]] = { + "custom_llm_provider": custom_llm_provider, + **({} if api_base is None else {"api_base": api_base}), + } + options: Final[dict[str, object]] = ( + {} if prompt_cache_options is None else {"prompt_cache_options": prompt_cache_options} + ) + chat_request: Final = OpenAIGPTConfig().transform_request( + model=model, + messages=_client_marked_messages(), + optional_params={"extra_body": dict(options)}, + litellm_params=dict(litellm_params), + headers={}, + ) + bridged_request: Final = LiteLLMResponsesTransformationHandler().transform_request( + model=model, + messages=_client_marked_messages(), + optional_params=dict(options), + litellm_params=dict(litellm_params), + headers={}, + litellm_logging_obj=Mock(), + ) + + chat_marks: Final = _marked_parts_of(chat_request["messages"]) + assert _marked_parts_of(bridged_request["input"]) == chat_marks + assert {(role, text) for role, text, _ in chat_marks} == ( + {("system", "Stable prefix"), ("user", "hi")} if reads_breakpoints else set() + ) + + +def test_transform_request_keeps_a_marked_string_system_message_in_input_and_folds_the_unmarked_one( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook + + monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) + messages: Final = _client_marked_messages() + chat_route_system: Final = AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(messages[1]) + (chat_route_part,) = _PARTS.validate_python(chat_route_system.get("content")) + + request: Final = LiteLLMResponsesTransformationHandler().transform_request( + model="openai.gpt-5.6-sol", + messages=messages, + optional_params={}, + litellm_params={"custom_llm_provider": "bedrock_mantle"}, + headers={}, + litellm_logging_obj=Mock(), + ) + + breakpoint_marker: Final = chat_route_part["prompt_cache_breakpoint"] + assert request["instructions"] == "Rules" + assert request["input"] == [ + { + "type": "message", + "role": "system", + "content": [{"type": "input_text", "text": "Stable prefix", "prompt_cache_breakpoint": breakpoint_marker}], + }, + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "hi", "prompt_cache_breakpoint": breakpoint_marker}], + }, + ] + assert messages == list(_CLIENT_MARKED_MESSAGES) + + +def test_transform_request_leaves_client_cache_control_unmapped_for_a_deployment_that_does_not_read_breakpoints( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem( + litellm.model_cost, "bedrock_mantle/gpt-6-luna", {**_MANTLE_GPT_ROW, "supports_prompt_cache_breakpoint": False} + ) + + request: Final = LiteLLMResponsesTransformationHandler().transform_request( + model="gpt-6-luna", + messages=_client_marked_messages(), + optional_params={"prompt_cache_options": _PINNED_EXPLICIT}, + litellm_params={"custom_llm_provider": "bedrock_mantle"}, + headers={}, + litellm_logging_obj=Mock(), + ) + + assert request["instructions"] == "Rules Stable prefix" + assert request["input"] == [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}] + + def test_mid_conversation_system_string_stays_in_input_after_a_user_turn(): handler: Final = LiteLLMResponsesTransformationHandler() @@ -5430,6 +5636,9 @@ def test_mid_conversation_system_reminder_as_string_and_as_text_block_produce_id assert as_string[1] == _system_input_item(reminder) +_HOST_WITHOUT_BREAKPOINTS: Final = {"custom_llm_provider": "openai", "api_base": "https://gateway.example.com/v1"} + + def test_claude_code_shaped_history_keeps_a_byte_stable_input_prefix_across_requests(): handler: Final = LiteLLMResponsesTransformationHandler() top_level_system: Final = [{"type": "text", "text": "You are Claude Code.", "cache_control": {"type": "ephemeral"}}] @@ -5469,7 +5678,7 @@ def test_claude_code_shaped_history_keeps_a_byte_stable_input_prefix_across_requ model="gpt-5.6-luna", messages=first_request_messages, optional_params={}, - litellm_params={}, + litellm_params=dict(_HOST_WITHOUT_BREAKPOINTS), headers={}, litellm_logging_obj=Mock(), ) @@ -5477,7 +5686,7 @@ def test_claude_code_shaped_history_keeps_a_byte_stable_input_prefix_across_requ model="gpt-5.6-luna", messages=second_request_messages, optional_params={}, - litellm_params={}, + litellm_params=dict(_HOST_WITHOUT_BREAKPOINTS), headers={}, litellm_logging_obj=Mock(), ) From 5a2a726a478eede66634d35db997c137a5673592 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 4 Nov 2026 13:38:51 -0800 Subject: [PATCH 2/9] fix(responses): keep system messages in order when a marked one comes first --- .../transformation.py | 14 ++++++++--- ...responses_transformation_transformation.py | 25 +++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 10c6a6c962e..7ab6d6293ff 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -293,6 +293,14 @@ def _get_reasoning_items( return [] +def _is_string_system_message(msg: "AllMessageValues") -> bool: + match msg: + case {"role": "system", "content": str()}: + return True + case _: + return False + + def _reasoning_input_items(msg: "AllMessageValues") -> list[dict[str, object]]: # mutable-ok: API message payload """Reasoning input items for an assistant message. @@ -944,8 +952,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): and isinstance(tool_call.get("custom"), dict) ) - leading_system_count: Final = next( - (index for index, msg in enumerate(messages) if msg.get("role") != "system"), + leading_system_string_count: Final = next( + (index for index, msg in enumerate(messages) if not _is_string_system_message(msg)), len(messages), ) @@ -959,7 +967,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): input_items.extend(replayed_turn.items) elif role == "system": # Extract system message as instructions - if isinstance(content, str) and index < leading_system_count: + if isinstance(content, str) and index < leading_system_string_count: if instructions: # Concatenate multiple system prompts with a space instructions = f"{instructions} {content}" diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 5201bdc9798..4c7abc965df 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -6111,6 +6111,31 @@ def test_transform_request_keeps_a_marked_string_system_message_in_input_and_fol assert messages == list(_CLIENT_MARKED_MESSAGES) +def test_transform_request_keeps_system_messages_in_order_when_a_marked_one_comes_first( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) + + request: Final = LiteLLMResponsesTransformationHandler().transform_request( + model="openai.gpt-5.6-sol", + messages=[ + {"role": "system", "content": "Stable prefix", "cache_control": _EPHEMERAL}, + {"role": "system", "content": "Rules"}, + {"role": "user", "content": "hi"}, + ], + optional_params={}, + litellm_params={"custom_llm_provider": "bedrock_mantle"}, + headers={}, + litellm_logging_obj=Mock(), + ) + + items: Final = (_MESSAGE.validate_python(item) for item in cast(list[object], request["input"])) # cast-ok: wire list + assert "instructions" not in request + assert [ + (item.get("role"), [part.get("text") for part in _PARTS.validate_python(item.get("content"))]) for item in items + ] == [("system", ["Stable prefix"]), ("system", ["Rules"]), ("user", ["hi"])] + + def test_transform_request_leaves_client_cache_control_unmapped_for_a_deployment_that_does_not_read_breakpoints( monkeypatch: pytest.MonkeyPatch, ) -> None: From a97c1b7a9e407547e0804bc5dea5e49b106229f0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:06:00 -0800 Subject: [PATCH 3/9] fix(responses): stop the system fold only at a marked message The previous commit stopped folding at any list system message, which moved a later string system message into input for requests with no cache marks. Unmarked requests now fold exactly as they did before this PR, and the fold stops only at a system message carrying a breakpoint. --- .../transformation.py | 15 +++--- .../anthropic_cache_control_hook.py | 8 ++-- ...responses_transformation_transformation.py | 46 ++++++++++++++++--- 3 files changed, 51 insertions(+), 18 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 7ab6d6293ff..8d01aa5cd88 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -36,6 +36,7 @@ from litellm.completion_extras.litellm_responses_transformation.history import ( ) from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, + message_carries_prompt_cache_breakpoint, routed_model_info, supports_openai_prompt_cache_breakpoint, ) @@ -293,12 +294,8 @@ def _get_reasoning_items( return [] -def _is_string_system_message(msg: "AllMessageValues") -> bool: - match msg: - case {"role": "system", "content": str()}: - return True - case _: - return False +def _is_unmarked_system_message(msg: "AllMessageValues") -> bool: + return msg.get("role") == "system" and not message_carries_prompt_cache_breakpoint(msg) def _reasoning_input_items(msg: "AllMessageValues") -> list[dict[str, object]]: # mutable-ok: API message payload @@ -952,8 +949,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): and isinstance(tool_call.get("custom"), dict) ) - leading_system_string_count: Final = next( - (index for index, msg in enumerate(messages) if not _is_string_system_message(msg)), + leading_unmarked_system_count: Final = next( + (index for index, msg in enumerate(messages) if not _is_unmarked_system_message(msg)), len(messages), ) @@ -967,7 +964,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): input_items.extend(replayed_turn.items) elif role == "system": # Extract system message as instructions - if isinstance(content, str) and index < leading_system_string_count: + if isinstance(content, str) and index < leading_unmarked_system_count: if instructions: # Concatenate multiple system prompts with a space instructions = f"{instructions} {content}" diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 950661f74a6..3f4601ed885 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -702,10 +702,12 @@ def _may_carry_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool: return message.get("role") != "assistant" +def message_carries_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool: + return any(_carries_prompt_cache_breakpoint(part) for part in _as_object_list(message.get("content")) or ()) + + def _tool_result_carries_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool: - return message.get("role") == "tool" and any( - _carries_prompt_cache_breakpoint(part) for part in _as_object_list(message.get("content")) or () - ) + return message.get("role") == "tool" and message_carries_prompt_cache_breakpoint(message) def _part_without_prompt_cache_breakpoint(part: object) -> object: diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 4c7abc965df..f56cde970b9 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -6111,7 +6111,44 @@ def test_transform_request_keeps_a_marked_string_system_message_in_input_and_fol assert messages == list(_CLIENT_MARKED_MESSAGES) +def _roles_and_texts(request_input: object) -> list[tuple[object, list[object]]]: + items: Final = (_MESSAGE.validate_python(item) for item in cast(list[object], request_input)) # cast-ok: wire list + return [ + (item.get("role"), [part.get("text") for part in _PARTS.validate_python(item.get("content"))]) for item in items + ] + + +@pytest.mark.parametrize( + "marked_system", + [ + {"role": "system", "content": "Stable prefix", "cache_control": _EPHEMERAL}, + {"role": "system", "content": [{"type": "text", "text": "Stable prefix", "cache_control": _EPHEMERAL}]}, + ], + ids=["marked_string", "marked_part"], +) def test_transform_request_keeps_system_messages_in_order_when_a_marked_one_comes_first( + monkeypatch: pytest.MonkeyPatch, marked_system: AllMessageValues +) -> None: + monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) + + request: Final = LiteLLMResponsesTransformationHandler().transform_request( + model="openai.gpt-5.6-sol", + messages=[marked_system, {"role": "system", "content": "Rules"}, {"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={"custom_llm_provider": "bedrock_mantle"}, + headers={}, + litellm_logging_obj=Mock(), + ) + + assert "instructions" not in request + assert _roles_and_texts(request["input"]) == [ + ("system", ["Stable prefix"]), + ("system", ["Rules"]), + ("user", ["hi"]), + ] + + +def test_transform_request_still_folds_a_string_system_message_behind_an_unmarked_list_one( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) @@ -6119,7 +6156,7 @@ def test_transform_request_keeps_system_messages_in_order_when_a_marked_one_come request: Final = LiteLLMResponsesTransformationHandler().transform_request( model="openai.gpt-5.6-sol", messages=[ - {"role": "system", "content": "Stable prefix", "cache_control": _EPHEMERAL}, + {"role": "system", "content": [{"type": "text", "text": "Stable prefix"}]}, {"role": "system", "content": "Rules"}, {"role": "user", "content": "hi"}, ], @@ -6129,11 +6166,8 @@ def test_transform_request_keeps_system_messages_in_order_when_a_marked_one_come litellm_logging_obj=Mock(), ) - items: Final = (_MESSAGE.validate_python(item) for item in cast(list[object], request["input"])) # cast-ok: wire list - assert "instructions" not in request - assert [ - (item.get("role"), [part.get("text") for part in _PARTS.validate_python(item.get("content"))]) for item in items - ] == [("system", ["Stable prefix"]), ("system", ["Rules"]), ("user", ["hi"])] + assert request["instructions"] == "Rules" + assert _roles_and_texts(request["input"]) == [("system", ["Stable prefix"]), ("user", ["hi"])] def test_transform_request_leaves_client_cache_control_unmapped_for_a_deployment_that_does_not_read_breakpoints( From e9130396b9c2b2f69eab627f50017be6aebb869d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 13:03:16 -0800 Subject: [PATCH 4/9] fix(cache): map /v1/messages marks for OpenAI-dialect targets and skip empty or malformed marks The /v1/messages chat adapter now keeps cache_control on system, user, and tool result blocks when the deployment reads OpenAI prompt cache breakpoints, so the chat bridge maps them. Empty text and non-object marks no longer become breakpoints, and a base_model alias maps client marks the way it keeps native ones --- .../transformation.py | 22 +++-- .../anthropic_cache_control_hook.py | 23 ++++- .../anthropic/messages/adapters/handler.py | 8 ++ .../messages/adapters/transformation.py | 22 +++-- .../messages/responses_adapters/handler.py | 15 +-- ...responses_transformation_transformation.py | 63 +++++++++++++ .../test_handler_prompt_cache_breakpoint.py | 91 +++++++++++++++++++ 7 files changed, 211 insertions(+), 33 deletions(-) create mode 100644 tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_breakpoint.py diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 8d01aa5cd88..30d9ab40ecc 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -239,19 +239,23 @@ def _drops_audio_input(model: str, litellm_params: Mapping[str, object]) -> bool def _messages_in_cache_mark_dialect( - model: str, + named_models: Sequence[str], messages: Sequence["AllMessageValues"], optional_params: Mapping[str, object], litellm_params: Mapping[str, object], model_info: Mapping[str, object] | None, ) -> Sequence["AllMessageValues"]: - if not AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( - model, - _served_provider(litellm_params), - litellm_params.get("api_base"), - optional_params.get("prompt_cache_options"), - model_info, - ): + reads_breakpoints: Final = any( + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( + name, + _served_provider(litellm_params), + litellm_params.get("api_base"), + optional_params.get("prompt_cache_options"), + model_info, + ) + for name in named_models + ) + if not reads_breakpoints: return messages return tuple(AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages) @@ -1200,7 +1204,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api( _messages_in_cache_mark_dialect( - model, + named_models, messages, cast(Mapping[str, object], optional_params), # cast-ok: upcast of the untyped params dict litellm_params, diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 3f4601ed885..52d8e51dd27 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -327,7 +327,7 @@ def whole_messages_within_cache_control_limit( def _part_with_cache_control_renamed(part: object) -> object: - if not _accepts_prompt_cache_breakpoint(part) or _attribute_or_key(part, "cache_control") is None: + if not _holds_prompt_cache_breakpoint(part) or not isinstance(_attribute_or_key(part, "cache_control"), Mapping): return part return cache_control_as_prompt_cache_breakpoint(_CONTENT_PART_ADAPTER.validate_python(part)) @@ -694,6 +694,10 @@ def _accepts_prompt_cache_breakpoint(block: object) -> bool: return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES +def _holds_prompt_cache_breakpoint(block: object) -> bool: + return _accepts_prompt_cache_breakpoint(block) and _attribute_or_key(block, "text") != "" + + def _carries_prompt_cache_breakpoint(block: object) -> bool: return _accepts_prompt_cache_breakpoint(block) and _attribute_or_key(block, "prompt_cache_breakpoint") is not None @@ -1256,7 +1260,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): }, ) ) - if fields.get("cache_control") is None: + if not isinstance(fields.get("cache_control"), Mapping): return renamed return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(renamed) @@ -1351,7 +1355,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): if not _may_carry_prompt_cache_breakpoint(message): return message message_content: Final = message.get("content", None) - if isinstance(message_content, str): + if isinstance(message_content, str) and message_content: marked: Final = copy.copy(message) marked["content"] = [ with_prompt_cache_breakpoint( @@ -1364,7 +1368,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): ( index for index in range(len(message_content) - 1, -1, -1) - if _accepts_prompt_cache_breakpoint(message_content[index]) + if _holds_prompt_cache_breakpoint(message_content[index]) ), None, ) @@ -2005,3 +2009,14 @@ def messages_within_reserved_cache_control_cap( MAX_CACHE_CONTROL_BLOCKS, ) return kept + + +def call_targets_openai_prompt_cache_breakpoint(model: str, call_kwargs: Mapping[str, object]) -> bool: + provider: Final = call_kwargs.get("custom_llm_provider") + return AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( + model, + provider if isinstance(provider, str) else None, + call_kwargs.get("api_base") or call_kwargs.get("base_url"), + call_kwargs.get("prompt_cache_options"), + routed_model_info(call_kwargs.get("model_info")), + ) diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index 31a4e9d9d73..12f0325f6c6 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -12,6 +12,7 @@ from typing_extensions import TypedDict import litellm from litellm._logging import verbose_logger +from litellm.integrations.anthropic_cache_control_hook import call_targets_openai_prompt_cache_breakpoint from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject @@ -397,6 +398,12 @@ ANTHROPIC_ADAPTER: Final = AnthropicAdapter() ######################################################## +def _target_reads_prompt_cache_breakpoint(model: str, call_kwargs: Mapping[str, object]) -> bool: + base_model: Final = call_kwargs.get("base_model") + named_models: Final = (model, base_model) if isinstance(base_model, str) and base_model else (model,) + return any(call_targets_openai_prompt_cache_breakpoint(name, call_kwargs) for name in named_models) + + def _record_bridged_model_response(logging_obj: LiteLLMLoggingObject | None, completion_response: object) -> None: if logging_obj is not None and isinstance(completion_response, ModelResponse): logging_obj.record_bridged_model_response(completion_response) @@ -678,6 +685,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: custom_llm_provider=target_provider, api_base=api_base if isinstance(api_base, str) else None, drop_params=drop_params, + keeps_cache_control=_target_reads_prompt_cache_breakpoint(model, extra_kwargs), ) if openai_request is None: diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 4b1b5a8a11a..8e6feced253 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -265,6 +265,7 @@ class AnthropicAdapter: custom_llm_provider: str | None = None, api_base: str | None = None, drop_params: bool = True, + keeps_cache_control: bool = False, ) -> tuple[ChatCompletionRequest | None, dict[str, str]]: """ Translate Anthropic request params to OpenAI format, returning tool name mapping. @@ -295,7 +296,7 @@ class AnthropicAdapter: ( translated_body, tool_name_mapping, - ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + ) = LiteLLMAnthropicMessagesAdapter(keeps_cache_control=keeps_cache_control).translate_anthropic_to_openai( anthropic_message_request=request_body, custom_llm_provider=custom_llm_provider, api_base=api_base, @@ -380,8 +381,8 @@ _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS: Final = frozenset({LlmProviders.KYMA.va class LiteLLMAnthropicMessagesAdapter: - def __init__(self): - pass + def __init__(self, keeps_cache_control: bool = False) -> None: + self._keeps_cache_control: Final = keeps_cache_control ### FOR [BETA] `/v1/messages` endpoint support @@ -419,6 +420,8 @@ class LiteLLMAnthropicMessagesAdapter: source: object, target: object, model: str | None, + *, + has_prompt_cache_breakpoint_slot: bool = True, ) -> None: """ Extract cache_control from source and add to target if it should be preserved. @@ -437,7 +440,8 @@ class LiteLLMAnthropicMessagesAdapter: cache_control: Final = ( source.get("cache_control") if isinstance(source, dict) else getattr(source, "cache_control", None) ) - if cache_control and model and self.target_consumes_cache_control(model): + keeps_as_breakpoint: Final = self._keeps_cache_control and has_prompt_cache_breakpoint_slot + if cache_control and model and (keeps_as_breakpoint or self.target_consumes_cache_control(model)): # TypedDict objects support dict operations at runtime # Use type ignore consistent with codebase pattern (see anthropic/chat/transformation.py:432) if isinstance(target, dict): @@ -592,7 +596,9 @@ class LiteLLMAnthropicMessagesAdapter: "type": "text", "text": content.get("text", ""), } - self._add_cache_control_if_applicable(content, text_block, model) + self._add_cache_control_if_applicable( + content, text_block, model, has_prompt_cache_breakpoint_slot=False + ) if "cache_control" in text_block: has_cache_control_in_text = True assistant_content_list.append(text_block) @@ -619,7 +625,9 @@ class LiteLLMAnthropicMessagesAdapter: type="function", function=function_chunk, ) - self._add_cache_control_if_applicable(content, tool_call, model) + self._add_cache_control_if_applicable( + content, tool_call, model, has_prompt_cache_breakpoint_slot=False + ) tool_calls.append(tool_call) elif content.get("type") == "thinking": # Anthropic's schema has no cache_control on thinking or @@ -883,7 +891,7 @@ class LiteLLMAnthropicMessagesAdapter: if k not in mapped_tool_params: # pass additional computer kwargs function_chunk.setdefault("parameters", {}).update({k: v}) tool_param = _chat_tool_param(function_chunk, tool) - self._add_cache_control_if_applicable(tool, tool_param, model) + self._add_cache_control_if_applicable(tool, tool_param, model, has_prompt_cache_breakpoint_slot=False) new_tools.append(tool_param) return new_tools, tool_name_mapping diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index ec09987837c..36efc28626d 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -12,7 +12,7 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm._logging import verbose_logger from litellm.integrations.anthropic_cache_control_hook import ( - AnthropicCacheControlHook, + call_targets_openai_prompt_cache_breakpoint, routed_model_info, supports_openai_prompt_cache_breakpoint, ) @@ -181,17 +181,6 @@ def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: objec return responses_config is not None and "include" in responses_config.get_supported_openai_params(provider_model) -def _reads_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[str, object]) -> bool: - provider: Final = forwarded_kwargs.get("custom_llm_provider") - return AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( - model, - provider if isinstance(provider, str) else None, - forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url"), - forwarded_kwargs.get("prompt_cache_options"), - routed_model_info(forwarded_kwargs.get("model_info")), - ) - - def _model_takes_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[str, object]) -> bool: provider: Final = forwarded_kwargs.get("custom_llm_provider") provider_name: Final = provider if isinstance(provider, str) else None @@ -221,7 +210,7 @@ def _prompt_cache_block_rewrite(model: str, forwarded_kwargs: Mapping[str, objec """Client cache marks become breakpoints for a deployment that reads them, and a model that takes no breakpoints drops raw ones under drop_params.""" drops_params: Final = drops_unsupported_params(forwarded_kwargs.get("drop_params")) - if _reads_prompt_cache_breakpoint(model, forwarded_kwargs): + if call_targets_openai_prompt_cache_breakpoint(model, forwarded_kwargs): return ( _cache_control_as_wire_safe_prompt_cache_breakpoint if drops_params diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index f56cde970b9..29baf93b32c 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -6170,6 +6170,69 @@ def test_transform_request_still_folds_a_string_system_message_behind_an_unmarke assert _roles_and_texts(request["input"]) == [("system", ["Stable prefix"]), ("user", ["hi"])] +def _mantle_marked_texts(model: str, messages: list[object], **litellm_params: object) -> set[tuple[object, object]]: + request: Final = LiteLLMResponsesTransformationHandler().transform_request( + model=model, + messages=cast(List[AllMessageValues], messages), # cast-ok: the test builds chat messages as plain mappings + optional_params={}, + litellm_params={"custom_llm_provider": "bedrock_mantle", **litellm_params}, + headers={}, + litellm_logging_obj=Mock(), + ) + return {(role, text) for role, text, _ in _marked_parts_of(request["input"])} + + +def test_transform_request_puts_no_breakpoint_on_empty_marked_text(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) + + marked: Final = _mantle_marked_texts( + "openai.gpt-5.6-sol", + [ + {"role": "system", "content": "", "cache_control": _EPHEMERAL}, + {"role": "user", "content": [{"type": "text", "text": "hi"}, {"type": "text", "text": "", "cache_control": _EPHEMERAL}]}, + { + "role": "user", + "content": [{"type": "text", "text": "there"}, {"type": "text", "text": ""}], + "cache_control": _EPHEMERAL, + }, + ], + ) + + assert marked == {("user", "there")} + + +def test_transform_request_maps_only_an_object_cache_control_to_a_breakpoint(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) + + marked: Final = _mantle_marked_texts( + "openai.gpt-5.6-sol", + [ + {"role": "system", "content": "Rules", "cache_control": "ephemeral"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "hi", "cache_control": "ephemeral"}, + {"type": "text", "text": "there", "cache_control": _EPHEMERAL}, + ], + }, + ], + ) + + assert marked == {("user", "there")} + + +def test_transform_request_maps_client_cache_control_for_an_alias_whose_base_model_reads_breakpoints( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) + + marked: Final = _mantle_marked_texts( + "my-alias", list(_client_marked_messages()), base_model="openai.gpt-5.6-sol" + ) + + assert marked == {("system", "Stable prefix"), ("user", "hi")} + + def test_transform_request_leaves_client_cache_control_unmapped_for_a_deployment_that_does_not_read_breakpoints( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_breakpoint.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_breakpoint.py new file mode 100644 index 00000000000..c1a37a16253 --- /dev/null +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_breakpoint.py @@ -0,0 +1,91 @@ +import json +from collections.abc import Iterator +from typing import Final, List, cast +from unittest.mock import Mock + +import pytest + +import litellm +from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, +) +from litellm.llms.anthropic.messages.adapters.handler import LiteLLMMessagesToCompletionTransformationHandler +from litellm.types.llms.openai import AllMessageValues + +_EPHEMERAL: Final = {"type": "ephemeral"} +_MANTLE_GPT_ROW: Final = { + "litellm_provider": "bedrock_mantle", + "mode": "responses", + "supports_prompt_cache_breakpoint": True, +} +_CLAUDE_CODE_MESSAGES: Final = ( + {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": _EPHEMERAL}]}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "checking", "cache_control": _EPHEMERAL}, + {"type": "tool_use", "id": "toolu_1", "name": "read", "input": {}, "cache_control": _EPHEMERAL}, + ], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "file body", "cache_control": _EPHEMERAL}], + }, +) +_SYSTEM: Final = [{"type": "text", "text": "Stable prefix", "cache_control": _EPHEMERAL}] +_TOOLS: Final = [{"name": "read", "description": "Read a file", "input_schema": {"type": "object"}, "cache_control": _EPHEMERAL}] + + +def _texts_with(key: str, value: object) -> Iterator[object]: + if isinstance(value, dict): + if key in value: + yield value.get("text", value.get("name")) + for nested in value.values(): + yield from _texts_with(key, nested) + if isinstance(value, list): + for item in value: + yield from _texts_with(key, item) + + +def _mantle_wire(model_info: dict[str, object] | None) -> tuple[dict[str, object], dict[str, object]]: + completion_kwargs, _ = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=64, + messages=json.loads(json.dumps(_CLAUDE_CODE_MESSAGES)), + model="bedrock_mantle/openai.gpt-5.6-sol", + system=json.loads(json.dumps(_SYSTEM)), + tools=json.loads(json.dumps(_TOOLS)), + extra_kwargs={"custom_llm_provider": "bedrock_mantle", "model_info": model_info}, + ) + wire: Final = LiteLLMResponsesTransformationHandler().transform_request( + model="openai.gpt-5.6-sol", + messages=cast(List[AllMessageValues], completion_kwargs["messages"]), # cast-ok: chat messages from the adapter + optional_params={"tools": completion_kwargs["tools"]}, + litellm_params={"custom_llm_provider": "bedrock_mantle", "model_info": model_info}, + headers={}, + litellm_logging_obj=Mock(), + ) + return dict(completion_kwargs), wire + + +def test_anthropic_messages_to_a_breakpoint_reading_target_carry_marks_where_the_openai_dialect_reads_them( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) + + completion_kwargs, wire = _mantle_wire(None) + + assert list(_texts_with("cache_control", completion_kwargs["tools"])) == [] + assert list(_texts_with("cache_control", cast(list[object], completion_kwargs["messages"])[2])) == [] + assert sorted(map(str, _texts_with("prompt_cache_breakpoint", wire["input"]))) == ["Stable prefix", "file body", "hi"] + assert list(_texts_with("cache_control", wire)) == [] + + +def test_anthropic_messages_to_a_deployment_that_opts_out_of_breakpoints_carry_no_marks( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) + + completion_kwargs, wire = _mantle_wire({"supports_prompt_cache_breakpoint": False}) + + assert list(_texts_with("cache_control", completion_kwargs)) == [] + assert list(_texts_with("prompt_cache_breakpoint", wire)) == [] From 555ac8eab8dbff60d816eb7f313018334595fd68 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:42:21 -0800 Subject: [PATCH 5/9] docs(anthropic): describe the breakpoint-slot flag on the cache mark helper --- litellm/llms/anthropic/messages/adapters/transformation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 8e6feced253..ca3a30d1a3f 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -435,6 +435,7 @@ class LiteLLMAnthropicMessagesAdapter: source: Dict or TypedDict containing potential cache_control field target: Dict or TypedDict to add cache_control to model: Model name to check if cache_control should be preserved + has_prompt_cache_breakpoint_slot: Whether the OpenAI cache dialect reads a breakpoint on this block """ # TypedDict objects are dicts at runtime, so .get() works cache_control: Final = ( From 756320e843eb060e749098f3a5d0424fba390b4d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 20:41:46 -0800 Subject: [PATCH 6/9] fix(cache): keep chat route marks on string and empty parts and fold system messages where breakpoints drop --- .../transformation.py | 24 +++-- .../anthropic_cache_control_hook.py | 37 ++++---- ...responses_transformation_transformation.py | 60 +++++++++++- .../test_handler_prompt_cache_breakpoint.py | 91 ------------------- .../adapters/test_handler_prompt_cache_key.py | 88 +++++++++++++++++- .../chat/test_openai_gpt_transformation.py | 13 +++ 6 files changed, 194 insertions(+), 119 deletions(-) delete mode 100644 tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_breakpoint.py diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index eb78fde2a21..f3a07ba27d9 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -36,6 +36,7 @@ from litellm.completion_extras.litellm_responses_transformation.history import ( ) from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, + holds_prompt_cache_breakpoint, message_carries_prompt_cache_breakpoint, routed_model_info, supports_openai_prompt_cache_breakpoint, @@ -259,7 +260,10 @@ def _messages_in_cache_mark_dialect( ) if not reads_breakpoints: return messages - return tuple(AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages) + return tuple( + AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message, holds_prompt_cache_breakpoint) + for message in messages + ) def responses_provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]: @@ -300,8 +304,10 @@ def _get_reasoning_items( return [] -def _is_unmarked_system_message(msg: "AllMessageValues") -> bool: - return msg.get("role") == "system" and not message_carries_prompt_cache_breakpoint(msg) +def _folds_into_instructions(msg: "AllMessageValues", keep_prompt_cache_breakpoints: bool) -> bool: + return msg.get("role") == "system" and not ( + keep_prompt_cache_breakpoints and message_carries_prompt_cache_breakpoint(msg) + ) def _reasoning_input_items(msg: "AllMessageValues") -> list[dict[str, object]]: # mutable-ok: API message payload @@ -915,6 +921,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): messages, drop_params=drop_params, target_trims_breakpoints=target_trims_breakpoints, + keep_prompt_cache_breakpoints=keep_prompt_cache_breakpoints, replay_tool_search=replay_tool_search, include_namespace=include_namespace, replay_openai_history=replay_openai_history, @@ -932,6 +939,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): *, drop_params: bool = False, target_trims_breakpoints: bool = True, + keep_prompt_cache_breakpoints: bool = False, replay_tool_search: bool = False, include_namespace: bool = False, replay_openai_history: bool = False, @@ -955,8 +963,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): and isinstance(tool_call.get("custom"), dict) ) - leading_unmarked_system_count: Final = next( - (index for index, msg in enumerate(messages) if not _is_unmarked_system_message(msg)), + leading_folded_system_count: Final = next( + ( + index + for index, msg in enumerate(messages) + if not _folds_into_instructions(msg, keep_prompt_cache_breakpoints) + ), len(messages), ) @@ -970,7 +982,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): input_items.extend(replayed_turn.items) elif role == "system": # Extract system message as instructions - if isinstance(content, str) and index < leading_unmarked_system_count: + if isinstance(content, str) and index < leading_folded_system_count: if instructions: # Concatenate multiple system prompts with a space instructions = f"{instructions} {content}" diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index be7146f263c..0900f885238 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -326,8 +326,8 @@ def whole_messages_within_cache_control_limit( return _messages_within_budget(messages, reserved_marks, _whole_message_kept) -def _part_with_cache_control_renamed(part: object) -> object: - if not _holds_prompt_cache_breakpoint(part) or not isinstance(_attribute_or_key(part, "cache_control"), Mapping): +def _part_with_cache_control_renamed(part: object, markable: Callable[[object], bool]) -> object: + if not markable(part) or _attribute_or_key(part, "cache_control") is None: return part return cache_control_as_prompt_cache_breakpoint(_CONTENT_PART_ADAPTER.validate_python(part)) @@ -694,7 +694,7 @@ def _accepts_prompt_cache_breakpoint(block: object) -> bool: return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES -def _holds_prompt_cache_breakpoint(block: object) -> bool: +def holds_prompt_cache_breakpoint(block: object) -> bool: return _accepts_prompt_cache_breakpoint(block) and _attribute_or_key(block, "text") != "" @@ -1242,7 +1242,9 @@ class AnthropicCacheControlHook(CustomPromptManagement): return message @staticmethod - def cache_control_as_prompt_cache_breakpoints(message: AllMessageValues) -> AllMessageValues: + def cache_control_as_prompt_cache_breakpoints( + message: AllMessageValues, markable: Callable[[object], bool] = _accepts_prompt_cache_breakpoint + ) -> AllMessageValues: """Every Anthropic ``cache_control`` on the message or its parts moved to OpenAI's ``prompt_cache_breakpoint``, placed where the injection hook would put one. Marks with no OpenAI slot are left for the caller to strip.""" fields: Final[Mapping[str, object]] = dict(message) @@ -1256,13 +1258,13 @@ class AnthropicCacheControlHook(CustomPromptManagement): AllMessageValues, { **fields, - "content": [_part_with_cache_control_renamed(part) for part in parts], + "content": [_part_with_cache_control_renamed(part, markable) for part in parts], }, ) ) - if not isinstance(fields.get("cache_control"), Mapping): + if fields.get("cache_control") is None: return renamed - return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(renamed) + return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(renamed, markable) @staticmethod def prompt_cache_breakpoints_as_cache_control(message: AllMessageValues, drop_params: bool) -> AllMessageValues: @@ -1351,25 +1353,22 @@ class AnthropicCacheControlHook(CustomPromptManagement): return capped_request(max(0, first_over_cap - 1)) @staticmethod - def _insert_prompt_cache_breakpoint_in_message(message: AllMessageValues) -> AllMessageValues: + def _insert_prompt_cache_breakpoint_in_message( + message: AllMessageValues, markable: Callable[[object], bool] = _accepts_prompt_cache_breakpoint + ) -> AllMessageValues: if not _may_carry_prompt_cache_breakpoint(message): return message message_content: Final = message.get("content", None) - if isinstance(message_content, str) and message_content: + if isinstance(message_content, str): + text_part: Final = ChatCompletionTextObject(type="text", text=message_content) + if not markable(text_part): + return message marked: Final = copy.copy(message) - marked["content"] = [ - with_prompt_cache_breakpoint( - ChatCompletionTextObject(type="text", text=message_content), PromptCacheBreakpoint(mode="explicit") - ) - ] + marked["content"] = [with_prompt_cache_breakpoint(text_part, PromptCacheBreakpoint(mode="explicit"))] return marked if isinstance(message_content, list): target_index: Final = next( - ( - index - for index in range(len(message_content) - 1, -1, -1) - if _holds_prompt_cache_breakpoint(message_content[index]) - ), + (index for index in range(len(message_content) - 1, -1, -1) if markable(message_content[index])), None, ) if target_index is not None: diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 3dfd9ae4854..551ac73eb64 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -6193,6 +6193,62 @@ def test_transform_request_keeps_system_messages_in_order_when_a_marked_one_come ] +@pytest.mark.parametrize( + ("row_key", "row", "model", "litellm_params"), + [ + pytest.param( + "luna-responses", + {"litellm_provider": "openai", "mode": "responses"}, + "luna-responses", + {"custom_llm_provider": "openai", "api_base": "https://api.openai.com/v1"}, + id="openai-model-without-breakpoints", + ), + pytest.param( + "luna-responses", + {"litellm_provider": "openai", "mode": "responses"}, + "luna-responses", + {"custom_llm_provider": "openai", "api_base": "https://api.openai.com/v1", "drop_params": True}, + id="openai-model-without-breakpoints-drop-params", + ), + pytest.param( + "bedrock_mantle/openai.gpt-5.6-sol", + _MANTLE_GPT_ROW, + "openai.gpt-5.6-sol", + {"custom_llm_provider": "bedrock_mantle", "model_info": {"supports_prompt_cache_breakpoint": False}}, + id="mantle-deployment-opted-out", + ), + ], +) +def test_transform_request_folds_system_messages_as_before_where_the_breakpoint_is_stripped( + monkeypatch: pytest.MonkeyPatch, + row_key: str, + row: Mapping[str, object], + model: str, + litellm_params: Mapping[str, object], +) -> None: + monkeypatch.setitem(litellm.model_cost, row_key, row) + + request: Final = LiteLLMResponsesTransformationHandler().transform_request( + model=model, + messages=[ + { + "role": "system", + "content": [{"type": "text", "text": "Stable prefix", "prompt_cache_breakpoint": _PINNED_EXPLICIT}], + }, + {"role": "system", "content": "Rules"}, + {"role": "user", "content": "hi"}, + ], + optional_params={}, + litellm_params=dict(litellm_params), + headers={}, + litellm_logging_obj=Mock(), + ) + + assert request["instructions"] == "Rules" + assert _roles_and_texts(request["input"]) == [("system", ["Stable prefix"]), ("user", ["hi"])] + assert list(_marked_parts_of(request["input"])) == [] + + def test_transform_request_still_folds_a_string_system_message_behind_an_unmarked_list_one( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -6246,7 +6302,7 @@ def test_transform_request_puts_no_breakpoint_on_empty_marked_text(monkeypatch: assert marked == {("user", "there")} -def test_transform_request_maps_only_an_object_cache_control_to_a_breakpoint(monkeypatch: pytest.MonkeyPatch) -> None: +def test_transform_request_maps_a_string_cache_control_like_an_object_one(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) marked: Final = _mantle_marked_texts( @@ -6263,7 +6319,7 @@ def test_transform_request_maps_only_an_object_cache_control_to_a_breakpoint(mon ], ) - assert marked == {("user", "there")} + assert marked == {("system", "Rules"), ("user", "hi"), ("user", "there")} def test_transform_request_maps_client_cache_control_for_an_alias_whose_base_model_reads_breakpoints( diff --git a/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_breakpoint.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_breakpoint.py deleted file mode 100644 index c1a37a16253..00000000000 --- a/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_breakpoint.py +++ /dev/null @@ -1,91 +0,0 @@ -import json -from collections.abc import Iterator -from typing import Final, List, cast -from unittest.mock import Mock - -import pytest - -import litellm -from litellm.completion_extras.litellm_responses_transformation.transformation import ( - LiteLLMResponsesTransformationHandler, -) -from litellm.llms.anthropic.messages.adapters.handler import LiteLLMMessagesToCompletionTransformationHandler -from litellm.types.llms.openai import AllMessageValues - -_EPHEMERAL: Final = {"type": "ephemeral"} -_MANTLE_GPT_ROW: Final = { - "litellm_provider": "bedrock_mantle", - "mode": "responses", - "supports_prompt_cache_breakpoint": True, -} -_CLAUDE_CODE_MESSAGES: Final = ( - {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": _EPHEMERAL}]}, - { - "role": "assistant", - "content": [ - {"type": "text", "text": "checking", "cache_control": _EPHEMERAL}, - {"type": "tool_use", "id": "toolu_1", "name": "read", "input": {}, "cache_control": _EPHEMERAL}, - ], - }, - { - "role": "user", - "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "file body", "cache_control": _EPHEMERAL}], - }, -) -_SYSTEM: Final = [{"type": "text", "text": "Stable prefix", "cache_control": _EPHEMERAL}] -_TOOLS: Final = [{"name": "read", "description": "Read a file", "input_schema": {"type": "object"}, "cache_control": _EPHEMERAL}] - - -def _texts_with(key: str, value: object) -> Iterator[object]: - if isinstance(value, dict): - if key in value: - yield value.get("text", value.get("name")) - for nested in value.values(): - yield from _texts_with(key, nested) - if isinstance(value, list): - for item in value: - yield from _texts_with(key, item) - - -def _mantle_wire(model_info: dict[str, object] | None) -> tuple[dict[str, object], dict[str, object]]: - completion_kwargs, _ = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( - max_tokens=64, - messages=json.loads(json.dumps(_CLAUDE_CODE_MESSAGES)), - model="bedrock_mantle/openai.gpt-5.6-sol", - system=json.loads(json.dumps(_SYSTEM)), - tools=json.loads(json.dumps(_TOOLS)), - extra_kwargs={"custom_llm_provider": "bedrock_mantle", "model_info": model_info}, - ) - wire: Final = LiteLLMResponsesTransformationHandler().transform_request( - model="openai.gpt-5.6-sol", - messages=cast(List[AllMessageValues], completion_kwargs["messages"]), # cast-ok: chat messages from the adapter - optional_params={"tools": completion_kwargs["tools"]}, - litellm_params={"custom_llm_provider": "bedrock_mantle", "model_info": model_info}, - headers={}, - litellm_logging_obj=Mock(), - ) - return dict(completion_kwargs), wire - - -def test_anthropic_messages_to_a_breakpoint_reading_target_carry_marks_where_the_openai_dialect_reads_them( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) - - completion_kwargs, wire = _mantle_wire(None) - - assert list(_texts_with("cache_control", completion_kwargs["tools"])) == [] - assert list(_texts_with("cache_control", cast(list[object], completion_kwargs["messages"])[2])) == [] - assert sorted(map(str, _texts_with("prompt_cache_breakpoint", wire["input"]))) == ["Stable prefix", "file body", "hi"] - assert list(_texts_with("cache_control", wire)) == [] - - -def test_anthropic_messages_to_a_deployment_that_opts_out_of_breakpoints_carry_no_marks( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) - - completion_kwargs, wire = _mantle_wire({"supports_prompt_cache_breakpoint": False}) - - assert list(_texts_with("cache_control", completion_kwargs)) == [] - assert list(_texts_with("prompt_cache_breakpoint", wire)) == [] diff --git a/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py index 77bdacc938f..9d08632e29b 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py @@ -1,15 +1,22 @@ import json import os import sys -from typing import Final +from collections.abc import Iterator +from typing import Final, List, cast +from unittest.mock import Mock import pytest sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../.."))) +import litellm +from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, +) from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) +from litellm.types.llms.openai import AllMessageValues MESSAGES = [{"role": "user", "content": "hello"}] CLAUDE_CODE_USER_ID = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "session-abc"}) @@ -138,3 +145,82 @@ def test_prepare_completion_kwargs_derives_no_prompt_cache_key_mistral_cannot_ta assert "prompt_cache_key" not in mistral_kwargs assert openai_kwargs["prompt_cache_key"] == "sitzung-ΓΌ" + + +_EPHEMERAL: Final = {"type": "ephemeral"} +_MANTLE_GPT_ROW: Final = { + "litellm_provider": "bedrock_mantle", + "mode": "responses", + "supports_prompt_cache_breakpoint": True, +} +_CLAUDE_CODE_MESSAGES: Final = ( + {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": _EPHEMERAL}]}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "checking", "cache_control": _EPHEMERAL}, + {"type": "tool_use", "id": "toolu_1", "name": "read", "input": {}, "cache_control": _EPHEMERAL}, + ], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "file body", "cache_control": _EPHEMERAL}], + }, +) +_SYSTEM: Final = [{"type": "text", "text": "Stable prefix", "cache_control": _EPHEMERAL}] +_TOOLS: Final = [{"name": "read", "description": "Read a file", "input_schema": {"type": "object"}, "cache_control": _EPHEMERAL}] + + +def _texts_with(key: str, value: object) -> Iterator[object]: + if isinstance(value, dict): + if key in value: + yield value.get("text", value.get("name")) + for nested in value.values(): + yield from _texts_with(key, nested) + if isinstance(value, list): + for item in value: + yield from _texts_with(key, item) + + +def _mantle_wire(model_info: dict[str, object] | None) -> tuple[dict[str, object], dict[str, object]]: + completion_kwargs, _ = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=64, + messages=json.loads(json.dumps(_CLAUDE_CODE_MESSAGES)), + model="bedrock_mantle/openai.gpt-5.6-sol", + system=json.loads(json.dumps(_SYSTEM)), + tools=json.loads(json.dumps(_TOOLS)), + extra_kwargs={"custom_llm_provider": "bedrock_mantle", "model_info": model_info}, + ) + wire: Final = LiteLLMResponsesTransformationHandler().transform_request( + model="openai.gpt-5.6-sol", + messages=cast(List[AllMessageValues], completion_kwargs["messages"]), # cast-ok: chat messages from the adapter + optional_params={"tools": completion_kwargs["tools"]}, + litellm_params={"custom_llm_provider": "bedrock_mantle", "model_info": model_info}, + headers={}, + litellm_logging_obj=Mock(), + ) + return dict(completion_kwargs), wire + + +def test_anthropic_messages_to_a_breakpoint_reading_target_carry_marks_where_the_openai_dialect_reads_them( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) + + completion_kwargs, wire = _mantle_wire(None) + + assert list(_texts_with("cache_control", completion_kwargs["tools"])) == [] + assert list(_texts_with("cache_control", cast(list[object], completion_kwargs["messages"])[2])) == [] + assert sorted(map(str, _texts_with("prompt_cache_breakpoint", wire["input"]))) == ["Stable prefix", "file body", "hi"] + assert list(_texts_with("cache_control", wire)) == [] + + +def test_anthropic_messages_to_a_deployment_that_opts_out_of_breakpoints_carry_no_marks( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) + + completion_kwargs, wire = _mantle_wire({"supports_prompt_cache_breakpoint": False}) + + assert list(_texts_with("cache_control", completion_kwargs)) == [] + assert list(_texts_with("prompt_cache_breakpoint", wire)) == [] diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 263c2f7618d..7959d9092bd 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -1056,6 +1056,19 @@ class TestCacheControlAsPromptCacheBreakpoint: {"role": "user", "content": "unmarked"}, ] + def test_gpt_5_6_rewrites_a_string_mark_and_a_mark_on_empty_text(self): + messages: list[dict[str, object]] = [ + {"role": "system", "content": "", "cache_control": _EPHEMERAL}, + {"role": "user", "content": [_text("Describe this"), {**_text("plain"), "cache_control": "ephemeral"}]}, + {"role": "user", "content": [_text("context"), _marked(_text(""))]}, + ] + body = self._transform("gpt-5.6", messages) + assert body["messages"] == [ + {"role": "system", "content": [_breakpoint(_text(""))]}, + {"role": "user", "content": [_text("Describe this"), _breakpoint(_text("plain"))]}, + {"role": "user", "content": [_text("context"), _breakpoint(_text(""))]}, + ] + def test_gpt_5_6_message_level_mark_on_list_content_marks_the_last_part(self): messages: list[dict[str, object]] = [ {"role": "user", "content": [_text("first"), _text("last")], "cache_control": _EPHEMERAL}, From 8d90603ba834b36d292da4fe626b05f431f6fa2f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:46:55 -0800 Subject: [PATCH 7/9] fix(cache): keep /v1/messages marks only where the chat route turns them into breakpoints --- .../anthropic/messages/adapters/handler.py | 13 ++++- .../adapters/test_handler_prompt_cache_key.py | 52 +++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index 329594c75d7..2d2c4680a95 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -397,9 +397,20 @@ ANTHROPIC_ADAPTER: Final = AnthropicAdapter() ######################################################## +_ROUTES_FORWARDING_CACHE_CONTROL_UNMAPPED: Final = frozenset(("azure",)) +_ROUTES_READING_ONLY_THE_SERVED_MODEL: Final = frozenset(("openai",)) + + def _target_reads_prompt_cache_breakpoint(model: str, call_kwargs: Mapping[str, object]) -> bool: + provider: Final = call_kwargs.get("custom_llm_provider") + if provider in _ROUTES_FORWARDING_CACHE_CONTROL_UNMAPPED: + return False base_model: Final = call_kwargs.get("base_model") - named_models: Final = (model, base_model) if isinstance(base_model, str) and base_model else (model,) + named_models: Final = ( + (model, base_model) + if isinstance(base_model, str) and base_model and provider not in _ROUTES_READING_ONLY_THE_SERVED_MODEL + else (model,) + ) return any(call_targets_openai_prompt_cache_breakpoint(name, call_kwargs) for name in named_models) diff --git a/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py index 9d08632e29b..11753a49aea 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py @@ -224,3 +224,55 @@ def test_anthropic_messages_to_a_deployment_that_opts_out_of_breakpoints_carry_n assert list(_texts_with("cache_control", completion_kwargs)) == [] assert list(_texts_with("prompt_cache_breakpoint", wire)) == [] + + +_OPENAI_GPT_ROW: Final = {"litellm_provider": "openai", "mode": "chat", "supports_prompt_cache_breakpoint": True} + + +@pytest.mark.parametrize( + ("model", "extra_kwargs", "keeps_marks"), + [ + pytest.param( + "azure/gpt-5.6-sol", + {"custom_llm_provider": "azure", "model_info": {"supports_prompt_cache_breakpoint": True}}, + False, + id="azure-chat-forwards-cache-control-unmapped", + ), + pytest.param( + "openai/team-alias", + { + "custom_llm_provider": "openai", + "base_model": "gpt-5.6-sol", + "api_base": "https://llm.example.com/v1", + "prompt_cache_options": {"mode": "explicit"}, + }, + False, + id="openai-chat-reads-only-the-served-model", + ), + pytest.param( + "bedrock_mantle/team-alias", + { + "custom_llm_provider": "bedrock_mantle", + "base_model": "openai.gpt-5.6-sol", + "prompt_cache_options": {"mode": "explicit"}, + }, + True, + id="bridge-reads-base-model", + ), + ], +) +def test_anthropic_messages_keep_marks_only_for_a_route_that_turns_them_into_breakpoints( + monkeypatch: pytest.MonkeyPatch, model: str, extra_kwargs: dict[str, object], keeps_marks: bool +) -> None: + monkeypatch.setitem(litellm.model_cost, "gpt-5.6-sol", _OPENAI_GPT_ROW) + monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) + + completion_kwargs, _ = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=64, + messages=json.loads(json.dumps(_CLAUDE_CODE_MESSAGES)), + model=model, + system=json.loads(json.dumps(_SYSTEM)), + extra_kwargs=extra_kwargs, + ) + + assert bool(list(_texts_with("cache_control", completion_kwargs["messages"]))) is keeps_marks From b6a67f8120e4b21c2f851c067b17a81f364be21f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 6 Nov 2026 01:33:12 -0800 Subject: [PATCH 8/9] fix(cache): keep /v1/messages marks only where the request's route turns them into breakpoints --- .../anthropic/messages/adapters/handler.py | 39 +++++++++---- .../llms/anthropic/messages/native/handler.py | 20 ++----- litellm/llms/azure_ai/chat/transformation.py | 4 +- litellm/responses/bridge_check.py | 9 +++ .../adapters/test_handler_prompt_cache_key.py | 55 ++++++++++++++++++- 5 files changed, 100 insertions(+), 27 deletions(-) diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index 2d2c4680a95..ce759d49736 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -12,7 +12,10 @@ from typing_extensions import TypedDict import litellm from litellm._logging import verbose_logger -from litellm.integrations.anthropic_cache_control_hook import call_targets_openai_prompt_cache_breakpoint +from litellm.integrations.anthropic_cache_control_hook import ( + call_targets_openai_prompt_cache_breakpoint, + routed_model_info, +) from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject @@ -45,6 +48,8 @@ from litellm.llms.anthropic.messages.utils import ( proxy_sub_call_logging_kwargs, proxy_sub_call_trace_id, ) +from litellm.llms.azure_ai.chat.transformation import deployment_reads_prompt_cache_breakpoints +from litellm.responses.bridge_check import responses_bridge_model from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, @@ -397,21 +402,35 @@ ANTHROPIC_ADAPTER: Final = AnthropicAdapter() ######################################################## -_ROUTES_FORWARDING_CACHE_CONTROL_UNMAPPED: Final = frozenset(("azure",)) -_ROUTES_READING_ONLY_THE_SERVED_MODEL: Final = frozenset(("openai",)) +_CHAT_ROUTES_RENAMING_MARKS_FOR_THE_SERVED_MODEL: Final = frozenset(("openai", "bedrock_mantle")) + + +def _deployment_base_model(call_kwargs: Mapping[str, object]) -> str | None: + model_info: Final = routed_model_info(call_kwargs.get("model_info")) + base_model: Final = call_kwargs.get("base_model") or (None if model_info is None else model_info.get("base_model")) + return base_model if isinstance(base_model, str) and base_model else None def _target_reads_prompt_cache_breakpoint(model: str, call_kwargs: Mapping[str, object]) -> bool: provider: Final = call_kwargs.get("custom_llm_provider") - if provider in _ROUTES_FORWARDING_CACHE_CONTROL_UNMAPPED: + if not isinstance(provider, str): return False - base_model: Final = call_kwargs.get("base_model") - named_models: Final = ( - (model, base_model) - if isinstance(base_model, str) and base_model and provider not in _ROUTES_READING_ONLY_THE_SERVED_MODEL - else (model,) + served_model: Final = model.removeprefix(f"{provider}/") + base_model: Final = _deployment_base_model(call_kwargs) + served_model_reads: Final = call_targets_openai_prompt_cache_breakpoint( + served_model.removeprefix("responses/"), call_kwargs ) - return any(call_targets_openai_prompt_cache_breakpoint(name, call_kwargs) for name in named_models) + if not served_model_reads and ( + base_model is None or not call_targets_openai_prompt_cache_breakpoint(base_model, call_kwargs) + ): + return False + if responses_bridge_model(served_model, provider, call_kwargs.get("model_info")) is not None: + return True + if provider == "azure_ai": + return deployment_reads_prompt_cache_breakpoints( + served_model, {"model_info": call_kwargs.get("model_info"), "base_model": base_model} + ) + return served_model_reads and provider in _CHAT_ROUTES_RENAMING_MARKS_FOR_THE_SERVED_MODEL def _record_bridged_model_response(logging_obj: LiteLLMLoggingObject | None, completion_response: object) -> None: diff --git a/litellm/llms/anthropic/messages/native/handler.py b/litellm/llms/anthropic/messages/native/handler.py index 922de386fe4..c70f30ad41c 100644 --- a/litellm/llms/anthropic/messages/native/handler.py +++ b/litellm/llms/anthropic/messages/native/handler.py @@ -69,17 +69,6 @@ _SAFEGUARD_KWARGS: Final = TypeAdapter(Mapping[str, object]) _SAFEGUARD_MESSAGES: Final = TypeAdapter(Sequence[Mapping[str, object]]) -def _bridges_to_responses_api(model: str, custom_llm_provider: str, deployment_model_info: object) -> bool: - from litellm.responses.bridge_check import responses_api_bridge_check, str_keyed_mapping - - model_info, _ = responses_api_bridge_check( - model=model, - custom_llm_provider=custom_llm_provider, - deployment_model_info=str_keyed_mapping(deployment_model_info), - ) - return model_info.get("mode") == "responses" - - def _responses_mode_is_lost_by_prefix_strip( requested_model: str, resolved_model: str, custom_llm_provider: str, deployment_model_info: object ) -> bool: @@ -89,11 +78,14 @@ def _responses_mode_is_lost_by_prefix_strip( deployment id such as ``perplexity/perplexity/sonar`` (mode ``responses``) is shadowed by the chat entry ``perplexity/sonar`` and would otherwise be sent to chat/completions. """ + from litellm.responses.bridge_check import responses_bridge_model + if requested_model == resolved_model: return False - return _bridges_to_responses_api( - requested_model, custom_llm_provider, deployment_model_info - ) and not _bridges_to_responses_api(resolved_model, custom_llm_provider, deployment_model_info) + return ( + responses_bridge_model(requested_model, custom_llm_provider, deployment_model_info) is not None + and responses_bridge_model(resolved_model, custom_llm_provider, deployment_model_info) is None + ) @dataclass(frozen=True, slots=True) diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 93ff6d8170a..592c2e1c7e9 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -130,7 +130,7 @@ def _with_breakpoints_for_foundry(message: AllMessageValues, drop_params: bool) return cast(AllMessageValues, foundry_message) # cast-ok: the same message, its breakpoints in Foundry's shape -def _deployment_reads_breakpoints(model: str, litellm_params: Mapping[str, object]) -> bool: +def deployment_reads_prompt_cache_breakpoints(model: str, litellm_params: Mapping[str, object]) -> bool: model_info: Final = routed_model_info(litellm_params.get("model_info")) declared: Final = declared_prompt_cache_breakpoint(model_info) if declared is not None: @@ -439,7 +439,7 @@ class AzureAIStudioConfig(OpenAIConfig): "messages": self._transform_messages( messages, model, - _deployment_reads_breakpoints(model, _FIELDS.validate_python(litellm_params)), + deployment_reads_prompt_cache_breakpoints(model, _FIELDS.validate_python(litellm_params)), bool(litellm_params.get("drop_params") or litellm.drop_params), ), **optional_params, diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index 826de8cfdc9..d8eda42b7cf 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -102,6 +102,15 @@ def str_keyed_mapping(value: object) -> Mapping[str, object] | None: ) +def responses_bridge_model(model: str, custom_llm_provider: str, deployment_model_info: object) -> str | None: + model_info, bridged_model = responses_api_bridge_check( + model=model, + custom_llm_provider=custom_llm_provider, + deployment_model_info=str_keyed_mapping(deployment_model_info), + ) + return bridged_model if model_info.get("mode") == "responses" else None + + def bridge_reasoning_effort( reasoning_effort: str | Mapping[str, object] | None, extra_body: object ) -> str | Mapping[str, object] | None: diff --git a/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py index 11753a49aea..1021888ed98 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py @@ -227,6 +227,7 @@ def test_anthropic_messages_to_a_deployment_that_opts_out_of_breakpoints_carry_n _OPENAI_GPT_ROW: Final = {"litellm_provider": "openai", "mode": "chat", "supports_prompt_cache_breakpoint": True} +_FOUNDRY_GPT_ROW: Final = {"litellm_provider": "azure_ai", "mode": "chat", "supports_prompt_cache_breakpoint": True} @pytest.mark.parametrize( @@ -249,6 +250,33 @@ _OPENAI_GPT_ROW: Final = {"litellm_provider": "openai", "mode": "chat", "support False, id="openai-chat-reads-only-the-served-model", ), + pytest.param( + "openai/responses/team-alias", + { + "custom_llm_provider": "openai", + "base_model": "gpt-5.6-sol", + "api_base": "https://llm.example.com/v1", + "prompt_cache_options": {"mode": "explicit"}, + }, + True, + id="openai-responses-alias-bridge-reads-base-model", + ), + pytest.param( + "azure/responses/gpt-5.6-sol", + {"custom_llm_provider": "azure", "model_info": {"supports_prompt_cache_breakpoint": True}}, + True, + id="azure-responses-bridge-renames-marks", + ), + pytest.param( + "bedrock_mantle/team-alias", + { + "custom_llm_provider": "bedrock_mantle", + "model_info": {"mode": "responses", "base_model": "openai.gpt-5.6-sol"}, + "prompt_cache_options": {"mode": "explicit"}, + }, + True, + id="pinned-bridge-reads-model-info-base-model", + ), pytest.param( "bedrock_mantle/team-alias", { @@ -256,8 +284,32 @@ _OPENAI_GPT_ROW: Final = {"litellm_provider": "openai", "mode": "chat", "support "base_model": "openai.gpt-5.6-sol", "prompt_cache_options": {"mode": "explicit"}, }, + False, + id="mantle-chat-reads-only-the-served-model", + ), + pytest.param( + "litellm_proxy/gpt-5.6-sol", + { + "custom_llm_provider": "litellm_proxy", + "model_info": {"supports_prompt_cache_breakpoint": True, "litellm_provider": "litellm_proxy"}, + }, + False, + id="litellm-proxy-chat-forwards-cache-control-unmapped", + ), + pytest.param( + "sap/gpt-5.6-sol", + { + "custom_llm_provider": "sap", + "model_info": {"supports_prompt_cache_breakpoint": True, "litellm_provider": "sap"}, + }, + False, + id="sap-chat-forwards-cache-control-unmapped", + ), + pytest.param( + "azure_ai/team-alias", + {"custom_llm_provider": "azure_ai", "model_info": {"base_model": "gpt-5.6-sol"}}, True, - id="bridge-reads-base-model", + id="foundry-chat-reads-model-info-base-model", ), ], ) @@ -265,6 +317,7 @@ def test_anthropic_messages_keep_marks_only_for_a_route_that_turns_them_into_bre monkeypatch: pytest.MonkeyPatch, model: str, extra_kwargs: dict[str, object], keeps_marks: bool ) -> None: monkeypatch.setitem(litellm.model_cost, "gpt-5.6-sol", _OPENAI_GPT_ROW) + litellm.register_model({"azure_ai/gpt-5.6-sol": _FOUNDRY_GPT_ROW}) monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) completion_kwargs, _ = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( From 25536769e9029aebcd0240a4ea5df40337debe28 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 6 Nov 2026 04:57:19 -0800 Subject: [PATCH 9/9] test(cache): pin that a base model declared in model_info reaches the Responses bridge --- ...responses_transformation_transformation.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 551ac73eb64..6401fcc7c9a 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -6334,6 +6334,46 @@ def test_transform_request_maps_client_cache_control_for_an_alias_whose_base_mod assert marked == {("system", "Stable prefix"), ("user", "hi")} +def _completion_marked_texts(model: str, model_info: Mapping[str, object]) -> set[tuple[object, object]]: + response_body: Final = { + "id": "resp_alias_marks", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": "gpt-5.6", + "output": [ + { + "type": "message", + "id": "msg_alias_marks", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + } + with respx.mock(assert_all_called=True) as router: + upstream: Final = router.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=response_body) + ) + litellm.completion( + model=model, + messages=_client_marked_messages(), + api_key="sk-test", + model_info=model_info, + cache={"no-cache": True}, + ) + request_body: Final = json.loads(upstream.calls[0].request.content) + return {(role, text) for role, text, _ in _marked_parts_of(request_body["input"])} + + +def test_completion_maps_client_cache_control_for_an_alias_whose_base_model_is_declared_in_model_info() -> None: + served_model_marks: Final = _completion_marked_texts("openai/responses/gpt-5.6", {}) + + assert served_model_marks + assert _completion_marked_texts("openai/responses/team-alias", {"base_model": "gpt-5.6"}) == served_model_marks + + def test_transform_request_leaves_client_cache_control_unmapped_for_a_deployment_that_does_not_read_breakpoints( monkeypatch: pytest.MonkeyPatch, ) -> None: