Merge 25536769e9 into febbe38326

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -34,7 +34,13 @@ from litellm.completion_extras.litellm_responses_transformation.history import (
     replay_items,
     requested_phase,
 )
-from litellm.integrations.anthropic_cache_control_hook import routed_model_info, supports_openai_prompt_cache_breakpoint
+from litellm.integrations.anthropic_cache_control_hook import (
+    AnthropicCacheControlHook,
+    holds_prompt_cache_breakpoint,
+    message_carries_prompt_cache_breakpoint,
+    routed_model_info,
+    supports_openai_prompt_cache_breakpoint,
+)
 from litellm.litellm_core_utils.allowed_tools_tool_choice import (
     parse_chat_allowed_tools_tool_choice,
     responses_allowed_tools_tool_choice,
@@ -235,6 +241,31 @@ def _drops_audio_input(model: str, litellm_params: Mapping[str, object]) -> bool
     return not _supports_audio_input(model, litellm_params)
 
 
+def _messages_in_cache_mark_dialect(
+    named_models: Sequence[str],
+    messages: Sequence["AllMessageValues"],
+    optional_params: Mapping[str, object],
+    litellm_params: Mapping[str, object],
+    model_info: Mapping[str, object] | None,
+) -> Sequence["AllMessageValues"]:
+    reads_breakpoints: Final = any(
+        AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
+            name,
+            _served_provider(litellm_params),
+            litellm_params.get("api_base"),
+            optional_params.get("prompt_cache_options"),
+            model_info,
+        )
+        for name in named_models
+    )
+    if not reads_breakpoints:
+        return messages
+    return tuple(
+        AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message, holds_prompt_cache_breakpoint)
+        for message in messages
+    )
+
+
 def responses_provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]:
     return MappingProxyType(
         {
@@ -273,6 +304,12 @@ def _get_reasoning_items(
     return []
 
 
+def _folds_into_instructions(msg: "AllMessageValues", keep_prompt_cache_breakpoints: bool) -> bool:
+    return msg.get("role") == "system" and not (
+        keep_prompt_cache_breakpoints and message_carries_prompt_cache_breakpoint(msg)
+    )
+
+
 def _reasoning_input_items(msg: "AllMessageValues") -> list[dict[str, object]]:  # mutable-ok: API message payload
     """Reasoning input items for an assistant message.
 
@@ -871,7 +908,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
     def convert_chat_completion_messages_to_responses_api(
         self,
-        messages: list["AllMessageValues"],
+        messages: Sequence["AllMessageValues"],
         *,
         drop_params: bool = False,
         target_trims_breakpoints: bool = True,
@@ -884,6 +921,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             messages,
             drop_params=drop_params,
             target_trims_breakpoints=target_trims_breakpoints,
+            keep_prompt_cache_breakpoints=keep_prompt_cache_breakpoints,
             replay_tool_search=replay_tool_search,
             include_namespace=include_namespace,
             replay_openai_history=replay_openai_history,
@@ -897,10 +935,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
     def _convert_chat_completion_messages_to_responses_input(
         self,
-        messages: list["AllMessageValues"],
+        messages: Sequence["AllMessageValues"],
         *,
         drop_params: bool = False,
         target_trims_breakpoints: bool = True,
+        keep_prompt_cache_breakpoints: bool = False,
         replay_tool_search: bool = False,
         include_namespace: bool = False,
         replay_openai_history: bool = False,
@@ -924,8 +963,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             and isinstance(tool_call.get("custom"), dict)
         )
 
-        leading_system_count: Final = next(
-            (index for index, msg in enumerate(messages) if msg.get("role") != "system"),
+        leading_folded_system_count: Final = next(
+            (
+                index
+                for index, msg in enumerate(messages)
+                if not _folds_into_instructions(msg, keep_prompt_cache_breakpoints)
+            ),
             len(messages),
         )
 
@@ -939,7 +982,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                 input_items.extend(replayed_turn.items)
             elif role == "system":
                 # Extract system message as instructions
-                if isinstance(content, str) and index < leading_system_count:
+                if isinstance(content, str) and index < leading_folded_system_count:
                     if instructions:
                         # Concatenate multiple system prompts with a space
                         instructions = f"{instructions} {content}"
@@ -1174,7 +1217,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             supports_openai_prompt_cache_breakpoint(name, provider, model_info) for name in named_models
         )
         converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api(
-            messages,
+            _messages_in_cache_mark_dialect(
+                named_models,
+                messages,
+                cast(Mapping[str, object], optional_params),  # cast-ok: upcast of the untyped params dict
+                litellm_params,
+                model_info,
+            ),
             drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params),
             target_trims_breakpoints=drops_unsupported_params(litellm_params.get("drop_params")),
             keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint,

litellm/integrations/anthropic_cache_control_hook.py

@@ -326,8 +326,8 @@ def whole_messages_within_cache_control_limit(
     return _messages_within_budget(messages, reserved_marks, _whole_message_kept)
 
 
-def _part_with_cache_control_renamed(part: object) -> object:
-    if not _accepts_prompt_cache_breakpoint(part):
+def _part_with_cache_control_renamed(part: object, markable: Callable[[object], bool]) -> object:
+    if not markable(part) or _attribute_or_key(part, "cache_control") is None:
         return part
     return cache_control_as_prompt_cache_breakpoint(_CONTENT_PART_ADAPTER.validate_python(part))
 
@@ -694,6 +694,10 @@ def _accepts_prompt_cache_breakpoint(block: object) -> bool:
     return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES
 
 
+def holds_prompt_cache_breakpoint(block: object) -> bool:
+    return _accepts_prompt_cache_breakpoint(block) and _attribute_or_key(block, "text") != ""
+
+
 def _carries_prompt_cache_breakpoint(block: object) -> bool:
     return _accepts_prompt_cache_breakpoint(block) and _attribute_or_key(block, "prompt_cache_breakpoint") is not None
 
@@ -702,10 +706,12 @@ def _may_carry_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool:
     return message.get("role") != "assistant"
 
 
+def message_carries_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool:
+    return any(_carries_prompt_cache_breakpoint(part) for part in _as_object_list(message.get("content")) or ())
+
+
 def _tool_result_carries_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool:
-    return message.get("role") == "tool" and any(
-        _carries_prompt_cache_breakpoint(part) for part in _as_object_list(message.get("content")) or ()
-    )
+    return message.get("role") == "tool" and message_carries_prompt_cache_breakpoint(message)
 
 
 def _part_without_prompt_cache_breakpoint(part: object) -> object:
@@ -1236,7 +1242,9 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         return message
 
     @staticmethod
-    def cache_control_as_prompt_cache_breakpoints(message: AllMessageValues) -> AllMessageValues:
+    def cache_control_as_prompt_cache_breakpoints(
+        message: AllMessageValues, markable: Callable[[object], bool] = _accepts_prompt_cache_breakpoint
+    ) -> AllMessageValues:
         """Every Anthropic ``cache_control`` on the message or its parts moved to OpenAI's ``prompt_cache_breakpoint``,
         placed where the injection hook would put one. Marks with no OpenAI slot are left for the caller to strip."""
         fields: Final[Mapping[str, object]] = dict(message)
@@ -1250,13 +1258,13 @@ class AnthropicCacheControlHook(CustomPromptManagement):
                 AllMessageValues,
                 {
                     **fields,
-                    "content": [_part_with_cache_control_renamed(part) for part in parts],
+                    "content": [_part_with_cache_control_renamed(part, markable) for part in parts],
                 },
             )
         )
         if fields.get("cache_control") is None:
             return renamed
-        return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(renamed)
+        return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(renamed, markable)
 
     @staticmethod
     def prompt_cache_breakpoints_as_cache_control(message: AllMessageValues, drop_params: bool) -> AllMessageValues:
@@ -1345,25 +1353,22 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         return capped_request(max(0, first_over_cap - 1))
 
     @staticmethod
-    def _insert_prompt_cache_breakpoint_in_message(message: AllMessageValues) -> AllMessageValues:
+    def _insert_prompt_cache_breakpoint_in_message(
+        message: AllMessageValues, markable: Callable[[object], bool] = _accepts_prompt_cache_breakpoint
+    ) -> AllMessageValues:
         if not _may_carry_prompt_cache_breakpoint(message):
             return message
         message_content: Final = message.get("content", None)
         if isinstance(message_content, str):
+            text_part: Final = ChatCompletionTextObject(type="text", text=message_content)
+            if not markable(text_part):
+                return message
             marked: Final = copy.copy(message)
-            marked["content"] = [
-                with_prompt_cache_breakpoint(
-                    ChatCompletionTextObject(type="text", text=message_content), PromptCacheBreakpoint(mode="explicit")
-                )
-            ]
+            marked["content"] = [with_prompt_cache_breakpoint(text_part, PromptCacheBreakpoint(mode="explicit"))]
             return marked
         if isinstance(message_content, list):
             target_index: Final = next(
-                (
-                    index
-                    for index in range(len(message_content) - 1, -1, -1)
-                    if _accepts_prompt_cache_breakpoint(message_content[index])
-                ),
+                (index for index in range(len(message_content) - 1, -1, -1) if markable(message_content[index])),
                 None,
             )
             if target_index is not None:
@@ -2003,3 +2008,14 @@ def messages_within_reserved_cache_control_cap(
             MAX_CACHE_CONTROL_BLOCKS,
         )
     return kept
+
+
+def call_targets_openai_prompt_cache_breakpoint(model: str, call_kwargs: Mapping[str, object]) -> bool:
+    provider: Final = call_kwargs.get("custom_llm_provider")
+    return AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
+        model,
+        provider if isinstance(provider, str) else None,
+        call_kwargs.get("api_base") or call_kwargs.get("base_url"),
+        call_kwargs.get("prompt_cache_options"),
+        routed_model_info(call_kwargs.get("model_info")),
+    )

litellm/llms/anthropic/messages/adapters/handler.py

@@ -12,6 +12,10 @@ from typing_extensions import TypedDict
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.integrations.anthropic_cache_control_hook import (
+    call_targets_openai_prompt_cache_breakpoint,
+    routed_model_info,
+)
 from litellm.litellm_core_utils.asyncify import run_async_function
 from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
@@ -44,6 +48,8 @@ from litellm.llms.anthropic.messages.utils import (
     proxy_sub_call_logging_kwargs,
     proxy_sub_call_trace_id,
 )
+from litellm.llms.azure_ai.chat.transformation import deployment_reads_prompt_cache_breakpoints
+from litellm.responses.bridge_check import responses_bridge_model
 from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
 from litellm.types.llms.anthropic_messages.anthropic_response import (
     AnthropicMessagesResponse,
@@ -396,6 +402,37 @@ ANTHROPIC_ADAPTER: Final = AnthropicAdapter()
 ########################################################
 
 
+_CHAT_ROUTES_RENAMING_MARKS_FOR_THE_SERVED_MODEL: Final = frozenset(("openai", "bedrock_mantle"))
+
+
+def _deployment_base_model(call_kwargs: Mapping[str, object]) -> str | None:
+    model_info: Final = routed_model_info(call_kwargs.get("model_info"))
+    base_model: Final = call_kwargs.get("base_model") or (None if model_info is None else model_info.get("base_model"))
+    return base_model if isinstance(base_model, str) and base_model else None
+
+
+def _target_reads_prompt_cache_breakpoint(model: str, call_kwargs: Mapping[str, object]) -> bool:
+    provider: Final = call_kwargs.get("custom_llm_provider")
+    if not isinstance(provider, str):
+        return False
+    served_model: Final = model.removeprefix(f"{provider}/")
+    base_model: Final = _deployment_base_model(call_kwargs)
+    served_model_reads: Final = call_targets_openai_prompt_cache_breakpoint(
+        served_model.removeprefix("responses/"), call_kwargs
+    )
+    if not served_model_reads and (
+        base_model is None or not call_targets_openai_prompt_cache_breakpoint(base_model, call_kwargs)
+    ):
+        return False
+    if responses_bridge_model(served_model, provider, call_kwargs.get("model_info")) is not None:
+        return True
+    if provider == "azure_ai":
+        return deployment_reads_prompt_cache_breakpoints(
+            served_model, {"model_info": call_kwargs.get("model_info"), "base_model": base_model}
+        )
+    return served_model_reads and provider in _CHAT_ROUTES_RENAMING_MARKS_FOR_THE_SERVED_MODEL
+
+
 def _record_bridged_model_response(logging_obj: LiteLLMLoggingObject | None, completion_response: object) -> None:
     if logging_obj is not None and isinstance(completion_response, ModelResponse):
         logging_obj.record_bridged_model_response(completion_response)
@@ -678,6 +715,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
             api_base=api_base if isinstance(api_base, str) else None,
             deployment_model_info=as_str_mapping(provider_kwargs.get("model_info")),
             drop_params=drop_params,
+            target_reads_prompt_cache_breakpoint=_target_reads_prompt_cache_breakpoint(model, extra_kwargs),
         )
 
         if openai_request is None:

litellm/llms/anthropic/messages/adapters/transformation.py

@@ -318,6 +318,7 @@ class AnthropicAdapter:
         api_base: str | None = None,
         deployment_model_info: Mapping[str, object] | None = None,
         drop_params: bool = True,
+        target_reads_prompt_cache_breakpoint: bool = False,
     ) -> tuple[ChatCompletionRequest | None, dict[str, str]]:
         """
         Translate Anthropic request params to OpenAI format, returning tool name mapping.
@@ -348,7 +349,9 @@ class AnthropicAdapter:
         (
             translated_body,
             tool_name_mapping,
-        ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
+        ) = LiteLLMAnthropicMessagesAdapter(
+            target_reads_prompt_cache_breakpoint=target_reads_prompt_cache_breakpoint
+        ).translate_anthropic_to_openai(
             anthropic_message_request=request_body,
             custom_llm_provider=custom_llm_provider,
             api_base=api_base,
@@ -434,8 +437,9 @@ _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS: Final = frozenset({LlmProviders.KYMA.va
 
 
 class LiteLLMAnthropicMessagesAdapter:
-    def __init__(self, *, keep_cache_control: bool = False) -> None:
+    def __init__(self, *, keep_cache_control: bool = False, target_reads_prompt_cache_breakpoint: bool = False) -> None:
         self.keep_cache_control: Final = keep_cache_control
+        self._target_reads_prompt_cache_breakpoint: Final = target_reads_prompt_cache_breakpoint
 
     ### FOR [BETA] `/v1/messages` endpoint support
 
@@ -473,6 +477,8 @@ class LiteLLMAnthropicMessagesAdapter:
         source: object,
         target: object,
         model: str | None,
+        *,
+        has_prompt_cache_breakpoint_slot: bool = True,
     ) -> None:
         """
         Extract cache_control from source and add to target if it should be preserved.
@@ -486,12 +492,15 @@ class LiteLLMAnthropicMessagesAdapter:
             source: Dict or TypedDict containing potential cache_control field
             target: Dict or TypedDict to add cache_control to
             model: Model name to check if cache_control should be preserved
+            has_prompt_cache_breakpoint_slot: Whether the OpenAI cache dialect reads a breakpoint on this block
         """
         # TypedDict objects are dicts at runtime, so .get() works
         cache_control: Final = (
             source.get("cache_control") if isinstance(source, dict) else getattr(source, "cache_control", None)
         )
-        if cache_control and (self.keep_cache_control or (model and self.target_consumes_cache_control(model))):
+        if cache_control and (
+            self.keep_cache_control or self._target_keeps_mark(model, has_prompt_cache_breakpoint_slot)
+        ):
             # TypedDict objects support dict operations at runtime
             # Use type ignore consistent with codebase pattern (see anthropic/chat/transformation.py:432)
             if isinstance(target, dict):
@@ -500,6 +509,12 @@ class LiteLLMAnthropicMessagesAdapter:
                 # Fallback for non-dict objects (shouldn't happen in practice)
                 cast(dict[str, object], target)["cache_control"] = cache_control
 
+    def _target_keeps_mark(self, model: str | None, has_prompt_cache_breakpoint_slot: bool) -> bool:
+        if not model:
+            return False
+        reads_as_breakpoint: Final = self._target_reads_prompt_cache_breakpoint and has_prompt_cache_breakpoint_slot
+        return reads_as_breakpoint or self.target_consumes_cache_control(model)
+
     @staticmethod
     def _add_prompt_cache_breakpoint_if_present(source: object, target: _BlockT) -> _BlockT:
         if isinstance(source, dict) and "prompt_cache_breakpoint" in source:
@@ -651,7 +666,9 @@ class LiteLLMAnthropicMessagesAdapter:
                                     "type": "text",
                                     "text": content.get("text", ""),
                                 }
-                                self._add_cache_control_if_applicable(content, text_block, model)
+                                self._add_cache_control_if_applicable(
+                                    content, text_block, model, has_prompt_cache_breakpoint_slot=False
+                                )
                                 if "cache_control" in text_block:
                                     has_cache_control_in_text = True
                                 assistant_content_list.append(text_block)
@@ -678,7 +695,9 @@ class LiteLLMAnthropicMessagesAdapter:
                                     type="function",
                                     function=function_chunk,
                                 )
-                                self._add_cache_control_if_applicable(content, tool_call, model)
+                                self._add_cache_control_if_applicable(
+                                    content, tool_call, model, has_prompt_cache_breakpoint_slot=False
+                                )
                                 tool_calls.append(tool_call)
                             elif content.get("type") == "thinking":
                                 # Anthropic's schema has no cache_control on thinking or
@@ -942,7 +961,7 @@ class LiteLLMAnthropicMessagesAdapter:
                 if k not in mapped_tool_params:  # pass additional computer kwargs
                     function_chunk.setdefault("parameters", {}).update({k: v})
             tool_param = _chat_tool_param(function_chunk, tool)
-            self._add_cache_control_if_applicable(tool, tool_param, model)
+            self._add_cache_control_if_applicable(tool, tool_param, model, has_prompt_cache_breakpoint_slot=False)
             new_tools.append(tool_param)
 
         return new_tools, tool_name_mapping

litellm/llms/anthropic/messages/native/handler.py

@@ -69,17 +69,6 @@ _SAFEGUARD_KWARGS: Final = TypeAdapter(Mapping[str, object])
 _SAFEGUARD_MESSAGES: Final = TypeAdapter(Sequence[Mapping[str, object]])
 
 
-def _bridges_to_responses_api(model: str, custom_llm_provider: str, deployment_model_info: object) -> bool:
-    from litellm.responses.bridge_check import responses_api_bridge_check, str_keyed_mapping
-
-    model_info, _ = responses_api_bridge_check(
-        model=model,
-        custom_llm_provider=custom_llm_provider,
-        deployment_model_info=str_keyed_mapping(deployment_model_info),
-    )
-    return model_info.get("mode") == "responses"
-
-
 def _responses_mode_is_lost_by_prefix_strip(
     requested_model: str, resolved_model: str, custom_llm_provider: str, deployment_model_info: object
 ) -> bool:
@@ -89,11 +78,14 @@ def _responses_mode_is_lost_by_prefix_strip(
     deployment id such as ``perplexity/perplexity/sonar`` (mode ``responses``) is shadowed by the
     chat entry ``perplexity/sonar`` and would otherwise be sent to chat/completions.
     """
+    from litellm.responses.bridge_check import responses_bridge_model
+
     if requested_model == resolved_model:
         return False
-    return _bridges_to_responses_api(
-        requested_model, custom_llm_provider, deployment_model_info
-    ) and not _bridges_to_responses_api(resolved_model, custom_llm_provider, deployment_model_info)
+    return (
+        responses_bridge_model(requested_model, custom_llm_provider, deployment_model_info) is not None
+        and responses_bridge_model(resolved_model, custom_llm_provider, deployment_model_info) is None
+    )
 
 
 @dataclass(frozen=True, slots=True)

litellm/llms/anthropic/messages/responses_adapters/handler.py

@@ -12,7 +12,7 @@ from pydantic import TypeAdapter, ValidationError
 import litellm
 from litellm._logging import verbose_logger
 from litellm.integrations.anthropic_cache_control_hook import (
-    AnthropicCacheControlHook,
+    call_targets_openai_prompt_cache_breakpoint,
     routed_model_info,
     supports_openai_prompt_cache_breakpoint,
 )
@@ -181,17 +181,6 @@ def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: objec
     return responses_config is not None and "include" in responses_config.get_supported_openai_params(provider_model)
 
 
-def _reads_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[str, object]) -> bool:
-    provider: Final = forwarded_kwargs.get("custom_llm_provider")
-    return AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
-        model,
-        provider if isinstance(provider, str) else None,
-        forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url"),
-        forwarded_kwargs.get("prompt_cache_options"),
-        routed_model_info(forwarded_kwargs.get("model_info")),
-    )
-
-
 def _model_takes_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[str, object]) -> bool:
     provider: Final = forwarded_kwargs.get("custom_llm_provider")
     provider_name: Final = provider if isinstance(provider, str) else None
@@ -221,7 +210,7 @@ def _prompt_cache_block_rewrite(model: str, forwarded_kwargs: Mapping[str, objec
     """Client cache marks become breakpoints for a deployment that reads them, and a model that takes no breakpoints
     drops raw ones under drop_params."""
     drops_params: Final = drops_unsupported_params(forwarded_kwargs.get("drop_params"))
-    if _reads_prompt_cache_breakpoint(model, forwarded_kwargs):
+    if call_targets_openai_prompt_cache_breakpoint(model, forwarded_kwargs):
         return (
             _cache_control_as_wire_safe_prompt_cache_breakpoint
             if drops_params

litellm/llms/azure_ai/chat/transformation.py

@@ -130,7 +130,7 @@ def _with_breakpoints_for_foundry(message: AllMessageValues, drop_params: bool)
     return cast(AllMessageValues, foundry_message)  # cast-ok: the same message, its breakpoints in Foundry's shape
 
 
-def _deployment_reads_breakpoints(model: str, litellm_params: Mapping[str, object]) -> bool:
+def deployment_reads_prompt_cache_breakpoints(model: str, litellm_params: Mapping[str, object]) -> bool:
     model_info: Final = routed_model_info(litellm_params.get("model_info"))
     declared: Final = declared_prompt_cache_breakpoint(model_info)
     if declared is not None:
@@ -439,7 +439,7 @@ class AzureAIStudioConfig(OpenAIConfig):
             "messages": self._transform_messages(
                 messages,
                 model,
-                _deployment_reads_breakpoints(model, _FIELDS.validate_python(litellm_params)),
+                deployment_reads_prompt_cache_breakpoints(model, _FIELDS.validate_python(litellm_params)),
                 bool(litellm_params.get("drop_params") or litellm.drop_params),
             ),
             **optional_params,

litellm/responses/bridge_check.py

@@ -102,6 +102,15 @@ def str_keyed_mapping(value: object) -> Mapping[str, object] | None:
     )
 
 
+def responses_bridge_model(model: str, custom_llm_provider: str, deployment_model_info: object) -> str | None:
+    model_info, bridged_model = responses_api_bridge_check(
+        model=model,
+        custom_llm_provider=custom_llm_provider,
+        deployment_model_info=str_keyed_mapping(deployment_model_info),
+    )
+    return bridged_model if model_info.get("mode") == "responses" else None
+
+
 def bridge_reasoning_effort(
     reasoning_effort: str | Mapping[str, object] | None, extra_body: object
 ) -> str | Mapping[str, object] | None:

tests/integration/providers/test_bedrock_mantle_gpt_prompt_cache_breakpoint_wire.py

@@ -64,6 +64,7 @@ from integration.providers._mantle_gpt_prompt_cache_support import (
     spend_rows,
     success_row,
     system_item,
+    user_item,
 )
 from pydantic import JsonValue
 
@@ -540,3 +541,55 @@ def test_f4_a_slow_mantle_stream_during_a_burst_completes_with_every_id_landing_
         name: Final = mantle_deployment(gateway, scenario, wire)
         wire.drain()
         assert_burst_landed(wire, name, burst(gateway, name, 12), marked=True)
+
+
+def client_marked_chat_body(name: str, prompt: str, *, stream: bool) -> dict[str, JsonValue]:
+    return request_body(
+        "chat",
+        name,
+        prompt,
+        stream=stream,
+        messages=[
+            {"role": "system", "content": SYSTEM, "cache_control": EPHEMERAL},
+            {"role": "user", "content": prompt},
+        ],
+    )
+
+
+@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
+def test_g1_a_clients_cache_control_on_a_string_system_message_reaches_mantle_as_its_breakpoint(
+    gateway: Gateway, stream: bool
+) -> None:
+    marker: Final = fresh_marker()
+    with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario:
+        name: Final = mantle_deployment(gateway, scenario, wire, points=[], prompt_cache_options=EXPLICIT)
+        outcome, received = observe(
+            gateway, wire, "chat", client_marked_chat_body(name, prompt_text(marker), stream=stream)
+        )
+        assert_answered(outcome, marker)
+        assert_wire(
+            received,
+            expected_wire(GPT, prompt_text(marker), endpoint="chat", marked=True, options=EXPLICIT),
+            streaming=stream,
+        )
+        success_row(name, marker)
+
+
+def test_g2_a_clients_cache_control_sends_no_mark_to_a_mantle_row_that_does_not_read_breakpoints(
+    gateway: Gateway,
+) -> None:
+    marker: Final = fresh_marker()
+    with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario:
+        name: Final = mantle_deployment(
+            gateway, scenario, wire, GPT_FLAGGED_ROW, model_info={"supports_prompt_cache_breakpoint": False}, points=[]
+        )
+        outcome, received = observe(
+            gateway, wire, "chat", client_marked_chat_body(name, prompt_text(marker), stream=False)
+        )
+        assert_answered(outcome, marker)
+        body: Final = body_of(received)
+        assert body["instructions"] == SYSTEM, received.body
+        assert body["input"] == [user_item(prompt_text(marker), "chat")], received.body
+        assert "prompt_cache_breakpoint" not in received.body.decode(), received.body
+        assert "cache_control" not in received.body.decode(), received.body
+        success_row(name, marker)

tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py

@@ -23,6 +23,7 @@ from openai.types.responses.response_reasoning_item import (
     ResponseReasoningItem,
     Summary,
 )
+from pydantic import TypeAdapter
 
 import litellm
 from litellm.completion_extras.litellm_responses_transformation.transformation import (
@@ -5935,6 +5936,464 @@ def test_transform_request_strips_the_breakpoint_for_a_served_provider_without_a
     ]
 
 
+_OPENAI_GPT_ROW: Final = {"litellm_provider": "openai", "mode": "chat", "supports_prompt_cache_breakpoint": True}
+_EPHEMERAL: Final = {"type": "ephemeral"}
+_CLIENT_MARKED_MESSAGES: Final = (
+    {"role": "system", "content": "Rules"},
+    {"role": "system", "content": "Stable prefix", "cache_control": _EPHEMERAL},
+    {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": _EPHEMERAL}]},
+)
+_PINNED_EXPLICIT: Final = {"mode": "explicit"}
+_MESSAGE: Final = TypeAdapter(Mapping[str, object])
+_PARTS: Final = TypeAdapter(list[Mapping[str, object]])
+
+
+def _client_marked_messages() -> list[AllMessageValues]:
+    return cast(List[AllMessageValues], copy.deepcopy(list(_CLIENT_MARKED_MESSAGES)))  # cast-ok: chat message dicts
+
+
+def _marked_parts(role: object, content: object) -> frozenset[tuple[object, object, str]]:
+    if not isinstance(content, list):
+        return frozenset()
+    return frozenset(
+        (role, part.get("text"), json.dumps(part["prompt_cache_breakpoint"], sort_keys=True))
+        for part in _PARTS.validate_python(content)
+        if "prompt_cache_breakpoint" in part
+    )
+
+
+def _marked_parts_of(items: object) -> frozenset[tuple[object, object, str]]:
+    messages: Final = (_MESSAGE.validate_python(item) for item in cast(list[object], items))  # cast-ok: wire list
+    return frozenset().union(*(_marked_parts(message.get("role"), message.get("content")) for message in messages))
+
+
+@pytest.mark.parametrize(
+    ("row_key", "row", "model", "custom_llm_provider", "api_base", "prompt_cache_options", "reads_breakpoints"),
+    [
+        pytest.param(
+            "bedrock_mantle/openai.gpt-5.6-sol",
+            _MANTLE_GPT_ROW,
+            "openai.gpt-5.6-sol",
+            "bedrock_mantle",
+            None,
+            None,
+            True,
+            id="flagged-hosted-row",
+        ),
+        pytest.param(
+            "bedrock_mantle/openai.gpt-5.6-sol",
+            _MANTLE_GPT_ROW,
+            "openai.gpt-5.6-sol",
+            "bedrock_mantle",
+            None,
+            _PINNED_EXPLICIT,
+            True,
+            id="flagged-hosted-row-pinned-explicit",
+        ),
+        pytest.param(
+            "bedrock_mantle/gpt-6-luna",
+            {**_MANTLE_GPT_ROW, "supports_prompt_cache_breakpoint": False},
+            "gpt-6-luna",
+            "bedrock_mantle",
+            None,
+            _PINNED_EXPLICIT,
+            False,
+            id="hosted-row-opted-out",
+        ),
+        pytest.param(
+            "bedrock_mantle/anthropic.claude-sonnet-test",
+            {"litellm_provider": "bedrock_mantle", "mode": "chat"},
+            "anthropic.claude-sonnet-test",
+            "bedrock_mantle",
+            None,
+            _PINNED_EXPLICIT,
+            False,
+            id="claude-on-the-same-host",
+        ),
+        pytest.param(
+            "gpt-6-luna",
+            _OPENAI_GPT_ROW,
+            "gpt-6-luna",
+            "openai",
+            "https://api.openai.com/v1",
+            None,
+            True,
+            id="openai-api",
+        ),
+        pytest.param(
+            "gpt-6-luna",
+            _OPENAI_GPT_ROW,
+            "gpt-6-luna",
+            "openai",
+            "https://gateway.example.com/v1",
+            None,
+            False,
+            id="third-party-host",
+        ),
+        pytest.param(
+            "gpt-6-luna",
+            _OPENAI_GPT_ROW,
+            "gpt-6-luna",
+            "openai",
+            "https://gateway.example.com/v1",
+            _PINNED_EXPLICIT,
+            True,
+            id="third-party-host-opted-in",
+        ),
+    ],
+)
+def test_transform_request_marks_the_parts_the_chat_route_marks_for_the_same_deployment(
+    monkeypatch: pytest.MonkeyPatch,
+    row_key: str,
+    row: Mapping[str, object],
+    model: str,
+    custom_llm_provider: str,
+    api_base: str | None,
+    prompt_cache_options: Mapping[str, object] | None,
+    reads_breakpoints: bool,
+) -> None:
+    from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
+
+    monkeypatch.setitem(litellm.model_cost, row_key, row)
+    litellm_params: Final[dict[str, object]] = {
+        "custom_llm_provider": custom_llm_provider,
+        **({} if api_base is None else {"api_base": api_base}),
+    }
+    options: Final[dict[str, object]] = (
+        {} if prompt_cache_options is None else {"prompt_cache_options": prompt_cache_options}
+    )
+    chat_request: Final = OpenAIGPTConfig().transform_request(
+        model=model,
+        messages=_client_marked_messages(),
+        optional_params={"extra_body": dict(options)},
+        litellm_params=dict(litellm_params),
+        headers={},
+    )
+    bridged_request: Final = LiteLLMResponsesTransformationHandler().transform_request(
+        model=model,
+        messages=_client_marked_messages(),
+        optional_params=dict(options),
+        litellm_params=dict(litellm_params),
+        headers={},
+        litellm_logging_obj=Mock(),
+    )
+
+    chat_marks: Final = _marked_parts_of(chat_request["messages"])
+    assert _marked_parts_of(bridged_request["input"]) == chat_marks
+    assert {(role, text) for role, text, _ in chat_marks} == (
+        {("system", "Stable prefix"), ("user", "hi")} if reads_breakpoints else set()
+    )
+
+
+@pytest.mark.parametrize("opted_in", [True, False])
+def test_transform_request_follows_the_deployments_own_breakpoint_opt_in_like_the_chat_route(
+    monkeypatch: pytest.MonkeyPatch, opted_in: bool
+) -> None:
+    from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
+
+    monkeypatch.setitem(litellm.model_cost, "luna-chat", {"litellm_provider": "openai", "mode": "chat"})
+    litellm_params: Final[dict[str, object]] = {
+        "custom_llm_provider": "openai",
+        "api_base": "https://api.openai.com/v1",
+        "model_info": {"supports_prompt_cache_breakpoint": opted_in},
+    }
+    chat_request: Final = OpenAIGPTConfig().transform_request(
+        model="luna-chat",
+        messages=_client_marked_messages(),
+        optional_params={},
+        litellm_params=dict(litellm_params),
+        headers={},
+    )
+    bridged_request: Final = LiteLLMResponsesTransformationHandler().transform_request(
+        model="luna-chat",
+        messages=_client_marked_messages(),
+        optional_params={},
+        litellm_params=dict(litellm_params),
+        headers={},
+        litellm_logging_obj=Mock(),
+    )
+
+    chat_marks: Final = _marked_parts_of(chat_request["messages"])
+    assert _marked_parts_of(bridged_request["input"]) == chat_marks
+    assert {(role, text) for role, text, _ in chat_marks} == (
+        {("system", "Stable prefix"), ("user", "hi")} if opted_in else set()
+    )
+
+
+def test_transform_request_keeps_a_marked_string_system_message_in_input_and_folds_the_unmarked_one(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
+
+    monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW)
+    messages: Final = _client_marked_messages()
+    chat_route_system: Final = AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(messages[1])
+    (chat_route_part,) = _PARTS.validate_python(chat_route_system.get("content"))
+
+    request: Final = LiteLLMResponsesTransformationHandler().transform_request(
+        model="openai.gpt-5.6-sol",
+        messages=messages,
+        optional_params={},
+        litellm_params={"custom_llm_provider": "bedrock_mantle"},
+        headers={},
+        litellm_logging_obj=Mock(),
+    )
+
+    breakpoint_marker: Final = chat_route_part["prompt_cache_breakpoint"]
+    assert request["instructions"] == "Rules"
+    assert request["input"] == [
+        {
+            "type": "message",
+            "role": "system",
+            "content": [{"type": "input_text", "text": "Stable prefix", "prompt_cache_breakpoint": breakpoint_marker}],
+        },
+        {
+            "type": "message",
+            "role": "user",
+            "content": [{"type": "input_text", "text": "hi", "prompt_cache_breakpoint": breakpoint_marker}],
+        },
+    ]
+    assert messages == list(_CLIENT_MARKED_MESSAGES)
+
+
+def _roles_and_texts(request_input: object) -> list[tuple[object, list[object]]]:
+    items: Final = (_MESSAGE.validate_python(item) for item in cast(list[object], request_input))  # cast-ok: wire list
+    return [
+        (item.get("role"), [part.get("text") for part in _PARTS.validate_python(item.get("content"))]) for item in items
+    ]
+
+
+@pytest.mark.parametrize(
+    "marked_system",
+    [
+        {"role": "system", "content": "Stable prefix", "cache_control": _EPHEMERAL},
+        {"role": "system", "content": [{"type": "text", "text": "Stable prefix", "cache_control": _EPHEMERAL}]},
+    ],
+    ids=["marked_string", "marked_part"],
+)
+def test_transform_request_keeps_system_messages_in_order_when_a_marked_one_comes_first(
+    monkeypatch: pytest.MonkeyPatch, marked_system: AllMessageValues
+) -> None:
+    monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW)
+
+    request: Final = LiteLLMResponsesTransformationHandler().transform_request(
+        model="openai.gpt-5.6-sol",
+        messages=[marked_system, {"role": "system", "content": "Rules"}, {"role": "user", "content": "hi"}],
+        optional_params={},
+        litellm_params={"custom_llm_provider": "bedrock_mantle"},
+        headers={},
+        litellm_logging_obj=Mock(),
+    )
+
+    assert "instructions" not in request
+    assert _roles_and_texts(request["input"]) == [
+        ("system", ["Stable prefix"]),
+        ("system", ["Rules"]),
+        ("user", ["hi"]),
+    ]
+
+
+@pytest.mark.parametrize(
+    ("row_key", "row", "model", "litellm_params"),
+    [
+        pytest.param(
+            "luna-responses",
+            {"litellm_provider": "openai", "mode": "responses"},
+            "luna-responses",
+            {"custom_llm_provider": "openai", "api_base": "https://api.openai.com/v1"},
+            id="openai-model-without-breakpoints",
+        ),
+        pytest.param(
+            "luna-responses",
+            {"litellm_provider": "openai", "mode": "responses"},
+            "luna-responses",
+            {"custom_llm_provider": "openai", "api_base": "https://api.openai.com/v1", "drop_params": True},
+            id="openai-model-without-breakpoints-drop-params",
+        ),
+        pytest.param(
+            "bedrock_mantle/openai.gpt-5.6-sol",
+            _MANTLE_GPT_ROW,
+            "openai.gpt-5.6-sol",
+            {"custom_llm_provider": "bedrock_mantle", "model_info": {"supports_prompt_cache_breakpoint": False}},
+            id="mantle-deployment-opted-out",
+        ),
+    ],
+)
+def test_transform_request_folds_system_messages_as_before_where_the_breakpoint_is_stripped(
+    monkeypatch: pytest.MonkeyPatch,
+    row_key: str,
+    row: Mapping[str, object],
+    model: str,
+    litellm_params: Mapping[str, object],
+) -> None:
+    monkeypatch.setitem(litellm.model_cost, row_key, row)
+
+    request: Final = LiteLLMResponsesTransformationHandler().transform_request(
+        model=model,
+        messages=[
+            {
+                "role": "system",
+                "content": [{"type": "text", "text": "Stable prefix", "prompt_cache_breakpoint": _PINNED_EXPLICIT}],
+            },
+            {"role": "system", "content": "Rules"},
+            {"role": "user", "content": "hi"},
+        ],
+        optional_params={},
+        litellm_params=dict(litellm_params),
+        headers={},
+        litellm_logging_obj=Mock(),
+    )
+
+    assert request["instructions"] == "Rules"
+    assert _roles_and_texts(request["input"]) == [("system", ["Stable prefix"]), ("user", ["hi"])]
+    assert list(_marked_parts_of(request["input"])) == []
+
+
+def test_transform_request_still_folds_a_string_system_message_behind_an_unmarked_list_one(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW)
+
+    request: Final = LiteLLMResponsesTransformationHandler().transform_request(
+        model="openai.gpt-5.6-sol",
+        messages=[
+            {"role": "system", "content": [{"type": "text", "text": "Stable prefix"}]},
+            {"role": "system", "content": "Rules"},
+            {"role": "user", "content": "hi"},
+        ],
+        optional_params={},
+        litellm_params={"custom_llm_provider": "bedrock_mantle"},
+        headers={},
+        litellm_logging_obj=Mock(),
+    )
+
+    assert request["instructions"] == "Rules"
+    assert _roles_and_texts(request["input"]) == [("system", ["Stable prefix"]), ("user", ["hi"])]
+
+
+def _mantle_marked_texts(model: str, messages: list[object], **litellm_params: object) -> set[tuple[object, object]]:
+    request: Final = LiteLLMResponsesTransformationHandler().transform_request(
+        model=model,
+        messages=cast(List[AllMessageValues], messages),  # cast-ok: the test builds chat messages as plain mappings
+        optional_params={},
+        litellm_params={"custom_llm_provider": "bedrock_mantle", **litellm_params},
+        headers={},
+        litellm_logging_obj=Mock(),
+    )
+    return {(role, text) for role, text, _ in _marked_parts_of(request["input"])}
+
+
+def test_transform_request_puts_no_breakpoint_on_empty_marked_text(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW)
+
+    marked: Final = _mantle_marked_texts(
+        "openai.gpt-5.6-sol",
+        [
+            {"role": "system", "content": "", "cache_control": _EPHEMERAL},
+            {"role": "user", "content": [{"type": "text", "text": "hi"}, {"type": "text", "text": "", "cache_control": _EPHEMERAL}]},
+            {
+                "role": "user",
+                "content": [{"type": "text", "text": "there"}, {"type": "text", "text": ""}],
+                "cache_control": _EPHEMERAL,
+            },
+        ],
+    )
+
+    assert marked == {("user", "there")}
+
+
+def test_transform_request_maps_a_string_cache_control_like_an_object_one(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW)
+
+    marked: Final = _mantle_marked_texts(
+        "openai.gpt-5.6-sol",
+        [
+            {"role": "system", "content": "Rules", "cache_control": "ephemeral"},
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "hi", "cache_control": "ephemeral"},
+                    {"type": "text", "text": "there", "cache_control": _EPHEMERAL},
+                ],
+            },
+        ],
+    )
+
+    assert marked == {("system", "Rules"), ("user", "hi"), ("user", "there")}
+
+
+def test_transform_request_maps_client_cache_control_for_an_alias_whose_base_model_reads_breakpoints(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW)
+
+    marked: Final = _mantle_marked_texts(
+        "my-alias", list(_client_marked_messages()), base_model="openai.gpt-5.6-sol"
+    )
+
+    assert marked == {("system", "Stable prefix"), ("user", "hi")}
+
+
+def _completion_marked_texts(model: str, model_info: Mapping[str, object]) -> set[tuple[object, object]]:
+    response_body: Final = {
+        "id": "resp_alias_marks",
+        "object": "response",
+        "created_at": 1789788253,
+        "status": "completed",
+        "model": "gpt-5.6",
+        "output": [
+            {
+                "type": "message",
+                "id": "msg_alias_marks",
+                "status": "completed",
+                "role": "assistant",
+                "content": [{"type": "output_text", "text": "ok", "annotations": []}],
+            }
+        ],
+        "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
+    }
+    with respx.mock(assert_all_called=True) as router:
+        upstream: Final = router.post("https://api.openai.com/v1/responses").mock(
+            return_value=httpx.Response(200, json=response_body)
+        )
+        litellm.completion(
+            model=model,
+            messages=_client_marked_messages(),
+            api_key="sk-test",
+            model_info=model_info,
+            cache={"no-cache": True},
+        )
+    request_body: Final = json.loads(upstream.calls[0].request.content)
+    return {(role, text) for role, text, _ in _marked_parts_of(request_body["input"])}
+
+
+def test_completion_maps_client_cache_control_for_an_alias_whose_base_model_is_declared_in_model_info() -> None:
+    served_model_marks: Final = _completion_marked_texts("openai/responses/gpt-5.6", {})
+
+    assert served_model_marks
+    assert _completion_marked_texts("openai/responses/team-alias", {"base_model": "gpt-5.6"}) == served_model_marks
+
+
+def test_transform_request_leaves_client_cache_control_unmapped_for_a_deployment_that_does_not_read_breakpoints(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setitem(
+        litellm.model_cost, "bedrock_mantle/gpt-6-luna", {**_MANTLE_GPT_ROW, "supports_prompt_cache_breakpoint": False}
+    )
+
+    request: Final = LiteLLMResponsesTransformationHandler().transform_request(
+        model="gpt-6-luna",
+        messages=_client_marked_messages(),
+        optional_params={"prompt_cache_options": _PINNED_EXPLICIT},
+        litellm_params={"custom_llm_provider": "bedrock_mantle"},
+        headers={},
+        litellm_logging_obj=Mock(),
+    )
+
+    assert request["instructions"] == "Rules Stable prefix"
+    assert request["input"] == [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}]
+
+
 def test_mid_conversation_system_string_stays_in_input_after_a_user_turn():
     handler: Final = LiteLLMResponsesTransformationHandler()
 
@@ -5992,6 +6451,9 @@ def test_mid_conversation_system_reminder_as_string_and_as_text_block_produce_id
     assert as_string[1] == _system_input_item(reminder)
 
 
+_HOST_WITHOUT_BREAKPOINTS: Final = {"custom_llm_provider": "openai", "api_base": "https://gateway.example.com/v1"}
+
+
 def test_claude_code_shaped_history_keeps_a_byte_stable_input_prefix_across_requests():
     handler: Final = LiteLLMResponsesTransformationHandler()
     top_level_system: Final = [{"type": "text", "text": "You are Claude Code.", "cache_control": {"type": "ephemeral"}}]
@@ -6031,7 +6493,7 @@ def test_claude_code_shaped_history_keeps_a_byte_stable_input_prefix_across_requ
         model="gpt-5.6-luna",
         messages=first_request_messages,
         optional_params={},
-        litellm_params={},
+        litellm_params=dict(_HOST_WITHOUT_BREAKPOINTS),
         headers={},
         litellm_logging_obj=Mock(),
     )
@@ -6039,7 +6501,7 @@ def test_claude_code_shaped_history_keeps_a_byte_stable_input_prefix_across_requ
         model="gpt-5.6-luna",
         messages=second_request_messages,
         optional_params={},
-        litellm_params={},
+        litellm_params=dict(_HOST_WITHOUT_BREAKPOINTS),
         headers={},
         litellm_logging_obj=Mock(),
     )

tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py

@@ -1,15 +1,22 @@
 import json
 import os
 import sys
-from typing import Final
+from collections.abc import Iterator
+from typing import Final, List, cast
+from unittest.mock import Mock
 
 import pytest
 
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../..")))
 
+import litellm
+from litellm.completion_extras.litellm_responses_transformation.transformation import (
+    LiteLLMResponsesTransformationHandler,
+)
 from litellm.llms.anthropic.messages.adapters.handler import (
     LiteLLMMessagesToCompletionTransformationHandler,
 )
+from litellm.types.llms.openai import AllMessageValues
 
 MESSAGES = [{"role": "user", "content": "hello"}]
 CLAUDE_CODE_USER_ID = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "session-abc"})
@@ -138,3 +145,187 @@ def test_prepare_completion_kwargs_derives_no_prompt_cache_key_mistral_cannot_ta
 
     assert "prompt_cache_key" not in mistral_kwargs
     assert openai_kwargs["prompt_cache_key"] == "sitzung-ü"
+
+
+_EPHEMERAL: Final = {"type": "ephemeral"}
+_MANTLE_GPT_ROW: Final = {
+    "litellm_provider": "bedrock_mantle",
+    "mode": "responses",
+    "supports_prompt_cache_breakpoint": True,
+}
+_CLAUDE_CODE_MESSAGES: Final = (
+    {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": _EPHEMERAL}]},
+    {
+        "role": "assistant",
+        "content": [
+            {"type": "text", "text": "checking", "cache_control": _EPHEMERAL},
+            {"type": "tool_use", "id": "toolu_1", "name": "read", "input": {}, "cache_control": _EPHEMERAL},
+        ],
+    },
+    {
+        "role": "user",
+        "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "file body", "cache_control": _EPHEMERAL}],
+    },
+)
+_SYSTEM: Final = [{"type": "text", "text": "Stable prefix", "cache_control": _EPHEMERAL}]
+_TOOLS: Final = [{"name": "read", "description": "Read a file", "input_schema": {"type": "object"}, "cache_control": _EPHEMERAL}]
+
+
+def _texts_with(key: str, value: object) -> Iterator[object]:
+    if isinstance(value, dict):
+        if key in value:
+            yield value.get("text", value.get("name"))
+        for nested in value.values():
+            yield from _texts_with(key, nested)
+    if isinstance(value, list):
+        for item in value:
+            yield from _texts_with(key, item)
+
+
+def _mantle_wire(model_info: dict[str, object] | None) -> tuple[dict[str, object], dict[str, object]]:
+    completion_kwargs, _ = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs(
+        max_tokens=64,
+        messages=json.loads(json.dumps(_CLAUDE_CODE_MESSAGES)),
+        model="bedrock_mantle/openai.gpt-5.6-sol",
+        system=json.loads(json.dumps(_SYSTEM)),
+        tools=json.loads(json.dumps(_TOOLS)),
+        extra_kwargs={"custom_llm_provider": "bedrock_mantle", "model_info": model_info},
+    )
+    wire: Final = LiteLLMResponsesTransformationHandler().transform_request(
+        model="openai.gpt-5.6-sol",
+        messages=cast(List[AllMessageValues], completion_kwargs["messages"]),  # cast-ok: chat messages from the adapter
+        optional_params={"tools": completion_kwargs["tools"]},
+        litellm_params={"custom_llm_provider": "bedrock_mantle", "model_info": model_info},
+        headers={},
+        litellm_logging_obj=Mock(),
+    )
+    return dict(completion_kwargs), wire
+
+
+def test_anthropic_messages_to_a_breakpoint_reading_target_carry_marks_where_the_openai_dialect_reads_them(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW)
+
+    completion_kwargs, wire = _mantle_wire(None)
+
+    assert list(_texts_with("cache_control", completion_kwargs["tools"])) == []
+    assert list(_texts_with("cache_control", cast(list[object], completion_kwargs["messages"])[2])) == []
+    assert sorted(map(str, _texts_with("prompt_cache_breakpoint", wire["input"]))) == ["Stable prefix", "file body", "hi"]
+    assert list(_texts_with("cache_control", wire)) == []
+
+
+def test_anthropic_messages_to_a_deployment_that_opts_out_of_breakpoints_carry_no_marks(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW)
+
+    completion_kwargs, wire = _mantle_wire({"supports_prompt_cache_breakpoint": False})
+
+    assert list(_texts_with("cache_control", completion_kwargs)) == []
+    assert list(_texts_with("prompt_cache_breakpoint", wire)) == []
+
+
+_OPENAI_GPT_ROW: Final = {"litellm_provider": "openai", "mode": "chat", "supports_prompt_cache_breakpoint": True}
+_FOUNDRY_GPT_ROW: Final = {"litellm_provider": "azure_ai", "mode": "chat", "supports_prompt_cache_breakpoint": True}
+
+
+@pytest.mark.parametrize(
+    ("model", "extra_kwargs", "keeps_marks"),
+    [
+        pytest.param(
+            "azure/gpt-5.6-sol",
+            {"custom_llm_provider": "azure", "model_info": {"supports_prompt_cache_breakpoint": True}},
+            False,
+            id="azure-chat-forwards-cache-control-unmapped",
+        ),
+        pytest.param(
+            "openai/team-alias",
+            {
+                "custom_llm_provider": "openai",
+                "base_model": "gpt-5.6-sol",
+                "api_base": "https://llm.example.com/v1",
+                "prompt_cache_options": {"mode": "explicit"},
+            },
+            False,
+            id="openai-chat-reads-only-the-served-model",
+        ),
+        pytest.param(
+            "openai/responses/team-alias",
+            {
+                "custom_llm_provider": "openai",
+                "base_model": "gpt-5.6-sol",
+                "api_base": "https://llm.example.com/v1",
+                "prompt_cache_options": {"mode": "explicit"},
+            },
+            True,
+            id="openai-responses-alias-bridge-reads-base-model",
+        ),
+        pytest.param(
+            "azure/responses/gpt-5.6-sol",
+            {"custom_llm_provider": "azure", "model_info": {"supports_prompt_cache_breakpoint": True}},
+            True,
+            id="azure-responses-bridge-renames-marks",
+        ),
+        pytest.param(
+            "bedrock_mantle/team-alias",
+            {
+                "custom_llm_provider": "bedrock_mantle",
+                "model_info": {"mode": "responses", "base_model": "openai.gpt-5.6-sol"},
+                "prompt_cache_options": {"mode": "explicit"},
+            },
+            True,
+            id="pinned-bridge-reads-model-info-base-model",
+        ),
+        pytest.param(
+            "bedrock_mantle/team-alias",
+            {
+                "custom_llm_provider": "bedrock_mantle",
+                "base_model": "openai.gpt-5.6-sol",
+                "prompt_cache_options": {"mode": "explicit"},
+            },
+            False,
+            id="mantle-chat-reads-only-the-served-model",
+        ),
+        pytest.param(
+            "litellm_proxy/gpt-5.6-sol",
+            {
+                "custom_llm_provider": "litellm_proxy",
+                "model_info": {"supports_prompt_cache_breakpoint": True, "litellm_provider": "litellm_proxy"},
+            },
+            False,
+            id="litellm-proxy-chat-forwards-cache-control-unmapped",
+        ),
+        pytest.param(
+            "sap/gpt-5.6-sol",
+            {
+                "custom_llm_provider": "sap",
+                "model_info": {"supports_prompt_cache_breakpoint": True, "litellm_provider": "sap"},
+            },
+            False,
+            id="sap-chat-forwards-cache-control-unmapped",
+        ),
+        pytest.param(
+            "azure_ai/team-alias",
+            {"custom_llm_provider": "azure_ai", "model_info": {"base_model": "gpt-5.6-sol"}},
+            True,
+            id="foundry-chat-reads-model-info-base-model",
+        ),
+    ],
+)
+def test_anthropic_messages_keep_marks_only_for_a_route_that_turns_them_into_breakpoints(
+    monkeypatch: pytest.MonkeyPatch, model: str, extra_kwargs: dict[str, object], keeps_marks: bool
+) -> None:
+    monkeypatch.setitem(litellm.model_cost, "gpt-5.6-sol", _OPENAI_GPT_ROW)
+    litellm.register_model({"azure_ai/gpt-5.6-sol": _FOUNDRY_GPT_ROW})
+    monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW)
+
+    completion_kwargs, _ = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs(
+        max_tokens=64,
+        messages=json.loads(json.dumps(_CLAUDE_CODE_MESSAGES)),
+        model=model,
+        system=json.loads(json.dumps(_SYSTEM)),
+        extra_kwargs=extra_kwargs,
+    )
+
+    assert bool(list(_texts_with("cache_control", completion_kwargs["messages"]))) is keeps_marks

tests/unit/llms/openai/chat/test_openai_gpt_transformation.py

@@ -1056,6 +1056,19 @@ class TestCacheControlAsPromptCacheBreakpoint:
             {"role": "user", "content": "unmarked"},
         ]
 
+    def test_gpt_5_6_rewrites_a_string_mark_and_a_mark_on_empty_text(self):
+        messages: list[dict[str, object]] = [
+            {"role": "system", "content": "", "cache_control": _EPHEMERAL},
+            {"role": "user", "content": [_text("Describe this"), {**_text("plain"), "cache_control": "ephemeral"}]},
+            {"role": "user", "content": [_text("context"), _marked(_text(""))]},
+        ]
+        body = self._transform("gpt-5.6", messages)
+        assert body["messages"] == [
+            {"role": "system", "content": [_breakpoint(_text(""))]},
+            {"role": "user", "content": [_text("Describe this"), _breakpoint(_text("plain"))]},
+            {"role": "user", "content": [_text("context"), _breakpoint(_text(""))]},
+        ]
+
     def test_gpt_5_6_message_level_mark_on_list_content_marks_the_last_part(self):
         messages: list[dict[str, object]] = [
             {"role": "user", "content": [_text("first"), _text("last")], "cache_control": _EPHEMERAL},