diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 9902fe1d8fc..f3a07ba27d9 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -34,7 +34,13 @@ from litellm.completion_extras.litellm_responses_transformation.history import ( replay_items, requested_phase, ) -from litellm.integrations.anthropic_cache_control_hook import routed_model_info, supports_openai_prompt_cache_breakpoint +from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + holds_prompt_cache_breakpoint, + message_carries_prompt_cache_breakpoint, + routed_model_info, + supports_openai_prompt_cache_breakpoint, +) from litellm.litellm_core_utils.allowed_tools_tool_choice import ( parse_chat_allowed_tools_tool_choice, responses_allowed_tools_tool_choice, @@ -235,6 +241,31 @@ def _drops_audio_input(model: str, litellm_params: Mapping[str, object]) -> bool return not _supports_audio_input(model, litellm_params) +def _messages_in_cache_mark_dialect( + named_models: Sequence[str], + messages: Sequence["AllMessageValues"], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + model_info: Mapping[str, object] | None, +) -> Sequence["AllMessageValues"]: + reads_breakpoints: Final = any( + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( + name, + _served_provider(litellm_params), + litellm_params.get("api_base"), + optional_params.get("prompt_cache_options"), + model_info, + ) + for name in named_models + ) + if not reads_breakpoints: + return messages + return tuple( + AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message, holds_prompt_cache_breakpoint) + for message in messages + ) + + def responses_provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]: return MappingProxyType( { @@ -273,6 +304,12 @@ def _get_reasoning_items( return [] +def _folds_into_instructions(msg: "AllMessageValues", keep_prompt_cache_breakpoints: bool) -> bool: + return msg.get("role") == "system" and not ( + keep_prompt_cache_breakpoints and message_carries_prompt_cache_breakpoint(msg) + ) + + def _reasoning_input_items(msg: "AllMessageValues") -> list[dict[str, object]]: # mutable-ok: API message payload """Reasoning input items for an assistant message. @@ -871,7 +908,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): def convert_chat_completion_messages_to_responses_api( self, - messages: list["AllMessageValues"], + messages: Sequence["AllMessageValues"], *, drop_params: bool = False, target_trims_breakpoints: bool = True, @@ -884,6 +921,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): messages, drop_params=drop_params, target_trims_breakpoints=target_trims_breakpoints, + keep_prompt_cache_breakpoints=keep_prompt_cache_breakpoints, replay_tool_search=replay_tool_search, include_namespace=include_namespace, replay_openai_history=replay_openai_history, @@ -897,10 +935,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): def _convert_chat_completion_messages_to_responses_input( self, - messages: list["AllMessageValues"], + messages: Sequence["AllMessageValues"], *, drop_params: bool = False, target_trims_breakpoints: bool = True, + keep_prompt_cache_breakpoints: bool = False, replay_tool_search: bool = False, include_namespace: bool = False, replay_openai_history: bool = False, @@ -924,8 +963,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): and isinstance(tool_call.get("custom"), dict) ) - leading_system_count: Final = next( - (index for index, msg in enumerate(messages) if msg.get("role") != "system"), + leading_folded_system_count: Final = next( + ( + index + for index, msg in enumerate(messages) + if not _folds_into_instructions(msg, keep_prompt_cache_breakpoints) + ), len(messages), ) @@ -939,7 +982,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): input_items.extend(replayed_turn.items) elif role == "system": # Extract system message as instructions - if isinstance(content, str) and index < leading_system_count: + if isinstance(content, str) and index < leading_folded_system_count: if instructions: # Concatenate multiple system prompts with a space instructions = f"{instructions} {content}" @@ -1174,7 +1217,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): supports_openai_prompt_cache_breakpoint(name, provider, model_info) for name in named_models ) converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api( - messages, + _messages_in_cache_mark_dialect( + named_models, + messages, + cast(Mapping[str, object], optional_params), # cast-ok: upcast of the untyped params dict + litellm_params, + model_info, + ), drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params), target_trims_breakpoints=drops_unsupported_params(litellm_params.get("drop_params")), keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint, diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index ea518d506fb..0900f885238 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -326,8 +326,8 @@ def whole_messages_within_cache_control_limit( return _messages_within_budget(messages, reserved_marks, _whole_message_kept) -def _part_with_cache_control_renamed(part: object) -> object: - if not _accepts_prompt_cache_breakpoint(part): +def _part_with_cache_control_renamed(part: object, markable: Callable[[object], bool]) -> object: + if not markable(part) or _attribute_or_key(part, "cache_control") is None: return part return cache_control_as_prompt_cache_breakpoint(_CONTENT_PART_ADAPTER.validate_python(part)) @@ -694,6 +694,10 @@ def _accepts_prompt_cache_breakpoint(block: object) -> bool: return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES +def holds_prompt_cache_breakpoint(block: object) -> bool: + return _accepts_prompt_cache_breakpoint(block) and _attribute_or_key(block, "text") != "" + + def _carries_prompt_cache_breakpoint(block: object) -> bool: return _accepts_prompt_cache_breakpoint(block) and _attribute_or_key(block, "prompt_cache_breakpoint") is not None @@ -702,10 +706,12 @@ def _may_carry_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool: return message.get("role") != "assistant" +def message_carries_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool: + return any(_carries_prompt_cache_breakpoint(part) for part in _as_object_list(message.get("content")) or ()) + + def _tool_result_carries_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool: - return message.get("role") == "tool" and any( - _carries_prompt_cache_breakpoint(part) for part in _as_object_list(message.get("content")) or () - ) + return message.get("role") == "tool" and message_carries_prompt_cache_breakpoint(message) def _part_without_prompt_cache_breakpoint(part: object) -> object: @@ -1236,7 +1242,9 @@ class AnthropicCacheControlHook(CustomPromptManagement): return message @staticmethod - def cache_control_as_prompt_cache_breakpoints(message: AllMessageValues) -> AllMessageValues: + def cache_control_as_prompt_cache_breakpoints( + message: AllMessageValues, markable: Callable[[object], bool] = _accepts_prompt_cache_breakpoint + ) -> AllMessageValues: """Every Anthropic ``cache_control`` on the message or its parts moved to OpenAI's ``prompt_cache_breakpoint``, placed where the injection hook would put one. Marks with no OpenAI slot are left for the caller to strip.""" fields: Final[Mapping[str, object]] = dict(message) @@ -1250,13 +1258,13 @@ class AnthropicCacheControlHook(CustomPromptManagement): AllMessageValues, { **fields, - "content": [_part_with_cache_control_renamed(part) for part in parts], + "content": [_part_with_cache_control_renamed(part, markable) for part in parts], }, ) ) if fields.get("cache_control") is None: return renamed - return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(renamed) + return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(renamed, markable) @staticmethod def prompt_cache_breakpoints_as_cache_control(message: AllMessageValues, drop_params: bool) -> AllMessageValues: @@ -1345,25 +1353,22 @@ class AnthropicCacheControlHook(CustomPromptManagement): return capped_request(max(0, first_over_cap - 1)) @staticmethod - def _insert_prompt_cache_breakpoint_in_message(message: AllMessageValues) -> AllMessageValues: + def _insert_prompt_cache_breakpoint_in_message( + message: AllMessageValues, markable: Callable[[object], bool] = _accepts_prompt_cache_breakpoint + ) -> AllMessageValues: if not _may_carry_prompt_cache_breakpoint(message): return message message_content: Final = message.get("content", None) if isinstance(message_content, str): + text_part: Final = ChatCompletionTextObject(type="text", text=message_content) + if not markable(text_part): + return message marked: Final = copy.copy(message) - marked["content"] = [ - with_prompt_cache_breakpoint( - ChatCompletionTextObject(type="text", text=message_content), PromptCacheBreakpoint(mode="explicit") - ) - ] + marked["content"] = [with_prompt_cache_breakpoint(text_part, PromptCacheBreakpoint(mode="explicit"))] return marked if isinstance(message_content, list): target_index: Final = next( - ( - index - for index in range(len(message_content) - 1, -1, -1) - if _accepts_prompt_cache_breakpoint(message_content[index]) - ), + (index for index in range(len(message_content) - 1, -1, -1) if markable(message_content[index])), None, ) if target_index is not None: @@ -2003,3 +2008,14 @@ def messages_within_reserved_cache_control_cap( MAX_CACHE_CONTROL_BLOCKS, ) return kept + + +def call_targets_openai_prompt_cache_breakpoint(model: str, call_kwargs: Mapping[str, object]) -> bool: + provider: Final = call_kwargs.get("custom_llm_provider") + return AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( + model, + provider if isinstance(provider, str) else None, + call_kwargs.get("api_base") or call_kwargs.get("base_url"), + call_kwargs.get("prompt_cache_options"), + routed_model_info(call_kwargs.get("model_info")), + ) diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index 64c7ac62d03..de4c361d9fb 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -12,6 +12,10 @@ from typing_extensions import TypedDict import litellm from litellm._logging import verbose_logger +from litellm.integrations.anthropic_cache_control_hook import ( + call_targets_openai_prompt_cache_breakpoint, + routed_model_info, +) from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject @@ -44,6 +48,8 @@ from litellm.llms.anthropic.messages.utils import ( proxy_sub_call_logging_kwargs, proxy_sub_call_trace_id, ) +from litellm.llms.azure_ai.chat.transformation import deployment_reads_prompt_cache_breakpoints +from litellm.responses.bridge_check import responses_bridge_model from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, @@ -396,6 +402,37 @@ ANTHROPIC_ADAPTER: Final = AnthropicAdapter() ######################################################## +_CHAT_ROUTES_RENAMING_MARKS_FOR_THE_SERVED_MODEL: Final = frozenset(("openai", "bedrock_mantle")) + + +def _deployment_base_model(call_kwargs: Mapping[str, object]) -> str | None: + model_info: Final = routed_model_info(call_kwargs.get("model_info")) + base_model: Final = call_kwargs.get("base_model") or (None if model_info is None else model_info.get("base_model")) + return base_model if isinstance(base_model, str) and base_model else None + + +def _target_reads_prompt_cache_breakpoint(model: str, call_kwargs: Mapping[str, object]) -> bool: + provider: Final = call_kwargs.get("custom_llm_provider") + if not isinstance(provider, str): + return False + served_model: Final = model.removeprefix(f"{provider}/") + base_model: Final = _deployment_base_model(call_kwargs) + served_model_reads: Final = call_targets_openai_prompt_cache_breakpoint( + served_model.removeprefix("responses/"), call_kwargs + ) + if not served_model_reads and ( + base_model is None or not call_targets_openai_prompt_cache_breakpoint(base_model, call_kwargs) + ): + return False + if responses_bridge_model(served_model, provider, call_kwargs.get("model_info")) is not None: + return True + if provider == "azure_ai": + return deployment_reads_prompt_cache_breakpoints( + served_model, {"model_info": call_kwargs.get("model_info"), "base_model": base_model} + ) + return served_model_reads and provider in _CHAT_ROUTES_RENAMING_MARKS_FOR_THE_SERVED_MODEL + + def _record_bridged_model_response(logging_obj: LiteLLMLoggingObject | None, completion_response: object) -> None: if logging_obj is not None and isinstance(completion_response, ModelResponse): logging_obj.record_bridged_model_response(completion_response) @@ -678,6 +715,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: api_base=api_base if isinstance(api_base, str) else None, deployment_model_info=as_str_mapping(provider_kwargs.get("model_info")), drop_params=drop_params, + target_reads_prompt_cache_breakpoint=_target_reads_prompt_cache_breakpoint(model, extra_kwargs), ) if openai_request is None: diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 43eaa37aba7..ef8d40aff09 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -318,6 +318,7 @@ class AnthropicAdapter: api_base: str | None = None, deployment_model_info: Mapping[str, object] | None = None, drop_params: bool = True, + target_reads_prompt_cache_breakpoint: bool = False, ) -> tuple[ChatCompletionRequest | None, dict[str, str]]: """ Translate Anthropic request params to OpenAI format, returning tool name mapping. @@ -348,7 +349,9 @@ class AnthropicAdapter: ( translated_body, tool_name_mapping, - ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + ) = LiteLLMAnthropicMessagesAdapter( + target_reads_prompt_cache_breakpoint=target_reads_prompt_cache_breakpoint + ).translate_anthropic_to_openai( anthropic_message_request=request_body, custom_llm_provider=custom_llm_provider, api_base=api_base, @@ -434,8 +437,9 @@ _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS: Final = frozenset({LlmProviders.KYMA.va class LiteLLMAnthropicMessagesAdapter: - def __init__(self, *, keep_cache_control: bool = False) -> None: + def __init__(self, *, keep_cache_control: bool = False, target_reads_prompt_cache_breakpoint: bool = False) -> None: self.keep_cache_control: Final = keep_cache_control + self._target_reads_prompt_cache_breakpoint: Final = target_reads_prompt_cache_breakpoint ### FOR [BETA] `/v1/messages` endpoint support @@ -473,6 +477,8 @@ class LiteLLMAnthropicMessagesAdapter: source: object, target: object, model: str | None, + *, + has_prompt_cache_breakpoint_slot: bool = True, ) -> None: """ Extract cache_control from source and add to target if it should be preserved. @@ -486,12 +492,15 @@ class LiteLLMAnthropicMessagesAdapter: source: Dict or TypedDict containing potential cache_control field target: Dict or TypedDict to add cache_control to model: Model name to check if cache_control should be preserved + has_prompt_cache_breakpoint_slot: Whether the OpenAI cache dialect reads a breakpoint on this block """ # TypedDict objects are dicts at runtime, so .get() works cache_control: Final = ( source.get("cache_control") if isinstance(source, dict) else getattr(source, "cache_control", None) ) - if cache_control and (self.keep_cache_control or (model and self.target_consumes_cache_control(model))): + if cache_control and ( + self.keep_cache_control or self._target_keeps_mark(model, has_prompt_cache_breakpoint_slot) + ): # TypedDict objects support dict operations at runtime # Use type ignore consistent with codebase pattern (see anthropic/chat/transformation.py:432) if isinstance(target, dict): @@ -500,6 +509,12 @@ class LiteLLMAnthropicMessagesAdapter: # Fallback for non-dict objects (shouldn't happen in practice) cast(dict[str, object], target)["cache_control"] = cache_control + def _target_keeps_mark(self, model: str | None, has_prompt_cache_breakpoint_slot: bool) -> bool: + if not model: + return False + reads_as_breakpoint: Final = self._target_reads_prompt_cache_breakpoint and has_prompt_cache_breakpoint_slot + return reads_as_breakpoint or self.target_consumes_cache_control(model) + @staticmethod def _add_prompt_cache_breakpoint_if_present(source: object, target: _BlockT) -> _BlockT: if isinstance(source, dict) and "prompt_cache_breakpoint" in source: @@ -651,7 +666,9 @@ class LiteLLMAnthropicMessagesAdapter: "type": "text", "text": content.get("text", ""), } - self._add_cache_control_if_applicable(content, text_block, model) + self._add_cache_control_if_applicable( + content, text_block, model, has_prompt_cache_breakpoint_slot=False + ) if "cache_control" in text_block: has_cache_control_in_text = True assistant_content_list.append(text_block) @@ -678,7 +695,9 @@ class LiteLLMAnthropicMessagesAdapter: type="function", function=function_chunk, ) - self._add_cache_control_if_applicable(content, tool_call, model) + self._add_cache_control_if_applicable( + content, tool_call, model, has_prompt_cache_breakpoint_slot=False + ) tool_calls.append(tool_call) elif content.get("type") == "thinking": # Anthropic's schema has no cache_control on thinking or @@ -942,7 +961,7 @@ class LiteLLMAnthropicMessagesAdapter: if k not in mapped_tool_params: # pass additional computer kwargs function_chunk.setdefault("parameters", {}).update({k: v}) tool_param = _chat_tool_param(function_chunk, tool) - self._add_cache_control_if_applicable(tool, tool_param, model) + self._add_cache_control_if_applicable(tool, tool_param, model, has_prompt_cache_breakpoint_slot=False) new_tools.append(tool_param) return new_tools, tool_name_mapping diff --git a/litellm/llms/anthropic/messages/native/handler.py b/litellm/llms/anthropic/messages/native/handler.py index 922de386fe4..c70f30ad41c 100644 --- a/litellm/llms/anthropic/messages/native/handler.py +++ b/litellm/llms/anthropic/messages/native/handler.py @@ -69,17 +69,6 @@ _SAFEGUARD_KWARGS: Final = TypeAdapter(Mapping[str, object]) _SAFEGUARD_MESSAGES: Final = TypeAdapter(Sequence[Mapping[str, object]]) -def _bridges_to_responses_api(model: str, custom_llm_provider: str, deployment_model_info: object) -> bool: - from litellm.responses.bridge_check import responses_api_bridge_check, str_keyed_mapping - - model_info, _ = responses_api_bridge_check( - model=model, - custom_llm_provider=custom_llm_provider, - deployment_model_info=str_keyed_mapping(deployment_model_info), - ) - return model_info.get("mode") == "responses" - - def _responses_mode_is_lost_by_prefix_strip( requested_model: str, resolved_model: str, custom_llm_provider: str, deployment_model_info: object ) -> bool: @@ -89,11 +78,14 @@ def _responses_mode_is_lost_by_prefix_strip( deployment id such as ``perplexity/perplexity/sonar`` (mode ``responses``) is shadowed by the chat entry ``perplexity/sonar`` and would otherwise be sent to chat/completions. """ + from litellm.responses.bridge_check import responses_bridge_model + if requested_model == resolved_model: return False - return _bridges_to_responses_api( - requested_model, custom_llm_provider, deployment_model_info - ) and not _bridges_to_responses_api(resolved_model, custom_llm_provider, deployment_model_info) + return ( + responses_bridge_model(requested_model, custom_llm_provider, deployment_model_info) is not None + and responses_bridge_model(resolved_model, custom_llm_provider, deployment_model_info) is None + ) @dataclass(frozen=True, slots=True) diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 85f042f621c..c639874c5ca 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -12,7 +12,7 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm._logging import verbose_logger from litellm.integrations.anthropic_cache_control_hook import ( - AnthropicCacheControlHook, + call_targets_openai_prompt_cache_breakpoint, routed_model_info, supports_openai_prompt_cache_breakpoint, ) @@ -181,17 +181,6 @@ def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: objec return responses_config is not None and "include" in responses_config.get_supported_openai_params(provider_model) -def _reads_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[str, object]) -> bool: - provider: Final = forwarded_kwargs.get("custom_llm_provider") - return AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( - model, - provider if isinstance(provider, str) else None, - forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url"), - forwarded_kwargs.get("prompt_cache_options"), - routed_model_info(forwarded_kwargs.get("model_info")), - ) - - def _model_takes_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[str, object]) -> bool: provider: Final = forwarded_kwargs.get("custom_llm_provider") provider_name: Final = provider if isinstance(provider, str) else None @@ -221,7 +210,7 @@ def _prompt_cache_block_rewrite(model: str, forwarded_kwargs: Mapping[str, objec """Client cache marks become breakpoints for a deployment that reads them, and a model that takes no breakpoints drops raw ones under drop_params.""" drops_params: Final = drops_unsupported_params(forwarded_kwargs.get("drop_params")) - if _reads_prompt_cache_breakpoint(model, forwarded_kwargs): + if call_targets_openai_prompt_cache_breakpoint(model, forwarded_kwargs): return ( _cache_control_as_wire_safe_prompt_cache_breakpoint if drops_params diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 93ff6d8170a..592c2e1c7e9 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -130,7 +130,7 @@ def _with_breakpoints_for_foundry(message: AllMessageValues, drop_params: bool) return cast(AllMessageValues, foundry_message) # cast-ok: the same message, its breakpoints in Foundry's shape -def _deployment_reads_breakpoints(model: str, litellm_params: Mapping[str, object]) -> bool: +def deployment_reads_prompt_cache_breakpoints(model: str, litellm_params: Mapping[str, object]) -> bool: model_info: Final = routed_model_info(litellm_params.get("model_info")) declared: Final = declared_prompt_cache_breakpoint(model_info) if declared is not None: @@ -439,7 +439,7 @@ class AzureAIStudioConfig(OpenAIConfig): "messages": self._transform_messages( messages, model, - _deployment_reads_breakpoints(model, _FIELDS.validate_python(litellm_params)), + deployment_reads_prompt_cache_breakpoints(model, _FIELDS.validate_python(litellm_params)), bool(litellm_params.get("drop_params") or litellm.drop_params), ), **optional_params, diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index 826de8cfdc9..d8eda42b7cf 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -102,6 +102,15 @@ def str_keyed_mapping(value: object) -> Mapping[str, object] | None: ) +def responses_bridge_model(model: str, custom_llm_provider: str, deployment_model_info: object) -> str | None: + model_info, bridged_model = responses_api_bridge_check( + model=model, + custom_llm_provider=custom_llm_provider, + deployment_model_info=str_keyed_mapping(deployment_model_info), + ) + return bridged_model if model_info.get("mode") == "responses" else None + + def bridge_reasoning_effort( reasoning_effort: str | Mapping[str, object] | None, extra_body: object ) -> str | Mapping[str, object] | None: diff --git a/tests/integration/providers/test_bedrock_mantle_gpt_prompt_cache_breakpoint_wire.py b/tests/integration/providers/test_bedrock_mantle_gpt_prompt_cache_breakpoint_wire.py index bc46cf5cde1..6a0cf4b5148 100644 --- a/tests/integration/providers/test_bedrock_mantle_gpt_prompt_cache_breakpoint_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_gpt_prompt_cache_breakpoint_wire.py @@ -64,6 +64,7 @@ from integration.providers._mantle_gpt_prompt_cache_support import ( spend_rows, success_row, system_item, + user_item, ) from pydantic import JsonValue @@ -540,3 +541,55 @@ def test_f4_a_slow_mantle_stream_during_a_burst_completes_with_every_id_landing_ name: Final = mantle_deployment(gateway, scenario, wire) wire.drain() assert_burst_landed(wire, name, burst(gateway, name, 12), marked=True) + + +def client_marked_chat_body(name: str, prompt: str, *, stream: bool) -> dict[str, JsonValue]: + return request_body( + "chat", + name, + prompt, + stream=stream, + messages=[ + {"role": "system", "content": SYSTEM, "cache_control": EPHEMERAL}, + {"role": "user", "content": prompt}, + ], + ) + + +@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"]) +def test_g1_a_clients_cache_control_on_a_string_system_message_reaches_mantle_as_its_breakpoint( + gateway: Gateway, stream: bool +) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment(gateway, scenario, wire, points=[], prompt_cache_options=EXPLICIT) + outcome, received = observe( + gateway, wire, "chat", client_marked_chat_body(name, prompt_text(marker), stream=stream) + ) + assert_answered(outcome, marker) + assert_wire( + received, + expected_wire(GPT, prompt_text(marker), endpoint="chat", marked=True, options=EXPLICIT), + streaming=stream, + ) + success_row(name, marker) + + +def test_g2_a_clients_cache_control_sends_no_mark_to_a_mantle_row_that_does_not_read_breakpoints( + gateway: Gateway, +) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment( + gateway, scenario, wire, GPT_FLAGGED_ROW, model_info={"supports_prompt_cache_breakpoint": False}, points=[] + ) + outcome, received = observe( + gateway, wire, "chat", client_marked_chat_body(name, prompt_text(marker), stream=False) + ) + assert_answered(outcome, marker) + body: Final = body_of(received) + assert body["instructions"] == SYSTEM, received.body + assert body["input"] == [user_item(prompt_text(marker), "chat")], received.body + assert "prompt_cache_breakpoint" not in received.body.decode(), received.body + assert "cache_control" not in received.body.decode(), received.body + success_row(name, marker) diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 19aefb430c7..6401fcc7c9a 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -23,6 +23,7 @@ from openai.types.responses.response_reasoning_item import ( ResponseReasoningItem, Summary, ) +from pydantic import TypeAdapter import litellm from litellm.completion_extras.litellm_responses_transformation.transformation import ( @@ -5935,6 +5936,464 @@ def test_transform_request_strips_the_breakpoint_for_a_served_provider_without_a ] +_OPENAI_GPT_ROW: Final = {"litellm_provider": "openai", "mode": "chat", "supports_prompt_cache_breakpoint": True} +_EPHEMERAL: Final = {"type": "ephemeral"} +_CLIENT_MARKED_MESSAGES: Final = ( + {"role": "system", "content": "Rules"}, + {"role": "system", "content": "Stable prefix", "cache_control": _EPHEMERAL}, + {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": _EPHEMERAL}]}, +) +_PINNED_EXPLICIT: Final = {"mode": "explicit"} +_MESSAGE: Final = TypeAdapter(Mapping[str, object]) +_PARTS: Final = TypeAdapter(list[Mapping[str, object]]) + + +def _client_marked_messages() -> list[AllMessageValues]: + return cast(List[AllMessageValues], copy.deepcopy(list(_CLIENT_MARKED_MESSAGES))) # cast-ok: chat message dicts + + +def _marked_parts(role: object, content: object) -> frozenset[tuple[object, object, str]]: + if not isinstance(content, list): + return frozenset() + return frozenset( + (role, part.get("text"), json.dumps(part["prompt_cache_breakpoint"], sort_keys=True)) + for part in _PARTS.validate_python(content) + if "prompt_cache_breakpoint" in part + ) + + +def _marked_parts_of(items: object) -> frozenset[tuple[object, object, str]]: + messages: Final = (_MESSAGE.validate_python(item) for item in cast(list[object], items)) # cast-ok: wire list + return frozenset().union(*(_marked_parts(message.get("role"), message.get("content")) for message in messages)) + + +@pytest.mark.parametrize( + ("row_key", "row", "model", "custom_llm_provider", "api_base", "prompt_cache_options", "reads_breakpoints"), + [ + pytest.param( + "bedrock_mantle/openai.gpt-5.6-sol", + _MANTLE_GPT_ROW, + "openai.gpt-5.6-sol", + "bedrock_mantle", + None, + None, + True, + id="flagged-hosted-row", + ), + pytest.param( + "bedrock_mantle/openai.gpt-5.6-sol", + _MANTLE_GPT_ROW, + "openai.gpt-5.6-sol", + "bedrock_mantle", + None, + _PINNED_EXPLICIT, + True, + id="flagged-hosted-row-pinned-explicit", + ), + pytest.param( + "bedrock_mantle/gpt-6-luna", + {**_MANTLE_GPT_ROW, "supports_prompt_cache_breakpoint": False}, + "gpt-6-luna", + "bedrock_mantle", + None, + _PINNED_EXPLICIT, + False, + id="hosted-row-opted-out", + ), + pytest.param( + "bedrock_mantle/anthropic.claude-sonnet-test", + {"litellm_provider": "bedrock_mantle", "mode": "chat"}, + "anthropic.claude-sonnet-test", + "bedrock_mantle", + None, + _PINNED_EXPLICIT, + False, + id="claude-on-the-same-host", + ), + pytest.param( + "gpt-6-luna", + _OPENAI_GPT_ROW, + "gpt-6-luna", + "openai", + "https://api.openai.com/v1", + None, + True, + id="openai-api", + ), + pytest.param( + "gpt-6-luna", + _OPENAI_GPT_ROW, + "gpt-6-luna", + "openai", + "https://gateway.example.com/v1", + None, + False, + id="third-party-host", + ), + pytest.param( + "gpt-6-luna", + _OPENAI_GPT_ROW, + "gpt-6-luna", + "openai", + "https://gateway.example.com/v1", + _PINNED_EXPLICIT, + True, + id="third-party-host-opted-in", + ), + ], +) +def test_transform_request_marks_the_parts_the_chat_route_marks_for_the_same_deployment( + monkeypatch: pytest.MonkeyPatch, + row_key: str, + row: Mapping[str, object], + model: str, + custom_llm_provider: str, + api_base: str | None, + prompt_cache_options: Mapping[str, object] | None, + reads_breakpoints: bool, +) -> None: + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + monkeypatch.setitem(litellm.model_cost, row_key, row) + litellm_params: Final[dict[str, object]] = { + "custom_llm_provider": custom_llm_provider, + **({} if api_base is None else {"api_base": api_base}), + } + options: Final[dict[str, object]] = ( + {} if prompt_cache_options is None else {"prompt_cache_options": prompt_cache_options} + ) + chat_request: Final = OpenAIGPTConfig().transform_request( + model=model, + messages=_client_marked_messages(), + optional_params={"extra_body": dict(options)}, + litellm_params=dict(litellm_params), + headers={}, + ) + bridged_request: Final = LiteLLMResponsesTransformationHandler().transform_request( + model=model, + messages=_client_marked_messages(), + optional_params=dict(options), + litellm_params=dict(litellm_params), + headers={}, + litellm_logging_obj=Mock(), + ) + + chat_marks: Final = _marked_parts_of(chat_request["messages"]) + assert _marked_parts_of(bridged_request["input"]) == chat_marks + assert {(role, text) for role, text, _ in chat_marks} == ( + {("system", "Stable prefix"), ("user", "hi")} if reads_breakpoints else set() + ) + + +@pytest.mark.parametrize("opted_in", [True, False]) +def test_transform_request_follows_the_deployments_own_breakpoint_opt_in_like_the_chat_route( + monkeypatch: pytest.MonkeyPatch, opted_in: bool +) -> None: + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + monkeypatch.setitem(litellm.model_cost, "luna-chat", {"litellm_provider": "openai", "mode": "chat"}) + litellm_params: Final[dict[str, object]] = { + "custom_llm_provider": "openai", + "api_base": "https://api.openai.com/v1", + "model_info": {"supports_prompt_cache_breakpoint": opted_in}, + } + chat_request: Final = OpenAIGPTConfig().transform_request( + model="luna-chat", + messages=_client_marked_messages(), + optional_params={}, + litellm_params=dict(litellm_params), + headers={}, + ) + bridged_request: Final = LiteLLMResponsesTransformationHandler().transform_request( + model="luna-chat", + messages=_client_marked_messages(), + optional_params={}, + litellm_params=dict(litellm_params), + headers={}, + litellm_logging_obj=Mock(), + ) + + chat_marks: Final = _marked_parts_of(chat_request["messages"]) + assert _marked_parts_of(bridged_request["input"]) == chat_marks + assert {(role, text) for role, text, _ in chat_marks} == ( + {("system", "Stable prefix"), ("user", "hi")} if opted_in else set() + ) + + +def test_transform_request_keeps_a_marked_string_system_message_in_input_and_folds_the_unmarked_one( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook + + monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) + messages: Final = _client_marked_messages() + chat_route_system: Final = AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(messages[1]) + (chat_route_part,) = _PARTS.validate_python(chat_route_system.get("content")) + + request: Final = LiteLLMResponsesTransformationHandler().transform_request( + model="openai.gpt-5.6-sol", + messages=messages, + optional_params={}, + litellm_params={"custom_llm_provider": "bedrock_mantle"}, + headers={}, + litellm_logging_obj=Mock(), + ) + + breakpoint_marker: Final = chat_route_part["prompt_cache_breakpoint"] + assert request["instructions"] == "Rules" + assert request["input"] == [ + { + "type": "message", + "role": "system", + "content": [{"type": "input_text", "text": "Stable prefix", "prompt_cache_breakpoint": breakpoint_marker}], + }, + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "hi", "prompt_cache_breakpoint": breakpoint_marker}], + }, + ] + assert messages == list(_CLIENT_MARKED_MESSAGES) + + +def _roles_and_texts(request_input: object) -> list[tuple[object, list[object]]]: + items: Final = (_MESSAGE.validate_python(item) for item in cast(list[object], request_input)) # cast-ok: wire list + return [ + (item.get("role"), [part.get("text") for part in _PARTS.validate_python(item.get("content"))]) for item in items + ] + + +@pytest.mark.parametrize( + "marked_system", + [ + {"role": "system", "content": "Stable prefix", "cache_control": _EPHEMERAL}, + {"role": "system", "content": [{"type": "text", "text": "Stable prefix", "cache_control": _EPHEMERAL}]}, + ], + ids=["marked_string", "marked_part"], +) +def test_transform_request_keeps_system_messages_in_order_when_a_marked_one_comes_first( + monkeypatch: pytest.MonkeyPatch, marked_system: AllMessageValues +) -> None: + monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) + + request: Final = LiteLLMResponsesTransformationHandler().transform_request( + model="openai.gpt-5.6-sol", + messages=[marked_system, {"role": "system", "content": "Rules"}, {"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={"custom_llm_provider": "bedrock_mantle"}, + headers={}, + litellm_logging_obj=Mock(), + ) + + assert "instructions" not in request + assert _roles_and_texts(request["input"]) == [ + ("system", ["Stable prefix"]), + ("system", ["Rules"]), + ("user", ["hi"]), + ] + + +@pytest.mark.parametrize( + ("row_key", "row", "model", "litellm_params"), + [ + pytest.param( + "luna-responses", + {"litellm_provider": "openai", "mode": "responses"}, + "luna-responses", + {"custom_llm_provider": "openai", "api_base": "https://api.openai.com/v1"}, + id="openai-model-without-breakpoints", + ), + pytest.param( + "luna-responses", + {"litellm_provider": "openai", "mode": "responses"}, + "luna-responses", + {"custom_llm_provider": "openai", "api_base": "https://api.openai.com/v1", "drop_params": True}, + id="openai-model-without-breakpoints-drop-params", + ), + pytest.param( + "bedrock_mantle/openai.gpt-5.6-sol", + _MANTLE_GPT_ROW, + "openai.gpt-5.6-sol", + {"custom_llm_provider": "bedrock_mantle", "model_info": {"supports_prompt_cache_breakpoint": False}}, + id="mantle-deployment-opted-out", + ), + ], +) +def test_transform_request_folds_system_messages_as_before_where_the_breakpoint_is_stripped( + monkeypatch: pytest.MonkeyPatch, + row_key: str, + row: Mapping[str, object], + model: str, + litellm_params: Mapping[str, object], +) -> None: + monkeypatch.setitem(litellm.model_cost, row_key, row) + + request: Final = LiteLLMResponsesTransformationHandler().transform_request( + model=model, + messages=[ + { + "role": "system", + "content": [{"type": "text", "text": "Stable prefix", "prompt_cache_breakpoint": _PINNED_EXPLICIT}], + }, + {"role": "system", "content": "Rules"}, + {"role": "user", "content": "hi"}, + ], + optional_params={}, + litellm_params=dict(litellm_params), + headers={}, + litellm_logging_obj=Mock(), + ) + + assert request["instructions"] == "Rules" + assert _roles_and_texts(request["input"]) == [("system", ["Stable prefix"]), ("user", ["hi"])] + assert list(_marked_parts_of(request["input"])) == [] + + +def test_transform_request_still_folds_a_string_system_message_behind_an_unmarked_list_one( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) + + request: Final = LiteLLMResponsesTransformationHandler().transform_request( + model="openai.gpt-5.6-sol", + messages=[ + {"role": "system", "content": [{"type": "text", "text": "Stable prefix"}]}, + {"role": "system", "content": "Rules"}, + {"role": "user", "content": "hi"}, + ], + optional_params={}, + litellm_params={"custom_llm_provider": "bedrock_mantle"}, + headers={}, + litellm_logging_obj=Mock(), + ) + + assert request["instructions"] == "Rules" + assert _roles_and_texts(request["input"]) == [("system", ["Stable prefix"]), ("user", ["hi"])] + + +def _mantle_marked_texts(model: str, messages: list[object], **litellm_params: object) -> set[tuple[object, object]]: + request: Final = LiteLLMResponsesTransformationHandler().transform_request( + model=model, + messages=cast(List[AllMessageValues], messages), # cast-ok: the test builds chat messages as plain mappings + optional_params={}, + litellm_params={"custom_llm_provider": "bedrock_mantle", **litellm_params}, + headers={}, + litellm_logging_obj=Mock(), + ) + return {(role, text) for role, text, _ in _marked_parts_of(request["input"])} + + +def test_transform_request_puts_no_breakpoint_on_empty_marked_text(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) + + marked: Final = _mantle_marked_texts( + "openai.gpt-5.6-sol", + [ + {"role": "system", "content": "", "cache_control": _EPHEMERAL}, + {"role": "user", "content": [{"type": "text", "text": "hi"}, {"type": "text", "text": "", "cache_control": _EPHEMERAL}]}, + { + "role": "user", + "content": [{"type": "text", "text": "there"}, {"type": "text", "text": ""}], + "cache_control": _EPHEMERAL, + }, + ], + ) + + assert marked == {("user", "there")} + + +def test_transform_request_maps_a_string_cache_control_like_an_object_one(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) + + marked: Final = _mantle_marked_texts( + "openai.gpt-5.6-sol", + [ + {"role": "system", "content": "Rules", "cache_control": "ephemeral"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "hi", "cache_control": "ephemeral"}, + {"type": "text", "text": "there", "cache_control": _EPHEMERAL}, + ], + }, + ], + ) + + assert marked == {("system", "Rules"), ("user", "hi"), ("user", "there")} + + +def test_transform_request_maps_client_cache_control_for_an_alias_whose_base_model_reads_breakpoints( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) + + marked: Final = _mantle_marked_texts( + "my-alias", list(_client_marked_messages()), base_model="openai.gpt-5.6-sol" + ) + + assert marked == {("system", "Stable prefix"), ("user", "hi")} + + +def _completion_marked_texts(model: str, model_info: Mapping[str, object]) -> set[tuple[object, object]]: + response_body: Final = { + "id": "resp_alias_marks", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": "gpt-5.6", + "output": [ + { + "type": "message", + "id": "msg_alias_marks", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + } + with respx.mock(assert_all_called=True) as router: + upstream: Final = router.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=response_body) + ) + litellm.completion( + model=model, + messages=_client_marked_messages(), + api_key="sk-test", + model_info=model_info, + cache={"no-cache": True}, + ) + request_body: Final = json.loads(upstream.calls[0].request.content) + return {(role, text) for role, text, _ in _marked_parts_of(request_body["input"])} + + +def test_completion_maps_client_cache_control_for_an_alias_whose_base_model_is_declared_in_model_info() -> None: + served_model_marks: Final = _completion_marked_texts("openai/responses/gpt-5.6", {}) + + assert served_model_marks + assert _completion_marked_texts("openai/responses/team-alias", {"base_model": "gpt-5.6"}) == served_model_marks + + +def test_transform_request_leaves_client_cache_control_unmapped_for_a_deployment_that_does_not_read_breakpoints( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem( + litellm.model_cost, "bedrock_mantle/gpt-6-luna", {**_MANTLE_GPT_ROW, "supports_prompt_cache_breakpoint": False} + ) + + request: Final = LiteLLMResponsesTransformationHandler().transform_request( + model="gpt-6-luna", + messages=_client_marked_messages(), + optional_params={"prompt_cache_options": _PINNED_EXPLICIT}, + litellm_params={"custom_llm_provider": "bedrock_mantle"}, + headers={}, + litellm_logging_obj=Mock(), + ) + + assert request["instructions"] == "Rules Stable prefix" + assert request["input"] == [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}] + + def test_mid_conversation_system_string_stays_in_input_after_a_user_turn(): handler: Final = LiteLLMResponsesTransformationHandler() @@ -5992,6 +6451,9 @@ def test_mid_conversation_system_reminder_as_string_and_as_text_block_produce_id assert as_string[1] == _system_input_item(reminder) +_HOST_WITHOUT_BREAKPOINTS: Final = {"custom_llm_provider": "openai", "api_base": "https://gateway.example.com/v1"} + + def test_claude_code_shaped_history_keeps_a_byte_stable_input_prefix_across_requests(): handler: Final = LiteLLMResponsesTransformationHandler() top_level_system: Final = [{"type": "text", "text": "You are Claude Code.", "cache_control": {"type": "ephemeral"}}] @@ -6031,7 +6493,7 @@ def test_claude_code_shaped_history_keeps_a_byte_stable_input_prefix_across_requ model="gpt-5.6-luna", messages=first_request_messages, optional_params={}, - litellm_params={}, + litellm_params=dict(_HOST_WITHOUT_BREAKPOINTS), headers={}, litellm_logging_obj=Mock(), ) @@ -6039,7 +6501,7 @@ def test_claude_code_shaped_history_keeps_a_byte_stable_input_prefix_across_requ model="gpt-5.6-luna", messages=second_request_messages, optional_params={}, - litellm_params={}, + litellm_params=dict(_HOST_WITHOUT_BREAKPOINTS), headers={}, litellm_logging_obj=Mock(), ) diff --git a/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py index 77bdacc938f..1021888ed98 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py @@ -1,15 +1,22 @@ import json import os import sys -from typing import Final +from collections.abc import Iterator +from typing import Final, List, cast +from unittest.mock import Mock import pytest sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../.."))) +import litellm +from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, +) from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) +from litellm.types.llms.openai import AllMessageValues MESSAGES = [{"role": "user", "content": "hello"}] CLAUDE_CODE_USER_ID = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "session-abc"}) @@ -138,3 +145,187 @@ def test_prepare_completion_kwargs_derives_no_prompt_cache_key_mistral_cannot_ta assert "prompt_cache_key" not in mistral_kwargs assert openai_kwargs["prompt_cache_key"] == "sitzung-ΓΌ" + + +_EPHEMERAL: Final = {"type": "ephemeral"} +_MANTLE_GPT_ROW: Final = { + "litellm_provider": "bedrock_mantle", + "mode": "responses", + "supports_prompt_cache_breakpoint": True, +} +_CLAUDE_CODE_MESSAGES: Final = ( + {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": _EPHEMERAL}]}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "checking", "cache_control": _EPHEMERAL}, + {"type": "tool_use", "id": "toolu_1", "name": "read", "input": {}, "cache_control": _EPHEMERAL}, + ], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "file body", "cache_control": _EPHEMERAL}], + }, +) +_SYSTEM: Final = [{"type": "text", "text": "Stable prefix", "cache_control": _EPHEMERAL}] +_TOOLS: Final = [{"name": "read", "description": "Read a file", "input_schema": {"type": "object"}, "cache_control": _EPHEMERAL}] + + +def _texts_with(key: str, value: object) -> Iterator[object]: + if isinstance(value, dict): + if key in value: + yield value.get("text", value.get("name")) + for nested in value.values(): + yield from _texts_with(key, nested) + if isinstance(value, list): + for item in value: + yield from _texts_with(key, item) + + +def _mantle_wire(model_info: dict[str, object] | None) -> tuple[dict[str, object], dict[str, object]]: + completion_kwargs, _ = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=64, + messages=json.loads(json.dumps(_CLAUDE_CODE_MESSAGES)), + model="bedrock_mantle/openai.gpt-5.6-sol", + system=json.loads(json.dumps(_SYSTEM)), + tools=json.loads(json.dumps(_TOOLS)), + extra_kwargs={"custom_llm_provider": "bedrock_mantle", "model_info": model_info}, + ) + wire: Final = LiteLLMResponsesTransformationHandler().transform_request( + model="openai.gpt-5.6-sol", + messages=cast(List[AllMessageValues], completion_kwargs["messages"]), # cast-ok: chat messages from the adapter + optional_params={"tools": completion_kwargs["tools"]}, + litellm_params={"custom_llm_provider": "bedrock_mantle", "model_info": model_info}, + headers={}, + litellm_logging_obj=Mock(), + ) + return dict(completion_kwargs), wire + + +def test_anthropic_messages_to_a_breakpoint_reading_target_carry_marks_where_the_openai_dialect_reads_them( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) + + completion_kwargs, wire = _mantle_wire(None) + + assert list(_texts_with("cache_control", completion_kwargs["tools"])) == [] + assert list(_texts_with("cache_control", cast(list[object], completion_kwargs["messages"])[2])) == [] + assert sorted(map(str, _texts_with("prompt_cache_breakpoint", wire["input"]))) == ["Stable prefix", "file body", "hi"] + assert list(_texts_with("cache_control", wire)) == [] + + +def test_anthropic_messages_to_a_deployment_that_opts_out_of_breakpoints_carry_no_marks( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) + + completion_kwargs, wire = _mantle_wire({"supports_prompt_cache_breakpoint": False}) + + assert list(_texts_with("cache_control", completion_kwargs)) == [] + assert list(_texts_with("prompt_cache_breakpoint", wire)) == [] + + +_OPENAI_GPT_ROW: Final = {"litellm_provider": "openai", "mode": "chat", "supports_prompt_cache_breakpoint": True} +_FOUNDRY_GPT_ROW: Final = {"litellm_provider": "azure_ai", "mode": "chat", "supports_prompt_cache_breakpoint": True} + + +@pytest.mark.parametrize( + ("model", "extra_kwargs", "keeps_marks"), + [ + pytest.param( + "azure/gpt-5.6-sol", + {"custom_llm_provider": "azure", "model_info": {"supports_prompt_cache_breakpoint": True}}, + False, + id="azure-chat-forwards-cache-control-unmapped", + ), + pytest.param( + "openai/team-alias", + { + "custom_llm_provider": "openai", + "base_model": "gpt-5.6-sol", + "api_base": "https://llm.example.com/v1", + "prompt_cache_options": {"mode": "explicit"}, + }, + False, + id="openai-chat-reads-only-the-served-model", + ), + pytest.param( + "openai/responses/team-alias", + { + "custom_llm_provider": "openai", + "base_model": "gpt-5.6-sol", + "api_base": "https://llm.example.com/v1", + "prompt_cache_options": {"mode": "explicit"}, + }, + True, + id="openai-responses-alias-bridge-reads-base-model", + ), + pytest.param( + "azure/responses/gpt-5.6-sol", + {"custom_llm_provider": "azure", "model_info": {"supports_prompt_cache_breakpoint": True}}, + True, + id="azure-responses-bridge-renames-marks", + ), + pytest.param( + "bedrock_mantle/team-alias", + { + "custom_llm_provider": "bedrock_mantle", + "model_info": {"mode": "responses", "base_model": "openai.gpt-5.6-sol"}, + "prompt_cache_options": {"mode": "explicit"}, + }, + True, + id="pinned-bridge-reads-model-info-base-model", + ), + pytest.param( + "bedrock_mantle/team-alias", + { + "custom_llm_provider": "bedrock_mantle", + "base_model": "openai.gpt-5.6-sol", + "prompt_cache_options": {"mode": "explicit"}, + }, + False, + id="mantle-chat-reads-only-the-served-model", + ), + pytest.param( + "litellm_proxy/gpt-5.6-sol", + { + "custom_llm_provider": "litellm_proxy", + "model_info": {"supports_prompt_cache_breakpoint": True, "litellm_provider": "litellm_proxy"}, + }, + False, + id="litellm-proxy-chat-forwards-cache-control-unmapped", + ), + pytest.param( + "sap/gpt-5.6-sol", + { + "custom_llm_provider": "sap", + "model_info": {"supports_prompt_cache_breakpoint": True, "litellm_provider": "sap"}, + }, + False, + id="sap-chat-forwards-cache-control-unmapped", + ), + pytest.param( + "azure_ai/team-alias", + {"custom_llm_provider": "azure_ai", "model_info": {"base_model": "gpt-5.6-sol"}}, + True, + id="foundry-chat-reads-model-info-base-model", + ), + ], +) +def test_anthropic_messages_keep_marks_only_for_a_route_that_turns_them_into_breakpoints( + monkeypatch: pytest.MonkeyPatch, model: str, extra_kwargs: dict[str, object], keeps_marks: bool +) -> None: + monkeypatch.setitem(litellm.model_cost, "gpt-5.6-sol", _OPENAI_GPT_ROW) + litellm.register_model({"azure_ai/gpt-5.6-sol": _FOUNDRY_GPT_ROW}) + monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW) + + completion_kwargs, _ = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=64, + messages=json.loads(json.dumps(_CLAUDE_CODE_MESSAGES)), + model=model, + system=json.loads(json.dumps(_SYSTEM)), + extra_kwargs=extra_kwargs, + ) + + assert bool(list(_texts_with("cache_control", completion_kwargs["messages"]))) is keeps_marks diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 263c2f7618d..7959d9092bd 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -1056,6 +1056,19 @@ class TestCacheControlAsPromptCacheBreakpoint: {"role": "user", "content": "unmarked"}, ] + def test_gpt_5_6_rewrites_a_string_mark_and_a_mark_on_empty_text(self): + messages: list[dict[str, object]] = [ + {"role": "system", "content": "", "cache_control": _EPHEMERAL}, + {"role": "user", "content": [_text("Describe this"), {**_text("plain"), "cache_control": "ephemeral"}]}, + {"role": "user", "content": [_text("context"), _marked(_text(""))]}, + ] + body = self._transform("gpt-5.6", messages) + assert body["messages"] == [ + {"role": "system", "content": [_breakpoint(_text(""))]}, + {"role": "user", "content": [_text("Describe this"), _breakpoint(_text("plain"))]}, + {"role": "user", "content": [_text("context"), _breakpoint(_text(""))]}, + ] + def test_gpt_5_6_message_level_mark_on_list_content_marks_the_last_part(self): messages: list[dict[str, object]] = [ {"role": "user", "content": [_text("first"), _text("last")], "cache_control": _EPHEMERAL},