From 5f6d22e7921de56e62524d2052e194300d80d2ee Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 04:16:22 -0700 Subject: [PATCH 1/3] Map cache_control_injection_points to OpenAI prompt_cache_breakpoint on GPT-5.6+ targets When the resolved deployment is provider openai and the model is GPT-5.6 or newer, the cache control hook now writes prompt_cache_breakpoint on the targeted content block and sets prompt_cache_options to explicit mode unless the caller already passed one. The /v1/messages bridges carry the marker through (the Responses bridge moves a marked system prompt into a developer message, since top-level instructions cannot hold one). Breakpoint counting and the stand-down check recognise both marker kinds, and client breakpoints already present in messages are no longer subtracted from the cap twice. Fixes #37509 --- .../anthropic_cache_control_hook.py | 216 ++++++-- .../prompt_templates/common_utils.py | 15 +- .../adapters/transformation.py | 10 + .../responses_adapters/transformation.py | 49 +- .../anthropic_cache_control_hook.py | 4 +- litellm/types/llms/anthropic.py | 4 + litellm/types/llms/openai.py | 11 + .../test_anthropic_cache_control_hook.py | 477 +++++++++++++++++- ...al_pass_through_adapters_transformation.py | 56 ++ .../test_handler_output_config_passthrough.py | 7 + .../test_responses_adapters_transformation.py | 152 ++++++ .../chat/test_openai_gpt_transformation.py | 45 ++ .../test_openai_responses_transformation.py | 58 +++ 13 files changed, 1052 insertions(+), 52 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 4df6fce74c0..05e2a37a230 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -10,17 +10,29 @@ Supported for both `v1/chat/completions` (via the prompt-management hook) and """ import copy +import re +from collections.abc import Iterable, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, cast from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.integrations.prompt_management_base import PromptManagementClient +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + with_prompt_cache_breakpoint, +) from litellm.types.integrations.anthropic_cache_control_hook import ( CacheControlInjectionPoint, CacheControlMessageInjectionPoint, ) -from litellm.types.llms.openai import AllMessageValues, ChatCompletionCachedContent +from litellm.types.llms.anthropic import AnthropicSystemMessageContent +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionCachedContent, + ChatCompletionTextObject, + PromptCacheBreakpoint, + PromptCacheOptions, +) from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.utils import StandardCallbackDynamicParams @@ -34,6 +46,27 @@ else: # breakpoints: "A maximum of 4 blocks with cache_control may be provided." MAX_CACHE_CONTROL_BLOCKS: Final = 4 +CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint") +OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6) +_GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?") +OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset({"text", "image", "image_url", "file", "input_audio"}) + + +def supports_openai_prompt_cache_breakpoint(model: str) -> bool: + version_match: Final = _GPT_VERSION_PATTERN.match(model.rsplit("/", 1)[-1].lower()) + if version_match is None: + return False + version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0)) + return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION + + +def _carries_cache_breakpoint(block: object) -> bool: + return isinstance(block, dict) and any(block.get(key) is not None for key in CACHE_BREAKPOINT_KEYS) + + +def _accepts_prompt_cache_breakpoint(block: object) -> bool: + return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES + class AnthropicCacheControlHook(CustomPromptManagement): def get_chat_completion_prompt( @@ -83,11 +116,21 @@ class AnthropicCacheControlHook(CustomPromptManagement): # limit, so reserve a slot for it here to leave room. reserved_blocks: Final = 1 if any(p.get("location") == "tool_config" for p in remaining_points) else 0 + openai_dialect: Final = AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( + model, injection_points[0].get("_litellm_provider") + ) + breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages) processed_messages = self._apply_message_injections( points=message_points, messages=processed_messages, max_blocks=MAX_CACHE_CONTROL_BLOCKS - reserved_blocks, + openai_dialect=openai_dialect, ) + if ( + openai_dialect + and AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages) > breakpoints_before + ): + non_default_params.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit")) # Pass through non-message injection points for provider-specific handling if remaining_points: @@ -97,11 +140,36 @@ class AnthropicCacheControlHook(CustomPromptManagement): return model, processed_messages, non_default_params + @staticmethod + def _targets_openai_prompt_cache_breakpoint(model: str | None, custom_llm_provider: str | None) -> bool: + if model is None or not supports_openai_prompt_cache_breakpoint(model): + return False + return (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) == "openai" + + @staticmethod + def _resolve_provider(model: str) -> str | None: + from litellm.exceptions import BadRequestError + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + try: + _, provider, _, _ = get_llm_provider(model=model) + except BadRequestError: + return None + return provider + + @staticmethod + def _count_request_cache_breakpoints(messages: Iterable[object], system: object = None) -> int: + system_blocks: Final = ( + sum(1 for block in system if _carries_cache_breakpoint(block)) if isinstance(system, list) else 0 + ) + return system_blocks + sum(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages) + @staticmethod def _apply_message_injections( points: list[CacheControlMessageInjectionPoint], messages: list[AllMessageValues], max_blocks: int, + openai_dialect: bool = False, ) -> list[AllMessageValues]: """Apply message-level cache control injection points in order. @@ -112,7 +180,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): ``max_blocks`` is reached. Injection points are honored in config order, so earlier points win when slots are scarce. """ - used_blocks = sum(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages) + used_blocks = AnthropicCacheControlHook._count_request_cache_breakpoints(messages) limit_reached = False for point in points: @@ -134,16 +202,17 @@ class AnthropicCacheControlHook(CustomPromptManagement): continue messages[target_index] = AnthropicCacheControlHook._safe_insert_cache_control_in_message( - messages[target_index], control + messages[target_index], control, openai_dialect ) - used_blocks += 1 + if AnthropicCacheControlHook._message_has_cache_control(messages[target_index]): + used_blocks += 1 if limit_reached: break if limit_reached: verbose_logger.warning( - "AnthropicCacheControlHook: Reached the Anthropic limit of %s cache_control blocks. Skipping further injection.", + "AnthropicCacheControlHook: Reached the provider limit of %s cache breakpoints. Skipping further injection.", MAX_CACHE_CONTROL_BLOCKS, ) @@ -189,16 +258,13 @@ class AnthropicCacheControlHook(CustomPromptManagement): return [] @staticmethod - def _count_cache_control_blocks(message: AllMessageValues) -> int: - """Count cache_control breakpoints on a message (message + content level).""" - count = 0 - if message.get("cache_control") is not None: - count += 1 + def _count_cache_control_blocks(message: object) -> int: + if not isinstance(message, dict): + return 0 + count = 1 if _carries_cache_breakpoint(message) else 0 content: Final = message.get("content") if isinstance(content, list): - for block in content: - if isinstance(block, dict) and block.get("cache_control") is not None: - count += 1 + count += sum(1 for block in content if _carries_cache_breakpoint(block)) return count @staticmethod @@ -208,7 +274,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): @staticmethod def _safe_insert_cache_control_in_message( - message: AllMessageValues, control: ChatCompletionCachedContent + message: AllMessageValues, control: ChatCompletionCachedContent, openai_dialect: bool = False ) -> AllMessageValues: """ Safe way to insert cache control in a message @@ -221,6 +287,9 @@ class AnthropicCacheControlHook(CustomPromptManagement): Per Anthropic's API specification, when using multiple content blocks, only the last content block can have cache_control. """ + if openai_dialect: + return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(message) + message_content: Final = message.get("content", None) # 1. if string, insert cache control in the message @@ -232,11 +301,43 @@ class AnthropicCacheControlHook(CustomPromptManagement): message_content[-1]["cache_control"] = control return message + @staticmethod + def _insert_prompt_cache_breakpoint_in_message(message: AllMessageValues) -> AllMessageValues: + if message.get("role") == "assistant": + return message + message_content: Final = message.get("content", None) + if isinstance(message_content, str): + marked: Final = copy.copy(message) + marked["content"] = [ + with_prompt_cache_breakpoint( + ChatCompletionTextObject(type="text", text=message_content), PromptCacheBreakpoint(mode="explicit") + ) + ] + return marked + if isinstance(message_content, list): + with_prompt_cache_breakpoint( + next((block for block in reversed(message_content) if _accepts_prompt_cache_breakpoint(block)), None), + PromptCacheBreakpoint(mode="explicit"), + ) + return message + + @staticmethod + def _system_block_with_breakpoint( + block: Mapping[str, object], control: ChatCompletionCachedContent, openai_dialect: bool + ) -> Mapping[str, object]: + marker: Final = ( + ("prompt_cache_breakpoint", PromptCacheBreakpoint(mode="explicit")) + if openai_dialect + else ("cache_control", control) + ) + return {**block, marker[0]: marker[1]} + @staticmethod def apply_to_anthropic_messages_request( messages: list[dict], system: str | list | None, injection_points: list[CacheControlInjectionPoint], + openai_dialect: bool = False, ) -> tuple[list[dict], str | list | None, list[CacheControlInjectionPoint]]: """Apply cache control injection for the Anthropic-native v1/messages endpoint. @@ -265,27 +366,27 @@ class AnthropicCacheControlHook(CustomPromptManagement): reserved_blocks: Final = 1 if any(p.get("location") == "tool_config" for p in remaining_points) else 0 max_blocks: Final = MAX_CACHE_CONTROL_BLOCKS - reserved_blocks - used_blocks = sum( - AnthropicCacheControlHook._count_cache_control_blocks(cast(AllMessageValues, msg)) - for msg in processed_messages - ) - if isinstance(processed_system, list): - used_blocks += sum( - 1 for b in processed_system if isinstance(b, dict) and b.get("cache_control") is not None - ) + message_blocks: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages) + system_blocks = AnthropicCacheControlHook._count_request_cache_breakpoints((), processed_system) - if system_points and processed_system is not None and used_blocks < max_blocks: + if system_points and processed_system is not None and message_blocks + system_blocks < max_blocks: system_already_has_cc: Final = isinstance(processed_system, list) and any( - isinstance(b, dict) and b.get("cache_control") is not None for b in processed_system + _carries_cache_breakpoint(b) for b in processed_system ) if not system_already_has_cc: control: Final = system_points[0].get("control") or ChatCompletionCachedContent(type="ephemeral") if isinstance(processed_system, str): - processed_system = [{"type": "text", "text": processed_system, "cache_control": control}] - used_blocks += 1 + processed_system = [ + AnthropicCacheControlHook._system_block_with_breakpoint( + AnthropicSystemMessageContent(type="text", text=processed_system), control, openai_dialect + ) + ] + system_blocks += 1 elif len(processed_system) > 0 and isinstance(processed_system[-1], dict): - processed_system[-1] = {**processed_system[-1], "cache_control": control} - used_blocks += 1 + processed_system[-1] = AnthropicCacheControlHook._system_block_with_breakpoint( + processed_system[-1], control, openai_dialect + ) + system_blocks += 1 for i, msg in enumerate(processed_messages): content = msg.get("content") @@ -295,7 +396,8 @@ class AnthropicCacheControlHook(CustomPromptManagement): processed_messages = AnthropicCacheControlHook._apply_message_injections( points=message_points, messages=cast(list[AllMessageValues], processed_messages), - max_blocks=max_blocks - used_blocks, + max_blocks=max_blocks - system_blocks, + openai_dialect=openai_dialect, ) return processed_messages, processed_system, remaining_points @@ -315,17 +417,43 @@ class AnthropicCacheControlHook(CustomPromptManagement): return ChatCompletionCachedContent(type="ephemeral") @staticmethod - def _stamped_as_judged(points: list[CacheControlInjectionPoint]) -> list[dict[str, object]]: + def _stamped_as_judged(points: Sequence[CacheControlInjectionPoint]) -> Sequence[Mapping[str, object]]: """Mark written-back points as having passed the client cache_control judgment. Builds copies because config-owned point dicts are shared across requests; mutating them would leak the stamp into future requests. """ - return [{**point, "_litellm_judged": True} for point in points] + return AnthropicCacheControlHook._stamped(points, "_litellm_judged", True) + + @staticmethod + def _judged_configured_points( + points: Sequence[CacheControlInjectionPoint], + messages: list[AllMessageValues], + tools: list[object] | None, + model: str, + custom_llm_provider: str | None, + ) -> Sequence[Mapping[str, object]] | None: + if AnthropicCacheControlHook._should_stand_down(points, messages, None, tools): + return None + return AnthropicCacheControlHook._stamped_with_provider(points, model, custom_llm_provider) + + @staticmethod + def _stamped_with_provider( + points: Sequence[CacheControlInjectionPoint], model: str, custom_llm_provider: str | None + ) -> Sequence[Mapping[str, object]]: + if custom_llm_provider is None or not supports_openai_prompt_cache_breakpoint(model): + return points + return AnthropicCacheControlHook._stamped(points, "_litellm_provider", custom_llm_provider) + + @staticmethod + def _stamped( + points: Sequence[CacheControlInjectionPoint], key: str, value: object + ) -> Sequence[Mapping[str, object]]: + return [{**point, key: value} for point in points] @staticmethod def _should_stand_down( - points: list[CacheControlInjectionPoint], + points: Sequence[CacheControlInjectionPoint], messages: list[AllMessageValues], system: str | list | None, tools: list | None, @@ -359,11 +487,8 @@ class AnthropicCacheControlHook(CustomPromptManagement): carry the mark either at the top level (Anthropic shape) or nested under ``function`` (OpenAI shape); the Anthropic chat transform accepts both. """ - if any(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages): + if AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) > 0: return True - if isinstance(system, list): - if any(isinstance(block, dict) and block.get("cache_control") is not None for block in system): - return True if tools is not None: return any( isinstance(tool, dict) @@ -452,10 +577,13 @@ class AnthropicCacheControlHook(CustomPromptManagement): unchanged. """ if non_default_params.get("cache_control_injection_points"): - if AnthropicCacheControlHook._should_stand_down( - non_default_params["cache_control_injection_points"], messages, None, tools - ): + judged: Final = AnthropicCacheControlHook._judged_configured_points( + non_default_params["cache_control_injection_points"], messages, tools, model, custom_llm_provider + ) + if judged is None: non_default_params.pop("cache_control_injection_points") + else: + non_default_params["cache_control_injection_points"] = judged return points: Final = AnthropicCacheControlHook.get_default_injection_points( messages=messages, @@ -513,11 +641,21 @@ class AnthropicCacheControlHook(CustomPromptManagement): if not injection_points: return messages, system + openai_dialect: Final = AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( + model, custom_llm_provider + ) + breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request( messages=messages, system=system, injection_points=injection_points, + openai_dialect=openai_dialect, ) + if ( + openai_dialect + and AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) > breakpoints_before + ): + kwargs.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit")) if remaining: kwargs["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(remaining) return messages, system diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index e4c1c9fc5cf..5904a14e79f 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -10,7 +10,7 @@ from collections.abc import Iterable, Mapping, Sequence from itertools import groupby from os import PathLike from pathlib import Path -from typing import TYPE_CHECKING, Any, Final, Literal, cast +from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast from openai.types.chat.chat_completion_custom_tool_param import ( CustomFormatGrammar, @@ -1325,6 +1325,19 @@ def check_is_function_call(logging_obj: "LoggingClass") -> bool: return False +_MarkedT = TypeVar("_MarkedT") + + +def _set_prompt_cache_breakpoint(target: object, marker: object) -> None: + if marker is not None and isinstance(target, dict): + target["prompt_cache_breakpoint"] = marker + + +def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT: + _set_prompt_cache_breakpoint(target, marker) + return target + + def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Any: """ Filters a value from a dictionary diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index e45414b4a73..2ee50e2a55b 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -61,6 +61,7 @@ from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingCho from litellm.litellm_core_utils.prompt_templates.common_utils import ( parse_tool_call_arguments, + with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, @@ -308,6 +309,11 @@ class LiteLLMAnthropicMessagesAdapter: # Fallback for non-dict objects (shouldn't happen in practice) cast(dict[str, object], target)["cache_control"] = cache_control + @staticmethod + def _add_prompt_cache_breakpoint_if_present(source: object, target: object) -> None: + if isinstance(source, dict) and "prompt_cache_breakpoint" in source: + with_prompt_cache_breakpoint(target, source["prompt_cache_breakpoint"]) + def translatable_anthropic_params(self) -> list[str]: """ Which anthropic params, we need to translate to the openai format. @@ -368,6 +374,7 @@ class LiteLLMAnthropicMessagesAdapter: if content.get("type") == "text": text_obj = ChatCompletionTextObject(type="text", text=content.get("text", "")) self._add_cache_control_if_applicable(content, text_obj, model) + self._add_prompt_cache_breakpoint_if_present(content, text_obj) new_user_content_list.append(text_obj) elif content.get("type") == "image": # Convert Anthropic image format to OpenAI format @@ -378,6 +385,7 @@ class LiteLLMAnthropicMessagesAdapter: image_url_obj = ChatCompletionImageUrlObject(url=openai_image_url) image_obj = ChatCompletionImageObject(type="image_url", image_url=image_url_obj) self._add_cache_control_if_applicable(content, image_obj, model) + self._add_prompt_cache_breakpoint_if_present(content, image_obj) new_user_content_list.append(image_obj) elif content.get("type") == "document": # Convert Anthropic document format (PDF, etc.) to OpenAI format @@ -869,6 +877,7 @@ class LiteLLMAnthropicMessagesAdapter: continue text_obj = ChatCompletionTextObject(type="text", text=text) self._add_cache_control_if_applicable(block, text_obj, model) + self._add_prompt_cache_breakpoint_if_present(block, text_obj) text_parts.append(text_obj) return ChatCompletionSystemMessage(role="system", content=text_parts) if text_parts else None @@ -900,6 +909,7 @@ class LiteLLMAnthropicMessagesAdapter: "text": block.get("text", ""), } self._add_cache_control_if_applicable(block, text_block, model_name) + self._add_prompt_cache_breakpoint_if_present(block, text_block) openai_system_content.append(text_block) if openai_system_content: new_messages.insert( diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 21a8cb9501e..18f33c63e4d 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -12,6 +12,7 @@ from typing import Any, Final, cast from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, + with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking_budget, @@ -82,7 +83,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: @staticmethod def _translate_midturn_system_content_to_responses( content: str | Iterable[AnthropicSystemMessageContent], - ) -> list[dict[str, str]]: # mutable-ok: API message payload + ) -> list[dict[str, object]]: # mutable-ok: API message payload """Convert in-sequence system content to Responses input-text parts.""" if isinstance(content, str): return ( @@ -91,7 +92,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if not isinstance(content, list): return [] # mutable-ok: API message payload return [ # mutable-ok: API message payload - {"type": "input_text", "text": text} # mutable-ok: API message payload + with_prompt_cache_breakpoint( + {"type": "input_text", "text": text}, block.get("prompt_cache_breakpoint") + ) # mutable-ok: API message payload for block in content if isinstance(block, dict) and block.get("type") == "text" and (text := block.get("text")) # pyright: ignore[reportUnnecessaryIsInstance] # untrusted client payload ] @@ -146,11 +149,20 @@ class LiteLLMAnthropicToResponsesAPIAdapter: continue btype = block.get("type") if btype == "text": - user_parts.append({"type": "input_text", "text": block.get("text", "")}) + user_parts.append( + with_prompt_cache_breakpoint( + {"type": "input_text", "text": block.get("text", "")}, + block.get("prompt_cache_breakpoint"), + ) + ) elif btype == "image": url = self._translate_anthropic_image_source_to_url(cast(dict, block.get("source", {}))) if url: - user_parts.append({"type": "input_image", "image_url": url}) + user_parts.append( + with_prompt_cache_breakpoint( + {"type": "input_image", "image_url": url}, block.get("prompt_cache_breakpoint") + ) + ) elif btype == "tool_result": tool_use_id = block.get("tool_use_id", "") inner = block.get("content") @@ -376,19 +388,36 @@ class LiteLLMAnthropicToResponsesAPIAdapter: anthropic_request["messages"], ) + input_items: Final = self.translate_messages_to_responses_input(messages_list) + system: Final = anthropic_request.get("system") + developer_parts: Final = ( + self._translate_midturn_system_content_to_responses(system) + if isinstance(system, list) + and any(isinstance(block, dict) and block.get("prompt_cache_breakpoint") is not None for block in system) + else () + ) + if developer_parts: + input_items.insert( + 0, + { # mutable-ok: API message payload + "type": "message", + "role": "developer", + "content": developer_parts, + }, + ) + responses_kwargs: Final[dict[str, Any]] = { "model": model, - "input": self.translate_messages_to_responses_input(messages_list), + "input": input_items, } - # system -> instructions - system: Final = anthropic_request.get("system") - if system: + if system and not developer_parts: if isinstance(system, str): responses_kwargs["instructions"] = system elif isinstance(system, list): - text_parts = [b.get("text", "") for b in system if isinstance(b, dict) and b.get("type") == "text"] - responses_kwargs["instructions"] = "\n".join(filter(None, text_parts)) + responses_kwargs["instructions"] = "\n".join( + filter(None, (b.get("text", "") for b in system if isinstance(b, dict) and b.get("type") == "text")) + ) # max_tokens -> max_output_tokens max_tokens: Final = anthropic_request.get("max_tokens") diff --git a/litellm/types/integrations/anthropic_cache_control_hook.py b/litellm/types/integrations/anthropic_cache_control_hook.py index da9b26ebbd8..3b3dcf5c86d 100644 --- a/litellm/types/integrations/anthropic_cache_control_hook.py +++ b/litellm/types/integrations/anthropic_cache_control_hook.py @@ -1,6 +1,6 @@ from typing import Literal -from typing_extensions import NotRequired, TypedDict +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.types.llms.openai import ChatCompletionCachedContent @@ -13,6 +13,7 @@ class CacheControlMessageInjectionPoint(TypedDict): index: int | str | None # Optional: target by specific index control: ChatCompletionCachedContent | None _litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran + _litellm_provider: NotRequired[ReadOnly[str]] class CacheControlToolConfigInjectionPoint(TypedDict): @@ -21,6 +22,7 @@ class CacheControlToolConfigInjectionPoint(TypedDict): location: Literal["tool_config"] control: ChatCompletionCachedContent | None _litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran + _litellm_provider: NotRequired[ReadOnly[str]] CacheControlInjectionPoint = CacheControlMessageInjectionPoint | CacheControlToolConfigInjectionPoint diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 5179210f942..43e1d3a4e11 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -9,6 +9,7 @@ from .openai import ( ChatCompletionCachedContent, ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock, + PromptCacheBreakpoint, ) @@ -201,6 +202,7 @@ class AnthropicMessagesTextParam(TypedDict, total=False): type: Required[Literal["text"]] text: Required[str] cache_control: dict | ChatCompletionCachedContent | None + prompt_cache_breakpoint: ReadOnly[PromptCacheBreakpoint] class AnthropicMessagesToolUseParam(TypedDict, total=False): @@ -261,6 +263,7 @@ class AnthropicMessagesImageParam(TypedDict, total=False): type: Required[Literal["image"]] source: Required[AnthropicContentParamSource | AnthropicContentParamSourceFileId | AnthropicContentParamSourceUrl] cache_control: dict | ChatCompletionCachedContent | None + prompt_cache_breakpoint: ReadOnly[PromptCacheBreakpoint] class CitationsObject(TypedDict): @@ -347,6 +350,7 @@ class AnthropicSystemMessageContent(TypedDict, total=False): type: str text: str cache_control: dict | ChatCompletionCachedContent | None + prompt_cache_breakpoint: ReadOnly[PromptCacheBreakpoint] class AnthropicMessagesSystemMessageParam(TypedDict, total=False): diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index edfc50c99f6..4c7f416ab8c 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -71,6 +71,7 @@ from pydantic import ( ) from typing_extensions import ( NotRequired, + ReadOnly, Required, TypedDict, override, @@ -510,6 +511,15 @@ class ChatCompletionCachedContent(TypedDict): ttl: NotRequired[Literal["5m", "1h"]] +class PromptCacheBreakpoint(TypedDict): + mode: ReadOnly[Literal["explicit"]] + + +class PromptCacheOptions(TypedDict, total=False): + mode: ReadOnly[Literal["implicit", "explicit"]] + ttl: ReadOnly[Literal["30m"]] + + class ChatCompletionThinkingBlock(TypedDict, total=False): type: Required[Literal["thinking"]] thinking: str @@ -1148,6 +1158,7 @@ class ResponsesAPIOptionalRequestParams(TypedDict, total=False): max_tool_calls: int | None prompt_cache_key: str | None prompt_cache_retention: str | None + prompt_cache_options: ReadOnly[PromptCacheOptions | None] stream_options: ResponsesAPIStreamOptions | None top_logprobs: int | None partial_images: int | None # Number of partial images to generate (1-3) for streaming image generation diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index cc43a424419..04876b9bf12 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -14,7 +14,10 @@ import pytest sys.path.insert(0, os.path.abspath("../..")) # Adds the parent directory to the system-path import litellm -from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook +from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + supports_openai_prompt_cache_breakpoint, +) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import StandardCallbackDynamicParams @@ -1996,3 +1999,475 @@ class TestAnthropicPromptCachingEnvVars: """An unparseable TTL must fall back to Anthropic's 5m default, never reach the provider verbatim.""" _, ttl = self._import_litellm_with_env({"LITELLM_ANTHROPIC_PROMPT_CACHING_TTL": value}) assert ttl is None + + +def _contains_key(value, key) -> bool: + if isinstance(value, dict): + return key in value or any(_contains_key(v, key) for v in value.values()) + if isinstance(value, list): + return any(_contains_key(v, key) for v in value) + return False + + +class TestOpenAIPromptCacheBreakpoint: + """OpenAI GPT-5.6+ targets get content-block `prompt_cache_breakpoint` markers and a + request-level `prompt_cache_options` instead of Anthropic `cache_control` (#37509).""" + + EXPLICIT = {"mode": "explicit"} + SYSTEM_POINT = [{"location": "message", "role": "system"}] + + @staticmethod + def _inject(messages, system, kwargs, model="openai/gpt-5.6", custom_llm_provider=None): + return AnthropicCacheControlHook.maybe_inject_cache_control( + copy.deepcopy(messages), + copy.deepcopy(system), + kwargs, + model=model, + custom_llm_provider=custom_llm_provider, + ) + + @staticmethod + def _chat(messages, params, model="openai/gpt-5.6"): + return AnthropicCacheControlHook().get_chat_completion_prompt( + model=model, + messages=copy.deepcopy(messages), + non_default_params=params, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + + @pytest.mark.parametrize( + "model,expected", + [ + ("gpt-5.6", True), + ("openai/gpt-5.6", True), + ("gpt-5.6-sol", True), + ("gpt-5.6-luna", True), + ("gpt-5.7", True), + ("gpt-6", True), + ("GPT-5.6", True), + ("gpt-5.5", False), + ("gpt-5", False), + ("gpt-5-chat-latest", False), + ("gpt-4.1", False), + ("o3", False), + ("claude-sonnet-4-5", False), + ], + ) + def test_model_support_truth_table(self, model, expected): + assert supports_openai_prompt_cache_breakpoint(model) is expected + + @pytest.mark.parametrize( + "model,provider,expected", + [ + ("openai/gpt-5.6", None, True), + ("gpt-5.6", None, True), + ("gpt-5.6", "openai", True), + ("gpt-5.6", "azure", False), + ("azure/gpt-5.6", None, False), + ("openai/gpt-4.1", None, False), + ("anthropic/claude-sonnet-4-5", None, False), + ("no-provider-can-route-this-model", None, False), + (None, "openai", False), + ], + ) + def test_dialect_resolution(self, model, provider, expected): + assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(model, provider) is expected + + def test_count_covers_both_marker_kinds(self): + message = { + "role": "user", + "cache_control": {"type": "ephemeral"}, + "content": [ + {"type": "text", "text": "a", "prompt_cache_breakpoint": self.EXPLICIT}, + {"type": "text", "text": "b", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "c"}, + ], + } + assert AnthropicCacheControlHook._count_cache_control_blocks(message) == 3 + + def test_v1_messages_string_system_gets_block_breakpoint(self): + kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + messages, system = self._inject([{"role": "user", "content": "hi"}], "sys", kwargs) + assert system == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}] + assert messages == [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + assert kwargs == {"prompt_cache_options": self.EXPLICIT} + assert not _contains_key(system, "cache_control") + + def test_v1_messages_list_system_marks_last_block_only(self): + kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + system = [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}] + _, result_system = self._inject([{"role": "user", "content": "hi"}], system, kwargs) + assert result_system == [ + {"type": "text", "text": "a"}, + {"type": "text", "text": "b", "prompt_cache_breakpoint": self.EXPLICIT}, + ] + assert kwargs["prompt_cache_options"] == self.EXPLICIT + + def test_v1_messages_targets_by_role(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "first"}, {"type": "text", "text": "second"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "reply"}]}, + {"role": "user", "content": "last"}, + ] + kwargs = {"cache_control_injection_points": [{"location": "message", "role": "user"}]} + result, _ = self._inject(messages, None, kwargs) + assert result[0]["content"] == [ + {"type": "text", "text": "first"}, + {"type": "text", "text": "second", "prompt_cache_breakpoint": self.EXPLICIT}, + ] + assert result[1] == messages[1] + assert result[2]["content"] == [{"type": "text", "text": "last", "prompt_cache_breakpoint": self.EXPLICIT}] + assert kwargs["prompt_cache_options"] == self.EXPLICIT + + def test_v1_messages_targets_by_index(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "first"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "reply"}]}, + {"role": "user", "content": [{"type": "text", "text": "last"}]}, + ] + kwargs = {"cache_control_injection_points": [{"location": "message", "index": -1}]} + result, _ = self._inject(messages, None, kwargs) + assert result[:2] == messages[:2] + assert result[2]["content"] == [{"type": "text", "text": "last", "prompt_cache_breakpoint": self.EXPLICIT}] + + def test_v1_messages_control_field_is_ignored(self): + ttl_control = {"type": "ephemeral", "ttl": "1h"} + kwargs = { + "cache_control_injection_points": [ + {"location": "message", "role": "system", "control": ttl_control}, + {"location": "message", "index": -1, "control": ttl_control}, + ] + } + messages, system = self._inject([{"role": "user", "content": "hi"}], "sys", kwargs) + assert system[0]["prompt_cache_breakpoint"] == self.EXPLICIT + assert messages[0]["content"][-1]["prompt_cache_breakpoint"] == self.EXPLICIT + assert not _contains_key(system, "cache_control") + assert not _contains_key(messages, "cache_control") + + def test_v1_messages_keeps_caller_prompt_cache_options(self): + caller_options = {"mode": "explicit", "ttl": "30m"} + kwargs = { + "cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT), + "prompt_cache_options": dict(caller_options), + } + _, system = self._inject([{"role": "user", "content": "hi"}], "sys", kwargs) + assert system[0]["prompt_cache_breakpoint"] == self.EXPLICIT + assert kwargs["prompt_cache_options"] == caller_options + + def test_v1_messages_no_prompt_cache_options_when_nothing_injected(self): + kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + messages, system = self._inject([{"role": "user", "content": "hi"}], None, kwargs) + assert system is None + assert "prompt_cache_options" not in kwargs + assert not _contains_key(messages, "prompt_cache_breakpoint") + + def test_v1_messages_anthropic_target_unchanged(self): + kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + _, system = self._inject( + [{"role": "user", "content": "hi"}], + "sys", + kwargs, + model="anthropic/claude-sonnet-4-5", + custom_llm_provider="anthropic", + ) + assert system == [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}] + assert kwargs == {} + + def test_v1_messages_older_openai_model_keeps_cache_control(self): + kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + _, system = self._inject([{"role": "user", "content": "hi"}], "sys", kwargs, model="openai/gpt-4.1") + assert system == [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}] + assert kwargs == {} + + def test_v1_messages_client_content_breakpoint_makes_configured_points_stand_down(self): + messages = [{"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]}] + kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + result, system = self._inject(messages, "sys", kwargs) + assert result == messages + assert system == "sys" + assert kwargs == {} + + def test_v1_messages_client_system_breakpoint_makes_configured_points_stand_down(self): + system = [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}] + messages = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + kwargs = {"cache_control_injection_points": [{"location": "message", "index": -1}]} + result, result_system = self._inject(messages, system, kwargs) + assert result == messages + assert result_system == system + assert kwargs == {} + + def test_chat_system_string_wrapped_with_block_breakpoint(self): + params = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + messages = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}] + _, processed, returned = self._chat(messages, params) + assert processed[0] == { + "role": "system", + "content": [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}], + } + assert processed[1] == {"role": "user", "content": "hi"} + assert returned is params + assert returned == {"prompt_cache_options": self.EXPLICIT} + + def test_chat_list_content_marks_last_block(self): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, + ], + } + ] + params = {"cache_control_injection_points": [{"location": "message", "index": -1}]} + _, processed, _ = self._chat(messages, params) + assert processed[0]["content"] == [ + {"type": "text", "text": "look"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/a.png"}, + "prompt_cache_breakpoint": self.EXPLICIT, + }, + ] + assert params["prompt_cache_options"] == self.EXPLICIT + + def test_chat_unprefixed_model_resolves_to_openai(self): + params = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + _, processed, _ = self._chat([{"role": "system", "content": "sys"}], params, model="gpt-5.6") + assert processed[0]["content"] == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}] + assert params["prompt_cache_options"] == self.EXPLICIT + + def test_chat_keeps_caller_prompt_cache_options(self): + params = { + "cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT), + "prompt_cache_options": {"mode": "implicit"}, + } + self._chat([{"role": "system", "content": "sys"}], params) + assert params["prompt_cache_options"] == {"mode": "implicit"} + + def test_chat_no_prompt_cache_options_when_nothing_injected(self): + params = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + messages = [{"role": "user", "content": "hi"}] + _, processed, _ = self._chat(messages, params) + assert processed == messages + assert params == {} + + @pytest.mark.parametrize("model", ["openai/gpt-4.1", "anthropic/claude-sonnet-4-5"]) + def test_chat_other_targets_keep_message_level_cache_control(self, model): + params = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + _, processed, _ = self._chat([{"role": "system", "content": "sys"}], params, model=model) + assert processed[0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}} + assert params == {} + + def test_chat_client_breakpoint_makes_seeded_points_stand_down(self): + params = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=[ + {"role": "system", "content": "sys"}, + {"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]}, + ], + model="openai/gpt-5.6", + custom_llm_provider="openai", + ) + assert params == {} + + def test_cap_counts_client_breakpoints_of_both_kinds(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "a", "prompt_cache_breakpoint": self.EXPLICIT}]}, + {"role": "user", "content": [{"type": "text", "text": "b", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": [{"type": "text", "text": "c", "prompt_cache_breakpoint": self.EXPLICIT}]}, + {"role": "user", "content": "d"}, + {"role": "user", "content": "e"}, + ] + result = AnthropicCacheControlHook._apply_message_injections( + points=[{"location": "message", "role": "user"}], + messages=copy.deepcopy(messages), + max_blocks=4, + openai_dialect=True, + ) + assert result[:3] == messages[:3] + assert result[3]["content"] == [{"type": "text", "text": "d", "prompt_cache_breakpoint": self.EXPLICIT}] + assert result[4] == {"role": "user", "content": "e"} + + +class TestOpenAIPromptCacheBreakpointPlacementRules: + """OpenAI dialect only marks blocks OpenAI (and the /v1/messages bridges) can carry (#37509).""" + + EXPLICIT = {"mode": "explicit"} + + def _chat(self, messages, points, model="openai/gpt-5.6"): + params = {"cache_control_injection_points": copy.deepcopy(points)} + _, out, params = AnthropicCacheControlHook().get_chat_completion_prompt( + model=model, + messages=copy.deepcopy(messages), + non_default_params=params, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + return out, params + + def test_assistant_message_is_never_marked_on_chat_path(self): + messages = [{"role": "user", "content": "q"}, {"role": "assistant", "content": "a"}] + out, params = self._chat(messages, [{"location": "message", "role": "assistant"}]) + assert out == messages + assert "prompt_cache_options" not in params + + def test_tool_message_text_is_marked_on_chat_path(self): + messages = [ + {"role": "user", "content": "weather?"}, + {"role": "assistant", "content": None, "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "w", "arguments": "{}"}}]}, + {"role": "tool", "tool_call_id": "c1", "content": "sunny"}, + ] + out, params = self._chat(messages, [{"location": "message", "index": -1}]) + assert out[2]["content"] == [{"type": "text", "text": "sunny", "prompt_cache_breakpoint": self.EXPLICIT}] + assert params["prompt_cache_options"] == self.EXPLICIT + + def test_tool_result_only_turn_is_skipped_on_v1_messages(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "q"}]}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "t1", "name": "w", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "sunny"}]}, + ] + kwargs = {"cache_control_injection_points": [{"location": "message", "index": -1}]} + out, system = AnthropicCacheControlHook.maybe_inject_cache_control( + copy.deepcopy(messages), None, kwargs, model="openai/gpt-5.6" + ) + assert out == messages + assert system is None + assert "prompt_cache_options" not in kwargs + + def test_assistant_turn_is_skipped_on_v1_messages(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "q"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "a"}]}, + ] + kwargs = {"cache_control_injection_points": [{"location": "message", "role": "assistant"}]} + out, _ = AnthropicCacheControlHook.maybe_inject_cache_control( + copy.deepcopy(messages), None, kwargs, model="openai/gpt-5.6" + ) + assert out == messages + assert "prompt_cache_options" not in kwargs + + def test_text_after_tool_result_is_marked(self): + messages = [ + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": "sunny"}, + {"type": "text", "text": "thanks"}, + ], + } + ] + kwargs = {"cache_control_injection_points": [{"location": "message", "index": -1}]} + out, _ = AnthropicCacheControlHook.maybe_inject_cache_control(messages, None, kwargs, model="openai/gpt-5.6") + assert out[0]["content"] == [ + {"type": "tool_result", "tool_use_id": "t1", "content": "sunny"}, + {"type": "text", "text": "thanks", "prompt_cache_breakpoint": self.EXPLICIT}, + ] + assert kwargs["prompt_cache_options"] == self.EXPLICIT + + def test_marker_walks_back_to_last_eligible_block(self): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "read this"}, + {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "doc"}}, + ], + } + ] + kwargs = {"cache_control_injection_points": [{"location": "message", "index": -1}]} + out, _ = AnthropicCacheControlHook.maybe_inject_cache_control(messages, None, kwargs, model="openai/gpt-5.6") + assert out[0]["content"][0] == {"type": "text", "text": "read this", "prompt_cache_breakpoint": self.EXPLICIT} + assert "prompt_cache_breakpoint" not in out[0]["content"][1] + + def test_skipped_block_does_not_consume_a_slot(self): + messages = [{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t0", "content": "r"}]}] + [ + {"role": "user", "content": [{"type": "text", "text": f"m{i}"}]} for i in range(4) + ] + kwargs = {"cache_control_injection_points": [{"location": "message", "role": "user"}]} + out, _ = AnthropicCacheControlHook.maybe_inject_cache_control(messages, None, kwargs, model="openai/gpt-5.6") + assert "prompt_cache_breakpoint" not in out[0]["content"][0] + assert all(msg["content"][0]["prompt_cache_breakpoint"] == self.EXPLICIT for msg in out[1:]) + + +class TestChatPathProviderStamp: + """The chat path learns the caller's custom_llm_provider through the seeded points (#37509).""" + + POINTS = [{"location": "message", "role": "system"}] + MESSAGES = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}] + + def _seed_and_run(self, model, custom_llm_provider): + params = {"cache_control_injection_points": copy.deepcopy(self.POINTS)} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=copy.deepcopy(self.MESSAGES), + model=model, + custom_llm_provider=custom_llm_provider, + ) + _, out, params = AnthropicCacheControlHook().get_chat_completion_prompt( + model=model, + messages=copy.deepcopy(self.MESSAGES), + non_default_params=params, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + return out, params + + def test_openai_compatible_provider_keeps_anthropic_style_markers(self): + out, params = self._seed_and_run("gpt-5.6", "hosted_vllm") + assert out[0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}} + assert "prompt_cache_options" not in params + + def test_explicit_openai_provider_uses_openai_dialect(self): + out, params = self._seed_and_run("gpt-5.6", "openai") + assert out[0]["content"] == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}] + assert params["prompt_cache_options"] == {"mode": "explicit"} + + def test_bare_gpt_model_without_provider_resolves_to_openai(self): + out, params = self._seed_and_run("gpt-5.6", None) + assert out[0]["content"] == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}] + assert params["prompt_cache_options"] == {"mode": "explicit"} + + def test_points_keep_identity_for_models_below_gpt_5_6(self): + points = copy.deepcopy(self.POINTS) + params = {"cache_control_injection_points": points} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=copy.deepcopy(self.MESSAGES), + model="anthropic/claude-sonnet-4-5", + custom_llm_provider="anthropic", + ) + assert params["cache_control_injection_points"] is points + + def test_provider_lookup_skipped_for_models_below_gpt_5_6(self): + from unittest.mock import patch + + with patch.object(AnthropicCacheControlHook, "_resolve_provider") as resolve: + assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("gpt-4.1", None) is False + assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("my-custom-model", None) is False + resolve.assert_not_called() + + +class TestClientBreakpointsCountedOnce: + def test_client_message_breakpoints_are_not_double_counted(self): + messages = [{"role": "user", "content": [{"type": "text", "text": "m0", "cache_control": {"type": "ephemeral"}}]}] + [ + {"role": "user", "content": [{"type": "text", "text": f"m{i}"}]} for i in range(1, 4) + ] + out, system, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request( + messages=messages, + system="sys", + injection_points=[ + {"location": "message", "role": "system"}, + {"location": "message", "index": -1}, + {"location": "message", "index": -2}, + {"location": "message", "index": -3}, + ], + ) + marked = [msg["content"][0].get("cache_control") is not None for msg in out] + assert marked == [True, False, True, True] + assert system[0]["cache_control"] == {"type": "ephemeral"} diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 1185893d428..abb976c3560 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -3741,3 +3741,59 @@ def test_tool_result_plain_text_unchanged_by_openai_transform(): assert len(tool_messages) == 1 assert tool_messages[0]["content"] == "42 files found" assert _image_urls_in_user_messages(result) == [] + + +def test_translate_anthropic_to_openai_carries_prompt_cache_breakpoint_on_system_and_user_blocks(): + explicit = {"mode": "explicit"} + openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + anthropic_message_request={ + "model": "gpt-5.6", + "max_tokens": 64, + "system": [{"type": "text", "text": "sys", "prompt_cache_breakpoint": explicit}], + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "hi", "prompt_cache_breakpoint": explicit}, + { + "type": "image", + "source": {"type": "url", "url": "https://example.com/a.png"}, + "prompt_cache_breakpoint": explicit, + }, + ], + } + ], + } + ) + assert openai_request["messages"][0] == { + "role": "system", + "content": [{"type": "text", "text": "sys", "prompt_cache_breakpoint": explicit}], + } + user_content = openai_request["messages"][1]["content"] + assert user_content[0] == {"type": "text", "text": "hi", "prompt_cache_breakpoint": explicit} + assert user_content[1]["type"] == "image_url" + assert user_content[1]["prompt_cache_breakpoint"] == explicit + + +def test_translate_anthropic_to_openai_without_prompt_cache_breakpoint_adds_nothing(): + openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + anthropic_message_request={ + "model": "gpt-5.6", + "max_tokens": 64, + "system": [{"type": "text", "text": "sys"}], + "messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}], + } + ) + assert openai_request["messages"][0] == {"role": "system", "content": [{"type": "text", "text": "sys"}]} + assert openai_request["messages"][1]["content"] == [{"type": "text", "text": "hi"}] + + +def test_translate_anthropic_messages_to_openai_carries_midturn_system_prompt_cache_breakpoint(): + explicit = {"mode": "explicit"} + result = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai( + messages=[{"role": "system", "content": [{"type": "text", "text": "fix", "prompt_cache_breakpoint": explicit}]}], + model="gpt-5.6", + ) + assert result == [ + {"role": "system", "content": [{"type": "text", "text": "fix", "prompt_cache_breakpoint": explicit}]} + ] diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py index 615dc5cfebc..a944afc6152 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py @@ -230,3 +230,10 @@ class TestEmptyExtraKwargsPath: # dict-like result. completion_kwargs = result[0] if isinstance(result, tuple) else result assert isinstance(completion_kwargs, dict) + + +class TestPromptCacheOptionsForwarded: + def test_prompt_cache_options_reaches_completion_kwargs(self): + result = _call_prepare(extra_kwargs={"prompt_cache_options": {"mode": "explicit"}}, model="gpt-5.6") + completion_kwargs = result[0] if isinstance(result, tuple) else result + assert completion_kwargs["prompt_cache_options"] == {"mode": "explicit"} diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 876213eda3f..8228d41cf82 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -1413,3 +1413,155 @@ class TestToolResultImages: outputs = [item for item in items if item.get("type") == "function_call_output"] assert outputs[0]["output"] == "screenshot saved" assert self._input_images(items) == [] + + +def _contains_key(value, key) -> bool: + if isinstance(value, dict): + return key in value or any(_contains_key(v, key) for v in value.values()) + if isinstance(value, list): + return any(_contains_key(v, key) for v in value) + return False + + +class TestPromptCacheBreakpointToResponses: + """OpenAI `prompt_cache_breakpoint` markers ride through the /v1/messages -> Responses bridge (#37509).""" + + EXPLICIT = {"mode": "explicit"} + + def test_system_with_breakpoint_becomes_leading_developer_message(self): + request = _make_request( + model="openai/gpt-5.6", + system=[ + {"type": "text", "text": "Be concise."}, + {"type": "text", "text": "Be helpful.", "prompt_cache_breakpoint": self.EXPLICIT}, + ], + ) + kwargs = _ADAPTER.translate_request(request) + assert "instructions" not in kwargs + assert kwargs["input"] == [ + { + "type": "message", + "role": "developer", + "content": [ + {"type": "input_text", "text": "Be concise."}, + {"type": "input_text", "text": "Be helpful.", "prompt_cache_breakpoint": self.EXPLICIT}, + ], + }, + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hello"}]}, + ] + + def test_system_without_breakpoint_still_becomes_instructions(self): + request = _make_request(system=[{"type": "text", "text": "Be concise."}, {"type": "text", "text": "Be helpful."}]) + kwargs = _ADAPTER.translate_request(request) + assert kwargs["instructions"] == "Be concise.\nBe helpful." + assert kwargs["input"] == [ + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hello"}]} + ] + + def test_system_string_still_becomes_instructions(self): + kwargs = _ADAPTER.translate_request(_make_request(system="Be concise.")) + assert kwargs["instructions"] == "Be concise." + assert kwargs["input"][0]["role"] == "user" + + def test_system_with_breakpoint_skips_non_text_blocks(self): + request = _make_request( + system=[ + {"type": "image", "source": {"type": "url", "url": "https://example.com/a.png"}}, + {"type": "text", "text": "only", "prompt_cache_breakpoint": self.EXPLICIT}, + ] + ) + kwargs = _ADAPTER.translate_request(request) + assert kwargs["input"][0] == { + "type": "message", + "role": "developer", + "content": [{"type": "input_text", "text": "only", "prompt_cache_breakpoint": self.EXPLICIT}], + } + + def test_user_text_and_image_blocks_carry_breakpoint(self): + items = _ADAPTER.translate_messages_to_responses_input( + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "look", "prompt_cache_breakpoint": self.EXPLICIT}, + { + "type": "image", + "source": {"type": "url", "url": "https://example.com/a.png"}, + "prompt_cache_breakpoint": self.EXPLICIT, + }, + ], + } + ] + ) + assert items == [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "look", "prompt_cache_breakpoint": self.EXPLICIT}, + { + "type": "input_image", + "image_url": "https://example.com/a.png", + "prompt_cache_breakpoint": self.EXPLICIT, + }, + ], + } + ] + + def test_user_blocks_without_breakpoint_are_unchanged(self): + items = _ADAPTER.translate_messages_to_responses_input( + [{"role": "user", "content": [{"type": "text", "text": "look"}]}] + ) + assert items == [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "look"}]}] + + def test_midturn_system_block_carries_breakpoint(self): + items = _ADAPTER.translate_messages_to_responses_input( + [{"role": "system", "content": [{"type": "text", "text": "fix", "prompt_cache_breakpoint": self.EXPLICIT}]}] + ) + assert items == [ + { + "type": "message", + "role": "system", + "content": [{"type": "input_text", "text": "fix", "prompt_cache_breakpoint": self.EXPLICIT}], + } + ] + + def test_assistant_and_tool_result_blocks_drop_breakpoint(self): + items = _ADAPTER.translate_messages_to_responses_input( + [ + {"role": "user", "content": [{"type": "text", "text": "q"}]}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "a", "prompt_cache_breakpoint": self.EXPLICIT}, + {"type": "tool_use", "id": "toolu_01", "name": "t", "input": {}}, + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": "r", + "prompt_cache_breakpoint": self.EXPLICIT, + } + ], + }, + ] + ) + assert len(items) == 4 + assert not _contains_key(items, "prompt_cache_breakpoint") + + def test_prompt_cache_options_forwarded_to_responses_kwargs(self): + from litellm.llms.anthropic.experimental_pass_through.responses_adapters.handler import ( + _build_responses_kwargs, + ) + + kwargs = _build_responses_kwargs( + max_tokens=16, + messages=[{"role": "user", "content": "hi"}], + model="openai/gpt-5.6", + extra_kwargs={"prompt_cache_options": {"mode": "explicit"}}, + ) + assert kwargs["prompt_cache_options"] == {"mode": "explicit"} diff --git a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py index 41c2e215c60..45f1bdbfa85 100644 --- a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py @@ -871,3 +871,48 @@ class TestToolMessageImageHoisting: result = request["messages"] assert [m.get("role") for m in result] == ["user", "assistant", "tool", "user"] assert result[3]["content"] == self.HOISTED_USER_CONTENT + + +class TestOpenAIPromptCacheBreakpointChatPath: + """Chat-path shape for OpenAI explicit prompt caching (#37509).""" + + EXPLICIT = {"mode": "explicit"} + + def test_prompt_cache_options_travels_in_extra_body(self): + optional_params = litellm.get_optional_params( + model="gpt-5.6", custom_llm_provider="openai", prompt_cache_options=self.EXPLICIT + ) + assert optional_params["extra_body"]["prompt_cache_options"] == self.EXPLICIT + assert "prompt_cache_options" not in optional_params + + def test_prompt_cache_options_is_not_a_supported_chat_param(self): + assert "prompt_cache_options" not in OpenAIGPT5Config().get_supported_openai_params("gpt-5.6") + assert "prompt_cache_options" not in OpenAIGPTConfig().get_supported_openai_params("gpt-4.1") + + def test_block_breakpoint_survives_transform_request(self): + request = OpenAIGPT5Config().transform_request( + model="gpt-5.6", + messages=[ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "sys", + "prompt_cache_breakpoint": self.EXPLICIT, + "cache_control": {"type": "ephemeral"}, + } + ], + }, + {"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]}, + ], + optional_params={"extra_body": {"prompt_cache_options": self.EXPLICIT}}, + litellm_params={}, + headers={}, + ) + assert request["messages"][0]["content"] == [ + {"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT} + ] + assert request["messages"][1]["content"] == [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}] + assert request["extra_body"] == {"prompt_cache_options": self.EXPLICIT} + assert "prompt_cache_options" not in request diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py index 151c51f1ca0..13b96dc9943 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -1540,3 +1540,61 @@ class TestPhaseParameter: assert validated[0]["phase"] == "commentary" assert validated[1]["phase"] == "final_answer" assert "phase" not in validated[2] + + +class TestPromptCacheOptionsOnResponsesPath: + """`prompt_cache_options` and block-level `prompt_cache_breakpoint` survive the Responses transformation (#37509).""" + + OPTIONS = {"mode": "explicit", "ttl": "30m"} + + def test_prompt_cache_options_survives_optional_param_filter(self): + from litellm.responses.utils import ResponsesAPIRequestUtils + + result = ResponsesAPIRequestUtils.get_requested_response_api_optional_param( + {"prompt_cache_options": dict(self.OPTIONS), "temperature": 0.2, "not_a_responses_param": 1} + ) + assert result["prompt_cache_options"] == self.OPTIONS + assert result["temperature"] == 0.2 + assert "not_a_responses_param" not in result + + def test_prompt_cache_options_reaches_transformed_request(self): + config = OpenAIResponsesAPIConfig() + mapped = config.map_openai_params( + response_api_optional_params={"prompt_cache_options": dict(self.OPTIONS)}, + model="gpt-5.6", + drop_params=False, + ) + result = config.transform_responses_api_request( + model="gpt-5.6", + input="hi", + response_api_optional_request_params=mapped, + litellm_params={}, + headers={}, + ) + assert result["prompt_cache_options"] == self.OPTIONS + + def test_prompt_cache_breakpoint_survives_cache_control_strip(self): + result = OpenAIResponsesAPIConfig().transform_responses_api_request( + model="gpt-5.6", + input=[ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hi", + "prompt_cache_breakpoint": {"mode": "explicit"}, + "cache_control": {"type": "ephemeral"}, + } + ], + } + ], + response_api_optional_request_params={}, + litellm_params={}, + headers={}, + ) + assert result["input"][0]["content"][0] == { + "type": "input_text", + "text": "hi", + "prompt_cache_breakpoint": {"mode": "explicit"}, + } From d9aaa959788efce33ad22c6887710028f0e2b27d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 05:17:13 -0700 Subject: [PATCH 2/3] Gate OpenAI prompt cache breakpoints on the real target and carry them through /v1/responses The cache control hook also runs on litellm.responses() input. On a GPT-5.6 deployment it wrapped a string-content item into a chat-shaped {"type": "text"} part, which the Responses API rejects, and it never marked input_text, input_image or input_file parts, so no breakpoint and no prompt_cache_options reached the provider. Add the Responses part types to the eligible block set and translate chat-shaped text parts on non-assistant items to input_text in ResponsesAPIRequestUtils.merge_prompt_management_input, which both the async and the sync prompt management sites go through. The dialect also fired for any GPT-5.6 name that resolved to provider openai, including deployments pointed at a custom api_base that does not understand prompt_cache_breakpoint. Decide it once per request from the provider, the model map and the resolved api_base (request, then litellm.api_base, then OPENAI_BASE_URL / OPENAI_API_BASE): only api.openai.com and *.api.openai.com hosts speak the dialect, a top-level prompt_cache_options opts a custom target in, and litellm_proxy/ targets never get it. maybe_seed_default_injection_points takes api_base and stamps the finished decision on the points as _litellm_openai_dialect so the sync completion() path, whose hook params do not carry api_base, honors it; maybe_inject_cache_control takes api_base from the /v1/messages handler. Eligibility now comes from a supports_prompt_cache_breakpoint model map flag on the OpenAI gpt-5.6 entries, exposed through litellm.utils.supports_prompt_cache_breakpoint, with the GPT version rule kept only for models the map does not know. The OpenAI dialect no longer reserves a slot for tool_config points, which OpenAI has no cache block for, and with_prompt_cache_breakpoint plus the chat bridge helper return a new block instead of mutating their input. --- .../anthropic_cache_control_hook.py | 113 ++++++-- .../prompt_templates/common_utils.py | 13 +- .../adapters/transformation.py | 26 +- .../messages/handler.py | 4 +- litellm/main.py | 2 + ...odel_prices_and_context_window_backup.json | 4 + litellm/responses/utils.py | 13 + .../anthropic_cache_control_hook.py | 4 +- litellm/types/utils.py | 1 + litellm/utils.py | 11 + model_prices_and_context_window.json | 4 + model_prices_and_context_window.schema.json | 3 + .../test_anthropic_cache_control_hook.py | 270 +++++++++++++++++- .../test_responses_api_request_body.py | 158 ++++++++++ .../responses/test_responses_utils.py | 62 ++++ tests/test_litellm/test_main.py | 46 +++ tests/test_litellm/test_utils.py | 1 + 17 files changed, 692 insertions(+), 43 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 05e2a37a230..f65decc5463 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -10,9 +10,11 @@ Supported for both `v1/chat/completions` (via the prompt-management hook) and """ import copy +import os import re from collections.abc import Iterable, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, cast +from urllib.parse import urlparse from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger @@ -49,10 +51,18 @@ MAX_CACHE_CONTROL_BLOCKS: Final = 4 CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint") OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6) _GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?") -OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset({"text", "image", "image_url", "file", "input_audio"}) +OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset( + {"text", "image", "image_url", "file", "input_audio", "input_text", "input_image", "input_file"} +) +OPENAI_API_HOST: Final = "api.openai.com" +OPENAI_API_BASE_ENV_VARS: Final = ("OPENAI_BASE_URL", "OPENAI_API_BASE") def supports_openai_prompt_cache_breakpoint(model: str) -> bool: + if _has_model_map_entry(model): + from litellm.utils import supports_prompt_cache_breakpoint + + return supports_prompt_cache_breakpoint(model) version_match: Final = _GPT_VERSION_PATTERN.match(model.rsplit("/", 1)[-1].lower()) if version_match is None: return False @@ -60,6 +70,25 @@ def supports_openai_prompt_cache_breakpoint(model: str) -> bool: return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION +def _has_model_map_entry(model: str) -> bool: + import litellm + + return model in litellm.model_cost or model.rsplit("/", 1)[-1] in litellm.model_cost + + +def targets_openai_api(api_base: object) -> bool: + import litellm + + resolved: Final = next( + (value for value in (api_base, litellm.api_base, *map(os.getenv, OPENAI_API_BASE_ENV_VARS)) if value), + None, + ) + if not isinstance(resolved, str): + return True + host: Final = urlparse(resolved).hostname + return host is not None and (host == OPENAI_API_HOST or host.endswith(f".{OPENAI_API_HOST}")) + + def _carries_cache_breakpoint(block: object) -> bool: return isinstance(block, dict) and any(block.get(key) is not None for key in CACHE_BREAKPOINT_KEYS) @@ -114,10 +143,19 @@ class AnthropicCacheControlHook(CustomPromptManagement): # provider transform, where each tool_config point appends at most one # cachePoint to the tools. That block also counts toward Anthropic's # limit, so reserve a slot for it here to leave room. - reserved_blocks: Final = 1 if any(p.get("location") == "tool_config" for p in remaining_points) else 0 - - openai_dialect: Final = AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( - model, injection_points[0].get("_litellm_provider") + stamped_dialect: Final = injection_points[0].get("_litellm_openai_dialect") + openai_dialect: Final = ( + stamped_dialect + if isinstance(stamped_dialect, bool) + else AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( + model, + non_default_params.get("custom_llm_provider"), + non_default_params.get("api_base"), + non_default_params.get("prompt_cache_options"), + ) + ) + reserved_blocks: Final = ( + 1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0 ) breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages) processed_messages = self._apply_message_injections( @@ -141,10 +179,17 @@ class AnthropicCacheControlHook(CustomPromptManagement): return model, processed_messages, non_default_params @staticmethod - def _targets_openai_prompt_cache_breakpoint(model: str | None, custom_llm_provider: str | None) -> bool: + def _targets_openai_prompt_cache_breakpoint( + model: str | None, + custom_llm_provider: str | None, + api_base: object = None, + prompt_cache_options: object = None, + ) -> bool: if model is None or not supports_openai_prompt_cache_breakpoint(model): return False - return (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) == "openai" + if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai": + return False + return prompt_cache_options is not None or targets_openai_api(api_base) @staticmethod def _resolve_provider(model: str) -> str | None: @@ -315,10 +360,18 @@ class AnthropicCacheControlHook(CustomPromptManagement): ] return marked if isinstance(message_content, list): - with_prompt_cache_breakpoint( - next((block for block in reversed(message_content) if _accepts_prompt_cache_breakpoint(block)), None), - PromptCacheBreakpoint(mode="explicit"), + target_index: Final = next( + ( + index + for index in range(len(message_content) - 1, -1, -1) + if _accepts_prompt_cache_breakpoint(message_content[index]) + ), + None, ) + if target_index is not None: + message_content[target_index] = with_prompt_cache_breakpoint( + message_content[target_index], PromptCacheBreakpoint(mode="explicit") + ) return message @staticmethod @@ -363,7 +416,9 @@ class AnthropicCacheControlHook(CustomPromptManagement): else: remaining_points.append(point) - reserved_blocks: Final = 1 if any(p.get("location") == "tool_config" for p in remaining_points) else 0 + reserved_blocks: Final = ( + 1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0 + ) max_blocks: Final = MAX_CACHE_CONTROL_BLOCKS - reserved_blocks message_blocks: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages) @@ -432,18 +487,32 @@ class AnthropicCacheControlHook(CustomPromptManagement): tools: list[object] | None, model: str, custom_llm_provider: str | None, + api_base: object, + prompt_cache_options: object, ) -> Sequence[Mapping[str, object]] | None: if AnthropicCacheControlHook._should_stand_down(points, messages, None, tools): return None - return AnthropicCacheControlHook._stamped_with_provider(points, model, custom_llm_provider) + return AnthropicCacheControlHook._stamped_with_dialect( + points, model, custom_llm_provider, api_base, prompt_cache_options + ) @staticmethod - def _stamped_with_provider( - points: Sequence[CacheControlInjectionPoint], model: str, custom_llm_provider: str | None + def _stamped_with_dialect( + points: Sequence[CacheControlInjectionPoint], + model: str, + custom_llm_provider: str | None, + api_base: object, + prompt_cache_options: object, ) -> Sequence[Mapping[str, object]]: - if custom_llm_provider is None or not supports_openai_prompt_cache_breakpoint(model): + if not supports_openai_prompt_cache_breakpoint(model): return points - return AnthropicCacheControlHook._stamped(points, "_litellm_provider", custom_llm_provider) + return AnthropicCacheControlHook._stamped( + points, + "_litellm_openai_dialect", + AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( + model, custom_llm_provider, api_base, prompt_cache_options + ), + ) @staticmethod def _stamped( @@ -563,6 +632,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): custom_llm_provider: str | None, tools: list | None = None, enable_prompt_caching: bool | None = None, + api_base: object = None, ) -> None: """For /chat/completions: resolve the injection points the request should carry. @@ -578,7 +648,13 @@ class AnthropicCacheControlHook(CustomPromptManagement): """ if non_default_params.get("cache_control_injection_points"): judged: Final = AnthropicCacheControlHook._judged_configured_points( - non_default_params["cache_control_injection_points"], messages, tools, model, custom_llm_provider + non_default_params["cache_control_injection_points"], + messages, + tools, + model, + custom_llm_provider, + api_base, + non_default_params.get("prompt_cache_options"), ) if judged is None: non_default_params.pop("cache_control_injection_points") @@ -604,6 +680,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): model: str | None = None, custom_llm_provider: str | None = None, tools: list[dict] | None = None, + api_base: str | None = None, ) -> tuple[list[dict], str | list | None]: """Extract cache_control_injection_points from kwargs and apply if present. @@ -642,7 +719,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): return messages, system openai_dialect: Final = AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( - model, custom_llm_provider + model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options") ) breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request( diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 5904a14e79f..2db5776047b 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1325,17 +1325,14 @@ def check_is_function_call(logging_obj: "LoggingClass") -> bool: return False -_MarkedT = TypeVar("_MarkedT") - - -def _set_prompt_cache_breakpoint(target: object, marker: object) -> None: - if marker is not None and isinstance(target, dict): - target["prompt_cache_breakpoint"] = marker +_MarkedT: Final = TypeVar("_MarkedT", bound=Mapping[str, object]) def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT: - _set_prompt_cache_breakpoint(target, marker) - return target + if marker is None: + return target + marked: Final = {**target, "prompt_cache_breakpoint": marker} # mutable-ok: API message payload + return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Any: diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 2ee50e2a55b..c6bfdb79002 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -2,7 +2,7 @@ import copy import hashlib import json from collections.abc import AsyncIterator, Iterator, Mapping -from typing import TYPE_CHECKING, Any, Final, Literal, cast +from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast from litellm.llms.anthropic.experimental_pass_through.utils import ( is_reasoning_auto_summary_enabled, @@ -246,6 +246,9 @@ class AnthropicAdapter: return anthropic_wrapper.anthropic_sse_wrapper() +_BlockT: Final = TypeVar("_BlockT", bound=Mapping[str, object]) + + class LiteLLMAnthropicMessagesAdapter: def __init__(self): pass @@ -310,9 +313,10 @@ class LiteLLMAnthropicMessagesAdapter: cast(dict[str, object], target)["cache_control"] = cache_control @staticmethod - def _add_prompt_cache_breakpoint_if_present(source: object, target: object) -> None: + def _add_prompt_cache_breakpoint_if_present(source: object, target: _BlockT) -> _BlockT: if isinstance(source, dict) and "prompt_cache_breakpoint" in source: - with_prompt_cache_breakpoint(target, source["prompt_cache_breakpoint"]) + return with_prompt_cache_breakpoint(target, source["prompt_cache_breakpoint"]) + return target def translatable_anthropic_params(self) -> list[str]: """ @@ -374,8 +378,9 @@ class LiteLLMAnthropicMessagesAdapter: if content.get("type") == "text": text_obj = ChatCompletionTextObject(type="text", text=content.get("text", "")) self._add_cache_control_if_applicable(content, text_obj, model) - self._add_prompt_cache_breakpoint_if_present(content, text_obj) - new_user_content_list.append(text_obj) + new_user_content_list.append( + self._add_prompt_cache_breakpoint_if_present(content, text_obj) + ) elif content.get("type") == "image": # Convert Anthropic image format to OpenAI format source = content.get("source", {}) @@ -385,8 +390,9 @@ class LiteLLMAnthropicMessagesAdapter: image_url_obj = ChatCompletionImageUrlObject(url=openai_image_url) image_obj = ChatCompletionImageObject(type="image_url", image_url=image_url_obj) self._add_cache_control_if_applicable(content, image_obj, model) - self._add_prompt_cache_breakpoint_if_present(content, image_obj) - new_user_content_list.append(image_obj) + new_user_content_list.append( + self._add_prompt_cache_breakpoint_if_present(content, image_obj) + ) elif content.get("type") == "document": # Convert Anthropic document format (PDF, etc.) to OpenAI format source = content.get("source", {}) @@ -877,8 +883,7 @@ class LiteLLMAnthropicMessagesAdapter: continue text_obj = ChatCompletionTextObject(type="text", text=text) self._add_cache_control_if_applicable(block, text_obj, model) - self._add_prompt_cache_breakpoint_if_present(block, text_obj) - text_parts.append(text_obj) + text_parts.append(self._add_prompt_cache_breakpoint_if_present(block, text_obj)) return ChatCompletionSystemMessage(role="system", content=text_parts) if text_parts else None def _add_system_message_to_messages( @@ -909,8 +914,7 @@ class LiteLLMAnthropicMessagesAdapter: "text": block.get("text", ""), } self._add_cache_control_if_applicable(block, text_block, model_name) - self._add_prompt_cache_breakpoint_if_present(block, text_block) - openai_system_content.append(text_block) + openai_system_content.append(self._add_prompt_cache_breakpoint_if_present(block, text_block)) if openai_system_content: new_messages.insert( 0, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index c4b5cc628e2..26aef666172 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -230,7 +230,7 @@ async def anthropic_messages( ) messages, system = AnthropicCacheControlHook.maybe_inject_cache_control( - messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools + messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools, api_base=api_base ) original_stream: Final = stream or kwargs.get("_websearch_interception_converted_stream", False) @@ -422,7 +422,7 @@ def anthropic_messages_handler( ) messages, system = AnthropicCacheControlHook.maybe_inject_cache_control( - messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools + messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools, api_base=api_base ) metadata = validate_anthropic_api_metadata(metadata) diff --git a/litellm/main.py b/litellm/main.py index f0b20eba9b6..87b32ab140c 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -507,6 +507,7 @@ async def acompletion( custom_llm_provider=cast(str | None, custom_llm_provider), # cast-ok: read from untyped kwargs tools=tools, enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs + api_base=kwargs.get("api_base"), ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( @@ -5171,6 +5172,7 @@ def completion( custom_llm_provider=cast(str | None, kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs tools=tools, enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs + api_base=kwargs.get("api_base"), ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3beb3ae4370..c92b4ccd18a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -25368,6 +25368,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -25430,6 +25431,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -25492,6 +25494,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -25554,6 +25557,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 4b5def790ed..504009802f2 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -40,12 +40,25 @@ def normalize_responses_api_stream_options( class ResponsesAPIRequestUtils: """Helper utils for constructing ResponseAPI requests""" + @staticmethod + def shape_prompt_managed_messages_for_responses(messages: Iterable[object]) -> None: + for message in messages: + if not isinstance(message, dict) or message.get("role") == "assistant": + continue + content: object = message.get("content") + if not isinstance(content, list): + continue + for part in content: + if isinstance(part, dict) and part.get("type") == "text": + part["type"] = "input_text" + @staticmethod def merge_prompt_management_input( original_input: str | ResponseInputParam, client_input: list[AllMessageValues], merged_input: list[AllMessageValues], ) -> list[object]: + ResponsesAPIRequestUtils.shape_prompt_managed_messages_for_responses(merged_input) if isinstance(original_input, str): return [*merged_input] diff --git a/litellm/types/integrations/anthropic_cache_control_hook.py b/litellm/types/integrations/anthropic_cache_control_hook.py index 3b3dcf5c86d..3ab0c02f28d 100644 --- a/litellm/types/integrations/anthropic_cache_control_hook.py +++ b/litellm/types/integrations/anthropic_cache_control_hook.py @@ -13,7 +13,7 @@ class CacheControlMessageInjectionPoint(TypedDict): index: int | str | None # Optional: target by specific index control: ChatCompletionCachedContent | None _litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran - _litellm_provider: NotRequired[ReadOnly[str]] + _litellm_openai_dialect: NotRequired[ReadOnly[bool]] class CacheControlToolConfigInjectionPoint(TypedDict): @@ -22,7 +22,7 @@ class CacheControlToolConfigInjectionPoint(TypedDict): location: Literal["tool_config"] control: ChatCompletionCachedContent | None _litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran - _litellm_provider: NotRequired[ReadOnly[str]] + _litellm_openai_dialect: NotRequired[ReadOnly[bool]] CacheControlInjectionPoint = CacheControlMessageInjectionPoint | CacheControlToolConfigInjectionPoint diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 96b9343353d..41210d18495 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -141,6 +141,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_tool_choice: bool | None supports_assistant_prefill: bool | None supports_prompt_caching: bool | None + supports_prompt_cache_breakpoint: ReadOnly[bool | None] supports_computer_use: bool | None supports_audio_input: bool | None supports_embedding_image_input: bool | None diff --git a/litellm/utils.py b/litellm/utils.py index d1b0cb882ac..867f7a93452 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2560,6 +2560,14 @@ def supports_prompt_caching(model: str, custom_llm_provider: str | None = None) ) +def supports_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool: + return _supports_factory( + model=model, + custom_llm_provider=custom_llm_provider, + key="supports_prompt_cache_breakpoint", + ) + + def supports_computer_use(model: str, custom_llm_provider: str | None = None) -> bool: """ Check if the given model supports computer use and return a boolean value. @@ -5473,6 +5481,7 @@ def _get_model_info_helper( supports_tool_choice=None, supports_assistant_prefill=None, supports_prompt_caching=None, + supports_prompt_cache_breakpoint=None, supports_computer_use=None, supports_pdf_input=None, ) @@ -5712,6 +5721,7 @@ def _get_model_info_helper( supports_tool_choice=_model_info.get("supports_tool_choice", None), supports_assistant_prefill=_model_info.get("supports_assistant_prefill", None), supports_prompt_caching=_model_info.get("supports_prompt_caching", None), + supports_prompt_cache_breakpoint=_model_info.get("supports_prompt_cache_breakpoint", None), supports_audio_input=_model_info.get("supports_audio_input", None), supports_audio_output=_model_info.get("supports_audio_output", None), supports_pdf_input=_model_info.get("supports_pdf_input", None), @@ -5846,6 +5856,7 @@ def get_model_info( supports_function_calling: Optional[bool] supports_tool_choice: Optional[bool] supports_prompt_caching: Optional[bool] + supports_prompt_cache_breakpoint: Optional[bool] supports_audio_input: Optional[bool] supports_audio_output: Optional[bool] supports_pdf_input: Optional[bool] diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3beb3ae4370..c92b4ccd18a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -25368,6 +25368,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -25430,6 +25431,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -25492,6 +25494,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -25554,6 +25557,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 82854a3b717..0991650d307 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -664,6 +664,9 @@ "supports_pdf_input": { "type": "boolean" }, + "supports_prompt_cache_breakpoint": { + "type": "boolean" + }, "supports_prompt_caching": { "type": "boolean" }, diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 04876b9bf12..4eb774af5bc 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -23,6 +23,13 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import StandardCallbackDynamicParams +@pytest.fixture(autouse=True) +def _no_openai_api_base_override(monkeypatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None) + + def _rendered_log_message(call): message = str(call.args[0]) values = call.args[1:] @@ -2395,19 +2402,28 @@ class TestOpenAIPromptCacheBreakpointPlacementRules: class TestChatPathProviderStamp: - """The chat path learns the caller's custom_llm_provider through the seeded points (#37509).""" + """The chat path learns the dialect decision (provider, api_base, opt-in) through the seeded points (#37509).""" POINTS = [{"location": "message", "role": "system"}] MESSAGES = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}] + ANTHROPIC_STYLE = {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}} + OPENAI_STYLE = [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}] + CUSTOM_API_BASE = "http://127.0.0.1:9/v1" - def _seed_and_run(self, model, custom_llm_provider): + def _seed_and_run(self, model, custom_llm_provider, api_base=None, prompt_cache_options=None): params = {"cache_control_injection_points": copy.deepcopy(self.POINTS)} + if prompt_cache_options is not None: + params["prompt_cache_options"] = prompt_cache_options AnthropicCacheControlHook.maybe_seed_default_injection_points( non_default_params=params, messages=copy.deepcopy(self.MESSAGES), model=model, custom_llm_provider=custom_llm_provider, + api_base=api_base, ) + return self._run(params, model) + + def _run(self, params, model): _, out, params = AnthropicCacheControlHook().get_chat_completion_prompt( model=model, messages=copy.deepcopy(self.MESSAGES), @@ -2452,6 +2468,84 @@ class TestChatPathProviderStamp: assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("my-custom-model", None) is False resolve.assert_not_called() + def test_litellm_proxy_target_keeps_anthropic_style_markers(self): + out, params = self._seed_and_run("litellm_proxy/gpt-5.6", None) + assert out[0] == self.ANTHROPIC_STYLE + assert "prompt_cache_options" not in params + + def test_custom_api_base_keeps_anthropic_style_markers(self): + out, params = self._seed_and_run("gpt-5.6", None, api_base=self.CUSTOM_API_BASE) + assert out[0] == self.ANTHROPIC_STYLE + assert "prompt_cache_options" not in params + + def test_custom_api_base_opts_in_through_prompt_cache_options(self): + out, params = self._seed_and_run( + "gpt-5.6", None, api_base=self.CUSTOM_API_BASE, prompt_cache_options={"mode": "explicit"} + ) + assert out[0]["content"] == self.OPENAI_STYLE + assert params["prompt_cache_options"] == {"mode": "explicit"} + + def test_regional_openai_api_base_uses_openai_dialect(self): + out, params = self._seed_and_run("gpt-5.6", None, api_base="https://eu.api.openai.com/v1") + assert out[0]["content"] == self.OPENAI_STYLE + assert params["prompt_cache_options"] == {"mode": "explicit"} + + @pytest.mark.parametrize("env_var", ["OPENAI_BASE_URL", "OPENAI_API_BASE"]) + def test_env_api_base_override_keeps_anthropic_style_markers(self, monkeypatch, env_var): + monkeypatch.setenv(env_var, self.CUSTOM_API_BASE) + out, params = self._seed_and_run("gpt-5.6", None) + assert out[0] == self.ANTHROPIC_STYLE + assert "prompt_cache_options" not in params + + def test_global_litellm_api_base_keeps_anthropic_style_markers(self, monkeypatch): + monkeypatch.setattr(litellm, "api_base", self.CUSTOM_API_BASE) + out, params = self._seed_and_run("gpt-5.6", None) + assert out[0] == self.ANTHROPIC_STYLE + assert "prompt_cache_options" not in params + + def test_request_api_base_wins_over_env_override(self, monkeypatch): + monkeypatch.setenv("OPENAI_BASE_URL", self.CUSTOM_API_BASE) + out, params = self._seed_and_run("gpt-5.6", None, api_base="https://api.openai.com/v1") + assert out[0]["content"] == self.OPENAI_STYLE + assert params["prompt_cache_options"] == {"mode": "explicit"} + + @pytest.mark.parametrize( + "api_base,expected", + [(None, True), ("http://127.0.0.1:9/v1", False), ("https://eu.api.openai.com/v1", True)], + ) + def test_seed_stamps_the_dialect_decision(self, api_base, expected): + params = {"cache_control_injection_points": copy.deepcopy(self.POINTS)} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=copy.deepcopy(self.MESSAGES), + model="gpt-5.6", + custom_llm_provider=None, + api_base=api_base, + ) + assert params["cache_control_injection_points"][0]["_litellm_openai_dialect"] is expected + + def test_stamp_is_authoritative_over_request_params(self): + points = [{**self.POINTS[0], "_litellm_openai_dialect": False}] + out, params = self._run({"cache_control_injection_points": points, "custom_llm_provider": "openai"}, "gpt-5.6") + assert out[0] == self.ANTHROPIC_STYLE + assert "prompt_cache_options" not in params + + def test_unstamped_points_read_api_base_from_request_params(self): + params = {"cache_control_injection_points": copy.deepcopy(self.POINTS), "api_base": self.CUSTOM_API_BASE} + out, params = self._run(params, "gpt-5.6") + assert out[0] == self.ANTHROPIC_STYLE + assert "prompt_cache_options" not in params + + def test_unstamped_points_read_prompt_cache_options_from_request_params(self): + params = { + "cache_control_injection_points": copy.deepcopy(self.POINTS), + "api_base": self.CUSTOM_API_BASE, + "prompt_cache_options": {"mode": "explicit"}, + } + out, params = self._run(params, "gpt-5.6") + assert out[0]["content"] == self.OPENAI_STYLE + assert params["prompt_cache_options"] == {"mode": "explicit"} + class TestClientBreakpointsCountedOnce: def test_client_message_breakpoints_are_not_double_counted(self): @@ -2471,3 +2565,175 @@ class TestClientBreakpointsCountedOnce: marked = [msg["content"][0].get("cache_control") is not None for msg in out] assert marked == [True, False, True, True] assert system[0]["cache_control"] == {"type": "ephemeral"} + + +class TestResponsesInputPartsEligible: + """Responses API input parts can carry prompt_cache_breakpoint on GPT-5.6+ (#37509).""" + + EXPLICIT = {"mode": "explicit"} + + def _chat(self, messages, points, model="openai/gpt-5.6"): + params = {"cache_control_injection_points": copy.deepcopy(points)} + _, out, params = AnthropicCacheControlHook().get_chat_completion_prompt( + model=model, + messages=copy.deepcopy(messages), + non_default_params=params, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + return out, params + + def test_marker_lands_on_last_input_text_part(self): + messages = [ + { + "role": "user", + "content": [{"type": "input_text", "text": "first"}, {"type": "input_text", "text": "second"}], + } + ] + out, params = self._chat(messages, [{"location": "message", "index": -1}]) + assert out[0]["content"][0] == {"type": "input_text", "text": "first"} + assert out[0]["content"][1] == { + "type": "input_text", + "text": "second", + "prompt_cache_breakpoint": self.EXPLICIT, + } + assert params["prompt_cache_options"] == self.EXPLICIT + + @pytest.mark.parametrize( + "part", + [ + {"type": "input_image", "image_url": "https://example.com/a.png"}, + {"type": "input_file", "file_id": "file_1"}, + ], + ) + def test_input_image_and_input_file_parts_are_eligible(self, part): + out, params = self._chat([{"role": "user", "content": [part]}], [{"location": "message", "index": -1}]) + assert out[0]["content"][0] == {**part, "prompt_cache_breakpoint": self.EXPLICIT} + assert params["prompt_cache_options"] == self.EXPLICIT + + +class TestMessagesPathApiBaseGate: + """/v1/messages only speaks the OpenAI dialect when the request really targets api.openai.com (#37509).""" + + EXPLICIT = {"mode": "explicit"} + USER_POINT = [{"location": "message", "role": "user"}] + MESSAGES = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + CUSTOM_API_BASE = "http://127.0.0.1:9/v1" + CACHE_CONTROL_BLOCK = {"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}} + BREAKPOINT_BLOCK = {"type": "text", "text": "hi", "prompt_cache_breakpoint": {"mode": "explicit"}} + + def _inject(self, model, api_base=None, prompt_cache_options=None, custom_llm_provider=None): + kwargs = {"cache_control_injection_points": copy.deepcopy(self.USER_POINT)} + if prompt_cache_options is not None: + kwargs["prompt_cache_options"] = prompt_cache_options + out, _ = AnthropicCacheControlHook.maybe_inject_cache_control( + copy.deepcopy(self.MESSAGES), + None, + kwargs, + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + ) + return out[0]["content"][0], kwargs + + def test_litellm_proxy_target_keeps_cache_control(self): + block, kwargs = self._inject("gpt-5.6", api_base=self.CUSTOM_API_BASE, custom_llm_provider="litellm_proxy") + assert block == self.CACHE_CONTROL_BLOCK + assert "prompt_cache_options" not in kwargs + + def test_custom_api_base_keeps_cache_control(self): + block, kwargs = self._inject("gpt-5.6", api_base=self.CUSTOM_API_BASE) + assert block == self.CACHE_CONTROL_BLOCK + assert "prompt_cache_options" not in kwargs + + def test_custom_api_base_opts_in_through_prompt_cache_options(self): + block, kwargs = self._inject("gpt-5.6", api_base=self.CUSTOM_API_BASE, prompt_cache_options=self.EXPLICIT) + assert block == self.BREAKPOINT_BLOCK + assert kwargs["prompt_cache_options"] == self.EXPLICIT + + def test_regional_openai_api_base_uses_openai_dialect(self): + block, kwargs = self._inject("gpt-5.6", api_base="https://eu.api.openai.com/v1") + assert block == self.BREAKPOINT_BLOCK + assert kwargs["prompt_cache_options"] == self.EXPLICIT + + def test_default_api_base_uses_openai_dialect(self): + block, kwargs = self._inject("openai/gpt-5.6") + assert block == self.BREAKPOINT_BLOCK + assert kwargs["prompt_cache_options"] == self.EXPLICIT + + +class TestToolConfigSlotInOpenAIDialect: + """OpenAI has no tool_config cache block, so the dialect does not hold a slot for one (#37509).""" + + EXPLICIT = {"mode": "explicit"} + MESSAGES = [{"role": "user", "content": [{"type": "text", "text": f"m{i}"}]} for i in range(4)] + POINTS = [{"location": "message", "index": i} for i in range(4)] + [{"location": "tool_config"}] + + def test_chat_path_marks_all_four_messages(self): + params = {"cache_control_injection_points": copy.deepcopy(self.POINTS)} + _, out, params = AnthropicCacheControlHook().get_chat_completion_prompt( + model="openai/gpt-5.6", + messages=copy.deepcopy(self.MESSAGES), + non_default_params=params, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + assert [msg["content"][0].get("prompt_cache_breakpoint") for msg in out] == [self.EXPLICIT] * 4 + assert params["prompt_cache_options"] == self.EXPLICIT + + def test_messages_path_marks_all_four_messages(self): + out, _, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request( + copy.deepcopy(self.MESSAGES), None, copy.deepcopy(self.POINTS), openai_dialect=True + ) + assert [msg["content"][0].get("prompt_cache_breakpoint") for msg in out] == [self.EXPLICIT] * 4 + + def test_anthropic_dialect_still_reserves_the_tool_config_slot(self): + out, _, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request( + copy.deepcopy(self.MESSAGES), None, copy.deepcopy(self.POINTS) + ) + assert sum(msg["content"][0].get("cache_control") is not None for msg in out) == 3 + + +class TestPromptCacheBreakpointCapability: + """Eligibility comes from the model map's supports_prompt_cache_breakpoint flag, with the GPT version + rule only for models the map does not know (#37509).""" + + @pytest.fixture(autouse=True) + def _fresh_model_info_cache(self): + litellm.utils._cached_get_model_info_helper.cache_clear() + yield + litellm.utils._cached_get_model_info_helper.cache_clear() + + def test_public_helper_reads_the_model_map(self): + from litellm.utils import supports_prompt_cache_breakpoint + + assert supports_prompt_cache_breakpoint("gpt-5.6") is True + assert supports_prompt_cache_breakpoint("openai/gpt-5.6-sol") is True + assert supports_prompt_cache_breakpoint("gpt-5.6", custom_llm_provider="openai") is True + assert supports_prompt_cache_breakpoint("gpt-4.1") is False + + @pytest.mark.parametrize("model", ["gpt-5.6", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"]) + def test_model_map_flags_every_openai_gpt_5_6_entry(self, model): + assert litellm.model_cost[model]["litellm_provider"] == "openai" + assert litellm.model_cost[model]["supports_prompt_cache_breakpoint"] is True + + def test_listed_model_uses_the_model_map_flag(self, monkeypatch): + flagged = {**litellm.model_cost["gpt-4.1"], "supports_prompt_cache_breakpoint": True} + monkeypatch.setitem(litellm.model_cost, "gpt-4.1", flagged) + assert supports_openai_prompt_cache_breakpoint("gpt-4.1") is True + + def test_listed_gpt_5_6_without_the_flag_is_not_eligible(self, monkeypatch): + unflagged = {k: v for k, v in litellm.model_cost["gpt-5.6"].items() if k != "supports_prompt_cache_breakpoint"} + monkeypatch.setitem(litellm.model_cost, "gpt-5.6", unflagged) + assert supports_openai_prompt_cache_breakpoint("gpt-5.6") is False + + def test_listed_gpt_model_without_the_flag_is_false(self): + assert "supports_prompt_cache_breakpoint" not in litellm.model_cost["gpt-4.1"] + assert supports_openai_prompt_cache_breakpoint("gpt-4.1") is False + + @pytest.mark.parametrize("model,expected", [("gpt-5.6-2026-01-01", True), ("gpt-5.5-preview-unlisted", False)]) + def test_unlisted_model_falls_back_to_the_version_rule(self, model, expected): + assert model not in litellm.model_cost + assert supports_openai_prompt_cache_breakpoint(model) is expected diff --git a/tests/test_litellm/responses/test_responses_api_request_body.py b/tests/test_litellm/responses/test_responses_api_request_body.py index ea5eb3afe9a..1402491a6b6 100644 --- a/tests/test_litellm/responses/test_responses_api_request_body.py +++ b/tests/test_litellm/responses/test_responses_api_request_body.py @@ -4,6 +4,7 @@ over the wire and surface provider errors correctly. Expected JSON bodies are st in expected_responses_api_request/. """ +import copy import json from pathlib import Path from unittest.mock import AsyncMock, patch @@ -422,3 +423,160 @@ async def test_aresponses_websocket_strips_responses_routing_prefix_from_openai_ mock_ws.assert_awaited_once() assert mock_ws.call_args.kwargs["model"] == "gpt-5.6" assert mock_ws.call_args.kwargs["custom_llm_provider"] == "openai" + + +_INJECTION_POINT_INPUT = [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "hi"}] +_SYSTEM_INJECTION_POINT = [{"location": "message", "role": "system"}] + + +def _sent_body(mock_post) -> dict: + kwargs = mock_post.call_args.kwargs + return kwargs["json"] if "json" in kwargs else json.loads(kwargs["data"]) + + +@pytest.mark.asyncio +async def test_aresponses_injection_point_marks_input_text_on_gpt_5_6(): + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_async", "gpt-5.6"), 200) + + await litellm.aresponses( + model="openai/gpt-5.6", + api_key="fake-api-key", + input=copy.deepcopy(_INJECTION_POINT_INPUT), + cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT), + ) + + body = _sent_body(mock_post) + assert body["input"][0]["content"][0] == { + "type": "input_text", + "text": "You are terse.", + "prompt_cache_breakpoint": {"mode": "explicit"}, + } + assert body["input"][1] == {"role": "user", "content": "hi"} + assert body["prompt_cache_options"] == {"mode": "explicit"} + + +def test_responses_injection_point_marks_input_text_on_gpt_5_6(): + with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post: + mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_sync", "gpt-5.6"), 200) + + litellm.responses( + model="openai/gpt-5.6", + api_key="fake-api-key", + input=copy.deepcopy(_INJECTION_POINT_INPUT), + cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT), + ) + + body = _sent_body(mock_post) + assert body["input"][0]["content"][0] == { + "type": "input_text", + "text": "You are terse.", + "prompt_cache_breakpoint": {"mode": "explicit"}, + } + assert body["input"][1] == {"role": "user", "content": "hi"} + assert body["prompt_cache_options"] == {"mode": "explicit"} + + +@pytest.mark.asyncio +async def test_aresponses_injection_point_sends_nothing_extra_below_gpt_5_6(): + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_old", "gpt-4.1"), 200) + + await litellm.aresponses( + model="openai/gpt-4.1", + api_key="fake-api-key", + input=copy.deepcopy(_INJECTION_POINT_INPUT), + cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT), + ) + + body = _sent_body(mock_post) + assert body["input"] == _INJECTION_POINT_INPUT + assert "prompt_cache_options" not in body + assert "cache_control" not in json.dumps(body) + + +@pytest.fixture +def _no_openai_api_base_override(monkeypatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None) + + +_CUSTOM_API_BASE = "http://127.0.0.1:9/v1" + + +async def _aresponses_body_with_system_point(**request_kwargs) -> dict: + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_gate", "gpt-5.6"), 200) + await litellm.aresponses( + api_key="fake-api-key", + input=copy.deepcopy(_INJECTION_POINT_INPUT), + cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT), + **request_kwargs, + ) + return _sent_body(mock_post) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_no_openai_api_base_override") +async def test_aresponses_litellm_proxy_target_sends_no_openai_markers(): + body = await _aresponses_body_with_system_point(model="litellm_proxy/gpt-5.6", api_base=_CUSTOM_API_BASE) + assert body["input"] == _INJECTION_POINT_INPUT + assert "prompt_cache_options" not in body + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_no_openai_api_base_override") +async def test_aresponses_custom_api_base_sends_no_openai_markers(): + body = await _aresponses_body_with_system_point(model="gpt-5.6", api_base=_CUSTOM_API_BASE) + assert body["input"] == _INJECTION_POINT_INPUT + assert "prompt_cache_options" not in body + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_no_openai_api_base_override") +async def test_aresponses_custom_api_base_opts_in_through_prompt_cache_options(): + body = await _aresponses_body_with_system_point( + model="gpt-5.6", api_base=_CUSTOM_API_BASE, prompt_cache_options={"mode": "explicit"} + ) + assert body["input"][0]["content"][0] == { + "type": "input_text", + "text": "You are terse.", + "prompt_cache_breakpoint": {"mode": "explicit"}, + } + assert body["prompt_cache_options"] == {"mode": "explicit"} + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_no_openai_api_base_override") +async def test_aresponses_regional_openai_api_base_marks_input_text(): + body = await _aresponses_body_with_system_point(model="gpt-5.6", api_base="https://eu.api.openai.com/v1") + assert body["input"][0]["content"][0]["prompt_cache_breakpoint"] == {"mode": "explicit"} + assert body["prompt_cache_options"] == {"mode": "explicit"} + + +@pytest.mark.usefixtures("_no_openai_api_base_override") +def test_responses_custom_api_base_sends_no_openai_markers(): + with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post: + mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_gate_sync", "gpt-5.6"), 200) + + litellm.responses( + model="gpt-5.6", + api_key="fake-api-key", + api_base=_CUSTOM_API_BASE, + input=copy.deepcopy(_INJECTION_POINT_INPUT), + cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT), + ) + + body = _sent_body(mock_post) + assert body["input"] == _INJECTION_POINT_INPUT + assert "prompt_cache_options" not in body diff --git a/tests/test_litellm/responses/test_responses_utils.py b/tests/test_litellm/responses/test_responses_utils.py index 2b9e6d34828..35bf99db6dc 100644 --- a/tests/test_litellm/responses/test_responses_utils.py +++ b/tests/test_litellm/responses/test_responses_utils.py @@ -638,3 +638,65 @@ def test_responses_maps_reasoning_effort_from_litellm_params_to_reasoning(): "effort": "high", "summary": "detailed", } + + +class TestMergePromptManagementInputReshape: + """Chat-shaped text parts produced by prompt management hooks become input_text parts (#37509).""" + + EXPLICIT = {"mode": "explicit"} + + def _run_cache_hook(self, client_input, points, model="openai/gpt-5.6"): + from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook + + _, merged, _ = AnthropicCacheControlHook().get_chat_completion_prompt( + model=model, + messages=client_input, + non_default_params={"cache_control_injection_points": points}, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + return merged + + def test_string_system_item_becomes_input_text_with_marker(self): + original_input = [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "hi"}] + merged = self._run_cache_hook(list(original_input), [{"location": "message", "role": "system"}]) + + result = ResponsesAPIRequestUtils.merge_prompt_management_input( + original_input=original_input, client_input=list(original_input), merged_input=merged + ) + + assert result[0]["content"] == [ + {"type": "input_text", "text": "You are terse.", "prompt_cache_breakpoint": self.EXPLICIT} + ] + assert result[1] == {"role": "user", "content": "hi"} + + def test_assistant_text_parts_are_left_alone(self): + merged = [ + {"role": "assistant", "content": [{"type": "text", "text": "earlier answer"}]}, + {"role": "user", "content": [{"type": "text", "text": "follow-up"}]}, + ] + + result = ResponsesAPIRequestUtils.merge_prompt_management_input( + original_input="ignored", client_input=[], merged_input=merged + ) + + assert result[0]["content"] == [{"type": "text", "text": "earlier answer"}] + assert result[1]["content"] == [{"type": "input_text", "text": "follow-up"}] + + def test_parts_already_in_responses_shape_are_unchanged(self): + merged = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "a", "prompt_cache_breakpoint": self.EXPLICIT}, + {"type": "input_image", "image_url": "https://example.com/a.png"}, + ], + } + ] + + result = ResponsesAPIRequestUtils.merge_prompt_management_input( + original_input="ignored", client_input=[], merged_input=merged + ) + + assert result == merged diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 68b8d1c62b5..786adf2d78b 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -2672,3 +2672,49 @@ def test_openai_model_without_a_provider_still_routes_to_openai(): ) mock_create.assert_called() + + +def _openai_chat_create_kwargs(client, **completion_kwargs): + with patch.object(client.chat.completions.with_raw_response, "create") as mock_client: + with contextlib.suppress(Exception): + litellm.completion( + messages=[{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}], + cache_control_injection_points=[{"location": "message", "role": "system"}], + client=client, + **completion_kwargs, + ) + + mock_client.assert_called_once() + return mock_client.call_args.kwargs + + +@pytest.fixture +def _no_openai_api_base_override(monkeypatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None) + + +@pytest.mark.usefixtures("_no_openai_api_base_override") +def test_completion_custom_api_base_sends_no_prompt_cache_breakpoint_for_gpt_5_6(): + from openai import OpenAI + + client = OpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:9/v1") + request_body = _openai_chat_create_kwargs(client, model="gpt-5.6", api_base="http://127.0.0.1:9/v1") + + assert request_body["messages"][0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}} + assert "prompt_cache_breakpoint" not in json.dumps(request_body["messages"]) + assert "prompt_cache_options" not in json.dumps(request_body) + + +@pytest.mark.usefixtures("_no_openai_api_base_override") +def test_completion_default_api_base_sends_prompt_cache_breakpoint_for_gpt_5_6(): + from openai import OpenAI + + client = OpenAI(api_key="fake-api-key") + request_body = _openai_chat_create_kwargs(client, model="gpt-5.6") + + assert request_body["messages"][0]["content"] == [ + {"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}} + ] + assert request_body["extra_body"]["prompt_cache_options"] == {"mode": "explicit"} diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 60453931595..cb58038e081 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -921,6 +921,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_parallel_tool_use_config": {"type": "boolean"}, "supports_pdf_input": {"type": "boolean"}, "prompt_cache_min_tokens": {"type": "number"}, + "supports_prompt_cache_breakpoint": {"type": "boolean"}, "supports_prompt_caching": {"type": "boolean"}, "supports_response_schema": {"type": "boolean"}, "supports_system_messages": {"type": "boolean"}, From e16cf5ed3ff1e7468f57186e94727c82ed278ec2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 05:47:19 -0700 Subject: [PATCH 3/3] Fall back to the GPT version rule when the cost map carries no breakpoint flag A proxy on the default remote cost map never produced a prompt cache breakpoint: the published map has the gpt-5.6 entries without supports_prompt_cache_breakpoint, so the model-map gate returned False for every listed model and only LITELLM_LOCAL_MODEL_COST_MAP=True (the repo .env, hence the passing unit tests) made the feature work. The hook now honors the flag when the entry carries one, True or False, and otherwise applies the GPT-5.6+ version rule to the model name, so a map that lags the flag still gets the OpenAI dialect. The model-map tests pin litellm.model_cost to the bundled backup map and a new test drives the hook against an unflagged gpt-5.6 entry. completion() and acompletion() take base_url as an alias for api_base that only lands on api_base after the cache control hook ran, so a GPT-5.6 call at a non-OpenAI gateway given through base_url still got the dialect. Both seed calls and the unstamped request-params read now look at base_url too. ResponsesAPIRequestUtils.merge_prompt_management_input reshaped hook output in place, retyping text parts to input_text on the caller's own message objects. The merge now shapes a copy of each message as it emits it, so the identity-based merge keeps working on the hook's objects and nothing the hook or the client owns is mutated. --- .../anthropic_cache_control_hook.py | 15 ++++--- litellm/main.py | 4 +- litellm/responses/utils.py | 40 ++++++++++------- .../test_anthropic_cache_control_hook.py | 45 ++++++++++++++++--- .../test_responses_api_request_body.py | 18 ++++++++ .../responses/test_responses_utils.py | 27 +++++++++++ tests/test_litellm/test_main.py | 36 +++++++++++++++ 7 files changed, 155 insertions(+), 30 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index f65decc5463..1258c7593b4 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -59,10 +59,9 @@ OPENAI_API_BASE_ENV_VARS: Final = ("OPENAI_BASE_URL", "OPENAI_API_BASE") def supports_openai_prompt_cache_breakpoint(model: str) -> bool: - if _has_model_map_entry(model): - from litellm.utils import supports_prompt_cache_breakpoint - - return supports_prompt_cache_breakpoint(model) + model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model) + if model_map_flag is not None: + return model_map_flag version_match: Final = _GPT_VERSION_PATTERN.match(model.rsplit("/", 1)[-1].lower()) if version_match is None: return False @@ -70,10 +69,12 @@ def supports_openai_prompt_cache_breakpoint(model: str) -> bool: return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION -def _has_model_map_entry(model: str) -> bool: +def _model_map_prompt_cache_breakpoint_flag(model: str) -> bool | None: import litellm - return model in litellm.model_cost or model.rsplit("/", 1)[-1] in litellm.model_cost + entries: Final = (litellm.model_cost.get(key) for key in (model, model.rsplit("/", 1)[-1])) + flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in entries if isinstance(entry, dict)) + return next((bool(flag) for flag in flags if flag is not None), None) def targets_openai_api(api_base: object) -> bool: @@ -150,7 +151,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): else AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( model, non_default_params.get("custom_llm_provider"), - non_default_params.get("api_base"), + non_default_params.get("api_base") or non_default_params.get("base_url"), non_default_params.get("prompt_cache_options"), ) ) diff --git a/litellm/main.py b/litellm/main.py index 87b32ab140c..98c220f94e0 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -507,7 +507,7 @@ async def acompletion( custom_llm_provider=cast(str | None, custom_llm_provider), # cast-ok: read from untyped kwargs tools=tools, enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs - api_base=kwargs.get("api_base"), + api_base=kwargs.get("api_base") or base_url, ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( @@ -5172,7 +5172,7 @@ def completion( custom_llm_provider=cast(str | None, kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs tools=tools, enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs - api_base=kwargs.get("api_base"), + api_base=kwargs.get("api_base") or base_url, ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 504009802f2..716a815547d 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -37,20 +37,28 @@ def normalize_responses_api_stream_options( return ResponsesAPIStreamOptions(include_obfuscation=include_obfuscation) +def _is_chat_text_part(part: object) -> bool: + return isinstance(part, dict) and part.get("type") == "text" + + +def _as_input_text_part(part: object) -> object: + if isinstance(part, dict) and part.get("type") == "text": + return {**part, "type": "input_text"} # mutable-ok: fresh part so the caller's block keeps its chat type + return part + + class ResponsesAPIRequestUtils: """Helper utils for constructing ResponseAPI requests""" @staticmethod - def shape_prompt_managed_messages_for_responses(messages: Iterable[object]) -> None: - for message in messages: - if not isinstance(message, dict) or message.get("role") == "assistant": - continue - content: object = message.get("content") - if not isinstance(content, list): - continue - for part in content: - if isinstance(part, dict) and part.get("type") == "text": - part["type"] = "input_text" + def shape_prompt_managed_message_for_responses(message: object) -> object: + if not isinstance(message, dict) or message.get("role") == "assistant": + return message + content: object = message.get("content") + if not isinstance(content, list) or not any(_is_chat_text_part(part) for part in content): + return message + shaped_content: Final = [_as_input_text_part(part) for part in content] # mutable-ok: Responses-shaped copy + return {**message, "content": shaped_content} # mutable-ok: copy, the hook's message stays untouched @staticmethod def merge_prompt_management_input( @@ -58,16 +66,16 @@ class ResponsesAPIRequestUtils: client_input: list[AllMessageValues], merged_input: list[AllMessageValues], ) -> list[object]: - ResponsesAPIRequestUtils.shape_prompt_managed_messages_for_responses(merged_input) + shape: Final = ResponsesAPIRequestUtils.shape_prompt_managed_message_for_responses if isinstance(original_input, str): - return [*merged_input] + return [shape(message) for message in merged_input] original_items: Final = tuple(original_input) client_item_ids: Final = frozenset(id(item) for item in client_input) message_positions = tuple(index for index, item in enumerate(original_items) if id(item) in client_item_ids) if len(message_positions) == len(original_items): - return [*merged_input] + return [shape(message) for message in merged_input] if not message_positions: verbose_logger.warning( "Prompt management hook returned messages without Responses API input messages; merged messages were ignored" @@ -82,7 +90,7 @@ class ResponsesAPIRequestUtils: if corresponding_messages: merged_by_position: Final = dict(zip(message_positions, merged_input)) return [ - merged_by_position[index] if index in merged_by_position else item + shape(merged_by_position[index]) if index in merged_by_position else item for index, item in enumerate(original_items) ] @@ -95,14 +103,14 @@ class ResponsesAPIRequestUtils: for index, position in enumerate(message_positions) } trailing_items: Final = original_items[message_positions[-1] + 1 :] - return [item for merged in merged_input for item in (*prefixes.get(id(merged), ()), merged)] + list( + return [item for merged in merged_input for item in (*prefixes.get(id(merged), ()), shape(merged))] + list( trailing_items ) verbose_logger.warning( "Prompt management hook replaced Responses API messages; non-message input items were dropped" ) - return [*merged_input] + return [shape(message) for message in merged_input] @staticmethod def merge_client_forwarded_headers( diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 4eb774af5bc..e92a368d24b 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -2697,11 +2697,14 @@ class TestToolConfigSlotInOpenAIDialect: class TestPromptCacheBreakpointCapability: - """Eligibility comes from the model map's supports_prompt_cache_breakpoint flag, with the GPT version - rule only for models the map does not know (#37509).""" + """Eligibility comes from the model map's supports_prompt_cache_breakpoint flag when the entry carries one, + with the GPT version rule for unlisted models and for entries the published map has not flagged yet (#37509).""" @pytest.fixture(autouse=True) - def _fresh_model_info_cache(self): + def _bundled_model_map(self, monkeypatch): + bundled = os.path.join(os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json") + with open(bundled) as handle: + monkeypatch.setattr(litellm, "model_cost", json.load(handle)) litellm.utils._cached_get_model_info_helper.cache_clear() yield litellm.utils._cached_get_model_info_helper.cache_clear() @@ -2724,15 +2727,47 @@ class TestPromptCacheBreakpointCapability: monkeypatch.setitem(litellm.model_cost, "gpt-4.1", flagged) assert supports_openai_prompt_cache_breakpoint("gpt-4.1") is True - def test_listed_gpt_5_6_without_the_flag_is_not_eligible(self, monkeypatch): + def test_listed_gpt_5_6_without_the_flag_falls_back_to_the_version_rule(self, monkeypatch): unflagged = {k: v for k, v in litellm.model_cost["gpt-5.6"].items() if k != "supports_prompt_cache_breakpoint"} monkeypatch.setitem(litellm.model_cost, "gpt-5.6", unflagged) + assert supports_openai_prompt_cache_breakpoint("gpt-5.6") is True + assert supports_openai_prompt_cache_breakpoint("openai/gpt-5.6") is True + + def test_listed_model_flagged_false_is_not_eligible(self, monkeypatch): + monkeypatch.setitem( + litellm.model_cost, "gpt-5.6", {**litellm.model_cost["gpt-5.6"], "supports_prompt_cache_breakpoint": False} + ) assert supports_openai_prompt_cache_breakpoint("gpt-5.6") is False - def test_listed_gpt_model_without_the_flag_is_false(self): + def test_listed_gpt_model_without_the_flag_follows_the_version_rule(self): assert "supports_prompt_cache_breakpoint" not in litellm.model_cost["gpt-4.1"] assert supports_openai_prompt_cache_breakpoint("gpt-4.1") is False + def test_published_map_without_the_flag_still_injects_on_gpt_5_6(self, monkeypatch): + unflagged = {k: v for k, v in litellm.model_cost["gpt-5.6"].items() if k != "supports_prompt_cache_breakpoint"} + monkeypatch.setitem(litellm.model_cost, "gpt-5.6", unflagged) + points = [{"location": "message", "role": "system"}] + + _, chat_messages, chat_params = AnthropicCacheControlHook().get_chat_completion_prompt( + model="openai/gpt-5.6", + messages=[{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}], + non_default_params={"cache_control_injection_points": copy.deepcopy(points)}, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + assert chat_messages[0]["content"] == [ + {"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}} + ] + assert chat_params["prompt_cache_options"] == {"mode": "explicit"} + + kwargs = {"cache_control_injection_points": copy.deepcopy(points)} + _, system = AnthropicCacheControlHook.maybe_inject_cache_control( + [{"role": "user", "content": "hi"}], "sys", kwargs, model="gpt-5.6", custom_llm_provider="openai" + ) + assert system == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}] + assert kwargs == {"prompt_cache_options": {"mode": "explicit"}} + @pytest.mark.parametrize("model,expected", [("gpt-5.6-2026-01-01", True), ("gpt-5.5-preview-unlisted", False)]) def test_unlisted_model_falls_back_to_the_version_rule(self, model, expected): assert model not in litellm.model_cost diff --git a/tests/test_litellm/responses/test_responses_api_request_body.py b/tests/test_litellm/responses/test_responses_api_request_body.py index 1402491a6b6..14eb9ab6e12 100644 --- a/tests/test_litellm/responses/test_responses_api_request_body.py +++ b/tests/test_litellm/responses/test_responses_api_request_body.py @@ -564,6 +564,24 @@ async def test_aresponses_regional_openai_api_base_marks_input_text(): assert body["prompt_cache_options"] == {"mode": "explicit"} +@pytest.mark.usefixtures("_no_openai_api_base_override") +def test_responses_custom_base_url_sends_no_openai_markers(): + with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post: + mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_gate_base_url", "gpt-5.6"), 200) + + litellm.responses( + model="gpt-5.6", + api_key="fake-api-key", + base_url=_CUSTOM_API_BASE, + input=copy.deepcopy(_INJECTION_POINT_INPUT), + cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT), + ) + + body = _sent_body(mock_post) + assert body["input"] == _INJECTION_POINT_INPUT + assert "prompt_cache_options" not in body + + @pytest.mark.usefixtures("_no_openai_api_base_override") def test_responses_custom_api_base_sends_no_openai_markers(): with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post: diff --git a/tests/test_litellm/responses/test_responses_utils.py b/tests/test_litellm/responses/test_responses_utils.py index 35bf99db6dc..2f4a699d307 100644 --- a/tests/test_litellm/responses/test_responses_utils.py +++ b/tests/test_litellm/responses/test_responses_utils.py @@ -671,6 +671,33 @@ class TestMergePromptManagementInputReshape: ] assert result[1] == {"role": "user", "content": "hi"} + def test_reshape_returns_copies_and_leaves_hook_output_untouched(self): + user_part = {"type": "text", "text": "follow-up"} + user_message = {"role": "user", "content": [user_part]} + merged = [user_message] + + result = ResponsesAPIRequestUtils.merge_prompt_management_input( + original_input="ignored", client_input=[], merged_input=merged + ) + + assert result == [{"role": "user", "content": [{"type": "input_text", "text": "follow-up"}]}] + assert user_part == {"type": "text", "text": "follow-up"} + assert user_message == {"role": "user", "content": [user_part]} + assert result[0] is not user_message + + def test_reshape_keeps_non_message_items_when_hook_returns_client_objects(self): + user_message = {"role": "user", "content": [{"type": "text", "text": "question"}]} + reference = {"type": "item_reference", "id": "msg_123"} + original_input = [reference, user_message] + + result = ResponsesAPIRequestUtils.merge_prompt_management_input( + original_input=original_input, client_input=[user_message], merged_input=[user_message] + ) + + assert result == [reference, {"role": "user", "content": [{"type": "input_text", "text": "question"}]}] + assert result[0] is reference + assert user_message["content"] == [{"type": "text", "text": "question"}] + def test_assistant_text_parts_are_left_alone(self): merged = [ {"role": "assistant", "content": [{"type": "text", "text": "earlier answer"}]}, diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 786adf2d78b..4b223a3a900 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -2707,6 +2707,42 @@ def test_completion_custom_api_base_sends_no_prompt_cache_breakpoint_for_gpt_5_6 assert "prompt_cache_options" not in json.dumps(request_body) +@pytest.mark.usefixtures("_no_openai_api_base_override") +def test_completion_custom_base_url_sends_no_prompt_cache_breakpoint_for_gpt_5_6(): + from openai import OpenAI + + client = OpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:9/v1") + request_body = _openai_chat_create_kwargs(client, model="gpt-5.6", base_url="http://127.0.0.1:9/v1") + + assert request_body["messages"][0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}} + assert "prompt_cache_breakpoint" not in json.dumps(request_body["messages"]) + assert "prompt_cache_options" not in json.dumps(request_body) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_no_openai_api_base_override") +async def test_acompletion_custom_base_url_sends_no_prompt_cache_breakpoint_for_gpt_5_6(): + from openai import AsyncOpenAI + + client = AsyncOpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:9/v1") + with patch.object(client.chat.completions.with_raw_response, "create") as mock_create: + with contextlib.suppress(Exception): + await litellm.acompletion( + model="gpt-5.6", + messages=[{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}], + cache_control_injection_points=[{"location": "message", "role": "system"}], + client=client, + base_url="http://127.0.0.1:9/v1", + ) + + mock_create.assert_called_once() + request_body = mock_create.call_args.kwargs + + assert request_body["messages"][0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}} + assert "prompt_cache_breakpoint" not in json.dumps(request_body["messages"]) + assert "prompt_cache_options" not in json.dumps(request_body) + + @pytest.mark.usefixtures("_no_openai_api_base_override") def test_completion_default_api_base_sends_prompt_cache_breakpoint_for_gpt_5_6(): from openai import OpenAI