diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 4df6fce74c0..1258c7593b4 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -10,17 +10,31 @@ Supported for both `v1/chat/completions` (via the prompt-management hook) and """ import copy +import os +import re +from collections.abc import Iterable, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, cast +from urllib.parse import urlparse from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.integrations.prompt_management_base import PromptManagementClient +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + with_prompt_cache_breakpoint, +) from litellm.types.integrations.anthropic_cache_control_hook import ( CacheControlInjectionPoint, CacheControlMessageInjectionPoint, ) -from litellm.types.llms.openai import AllMessageValues, ChatCompletionCachedContent +from litellm.types.llms.anthropic import AnthropicSystemMessageContent +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionCachedContent, + ChatCompletionTextObject, + PromptCacheBreakpoint, + PromptCacheOptions, +) from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.utils import StandardCallbackDynamicParams @@ -34,6 +48,55 @@ else: # breakpoints: "A maximum of 4 blocks with cache_control may be provided." MAX_CACHE_CONTROL_BLOCKS: Final = 4 +CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint") +OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6) +_GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?") +OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset( + {"text", "image", "image_url", "file", "input_audio", "input_text", "input_image", "input_file"} +) +OPENAI_API_HOST: Final = "api.openai.com" +OPENAI_API_BASE_ENV_VARS: Final = ("OPENAI_BASE_URL", "OPENAI_API_BASE") + + +def supports_openai_prompt_cache_breakpoint(model: str) -> bool: + model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model) + if model_map_flag is not None: + return model_map_flag + version_match: Final = _GPT_VERSION_PATTERN.match(model.rsplit("/", 1)[-1].lower()) + if version_match is None: + return False + version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0)) + return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION + + +def _model_map_prompt_cache_breakpoint_flag(model: str) -> bool | None: + import litellm + + entries: Final = (litellm.model_cost.get(key) for key in (model, model.rsplit("/", 1)[-1])) + flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in entries if isinstance(entry, dict)) + return next((bool(flag) for flag in flags if flag is not None), None) + + +def targets_openai_api(api_base: object) -> bool: + import litellm + + resolved: Final = next( + (value for value in (api_base, litellm.api_base, *map(os.getenv, OPENAI_API_BASE_ENV_VARS)) if value), + None, + ) + if not isinstance(resolved, str): + return True + host: Final = urlparse(resolved).hostname + return host is not None and (host == OPENAI_API_HOST or host.endswith(f".{OPENAI_API_HOST}")) + + +def _carries_cache_breakpoint(block: object) -> bool: + return isinstance(block, dict) and any(block.get(key) is not None for key in CACHE_BREAKPOINT_KEYS) + + +def _accepts_prompt_cache_breakpoint(block: object) -> bool: + return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES + class AnthropicCacheControlHook(CustomPromptManagement): def get_chat_completion_prompt( @@ -81,13 +144,32 @@ class AnthropicCacheControlHook(CustomPromptManagement): # provider transform, where each tool_config point appends at most one # cachePoint to the tools. That block also counts toward Anthropic's # limit, so reserve a slot for it here to leave room. - reserved_blocks: Final = 1 if any(p.get("location") == "tool_config" for p in remaining_points) else 0 - + stamped_dialect: Final = injection_points[0].get("_litellm_openai_dialect") + openai_dialect: Final = ( + stamped_dialect + if isinstance(stamped_dialect, bool) + else AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( + model, + non_default_params.get("custom_llm_provider"), + non_default_params.get("api_base") or non_default_params.get("base_url"), + non_default_params.get("prompt_cache_options"), + ) + ) + reserved_blocks: Final = ( + 1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0 + ) + breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages) processed_messages = self._apply_message_injections( points=message_points, messages=processed_messages, max_blocks=MAX_CACHE_CONTROL_BLOCKS - reserved_blocks, + openai_dialect=openai_dialect, ) + if ( + openai_dialect + and AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages) > breakpoints_before + ): + non_default_params.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit")) # Pass through non-message injection points for provider-specific handling if remaining_points: @@ -97,11 +179,43 @@ class AnthropicCacheControlHook(CustomPromptManagement): return model, processed_messages, non_default_params + @staticmethod + def _targets_openai_prompt_cache_breakpoint( + model: str | None, + custom_llm_provider: str | None, + api_base: object = None, + prompt_cache_options: object = None, + ) -> bool: + if model is None or not supports_openai_prompt_cache_breakpoint(model): + return False + if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai": + return False + return prompt_cache_options is not None or targets_openai_api(api_base) + + @staticmethod + def _resolve_provider(model: str) -> str | None: + from litellm.exceptions import BadRequestError + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + try: + _, provider, _, _ = get_llm_provider(model=model) + except BadRequestError: + return None + return provider + + @staticmethod + def _count_request_cache_breakpoints(messages: Iterable[object], system: object = None) -> int: + system_blocks: Final = ( + sum(1 for block in system if _carries_cache_breakpoint(block)) if isinstance(system, list) else 0 + ) + return system_blocks + sum(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages) + @staticmethod def _apply_message_injections( points: list[CacheControlMessageInjectionPoint], messages: list[AllMessageValues], max_blocks: int, + openai_dialect: bool = False, ) -> list[AllMessageValues]: """Apply message-level cache control injection points in order. @@ -112,7 +226,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): ``max_blocks`` is reached. Injection points are honored in config order, so earlier points win when slots are scarce. """ - used_blocks = sum(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages) + used_blocks = AnthropicCacheControlHook._count_request_cache_breakpoints(messages) limit_reached = False for point in points: @@ -134,16 +248,17 @@ class AnthropicCacheControlHook(CustomPromptManagement): continue messages[target_index] = AnthropicCacheControlHook._safe_insert_cache_control_in_message( - messages[target_index], control + messages[target_index], control, openai_dialect ) - used_blocks += 1 + if AnthropicCacheControlHook._message_has_cache_control(messages[target_index]): + used_blocks += 1 if limit_reached: break if limit_reached: verbose_logger.warning( - "AnthropicCacheControlHook: Reached the Anthropic limit of %s cache_control blocks. Skipping further injection.", + "AnthropicCacheControlHook: Reached the provider limit of %s cache breakpoints. Skipping further injection.", MAX_CACHE_CONTROL_BLOCKS, ) @@ -189,16 +304,13 @@ class AnthropicCacheControlHook(CustomPromptManagement): return [] @staticmethod - def _count_cache_control_blocks(message: AllMessageValues) -> int: - """Count cache_control breakpoints on a message (message + content level).""" - count = 0 - if message.get("cache_control") is not None: - count += 1 + def _count_cache_control_blocks(message: object) -> int: + if not isinstance(message, dict): + return 0 + count = 1 if _carries_cache_breakpoint(message) else 0 content: Final = message.get("content") if isinstance(content, list): - for block in content: - if isinstance(block, dict) and block.get("cache_control") is not None: - count += 1 + count += sum(1 for block in content if _carries_cache_breakpoint(block)) return count @staticmethod @@ -208,7 +320,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): @staticmethod def _safe_insert_cache_control_in_message( - message: AllMessageValues, control: ChatCompletionCachedContent + message: AllMessageValues, control: ChatCompletionCachedContent, openai_dialect: bool = False ) -> AllMessageValues: """ Safe way to insert cache control in a message @@ -221,6 +333,9 @@ class AnthropicCacheControlHook(CustomPromptManagement): Per Anthropic's API specification, when using multiple content blocks, only the last content block can have cache_control. """ + if openai_dialect: + return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(message) + message_content: Final = message.get("content", None) # 1. if string, insert cache control in the message @@ -232,11 +347,51 @@ class AnthropicCacheControlHook(CustomPromptManagement): message_content[-1]["cache_control"] = control return message + @staticmethod + def _insert_prompt_cache_breakpoint_in_message(message: AllMessageValues) -> AllMessageValues: + if message.get("role") == "assistant": + return message + message_content: Final = message.get("content", None) + if isinstance(message_content, str): + marked: Final = copy.copy(message) + marked["content"] = [ + with_prompt_cache_breakpoint( + ChatCompletionTextObject(type="text", text=message_content), PromptCacheBreakpoint(mode="explicit") + ) + ] + return marked + if isinstance(message_content, list): + target_index: Final = next( + ( + index + for index in range(len(message_content) - 1, -1, -1) + if _accepts_prompt_cache_breakpoint(message_content[index]) + ), + None, + ) + if target_index is not None: + message_content[target_index] = with_prompt_cache_breakpoint( + message_content[target_index], PromptCacheBreakpoint(mode="explicit") + ) + return message + + @staticmethod + def _system_block_with_breakpoint( + block: Mapping[str, object], control: ChatCompletionCachedContent, openai_dialect: bool + ) -> Mapping[str, object]: + marker: Final = ( + ("prompt_cache_breakpoint", PromptCacheBreakpoint(mode="explicit")) + if openai_dialect + else ("cache_control", control) + ) + return {**block, marker[0]: marker[1]} + @staticmethod def apply_to_anthropic_messages_request( messages: list[dict], system: str | list | None, injection_points: list[CacheControlInjectionPoint], + openai_dialect: bool = False, ) -> tuple[list[dict], str | list | None, list[CacheControlInjectionPoint]]: """Apply cache control injection for the Anthropic-native v1/messages endpoint. @@ -262,30 +417,32 @@ class AnthropicCacheControlHook(CustomPromptManagement): else: remaining_points.append(point) - reserved_blocks: Final = 1 if any(p.get("location") == "tool_config" for p in remaining_points) else 0 + reserved_blocks: Final = ( + 1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0 + ) max_blocks: Final = MAX_CACHE_CONTROL_BLOCKS - reserved_blocks - used_blocks = sum( - AnthropicCacheControlHook._count_cache_control_blocks(cast(AllMessageValues, msg)) - for msg in processed_messages - ) - if isinstance(processed_system, list): - used_blocks += sum( - 1 for b in processed_system if isinstance(b, dict) and b.get("cache_control") is not None - ) + message_blocks: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages) + system_blocks = AnthropicCacheControlHook._count_request_cache_breakpoints((), processed_system) - if system_points and processed_system is not None and used_blocks < max_blocks: + if system_points and processed_system is not None and message_blocks + system_blocks < max_blocks: system_already_has_cc: Final = isinstance(processed_system, list) and any( - isinstance(b, dict) and b.get("cache_control") is not None for b in processed_system + _carries_cache_breakpoint(b) for b in processed_system ) if not system_already_has_cc: control: Final = system_points[0].get("control") or ChatCompletionCachedContent(type="ephemeral") if isinstance(processed_system, str): - processed_system = [{"type": "text", "text": processed_system, "cache_control": control}] - used_blocks += 1 + processed_system = [ + AnthropicCacheControlHook._system_block_with_breakpoint( + AnthropicSystemMessageContent(type="text", text=processed_system), control, openai_dialect + ) + ] + system_blocks += 1 elif len(processed_system) > 0 and isinstance(processed_system[-1], dict): - processed_system[-1] = {**processed_system[-1], "cache_control": control} - used_blocks += 1 + processed_system[-1] = AnthropicCacheControlHook._system_block_with_breakpoint( + processed_system[-1], control, openai_dialect + ) + system_blocks += 1 for i, msg in enumerate(processed_messages): content = msg.get("content") @@ -295,7 +452,8 @@ class AnthropicCacheControlHook(CustomPromptManagement): processed_messages = AnthropicCacheControlHook._apply_message_injections( points=message_points, messages=cast(list[AllMessageValues], processed_messages), - max_blocks=max_blocks - used_blocks, + max_blocks=max_blocks - system_blocks, + openai_dialect=openai_dialect, ) return processed_messages, processed_system, remaining_points @@ -315,17 +473,57 @@ class AnthropicCacheControlHook(CustomPromptManagement): return ChatCompletionCachedContent(type="ephemeral") @staticmethod - def _stamped_as_judged(points: list[CacheControlInjectionPoint]) -> list[dict[str, object]]: + def _stamped_as_judged(points: Sequence[CacheControlInjectionPoint]) -> Sequence[Mapping[str, object]]: """Mark written-back points as having passed the client cache_control judgment. Builds copies because config-owned point dicts are shared across requests; mutating them would leak the stamp into future requests. """ - return [{**point, "_litellm_judged": True} for point in points] + return AnthropicCacheControlHook._stamped(points, "_litellm_judged", True) + + @staticmethod + def _judged_configured_points( + points: Sequence[CacheControlInjectionPoint], + messages: list[AllMessageValues], + tools: list[object] | None, + model: str, + custom_llm_provider: str | None, + api_base: object, + prompt_cache_options: object, + ) -> Sequence[Mapping[str, object]] | None: + if AnthropicCacheControlHook._should_stand_down(points, messages, None, tools): + return None + return AnthropicCacheControlHook._stamped_with_dialect( + points, model, custom_llm_provider, api_base, prompt_cache_options + ) + + @staticmethod + def _stamped_with_dialect( + points: Sequence[CacheControlInjectionPoint], + model: str, + custom_llm_provider: str | None, + api_base: object, + prompt_cache_options: object, + ) -> Sequence[Mapping[str, object]]: + if not supports_openai_prompt_cache_breakpoint(model): + return points + return AnthropicCacheControlHook._stamped( + points, + "_litellm_openai_dialect", + AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( + model, custom_llm_provider, api_base, prompt_cache_options + ), + ) + + @staticmethod + def _stamped( + points: Sequence[CacheControlInjectionPoint], key: str, value: object + ) -> Sequence[Mapping[str, object]]: + return [{**point, key: value} for point in points] @staticmethod def _should_stand_down( - points: list[CacheControlInjectionPoint], + points: Sequence[CacheControlInjectionPoint], messages: list[AllMessageValues], system: str | list | None, tools: list | None, @@ -359,11 +557,8 @@ class AnthropicCacheControlHook(CustomPromptManagement): carry the mark either at the top level (Anthropic shape) or nested under ``function`` (OpenAI shape); the Anthropic chat transform accepts both. """ - if any(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages): + if AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) > 0: return True - if isinstance(system, list): - if any(isinstance(block, dict) and block.get("cache_control") is not None for block in system): - return True if tools is not None: return any( isinstance(tool, dict) @@ -438,6 +633,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): custom_llm_provider: str | None, tools: list | None = None, enable_prompt_caching: bool | None = None, + api_base: object = None, ) -> None: """For /chat/completions: resolve the injection points the request should carry. @@ -452,10 +648,19 @@ class AnthropicCacheControlHook(CustomPromptManagement): unchanged. """ if non_default_params.get("cache_control_injection_points"): - if AnthropicCacheControlHook._should_stand_down( - non_default_params["cache_control_injection_points"], messages, None, tools - ): + judged: Final = AnthropicCacheControlHook._judged_configured_points( + non_default_params["cache_control_injection_points"], + messages, + tools, + model, + custom_llm_provider, + api_base, + non_default_params.get("prompt_cache_options"), + ) + if judged is None: non_default_params.pop("cache_control_injection_points") + else: + non_default_params["cache_control_injection_points"] = judged return points: Final = AnthropicCacheControlHook.get_default_injection_points( messages=messages, @@ -476,6 +681,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): model: str | None = None, custom_llm_provider: str | None = None, tools: list[dict] | None = None, + api_base: str | None = None, ) -> tuple[list[dict], str | list | None]: """Extract cache_control_injection_points from kwargs and apply if present. @@ -513,11 +719,21 @@ class AnthropicCacheControlHook(CustomPromptManagement): if not injection_points: return messages, system + openai_dialect: Final = AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( + model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options") + ) + breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request( messages=messages, system=system, injection_points=injection_points, + openai_dialect=openai_dialect, ) + if ( + openai_dialect + and AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) > breakpoints_before + ): + kwargs.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit")) if remaining: kwargs["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(remaining) return messages, system diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index e4c1c9fc5cf..2db5776047b 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -10,7 +10,7 @@ from collections.abc import Iterable, Mapping, Sequence from itertools import groupby from os import PathLike from pathlib import Path -from typing import TYPE_CHECKING, Any, Final, Literal, cast +from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast from openai.types.chat.chat_completion_custom_tool_param import ( CustomFormatGrammar, @@ -1325,6 +1325,16 @@ def check_is_function_call(logging_obj: "LoggingClass") -> bool: return False +_MarkedT: Final = TypeVar("_MarkedT", bound=Mapping[str, object]) + + +def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT: + if marker is None: + return target + marked: Final = {**target, "prompt_cache_breakpoint": marker} # mutable-ok: API message payload + return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key + + def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Any: """ Filters a value from a dictionary diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index e45414b4a73..c6bfdb79002 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -2,7 +2,7 @@ import copy import hashlib import json from collections.abc import AsyncIterator, Iterator, Mapping -from typing import TYPE_CHECKING, Any, Final, Literal, cast +from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast from litellm.llms.anthropic.experimental_pass_through.utils import ( is_reasoning_auto_summary_enabled, @@ -61,6 +61,7 @@ from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingCho from litellm.litellm_core_utils.prompt_templates.common_utils import ( parse_tool_call_arguments, + with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, @@ -245,6 +246,9 @@ class AnthropicAdapter: return anthropic_wrapper.anthropic_sse_wrapper() +_BlockT: Final = TypeVar("_BlockT", bound=Mapping[str, object]) + + class LiteLLMAnthropicMessagesAdapter: def __init__(self): pass @@ -308,6 +312,12 @@ class LiteLLMAnthropicMessagesAdapter: # Fallback for non-dict objects (shouldn't happen in practice) cast(dict[str, object], target)["cache_control"] = cache_control + @staticmethod + def _add_prompt_cache_breakpoint_if_present(source: object, target: _BlockT) -> _BlockT: + if isinstance(source, dict) and "prompt_cache_breakpoint" in source: + return with_prompt_cache_breakpoint(target, source["prompt_cache_breakpoint"]) + return target + def translatable_anthropic_params(self) -> list[str]: """ Which anthropic params, we need to translate to the openai format. @@ -368,7 +378,9 @@ class LiteLLMAnthropicMessagesAdapter: if content.get("type") == "text": text_obj = ChatCompletionTextObject(type="text", text=content.get("text", "")) self._add_cache_control_if_applicable(content, text_obj, model) - new_user_content_list.append(text_obj) + new_user_content_list.append( + self._add_prompt_cache_breakpoint_if_present(content, text_obj) + ) elif content.get("type") == "image": # Convert Anthropic image format to OpenAI format source = content.get("source", {}) @@ -378,7 +390,9 @@ class LiteLLMAnthropicMessagesAdapter: image_url_obj = ChatCompletionImageUrlObject(url=openai_image_url) image_obj = ChatCompletionImageObject(type="image_url", image_url=image_url_obj) self._add_cache_control_if_applicable(content, image_obj, model) - new_user_content_list.append(image_obj) + new_user_content_list.append( + self._add_prompt_cache_breakpoint_if_present(content, image_obj) + ) elif content.get("type") == "document": # Convert Anthropic document format (PDF, etc.) to OpenAI format source = content.get("source", {}) @@ -869,7 +883,7 @@ class LiteLLMAnthropicMessagesAdapter: continue text_obj = ChatCompletionTextObject(type="text", text=text) self._add_cache_control_if_applicable(block, text_obj, model) - text_parts.append(text_obj) + text_parts.append(self._add_prompt_cache_breakpoint_if_present(block, text_obj)) return ChatCompletionSystemMessage(role="system", content=text_parts) if text_parts else None def _add_system_message_to_messages( @@ -900,7 +914,7 @@ class LiteLLMAnthropicMessagesAdapter: "text": block.get("text", ""), } self._add_cache_control_if_applicable(block, text_block, model_name) - openai_system_content.append(text_block) + openai_system_content.append(self._add_prompt_cache_breakpoint_if_present(block, text_block)) if openai_system_content: new_messages.insert( 0, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index c4b5cc628e2..26aef666172 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -230,7 +230,7 @@ async def anthropic_messages( ) messages, system = AnthropicCacheControlHook.maybe_inject_cache_control( - messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools + messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools, api_base=api_base ) original_stream: Final = stream or kwargs.get("_websearch_interception_converted_stream", False) @@ -422,7 +422,7 @@ def anthropic_messages_handler( ) messages, system = AnthropicCacheControlHook.maybe_inject_cache_control( - messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools + messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools, api_base=api_base ) metadata = validate_anthropic_api_metadata(metadata) diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 21a8cb9501e..18f33c63e4d 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -12,6 +12,7 @@ from typing import Any, Final, cast from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, + with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking_budget, @@ -82,7 +83,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: @staticmethod def _translate_midturn_system_content_to_responses( content: str | Iterable[AnthropicSystemMessageContent], - ) -> list[dict[str, str]]: # mutable-ok: API message payload + ) -> list[dict[str, object]]: # mutable-ok: API message payload """Convert in-sequence system content to Responses input-text parts.""" if isinstance(content, str): return ( @@ -91,7 +92,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if not isinstance(content, list): return [] # mutable-ok: API message payload return [ # mutable-ok: API message payload - {"type": "input_text", "text": text} # mutable-ok: API message payload + with_prompt_cache_breakpoint( + {"type": "input_text", "text": text}, block.get("prompt_cache_breakpoint") + ) # mutable-ok: API message payload for block in content if isinstance(block, dict) and block.get("type") == "text" and (text := block.get("text")) # pyright: ignore[reportUnnecessaryIsInstance] # untrusted client payload ] @@ -146,11 +149,20 @@ class LiteLLMAnthropicToResponsesAPIAdapter: continue btype = block.get("type") if btype == "text": - user_parts.append({"type": "input_text", "text": block.get("text", "")}) + user_parts.append( + with_prompt_cache_breakpoint( + {"type": "input_text", "text": block.get("text", "")}, + block.get("prompt_cache_breakpoint"), + ) + ) elif btype == "image": url = self._translate_anthropic_image_source_to_url(cast(dict, block.get("source", {}))) if url: - user_parts.append({"type": "input_image", "image_url": url}) + user_parts.append( + with_prompt_cache_breakpoint( + {"type": "input_image", "image_url": url}, block.get("prompt_cache_breakpoint") + ) + ) elif btype == "tool_result": tool_use_id = block.get("tool_use_id", "") inner = block.get("content") @@ -376,19 +388,36 @@ class LiteLLMAnthropicToResponsesAPIAdapter: anthropic_request["messages"], ) + input_items: Final = self.translate_messages_to_responses_input(messages_list) + system: Final = anthropic_request.get("system") + developer_parts: Final = ( + self._translate_midturn_system_content_to_responses(system) + if isinstance(system, list) + and any(isinstance(block, dict) and block.get("prompt_cache_breakpoint") is not None for block in system) + else () + ) + if developer_parts: + input_items.insert( + 0, + { # mutable-ok: API message payload + "type": "message", + "role": "developer", + "content": developer_parts, + }, + ) + responses_kwargs: Final[dict[str, Any]] = { "model": model, - "input": self.translate_messages_to_responses_input(messages_list), + "input": input_items, } - # system -> instructions - system: Final = anthropic_request.get("system") - if system: + if system and not developer_parts: if isinstance(system, str): responses_kwargs["instructions"] = system elif isinstance(system, list): - text_parts = [b.get("text", "") for b in system if isinstance(b, dict) and b.get("type") == "text"] - responses_kwargs["instructions"] = "\n".join(filter(None, text_parts)) + responses_kwargs["instructions"] = "\n".join( + filter(None, (b.get("text", "") for b in system if isinstance(b, dict) and b.get("type") == "text")) + ) # max_tokens -> max_output_tokens max_tokens: Final = anthropic_request.get("max_tokens") diff --git a/litellm/main.py b/litellm/main.py index f0b20eba9b6..98c220f94e0 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -507,6 +507,7 @@ async def acompletion( custom_llm_provider=cast(str | None, custom_llm_provider), # cast-ok: read from untyped kwargs tools=tools, enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs + api_base=kwargs.get("api_base") or base_url, ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( @@ -5171,6 +5172,7 @@ def completion( custom_llm_provider=cast(str | None, kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs tools=tools, enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs + api_base=kwargs.get("api_base") or base_url, ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 34c9f98077b..db6627bad58 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -25368,6 +25368,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -25430,6 +25431,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -25492,6 +25494,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -25554,6 +25557,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 4b5def790ed..716a815547d 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -37,24 +37,45 @@ def normalize_responses_api_stream_options( return ResponsesAPIStreamOptions(include_obfuscation=include_obfuscation) +def _is_chat_text_part(part: object) -> bool: + return isinstance(part, dict) and part.get("type") == "text" + + +def _as_input_text_part(part: object) -> object: + if isinstance(part, dict) and part.get("type") == "text": + return {**part, "type": "input_text"} # mutable-ok: fresh part so the caller's block keeps its chat type + return part + + class ResponsesAPIRequestUtils: """Helper utils for constructing ResponseAPI requests""" + @staticmethod + def shape_prompt_managed_message_for_responses(message: object) -> object: + if not isinstance(message, dict) or message.get("role") == "assistant": + return message + content: object = message.get("content") + if not isinstance(content, list) or not any(_is_chat_text_part(part) for part in content): + return message + shaped_content: Final = [_as_input_text_part(part) for part in content] # mutable-ok: Responses-shaped copy + return {**message, "content": shaped_content} # mutable-ok: copy, the hook's message stays untouched + @staticmethod def merge_prompt_management_input( original_input: str | ResponseInputParam, client_input: list[AllMessageValues], merged_input: list[AllMessageValues], ) -> list[object]: + shape: Final = ResponsesAPIRequestUtils.shape_prompt_managed_message_for_responses if isinstance(original_input, str): - return [*merged_input] + return [shape(message) for message in merged_input] original_items: Final = tuple(original_input) client_item_ids: Final = frozenset(id(item) for item in client_input) message_positions = tuple(index for index, item in enumerate(original_items) if id(item) in client_item_ids) if len(message_positions) == len(original_items): - return [*merged_input] + return [shape(message) for message in merged_input] if not message_positions: verbose_logger.warning( "Prompt management hook returned messages without Responses API input messages; merged messages were ignored" @@ -69,7 +90,7 @@ class ResponsesAPIRequestUtils: if corresponding_messages: merged_by_position: Final = dict(zip(message_positions, merged_input)) return [ - merged_by_position[index] if index in merged_by_position else item + shape(merged_by_position[index]) if index in merged_by_position else item for index, item in enumerate(original_items) ] @@ -82,14 +103,14 @@ class ResponsesAPIRequestUtils: for index, position in enumerate(message_positions) } trailing_items: Final = original_items[message_positions[-1] + 1 :] - return [item for merged in merged_input for item in (*prefixes.get(id(merged), ()), merged)] + list( + return [item for merged in merged_input for item in (*prefixes.get(id(merged), ()), shape(merged))] + list( trailing_items ) verbose_logger.warning( "Prompt management hook replaced Responses API messages; non-message input items were dropped" ) - return [*merged_input] + return [shape(message) for message in merged_input] @staticmethod def merge_client_forwarded_headers( diff --git a/litellm/types/integrations/anthropic_cache_control_hook.py b/litellm/types/integrations/anthropic_cache_control_hook.py index da9b26ebbd8..3ab0c02f28d 100644 --- a/litellm/types/integrations/anthropic_cache_control_hook.py +++ b/litellm/types/integrations/anthropic_cache_control_hook.py @@ -1,6 +1,6 @@ from typing import Literal -from typing_extensions import NotRequired, TypedDict +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.types.llms.openai import ChatCompletionCachedContent @@ -13,6 +13,7 @@ class CacheControlMessageInjectionPoint(TypedDict): index: int | str | None # Optional: target by specific index control: ChatCompletionCachedContent | None _litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran + _litellm_openai_dialect: NotRequired[ReadOnly[bool]] class CacheControlToolConfigInjectionPoint(TypedDict): @@ -21,6 +22,7 @@ class CacheControlToolConfigInjectionPoint(TypedDict): location: Literal["tool_config"] control: ChatCompletionCachedContent | None _litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran + _litellm_openai_dialect: NotRequired[ReadOnly[bool]] CacheControlInjectionPoint = CacheControlMessageInjectionPoint | CacheControlToolConfigInjectionPoint diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 5179210f942..43e1d3a4e11 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -9,6 +9,7 @@ from .openai import ( ChatCompletionCachedContent, ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock, + PromptCacheBreakpoint, ) @@ -201,6 +202,7 @@ class AnthropicMessagesTextParam(TypedDict, total=False): type: Required[Literal["text"]] text: Required[str] cache_control: dict | ChatCompletionCachedContent | None + prompt_cache_breakpoint: ReadOnly[PromptCacheBreakpoint] class AnthropicMessagesToolUseParam(TypedDict, total=False): @@ -261,6 +263,7 @@ class AnthropicMessagesImageParam(TypedDict, total=False): type: Required[Literal["image"]] source: Required[AnthropicContentParamSource | AnthropicContentParamSourceFileId | AnthropicContentParamSourceUrl] cache_control: dict | ChatCompletionCachedContent | None + prompt_cache_breakpoint: ReadOnly[PromptCacheBreakpoint] class CitationsObject(TypedDict): @@ -347,6 +350,7 @@ class AnthropicSystemMessageContent(TypedDict, total=False): type: str text: str cache_control: dict | ChatCompletionCachedContent | None + prompt_cache_breakpoint: ReadOnly[PromptCacheBreakpoint] class AnthropicMessagesSystemMessageParam(TypedDict, total=False): diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index edfc50c99f6..4c7f416ab8c 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -71,6 +71,7 @@ from pydantic import ( ) from typing_extensions import ( NotRequired, + ReadOnly, Required, TypedDict, override, @@ -510,6 +511,15 @@ class ChatCompletionCachedContent(TypedDict): ttl: NotRequired[Literal["5m", "1h"]] +class PromptCacheBreakpoint(TypedDict): + mode: ReadOnly[Literal["explicit"]] + + +class PromptCacheOptions(TypedDict, total=False): + mode: ReadOnly[Literal["implicit", "explicit"]] + ttl: ReadOnly[Literal["30m"]] + + class ChatCompletionThinkingBlock(TypedDict, total=False): type: Required[Literal["thinking"]] thinking: str @@ -1148,6 +1158,7 @@ class ResponsesAPIOptionalRequestParams(TypedDict, total=False): max_tool_calls: int | None prompt_cache_key: str | None prompt_cache_retention: str | None + prompt_cache_options: ReadOnly[PromptCacheOptions | None] stream_options: ResponsesAPIStreamOptions | None top_logprobs: int | None partial_images: int | None # Number of partial images to generate (1-3) for streaming image generation diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 96b9343353d..41210d18495 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -141,6 +141,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_tool_choice: bool | None supports_assistant_prefill: bool | None supports_prompt_caching: bool | None + supports_prompt_cache_breakpoint: ReadOnly[bool | None] supports_computer_use: bool | None supports_audio_input: bool | None supports_embedding_image_input: bool | None diff --git a/litellm/utils.py b/litellm/utils.py index d1b0cb882ac..867f7a93452 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2560,6 +2560,14 @@ def supports_prompt_caching(model: str, custom_llm_provider: str | None = None) ) +def supports_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool: + return _supports_factory( + model=model, + custom_llm_provider=custom_llm_provider, + key="supports_prompt_cache_breakpoint", + ) + + def supports_computer_use(model: str, custom_llm_provider: str | None = None) -> bool: """ Check if the given model supports computer use and return a boolean value. @@ -5473,6 +5481,7 @@ def _get_model_info_helper( supports_tool_choice=None, supports_assistant_prefill=None, supports_prompt_caching=None, + supports_prompt_cache_breakpoint=None, supports_computer_use=None, supports_pdf_input=None, ) @@ -5712,6 +5721,7 @@ def _get_model_info_helper( supports_tool_choice=_model_info.get("supports_tool_choice", None), supports_assistant_prefill=_model_info.get("supports_assistant_prefill", None), supports_prompt_caching=_model_info.get("supports_prompt_caching", None), + supports_prompt_cache_breakpoint=_model_info.get("supports_prompt_cache_breakpoint", None), supports_audio_input=_model_info.get("supports_audio_input", None), supports_audio_output=_model_info.get("supports_audio_output", None), supports_pdf_input=_model_info.get("supports_pdf_input", None), @@ -5846,6 +5856,7 @@ def get_model_info( supports_function_calling: Optional[bool] supports_tool_choice: Optional[bool] supports_prompt_caching: Optional[bool] + supports_prompt_cache_breakpoint: Optional[bool] supports_audio_input: Optional[bool] supports_audio_output: Optional[bool] supports_pdf_input: Optional[bool] diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 34c9f98077b..db6627bad58 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -25368,6 +25368,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -25430,6 +25431,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -25492,6 +25494,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -25554,6 +25557,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 82854a3b717..0991650d307 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -664,6 +664,9 @@ "supports_pdf_input": { "type": "boolean" }, + "supports_prompt_cache_breakpoint": { + "type": "boolean" + }, "supports_prompt_caching": { "type": "boolean" }, diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index cc43a424419..e92a368d24b 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -14,12 +14,22 @@ import pytest sys.path.insert(0, os.path.abspath("../..")) # Adds the parent directory to the system-path import litellm -from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook +from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + supports_openai_prompt_cache_breakpoint, +) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import StandardCallbackDynamicParams +@pytest.fixture(autouse=True) +def _no_openai_api_base_override(monkeypatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None) + + def _rendered_log_message(call): message = str(call.args[0]) values = call.args[1:] @@ -1996,3 +2006,769 @@ class TestAnthropicPromptCachingEnvVars: """An unparseable TTL must fall back to Anthropic's 5m default, never reach the provider verbatim.""" _, ttl = self._import_litellm_with_env({"LITELLM_ANTHROPIC_PROMPT_CACHING_TTL": value}) assert ttl is None + + +def _contains_key(value, key) -> bool: + if isinstance(value, dict): + return key in value or any(_contains_key(v, key) for v in value.values()) + if isinstance(value, list): + return any(_contains_key(v, key) for v in value) + return False + + +class TestOpenAIPromptCacheBreakpoint: + """OpenAI GPT-5.6+ targets get content-block `prompt_cache_breakpoint` markers and a + request-level `prompt_cache_options` instead of Anthropic `cache_control` (#37509).""" + + EXPLICIT = {"mode": "explicit"} + SYSTEM_POINT = [{"location": "message", "role": "system"}] + + @staticmethod + def _inject(messages, system, kwargs, model="openai/gpt-5.6", custom_llm_provider=None): + return AnthropicCacheControlHook.maybe_inject_cache_control( + copy.deepcopy(messages), + copy.deepcopy(system), + kwargs, + model=model, + custom_llm_provider=custom_llm_provider, + ) + + @staticmethod + def _chat(messages, params, model="openai/gpt-5.6"): + return AnthropicCacheControlHook().get_chat_completion_prompt( + model=model, + messages=copy.deepcopy(messages), + non_default_params=params, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + + @pytest.mark.parametrize( + "model,expected", + [ + ("gpt-5.6", True), + ("openai/gpt-5.6", True), + ("gpt-5.6-sol", True), + ("gpt-5.6-luna", True), + ("gpt-5.7", True), + ("gpt-6", True), + ("GPT-5.6", True), + ("gpt-5.5", False), + ("gpt-5", False), + ("gpt-5-chat-latest", False), + ("gpt-4.1", False), + ("o3", False), + ("claude-sonnet-4-5", False), + ], + ) + def test_model_support_truth_table(self, model, expected): + assert supports_openai_prompt_cache_breakpoint(model) is expected + + @pytest.mark.parametrize( + "model,provider,expected", + [ + ("openai/gpt-5.6", None, True), + ("gpt-5.6", None, True), + ("gpt-5.6", "openai", True), + ("gpt-5.6", "azure", False), + ("azure/gpt-5.6", None, False), + ("openai/gpt-4.1", None, False), + ("anthropic/claude-sonnet-4-5", None, False), + ("no-provider-can-route-this-model", None, False), + (None, "openai", False), + ], + ) + def test_dialect_resolution(self, model, provider, expected): + assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(model, provider) is expected + + def test_count_covers_both_marker_kinds(self): + message = { + "role": "user", + "cache_control": {"type": "ephemeral"}, + "content": [ + {"type": "text", "text": "a", "prompt_cache_breakpoint": self.EXPLICIT}, + {"type": "text", "text": "b", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "c"}, + ], + } + assert AnthropicCacheControlHook._count_cache_control_blocks(message) == 3 + + def test_v1_messages_string_system_gets_block_breakpoint(self): + kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + messages, system = self._inject([{"role": "user", "content": "hi"}], "sys", kwargs) + assert system == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}] + assert messages == [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + assert kwargs == {"prompt_cache_options": self.EXPLICIT} + assert not _contains_key(system, "cache_control") + + def test_v1_messages_list_system_marks_last_block_only(self): + kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + system = [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}] + _, result_system = self._inject([{"role": "user", "content": "hi"}], system, kwargs) + assert result_system == [ + {"type": "text", "text": "a"}, + {"type": "text", "text": "b", "prompt_cache_breakpoint": self.EXPLICIT}, + ] + assert kwargs["prompt_cache_options"] == self.EXPLICIT + + def test_v1_messages_targets_by_role(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "first"}, {"type": "text", "text": "second"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "reply"}]}, + {"role": "user", "content": "last"}, + ] + kwargs = {"cache_control_injection_points": [{"location": "message", "role": "user"}]} + result, _ = self._inject(messages, None, kwargs) + assert result[0]["content"] == [ + {"type": "text", "text": "first"}, + {"type": "text", "text": "second", "prompt_cache_breakpoint": self.EXPLICIT}, + ] + assert result[1] == messages[1] + assert result[2]["content"] == [{"type": "text", "text": "last", "prompt_cache_breakpoint": self.EXPLICIT}] + assert kwargs["prompt_cache_options"] == self.EXPLICIT + + def test_v1_messages_targets_by_index(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "first"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "reply"}]}, + {"role": "user", "content": [{"type": "text", "text": "last"}]}, + ] + kwargs = {"cache_control_injection_points": [{"location": "message", "index": -1}]} + result, _ = self._inject(messages, None, kwargs) + assert result[:2] == messages[:2] + assert result[2]["content"] == [{"type": "text", "text": "last", "prompt_cache_breakpoint": self.EXPLICIT}] + + def test_v1_messages_control_field_is_ignored(self): + ttl_control = {"type": "ephemeral", "ttl": "1h"} + kwargs = { + "cache_control_injection_points": [ + {"location": "message", "role": "system", "control": ttl_control}, + {"location": "message", "index": -1, "control": ttl_control}, + ] + } + messages, system = self._inject([{"role": "user", "content": "hi"}], "sys", kwargs) + assert system[0]["prompt_cache_breakpoint"] == self.EXPLICIT + assert messages[0]["content"][-1]["prompt_cache_breakpoint"] == self.EXPLICIT + assert not _contains_key(system, "cache_control") + assert not _contains_key(messages, "cache_control") + + def test_v1_messages_keeps_caller_prompt_cache_options(self): + caller_options = {"mode": "explicit", "ttl": "30m"} + kwargs = { + "cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT), + "prompt_cache_options": dict(caller_options), + } + _, system = self._inject([{"role": "user", "content": "hi"}], "sys", kwargs) + assert system[0]["prompt_cache_breakpoint"] == self.EXPLICIT + assert kwargs["prompt_cache_options"] == caller_options + + def test_v1_messages_no_prompt_cache_options_when_nothing_injected(self): + kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + messages, system = self._inject([{"role": "user", "content": "hi"}], None, kwargs) + assert system is None + assert "prompt_cache_options" not in kwargs + assert not _contains_key(messages, "prompt_cache_breakpoint") + + def test_v1_messages_anthropic_target_unchanged(self): + kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + _, system = self._inject( + [{"role": "user", "content": "hi"}], + "sys", + kwargs, + model="anthropic/claude-sonnet-4-5", + custom_llm_provider="anthropic", + ) + assert system == [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}] + assert kwargs == {} + + def test_v1_messages_older_openai_model_keeps_cache_control(self): + kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + _, system = self._inject([{"role": "user", "content": "hi"}], "sys", kwargs, model="openai/gpt-4.1") + assert system == [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}] + assert kwargs == {} + + def test_v1_messages_client_content_breakpoint_makes_configured_points_stand_down(self): + messages = [{"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]}] + kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + result, system = self._inject(messages, "sys", kwargs) + assert result == messages + assert system == "sys" + assert kwargs == {} + + def test_v1_messages_client_system_breakpoint_makes_configured_points_stand_down(self): + system = [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}] + messages = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + kwargs = {"cache_control_injection_points": [{"location": "message", "index": -1}]} + result, result_system = self._inject(messages, system, kwargs) + assert result == messages + assert result_system == system + assert kwargs == {} + + def test_chat_system_string_wrapped_with_block_breakpoint(self): + params = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + messages = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}] + _, processed, returned = self._chat(messages, params) + assert processed[0] == { + "role": "system", + "content": [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}], + } + assert processed[1] == {"role": "user", "content": "hi"} + assert returned is params + assert returned == {"prompt_cache_options": self.EXPLICIT} + + def test_chat_list_content_marks_last_block(self): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, + ], + } + ] + params = {"cache_control_injection_points": [{"location": "message", "index": -1}]} + _, processed, _ = self._chat(messages, params) + assert processed[0]["content"] == [ + {"type": "text", "text": "look"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/a.png"}, + "prompt_cache_breakpoint": self.EXPLICIT, + }, + ] + assert params["prompt_cache_options"] == self.EXPLICIT + + def test_chat_unprefixed_model_resolves_to_openai(self): + params = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + _, processed, _ = self._chat([{"role": "system", "content": "sys"}], params, model="gpt-5.6") + assert processed[0]["content"] == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}] + assert params["prompt_cache_options"] == self.EXPLICIT + + def test_chat_keeps_caller_prompt_cache_options(self): + params = { + "cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT), + "prompt_cache_options": {"mode": "implicit"}, + } + self._chat([{"role": "system", "content": "sys"}], params) + assert params["prompt_cache_options"] == {"mode": "implicit"} + + def test_chat_no_prompt_cache_options_when_nothing_injected(self): + params = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + messages = [{"role": "user", "content": "hi"}] + _, processed, _ = self._chat(messages, params) + assert processed == messages + assert params == {} + + @pytest.mark.parametrize("model", ["openai/gpt-4.1", "anthropic/claude-sonnet-4-5"]) + def test_chat_other_targets_keep_message_level_cache_control(self, model): + params = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + _, processed, _ = self._chat([{"role": "system", "content": "sys"}], params, model=model) + assert processed[0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}} + assert params == {} + + def test_chat_client_breakpoint_makes_seeded_points_stand_down(self): + params = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=[ + {"role": "system", "content": "sys"}, + {"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]}, + ], + model="openai/gpt-5.6", + custom_llm_provider="openai", + ) + assert params == {} + + def test_cap_counts_client_breakpoints_of_both_kinds(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "a", "prompt_cache_breakpoint": self.EXPLICIT}]}, + {"role": "user", "content": [{"type": "text", "text": "b", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": [{"type": "text", "text": "c", "prompt_cache_breakpoint": self.EXPLICIT}]}, + {"role": "user", "content": "d"}, + {"role": "user", "content": "e"}, + ] + result = AnthropicCacheControlHook._apply_message_injections( + points=[{"location": "message", "role": "user"}], + messages=copy.deepcopy(messages), + max_blocks=4, + openai_dialect=True, + ) + assert result[:3] == messages[:3] + assert result[3]["content"] == [{"type": "text", "text": "d", "prompt_cache_breakpoint": self.EXPLICIT}] + assert result[4] == {"role": "user", "content": "e"} + + +class TestOpenAIPromptCacheBreakpointPlacementRules: + """OpenAI dialect only marks blocks OpenAI (and the /v1/messages bridges) can carry (#37509).""" + + EXPLICIT = {"mode": "explicit"} + + def _chat(self, messages, points, model="openai/gpt-5.6"): + params = {"cache_control_injection_points": copy.deepcopy(points)} + _, out, params = AnthropicCacheControlHook().get_chat_completion_prompt( + model=model, + messages=copy.deepcopy(messages), + non_default_params=params, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + return out, params + + def test_assistant_message_is_never_marked_on_chat_path(self): + messages = [{"role": "user", "content": "q"}, {"role": "assistant", "content": "a"}] + out, params = self._chat(messages, [{"location": "message", "role": "assistant"}]) + assert out == messages + assert "prompt_cache_options" not in params + + def test_tool_message_text_is_marked_on_chat_path(self): + messages = [ + {"role": "user", "content": "weather?"}, + {"role": "assistant", "content": None, "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "w", "arguments": "{}"}}]}, + {"role": "tool", "tool_call_id": "c1", "content": "sunny"}, + ] + out, params = self._chat(messages, [{"location": "message", "index": -1}]) + assert out[2]["content"] == [{"type": "text", "text": "sunny", "prompt_cache_breakpoint": self.EXPLICIT}] + assert params["prompt_cache_options"] == self.EXPLICIT + + def test_tool_result_only_turn_is_skipped_on_v1_messages(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "q"}]}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "t1", "name": "w", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "sunny"}]}, + ] + kwargs = {"cache_control_injection_points": [{"location": "message", "index": -1}]} + out, system = AnthropicCacheControlHook.maybe_inject_cache_control( + copy.deepcopy(messages), None, kwargs, model="openai/gpt-5.6" + ) + assert out == messages + assert system is None + assert "prompt_cache_options" not in kwargs + + def test_assistant_turn_is_skipped_on_v1_messages(self): + messages = [ + {"role": "user", "content": [{"type": "text", "text": "q"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "a"}]}, + ] + kwargs = {"cache_control_injection_points": [{"location": "message", "role": "assistant"}]} + out, _ = AnthropicCacheControlHook.maybe_inject_cache_control( + copy.deepcopy(messages), None, kwargs, model="openai/gpt-5.6" + ) + assert out == messages + assert "prompt_cache_options" not in kwargs + + def test_text_after_tool_result_is_marked(self): + messages = [ + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": "sunny"}, + {"type": "text", "text": "thanks"}, + ], + } + ] + kwargs = {"cache_control_injection_points": [{"location": "message", "index": -1}]} + out, _ = AnthropicCacheControlHook.maybe_inject_cache_control(messages, None, kwargs, model="openai/gpt-5.6") + assert out[0]["content"] == [ + {"type": "tool_result", "tool_use_id": "t1", "content": "sunny"}, + {"type": "text", "text": "thanks", "prompt_cache_breakpoint": self.EXPLICIT}, + ] + assert kwargs["prompt_cache_options"] == self.EXPLICIT + + def test_marker_walks_back_to_last_eligible_block(self): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "read this"}, + {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "doc"}}, + ], + } + ] + kwargs = {"cache_control_injection_points": [{"location": "message", "index": -1}]} + out, _ = AnthropicCacheControlHook.maybe_inject_cache_control(messages, None, kwargs, model="openai/gpt-5.6") + assert out[0]["content"][0] == {"type": "text", "text": "read this", "prompt_cache_breakpoint": self.EXPLICIT} + assert "prompt_cache_breakpoint" not in out[0]["content"][1] + + def test_skipped_block_does_not_consume_a_slot(self): + messages = [{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t0", "content": "r"}]}] + [ + {"role": "user", "content": [{"type": "text", "text": f"m{i}"}]} for i in range(4) + ] + kwargs = {"cache_control_injection_points": [{"location": "message", "role": "user"}]} + out, _ = AnthropicCacheControlHook.maybe_inject_cache_control(messages, None, kwargs, model="openai/gpt-5.6") + assert "prompt_cache_breakpoint" not in out[0]["content"][0] + assert all(msg["content"][0]["prompt_cache_breakpoint"] == self.EXPLICIT for msg in out[1:]) + + +class TestChatPathProviderStamp: + """The chat path learns the dialect decision (provider, api_base, opt-in) through the seeded points (#37509).""" + + POINTS = [{"location": "message", "role": "system"}] + MESSAGES = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}] + ANTHROPIC_STYLE = {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}} + OPENAI_STYLE = [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}] + CUSTOM_API_BASE = "http://127.0.0.1:9/v1" + + def _seed_and_run(self, model, custom_llm_provider, api_base=None, prompt_cache_options=None): + params = {"cache_control_injection_points": copy.deepcopy(self.POINTS)} + if prompt_cache_options is not None: + params["prompt_cache_options"] = prompt_cache_options + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=copy.deepcopy(self.MESSAGES), + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + ) + return self._run(params, model) + + def _run(self, params, model): + _, out, params = AnthropicCacheControlHook().get_chat_completion_prompt( + model=model, + messages=copy.deepcopy(self.MESSAGES), + non_default_params=params, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + return out, params + + def test_openai_compatible_provider_keeps_anthropic_style_markers(self): + out, params = self._seed_and_run("gpt-5.6", "hosted_vllm") + assert out[0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}} + assert "prompt_cache_options" not in params + + def test_explicit_openai_provider_uses_openai_dialect(self): + out, params = self._seed_and_run("gpt-5.6", "openai") + assert out[0]["content"] == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}] + assert params["prompt_cache_options"] == {"mode": "explicit"} + + def test_bare_gpt_model_without_provider_resolves_to_openai(self): + out, params = self._seed_and_run("gpt-5.6", None) + assert out[0]["content"] == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}] + assert params["prompt_cache_options"] == {"mode": "explicit"} + + def test_points_keep_identity_for_models_below_gpt_5_6(self): + points = copy.deepcopy(self.POINTS) + params = {"cache_control_injection_points": points} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=copy.deepcopy(self.MESSAGES), + model="anthropic/claude-sonnet-4-5", + custom_llm_provider="anthropic", + ) + assert params["cache_control_injection_points"] is points + + def test_provider_lookup_skipped_for_models_below_gpt_5_6(self): + from unittest.mock import patch + + with patch.object(AnthropicCacheControlHook, "_resolve_provider") as resolve: + assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("gpt-4.1", None) is False + assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("my-custom-model", None) is False + resolve.assert_not_called() + + def test_litellm_proxy_target_keeps_anthropic_style_markers(self): + out, params = self._seed_and_run("litellm_proxy/gpt-5.6", None) + assert out[0] == self.ANTHROPIC_STYLE + assert "prompt_cache_options" not in params + + def test_custom_api_base_keeps_anthropic_style_markers(self): + out, params = self._seed_and_run("gpt-5.6", None, api_base=self.CUSTOM_API_BASE) + assert out[0] == self.ANTHROPIC_STYLE + assert "prompt_cache_options" not in params + + def test_custom_api_base_opts_in_through_prompt_cache_options(self): + out, params = self._seed_and_run( + "gpt-5.6", None, api_base=self.CUSTOM_API_BASE, prompt_cache_options={"mode": "explicit"} + ) + assert out[0]["content"] == self.OPENAI_STYLE + assert params["prompt_cache_options"] == {"mode": "explicit"} + + def test_regional_openai_api_base_uses_openai_dialect(self): + out, params = self._seed_and_run("gpt-5.6", None, api_base="https://eu.api.openai.com/v1") + assert out[0]["content"] == self.OPENAI_STYLE + assert params["prompt_cache_options"] == {"mode": "explicit"} + + @pytest.mark.parametrize("env_var", ["OPENAI_BASE_URL", "OPENAI_API_BASE"]) + def test_env_api_base_override_keeps_anthropic_style_markers(self, monkeypatch, env_var): + monkeypatch.setenv(env_var, self.CUSTOM_API_BASE) + out, params = self._seed_and_run("gpt-5.6", None) + assert out[0] == self.ANTHROPIC_STYLE + assert "prompt_cache_options" not in params + + def test_global_litellm_api_base_keeps_anthropic_style_markers(self, monkeypatch): + monkeypatch.setattr(litellm, "api_base", self.CUSTOM_API_BASE) + out, params = self._seed_and_run("gpt-5.6", None) + assert out[0] == self.ANTHROPIC_STYLE + assert "prompt_cache_options" not in params + + def test_request_api_base_wins_over_env_override(self, monkeypatch): + monkeypatch.setenv("OPENAI_BASE_URL", self.CUSTOM_API_BASE) + out, params = self._seed_and_run("gpt-5.6", None, api_base="https://api.openai.com/v1") + assert out[0]["content"] == self.OPENAI_STYLE + assert params["prompt_cache_options"] == {"mode": "explicit"} + + @pytest.mark.parametrize( + "api_base,expected", + [(None, True), ("http://127.0.0.1:9/v1", False), ("https://eu.api.openai.com/v1", True)], + ) + def test_seed_stamps_the_dialect_decision(self, api_base, expected): + params = {"cache_control_injection_points": copy.deepcopy(self.POINTS)} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=copy.deepcopy(self.MESSAGES), + model="gpt-5.6", + custom_llm_provider=None, + api_base=api_base, + ) + assert params["cache_control_injection_points"][0]["_litellm_openai_dialect"] is expected + + def test_stamp_is_authoritative_over_request_params(self): + points = [{**self.POINTS[0], "_litellm_openai_dialect": False}] + out, params = self._run({"cache_control_injection_points": points, "custom_llm_provider": "openai"}, "gpt-5.6") + assert out[0] == self.ANTHROPIC_STYLE + assert "prompt_cache_options" not in params + + def test_unstamped_points_read_api_base_from_request_params(self): + params = {"cache_control_injection_points": copy.deepcopy(self.POINTS), "api_base": self.CUSTOM_API_BASE} + out, params = self._run(params, "gpt-5.6") + assert out[0] == self.ANTHROPIC_STYLE + assert "prompt_cache_options" not in params + + def test_unstamped_points_read_prompt_cache_options_from_request_params(self): + params = { + "cache_control_injection_points": copy.deepcopy(self.POINTS), + "api_base": self.CUSTOM_API_BASE, + "prompt_cache_options": {"mode": "explicit"}, + } + out, params = self._run(params, "gpt-5.6") + assert out[0]["content"] == self.OPENAI_STYLE + assert params["prompt_cache_options"] == {"mode": "explicit"} + + +class TestClientBreakpointsCountedOnce: + def test_client_message_breakpoints_are_not_double_counted(self): + messages = [{"role": "user", "content": [{"type": "text", "text": "m0", "cache_control": {"type": "ephemeral"}}]}] + [ + {"role": "user", "content": [{"type": "text", "text": f"m{i}"}]} for i in range(1, 4) + ] + out, system, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request( + messages=messages, + system="sys", + injection_points=[ + {"location": "message", "role": "system"}, + {"location": "message", "index": -1}, + {"location": "message", "index": -2}, + {"location": "message", "index": -3}, + ], + ) + marked = [msg["content"][0].get("cache_control") is not None for msg in out] + assert marked == [True, False, True, True] + assert system[0]["cache_control"] == {"type": "ephemeral"} + + +class TestResponsesInputPartsEligible: + """Responses API input parts can carry prompt_cache_breakpoint on GPT-5.6+ (#37509).""" + + EXPLICIT = {"mode": "explicit"} + + def _chat(self, messages, points, model="openai/gpt-5.6"): + params = {"cache_control_injection_points": copy.deepcopy(points)} + _, out, params = AnthropicCacheControlHook().get_chat_completion_prompt( + model=model, + messages=copy.deepcopy(messages), + non_default_params=params, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + return out, params + + def test_marker_lands_on_last_input_text_part(self): + messages = [ + { + "role": "user", + "content": [{"type": "input_text", "text": "first"}, {"type": "input_text", "text": "second"}], + } + ] + out, params = self._chat(messages, [{"location": "message", "index": -1}]) + assert out[0]["content"][0] == {"type": "input_text", "text": "first"} + assert out[0]["content"][1] == { + "type": "input_text", + "text": "second", + "prompt_cache_breakpoint": self.EXPLICIT, + } + assert params["prompt_cache_options"] == self.EXPLICIT + + @pytest.mark.parametrize( + "part", + [ + {"type": "input_image", "image_url": "https://example.com/a.png"}, + {"type": "input_file", "file_id": "file_1"}, + ], + ) + def test_input_image_and_input_file_parts_are_eligible(self, part): + out, params = self._chat([{"role": "user", "content": [part]}], [{"location": "message", "index": -1}]) + assert out[0]["content"][0] == {**part, "prompt_cache_breakpoint": self.EXPLICIT} + assert params["prompt_cache_options"] == self.EXPLICIT + + +class TestMessagesPathApiBaseGate: + """/v1/messages only speaks the OpenAI dialect when the request really targets api.openai.com (#37509).""" + + EXPLICIT = {"mode": "explicit"} + USER_POINT = [{"location": "message", "role": "user"}] + MESSAGES = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + CUSTOM_API_BASE = "http://127.0.0.1:9/v1" + CACHE_CONTROL_BLOCK = {"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}} + BREAKPOINT_BLOCK = {"type": "text", "text": "hi", "prompt_cache_breakpoint": {"mode": "explicit"}} + + def _inject(self, model, api_base=None, prompt_cache_options=None, custom_llm_provider=None): + kwargs = {"cache_control_injection_points": copy.deepcopy(self.USER_POINT)} + if prompt_cache_options is not None: + kwargs["prompt_cache_options"] = prompt_cache_options + out, _ = AnthropicCacheControlHook.maybe_inject_cache_control( + copy.deepcopy(self.MESSAGES), + None, + kwargs, + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + ) + return out[0]["content"][0], kwargs + + def test_litellm_proxy_target_keeps_cache_control(self): + block, kwargs = self._inject("gpt-5.6", api_base=self.CUSTOM_API_BASE, custom_llm_provider="litellm_proxy") + assert block == self.CACHE_CONTROL_BLOCK + assert "prompt_cache_options" not in kwargs + + def test_custom_api_base_keeps_cache_control(self): + block, kwargs = self._inject("gpt-5.6", api_base=self.CUSTOM_API_BASE) + assert block == self.CACHE_CONTROL_BLOCK + assert "prompt_cache_options" not in kwargs + + def test_custom_api_base_opts_in_through_prompt_cache_options(self): + block, kwargs = self._inject("gpt-5.6", api_base=self.CUSTOM_API_BASE, prompt_cache_options=self.EXPLICIT) + assert block == self.BREAKPOINT_BLOCK + assert kwargs["prompt_cache_options"] == self.EXPLICIT + + def test_regional_openai_api_base_uses_openai_dialect(self): + block, kwargs = self._inject("gpt-5.6", api_base="https://eu.api.openai.com/v1") + assert block == self.BREAKPOINT_BLOCK + assert kwargs["prompt_cache_options"] == self.EXPLICIT + + def test_default_api_base_uses_openai_dialect(self): + block, kwargs = self._inject("openai/gpt-5.6") + assert block == self.BREAKPOINT_BLOCK + assert kwargs["prompt_cache_options"] == self.EXPLICIT + + +class TestToolConfigSlotInOpenAIDialect: + """OpenAI has no tool_config cache block, so the dialect does not hold a slot for one (#37509).""" + + EXPLICIT = {"mode": "explicit"} + MESSAGES = [{"role": "user", "content": [{"type": "text", "text": f"m{i}"}]} for i in range(4)] + POINTS = [{"location": "message", "index": i} for i in range(4)] + [{"location": "tool_config"}] + + def test_chat_path_marks_all_four_messages(self): + params = {"cache_control_injection_points": copy.deepcopy(self.POINTS)} + _, out, params = AnthropicCacheControlHook().get_chat_completion_prompt( + model="openai/gpt-5.6", + messages=copy.deepcopy(self.MESSAGES), + non_default_params=params, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + assert [msg["content"][0].get("prompt_cache_breakpoint") for msg in out] == [self.EXPLICIT] * 4 + assert params["prompt_cache_options"] == self.EXPLICIT + + def test_messages_path_marks_all_four_messages(self): + out, _, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request( + copy.deepcopy(self.MESSAGES), None, copy.deepcopy(self.POINTS), openai_dialect=True + ) + assert [msg["content"][0].get("prompt_cache_breakpoint") for msg in out] == [self.EXPLICIT] * 4 + + def test_anthropic_dialect_still_reserves_the_tool_config_slot(self): + out, _, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request( + copy.deepcopy(self.MESSAGES), None, copy.deepcopy(self.POINTS) + ) + assert sum(msg["content"][0].get("cache_control") is not None for msg in out) == 3 + + +class TestPromptCacheBreakpointCapability: + """Eligibility comes from the model map's supports_prompt_cache_breakpoint flag when the entry carries one, + with the GPT version rule for unlisted models and for entries the published map has not flagged yet (#37509).""" + + @pytest.fixture(autouse=True) + def _bundled_model_map(self, monkeypatch): + bundled = os.path.join(os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json") + with open(bundled) as handle: + monkeypatch.setattr(litellm, "model_cost", json.load(handle)) + litellm.utils._cached_get_model_info_helper.cache_clear() + yield + litellm.utils._cached_get_model_info_helper.cache_clear() + + def test_public_helper_reads_the_model_map(self): + from litellm.utils import supports_prompt_cache_breakpoint + + assert supports_prompt_cache_breakpoint("gpt-5.6") is True + assert supports_prompt_cache_breakpoint("openai/gpt-5.6-sol") is True + assert supports_prompt_cache_breakpoint("gpt-5.6", custom_llm_provider="openai") is True + assert supports_prompt_cache_breakpoint("gpt-4.1") is False + + @pytest.mark.parametrize("model", ["gpt-5.6", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"]) + def test_model_map_flags_every_openai_gpt_5_6_entry(self, model): + assert litellm.model_cost[model]["litellm_provider"] == "openai" + assert litellm.model_cost[model]["supports_prompt_cache_breakpoint"] is True + + def test_listed_model_uses_the_model_map_flag(self, monkeypatch): + flagged = {**litellm.model_cost["gpt-4.1"], "supports_prompt_cache_breakpoint": True} + monkeypatch.setitem(litellm.model_cost, "gpt-4.1", flagged) + assert supports_openai_prompt_cache_breakpoint("gpt-4.1") is True + + def test_listed_gpt_5_6_without_the_flag_falls_back_to_the_version_rule(self, monkeypatch): + unflagged = {k: v for k, v in litellm.model_cost["gpt-5.6"].items() if k != "supports_prompt_cache_breakpoint"} + monkeypatch.setitem(litellm.model_cost, "gpt-5.6", unflagged) + assert supports_openai_prompt_cache_breakpoint("gpt-5.6") is True + assert supports_openai_prompt_cache_breakpoint("openai/gpt-5.6") is True + + def test_listed_model_flagged_false_is_not_eligible(self, monkeypatch): + monkeypatch.setitem( + litellm.model_cost, "gpt-5.6", {**litellm.model_cost["gpt-5.6"], "supports_prompt_cache_breakpoint": False} + ) + assert supports_openai_prompt_cache_breakpoint("gpt-5.6") is False + + def test_listed_gpt_model_without_the_flag_follows_the_version_rule(self): + assert "supports_prompt_cache_breakpoint" not in litellm.model_cost["gpt-4.1"] + assert supports_openai_prompt_cache_breakpoint("gpt-4.1") is False + + def test_published_map_without_the_flag_still_injects_on_gpt_5_6(self, monkeypatch): + unflagged = {k: v for k, v in litellm.model_cost["gpt-5.6"].items() if k != "supports_prompt_cache_breakpoint"} + monkeypatch.setitem(litellm.model_cost, "gpt-5.6", unflagged) + points = [{"location": "message", "role": "system"}] + + _, chat_messages, chat_params = AnthropicCacheControlHook().get_chat_completion_prompt( + model="openai/gpt-5.6", + messages=[{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}], + non_default_params={"cache_control_injection_points": copy.deepcopy(points)}, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + assert chat_messages[0]["content"] == [ + {"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}} + ] + assert chat_params["prompt_cache_options"] == {"mode": "explicit"} + + kwargs = {"cache_control_injection_points": copy.deepcopy(points)} + _, system = AnthropicCacheControlHook.maybe_inject_cache_control( + [{"role": "user", "content": "hi"}], "sys", kwargs, model="gpt-5.6", custom_llm_provider="openai" + ) + assert system == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}] + assert kwargs == {"prompt_cache_options": {"mode": "explicit"}} + + @pytest.mark.parametrize("model,expected", [("gpt-5.6-2026-01-01", True), ("gpt-5.5-preview-unlisted", False)]) + def test_unlisted_model_falls_back_to_the_version_rule(self, model, expected): + assert model not in litellm.model_cost + assert supports_openai_prompt_cache_breakpoint(model) is expected diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 1185893d428..abb976c3560 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -3741,3 +3741,59 @@ def test_tool_result_plain_text_unchanged_by_openai_transform(): assert len(tool_messages) == 1 assert tool_messages[0]["content"] == "42 files found" assert _image_urls_in_user_messages(result) == [] + + +def test_translate_anthropic_to_openai_carries_prompt_cache_breakpoint_on_system_and_user_blocks(): + explicit = {"mode": "explicit"} + openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + anthropic_message_request={ + "model": "gpt-5.6", + "max_tokens": 64, + "system": [{"type": "text", "text": "sys", "prompt_cache_breakpoint": explicit}], + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "hi", "prompt_cache_breakpoint": explicit}, + { + "type": "image", + "source": {"type": "url", "url": "https://example.com/a.png"}, + "prompt_cache_breakpoint": explicit, + }, + ], + } + ], + } + ) + assert openai_request["messages"][0] == { + "role": "system", + "content": [{"type": "text", "text": "sys", "prompt_cache_breakpoint": explicit}], + } + user_content = openai_request["messages"][1]["content"] + assert user_content[0] == {"type": "text", "text": "hi", "prompt_cache_breakpoint": explicit} + assert user_content[1]["type"] == "image_url" + assert user_content[1]["prompt_cache_breakpoint"] == explicit + + +def test_translate_anthropic_to_openai_without_prompt_cache_breakpoint_adds_nothing(): + openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + anthropic_message_request={ + "model": "gpt-5.6", + "max_tokens": 64, + "system": [{"type": "text", "text": "sys"}], + "messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}], + } + ) + assert openai_request["messages"][0] == {"role": "system", "content": [{"type": "text", "text": "sys"}]} + assert openai_request["messages"][1]["content"] == [{"type": "text", "text": "hi"}] + + +def test_translate_anthropic_messages_to_openai_carries_midturn_system_prompt_cache_breakpoint(): + explicit = {"mode": "explicit"} + result = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai( + messages=[{"role": "system", "content": [{"type": "text", "text": "fix", "prompt_cache_breakpoint": explicit}]}], + model="gpt-5.6", + ) + assert result == [ + {"role": "system", "content": [{"type": "text", "text": "fix", "prompt_cache_breakpoint": explicit}]} + ] diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py index 615dc5cfebc..a944afc6152 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py @@ -230,3 +230,10 @@ class TestEmptyExtraKwargsPath: # dict-like result. completion_kwargs = result[0] if isinstance(result, tuple) else result assert isinstance(completion_kwargs, dict) + + +class TestPromptCacheOptionsForwarded: + def test_prompt_cache_options_reaches_completion_kwargs(self): + result = _call_prepare(extra_kwargs={"prompt_cache_options": {"mode": "explicit"}}, model="gpt-5.6") + completion_kwargs = result[0] if isinstance(result, tuple) else result + assert completion_kwargs["prompt_cache_options"] == {"mode": "explicit"} diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 876213eda3f..8228d41cf82 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -1413,3 +1413,155 @@ class TestToolResultImages: outputs = [item for item in items if item.get("type") == "function_call_output"] assert outputs[0]["output"] == "screenshot saved" assert self._input_images(items) == [] + + +def _contains_key(value, key) -> bool: + if isinstance(value, dict): + return key in value or any(_contains_key(v, key) for v in value.values()) + if isinstance(value, list): + return any(_contains_key(v, key) for v in value) + return False + + +class TestPromptCacheBreakpointToResponses: + """OpenAI `prompt_cache_breakpoint` markers ride through the /v1/messages -> Responses bridge (#37509).""" + + EXPLICIT = {"mode": "explicit"} + + def test_system_with_breakpoint_becomes_leading_developer_message(self): + request = _make_request( + model="openai/gpt-5.6", + system=[ + {"type": "text", "text": "Be concise."}, + {"type": "text", "text": "Be helpful.", "prompt_cache_breakpoint": self.EXPLICIT}, + ], + ) + kwargs = _ADAPTER.translate_request(request) + assert "instructions" not in kwargs + assert kwargs["input"] == [ + { + "type": "message", + "role": "developer", + "content": [ + {"type": "input_text", "text": "Be concise."}, + {"type": "input_text", "text": "Be helpful.", "prompt_cache_breakpoint": self.EXPLICIT}, + ], + }, + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hello"}]}, + ] + + def test_system_without_breakpoint_still_becomes_instructions(self): + request = _make_request(system=[{"type": "text", "text": "Be concise."}, {"type": "text", "text": "Be helpful."}]) + kwargs = _ADAPTER.translate_request(request) + assert kwargs["instructions"] == "Be concise.\nBe helpful." + assert kwargs["input"] == [ + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hello"}]} + ] + + def test_system_string_still_becomes_instructions(self): + kwargs = _ADAPTER.translate_request(_make_request(system="Be concise.")) + assert kwargs["instructions"] == "Be concise." + assert kwargs["input"][0]["role"] == "user" + + def test_system_with_breakpoint_skips_non_text_blocks(self): + request = _make_request( + system=[ + {"type": "image", "source": {"type": "url", "url": "https://example.com/a.png"}}, + {"type": "text", "text": "only", "prompt_cache_breakpoint": self.EXPLICIT}, + ] + ) + kwargs = _ADAPTER.translate_request(request) + assert kwargs["input"][0] == { + "type": "message", + "role": "developer", + "content": [{"type": "input_text", "text": "only", "prompt_cache_breakpoint": self.EXPLICIT}], + } + + def test_user_text_and_image_blocks_carry_breakpoint(self): + items = _ADAPTER.translate_messages_to_responses_input( + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "look", "prompt_cache_breakpoint": self.EXPLICIT}, + { + "type": "image", + "source": {"type": "url", "url": "https://example.com/a.png"}, + "prompt_cache_breakpoint": self.EXPLICIT, + }, + ], + } + ] + ) + assert items == [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "look", "prompt_cache_breakpoint": self.EXPLICIT}, + { + "type": "input_image", + "image_url": "https://example.com/a.png", + "prompt_cache_breakpoint": self.EXPLICIT, + }, + ], + } + ] + + def test_user_blocks_without_breakpoint_are_unchanged(self): + items = _ADAPTER.translate_messages_to_responses_input( + [{"role": "user", "content": [{"type": "text", "text": "look"}]}] + ) + assert items == [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "look"}]}] + + def test_midturn_system_block_carries_breakpoint(self): + items = _ADAPTER.translate_messages_to_responses_input( + [{"role": "system", "content": [{"type": "text", "text": "fix", "prompt_cache_breakpoint": self.EXPLICIT}]}] + ) + assert items == [ + { + "type": "message", + "role": "system", + "content": [{"type": "input_text", "text": "fix", "prompt_cache_breakpoint": self.EXPLICIT}], + } + ] + + def test_assistant_and_tool_result_blocks_drop_breakpoint(self): + items = _ADAPTER.translate_messages_to_responses_input( + [ + {"role": "user", "content": [{"type": "text", "text": "q"}]}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "a", "prompt_cache_breakpoint": self.EXPLICIT}, + {"type": "tool_use", "id": "toolu_01", "name": "t", "input": {}}, + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": "r", + "prompt_cache_breakpoint": self.EXPLICIT, + } + ], + }, + ] + ) + assert len(items) == 4 + assert not _contains_key(items, "prompt_cache_breakpoint") + + def test_prompt_cache_options_forwarded_to_responses_kwargs(self): + from litellm.llms.anthropic.experimental_pass_through.responses_adapters.handler import ( + _build_responses_kwargs, + ) + + kwargs = _build_responses_kwargs( + max_tokens=16, + messages=[{"role": "user", "content": "hi"}], + model="openai/gpt-5.6", + extra_kwargs={"prompt_cache_options": {"mode": "explicit"}}, + ) + assert kwargs["prompt_cache_options"] == {"mode": "explicit"} diff --git a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py index 41c2e215c60..45f1bdbfa85 100644 --- a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py @@ -871,3 +871,48 @@ class TestToolMessageImageHoisting: result = request["messages"] assert [m.get("role") for m in result] == ["user", "assistant", "tool", "user"] assert result[3]["content"] == self.HOISTED_USER_CONTENT + + +class TestOpenAIPromptCacheBreakpointChatPath: + """Chat-path shape for OpenAI explicit prompt caching (#37509).""" + + EXPLICIT = {"mode": "explicit"} + + def test_prompt_cache_options_travels_in_extra_body(self): + optional_params = litellm.get_optional_params( + model="gpt-5.6", custom_llm_provider="openai", prompt_cache_options=self.EXPLICIT + ) + assert optional_params["extra_body"]["prompt_cache_options"] == self.EXPLICIT + assert "prompt_cache_options" not in optional_params + + def test_prompt_cache_options_is_not_a_supported_chat_param(self): + assert "prompt_cache_options" not in OpenAIGPT5Config().get_supported_openai_params("gpt-5.6") + assert "prompt_cache_options" not in OpenAIGPTConfig().get_supported_openai_params("gpt-4.1") + + def test_block_breakpoint_survives_transform_request(self): + request = OpenAIGPT5Config().transform_request( + model="gpt-5.6", + messages=[ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "sys", + "prompt_cache_breakpoint": self.EXPLICIT, + "cache_control": {"type": "ephemeral"}, + } + ], + }, + {"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]}, + ], + optional_params={"extra_body": {"prompt_cache_options": self.EXPLICIT}}, + litellm_params={}, + headers={}, + ) + assert request["messages"][0]["content"] == [ + {"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT} + ] + assert request["messages"][1]["content"] == [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}] + assert request["extra_body"] == {"prompt_cache_options": self.EXPLICIT} + assert "prompt_cache_options" not in request diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py index 151c51f1ca0..13b96dc9943 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -1540,3 +1540,61 @@ class TestPhaseParameter: assert validated[0]["phase"] == "commentary" assert validated[1]["phase"] == "final_answer" assert "phase" not in validated[2] + + +class TestPromptCacheOptionsOnResponsesPath: + """`prompt_cache_options` and block-level `prompt_cache_breakpoint` survive the Responses transformation (#37509).""" + + OPTIONS = {"mode": "explicit", "ttl": "30m"} + + def test_prompt_cache_options_survives_optional_param_filter(self): + from litellm.responses.utils import ResponsesAPIRequestUtils + + result = ResponsesAPIRequestUtils.get_requested_response_api_optional_param( + {"prompt_cache_options": dict(self.OPTIONS), "temperature": 0.2, "not_a_responses_param": 1} + ) + assert result["prompt_cache_options"] == self.OPTIONS + assert result["temperature"] == 0.2 + assert "not_a_responses_param" not in result + + def test_prompt_cache_options_reaches_transformed_request(self): + config = OpenAIResponsesAPIConfig() + mapped = config.map_openai_params( + response_api_optional_params={"prompt_cache_options": dict(self.OPTIONS)}, + model="gpt-5.6", + drop_params=False, + ) + result = config.transform_responses_api_request( + model="gpt-5.6", + input="hi", + response_api_optional_request_params=mapped, + litellm_params={}, + headers={}, + ) + assert result["prompt_cache_options"] == self.OPTIONS + + def test_prompt_cache_breakpoint_survives_cache_control_strip(self): + result = OpenAIResponsesAPIConfig().transform_responses_api_request( + model="gpt-5.6", + input=[ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hi", + "prompt_cache_breakpoint": {"mode": "explicit"}, + "cache_control": {"type": "ephemeral"}, + } + ], + } + ], + response_api_optional_request_params={}, + litellm_params={}, + headers={}, + ) + assert result["input"][0]["content"][0] == { + "type": "input_text", + "text": "hi", + "prompt_cache_breakpoint": {"mode": "explicit"}, + } diff --git a/tests/test_litellm/responses/test_responses_api_request_body.py b/tests/test_litellm/responses/test_responses_api_request_body.py index ea5eb3afe9a..14eb9ab6e12 100644 --- a/tests/test_litellm/responses/test_responses_api_request_body.py +++ b/tests/test_litellm/responses/test_responses_api_request_body.py @@ -4,6 +4,7 @@ over the wire and surface provider errors correctly. Expected JSON bodies are st in expected_responses_api_request/. """ +import copy import json from pathlib import Path from unittest.mock import AsyncMock, patch @@ -422,3 +423,178 @@ async def test_aresponses_websocket_strips_responses_routing_prefix_from_openai_ mock_ws.assert_awaited_once() assert mock_ws.call_args.kwargs["model"] == "gpt-5.6" assert mock_ws.call_args.kwargs["custom_llm_provider"] == "openai" + + +_INJECTION_POINT_INPUT = [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "hi"}] +_SYSTEM_INJECTION_POINT = [{"location": "message", "role": "system"}] + + +def _sent_body(mock_post) -> dict: + kwargs = mock_post.call_args.kwargs + return kwargs["json"] if "json" in kwargs else json.loads(kwargs["data"]) + + +@pytest.mark.asyncio +async def test_aresponses_injection_point_marks_input_text_on_gpt_5_6(): + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_async", "gpt-5.6"), 200) + + await litellm.aresponses( + model="openai/gpt-5.6", + api_key="fake-api-key", + input=copy.deepcopy(_INJECTION_POINT_INPUT), + cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT), + ) + + body = _sent_body(mock_post) + assert body["input"][0]["content"][0] == { + "type": "input_text", + "text": "You are terse.", + "prompt_cache_breakpoint": {"mode": "explicit"}, + } + assert body["input"][1] == {"role": "user", "content": "hi"} + assert body["prompt_cache_options"] == {"mode": "explicit"} + + +def test_responses_injection_point_marks_input_text_on_gpt_5_6(): + with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post: + mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_sync", "gpt-5.6"), 200) + + litellm.responses( + model="openai/gpt-5.6", + api_key="fake-api-key", + input=copy.deepcopy(_INJECTION_POINT_INPUT), + cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT), + ) + + body = _sent_body(mock_post) + assert body["input"][0]["content"][0] == { + "type": "input_text", + "text": "You are terse.", + "prompt_cache_breakpoint": {"mode": "explicit"}, + } + assert body["input"][1] == {"role": "user", "content": "hi"} + assert body["prompt_cache_options"] == {"mode": "explicit"} + + +@pytest.mark.asyncio +async def test_aresponses_injection_point_sends_nothing_extra_below_gpt_5_6(): + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_old", "gpt-4.1"), 200) + + await litellm.aresponses( + model="openai/gpt-4.1", + api_key="fake-api-key", + input=copy.deepcopy(_INJECTION_POINT_INPUT), + cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT), + ) + + body = _sent_body(mock_post) + assert body["input"] == _INJECTION_POINT_INPUT + assert "prompt_cache_options" not in body + assert "cache_control" not in json.dumps(body) + + +@pytest.fixture +def _no_openai_api_base_override(monkeypatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None) + + +_CUSTOM_API_BASE = "http://127.0.0.1:9/v1" + + +async def _aresponses_body_with_system_point(**request_kwargs) -> dict: + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_gate", "gpt-5.6"), 200) + await litellm.aresponses( + api_key="fake-api-key", + input=copy.deepcopy(_INJECTION_POINT_INPUT), + cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT), + **request_kwargs, + ) + return _sent_body(mock_post) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_no_openai_api_base_override") +async def test_aresponses_litellm_proxy_target_sends_no_openai_markers(): + body = await _aresponses_body_with_system_point(model="litellm_proxy/gpt-5.6", api_base=_CUSTOM_API_BASE) + assert body["input"] == _INJECTION_POINT_INPUT + assert "prompt_cache_options" not in body + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_no_openai_api_base_override") +async def test_aresponses_custom_api_base_sends_no_openai_markers(): + body = await _aresponses_body_with_system_point(model="gpt-5.6", api_base=_CUSTOM_API_BASE) + assert body["input"] == _INJECTION_POINT_INPUT + assert "prompt_cache_options" not in body + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_no_openai_api_base_override") +async def test_aresponses_custom_api_base_opts_in_through_prompt_cache_options(): + body = await _aresponses_body_with_system_point( + model="gpt-5.6", api_base=_CUSTOM_API_BASE, prompt_cache_options={"mode": "explicit"} + ) + assert body["input"][0]["content"][0] == { + "type": "input_text", + "text": "You are terse.", + "prompt_cache_breakpoint": {"mode": "explicit"}, + } + assert body["prompt_cache_options"] == {"mode": "explicit"} + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_no_openai_api_base_override") +async def test_aresponses_regional_openai_api_base_marks_input_text(): + body = await _aresponses_body_with_system_point(model="gpt-5.6", api_base="https://eu.api.openai.com/v1") + assert body["input"][0]["content"][0]["prompt_cache_breakpoint"] == {"mode": "explicit"} + assert body["prompt_cache_options"] == {"mode": "explicit"} + + +@pytest.mark.usefixtures("_no_openai_api_base_override") +def test_responses_custom_base_url_sends_no_openai_markers(): + with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post: + mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_gate_base_url", "gpt-5.6"), 200) + + litellm.responses( + model="gpt-5.6", + api_key="fake-api-key", + base_url=_CUSTOM_API_BASE, + input=copy.deepcopy(_INJECTION_POINT_INPUT), + cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT), + ) + + body = _sent_body(mock_post) + assert body["input"] == _INJECTION_POINT_INPUT + assert "prompt_cache_options" not in body + + +@pytest.mark.usefixtures("_no_openai_api_base_override") +def test_responses_custom_api_base_sends_no_openai_markers(): + with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post: + mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_gate_sync", "gpt-5.6"), 200) + + litellm.responses( + model="gpt-5.6", + api_key="fake-api-key", + api_base=_CUSTOM_API_BASE, + input=copy.deepcopy(_INJECTION_POINT_INPUT), + cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT), + ) + + body = _sent_body(mock_post) + assert body["input"] == _INJECTION_POINT_INPUT + assert "prompt_cache_options" not in body diff --git a/tests/test_litellm/responses/test_responses_utils.py b/tests/test_litellm/responses/test_responses_utils.py index 2b9e6d34828..2f4a699d307 100644 --- a/tests/test_litellm/responses/test_responses_utils.py +++ b/tests/test_litellm/responses/test_responses_utils.py @@ -638,3 +638,92 @@ def test_responses_maps_reasoning_effort_from_litellm_params_to_reasoning(): "effort": "high", "summary": "detailed", } + + +class TestMergePromptManagementInputReshape: + """Chat-shaped text parts produced by prompt management hooks become input_text parts (#37509).""" + + EXPLICIT = {"mode": "explicit"} + + def _run_cache_hook(self, client_input, points, model="openai/gpt-5.6"): + from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook + + _, merged, _ = AnthropicCacheControlHook().get_chat_completion_prompt( + model=model, + messages=client_input, + non_default_params={"cache_control_injection_points": points}, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + return merged + + def test_string_system_item_becomes_input_text_with_marker(self): + original_input = [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "hi"}] + merged = self._run_cache_hook(list(original_input), [{"location": "message", "role": "system"}]) + + result = ResponsesAPIRequestUtils.merge_prompt_management_input( + original_input=original_input, client_input=list(original_input), merged_input=merged + ) + + assert result[0]["content"] == [ + {"type": "input_text", "text": "You are terse.", "prompt_cache_breakpoint": self.EXPLICIT} + ] + assert result[1] == {"role": "user", "content": "hi"} + + def test_reshape_returns_copies_and_leaves_hook_output_untouched(self): + user_part = {"type": "text", "text": "follow-up"} + user_message = {"role": "user", "content": [user_part]} + merged = [user_message] + + result = ResponsesAPIRequestUtils.merge_prompt_management_input( + original_input="ignored", client_input=[], merged_input=merged + ) + + assert result == [{"role": "user", "content": [{"type": "input_text", "text": "follow-up"}]}] + assert user_part == {"type": "text", "text": "follow-up"} + assert user_message == {"role": "user", "content": [user_part]} + assert result[0] is not user_message + + def test_reshape_keeps_non_message_items_when_hook_returns_client_objects(self): + user_message = {"role": "user", "content": [{"type": "text", "text": "question"}]} + reference = {"type": "item_reference", "id": "msg_123"} + original_input = [reference, user_message] + + result = ResponsesAPIRequestUtils.merge_prompt_management_input( + original_input=original_input, client_input=[user_message], merged_input=[user_message] + ) + + assert result == [reference, {"role": "user", "content": [{"type": "input_text", "text": "question"}]}] + assert result[0] is reference + assert user_message["content"] == [{"type": "text", "text": "question"}] + + def test_assistant_text_parts_are_left_alone(self): + merged = [ + {"role": "assistant", "content": [{"type": "text", "text": "earlier answer"}]}, + {"role": "user", "content": [{"type": "text", "text": "follow-up"}]}, + ] + + result = ResponsesAPIRequestUtils.merge_prompt_management_input( + original_input="ignored", client_input=[], merged_input=merged + ) + + assert result[0]["content"] == [{"type": "text", "text": "earlier answer"}] + assert result[1]["content"] == [{"type": "input_text", "text": "follow-up"}] + + def test_parts_already_in_responses_shape_are_unchanged(self): + merged = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "a", "prompt_cache_breakpoint": self.EXPLICIT}, + {"type": "input_image", "image_url": "https://example.com/a.png"}, + ], + } + ] + + result = ResponsesAPIRequestUtils.merge_prompt_management_input( + original_input="ignored", client_input=[], merged_input=merged + ) + + assert result == merged diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 68b8d1c62b5..4b223a3a900 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -2672,3 +2672,85 @@ def test_openai_model_without_a_provider_still_routes_to_openai(): ) mock_create.assert_called() + + +def _openai_chat_create_kwargs(client, **completion_kwargs): + with patch.object(client.chat.completions.with_raw_response, "create") as mock_client: + with contextlib.suppress(Exception): + litellm.completion( + messages=[{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}], + cache_control_injection_points=[{"location": "message", "role": "system"}], + client=client, + **completion_kwargs, + ) + + mock_client.assert_called_once() + return mock_client.call_args.kwargs + + +@pytest.fixture +def _no_openai_api_base_override(monkeypatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None) + + +@pytest.mark.usefixtures("_no_openai_api_base_override") +def test_completion_custom_api_base_sends_no_prompt_cache_breakpoint_for_gpt_5_6(): + from openai import OpenAI + + client = OpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:9/v1") + request_body = _openai_chat_create_kwargs(client, model="gpt-5.6", api_base="http://127.0.0.1:9/v1") + + assert request_body["messages"][0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}} + assert "prompt_cache_breakpoint" not in json.dumps(request_body["messages"]) + assert "prompt_cache_options" not in json.dumps(request_body) + + +@pytest.mark.usefixtures("_no_openai_api_base_override") +def test_completion_custom_base_url_sends_no_prompt_cache_breakpoint_for_gpt_5_6(): + from openai import OpenAI + + client = OpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:9/v1") + request_body = _openai_chat_create_kwargs(client, model="gpt-5.6", base_url="http://127.0.0.1:9/v1") + + assert request_body["messages"][0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}} + assert "prompt_cache_breakpoint" not in json.dumps(request_body["messages"]) + assert "prompt_cache_options" not in json.dumps(request_body) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_no_openai_api_base_override") +async def test_acompletion_custom_base_url_sends_no_prompt_cache_breakpoint_for_gpt_5_6(): + from openai import AsyncOpenAI + + client = AsyncOpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:9/v1") + with patch.object(client.chat.completions.with_raw_response, "create") as mock_create: + with contextlib.suppress(Exception): + await litellm.acompletion( + model="gpt-5.6", + messages=[{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}], + cache_control_injection_points=[{"location": "message", "role": "system"}], + client=client, + base_url="http://127.0.0.1:9/v1", + ) + + mock_create.assert_called_once() + request_body = mock_create.call_args.kwargs + + assert request_body["messages"][0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}} + assert "prompt_cache_breakpoint" not in json.dumps(request_body["messages"]) + assert "prompt_cache_options" not in json.dumps(request_body) + + +@pytest.mark.usefixtures("_no_openai_api_base_override") +def test_completion_default_api_base_sends_prompt_cache_breakpoint_for_gpt_5_6(): + from openai import OpenAI + + client = OpenAI(api_key="fake-api-key") + request_body = _openai_chat_create_kwargs(client, model="gpt-5.6") + + assert request_body["messages"][0]["content"] == [ + {"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}} + ] + assert request_body["extra_body"]["prompt_cache_options"] == {"mode": "explicit"} diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 60453931595..cb58038e081 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -921,6 +921,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_parallel_tool_use_config": {"type": "boolean"}, "supports_pdf_input": {"type": "boolean"}, "prompt_cache_min_tokens": {"type": "number"}, + "supports_prompt_cache_breakpoint": {"type": "boolean"}, "supports_prompt_caching": {"type": "boolean"}, "supports_response_schema": {"type": "boolean"}, "supports_system_messages": {"type": "boolean"},