diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 05e2a37a230..f65decc5463 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -10,9 +10,11 @@ Supported for both `v1/chat/completions` (via the prompt-management hook) and """ import copy +import os import re from collections.abc import Iterable, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, cast +from urllib.parse import urlparse from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger @@ -49,10 +51,18 @@ MAX_CACHE_CONTROL_BLOCKS: Final = 4 CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint") OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6) _GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?") -OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset({"text", "image", "image_url", "file", "input_audio"}) +OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset( + {"text", "image", "image_url", "file", "input_audio", "input_text", "input_image", "input_file"} +) +OPENAI_API_HOST: Final = "api.openai.com" +OPENAI_API_BASE_ENV_VARS: Final = ("OPENAI_BASE_URL", "OPENAI_API_BASE") def supports_openai_prompt_cache_breakpoint(model: str) -> bool: + if _has_model_map_entry(model): + from litellm.utils import supports_prompt_cache_breakpoint + + return supports_prompt_cache_breakpoint(model) version_match: Final = _GPT_VERSION_PATTERN.match(model.rsplit("/", 1)[-1].lower()) if version_match is None: return False @@ -60,6 +70,25 @@ def supports_openai_prompt_cache_breakpoint(model: str) -> bool: return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION +def _has_model_map_entry(model: str) -> bool: + import litellm + + return model in litellm.model_cost or model.rsplit("/", 1)[-1] in litellm.model_cost + + +def targets_openai_api(api_base: object) -> bool: + import litellm + + resolved: Final = next( + (value for value in (api_base, litellm.api_base, *map(os.getenv, OPENAI_API_BASE_ENV_VARS)) if value), + None, + ) + if not isinstance(resolved, str): + return True + host: Final = urlparse(resolved).hostname + return host is not None and (host == OPENAI_API_HOST or host.endswith(f".{OPENAI_API_HOST}")) + + def _carries_cache_breakpoint(block: object) -> bool: return isinstance(block, dict) and any(block.get(key) is not None for key in CACHE_BREAKPOINT_KEYS) @@ -114,10 +143,19 @@ class AnthropicCacheControlHook(CustomPromptManagement): # provider transform, where each tool_config point appends at most one # cachePoint to the tools. That block also counts toward Anthropic's # limit, so reserve a slot for it here to leave room. - reserved_blocks: Final = 1 if any(p.get("location") == "tool_config" for p in remaining_points) else 0 - - openai_dialect: Final = AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( - model, injection_points[0].get("_litellm_provider") + stamped_dialect: Final = injection_points[0].get("_litellm_openai_dialect") + openai_dialect: Final = ( + stamped_dialect + if isinstance(stamped_dialect, bool) + else AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( + model, + non_default_params.get("custom_llm_provider"), + non_default_params.get("api_base"), + non_default_params.get("prompt_cache_options"), + ) + ) + reserved_blocks: Final = ( + 1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0 ) breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages) processed_messages = self._apply_message_injections( @@ -141,10 +179,17 @@ class AnthropicCacheControlHook(CustomPromptManagement): return model, processed_messages, non_default_params @staticmethod - def _targets_openai_prompt_cache_breakpoint(model: str | None, custom_llm_provider: str | None) -> bool: + def _targets_openai_prompt_cache_breakpoint( + model: str | None, + custom_llm_provider: str | None, + api_base: object = None, + prompt_cache_options: object = None, + ) -> bool: if model is None or not supports_openai_prompt_cache_breakpoint(model): return False - return (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) == "openai" + if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai": + return False + return prompt_cache_options is not None or targets_openai_api(api_base) @staticmethod def _resolve_provider(model: str) -> str | None: @@ -315,10 +360,18 @@ class AnthropicCacheControlHook(CustomPromptManagement): ] return marked if isinstance(message_content, list): - with_prompt_cache_breakpoint( - next((block for block in reversed(message_content) if _accepts_prompt_cache_breakpoint(block)), None), - PromptCacheBreakpoint(mode="explicit"), + target_index: Final = next( + ( + index + for index in range(len(message_content) - 1, -1, -1) + if _accepts_prompt_cache_breakpoint(message_content[index]) + ), + None, ) + if target_index is not None: + message_content[target_index] = with_prompt_cache_breakpoint( + message_content[target_index], PromptCacheBreakpoint(mode="explicit") + ) return message @staticmethod @@ -363,7 +416,9 @@ class AnthropicCacheControlHook(CustomPromptManagement): else: remaining_points.append(point) - reserved_blocks: Final = 1 if any(p.get("location") == "tool_config" for p in remaining_points) else 0 + reserved_blocks: Final = ( + 1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0 + ) max_blocks: Final = MAX_CACHE_CONTROL_BLOCKS - reserved_blocks message_blocks: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages) @@ -432,18 +487,32 @@ class AnthropicCacheControlHook(CustomPromptManagement): tools: list[object] | None, model: str, custom_llm_provider: str | None, + api_base: object, + prompt_cache_options: object, ) -> Sequence[Mapping[str, object]] | None: if AnthropicCacheControlHook._should_stand_down(points, messages, None, tools): return None - return AnthropicCacheControlHook._stamped_with_provider(points, model, custom_llm_provider) + return AnthropicCacheControlHook._stamped_with_dialect( + points, model, custom_llm_provider, api_base, prompt_cache_options + ) @staticmethod - def _stamped_with_provider( - points: Sequence[CacheControlInjectionPoint], model: str, custom_llm_provider: str | None + def _stamped_with_dialect( + points: Sequence[CacheControlInjectionPoint], + model: str, + custom_llm_provider: str | None, + api_base: object, + prompt_cache_options: object, ) -> Sequence[Mapping[str, object]]: - if custom_llm_provider is None or not supports_openai_prompt_cache_breakpoint(model): + if not supports_openai_prompt_cache_breakpoint(model): return points - return AnthropicCacheControlHook._stamped(points, "_litellm_provider", custom_llm_provider) + return AnthropicCacheControlHook._stamped( + points, + "_litellm_openai_dialect", + AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( + model, custom_llm_provider, api_base, prompt_cache_options + ), + ) @staticmethod def _stamped( @@ -563,6 +632,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): custom_llm_provider: str | None, tools: list | None = None, enable_prompt_caching: bool | None = None, + api_base: object = None, ) -> None: """For /chat/completions: resolve the injection points the request should carry. @@ -578,7 +648,13 @@ class AnthropicCacheControlHook(CustomPromptManagement): """ if non_default_params.get("cache_control_injection_points"): judged: Final = AnthropicCacheControlHook._judged_configured_points( - non_default_params["cache_control_injection_points"], messages, tools, model, custom_llm_provider + non_default_params["cache_control_injection_points"], + messages, + tools, + model, + custom_llm_provider, + api_base, + non_default_params.get("prompt_cache_options"), ) if judged is None: non_default_params.pop("cache_control_injection_points") @@ -604,6 +680,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): model: str | None = None, custom_llm_provider: str | None = None, tools: list[dict] | None = None, + api_base: str | None = None, ) -> tuple[list[dict], str | list | None]: """Extract cache_control_injection_points from kwargs and apply if present. @@ -642,7 +719,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): return messages, system openai_dialect: Final = AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( - model, custom_llm_provider + model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options") ) breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request( diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 5904a14e79f..2db5776047b 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1325,17 +1325,14 @@ def check_is_function_call(logging_obj: "LoggingClass") -> bool: return False -_MarkedT = TypeVar("_MarkedT") - - -def _set_prompt_cache_breakpoint(target: object, marker: object) -> None: - if marker is not None and isinstance(target, dict): - target["prompt_cache_breakpoint"] = marker +_MarkedT: Final = TypeVar("_MarkedT", bound=Mapping[str, object]) def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT: - _set_prompt_cache_breakpoint(target, marker) - return target + if marker is None: + return target + marked: Final = {**target, "prompt_cache_breakpoint": marker} # mutable-ok: API message payload + return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Any: diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 2ee50e2a55b..c6bfdb79002 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -2,7 +2,7 @@ import copy import hashlib import json from collections.abc import AsyncIterator, Iterator, Mapping -from typing import TYPE_CHECKING, Any, Final, Literal, cast +from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast from litellm.llms.anthropic.experimental_pass_through.utils import ( is_reasoning_auto_summary_enabled, @@ -246,6 +246,9 @@ class AnthropicAdapter: return anthropic_wrapper.anthropic_sse_wrapper() +_BlockT: Final = TypeVar("_BlockT", bound=Mapping[str, object]) + + class LiteLLMAnthropicMessagesAdapter: def __init__(self): pass @@ -310,9 +313,10 @@ class LiteLLMAnthropicMessagesAdapter: cast(dict[str, object], target)["cache_control"] = cache_control @staticmethod - def _add_prompt_cache_breakpoint_if_present(source: object, target: object) -> None: + def _add_prompt_cache_breakpoint_if_present(source: object, target: _BlockT) -> _BlockT: if isinstance(source, dict) and "prompt_cache_breakpoint" in source: - with_prompt_cache_breakpoint(target, source["prompt_cache_breakpoint"]) + return with_prompt_cache_breakpoint(target, source["prompt_cache_breakpoint"]) + return target def translatable_anthropic_params(self) -> list[str]: """ @@ -374,8 +378,9 @@ class LiteLLMAnthropicMessagesAdapter: if content.get("type") == "text": text_obj = ChatCompletionTextObject(type="text", text=content.get("text", "")) self._add_cache_control_if_applicable(content, text_obj, model) - self._add_prompt_cache_breakpoint_if_present(content, text_obj) - new_user_content_list.append(text_obj) + new_user_content_list.append( + self._add_prompt_cache_breakpoint_if_present(content, text_obj) + ) elif content.get("type") == "image": # Convert Anthropic image format to OpenAI format source = content.get("source", {}) @@ -385,8 +390,9 @@ class LiteLLMAnthropicMessagesAdapter: image_url_obj = ChatCompletionImageUrlObject(url=openai_image_url) image_obj = ChatCompletionImageObject(type="image_url", image_url=image_url_obj) self._add_cache_control_if_applicable(content, image_obj, model) - self._add_prompt_cache_breakpoint_if_present(content, image_obj) - new_user_content_list.append(image_obj) + new_user_content_list.append( + self._add_prompt_cache_breakpoint_if_present(content, image_obj) + ) elif content.get("type") == "document": # Convert Anthropic document format (PDF, etc.) to OpenAI format source = content.get("source", {}) @@ -877,8 +883,7 @@ class LiteLLMAnthropicMessagesAdapter: continue text_obj = ChatCompletionTextObject(type="text", text=text) self._add_cache_control_if_applicable(block, text_obj, model) - self._add_prompt_cache_breakpoint_if_present(block, text_obj) - text_parts.append(text_obj) + text_parts.append(self._add_prompt_cache_breakpoint_if_present(block, text_obj)) return ChatCompletionSystemMessage(role="system", content=text_parts) if text_parts else None def _add_system_message_to_messages( @@ -909,8 +914,7 @@ class LiteLLMAnthropicMessagesAdapter: "text": block.get("text", ""), } self._add_cache_control_if_applicable(block, text_block, model_name) - self._add_prompt_cache_breakpoint_if_present(block, text_block) - openai_system_content.append(text_block) + openai_system_content.append(self._add_prompt_cache_breakpoint_if_present(block, text_block)) if openai_system_content: new_messages.insert( 0, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index c4b5cc628e2..26aef666172 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -230,7 +230,7 @@ async def anthropic_messages( ) messages, system = AnthropicCacheControlHook.maybe_inject_cache_control( - messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools + messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools, api_base=api_base ) original_stream: Final = stream or kwargs.get("_websearch_interception_converted_stream", False) @@ -422,7 +422,7 @@ def anthropic_messages_handler( ) messages, system = AnthropicCacheControlHook.maybe_inject_cache_control( - messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools + messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools, api_base=api_base ) metadata = validate_anthropic_api_metadata(metadata) diff --git a/litellm/main.py b/litellm/main.py index f0b20eba9b6..87b32ab140c 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -507,6 +507,7 @@ async def acompletion( custom_llm_provider=cast(str | None, custom_llm_provider), # cast-ok: read from untyped kwargs tools=tools, enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs + api_base=kwargs.get("api_base"), ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( @@ -5171,6 +5172,7 @@ def completion( custom_llm_provider=cast(str | None, kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs tools=tools, enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs + api_base=kwargs.get("api_base"), ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3beb3ae4370..c92b4ccd18a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -25368,6 +25368,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -25430,6 +25431,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -25492,6 +25494,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -25554,6 +25557,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 4b5def790ed..504009802f2 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -40,12 +40,25 @@ def normalize_responses_api_stream_options( class ResponsesAPIRequestUtils: """Helper utils for constructing ResponseAPI requests""" + @staticmethod + def shape_prompt_managed_messages_for_responses(messages: Iterable[object]) -> None: + for message in messages: + if not isinstance(message, dict) or message.get("role") == "assistant": + continue + content: object = message.get("content") + if not isinstance(content, list): + continue + for part in content: + if isinstance(part, dict) and part.get("type") == "text": + part["type"] = "input_text" + @staticmethod def merge_prompt_management_input( original_input: str | ResponseInputParam, client_input: list[AllMessageValues], merged_input: list[AllMessageValues], ) -> list[object]: + ResponsesAPIRequestUtils.shape_prompt_managed_messages_for_responses(merged_input) if isinstance(original_input, str): return [*merged_input] diff --git a/litellm/types/integrations/anthropic_cache_control_hook.py b/litellm/types/integrations/anthropic_cache_control_hook.py index 3b3dcf5c86d..3ab0c02f28d 100644 --- a/litellm/types/integrations/anthropic_cache_control_hook.py +++ b/litellm/types/integrations/anthropic_cache_control_hook.py @@ -13,7 +13,7 @@ class CacheControlMessageInjectionPoint(TypedDict): index: int | str | None # Optional: target by specific index control: ChatCompletionCachedContent | None _litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran - _litellm_provider: NotRequired[ReadOnly[str]] + _litellm_openai_dialect: NotRequired[ReadOnly[bool]] class CacheControlToolConfigInjectionPoint(TypedDict): @@ -22,7 +22,7 @@ class CacheControlToolConfigInjectionPoint(TypedDict): location: Literal["tool_config"] control: ChatCompletionCachedContent | None _litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran - _litellm_provider: NotRequired[ReadOnly[str]] + _litellm_openai_dialect: NotRequired[ReadOnly[bool]] CacheControlInjectionPoint = CacheControlMessageInjectionPoint | CacheControlToolConfigInjectionPoint diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 96b9343353d..41210d18495 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -141,6 +141,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_tool_choice: bool | None supports_assistant_prefill: bool | None supports_prompt_caching: bool | None + supports_prompt_cache_breakpoint: ReadOnly[bool | None] supports_computer_use: bool | None supports_audio_input: bool | None supports_embedding_image_input: bool | None diff --git a/litellm/utils.py b/litellm/utils.py index d1b0cb882ac..867f7a93452 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2560,6 +2560,14 @@ def supports_prompt_caching(model: str, custom_llm_provider: str | None = None) ) +def supports_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool: + return _supports_factory( + model=model, + custom_llm_provider=custom_llm_provider, + key="supports_prompt_cache_breakpoint", + ) + + def supports_computer_use(model: str, custom_llm_provider: str | None = None) -> bool: """ Check if the given model supports computer use and return a boolean value. @@ -5473,6 +5481,7 @@ def _get_model_info_helper( supports_tool_choice=None, supports_assistant_prefill=None, supports_prompt_caching=None, + supports_prompt_cache_breakpoint=None, supports_computer_use=None, supports_pdf_input=None, ) @@ -5712,6 +5721,7 @@ def _get_model_info_helper( supports_tool_choice=_model_info.get("supports_tool_choice", None), supports_assistant_prefill=_model_info.get("supports_assistant_prefill", None), supports_prompt_caching=_model_info.get("supports_prompt_caching", None), + supports_prompt_cache_breakpoint=_model_info.get("supports_prompt_cache_breakpoint", None), supports_audio_input=_model_info.get("supports_audio_input", None), supports_audio_output=_model_info.get("supports_audio_output", None), supports_pdf_input=_model_info.get("supports_pdf_input", None), @@ -5846,6 +5856,7 @@ def get_model_info( supports_function_calling: Optional[bool] supports_tool_choice: Optional[bool] supports_prompt_caching: Optional[bool] + supports_prompt_cache_breakpoint: Optional[bool] supports_audio_input: Optional[bool] supports_audio_output: Optional[bool] supports_pdf_input: Optional[bool] diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3beb3ae4370..c92b4ccd18a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -25368,6 +25368,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -25430,6 +25431,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -25492,6 +25494,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -25554,6 +25557,7 @@ "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 82854a3b717..0991650d307 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -664,6 +664,9 @@ "supports_pdf_input": { "type": "boolean" }, + "supports_prompt_cache_breakpoint": { + "type": "boolean" + }, "supports_prompt_caching": { "type": "boolean" }, diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 04876b9bf12..4eb774af5bc 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -23,6 +23,13 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import StandardCallbackDynamicParams +@pytest.fixture(autouse=True) +def _no_openai_api_base_override(monkeypatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None) + + def _rendered_log_message(call): message = str(call.args[0]) values = call.args[1:] @@ -2395,19 +2402,28 @@ class TestOpenAIPromptCacheBreakpointPlacementRules: class TestChatPathProviderStamp: - """The chat path learns the caller's custom_llm_provider through the seeded points (#37509).""" + """The chat path learns the dialect decision (provider, api_base, opt-in) through the seeded points (#37509).""" POINTS = [{"location": "message", "role": "system"}] MESSAGES = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}] + ANTHROPIC_STYLE = {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}} + OPENAI_STYLE = [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}] + CUSTOM_API_BASE = "http://127.0.0.1:9/v1" - def _seed_and_run(self, model, custom_llm_provider): + def _seed_and_run(self, model, custom_llm_provider, api_base=None, prompt_cache_options=None): params = {"cache_control_injection_points": copy.deepcopy(self.POINTS)} + if prompt_cache_options is not None: + params["prompt_cache_options"] = prompt_cache_options AnthropicCacheControlHook.maybe_seed_default_injection_points( non_default_params=params, messages=copy.deepcopy(self.MESSAGES), model=model, custom_llm_provider=custom_llm_provider, + api_base=api_base, ) + return self._run(params, model) + + def _run(self, params, model): _, out, params = AnthropicCacheControlHook().get_chat_completion_prompt( model=model, messages=copy.deepcopy(self.MESSAGES), @@ -2452,6 +2468,84 @@ class TestChatPathProviderStamp: assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("my-custom-model", None) is False resolve.assert_not_called() + def test_litellm_proxy_target_keeps_anthropic_style_markers(self): + out, params = self._seed_and_run("litellm_proxy/gpt-5.6", None) + assert out[0] == self.ANTHROPIC_STYLE + assert "prompt_cache_options" not in params + + def test_custom_api_base_keeps_anthropic_style_markers(self): + out, params = self._seed_and_run("gpt-5.6", None, api_base=self.CUSTOM_API_BASE) + assert out[0] == self.ANTHROPIC_STYLE + assert "prompt_cache_options" not in params + + def test_custom_api_base_opts_in_through_prompt_cache_options(self): + out, params = self._seed_and_run( + "gpt-5.6", None, api_base=self.CUSTOM_API_BASE, prompt_cache_options={"mode": "explicit"} + ) + assert out[0]["content"] == self.OPENAI_STYLE + assert params["prompt_cache_options"] == {"mode": "explicit"} + + def test_regional_openai_api_base_uses_openai_dialect(self): + out, params = self._seed_and_run("gpt-5.6", None, api_base="https://eu.api.openai.com/v1") + assert out[0]["content"] == self.OPENAI_STYLE + assert params["prompt_cache_options"] == {"mode": "explicit"} + + @pytest.mark.parametrize("env_var", ["OPENAI_BASE_URL", "OPENAI_API_BASE"]) + def test_env_api_base_override_keeps_anthropic_style_markers(self, monkeypatch, env_var): + monkeypatch.setenv(env_var, self.CUSTOM_API_BASE) + out, params = self._seed_and_run("gpt-5.6", None) + assert out[0] == self.ANTHROPIC_STYLE + assert "prompt_cache_options" not in params + + def test_global_litellm_api_base_keeps_anthropic_style_markers(self, monkeypatch): + monkeypatch.setattr(litellm, "api_base", self.CUSTOM_API_BASE) + out, params = self._seed_and_run("gpt-5.6", None) + assert out[0] == self.ANTHROPIC_STYLE + assert "prompt_cache_options" not in params + + def test_request_api_base_wins_over_env_override(self, monkeypatch): + monkeypatch.setenv("OPENAI_BASE_URL", self.CUSTOM_API_BASE) + out, params = self._seed_and_run("gpt-5.6", None, api_base="https://api.openai.com/v1") + assert out[0]["content"] == self.OPENAI_STYLE + assert params["prompt_cache_options"] == {"mode": "explicit"} + + @pytest.mark.parametrize( + "api_base,expected", + [(None, True), ("http://127.0.0.1:9/v1", False), ("https://eu.api.openai.com/v1", True)], + ) + def test_seed_stamps_the_dialect_decision(self, api_base, expected): + params = {"cache_control_injection_points": copy.deepcopy(self.POINTS)} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=copy.deepcopy(self.MESSAGES), + model="gpt-5.6", + custom_llm_provider=None, + api_base=api_base, + ) + assert params["cache_control_injection_points"][0]["_litellm_openai_dialect"] is expected + + def test_stamp_is_authoritative_over_request_params(self): + points = [{**self.POINTS[0], "_litellm_openai_dialect": False}] + out, params = self._run({"cache_control_injection_points": points, "custom_llm_provider": "openai"}, "gpt-5.6") + assert out[0] == self.ANTHROPIC_STYLE + assert "prompt_cache_options" not in params + + def test_unstamped_points_read_api_base_from_request_params(self): + params = {"cache_control_injection_points": copy.deepcopy(self.POINTS), "api_base": self.CUSTOM_API_BASE} + out, params = self._run(params, "gpt-5.6") + assert out[0] == self.ANTHROPIC_STYLE + assert "prompt_cache_options" not in params + + def test_unstamped_points_read_prompt_cache_options_from_request_params(self): + params = { + "cache_control_injection_points": copy.deepcopy(self.POINTS), + "api_base": self.CUSTOM_API_BASE, + "prompt_cache_options": {"mode": "explicit"}, + } + out, params = self._run(params, "gpt-5.6") + assert out[0]["content"] == self.OPENAI_STYLE + assert params["prompt_cache_options"] == {"mode": "explicit"} + class TestClientBreakpointsCountedOnce: def test_client_message_breakpoints_are_not_double_counted(self): @@ -2471,3 +2565,175 @@ class TestClientBreakpointsCountedOnce: marked = [msg["content"][0].get("cache_control") is not None for msg in out] assert marked == [True, False, True, True] assert system[0]["cache_control"] == {"type": "ephemeral"} + + +class TestResponsesInputPartsEligible: + """Responses API input parts can carry prompt_cache_breakpoint on GPT-5.6+ (#37509).""" + + EXPLICIT = {"mode": "explicit"} + + def _chat(self, messages, points, model="openai/gpt-5.6"): + params = {"cache_control_injection_points": copy.deepcopy(points)} + _, out, params = AnthropicCacheControlHook().get_chat_completion_prompt( + model=model, + messages=copy.deepcopy(messages), + non_default_params=params, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + return out, params + + def test_marker_lands_on_last_input_text_part(self): + messages = [ + { + "role": "user", + "content": [{"type": "input_text", "text": "first"}, {"type": "input_text", "text": "second"}], + } + ] + out, params = self._chat(messages, [{"location": "message", "index": -1}]) + assert out[0]["content"][0] == {"type": "input_text", "text": "first"} + assert out[0]["content"][1] == { + "type": "input_text", + "text": "second", + "prompt_cache_breakpoint": self.EXPLICIT, + } + assert params["prompt_cache_options"] == self.EXPLICIT + + @pytest.mark.parametrize( + "part", + [ + {"type": "input_image", "image_url": "https://example.com/a.png"}, + {"type": "input_file", "file_id": "file_1"}, + ], + ) + def test_input_image_and_input_file_parts_are_eligible(self, part): + out, params = self._chat([{"role": "user", "content": [part]}], [{"location": "message", "index": -1}]) + assert out[0]["content"][0] == {**part, "prompt_cache_breakpoint": self.EXPLICIT} + assert params["prompt_cache_options"] == self.EXPLICIT + + +class TestMessagesPathApiBaseGate: + """/v1/messages only speaks the OpenAI dialect when the request really targets api.openai.com (#37509).""" + + EXPLICIT = {"mode": "explicit"} + USER_POINT = [{"location": "message", "role": "user"}] + MESSAGES = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + CUSTOM_API_BASE = "http://127.0.0.1:9/v1" + CACHE_CONTROL_BLOCK = {"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}} + BREAKPOINT_BLOCK = {"type": "text", "text": "hi", "prompt_cache_breakpoint": {"mode": "explicit"}} + + def _inject(self, model, api_base=None, prompt_cache_options=None, custom_llm_provider=None): + kwargs = {"cache_control_injection_points": copy.deepcopy(self.USER_POINT)} + if prompt_cache_options is not None: + kwargs["prompt_cache_options"] = prompt_cache_options + out, _ = AnthropicCacheControlHook.maybe_inject_cache_control( + copy.deepcopy(self.MESSAGES), + None, + kwargs, + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + ) + return out[0]["content"][0], kwargs + + def test_litellm_proxy_target_keeps_cache_control(self): + block, kwargs = self._inject("gpt-5.6", api_base=self.CUSTOM_API_BASE, custom_llm_provider="litellm_proxy") + assert block == self.CACHE_CONTROL_BLOCK + assert "prompt_cache_options" not in kwargs + + def test_custom_api_base_keeps_cache_control(self): + block, kwargs = self._inject("gpt-5.6", api_base=self.CUSTOM_API_BASE) + assert block == self.CACHE_CONTROL_BLOCK + assert "prompt_cache_options" not in kwargs + + def test_custom_api_base_opts_in_through_prompt_cache_options(self): + block, kwargs = self._inject("gpt-5.6", api_base=self.CUSTOM_API_BASE, prompt_cache_options=self.EXPLICIT) + assert block == self.BREAKPOINT_BLOCK + assert kwargs["prompt_cache_options"] == self.EXPLICIT + + def test_regional_openai_api_base_uses_openai_dialect(self): + block, kwargs = self._inject("gpt-5.6", api_base="https://eu.api.openai.com/v1") + assert block == self.BREAKPOINT_BLOCK + assert kwargs["prompt_cache_options"] == self.EXPLICIT + + def test_default_api_base_uses_openai_dialect(self): + block, kwargs = self._inject("openai/gpt-5.6") + assert block == self.BREAKPOINT_BLOCK + assert kwargs["prompt_cache_options"] == self.EXPLICIT + + +class TestToolConfigSlotInOpenAIDialect: + """OpenAI has no tool_config cache block, so the dialect does not hold a slot for one (#37509).""" + + EXPLICIT = {"mode": "explicit"} + MESSAGES = [{"role": "user", "content": [{"type": "text", "text": f"m{i}"}]} for i in range(4)] + POINTS = [{"location": "message", "index": i} for i in range(4)] + [{"location": "tool_config"}] + + def test_chat_path_marks_all_four_messages(self): + params = {"cache_control_injection_points": copy.deepcopy(self.POINTS)} + _, out, params = AnthropicCacheControlHook().get_chat_completion_prompt( + model="openai/gpt-5.6", + messages=copy.deepcopy(self.MESSAGES), + non_default_params=params, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + assert [msg["content"][0].get("prompt_cache_breakpoint") for msg in out] == [self.EXPLICIT] * 4 + assert params["prompt_cache_options"] == self.EXPLICIT + + def test_messages_path_marks_all_four_messages(self): + out, _, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request( + copy.deepcopy(self.MESSAGES), None, copy.deepcopy(self.POINTS), openai_dialect=True + ) + assert [msg["content"][0].get("prompt_cache_breakpoint") for msg in out] == [self.EXPLICIT] * 4 + + def test_anthropic_dialect_still_reserves_the_tool_config_slot(self): + out, _, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request( + copy.deepcopy(self.MESSAGES), None, copy.deepcopy(self.POINTS) + ) + assert sum(msg["content"][0].get("cache_control") is not None for msg in out) == 3 + + +class TestPromptCacheBreakpointCapability: + """Eligibility comes from the model map's supports_prompt_cache_breakpoint flag, with the GPT version + rule only for models the map does not know (#37509).""" + + @pytest.fixture(autouse=True) + def _fresh_model_info_cache(self): + litellm.utils._cached_get_model_info_helper.cache_clear() + yield + litellm.utils._cached_get_model_info_helper.cache_clear() + + def test_public_helper_reads_the_model_map(self): + from litellm.utils import supports_prompt_cache_breakpoint + + assert supports_prompt_cache_breakpoint("gpt-5.6") is True + assert supports_prompt_cache_breakpoint("openai/gpt-5.6-sol") is True + assert supports_prompt_cache_breakpoint("gpt-5.6", custom_llm_provider="openai") is True + assert supports_prompt_cache_breakpoint("gpt-4.1") is False + + @pytest.mark.parametrize("model", ["gpt-5.6", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"]) + def test_model_map_flags_every_openai_gpt_5_6_entry(self, model): + assert litellm.model_cost[model]["litellm_provider"] == "openai" + assert litellm.model_cost[model]["supports_prompt_cache_breakpoint"] is True + + def test_listed_model_uses_the_model_map_flag(self, monkeypatch): + flagged = {**litellm.model_cost["gpt-4.1"], "supports_prompt_cache_breakpoint": True} + monkeypatch.setitem(litellm.model_cost, "gpt-4.1", flagged) + assert supports_openai_prompt_cache_breakpoint("gpt-4.1") is True + + def test_listed_gpt_5_6_without_the_flag_is_not_eligible(self, monkeypatch): + unflagged = {k: v for k, v in litellm.model_cost["gpt-5.6"].items() if k != "supports_prompt_cache_breakpoint"} + monkeypatch.setitem(litellm.model_cost, "gpt-5.6", unflagged) + assert supports_openai_prompt_cache_breakpoint("gpt-5.6") is False + + def test_listed_gpt_model_without_the_flag_is_false(self): + assert "supports_prompt_cache_breakpoint" not in litellm.model_cost["gpt-4.1"] + assert supports_openai_prompt_cache_breakpoint("gpt-4.1") is False + + @pytest.mark.parametrize("model,expected", [("gpt-5.6-2026-01-01", True), ("gpt-5.5-preview-unlisted", False)]) + def test_unlisted_model_falls_back_to_the_version_rule(self, model, expected): + assert model not in litellm.model_cost + assert supports_openai_prompt_cache_breakpoint(model) is expected diff --git a/tests/test_litellm/responses/test_responses_api_request_body.py b/tests/test_litellm/responses/test_responses_api_request_body.py index ea5eb3afe9a..1402491a6b6 100644 --- a/tests/test_litellm/responses/test_responses_api_request_body.py +++ b/tests/test_litellm/responses/test_responses_api_request_body.py @@ -4,6 +4,7 @@ over the wire and surface provider errors correctly. Expected JSON bodies are st in expected_responses_api_request/. """ +import copy import json from pathlib import Path from unittest.mock import AsyncMock, patch @@ -422,3 +423,160 @@ async def test_aresponses_websocket_strips_responses_routing_prefix_from_openai_ mock_ws.assert_awaited_once() assert mock_ws.call_args.kwargs["model"] == "gpt-5.6" assert mock_ws.call_args.kwargs["custom_llm_provider"] == "openai" + + +_INJECTION_POINT_INPUT = [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "hi"}] +_SYSTEM_INJECTION_POINT = [{"location": "message", "role": "system"}] + + +def _sent_body(mock_post) -> dict: + kwargs = mock_post.call_args.kwargs + return kwargs["json"] if "json" in kwargs else json.loads(kwargs["data"]) + + +@pytest.mark.asyncio +async def test_aresponses_injection_point_marks_input_text_on_gpt_5_6(): + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_async", "gpt-5.6"), 200) + + await litellm.aresponses( + model="openai/gpt-5.6", + api_key="fake-api-key", + input=copy.deepcopy(_INJECTION_POINT_INPUT), + cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT), + ) + + body = _sent_body(mock_post) + assert body["input"][0]["content"][0] == { + "type": "input_text", + "text": "You are terse.", + "prompt_cache_breakpoint": {"mode": "explicit"}, + } + assert body["input"][1] == {"role": "user", "content": "hi"} + assert body["prompt_cache_options"] == {"mode": "explicit"} + + +def test_responses_injection_point_marks_input_text_on_gpt_5_6(): + with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post: + mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_sync", "gpt-5.6"), 200) + + litellm.responses( + model="openai/gpt-5.6", + api_key="fake-api-key", + input=copy.deepcopy(_INJECTION_POINT_INPUT), + cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT), + ) + + body = _sent_body(mock_post) + assert body["input"][0]["content"][0] == { + "type": "input_text", + "text": "You are terse.", + "prompt_cache_breakpoint": {"mode": "explicit"}, + } + assert body["input"][1] == {"role": "user", "content": "hi"} + assert body["prompt_cache_options"] == {"mode": "explicit"} + + +@pytest.mark.asyncio +async def test_aresponses_injection_point_sends_nothing_extra_below_gpt_5_6(): + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_old", "gpt-4.1"), 200) + + await litellm.aresponses( + model="openai/gpt-4.1", + api_key="fake-api-key", + input=copy.deepcopy(_INJECTION_POINT_INPUT), + cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT), + ) + + body = _sent_body(mock_post) + assert body["input"] == _INJECTION_POINT_INPUT + assert "prompt_cache_options" not in body + assert "cache_control" not in json.dumps(body) + + +@pytest.fixture +def _no_openai_api_base_override(monkeypatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None) + + +_CUSTOM_API_BASE = "http://127.0.0.1:9/v1" + + +async def _aresponses_body_with_system_point(**request_kwargs) -> dict: + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_gate", "gpt-5.6"), 200) + await litellm.aresponses( + api_key="fake-api-key", + input=copy.deepcopy(_INJECTION_POINT_INPUT), + cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT), + **request_kwargs, + ) + return _sent_body(mock_post) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_no_openai_api_base_override") +async def test_aresponses_litellm_proxy_target_sends_no_openai_markers(): + body = await _aresponses_body_with_system_point(model="litellm_proxy/gpt-5.6", api_base=_CUSTOM_API_BASE) + assert body["input"] == _INJECTION_POINT_INPUT + assert "prompt_cache_options" not in body + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_no_openai_api_base_override") +async def test_aresponses_custom_api_base_sends_no_openai_markers(): + body = await _aresponses_body_with_system_point(model="gpt-5.6", api_base=_CUSTOM_API_BASE) + assert body["input"] == _INJECTION_POINT_INPUT + assert "prompt_cache_options" not in body + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_no_openai_api_base_override") +async def test_aresponses_custom_api_base_opts_in_through_prompt_cache_options(): + body = await _aresponses_body_with_system_point( + model="gpt-5.6", api_base=_CUSTOM_API_BASE, prompt_cache_options={"mode": "explicit"} + ) + assert body["input"][0]["content"][0] == { + "type": "input_text", + "text": "You are terse.", + "prompt_cache_breakpoint": {"mode": "explicit"}, + } + assert body["prompt_cache_options"] == {"mode": "explicit"} + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_no_openai_api_base_override") +async def test_aresponses_regional_openai_api_base_marks_input_text(): + body = await _aresponses_body_with_system_point(model="gpt-5.6", api_base="https://eu.api.openai.com/v1") + assert body["input"][0]["content"][0]["prompt_cache_breakpoint"] == {"mode": "explicit"} + assert body["prompt_cache_options"] == {"mode": "explicit"} + + +@pytest.mark.usefixtures("_no_openai_api_base_override") +def test_responses_custom_api_base_sends_no_openai_markers(): + with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post: + mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_gate_sync", "gpt-5.6"), 200) + + litellm.responses( + model="gpt-5.6", + api_key="fake-api-key", + api_base=_CUSTOM_API_BASE, + input=copy.deepcopy(_INJECTION_POINT_INPUT), + cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT), + ) + + body = _sent_body(mock_post) + assert body["input"] == _INJECTION_POINT_INPUT + assert "prompt_cache_options" not in body diff --git a/tests/test_litellm/responses/test_responses_utils.py b/tests/test_litellm/responses/test_responses_utils.py index 2b9e6d34828..35bf99db6dc 100644 --- a/tests/test_litellm/responses/test_responses_utils.py +++ b/tests/test_litellm/responses/test_responses_utils.py @@ -638,3 +638,65 @@ def test_responses_maps_reasoning_effort_from_litellm_params_to_reasoning(): "effort": "high", "summary": "detailed", } + + +class TestMergePromptManagementInputReshape: + """Chat-shaped text parts produced by prompt management hooks become input_text parts (#37509).""" + + EXPLICIT = {"mode": "explicit"} + + def _run_cache_hook(self, client_input, points, model="openai/gpt-5.6"): + from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook + + _, merged, _ = AnthropicCacheControlHook().get_chat_completion_prompt( + model=model, + messages=client_input, + non_default_params={"cache_control_injection_points": points}, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + return merged + + def test_string_system_item_becomes_input_text_with_marker(self): + original_input = [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "hi"}] + merged = self._run_cache_hook(list(original_input), [{"location": "message", "role": "system"}]) + + result = ResponsesAPIRequestUtils.merge_prompt_management_input( + original_input=original_input, client_input=list(original_input), merged_input=merged + ) + + assert result[0]["content"] == [ + {"type": "input_text", "text": "You are terse.", "prompt_cache_breakpoint": self.EXPLICIT} + ] + assert result[1] == {"role": "user", "content": "hi"} + + def test_assistant_text_parts_are_left_alone(self): + merged = [ + {"role": "assistant", "content": [{"type": "text", "text": "earlier answer"}]}, + {"role": "user", "content": [{"type": "text", "text": "follow-up"}]}, + ] + + result = ResponsesAPIRequestUtils.merge_prompt_management_input( + original_input="ignored", client_input=[], merged_input=merged + ) + + assert result[0]["content"] == [{"type": "text", "text": "earlier answer"}] + assert result[1]["content"] == [{"type": "input_text", "text": "follow-up"}] + + def test_parts_already_in_responses_shape_are_unchanged(self): + merged = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "a", "prompt_cache_breakpoint": self.EXPLICIT}, + {"type": "input_image", "image_url": "https://example.com/a.png"}, + ], + } + ] + + result = ResponsesAPIRequestUtils.merge_prompt_management_input( + original_input="ignored", client_input=[], merged_input=merged + ) + + assert result == merged diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 68b8d1c62b5..786adf2d78b 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -2672,3 +2672,49 @@ def test_openai_model_without_a_provider_still_routes_to_openai(): ) mock_create.assert_called() + + +def _openai_chat_create_kwargs(client, **completion_kwargs): + with patch.object(client.chat.completions.with_raw_response, "create") as mock_client: + with contextlib.suppress(Exception): + litellm.completion( + messages=[{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}], + cache_control_injection_points=[{"location": "message", "role": "system"}], + client=client, + **completion_kwargs, + ) + + mock_client.assert_called_once() + return mock_client.call_args.kwargs + + +@pytest.fixture +def _no_openai_api_base_override(monkeypatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None) + + +@pytest.mark.usefixtures("_no_openai_api_base_override") +def test_completion_custom_api_base_sends_no_prompt_cache_breakpoint_for_gpt_5_6(): + from openai import OpenAI + + client = OpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:9/v1") + request_body = _openai_chat_create_kwargs(client, model="gpt-5.6", api_base="http://127.0.0.1:9/v1") + + assert request_body["messages"][0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}} + assert "prompt_cache_breakpoint" not in json.dumps(request_body["messages"]) + assert "prompt_cache_options" not in json.dumps(request_body) + + +@pytest.mark.usefixtures("_no_openai_api_base_override") +def test_completion_default_api_base_sends_prompt_cache_breakpoint_for_gpt_5_6(): + from openai import OpenAI + + client = OpenAI(api_key="fake-api-key") + request_body = _openai_chat_create_kwargs(client, model="gpt-5.6") + + assert request_body["messages"][0]["content"] == [ + {"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}} + ] + assert request_body["extra_body"]["prompt_cache_options"] == {"mode": "explicit"} diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 60453931595..cb58038e081 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -921,6 +921,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_parallel_tool_use_config": {"type": "boolean"}, "supports_pdf_input": {"type": "boolean"}, "prompt_cache_min_tokens": {"type": "number"}, + "supports_prompt_cache_breakpoint": {"type": "boolean"}, "supports_prompt_caching": {"type": "boolean"}, "supports_response_schema": {"type": "boolean"}, "supports_system_messages": {"type": "boolean"},