mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
Gate OpenAI prompt cache breakpoints on the real target and carry them through /v1/responses
The cache control hook also runs on litellm.responses() input. On a
GPT-5.6 deployment it wrapped a string-content item into a chat-shaped
{"type": "text"} part, which the Responses API rejects, and it never
marked input_text, input_image or input_file parts, so no breakpoint and
no prompt_cache_options reached the provider. Add the Responses part
types to the eligible block set and translate chat-shaped text parts on
non-assistant items to input_text in
ResponsesAPIRequestUtils.merge_prompt_management_input, which both the
async and the sync prompt management sites go through.
The dialect also fired for any GPT-5.6 name that resolved to provider
openai, including deployments pointed at a custom api_base that does not
understand prompt_cache_breakpoint. Decide it once per request from the
provider, the model map and the resolved api_base (request, then
litellm.api_base, then OPENAI_BASE_URL / OPENAI_API_BASE): only
api.openai.com and *.api.openai.com hosts speak the dialect, a top-level
prompt_cache_options opts a custom target in, and litellm_proxy/ targets
never get it. maybe_seed_default_injection_points takes api_base and
stamps the finished decision on the points as _litellm_openai_dialect so
the sync completion() path, whose hook params do not carry api_base,
honors it; maybe_inject_cache_control takes api_base from the
/v1/messages handler.
Eligibility now comes from a supports_prompt_cache_breakpoint model map
flag on the OpenAI gpt-5.6 entries, exposed through
litellm.utils.supports_prompt_cache_breakpoint, with the GPT version rule
kept only for models the map does not know. The OpenAI dialect no longer
reserves a slot for tool_config points, which OpenAI has no cache block
for, and with_prompt_cache_breakpoint plus the chat bridge helper return
a new block instead of mutating their input.
This commit is contained in:
mateo-berri
parent
5f6d22e792
commit
d9aaa95978
17 changed files with 692 additions and 43 deletions
|
|
@ -10,9 +10,11 @@ Supported for both `v1/chat/completions` (via the prompt-management hook) and
|
|||
"""
|
||||
|
||||
import copy
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
|
@ -49,10 +51,18 @@ MAX_CACHE_CONTROL_BLOCKS: Final = 4
|
|||
CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint")
|
||||
OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6)
|
||||
_GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?")
|
||||
OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset({"text", "image", "image_url", "file", "input_audio"})
|
||||
OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset(
|
||||
{"text", "image", "image_url", "file", "input_audio", "input_text", "input_image", "input_file"}
|
||||
)
|
||||
OPENAI_API_HOST: Final = "api.openai.com"
|
||||
OPENAI_API_BASE_ENV_VARS: Final = ("OPENAI_BASE_URL", "OPENAI_API_BASE")
|
||||
|
||||
|
||||
def supports_openai_prompt_cache_breakpoint(model: str) -> bool:
|
||||
if _has_model_map_entry(model):
|
||||
from litellm.utils import supports_prompt_cache_breakpoint
|
||||
|
||||
return supports_prompt_cache_breakpoint(model)
|
||||
version_match: Final = _GPT_VERSION_PATTERN.match(model.rsplit("/", 1)[-1].lower())
|
||||
if version_match is None:
|
||||
return False
|
||||
|
|
@ -60,6 +70,25 @@ def supports_openai_prompt_cache_breakpoint(model: str) -> bool:
|
|||
return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION
|
||||
|
||||
|
||||
def _has_model_map_entry(model: str) -> bool:
|
||||
import litellm
|
||||
|
||||
return model in litellm.model_cost or model.rsplit("/", 1)[-1] in litellm.model_cost
|
||||
|
||||
|
||||
def targets_openai_api(api_base: object) -> bool:
|
||||
import litellm
|
||||
|
||||
resolved: Final = next(
|
||||
(value for value in (api_base, litellm.api_base, *map(os.getenv, OPENAI_API_BASE_ENV_VARS)) if value),
|
||||
None,
|
||||
)
|
||||
if not isinstance(resolved, str):
|
||||
return True
|
||||
host: Final = urlparse(resolved).hostname
|
||||
return host is not None and (host == OPENAI_API_HOST or host.endswith(f".{OPENAI_API_HOST}"))
|
||||
|
||||
|
||||
def _carries_cache_breakpoint(block: object) -> bool:
|
||||
return isinstance(block, dict) and any(block.get(key) is not None for key in CACHE_BREAKPOINT_KEYS)
|
||||
|
||||
|
|
@ -114,10 +143,19 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
# provider transform, where each tool_config point appends at most one
|
||||
# cachePoint to the tools. That block also counts toward Anthropic's
|
||||
# limit, so reserve a slot for it here to leave room.
|
||||
reserved_blocks: Final = 1 if any(p.get("location") == "tool_config" for p in remaining_points) else 0
|
||||
|
||||
openai_dialect: Final = AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
|
||||
model, injection_points[0].get("_litellm_provider")
|
||||
stamped_dialect: Final = injection_points[0].get("_litellm_openai_dialect")
|
||||
openai_dialect: Final = (
|
||||
stamped_dialect
|
||||
if isinstance(stamped_dialect, bool)
|
||||
else AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
|
||||
model,
|
||||
non_default_params.get("custom_llm_provider"),
|
||||
non_default_params.get("api_base"),
|
||||
non_default_params.get("prompt_cache_options"),
|
||||
)
|
||||
)
|
||||
reserved_blocks: Final = (
|
||||
1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0
|
||||
)
|
||||
breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages)
|
||||
processed_messages = self._apply_message_injections(
|
||||
|
|
@ -141,10 +179,17 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
return model, processed_messages, non_default_params
|
||||
|
||||
@staticmethod
|
||||
def _targets_openai_prompt_cache_breakpoint(model: str | None, custom_llm_provider: str | None) -> bool:
|
||||
def _targets_openai_prompt_cache_breakpoint(
|
||||
model: str | None,
|
||||
custom_llm_provider: str | None,
|
||||
api_base: object = None,
|
||||
prompt_cache_options: object = None,
|
||||
) -> bool:
|
||||
if model is None or not supports_openai_prompt_cache_breakpoint(model):
|
||||
return False
|
||||
return (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) == "openai"
|
||||
if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai":
|
||||
return False
|
||||
return prompt_cache_options is not None or targets_openai_api(api_base)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_provider(model: str) -> str | None:
|
||||
|
|
@ -315,10 +360,18 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
]
|
||||
return marked
|
||||
if isinstance(message_content, list):
|
||||
with_prompt_cache_breakpoint(
|
||||
next((block for block in reversed(message_content) if _accepts_prompt_cache_breakpoint(block)), None),
|
||||
PromptCacheBreakpoint(mode="explicit"),
|
||||
target_index: Final = next(
|
||||
(
|
||||
index
|
||||
for index in range(len(message_content) - 1, -1, -1)
|
||||
if _accepts_prompt_cache_breakpoint(message_content[index])
|
||||
),
|
||||
None,
|
||||
)
|
||||
if target_index is not None:
|
||||
message_content[target_index] = with_prompt_cache_breakpoint(
|
||||
message_content[target_index], PromptCacheBreakpoint(mode="explicit")
|
||||
)
|
||||
return message
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -363,7 +416,9 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
else:
|
||||
remaining_points.append(point)
|
||||
|
||||
reserved_blocks: Final = 1 if any(p.get("location") == "tool_config" for p in remaining_points) else 0
|
||||
reserved_blocks: Final = (
|
||||
1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0
|
||||
)
|
||||
max_blocks: Final = MAX_CACHE_CONTROL_BLOCKS - reserved_blocks
|
||||
|
||||
message_blocks: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages)
|
||||
|
|
@ -432,18 +487,32 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
tools: list[object] | None,
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
api_base: object,
|
||||
prompt_cache_options: object,
|
||||
) -> Sequence[Mapping[str, object]] | None:
|
||||
if AnthropicCacheControlHook._should_stand_down(points, messages, None, tools):
|
||||
return None
|
||||
return AnthropicCacheControlHook._stamped_with_provider(points, model, custom_llm_provider)
|
||||
return AnthropicCacheControlHook._stamped_with_dialect(
|
||||
points, model, custom_llm_provider, api_base, prompt_cache_options
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _stamped_with_provider(
|
||||
points: Sequence[CacheControlInjectionPoint], model: str, custom_llm_provider: str | None
|
||||
def _stamped_with_dialect(
|
||||
points: Sequence[CacheControlInjectionPoint],
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
api_base: object,
|
||||
prompt_cache_options: object,
|
||||
) -> Sequence[Mapping[str, object]]:
|
||||
if custom_llm_provider is None or not supports_openai_prompt_cache_breakpoint(model):
|
||||
if not supports_openai_prompt_cache_breakpoint(model):
|
||||
return points
|
||||
return AnthropicCacheControlHook._stamped(points, "_litellm_provider", custom_llm_provider)
|
||||
return AnthropicCacheControlHook._stamped(
|
||||
points,
|
||||
"_litellm_openai_dialect",
|
||||
AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
|
||||
model, custom_llm_provider, api_base, prompt_cache_options
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _stamped(
|
||||
|
|
@ -563,6 +632,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
custom_llm_provider: str | None,
|
||||
tools: list | None = None,
|
||||
enable_prompt_caching: bool | None = None,
|
||||
api_base: object = None,
|
||||
) -> None:
|
||||
"""For /chat/completions: resolve the injection points the request should carry.
|
||||
|
||||
|
|
@ -578,7 +648,13 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
"""
|
||||
if non_default_params.get("cache_control_injection_points"):
|
||||
judged: Final = AnthropicCacheControlHook._judged_configured_points(
|
||||
non_default_params["cache_control_injection_points"], messages, tools, model, custom_llm_provider
|
||||
non_default_params["cache_control_injection_points"],
|
||||
messages,
|
||||
tools,
|
||||
model,
|
||||
custom_llm_provider,
|
||||
api_base,
|
||||
non_default_params.get("prompt_cache_options"),
|
||||
)
|
||||
if judged is None:
|
||||
non_default_params.pop("cache_control_injection_points")
|
||||
|
|
@ -604,6 +680,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
model: str | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
tools: list[dict] | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> tuple[list[dict], str | list | None]:
|
||||
"""Extract cache_control_injection_points from kwargs and apply if present.
|
||||
|
||||
|
|
@ -642,7 +719,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
return messages, system
|
||||
|
||||
openai_dialect: Final = AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
|
||||
model, custom_llm_provider
|
||||
model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options")
|
||||
)
|
||||
breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system)
|
||||
messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
|
||||
|
|
|
|||
|
|
@ -1325,17 +1325,14 @@ def check_is_function_call(logging_obj: "LoggingClass") -> bool:
|
|||
return False
|
||||
|
||||
|
||||
_MarkedT = TypeVar("_MarkedT")
|
||||
|
||||
|
||||
def _set_prompt_cache_breakpoint(target: object, marker: object) -> None:
|
||||
if marker is not None and isinstance(target, dict):
|
||||
target["prompt_cache_breakpoint"] = marker
|
||||
_MarkedT: Final = TypeVar("_MarkedT", bound=Mapping[str, object])
|
||||
|
||||
|
||||
def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
|
||||
_set_prompt_cache_breakpoint(target, marker)
|
||||
return target
|
||||
if marker is None:
|
||||
return target
|
||||
marked: Final = {**target, "prompt_cache_breakpoint": marker} # mutable-ok: API message payload
|
||||
return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key
|
||||
|
||||
|
||||
def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Any:
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import copy
|
|||
import hashlib
|
||||
import json
|
||||
from collections.abc import AsyncIterator, Iterator, Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, cast
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast
|
||||
|
||||
from litellm.llms.anthropic.experimental_pass_through.utils import (
|
||||
is_reasoning_auto_summary_enabled,
|
||||
|
|
@ -246,6 +246,9 @@ class AnthropicAdapter:
|
|||
return anthropic_wrapper.anthropic_sse_wrapper()
|
||||
|
||||
|
||||
_BlockT: Final = TypeVar("_BlockT", bound=Mapping[str, object])
|
||||
|
||||
|
||||
class LiteLLMAnthropicMessagesAdapter:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
|
@ -310,9 +313,10 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
cast(dict[str, object], target)["cache_control"] = cache_control
|
||||
|
||||
@staticmethod
|
||||
def _add_prompt_cache_breakpoint_if_present(source: object, target: object) -> None:
|
||||
def _add_prompt_cache_breakpoint_if_present(source: object, target: _BlockT) -> _BlockT:
|
||||
if isinstance(source, dict) and "prompt_cache_breakpoint" in source:
|
||||
with_prompt_cache_breakpoint(target, source["prompt_cache_breakpoint"])
|
||||
return with_prompt_cache_breakpoint(target, source["prompt_cache_breakpoint"])
|
||||
return target
|
||||
|
||||
def translatable_anthropic_params(self) -> list[str]:
|
||||
"""
|
||||
|
|
@ -374,8 +378,9 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
if content.get("type") == "text":
|
||||
text_obj = ChatCompletionTextObject(type="text", text=content.get("text", ""))
|
||||
self._add_cache_control_if_applicable(content, text_obj, model)
|
||||
self._add_prompt_cache_breakpoint_if_present(content, text_obj)
|
||||
new_user_content_list.append(text_obj)
|
||||
new_user_content_list.append(
|
||||
self._add_prompt_cache_breakpoint_if_present(content, text_obj)
|
||||
)
|
||||
elif content.get("type") == "image":
|
||||
# Convert Anthropic image format to OpenAI format
|
||||
source = content.get("source", {})
|
||||
|
|
@ -385,8 +390,9 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
image_url_obj = ChatCompletionImageUrlObject(url=openai_image_url)
|
||||
image_obj = ChatCompletionImageObject(type="image_url", image_url=image_url_obj)
|
||||
self._add_cache_control_if_applicable(content, image_obj, model)
|
||||
self._add_prompt_cache_breakpoint_if_present(content, image_obj)
|
||||
new_user_content_list.append(image_obj)
|
||||
new_user_content_list.append(
|
||||
self._add_prompt_cache_breakpoint_if_present(content, image_obj)
|
||||
)
|
||||
elif content.get("type") == "document":
|
||||
# Convert Anthropic document format (PDF, etc.) to OpenAI format
|
||||
source = content.get("source", {})
|
||||
|
|
@ -877,8 +883,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
continue
|
||||
text_obj = ChatCompletionTextObject(type="text", text=text)
|
||||
self._add_cache_control_if_applicable(block, text_obj, model)
|
||||
self._add_prompt_cache_breakpoint_if_present(block, text_obj)
|
||||
text_parts.append(text_obj)
|
||||
text_parts.append(self._add_prompt_cache_breakpoint_if_present(block, text_obj))
|
||||
return ChatCompletionSystemMessage(role="system", content=text_parts) if text_parts else None
|
||||
|
||||
def _add_system_message_to_messages(
|
||||
|
|
@ -909,8 +914,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
"text": block.get("text", ""),
|
||||
}
|
||||
self._add_cache_control_if_applicable(block, text_block, model_name)
|
||||
self._add_prompt_cache_breakpoint_if_present(block, text_block)
|
||||
openai_system_content.append(text_block)
|
||||
openai_system_content.append(self._add_prompt_cache_breakpoint_if_present(block, text_block))
|
||||
if openai_system_content:
|
||||
new_messages.insert(
|
||||
0,
|
||||
|
|
|
|||
|
|
@ -230,7 +230,7 @@ async def anthropic_messages(
|
|||
)
|
||||
|
||||
messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(
|
||||
messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools
|
||||
messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools, api_base=api_base
|
||||
)
|
||||
|
||||
original_stream: Final = stream or kwargs.get("_websearch_interception_converted_stream", False)
|
||||
|
|
@ -422,7 +422,7 @@ def anthropic_messages_handler(
|
|||
)
|
||||
|
||||
messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(
|
||||
messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools
|
||||
messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools, api_base=api_base
|
||||
)
|
||||
|
||||
metadata = validate_anthropic_api_metadata(metadata)
|
||||
|
|
|
|||
|
|
@ -507,6 +507,7 @@ async def acompletion(
|
|||
custom_llm_provider=cast(str | None, custom_llm_provider), # cast-ok: read from untyped kwargs
|
||||
tools=tools,
|
||||
enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs
|
||||
api_base=kwargs.get("api_base"),
|
||||
)
|
||||
|
||||
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
|
||||
|
|
@ -5171,6 +5172,7 @@ def completion(
|
|||
custom_llm_provider=cast(str | None, kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs
|
||||
tools=tools,
|
||||
enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs
|
||||
api_base=kwargs.get("api_base"),
|
||||
)
|
||||
|
||||
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
|
||||
|
|
|
|||
|
|
@ -25368,6 +25368,7 @@
|
|||
"supports_none_reasoning_effort": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_cache_breakpoint": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
@ -25430,6 +25431,7 @@
|
|||
"supports_none_reasoning_effort": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_cache_breakpoint": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
@ -25492,6 +25494,7 @@
|
|||
"supports_none_reasoning_effort": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_cache_breakpoint": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
@ -25554,6 +25557,7 @@
|
|||
"supports_none_reasoning_effort": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_cache_breakpoint": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
|
|||
|
|
@ -40,12 +40,25 @@ def normalize_responses_api_stream_options(
|
|||
class ResponsesAPIRequestUtils:
|
||||
"""Helper utils for constructing ResponseAPI requests"""
|
||||
|
||||
@staticmethod
|
||||
def shape_prompt_managed_messages_for_responses(messages: Iterable[object]) -> None:
|
||||
for message in messages:
|
||||
if not isinstance(message, dict) or message.get("role") == "assistant":
|
||||
continue
|
||||
content: object = message.get("content")
|
||||
if not isinstance(content, list):
|
||||
continue
|
||||
for part in content:
|
||||
if isinstance(part, dict) and part.get("type") == "text":
|
||||
part["type"] = "input_text"
|
||||
|
||||
@staticmethod
|
||||
def merge_prompt_management_input(
|
||||
original_input: str | ResponseInputParam,
|
||||
client_input: list[AllMessageValues],
|
||||
merged_input: list[AllMessageValues],
|
||||
) -> list[object]:
|
||||
ResponsesAPIRequestUtils.shape_prompt_managed_messages_for_responses(merged_input)
|
||||
if isinstance(original_input, str):
|
||||
return [*merged_input]
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ class CacheControlMessageInjectionPoint(TypedDict):
|
|||
index: int | str | None # Optional: target by specific index
|
||||
control: ChatCompletionCachedContent | None
|
||||
_litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran
|
||||
_litellm_provider: NotRequired[ReadOnly[str]]
|
||||
_litellm_openai_dialect: NotRequired[ReadOnly[bool]]
|
||||
|
||||
|
||||
class CacheControlToolConfigInjectionPoint(TypedDict):
|
||||
|
|
@ -22,7 +22,7 @@ class CacheControlToolConfigInjectionPoint(TypedDict):
|
|||
location: Literal["tool_config"]
|
||||
control: ChatCompletionCachedContent | None
|
||||
_litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran
|
||||
_litellm_provider: NotRequired[ReadOnly[str]]
|
||||
_litellm_openai_dialect: NotRequired[ReadOnly[bool]]
|
||||
|
||||
|
||||
CacheControlInjectionPoint = CacheControlMessageInjectionPoint | CacheControlToolConfigInjectionPoint
|
||||
|
|
|
|||
|
|
@ -141,6 +141,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
|
|||
supports_tool_choice: bool | None
|
||||
supports_assistant_prefill: bool | None
|
||||
supports_prompt_caching: bool | None
|
||||
supports_prompt_cache_breakpoint: ReadOnly[bool | None]
|
||||
supports_computer_use: bool | None
|
||||
supports_audio_input: bool | None
|
||||
supports_embedding_image_input: bool | None
|
||||
|
|
|
|||
|
|
@ -2560,6 +2560,14 @@ def supports_prompt_caching(model: str, custom_llm_provider: str | None = None)
|
|||
)
|
||||
|
||||
|
||||
def supports_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool:
|
||||
return _supports_factory(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
key="supports_prompt_cache_breakpoint",
|
||||
)
|
||||
|
||||
|
||||
def supports_computer_use(model: str, custom_llm_provider: str | None = None) -> bool:
|
||||
"""
|
||||
Check if the given model supports computer use and return a boolean value.
|
||||
|
|
@ -5473,6 +5481,7 @@ def _get_model_info_helper(
|
|||
supports_tool_choice=None,
|
||||
supports_assistant_prefill=None,
|
||||
supports_prompt_caching=None,
|
||||
supports_prompt_cache_breakpoint=None,
|
||||
supports_computer_use=None,
|
||||
supports_pdf_input=None,
|
||||
)
|
||||
|
|
@ -5712,6 +5721,7 @@ def _get_model_info_helper(
|
|||
supports_tool_choice=_model_info.get("supports_tool_choice", None),
|
||||
supports_assistant_prefill=_model_info.get("supports_assistant_prefill", None),
|
||||
supports_prompt_caching=_model_info.get("supports_prompt_caching", None),
|
||||
supports_prompt_cache_breakpoint=_model_info.get("supports_prompt_cache_breakpoint", None),
|
||||
supports_audio_input=_model_info.get("supports_audio_input", None),
|
||||
supports_audio_output=_model_info.get("supports_audio_output", None),
|
||||
supports_pdf_input=_model_info.get("supports_pdf_input", None),
|
||||
|
|
@ -5846,6 +5856,7 @@ def get_model_info(
|
|||
supports_function_calling: Optional[bool]
|
||||
supports_tool_choice: Optional[bool]
|
||||
supports_prompt_caching: Optional[bool]
|
||||
supports_prompt_cache_breakpoint: Optional[bool]
|
||||
supports_audio_input: Optional[bool]
|
||||
supports_audio_output: Optional[bool]
|
||||
supports_pdf_input: Optional[bool]
|
||||
|
|
|
|||
|
|
@ -25368,6 +25368,7 @@
|
|||
"supports_none_reasoning_effort": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_cache_breakpoint": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
@ -25430,6 +25431,7 @@
|
|||
"supports_none_reasoning_effort": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_cache_breakpoint": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
@ -25492,6 +25494,7 @@
|
|||
"supports_none_reasoning_effort": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_cache_breakpoint": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
@ -25554,6 +25557,7 @@
|
|||
"supports_none_reasoning_effort": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_cache_breakpoint": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
|
|||
|
|
@ -664,6 +664,9 @@
|
|||
"supports_pdf_input": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_prompt_cache_breakpoint": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_prompt_caching": {
|
||||
"type": "boolean"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -23,6 +23,13 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_openai_api_base_override(monkeypatch):
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", None)
|
||||
|
||||
|
||||
def _rendered_log_message(call):
|
||||
message = str(call.args[0])
|
||||
values = call.args[1:]
|
||||
|
|
@ -2395,19 +2402,28 @@ class TestOpenAIPromptCacheBreakpointPlacementRules:
|
|||
|
||||
|
||||
class TestChatPathProviderStamp:
|
||||
"""The chat path learns the caller's custom_llm_provider through the seeded points (#37509)."""
|
||||
"""The chat path learns the dialect decision (provider, api_base, opt-in) through the seeded points (#37509)."""
|
||||
|
||||
POINTS = [{"location": "message", "role": "system"}]
|
||||
MESSAGES = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}]
|
||||
ANTHROPIC_STYLE = {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}}
|
||||
OPENAI_STYLE = [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}]
|
||||
CUSTOM_API_BASE = "http://127.0.0.1:9/v1"
|
||||
|
||||
def _seed_and_run(self, model, custom_llm_provider):
|
||||
def _seed_and_run(self, model, custom_llm_provider, api_base=None, prompt_cache_options=None):
|
||||
params = {"cache_control_injection_points": copy.deepcopy(self.POINTS)}
|
||||
if prompt_cache_options is not None:
|
||||
params["prompt_cache_options"] = prompt_cache_options
|
||||
AnthropicCacheControlHook.maybe_seed_default_injection_points(
|
||||
non_default_params=params,
|
||||
messages=copy.deepcopy(self.MESSAGES),
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
)
|
||||
return self._run(params, model)
|
||||
|
||||
def _run(self, params, model):
|
||||
_, out, params = AnthropicCacheControlHook().get_chat_completion_prompt(
|
||||
model=model,
|
||||
messages=copy.deepcopy(self.MESSAGES),
|
||||
|
|
@ -2452,6 +2468,84 @@ class TestChatPathProviderStamp:
|
|||
assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("my-custom-model", None) is False
|
||||
resolve.assert_not_called()
|
||||
|
||||
def test_litellm_proxy_target_keeps_anthropic_style_markers(self):
|
||||
out, params = self._seed_and_run("litellm_proxy/gpt-5.6", None)
|
||||
assert out[0] == self.ANTHROPIC_STYLE
|
||||
assert "prompt_cache_options" not in params
|
||||
|
||||
def test_custom_api_base_keeps_anthropic_style_markers(self):
|
||||
out, params = self._seed_and_run("gpt-5.6", None, api_base=self.CUSTOM_API_BASE)
|
||||
assert out[0] == self.ANTHROPIC_STYLE
|
||||
assert "prompt_cache_options" not in params
|
||||
|
||||
def test_custom_api_base_opts_in_through_prompt_cache_options(self):
|
||||
out, params = self._seed_and_run(
|
||||
"gpt-5.6", None, api_base=self.CUSTOM_API_BASE, prompt_cache_options={"mode": "explicit"}
|
||||
)
|
||||
assert out[0]["content"] == self.OPENAI_STYLE
|
||||
assert params["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
||||
def test_regional_openai_api_base_uses_openai_dialect(self):
|
||||
out, params = self._seed_and_run("gpt-5.6", None, api_base="https://eu.api.openai.com/v1")
|
||||
assert out[0]["content"] == self.OPENAI_STYLE
|
||||
assert params["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
||||
@pytest.mark.parametrize("env_var", ["OPENAI_BASE_URL", "OPENAI_API_BASE"])
|
||||
def test_env_api_base_override_keeps_anthropic_style_markers(self, monkeypatch, env_var):
|
||||
monkeypatch.setenv(env_var, self.CUSTOM_API_BASE)
|
||||
out, params = self._seed_and_run("gpt-5.6", None)
|
||||
assert out[0] == self.ANTHROPIC_STYLE
|
||||
assert "prompt_cache_options" not in params
|
||||
|
||||
def test_global_litellm_api_base_keeps_anthropic_style_markers(self, monkeypatch):
|
||||
monkeypatch.setattr(litellm, "api_base", self.CUSTOM_API_BASE)
|
||||
out, params = self._seed_and_run("gpt-5.6", None)
|
||||
assert out[0] == self.ANTHROPIC_STYLE
|
||||
assert "prompt_cache_options" not in params
|
||||
|
||||
def test_request_api_base_wins_over_env_override(self, monkeypatch):
|
||||
monkeypatch.setenv("OPENAI_BASE_URL", self.CUSTOM_API_BASE)
|
||||
out, params = self._seed_and_run("gpt-5.6", None, api_base="https://api.openai.com/v1")
|
||||
assert out[0]["content"] == self.OPENAI_STYLE
|
||||
assert params["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api_base,expected",
|
||||
[(None, True), ("http://127.0.0.1:9/v1", False), ("https://eu.api.openai.com/v1", True)],
|
||||
)
|
||||
def test_seed_stamps_the_dialect_decision(self, api_base, expected):
|
||||
params = {"cache_control_injection_points": copy.deepcopy(self.POINTS)}
|
||||
AnthropicCacheControlHook.maybe_seed_default_injection_points(
|
||||
non_default_params=params,
|
||||
messages=copy.deepcopy(self.MESSAGES),
|
||||
model="gpt-5.6",
|
||||
custom_llm_provider=None,
|
||||
api_base=api_base,
|
||||
)
|
||||
assert params["cache_control_injection_points"][0]["_litellm_openai_dialect"] is expected
|
||||
|
||||
def test_stamp_is_authoritative_over_request_params(self):
|
||||
points = [{**self.POINTS[0], "_litellm_openai_dialect": False}]
|
||||
out, params = self._run({"cache_control_injection_points": points, "custom_llm_provider": "openai"}, "gpt-5.6")
|
||||
assert out[0] == self.ANTHROPIC_STYLE
|
||||
assert "prompt_cache_options" not in params
|
||||
|
||||
def test_unstamped_points_read_api_base_from_request_params(self):
|
||||
params = {"cache_control_injection_points": copy.deepcopy(self.POINTS), "api_base": self.CUSTOM_API_BASE}
|
||||
out, params = self._run(params, "gpt-5.6")
|
||||
assert out[0] == self.ANTHROPIC_STYLE
|
||||
assert "prompt_cache_options" not in params
|
||||
|
||||
def test_unstamped_points_read_prompt_cache_options_from_request_params(self):
|
||||
params = {
|
||||
"cache_control_injection_points": copy.deepcopy(self.POINTS),
|
||||
"api_base": self.CUSTOM_API_BASE,
|
||||
"prompt_cache_options": {"mode": "explicit"},
|
||||
}
|
||||
out, params = self._run(params, "gpt-5.6")
|
||||
assert out[0]["content"] == self.OPENAI_STYLE
|
||||
assert params["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
||||
|
||||
class TestClientBreakpointsCountedOnce:
|
||||
def test_client_message_breakpoints_are_not_double_counted(self):
|
||||
|
|
@ -2471,3 +2565,175 @@ class TestClientBreakpointsCountedOnce:
|
|||
marked = [msg["content"][0].get("cache_control") is not None for msg in out]
|
||||
assert marked == [True, False, True, True]
|
||||
assert system[0]["cache_control"] == {"type": "ephemeral"}
|
||||
|
||||
|
||||
class TestResponsesInputPartsEligible:
|
||||
"""Responses API input parts can carry prompt_cache_breakpoint on GPT-5.6+ (#37509)."""
|
||||
|
||||
EXPLICIT = {"mode": "explicit"}
|
||||
|
||||
def _chat(self, messages, points, model="openai/gpt-5.6"):
|
||||
params = {"cache_control_injection_points": copy.deepcopy(points)}
|
||||
_, out, params = AnthropicCacheControlHook().get_chat_completion_prompt(
|
||||
model=model,
|
||||
messages=copy.deepcopy(messages),
|
||||
non_default_params=params,
|
||||
prompt_id=None,
|
||||
prompt_variables=None,
|
||||
dynamic_callback_params={},
|
||||
)
|
||||
return out, params
|
||||
|
||||
def test_marker_lands_on_last_input_text_part(self):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": "first"}, {"type": "input_text", "text": "second"}],
|
||||
}
|
||||
]
|
||||
out, params = self._chat(messages, [{"location": "message", "index": -1}])
|
||||
assert out[0]["content"][0] == {"type": "input_text", "text": "first"}
|
||||
assert out[0]["content"][1] == {
|
||||
"type": "input_text",
|
||||
"text": "second",
|
||||
"prompt_cache_breakpoint": self.EXPLICIT,
|
||||
}
|
||||
assert params["prompt_cache_options"] == self.EXPLICIT
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"part",
|
||||
[
|
||||
{"type": "input_image", "image_url": "https://example.com/a.png"},
|
||||
{"type": "input_file", "file_id": "file_1"},
|
||||
],
|
||||
)
|
||||
def test_input_image_and_input_file_parts_are_eligible(self, part):
|
||||
out, params = self._chat([{"role": "user", "content": [part]}], [{"location": "message", "index": -1}])
|
||||
assert out[0]["content"][0] == {**part, "prompt_cache_breakpoint": self.EXPLICIT}
|
||||
assert params["prompt_cache_options"] == self.EXPLICIT
|
||||
|
||||
|
||||
class TestMessagesPathApiBaseGate:
|
||||
"""/v1/messages only speaks the OpenAI dialect when the request really targets api.openai.com (#37509)."""
|
||||
|
||||
EXPLICIT = {"mode": "explicit"}
|
||||
USER_POINT = [{"location": "message", "role": "user"}]
|
||||
MESSAGES = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]
|
||||
CUSTOM_API_BASE = "http://127.0.0.1:9/v1"
|
||||
CACHE_CONTROL_BLOCK = {"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}
|
||||
BREAKPOINT_BLOCK = {"type": "text", "text": "hi", "prompt_cache_breakpoint": {"mode": "explicit"}}
|
||||
|
||||
def _inject(self, model, api_base=None, prompt_cache_options=None, custom_llm_provider=None):
|
||||
kwargs = {"cache_control_injection_points": copy.deepcopy(self.USER_POINT)}
|
||||
if prompt_cache_options is not None:
|
||||
kwargs["prompt_cache_options"] = prompt_cache_options
|
||||
out, _ = AnthropicCacheControlHook.maybe_inject_cache_control(
|
||||
copy.deepcopy(self.MESSAGES),
|
||||
None,
|
||||
kwargs,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
)
|
||||
return out[0]["content"][0], kwargs
|
||||
|
||||
def test_litellm_proxy_target_keeps_cache_control(self):
|
||||
block, kwargs = self._inject("gpt-5.6", api_base=self.CUSTOM_API_BASE, custom_llm_provider="litellm_proxy")
|
||||
assert block == self.CACHE_CONTROL_BLOCK
|
||||
assert "prompt_cache_options" not in kwargs
|
||||
|
||||
def test_custom_api_base_keeps_cache_control(self):
|
||||
block, kwargs = self._inject("gpt-5.6", api_base=self.CUSTOM_API_BASE)
|
||||
assert block == self.CACHE_CONTROL_BLOCK
|
||||
assert "prompt_cache_options" not in kwargs
|
||||
|
||||
def test_custom_api_base_opts_in_through_prompt_cache_options(self):
|
||||
block, kwargs = self._inject("gpt-5.6", api_base=self.CUSTOM_API_BASE, prompt_cache_options=self.EXPLICIT)
|
||||
assert block == self.BREAKPOINT_BLOCK
|
||||
assert kwargs["prompt_cache_options"] == self.EXPLICIT
|
||||
|
||||
def test_regional_openai_api_base_uses_openai_dialect(self):
|
||||
block, kwargs = self._inject("gpt-5.6", api_base="https://eu.api.openai.com/v1")
|
||||
assert block == self.BREAKPOINT_BLOCK
|
||||
assert kwargs["prompt_cache_options"] == self.EXPLICIT
|
||||
|
||||
def test_default_api_base_uses_openai_dialect(self):
|
||||
block, kwargs = self._inject("openai/gpt-5.6")
|
||||
assert block == self.BREAKPOINT_BLOCK
|
||||
assert kwargs["prompt_cache_options"] == self.EXPLICIT
|
||||
|
||||
|
||||
class TestToolConfigSlotInOpenAIDialect:
|
||||
"""OpenAI has no tool_config cache block, so the dialect does not hold a slot for one (#37509)."""
|
||||
|
||||
EXPLICIT = {"mode": "explicit"}
|
||||
MESSAGES = [{"role": "user", "content": [{"type": "text", "text": f"m{i}"}]} for i in range(4)]
|
||||
POINTS = [{"location": "message", "index": i} for i in range(4)] + [{"location": "tool_config"}]
|
||||
|
||||
def test_chat_path_marks_all_four_messages(self):
|
||||
params = {"cache_control_injection_points": copy.deepcopy(self.POINTS)}
|
||||
_, out, params = AnthropicCacheControlHook().get_chat_completion_prompt(
|
||||
model="openai/gpt-5.6",
|
||||
messages=copy.deepcopy(self.MESSAGES),
|
||||
non_default_params=params,
|
||||
prompt_id=None,
|
||||
prompt_variables=None,
|
||||
dynamic_callback_params={},
|
||||
)
|
||||
assert [msg["content"][0].get("prompt_cache_breakpoint") for msg in out] == [self.EXPLICIT] * 4
|
||||
assert params["prompt_cache_options"] == self.EXPLICIT
|
||||
|
||||
def test_messages_path_marks_all_four_messages(self):
|
||||
out, _, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
|
||||
copy.deepcopy(self.MESSAGES), None, copy.deepcopy(self.POINTS), openai_dialect=True
|
||||
)
|
||||
assert [msg["content"][0].get("prompt_cache_breakpoint") for msg in out] == [self.EXPLICIT] * 4
|
||||
|
||||
def test_anthropic_dialect_still_reserves_the_tool_config_slot(self):
|
||||
out, _, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
|
||||
copy.deepcopy(self.MESSAGES), None, copy.deepcopy(self.POINTS)
|
||||
)
|
||||
assert sum(msg["content"][0].get("cache_control") is not None for msg in out) == 3
|
||||
|
||||
|
||||
class TestPromptCacheBreakpointCapability:
|
||||
"""Eligibility comes from the model map's supports_prompt_cache_breakpoint flag, with the GPT version
|
||||
rule only for models the map does not know (#37509)."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _fresh_model_info_cache(self):
|
||||
litellm.utils._cached_get_model_info_helper.cache_clear()
|
||||
yield
|
||||
litellm.utils._cached_get_model_info_helper.cache_clear()
|
||||
|
||||
def test_public_helper_reads_the_model_map(self):
|
||||
from litellm.utils import supports_prompt_cache_breakpoint
|
||||
|
||||
assert supports_prompt_cache_breakpoint("gpt-5.6") is True
|
||||
assert supports_prompt_cache_breakpoint("openai/gpt-5.6-sol") is True
|
||||
assert supports_prompt_cache_breakpoint("gpt-5.6", custom_llm_provider="openai") is True
|
||||
assert supports_prompt_cache_breakpoint("gpt-4.1") is False
|
||||
|
||||
@pytest.mark.parametrize("model", ["gpt-5.6", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"])
|
||||
def test_model_map_flags_every_openai_gpt_5_6_entry(self, model):
|
||||
assert litellm.model_cost[model]["litellm_provider"] == "openai"
|
||||
assert litellm.model_cost[model]["supports_prompt_cache_breakpoint"] is True
|
||||
|
||||
def test_listed_model_uses_the_model_map_flag(self, monkeypatch):
|
||||
flagged = {**litellm.model_cost["gpt-4.1"], "supports_prompt_cache_breakpoint": True}
|
||||
monkeypatch.setitem(litellm.model_cost, "gpt-4.1", flagged)
|
||||
assert supports_openai_prompt_cache_breakpoint("gpt-4.1") is True
|
||||
|
||||
def test_listed_gpt_5_6_without_the_flag_is_not_eligible(self, monkeypatch):
|
||||
unflagged = {k: v for k, v in litellm.model_cost["gpt-5.6"].items() if k != "supports_prompt_cache_breakpoint"}
|
||||
monkeypatch.setitem(litellm.model_cost, "gpt-5.6", unflagged)
|
||||
assert supports_openai_prompt_cache_breakpoint("gpt-5.6") is False
|
||||
|
||||
def test_listed_gpt_model_without_the_flag_is_false(self):
|
||||
assert "supports_prompt_cache_breakpoint" not in litellm.model_cost["gpt-4.1"]
|
||||
assert supports_openai_prompt_cache_breakpoint("gpt-4.1") is False
|
||||
|
||||
@pytest.mark.parametrize("model,expected", [("gpt-5.6-2026-01-01", True), ("gpt-5.5-preview-unlisted", False)])
|
||||
def test_unlisted_model_falls_back_to_the_version_rule(self, model, expected):
|
||||
assert model not in litellm.model_cost
|
||||
assert supports_openai_prompt_cache_breakpoint(model) is expected
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ over the wire and surface provider errors correctly. Expected JSON bodies are st
|
|||
in expected_responses_api_request/.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
|
@ -422,3 +423,160 @@ async def test_aresponses_websocket_strips_responses_routing_prefix_from_openai_
|
|||
mock_ws.assert_awaited_once()
|
||||
assert mock_ws.call_args.kwargs["model"] == "gpt-5.6"
|
||||
assert mock_ws.call_args.kwargs["custom_llm_provider"] == "openai"
|
||||
|
||||
|
||||
_INJECTION_POINT_INPUT = [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "hi"}]
|
||||
_SYSTEM_INJECTION_POINT = [{"location": "message", "role": "system"}]
|
||||
|
||||
|
||||
def _sent_body(mock_post) -> dict:
|
||||
kwargs = mock_post.call_args.kwargs
|
||||
return kwargs["json"] if "json" in kwargs else json.loads(kwargs["data"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aresponses_injection_point_marks_input_text_on_gpt_5_6():
|
||||
with patch(
|
||||
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_post:
|
||||
mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_async", "gpt-5.6"), 200)
|
||||
|
||||
await litellm.aresponses(
|
||||
model="openai/gpt-5.6",
|
||||
api_key="fake-api-key",
|
||||
input=copy.deepcopy(_INJECTION_POINT_INPUT),
|
||||
cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT),
|
||||
)
|
||||
|
||||
body = _sent_body(mock_post)
|
||||
assert body["input"][0]["content"][0] == {
|
||||
"type": "input_text",
|
||||
"text": "You are terse.",
|
||||
"prompt_cache_breakpoint": {"mode": "explicit"},
|
||||
}
|
||||
assert body["input"][1] == {"role": "user", "content": "hi"}
|
||||
assert body["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
||||
|
||||
def test_responses_injection_point_marks_input_text_on_gpt_5_6():
|
||||
with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post:
|
||||
mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_sync", "gpt-5.6"), 200)
|
||||
|
||||
litellm.responses(
|
||||
model="openai/gpt-5.6",
|
||||
api_key="fake-api-key",
|
||||
input=copy.deepcopy(_INJECTION_POINT_INPUT),
|
||||
cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT),
|
||||
)
|
||||
|
||||
body = _sent_body(mock_post)
|
||||
assert body["input"][0]["content"][0] == {
|
||||
"type": "input_text",
|
||||
"text": "You are terse.",
|
||||
"prompt_cache_breakpoint": {"mode": "explicit"},
|
||||
}
|
||||
assert body["input"][1] == {"role": "user", "content": "hi"}
|
||||
assert body["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aresponses_injection_point_sends_nothing_extra_below_gpt_5_6():
|
||||
with patch(
|
||||
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_post:
|
||||
mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_old", "gpt-4.1"), 200)
|
||||
|
||||
await litellm.aresponses(
|
||||
model="openai/gpt-4.1",
|
||||
api_key="fake-api-key",
|
||||
input=copy.deepcopy(_INJECTION_POINT_INPUT),
|
||||
cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT),
|
||||
)
|
||||
|
||||
body = _sent_body(mock_post)
|
||||
assert body["input"] == _INJECTION_POINT_INPUT
|
||||
assert "prompt_cache_options" not in body
|
||||
assert "cache_control" not in json.dumps(body)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _no_openai_api_base_override(monkeypatch):
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", None)
|
||||
|
||||
|
||||
_CUSTOM_API_BASE = "http://127.0.0.1:9/v1"
|
||||
|
||||
|
||||
async def _aresponses_body_with_system_point(**request_kwargs) -> dict:
|
||||
with patch(
|
||||
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_post:
|
||||
mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_gate", "gpt-5.6"), 200)
|
||||
await litellm.aresponses(
|
||||
api_key="fake-api-key",
|
||||
input=copy.deepcopy(_INJECTION_POINT_INPUT),
|
||||
cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT),
|
||||
**request_kwargs,
|
||||
)
|
||||
return _sent_body(mock_post)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("_no_openai_api_base_override")
|
||||
async def test_aresponses_litellm_proxy_target_sends_no_openai_markers():
|
||||
body = await _aresponses_body_with_system_point(model="litellm_proxy/gpt-5.6", api_base=_CUSTOM_API_BASE)
|
||||
assert body["input"] == _INJECTION_POINT_INPUT
|
||||
assert "prompt_cache_options" not in body
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("_no_openai_api_base_override")
|
||||
async def test_aresponses_custom_api_base_sends_no_openai_markers():
|
||||
body = await _aresponses_body_with_system_point(model="gpt-5.6", api_base=_CUSTOM_API_BASE)
|
||||
assert body["input"] == _INJECTION_POINT_INPUT
|
||||
assert "prompt_cache_options" not in body
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("_no_openai_api_base_override")
|
||||
async def test_aresponses_custom_api_base_opts_in_through_prompt_cache_options():
|
||||
body = await _aresponses_body_with_system_point(
|
||||
model="gpt-5.6", api_base=_CUSTOM_API_BASE, prompt_cache_options={"mode": "explicit"}
|
||||
)
|
||||
assert body["input"][0]["content"][0] == {
|
||||
"type": "input_text",
|
||||
"text": "You are terse.",
|
||||
"prompt_cache_breakpoint": {"mode": "explicit"},
|
||||
}
|
||||
assert body["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("_no_openai_api_base_override")
|
||||
async def test_aresponses_regional_openai_api_base_marks_input_text():
|
||||
body = await _aresponses_body_with_system_point(model="gpt-5.6", api_base="https://eu.api.openai.com/v1")
|
||||
assert body["input"][0]["content"][0]["prompt_cache_breakpoint"] == {"mode": "explicit"}
|
||||
assert body["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_no_openai_api_base_override")
|
||||
def test_responses_custom_api_base_sends_no_openai_markers():
|
||||
with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post:
|
||||
mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_gate_sync", "gpt-5.6"), 200)
|
||||
|
||||
litellm.responses(
|
||||
model="gpt-5.6",
|
||||
api_key="fake-api-key",
|
||||
api_base=_CUSTOM_API_BASE,
|
||||
input=copy.deepcopy(_INJECTION_POINT_INPUT),
|
||||
cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT),
|
||||
)
|
||||
|
||||
body = _sent_body(mock_post)
|
||||
assert body["input"] == _INJECTION_POINT_INPUT
|
||||
assert "prompt_cache_options" not in body
|
||||
|
|
|
|||
|
|
@ -638,3 +638,65 @@ def test_responses_maps_reasoning_effort_from_litellm_params_to_reasoning():
|
|||
"effort": "high",
|
||||
"summary": "detailed",
|
||||
}
|
||||
|
||||
|
||||
class TestMergePromptManagementInputReshape:
|
||||
"""Chat-shaped text parts produced by prompt management hooks become input_text parts (#37509)."""
|
||||
|
||||
EXPLICIT = {"mode": "explicit"}
|
||||
|
||||
def _run_cache_hook(self, client_input, points, model="openai/gpt-5.6"):
|
||||
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
|
||||
|
||||
_, merged, _ = AnthropicCacheControlHook().get_chat_completion_prompt(
|
||||
model=model,
|
||||
messages=client_input,
|
||||
non_default_params={"cache_control_injection_points": points},
|
||||
prompt_id=None,
|
||||
prompt_variables=None,
|
||||
dynamic_callback_params={},
|
||||
)
|
||||
return merged
|
||||
|
||||
def test_string_system_item_becomes_input_text_with_marker(self):
|
||||
original_input = [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "hi"}]
|
||||
merged = self._run_cache_hook(list(original_input), [{"location": "message", "role": "system"}])
|
||||
|
||||
result = ResponsesAPIRequestUtils.merge_prompt_management_input(
|
||||
original_input=original_input, client_input=list(original_input), merged_input=merged
|
||||
)
|
||||
|
||||
assert result[0]["content"] == [
|
||||
{"type": "input_text", "text": "You are terse.", "prompt_cache_breakpoint": self.EXPLICIT}
|
||||
]
|
||||
assert result[1] == {"role": "user", "content": "hi"}
|
||||
|
||||
def test_assistant_text_parts_are_left_alone(self):
|
||||
merged = [
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "earlier answer"}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "follow-up"}]},
|
||||
]
|
||||
|
||||
result = ResponsesAPIRequestUtils.merge_prompt_management_input(
|
||||
original_input="ignored", client_input=[], merged_input=merged
|
||||
)
|
||||
|
||||
assert result[0]["content"] == [{"type": "text", "text": "earlier answer"}]
|
||||
assert result[1]["content"] == [{"type": "input_text", "text": "follow-up"}]
|
||||
|
||||
def test_parts_already_in_responses_shape_are_unchanged(self):
|
||||
merged = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "a", "prompt_cache_breakpoint": self.EXPLICIT},
|
||||
{"type": "input_image", "image_url": "https://example.com/a.png"},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
result = ResponsesAPIRequestUtils.merge_prompt_management_input(
|
||||
original_input="ignored", client_input=[], merged_input=merged
|
||||
)
|
||||
|
||||
assert result == merged
|
||||
|
|
|
|||
|
|
@ -2672,3 +2672,49 @@ def test_openai_model_without_a_provider_still_routes_to_openai():
|
|||
)
|
||||
|
||||
mock_create.assert_called()
|
||||
|
||||
|
||||
def _openai_chat_create_kwargs(client, **completion_kwargs):
|
||||
with patch.object(client.chat.completions.with_raw_response, "create") as mock_client:
|
||||
with contextlib.suppress(Exception):
|
||||
litellm.completion(
|
||||
messages=[{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}],
|
||||
cache_control_injection_points=[{"location": "message", "role": "system"}],
|
||||
client=client,
|
||||
**completion_kwargs,
|
||||
)
|
||||
|
||||
mock_client.assert_called_once()
|
||||
return mock_client.call_args.kwargs
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _no_openai_api_base_override(monkeypatch):
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", None)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_no_openai_api_base_override")
|
||||
def test_completion_custom_api_base_sends_no_prompt_cache_breakpoint_for_gpt_5_6():
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:9/v1")
|
||||
request_body = _openai_chat_create_kwargs(client, model="gpt-5.6", api_base="http://127.0.0.1:9/v1")
|
||||
|
||||
assert request_body["messages"][0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}}
|
||||
assert "prompt_cache_breakpoint" not in json.dumps(request_body["messages"])
|
||||
assert "prompt_cache_options" not in json.dumps(request_body)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_no_openai_api_base_override")
|
||||
def test_completion_default_api_base_sends_prompt_cache_breakpoint_for_gpt_5_6():
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(api_key="fake-api-key")
|
||||
request_body = _openai_chat_create_kwargs(client, model="gpt-5.6")
|
||||
|
||||
assert request_body["messages"][0]["content"] == [
|
||||
{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}
|
||||
]
|
||||
assert request_body["extra_body"]["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
|
|
|||
|
|
@ -921,6 +921,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
|
|||
"supports_parallel_tool_use_config": {"type": "boolean"},
|
||||
"supports_pdf_input": {"type": "boolean"},
|
||||
"prompt_cache_min_tokens": {"type": "number"},
|
||||
"supports_prompt_cache_breakpoint": {"type": "boolean"},
|
||||
"supports_prompt_caching": {"type": "boolean"},
|
||||
"supports_response_schema": {"type": "boolean"},
|
||||
"supports_system_messages": {"type": "boolean"},
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue