mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
Merge pull request #37628 from BerriAI/litellm_lit5876_openai_prompt_cache_breakpoint
feat(prompt-caching): map cache_control_injection_points to OpenAI prompt_cache_breakpoint on GPT-5.6+ targets
This commit is contained in:
Mateo Wang
GitHub
commit
e51addb802
25 changed files with 1841 additions and 67 deletions
|
|
@ -10,17 +10,31 @@ Supported for both `v1/chat/completions` (via the prompt-management hook) and
|
|||
"""
|
||||
|
||||
import copy
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.custom_prompt_management import CustomPromptManagement
|
||||
from litellm.integrations.prompt_management_base import PromptManagementClient
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
with_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.types.integrations.anthropic_cache_control_hook import (
|
||||
CacheControlInjectionPoint,
|
||||
CacheControlMessageInjectionPoint,
|
||||
)
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionCachedContent
|
||||
from litellm.types.llms.anthropic import AnthropicSystemMessageContent
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionCachedContent,
|
||||
ChatCompletionTextObject,
|
||||
PromptCacheBreakpoint,
|
||||
PromptCacheOptions,
|
||||
)
|
||||
from litellm.types.prompts.init_prompts import PromptSpec
|
||||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
|
||||
|
|
@ -34,6 +48,55 @@ else:
|
|||
# breakpoints: "A maximum of 4 blocks with cache_control may be provided."
|
||||
MAX_CACHE_CONTROL_BLOCKS: Final = 4
|
||||
|
||||
CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint")
|
||||
OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6)
|
||||
_GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?")
|
||||
OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset(
|
||||
{"text", "image", "image_url", "file", "input_audio", "input_text", "input_image", "input_file"}
|
||||
)
|
||||
OPENAI_API_HOST: Final = "api.openai.com"
|
||||
OPENAI_API_BASE_ENV_VARS: Final = ("OPENAI_BASE_URL", "OPENAI_API_BASE")
|
||||
|
||||
|
||||
def supports_openai_prompt_cache_breakpoint(model: str) -> bool:
|
||||
model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model)
|
||||
if model_map_flag is not None:
|
||||
return model_map_flag
|
||||
version_match: Final = _GPT_VERSION_PATTERN.match(model.rsplit("/", 1)[-1].lower())
|
||||
if version_match is None:
|
||||
return False
|
||||
version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0))
|
||||
return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION
|
||||
|
||||
|
||||
def _model_map_prompt_cache_breakpoint_flag(model: str) -> bool | None:
|
||||
import litellm
|
||||
|
||||
entries: Final = (litellm.model_cost.get(key) for key in (model, model.rsplit("/", 1)[-1]))
|
||||
flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in entries if isinstance(entry, dict))
|
||||
return next((bool(flag) for flag in flags if flag is not None), None)
|
||||
|
||||
|
||||
def targets_openai_api(api_base: object) -> bool:
|
||||
import litellm
|
||||
|
||||
resolved: Final = next(
|
||||
(value for value in (api_base, litellm.api_base, *map(os.getenv, OPENAI_API_BASE_ENV_VARS)) if value),
|
||||
None,
|
||||
)
|
||||
if not isinstance(resolved, str):
|
||||
return True
|
||||
host: Final = urlparse(resolved).hostname
|
||||
return host is not None and (host == OPENAI_API_HOST or host.endswith(f".{OPENAI_API_HOST}"))
|
||||
|
||||
|
||||
def _carries_cache_breakpoint(block: object) -> bool:
|
||||
return isinstance(block, dict) and any(block.get(key) is not None for key in CACHE_BREAKPOINT_KEYS)
|
||||
|
||||
|
||||
def _accepts_prompt_cache_breakpoint(block: object) -> bool:
|
||||
return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES
|
||||
|
||||
|
||||
class AnthropicCacheControlHook(CustomPromptManagement):
|
||||
def get_chat_completion_prompt(
|
||||
|
|
@ -81,13 +144,32 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
# provider transform, where each tool_config point appends at most one
|
||||
# cachePoint to the tools. That block also counts toward Anthropic's
|
||||
# limit, so reserve a slot for it here to leave room.
|
||||
reserved_blocks: Final = 1 if any(p.get("location") == "tool_config" for p in remaining_points) else 0
|
||||
|
||||
stamped_dialect: Final = injection_points[0].get("_litellm_openai_dialect")
|
||||
openai_dialect: Final = (
|
||||
stamped_dialect
|
||||
if isinstance(stamped_dialect, bool)
|
||||
else AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
|
||||
model,
|
||||
non_default_params.get("custom_llm_provider"),
|
||||
non_default_params.get("api_base") or non_default_params.get("base_url"),
|
||||
non_default_params.get("prompt_cache_options"),
|
||||
)
|
||||
)
|
||||
reserved_blocks: Final = (
|
||||
1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0
|
||||
)
|
||||
breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages)
|
||||
processed_messages = self._apply_message_injections(
|
||||
points=message_points,
|
||||
messages=processed_messages,
|
||||
max_blocks=MAX_CACHE_CONTROL_BLOCKS - reserved_blocks,
|
||||
openai_dialect=openai_dialect,
|
||||
)
|
||||
if (
|
||||
openai_dialect
|
||||
and AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages) > breakpoints_before
|
||||
):
|
||||
non_default_params.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit"))
|
||||
|
||||
# Pass through non-message injection points for provider-specific handling
|
||||
if remaining_points:
|
||||
|
|
@ -97,11 +179,43 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
|
||||
return model, processed_messages, non_default_params
|
||||
|
||||
@staticmethod
|
||||
def _targets_openai_prompt_cache_breakpoint(
|
||||
model: str | None,
|
||||
custom_llm_provider: str | None,
|
||||
api_base: object = None,
|
||||
prompt_cache_options: object = None,
|
||||
) -> bool:
|
||||
if model is None or not supports_openai_prompt_cache_breakpoint(model):
|
||||
return False
|
||||
if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai":
|
||||
return False
|
||||
return prompt_cache_options is not None or targets_openai_api(api_base)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_provider(model: str) -> str | None:
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
try:
|
||||
_, provider, _, _ = get_llm_provider(model=model)
|
||||
except BadRequestError:
|
||||
return None
|
||||
return provider
|
||||
|
||||
@staticmethod
|
||||
def _count_request_cache_breakpoints(messages: Iterable[object], system: object = None) -> int:
|
||||
system_blocks: Final = (
|
||||
sum(1 for block in system if _carries_cache_breakpoint(block)) if isinstance(system, list) else 0
|
||||
)
|
||||
return system_blocks + sum(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages)
|
||||
|
||||
@staticmethod
|
||||
def _apply_message_injections(
|
||||
points: list[CacheControlMessageInjectionPoint],
|
||||
messages: list[AllMessageValues],
|
||||
max_blocks: int,
|
||||
openai_dialect: bool = False,
|
||||
) -> list[AllMessageValues]:
|
||||
"""Apply message-level cache control injection points in order.
|
||||
|
||||
|
|
@ -112,7 +226,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
``max_blocks`` is reached. Injection points are honored in config order,
|
||||
so earlier points win when slots are scarce.
|
||||
"""
|
||||
used_blocks = sum(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages)
|
||||
used_blocks = AnthropicCacheControlHook._count_request_cache_breakpoints(messages)
|
||||
|
||||
limit_reached = False
|
||||
for point in points:
|
||||
|
|
@ -134,16 +248,17 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
continue
|
||||
|
||||
messages[target_index] = AnthropicCacheControlHook._safe_insert_cache_control_in_message(
|
||||
messages[target_index], control
|
||||
messages[target_index], control, openai_dialect
|
||||
)
|
||||
used_blocks += 1
|
||||
if AnthropicCacheControlHook._message_has_cache_control(messages[target_index]):
|
||||
used_blocks += 1
|
||||
|
||||
if limit_reached:
|
||||
break
|
||||
|
||||
if limit_reached:
|
||||
verbose_logger.warning(
|
||||
"AnthropicCacheControlHook: Reached the Anthropic limit of %s cache_control blocks. Skipping further injection.",
|
||||
"AnthropicCacheControlHook: Reached the provider limit of %s cache breakpoints. Skipping further injection.",
|
||||
MAX_CACHE_CONTROL_BLOCKS,
|
||||
)
|
||||
|
||||
|
|
@ -189,16 +304,13 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
return []
|
||||
|
||||
@staticmethod
|
||||
def _count_cache_control_blocks(message: AllMessageValues) -> int:
|
||||
"""Count cache_control breakpoints on a message (message + content level)."""
|
||||
count = 0
|
||||
if message.get("cache_control") is not None:
|
||||
count += 1
|
||||
def _count_cache_control_blocks(message: object) -> int:
|
||||
if not isinstance(message, dict):
|
||||
return 0
|
||||
count = 1 if _carries_cache_breakpoint(message) else 0
|
||||
content: Final = message.get("content")
|
||||
if isinstance(content, list):
|
||||
for block in content:
|
||||
if isinstance(block, dict) and block.get("cache_control") is not None:
|
||||
count += 1
|
||||
count += sum(1 for block in content if _carries_cache_breakpoint(block))
|
||||
return count
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -208,7 +320,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
|
||||
@staticmethod
|
||||
def _safe_insert_cache_control_in_message(
|
||||
message: AllMessageValues, control: ChatCompletionCachedContent
|
||||
message: AllMessageValues, control: ChatCompletionCachedContent, openai_dialect: bool = False
|
||||
) -> AllMessageValues:
|
||||
"""
|
||||
Safe way to insert cache control in a message
|
||||
|
|
@ -221,6 +333,9 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
Per Anthropic's API specification, when using multiple content blocks,
|
||||
only the last content block can have cache_control.
|
||||
"""
|
||||
if openai_dialect:
|
||||
return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(message)
|
||||
|
||||
message_content: Final = message.get("content", None)
|
||||
|
||||
# 1. if string, insert cache control in the message
|
||||
|
|
@ -232,11 +347,51 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
message_content[-1]["cache_control"] = control
|
||||
return message
|
||||
|
||||
@staticmethod
|
||||
def _insert_prompt_cache_breakpoint_in_message(message: AllMessageValues) -> AllMessageValues:
|
||||
if message.get("role") == "assistant":
|
||||
return message
|
||||
message_content: Final = message.get("content", None)
|
||||
if isinstance(message_content, str):
|
||||
marked: Final = copy.copy(message)
|
||||
marked["content"] = [
|
||||
with_prompt_cache_breakpoint(
|
||||
ChatCompletionTextObject(type="text", text=message_content), PromptCacheBreakpoint(mode="explicit")
|
||||
)
|
||||
]
|
||||
return marked
|
||||
if isinstance(message_content, list):
|
||||
target_index: Final = next(
|
||||
(
|
||||
index
|
||||
for index in range(len(message_content) - 1, -1, -1)
|
||||
if _accepts_prompt_cache_breakpoint(message_content[index])
|
||||
),
|
||||
None,
|
||||
)
|
||||
if target_index is not None:
|
||||
message_content[target_index] = with_prompt_cache_breakpoint(
|
||||
message_content[target_index], PromptCacheBreakpoint(mode="explicit")
|
||||
)
|
||||
return message
|
||||
|
||||
@staticmethod
|
||||
def _system_block_with_breakpoint(
|
||||
block: Mapping[str, object], control: ChatCompletionCachedContent, openai_dialect: bool
|
||||
) -> Mapping[str, object]:
|
||||
marker: Final = (
|
||||
("prompt_cache_breakpoint", PromptCacheBreakpoint(mode="explicit"))
|
||||
if openai_dialect
|
||||
else ("cache_control", control)
|
||||
)
|
||||
return {**block, marker[0]: marker[1]}
|
||||
|
||||
@staticmethod
|
||||
def apply_to_anthropic_messages_request(
|
||||
messages: list[dict],
|
||||
system: str | list | None,
|
||||
injection_points: list[CacheControlInjectionPoint],
|
||||
openai_dialect: bool = False,
|
||||
) -> tuple[list[dict], str | list | None, list[CacheControlInjectionPoint]]:
|
||||
"""Apply cache control injection for the Anthropic-native v1/messages endpoint.
|
||||
|
||||
|
|
@ -262,30 +417,32 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
else:
|
||||
remaining_points.append(point)
|
||||
|
||||
reserved_blocks: Final = 1 if any(p.get("location") == "tool_config" for p in remaining_points) else 0
|
||||
reserved_blocks: Final = (
|
||||
1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0
|
||||
)
|
||||
max_blocks: Final = MAX_CACHE_CONTROL_BLOCKS - reserved_blocks
|
||||
|
||||
used_blocks = sum(
|
||||
AnthropicCacheControlHook._count_cache_control_blocks(cast(AllMessageValues, msg))
|
||||
for msg in processed_messages
|
||||
)
|
||||
if isinstance(processed_system, list):
|
||||
used_blocks += sum(
|
||||
1 for b in processed_system if isinstance(b, dict) and b.get("cache_control") is not None
|
||||
)
|
||||
message_blocks: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages)
|
||||
system_blocks = AnthropicCacheControlHook._count_request_cache_breakpoints((), processed_system)
|
||||
|
||||
if system_points and processed_system is not None and used_blocks < max_blocks:
|
||||
if system_points and processed_system is not None and message_blocks + system_blocks < max_blocks:
|
||||
system_already_has_cc: Final = isinstance(processed_system, list) and any(
|
||||
isinstance(b, dict) and b.get("cache_control") is not None for b in processed_system
|
||||
_carries_cache_breakpoint(b) for b in processed_system
|
||||
)
|
||||
if not system_already_has_cc:
|
||||
control: Final = system_points[0].get("control") or ChatCompletionCachedContent(type="ephemeral")
|
||||
if isinstance(processed_system, str):
|
||||
processed_system = [{"type": "text", "text": processed_system, "cache_control": control}]
|
||||
used_blocks += 1
|
||||
processed_system = [
|
||||
AnthropicCacheControlHook._system_block_with_breakpoint(
|
||||
AnthropicSystemMessageContent(type="text", text=processed_system), control, openai_dialect
|
||||
)
|
||||
]
|
||||
system_blocks += 1
|
||||
elif len(processed_system) > 0 and isinstance(processed_system[-1], dict):
|
||||
processed_system[-1] = {**processed_system[-1], "cache_control": control}
|
||||
used_blocks += 1
|
||||
processed_system[-1] = AnthropicCacheControlHook._system_block_with_breakpoint(
|
||||
processed_system[-1], control, openai_dialect
|
||||
)
|
||||
system_blocks += 1
|
||||
|
||||
for i, msg in enumerate(processed_messages):
|
||||
content = msg.get("content")
|
||||
|
|
@ -295,7 +452,8 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
processed_messages = AnthropicCacheControlHook._apply_message_injections(
|
||||
points=message_points,
|
||||
messages=cast(list[AllMessageValues], processed_messages),
|
||||
max_blocks=max_blocks - used_blocks,
|
||||
max_blocks=max_blocks - system_blocks,
|
||||
openai_dialect=openai_dialect,
|
||||
)
|
||||
|
||||
return processed_messages, processed_system, remaining_points
|
||||
|
|
@ -315,17 +473,57 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
return ChatCompletionCachedContent(type="ephemeral")
|
||||
|
||||
@staticmethod
|
||||
def _stamped_as_judged(points: list[CacheControlInjectionPoint]) -> list[dict[str, object]]:
|
||||
def _stamped_as_judged(points: Sequence[CacheControlInjectionPoint]) -> Sequence[Mapping[str, object]]:
|
||||
"""Mark written-back points as having passed the client cache_control judgment.
|
||||
|
||||
Builds copies because config-owned point dicts are shared across
|
||||
requests; mutating them would leak the stamp into future requests.
|
||||
"""
|
||||
return [{**point, "_litellm_judged": True} for point in points]
|
||||
return AnthropicCacheControlHook._stamped(points, "_litellm_judged", True)
|
||||
|
||||
@staticmethod
|
||||
def _judged_configured_points(
|
||||
points: Sequence[CacheControlInjectionPoint],
|
||||
messages: list[AllMessageValues],
|
||||
tools: list[object] | None,
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
api_base: object,
|
||||
prompt_cache_options: object,
|
||||
) -> Sequence[Mapping[str, object]] | None:
|
||||
if AnthropicCacheControlHook._should_stand_down(points, messages, None, tools):
|
||||
return None
|
||||
return AnthropicCacheControlHook._stamped_with_dialect(
|
||||
points, model, custom_llm_provider, api_base, prompt_cache_options
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _stamped_with_dialect(
|
||||
points: Sequence[CacheControlInjectionPoint],
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
api_base: object,
|
||||
prompt_cache_options: object,
|
||||
) -> Sequence[Mapping[str, object]]:
|
||||
if not supports_openai_prompt_cache_breakpoint(model):
|
||||
return points
|
||||
return AnthropicCacheControlHook._stamped(
|
||||
points,
|
||||
"_litellm_openai_dialect",
|
||||
AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
|
||||
model, custom_llm_provider, api_base, prompt_cache_options
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _stamped(
|
||||
points: Sequence[CacheControlInjectionPoint], key: str, value: object
|
||||
) -> Sequence[Mapping[str, object]]:
|
||||
return [{**point, key: value} for point in points]
|
||||
|
||||
@staticmethod
|
||||
def _should_stand_down(
|
||||
points: list[CacheControlInjectionPoint],
|
||||
points: Sequence[CacheControlInjectionPoint],
|
||||
messages: list[AllMessageValues],
|
||||
system: str | list | None,
|
||||
tools: list | None,
|
||||
|
|
@ -359,11 +557,8 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
carry the mark either at the top level (Anthropic shape) or nested under
|
||||
``function`` (OpenAI shape); the Anthropic chat transform accepts both.
|
||||
"""
|
||||
if any(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages):
|
||||
if AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) > 0:
|
||||
return True
|
||||
if isinstance(system, list):
|
||||
if any(isinstance(block, dict) and block.get("cache_control") is not None for block in system):
|
||||
return True
|
||||
if tools is not None:
|
||||
return any(
|
||||
isinstance(tool, dict)
|
||||
|
|
@ -438,6 +633,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
custom_llm_provider: str | None,
|
||||
tools: list | None = None,
|
||||
enable_prompt_caching: bool | None = None,
|
||||
api_base: object = None,
|
||||
) -> None:
|
||||
"""For /chat/completions: resolve the injection points the request should carry.
|
||||
|
||||
|
|
@ -452,10 +648,19 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
unchanged.
|
||||
"""
|
||||
if non_default_params.get("cache_control_injection_points"):
|
||||
if AnthropicCacheControlHook._should_stand_down(
|
||||
non_default_params["cache_control_injection_points"], messages, None, tools
|
||||
):
|
||||
judged: Final = AnthropicCacheControlHook._judged_configured_points(
|
||||
non_default_params["cache_control_injection_points"],
|
||||
messages,
|
||||
tools,
|
||||
model,
|
||||
custom_llm_provider,
|
||||
api_base,
|
||||
non_default_params.get("prompt_cache_options"),
|
||||
)
|
||||
if judged is None:
|
||||
non_default_params.pop("cache_control_injection_points")
|
||||
else:
|
||||
non_default_params["cache_control_injection_points"] = judged
|
||||
return
|
||||
points: Final = AnthropicCacheControlHook.get_default_injection_points(
|
||||
messages=messages,
|
||||
|
|
@ -476,6 +681,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
model: str | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
tools: list[dict] | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> tuple[list[dict], str | list | None]:
|
||||
"""Extract cache_control_injection_points from kwargs and apply if present.
|
||||
|
||||
|
|
@ -513,11 +719,21 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
if not injection_points:
|
||||
return messages, system
|
||||
|
||||
openai_dialect: Final = AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
|
||||
model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options")
|
||||
)
|
||||
breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system)
|
||||
messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
|
||||
messages=messages,
|
||||
system=system,
|
||||
injection_points=injection_points,
|
||||
openai_dialect=openai_dialect,
|
||||
)
|
||||
if (
|
||||
openai_dialect
|
||||
and AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) > breakpoints_before
|
||||
):
|
||||
kwargs.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit"))
|
||||
if remaining:
|
||||
kwargs["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(remaining)
|
||||
return messages, system
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from collections.abc import Iterable, Mapping, Sequence
|
|||
from itertools import groupby
|
||||
from os import PathLike
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, cast
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast
|
||||
|
||||
from openai.types.chat.chat_completion_custom_tool_param import (
|
||||
CustomFormatGrammar,
|
||||
|
|
@ -1325,6 +1325,16 @@ def check_is_function_call(logging_obj: "LoggingClass") -> bool:
|
|||
return False
|
||||
|
||||
|
||||
_MarkedT: Final = TypeVar("_MarkedT", bound=Mapping[str, object])
|
||||
|
||||
|
||||
def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
|
||||
if marker is None:
|
||||
return target
|
||||
marked: Final = {**target, "prompt_cache_breakpoint": marker} # mutable-ok: API message payload
|
||||
return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key
|
||||
|
||||
|
||||
def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Any:
|
||||
"""
|
||||
Filters a value from a dictionary
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import copy
|
|||
import hashlib
|
||||
import json
|
||||
from collections.abc import AsyncIterator, Iterator, Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, cast
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast
|
||||
|
||||
from litellm.llms.anthropic.experimental_pass_through.utils import (
|
||||
is_reasoning_auto_summary_enabled,
|
||||
|
|
@ -61,6 +61,7 @@ from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingCho
|
|||
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
parse_tool_call_arguments,
|
||||
with_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
THOUGHT_SIGNATURE_SEPARATOR,
|
||||
|
|
@ -245,6 +246,9 @@ class AnthropicAdapter:
|
|||
return anthropic_wrapper.anthropic_sse_wrapper()
|
||||
|
||||
|
||||
_BlockT: Final = TypeVar("_BlockT", bound=Mapping[str, object])
|
||||
|
||||
|
||||
class LiteLLMAnthropicMessagesAdapter:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
|
@ -308,6 +312,12 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
# Fallback for non-dict objects (shouldn't happen in practice)
|
||||
cast(dict[str, object], target)["cache_control"] = cache_control
|
||||
|
||||
@staticmethod
|
||||
def _add_prompt_cache_breakpoint_if_present(source: object, target: _BlockT) -> _BlockT:
|
||||
if isinstance(source, dict) and "prompt_cache_breakpoint" in source:
|
||||
return with_prompt_cache_breakpoint(target, source["prompt_cache_breakpoint"])
|
||||
return target
|
||||
|
||||
def translatable_anthropic_params(self) -> list[str]:
|
||||
"""
|
||||
Which anthropic params, we need to translate to the openai format.
|
||||
|
|
@ -368,7 +378,9 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
if content.get("type") == "text":
|
||||
text_obj = ChatCompletionTextObject(type="text", text=content.get("text", ""))
|
||||
self._add_cache_control_if_applicable(content, text_obj, model)
|
||||
new_user_content_list.append(text_obj)
|
||||
new_user_content_list.append(
|
||||
self._add_prompt_cache_breakpoint_if_present(content, text_obj)
|
||||
)
|
||||
elif content.get("type") == "image":
|
||||
# Convert Anthropic image format to OpenAI format
|
||||
source = content.get("source", {})
|
||||
|
|
@ -378,7 +390,9 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
image_url_obj = ChatCompletionImageUrlObject(url=openai_image_url)
|
||||
image_obj = ChatCompletionImageObject(type="image_url", image_url=image_url_obj)
|
||||
self._add_cache_control_if_applicable(content, image_obj, model)
|
||||
new_user_content_list.append(image_obj)
|
||||
new_user_content_list.append(
|
||||
self._add_prompt_cache_breakpoint_if_present(content, image_obj)
|
||||
)
|
||||
elif content.get("type") == "document":
|
||||
# Convert Anthropic document format (PDF, etc.) to OpenAI format
|
||||
source = content.get("source", {})
|
||||
|
|
@ -869,7 +883,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
continue
|
||||
text_obj = ChatCompletionTextObject(type="text", text=text)
|
||||
self._add_cache_control_if_applicable(block, text_obj, model)
|
||||
text_parts.append(text_obj)
|
||||
text_parts.append(self._add_prompt_cache_breakpoint_if_present(block, text_obj))
|
||||
return ChatCompletionSystemMessage(role="system", content=text_parts) if text_parts else None
|
||||
|
||||
def _add_system_message_to_messages(
|
||||
|
|
@ -900,7 +914,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
"text": block.get("text", ""),
|
||||
}
|
||||
self._add_cache_control_if_applicable(block, text_block, model_name)
|
||||
openai_system_content.append(text_block)
|
||||
openai_system_content.append(self._add_prompt_cache_breakpoint_if_present(block, text_block))
|
||||
if openai_system_content:
|
||||
new_messages.insert(
|
||||
0,
|
||||
|
|
|
|||
|
|
@ -230,7 +230,7 @@ async def anthropic_messages(
|
|||
)
|
||||
|
||||
messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(
|
||||
messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools
|
||||
messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools, api_base=api_base
|
||||
)
|
||||
|
||||
original_stream: Final = stream or kwargs.get("_websearch_interception_converted_stream", False)
|
||||
|
|
@ -422,7 +422,7 @@ def anthropic_messages_handler(
|
|||
)
|
||||
|
||||
messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(
|
||||
messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools
|
||||
messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools, api_base=api_base
|
||||
)
|
||||
|
||||
metadata = validate_anthropic_api_metadata(metadata)
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from typing import Any, Final, cast
|
|||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
TOOL_RESULT_IMAGE_BOUNDARY,
|
||||
TOOL_RESULT_IMAGE_PLACEHOLDER,
|
||||
with_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import (
|
||||
reasoning_effort_from_thinking_budget,
|
||||
|
|
@ -82,7 +83,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
@staticmethod
|
||||
def _translate_midturn_system_content_to_responses(
|
||||
content: str | Iterable[AnthropicSystemMessageContent],
|
||||
) -> list[dict[str, str]]: # mutable-ok: API message payload
|
||||
) -> list[dict[str, object]]: # mutable-ok: API message payload
|
||||
"""Convert in-sequence system content to Responses input-text parts."""
|
||||
if isinstance(content, str):
|
||||
return (
|
||||
|
|
@ -91,7 +92,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
if not isinstance(content, list):
|
||||
return [] # mutable-ok: API message payload
|
||||
return [ # mutable-ok: API message payload
|
||||
{"type": "input_text", "text": text} # mutable-ok: API message payload
|
||||
with_prompt_cache_breakpoint(
|
||||
{"type": "input_text", "text": text}, block.get("prompt_cache_breakpoint")
|
||||
) # mutable-ok: API message payload
|
||||
for block in content
|
||||
if isinstance(block, dict) and block.get("type") == "text" and (text := block.get("text")) # pyright: ignore[reportUnnecessaryIsInstance] # untrusted client payload
|
||||
]
|
||||
|
|
@ -146,11 +149,20 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
continue
|
||||
btype = block.get("type")
|
||||
if btype == "text":
|
||||
user_parts.append({"type": "input_text", "text": block.get("text", "")})
|
||||
user_parts.append(
|
||||
with_prompt_cache_breakpoint(
|
||||
{"type": "input_text", "text": block.get("text", "")},
|
||||
block.get("prompt_cache_breakpoint"),
|
||||
)
|
||||
)
|
||||
elif btype == "image":
|
||||
url = self._translate_anthropic_image_source_to_url(cast(dict, block.get("source", {})))
|
||||
if url:
|
||||
user_parts.append({"type": "input_image", "image_url": url})
|
||||
user_parts.append(
|
||||
with_prompt_cache_breakpoint(
|
||||
{"type": "input_image", "image_url": url}, block.get("prompt_cache_breakpoint")
|
||||
)
|
||||
)
|
||||
elif btype == "tool_result":
|
||||
tool_use_id = block.get("tool_use_id", "")
|
||||
inner = block.get("content")
|
||||
|
|
@ -376,19 +388,36 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
anthropic_request["messages"],
|
||||
)
|
||||
|
||||
input_items: Final = self.translate_messages_to_responses_input(messages_list)
|
||||
system: Final = anthropic_request.get("system")
|
||||
developer_parts: Final = (
|
||||
self._translate_midturn_system_content_to_responses(system)
|
||||
if isinstance(system, list)
|
||||
and any(isinstance(block, dict) and block.get("prompt_cache_breakpoint") is not None for block in system)
|
||||
else ()
|
||||
)
|
||||
if developer_parts:
|
||||
input_items.insert(
|
||||
0,
|
||||
{ # mutable-ok: API message payload
|
||||
"type": "message",
|
||||
"role": "developer",
|
||||
"content": developer_parts,
|
||||
},
|
||||
)
|
||||
|
||||
responses_kwargs: Final[dict[str, Any]] = {
|
||||
"model": model,
|
||||
"input": self.translate_messages_to_responses_input(messages_list),
|
||||
"input": input_items,
|
||||
}
|
||||
|
||||
# system -> instructions
|
||||
system: Final = anthropic_request.get("system")
|
||||
if system:
|
||||
if system and not developer_parts:
|
||||
if isinstance(system, str):
|
||||
responses_kwargs["instructions"] = system
|
||||
elif isinstance(system, list):
|
||||
text_parts = [b.get("text", "") for b in system if isinstance(b, dict) and b.get("type") == "text"]
|
||||
responses_kwargs["instructions"] = "\n".join(filter(None, text_parts))
|
||||
responses_kwargs["instructions"] = "\n".join(
|
||||
filter(None, (b.get("text", "") for b in system if isinstance(b, dict) and b.get("type") == "text"))
|
||||
)
|
||||
|
||||
# max_tokens -> max_output_tokens
|
||||
max_tokens: Final = anthropic_request.get("max_tokens")
|
||||
|
|
|
|||
|
|
@ -507,6 +507,7 @@ async def acompletion(
|
|||
custom_llm_provider=cast(str | None, custom_llm_provider), # cast-ok: read from untyped kwargs
|
||||
tools=tools,
|
||||
enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs
|
||||
api_base=kwargs.get("api_base") or base_url,
|
||||
)
|
||||
|
||||
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
|
||||
|
|
@ -5171,6 +5172,7 @@ def completion(
|
|||
custom_llm_provider=cast(str | None, kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs
|
||||
tools=tools,
|
||||
enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs
|
||||
api_base=kwargs.get("api_base") or base_url,
|
||||
)
|
||||
|
||||
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
|
||||
|
|
|
|||
|
|
@ -25368,6 +25368,7 @@
|
|||
"supports_none_reasoning_effort": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_cache_breakpoint": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
@ -25430,6 +25431,7 @@
|
|||
"supports_none_reasoning_effort": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_cache_breakpoint": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
@ -25492,6 +25494,7 @@
|
|||
"supports_none_reasoning_effort": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_cache_breakpoint": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
@ -25554,6 +25557,7 @@
|
|||
"supports_none_reasoning_effort": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_cache_breakpoint": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
|
|||
|
|
@ -37,24 +37,45 @@ def normalize_responses_api_stream_options(
|
|||
return ResponsesAPIStreamOptions(include_obfuscation=include_obfuscation)
|
||||
|
||||
|
||||
def _is_chat_text_part(part: object) -> bool:
|
||||
return isinstance(part, dict) and part.get("type") == "text"
|
||||
|
||||
|
||||
def _as_input_text_part(part: object) -> object:
|
||||
if isinstance(part, dict) and part.get("type") == "text":
|
||||
return {**part, "type": "input_text"} # mutable-ok: fresh part so the caller's block keeps its chat type
|
||||
return part
|
||||
|
||||
|
||||
class ResponsesAPIRequestUtils:
|
||||
"""Helper utils for constructing ResponseAPI requests"""
|
||||
|
||||
@staticmethod
|
||||
def shape_prompt_managed_message_for_responses(message: object) -> object:
|
||||
if not isinstance(message, dict) or message.get("role") == "assistant":
|
||||
return message
|
||||
content: object = message.get("content")
|
||||
if not isinstance(content, list) or not any(_is_chat_text_part(part) for part in content):
|
||||
return message
|
||||
shaped_content: Final = [_as_input_text_part(part) for part in content] # mutable-ok: Responses-shaped copy
|
||||
return {**message, "content": shaped_content} # mutable-ok: copy, the hook's message stays untouched
|
||||
|
||||
@staticmethod
|
||||
def merge_prompt_management_input(
|
||||
original_input: str | ResponseInputParam,
|
||||
client_input: list[AllMessageValues],
|
||||
merged_input: list[AllMessageValues],
|
||||
) -> list[object]:
|
||||
shape: Final = ResponsesAPIRequestUtils.shape_prompt_managed_message_for_responses
|
||||
if isinstance(original_input, str):
|
||||
return [*merged_input]
|
||||
return [shape(message) for message in merged_input]
|
||||
|
||||
original_items: Final = tuple(original_input)
|
||||
client_item_ids: Final = frozenset(id(item) for item in client_input)
|
||||
message_positions = tuple(index for index, item in enumerate(original_items) if id(item) in client_item_ids)
|
||||
|
||||
if len(message_positions) == len(original_items):
|
||||
return [*merged_input]
|
||||
return [shape(message) for message in merged_input]
|
||||
if not message_positions:
|
||||
verbose_logger.warning(
|
||||
"Prompt management hook returned messages without Responses API input messages; merged messages were ignored"
|
||||
|
|
@ -69,7 +90,7 @@ class ResponsesAPIRequestUtils:
|
|||
if corresponding_messages:
|
||||
merged_by_position: Final = dict(zip(message_positions, merged_input))
|
||||
return [
|
||||
merged_by_position[index] if index in merged_by_position else item
|
||||
shape(merged_by_position[index]) if index in merged_by_position else item
|
||||
for index, item in enumerate(original_items)
|
||||
]
|
||||
|
||||
|
|
@ -82,14 +103,14 @@ class ResponsesAPIRequestUtils:
|
|||
for index, position in enumerate(message_positions)
|
||||
}
|
||||
trailing_items: Final = original_items[message_positions[-1] + 1 :]
|
||||
return [item for merged in merged_input for item in (*prefixes.get(id(merged), ()), merged)] + list(
|
||||
return [item for merged in merged_input for item in (*prefixes.get(id(merged), ()), shape(merged))] + list(
|
||||
trailing_items
|
||||
)
|
||||
|
||||
verbose_logger.warning(
|
||||
"Prompt management hook replaced Responses API messages; non-message input items were dropped"
|
||||
)
|
||||
return [*merged_input]
|
||||
return [shape(message) for message in merged_input]
|
||||
|
||||
@staticmethod
|
||||
def merge_client_forwarded_headers(
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from typing import Literal
|
||||
|
||||
from typing_extensions import NotRequired, TypedDict
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm.types.llms.openai import ChatCompletionCachedContent
|
||||
|
||||
|
|
@ -13,6 +13,7 @@ class CacheControlMessageInjectionPoint(TypedDict):
|
|||
index: int | str | None # Optional: target by specific index
|
||||
control: ChatCompletionCachedContent | None
|
||||
_litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran
|
||||
_litellm_openai_dialect: NotRequired[ReadOnly[bool]]
|
||||
|
||||
|
||||
class CacheControlToolConfigInjectionPoint(TypedDict):
|
||||
|
|
@ -21,6 +22,7 @@ class CacheControlToolConfigInjectionPoint(TypedDict):
|
|||
location: Literal["tool_config"]
|
||||
control: ChatCompletionCachedContent | None
|
||||
_litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran
|
||||
_litellm_openai_dialect: NotRequired[ReadOnly[bool]]
|
||||
|
||||
|
||||
CacheControlInjectionPoint = CacheControlMessageInjectionPoint | CacheControlToolConfigInjectionPoint
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from .openai import (
|
|||
ChatCompletionCachedContent,
|
||||
ChatCompletionRedactedThinkingBlock,
|
||||
ChatCompletionThinkingBlock,
|
||||
PromptCacheBreakpoint,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -201,6 +202,7 @@ class AnthropicMessagesTextParam(TypedDict, total=False):
|
|||
type: Required[Literal["text"]]
|
||||
text: Required[str]
|
||||
cache_control: dict | ChatCompletionCachedContent | None
|
||||
prompt_cache_breakpoint: ReadOnly[PromptCacheBreakpoint]
|
||||
|
||||
|
||||
class AnthropicMessagesToolUseParam(TypedDict, total=False):
|
||||
|
|
@ -261,6 +263,7 @@ class AnthropicMessagesImageParam(TypedDict, total=False):
|
|||
type: Required[Literal["image"]]
|
||||
source: Required[AnthropicContentParamSource | AnthropicContentParamSourceFileId | AnthropicContentParamSourceUrl]
|
||||
cache_control: dict | ChatCompletionCachedContent | None
|
||||
prompt_cache_breakpoint: ReadOnly[PromptCacheBreakpoint]
|
||||
|
||||
|
||||
class CitationsObject(TypedDict):
|
||||
|
|
@ -347,6 +350,7 @@ class AnthropicSystemMessageContent(TypedDict, total=False):
|
|||
type: str
|
||||
text: str
|
||||
cache_control: dict | ChatCompletionCachedContent | None
|
||||
prompt_cache_breakpoint: ReadOnly[PromptCacheBreakpoint]
|
||||
|
||||
|
||||
class AnthropicMessagesSystemMessageParam(TypedDict, total=False):
|
||||
|
|
|
|||
|
|
@ -71,6 +71,7 @@ from pydantic import (
|
|||
)
|
||||
from typing_extensions import (
|
||||
NotRequired,
|
||||
ReadOnly,
|
||||
Required,
|
||||
TypedDict,
|
||||
override,
|
||||
|
|
@ -510,6 +511,15 @@ class ChatCompletionCachedContent(TypedDict):
|
|||
ttl: NotRequired[Literal["5m", "1h"]]
|
||||
|
||||
|
||||
class PromptCacheBreakpoint(TypedDict):
|
||||
mode: ReadOnly[Literal["explicit"]]
|
||||
|
||||
|
||||
class PromptCacheOptions(TypedDict, total=False):
|
||||
mode: ReadOnly[Literal["implicit", "explicit"]]
|
||||
ttl: ReadOnly[Literal["30m"]]
|
||||
|
||||
|
||||
class ChatCompletionThinkingBlock(TypedDict, total=False):
|
||||
type: Required[Literal["thinking"]]
|
||||
thinking: str
|
||||
|
|
@ -1148,6 +1158,7 @@ class ResponsesAPIOptionalRequestParams(TypedDict, total=False):
|
|||
max_tool_calls: int | None
|
||||
prompt_cache_key: str | None
|
||||
prompt_cache_retention: str | None
|
||||
prompt_cache_options: ReadOnly[PromptCacheOptions | None]
|
||||
stream_options: ResponsesAPIStreamOptions | None
|
||||
top_logprobs: int | None
|
||||
partial_images: int | None # Number of partial images to generate (1-3) for streaming image generation
|
||||
|
|
|
|||
|
|
@ -141,6 +141,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
|
|||
supports_tool_choice: bool | None
|
||||
supports_assistant_prefill: bool | None
|
||||
supports_prompt_caching: bool | None
|
||||
supports_prompt_cache_breakpoint: ReadOnly[bool | None]
|
||||
supports_computer_use: bool | None
|
||||
supports_audio_input: bool | None
|
||||
supports_embedding_image_input: bool | None
|
||||
|
|
|
|||
|
|
@ -2560,6 +2560,14 @@ def supports_prompt_caching(model: str, custom_llm_provider: str | None = None)
|
|||
)
|
||||
|
||||
|
||||
def supports_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool:
|
||||
return _supports_factory(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
key="supports_prompt_cache_breakpoint",
|
||||
)
|
||||
|
||||
|
||||
def supports_computer_use(model: str, custom_llm_provider: str | None = None) -> bool:
|
||||
"""
|
||||
Check if the given model supports computer use and return a boolean value.
|
||||
|
|
@ -5473,6 +5481,7 @@ def _get_model_info_helper(
|
|||
supports_tool_choice=None,
|
||||
supports_assistant_prefill=None,
|
||||
supports_prompt_caching=None,
|
||||
supports_prompt_cache_breakpoint=None,
|
||||
supports_computer_use=None,
|
||||
supports_pdf_input=None,
|
||||
)
|
||||
|
|
@ -5712,6 +5721,7 @@ def _get_model_info_helper(
|
|||
supports_tool_choice=_model_info.get("supports_tool_choice", None),
|
||||
supports_assistant_prefill=_model_info.get("supports_assistant_prefill", None),
|
||||
supports_prompt_caching=_model_info.get("supports_prompt_caching", None),
|
||||
supports_prompt_cache_breakpoint=_model_info.get("supports_prompt_cache_breakpoint", None),
|
||||
supports_audio_input=_model_info.get("supports_audio_input", None),
|
||||
supports_audio_output=_model_info.get("supports_audio_output", None),
|
||||
supports_pdf_input=_model_info.get("supports_pdf_input", None),
|
||||
|
|
@ -5846,6 +5856,7 @@ def get_model_info(
|
|||
supports_function_calling: Optional[bool]
|
||||
supports_tool_choice: Optional[bool]
|
||||
supports_prompt_caching: Optional[bool]
|
||||
supports_prompt_cache_breakpoint: Optional[bool]
|
||||
supports_audio_input: Optional[bool]
|
||||
supports_audio_output: Optional[bool]
|
||||
supports_pdf_input: Optional[bool]
|
||||
|
|
|
|||
|
|
@ -25368,6 +25368,7 @@
|
|||
"supports_none_reasoning_effort": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_cache_breakpoint": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
@ -25430,6 +25431,7 @@
|
|||
"supports_none_reasoning_effort": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_cache_breakpoint": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
@ -25492,6 +25494,7 @@
|
|||
"supports_none_reasoning_effort": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_cache_breakpoint": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
@ -25554,6 +25557,7 @@
|
|||
"supports_none_reasoning_effort": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_cache_breakpoint": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
|
|||
|
|
@ -664,6 +664,9 @@
|
|||
"supports_pdf_input": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_prompt_cache_breakpoint": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_prompt_caching": {
|
||||
"type": "boolean"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -14,12 +14,22 @@ import pytest
|
|||
|
||||
sys.path.insert(0, os.path.abspath("../..")) # Adds the parent directory to the system-path
|
||||
import litellm
|
||||
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
AnthropicCacheControlHook,
|
||||
supports_openai_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_openai_api_base_override(monkeypatch):
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", None)
|
||||
|
||||
|
||||
def _rendered_log_message(call):
|
||||
message = str(call.args[0])
|
||||
values = call.args[1:]
|
||||
|
|
@ -1996,3 +2006,769 @@ class TestAnthropicPromptCachingEnvVars:
|
|||
"""An unparseable TTL must fall back to Anthropic's 5m default, never reach the provider verbatim."""
|
||||
_, ttl = self._import_litellm_with_env({"LITELLM_ANTHROPIC_PROMPT_CACHING_TTL": value})
|
||||
assert ttl is None
|
||||
|
||||
|
||||
def _contains_key(value, key) -> bool:
|
||||
if isinstance(value, dict):
|
||||
return key in value or any(_contains_key(v, key) for v in value.values())
|
||||
if isinstance(value, list):
|
||||
return any(_contains_key(v, key) for v in value)
|
||||
return False
|
||||
|
||||
|
||||
class TestOpenAIPromptCacheBreakpoint:
|
||||
"""OpenAI GPT-5.6+ targets get content-block `prompt_cache_breakpoint` markers and a
|
||||
request-level `prompt_cache_options` instead of Anthropic `cache_control` (#37509)."""
|
||||
|
||||
EXPLICIT = {"mode": "explicit"}
|
||||
SYSTEM_POINT = [{"location": "message", "role": "system"}]
|
||||
|
||||
@staticmethod
|
||||
def _inject(messages, system, kwargs, model="openai/gpt-5.6", custom_llm_provider=None):
|
||||
return AnthropicCacheControlHook.maybe_inject_cache_control(
|
||||
copy.deepcopy(messages),
|
||||
copy.deepcopy(system),
|
||||
kwargs,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _chat(messages, params, model="openai/gpt-5.6"):
|
||||
return AnthropicCacheControlHook().get_chat_completion_prompt(
|
||||
model=model,
|
||||
messages=copy.deepcopy(messages),
|
||||
non_default_params=params,
|
||||
prompt_id=None,
|
||||
prompt_variables=None,
|
||||
dynamic_callback_params={},
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model,expected",
|
||||
[
|
||||
("gpt-5.6", True),
|
||||
("openai/gpt-5.6", True),
|
||||
("gpt-5.6-sol", True),
|
||||
("gpt-5.6-luna", True),
|
||||
("gpt-5.7", True),
|
||||
("gpt-6", True),
|
||||
("GPT-5.6", True),
|
||||
("gpt-5.5", False),
|
||||
("gpt-5", False),
|
||||
("gpt-5-chat-latest", False),
|
||||
("gpt-4.1", False),
|
||||
("o3", False),
|
||||
("claude-sonnet-4-5", False),
|
||||
],
|
||||
)
|
||||
def test_model_support_truth_table(self, model, expected):
|
||||
assert supports_openai_prompt_cache_breakpoint(model) is expected
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model,provider,expected",
|
||||
[
|
||||
("openai/gpt-5.6", None, True),
|
||||
("gpt-5.6", None, True),
|
||||
("gpt-5.6", "openai", True),
|
||||
("gpt-5.6", "azure", False),
|
||||
("azure/gpt-5.6", None, False),
|
||||
("openai/gpt-4.1", None, False),
|
||||
("anthropic/claude-sonnet-4-5", None, False),
|
||||
("no-provider-can-route-this-model", None, False),
|
||||
(None, "openai", False),
|
||||
],
|
||||
)
|
||||
def test_dialect_resolution(self, model, provider, expected):
|
||||
assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(model, provider) is expected
|
||||
|
||||
def test_count_covers_both_marker_kinds(self):
|
||||
message = {
|
||||
"role": "user",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
"content": [
|
||||
{"type": "text", "text": "a", "prompt_cache_breakpoint": self.EXPLICIT},
|
||||
{"type": "text", "text": "b", "cache_control": {"type": "ephemeral"}},
|
||||
{"type": "text", "text": "c"},
|
||||
],
|
||||
}
|
||||
assert AnthropicCacheControlHook._count_cache_control_blocks(message) == 3
|
||||
|
||||
def test_v1_messages_string_system_gets_block_breakpoint(self):
|
||||
kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)}
|
||||
messages, system = self._inject([{"role": "user", "content": "hi"}], "sys", kwargs)
|
||||
assert system == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}]
|
||||
assert messages == [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]
|
||||
assert kwargs == {"prompt_cache_options": self.EXPLICIT}
|
||||
assert not _contains_key(system, "cache_control")
|
||||
|
||||
def test_v1_messages_list_system_marks_last_block_only(self):
|
||||
kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)}
|
||||
system = [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}]
|
||||
_, result_system = self._inject([{"role": "user", "content": "hi"}], system, kwargs)
|
||||
assert result_system == [
|
||||
{"type": "text", "text": "a"},
|
||||
{"type": "text", "text": "b", "prompt_cache_breakpoint": self.EXPLICIT},
|
||||
]
|
||||
assert kwargs["prompt_cache_options"] == self.EXPLICIT
|
||||
|
||||
def test_v1_messages_targets_by_role(self):
|
||||
messages = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "first"}, {"type": "text", "text": "second"}]},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "reply"}]},
|
||||
{"role": "user", "content": "last"},
|
||||
]
|
||||
kwargs = {"cache_control_injection_points": [{"location": "message", "role": "user"}]}
|
||||
result, _ = self._inject(messages, None, kwargs)
|
||||
assert result[0]["content"] == [
|
||||
{"type": "text", "text": "first"},
|
||||
{"type": "text", "text": "second", "prompt_cache_breakpoint": self.EXPLICIT},
|
||||
]
|
||||
assert result[1] == messages[1]
|
||||
assert result[2]["content"] == [{"type": "text", "text": "last", "prompt_cache_breakpoint": self.EXPLICIT}]
|
||||
assert kwargs["prompt_cache_options"] == self.EXPLICIT
|
||||
|
||||
def test_v1_messages_targets_by_index(self):
|
||||
messages = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "first"}]},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "reply"}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "last"}]},
|
||||
]
|
||||
kwargs = {"cache_control_injection_points": [{"location": "message", "index": -1}]}
|
||||
result, _ = self._inject(messages, None, kwargs)
|
||||
assert result[:2] == messages[:2]
|
||||
assert result[2]["content"] == [{"type": "text", "text": "last", "prompt_cache_breakpoint": self.EXPLICIT}]
|
||||
|
||||
def test_v1_messages_control_field_is_ignored(self):
|
||||
ttl_control = {"type": "ephemeral", "ttl": "1h"}
|
||||
kwargs = {
|
||||
"cache_control_injection_points": [
|
||||
{"location": "message", "role": "system", "control": ttl_control},
|
||||
{"location": "message", "index": -1, "control": ttl_control},
|
||||
]
|
||||
}
|
||||
messages, system = self._inject([{"role": "user", "content": "hi"}], "sys", kwargs)
|
||||
assert system[0]["prompt_cache_breakpoint"] == self.EXPLICIT
|
||||
assert messages[0]["content"][-1]["prompt_cache_breakpoint"] == self.EXPLICIT
|
||||
assert not _contains_key(system, "cache_control")
|
||||
assert not _contains_key(messages, "cache_control")
|
||||
|
||||
def test_v1_messages_keeps_caller_prompt_cache_options(self):
|
||||
caller_options = {"mode": "explicit", "ttl": "30m"}
|
||||
kwargs = {
|
||||
"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT),
|
||||
"prompt_cache_options": dict(caller_options),
|
||||
}
|
||||
_, system = self._inject([{"role": "user", "content": "hi"}], "sys", kwargs)
|
||||
assert system[0]["prompt_cache_breakpoint"] == self.EXPLICIT
|
||||
assert kwargs["prompt_cache_options"] == caller_options
|
||||
|
||||
def test_v1_messages_no_prompt_cache_options_when_nothing_injected(self):
|
||||
kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)}
|
||||
messages, system = self._inject([{"role": "user", "content": "hi"}], None, kwargs)
|
||||
assert system is None
|
||||
assert "prompt_cache_options" not in kwargs
|
||||
assert not _contains_key(messages, "prompt_cache_breakpoint")
|
||||
|
||||
def test_v1_messages_anthropic_target_unchanged(self):
|
||||
kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)}
|
||||
_, system = self._inject(
|
||||
[{"role": "user", "content": "hi"}],
|
||||
"sys",
|
||||
kwargs,
|
||||
model="anthropic/claude-sonnet-4-5",
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
assert system == [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]
|
||||
assert kwargs == {}
|
||||
|
||||
def test_v1_messages_older_openai_model_keeps_cache_control(self):
|
||||
kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)}
|
||||
_, system = self._inject([{"role": "user", "content": "hi"}], "sys", kwargs, model="openai/gpt-4.1")
|
||||
assert system == [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]
|
||||
assert kwargs == {}
|
||||
|
||||
def test_v1_messages_client_content_breakpoint_makes_configured_points_stand_down(self):
|
||||
messages = [{"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]}]
|
||||
kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)}
|
||||
result, system = self._inject(messages, "sys", kwargs)
|
||||
assert result == messages
|
||||
assert system == "sys"
|
||||
assert kwargs == {}
|
||||
|
||||
def test_v1_messages_client_system_breakpoint_makes_configured_points_stand_down(self):
|
||||
system = [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}]
|
||||
messages = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]
|
||||
kwargs = {"cache_control_injection_points": [{"location": "message", "index": -1}]}
|
||||
result, result_system = self._inject(messages, system, kwargs)
|
||||
assert result == messages
|
||||
assert result_system == system
|
||||
assert kwargs == {}
|
||||
|
||||
def test_chat_system_string_wrapped_with_block_breakpoint(self):
|
||||
params = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)}
|
||||
messages = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}]
|
||||
_, processed, returned = self._chat(messages, params)
|
||||
assert processed[0] == {
|
||||
"role": "system",
|
||||
"content": [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}],
|
||||
}
|
||||
assert processed[1] == {"role": "user", "content": "hi"}
|
||||
assert returned is params
|
||||
assert returned == {"prompt_cache_options": self.EXPLICIT}
|
||||
|
||||
def test_chat_list_content_marks_last_block(self):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "look"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/a.png"}},
|
||||
],
|
||||
}
|
||||
]
|
||||
params = {"cache_control_injection_points": [{"location": "message", "index": -1}]}
|
||||
_, processed, _ = self._chat(messages, params)
|
||||
assert processed[0]["content"] == [
|
||||
{"type": "text", "text": "look"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "https://example.com/a.png"},
|
||||
"prompt_cache_breakpoint": self.EXPLICIT,
|
||||
},
|
||||
]
|
||||
assert params["prompt_cache_options"] == self.EXPLICIT
|
||||
|
||||
def test_chat_unprefixed_model_resolves_to_openai(self):
|
||||
params = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)}
|
||||
_, processed, _ = self._chat([{"role": "system", "content": "sys"}], params, model="gpt-5.6")
|
||||
assert processed[0]["content"] == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}]
|
||||
assert params["prompt_cache_options"] == self.EXPLICIT
|
||||
|
||||
def test_chat_keeps_caller_prompt_cache_options(self):
|
||||
params = {
|
||||
"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT),
|
||||
"prompt_cache_options": {"mode": "implicit"},
|
||||
}
|
||||
self._chat([{"role": "system", "content": "sys"}], params)
|
||||
assert params["prompt_cache_options"] == {"mode": "implicit"}
|
||||
|
||||
def test_chat_no_prompt_cache_options_when_nothing_injected(self):
|
||||
params = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)}
|
||||
messages = [{"role": "user", "content": "hi"}]
|
||||
_, processed, _ = self._chat(messages, params)
|
||||
assert processed == messages
|
||||
assert params == {}
|
||||
|
||||
@pytest.mark.parametrize("model", ["openai/gpt-4.1", "anthropic/claude-sonnet-4-5"])
|
||||
def test_chat_other_targets_keep_message_level_cache_control(self, model):
|
||||
params = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)}
|
||||
_, processed, _ = self._chat([{"role": "system", "content": "sys"}], params, model=model)
|
||||
assert processed[0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}}
|
||||
assert params == {}
|
||||
|
||||
def test_chat_client_breakpoint_makes_seeded_points_stand_down(self):
|
||||
params = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)}
|
||||
AnthropicCacheControlHook.maybe_seed_default_injection_points(
|
||||
non_default_params=params,
|
||||
messages=[
|
||||
{"role": "system", "content": "sys"},
|
||||
{"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]},
|
||||
],
|
||||
model="openai/gpt-5.6",
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
assert params == {}
|
||||
|
||||
def test_cap_counts_client_breakpoints_of_both_kinds(self):
|
||||
messages = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "a", "prompt_cache_breakpoint": self.EXPLICIT}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "b", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "c", "prompt_cache_breakpoint": self.EXPLICIT}]},
|
||||
{"role": "user", "content": "d"},
|
||||
{"role": "user", "content": "e"},
|
||||
]
|
||||
result = AnthropicCacheControlHook._apply_message_injections(
|
||||
points=[{"location": "message", "role": "user"}],
|
||||
messages=copy.deepcopy(messages),
|
||||
max_blocks=4,
|
||||
openai_dialect=True,
|
||||
)
|
||||
assert result[:3] == messages[:3]
|
||||
assert result[3]["content"] == [{"type": "text", "text": "d", "prompt_cache_breakpoint": self.EXPLICIT}]
|
||||
assert result[4] == {"role": "user", "content": "e"}
|
||||
|
||||
|
||||
class TestOpenAIPromptCacheBreakpointPlacementRules:
|
||||
"""OpenAI dialect only marks blocks OpenAI (and the /v1/messages bridges) can carry (#37509)."""
|
||||
|
||||
EXPLICIT = {"mode": "explicit"}
|
||||
|
||||
def _chat(self, messages, points, model="openai/gpt-5.6"):
|
||||
params = {"cache_control_injection_points": copy.deepcopy(points)}
|
||||
_, out, params = AnthropicCacheControlHook().get_chat_completion_prompt(
|
||||
model=model,
|
||||
messages=copy.deepcopy(messages),
|
||||
non_default_params=params,
|
||||
prompt_id=None,
|
||||
prompt_variables=None,
|
||||
dynamic_callback_params={},
|
||||
)
|
||||
return out, params
|
||||
|
||||
def test_assistant_message_is_never_marked_on_chat_path(self):
|
||||
messages = [{"role": "user", "content": "q"}, {"role": "assistant", "content": "a"}]
|
||||
out, params = self._chat(messages, [{"location": "message", "role": "assistant"}])
|
||||
assert out == messages
|
||||
assert "prompt_cache_options" not in params
|
||||
|
||||
def test_tool_message_text_is_marked_on_chat_path(self):
|
||||
messages = [
|
||||
{"role": "user", "content": "weather?"},
|
||||
{"role": "assistant", "content": None, "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "w", "arguments": "{}"}}]},
|
||||
{"role": "tool", "tool_call_id": "c1", "content": "sunny"},
|
||||
]
|
||||
out, params = self._chat(messages, [{"location": "message", "index": -1}])
|
||||
assert out[2]["content"] == [{"type": "text", "text": "sunny", "prompt_cache_breakpoint": self.EXPLICIT}]
|
||||
assert params["prompt_cache_options"] == self.EXPLICIT
|
||||
|
||||
def test_tool_result_only_turn_is_skipped_on_v1_messages(self):
|
||||
messages = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "q"}]},
|
||||
{"role": "assistant", "content": [{"type": "tool_use", "id": "t1", "name": "w", "input": {}}]},
|
||||
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "sunny"}]},
|
||||
]
|
||||
kwargs = {"cache_control_injection_points": [{"location": "message", "index": -1}]}
|
||||
out, system = AnthropicCacheControlHook.maybe_inject_cache_control(
|
||||
copy.deepcopy(messages), None, kwargs, model="openai/gpt-5.6"
|
||||
)
|
||||
assert out == messages
|
||||
assert system is None
|
||||
assert "prompt_cache_options" not in kwargs
|
||||
|
||||
def test_assistant_turn_is_skipped_on_v1_messages(self):
|
||||
messages = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "q"}]},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "a"}]},
|
||||
]
|
||||
kwargs = {"cache_control_injection_points": [{"location": "message", "role": "assistant"}]}
|
||||
out, _ = AnthropicCacheControlHook.maybe_inject_cache_control(
|
||||
copy.deepcopy(messages), None, kwargs, model="openai/gpt-5.6"
|
||||
)
|
||||
assert out == messages
|
||||
assert "prompt_cache_options" not in kwargs
|
||||
|
||||
def test_text_after_tool_result_is_marked(self):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "tool_result", "tool_use_id": "t1", "content": "sunny"},
|
||||
{"type": "text", "text": "thanks"},
|
||||
],
|
||||
}
|
||||
]
|
||||
kwargs = {"cache_control_injection_points": [{"location": "message", "index": -1}]}
|
||||
out, _ = AnthropicCacheControlHook.maybe_inject_cache_control(messages, None, kwargs, model="openai/gpt-5.6")
|
||||
assert out[0]["content"] == [
|
||||
{"type": "tool_result", "tool_use_id": "t1", "content": "sunny"},
|
||||
{"type": "text", "text": "thanks", "prompt_cache_breakpoint": self.EXPLICIT},
|
||||
]
|
||||
assert kwargs["prompt_cache_options"] == self.EXPLICIT
|
||||
|
||||
def test_marker_walks_back_to_last_eligible_block(self):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "read this"},
|
||||
{"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "doc"}},
|
||||
],
|
||||
}
|
||||
]
|
||||
kwargs = {"cache_control_injection_points": [{"location": "message", "index": -1}]}
|
||||
out, _ = AnthropicCacheControlHook.maybe_inject_cache_control(messages, None, kwargs, model="openai/gpt-5.6")
|
||||
assert out[0]["content"][0] == {"type": "text", "text": "read this", "prompt_cache_breakpoint": self.EXPLICIT}
|
||||
assert "prompt_cache_breakpoint" not in out[0]["content"][1]
|
||||
|
||||
def test_skipped_block_does_not_consume_a_slot(self):
|
||||
messages = [{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t0", "content": "r"}]}] + [
|
||||
{"role": "user", "content": [{"type": "text", "text": f"m{i}"}]} for i in range(4)
|
||||
]
|
||||
kwargs = {"cache_control_injection_points": [{"location": "message", "role": "user"}]}
|
||||
out, _ = AnthropicCacheControlHook.maybe_inject_cache_control(messages, None, kwargs, model="openai/gpt-5.6")
|
||||
assert "prompt_cache_breakpoint" not in out[0]["content"][0]
|
||||
assert all(msg["content"][0]["prompt_cache_breakpoint"] == self.EXPLICIT for msg in out[1:])
|
||||
|
||||
|
||||
class TestChatPathProviderStamp:
|
||||
"""The chat path learns the dialect decision (provider, api_base, opt-in) through the seeded points (#37509)."""
|
||||
|
||||
POINTS = [{"location": "message", "role": "system"}]
|
||||
MESSAGES = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}]
|
||||
ANTHROPIC_STYLE = {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}}
|
||||
OPENAI_STYLE = [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}]
|
||||
CUSTOM_API_BASE = "http://127.0.0.1:9/v1"
|
||||
|
||||
def _seed_and_run(self, model, custom_llm_provider, api_base=None, prompt_cache_options=None):
|
||||
params = {"cache_control_injection_points": copy.deepcopy(self.POINTS)}
|
||||
if prompt_cache_options is not None:
|
||||
params["prompt_cache_options"] = prompt_cache_options
|
||||
AnthropicCacheControlHook.maybe_seed_default_injection_points(
|
||||
non_default_params=params,
|
||||
messages=copy.deepcopy(self.MESSAGES),
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
)
|
||||
return self._run(params, model)
|
||||
|
||||
def _run(self, params, model):
|
||||
_, out, params = AnthropicCacheControlHook().get_chat_completion_prompt(
|
||||
model=model,
|
||||
messages=copy.deepcopy(self.MESSAGES),
|
||||
non_default_params=params,
|
||||
prompt_id=None,
|
||||
prompt_variables=None,
|
||||
dynamic_callback_params={},
|
||||
)
|
||||
return out, params
|
||||
|
||||
def test_openai_compatible_provider_keeps_anthropic_style_markers(self):
|
||||
out, params = self._seed_and_run("gpt-5.6", "hosted_vllm")
|
||||
assert out[0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}}
|
||||
assert "prompt_cache_options" not in params
|
||||
|
||||
def test_explicit_openai_provider_uses_openai_dialect(self):
|
||||
out, params = self._seed_and_run("gpt-5.6", "openai")
|
||||
assert out[0]["content"] == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}]
|
||||
assert params["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
||||
def test_bare_gpt_model_without_provider_resolves_to_openai(self):
|
||||
out, params = self._seed_and_run("gpt-5.6", None)
|
||||
assert out[0]["content"] == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}]
|
||||
assert params["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
||||
def test_points_keep_identity_for_models_below_gpt_5_6(self):
|
||||
points = copy.deepcopy(self.POINTS)
|
||||
params = {"cache_control_injection_points": points}
|
||||
AnthropicCacheControlHook.maybe_seed_default_injection_points(
|
||||
non_default_params=params,
|
||||
messages=copy.deepcopy(self.MESSAGES),
|
||||
model="anthropic/claude-sonnet-4-5",
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
assert params["cache_control_injection_points"] is points
|
||||
|
||||
def test_provider_lookup_skipped_for_models_below_gpt_5_6(self):
|
||||
from unittest.mock import patch
|
||||
|
||||
with patch.object(AnthropicCacheControlHook, "_resolve_provider") as resolve:
|
||||
assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("gpt-4.1", None) is False
|
||||
assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("my-custom-model", None) is False
|
||||
resolve.assert_not_called()
|
||||
|
||||
def test_litellm_proxy_target_keeps_anthropic_style_markers(self):
|
||||
out, params = self._seed_and_run("litellm_proxy/gpt-5.6", None)
|
||||
assert out[0] == self.ANTHROPIC_STYLE
|
||||
assert "prompt_cache_options" not in params
|
||||
|
||||
def test_custom_api_base_keeps_anthropic_style_markers(self):
|
||||
out, params = self._seed_and_run("gpt-5.6", None, api_base=self.CUSTOM_API_BASE)
|
||||
assert out[0] == self.ANTHROPIC_STYLE
|
||||
assert "prompt_cache_options" not in params
|
||||
|
||||
def test_custom_api_base_opts_in_through_prompt_cache_options(self):
|
||||
out, params = self._seed_and_run(
|
||||
"gpt-5.6", None, api_base=self.CUSTOM_API_BASE, prompt_cache_options={"mode": "explicit"}
|
||||
)
|
||||
assert out[0]["content"] == self.OPENAI_STYLE
|
||||
assert params["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
||||
def test_regional_openai_api_base_uses_openai_dialect(self):
|
||||
out, params = self._seed_and_run("gpt-5.6", None, api_base="https://eu.api.openai.com/v1")
|
||||
assert out[0]["content"] == self.OPENAI_STYLE
|
||||
assert params["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
||||
@pytest.mark.parametrize("env_var", ["OPENAI_BASE_URL", "OPENAI_API_BASE"])
|
||||
def test_env_api_base_override_keeps_anthropic_style_markers(self, monkeypatch, env_var):
|
||||
monkeypatch.setenv(env_var, self.CUSTOM_API_BASE)
|
||||
out, params = self._seed_and_run("gpt-5.6", None)
|
||||
assert out[0] == self.ANTHROPIC_STYLE
|
||||
assert "prompt_cache_options" not in params
|
||||
|
||||
def test_global_litellm_api_base_keeps_anthropic_style_markers(self, monkeypatch):
|
||||
monkeypatch.setattr(litellm, "api_base", self.CUSTOM_API_BASE)
|
||||
out, params = self._seed_and_run("gpt-5.6", None)
|
||||
assert out[0] == self.ANTHROPIC_STYLE
|
||||
assert "prompt_cache_options" not in params
|
||||
|
||||
def test_request_api_base_wins_over_env_override(self, monkeypatch):
|
||||
monkeypatch.setenv("OPENAI_BASE_URL", self.CUSTOM_API_BASE)
|
||||
out, params = self._seed_and_run("gpt-5.6", None, api_base="https://api.openai.com/v1")
|
||||
assert out[0]["content"] == self.OPENAI_STYLE
|
||||
assert params["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api_base,expected",
|
||||
[(None, True), ("http://127.0.0.1:9/v1", False), ("https://eu.api.openai.com/v1", True)],
|
||||
)
|
||||
def test_seed_stamps_the_dialect_decision(self, api_base, expected):
|
||||
params = {"cache_control_injection_points": copy.deepcopy(self.POINTS)}
|
||||
AnthropicCacheControlHook.maybe_seed_default_injection_points(
|
||||
non_default_params=params,
|
||||
messages=copy.deepcopy(self.MESSAGES),
|
||||
model="gpt-5.6",
|
||||
custom_llm_provider=None,
|
||||
api_base=api_base,
|
||||
)
|
||||
assert params["cache_control_injection_points"][0]["_litellm_openai_dialect"] is expected
|
||||
|
||||
def test_stamp_is_authoritative_over_request_params(self):
|
||||
points = [{**self.POINTS[0], "_litellm_openai_dialect": False}]
|
||||
out, params = self._run({"cache_control_injection_points": points, "custom_llm_provider": "openai"}, "gpt-5.6")
|
||||
assert out[0] == self.ANTHROPIC_STYLE
|
||||
assert "prompt_cache_options" not in params
|
||||
|
||||
def test_unstamped_points_read_api_base_from_request_params(self):
|
||||
params = {"cache_control_injection_points": copy.deepcopy(self.POINTS), "api_base": self.CUSTOM_API_BASE}
|
||||
out, params = self._run(params, "gpt-5.6")
|
||||
assert out[0] == self.ANTHROPIC_STYLE
|
||||
assert "prompt_cache_options" not in params
|
||||
|
||||
def test_unstamped_points_read_prompt_cache_options_from_request_params(self):
|
||||
params = {
|
||||
"cache_control_injection_points": copy.deepcopy(self.POINTS),
|
||||
"api_base": self.CUSTOM_API_BASE,
|
||||
"prompt_cache_options": {"mode": "explicit"},
|
||||
}
|
||||
out, params = self._run(params, "gpt-5.6")
|
||||
assert out[0]["content"] == self.OPENAI_STYLE
|
||||
assert params["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
||||
|
||||
class TestClientBreakpointsCountedOnce:
|
||||
def test_client_message_breakpoints_are_not_double_counted(self):
|
||||
messages = [{"role": "user", "content": [{"type": "text", "text": "m0", "cache_control": {"type": "ephemeral"}}]}] + [
|
||||
{"role": "user", "content": [{"type": "text", "text": f"m{i}"}]} for i in range(1, 4)
|
||||
]
|
||||
out, system, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
|
||||
messages=messages,
|
||||
system="sys",
|
||||
injection_points=[
|
||||
{"location": "message", "role": "system"},
|
||||
{"location": "message", "index": -1},
|
||||
{"location": "message", "index": -2},
|
||||
{"location": "message", "index": -3},
|
||||
],
|
||||
)
|
||||
marked = [msg["content"][0].get("cache_control") is not None for msg in out]
|
||||
assert marked == [True, False, True, True]
|
||||
assert system[0]["cache_control"] == {"type": "ephemeral"}
|
||||
|
||||
|
||||
class TestResponsesInputPartsEligible:
|
||||
"""Responses API input parts can carry prompt_cache_breakpoint on GPT-5.6+ (#37509)."""
|
||||
|
||||
EXPLICIT = {"mode": "explicit"}
|
||||
|
||||
def _chat(self, messages, points, model="openai/gpt-5.6"):
|
||||
params = {"cache_control_injection_points": copy.deepcopy(points)}
|
||||
_, out, params = AnthropicCacheControlHook().get_chat_completion_prompt(
|
||||
model=model,
|
||||
messages=copy.deepcopy(messages),
|
||||
non_default_params=params,
|
||||
prompt_id=None,
|
||||
prompt_variables=None,
|
||||
dynamic_callback_params={},
|
||||
)
|
||||
return out, params
|
||||
|
||||
def test_marker_lands_on_last_input_text_part(self):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": "first"}, {"type": "input_text", "text": "second"}],
|
||||
}
|
||||
]
|
||||
out, params = self._chat(messages, [{"location": "message", "index": -1}])
|
||||
assert out[0]["content"][0] == {"type": "input_text", "text": "first"}
|
||||
assert out[0]["content"][1] == {
|
||||
"type": "input_text",
|
||||
"text": "second",
|
||||
"prompt_cache_breakpoint": self.EXPLICIT,
|
||||
}
|
||||
assert params["prompt_cache_options"] == self.EXPLICIT
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"part",
|
||||
[
|
||||
{"type": "input_image", "image_url": "https://example.com/a.png"},
|
||||
{"type": "input_file", "file_id": "file_1"},
|
||||
],
|
||||
)
|
||||
def test_input_image_and_input_file_parts_are_eligible(self, part):
|
||||
out, params = self._chat([{"role": "user", "content": [part]}], [{"location": "message", "index": -1}])
|
||||
assert out[0]["content"][0] == {**part, "prompt_cache_breakpoint": self.EXPLICIT}
|
||||
assert params["prompt_cache_options"] == self.EXPLICIT
|
||||
|
||||
|
||||
class TestMessagesPathApiBaseGate:
|
||||
"""/v1/messages only speaks the OpenAI dialect when the request really targets api.openai.com (#37509)."""
|
||||
|
||||
EXPLICIT = {"mode": "explicit"}
|
||||
USER_POINT = [{"location": "message", "role": "user"}]
|
||||
MESSAGES = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]
|
||||
CUSTOM_API_BASE = "http://127.0.0.1:9/v1"
|
||||
CACHE_CONTROL_BLOCK = {"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}
|
||||
BREAKPOINT_BLOCK = {"type": "text", "text": "hi", "prompt_cache_breakpoint": {"mode": "explicit"}}
|
||||
|
||||
def _inject(self, model, api_base=None, prompt_cache_options=None, custom_llm_provider=None):
|
||||
kwargs = {"cache_control_injection_points": copy.deepcopy(self.USER_POINT)}
|
||||
if prompt_cache_options is not None:
|
||||
kwargs["prompt_cache_options"] = prompt_cache_options
|
||||
out, _ = AnthropicCacheControlHook.maybe_inject_cache_control(
|
||||
copy.deepcopy(self.MESSAGES),
|
||||
None,
|
||||
kwargs,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
)
|
||||
return out[0]["content"][0], kwargs
|
||||
|
||||
def test_litellm_proxy_target_keeps_cache_control(self):
|
||||
block, kwargs = self._inject("gpt-5.6", api_base=self.CUSTOM_API_BASE, custom_llm_provider="litellm_proxy")
|
||||
assert block == self.CACHE_CONTROL_BLOCK
|
||||
assert "prompt_cache_options" not in kwargs
|
||||
|
||||
def test_custom_api_base_keeps_cache_control(self):
|
||||
block, kwargs = self._inject("gpt-5.6", api_base=self.CUSTOM_API_BASE)
|
||||
assert block == self.CACHE_CONTROL_BLOCK
|
||||
assert "prompt_cache_options" not in kwargs
|
||||
|
||||
def test_custom_api_base_opts_in_through_prompt_cache_options(self):
|
||||
block, kwargs = self._inject("gpt-5.6", api_base=self.CUSTOM_API_BASE, prompt_cache_options=self.EXPLICIT)
|
||||
assert block == self.BREAKPOINT_BLOCK
|
||||
assert kwargs["prompt_cache_options"] == self.EXPLICIT
|
||||
|
||||
def test_regional_openai_api_base_uses_openai_dialect(self):
|
||||
block, kwargs = self._inject("gpt-5.6", api_base="https://eu.api.openai.com/v1")
|
||||
assert block == self.BREAKPOINT_BLOCK
|
||||
assert kwargs["prompt_cache_options"] == self.EXPLICIT
|
||||
|
||||
def test_default_api_base_uses_openai_dialect(self):
|
||||
block, kwargs = self._inject("openai/gpt-5.6")
|
||||
assert block == self.BREAKPOINT_BLOCK
|
||||
assert kwargs["prompt_cache_options"] == self.EXPLICIT
|
||||
|
||||
|
||||
class TestToolConfigSlotInOpenAIDialect:
|
||||
"""OpenAI has no tool_config cache block, so the dialect does not hold a slot for one (#37509)."""
|
||||
|
||||
EXPLICIT = {"mode": "explicit"}
|
||||
MESSAGES = [{"role": "user", "content": [{"type": "text", "text": f"m{i}"}]} for i in range(4)]
|
||||
POINTS = [{"location": "message", "index": i} for i in range(4)] + [{"location": "tool_config"}]
|
||||
|
||||
def test_chat_path_marks_all_four_messages(self):
|
||||
params = {"cache_control_injection_points": copy.deepcopy(self.POINTS)}
|
||||
_, out, params = AnthropicCacheControlHook().get_chat_completion_prompt(
|
||||
model="openai/gpt-5.6",
|
||||
messages=copy.deepcopy(self.MESSAGES),
|
||||
non_default_params=params,
|
||||
prompt_id=None,
|
||||
prompt_variables=None,
|
||||
dynamic_callback_params={},
|
||||
)
|
||||
assert [msg["content"][0].get("prompt_cache_breakpoint") for msg in out] == [self.EXPLICIT] * 4
|
||||
assert params["prompt_cache_options"] == self.EXPLICIT
|
||||
|
||||
def test_messages_path_marks_all_four_messages(self):
|
||||
out, _, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
|
||||
copy.deepcopy(self.MESSAGES), None, copy.deepcopy(self.POINTS), openai_dialect=True
|
||||
)
|
||||
assert [msg["content"][0].get("prompt_cache_breakpoint") for msg in out] == [self.EXPLICIT] * 4
|
||||
|
||||
def test_anthropic_dialect_still_reserves_the_tool_config_slot(self):
|
||||
out, _, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
|
||||
copy.deepcopy(self.MESSAGES), None, copy.deepcopy(self.POINTS)
|
||||
)
|
||||
assert sum(msg["content"][0].get("cache_control") is not None for msg in out) == 3
|
||||
|
||||
|
||||
class TestPromptCacheBreakpointCapability:
|
||||
"""Eligibility comes from the model map's supports_prompt_cache_breakpoint flag when the entry carries one,
|
||||
with the GPT version rule for unlisted models and for entries the published map has not flagged yet (#37509)."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _bundled_model_map(self, monkeypatch):
|
||||
bundled = os.path.join(os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json")
|
||||
with open(bundled) as handle:
|
||||
monkeypatch.setattr(litellm, "model_cost", json.load(handle))
|
||||
litellm.utils._cached_get_model_info_helper.cache_clear()
|
||||
yield
|
||||
litellm.utils._cached_get_model_info_helper.cache_clear()
|
||||
|
||||
def test_public_helper_reads_the_model_map(self):
|
||||
from litellm.utils import supports_prompt_cache_breakpoint
|
||||
|
||||
assert supports_prompt_cache_breakpoint("gpt-5.6") is True
|
||||
assert supports_prompt_cache_breakpoint("openai/gpt-5.6-sol") is True
|
||||
assert supports_prompt_cache_breakpoint("gpt-5.6", custom_llm_provider="openai") is True
|
||||
assert supports_prompt_cache_breakpoint("gpt-4.1") is False
|
||||
|
||||
@pytest.mark.parametrize("model", ["gpt-5.6", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"])
|
||||
def test_model_map_flags_every_openai_gpt_5_6_entry(self, model):
|
||||
assert litellm.model_cost[model]["litellm_provider"] == "openai"
|
||||
assert litellm.model_cost[model]["supports_prompt_cache_breakpoint"] is True
|
||||
|
||||
def test_listed_model_uses_the_model_map_flag(self, monkeypatch):
|
||||
flagged = {**litellm.model_cost["gpt-4.1"], "supports_prompt_cache_breakpoint": True}
|
||||
monkeypatch.setitem(litellm.model_cost, "gpt-4.1", flagged)
|
||||
assert supports_openai_prompt_cache_breakpoint("gpt-4.1") is True
|
||||
|
||||
def test_listed_gpt_5_6_without_the_flag_falls_back_to_the_version_rule(self, monkeypatch):
|
||||
unflagged = {k: v for k, v in litellm.model_cost["gpt-5.6"].items() if k != "supports_prompt_cache_breakpoint"}
|
||||
monkeypatch.setitem(litellm.model_cost, "gpt-5.6", unflagged)
|
||||
assert supports_openai_prompt_cache_breakpoint("gpt-5.6") is True
|
||||
assert supports_openai_prompt_cache_breakpoint("openai/gpt-5.6") is True
|
||||
|
||||
def test_listed_model_flagged_false_is_not_eligible(self, monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost, "gpt-5.6", {**litellm.model_cost["gpt-5.6"], "supports_prompt_cache_breakpoint": False}
|
||||
)
|
||||
assert supports_openai_prompt_cache_breakpoint("gpt-5.6") is False
|
||||
|
||||
def test_listed_gpt_model_without_the_flag_follows_the_version_rule(self):
|
||||
assert "supports_prompt_cache_breakpoint" not in litellm.model_cost["gpt-4.1"]
|
||||
assert supports_openai_prompt_cache_breakpoint("gpt-4.1") is False
|
||||
|
||||
def test_published_map_without_the_flag_still_injects_on_gpt_5_6(self, monkeypatch):
|
||||
unflagged = {k: v for k, v in litellm.model_cost["gpt-5.6"].items() if k != "supports_prompt_cache_breakpoint"}
|
||||
monkeypatch.setitem(litellm.model_cost, "gpt-5.6", unflagged)
|
||||
points = [{"location": "message", "role": "system"}]
|
||||
|
||||
_, chat_messages, chat_params = AnthropicCacheControlHook().get_chat_completion_prompt(
|
||||
model="openai/gpt-5.6",
|
||||
messages=[{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}],
|
||||
non_default_params={"cache_control_injection_points": copy.deepcopy(points)},
|
||||
prompt_id=None,
|
||||
prompt_variables=None,
|
||||
dynamic_callback_params={},
|
||||
)
|
||||
assert chat_messages[0]["content"] == [
|
||||
{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}
|
||||
]
|
||||
assert chat_params["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
||||
kwargs = {"cache_control_injection_points": copy.deepcopy(points)}
|
||||
_, system = AnthropicCacheControlHook.maybe_inject_cache_control(
|
||||
[{"role": "user", "content": "hi"}], "sys", kwargs, model="gpt-5.6", custom_llm_provider="openai"
|
||||
)
|
||||
assert system == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}]
|
||||
assert kwargs == {"prompt_cache_options": {"mode": "explicit"}}
|
||||
|
||||
@pytest.mark.parametrize("model,expected", [("gpt-5.6-2026-01-01", True), ("gpt-5.5-preview-unlisted", False)])
|
||||
def test_unlisted_model_falls_back_to_the_version_rule(self, model, expected):
|
||||
assert model not in litellm.model_cost
|
||||
assert supports_openai_prompt_cache_breakpoint(model) is expected
|
||||
|
|
|
|||
|
|
@ -3741,3 +3741,59 @@ def test_tool_result_plain_text_unchanged_by_openai_transform():
|
|||
assert len(tool_messages) == 1
|
||||
assert tool_messages[0]["content"] == "42 files found"
|
||||
assert _image_urls_in_user_messages(result) == []
|
||||
|
||||
|
||||
def test_translate_anthropic_to_openai_carries_prompt_cache_breakpoint_on_system_and_user_blocks():
|
||||
explicit = {"mode": "explicit"}
|
||||
openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
|
||||
anthropic_message_request={
|
||||
"model": "gpt-5.6",
|
||||
"max_tokens": 64,
|
||||
"system": [{"type": "text", "text": "sys", "prompt_cache_breakpoint": explicit}],
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "hi", "prompt_cache_breakpoint": explicit},
|
||||
{
|
||||
"type": "image",
|
||||
"source": {"type": "url", "url": "https://example.com/a.png"},
|
||||
"prompt_cache_breakpoint": explicit,
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
assert openai_request["messages"][0] == {
|
||||
"role": "system",
|
||||
"content": [{"type": "text", "text": "sys", "prompt_cache_breakpoint": explicit}],
|
||||
}
|
||||
user_content = openai_request["messages"][1]["content"]
|
||||
assert user_content[0] == {"type": "text", "text": "hi", "prompt_cache_breakpoint": explicit}
|
||||
assert user_content[1]["type"] == "image_url"
|
||||
assert user_content[1]["prompt_cache_breakpoint"] == explicit
|
||||
|
||||
|
||||
def test_translate_anthropic_to_openai_without_prompt_cache_breakpoint_adds_nothing():
|
||||
openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
|
||||
anthropic_message_request={
|
||||
"model": "gpt-5.6",
|
||||
"max_tokens": 64,
|
||||
"system": [{"type": "text", "text": "sys"}],
|
||||
"messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}],
|
||||
}
|
||||
)
|
||||
assert openai_request["messages"][0] == {"role": "system", "content": [{"type": "text", "text": "sys"}]}
|
||||
assert openai_request["messages"][1]["content"] == [{"type": "text", "text": "hi"}]
|
||||
|
||||
|
||||
def test_translate_anthropic_messages_to_openai_carries_midturn_system_prompt_cache_breakpoint():
|
||||
explicit = {"mode": "explicit"}
|
||||
result = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai(
|
||||
messages=[{"role": "system", "content": [{"type": "text", "text": "fix", "prompt_cache_breakpoint": explicit}]}],
|
||||
model="gpt-5.6",
|
||||
)
|
||||
assert result == [
|
||||
{"role": "system", "content": [{"type": "text", "text": "fix", "prompt_cache_breakpoint": explicit}]}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -230,3 +230,10 @@ class TestEmptyExtraKwargsPath:
|
|||
# dict-like result.
|
||||
completion_kwargs = result[0] if isinstance(result, tuple) else result
|
||||
assert isinstance(completion_kwargs, dict)
|
||||
|
||||
|
||||
class TestPromptCacheOptionsForwarded:
|
||||
def test_prompt_cache_options_reaches_completion_kwargs(self):
|
||||
result = _call_prepare(extra_kwargs={"prompt_cache_options": {"mode": "explicit"}}, model="gpt-5.6")
|
||||
completion_kwargs = result[0] if isinstance(result, tuple) else result
|
||||
assert completion_kwargs["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
|
|
|||
|
|
@ -1413,3 +1413,155 @@ class TestToolResultImages:
|
|||
outputs = [item for item in items if item.get("type") == "function_call_output"]
|
||||
assert outputs[0]["output"] == "screenshot saved"
|
||||
assert self._input_images(items) == []
|
||||
|
||||
|
||||
def _contains_key(value, key) -> bool:
|
||||
if isinstance(value, dict):
|
||||
return key in value or any(_contains_key(v, key) for v in value.values())
|
||||
if isinstance(value, list):
|
||||
return any(_contains_key(v, key) for v in value)
|
||||
return False
|
||||
|
||||
|
||||
class TestPromptCacheBreakpointToResponses:
|
||||
"""OpenAI `prompt_cache_breakpoint` markers ride through the /v1/messages -> Responses bridge (#37509)."""
|
||||
|
||||
EXPLICIT = {"mode": "explicit"}
|
||||
|
||||
def test_system_with_breakpoint_becomes_leading_developer_message(self):
|
||||
request = _make_request(
|
||||
model="openai/gpt-5.6",
|
||||
system=[
|
||||
{"type": "text", "text": "Be concise."},
|
||||
{"type": "text", "text": "Be helpful.", "prompt_cache_breakpoint": self.EXPLICIT},
|
||||
],
|
||||
)
|
||||
kwargs = _ADAPTER.translate_request(request)
|
||||
assert "instructions" not in kwargs
|
||||
assert kwargs["input"] == [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "developer",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "Be concise."},
|
||||
{"type": "input_text", "text": "Be helpful.", "prompt_cache_breakpoint": self.EXPLICIT},
|
||||
],
|
||||
},
|
||||
{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hello"}]},
|
||||
]
|
||||
|
||||
def test_system_without_breakpoint_still_becomes_instructions(self):
|
||||
request = _make_request(system=[{"type": "text", "text": "Be concise."}, {"type": "text", "text": "Be helpful."}])
|
||||
kwargs = _ADAPTER.translate_request(request)
|
||||
assert kwargs["instructions"] == "Be concise.\nBe helpful."
|
||||
assert kwargs["input"] == [
|
||||
{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hello"}]}
|
||||
]
|
||||
|
||||
def test_system_string_still_becomes_instructions(self):
|
||||
kwargs = _ADAPTER.translate_request(_make_request(system="Be concise."))
|
||||
assert kwargs["instructions"] == "Be concise."
|
||||
assert kwargs["input"][0]["role"] == "user"
|
||||
|
||||
def test_system_with_breakpoint_skips_non_text_blocks(self):
|
||||
request = _make_request(
|
||||
system=[
|
||||
{"type": "image", "source": {"type": "url", "url": "https://example.com/a.png"}},
|
||||
{"type": "text", "text": "only", "prompt_cache_breakpoint": self.EXPLICIT},
|
||||
]
|
||||
)
|
||||
kwargs = _ADAPTER.translate_request(request)
|
||||
assert kwargs["input"][0] == {
|
||||
"type": "message",
|
||||
"role": "developer",
|
||||
"content": [{"type": "input_text", "text": "only", "prompt_cache_breakpoint": self.EXPLICIT}],
|
||||
}
|
||||
|
||||
def test_user_text_and_image_blocks_carry_breakpoint(self):
|
||||
items = _ADAPTER.translate_messages_to_responses_input(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "look", "prompt_cache_breakpoint": self.EXPLICIT},
|
||||
{
|
||||
"type": "image",
|
||||
"source": {"type": "url", "url": "https://example.com/a.png"},
|
||||
"prompt_cache_breakpoint": self.EXPLICIT,
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
)
|
||||
assert items == [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "look", "prompt_cache_breakpoint": self.EXPLICIT},
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "https://example.com/a.png",
|
||||
"prompt_cache_breakpoint": self.EXPLICIT,
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
def test_user_blocks_without_breakpoint_are_unchanged(self):
|
||||
items = _ADAPTER.translate_messages_to_responses_input(
|
||||
[{"role": "user", "content": [{"type": "text", "text": "look"}]}]
|
||||
)
|
||||
assert items == [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "look"}]}]
|
||||
|
||||
def test_midturn_system_block_carries_breakpoint(self):
|
||||
items = _ADAPTER.translate_messages_to_responses_input(
|
||||
[{"role": "system", "content": [{"type": "text", "text": "fix", "prompt_cache_breakpoint": self.EXPLICIT}]}]
|
||||
)
|
||||
assert items == [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "system",
|
||||
"content": [{"type": "input_text", "text": "fix", "prompt_cache_breakpoint": self.EXPLICIT}],
|
||||
}
|
||||
]
|
||||
|
||||
def test_assistant_and_tool_result_blocks_drop_breakpoint(self):
|
||||
items = _ADAPTER.translate_messages_to_responses_input(
|
||||
[
|
||||
{"role": "user", "content": [{"type": "text", "text": "q"}]},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "text", "text": "a", "prompt_cache_breakpoint": self.EXPLICIT},
|
||||
{"type": "tool_use", "id": "toolu_01", "name": "t", "input": {}},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_01",
|
||||
"content": "r",
|
||||
"prompt_cache_breakpoint": self.EXPLICIT,
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
)
|
||||
assert len(items) == 4
|
||||
assert not _contains_key(items, "prompt_cache_breakpoint")
|
||||
|
||||
def test_prompt_cache_options_forwarded_to_responses_kwargs(self):
|
||||
from litellm.llms.anthropic.experimental_pass_through.responses_adapters.handler import (
|
||||
_build_responses_kwargs,
|
||||
)
|
||||
|
||||
kwargs = _build_responses_kwargs(
|
||||
max_tokens=16,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
model="openai/gpt-5.6",
|
||||
extra_kwargs={"prompt_cache_options": {"mode": "explicit"}},
|
||||
)
|
||||
assert kwargs["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
|
|
|||
|
|
@ -871,3 +871,48 @@ class TestToolMessageImageHoisting:
|
|||
result = request["messages"]
|
||||
assert [m.get("role") for m in result] == ["user", "assistant", "tool", "user"]
|
||||
assert result[3]["content"] == self.HOISTED_USER_CONTENT
|
||||
|
||||
|
||||
class TestOpenAIPromptCacheBreakpointChatPath:
|
||||
"""Chat-path shape for OpenAI explicit prompt caching (#37509)."""
|
||||
|
||||
EXPLICIT = {"mode": "explicit"}
|
||||
|
||||
def test_prompt_cache_options_travels_in_extra_body(self):
|
||||
optional_params = litellm.get_optional_params(
|
||||
model="gpt-5.6", custom_llm_provider="openai", prompt_cache_options=self.EXPLICIT
|
||||
)
|
||||
assert optional_params["extra_body"]["prompt_cache_options"] == self.EXPLICIT
|
||||
assert "prompt_cache_options" not in optional_params
|
||||
|
||||
def test_prompt_cache_options_is_not_a_supported_chat_param(self):
|
||||
assert "prompt_cache_options" not in OpenAIGPT5Config().get_supported_openai_params("gpt-5.6")
|
||||
assert "prompt_cache_options" not in OpenAIGPTConfig().get_supported_openai_params("gpt-4.1")
|
||||
|
||||
def test_block_breakpoint_survives_transform_request(self):
|
||||
request = OpenAIGPT5Config().transform_request(
|
||||
model="gpt-5.6",
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "sys",
|
||||
"prompt_cache_breakpoint": self.EXPLICIT,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]},
|
||||
],
|
||||
optional_params={"extra_body": {"prompt_cache_options": self.EXPLICIT}},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
assert request["messages"][0]["content"] == [
|
||||
{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}
|
||||
]
|
||||
assert request["messages"][1]["content"] == [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]
|
||||
assert request["extra_body"] == {"prompt_cache_options": self.EXPLICIT}
|
||||
assert "prompt_cache_options" not in request
|
||||
|
|
|
|||
|
|
@ -1540,3 +1540,61 @@ class TestPhaseParameter:
|
|||
assert validated[0]["phase"] == "commentary"
|
||||
assert validated[1]["phase"] == "final_answer"
|
||||
assert "phase" not in validated[2]
|
||||
|
||||
|
||||
class TestPromptCacheOptionsOnResponsesPath:
|
||||
"""`prompt_cache_options` and block-level `prompt_cache_breakpoint` survive the Responses transformation (#37509)."""
|
||||
|
||||
OPTIONS = {"mode": "explicit", "ttl": "30m"}
|
||||
|
||||
def test_prompt_cache_options_survives_optional_param_filter(self):
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
|
||||
result = ResponsesAPIRequestUtils.get_requested_response_api_optional_param(
|
||||
{"prompt_cache_options": dict(self.OPTIONS), "temperature": 0.2, "not_a_responses_param": 1}
|
||||
)
|
||||
assert result["prompt_cache_options"] == self.OPTIONS
|
||||
assert result["temperature"] == 0.2
|
||||
assert "not_a_responses_param" not in result
|
||||
|
||||
def test_prompt_cache_options_reaches_transformed_request(self):
|
||||
config = OpenAIResponsesAPIConfig()
|
||||
mapped = config.map_openai_params(
|
||||
response_api_optional_params={"prompt_cache_options": dict(self.OPTIONS)},
|
||||
model="gpt-5.6",
|
||||
drop_params=False,
|
||||
)
|
||||
result = config.transform_responses_api_request(
|
||||
model="gpt-5.6",
|
||||
input="hi",
|
||||
response_api_optional_request_params=mapped,
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
assert result["prompt_cache_options"] == self.OPTIONS
|
||||
|
||||
def test_prompt_cache_breakpoint_survives_cache_control_strip(self):
|
||||
result = OpenAIResponsesAPIConfig().transform_responses_api_request(
|
||||
model="gpt-5.6",
|
||||
input=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": "hi",
|
||||
"prompt_cache_breakpoint": {"mode": "explicit"},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
response_api_optional_request_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
assert result["input"][0]["content"][0] == {
|
||||
"type": "input_text",
|
||||
"text": "hi",
|
||||
"prompt_cache_breakpoint": {"mode": "explicit"},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ over the wire and surface provider errors correctly. Expected JSON bodies are st
|
|||
in expected_responses_api_request/.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
|
@ -422,3 +423,178 @@ async def test_aresponses_websocket_strips_responses_routing_prefix_from_openai_
|
|||
mock_ws.assert_awaited_once()
|
||||
assert mock_ws.call_args.kwargs["model"] == "gpt-5.6"
|
||||
assert mock_ws.call_args.kwargs["custom_llm_provider"] == "openai"
|
||||
|
||||
|
||||
_INJECTION_POINT_INPUT = [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "hi"}]
|
||||
_SYSTEM_INJECTION_POINT = [{"location": "message", "role": "system"}]
|
||||
|
||||
|
||||
def _sent_body(mock_post) -> dict:
|
||||
kwargs = mock_post.call_args.kwargs
|
||||
return kwargs["json"] if "json" in kwargs else json.loads(kwargs["data"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aresponses_injection_point_marks_input_text_on_gpt_5_6():
|
||||
with patch(
|
||||
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_post:
|
||||
mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_async", "gpt-5.6"), 200)
|
||||
|
||||
await litellm.aresponses(
|
||||
model="openai/gpt-5.6",
|
||||
api_key="fake-api-key",
|
||||
input=copy.deepcopy(_INJECTION_POINT_INPUT),
|
||||
cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT),
|
||||
)
|
||||
|
||||
body = _sent_body(mock_post)
|
||||
assert body["input"][0]["content"][0] == {
|
||||
"type": "input_text",
|
||||
"text": "You are terse.",
|
||||
"prompt_cache_breakpoint": {"mode": "explicit"},
|
||||
}
|
||||
assert body["input"][1] == {"role": "user", "content": "hi"}
|
||||
assert body["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
||||
|
||||
def test_responses_injection_point_marks_input_text_on_gpt_5_6():
|
||||
with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post:
|
||||
mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_sync", "gpt-5.6"), 200)
|
||||
|
||||
litellm.responses(
|
||||
model="openai/gpt-5.6",
|
||||
api_key="fake-api-key",
|
||||
input=copy.deepcopy(_INJECTION_POINT_INPUT),
|
||||
cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT),
|
||||
)
|
||||
|
||||
body = _sent_body(mock_post)
|
||||
assert body["input"][0]["content"][0] == {
|
||||
"type": "input_text",
|
||||
"text": "You are terse.",
|
||||
"prompt_cache_breakpoint": {"mode": "explicit"},
|
||||
}
|
||||
assert body["input"][1] == {"role": "user", "content": "hi"}
|
||||
assert body["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aresponses_injection_point_sends_nothing_extra_below_gpt_5_6():
|
||||
with patch(
|
||||
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_post:
|
||||
mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_old", "gpt-4.1"), 200)
|
||||
|
||||
await litellm.aresponses(
|
||||
model="openai/gpt-4.1",
|
||||
api_key="fake-api-key",
|
||||
input=copy.deepcopy(_INJECTION_POINT_INPUT),
|
||||
cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT),
|
||||
)
|
||||
|
||||
body = _sent_body(mock_post)
|
||||
assert body["input"] == _INJECTION_POINT_INPUT
|
||||
assert "prompt_cache_options" not in body
|
||||
assert "cache_control" not in json.dumps(body)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _no_openai_api_base_override(monkeypatch):
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", None)
|
||||
|
||||
|
||||
_CUSTOM_API_BASE = "http://127.0.0.1:9/v1"
|
||||
|
||||
|
||||
async def _aresponses_body_with_system_point(**request_kwargs) -> dict:
|
||||
with patch(
|
||||
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_post:
|
||||
mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_gate", "gpt-5.6"), 200)
|
||||
await litellm.aresponses(
|
||||
api_key="fake-api-key",
|
||||
input=copy.deepcopy(_INJECTION_POINT_INPUT),
|
||||
cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT),
|
||||
**request_kwargs,
|
||||
)
|
||||
return _sent_body(mock_post)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("_no_openai_api_base_override")
|
||||
async def test_aresponses_litellm_proxy_target_sends_no_openai_markers():
|
||||
body = await _aresponses_body_with_system_point(model="litellm_proxy/gpt-5.6", api_base=_CUSTOM_API_BASE)
|
||||
assert body["input"] == _INJECTION_POINT_INPUT
|
||||
assert "prompt_cache_options" not in body
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("_no_openai_api_base_override")
|
||||
async def test_aresponses_custom_api_base_sends_no_openai_markers():
|
||||
body = await _aresponses_body_with_system_point(model="gpt-5.6", api_base=_CUSTOM_API_BASE)
|
||||
assert body["input"] == _INJECTION_POINT_INPUT
|
||||
assert "prompt_cache_options" not in body
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("_no_openai_api_base_override")
|
||||
async def test_aresponses_custom_api_base_opts_in_through_prompt_cache_options():
|
||||
body = await _aresponses_body_with_system_point(
|
||||
model="gpt-5.6", api_base=_CUSTOM_API_BASE, prompt_cache_options={"mode": "explicit"}
|
||||
)
|
||||
assert body["input"][0]["content"][0] == {
|
||||
"type": "input_text",
|
||||
"text": "You are terse.",
|
||||
"prompt_cache_breakpoint": {"mode": "explicit"},
|
||||
}
|
||||
assert body["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("_no_openai_api_base_override")
|
||||
async def test_aresponses_regional_openai_api_base_marks_input_text():
|
||||
body = await _aresponses_body_with_system_point(model="gpt-5.6", api_base="https://eu.api.openai.com/v1")
|
||||
assert body["input"][0]["content"][0]["prompt_cache_breakpoint"] == {"mode": "explicit"}
|
||||
assert body["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_no_openai_api_base_override")
|
||||
def test_responses_custom_base_url_sends_no_openai_markers():
|
||||
with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post:
|
||||
mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_gate_base_url", "gpt-5.6"), 200)
|
||||
|
||||
litellm.responses(
|
||||
model="gpt-5.6",
|
||||
api_key="fake-api-key",
|
||||
base_url=_CUSTOM_API_BASE,
|
||||
input=copy.deepcopy(_INJECTION_POINT_INPUT),
|
||||
cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT),
|
||||
)
|
||||
|
||||
body = _sent_body(mock_post)
|
||||
assert body["input"] == _INJECTION_POINT_INPUT
|
||||
assert "prompt_cache_options" not in body
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_no_openai_api_base_override")
|
||||
def test_responses_custom_api_base_sends_no_openai_markers():
|
||||
with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post:
|
||||
mock_post.return_value = MockResponse(_minimal_responses_api_payload("resp_pcb_gate_sync", "gpt-5.6"), 200)
|
||||
|
||||
litellm.responses(
|
||||
model="gpt-5.6",
|
||||
api_key="fake-api-key",
|
||||
api_base=_CUSTOM_API_BASE,
|
||||
input=copy.deepcopy(_INJECTION_POINT_INPUT),
|
||||
cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT),
|
||||
)
|
||||
|
||||
body = _sent_body(mock_post)
|
||||
assert body["input"] == _INJECTION_POINT_INPUT
|
||||
assert "prompt_cache_options" not in body
|
||||
|
|
|
|||
|
|
@ -638,3 +638,92 @@ def test_responses_maps_reasoning_effort_from_litellm_params_to_reasoning():
|
|||
"effort": "high",
|
||||
"summary": "detailed",
|
||||
}
|
||||
|
||||
|
||||
class TestMergePromptManagementInputReshape:
|
||||
"""Chat-shaped text parts produced by prompt management hooks become input_text parts (#37509)."""
|
||||
|
||||
EXPLICIT = {"mode": "explicit"}
|
||||
|
||||
def _run_cache_hook(self, client_input, points, model="openai/gpt-5.6"):
|
||||
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
|
||||
|
||||
_, merged, _ = AnthropicCacheControlHook().get_chat_completion_prompt(
|
||||
model=model,
|
||||
messages=client_input,
|
||||
non_default_params={"cache_control_injection_points": points},
|
||||
prompt_id=None,
|
||||
prompt_variables=None,
|
||||
dynamic_callback_params={},
|
||||
)
|
||||
return merged
|
||||
|
||||
def test_string_system_item_becomes_input_text_with_marker(self):
|
||||
original_input = [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "hi"}]
|
||||
merged = self._run_cache_hook(list(original_input), [{"location": "message", "role": "system"}])
|
||||
|
||||
result = ResponsesAPIRequestUtils.merge_prompt_management_input(
|
||||
original_input=original_input, client_input=list(original_input), merged_input=merged
|
||||
)
|
||||
|
||||
assert result[0]["content"] == [
|
||||
{"type": "input_text", "text": "You are terse.", "prompt_cache_breakpoint": self.EXPLICIT}
|
||||
]
|
||||
assert result[1] == {"role": "user", "content": "hi"}
|
||||
|
||||
def test_reshape_returns_copies_and_leaves_hook_output_untouched(self):
|
||||
user_part = {"type": "text", "text": "follow-up"}
|
||||
user_message = {"role": "user", "content": [user_part]}
|
||||
merged = [user_message]
|
||||
|
||||
result = ResponsesAPIRequestUtils.merge_prompt_management_input(
|
||||
original_input="ignored", client_input=[], merged_input=merged
|
||||
)
|
||||
|
||||
assert result == [{"role": "user", "content": [{"type": "input_text", "text": "follow-up"}]}]
|
||||
assert user_part == {"type": "text", "text": "follow-up"}
|
||||
assert user_message == {"role": "user", "content": [user_part]}
|
||||
assert result[0] is not user_message
|
||||
|
||||
def test_reshape_keeps_non_message_items_when_hook_returns_client_objects(self):
|
||||
user_message = {"role": "user", "content": [{"type": "text", "text": "question"}]}
|
||||
reference = {"type": "item_reference", "id": "msg_123"}
|
||||
original_input = [reference, user_message]
|
||||
|
||||
result = ResponsesAPIRequestUtils.merge_prompt_management_input(
|
||||
original_input=original_input, client_input=[user_message], merged_input=[user_message]
|
||||
)
|
||||
|
||||
assert result == [reference, {"role": "user", "content": [{"type": "input_text", "text": "question"}]}]
|
||||
assert result[0] is reference
|
||||
assert user_message["content"] == [{"type": "text", "text": "question"}]
|
||||
|
||||
def test_assistant_text_parts_are_left_alone(self):
|
||||
merged = [
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "earlier answer"}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "follow-up"}]},
|
||||
]
|
||||
|
||||
result = ResponsesAPIRequestUtils.merge_prompt_management_input(
|
||||
original_input="ignored", client_input=[], merged_input=merged
|
||||
)
|
||||
|
||||
assert result[0]["content"] == [{"type": "text", "text": "earlier answer"}]
|
||||
assert result[1]["content"] == [{"type": "input_text", "text": "follow-up"}]
|
||||
|
||||
def test_parts_already_in_responses_shape_are_unchanged(self):
|
||||
merged = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "a", "prompt_cache_breakpoint": self.EXPLICIT},
|
||||
{"type": "input_image", "image_url": "https://example.com/a.png"},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
result = ResponsesAPIRequestUtils.merge_prompt_management_input(
|
||||
original_input="ignored", client_input=[], merged_input=merged
|
||||
)
|
||||
|
||||
assert result == merged
|
||||
|
|
|
|||
|
|
@ -2672,3 +2672,85 @@ def test_openai_model_without_a_provider_still_routes_to_openai():
|
|||
)
|
||||
|
||||
mock_create.assert_called()
|
||||
|
||||
|
||||
def _openai_chat_create_kwargs(client, **completion_kwargs):
|
||||
with patch.object(client.chat.completions.with_raw_response, "create") as mock_client:
|
||||
with contextlib.suppress(Exception):
|
||||
litellm.completion(
|
||||
messages=[{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}],
|
||||
cache_control_injection_points=[{"location": "message", "role": "system"}],
|
||||
client=client,
|
||||
**completion_kwargs,
|
||||
)
|
||||
|
||||
mock_client.assert_called_once()
|
||||
return mock_client.call_args.kwargs
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _no_openai_api_base_override(monkeypatch):
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", None)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_no_openai_api_base_override")
|
||||
def test_completion_custom_api_base_sends_no_prompt_cache_breakpoint_for_gpt_5_6():
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:9/v1")
|
||||
request_body = _openai_chat_create_kwargs(client, model="gpt-5.6", api_base="http://127.0.0.1:9/v1")
|
||||
|
||||
assert request_body["messages"][0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}}
|
||||
assert "prompt_cache_breakpoint" not in json.dumps(request_body["messages"])
|
||||
assert "prompt_cache_options" not in json.dumps(request_body)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_no_openai_api_base_override")
|
||||
def test_completion_custom_base_url_sends_no_prompt_cache_breakpoint_for_gpt_5_6():
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:9/v1")
|
||||
request_body = _openai_chat_create_kwargs(client, model="gpt-5.6", base_url="http://127.0.0.1:9/v1")
|
||||
|
||||
assert request_body["messages"][0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}}
|
||||
assert "prompt_cache_breakpoint" not in json.dumps(request_body["messages"])
|
||||
assert "prompt_cache_options" not in json.dumps(request_body)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("_no_openai_api_base_override")
|
||||
async def test_acompletion_custom_base_url_sends_no_prompt_cache_breakpoint_for_gpt_5_6():
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
client = AsyncOpenAI(api_key="fake-api-key", base_url="http://127.0.0.1:9/v1")
|
||||
with patch.object(client.chat.completions.with_raw_response, "create") as mock_create:
|
||||
with contextlib.suppress(Exception):
|
||||
await litellm.acompletion(
|
||||
model="gpt-5.6",
|
||||
messages=[{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}],
|
||||
cache_control_injection_points=[{"location": "message", "role": "system"}],
|
||||
client=client,
|
||||
base_url="http://127.0.0.1:9/v1",
|
||||
)
|
||||
|
||||
mock_create.assert_called_once()
|
||||
request_body = mock_create.call_args.kwargs
|
||||
|
||||
assert request_body["messages"][0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}}
|
||||
assert "prompt_cache_breakpoint" not in json.dumps(request_body["messages"])
|
||||
assert "prompt_cache_options" not in json.dumps(request_body)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_no_openai_api_base_override")
|
||||
def test_completion_default_api_base_sends_prompt_cache_breakpoint_for_gpt_5_6():
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(api_key="fake-api-key")
|
||||
request_body = _openai_chat_create_kwargs(client, model="gpt-5.6")
|
||||
|
||||
assert request_body["messages"][0]["content"] == [
|
||||
{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}
|
||||
]
|
||||
assert request_body["extra_body"]["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
|
|
|||
|
|
@ -921,6 +921,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
|
|||
"supports_parallel_tool_use_config": {"type": "boolean"},
|
||||
"supports_pdf_input": {"type": "boolean"},
|
||||
"prompt_cache_min_tokens": {"type": "number"},
|
||||
"supports_prompt_cache_breakpoint": {"type": "boolean"},
|
||||
"supports_prompt_caching": {"type": "boolean"},
|
||||
"supports_response_schema": {"type": "boolean"},
|
||||
"supports_system_messages": {"type": "boolean"},
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue