mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-01 07:03:02 +00:00
fix(openai): stop stripping cache_control from the caller's own messages and tools (#35602)
* fix(openai): stop stripping cache_control from the caller's messages Copy before removing the field, so the caller's own message objects are left alone. * fix(fireworks_ai): strip cache_control on a copy so a fallback keeps the caller's hints * fix: keep the caller's request intact on more providers and trim extra Claude cache hints under drop_params * fix: drop only the oldest extra cache hints inside one message when trimming to the Claude limit * fix: keep replayed session turns yielding their hints as whole turns while a request's own message keeps its newest hints * fix(cache_control): count only hints that reach Claude and cap hints on OpenAI-compatible Claude routes under drop_params * fix(cache_control): keep the capped Claude messages a list in OpenAI-compatible request bodies * fix(caching): cap Claude hints for string drop_params, a Claude base_model, and Converse tool messages The 4-hint cap now reads drop_params through drops_unsupported_params on chat, Converse, and Anthropic routes, names Claude by base_model on OpenAI-compatible routes, and counts a Converse tool message's hints as the one cache point it sends. * fix(caching): trim Claude cache hints in one wire-order pass and reserve a top-level cache_control on OpenAI-compatible routes Dropping the oldest hint one at a time re-walked the whole request per hint, which took seconds on long tool histories. The cap now drops each message's oldest hints in the order the provider counts them, so a string assistant turn loses its text hint before its tool call hints. OpenAI-compatible Claude routes also keep room for a request-level or extra_body cache_control under drop_params. * test: drop narrating comments and stacked generator clauses from cache_control tests * fix(vertex_ai): put every system message in the Gemini context cache so generateContent never sends it beside cachedContent --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
Vineeth Sai Varikuntla
GitHub
parent
e5aa3d32cd
commit
a7b66dd8b9
34 changed files with 1463 additions and 278 deletions
|
|
@ -14,7 +14,7 @@ import itertools
|
|||
import os
|
||||
import re
|
||||
from collections.abc import Callable, Container, Iterable, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
|
@ -102,6 +102,192 @@ def _validated_object_list(value: object) -> list[object] | None:
|
|||
return None
|
||||
|
||||
|
||||
_MessageT: Final = TypeVar("_MessageT")
|
||||
_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None]
|
||||
_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls")
|
||||
|
||||
|
||||
def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]:
|
||||
lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS)
|
||||
return tuple(
|
||||
itertools.chain.from_iterable(
|
||||
_OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _block_carries_cache_control(block: object) -> bool:
|
||||
return isinstance(block, dict) and _CONTENT_PART_ADAPTER.validate_python(block).get("cache_control") is not None
|
||||
|
||||
|
||||
_ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT: Final = frozenset({"tool", "function"})
|
||||
|
||||
|
||||
def _message_mark_reaches_provider(fields: Mapping[str, object]) -> bool:
|
||||
if fields.get("cache_control") is None:
|
||||
return False
|
||||
return (
|
||||
not isinstance(fields.get("content"), list)
|
||||
or fields.get("role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT
|
||||
)
|
||||
|
||||
|
||||
def cache_control_count(message: object) -> int:
|
||||
if not isinstance(message, dict):
|
||||
return 0
|
||||
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
|
||||
block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block))
|
||||
return int(_message_mark_reaches_provider(fields)) + block_marks
|
||||
|
||||
|
||||
def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]:
|
||||
others: Final = {key: value for key, value in fields.items() if key != "cache_control"}
|
||||
if "cache_control" not in fields:
|
||||
return others
|
||||
cache_control: Final = rewrite(fields["cache_control"])
|
||||
return others if cache_control is None else {**others, "cache_control": cache_control}
|
||||
|
||||
|
||||
def block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object:
|
||||
if not isinstance(block, dict):
|
||||
return block
|
||||
return _fields_with_cache_control(_CONTENT_PART_ADAPTER.validate_python(block), rewrite)
|
||||
|
||||
|
||||
def with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT:
|
||||
if not isinstance(message, dict):
|
||||
return message
|
||||
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
|
||||
blocks_rewritten: Final = {
|
||||
key: (
|
||||
[block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)]
|
||||
if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list)
|
||||
else value
|
||||
)
|
||||
for key, value in fields.items()
|
||||
}
|
||||
return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped
|
||||
_MessageT, _fields_with_cache_control(blocks_rewritten, rewrite)
|
||||
)
|
||||
|
||||
|
||||
def dropped_cache_control(_: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _first_marked_index(value: object) -> int | None:
|
||||
if not isinstance(value, list):
|
||||
return None
|
||||
blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value)
|
||||
return next((index for index, block in enumerate(blocks) if _block_carries_cache_control(block)), None)
|
||||
|
||||
|
||||
_MarkSlot: TypeAlias = tuple[str, int]
|
||||
_MESSAGE_MARK_SLOT: Final[_MarkSlot] = ("cache_control", -1)
|
||||
|
||||
|
||||
def _block_mark_slots(fields: Mapping[str, object], key: str) -> tuple[_MarkSlot, ...]:
|
||||
value: Final = fields.get(key)
|
||||
if not isinstance(value, list):
|
||||
return ()
|
||||
blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value)
|
||||
return tuple((key, index) for index, block in enumerate(blocks) if _block_carries_cache_control(block))
|
||||
|
||||
|
||||
def _mark_slots_in_wire_order(fields: Mapping[str, object]) -> tuple[_MarkSlot, ...]:
|
||||
content_slots: Final = _block_mark_slots(fields, "content")
|
||||
tool_call_slots: Final = _block_mark_slots(fields, "tool_calls")
|
||||
if not _message_mark_reaches_provider(fields):
|
||||
return content_slots + tool_call_slots
|
||||
if isinstance(fields.get("content"), list):
|
||||
return content_slots + (_MESSAGE_MARK_SLOT,) + tool_call_slots
|
||||
return (_MESSAGE_MARK_SLOT,) + content_slots + tool_call_slots
|
||||
|
||||
|
||||
def _blocks_without_marks(key: str, value: object, dropped_slots: frozenset[_MarkSlot]) -> object:
|
||||
if key not in _MARK_CARRYING_MESSAGE_FIELDS or not isinstance(value, list):
|
||||
return value
|
||||
return [
|
||||
block_with_cache_control(block, dropped_cache_control) if (key, index) in dropped_slots else block
|
||||
for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(value))
|
||||
]
|
||||
|
||||
|
||||
def _without_oldest_marks(message: _MessageT, dropped: int) -> _MessageT:
|
||||
if dropped == 0 or not isinstance(message, dict):
|
||||
return message
|
||||
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
|
||||
dropped_slots: Final = frozenset(_mark_slots_in_wire_order(fields)[:dropped])
|
||||
blocks_unmarked: Final = {key: _blocks_without_marks(key, value, dropped_slots) for key, value in fields.items()}
|
||||
return cast( # cast-ok: the same message with its oldest wire-order marks dropped
|
||||
_MessageT,
|
||||
_fields_with_cache_control(blocks_unmarked, dropped_cache_control)
|
||||
if _MESSAGE_MARK_SLOT in dropped_slots
|
||||
else blocks_unmarked,
|
||||
)
|
||||
|
||||
|
||||
def tool_message_with_single_cache_mark(message: _MessageT) -> _MessageT:
|
||||
if not isinstance(message, dict):
|
||||
return message
|
||||
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
|
||||
content: Final = fields.get("content")
|
||||
if fields.get("role") != "tool" or not isinstance(content, list):
|
||||
return message
|
||||
kept_index: Final = None if fields.get("cache_control") is not None else _first_marked_index(content)
|
||||
return cast( # cast-ok: the same tool message keeping only the mark its single cache point reads
|
||||
_MessageT,
|
||||
{
|
||||
**fields,
|
||||
"content": [
|
||||
block if index == kept_index else block_with_cache_control(block, dropped_cache_control)
|
||||
for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(content))
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
_BudgetSpend: TypeAlias = Callable[[tuple[int, int], int], tuple[int, int]]
|
||||
|
||||
|
||||
def _newest_marks_kept(state: tuple[int, int], marks: int) -> tuple[int, int]:
|
||||
remaining, _ = state
|
||||
kept: Final = min(marks, remaining)
|
||||
return remaining - kept, marks - kept
|
||||
|
||||
|
||||
def _whole_message_kept(state: tuple[int, int], marks: int) -> tuple[int, int]:
|
||||
remaining, _ = state
|
||||
if marks > remaining:
|
||||
return remaining, marks
|
||||
return remaining - marks, 0
|
||||
|
||||
|
||||
def _messages_within_budget(
|
||||
messages: Sequence[_MessageT], reserved_marks: int, spend: _BudgetSpend
|
||||
) -> Sequence[_MessageT]:
|
||||
budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0)
|
||||
newest_first: Final = tuple(reversed(messages))
|
||||
budget_after_each: Final = itertools.accumulate(
|
||||
(cache_control_count(message) for message in newest_first), spend, initial=(budget, 0)
|
||||
)
|
||||
dropped_counts: Final = tuple(dropped for _, dropped in itertools.islice(budget_after_each, 1, None))
|
||||
kept_newest_first: Final = tuple(
|
||||
_without_oldest_marks(message, dropped) for message, dropped in zip(newest_first, dropped_counts)
|
||||
)
|
||||
return tuple(reversed(kept_newest_first))
|
||||
|
||||
|
||||
def newest_marks_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]:
|
||||
return _messages_within_budget(messages, reserved_marks, _newest_marks_kept)
|
||||
|
||||
|
||||
def whole_messages_within_cache_control_limit(
|
||||
messages: Sequence[_MessageT], reserved_marks: int
|
||||
) -> Sequence[_MessageT]:
|
||||
return _messages_within_budget(messages, reserved_marks, _whole_message_kept)
|
||||
|
||||
|
||||
def _part_with_cache_control_renamed(part: object) -> object:
|
||||
if not _accepts_prompt_cache_breakpoint(part):
|
||||
return part
|
||||
|
|
@ -1439,3 +1625,25 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
llm_router=None,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def messages_within_cache_control_cap(
|
||||
messages: Sequence[_MessageT], tools: object, cache_control: object = None, request_kwargs: object = None
|
||||
) -> Sequence[_MessageT]:
|
||||
"""Under drop_params, drop the oldest message hints the provider's cache_control cap has no room for.
|
||||
|
||||
Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first.
|
||||
"""
|
||||
reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints(
|
||||
_validated_object_list(tools), cache_control, request_kwargs
|
||||
)
|
||||
kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks)
|
||||
dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept))
|
||||
if dropped > 0:
|
||||
verbose_logger.warning(
|
||||
"Dropped %d cache_control hint(s) from the oldest messages because drop_params is on and the "
|
||||
"provider accepts at most %d cache_control blocks per request",
|
||||
dropped,
|
||||
MAX_CACHE_CONTROL_BLOCKS,
|
||||
)
|
||||
return kept
|
||||
|
|
|
|||
|
|
@ -88,17 +88,22 @@ def handle_any_messages_to_chat_completion_str_messages_conversion(
|
|||
return [{"input": str(messages)}]
|
||||
|
||||
|
||||
def _message_with_str_content(message: AllMessageValues) -> AllMessageValues:
|
||||
texts: Final = convert_content_list_to_str(message=message)
|
||||
if not texts:
|
||||
return message
|
||||
return cast( # cast-ok: the same message with its content flattened to one string
|
||||
AllMessageValues, {**message, "content": texts}
|
||||
)
|
||||
|
||||
|
||||
def handle_messages_with_content_list_to_str_conversion(
|
||||
messages: list[AllMessageValues],
|
||||
) -> list[AllMessageValues]:
|
||||
"""
|
||||
Handles messages with content list conversion
|
||||
Returns new messages with each content list flattened to a string, leaving the caller's messages untouched
|
||||
"""
|
||||
for message in messages:
|
||||
texts = convert_content_list_to_str(message=message)
|
||||
if texts:
|
||||
message["content"] = texts
|
||||
return messages
|
||||
return [_message_with_str_content(message) for message in messages]
|
||||
|
||||
|
||||
def strip_name_from_message(message: AllMessageValues, allowed_name_roles: list[str] = ["user"]) -> AllMessageValues:
|
||||
|
|
|
|||
|
|
@ -167,6 +167,7 @@ _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool)
|
|||
_TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_TOOLS_ADAPTER: Final = TypeAdapter(list[object])
|
||||
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_REQUEST_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
|
||||
("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
|
||||
)
|
||||
|
|
@ -2254,6 +2255,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
headers=headers,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _messages_within_cache_control_cap(
|
||||
messages: Sequence[AllMessageValues], optional_params: Mapping[str, object]
|
||||
) -> Sequence[AllMessageValues]:
|
||||
from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap
|
||||
|
||||
return messages_within_cache_control_cap(
|
||||
messages, optional_params.get("tools"), optional_params.get("cache_control"), optional_params
|
||||
)
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -2271,9 +2282,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
anthropic_messages_pt,
|
||||
)
|
||||
|
||||
drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True
|
||||
drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params"))
|
||||
capped_messages: Final = (
|
||||
list(
|
||||
self._messages_within_cache_control_cap(
|
||||
messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params)
|
||||
)
|
||||
)
|
||||
if drop_params
|
||||
else messages
|
||||
)
|
||||
|
||||
if "tools" not in optional_params and messages is not None and has_tool_call_blocks(messages):
|
||||
if "tools" not in optional_params and has_tool_call_blocks(capped_messages):
|
||||
optional_params["tools"], _ = self._map_tools(add_dummy_tool(custom_llm_provider="anthropic"))
|
||||
|
||||
# Drop thinking param if thinking is enabled but thinking_blocks are missing
|
||||
|
|
@ -2285,9 +2305,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
# Related issue: https://github.com/BerriAI/litellm/issues/18926
|
||||
if (
|
||||
optional_params.get("thinking") is not None
|
||||
and messages is not None
|
||||
and last_assistant_with_tool_calls_has_no_thinking_blocks(messages)
|
||||
and not any_assistant_message_has_thinking_blocks(messages)
|
||||
and last_assistant_with_tool_calls_has_no_thinking_blocks(capped_messages)
|
||||
and not any_assistant_message_has_thinking_blocks(capped_messages)
|
||||
):
|
||||
if litellm.modify_params:
|
||||
optional_params.pop("thinking", None)
|
||||
|
|
@ -2332,15 +2351,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
_name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request(
|
||||
optional_params=optional_params,
|
||||
)
|
||||
if _name_forward_map:
|
||||
messages = self._rewrite_tool_names_in_messages(messages, _name_forward_map)
|
||||
request_messages: Final = (
|
||||
self._rewrite_tool_names_in_messages(capped_messages, _name_forward_map)
|
||||
if _name_forward_map
|
||||
else capped_messages
|
||||
)
|
||||
if _name_reverse_map and isinstance(litellm_params, dict):
|
||||
litellm_params[ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY] = _name_reverse_map
|
||||
|
||||
# Only the leading system run becomes the top-level system prompt. A later
|
||||
# system message stays in the conversation: hoisting it rewrites the cached
|
||||
# prefix and re-bills the whole history at cache-write pricing (#36559).
|
||||
leading_system_run, later_messages = split_leading_system_run(messages)
|
||||
leading_system_run, later_messages = split_leading_system_run(request_messages)
|
||||
anthropic_system_message_list: Final = self.translate_system_message(messages=list(leading_system_run))
|
||||
# Handling anthropic API Prompt Caching
|
||||
if len(anthropic_system_message_list) > 0:
|
||||
|
|
@ -2362,7 +2384,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
except Exception as e:
|
||||
raise AnthropicError(
|
||||
status_code=400,
|
||||
message=f"{e}\nReceived Messages={messages}",
|
||||
message=f"{e}\nReceived Messages={request_messages}",
|
||||
) # don't use verbose_logger.exception, if exception is raised
|
||||
|
||||
self.update_headers_with_optional_anthropic_beta(
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from litellm.constants import (
|
|||
)
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
as_str_mapping,
|
||||
drops_unsupported_params,
|
||||
filter_exceptions_from_params,
|
||||
filter_internal_params,
|
||||
map_finish_reason,
|
||||
|
|
@ -2129,6 +2130,24 @@ class AmazonConverseConfig(BaseConfig):
|
|||
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def _claude_messages_within_cache_control_cap(
|
||||
messages: Sequence[AllMessageValues],
|
||||
capability_model: str,
|
||||
optional_params: Mapping[str, object],
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> Sequence[AllMessageValues]:
|
||||
if not drops_unsupported_params(litellm_params.get("drop_params")) or "claude" not in capability_model.lower():
|
||||
return messages
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
messages_within_cache_control_cap,
|
||||
tool_message_with_single_cache_mark,
|
||||
)
|
||||
|
||||
return messages_within_cache_control_cap(
|
||||
tuple(map(tool_message_with_single_cache_mark, messages)), optional_params.get("tools")
|
||||
)
|
||||
|
||||
async def _async_transform_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -2140,7 +2159,13 @@ class AmazonConverseConfig(BaseConfig):
|
|||
base_model: Final = litellm_params.get("base_model")
|
||||
capability_model: Final = bedrock_capability_model(model, base_model)
|
||||
messages, system_content_blocks = self._transform_system_message(
|
||||
messages, model=capability_model, deployment_model=model
|
||||
list(
|
||||
self._claude_messages_within_cache_control_cap(
|
||||
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
|
||||
)
|
||||
),
|
||||
model=capability_model,
|
||||
deployment_model=model,
|
||||
)
|
||||
|
||||
# Convert last user message to guarded_text if guardrailConfig is present
|
||||
|
|
@ -2221,7 +2246,13 @@ class AmazonConverseConfig(BaseConfig):
|
|||
base_model: Final = litellm_params.get("base_model")
|
||||
capability_model: Final = bedrock_capability_model(model, base_model)
|
||||
messages, system_content_blocks = self._transform_system_message(
|
||||
messages, model=capability_model, deployment_model=model
|
||||
list(
|
||||
self._claude_messages_within_cache_control_cap(
|
||||
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
|
||||
)
|
||||
),
|
||||
model=capability_model,
|
||||
deployment_model=model,
|
||||
)
|
||||
|
||||
# Convert last user message to guarded_text if guardrailConfig is present
|
||||
|
|
|
|||
|
|
@ -130,7 +130,8 @@ class DeepInfraConfig(OpenAIGPTConfig):
|
|||
- Input: {"role": "tool", "content": [{"type": "text", "text": "result"}]}
|
||||
- Output: {"role": "tool", "content": "[{\"type\": \"text\", \"text\": \"result\"}]"}
|
||||
"""
|
||||
for message in messages:
|
||||
copied_messages: Final = [message.copy() for message in messages]
|
||||
for message in copied_messages:
|
||||
if message.get("role") == "tool":
|
||||
content = message.get("content")
|
||||
|
||||
|
|
@ -149,7 +150,7 @@ class DeepInfraConfig(OpenAIGPTConfig):
|
|||
# For complex content, serialize the entire array as JSON string
|
||||
message["content"] = json.dumps(content)
|
||||
|
||||
return messages
|
||||
return copied_messages
|
||||
|
||||
@overload
|
||||
def _transform_messages(
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from litellm.secret_managers.main import get_secret_str
|
|||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionToolParam,
|
||||
ChatCompletionToolParamFunctionChunk,
|
||||
OpenAIChatCompletionToolParam,
|
||||
)
|
||||
from litellm.types.utils import (
|
||||
|
|
@ -45,6 +46,7 @@ from ...openai.chat.gpt_transformation import (
|
|||
OpenAIChatCompletionStreamingHandler,
|
||||
OpenAIGPTConfig,
|
||||
)
|
||||
from ...openai.common_utils import without_cache_control
|
||||
from ..common_utils import (
|
||||
FIREROUTER,
|
||||
FireworksAIException,
|
||||
|
|
@ -480,15 +482,23 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
|
|||
return (("response_format", _json_schema_response_format(choice_schema, "choice")),)
|
||||
|
||||
def _transform_tools(self, tools: list[OpenAIChatCompletionToolParam]) -> list[OpenAIChatCompletionToolParam]:
|
||||
for tool in tools:
|
||||
if tool.get("type") != "function":
|
||||
continue
|
||||
function = tool["function"]
|
||||
function.pop("strict", None)
|
||||
params = function.get("parameters")
|
||||
if isinstance(params, dict):
|
||||
unpack_legacy_defs(params)
|
||||
return tools
|
||||
return [self._fireworks_tool(tool) for tool in tools]
|
||||
|
||||
@staticmethod
|
||||
def _fireworks_tool(tool: OpenAIChatCompletionToolParam) -> OpenAIChatCompletionToolParam:
|
||||
if tool.get("type") != "function":
|
||||
return tool
|
||||
function: Final = tool["function"]
|
||||
params: Final = function.get("parameters")
|
||||
unstrict_function: Final = cast( # cast-ok: the same function definition minus `strict`
|
||||
ChatCompletionToolParamFunctionChunk, {key: value for key, value in function.items() if key != "strict"}
|
||||
)
|
||||
new_function: Final[ChatCompletionToolParamFunctionChunk] = (
|
||||
{**unstrict_function, "parameters": unpack_legacy_defs(params, copy=True)}
|
||||
if isinstance(params, dict)
|
||||
else unstrict_function
|
||||
)
|
||||
return {**tool, "function": new_function}
|
||||
|
||||
def _transform_messages_helper(
|
||||
self, messages: list[AllMessageValues], model: str, litellm_params: dict
|
||||
|
|
@ -496,10 +506,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
|
|||
"""
|
||||
Strip fields not permitted by FireworksAI from messages.
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
filter_value_from_dict,
|
||||
)
|
||||
|
||||
supports_vision_value: Final = self._get_model_cost_capability_exact(model=model, capability="supports_vision")
|
||||
for message in messages:
|
||||
if message["role"] == "user":
|
||||
|
|
@ -530,9 +536,11 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
|
|||
model=model,
|
||||
llm_provider="fireworks_ai",
|
||||
)
|
||||
filter_value_from_dict(cast(dict, message), "cache_control")
|
||||
|
||||
return [strip_litellm_response_only_fields(message) for message in messages]
|
||||
return [
|
||||
strip_litellm_response_only_fields(cast(AllMessageValues, without_cache_control(message)))
|
||||
for message in messages
|
||||
]
|
||||
|
||||
# Cached index of fireworks_ai/* entries from litellm.model_cost. Building
|
||||
# this index requires a full scan of model_cost (tens of thousands of
|
||||
|
|
|
|||
|
|
@ -140,7 +140,8 @@ class GroqChatConfig(OpenAILikeChatConfig):
|
|||
def _transform_messages(
|
||||
self, messages: list[AllMessageValues], model: str, is_async: bool = False
|
||||
) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]:
|
||||
for idx, message in enumerate(messages):
|
||||
transformed_messages: Final = list(messages)
|
||||
for idx, message in enumerate(transformed_messages):
|
||||
"""
|
||||
1. Don't pass 'null' function_call assistant message to groq - https://github.com/BerriAI/litellm/issues/5839
|
||||
"""
|
||||
|
|
@ -154,12 +155,12 @@ class GroqChatConfig(OpenAILikeChatConfig):
|
|||
for k, v in _message.items():
|
||||
if v is not None:
|
||||
new_message[k] = v
|
||||
messages[idx] = strip_litellm_internal_message_fields(new_message)
|
||||
transformed_messages[idx] = strip_litellm_internal_message_fields(new_message)
|
||||
|
||||
if is_async:
|
||||
return super()._transform_messages(messages=messages, model=model, is_async=True)
|
||||
return super()._transform_messages(messages=transformed_messages, model=model, is_async=True)
|
||||
else:
|
||||
return super()._transform_messages(messages=messages, model=model, is_async=False)
|
||||
return super()._transform_messages(messages=transformed_messages, model=model, is_async=False)
|
||||
|
||||
def transform_messages(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
|
||||
"""
|
||||
|
||||
import copy
|
||||
import json
|
||||
from collections.abc import Coroutine
|
||||
from typing import Final, Literal, cast, overload
|
||||
|
|
@ -88,7 +89,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
|
|||
) -> dict:
|
||||
_tools = non_default_params.pop("tools", None)
|
||||
if _tools is not None:
|
||||
_tools = remove_additional_properties(_tools)
|
||||
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
|
||||
_tools = remove_strict_from_schema(_tools)
|
||||
if isinstance(_tools, list):
|
||||
_tools = self._convert_custom_tools_to_function_tools(_tools)
|
||||
|
|
|
|||
|
|
@ -755,7 +755,8 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
if _name is not None:
|
||||
# Remove name if not a tool message
|
||||
if message["role"] != "tool" or isinstance(_name, str) and len(_name.strip()) == 0:
|
||||
message.pop("name", None)
|
||||
message_without_name: Final = {k: v for k, v in message.items() if k != "name"}
|
||||
return cast(AllMessageValues, message_without_name) # cast-ok: the same message minus name
|
||||
|
||||
return message
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from pydantic import TypeAdapter, ValidationError
|
|||
|
||||
import litellm
|
||||
from litellm.constants import OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS
|
||||
from litellm.litellm_core_utils.core_helpers import map_finish_reason
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
extract_reasoning_content,
|
||||
handle_invalid_parallel_tool_calls,
|
||||
|
|
@ -64,7 +64,7 @@ from litellm.types.utils import (
|
|||
)
|
||||
from litellm.utils import convert_to_model_response_object, declared_mid_conversation_system_support
|
||||
|
||||
from ..common_utils import OpenAIError
|
||||
from ..common_utils import OpenAIError, without_cache_control
|
||||
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -128,6 +128,29 @@ def _litellm_params_str(litellm_params: Mapping[str, object] | None, key: str) -
|
|||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
def _drops_params(litellm_params: Mapping[str, object]) -> bool:
|
||||
return drops_unsupported_params(litellm_params.get("drop_params"))
|
||||
|
||||
|
||||
def _names_claude(model: str, litellm_params: Mapping[str, object]) -> bool:
|
||||
base_model: Final = litellm_params.get("base_model")
|
||||
return "claude" in model.lower() or (isinstance(base_model, str) and "claude" in base_model.lower())
|
||||
|
||||
|
||||
def _claude_messages_within_cache_control_cap(
|
||||
model: str,
|
||||
messages: Sequence[AllMessageValues],
|
||||
tools: Sequence["ChatCompletionToolParam"] | None,
|
||||
litellm_params: Mapping[str, object],
|
||||
request_params: Mapping[str, object],
|
||||
) -> Sequence[AllMessageValues]:
|
||||
if not _names_claude(model, litellm_params) or not _drops_params(litellm_params):
|
||||
return messages
|
||||
from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap
|
||||
|
||||
return list(messages_within_cache_control_cap(messages, tools, request_params.get("cache_control"), request_params))
|
||||
|
||||
|
||||
def _prompt_cache_options(optional_params: Mapping[str, object]) -> object | None:
|
||||
try:
|
||||
extra_body: Final = _OBJECT_MAPPING.validate_python(optional_params.get("extra_body"))
|
||||
|
|
@ -311,52 +334,58 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
return content_copy
|
||||
|
||||
async def _async_handle_pdf_url(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile:
|
||||
file_id: Final = content_item.get("file_id")
|
||||
content_copy: Final = content_item.copy()
|
||||
file_id: Final = content_copy.get("file_id")
|
||||
if file_id is not None: # check for file id being url done in _handle_pdf_url
|
||||
base64_data: Final = await async_convert_url_to_base64(file_id)
|
||||
content_item["file_data"] = base64_data
|
||||
content_item["filename"] = "my_file.pdf"
|
||||
content_item.pop("file_id")
|
||||
return content_item
|
||||
content_copy["file_data"] = base64_data
|
||||
content_copy["filename"] = "my_file.pdf"
|
||||
content_copy.pop("file_id")
|
||||
return content_copy
|
||||
|
||||
def _common_file_data_check(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile:
|
||||
file_data: Final = content_item.get("file_data")
|
||||
filename: Final = content_item.get("filename")
|
||||
if file_data is not None and filename is None:
|
||||
content_item["filename"] = "my_file.pdf"
|
||||
return {**content_item, "filename": "my_file.pdf"}
|
||||
return content_item
|
||||
|
||||
@staticmethod
|
||||
def _image_item_for_openai(content_item: ChatCompletionImageObject) -> ChatCompletionImageObject:
|
||||
image_url: Final = content_item["image_url"]
|
||||
if isinstance(image_url, str):
|
||||
return {**content_item, "image_url": {"url": image_url}}
|
||||
if not isinstance(image_url, dict):
|
||||
return content_item
|
||||
new_image_url_obj: Final = cast(
|
||||
ChatCompletionImageUrlObject,
|
||||
{k: v for k, v in image_url.items() if k not in {"format", "video_metadata"}},
|
||||
)
|
||||
return {**content_item, "image_url": new_image_url_obj}
|
||||
|
||||
@staticmethod
|
||||
def _file_item_for_openai(content_item: ChatCompletionFileObject) -> ChatCompletionFileObject:
|
||||
litellm_specific_params: Final = {"format", "detail", "video_metadata"}
|
||||
file_obj: Final = content_item.get("file")
|
||||
if file_obj is None:
|
||||
raise litellm.BadRequestError(
|
||||
message="Content block has type='file' but is missing the required 'file' field",
|
||||
model=None,
|
||||
llm_provider="openai",
|
||||
)
|
||||
new_file_obj: Final = ChatCompletionFileObjectFile(
|
||||
**{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
|
||||
)
|
||||
return {**content_item, "file": new_file_obj}
|
||||
|
||||
def _apply_common_transform_content_item(
|
||||
self,
|
||||
content_item: OpenAIMessageContentListBlock,
|
||||
) -> OpenAIMessageContentListBlock:
|
||||
litellm_specific_params: Final = {"format", "detail", "video_metadata"}
|
||||
if content_item.get("type") == "image_url":
|
||||
content_item = cast(ChatCompletionImageObject, content_item)
|
||||
if isinstance(content_item["image_url"], str):
|
||||
content_item["image_url"] = {
|
||||
"url": content_item["image_url"],
|
||||
}
|
||||
elif isinstance(content_item["image_url"], dict):
|
||||
new_image_url_obj: Final = cast(
|
||||
ChatCompletionImageUrlObject,
|
||||
{k: v for k, v in content_item["image_url"].items() if k not in {"format", "video_metadata"}},
|
||||
)
|
||||
content_item["image_url"] = new_image_url_obj
|
||||
elif content_item.get("type") == "file":
|
||||
content_item = cast(ChatCompletionFileObject, content_item)
|
||||
file_obj: Final = content_item.get("file")
|
||||
if file_obj is None:
|
||||
raise litellm.BadRequestError(
|
||||
message="Content block has type='file' but is missing the required 'file' field",
|
||||
model=None,
|
||||
llm_provider="openai",
|
||||
)
|
||||
new_file_obj: Final = ChatCompletionFileObjectFile(
|
||||
**{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
|
||||
)
|
||||
content_item["file"] = new_file_obj
|
||||
|
||||
return self._image_item_for_openai(cast(ChatCompletionImageObject, content_item))
|
||||
if content_item.get("type") == "file":
|
||||
return self._file_item_for_openai(cast(ChatCompletionFileObject, content_item))
|
||||
return content_item
|
||||
|
||||
def _transform_content_item(
|
||||
|
|
@ -419,31 +448,34 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
hoisted_messages: Final = hoist_images_from_tool_messages(stripped_messages)
|
||||
|
||||
async def _async_transform():
|
||||
for message in hoisted_messages:
|
||||
message_content = message.get("content")
|
||||
message_role = message.get("role")
|
||||
|
||||
if message_role == "user" and message_content and isinstance(message_content, list):
|
||||
message_content_types = cast(list[OpenAIMessageContentListBlock], message_content)
|
||||
for i, content_item in enumerate(message_content_types):
|
||||
message_content_types[i] = await self._async_transform_content_item(
|
||||
cast(OpenAIMessageContentListBlock, content_item),
|
||||
)
|
||||
return hoisted_messages
|
||||
return [await self._async_user_message_for_openai(message) for message in hoisted_messages]
|
||||
|
||||
if is_async:
|
||||
return _async_transform()
|
||||
else:
|
||||
for message in hoisted_messages:
|
||||
message_content = message.get("content")
|
||||
message_role = message.get("role")
|
||||
if message_role == "user" and message_content and isinstance(message_content, list):
|
||||
message_content_types = cast(list[OpenAIMessageContentListBlock], message_content)
|
||||
for i, content_item in enumerate(message_content):
|
||||
message_content_types[i] = self._transform_content_item(
|
||||
cast(OpenAIMessageContentListBlock, content_item)
|
||||
)
|
||||
return hoisted_messages
|
||||
return [self._user_message_for_openai(message) for message in hoisted_messages]
|
||||
|
||||
@staticmethod
|
||||
def _user_content_parts(message: AllMessageValues) -> Sequence[OpenAIMessageContentListBlock] | None:
|
||||
message_content: Final = message.get("content")
|
||||
if message.get("role") != "user" or not message_content or not isinstance(message_content, list):
|
||||
return None
|
||||
return cast(Sequence[OpenAIMessageContentListBlock], message_content)
|
||||
|
||||
def _user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues:
|
||||
parts: Final = self._user_content_parts(message)
|
||||
if parts is None:
|
||||
return message
|
||||
return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI
|
||||
AllMessageValues, {**message, "content": [self._transform_content_item(part) for part in parts]}
|
||||
)
|
||||
|
||||
async def _async_user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues:
|
||||
parts: Final = self._user_content_parts(message)
|
||||
if parts is None:
|
||||
return message
|
||||
return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI
|
||||
AllMessageValues, {**message, "content": [await self._async_transform_content_item(part) for part in parts]}
|
||||
)
|
||||
|
||||
def transform_messages(
|
||||
self,
|
||||
|
|
@ -462,23 +494,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
tools: list["ChatCompletionToolParam"] | None = None,
|
||||
) -> tuple[list[AllMessageValues], list["ChatCompletionToolParam"] | None]:
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
filter_value_from_dict,
|
||||
)
|
||||
from litellm.types.llms.openai import ChatCompletionToolParam
|
||||
|
||||
for i, message in enumerate(messages):
|
||||
messages[i] = cast(
|
||||
AllMessageValues,
|
||||
filter_value_from_dict(message, "cache_control"),
|
||||
)
|
||||
if tools is not None:
|
||||
for i, tool in enumerate(tools):
|
||||
tools[i] = cast(
|
||||
ChatCompletionToolParam,
|
||||
filter_value_from_dict(tool, "cache_control"),
|
||||
)
|
||||
return messages, tools
|
||||
new_messages: Final = [cast(AllMessageValues, without_cache_control(message)) for message in messages]
|
||||
new_tools: Final = (
|
||||
[cast(ChatCompletionToolParam, without_cache_control(tool)) for tool in tools]
|
||||
if tools is not None
|
||||
else None
|
||||
)
|
||||
return new_messages, new_tools
|
||||
|
||||
def translate_developer_role_to_system_role(
|
||||
self,
|
||||
|
|
@ -527,6 +551,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
tools: Sequence["ChatCompletionToolParam"] | None,
|
||||
prompt_cache_options: object,
|
||||
litellm_params: Mapping[str, object],
|
||||
request_params: Mapping[str, object],
|
||||
) -> tuple[Sequence[AllMessageValues], Sequence["ChatCompletionToolParam"] | None]:
|
||||
"""
|
||||
A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on
|
||||
|
|
@ -543,10 +568,11 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
model, custom_llm_provider, api_base, prompt_cache_options
|
||||
)
|
||||
if not reads_breakpoints and self._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base):
|
||||
return messages, tools
|
||||
drops_breakpoints: Final = not reads_breakpoints and (
|
||||
litellm.drop_params is True or litellm_params.get("drop_params") is True
|
||||
)
|
||||
return (
|
||||
_claude_messages_within_cache_control_cap(model, messages, tools, litellm_params, request_params),
|
||||
tools,
|
||||
)
|
||||
drops_breakpoints: Final = not reads_breakpoints and _drops_params(litellm_params)
|
||||
renamed: Final = (
|
||||
[AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages]
|
||||
if reads_breakpoints
|
||||
|
|
@ -561,7 +587,13 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
if drops_breakpoints
|
||||
else list(tools)
|
||||
)
|
||||
return self.remove_cache_control_flag_from_messages_and_tools(model=model, messages=renamed, tools=kept_tools)
|
||||
sent_messages, sent_tools = self.remove_cache_control_flag_from_messages_and_tools(
|
||||
model=model, messages=renamed, tools=kept_tools
|
||||
)
|
||||
return (
|
||||
_claude_messages_within_cache_control_cap(model, sent_messages, sent_tools, litellm_params, request_params),
|
||||
sent_tools,
|
||||
)
|
||||
|
||||
def _sanitized_tools_update_for_openai(
|
||||
self,
|
||||
|
|
@ -623,6 +655,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
tools=optional_params.get("tools", []),
|
||||
prompt_cache_options=_prompt_cache_options(optional_params),
|
||||
litellm_params=litellm_params,
|
||||
request_params=optional_params,
|
||||
)
|
||||
if tools is not None and len(tools) > 0:
|
||||
optional_params["tools"] = tools
|
||||
|
|
@ -654,6 +687,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
tools=optional_params.get("tools", []),
|
||||
prompt_cache_options=_prompt_cache_options(optional_params),
|
||||
litellm_params=litellm_params,
|
||||
request_params=optional_params,
|
||||
)
|
||||
if tools is not None and len(tools) > 0:
|
||||
optional_params["tools"] = tools
|
||||
|
|
|
|||
|
|
@ -153,12 +153,13 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig):
|
|||
- role: system ==> translate to role 'user'
|
||||
"""
|
||||
_supports_system_messages: Final = supports_system_messages(model, "openai")
|
||||
for i, message in enumerate(messages):
|
||||
transformed_messages: Final = list(messages)
|
||||
for i, message in enumerate(transformed_messages):
|
||||
if message["role"] == "system" and not _supports_system_messages:
|
||||
new_message = ChatCompletionUserMessage(content=message["content"], role="user")
|
||||
messages[i] = new_message # Replace the old message with the new one
|
||||
transformed_messages[i] = new_message # Replace the old message with the new one
|
||||
|
||||
if is_async:
|
||||
return super()._transform_messages(messages, model, is_async=cast(Literal[True], True))
|
||||
return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[True], True))
|
||||
else:
|
||||
return super()._transform_messages(messages, model, is_async=cast(Literal[False], False))
|
||||
return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[False], False))
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Common helpers / utils across al OpenAI endpoints
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import hashlib
|
||||
import inspect
|
||||
import json
|
||||
|
|
@ -31,7 +32,7 @@ if TYPE_CHECKING:
|
|||
import litellm
|
||||
from litellm.exceptions import error_response_without_credentials
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, iter_items
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params
|
||||
from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
|
@ -43,6 +44,47 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
)
|
||||
|
||||
|
||||
def _is_plain_dict(value: object) -> bool:
|
||||
return isinstance(value, dict)
|
||||
|
||||
|
||||
def _is_list(value: object) -> bool:
|
||||
return isinstance(value, list)
|
||||
|
||||
|
||||
def _contains_cache_control(value: object) -> bool:
|
||||
pending: Final[list[object]] = [value] # mutable-ok: a local worklist, never returned or stored
|
||||
while pending:
|
||||
node = pending.pop()
|
||||
mapping = as_str_mapping(node)
|
||||
if mapping is not None:
|
||||
if "cache_control" in mapping:
|
||||
return True
|
||||
pending.extend(mapping.values())
|
||||
elif _is_list(node):
|
||||
pending.extend(iter_items(node))
|
||||
return False
|
||||
|
||||
|
||||
def without_cache_control(item: object) -> object:
|
||||
"""Return `item` with every `cache_control` key removed, without touching the original.
|
||||
|
||||
`filter_value_from_dict` deletes the key in place and recurses into nested dicts and
|
||||
lists, so the caller's own message, tool or input object has to be copied first: it
|
||||
may reuse the same list on a provider that does support prompt caching. The copy is
|
||||
skipped when there is nothing to strip, which keeps the object identity the
|
||||
pass-through paths rely on and avoids a deep copy on every request.
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import filter_value_from_dict
|
||||
|
||||
if not _is_plain_dict(item) or not _contains_cache_control(item):
|
||||
return item
|
||||
copied: Final = copy.deepcopy(item)
|
||||
if isinstance(copied, dict):
|
||||
filter_value_from_dict(copied, "cache_control")
|
||||
return copied
|
||||
|
||||
|
||||
def _get_client_init_params(cls: type) -> tuple[str, ...]:
|
||||
"""Extract __init__ parameter names (excluding 'self') from a class."""
|
||||
return tuple(p for p in inspect.signature(cls.__init__).parameters if p != "self")
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ from litellm.types.router import GenericLiteLLMParams
|
|||
from litellm.types.utils import LlmProviders
|
||||
from litellm.types.workload_identity import OPENAI_WIF_KWARGS_KEYS
|
||||
|
||||
from ..common_utils import OpenAIError
|
||||
from ..common_utils import OpenAIError, without_cache_control
|
||||
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
|
||||
|
||||
OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16
|
||||
|
|
@ -658,13 +658,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
input=self._replayable_input(marker_safe_input, reads_breakpoints, maps_anthropic_tool_search),
|
||||
tools=mapped_tools,
|
||||
)
|
||||
if marker_safe_input is not validated_input:
|
||||
self._strip_caller_input_as_if_unmarked(
|
||||
model=model,
|
||||
caller_input=validated_input,
|
||||
reads_breakpoints=reads_breakpoints,
|
||||
maps_anthropic_tool_search=maps_anthropic_tool_search,
|
||||
)
|
||||
object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools)
|
||||
sanitized_tools: Final = self._sanitized_tool_schemas_for_openai(
|
||||
model=model, tools=object_schema_tools, litellm_params=litellm_params
|
||||
|
|
@ -679,17 +672,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
return marked_input
|
||||
return input_without_unanswered_anthropic_tool_search_calls(marked_input)
|
||||
|
||||
def _strip_caller_input_as_if_unmarked(
|
||||
self,
|
||||
model: str,
|
||||
caller_input: str | ResponseInputParam,
|
||||
reads_breakpoints: bool,
|
||||
maps_anthropic_tool_search: bool,
|
||||
) -> None:
|
||||
self.remove_cache_control_flag_from_input_and_tools(
|
||||
model=model, input=self._replayable_input(caller_input, reads_breakpoints, maps_anthropic_tool_search)
|
||||
)
|
||||
|
||||
def _input_with_wire_safe_prompt_cache_breakpoints(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -800,24 +782,29 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
the chat path. Strips Anthropic-only `cache_control` markers from
|
||||
Responses API input content blocks and tools.
|
||||
|
||||
`filter_value_from_dict` mutates each dict in place, so the same
|
||||
objects are returned.
|
||||
`filter_value_from_dict` deletes the key in place and recurses, so each
|
||||
item is copied first: the caller keeps its own input list and may reuse
|
||||
it on a provider that does support prompt caching.
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
filter_value_from_dict,
|
||||
new_input: Final = (
|
||||
cast( # cast-ok: the comprehension rebuilds the input item for item
|
||||
ResponseInputParam,
|
||||
[without_cache_control(item) for item in input],
|
||||
)
|
||||
if isinstance(input, list)
|
||||
else input
|
||||
)
|
||||
|
||||
if isinstance(input, list):
|
||||
for item in input:
|
||||
if isinstance(item, dict):
|
||||
filter_value_from_dict(cast(dict, item), "cache_control")
|
||||
new_tools: Final = (
|
||||
cast( # cast-ok: the comprehension rebuilds the tool list item for item
|
||||
Sequence[ALL_RESPONSES_API_TOOL_PARAMS],
|
||||
[without_cache_control(tool) for tool in tools],
|
||||
)
|
||||
if tools is not None
|
||||
else None
|
||||
)
|
||||
|
||||
if tools is not None:
|
||||
for tool in tools:
|
||||
if isinstance(tool, dict):
|
||||
filter_value_from_dict(cast(dict, tool), "cache_control")
|
||||
|
||||
return input, tools
|
||||
return new_input, new_tools
|
||||
|
||||
def _drop_bridge_minted_reasoning_items(self, input: str | ResponseInputParam) -> str | ResponseInputParam:
|
||||
if self.custom_llm_provider not in _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING or not isinstance(input, list):
|
||||
|
|
|
|||
|
|
@ -110,38 +110,23 @@ def separate_cached_messages(
|
|||
messages: list[AllMessageValues],
|
||||
) -> tuple[list[AllMessageValues], list[AllMessageValues]]:
|
||||
"""
|
||||
Returns separated cached and non-cached messages.
|
||||
|
||||
Args:
|
||||
messages: List of messages to be separated.
|
||||
|
||||
Returns:
|
||||
Tuple containing:
|
||||
- cached_messages: List of cached messages.
|
||||
- non_cached_messages: List of non-cached messages.
|
||||
Returns the first continuous block of cached messages, plus every system message once
|
||||
any message is cached, since generateContent rejects a system_instruction beside a
|
||||
cachedContent, and the remaining messages.
|
||||
"""
|
||||
cached_messages: list[AllMessageValues] = []
|
||||
non_cached_messages: list[AllMessageValues] = []
|
||||
filtered_messages: Final = [(idx, message) for idx, message in enumerate(messages) if is_cached_message(message)]
|
||||
if not filtered_messages:
|
||||
return [], messages
|
||||
|
||||
# Extract cached messages and their indices
|
||||
filtered_messages: Final[list[tuple[int, AllMessageValues]]] = []
|
||||
for idx, message in enumerate(messages):
|
||||
if is_cached_message(message=message):
|
||||
filtered_messages.append((idx, message))
|
||||
|
||||
# Validate only one block of continuous cached messages
|
||||
last_continuous_block_idx: Final = get_first_continuous_block_idx(filtered_messages)
|
||||
# Separate messages based on the block of cached messages
|
||||
if filtered_messages and last_continuous_block_idx is not None:
|
||||
first_cached_idx: Final = filtered_messages[0][0]
|
||||
last_cached_idx: Final = filtered_messages[last_continuous_block_idx][0]
|
||||
|
||||
cached_messages = messages[first_cached_idx : last_cached_idx + 1]
|
||||
non_cached_messages = messages[:first_cached_idx] + messages[last_cached_idx + 1 :]
|
||||
else:
|
||||
non_cached_messages = messages
|
||||
|
||||
return cached_messages, non_cached_messages
|
||||
cached_block: Final = range(filtered_messages[0][0], filtered_messages[last_continuous_block_idx][0] + 1)
|
||||
goes_in_cache: Final = tuple(
|
||||
idx in cached_block or message.get("role") == "system" for idx, message in enumerate(messages)
|
||||
)
|
||||
return (
|
||||
[message for message, cached in zip(messages, goes_in_cache) if cached],
|
||||
[message for message, cached in zip(messages, goes_in_cache) if not cached],
|
||||
)
|
||||
|
||||
|
||||
def cached_messages_end_on_supported_turn(cached_messages: Sequence[AllMessageValues]) -> bool:
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ Translation from OpenAI's `/chat/completions` endpoint to IBM WatsonX's `/text/c
|
|||
Docs: https://cloud.ibm.com/apidocs/watsonx-ai#text-chat
|
||||
"""
|
||||
|
||||
import copy
|
||||
from typing import Final
|
||||
|
||||
from litellm import verbose_logger
|
||||
|
|
@ -57,7 +58,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig):
|
|||
_tools = non_default_params.pop("tools", None)
|
||||
if _tools is not None:
|
||||
# remove 'additionalProperties' from tools
|
||||
_tools = remove_additional_properties(_tools)
|
||||
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
|
||||
# remove 'strict' from tools
|
||||
_tools = remove_strict_from_schema(_tools)
|
||||
if _tools is not None:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
from collections.abc import AsyncIterator, Iterator, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final
|
||||
|
|
@ -208,7 +209,8 @@ class XAIChatConfig(OpenAIGPTConfig):
|
|||
tools = [
|
||||
filtered_tool
|
||||
for filtered_tool in (
|
||||
_FILTERED_TOOL.validate_python(filter_value_from_dict(tool, "strict")) for tool in value
|
||||
_FILTERED_TOOL.validate_python(filter_value_from_dict(copy.deepcopy(tool), "strict")) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
|
||||
for tool in value
|
||||
)
|
||||
if filtered_tool is not None
|
||||
]
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import uuid
|
|||
from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from itertools import accumulate, chain, groupby, islice
|
||||
from itertools import chain, groupby
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
|
|
@ -45,7 +45,15 @@ from litellm.constants import (
|
|||
REDACTED_BY_LITELLM,
|
||||
REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER,
|
||||
)
|
||||
from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
AnthropicCacheControlHook,
|
||||
block_with_cache_control,
|
||||
cache_control_count,
|
||||
dropped_cache_control,
|
||||
newest_marks_within_cache_control_limit,
|
||||
whole_messages_within_cache_control_limit,
|
||||
with_cache_control,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params
|
||||
from litellm.litellm_core_utils.get_supported_openai_params import (
|
||||
get_supported_openai_params,
|
||||
|
|
@ -603,10 +611,6 @@ def _validated_cache_control(value: object) -> ChatCompletionCachedContent | Non
|
|||
return None
|
||||
|
||||
|
||||
def _dropped_cache_control(_: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _without_cache_control_key(item: object) -> object:
|
||||
if not isinstance(item, dict):
|
||||
return item
|
||||
|
|
@ -694,83 +698,6 @@ def _content_blocks_carrying(content: object, cache_control: ChatCompletionCache
|
|||
return _with_cache_control_on_last_block(_OBJECT_LIST_ADAPTER.validate_python(content), cache_control)
|
||||
|
||||
|
||||
_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls")
|
||||
|
||||
|
||||
def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]:
|
||||
lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS)
|
||||
return tuple(
|
||||
chain.from_iterable(_OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list))
|
||||
)
|
||||
|
||||
|
||||
def _block_carries_cache_control(block: object) -> bool:
|
||||
return isinstance(block, dict) and _STR_KEY_DICT_ADAPTER.validate_python(block).get("cache_control") is not None
|
||||
|
||||
|
||||
def _cache_control_count(message: object) -> int:
|
||||
if not isinstance(message, dict):
|
||||
return 0
|
||||
fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message)
|
||||
block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block))
|
||||
return int(fields.get("cache_control") is not None) + block_marks
|
||||
|
||||
|
||||
_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None]
|
||||
|
||||
|
||||
def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]:
|
||||
others: Final = {key: value for key, value in fields.items() if key != "cache_control"}
|
||||
if "cache_control" not in fields:
|
||||
return others
|
||||
cache_control: Final = rewrite(fields["cache_control"])
|
||||
return others if cache_control is None else {**others, "cache_control": cache_control}
|
||||
|
||||
|
||||
def _block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object:
|
||||
if not isinstance(block, dict):
|
||||
return block
|
||||
return _fields_with_cache_control(_STR_KEY_DICT_ADAPTER.validate_python(block), rewrite)
|
||||
|
||||
|
||||
def _with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT:
|
||||
if not isinstance(message, dict):
|
||||
return message
|
||||
fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message)
|
||||
blocks_rewritten: Final = {
|
||||
key: (
|
||||
[_block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)]
|
||||
if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list)
|
||||
else value
|
||||
)
|
||||
for key, value in fields.items()
|
||||
}
|
||||
return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped
|
||||
_MessageT, _fields_with_cache_control(blocks_rewritten, rewrite)
|
||||
)
|
||||
|
||||
|
||||
def _kept_within_budget(state: tuple[int, bool], marks: int) -> tuple[int, bool]:
|
||||
remaining, _ = state
|
||||
if marks > remaining:
|
||||
return remaining, False
|
||||
return remaining - marks, True
|
||||
|
||||
|
||||
def _messages_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]:
|
||||
budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0)
|
||||
newest_first: Final = tuple(reversed(messages))
|
||||
budget_after_each: Final = accumulate(
|
||||
(_cache_control_count(message) for message in newest_first), _kept_within_budget, initial=(budget, True)
|
||||
)
|
||||
kept_flags: Final = tuple(kept for _, kept in islice(budget_after_each, 1, None))
|
||||
kept_newest_first: Final = tuple(
|
||||
message if kept else _with_cache_control(message, _dropped_cache_control)
|
||||
for message, kept in zip(newest_first, kept_flags)
|
||||
)
|
||||
return tuple(reversed(kept_newest_first))
|
||||
|
||||
|
||||
_SYSTEM_MESSAGE_FIELDS: Final = frozenset(("content", "cache_control"))
|
||||
|
||||
|
||||
|
|
@ -799,7 +726,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT:
|
|||
if fields is None:
|
||||
return message
|
||||
if fields.get("role") == "tool":
|
||||
return _with_cache_control(message, _dropped_cache_control)
|
||||
return with_cache_control(message, dropped_cache_control)
|
||||
tool_calls: Final = fields.get("tool_calls")
|
||||
if not isinstance(tool_calls, list):
|
||||
return message
|
||||
|
|
@ -808,7 +735,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT:
|
|||
{
|
||||
**fields,
|
||||
"tool_calls": [
|
||||
_block_with_cache_control(tool_call, _dropped_cache_control)
|
||||
block_with_cache_control(tool_call, dropped_cache_control)
|
||||
for tool_call in _OBJECT_LIST_ADAPTER.validate_python(tool_calls)
|
||||
],
|
||||
},
|
||||
|
|
@ -822,7 +749,7 @@ class _ItemMarkRules:
|
|||
external_marks: int
|
||||
|
||||
def sanitized(self, message: _MessageT) -> _MessageT:
|
||||
valid: Final = _with_cache_control(message, _validated_cache_control)
|
||||
valid: Final = with_cache_control(message, _validated_cache_control)
|
||||
return _without_tool_block_marks(valid) if self.refuses_tool_block_marks else valid
|
||||
|
||||
def checked(self, message: _MessageT) -> _MessageT:
|
||||
|
|
@ -845,15 +772,15 @@ def _request_messages_for_cache_points(messages: Sequence[_MessageT], rules: _It
|
|||
as_system: Final = tuple(_developer_message_as_system_message(message) for message in messages)
|
||||
if not rules.drop_params:
|
||||
return as_system
|
||||
return _messages_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks)
|
||||
return newest_marks_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks)
|
||||
|
||||
|
||||
def _replayed_messages_for_cache_points(
|
||||
replayed_messages: Sequence[_MessageT], current_messages: Sequence[object], rules: _ItemMarkRules
|
||||
) -> Sequence[_MessageT]:
|
||||
as_system: Final = tuple(_developer_message_as_system_message(message) for message in replayed_messages)
|
||||
current_marks: Final = sum(_cache_control_count(message) for message in current_messages)
|
||||
return _messages_within_cache_control_limit(
|
||||
current_marks: Final = sum(cache_control_count(message) for message in current_messages)
|
||||
return whole_messages_within_cache_control_limit(
|
||||
tuple(map(rules.sanitized, as_system)), current_marks + rules.external_marks
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
get_semantic_cache_prompt_from_messages,
|
||||
get_str_from_messages,
|
||||
handle_any_messages_to_chat_completion_str_messages_conversion,
|
||||
handle_messages_with_content_list_to_str_conversion,
|
||||
hoist_images_from_tool_messages,
|
||||
is_encrypted_reasoning_block,
|
||||
merge_consecutive_system_messages,
|
||||
|
|
@ -2675,3 +2676,16 @@ def test_tpm_projection_preserves_non_media_identity_and_opaque_values() -> None
|
|||
projected, media = split_media_for_tpm(request)
|
||||
assert projected is request
|
||||
assert media == ()
|
||||
|
||||
|
||||
def test_content_list_flattening_returns_new_messages_and_leaves_the_callers_untouched() -> None:
|
||||
messages: Final = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "Hello"}, {"type": "text", "text": " world"}]},
|
||||
{"role": "assistant", "content": "Hi"},
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
flattened: Final = handle_messages_with_content_list_to_str_conversion(messages)
|
||||
|
||||
assert flattened == [{"role": "user", "content": "Hello world"}, {"role": "assistant", "content": "Hi"}]
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import copy
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Iterator
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
|
@ -9849,3 +9850,186 @@ def test_transform_request_tool_config_point_without_tools_drops_the_key():
|
|||
assert "cache_control_injection_points" not in data
|
||||
assert "tools" not in data
|
||||
assert litellm_params["metadata"] == {}
|
||||
|
||||
|
||||
def _five_cache_hint_conversation() -> list[dict[str, object]]:
|
||||
return [
|
||||
{"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "assistant", "content": "answer 2"},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]},
|
||||
]
|
||||
|
||||
|
||||
def _cached_texts(node: object) -> Iterator[str]:
|
||||
if isinstance(node, dict):
|
||||
yield from ((node["text"],) if "cache_control" in node and "text" in node else ())
|
||||
yield from chain.from_iterable(map(_cached_texts, node.values()))
|
||||
elif isinstance(node, list):
|
||||
yield from chain.from_iterable(map(_cached_texts, node))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("config", "model"),
|
||||
[
|
||||
(AnthropicConfig(), "claude-haiku-5-5"),
|
||||
(VertexAIAnthropicConfig(), "claude-haiku-5-5"),
|
||||
(AmazonAnthropicClaudeConfig(), "anthropic.claude-haiku-5-5"),
|
||||
(AzureAnthropicConfig(), "claude-haiku-5-5"),
|
||||
],
|
||||
ids=["anthropic", "vertex_ai", "bedrock_invoke", "azure_ai"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("drop_params", "expected_cached_texts"),
|
||||
[
|
||||
(True, ["answer 1", "turn 1", "turn 2", "turn 3"]),
|
||||
("true", ["answer 1", "turn 1", "turn 2", "turn 3"]),
|
||||
(False, ["answer 1", "system prompt", "turn 1", "turn 2", "turn 3"]),
|
||||
],
|
||||
ids=[
|
||||
"drop_params_keeps_the_newest_four",
|
||||
"string_drop_params_keeps_the_newest_four",
|
||||
"without_drop_params_all_five_pass_through",
|
||||
],
|
||||
)
|
||||
def test_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params(
|
||||
config: AnthropicConfig,
|
||||
model: str,
|
||||
drop_params: bool | str,
|
||||
expected_cached_texts: list[str],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
messages: Final = _five_cache_hint_conversation()
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = config.transform_request(
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={"drop_params": drop_params},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert sorted(_cached_texts(body)) == expected_cached_texts
|
||||
assert json.dumps(body).count('"cache_control"') == len(expected_cached_texts)
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
def test_claude_cache_control_cap_keeps_a_cached_tool_and_drops_the_oldest_message_hint(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
messages: Final = _five_cache_hint_conversation()[1:]
|
||||
tool: Final = {
|
||||
"name": "get_weather",
|
||||
"description": "Get the weather",
|
||||
"input_schema": {"type": "object", "properties": {}},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
|
||||
body: Final = AnthropicConfig().transform_request(
|
||||
model="claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={"tools": [tool]},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert body["tools"][0]["cache_control"] == {"type": "ephemeral"}
|
||||
assert sorted(_cached_texts(body["messages"])) == ["answer 1", "turn 2", "turn 3"]
|
||||
|
||||
|
||||
def test_claude_cache_control_cap_drops_a_string_assistant_turns_text_hint_before_its_tool_call_hints(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
hint: Final = {"type": "ephemeral"}
|
||||
tool_calls: Final = [
|
||||
{"id": f"call_{name}", "type": "function", "function": {"name": name, "arguments": "{}"}, "cache_control": hint}
|
||||
for name in ("lookup", "search")
|
||||
]
|
||||
follow_up: Final = [
|
||||
{"type": "text", "text": "follow-up", "cache_control": hint},
|
||||
{"type": "text", "text": "and more", "cache_control": hint},
|
||||
]
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "question"},
|
||||
{"role": "assistant", "content": "checking both", "cache_control": hint, "tool_calls": tool_calls},
|
||||
{"role": "tool", "tool_call_id": "call_lookup", "content": "found"},
|
||||
{"role": "tool", "tool_call_id": "call_search", "content": "none"},
|
||||
{"role": "user", "content": follow_up},
|
||||
]
|
||||
|
||||
body: Final = AnthropicConfig().transform_request(
|
||||
model="claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assistant_blocks: Final = body["messages"][1]["content"]
|
||||
assert [block["type"] for block in assistant_blocks] == ["text", "tool_use", "tool_use"]
|
||||
assert ["cache_control" in block for block in assistant_blocks] == [False, True, True]
|
||||
assert json.dumps(body).count('"cache_control"') == 4
|
||||
assert json.dumps(body).count('"cache_control"') == 4
|
||||
|
||||
|
||||
def test_claude_cache_control_cap_keeps_the_newest_hints_inside_one_message(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
messages: Final = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
*({"type": "text", "text": f"part {n}", "cache_control": {"type": "ephemeral"}} for n in range(1, 6)),
|
||||
{"type": "text", "text": "Reply with the single word OK."},
|
||||
],
|
||||
}
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = AnthropicConfig().transform_request(
|
||||
model="claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert sorted(_cached_texts(body["messages"])) == ["part 2", "part 3", "part 4", "part 5"]
|
||||
assert json.dumps(body).count('"cache_control"') == 4
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
def test_claude_cache_control_cap_does_not_count_a_message_level_hint_that_list_content_never_sends(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
messages: Final = [
|
||||
{"role": "system", "content": "system prompt", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "user", "content": "turn 1", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "assistant", "content": "answer 1"},
|
||||
{"role": "user", "content": "turn 2", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "assistant", "content": "answer 2"},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}],
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
]
|
||||
|
||||
body: Final = AnthropicConfig().transform_request(
|
||||
model="claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={"drop_params": True},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert sorted(_cached_texts(body)) == ["system prompt", "turn 1", "turn 2", "turn 3"]
|
||||
assert json.dumps(body).count('"cache_control"') == 4
|
||||
|
|
|
|||
|
|
@ -11259,3 +11259,86 @@ def test_converse_sends_only_the_header_betas_when_header_and_body_both_carry_be
|
|||
"thinking-binding-controls-2026-08-01",
|
||||
"thinking-display-updates-2026-08-18",
|
||||
]
|
||||
|
||||
|
||||
def _five_cache_hint_conversation() -> list[dict[str, object]]:
|
||||
return [
|
||||
{"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "assistant", "content": "answer 2"},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("drop_params", "expected_system_cache_points"),
|
||||
[(True, 0), ("true", 0), (False, 1)],
|
||||
ids=[
|
||||
"drop_params_drops_the_oldest_hint",
|
||||
"string_drop_params_drops_the_oldest_hint",
|
||||
"without_drop_params_all_five_pass_through",
|
||||
],
|
||||
)
|
||||
def test_converse_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params(
|
||||
drop_params: bool | str, expected_system_cache_points: int, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
messages: Final = _five_cache_hint_conversation()
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
request: Final = AmazonConverseConfig().transform_request(
|
||||
model="anthropic.claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={"drop_params": drop_params},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert len(list(_cache_points(request["system"]))) == expected_system_cache_points
|
||||
assert [len(list(_cache_points(message))) for message in request["messages"]] == [1, 1, 1, 0, 1]
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
def test_converse_claude_tool_message_with_several_marked_blocks_counts_as_one_cache_point_against_the_cap(
|
||||
monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
messages: Final = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "policy", "cache_control": {"type": "ephemeral"}},
|
||||
{"type": "text", "text": "question", "cache_control": {"type": "ephemeral"}},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_1",
|
||||
"content": [
|
||||
{"type": "text", "text": "part 1", "cache_control": {"type": "ephemeral"}},
|
||||
{"type": "text", "text": "part 2", "cache_control": {"type": "ephemeral"}},
|
||||
{"type": "text", "text": "part 3", "cache_control": {"type": "ephemeral"}},
|
||||
],
|
||||
},
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
request: Final = AmazonConverseConfig().transform_request(
|
||||
model="anthropic.claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={
|
||||
"tools": [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}]
|
||||
},
|
||||
litellm_params={"drop_params": True},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert len(list(_cache_points(request["messages"]))) == 3
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -2773,3 +2773,66 @@ def test_claude_json_schema_stream_split_across_json_tools_keeps_the_non_stream_
|
|||
choices: Final = _json_schema_stream_choices(respx_mock, call, split_stream)
|
||||
|
||||
assert [choice.finish_reason for choice in choices if choice.finish_reason] == [answer.finish_reason]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "base_model"),
|
||||
[("databricks-claude-haiku-5-5", None), ("support-bot-endpoint", "databricks/databricks-claude-haiku-5-5")],
|
||||
ids=["claude_in_the_endpoint_name", "claude_named_only_by_base_model"],
|
||||
)
|
||||
def test_databricks_claude_request_over_the_cache_control_cap_keeps_the_newest_four_hints_under_drop_params(
|
||||
model: str, base_model: str | None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
hint: Final = {"type": "ephemeral"}
|
||||
messages: Final = [
|
||||
{"role": "system", "content": "system prompt", "cache_control": hint},
|
||||
{"role": "user", "content": "turn 1", "cache_control": hint},
|
||||
{"role": "assistant", "content": "answer 1", "cache_control": hint},
|
||||
{"role": "user", "content": "turn 2", "cache_control": hint},
|
||||
{"role": "assistant", "content": "answer 2"},
|
||||
{"role": "user", "content": "turn 3", "cache_control": hint},
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = DatabricksConfig().transform_request(
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={"drop_params": True, "custom_llm_provider": "databricks", "base_model": base_model},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert "cache_control" not in json.dumps(body["messages"][0])
|
||||
assert json.dumps(body["messages"]).count('"cache_control"') == 4
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"optional_params",
|
||||
[{"cache_control": {"type": "ephemeral"}}, {"extra_body": {"cache_control": {"type": "ephemeral"}}}],
|
||||
ids=["top_level_cache_control", "extra_body_cache_control"],
|
||||
)
|
||||
def test_databricks_claude_cache_control_cap_keeps_room_for_a_top_level_cache_control_under_drop_params(
|
||||
optional_params: dict[str, object], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
hint: Final = {"type": "ephemeral"}
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "turn 1", "cache_control": hint},
|
||||
{"role": "assistant", "content": "answer 1", "cache_control": hint},
|
||||
{"role": "user", "content": "turn 2", "cache_control": hint},
|
||||
{"role": "assistant", "content": "answer 2", "cache_control": hint},
|
||||
{"role": "user", "content": "turn 3"},
|
||||
]
|
||||
|
||||
body: Final = DatabricksConfig().transform_request(
|
||||
model="databricks-claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params=copy.deepcopy(optional_params),
|
||||
litellm_params={"drop_params": True, "custom_llm_provider": "databricks"},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert "cache_control" not in json.dumps(body["messages"][0])
|
||||
assert json.dumps(body["messages"]).count('"cache_control"') == 3
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -200,3 +202,38 @@ async def test_deepinfra_tool_message_content_transformation_async():
|
|||
print(f"✓ Async test passed: {tool_message['content']}")
|
||||
|
||||
print("\n✅ DeepInfra async tool message transformation test passed!")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"])
|
||||
async def test_deepinfra_request_stringifies_tool_content_without_touching_the_callers_message(is_async: bool) -> None:
|
||||
from litellm.llms.deepinfra.chat.transformation import DeepInfraConfig
|
||||
|
||||
config: Final = DeepInfraConfig()
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "Calculate 10 + 10"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{"id": "call_1", "type": "function", "function": {"name": "calculator", "arguments": "{}"}}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": [{"type": "text", "text": "20"}]},
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
request_kwargs: Final = {
|
||||
"model": "deepseek-ai/DeepSeek-V3.2",
|
||||
"messages": messages,
|
||||
"optional_params": {},
|
||||
"litellm_params": {},
|
||||
"headers": {},
|
||||
}
|
||||
|
||||
body: Final = (
|
||||
await config.async_transform_request(**request_kwargs)
|
||||
if is_async
|
||||
else config.transform_request(**request_kwargs)
|
||||
)
|
||||
|
||||
assert body["messages"][2]["content"] == "20"
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
|
|
@ -537,6 +538,26 @@ def test_tool_search_reply_sent_to_fireworks_drops_litellm_response_fields():
|
|||
}
|
||||
|
||||
|
||||
def test_cache_control_is_stripped_from_the_request_but_kept_on_the_callers_messages():
|
||||
hint: Final = {"type": "ephemeral"}
|
||||
messages: Final = [
|
||||
{"role": "system", "content": [{"type": "text", "text": "a long cached prompt", "cache_control": hint}]},
|
||||
{"role": "user", "content": "hi", "cache_control": hint},
|
||||
]
|
||||
caller_messages_before: Final = json.dumps(messages)
|
||||
|
||||
request: Final = FireworksAIConfig().transform_request(
|
||||
model="accounts/fireworks/models/gpt-oss-120b",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert "cache_control" not in json.dumps(request["messages"])
|
||||
assert json.dumps(messages) == caller_messages_before
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Regression tests for legacy / OpenAPI $ref defs in tool parameters.
|
||||
#
|
||||
|
|
@ -2477,3 +2498,43 @@ def test_translate_developer_role_follows_the_deployment_model_info_over_the_cos
|
|||
)
|
||||
|
||||
assert [message["role"] for message in messages] == expected_roles
|
||||
|
||||
|
||||
def test_fireworks_request_rewrites_tools_without_touching_the_callers_tool() -> None:
|
||||
tools: Final = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "create_thing",
|
||||
"strict": True,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"thing": {"$ref": "#/definitions/Thing"}},
|
||||
"definitions": {"Thing": {"type": "object", "properties": {"id": {"type": "string"}}}},
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
tools_before: Final = copy.deepcopy(tools)
|
||||
|
||||
request: Final = FireworksAIConfig().transform_request(
|
||||
model="accounts/fireworks/models/gpt-oss-120b",
|
||||
messages=[{"role": "user", "content": "Make a thing"}],
|
||||
optional_params={"tools": tools},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request["tools"] == [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "create_thing",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"thing": {"type": "object", "properties": {"id": {"type": "string"}}}},
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
assert tools == tools_before
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import logging
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
|
@ -589,3 +590,19 @@ async def test_streamed_reasoning_reply_can_be_replayed(is_async: bool, with_too
|
|||
}
|
||||
]
|
||||
assert reply.reasoning_content == "I should check the weather."
|
||||
|
||||
|
||||
def test_groq_request_drops_null_assistant_fields_without_touching_the_callers_messages() -> None:
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "Hi"},
|
||||
{"role": "assistant", "content": "Hello", "function_call": None, "tool_calls": None},
|
||||
{"role": "user", "content": "Again"},
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = GroqChatConfig().transform_request(
|
||||
model="openai/gpt-oss-120b", messages=messages, optional_params={}, litellm_params={}, headers={}
|
||||
)
|
||||
|
||||
assert body["messages"][1] == {"role": "assistant", "content": "Hello"}
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
import copy
|
||||
import json
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -397,3 +399,43 @@ def test_hosted_vllm_custom_tools_use_top_level_input_schema():
|
|||
assert tools[0]["function"]["name"] == "search"
|
||||
assert tools[0]["function"]["description"] == "Search docs"
|
||||
assert tools[0]["function"]["parameters"] == input_schema
|
||||
|
||||
|
||||
def _strict_weather_tool() -> dict[str, object]:
|
||||
return {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"strict": True,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_hosted_vllm_tools_are_rewritten_without_touching_the_callers_tool() -> None:
|
||||
tools: Final = [_strict_weather_tool()]
|
||||
tools_before: Final = copy.deepcopy(tools)
|
||||
|
||||
optional_params: Final = HostedVLLMChatConfig().map_openai_params(
|
||||
non_default_params={"tools": tools}, optional_params={}, model="gpt-oss-120b", drop_params=False
|
||||
)
|
||||
|
||||
assert optional_params["tools"] == [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
assert tools == tools_before
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Mapping
|
||||
|
|
@ -1212,7 +1213,10 @@ def test_mistral_sends_only_the_text_items_of_a_native_thinking_chunk_beside_an_
|
|||
"content": [
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": [{"type": "text", "text": "It has whiskers."}, {"type": "reference", "reference_ids": [1]}],
|
||||
"thinking": [
|
||||
{"type": "text", "text": "It has whiskers."},
|
||||
{"type": "reference", "reference_ids": [1]},
|
||||
],
|
||||
},
|
||||
{"type": "text", "text": "A cat."},
|
||||
],
|
||||
|
|
@ -1672,6 +1676,18 @@ def test_mistral_transform_request_drops_an_empty_tool_message_whose_call_id_is_
|
|||
assert [m.get("role") for m in request["messages"]] == ["user", "assistant"]
|
||||
|
||||
|
||||
def test_mistral_request_drops_a_user_name_without_touching_the_callers_message() -> None:
|
||||
messages: Final = [{"role": "user", "content": "Hi"}, {"role": "user", "content": "", "name": "alice"}]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = MistralConfig().transform_request(
|
||||
model="mistral-medium-latest", messages=messages, optional_params={}, litellm_params={}, headers={}
|
||||
)
|
||||
|
||||
assert "name" not in json.dumps(body["messages"])
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage()
|
||||
|
|
|
|||
|
|
@ -1824,6 +1824,94 @@ class TestToolSchemaCombinatorFlatteningForOpenAI:
|
|||
assert request["tools"][0] is tool
|
||||
|
||||
|
||||
class TestCacheControlStrippingDoesNotMutateCallerInput:
|
||||
"""
|
||||
Stripping cache_control for a provider that cannot use it must not reach back
|
||||
into the caller's own message and tool objects.
|
||||
|
||||
filter_value_from_dict deletes the key in place and recurses into nested dicts
|
||||
and lists, and remove_cache_control_flag_from_messages_and_tools assigned the
|
||||
result back into the caller's list, so one call to any OpenAI-compatible
|
||||
provider permanently stripped cache_control from a list the caller still holds.
|
||||
Reusing that list on Anthropic or Bedrock afterwards then silently lost prompt
|
||||
caching, with no error and full-price billing.
|
||||
"""
|
||||
|
||||
def setup_method(self):
|
||||
self.config = OpenAIGPTConfig()
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_openai_base_env(self, monkeypatch):
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", None, raising=False)
|
||||
|
||||
@staticmethod
|
||||
def _messages():
|
||||
return [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "a long cached system prompt",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hello",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _tools():
|
||||
return [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "parameters": {"type": "object"}},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
]
|
||||
|
||||
def _transform(self, messages, tools):
|
||||
return self.config.transform_request(
|
||||
model="gpt-4o",
|
||||
messages=messages,
|
||||
optional_params={"tools": tools},
|
||||
litellm_params={"custom_llm_provider": "openai", "api_base": None},
|
||||
headers={},
|
||||
)
|
||||
|
||||
def test_caller_messages_and_tools_keep_cache_control(self):
|
||||
messages, tools = self._messages(), self._tools()
|
||||
messages_before, tools_before = copy.deepcopy(messages), copy.deepcopy(tools)
|
||||
|
||||
request = self._transform(messages, tools)
|
||||
|
||||
assert "cache_control" not in json.dumps(request)
|
||||
assert messages == messages_before
|
||||
assert tools == tools_before
|
||||
|
||||
def test_a_later_anthropic_call_still_sees_cache_control(self):
|
||||
"""The user-visible consequence: the same message list reused on a provider
|
||||
that does support caching must still carry the cache breakpoints."""
|
||||
messages = self._messages()
|
||||
|
||||
self._transform(messages, self._tools())
|
||||
|
||||
anthropic_body = litellm.AnthropicConfig().transform_request(
|
||||
model="claude-3-5-sonnet-20240620",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
assert "cache_control" in json.dumps(anthropic_body)
|
||||
|
||||
|
||||
def _developer_message_after_a_user_turn() -> list[dict[str, str]]:
|
||||
return [
|
||||
{"role": "system", "content": "You are terse."},
|
||||
|
|
@ -1916,3 +2004,53 @@ def test_completion_leaves_the_callers_closing_developer_message_unchanged_on_a_
|
|||
)
|
||||
assert json.loads(chat.calls.last.request.content)["messages"][-1]["role"] == "user"
|
||||
assert messages == sent
|
||||
|
||||
|
||||
def _user_turn_with_litellm_only_media_fields() -> list[dict[str, object]]:
|
||||
return [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe these"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png", "format": "image/png"}},
|
||||
{"type": "image_url", "image_url": "https://example.com/dog.png"},
|
||||
{
|
||||
"type": "file",
|
||||
"file": {"file_data": "data:application/pdf;base64,JVBERi0=", "format": "application/pdf"},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"])
|
||||
async def test_openai_request_rewrites_media_parts_without_touching_the_callers_message(
|
||||
is_async: bool, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", None, raising=False)
|
||||
config: Final = OpenAIGPTConfig()
|
||||
messages: Final = _user_turn_with_litellm_only_media_fields()
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
request_kwargs: Final = {
|
||||
"model": "gpt-5.5",
|
||||
"messages": messages,
|
||||
"optional_params": {},
|
||||
"litellm_params": {"custom_llm_provider": "openai", "api_base": None},
|
||||
"headers": {},
|
||||
}
|
||||
|
||||
body: Final = (
|
||||
await config.async_transform_request(**request_kwargs)
|
||||
if is_async
|
||||
else config.transform_request(**request_kwargs)
|
||||
)
|
||||
|
||||
assert body["messages"][0]["content"] == [
|
||||
{"type": "text", "text": "Describe these"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/dog.png"}},
|
||||
{"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0=", "filename": "my_file.pdf"}},
|
||||
]
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Callable, Mapping
|
||||
|
|
@ -541,6 +542,50 @@ class TestOpenAIResponsesAPIConfig:
|
|||
assert "cache_control" not in result["tools"][0]
|
||||
assert result["tools"][0]["name"] == "get_weather"
|
||||
|
||||
def test_transform_does_not_strip_cache_control_from_the_callers_input(self):
|
||||
"""Stripping for OpenAI must not reach back into the caller's own objects.
|
||||
|
||||
`filter_value_from_dict` deletes the key in place and recurses, and
|
||||
`_validate_input_param` passes plain dict items through by reference, so
|
||||
the caller's input list used to lose its cache breakpoints. Reusing that
|
||||
list on Anthropic or Bedrock afterwards then silently lost prompt caching.
|
||||
Same defect as the Chat Completions path.
|
||||
"""
|
||||
input_with_cache_control = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": "a long cached system prompt",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
tools_with_cache_control = [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object"},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
]
|
||||
input_before = copy.deepcopy(input_with_cache_control)
|
||||
tools_before = copy.deepcopy(tools_with_cache_control)
|
||||
|
||||
result = self.config.transform_responses_api_request(
|
||||
model=self.model,
|
||||
input=input_with_cache_control,
|
||||
response_api_optional_request_params={"tools": tools_with_cache_control},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert "cache_control" not in json.dumps(result)
|
||||
assert input_with_cache_control == input_before
|
||||
assert tools_with_cache_control == tools_before
|
||||
|
||||
def test_transform_preserves_input_without_cache_control(self):
|
||||
"""Inputs without cache_control must pass through unmodified."""
|
||||
input_clean = [
|
||||
|
|
@ -4607,10 +4652,10 @@ def test_cache_control_still_marks_a_part_whose_own_marker_is_malformed_under_dr
|
|||
}
|
||||
|
||||
|
||||
def _caller_input_after_transform(model: str, marker: object) -> str:
|
||||
def _hinted_caller_input(marker: object) -> list[dict[str, object]]:
|
||||
hint: Final = {"type": "ephemeral"}
|
||||
marked: Final = {} if marker is None else {"prompt_cache_breakpoint": marker}
|
||||
input: Final = [
|
||||
return [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
|
|
@ -4626,6 +4671,10 @@ def _caller_input_after_transform(model: str, marker: object) -> str:
|
|||
"content": [{"type": "output_text", "text": "ok", "cache_control": hint, **marked}],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _caller_input_after_transform(model: str, marker: object) -> str:
|
||||
input: Final = _hinted_caller_input(marker)
|
||||
_CONFIGS[0].transform_responses_api_request(
|
||||
model=model,
|
||||
input=cast("ResponseInputParam", input), # cast-ok: plain dicts shaped like the SDK's input items
|
||||
|
|
@ -4647,6 +4696,15 @@ def test_a_marker_leaves_the_caller_request_that_fallbacks_and_logs_read_as_an_u
|
|||
assert _caller_input_after_transform(model, marker).replace(marker_json, "") == unmarked
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ("gpt-4.1", "gpt-6.1-sol"))
|
||||
@pytest.mark.parametrize("marker", (None, _VALID_MARKERS[0], "yes"), ids=("unmarked", "explicit", "malformed"))
|
||||
def test_the_caller_input_keeps_its_cache_control_hints_for_a_fallback_deployment(
|
||||
monkeypatch: pytest.MonkeyPatch, model: str, marker: object
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
assert _caller_input_after_transform(model, marker) == json.dumps(_hinted_caller_input(marker))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures(httpx_transport.__name__)
|
||||
async def test_aresponses_forwards_previous_response_id_to_openai() -> None:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -198,3 +199,23 @@ def test_translate_developer_role_keeps_developer_messages_untouched() -> None:
|
|||
)
|
||||
== messages
|
||||
)
|
||||
|
||||
|
||||
def test_o_series_request_turns_system_into_user_without_touching_the_callers_messages(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
model: Final = "o-series-test-model-without-system-role"
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
model,
|
||||
{"litellm_provider": "openai", "mode": "chat", "supports_system_messages": False},
|
||||
)
|
||||
messages: Final = [{"role": "system", "content": "Be terse"}, {"role": "user", "content": "Hi"}]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = OpenAIOSeriesConfig().transform_request(
|
||||
model=model, messages=messages, optional_params={}, litellm_params={}, headers={}
|
||||
)
|
||||
|
||||
assert [message["role"] for message in body["messages"]] == ["user", "user"]
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
|
||||
import copy
|
||||
import json
|
||||
from collections.abc import Iterator
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
|
|
@ -874,6 +877,58 @@ def test_openrouter_streaming_reasoning_is_preserved() -> None:
|
|||
assert assembled.choices[0].message.reasoning_content == "Consider the evidence. Apply the rule."
|
||||
|
||||
|
||||
def _five_message_level_cache_hints() -> list[dict[str, object]]:
|
||||
return [
|
||||
{"role": "system", "content": "system prompt", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "user", "content": "turn 1", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "assistant", "content": "answer 1", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "user", "content": "turn 2", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "assistant", "content": "answer 2"},
|
||||
{"role": "user", "content": "turn 3", "cache_control": {"type": "ephemeral"}},
|
||||
]
|
||||
|
||||
|
||||
def _cached_texts(node: object) -> Iterator[str]:
|
||||
if isinstance(node, dict):
|
||||
yield from ((node["text"],) if "cache_control" in node and "text" in node else ())
|
||||
yield from chain.from_iterable(map(_cached_texts, node.values()))
|
||||
elif isinstance(node, list):
|
||||
yield from chain.from_iterable(map(_cached_texts, node))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("drop_params", "expected_cached_texts"),
|
||||
[
|
||||
(True, ["answer 1", "turn 1", "turn 2", "turn 3"]),
|
||||
("true", ["answer 1", "turn 1", "turn 2", "turn 3"]),
|
||||
(False, ["answer 1", "system prompt", "turn 1", "turn 2", "turn 3"]),
|
||||
],
|
||||
ids=[
|
||||
"drop_params_keeps_the_newest_four",
|
||||
"string_drop_params_keeps_the_newest_four",
|
||||
"without_drop_params_all_five_pass_through",
|
||||
],
|
||||
)
|
||||
def test_openrouter_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params(
|
||||
drop_params: bool | str, expected_cached_texts: list[str], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
messages: Final = _five_message_level_cache_hints()
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = OpenrouterConfig().transform_request(
|
||||
model="anthropic/claude-haiku-5.5",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={"drop_params": drop_params, "custom_llm_provider": "openrouter"},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert sorted(_cached_texts(body["messages"])) == expected_cached_texts
|
||||
assert json.dumps(body).count('"cache_control"') == len(expected_cached_texts)
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
def _transform_openrouter_response(body: dict[str, object]) -> litellm.ModelResponse:
|
||||
return OpenrouterConfig().transform_response(
|
||||
model="openrouter/openai/gpt-4o-mini",
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import json
|
||||
from typing import Final, Literal
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -2507,3 +2508,46 @@ def test_check_and_create_cache_clamps_negative_creation_tokens() -> None:
|
|||
|
||||
assert cache_name == "cachedContents/created"
|
||||
assert logging_obj.vertex_ai_cache_creation_input_tokens == 0
|
||||
|
||||
|
||||
def _recording_gemini_client(calls: list[httpx.Request]) -> HTTPHandler:
|
||||
def route(request: httpx.Request) -> httpx.Response:
|
||||
calls.append(request)
|
||||
if request.url.path.endswith("/cachedContents"):
|
||||
return httpx.Response(200, json={} if request.method == "GET" else _created_cache_body(3000))
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"candidates": [{"content": {"role": "model", "parts": [{"text": "OK"}]}, "finishReason": "STOP"}],
|
||||
"usageMetadata": {"promptTokenCount": 3000, "candidatesTokenCount": 1, "totalTokenCount": 3001},
|
||||
},
|
||||
)
|
||||
|
||||
return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(route)))
|
||||
|
||||
|
||||
def test_a_plain_system_prompt_goes_into_the_gemini_cache_instead_of_beside_it(local_model_cost_map) -> None:
|
||||
calls: Final[list[httpx.Request]] = []
|
||||
litellm.completion(
|
||||
model="gemini/gemini-flash-latest",
|
||||
messages=[
|
||||
{"role": "system", "content": "You are terse."},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "plain system prefix " * 3000, "cache_control": {"type": "ephemeral"}},
|
||||
{"type": "text", "text": "Reply with OK only."},
|
||||
],
|
||||
},
|
||||
],
|
||||
api_key="test-key",
|
||||
client=_recording_gemini_client(calls),
|
||||
)
|
||||
|
||||
created: Final = next(
|
||||
json.loads(call.content) for call in calls if call.method == "POST" and call.url.path.endswith("/cachedContents")
|
||||
)
|
||||
generated: Final = next(json.loads(call.content) for call in calls if call.url.path.endswith(":generateContent"))
|
||||
assert created["system_instruction"] == {"parts": [{"text": "You are terse."}]}
|
||||
assert generated["cachedContent"] == "cachedContents/created"
|
||||
assert "system_instruction" not in generated
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
import copy
|
||||
import importlib
|
||||
import json
|
||||
from typing import Final, Optional, cast
|
||||
|
|
@ -10,6 +11,7 @@ import pytest
|
|||
import litellm
|
||||
from litellm import completion, embedding
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.llms.watsonx.chat.transformation import IBMWatsonXChatConfig
|
||||
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
|
||||
|
||||
|
||||
|
|
@ -542,3 +544,43 @@ def test_watsonx_deployment_space_id_embedding(monkeypatch, watsonx_embedding_ca
|
|||
|
||||
# nor space_id or project_id is required by wx.ai API when inferencing deployment
|
||||
assert "project_id" not in json_data and "space_id" not in json_data
|
||||
|
||||
|
||||
def _strict_weather_tool() -> dict[str, object]:
|
||||
return {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"strict": True,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_watsonx_chat_tools_are_rewritten_without_touching_the_callers_tool() -> None:
|
||||
tools: Final = [_strict_weather_tool()]
|
||||
tools_before: Final = copy.deepcopy(tools)
|
||||
|
||||
optional_params: Final = IBMWatsonXChatConfig().map_openai_params(
|
||||
non_default_params={"tools": tools}, optional_params={}, model="ibm/granite-4-h-small", drop_params=False
|
||||
)
|
||||
|
||||
assert optional_params["tools"] == [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
assert tools == tools_before
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import json
|
||||
import os
|
||||
from typing import Final
|
||||
|
|
@ -581,3 +582,44 @@ def test_xai_grok_4_frequency_penalty_not_supported(model):
|
|||
"""
|
||||
supported_params = XAIChatConfig().get_supported_openai_params(model=model)
|
||||
assert "frequency_penalty" not in supported_params
|
||||
|
||||
|
||||
def _strict_weather_tool() -> dict[str, object]:
|
||||
return {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"strict": True,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_xai_tools_are_rewritten_without_touching_the_callers_tool() -> None:
|
||||
tools: Final = [_strict_weather_tool()]
|
||||
tools_before: Final = copy.deepcopy(tools)
|
||||
|
||||
optional_params: Final = XAIChatConfig().map_openai_params(
|
||||
non_default_params={"tools": tools}, optional_params={}, model="grok-4.3", drop_params=False
|
||||
)
|
||||
|
||||
assert optional_params["tools"] == [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
assert tools == tools_before
|
||||
|
|
|
|||
|
|
@ -6314,6 +6314,17 @@ def test_marks_past_the_provider_cap_keep_the_newest_under_drop_params(drop_para
|
|||
assert messages[1] == {"role": "user", "content": "turn 0"}
|
||||
|
||||
|
||||
def test_marks_past_the_provider_cap_inside_one_message_keep_the_newest_under_drop_params():
|
||||
parts: Final = tuple(
|
||||
{"type": "input_text", "text": f"part {index}", "cache_control": _EPHEMERAL} for index in range(1, 6)
|
||||
)
|
||||
|
||||
messages: Final = _bridged_request({"role": "user", "content": list(parts)}, drop_params=True)
|
||||
|
||||
content: Final = TypeAdapter(list[dict[str, object]]).validate_python(messages[-1]["content"])
|
||||
assert ["cache_control" in part for part in content] == [False, True, True, True, True]
|
||||
|
||||
|
||||
def test_global_drop_params_keeps_the_newest_marks(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue