Merge remote-tracking branch 'origin/main' into litellm_fix_claude_stop_reasons_pause_turn_refusal

# Conflicts:
#	litellm/llms/anthropic/chat/transformation.py
This commit is contained in:
mateo-berri 2026-10-31 01:00:11 -07:00
commit a1b098f4c4
67 changed files with 3023 additions and 328 deletions

View file

@ -634,6 +634,8 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60
BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024)
# Anthropic's Messages API rejects thinking.budget_tokens < 1024.
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024
ANTHROPIC_MAX_USER_ID_LENGTH: Final = 512
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH: Final = 255
ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:"
REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5)
DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000)

View file

@ -14,7 +14,7 @@ import itertools
import os
import re
from collections.abc import Callable, Container, Iterable, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast
from urllib.parse import urlparse
from pydantic import TypeAdapter, ValidationError
@ -102,6 +102,192 @@ def _validated_object_list(value: object) -> list[object] | None:
return None
_MessageT: Final = TypeVar("_MessageT")
_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None]
_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls")
def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]:
lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS)
return tuple(
itertools.chain.from_iterable(
_OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list)
)
)
def _block_carries_cache_control(block: object) -> bool:
return isinstance(block, dict) and _CONTENT_PART_ADAPTER.validate_python(block).get("cache_control") is not None
_ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT: Final = frozenset({"tool", "function"})
def _message_mark_reaches_provider(fields: Mapping[str, object]) -> bool:
if fields.get("cache_control") is None:
return False
return (
not isinstance(fields.get("content"), list)
or fields.get("role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT
)
def cache_control_count(message: object) -> int:
if not isinstance(message, dict):
return 0
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block))
return int(_message_mark_reaches_provider(fields)) + block_marks
def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]:
others: Final = {key: value for key, value in fields.items() if key != "cache_control"}
if "cache_control" not in fields:
return others
cache_control: Final = rewrite(fields["cache_control"])
return others if cache_control is None else {**others, "cache_control": cache_control}
def block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object:
if not isinstance(block, dict):
return block
return _fields_with_cache_control(_CONTENT_PART_ADAPTER.validate_python(block), rewrite)
def with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT:
if not isinstance(message, dict):
return message
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
blocks_rewritten: Final = {
key: (
[block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)]
if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list)
else value
)
for key, value in fields.items()
}
return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped
_MessageT, _fields_with_cache_control(blocks_rewritten, rewrite)
)
def dropped_cache_control(_: object) -> None:
return None
def _first_marked_index(value: object) -> int | None:
if not isinstance(value, list):
return None
blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value)
return next((index for index, block in enumerate(blocks) if _block_carries_cache_control(block)), None)
_MarkSlot: TypeAlias = tuple[str, int]
_MESSAGE_MARK_SLOT: Final[_MarkSlot] = ("cache_control", -1)
def _block_mark_slots(fields: Mapping[str, object], key: str) -> tuple[_MarkSlot, ...]:
value: Final = fields.get(key)
if not isinstance(value, list):
return ()
blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value)
return tuple((key, index) for index, block in enumerate(blocks) if _block_carries_cache_control(block))
def _mark_slots_in_wire_order(fields: Mapping[str, object]) -> tuple[_MarkSlot, ...]:
content_slots: Final = _block_mark_slots(fields, "content")
tool_call_slots: Final = _block_mark_slots(fields, "tool_calls")
if not _message_mark_reaches_provider(fields):
return content_slots + tool_call_slots
if isinstance(fields.get("content"), list):
return content_slots + (_MESSAGE_MARK_SLOT,) + tool_call_slots
return (_MESSAGE_MARK_SLOT,) + content_slots + tool_call_slots
def _blocks_without_marks(key: str, value: object, dropped_slots: frozenset[_MarkSlot]) -> object:
if key not in _MARK_CARRYING_MESSAGE_FIELDS or not isinstance(value, list):
return value
return [
block_with_cache_control(block, dropped_cache_control) if (key, index) in dropped_slots else block
for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(value))
]
def _without_oldest_marks(message: _MessageT, dropped: int) -> _MessageT:
if dropped == 0 or not isinstance(message, dict):
return message
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
dropped_slots: Final = frozenset(_mark_slots_in_wire_order(fields)[:dropped])
blocks_unmarked: Final = {key: _blocks_without_marks(key, value, dropped_slots) for key, value in fields.items()}
return cast( # cast-ok: the same message with its oldest wire-order marks dropped
_MessageT,
_fields_with_cache_control(blocks_unmarked, dropped_cache_control)
if _MESSAGE_MARK_SLOT in dropped_slots
else blocks_unmarked,
)
def tool_message_with_single_cache_mark(message: _MessageT) -> _MessageT:
if not isinstance(message, dict):
return message
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
content: Final = fields.get("content")
if fields.get("role") != "tool" or not isinstance(content, list):
return message
kept_index: Final = None if fields.get("cache_control") is not None else _first_marked_index(content)
return cast( # cast-ok: the same tool message keeping only the mark its single cache point reads
_MessageT,
{
**fields,
"content": [
block if index == kept_index else block_with_cache_control(block, dropped_cache_control)
for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(content))
],
},
)
_BudgetSpend: TypeAlias = Callable[[tuple[int, int], int], tuple[int, int]]
def _newest_marks_kept(state: tuple[int, int], marks: int) -> tuple[int, int]:
remaining, _ = state
kept: Final = min(marks, remaining)
return remaining - kept, marks - kept
def _whole_message_kept(state: tuple[int, int], marks: int) -> tuple[int, int]:
remaining, _ = state
if marks > remaining:
return remaining, marks
return remaining - marks, 0
def _messages_within_budget(
messages: Sequence[_MessageT], reserved_marks: int, spend: _BudgetSpend
) -> Sequence[_MessageT]:
budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0)
newest_first: Final = tuple(reversed(messages))
budget_after_each: Final = itertools.accumulate(
(cache_control_count(message) for message in newest_first), spend, initial=(budget, 0)
)
dropped_counts: Final = tuple(dropped for _, dropped in itertools.islice(budget_after_each, 1, None))
kept_newest_first: Final = tuple(
_without_oldest_marks(message, dropped) for message, dropped in zip(newest_first, dropped_counts)
)
return tuple(reversed(kept_newest_first))
def newest_marks_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]:
return _messages_within_budget(messages, reserved_marks, _newest_marks_kept)
def whole_messages_within_cache_control_limit(
messages: Sequence[_MessageT], reserved_marks: int
) -> Sequence[_MessageT]:
return _messages_within_budget(messages, reserved_marks, _whole_message_kept)
def _part_with_cache_control_renamed(part: object) -> object:
if not _accepts_prompt_cache_breakpoint(part):
return part
@ -1439,3 +1625,25 @@ class AnthropicCacheControlHook(CustomPromptManagement):
llm_router=None,
)
return None
def messages_within_cache_control_cap(
messages: Sequence[_MessageT], tools: object, cache_control: object = None, request_kwargs: object = None
) -> Sequence[_MessageT]:
"""Under drop_params, drop the oldest message hints the provider's cache_control cap has no room for.
Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first.
"""
reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints(
_validated_object_list(tools), cache_control, request_kwargs
)
kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks)
dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept))
if dropped > 0:
verbose_logger.warning(
"Dropped %d cache_control hint(s) from the oldest messages because drop_params is on and the "
"provider accepts at most %d cache_control blocks per request",
dropped,
MAX_CACHE_CONTROL_BLOCKS,
)
return kept

View file

@ -208,11 +208,14 @@ class StandardBuiltInToolCostTracking:
usage=usage, response_object=response_object
)
response_model: Final = _output_item_field(response_object, "model")
if model_info is not None and resolved_usage is not None and resolved_provider is not None:
result: Final = get_cost_for_web_search_request(
custom_llm_provider=resolved_provider,
usage=resolved_usage,
model_info=model_info,
response_model=response_model if isinstance(response_model, str) else None,
lookup_model_info=StandardBuiltInToolCostTracking._safe_get_model_info,
)
if result is not None:
return result

View file

@ -88,17 +88,22 @@ def handle_any_messages_to_chat_completion_str_messages_conversion(
return [{"input": str(messages)}]
def _message_with_str_content(message: AllMessageValues) -> AllMessageValues:
texts: Final = convert_content_list_to_str(message=message)
if not texts:
return message
return cast( # cast-ok: the same message with its content flattened to one string
AllMessageValues, {**message, "content": texts}
)
def handle_messages_with_content_list_to_str_conversion(
messages: list[AllMessageValues],
) -> list[AllMessageValues]:
"""
Handles messages with content list conversion
Returns new messages with each content list flattened to a string, leaving the caller's messages untouched
"""
for message in messages:
texts = convert_content_list_to_str(message=message)
if texts:
message["content"] = texts
return messages
return [_message_with_str_content(message) for message in messages]
def strip_name_from_message(message: AllMessageValues, allowed_name_roles: list[str] = ["user"]) -> AllMessageValues:

View file

@ -9,6 +9,8 @@ from litellm.types.utils import CallTypes
from . import *
if TYPE_CHECKING:
from collections.abc import Callable
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
)
@ -30,7 +32,29 @@ def get_cost_for_google_maps_grounding_request(
return cost_per_google_maps_grounding_request(usage=usage, model_info=model_info)
def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", model_info: "ModelInfo") -> float | None:
def _with_served_model_search_pricing(
model_info: "ModelInfo",
response_model: str | None,
served_model_provider: str,
lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None",
) -> "ModelInfo":
if model_info.get("search_context_cost_per_query") is not None or response_model is None:
return model_info
served_model_info: Final = (
lookup_model_info(response_model, served_model_provider) if lookup_model_info is not None else None
)
if served_model_info is None:
return model_info
return {**model_info, "search_context_cost_per_query": served_model_info.get("search_context_cost_per_query")}
def get_cost_for_web_search_request(
custom_llm_provider: str,
usage: "Usage",
model_info: "ModelInfo",
response_model: str | None = None,
lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None" = None,
) -> float | None:
"""
Get the cost for a web search request for a given model.
@ -38,6 +62,8 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
custom_llm_provider: The custom LLM provider.
usage: The usage object.
model_info: The model info.
response_model: The model the provider response names.
lookup_model_info: Looks up a model's cost map entry, returning None when it is not mapped.
"""
if custom_llm_provider == "gemini":
from .gemini.cost_calculator import cost_per_web_search_request
@ -46,17 +72,25 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
elif custom_llm_provider == "anthropic":
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
return get_cost_for_anthropic_web_search(
model_info=_with_served_model_search_pricing(model_info, response_model, "anthropic", lookup_model_info),
usage=usage,
)
elif custom_llm_provider.startswith("vertex_ai"):
# Anthropic Claude models on Vertex AI populate server_tool_use.web_search_requests
# (same as the direct Anthropic API), not prompt_tokens_details.web_search_requests
# (which is the Gemini field). Route claude-* models to the Anthropic calculator.
model_key: Final[str] = model_info.get("key", "") if model_info else ""
if "claude" in model_key.lower():
if "claude" in (response_model or model_key).lower():
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
verbose_logger.debug("vertex_ai/claude model detected — routing web search cost to Anthropic calculator")
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
return get_cost_for_anthropic_web_search(
model_info=_with_served_model_search_pricing(
model_info, response_model, "vertex_ai", lookup_model_info
),
usage=usage,
)
from .vertex_ai.gemini.cost_calculator import (
cost_per_web_search_request as cost_per_web_search_request_vertex_ai,
@ -71,7 +105,10 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
return None
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
return get_cost_for_anthropic_web_search(
model_info=_with_served_model_search_pricing(model_info, response_model, "azure_ai", lookup_model_info),
usage=usage,
)
elif custom_llm_provider == "perplexity":
# Perplexity handles search costs internally in its own cost calculator
# Return 0.0 to indicate costs are already accounted for

View file

@ -170,6 +170,7 @@ _TOOLS_ADAPTER: Final = TypeAdapter(list[object])
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_RESPONSE_BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
_PENDING_WEB_SEARCH_RESULT: Final = MappingProxyType({"content": ()})
_REQUEST_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
)
@ -2257,6 +2258,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
headers=headers,
)
@staticmethod
def _messages_within_cache_control_cap(
messages: Sequence[AllMessageValues], optional_params: Mapping[str, object]
) -> Sequence[AllMessageValues]:
from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap
return messages_within_cache_control_cap(
messages, optional_params.get("tools"), optional_params.get("cache_control"), optional_params
)
def transform_request(
self,
model: str,
@ -2274,9 +2285,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
anthropic_messages_pt,
)
drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True
drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params"))
capped_messages: Final = (
list(
self._messages_within_cache_control_cap(
messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params)
)
)
if drop_params
else messages
)
if "tools" not in optional_params and messages is not None and has_tool_call_blocks(messages):
if "tools" not in optional_params and has_tool_call_blocks(capped_messages):
optional_params["tools"], _ = self._map_tools(add_dummy_tool(custom_llm_provider="anthropic"))
# Drop thinking param if thinking is enabled but thinking_blocks are missing
@ -2288,9 +2308,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# Related issue: https://github.com/BerriAI/litellm/issues/18926
if (
optional_params.get("thinking") is not None
and messages is not None
and last_assistant_with_tool_calls_has_no_thinking_blocks(messages)
and not any_assistant_message_has_thinking_blocks(messages)
and last_assistant_with_tool_calls_has_no_thinking_blocks(capped_messages)
and not any_assistant_message_has_thinking_blocks(capped_messages)
):
if litellm.modify_params:
optional_params.pop("thinking", None)
@ -2335,15 +2354,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
_name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request(
optional_params=optional_params,
)
if _name_forward_map:
messages = self._rewrite_tool_names_in_messages(messages, _name_forward_map)
request_messages: Final = (
self._rewrite_tool_names_in_messages(capped_messages, _name_forward_map)
if _name_forward_map
else capped_messages
)
if _name_reverse_map and isinstance(litellm_params, dict):
litellm_params[ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY] = _name_reverse_map
# Only the leading system run becomes the top-level system prompt. A later
# system message stays in the conversation: hoisting it rewrites the cached
# prefix and re-bills the whole history at cache-write pricing (#36559).
leading_system_run, later_messages = split_leading_system_run(messages)
leading_system_run, later_messages = split_leading_system_run(request_messages)
anthropic_system_message_list: Final = self.translate_system_message(messages=list(leading_system_run))
# Handling anthropic API Prompt Caching
if len(anthropic_system_message_list) > 0:
@ -2365,7 +2387,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
except Exception as e:
raise AnthropicError(
status_code=400,
message=f"{e}\nReceived Messages={messages}",
message=f"{e}\nReceived Messages={request_messages}",
) # don't use verbose_logger.exception, if exception is raised
self.update_headers_with_optional_anthropic_beta(

View file

@ -4,6 +4,7 @@ This file contains common utils for anthropic calls.
import copy
import json
import os
import re
from collections.abc import Container, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
@ -19,6 +20,8 @@ from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm.constants import (
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH,
ANTHROPIC_MAX_USER_ID_LENGTH,
DEFAULT_MODEL_CREATED_AT_TIME,
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
@ -178,6 +181,40 @@ def tools_without_custom_field(tools: Sequence[object], hoist_defer_loading: boo
return [_tool_without_custom_field(tool, hoist_defer_loading, drop_custom) for tool in tools]
def anthropic_pass_through_api_base() -> str:
return os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
def _same_host_and_port(url: httpx.URL, other: str) -> bool:
try:
other_url: Final = httpx.URL(other)
except httpx.InvalidURL:
return False
return (url.host, url.port) == (other_url.host, other_url.port)
def anthropic_upstream_user_id_limit(url: str) -> int | None:
upstream: Final = httpx.URL(url)
if not upstream.path.removesuffix("/").endswith("/v1/messages"):
return None
if upstream.host == "api.anthropic.com":
return ANTHROPIC_MAX_USER_ID_LENGTH
return (
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH
if _same_host_and_port(upstream, anthropic_pass_through_api_base())
else None
)
def _has_email_shape(token: str) -> bool:
at_sign: Final = token.find("@", 1)
return at_sign != -1 and token.find(".", at_sign + 2, -1) != -1
def anthropic_accepts_user_id(user_id: str, max_length: int) -> bool:
return len(user_id) <= max_length and user_id.isprintable() and not any(map(_has_email_shape, user_id.split()))
def is_anthropic_messages_url(url: str) -> bool:
"""Check whether a URL addresses Anthropic's Messages API."""
parsed_url: Final = urlparse(url)

View file

@ -25,6 +25,7 @@ from litellm.constants import (
)
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
drops_unsupported_params,
filter_exceptions_from_params,
filter_internal_params,
map_finish_reason,
@ -2129,6 +2130,24 @@ class AmazonConverseConfig(BaseConfig):
return data
@staticmethod
def _claude_messages_within_cache_control_cap(
messages: Sequence[AllMessageValues],
capability_model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> Sequence[AllMessageValues]:
if not drops_unsupported_params(litellm_params.get("drop_params")) or "claude" not in capability_model.lower():
return messages
from litellm.integrations.anthropic_cache_control_hook import (
messages_within_cache_control_cap,
tool_message_with_single_cache_mark,
)
return messages_within_cache_control_cap(
tuple(map(tool_message_with_single_cache_mark, messages)), optional_params.get("tools")
)
async def _async_transform_request(
self,
model: str,
@ -2140,7 +2159,13 @@ class AmazonConverseConfig(BaseConfig):
base_model: Final = litellm_params.get("base_model")
capability_model: Final = bedrock_capability_model(model, base_model)
messages, system_content_blocks = self._transform_system_message(
messages, model=capability_model, deployment_model=model
list(
self._claude_messages_within_cache_control_cap(
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
)
),
model=capability_model,
deployment_model=model,
)
# Convert last user message to guarded_text if guardrailConfig is present
@ -2221,7 +2246,13 @@ class AmazonConverseConfig(BaseConfig):
base_model: Final = litellm_params.get("base_model")
capability_model: Final = bedrock_capability_model(model, base_model)
messages, system_content_blocks = self._transform_system_message(
messages, model=capability_model, deployment_model=model
list(
self._claude_messages_within_cache_control_cap(
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
)
),
model=capability_model,
deployment_model=model,
)
# Convert last user message to guarded_text if guardrailConfig is present

View file

@ -130,7 +130,8 @@ class DeepInfraConfig(OpenAIGPTConfig):
- Input: {"role": "tool", "content": [{"type": "text", "text": "result"}]}
- Output: {"role": "tool", "content": "[{\"type\": \"text\", \"text\": \"result\"}]"}
"""
for message in messages:
copied_messages: Final = [message.copy() for message in messages]
for message in copied_messages:
if message.get("role") == "tool":
content = message.get("content")
@ -149,7 +150,7 @@ class DeepInfraConfig(OpenAIGPTConfig):
# For complex content, serialize the entire array as JSON string
message["content"] = json.dumps(content)
return messages
return copied_messages
@overload
def _transform_messages(

View file

@ -23,6 +23,7 @@ from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionToolParam,
ChatCompletionToolParamFunctionChunk,
OpenAIChatCompletionToolParam,
)
from litellm.types.utils import (
@ -45,6 +46,7 @@ from ...openai.chat.gpt_transformation import (
OpenAIChatCompletionStreamingHandler,
OpenAIGPTConfig,
)
from ...openai.common_utils import without_cache_control
from ..common_utils import (
FIREROUTER,
FireworksAIException,
@ -480,15 +482,23 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
return (("response_format", _json_schema_response_format(choice_schema, "choice")),)
def _transform_tools(self, tools: list[OpenAIChatCompletionToolParam]) -> list[OpenAIChatCompletionToolParam]:
for tool in tools:
if tool.get("type") != "function":
continue
function = tool["function"]
function.pop("strict", None)
params = function.get("parameters")
if isinstance(params, dict):
unpack_legacy_defs(params)
return tools
return [self._fireworks_tool(tool) for tool in tools]
@staticmethod
def _fireworks_tool(tool: OpenAIChatCompletionToolParam) -> OpenAIChatCompletionToolParam:
if tool.get("type") != "function":
return tool
function: Final = tool["function"]
params: Final = function.get("parameters")
unstrict_function: Final = cast( # cast-ok: the same function definition minus `strict`
ChatCompletionToolParamFunctionChunk, {key: value for key, value in function.items() if key != "strict"}
)
new_function: Final[ChatCompletionToolParamFunctionChunk] = (
{**unstrict_function, "parameters": unpack_legacy_defs(params, copy=True)}
if isinstance(params, dict)
else unstrict_function
)
return {**tool, "function": new_function}
def _transform_messages_helper(
self, messages: list[AllMessageValues], model: str, litellm_params: dict
@ -496,10 +506,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
"""
Strip fields not permitted by FireworksAI from messages.
"""
from litellm.litellm_core_utils.prompt_templates.common_utils import (
filter_value_from_dict,
)
supports_vision_value: Final = self._get_model_cost_capability_exact(model=model, capability="supports_vision")
for message in messages:
if message["role"] == "user":
@ -530,9 +536,11 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
model=model,
llm_provider="fireworks_ai",
)
filter_value_from_dict(cast(dict, message), "cache_control")
return [strip_litellm_response_only_fields(message) for message in messages]
return [
strip_litellm_response_only_fields(cast(AllMessageValues, without_cache_control(message)))
for message in messages
]
# Cached index of fireworks_ai/* entries from litellm.model_cost. Building
# this index requires a full scan of model_cost (tens of thousands of

View file

@ -140,7 +140,8 @@ class GroqChatConfig(OpenAILikeChatConfig):
def _transform_messages(
self, messages: list[AllMessageValues], model: str, is_async: bool = False
) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]:
for idx, message in enumerate(messages):
transformed_messages: Final = list(messages)
for idx, message in enumerate(transformed_messages):
"""
1. Don't pass 'null' function_call assistant message to groq - https://github.com/BerriAI/litellm/issues/5839
"""
@ -154,12 +155,12 @@ class GroqChatConfig(OpenAILikeChatConfig):
for k, v in _message.items():
if v is not None:
new_message[k] = v
messages[idx] = strip_litellm_internal_message_fields(new_message)
transformed_messages[idx] = strip_litellm_internal_message_fields(new_message)
if is_async:
return super()._transform_messages(messages=messages, model=model, is_async=True)
return super()._transform_messages(messages=transformed_messages, model=model, is_async=True)
else:
return super()._transform_messages(messages=messages, model=model, is_async=False)
return super()._transform_messages(messages=transformed_messages, model=model, is_async=False)
def transform_messages(
self,

View file

@ -2,6 +2,7 @@
Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
"""
import copy
import json
from collections.abc import Coroutine
from typing import Final, Literal, cast, overload
@ -88,7 +89,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
) -> dict:
_tools = non_default_params.pop("tools", None)
if _tools is not None:
_tools = remove_additional_properties(_tools)
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
_tools = remove_strict_from_schema(_tools)
if isinstance(_tools, list):
_tools = self._convert_custom_tools_to_function_tools(_tools)

View file

@ -755,7 +755,8 @@ class MistralConfig(OpenAIGPTConfig):
if _name is not None:
# Remove name if not a tool message
if message["role"] != "tool" or isinstance(_name, str) and len(_name.strip()) == 0:
message.pop("name", None)
message_without_name: Final = {k: v for k, v in message.items() if k != "name"}
return cast(AllMessageValues, message_without_name) # cast-ok: the same message minus name
return message

View file

@ -14,7 +14,7 @@ from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.constants import OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
extract_reasoning_content,
handle_invalid_parallel_tool_calls,
@ -64,7 +64,7 @@ from litellm.types.utils import (
)
from litellm.utils import convert_to_model_response_object, declared_mid_conversation_system_support
from ..common_utils import OpenAIError
from ..common_utils import OpenAIError, without_cache_control
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
if TYPE_CHECKING:
@ -128,6 +128,29 @@ def _litellm_params_str(litellm_params: Mapping[str, object] | None, key: str) -
return value if isinstance(value, str) else None
def _drops_params(litellm_params: Mapping[str, object]) -> bool:
return drops_unsupported_params(litellm_params.get("drop_params"))
def _names_claude(model: str, litellm_params: Mapping[str, object]) -> bool:
base_model: Final = litellm_params.get("base_model")
return "claude" in model.lower() or (isinstance(base_model, str) and "claude" in base_model.lower())
def _claude_messages_within_cache_control_cap(
model: str,
messages: Sequence[AllMessageValues],
tools: Sequence["ChatCompletionToolParam"] | None,
litellm_params: Mapping[str, object],
request_params: Mapping[str, object],
) -> Sequence[AllMessageValues]:
if not _names_claude(model, litellm_params) or not _drops_params(litellm_params):
return messages
from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap
return list(messages_within_cache_control_cap(messages, tools, request_params.get("cache_control"), request_params))
def _prompt_cache_options(optional_params: Mapping[str, object]) -> object | None:
try:
extra_body: Final = _OBJECT_MAPPING.validate_python(optional_params.get("extra_body"))
@ -311,52 +334,58 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
return content_copy
async def _async_handle_pdf_url(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile:
file_id: Final = content_item.get("file_id")
content_copy: Final = content_item.copy()
file_id: Final = content_copy.get("file_id")
if file_id is not None: # check for file id being url done in _handle_pdf_url
base64_data: Final = await async_convert_url_to_base64(file_id)
content_item["file_data"] = base64_data
content_item["filename"] = "my_file.pdf"
content_item.pop("file_id")
return content_item
content_copy["file_data"] = base64_data
content_copy["filename"] = "my_file.pdf"
content_copy.pop("file_id")
return content_copy
def _common_file_data_check(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile:
file_data: Final = content_item.get("file_data")
filename: Final = content_item.get("filename")
if file_data is not None and filename is None:
content_item["filename"] = "my_file.pdf"
return {**content_item, "filename": "my_file.pdf"}
return content_item
@staticmethod
def _image_item_for_openai(content_item: ChatCompletionImageObject) -> ChatCompletionImageObject:
image_url: Final = content_item["image_url"]
if isinstance(image_url, str):
return {**content_item, "image_url": {"url": image_url}}
if not isinstance(image_url, dict):
return content_item
new_image_url_obj: Final = cast(
ChatCompletionImageUrlObject,
{k: v for k, v in image_url.items() if k not in {"format", "video_metadata"}},
)
return {**content_item, "image_url": new_image_url_obj}
@staticmethod
def _file_item_for_openai(content_item: ChatCompletionFileObject) -> ChatCompletionFileObject:
litellm_specific_params: Final = {"format", "detail", "video_metadata"}
file_obj: Final = content_item.get("file")
if file_obj is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=None,
llm_provider="openai",
)
new_file_obj: Final = ChatCompletionFileObjectFile(
**{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
)
return {**content_item, "file": new_file_obj}
def _apply_common_transform_content_item(
self,
content_item: OpenAIMessageContentListBlock,
) -> OpenAIMessageContentListBlock:
litellm_specific_params: Final = {"format", "detail", "video_metadata"}
if content_item.get("type") == "image_url":
content_item = cast(ChatCompletionImageObject, content_item)
if isinstance(content_item["image_url"], str):
content_item["image_url"] = {
"url": content_item["image_url"],
}
elif isinstance(content_item["image_url"], dict):
new_image_url_obj: Final = cast(
ChatCompletionImageUrlObject,
{k: v for k, v in content_item["image_url"].items() if k not in {"format", "video_metadata"}},
)
content_item["image_url"] = new_image_url_obj
elif content_item.get("type") == "file":
content_item = cast(ChatCompletionFileObject, content_item)
file_obj: Final = content_item.get("file")
if file_obj is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=None,
llm_provider="openai",
)
new_file_obj: Final = ChatCompletionFileObjectFile(
**{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
)
content_item["file"] = new_file_obj
return self._image_item_for_openai(cast(ChatCompletionImageObject, content_item))
if content_item.get("type") == "file":
return self._file_item_for_openai(cast(ChatCompletionFileObject, content_item))
return content_item
def _transform_content_item(
@ -419,31 +448,34 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
hoisted_messages: Final = hoist_images_from_tool_messages(stripped_messages)
async def _async_transform():
for message in hoisted_messages:
message_content = message.get("content")
message_role = message.get("role")
if message_role == "user" and message_content and isinstance(message_content, list):
message_content_types = cast(list[OpenAIMessageContentListBlock], message_content)
for i, content_item in enumerate(message_content_types):
message_content_types[i] = await self._async_transform_content_item(
cast(OpenAIMessageContentListBlock, content_item),
)
return hoisted_messages
return [await self._async_user_message_for_openai(message) for message in hoisted_messages]
if is_async:
return _async_transform()
else:
for message in hoisted_messages:
message_content = message.get("content")
message_role = message.get("role")
if message_role == "user" and message_content and isinstance(message_content, list):
message_content_types = cast(list[OpenAIMessageContentListBlock], message_content)
for i, content_item in enumerate(message_content):
message_content_types[i] = self._transform_content_item(
cast(OpenAIMessageContentListBlock, content_item)
)
return hoisted_messages
return [self._user_message_for_openai(message) for message in hoisted_messages]
@staticmethod
def _user_content_parts(message: AllMessageValues) -> Sequence[OpenAIMessageContentListBlock] | None:
message_content: Final = message.get("content")
if message.get("role") != "user" or not message_content or not isinstance(message_content, list):
return None
return cast(Sequence[OpenAIMessageContentListBlock], message_content)
def _user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues:
parts: Final = self._user_content_parts(message)
if parts is None:
return message
return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI
AllMessageValues, {**message, "content": [self._transform_content_item(part) for part in parts]}
)
async def _async_user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues:
parts: Final = self._user_content_parts(message)
if parts is None:
return message
return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI
AllMessageValues, {**message, "content": [await self._async_transform_content_item(part) for part in parts]}
)
def transform_messages(
self,
@ -462,23 +494,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
messages: list[AllMessageValues],
tools: list["ChatCompletionToolParam"] | None = None,
) -> tuple[list[AllMessageValues], list["ChatCompletionToolParam"] | None]:
from litellm.litellm_core_utils.prompt_templates.common_utils import (
filter_value_from_dict,
)
from litellm.types.llms.openai import ChatCompletionToolParam
for i, message in enumerate(messages):
messages[i] = cast(
AllMessageValues,
filter_value_from_dict(message, "cache_control"),
)
if tools is not None:
for i, tool in enumerate(tools):
tools[i] = cast(
ChatCompletionToolParam,
filter_value_from_dict(tool, "cache_control"),
)
return messages, tools
new_messages: Final = [cast(AllMessageValues, without_cache_control(message)) for message in messages]
new_tools: Final = (
[cast(ChatCompletionToolParam, without_cache_control(tool)) for tool in tools]
if tools is not None
else None
)
return new_messages, new_tools
def translate_developer_role_to_system_role(
self,
@ -527,6 +551,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
tools: Sequence["ChatCompletionToolParam"] | None,
prompt_cache_options: object,
litellm_params: Mapping[str, object],
request_params: Mapping[str, object],
) -> tuple[Sequence[AllMessageValues], Sequence["ChatCompletionToolParam"] | None]:
"""
A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on
@ -543,10 +568,11 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
model, custom_llm_provider, api_base, prompt_cache_options
)
if not reads_breakpoints and self._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base):
return messages, tools
drops_breakpoints: Final = not reads_breakpoints and (
litellm.drop_params is True or litellm_params.get("drop_params") is True
)
return (
_claude_messages_within_cache_control_cap(model, messages, tools, litellm_params, request_params),
tools,
)
drops_breakpoints: Final = not reads_breakpoints and _drops_params(litellm_params)
renamed: Final = (
[AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages]
if reads_breakpoints
@ -561,7 +587,13 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
if drops_breakpoints
else list(tools)
)
return self.remove_cache_control_flag_from_messages_and_tools(model=model, messages=renamed, tools=kept_tools)
sent_messages, sent_tools = self.remove_cache_control_flag_from_messages_and_tools(
model=model, messages=renamed, tools=kept_tools
)
return (
_claude_messages_within_cache_control_cap(model, sent_messages, sent_tools, litellm_params, request_params),
sent_tools,
)
def _sanitized_tools_update_for_openai(
self,
@ -623,6 +655,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
tools=optional_params.get("tools", []),
prompt_cache_options=_prompt_cache_options(optional_params),
litellm_params=litellm_params,
request_params=optional_params,
)
if tools is not None and len(tools) > 0:
optional_params["tools"] = tools
@ -654,6 +687,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
tools=optional_params.get("tools", []),
prompt_cache_options=_prompt_cache_options(optional_params),
litellm_params=litellm_params,
request_params=optional_params,
)
if tools is not None and len(tools) > 0:
optional_params["tools"] = tools

View file

@ -153,12 +153,13 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig):
- role: system ==> translate to role 'user'
"""
_supports_system_messages: Final = supports_system_messages(model, "openai")
for i, message in enumerate(messages):
transformed_messages: Final = list(messages)
for i, message in enumerate(transformed_messages):
if message["role"] == "system" and not _supports_system_messages:
new_message = ChatCompletionUserMessage(content=message["content"], role="user")
messages[i] = new_message # Replace the old message with the new one
transformed_messages[i] = new_message # Replace the old message with the new one
if is_async:
return super()._transform_messages(messages, model, is_async=cast(Literal[True], True))
return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[True], True))
else:
return super()._transform_messages(messages, model, is_async=cast(Literal[False], False))
return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[False], False))

View file

@ -3,6 +3,7 @@ Common helpers / utils across al OpenAI endpoints
"""
import asyncio
import copy
import hashlib
import inspect
import json
@ -31,7 +32,7 @@ if TYPE_CHECKING:
import litellm
from litellm.exceptions import error_response_without_credentials
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, iter_items
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params
from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -43,6 +44,47 @@ from litellm.llms.custom_httpx.http_handler import (
)
def _is_plain_dict(value: object) -> bool:
return isinstance(value, dict)
def _is_list(value: object) -> bool:
return isinstance(value, list)
def _contains_cache_control(value: object) -> bool:
pending: Final[list[object]] = [value] # mutable-ok: a local worklist, never returned or stored
while pending:
node = pending.pop()
mapping = as_str_mapping(node)
if mapping is not None:
if "cache_control" in mapping:
return True
pending.extend(mapping.values())
elif _is_list(node):
pending.extend(iter_items(node))
return False
def without_cache_control(item: object) -> object:
"""Return `item` with every `cache_control` key removed, without touching the original.
`filter_value_from_dict` deletes the key in place and recurses into nested dicts and
lists, so the caller's own message, tool or input object has to be copied first: it
may reuse the same list on a provider that does support prompt caching. The copy is
skipped when there is nothing to strip, which keeps the object identity the
pass-through paths rely on and avoids a deep copy on every request.
"""
from litellm.litellm_core_utils.prompt_templates.common_utils import filter_value_from_dict
if not _is_plain_dict(item) or not _contains_cache_control(item):
return item
copied: Final = copy.deepcopy(item)
if isinstance(copied, dict):
filter_value_from_dict(copied, "cache_control")
return copied
def _get_client_init_params(cls: type) -> tuple[str, ...]:
"""Extract __init__ parameter names (excluding 'self') from a class."""
return tuple(p for p in inspect.signature(cls.__init__).parameters if p != "self")

View file

@ -16,6 +16,7 @@ from openai._base_client import make_request_options
from openai._constants import RAW_RESPONSE_HEADER
from openai._legacy_response import LegacyAPIResponse
from openai._types import RequestOptions
from openai.pagination import SyncCursorPage
from openai.types import CreateEmbeddingResponse
from openai.types.beta.assistant_deleted import AssistantDeleted
from openai.types.chat import ChatCompletion, ChatCompletionChunk
@ -40,6 +41,7 @@ from litellm.types.utils import (
LiteLLMBatch,
ModelResponse,
ModelResponseStream,
openai_batch_status,
)
from litellm.utils import (
CustomStreamWrapper,
@ -2182,6 +2184,18 @@ class OpenAIFilesAPI(BaseLLM):
return response
def _with_openai_batch_fields(batches: Sequence[Batch]) -> Sequence[Batch]:
return [
batch.model_copy(
update={
"object": batch.object or "batch",
"status": openai_batch_status(batch.status),
}
)
for batch in batches
]
class OpenAIBatchesAPI(BaseLLM):
"""
OpenAI methods to support for batches
@ -2372,7 +2386,7 @@ class OpenAIBatchesAPI(BaseLLM):
):
verbose_logger.debug("listing batches, after= %s, limit= %s", after, limit)
response: Final = await openai_client.batches.list(after=after, limit=limit)
return response
return response.model_copy(update={"data": _with_openai_batch_fields(response.data)})
def list_batches(
self,
@ -2384,7 +2398,7 @@ class OpenAIBatchesAPI(BaseLLM):
organization: str | None,
after: str | None = None,
limit: int | None = None,
client: OpenAI | None = None,
client: OpenAI | AsyncOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
):
openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
@ -2409,7 +2423,9 @@ class OpenAIBatchesAPI(BaseLLM):
)
return self.alist_batches(openai_client=openai_client, after=after, limit=limit)
response: Final = openai_client.batches.list(after=after, limit=limit)
return response
if not isinstance(response, SyncCursorPage):
return response
return response.model_copy(update={"data": _with_openai_batch_fields(response.data)})
class OpenAIAssistantsAPI(BaseLLM):

View file

@ -52,7 +52,7 @@ from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
from litellm.types.workload_identity import OPENAI_WIF_KWARGS_KEYS
from ..common_utils import OpenAIError
from ..common_utils import OpenAIError, without_cache_control
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16
@ -658,13 +658,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
input=self._replayable_input(marker_safe_input, reads_breakpoints, maps_anthropic_tool_search),
tools=mapped_tools,
)
if marker_safe_input is not validated_input:
self._strip_caller_input_as_if_unmarked(
model=model,
caller_input=validated_input,
reads_breakpoints=reads_breakpoints,
maps_anthropic_tool_search=maps_anthropic_tool_search,
)
object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools)
sanitized_tools: Final = self._sanitized_tool_schemas_for_openai(
model=model, tools=object_schema_tools, litellm_params=litellm_params
@ -679,17 +672,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
return marked_input
return input_without_unanswered_anthropic_tool_search_calls(marked_input)
def _strip_caller_input_as_if_unmarked(
self,
model: str,
caller_input: str | ResponseInputParam,
reads_breakpoints: bool,
maps_anthropic_tool_search: bool,
) -> None:
self.remove_cache_control_flag_from_input_and_tools(
model=model, input=self._replayable_input(caller_input, reads_breakpoints, maps_anthropic_tool_search)
)
def _input_with_wire_safe_prompt_cache_breakpoints(
self,
model: str,
@ -800,24 +782,29 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
the chat path. Strips Anthropic-only `cache_control` markers from
Responses API input content blocks and tools.
`filter_value_from_dict` mutates each dict in place, so the same
objects are returned.
`filter_value_from_dict` deletes the key in place and recurses, so each
item is copied first: the caller keeps its own input list and may reuse
it on a provider that does support prompt caching.
"""
from litellm.litellm_core_utils.prompt_templates.common_utils import (
filter_value_from_dict,
new_input: Final = (
cast( # cast-ok: the comprehension rebuilds the input item for item
ResponseInputParam,
[without_cache_control(item) for item in input],
)
if isinstance(input, list)
else input
)
if isinstance(input, list):
for item in input:
if isinstance(item, dict):
filter_value_from_dict(cast(dict, item), "cache_control")
new_tools: Final = (
cast( # cast-ok: the comprehension rebuilds the tool list item for item
Sequence[ALL_RESPONSES_API_TOOL_PARAMS],
[without_cache_control(tool) for tool in tools],
)
if tools is not None
else None
)
if tools is not None:
for tool in tools:
if isinstance(tool, dict):
filter_value_from_dict(cast(dict, tool), "cache_control")
return input, tools
return new_input, new_tools
def _drop_bridge_minted_reasoning_items(self, input: str | ResponseInputParam) -> str | ResponseInputParam:
if self.custom_llm_provider not in _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING or not isinstance(input, list):

View file

@ -110,38 +110,23 @@ def separate_cached_messages(
messages: list[AllMessageValues],
) -> tuple[list[AllMessageValues], list[AllMessageValues]]:
"""
Returns separated cached and non-cached messages.
Args:
messages: List of messages to be separated.
Returns:
Tuple containing:
- cached_messages: List of cached messages.
- non_cached_messages: List of non-cached messages.
Returns the first continuous block of cached messages, plus every system message once
any message is cached, since generateContent rejects a system_instruction beside a
cachedContent, and the remaining messages.
"""
cached_messages: list[AllMessageValues] = []
non_cached_messages: list[AllMessageValues] = []
filtered_messages: Final = [(idx, message) for idx, message in enumerate(messages) if is_cached_message(message)]
if not filtered_messages:
return [], messages
# Extract cached messages and their indices
filtered_messages: Final[list[tuple[int, AllMessageValues]]] = []
for idx, message in enumerate(messages):
if is_cached_message(message=message):
filtered_messages.append((idx, message))
# Validate only one block of continuous cached messages
last_continuous_block_idx: Final = get_first_continuous_block_idx(filtered_messages)
# Separate messages based on the block of cached messages
if filtered_messages and last_continuous_block_idx is not None:
first_cached_idx: Final = filtered_messages[0][0]
last_cached_idx: Final = filtered_messages[last_continuous_block_idx][0]
cached_messages = messages[first_cached_idx : last_cached_idx + 1]
non_cached_messages = messages[:first_cached_idx] + messages[last_cached_idx + 1 :]
else:
non_cached_messages = messages
return cached_messages, non_cached_messages
cached_block: Final = range(filtered_messages[0][0], filtered_messages[last_continuous_block_idx][0] + 1)
goes_in_cache: Final = tuple(
idx in cached_block or message.get("role") == "system" for idx, message in enumerate(messages)
)
return (
[message for message, cached in zip(messages, goes_in_cache) if cached],
[message for message, cached in zip(messages, goes_in_cache) if not cached],
)
def cached_messages_end_on_supported_turn(cached_messages: Sequence[AllMessageValues]) -> bool:

View file

@ -4,6 +4,7 @@ Translation from OpenAI's `/chat/completions` endpoint to IBM WatsonX's `/text/c
Docs: https://cloud.ibm.com/apidocs/watsonx-ai#text-chat
"""
import copy
from typing import Final
from litellm import verbose_logger
@ -57,7 +58,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig):
_tools = non_default_params.pop("tools", None)
if _tools is not None:
# remove 'additionalProperties' from tools
_tools = remove_additional_properties(_tools)
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
# remove 'strict' from tools
_tools = remove_strict_from_schema(_tools)
if _tools is not None:

View file

@ -1,3 +1,4 @@
import copy
from collections.abc import AsyncIterator, Iterator, Mapping
from types import MappingProxyType
from typing import Any, Final
@ -208,7 +209,8 @@ class XAIChatConfig(OpenAIGPTConfig):
tools = [
filtered_tool
for filtered_tool in (
_FILTERED_TOOL.validate_python(filter_value_from_dict(tool, "strict")) for tool in value
_FILTERED_TOOL.validate_python(filter_value_from_dict(copy.deepcopy(tool), "strict")) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
for tool in value
)
if filtered_tool is not None
]

View file

@ -51387,6 +51387,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 5e-06,
"output_cost_per_token_batches": 2.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -51415,6 +51420,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 5e-06,
"output_cost_per_token_batches": 2.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -52161,6 +52171,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_batches": 7.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -52266,6 +52281,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_batches": 7.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,

View file

@ -5031,8 +5031,9 @@ class ProxyBaseLLMRequestProcessing:
if stream_usage is None:
return None
service_tier: Final = obj.get("service_tier")
served_model: Final = litellm_logging_obj.model if litellm_logging_obj is not None else None
cost_val: Final = ProxyBaseLLMRequestProcessing._streamed_usage_cost(
ModelResponse(usage=stream_usage),
ModelResponse(model=served_model, usage=stream_usage),
model_name,
service_tier if isinstance(service_tier, str) else None,
litellm_logging_obj,

View file

@ -52,7 +52,11 @@ from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming, is_guardrail_intervention
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.fallback_generalizations import match_routing_generalization
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers
from litellm.llms.anthropic.common_utils import (
AnthropicModelInfo,
anthropic_pass_through_api_base,
merge_anthropic_beta_headers,
)
from litellm.llms.azure.passthrough.transformation import (
foreign_azure_deployment,
is_azure_body_model_inference_endpoint,
@ -1047,7 +1051,7 @@ async def anthropic_proxy_route(
"""
[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)
"""
base_target_url = os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
base_target_url = anthropic_pass_through_api_base()
encoded_endpoint = httpx.URL(endpoint).path
# Ensure endpoint starts with '/' for proper URL construction

View file

@ -71,7 +71,11 @@ from litellm.litellm_core_utils.litellm_logging import get_masked_values
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.anthropic.common_utils import is_anthropic_messages_url
from litellm.llms.anthropic.common_utils import (
anthropic_accepts_user_id,
anthropic_upstream_user_id_limit,
is_anthropic_messages_url,
)
from litellm.llms.base_llm.managed_resources.utils import (
resolve_passthrough_managed_id_provider,
)
@ -148,6 +152,8 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
EndpointType,
PassthroughStandardLoggingPayload,
WebSocketSessionMeter,
WebSocketSessionUsage,
)
from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, CallTypes, Usage
@ -501,7 +507,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
url: httpx.URL,
headers: dict,
requested_query_params: dict | None = None,
_parsed_body: dict | None = None,
_parsed_body: Mapping[str, object] | None = None,
forward_multipart: bool = False,
) -> httpx.Response:
"""
@ -715,8 +721,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
# unenforced. Set after the client merge so a request body cannot supply its own budget.
#
# Only for the built-in provider routes. `get_model_from_request` returns
# None for a user-defined pass-through, deliberately: its body is forwarded
# verbatim, so `model` there names an UPSTREAM model rather than a
# None for a user-defined pass-through, deliberately: the client's `model`
# goes upstream unchanged, so it names an UPSTREAM model rather than a
# LiteLLM-managed one. Enforcement is therefore skipped on those routes, and
# charging a counter anyway would track spend that nothing can refuse, and
# would attribute it to a budget the operator scoped to a LiteLLM model that
@ -1185,6 +1191,36 @@ def _unfiltered_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
return headers
def _json_object(value: object) -> Mapping[str, object]:
return cast("Mapping[str, object]", value) if isinstance(value, Mapping) else MappingProxyType({}) # cast-ok: json
def _anthropic_user_id_to_forward(
body: Mapping[str, object],
url: str,
has_pre_call_guardrails: Callable[[Mapping[str, object]], bool],
) -> str | None:
user_id: Final = _json_object(body.get("metadata")).get("user_id")
if not isinstance(user_id, str):
return None
max_length: Final = anthropic_upstream_user_id_limit(url)
if max_length is None:
return None
if not anthropic_accepts_user_id(user_id, max_length):
verbose_proxy_logger.warning("Not forwarding metadata.user_id because the upstream would refuse it")
return None
if has_pre_call_guardrails(body):
verbose_proxy_logger.debug("Not forwarding metadata.user_id because a pre-call guardrail cannot scan it")
return None
return user_id
def _with_anthropic_user_id(body: Mapping[str, object] | None, user_id: str | None) -> Mapping[str, object] | None:
if body is None or user_id is None:
return body
return {**body, "metadata": {"user_id": user_id}}
def _eu_jev_client_body(custom_llm_provider: str | None, body: Mapping[str, object]) -> Mapping[str, object] | None:
return MappingProxyType(dict(body)) if custom_llm_provider == "eu_jev" else None
@ -1421,6 +1457,10 @@ async def pass_through_request(
request_method=getattr(request, "method", None),
cost_per_request=cost_per_request,
)
hooked_body: Final = _json_object(_parsed_body)
client_user_id: Final = _anthropic_user_id_to_forward(
hooked_body, str(url), proxy_logging_obj.request_has_pre_call_guardrails
)
kwargs = HttpPassThroughEndpointHelpers.init_kwargs_for_pass_through_endpoint( # rebind-ok: pre-existing rebinding on a rename-only line
user_api_key_dict=user_api_key_dict,
_parsed_body=_parsed_body,
@ -1588,11 +1628,13 @@ async def pass_through_request(
else:
logging_url = str(url) + "?" + requested_query_params_str
outgoing_body: Final = cast("Mapping[str, object] | None", _parsed_body) # cast-ok: json
upstream_body: Final = _with_anthropic_user_id(outgoing_body, client_user_id)
logging_obj.pre_call(
input=[{"role": "user", "content": safe_dumps(_parsed_body)}],
api_key="",
additional_args={
"complete_input_dict": _parsed_body,
"complete_input_dict": upstream_body,
"api_base": str(logging_url),
"headers": redact_credential_headers(upstream_headers),
},
@ -1632,7 +1674,7 @@ async def pass_through_request(
url,
params=requested_query_params,
headers=upstream_headers,
json=_parsed_body,
json=upstream_body,
)
)
@ -1731,7 +1773,7 @@ async def pass_through_request(
url=url,
headers=upstream_headers,
requested_query_params=requested_query_params,
_parsed_body=_parsed_body,
_parsed_body=upstream_body,
forward_multipart=is_multipart,
)
verbose_proxy_logger.debug("response.headers= %s", response.headers)
@ -2595,6 +2637,13 @@ def _with_trace_context(headers: Mapping[str, str], parent_span: object) -> dict
return inject_trace_context(headers, parent_span=parent_span)
def _log_session_usage(logging_obj: LiteLLMLoggingObj, usage: WebSocketSessionUsage) -> None:
logging_obj.model = usage.model # rebind-ok: the spend logger reads model and cost off the shared logging object
logging_obj.model_call_details["model"] = usage.model # rebind-ok: same shared logging object
logging_obj.model_call_details["custom_llm_provider"] = usage.custom_llm_provider # rebind-ok: same
logging_obj.model_call_details["response_cost"] = usage.response_cost # rebind-ok: same
async def websocket_passthrough_request(
websocket: WebSocket,
target: str,
@ -2607,6 +2656,7 @@ async def websocket_passthrough_request(
setup_model_rewriter: Callable[[str], str] | None = None,
builtin_provider_route: bool = False,
model: str | None = None,
meter: WebSocketSessionMeter | None = None,
):
"""
WebSocket passthrough request handler.
@ -2622,6 +2672,7 @@ async def websocket_passthrough_request(
setup_model_rewriter: Optional rewrite of the setup frame's model before it reaches the upstream
builtin_provider_route: Whether a built-in provider route serves the connection, so its URL names the provider
model: The model the caller named before connecting, logged on a failure before the setup frame arrives
meter: Prices the session from the client text frames delivered upstream, for an upstream that reports no usage
"""
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy.proxy_server import proxy_config, proxy_logging_obj
@ -2632,6 +2683,7 @@ async def websocket_passthrough_request(
# Initialize tracking variables
start_time: Final = datetime.now()
websocket_messages: Final[list[dict[str, object]]] = []
metered_total = 0 # rebind-ok: forward_client_to_upstream adds the measure of each client text frame it delivers
litellm_call_id: Final = str(uuid.uuid4())
verbose_proxy_logger.info("WebSocket passthrough (%s): Starting WebSocket connection to %s", endpoint, target)
@ -2750,6 +2802,7 @@ async def websocket_passthrough_request(
async def forward_client_to_upstream() -> None:
"""Forward messages from client to upstream WebSocket"""
nonlocal metered_total
try:
while True:
message = await websocket.receive()
@ -2819,6 +2872,8 @@ async def websocket_passthrough_request(
# Not a JSON message or doesn't contain setup data
await upstream_ws.send(_rewrite_vertex_live_setup_model(text_data, setup_model_rewriter))
if meter is not None:
metered_total += meter.measure(text_data)
elif bytes_data is not None:
await upstream_ws.send(bytes_data)
except asyncio.CancelledError:
@ -2945,6 +3000,8 @@ async def websocket_passthrough_request(
self.url = target_url
mock_response: Final = MockWebSocketResponse(target)
if meter is not None:
_log_session_usage(logging_obj, meter.price(metered_total))
# Use the same success handler as HTTP passthrough endpoints
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(

View file

@ -3031,14 +3031,21 @@ class ProxyLogging:
hook to consult, but it judges the payload the same way, so a proxy configured only with
one of those still has something to say about every record.
"""
if request_metadata.get("_guardrail_pipelines"):
return self.request_has_pre_call_guardrails({"metadata": dict(request_metadata)}, event_type)
def request_has_pre_call_guardrails(
self,
request_data: Mapping[str, object],
event_type: GuardrailEventHooks = GuardrailEventHooks.pre_call,
) -> bool:
if _policy_state_metadata(request_data).get("_guardrail_pipelines"):
return True
caps: Final = ProxyLogging.callback_capabilities()
if caps.has_content_enforcer:
return True
probe: Final = {"metadata": dict(request_metadata)}
return any(
isinstance(callback, CustomGuardrail) and callback.should_run_guardrail(data=probe, event_type=event_type)
isinstance(callback, CustomGuardrail)
and callback.should_run_guardrail(data=request_data, event_type=event_type)
for callback in caps.resolved_callbacks
)

View file

@ -8,7 +8,7 @@ import uuid
from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from functools import partial
from itertools import accumulate, chain, groupby, islice
from itertools import accumulate, chain, groupby
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
@ -45,7 +45,15 @@ from litellm.constants import (
REDACTED_BY_LITELLM,
REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER,
)
from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook
from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
block_with_cache_control,
cache_control_count,
dropped_cache_control,
newest_marks_within_cache_control_limit,
whole_messages_within_cache_control_limit,
with_cache_control,
)
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params
from litellm.litellm_core_utils.get_supported_openai_params import (
get_supported_openai_params,
@ -630,10 +638,6 @@ def _validated_cache_control(value: object) -> ChatCompletionCachedContent | Non
return None
def _dropped_cache_control(_: object) -> None:
return None
def _without_cache_control_key(item: object) -> object:
if not isinstance(item, dict):
return item
@ -721,83 +725,6 @@ def _content_blocks_carrying(content: object, cache_control: ChatCompletionCache
return _with_cache_control_on_last_block(_OBJECT_LIST_ADAPTER.validate_python(content), cache_control)
_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls")
def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]:
lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS)
return tuple(
chain.from_iterable(_OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list))
)
def _block_carries_cache_control(block: object) -> bool:
return isinstance(block, dict) and _STR_KEY_DICT_ADAPTER.validate_python(block).get("cache_control") is not None
def _cache_control_count(message: object) -> int:
if not isinstance(message, dict):
return 0
fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message)
block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block))
return int(fields.get("cache_control") is not None) + block_marks
_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None]
def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]:
others: Final = {key: value for key, value in fields.items() if key != "cache_control"}
if "cache_control" not in fields:
return others
cache_control: Final = rewrite(fields["cache_control"])
return others if cache_control is None else {**others, "cache_control": cache_control}
def _block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object:
if not isinstance(block, dict):
return block
return _fields_with_cache_control(_STR_KEY_DICT_ADAPTER.validate_python(block), rewrite)
def _with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT:
if not isinstance(message, dict):
return message
fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message)
blocks_rewritten: Final = {
key: (
[_block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)]
if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list)
else value
)
for key, value in fields.items()
}
return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped
_MessageT, _fields_with_cache_control(blocks_rewritten, rewrite)
)
def _kept_within_budget(state: tuple[int, bool], marks: int) -> tuple[int, bool]:
remaining, _ = state
if marks > remaining:
return remaining, False
return remaining - marks, True
def _messages_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]:
budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0)
newest_first: Final = tuple(reversed(messages))
budget_after_each: Final = accumulate(
(_cache_control_count(message) for message in newest_first), _kept_within_budget, initial=(budget, True)
)
kept_flags: Final = tuple(kept for _, kept in islice(budget_after_each, 1, None))
kept_newest_first: Final = tuple(
message if kept else _with_cache_control(message, _dropped_cache_control)
for message, kept in zip(newest_first, kept_flags)
)
return tuple(reversed(kept_newest_first))
_SYSTEM_MESSAGE_FIELDS: Final = frozenset(("content", "cache_control"))
@ -826,7 +753,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT:
if fields is None:
return message
if fields.get("role") == "tool":
return _with_cache_control(message, _dropped_cache_control)
return with_cache_control(message, dropped_cache_control)
tool_calls: Final = fields.get("tool_calls")
if not isinstance(tool_calls, list):
return message
@ -835,7 +762,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT:
{
**fields,
"tool_calls": [
_block_with_cache_control(tool_call, _dropped_cache_control)
block_with_cache_control(tool_call, dropped_cache_control)
for tool_call in _OBJECT_LIST_ADAPTER.validate_python(tool_calls)
],
},
@ -849,7 +776,7 @@ class _ItemMarkRules:
external_marks: int
def sanitized(self, message: _MessageT) -> _MessageT:
valid: Final = _with_cache_control(message, _validated_cache_control)
valid: Final = with_cache_control(message, _validated_cache_control)
return _without_tool_block_marks(valid) if self.refuses_tool_block_marks else valid
def checked(self, message: _MessageT) -> _MessageT:
@ -872,15 +799,15 @@ def _request_messages_for_cache_points(messages: Sequence[_MessageT], rules: _It
as_system: Final = tuple(_developer_message_as_system_message(message) for message in messages)
if not rules.drop_params:
return as_system
return _messages_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks)
return newest_marks_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks)
def _replayed_messages_for_cache_points(
replayed_messages: Sequence[_MessageT], current_messages: Sequence[object], rules: _ItemMarkRules
) -> Sequence[_MessageT]:
as_system: Final = tuple(_developer_message_as_system_message(message) for message in replayed_messages)
current_marks: Final = sum(_cache_control_count(message) for message in current_messages)
return _messages_within_cache_control_limit(
current_marks: Final = sum(cache_control_count(message) for message in current_messages)
return whole_messages_within_cache_control_limit(
tuple(map(rules.sanitized, as_system)), current_marks + rules.external_marks
)

View file

@ -403,7 +403,7 @@ class OpenAIFileObject(LiteLLMBaseModel):
filename: str
"""The name of the file."""
object: Literal["file"]
object: Literal["file"] = "file"
"""The object type, which is always `file`."""
purpose: OpenAIFilesPurpose
@ -441,6 +441,11 @@ class OpenAIFileObject(LiteLLMBaseModel):
_hidden_params: dict = PrivateAttr(default={"response_cost": 0.0}) # no cost for writing a file
@field_validator("object", mode="before")
@classmethod
def _default_missing_object_type(cls, value: _JsonValue) -> _JsonValue:
return "file" if value is None else value
@property
def hidden_params(self) -> dict[str, builtins.object]: # mutable-ok: API requires mutation
return self._hidden_params

View file

@ -1,5 +1,6 @@
from dataclasses import dataclass
from enum import Enum
from typing import Final
from typing import Final, Protocol
from pydantic import Field
from typing_extensions import TypedDict
@ -75,6 +76,19 @@ class PassthroughStandardLoggingPayload(TypedDict, total=False):
"""
@dataclass(frozen=True, slots=True)
class WebSocketSessionUsage:
model: str
custom_llm_provider: str
response_cost: float | None
class WebSocketSessionMeter(Protocol):
def measure(self, client_text: str) -> int: ...
def price(self, total: int) -> WebSocketSessionUsage: ...
class PassThroughGuardrailSettings(LiteLLMPydanticObjectBase):
"""
Settings for a specific guardrail on a passthrough endpoint.

View file

@ -106,6 +106,7 @@ from .llms.openai import (
OpenAIRealtimeStreamList,
ResponsesAPIResponse,
WebSearchOptions,
_JsonValue,
)
from .rerank import RerankResponse as RerankResponse
@ -4963,29 +4964,50 @@ def _batch_usage_from_chat_usage(usage: Usage) -> BatchUsage:
)
OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS: Final = MappingProxyType(
{"pending": "validating", "running": "in_progress"}
)
def openai_batch_status(status: _JsonValue) -> _JsonValue:
return OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(status, status) if isinstance(status, str) else status
class LiteLLMBatch(Batch):
_hidden_params: dict = {}
object: Literal["batch"] = "batch"
@field_validator("object", mode="before")
@classmethod
def _default_missing_object_type(cls, value: _JsonValue) -> _JsonValue:
return "batch" if value is None else value
@field_validator("status", mode="before")
@classmethod
def _map_vllm_router_status(cls, value: _JsonValue) -> _JsonValue:
return openai_batch_status(value)
@property
def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation
def hidden_params(self) -> dict[str, _JsonValue]: # mutable-ok: API requires mutation
return self._hidden_params
@hidden_params.setter
def hidden_params(self, hidden_params: dict[str, object]) -> None: # mutable-ok: API requires mutation
def hidden_params(self, hidden_params: dict[str, _JsonValue]) -> None: # mutable-ok: API requires mutation
self._hidden_params = hidden_params
usage: Usage | None = None
@field_validator("usage", mode="before")
@classmethod
def _usage_from_batch_token_counts(cls, value: object) -> object:
def _usage_from_batch_token_counts(cls, value: _JsonValue) -> _JsonValue:
counts: Final = _parse_batch_token_counts(value)
return value if counts is None else _usage_with_batch_token_counts(Usage.model_validate(value), counts)
@field_serializer("usage")
def _serialize_usage_with_batch_fields(
self, usage: Usage | None, info: FieldSerializationInfo
) -> Mapping[str, object] | None:
) -> Mapping[str, _JsonValue] | None:
if not isinstance(usage, Usage):
return usage
chat_fields: Final = usage.model_dump(

View file

@ -51387,6 +51387,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 5e-06,
"output_cost_per_token_batches": 2.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -51415,6 +51420,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 5e-06,
"output_cost_per_token_batches": 2.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -52161,6 +52171,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_batches": 7.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -52266,6 +52281,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_batches": 7.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,

View file

@ -383,6 +383,8 @@ exclude-newer = "3 days"
[tool.uv.exclude-newer-package]
# TODO: remove after 2026-10-25, once brotli 1.2.1 (CVE-2026-111497) clears the 3-day window
brotli = "2026-10-22T14:01:00Z"
# TODO: remove after 2026-10-31, once pypdf 6.21.4 clears the 3-day window
pypdf = "2026-10-28T17:01:00Z"
[tool.uv.sources]
litellm-proxy-extras = { workspace = true }

View file

@ -3,7 +3,7 @@
PARITY pairs the streamed case with its plain twin where both bill the same row."""
import json
from typing import Final, cast
from typing import Final, Literal, cast
from pydantic import JsonValue
@ -41,6 +41,7 @@ from integration.cost_calculation.cases.anthropic import (
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs
from integration.cost_calculation.cost_tracking_case import (
CostTrackingTestCase,
Deployment,
ExactExpected,
JsonResponse,
RecountExpected,
@ -3387,6 +3388,232 @@ VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS: Final = bridge_pairs(
)
VERTEX_AI_CLAUDE_DEPLOYMENT_RATES: Final = Deployment(
input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.01
)
def _vertex_claude_web_search_message(web_search_requests: int) -> JsonResponse:
return JsonResponse(
content_type="application/json",
body={
"id": "msg_$REQUEST_ID",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5-5",
"content": [{"type": "text", "text": "scripted answer for cost fixtures"}],
"stop_reason": "end_turn",
"usage": {
"input_tokens": 1840,
"output_tokens": 412,
"server_tool_use": {"web_search_requests": web_search_requests},
},
},
)
def _vertex_claude_web_search_stream(web_search_requests: int) -> SseResponse:
return SseResponse(
content_type="text/event-stream",
frames=sse_frames(
{
"type": "message_start",
"message": {
"id": "msg_$REQUEST_ID",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5-5",
"content": [],
"stop_reason": None,
"usage": {"input_tokens": 1840},
},
},
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
{
"type": "content_block_delta",
"index": 0,
"delta": {"type": "text_delta", "text": "scripted answer for cost fixtures"},
},
{"type": "content_block_stop", "index": 0},
{
"type": "message_delta",
"delta": {"stop_reason": "end_turn"},
"usage": {"output_tokens": 412, "server_tool_use": {"web_search_requests": web_search_requests}},
},
{"type": "message_stop"},
),
)
def _vertex_claude_deployment_web_search_case(
name: str,
endpoint: Literal["/v1/chat/completions", "/v1/responses", "/v1/messages"],
request: dict[str, JsonValue],
response: JsonResponse | SseResponse,
tool_usage_cost: float,
deployment: Deployment = VERTEX_AI_CLAUDE_DEPLOYMENT_RATES,
) -> CostTrackingTestCase:
return CostTrackingTestCase(
name=name,
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="vertex_ai/claude-sonnet-5-5",
endpoint=endpoint,
deployment=deployment,
request={"model": "$MODEL", **request},
response=response,
expected=ExactExpected(
spend=round(0.0039 + tool_usage_cost, 10),
input_cost=0.00184,
output_cost=0.00206,
prompt_tokens=1840,
completion_tokens=412,
tool_usage_cost=tool_usage_cost,
),
)
VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL: Final[dict[str, JsonValue]] = {
"type": "web_search_20250305",
"name": "web_search",
"max_uses": 3,
}
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_rates_messages_web_search_two_requests",
"/v1/messages",
{
"max_tokens": 412,
"messages": [{"role": "user", "content": "vertex claude deployment rates messages web search"}],
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
},
_vertex_claude_web_search_message(2),
0.02,
)
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM: Final = _vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_rates_messages_web_search_two_requests_stream",
"/v1/messages",
{
"max_tokens": 412,
"messages": [{"role": "user", "content": "vertex claude deployment rates messages web search stream"}],
"stream": True,
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
},
_vertex_claude_web_search_stream(2),
0.02,
)
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE: Final = (
_vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_zero_search_rate_messages_two_requests",
"/v1/messages",
{
"max_tokens": 412,
"messages": [{"role": "user", "content": "vertex claude deployment zero search rate messages"}],
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
},
_vertex_claude_web_search_message(2),
0.0,
Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.0),
)
)
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY: Final = (
_vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_token_rates_only_messages_web_search_two_requests",
"/v1/messages",
{
"max_tokens": 412,
"messages": [{"role": "user", "content": "vertex claude deployment token rates only messages web search"}],
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
},
_vertex_claude_web_search_message(2),
0.034,
Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06),
)
)
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_rates_chat_web_search_three_requests",
"/v1/chat/completions",
{
"messages": [{"role": "user", "content": "vertex claude deployment rates chat web search"}],
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
"allowed_openai_params": ["web_search_options"],
},
_vertex_claude_web_search_message(3),
0.03,
)
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH_STREAM: Final = _vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_rates_chat_web_search_two_requests_stream",
"/v1/chat/completions",
{
"messages": [{"role": "user", "content": "vertex claude deployment rates chat web search stream"}],
"stream": True,
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
"allowed_openai_params": ["web_search_options"],
},
_vertex_claude_web_search_stream(2),
0.02,
)
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_RESPONSES_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_rates_responses_web_search_two_requests",
"/v1/responses",
{"input": "vertex claude deployment rates responses web search", "tools": [{"type": "web_search"}]},
_vertex_claude_web_search_message(2),
0.02,
)
GEMINI_3_1_PRO_PREVIEW_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM: Final = CostTrackingTestCase(
name="vertex_ai/gemini-3.1-pro-preview-deployment_rates_messages_web_search_two_queries_stream",
covers="quota_management.spend_tracking.scripted_wire.logs_cost",
model="vertex_ai/gemini-3.1-pro-preview",
endpoint="/v1/messages",
deployment=Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.03),
request={
"model": "$MODEL",
"max_tokens": 412,
"messages": [{"role": "user", "content": "vertex gemini deployment rates messages web search stream"}],
"stream": True,
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
},
response=SseResponse(
content_type="text/event-stream",
frames=sse_frames(
{
"candidates": [
{
"content": {"parts": [{"text": "scripted answer for cost fixtures"}], "role": "model"},
"index": 0,
}
],
"modelVersion": "gemini-3.1-pro-preview",
},
{
"candidates": [
{
"content": {"parts": [{"text": ""}], "role": "model"},
"finishReason": "STOP",
"index": 0,
"groundingMetadata": {"webSearchQueries": ["query 0", "query 1"]},
}
],
"usageMetadata": {
"promptTokenCount": 1840,
"candidatesTokenCount": 412,
"totalTokenCount": 2252,
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}],
},
"modelVersion": "gemini-3.1-pro-preview",
},
),
),
expected=ExactExpected(
spend=0.0339,
input_cost=0.00184,
output_cost=0.00206,
prompt_tokens=1840,
completion_tokens=412,
tool_usage_cost=0.03,
),
)
GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT: Final = _gemini_case(
GEMINI_3_8_FLASH_INPUT_TEXT, "vertex_ai/gemini-3.1-pro-preview-input_text", "vertex_ai/gemini-3.1-pro-preview",
ExactExpected(spend=0.0094864, input_cost=0.004048, output_cost=0.0054384, prompt_tokens=1840, completion_tokens=412),
@ -3725,6 +3952,14 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT,
VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_STREAM,
*(pair.bridged for pair in VERTEX_AI_CLAUDE_SONNET_5_5_BRIDGE_PAIRS),
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH_STREAM,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_RESPONSES_WEB_SEARCH,
GEMINI_3_1_PRO_PREVIEW_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM,
GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT,
GEMINI_3_1_PRO_PREVIEW_STREAM,
GEMINI_3_1_PRO_PREVIEW_CACHE_READ,

View file

@ -317,7 +317,6 @@ def test_credentialless_proxy_forwards_the_caller_anthropic_credential_and_never
def test_anthropic_sdk_metadata_user_id_reaches_upstream(api_key_proxy: Gateway, wire: Wire) -> None:
pytest.skip("BUG: /anthropic/v1/messages drops the native metadata.user_id field before forwarding upstream")
with api_key_proxy.scenario() as scenario:
key: Final = scenario.key()
sent: Final[list[httpx.Request]] = []

View file

@ -37,13 +37,15 @@ import litellm.batches.main as bm
import asyncio
import datetime
import json
from collections.abc import Mapping
from collections.abc import Awaitable, Callable, Mapping, Sequence
from typing import Final
import httpx
import respx
from pydantic import TypeAdapter
from pydantic import TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
from litellm.integrations.custom_logger import CustomLogger
from openai.types.batch import Batch
from litellm.types.utils import LiteLLMBatch
# --------------------------------------------------------------------------- #
@ -1204,3 +1206,183 @@ async def test_delete_batch_output_file(respx_mock: respx.MockRouter, monkeypatc
assert delete_route.call_count == 1
assert delete_response.id == "file-output123"
assert delete_response.deleted is True
VLLM_ROUTER_API_BASE: Final = "http://vllm-router.test/v1"
VLLM_ROUTER_BATCH: Final = MappingProxyType(
{
"id": "batch_5f8dac",
"status": "pending",
"input_file_id": "file-b73f4c",
"created_at": 1793239207,
"endpoint": "/v1/chat/completions",
"completion_window": "24h",
"output_file_id": None,
"total_requests": None,
"completed_requests": 0,
"failed_requests": 0,
"metadata": {},
}
)
async def _create_on_vllm_router() -> LiteLLMBatch:
return await litellm.acreate_batch(
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id="file-b73f4c",
custom_llm_provider="hosted_vllm",
api_key="sk-test",
api_base=VLLM_ROUTER_API_BASE,
)
async def _retrieve_from_vllm_router() -> LiteLLMBatch:
return await litellm.aretrieve_batch(
batch_id="batch_5f8dac", custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE
)
VLLM_ROUTER_BATCH_CALLS: Final = (
pytest.param("POST", "/batches", _create_on_vllm_router, id="acreate_batch"),
pytest.param("GET", "/batches/batch_5f8dac", _retrieve_from_vllm_router, id="aretrieve_batch"),
)
@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_BATCH_CALLS)
@pytest.mark.parametrize(
"router_status, openai_status",
(
("pending", "validating"),
("running", "in_progress"),
("completed", "completed"),
("failed", "failed"),
("cancelled", "cancelled"),
),
)
async def test_hosted_vllm_router_batch_comes_back_as_an_openai_batch(
respx_mock: respx.MockRouter,
method: str,
path: str,
call_router: Callable[[], Awaitable[LiteLLMBatch]],
router_status: str,
openai_status: str,
):
respx_mock.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond(
200, json={**VLLM_ROUTER_BATCH, "status": router_status}
)
batch: Final = await call_router()
assert (batch.id, batch.object, batch.status) == ("batch_5f8dac", "batch", openai_status)
@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_BATCH_CALLS)
@pytest.mark.parametrize("field, value", (("status", "queued"), ("object", "file")))
async def test_hosted_vllm_router_batch_with_an_unknown_value_is_rejected(
respx_mock: respx.MockRouter,
method: str,
path: str,
call_router: Callable[[], Awaitable[LiteLLMBatch]],
field: str,
value: str,
):
respx_mock.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond(
200, json={**VLLM_ROUTER_BATCH, field: value}
)
with pytest.raises(ValidationError) as exc_info:
await call_router()
assert [error["loc"] for error in exc_info.value.errors()] == [(field,)]
async def _list_on_vllm_router() -> Sequence[Batch]:
page: Final = await litellm.alist_batches(
custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE
)
return page.data
async def _list_on_vllm_router_sync() -> Sequence[Batch]:
return litellm.list_batches(custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE).data
@pytest.mark.parametrize("list_router", (_list_on_vllm_router, _list_on_vllm_router_sync))
async def test_hosted_vllm_router_batch_list_comes_back_as_openai_batches(
respx_mock: respx.MockRouter, list_router: Callable[[], Awaitable[Sequence[Batch]]]
):
respx_mock.get(f"{VLLM_ROUTER_API_BASE}/batches").respond(
200,
json={
"object": "list",
"data": [
{**VLLM_ROUTER_BATCH, "id": "batch_5f8dac", "status": "pending"},
{**VLLM_ROUTER_BATCH, "id": "batch_c74802", "status": "running"},
],
"first_id": "batch_5f8dac",
"last_id": "batch_c74802",
"has_more": False,
},
)
batches: Final = await list_router()
assert [(batch.id, batch.object, batch.status) for batch in batches] == [
("batch_5f8dac", "batch", "validating"),
("batch_c74802", "batch", "in_progress"),
]
async def test_hosted_vllm_router_batch_list_passes_through_a_status_that_is_not_a_string(
respx_mock: respx.MockRouter,
):
respx_mock.get(f"{VLLM_ROUTER_API_BASE}/batches").respond(
200,
json={
"object": "list",
"data": [{**VLLM_ROUTER_BATCH, "status": ["pending"]}],
"first_id": "batch_5f8dac",
"last_id": "batch_5f8dac",
"has_more": False,
},
)
batches: Final = await _list_on_vllm_router()
assert [(batch.id, batch.object, batch.status) for batch in batches] == [("batch_5f8dac", "batch", ["pending"])]
async def test_openai_batch_list_items_come_back_as_openai_sent_them(respx_mock: respx.MockRouter):
respx_mock.get("http://openai.test/v1/batches").respond(
200,
json={
"object": "list",
"data": [
{
**_OPENAI_BATCH_JSON,
"status": "completed",
"usage": {
"input_tokens": 10,
"input_tokens_details": {"cached_tokens": 4},
"output_tokens": 5,
"output_tokens_details": {"reasoning_tokens": 2},
"total_tokens": 15,
},
},
{**_OPENAI_BATCH_JSON, "id": "batch_def456", "status": "queued"},
],
"first_id": "batch_abc123",
"last_id": "batch_def456",
"has_more": False,
},
)
page: Final = await litellm.alist_batches(
custom_llm_provider="openai", api_key="sk-test", api_base="http://openai.test/v1"
)
first, second = page.data
assert first.usage is not None
assert (first.usage.input_tokens_details.cached_tokens, first.usage.output_tokens_details.reasoning_tokens) == (4, 2)
assert (second.id, second.object, second.status) == ("batch_def456", "batch", "queued")

View file

@ -1,3 +1,4 @@
from collections.abc import Awaitable, Callable
from types import MappingProxyType
from typing import Final
from urllib.parse import parse_qs, urlparse
@ -124,6 +125,61 @@ async def test_afile_retrieve_rejects_a_provider_file_without_its_size():
assert [error["loc"] for error in exc_info.value.errors()] == [("bytes",)]
VLLM_ROUTER_API_BASE: Final = "http://vllm-router.test/v1"
VLLM_ROUTER_FILE: Final = MappingProxyType(
{"id": "file-b73f4c", "bytes": 156, "created_at": 1793238707, "filename": "batch_requests.jsonl", "purpose": "batch"}
)
async def _upload_to_vllm_router() -> OpenAIFileObject:
return await litellm.acreate_file(
file=("batch_requests.jsonl", b'{"custom_id": "request-1"}'),
purpose="batch",
custom_llm_provider="hosted_vllm",
api_key="sk-test",
api_base=VLLM_ROUTER_API_BASE,
)
async def _retrieve_from_vllm_router() -> OpenAIFileObject:
return await litellm.afile_retrieve(
file_id="file-b73f4c", custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE
)
VLLM_ROUTER_FILE_CALLS: Final = (
pytest.param("POST", "/files", _upload_to_vllm_router, id="acreate_file"),
pytest.param("GET", "/files/file-b73f4c", _retrieve_from_vllm_router, id="afile_retrieve"),
)
@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_FILE_CALLS)
@respx.mock
async def test_hosted_vllm_file_without_an_object_type_comes_back_as_a_file(
method: str, path: str, call_router: Callable[[], Awaitable[OpenAIFileObject]]
):
respx.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond(200, json=dict(VLLM_ROUTER_FILE))
file_object: Final = await call_router()
assert file_object.model_dump(exclude_none=True) == {**VLLM_ROUTER_FILE, "object": "file"}
@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_FILE_CALLS)
@respx.mock
async def test_hosted_vllm_file_with_a_different_object_type_is_rejected(
method: str, path: str, call_router: Callable[[], Awaitable[OpenAIFileObject]]
):
respx.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond(
200, json={**VLLM_ROUTER_FILE, "object": "batch"}
)
with pytest.raises(ValidationError) as exc_info:
await call_router()
assert [error["loc"] for error in exc_info.value.errors()] == [("object",)]
_FILE_BODY: Final = b'{"prompt": "Hello", "completion": "Hi"}'
_FINE_TUNE_FILE_JSON: Final = MappingProxyType(
{

View file

@ -1,3 +1,4 @@
from collections.abc import Callable
from typing import Final
import pytest
@ -7,7 +8,7 @@ from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
StandardBuiltInToolCostTracking,
)
from litellm.types.llms.openai import FileSearchTool, ResponsesAPIResponse, WebSearchOptions
from litellm.types.utils import ModelResponse, StandardBuiltInToolsParams
from litellm.types.utils import ModelResponse, ServerToolUse, StandardBuiltInToolsParams, Usage
def test_web_search_cost_low():
@ -1052,6 +1053,175 @@ def test_azure_ai_non_claude_web_search_keeps_generic_fallback() -> None:
assert get_cost_for_web_search_request(custom_llm_provider="azure_ai", usage=usage, model_info=model_info) is None
def _register_priced_deployment(
model: str, search_cost_per_query: float | None, deployment_id: str | None = None
) -> str:
from litellm import Router
token_rates: Final = {"input_cost_per_token": 3e-6, "output_cost_per_token": 1.5e-5}
search_rates: Final = (
{}
if search_cost_per_query is None
else {
"search_context_cost_per_query": {
"search_context_size_low": search_cost_per_query,
"search_context_size_medium": search_cost_per_query,
"search_context_size_high": search_cost_per_query,
}
}
)
router: Final = Router(
model_list=[
{
"model_name": "priced-deployment",
"litellm_params": {"model": model, "api_key": "k", "vertex_project": "p", "vertex_location": "global"},
"model_info": {**token_rates, **search_rates, **({} if deployment_id is None else {"id": deployment_id})},
}
]
)
return router.model_list[0]["model_info"]["id"]
def _claude_chat_response(
web_search_requests: int, model: str = "claude-sonnet-5-5"
) -> tuple[object, Usage | None]:
usage: Final = Usage(
prompt_tokens=1840,
completion_tokens=412,
total_tokens=2252,
server_tool_use=ServerToolUse(web_search_requests=web_search_requests),
)
return ModelResponse(model=model, usage=usage), usage
def _claude_messages_response(
web_search_requests: int, model: str = "claude-sonnet-5-5"
) -> tuple[object, Usage | None]:
return {
"id": "msg_vertex_claude_web_search",
"type": "message",
"role": "assistant",
"model": model,
"content": [{"type": "text", "text": "scripted answer"}],
"usage": {
"input_tokens": 1840,
"output_tokens": 412,
"server_tool_use": {"web_search_requests": web_search_requests},
},
}, None
@pytest.mark.parametrize("build_response", [_claude_chat_response, _claude_messages_response])
@pytest.mark.parametrize(("web_search_requests", "search_cost_per_query"), [(2, 0.01), (3, 0.01), (2, 0.0)])
def test_vertex_claude_deployment_priced_web_search_billed_per_request(
local_model_cost_map: None,
build_response: Callable[[int], tuple[object, Usage | None]],
web_search_requests: int,
search_cost_per_query: float,
) -> None:
deployment_id: Final = _register_priced_deployment("vertex_ai/claude-sonnet-5-5", search_cost_per_query)
response_object, usage = build_response(web_search_requests)
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model=deployment_id,
response_object=response_object,
usage=usage,
custom_llm_provider="vertex_ai",
standard_built_in_tools_params=None,
)
assert cost == pytest.approx(web_search_requests * search_cost_per_query)
@pytest.mark.parametrize("build_response", [_claude_chat_response, _claude_messages_response])
@pytest.mark.parametrize(
("deployment_model", "served_model", "custom_llm_provider"),
# Google bills Claude web search on Sonnet 4.5 and Haiku 4.5 too:
# https://cloud.google.com/model-garden/partner-models/pricing (read 2026-10-30)
[
("vertex_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "vertex_ai"),
("vertex_ai/claude-sonnet-4-5@20250929", "claude-sonnet-4-5-20250929", "vertex_ai"),
("vertex_ai/claude-haiku-4-5@20251001", "claude-haiku-4-5-20251001", "vertex_ai"),
("anthropic/claude-sonnet-5-5", "claude-sonnet-5-5", "anthropic"),
("azure_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "azure_ai"),
],
)
@pytest.mark.parametrize("deployment_id", [None, "claude-token-priced"], ids=["hashed_id", "claude_in_id"])
def test_claude_deployment_with_token_rates_only_bills_served_model_search_price(
local_model_cost_map: None,
build_response: Callable[[int, str], tuple[object, Usage | None]],
deployment_model: str,
served_model: str,
custom_llm_provider: str,
deployment_id: str | None,
) -> None:
served_search_pricing: Final = (
litellm.get_model_info(model=deployment_model).get("search_context_cost_per_query") or {}
)
served_search_price: Final = served_search_pricing.get("search_context_size_medium") or 0.0
registered_id: Final = _register_priced_deployment(deployment_model, None, deployment_id)
response_object, usage = build_response(2, served_model)
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model=registered_id,
response_object=response_object,
usage=usage,
custom_llm_provider=custom_llm_provider,
standard_built_in_tools_params=None,
)
assert served_search_price > 0
assert cost == pytest.approx(2 * served_search_price)
def test_vertex_claude_deployment_with_token_rates_only_and_unmapped_served_model_bills_no_search(
local_model_cost_map: None,
) -> None:
deployment_id: Final = _register_priced_deployment("vertex_ai/claude-sonnet-5-5", None)
usage: Final = Usage(
prompt_tokens=1840,
completion_tokens=412,
total_tokens=2252,
server_tool_use=ServerToolUse(web_search_requests=2),
)
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model=deployment_id,
response_object=ModelResponse(model="claude-unmapped-preview", usage=usage),
usage=usage,
custom_llm_provider="vertex_ai",
standard_built_in_tools_params=None,
)
assert cost == 0.0
@pytest.mark.parametrize("deployment_id", [None, "claude-code-gemini"], ids=["hashed_id", "claude_in_id"])
def test_vertex_gemini_deployment_priced_web_search_stays_per_prompt(
local_model_cost_map: None, deployment_id: str | None
) -> None:
search_cost_per_prompt: Final = 0.03
registered_id: Final = _register_priced_deployment(
"vertex_ai/gemini-2.5-flash", search_cost_per_prompt, deployment_id
)
usage: Final = Usage(
prompt_tokens=1840,
completion_tokens=412,
total_tokens=2252,
server_tool_use=ServerToolUse(web_search_requests=2),
)
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model=registered_id,
response_object=ModelResponse(model="gemini-2.5-flash", usage=usage),
usage=usage,
custom_llm_provider="vertex_ai",
standard_built_in_tools_params=None,
)
assert cost == pytest.approx(search_cost_per_prompt)
@pytest.mark.parametrize(
"kwargs",
[{"tools": 5}, {"web_search_options": None}, {"web_search_options": 5}],

View file

@ -25,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_semantic_cache_prompt_from_messages,
get_str_from_messages,
handle_any_messages_to_chat_completion_str_messages_conversion,
handle_messages_with_content_list_to_str_conversion,
hoist_images_from_tool_messages,
is_encrypted_reasoning_block,
merge_consecutive_system_messages,
@ -2675,3 +2676,16 @@ def test_tpm_projection_preserves_non_media_identity_and_opaque_values() -> None
projected, media = split_media_for_tpm(request)
assert projected is request
assert media == ()
def test_content_list_flattening_returns_new_messages_and_leaves_the_callers_untouched() -> None:
messages: Final = [
{"role": "user", "content": [{"type": "text", "text": "Hello"}, {"type": "text", "text": " world"}]},
{"role": "assistant", "content": "Hi"},
]
messages_before: Final = copy.deepcopy(messages)
flattened: Final = handle_messages_with_content_list_to_str_conversion(messages)
assert flattened == [{"role": "user", "content": "Hello world"}, {"role": "assistant", "content": "Hi"}]
assert messages == messages_before

View file

@ -1,6 +1,7 @@
import copy
import json
import logging
from collections.abc import Iterator
from itertools import chain
from typing import Final
from unittest.mock import MagicMock, patch
@ -9919,3 +9920,186 @@ def test_transform_request_tool_config_point_without_tools_drops_the_key():
assert "cache_control_injection_points" not in data
assert "tools" not in data
assert litellm_params["metadata"] == {}
def _five_cache_hint_conversation() -> list[dict[str, object]]:
return [
{"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]},
{"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]},
{"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]},
{"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]},
{"role": "assistant", "content": "answer 2"},
{"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]},
]
def _cached_texts(node: object) -> Iterator[str]:
if isinstance(node, dict):
yield from ((node["text"],) if "cache_control" in node and "text" in node else ())
yield from chain.from_iterable(map(_cached_texts, node.values()))
elif isinstance(node, list):
yield from chain.from_iterable(map(_cached_texts, node))
@pytest.mark.parametrize(
("config", "model"),
[
(AnthropicConfig(), "claude-haiku-5-5"),
(VertexAIAnthropicConfig(), "claude-haiku-5-5"),
(AmazonAnthropicClaudeConfig(), "anthropic.claude-haiku-5-5"),
(AzureAnthropicConfig(), "claude-haiku-5-5"),
],
ids=["anthropic", "vertex_ai", "bedrock_invoke", "azure_ai"],
)
@pytest.mark.parametrize(
("drop_params", "expected_cached_texts"),
[
(True, ["answer 1", "turn 1", "turn 2", "turn 3"]),
("true", ["answer 1", "turn 1", "turn 2", "turn 3"]),
(False, ["answer 1", "system prompt", "turn 1", "turn 2", "turn 3"]),
],
ids=[
"drop_params_keeps_the_newest_four",
"string_drop_params_keeps_the_newest_four",
"without_drop_params_all_five_pass_through",
],
)
def test_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params(
config: AnthropicConfig,
model: str,
drop_params: bool | str,
expected_cached_texts: list[str],
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
messages: Final = _five_cache_hint_conversation()
messages_before: Final = copy.deepcopy(messages)
body: Final = config.transform_request(
model=model,
messages=messages,
optional_params={},
litellm_params={"drop_params": drop_params},
headers={},
)
assert sorted(_cached_texts(body)) == expected_cached_texts
assert json.dumps(body).count('"cache_control"') == len(expected_cached_texts)
assert messages == messages_before
def test_claude_cache_control_cap_keeps_a_cached_tool_and_drops_the_oldest_message_hint(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "drop_params", True)
messages: Final = _five_cache_hint_conversation()[1:]
tool: Final = {
"name": "get_weather",
"description": "Get the weather",
"input_schema": {"type": "object", "properties": {}},
"cache_control": {"type": "ephemeral"},
}
body: Final = AnthropicConfig().transform_request(
model="claude-haiku-5-5",
messages=messages,
optional_params={"tools": [tool]},
litellm_params={},
headers={},
)
assert body["tools"][0]["cache_control"] == {"type": "ephemeral"}
assert sorted(_cached_texts(body["messages"])) == ["answer 1", "turn 2", "turn 3"]
def test_claude_cache_control_cap_drops_a_string_assistant_turns_text_hint_before_its_tool_call_hints(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "drop_params", True)
hint: Final = {"type": "ephemeral"}
tool_calls: Final = [
{"id": f"call_{name}", "type": "function", "function": {"name": name, "arguments": "{}"}, "cache_control": hint}
for name in ("lookup", "search")
]
follow_up: Final = [
{"type": "text", "text": "follow-up", "cache_control": hint},
{"type": "text", "text": "and more", "cache_control": hint},
]
messages: Final = [
{"role": "user", "content": "question"},
{"role": "assistant", "content": "checking both", "cache_control": hint, "tool_calls": tool_calls},
{"role": "tool", "tool_call_id": "call_lookup", "content": "found"},
{"role": "tool", "tool_call_id": "call_search", "content": "none"},
{"role": "user", "content": follow_up},
]
body: Final = AnthropicConfig().transform_request(
model="claude-haiku-5-5",
messages=messages,
optional_params={},
litellm_params={},
headers={},
)
assistant_blocks: Final = body["messages"][1]["content"]
assert [block["type"] for block in assistant_blocks] == ["text", "tool_use", "tool_use"]
assert ["cache_control" in block for block in assistant_blocks] == [False, True, True]
assert json.dumps(body).count('"cache_control"') == 4
assert json.dumps(body).count('"cache_control"') == 4
def test_claude_cache_control_cap_keeps_the_newest_hints_inside_one_message(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "drop_params", True)
messages: Final = [
{
"role": "user",
"content": [
*({"type": "text", "text": f"part {n}", "cache_control": {"type": "ephemeral"}} for n in range(1, 6)),
{"type": "text", "text": "Reply with the single word OK."},
],
}
]
messages_before: Final = copy.deepcopy(messages)
body: Final = AnthropicConfig().transform_request(
model="claude-haiku-5-5",
messages=messages,
optional_params={},
litellm_params={},
headers={},
)
assert sorted(_cached_texts(body["messages"])) == ["part 2", "part 3", "part 4", "part 5"]
assert json.dumps(body).count('"cache_control"') == 4
assert messages == messages_before
def test_claude_cache_control_cap_does_not_count_a_message_level_hint_that_list_content_never_sends(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
messages: Final = [
{"role": "system", "content": "system prompt", "cache_control": {"type": "ephemeral"}},
{"role": "user", "content": "turn 1", "cache_control": {"type": "ephemeral"}},
{"role": "assistant", "content": "answer 1"},
{"role": "user", "content": "turn 2", "cache_control": {"type": "ephemeral"}},
{"role": "assistant", "content": "answer 2"},
{
"role": "user",
"content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}],
"cache_control": {"type": "ephemeral"},
},
]
body: Final = AnthropicConfig().transform_request(
model="claude-haiku-5-5",
messages=messages,
optional_params={},
litellm_params={"drop_params": True},
headers={},
)
assert sorted(_cached_texts(body)) == ["system prompt", "turn 1", "turn 2", "turn 3"]
assert json.dumps(body).count('"cache_control"') == 4

View file

@ -56,6 +56,98 @@ def test_is_anthropic_messages_url(url: str, expected: bool) -> None:
assert is_anthropic_messages_url(url) is expected
@pytest.mark.parametrize(
("configured_base", "url", "expected"),
[
pytest.param(None, "https://api.anthropic.com/v1/messages", 512, id="default-upstream"),
pytest.param(None, "https://api.anthropic.com:443/v1/messages/", 512, id="default-upstream-explicit-port"),
pytest.param("http://relay.test:8080", "https://api.anthropic.com/v1/messages", 512, id="direct-target"),
pytest.param("https://api.anthropic.com", "https://api.anthropic.com/v1/messages", 512, id="configured-default"),
pytest.param("http://relay.test:8080", "http://relay.test:8080/v1/messages", 255, id="configured-upstream"),
pytest.param("http://relay.test:8080/a", "http://relay.test:8080/a/v1/messages", 255, id="configured-prefix"),
pytest.param("https://Relay.Test", "https://relay.test/v1/messages", 255, id="configured-upper-case-host"),
pytest.param("https://relay.test:443", "https://relay.test/v1/messages", 255, id="configured-https-port"),
pytest.param("http://relay.test:80/a", "http://relay.test/a/v1/messages", 255, id="configured-http-port"),
pytest.param("http://relay.test:8080", "http://relay.test:8081/v1/messages", None, id="other-port"),
pytest.param("http://relay.test:notaport", "http://relay.test/v1/messages", None, id="invalid-configured-base"),
pytest.param(None, "https://other-vendor.test/v1/messages", None, id="other-vendor"),
pytest.param(None, "https://api.anthropic.com/v1/messages/count_tokens", None, id="count-tokens"),
],
)
def test_anthropic_upstream_user_id_limit(
monkeypatch: pytest.MonkeyPatch, configured_base: str | None, url: str, expected: int | None
) -> None:
from litellm.llms.anthropic.common_utils import anthropic_upstream_user_id_limit
monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False)
monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False)
if configured_base is not None:
monkeypatch.setenv("ANTHROPIC_API_BASE", configured_base)
assert anthropic_upstream_user_id_limit(url) == expected
def test_anthropic_pass_through_api_base_prefers_the_api_base_variable(monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.llms.anthropic.common_utils import anthropic_pass_through_api_base
monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://fallback.test")
monkeypatch.setenv("ANTHROPIC_API_BASE", "http://preferred.test")
assert anthropic_pass_through_api_base() == "http://preferred.test"
monkeypatch.delenv("ANTHROPIC_API_BASE")
assert anthropic_pass_through_api_base() == "http://fallback.test"
# Each row was sent to https://api.anthropic.com/v1/messages on 2026-10-30, False rows answered 400.
@pytest.mark.parametrize(
("user_id", "expected"),
[
pytest.param("user-1", True, id="opaque"),
pytest.param("", True, id="empty"),
pytest.param('{"device_id":"abc","session_id":"def"}', True, id="claude-code-shape"),
pytest.param("1234567890", True, id="digits"),
pytest.param("+14155550100", True, id="phone-shape"),
pytest.param("a@b", True, id="no-dot"),
pytest.param("@b.co", True, id="nothing-before-at-sign"),
pytest.param("a@.co", True, id="nothing-between-at-sign-and-dot"),
pytest.param("a@b.", True, id="nothing-after-dot"),
pytest.param("a @ b.co", True, id="spaces-around-at-sign"),
pytest.param("a\u00a0@b.co", False, id="no-break-space-before-at-sign"),
pytest.param("a@b\n.co", False, id="newline-before-dot"),
pytest.param("line one\nline two", False, id="newline"),
pytest.param("a\tb", False, id="tab"),
pytest.param("a\x1f@b.co", False, id="unit-separator-before-at-sign"),
pytest.param("a@\x1cb.co", False, id="file-separator-after-at-sign"),
pytest.param("a\u200bb", False, id="zero-width-space"),
pytest.param("a@b.co", False, id="email"),
pytest.param("user a@b.co here", False, id="email-inside-text"),
pytest.param('{"device_id":"a@b.co"}', False, id="email-inside-json"),
pytest.param("a@@example.com", False, id="doubled-at-sign"),
pytest.param("@@example.com", False, id="leading-doubled-at-sign"),
pytest.param("a@b@c.co", False, id="two-at-signs"),
pytest.param("a@b..co", False, id="doubled-dot"),
pytest.param("a@.b.co", False, id="dot-after-at-sign"),
pytest.param("a@b.c d", False, id="email-before-space"),
],
)
def test_anthropic_accepts_user_id(user_id: str, expected: bool) -> None:
from litellm.constants import ANTHROPIC_MAX_USER_ID_LENGTH
from litellm.llms.anthropic.common_utils import anthropic_accepts_user_id
assert anthropic_accepts_user_id(user_id, ANTHROPIC_MAX_USER_ID_LENGTH) is expected
@pytest.mark.parametrize("character", ["a", "\u00e9", "\U0001f600"], ids=["ascii", "two-bytes", "four-bytes"])
def test_anthropic_accepts_user_id_counts_characters_up_to_the_limit(character: str) -> None:
from litellm.constants import ANTHROPIC_MAX_USER_ID_LENGTH
from litellm.llms.anthropic.common_utils import anthropic_accepts_user_id
assert anthropic_accepts_user_id(character * ANTHROPIC_MAX_USER_ID_LENGTH, ANTHROPIC_MAX_USER_ID_LENGTH) is True
assert (
anthropic_accepts_user_id(character * (ANTHROPIC_MAX_USER_ID_LENGTH + 1), ANTHROPIC_MAX_USER_ID_LENGTH) is False
)
def test_anthropic_model_not_found_maps_to_not_found_but_authentication_error_does_not() -> None:
with respx.mock(assert_all_called=True) as router:
router.post("https://api.anthropic.com/v1/messages").mock(

View file

@ -11259,3 +11259,86 @@ def test_converse_sends_only_the_header_betas_when_header_and_body_both_carry_be
"thinking-binding-controls-2026-08-01",
"thinking-display-updates-2026-08-18",
]
def _five_cache_hint_conversation() -> list[dict[str, object]]:
return [
{"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]},
{"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]},
{"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]},
{"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]},
{"role": "assistant", "content": "answer 2"},
{"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]},
]
@pytest.mark.parametrize(
("drop_params", "expected_system_cache_points"),
[(True, 0), ("true", 0), (False, 1)],
ids=[
"drop_params_drops_the_oldest_hint",
"string_drop_params_drops_the_oldest_hint",
"without_drop_params_all_five_pass_through",
],
)
def test_converse_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params(
drop_params: bool | str, expected_system_cache_points: int, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
messages: Final = _five_cache_hint_conversation()
messages_before: Final = copy.deepcopy(messages)
request: Final = AmazonConverseConfig().transform_request(
model="anthropic.claude-haiku-5-5",
messages=messages,
optional_params={},
litellm_params={"drop_params": drop_params},
headers={},
)
assert len(list(_cache_points(request["system"]))) == expected_system_cache_points
assert [len(list(_cache_points(message))) for message in request["messages"]] == [1, 1, 1, 0, 1]
assert messages == messages_before
def test_converse_claude_tool_message_with_several_marked_blocks_counts_as_one_cache_point_against_the_cap(
monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
messages: Final = [
{
"role": "user",
"content": [
{"type": "text", "text": "policy", "cache_control": {"type": "ephemeral"}},
{"type": "text", "text": "question", "cache_control": {"type": "ephemeral"}},
],
},
{
"role": "assistant",
"content": None,
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}],
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": [
{"type": "text", "text": "part 1", "cache_control": {"type": "ephemeral"}},
{"type": "text", "text": "part 2", "cache_control": {"type": "ephemeral"}},
{"type": "text", "text": "part 3", "cache_control": {"type": "ephemeral"}},
],
},
]
messages_before: Final = copy.deepcopy(messages)
request: Final = AmazonConverseConfig().transform_request(
model="anthropic.claude-haiku-5-5",
messages=messages,
optional_params={
"tools": [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}]
},
litellm_params={"drop_params": True},
headers={},
)
assert len(list(_cache_points(request["messages"]))) == 3
assert messages == messages_before

View file

@ -2773,3 +2773,66 @@ def test_claude_json_schema_stream_split_across_json_tools_keeps_the_non_stream_
choices: Final = _json_schema_stream_choices(respx_mock, call, split_stream)
assert [choice.finish_reason for choice in choices if choice.finish_reason] == [answer.finish_reason]
@pytest.mark.parametrize(
("model", "base_model"),
[("databricks-claude-haiku-5-5", None), ("support-bot-endpoint", "databricks/databricks-claude-haiku-5-5")],
ids=["claude_in_the_endpoint_name", "claude_named_only_by_base_model"],
)
def test_databricks_claude_request_over_the_cache_control_cap_keeps_the_newest_four_hints_under_drop_params(
model: str, base_model: str | None, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
hint: Final = {"type": "ephemeral"}
messages: Final = [
{"role": "system", "content": "system prompt", "cache_control": hint},
{"role": "user", "content": "turn 1", "cache_control": hint},
{"role": "assistant", "content": "answer 1", "cache_control": hint},
{"role": "user", "content": "turn 2", "cache_control": hint},
{"role": "assistant", "content": "answer 2"},
{"role": "user", "content": "turn 3", "cache_control": hint},
]
messages_before: Final = copy.deepcopy(messages)
body: Final = DatabricksConfig().transform_request(
model=model,
messages=messages,
optional_params={},
litellm_params={"drop_params": True, "custom_llm_provider": "databricks", "base_model": base_model},
headers={},
)
assert "cache_control" not in json.dumps(body["messages"][0])
assert json.dumps(body["messages"]).count('"cache_control"') == 4
assert messages == messages_before
@pytest.mark.parametrize(
"optional_params",
[{"cache_control": {"type": "ephemeral"}}, {"extra_body": {"cache_control": {"type": "ephemeral"}}}],
ids=["top_level_cache_control", "extra_body_cache_control"],
)
def test_databricks_claude_cache_control_cap_keeps_room_for_a_top_level_cache_control_under_drop_params(
optional_params: dict[str, object], monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
hint: Final = {"type": "ephemeral"}
messages: Final = [
{"role": "user", "content": "turn 1", "cache_control": hint},
{"role": "assistant", "content": "answer 1", "cache_control": hint},
{"role": "user", "content": "turn 2", "cache_control": hint},
{"role": "assistant", "content": "answer 2", "cache_control": hint},
{"role": "user", "content": "turn 3"},
]
body: Final = DatabricksConfig().transform_request(
model="databricks-claude-haiku-5-5",
messages=messages,
optional_params=copy.deepcopy(optional_params),
litellm_params={"drop_params": True, "custom_llm_provider": "databricks"},
headers={},
)
assert "cache_control" not in json.dumps(body["messages"][0])
assert json.dumps(body["messages"]).count('"cache_control"') == 3

View file

@ -1,6 +1,8 @@
import asyncio
import copy
import json
import os
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@ -200,3 +202,38 @@ async def test_deepinfra_tool_message_content_transformation_async():
print(f"✓ Async test passed: {tool_message['content']}")
print("\n✅ DeepInfra async tool message transformation test passed!")
@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"])
async def test_deepinfra_request_stringifies_tool_content_without_touching_the_callers_message(is_async: bool) -> None:
from litellm.llms.deepinfra.chat.transformation import DeepInfraConfig
config: Final = DeepInfraConfig()
messages: Final = [
{"role": "user", "content": "Calculate 10 + 10"},
{
"role": "assistant",
"content": "",
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "calculator", "arguments": "{}"}}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": [{"type": "text", "text": "20"}]},
]
messages_before: Final = copy.deepcopy(messages)
request_kwargs: Final = {
"model": "deepseek-ai/DeepSeek-V3.2",
"messages": messages,
"optional_params": {},
"litellm_params": {},
"headers": {},
}
body: Final = (
await config.async_transform_request(**request_kwargs)
if is_async
else config.transform_request(**request_kwargs)
)
assert body["messages"][2]["content"] == "20"
assert messages == messages_before

View file

@ -1,3 +1,4 @@
import copy
import json
import logging
from collections.abc import Callable
@ -537,6 +538,26 @@ def test_tool_search_reply_sent_to_fireworks_drops_litellm_response_fields():
}
def test_cache_control_is_stripped_from_the_request_but_kept_on_the_callers_messages():
hint: Final = {"type": "ephemeral"}
messages: Final = [
{"role": "system", "content": [{"type": "text", "text": "a long cached prompt", "cache_control": hint}]},
{"role": "user", "content": "hi", "cache_control": hint},
]
caller_messages_before: Final = json.dumps(messages)
request: Final = FireworksAIConfig().transform_request(
model="accounts/fireworks/models/gpt-oss-120b",
messages=messages,
optional_params={},
litellm_params={},
headers={},
)
assert "cache_control" not in json.dumps(request["messages"])
assert json.dumps(messages) == caller_messages_before
# -----------------------------------------------------------------------------
# Regression tests for legacy / OpenAPI $ref defs in tool parameters.
#
@ -2477,3 +2498,43 @@ def test_translate_developer_role_follows_the_deployment_model_info_over_the_cos
)
assert [message["role"] for message in messages] == expected_roles
def test_fireworks_request_rewrites_tools_without_touching_the_callers_tool() -> None:
tools: Final = [
{
"type": "function",
"function": {
"name": "create_thing",
"strict": True,
"parameters": {
"type": "object",
"properties": {"thing": {"$ref": "#/definitions/Thing"}},
"definitions": {"Thing": {"type": "object", "properties": {"id": {"type": "string"}}}},
},
},
}
]
tools_before: Final = copy.deepcopy(tools)
request: Final = FireworksAIConfig().transform_request(
model="accounts/fireworks/models/gpt-oss-120b",
messages=[{"role": "user", "content": "Make a thing"}],
optional_params={"tools": tools},
litellm_params={},
headers={},
)
assert request["tools"] == [
{
"type": "function",
"function": {
"name": "create_thing",
"parameters": {
"type": "object",
"properties": {"thing": {"type": "object", "properties": {"id": {"type": "string"}}}},
},
},
}
]
assert tools == tools_before

View file

@ -1,3 +1,4 @@
import copy
import logging
from typing import Final
from unittest.mock import patch
@ -589,3 +590,19 @@ async def test_streamed_reasoning_reply_can_be_replayed(is_async: bool, with_too
}
]
assert reply.reasoning_content == "I should check the weather."
def test_groq_request_drops_null_assistant_fields_without_touching_the_callers_messages() -> None:
messages: Final = [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hello", "function_call": None, "tool_calls": None},
{"role": "user", "content": "Again"},
]
messages_before: Final = copy.deepcopy(messages)
body: Final = GroqChatConfig().transform_request(
model="openai/gpt-oss-120b", messages=messages, optional_params={}, litellm_params={}, headers={}
)
assert body["messages"][1] == {"role": "assistant", "content": "Hello"}
assert messages == messages_before

View file

@ -1,4 +1,6 @@
import copy
import json
from typing import Final
from unittest.mock import MagicMock, patch
import pytest
@ -397,3 +399,43 @@ def test_hosted_vllm_custom_tools_use_top_level_input_schema():
assert tools[0]["function"]["name"] == "search"
assert tools[0]["function"]["description"] == "Search docs"
assert tools[0]["function"]["parameters"] == input_schema
def _strict_weather_tool() -> dict[str, object]:
return {
"type": "function",
"function": {
"name": "get_weather",
"strict": True,
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
"additionalProperties": False,
},
},
}
def test_hosted_vllm_tools_are_rewritten_without_touching_the_callers_tool() -> None:
tools: Final = [_strict_weather_tool()]
tools_before: Final = copy.deepcopy(tools)
optional_params: Final = HostedVLLMChatConfig().map_openai_params(
non_default_params={"tools": tools}, optional_params={}, model="gpt-oss-120b", drop_params=False
)
assert optional_params["tools"] == [
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
}
]
assert tools == tools_before

View file

@ -1,3 +1,4 @@
import copy
import json
import logging
from collections.abc import Mapping
@ -1212,7 +1213,10 @@ def test_mistral_sends_only_the_text_items_of_a_native_thinking_chunk_beside_an_
"content": [
{
"type": "thinking",
"thinking": [{"type": "text", "text": "It has whiskers."}, {"type": "reference", "reference_ids": [1]}],
"thinking": [
{"type": "text", "text": "It has whiskers."},
{"type": "reference", "reference_ids": [1]},
],
},
{"type": "text", "text": "A cat."},
],
@ -1672,6 +1676,18 @@ def test_mistral_transform_request_drops_an_empty_tool_message_whose_call_id_is_
assert [m.get("role") for m in request["messages"]] == ["user", "assistant"]
def test_mistral_request_drops_a_user_name_without_touching_the_callers_message() -> None:
messages: Final = [{"role": "user", "content": "Hi"}, {"role": "user", "content": "", "name": "alice"}]
messages_before: Final = copy.deepcopy(messages)
body: Final = MistralConfig().transform_request(
model="mistral-medium-latest", messages=messages, optional_params={}, litellm_params={}, headers={}
)
assert "name" not in json.dumps(body["messages"])
assert messages == messages_before
def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]:
return tuple(
r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage()

View file

@ -1824,6 +1824,94 @@ class TestToolSchemaCombinatorFlatteningForOpenAI:
assert request["tools"][0] is tool
class TestCacheControlStrippingDoesNotMutateCallerInput:
"""
Stripping cache_control for a provider that cannot use it must not reach back
into the caller's own message and tool objects.
filter_value_from_dict deletes the key in place and recurses into nested dicts
and lists, and remove_cache_control_flag_from_messages_and_tools assigned the
result back into the caller's list, so one call to any OpenAI-compatible
provider permanently stripped cache_control from a list the caller still holds.
Reusing that list on Anthropic or Bedrock afterwards then silently lost prompt
caching, with no error and full-price billing.
"""
def setup_method(self):
self.config = OpenAIGPTConfig()
@pytest.fixture(autouse=True)
def _clean_openai_base_env(self, monkeypatch):
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
monkeypatch.setattr(litellm, "api_base", None, raising=False)
@staticmethod
def _messages():
return [
{
"role": "system",
"content": [
{
"type": "text",
"text": "a long cached system prompt",
"cache_control": {"type": "ephemeral"},
}
],
},
{
"role": "user",
"content": "Hello",
"cache_control": {"type": "ephemeral"},
},
]
@staticmethod
def _tools():
return [
{
"type": "function",
"function": {"name": "get_weather", "parameters": {"type": "object"}},
"cache_control": {"type": "ephemeral"},
}
]
def _transform(self, messages, tools):
return self.config.transform_request(
model="gpt-4o",
messages=messages,
optional_params={"tools": tools},
litellm_params={"custom_llm_provider": "openai", "api_base": None},
headers={},
)
def test_caller_messages_and_tools_keep_cache_control(self):
messages, tools = self._messages(), self._tools()
messages_before, tools_before = copy.deepcopy(messages), copy.deepcopy(tools)
request = self._transform(messages, tools)
assert "cache_control" not in json.dumps(request)
assert messages == messages_before
assert tools == tools_before
def test_a_later_anthropic_call_still_sees_cache_control(self):
"""The user-visible consequence: the same message list reused on a provider
that does support caching must still carry the cache breakpoints."""
messages = self._messages()
self._transform(messages, self._tools())
anthropic_body = litellm.AnthropicConfig().transform_request(
model="claude-3-5-sonnet-20240620",
messages=messages,
optional_params={},
litellm_params={},
headers={},
)
assert "cache_control" in json.dumps(anthropic_body)
def _developer_message_after_a_user_turn() -> list[dict[str, str]]:
return [
{"role": "system", "content": "You are terse."},
@ -1916,3 +2004,53 @@ def test_completion_leaves_the_callers_closing_developer_message_unchanged_on_a_
)
assert json.loads(chat.calls.last.request.content)["messages"][-1]["role"] == "user"
assert messages == sent
def _user_turn_with_litellm_only_media_fields() -> list[dict[str, object]]:
return [
{
"role": "user",
"content": [
{"type": "text", "text": "Describe these"},
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png", "format": "image/png"}},
{"type": "image_url", "image_url": "https://example.com/dog.png"},
{
"type": "file",
"file": {"file_data": "data:application/pdf;base64,JVBERi0=", "format": "application/pdf"},
},
],
}
]
@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"])
async def test_openai_request_rewrites_media_parts_without_touching_the_callers_message(
is_async: bool, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
monkeypatch.setattr(litellm, "api_base", None, raising=False)
config: Final = OpenAIGPTConfig()
messages: Final = _user_turn_with_litellm_only_media_fields()
messages_before: Final = copy.deepcopy(messages)
request_kwargs: Final = {
"model": "gpt-5.5",
"messages": messages,
"optional_params": {},
"litellm_params": {"custom_llm_provider": "openai", "api_base": None},
"headers": {},
}
body: Final = (
await config.async_transform_request(**request_kwargs)
if is_async
else config.transform_request(**request_kwargs)
)
assert body["messages"][0]["content"] == [
{"type": "text", "text": "Describe these"},
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}},
{"type": "image_url", "image_url": {"url": "https://example.com/dog.png"}},
{"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0=", "filename": "my_file.pdf"}},
]
assert messages == messages_before

View file

@ -1,3 +1,4 @@
import copy
import json
import logging
from collections.abc import Callable, Mapping
@ -541,6 +542,50 @@ class TestOpenAIResponsesAPIConfig:
assert "cache_control" not in result["tools"][0]
assert result["tools"][0]["name"] == "get_weather"
def test_transform_does_not_strip_cache_control_from_the_callers_input(self):
"""Stripping for OpenAI must not reach back into the caller's own objects.
`filter_value_from_dict` deletes the key in place and recurses, and
`_validate_input_param` passes plain dict items through by reference, so
the caller's input list used to lose its cache breakpoints. Reusing that
list on Anthropic or Bedrock afterwards then silently lost prompt caching.
Same defect as the Chat Completions path.
"""
input_with_cache_control = [
{
"role": "system",
"content": [
{
"type": "input_text",
"text": "a long cached system prompt",
"cache_control": {"type": "ephemeral"},
}
],
}
]
tools_with_cache_control = [
{
"type": "function",
"name": "get_weather",
"parameters": {"type": "object"},
"cache_control": {"type": "ephemeral"},
}
]
input_before = copy.deepcopy(input_with_cache_control)
tools_before = copy.deepcopy(tools_with_cache_control)
result = self.config.transform_responses_api_request(
model=self.model,
input=input_with_cache_control,
response_api_optional_request_params={"tools": tools_with_cache_control},
litellm_params={},
headers={},
)
assert "cache_control" not in json.dumps(result)
assert input_with_cache_control == input_before
assert tools_with_cache_control == tools_before
def test_transform_preserves_input_without_cache_control(self):
"""Inputs without cache_control must pass through unmodified."""
input_clean = [
@ -4607,10 +4652,10 @@ def test_cache_control_still_marks_a_part_whose_own_marker_is_malformed_under_dr
}
def _caller_input_after_transform(model: str, marker: object) -> str:
def _hinted_caller_input(marker: object) -> list[dict[str, object]]:
hint: Final = {"type": "ephemeral"}
marked: Final = {} if marker is None else {"prompt_cache_breakpoint": marker}
input: Final = [
return [
{
"type": "message",
"role": "user",
@ -4626,6 +4671,10 @@ def _caller_input_after_transform(model: str, marker: object) -> str:
"content": [{"type": "output_text", "text": "ok", "cache_control": hint, **marked}],
},
]
def _caller_input_after_transform(model: str, marker: object) -> str:
input: Final = _hinted_caller_input(marker)
_CONFIGS[0].transform_responses_api_request(
model=model,
input=cast("ResponseInputParam", input), # cast-ok: plain dicts shaped like the SDK's input items
@ -4647,6 +4696,15 @@ def test_a_marker_leaves_the_caller_request_that_fallbacks_and_logs_read_as_an_u
assert _caller_input_after_transform(model, marker).replace(marker_json, "") == unmarked
@pytest.mark.parametrize("model", ("gpt-4.1", "gpt-6.1-sol"))
@pytest.mark.parametrize("marker", (None, _VALID_MARKERS[0], "yes"), ids=("unmarked", "explicit", "malformed"))
def test_the_caller_input_keeps_its_cache_control_hints_for_a_fallback_deployment(
monkeypatch: pytest.MonkeyPatch, model: str, marker: object
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
assert _caller_input_after_transform(model, marker) == json.dumps(_hinted_caller_input(marker))
@pytest.mark.asyncio
@pytest.mark.usefixtures(httpx_transport.__name__)
async def test_aresponses_forwards_previous_response_id_to_openai() -> None:

View file

@ -1,3 +1,4 @@
import copy
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
@ -198,3 +199,23 @@ def test_translate_developer_role_keeps_developer_messages_untouched() -> None:
)
== messages
)
def test_o_series_request_turns_system_into_user_without_touching_the_callers_messages(
monkeypatch: pytest.MonkeyPatch,
) -> None:
model: Final = "o-series-test-model-without-system-role"
monkeypatch.setitem(
litellm.model_cost,
model,
{"litellm_provider": "openai", "mode": "chat", "supports_system_messages": False},
)
messages: Final = [{"role": "system", "content": "Be terse"}, {"role": "user", "content": "Hi"}]
messages_before: Final = copy.deepcopy(messages)
body: Final = OpenAIOSeriesConfig().transform_request(
model=model, messages=messages, optional_params={}, litellm_params={}, headers={}
)
assert [message["role"] for message in body["messages"]] == ["user", "user"]
assert messages == messages_before

View file

@ -24,6 +24,7 @@ from litellm.llms.openai.openai import (
AsyncAssistantEventHandler,
AsyncCursorPage,
MessageData,
OpenAIBatchesAPI,
OpenAIChatCompletion,
OpenAIConfig,
OpenAIMessage as Message,
@ -2320,3 +2321,43 @@ def test_openai_config_hoists_a_later_developer_message_only_for_a_model_off_ope
model_info=None,
)
assert [message["role"] for message in messages] == expected_roles
@pytest.mark.asyncio
async def test_sync_batch_list_given_an_async_client_returns_a_page_the_caller_can_await(
http_backend: ModuleType,
):
def respond(request: _Request) -> httpx.Response | SDKResponse:
assert str(request.url).startswith("http://vllm-router.test/v1/batches")
return http_backend.Response(
200,
json={
"object": "list",
"data": [
{
"id": "batch_5f8dac",
"status": "pending",
"input_file_id": "file-b73f4c",
"created_at": 1793239207,
"endpoint": "/v1/chat/completions",
"completion_window": "24h",
}
],
"first_id": "batch_5f8dac",
"last_id": "batch_5f8dac",
"has_more": False,
},
)
async with http_backend.AsyncClient(transport=http_backend.MockTransport(respond)) as http_client:
page: Final = OpenAIBatchesAPI().list_batches(
_is_async=False,
api_key="sk-test",
api_base="http://vllm-router.test/v1",
timeout=5.0,
max_retries=0,
organization=None,
client=AsyncOpenAI(api_key="sk-test", base_url="http://vllm-router.test/v1", http_client=http_client),
)
assert [(batch.id, batch.status) for batch in (await page).data] == [("batch_5f8dac", "pending")]

View file

@ -1,5 +1,8 @@
import copy
import json
from collections.abc import Iterator
from itertools import chain
from typing import Final
from unittest.mock import MagicMock
@ -874,6 +877,58 @@ def test_openrouter_streaming_reasoning_is_preserved() -> None:
assert assembled.choices[0].message.reasoning_content == "Consider the evidence. Apply the rule."
def _five_message_level_cache_hints() -> list[dict[str, object]]:
return [
{"role": "system", "content": "system prompt", "cache_control": {"type": "ephemeral"}},
{"role": "user", "content": "turn 1", "cache_control": {"type": "ephemeral"}},
{"role": "assistant", "content": "answer 1", "cache_control": {"type": "ephemeral"}},
{"role": "user", "content": "turn 2", "cache_control": {"type": "ephemeral"}},
{"role": "assistant", "content": "answer 2"},
{"role": "user", "content": "turn 3", "cache_control": {"type": "ephemeral"}},
]
def _cached_texts(node: object) -> Iterator[str]:
if isinstance(node, dict):
yield from ((node["text"],) if "cache_control" in node and "text" in node else ())
yield from chain.from_iterable(map(_cached_texts, node.values()))
elif isinstance(node, list):
yield from chain.from_iterable(map(_cached_texts, node))
@pytest.mark.parametrize(
("drop_params", "expected_cached_texts"),
[
(True, ["answer 1", "turn 1", "turn 2", "turn 3"]),
("true", ["answer 1", "turn 1", "turn 2", "turn 3"]),
(False, ["answer 1", "system prompt", "turn 1", "turn 2", "turn 3"]),
],
ids=[
"drop_params_keeps_the_newest_four",
"string_drop_params_keeps_the_newest_four",
"without_drop_params_all_five_pass_through",
],
)
def test_openrouter_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params(
drop_params: bool | str, expected_cached_texts: list[str], monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
messages: Final = _five_message_level_cache_hints()
messages_before: Final = copy.deepcopy(messages)
body: Final = OpenrouterConfig().transform_request(
model="anthropic/claude-haiku-5.5",
messages=messages,
optional_params={},
litellm_params={"drop_params": drop_params, "custom_llm_provider": "openrouter"},
headers={},
)
assert sorted(_cached_texts(body["messages"])) == expected_cached_texts
assert json.dumps(body).count('"cache_control"') == len(expected_cached_texts)
assert messages == messages_before
def _transform_openrouter_response(body: dict[str, object]) -> litellm.ModelResponse:
return OpenrouterConfig().transform_response(
model="openrouter/openai/gpt-4o-mini",

View file

@ -1,3 +1,4 @@
import json
from typing import Final, Literal
from unittest.mock import AsyncMock, MagicMock, patch
@ -2507,3 +2508,46 @@ def test_check_and_create_cache_clamps_negative_creation_tokens() -> None:
assert cache_name == "cachedContents/created"
assert logging_obj.vertex_ai_cache_creation_input_tokens == 0
def _recording_gemini_client(calls: list[httpx.Request]) -> HTTPHandler:
def route(request: httpx.Request) -> httpx.Response:
calls.append(request)
if request.url.path.endswith("/cachedContents"):
return httpx.Response(200, json={} if request.method == "GET" else _created_cache_body(3000))
return httpx.Response(
200,
json={
"candidates": [{"content": {"role": "model", "parts": [{"text": "OK"}]}, "finishReason": "STOP"}],
"usageMetadata": {"promptTokenCount": 3000, "candidatesTokenCount": 1, "totalTokenCount": 3001},
},
)
return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(route)))
def test_a_plain_system_prompt_goes_into_the_gemini_cache_instead_of_beside_it(local_model_cost_map) -> None:
calls: Final[list[httpx.Request]] = []
litellm.completion(
model="gemini/gemini-flash-latest",
messages=[
{"role": "system", "content": "You are terse."},
{
"role": "user",
"content": [
{"type": "text", "text": "plain system prefix " * 3000, "cache_control": {"type": "ephemeral"}},
{"type": "text", "text": "Reply with OK only."},
],
},
],
api_key="test-key",
client=_recording_gemini_client(calls),
)
created: Final = next(
json.loads(call.content) for call in calls if call.method == "POST" and call.url.path.endswith("/cachedContents")
)
generated: Final = next(json.loads(call.content) for call in calls if call.url.path.endswith(":generateContent"))
assert created["system_instruction"] == {"parts": [{"text": "You are terse."}]}
assert generated["cachedContent"] == "cachedContents/created"
assert "system_instruction" not in generated

View file

@ -1,4 +1,5 @@
import asyncio
import copy
import importlib
import json
from typing import Final, Optional, cast
@ -10,6 +11,7 @@ import pytest
import litellm
from litellm import completion, embedding
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.watsonx.chat.transformation import IBMWatsonXChatConfig
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
@ -542,3 +544,43 @@ def test_watsonx_deployment_space_id_embedding(monkeypatch, watsonx_embedding_ca
# nor space_id or project_id is required by wx.ai API when inferencing deployment
assert "project_id" not in json_data and "space_id" not in json_data
def _strict_weather_tool() -> dict[str, object]:
return {
"type": "function",
"function": {
"name": "get_weather",
"strict": True,
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
"additionalProperties": False,
},
},
}
def test_watsonx_chat_tools_are_rewritten_without_touching_the_callers_tool() -> None:
tools: Final = [_strict_weather_tool()]
tools_before: Final = copy.deepcopy(tools)
optional_params: Final = IBMWatsonXChatConfig().map_openai_params(
non_default_params={"tools": tools}, optional_params={}, model="ibm/granite-4-h-small", drop_params=False
)
assert optional_params["tools"] == [
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
}
]
assert tools == tools_before

View file

@ -1,3 +1,4 @@
import copy
import json
import os
from typing import Final
@ -581,3 +582,44 @@ def test_xai_grok_4_frequency_penalty_not_supported(model):
"""
supported_params = XAIChatConfig().get_supported_openai_params(model=model)
assert "frequency_penalty" not in supported_params
def _strict_weather_tool() -> dict[str, object]:
return {
"type": "function",
"function": {
"name": "get_weather",
"strict": True,
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
"additionalProperties": False,
},
},
}
def test_xai_tools_are_rewritten_without_touching_the_callers_tool() -> None:
tools: Final = [_strict_weather_tool()]
tools_before: Final = copy.deepcopy(tools)
optional_params: Final = XAIChatConfig().map_openai_params(
non_default_params={"tools": tools}, optional_params={}, model="grok-4.3", drop_params=False
)
assert optional_params["tools"] == [
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
"additionalProperties": False,
},
},
}
]
assert tools == tools_before

View file

@ -31,7 +31,7 @@ from starlette.datastructures import UploadFile as StarletteUploadFile
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.caching.dual_cache import DualCache
from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS
from litellm.constants import ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH, ANTHROPIC_MAX_USER_ID_LENGTH, DEFAULT_REQUEST_TIMEOUT_SECONDS
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.prometheus import PrometheusLogger
@ -77,6 +77,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
EndpointType,
PassthroughStandardLoggingPayload,
WebSocketSessionUsage,
)
from tests._master_key import MASTER_KEY
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
@ -6620,6 +6621,272 @@ async def test_pass_through_body_stays_byte_identical_outside_the_opt_in(
assert "x-litellm-call-id" in response.headers
def _echo_upstream_body(upstream_request: httpx.Request) -> httpx.Response:
return httpx.Response(
200, content=upstream_request.content, headers={"content-type": "application/json"}, request=upstream_request
)
async def _relayed_body(response: Response) -> bytes:
if isinstance(response, StreamingResponse):
return b"".join([chunk async for chunk in response.body_iterator])
return bytes(response.body)
async def _relay_through_pass_through(
client_body: Mapping[str, object],
target: str,
guardrails_config: dict[str, object] | None,
custom_body: dict[str, object] | None = None,
) -> Mapping[str, object]:
async def receive() -> Mapping[str, object]:
return {"type": "http.request", "body": json.dumps(dict(client_body)).encode()}
request: Final = Request(
{
"type": "http",
"method": "POST",
"path": "/my-claude-route",
"query_string": b"",
"headers": [(b"content-type", b"application/json")],
},
receive,
)
fake_client, cleanup = _inject_fake_passthrough_client(httpx.MockTransport(_echo_upstream_body), timeout=None)
try:
with ExitStack() as stack:
_enter_relay_logging_mocks(stack, {})
stack.enter_context(
patch("litellm.proxy.proxy_server.proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache()))
)
response: Final = await pass_through_request(
request=request,
target=target,
custom_headers={},
user_api_key_dict=UserAPIKeyAuth(api_key="sk-relay-test"),
custom_body=custom_body,
guardrails_config=guardrails_config,
)
relayed: Final = await _relayed_body(response)
finally:
cleanup()
await fake_client.aclose()
assert response.status_code == 200
return MappingProxyType(json.loads(relayed))
_MESSAGES_BODY: Final = MappingProxyType(
{"model": "claude-sonnet-4-5", "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]}
)
_ANTHROPIC_MESSAGES: Final = "https://api.anthropic.com/v1/messages"
@pytest.fixture
def no_configured_anthropic_base(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False)
monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False)
@pytest.mark.asyncio
@pytest.mark.usefixtures("no_configured_anthropic_base")
@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"])
@pytest.mark.parametrize(
"client_metadata, target, forwarded_metadata",
[
({"user_id": "user-1"}, _ANTHROPIC_MESSAGES, {"user_id": "user-1"}),
({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/", {"user_id": "user-1"}),
({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}?beta=true", {"user_id": "user-1"}),
(
{"user_id": "user-1", "tags": ["a"], "trace_id": "t", "generation_name": "g", "foo": "bar"},
_ANTHROPIC_MESSAGES,
{"user_id": "user-1"},
),
({"user_id": "1234567890"}, _ANTHROPIC_MESSAGES, {"user_id": "1234567890"}),
({"tags": ["a"], "foo": "bar"}, _ANTHROPIC_MESSAGES, None),
({"user_id": "a@example.com"}, _ANTHROPIC_MESSAGES, None),
({"user_id": "x" * (ANTHROPIC_MAX_USER_ID_LENGTH + 1)}, _ANTHROPIC_MESSAGES, None),
({"user_id": 42}, _ANTHROPIC_MESSAGES, None),
({"user_id": "line one\nline two"}, _ANTHROPIC_MESSAGES, None),
({"user_id": "a\x1f@example.com"}, _ANTHROPIC_MESSAGES, None),
({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/count_tokens", None),
({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/batches", None),
({"user_id": "user-1"}, "https://api.anthropic.com/v1/models", None),
({"user_id": "user-1"}, "https://other-vendor.test/anthropic/v1/messages", None),
({"user_id": "user-1"}, "https://other-vendor.test/v1/responses", None),
({"user_id": "user-1"}, "https://other-vendor.test/v1/chat/completions", None),
],
ids=[
"messages",
"messages-trailing-slash",
"messages-with-query",
"user-id-beside-litellm-and-unknown-keys",
"digits-user-id",
"no-user-id",
"email-user-id",
"over-long-user-id",
"integer-user-id",
"newline-user-id",
"control-character-before-at-sign",
"count-tokens",
"batches",
"other-anthropic-path",
"other-vendor-messages",
"other-vendor-responses",
"other-vendor-chat-completions",
],
)
async def test_pass_through_forwards_only_an_acceptable_user_id_to_anthropic_messages(
stream: bool, client_metadata: object, target: str, forwarded_metadata: Mapping[str, object] | None
) -> None:
client_body: Final = MappingProxyType(
{**_MESSAGES_BODY, "metadata": client_metadata, **({"stream": True} if stream else {})}
)
body_without_metadata: Final = {k: v for k, v in client_body.items() if k != "metadata"}
relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None)
assert relayed == (
body_without_metadata if forwarded_metadata is None else {**body_without_metadata, "metadata": forwarded_metadata}
)
@pytest.mark.asyncio
@pytest.mark.usefixtures("no_configured_anthropic_base")
@pytest.mark.parametrize("base_env", ["ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"])
@pytest.mark.parametrize(
"target, forwarded",
[
("http://gateway.test:8080/anthropic/v1/messages", True),
("http://gateway.test:9090/v1/messages", False),
("http://other-gateway.test:8080/v1/messages", False),
],
ids=["configured-upstream", "other-port", "other-host"],
)
async def test_pass_through_treats_the_configured_anthropic_base_as_anthropic(
monkeypatch: pytest.MonkeyPatch, base_env: str, target: str, forwarded: bool
) -> None:
monkeypatch.setenv(base_env, "http://gateway.test:8080/anthropic")
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}})
relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None)
assert relayed == (client_body if forwarded else _MESSAGES_BODY)
@pytest.mark.asyncio
@pytest.mark.usefixtures("no_configured_anthropic_base")
@pytest.mark.parametrize(
"user_id_length, target, forwarded",
[
(ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH, "http://gateway.test/v1/messages", True),
(ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH + 1, "http://gateway.test/v1/messages", False),
(ANTHROPIC_MAX_USER_ID_LENGTH, _ANTHROPIC_MESSAGES, True),
],
ids=["at-the-configured-base-limit", "over-the-configured-base-limit", "direct-anthropic-keeps-its-limit"],
)
async def test_pass_through_caps_the_user_id_for_a_configured_anthropic_compatible_base(
monkeypatch: pytest.MonkeyPatch, user_id_length: int, target: str, forwarded: bool
) -> None:
monkeypatch.setenv("ANTHROPIC_API_BASE", "http://gateway.test")
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "u" * user_id_length}})
relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None)
assert relayed == (client_body if forwarded else _MESSAGES_BODY)
@pytest.mark.asyncio
@pytest.mark.usefixtures("no_configured_anthropic_base")
@pytest.mark.parametrize(
"default_on, event_hook, guardrails_config, client_fields, stream_scope, forwarded",
[
(False, GuardrailEventHooks.pre_call, None, {}, None, True),
(False, GuardrailEventHooks.pre_call, {"unregistered-guard": {}}, {}, None, True),
(True, GuardrailEventHooks.post_call, None, {}, None, True),
(True, GuardrailEventHooks.pre_call, None, {}, None, False),
(False, GuardrailEventHooks.pre_call, {"pii-guard": {}}, {}, None, False),
(False, GuardrailEventHooks.pre_call, None, {"litellm_metadata": {"guardrails": ["pii-guard"]}}, None, False),
(False, GuardrailEventHooks.pre_call, {"pii-guard": {}}, {"litellm_metadata": {"guardrails": []}}, None, False),
(True, GuardrailEventHooks.pre_call, None, {"stream": True}, "streaming", False),
(True, GuardrailEventHooks.pre_call, None, {}, "streaming", True),
(True, GuardrailEventHooks.pre_call, None, {"stream": True}, "non_streaming", True),
(True, GuardrailEventHooks.pre_call, None, {}, "non_streaming", False),
],
ids=[
"guardrail-not-requested",
"guardrail-not-registered",
"default-on-post-call-guardrail",
"default-on-pre-call-guardrail",
"guardrail-on-the-route",
"guardrail-in-litellm-metadata",
"route-guardrail-beside-an-empty-client-list",
"streaming-guardrail-on-a-stream",
"streaming-guardrail-off-a-stream",
"non-streaming-guardrail-on-a-stream",
"non-streaming-guardrail-off-a-stream",
],
)
async def test_pass_through_keeps_the_user_id_from_routes_a_guardrail_scans(
default_on: bool,
event_hook: GuardrailEventHooks,
guardrails_config: dict[str, object] | None,
client_fields: Mapping[str, object],
stream_scope: str | None,
forwarded: bool,
) -> None:
guardrail: Final = CustomGuardrail(
guardrail_name="pii-guard", default_on=default_on, event_hook=event_hook, stream_scope=stream_scope
)
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}, **client_fields})
body_without_client_fields: Final = {k: v for k, v in client_body.items() if k not in ("metadata", "litellm_metadata")}
with patch.object(litellm, "callbacks", [guardrail]):
relayed: Final = await _relay_through_pass_through(
client_body, _ANTHROPIC_MESSAGES, guardrails_config=guardrails_config
)
assert relayed == (
{**body_without_client_fields, "metadata": {"user_id": "user-1"}} if forwarded else body_without_client_fields
)
@pytest.mark.asyncio
@pytest.mark.usefixtures("no_configured_anthropic_base")
async def test_pass_through_warns_without_the_value_when_it_drops_a_refused_user_id(
caplog: pytest.LogCaptureFixture,
) -> None:
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "person@example.com"}})
with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name):
await _relay_through_pass_through(client_body, _ANTHROPIC_MESSAGES, guardrails_config=None)
assert "Not forwarding metadata.user_id" in caplog.text
assert "person@example.com" not in caplog.text
@pytest.mark.asyncio
@pytest.mark.usefixtures("no_configured_anthropic_base")
async def test_pass_through_drops_metadata_that_is_not_an_object() -> None:
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": "user-1"})
relayed: Final = await _relay_through_pass_through(client_body, _ANTHROPIC_MESSAGES, guardrails_config=None)
assert relayed == _MESSAGES_BODY
@pytest.mark.asyncio
@pytest.mark.usefixtures("no_configured_anthropic_base")
async def test_pass_through_custom_body_replaces_the_client_user_id() -> None:
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}})
relayed: Final = await _relay_through_pass_through(
client_body, _ANTHROPIC_MESSAGES, guardrails_config=None, custom_body=dict(_MESSAGES_BODY)
)
assert relayed == _MESSAGES_BODY
_PARTIAL_RELAY_WARNING_MARKER = "ended before upstream body was fully relayed"
@ -7736,6 +8003,102 @@ async def test_websocket_passthrough_sends_deepgram_audio_bytes_and_control_text
upstream_ws.close.assert_awaited_once()
@dataclass(frozen=True, slots=True)
class _LengthMeter:
rate: float
def measure(self, client_text: str) -> int:
return len(client_text)
def price(self, total: int) -> WebSocketSessionUsage:
return WebSocketSessionUsage(model="voice-1", custom_llm_provider="tts-provider", response_cost=total * self.rate)
@pytest.mark.asyncio
async def test_websocket_passthrough_logs_the_meters_price_for_the_client_text_it_delivered():
"""Text frames reach the upstream unchanged and are measured, binary frames are not, and the session is logged at
the model, provider and cost the meter prices for the total."""
frames = ("hello", b"\x00\x01binary", "world!")
upstream_ws = RecordingUpstreamWebSocket()
websocket = _client_websocket(
AsyncMock(
side_effect=[
*(
{"type": "websocket.receive", "bytes": frame}
if isinstance(frame, bytes)
else {"type": "websocket.receive", "text": frame}
for frame in frames
),
{"type": "websocket.disconnect"},
]
)
)
with (
_patched_websocket_passthrough_environment(upstream_ws),
patch( # test-quality-ok: pass_through_endpoint_logging is a module global read inside websocket_passthrough_request; there is no injection seam
"litellm.proxy.pass_through_endpoints.pass_through_endpoints."
"pass_through_endpoint_logging.pass_through_async_success_handler",
new=AsyncMock(),
) as success_handler,
):
await websocket_passthrough_request(
websocket=websocket,
target="wss://upstream.example/v1/stream",
custom_headers={},
user_api_key_dict=UserAPIKeyAuth(),
forward_headers=False,
endpoint="/provider/v1/stream",
accept_websocket=False,
meter=_LengthMeter(rate=0.5),
)
assert [call.args[0] for call in upstream_ws.send.await_args_list] == list(frames)
logging_obj = success_handler.call_args.kwargs["logging_obj"]
assert logging_obj.model == "voice-1"
assert logging_obj.model_call_details["model"] == "voice-1"
assert logging_obj.model_call_details["custom_llm_provider"] == "tts-provider"
assert logging_obj.model_call_details["response_cost"] == (len("hello") + len("world!")) * 0.5
@pytest.mark.asyncio
async def test_websocket_passthrough_meters_only_client_text_the_upstream_accepted():
upstream_ws = RecordingUpstreamWebSocket()
upstream_ws.send = AsyncMock(side_effect=[None, ConnectionError("upstream went away")])
websocket = _client_websocket(
AsyncMock(
side_effect=[
{"type": "websocket.receive", "text": "hello"},
{"type": "websocket.receive", "text": "never delivered"},
{"type": "websocket.disconnect"},
]
)
)
with (
_patched_websocket_passthrough_environment(upstream_ws),
patch( # test-quality-ok: pass_through_endpoint_logging is a module global read inside websocket_passthrough_request; there is no injection seam
"litellm.proxy.pass_through_endpoints.pass_through_endpoints."
"pass_through_endpoint_logging.pass_through_async_success_handler",
new=AsyncMock(),
) as success_handler,
):
await websocket_passthrough_request(
websocket=websocket,
target="wss://upstream.example/v1/stream",
custom_headers={},
user_api_key_dict=UserAPIKeyAuth(),
forward_headers=False,
endpoint="/provider/v1/stream",
accept_websocket=False,
meter=_LengthMeter(rate=0.5),
)
logging_obj = success_handler.call_args.kwargs["logging_obj"]
assert [call.args[0] for call in upstream_ws.send.await_args_list] == ["hello", "never delivered"]
assert logging_obj.model_call_details["response_cost"] == len("hello") * 0.5
@pytest.mark.asyncio
async def test_websocket_passthrough_vertex_live_setup_ack_names_the_model_but_is_not_billed_as_usage():
"""Vertex Live keeps its special first frame: the setup acknowledgement is forwarded verbatim, read for the

View file

@ -41,12 +41,13 @@ def _make_streaming_response(chunks):
return mock
def _unarmed_logging_obj():
def _unarmed_logging_obj(model: str = "unknown"):
"""Real Logging objects only carry _on_deferred_stream_complete when the
proxy arms deferred dispatch; a bare MagicMock's auto-attribute is truthy
and would spuriously trigger the deferral branch."""
obj = MagicMock()
obj._on_deferred_stream_complete = None
obj.model = model
return obj
@ -409,7 +410,7 @@ async def _collect_openai_passthrough_chunks(chunks, endpoint_type):
async for chunk in PassThroughStreamingHandler.chunk_processor(
response=response,
request_body={"model": "gpt-4o-mini", "stream": True},
litellm_logging_obj=_unarmed_logging_obj(),
litellm_logging_obj=_unarmed_logging_obj(model="gpt-4o-mini"),
endpoint_type=endpoint_type,
start_time=datetime(2026, 1, 1),
passthrough_success_handler_obj=MagicMock(),

View file

@ -346,6 +346,7 @@ async def test_vertex_ai_anthropic_streaming_cost_injection_enabled():
# Setup logging object with model info
litellm_logging_obj = MagicMock(spec=LiteLLMLoggingObj)
litellm_logging_obj.model = "claude-sonnet-4@20250514"
litellm_logging_obj.litellm_params = {}
litellm_logging_obj.model_call_details = {"model": "claude-sonnet-4@20250514"}
litellm_logging_obj.completion_start_time = None
@ -544,6 +545,7 @@ async def test_vertex_ai_anthropic_streaming_model_extraction():
response.aiter_bytes = mock_aiter_bytes
litellm_logging_obj = MagicMock(spec=LiteLLMLoggingObj)
litellm_logging_obj.model = "claude-sonnet-4@20250514"
litellm_logging_obj.litellm_params = {}
litellm_logging_obj.model_call_details = {}
litellm_logging_obj.completion_start_time = None

View file

@ -74,6 +74,7 @@ from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitErro
from litellm.proxy.common_utils.sse_keepalive import ANTHROPIC_PING_SSE_CHUNK
from litellm.proxy.dd_span_tagger import DDSpanTagger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.proxy._types import Litellm_EntityType, ProxyErrorTypes, ProxyException
from litellm.proxy._types import UserAPIKeyAuth as ProxyUserAPIKeyAuth
from litellm.proxy.auth.fallback_budget import RouterFallbackBudgetCheck
@ -10221,6 +10222,8 @@ class TestInjectCostIntoUsageDict:
discount streamed a ``usage.cost`` that disagreed with the callback's ``response_cost``."""
class _StubLoggingObj:
model = "claude-haiku-4-5"
def __init__(self, cost):
self._cost = cost
self.captured_result = None
@ -10257,6 +10260,8 @@ class TestInjectCostIntoUsageDict:
def test_message_delta_falls_back_to_model_pricing_when_the_logging_obj_returns_no_cost(self):
class _StubLoggingObj:
model = "claude-haiku-4-5"
def response_cost_calculator(self, result):
return None
@ -10282,6 +10287,8 @@ class TestInjectCostIntoUsageDict:
model-name pricing rather than propagating into the response body."""
class _StubLoggingObj:
model = "claude-haiku-4-5"
def response_cost_calculator(self, result):
raise ValueError("no pricing for this deployment")
@ -10363,6 +10370,8 @@ class TestInjectCostIntoUsageDict:
streaming /v1/chat/completions gets its negotiated price instead of sticker."""
class _StubLoggingObj:
model = "gpt-4o-mini"
def __init__(self, cost):
self._cost = cost
self.captured_result = None
@ -10391,6 +10400,8 @@ class TestInjectCostIntoUsageDict:
def test_openai_chunk_falls_back_to_model_pricing_when_the_logging_obj_returns_no_cost(self):
class _StubLoggingObj:
model = "gpt-4o-mini"
def response_cost_calculator(self, result):
return None
@ -10406,6 +10417,60 @@ class TestInjectCostIntoUsageDict:
assert result is not None
assert result["usage"]["cost"] == pytest.approx(self._expected_cost("gpt-4o-mini", 11, 4))
@pytest.mark.parametrize(
("deployment_model", "served_model", "custom_llm_provider"),
[
("vertex_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "vertex_ai"),
("anthropic/claude-sonnet-5-5", "claude-sonnet-5-5", "anthropic"),
("azure_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "azure_ai"),
],
)
def test_message_delta_bills_search_like_the_logged_response_on_a_token_priced_claude_deployment(
self, local_model_cost_map: None, deployment_model: str, served_model: str, custom_llm_provider: str
) -> None:
token_rates: Final = {"input_cost_per_token": 3e-6, "output_cost_per_token": 1.5e-5}
router: Final = Router(
model_list=[
{
"model_name": "priced-claude",
"litellm_params": {
"model": deployment_model,
"api_key": "k",
"vertex_project": "p",
"vertex_location": "global",
},
"model_info": token_rates,
}
]
)
deployment_id: Final = router.model_list[0]["model_info"]["id"]
logging_obj: Final = LiteLLMLoggingObj(
model=deployment_model,
messages=[{"role": "user", "content": "test"}],
stream=True,
call_type="anthropic_messages",
start_time=None,
litellm_call_id="token-priced-claude-search",
function_id="token-priced-claude-search",
)
logging_obj.update_environment_variables(
litellm_params={"metadata": {"model_info": {"id": deployment_id, **token_rates}}}, optional_params={}
)
logging_obj.model_call_details["custom_llm_provider"] = custom_llm_provider
wire_usage: Final = {"input_tokens": 1840, "output_tokens": 412, "server_tool_use": {"web_search_requests": 2}}
event: Final = {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": wire_usage}
result: Final = ProxyBaseLLMRequestProcessing._inject_cost_into_usage_dict(event, "priced-claude", logging_obj)
stream_usage: Final = AnthropicConfig().calculate_usage(usage_object=wire_usage, reasoning_content=None)
logged_cost: Final = logging_obj.response_cost_calculator(
result=litellm.ModelResponse(model=served_model, usage=stream_usage)
)
token_cost: Final = 1840 * token_rates["input_cost_per_token"] + 412 * token_rates["output_cost_per_token"]
assert logged_cost is not None and logged_cost > token_cost
assert result is not None
assert result["usage"]["cost"] == pytest.approx(logged_cost)
class TestProcessChunkWithCostInjection:
@pytest.mark.parametrize("model", ["claude-opus-5-5", "vertex_ai/claude-opus-5-5"])
@ -10472,6 +10537,8 @@ class TestProcessChunkWithCostInjection:
monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True)
class _StubLoggingObj:
model = "claude-haiku-4-5"
def response_cost_calculator(self, result):
return 0.00042

View file

@ -996,3 +996,21 @@ async def test_guardrails_only_and_skip_guardrails_are_mutually_exclusive(
guardrails_only=True,
skip_guardrails=True,
)
@pytest.mark.parametrize(
"request_data, expected",
[
({"metadata": {"_guardrail_pipelines": ["p1"]}}, True),
({"metadata": {"user_id": "u"}, "litellm_metadata": {"_guardrail_pipelines": ["p1"]}}, True),
({"metadata": {"_guardrail_pipelines": 5}}, True),
({"metadata": {"user_id": "u"}}, False),
],
ids=["pipeline-in-metadata", "pipeline-in-litellm-metadata", "non-list-pipeline-value", "no-pipeline"],
)
def test_request_has_pre_call_guardrails_reads_pipelines_from_both_metadata_buckets(
proxy_logging, monkeypatch, request_data, expected
):
monkeypatch.setattr(litellm, "callbacks", [])
assert proxy_logging.request_has_pre_call_guardrails(request_data) is expected

View file

@ -6315,6 +6315,17 @@ def test_marks_past_the_provider_cap_keep_the_newest_under_drop_params(drop_para
assert messages[1] == {"role": "user", "content": "turn 0"}
def test_marks_past_the_provider_cap_inside_one_message_keep_the_newest_under_drop_params():
parts: Final = tuple(
{"type": "input_text", "text": f"part {index}", "cache_control": _EPHEMERAL} for index in range(1, 6)
)
messages: Final = _bridged_request({"role": "user", "content": list(parts)}, drop_params=True)
content: Final = TypeAdapter(list[dict[str, object]]).validate_python(messages[-1]["content"])
assert ["cache_control" in part for part in content] == [False, True, True, True, True]
def test_global_drop_params_keeps_the_newest_marks(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "drop_params", True)

View file

@ -607,6 +607,23 @@ def test_openai_file_object_accepts_pending_status():
assert file_obj.status == "pending"
@pytest.mark.parametrize("object_type", ({}, {"object": None}), ids=["missing", "null"])
def test_openai_file_object_without_an_object_type_validates_as_a_file(object_type: dict[str, None]) -> None:
from litellm.types.llms.openai import OpenAIFileObject
server_file: Final = {
"id": "file-b73f4c",
"bytes": 156,
"created_at": 1793238707,
"filename": "batch_requests.jsonl",
"purpose": "batch",
}
file_obj: Final = OpenAIFileObject.model_validate({**server_file, **object_type})
assert file_obj.model_dump(exclude_none=True) == {**server_file, "object": "file"}
class TestOpenAIFileObjectBatchGuardrailSerialization:
"""The proxy-only `litellm_batch_guardrail` key must reach the wire only when something set it."""

View file

@ -2037,6 +2037,13 @@
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@isaacs/nonrecursive-glob-match": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.0.2.tgz",
"integrity": "sha512-GJPHIgPPtFl+nLhFHyF0zZoFcOFLsZWbHFFzo+Zr+PWDXJbl9hHxb2w+9SiF2fHmLgKaXIpUzQo3SbBBZ9WObw==",
"dev": true,
"license": "BlueOak-1.0.0"
},
"node_modules/@jridgewell/gen-mapping": {
"version": "0.3.13",
"resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz",
@ -9657,13 +9664,14 @@
}
},
"node_modules/minimatch": {
"version": "10.2.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz",
"integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==",
"version": "11.0.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.2.tgz",
"integrity": "sha512-pWTgz5CN3tjqyZNSK2tePlSmu9IwM41q8yn8Usdqqwe9TIHh+9dEpLLjF4uApsfPEAZY71qtSiA2B2APEr+1Yw==",
"dev": true,
"license": "BlueOak-1.0.0",
"dependencies": {
"brace-expansion": "^5.0.2"
"@isaacs/nonrecursive-glob-match": "^1.0.2",
"brace-expansion": "^5.0.12"
},
"engines": {
"node": "18 || 20 || >=22"

View file

@ -117,7 +117,7 @@
"js-yaml": "4.3.2",
"brace-expansion": "5.0.12",
"glob": "13.0.0",
"minimatch": "10.2.4",
"minimatch": "11.0.2",
"ws": "8.21.0",
"braces": "3.0.3",
"axios": "1.13.6",

13
uv.lock generated
View file

@ -16,6 +16,7 @@ exclude-newer-span = "P3D"
[options.exclude-newer-package]
brotli = "2026-10-22T14:01:00Z"
pypdf = "2026-10-28T17:01:00Z"
[manifest]
members = [
@ -8372,11 +8373,11 @@ wheels = [
[[package]]
name = "pyasn1"
version = "0.6.4"
version = "0.6.5"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" }
sdist = { url = "https://files.pythonhosted.org/packages/88/7e/3c0da41061a2021cac7cd524fd57092adbe5b515e8bf9005702e366802e2/pyasn1-0.6.5.tar.gz", hash = "sha256:6b11616a6bcf3a40e635f082bb232fa57b5e0dd7f517bd7ddd69ec3794508a47", size = 155152, upload-time = "2026-10-27T06:32:49.125Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" },
{ url = "https://files.pythonhosted.org/packages/19/1f/3eefd5c84c35cae44549ce1c7f7e58f7945b34f5a6bce07a135438522073/pyasn1-0.6.5-py3-none-any.whl", hash = "sha256:4017b0fce125096b8573b39c989fdb1a4b19cf761e683398184550fea0e66587", size = 84807, upload-time = "2026-10-27T06:32:48.045Z" },
]
[[package]]
@ -8869,14 +8870,14 @@ wheels = [
[[package]]
name = "pypdf"
version = "6.21.2"
version = "6.21.4"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/06/28/081f1cbcca7042441dafc8de00f48ba60853f89e5bff92c5f26477346d81/pypdf-6.21.2.tar.gz", hash = "sha256:0a3e1c5c886a1be03646c16a04827cc166c2a0575c971b26a4bf89686551c2d4", size = 7144571, upload-time = "2026-10-21T16:44:55.272Z" }
sdist = { url = "https://files.pythonhosted.org/packages/1e/e1/ec8f4d155c997c45917153a5521e60d6ee6389d68abdcbc8e44e564caffc/pypdf-6.21.4.tar.gz", hash = "sha256:c8817e3f888d47a087872ded69812c1ef30cacb871d6e053e0c00bbaf0bc2163", size = 7109255, upload-time = "2026-10-28T17:00:54.81Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/1b/4d/c9d2d67f204eff463237a1a8f5e25a6923b7b1043982b91fcaf104c8ef60/pypdf-6.21.2-py3-none-any.whl", hash = "sha256:1d44d1a3a200f25062adff0085b02040e4646431ac2ccac22e42679e91466084", size = 407766, upload-time = "2026-10-21T16:44:53.626Z" },
{ url = "https://files.pythonhosted.org/packages/3f/4b/77e8ad4ac42c9d93600f40541a50e9feb3a61d2bbee2d76f659346072051/pypdf-6.21.4-py3-none-any.whl", hash = "sha256:33e4fb41ca352810eb4e9838aa0d82d6888a5f8bf682f43e99cce5faa666255c", size = 410430, upload-time = "2026-10-28T17:00:53.146Z" },
]
[[package]]

View file

@ -644,6 +644,13 @@
"node": ">=18"
}
},
"node_modules/@isaacs/nonrecursive-glob-match": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.0.2.tgz",
"integrity": "sha512-GJPHIgPPtFl+nLhFHyF0zZoFcOFLsZWbHFFzo+Zr+PWDXJbl9hHxb2w+9SiF2fHmLgKaXIpUzQo3SbBBZ9WObw==",
"dev": true,
"license": "BlueOak-1.0.0"
},
"node_modules/@jridgewell/sourcemap-codec": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz",
@ -2700,13 +2707,14 @@
}
},
"node_modules/minimatch": {
"version": "10.2.6",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz",
"integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==",
"version": "11.0.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.2.tgz",
"integrity": "sha512-pWTgz5CN3tjqyZNSK2tePlSmu9IwM41q8yn8Usdqqwe9TIHh+9dEpLLjF4uApsfPEAZY71qtSiA2B2APEr+1Yw==",
"dev": true,
"license": "BlueOak-1.0.0",
"dependencies": {
"brace-expansion": "^5.0.8"
"@isaacs/nonrecursive-glob-match": "^1.0.2",
"brace-expansion": "^5.0.12"
},
"engines": {
"node": "18 || 20 || >=22"

View file

@ -83,5 +83,8 @@
"esbuild": "^0.28.2",
"typescript": "^5.9.3",
"vitest": "^4.1.12"
},
"overrides": {
"minimatch": "11.0.2"
}
}