Merge remote-tracking branch 'origin/main' into litellm_gemini_tool_call_id_length

# Conflicts:
#	litellm/responses/litellm_completion_transformation/transformation.py
This commit is contained in:
mateo-berri 2026-10-31 02:01:07 -07:00
commit 0cf5d818db
126 changed files with 8055 additions and 662 deletions

View file

@ -177,6 +177,8 @@ MID_STREAM_FAILURE_LOG_WAIT_SECONDS: Final = 1.0
# Set to 0 to disable truncation.
MAX_BASE64_LENGTH_FOR_LOGGING: Final = get_env_int("MAX_BASE64_LENGTH_FOR_LOGGING", 64)
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS: Final = 256 * 1024
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS: Final = 256
BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS: Final = 8 * 1024 * 1024
REDACTED_BY_LITELLM: Final = "redacted-by-litellm"
# in-memory stand-in handed to provider converters for redacted arguments; never stored
REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}"
@ -634,6 +636,8 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60
BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024)
# Anthropic's Messages API rejects thinking.budget_tokens < 1024.
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024
ANTHROPIC_MAX_USER_ID_LENGTH: Final = 512
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH: Final = 255
ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:"
REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5)
DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000)
@ -1163,6 +1167,7 @@ openai_compatible_providers: Final[list[str]] = [
"cortecs",
"scx-ai",
"prism",
"hostinger_router",
"sail",
"reka",
"streamlake",

View file

@ -3,10 +3,10 @@ from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field, replace
from functools import partial
from types import MappingProxyType
from typing import Final, Literal, TypeAlias
from typing import Annotated, Final, Literal, TypeAlias
import httpx
from pydantic import BaseModel, ConfigDict, StrictBool, TypeAdapter, ValidationError
from pydantic import BaseModel, BeforeValidator, ConfigDict, TypeAdapter, ValidationError
from pydantic_core import to_jsonable_python
from typing_extensions import assert_never
@ -23,6 +23,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.decisions.transformation import (
BaseDecisionsConfig,
DecisionsResponseValidationError,
catalog_mode,
image_media_type,
image_param,
input_images,
@ -61,7 +62,7 @@ from litellm.types.decisions import (
UnsupportedDecisionsRequest,
)
from litellm.types.llms.openai import ChatCompletionUserMessage
from litellm.types.utils import LlmProviders, ProviderSpecificHeader
from litellm.types.utils import LlmProviders, ProviderSpecificHeader, is_decisions_model_mode
from litellm.utils import ProviderConfigManager, get_model_cost_key
SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]]
@ -257,19 +258,105 @@ def _request_safety_identifier(
return None
def _fields_without_unsupported_media(
fields: RequestFields,
def _bool_or_none(value: object) -> bool | None:
return value if isinstance(value, bool) else None
def _str_or_none(value: object) -> str | None:
return value if isinstance(value, str) else None
_DeploymentFlag: TypeAlias = Annotated[bool | None, BeforeValidator(_bool_or_none)]
class _DeploymentCapabilities(BaseModel):
model_config = ConfigDict(frozen=True, from_attributes=True)
mode: Annotated[str | None, BeforeValidator(_str_or_none)] = None
supports_box_questions: _DeploymentFlag = None
supports_audio_input: _DeploymentFlag = None
supports_video_input: _DeploymentFlag = None
def decisions_media_flag(self, flag: bool | None, model: str, custom_llm_provider: str) -> bool | None:
if flag is None:
return None
mode: Final = self.mode if self.mode is not None else catalog_mode(model, custom_llm_provider)
return flag if mode is None or is_decisions_model_mode(mode) else None
@dataclass(frozen=True, slots=True)
class _ResolvedSupport:
box_questions: bool
audio: bool
videos: bool
def _takes_audio(
provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities
) -> bool:
return provider_config.supports_audio_input(
model,
custom_llm_provider,
deployment.decisions_media_flag(deployment.supports_audio_input, model, custom_llm_provider),
)
def _takes_videos(
provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities
) -> bool:
return provider_config.supports_video_input(
model,
custom_llm_provider,
deployment.decisions_media_flag(deployment.supports_video_input, model, custom_llm_provider),
)
def deployment_refuses_media(
model: str,
custom_llm_provider: str,
model_info: Mapping[str, object],
call_kwargs: Mapping[str, object],
*,
audio: bool,
videos: bool,
) -> bool:
if _drops_params(call_kwargs):
return False
provider_config: Final = get_decisions_provider_config(model, custom_llm_provider)
canonical_model: Final = provider_config.canonical_model(model)
deployment: Final = _DeploymentCapabilities.model_validate(model_info)
return (audio and not _takes_audio(provider_config, canonical_model, custom_llm_provider, deployment)) or (
videos and not _takes_videos(provider_config, canonical_model, custom_llm_provider, deployment)
)
def _resolved_support(
provider_config: BaseDecisionsConfig,
model: str,
custom_llm_provider: str,
headers: Mapping[str, str],
kwargs: Mapping[str, object],
fields: RequestFields,
) -> _ResolvedSupport:
deployment: Final = _DeploymentCapabilities.model_validate(kwargs.get("model_info") or {})
return _ResolvedSupport(
box_questions=provider_config.supports_box_questions(
model, custom_llm_provider, headers, deployment.supports_box_questions
),
audio=bool(fields.audio) and _takes_audio(provider_config, model, custom_llm_provider, deployment),
videos=bool(fields.videos) and _takes_videos(provider_config, model, custom_llm_provider, deployment),
)
def _fields_without_unsupported_media(
fields: RequestFields, support: _ResolvedSupport, kwargs: Mapping[str, object]
) -> RequestFields:
if not _drops_params(kwargs):
return fields
return replace(
fields,
audio=fields.audio if provider_config.supports_audio_input(model, custom_llm_provider) else None,
videos=fields.videos if provider_config.supports_video_input(model, custom_llm_provider) else None,
audio=fields.audio if support.audio else None,
videos=fields.videos if support.videos else None,
)
@ -292,10 +379,8 @@ class _MalformedImages:
def _unsupported_params(
ir_request: DecisionsIRRequest,
provider_config: BaseDecisionsConfig,
model: str,
custom_llm_provider: str,
headers: Mapping[str, str],
supports_box_questions: bool,
support: _ResolvedSupport,
) -> tuple[str, ...]:
refuses_safety_identifier: Final = (
ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier
@ -304,13 +389,9 @@ def _unsupported_params(
None if provider_config.supports_images_in(ir_request.input, headers) else image_param(ir_request.input)
)
refuses_assets: Final = bool(ir_request.assets) and not provider_config.supports_assets(headers)
refuses_box_questions: Final = _has_box_question(ir_request.questions) and not supports_box_questions
refuses_audio: Final = bool(ir_request.audio) and not provider_config.supports_audio_input(
model, custom_llm_provider
)
refuses_videos: Final = bool(ir_request.videos) and not provider_config.supports_video_input(
model, custom_llm_provider
)
refuses_box_questions: Final = _has_box_question(ir_request.questions) and not support.box_questions
refuses_audio: Final = bool(ir_request.audio) and not support.audio
refuses_videos: Final = bool(ir_request.videos) and not support.videos
return (
*(("safety_identifier",) if refuses_safety_identifier else ()),
*(() if refused_image_param is None else (refused_image_param,)),
@ -333,7 +414,7 @@ def _without_unsupported_params(
ir_request: DecisionsIRRequest,
provider_config: BaseDecisionsConfig,
headers: Mapping[str, str],
supports_box_questions: bool,
support: _ResolvedSupport,
) -> DecisionsIRRequest:
return replace(
ir_request,
@ -345,9 +426,7 @@ def _without_unsupported_params(
safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None,
assets=ir_request.assets if provider_config.supports_assets(headers) else MappingProxyType({}),
questions=(
ir_request.questions
if supports_box_questions
else tuple(filter(_is_not_box_question, ir_request.questions))
ir_request.questions if support.box_questions else tuple(filter(_is_not_box_question, ir_request.questions))
),
)
@ -368,39 +447,20 @@ def _without_malformed_images(
return well_formed
class _DeploymentBoxQuestions(BaseModel):
model_config = ConfigDict(frozen=True, from_attributes=True)
supports_box_questions: StrictBool | None = None
def _deployment_supports_box_questions(kwargs: Mapping[str, object]) -> bool | None:
try:
return _DeploymentBoxQuestions.model_validate(kwargs.get("model_info") or {}).supports_box_questions
except ValidationError:
return None
def _provider_ir_request(
ir_request: DecisionsIRRequest,
*,
provider_config: BaseDecisionsConfig,
model: str,
custom_llm_provider: str,
headers: Mapping[str, str],
support: _ResolvedSupport,
kwargs: Mapping[str, object],
) -> DecisionsIRRequest | _UnsupportedParams | _NothingLeftAfterDrop:
supports_box_questions: Final = provider_config.supports_box_questions(
model, custom_llm_provider, headers, _deployment_supports_box_questions(kwargs)
)
unsupported: Final = _unsupported_params(
ir_request, provider_config, model, custom_llm_provider, headers, supports_box_questions
)
unsupported: Final = _unsupported_params(ir_request, provider_config, headers, support)
if not unsupported:
return ir_request
if not _drops_params(kwargs):
return _UnsupportedParams(names=unsupported)
supported: Final = _without_unsupported_params(ir_request, provider_config, headers, supports_box_questions)
supported: Final = _without_unsupported_params(ir_request, provider_config, headers, support)
if _leaves_no_input(supported.input):
return _NothingLeftAfterDrop(names=unsupported, missing="input to decide on")
if not supported.questions:
@ -479,10 +539,11 @@ def prepare_call(
raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error
try:
headers: Final = decisions_outbound_headers(extra_headers, provider, kwargs)
support: Final = _resolved_support(provider_config, canonical_model, provider, headers, kwargs, fields)
request_safety_identifier: Final = _request_safety_identifier(safety_identifier, provider_config, kwargs)
request: Final = _fields_without_unsupported_media(
fields, provider_config, canonical_model, provider, kwargs
).validate(safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs))
request: Final = _fields_without_unsupported_media(fields, support, kwargs).validate(
safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs)
)
except ValidationError as error:
raise litellm.BadRequestError(
message=f"Invalid Decisions request: {error}",
@ -523,12 +584,7 @@ def prepare_call(
llm_provider=provider,
)
ir_request: Final = _provider_ir_request(
caller_ir_request,
provider_config=provider_config,
model=canonical_model,
custom_llm_provider=provider,
headers=headers,
kwargs=kwargs,
caller_ir_request, provider_config=provider_config, headers=headers, support=support, kwargs=kwargs
)
if isinstance(ir_request, _UnsupportedParams):
raise litellm.UnsupportedParamsError(

View file

@ -14,7 +14,7 @@ import itertools
import os
import re
from collections.abc import Callable, Container, Iterable, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast
from urllib.parse import urlparse
from pydantic import TypeAdapter, ValidationError
@ -102,6 +102,192 @@ def _validated_object_list(value: object) -> list[object] | None:
return None
_MessageT: Final = TypeVar("_MessageT")
_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None]
_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls")
def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]:
lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS)
return tuple(
itertools.chain.from_iterable(
_OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list)
)
)
def _block_carries_cache_control(block: object) -> bool:
return isinstance(block, dict) and _CONTENT_PART_ADAPTER.validate_python(block).get("cache_control") is not None
_ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT: Final = frozenset({"tool", "function"})
def _message_mark_reaches_provider(fields: Mapping[str, object]) -> bool:
if fields.get("cache_control") is None:
return False
return (
not isinstance(fields.get("content"), list)
or fields.get("role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT
)
def cache_control_count(message: object) -> int:
if not isinstance(message, dict):
return 0
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block))
return int(_message_mark_reaches_provider(fields)) + block_marks
def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]:
others: Final = {key: value for key, value in fields.items() if key != "cache_control"}
if "cache_control" not in fields:
return others
cache_control: Final = rewrite(fields["cache_control"])
return others if cache_control is None else {**others, "cache_control": cache_control}
def block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object:
if not isinstance(block, dict):
return block
return _fields_with_cache_control(_CONTENT_PART_ADAPTER.validate_python(block), rewrite)
def with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT:
if not isinstance(message, dict):
return message
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
blocks_rewritten: Final = {
key: (
[block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)]
if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list)
else value
)
for key, value in fields.items()
}
return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped
_MessageT, _fields_with_cache_control(blocks_rewritten, rewrite)
)
def dropped_cache_control(_: object) -> None:
return None
def _first_marked_index(value: object) -> int | None:
if not isinstance(value, list):
return None
blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value)
return next((index for index, block in enumerate(blocks) if _block_carries_cache_control(block)), None)
_MarkSlot: TypeAlias = tuple[str, int]
_MESSAGE_MARK_SLOT: Final[_MarkSlot] = ("cache_control", -1)
def _block_mark_slots(fields: Mapping[str, object], key: str) -> tuple[_MarkSlot, ...]:
value: Final = fields.get(key)
if not isinstance(value, list):
return ()
blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value)
return tuple((key, index) for index, block in enumerate(blocks) if _block_carries_cache_control(block))
def _mark_slots_in_wire_order(fields: Mapping[str, object]) -> tuple[_MarkSlot, ...]:
content_slots: Final = _block_mark_slots(fields, "content")
tool_call_slots: Final = _block_mark_slots(fields, "tool_calls")
if not _message_mark_reaches_provider(fields):
return content_slots + tool_call_slots
if isinstance(fields.get("content"), list):
return content_slots + (_MESSAGE_MARK_SLOT,) + tool_call_slots
return (_MESSAGE_MARK_SLOT,) + content_slots + tool_call_slots
def _blocks_without_marks(key: str, value: object, dropped_slots: frozenset[_MarkSlot]) -> object:
if key not in _MARK_CARRYING_MESSAGE_FIELDS or not isinstance(value, list):
return value
return [
block_with_cache_control(block, dropped_cache_control) if (key, index) in dropped_slots else block
for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(value))
]
def _without_oldest_marks(message: _MessageT, dropped: int) -> _MessageT:
if dropped == 0 or not isinstance(message, dict):
return message
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
dropped_slots: Final = frozenset(_mark_slots_in_wire_order(fields)[:dropped])
blocks_unmarked: Final = {key: _blocks_without_marks(key, value, dropped_slots) for key, value in fields.items()}
return cast( # cast-ok: the same message with its oldest wire-order marks dropped
_MessageT,
_fields_with_cache_control(blocks_unmarked, dropped_cache_control)
if _MESSAGE_MARK_SLOT in dropped_slots
else blocks_unmarked,
)
def tool_message_with_single_cache_mark(message: _MessageT) -> _MessageT:
if not isinstance(message, dict):
return message
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
content: Final = fields.get("content")
if fields.get("role") != "tool" or not isinstance(content, list):
return message
kept_index: Final = None if fields.get("cache_control") is not None else _first_marked_index(content)
return cast( # cast-ok: the same tool message keeping only the mark its single cache point reads
_MessageT,
{
**fields,
"content": [
block if index == kept_index else block_with_cache_control(block, dropped_cache_control)
for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(content))
],
},
)
_BudgetSpend: TypeAlias = Callable[[tuple[int, int], int], tuple[int, int]]
def _newest_marks_kept(state: tuple[int, int], marks: int) -> tuple[int, int]:
remaining, _ = state
kept: Final = min(marks, remaining)
return remaining - kept, marks - kept
def _whole_message_kept(state: tuple[int, int], marks: int) -> tuple[int, int]:
remaining, _ = state
if marks > remaining:
return remaining, marks
return remaining - marks, 0
def _messages_within_budget(
messages: Sequence[_MessageT], reserved_marks: int, spend: _BudgetSpend
) -> Sequence[_MessageT]:
budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0)
newest_first: Final = tuple(reversed(messages))
budget_after_each: Final = itertools.accumulate(
(cache_control_count(message) for message in newest_first), spend, initial=(budget, 0)
)
dropped_counts: Final = tuple(dropped for _, dropped in itertools.islice(budget_after_each, 1, None))
kept_newest_first: Final = tuple(
_without_oldest_marks(message, dropped) for message, dropped in zip(newest_first, dropped_counts)
)
return tuple(reversed(kept_newest_first))
def newest_marks_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]:
return _messages_within_budget(messages, reserved_marks, _newest_marks_kept)
def whole_messages_within_cache_control_limit(
messages: Sequence[_MessageT], reserved_marks: int
) -> Sequence[_MessageT]:
return _messages_within_budget(messages, reserved_marks, _whole_message_kept)
def _part_with_cache_control_renamed(part: object) -> object:
if not _accepts_prompt_cache_breakpoint(part):
return part
@ -1439,3 +1625,25 @@ class AnthropicCacheControlHook(CustomPromptManagement):
llm_router=None,
)
return None
def messages_within_cache_control_cap(
messages: Sequence[_MessageT], tools: object, cache_control: object = None, request_kwargs: object = None
) -> Sequence[_MessageT]:
"""Under drop_params, drop the oldest message hints the provider's cache_control cap has no room for.
Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first.
"""
reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints(
_validated_object_list(tools), cache_control, request_kwargs
)
kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks)
dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept))
if dropped > 0:
verbose_logger.warning(
"Dropped %d cache_control hint(s) from the oldest messages because drop_params is on and the "
"provider accepts at most %d cache_control blocks per request",
dropped,
MAX_CACHE_CONTROL_BLOCKS,
)
return kept

View file

@ -208,11 +208,14 @@ class StandardBuiltInToolCostTracking:
usage=usage, response_object=response_object
)
response_model: Final = _output_item_field(response_object, "model")
if model_info is not None and resolved_usage is not None and resolved_provider is not None:
result: Final = get_cost_for_web_search_request(
custom_llm_provider=resolved_provider,
usage=resolved_usage,
model_info=model_info,
response_model=response_model if isinstance(response_model, str) else None,
lookup_model_info=StandardBuiltInToolCostTracking._safe_get_model_info,
)
if result is not None:
return result

View file

@ -3,13 +3,15 @@ import functools
import inspect
import re
import time
from collections.abc import Iterator, Mapping, Sequence
from collections.abc import Callable, Iterator, Mapping, Sequence
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final, TypeAlias, cast
from litellm._logging import format_base64_size, verbose_logger
from litellm.constants import (
BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS,
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS,
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS,
MAX_BASE64_LENGTH_FOR_LOGGING,
)
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
@ -44,6 +46,8 @@ Helper utils used for logging callbacks
# Regex matching data-URI base64 content: "data:<mime>;base64,<payload>"
# Captures: group(1)=mime_type, group(2)=base64_payload
_DATA_URI_RE: Final = re.compile(r"data:([^;,\s]{1,255});base64,([A-Za-z0-9+/=]+)")
_DATA_URI_PREFIX: Final = "data:"
_DATA_URI_BASE64_MARKER: Final = ";base64,"
# Maximum nesting depth for _truncate_base64_in_value to guard against
# pathological payloads. OpenAI message format is typically 3-4 levels deep.
@ -60,15 +64,20 @@ def _base64_data_uri_replacer(match: re.Match[str]) -> str:
return f"data:{mime_type};base64,[base64_data truncated: {size_str}]"
def _may_contain_base64_data_uri(value: str) -> bool:
return ";" in value and _DATA_URI_PREFIX in value and _DATA_URI_BASE64_MARKER in value
def _truncate_base64_in_string(value: str) -> str:
"""Replace long base64 data-URI payloads in a string with a size placeholder."""
if MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
if MAX_BASE64_LENGTH_FOR_LOGGING <= 0 or not _may_contain_base64_data_uri(value):
return value
return _DATA_URI_RE.sub(_base64_data_uri_replacer, value)
def _truncate_base64_in_value(
value: str | dict[str, object] | list[object] | None,
truncate_string: Callable[[str], str],
) -> str | dict[str, object] | list[object] | None:
"""Iteratively truncate base64 data URIs in a JSON-like value (str/list/dict).
@ -78,13 +87,13 @@ def _truncate_base64_in_value(
# Stack entries: (source_value, depth, parent_container, key_or_index)
# We mutate *copies* of dicts/lists in-place via parent references.
if isinstance(value, str):
return _truncate_base64_in_string(value)
return truncate_string(value)
if not isinstance(value, (dict, list)):
return value
# Shallow-copy the root so we don't mutate the caller's data.
root: Final = {k: v for k, v in value.items()} if isinstance(value, dict) else list(value)
stack: Final[list] = [(root, 0)]
stack: Final[list[tuple[dict[str, object] | list[object], int]]] = [(root, 0)] # mutable-ok: explicit stack
while stack:
container, depth = stack.pop()
@ -93,19 +102,19 @@ def _truncate_base64_in_value(
if isinstance(container, dict):
for k, v in container.items():
if isinstance(v, str):
container[k] = _truncate_base64_in_string(v)
container[k] = truncate_string(v)
elif isinstance(v, dict):
copy: dict | list = {ck: cv for ck, cv in v.items()}
copy: dict[str, object] | list[object] = {ck: cv for ck, cv in v.items()}
container[k] = copy
stack.append((copy, depth + 1))
elif isinstance(v, list):
copy = list(v)
container[k] = copy
stack.append((copy, depth + 1))
elif isinstance(container, list):
else:
for i, v in enumerate(container):
if isinstance(v, str):
container[i] = _truncate_base64_in_string(v)
container[i] = truncate_string(v)
elif isinstance(v, dict):
copy = {ck: cv for ck, cv in v.items()}
container[i] = copy
@ -127,8 +136,15 @@ def truncate_base64_in_messages(
"""
if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
return messages
return _copy_messages(messages, _truncate_base64_in_string)
def _copy_messages(
messages: str | dict[str, object] | list[object], # mutable-ok: same contract as truncate_base64_in_messages
truncate_string: Callable[[str], str],
) -> str | dict[str, object] | list[object] | None: # mutable-ok: same contract as truncate_base64_in_messages
try:
return _truncate_base64_in_value(messages)
return _truncate_base64_in_value(messages, truncate_string)
except Exception as e:
verbose_logger.debug("Failed to truncate base64 in messages: %s", e)
return messages
@ -151,19 +167,31 @@ def _iter_string_leaves(value: _StringTree) -> Iterator[str]:
pass
def _is_few_plain_strings(leaves: tuple[str, ...]) -> bool:
return len(leaves) < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS and not any(
_may_contain_base64_data_uri(leaf) for leaf in leaves
)
async def truncate_base64_in_messages_async(
messages: str | list | dict | None, # mutable-ok: same contract as truncate_base64_in_messages
) -> str | list | dict | None: # mutable-ok: same contract as truncate_base64_in_messages
"""
Same result as truncate_base64_in_messages, but payloads whose string content
reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are scanned in a worker
thread so the regex pass over multi-MB base64 images does not block the event loop.
reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are copied in a worker thread,
so neither the regex pass over multi-MB base64 images nor the copy of a long
history blocks the event loop. A few strings with no data URI, under
BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS in total, are copied inline, since
checking them for data URIs and copying them costs less than the thread hop.
"""
if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
return messages
total_chars: Final = sum(len(leaf) for leaf in _iter_string_leaves(messages))
leaves: Final = tuple(_iter_string_leaves(messages))
total_chars: Final = sum(len(leaf) for leaf in leaves)
if total_chars < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS:
return truncate_base64_in_messages(messages)
if total_chars < BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS and _is_few_plain_strings(leaves):
return _copy_messages(messages, lambda text: text)
return await asyncio.to_thread(truncate_base64_in_messages, messages)

View file

@ -88,17 +88,22 @@ def handle_any_messages_to_chat_completion_str_messages_conversion(
return [{"input": str(messages)}]
def _message_with_str_content(message: AllMessageValues) -> AllMessageValues:
texts: Final = convert_content_list_to_str(message=message)
if not texts:
return message
return cast( # cast-ok: the same message with its content flattened to one string
AllMessageValues, {**message, "content": texts}
)
def handle_messages_with_content_list_to_str_conversion(
messages: list[AllMessageValues],
) -> list[AllMessageValues]:
"""
Handles messages with content list conversion
Returns new messages with each content list flattened to a string, leaving the caller's messages untouched
"""
for message in messages:
texts = convert_content_list_to_str(message=message)
if texts:
message["content"] = texts
return messages
return [_message_with_str_content(message) for message in messages]
def strip_name_from_message(message: AllMessageValues, allowed_name_roles: list[str] = ["user"]) -> AllMessageValues:

View file

@ -9,6 +9,8 @@ from litellm.types.utils import CallTypes
from . import *
if TYPE_CHECKING:
from collections.abc import Callable
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
)
@ -30,7 +32,29 @@ def get_cost_for_google_maps_grounding_request(
return cost_per_google_maps_grounding_request(usage=usage, model_info=model_info)
def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", model_info: "ModelInfo") -> float | None:
def _with_served_model_search_pricing(
model_info: "ModelInfo",
response_model: str | None,
served_model_provider: str,
lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None",
) -> "ModelInfo":
if model_info.get("search_context_cost_per_query") is not None or response_model is None:
return model_info
served_model_info: Final = (
lookup_model_info(response_model, served_model_provider) if lookup_model_info is not None else None
)
if served_model_info is None:
return model_info
return {**model_info, "search_context_cost_per_query": served_model_info.get("search_context_cost_per_query")}
def get_cost_for_web_search_request(
custom_llm_provider: str,
usage: "Usage",
model_info: "ModelInfo",
response_model: str | None = None,
lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None" = None,
) -> float | None:
"""
Get the cost for a web search request for a given model.
@ -38,6 +62,8 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
custom_llm_provider: The custom LLM provider.
usage: The usage object.
model_info: The model info.
response_model: The model the provider response names.
lookup_model_info: Looks up a model's cost map entry, returning None when it is not mapped.
"""
if custom_llm_provider == "gemini":
from .gemini.cost_calculator import cost_per_web_search_request
@ -46,17 +72,25 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
elif custom_llm_provider == "anthropic":
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
return get_cost_for_anthropic_web_search(
model_info=_with_served_model_search_pricing(model_info, response_model, "anthropic", lookup_model_info),
usage=usage,
)
elif custom_llm_provider.startswith("vertex_ai"):
# Anthropic Claude models on Vertex AI populate server_tool_use.web_search_requests
# (same as the direct Anthropic API), not prompt_tokens_details.web_search_requests
# (which is the Gemini field). Route claude-* models to the Anthropic calculator.
model_key: Final[str] = model_info.get("key", "") if model_info else ""
if "claude" in model_key.lower():
if "claude" in (response_model or model_key).lower():
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
verbose_logger.debug("vertex_ai/claude model detected — routing web search cost to Anthropic calculator")
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
return get_cost_for_anthropic_web_search(
model_info=_with_served_model_search_pricing(
model_info, response_model, "vertex_ai", lookup_model_info
),
usage=usage,
)
from .vertex_ai.gemini.cost_calculator import (
cost_per_web_search_request as cost_per_web_search_request_vertex_ai,
@ -71,7 +105,10 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
return None
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
return get_cost_for_anthropic_web_search(
model_info=_with_served_model_search_pricing(model_info, response_model, "azure_ai", lookup_model_info),
usage=usage,
)
elif custom_llm_provider == "perplexity":
# Perplexity handles search costs internally in its own cost calculator
# Return 0.0 to indicate costs are already accounted for

View file

@ -167,6 +167,7 @@ _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool)
_TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_TOOLS_ADAPTER: Final = TypeAdapter(list[object])
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_REQUEST_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
)
@ -2254,6 +2255,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
headers=headers,
)
@staticmethod
def _messages_within_cache_control_cap(
messages: Sequence[AllMessageValues], optional_params: Mapping[str, object]
) -> Sequence[AllMessageValues]:
from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap
return messages_within_cache_control_cap(
messages, optional_params.get("tools"), optional_params.get("cache_control"), optional_params
)
def transform_request(
self,
model: str,
@ -2271,9 +2282,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
anthropic_messages_pt,
)
drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True
drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params"))
capped_messages: Final = (
list(
self._messages_within_cache_control_cap(
messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params)
)
)
if drop_params
else messages
)
if "tools" not in optional_params and messages is not None and has_tool_call_blocks(messages):
if "tools" not in optional_params and has_tool_call_blocks(capped_messages):
optional_params["tools"], _ = self._map_tools(add_dummy_tool(custom_llm_provider="anthropic"))
# Drop thinking param if thinking is enabled but thinking_blocks are missing
@ -2285,9 +2305,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# Related issue: https://github.com/BerriAI/litellm/issues/18926
if (
optional_params.get("thinking") is not None
and messages is not None
and last_assistant_with_tool_calls_has_no_thinking_blocks(messages)
and not any_assistant_message_has_thinking_blocks(messages)
and last_assistant_with_tool_calls_has_no_thinking_blocks(capped_messages)
and not any_assistant_message_has_thinking_blocks(capped_messages)
):
if litellm.modify_params:
optional_params.pop("thinking", None)
@ -2332,15 +2351,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
_name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request(
optional_params=optional_params,
)
if _name_forward_map:
messages = self._rewrite_tool_names_in_messages(messages, _name_forward_map)
request_messages: Final = (
self._rewrite_tool_names_in_messages(capped_messages, _name_forward_map)
if _name_forward_map
else capped_messages
)
if _name_reverse_map and isinstance(litellm_params, dict):
litellm_params[ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY] = _name_reverse_map
# Only the leading system run becomes the top-level system prompt. A later
# system message stays in the conversation: hoisting it rewrites the cached
# prefix and re-bills the whole history at cache-write pricing (#36559).
leading_system_run, later_messages = split_leading_system_run(messages)
leading_system_run, later_messages = split_leading_system_run(request_messages)
anthropic_system_message_list: Final = self.translate_system_message(messages=list(leading_system_run))
# Handling anthropic API Prompt Caching
if len(anthropic_system_message_list) > 0:
@ -2362,7 +2384,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
except Exception as e:
raise AnthropicError(
status_code=400,
message=f"{e}\nReceived Messages={messages}",
message=f"{e}\nReceived Messages={request_messages}",
) # don't use verbose_logger.exception, if exception is raised
self.update_headers_with_optional_anthropic_beta(

View file

@ -4,6 +4,7 @@ This file contains common utils for anthropic calls.
import copy
import json
import os
import re
from collections.abc import Container, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
@ -19,6 +20,8 @@ from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm.constants import (
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH,
ANTHROPIC_MAX_USER_ID_LENGTH,
DEFAULT_MODEL_CREATED_AT_TIME,
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
@ -178,6 +181,40 @@ def tools_without_custom_field(tools: Sequence[object], hoist_defer_loading: boo
return [_tool_without_custom_field(tool, hoist_defer_loading, drop_custom) for tool in tools]
def anthropic_pass_through_api_base() -> str:
return os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
def _same_host_and_port(url: httpx.URL, other: str) -> bool:
try:
other_url: Final = httpx.URL(other)
except httpx.InvalidURL:
return False
return (url.host, url.port) == (other_url.host, other_url.port)
def anthropic_upstream_user_id_limit(url: str) -> int | None:
upstream: Final = httpx.URL(url)
if not upstream.path.removesuffix("/").endswith("/v1/messages"):
return None
if upstream.host == "api.anthropic.com":
return ANTHROPIC_MAX_USER_ID_LENGTH
return (
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH
if _same_host_and_port(upstream, anthropic_pass_through_api_base())
else None
)
def _has_email_shape(token: str) -> bool:
at_sign: Final = token.find("@", 1)
return at_sign != -1 and token.find(".", at_sign + 2, -1) != -1
def anthropic_accepts_user_id(user_id: str, max_length: int) -> bool:
return len(user_id) <= max_length and user_id.isprintable() and not any(map(_has_email_shape, user_id.split()))
def is_anthropic_messages_url(url: str) -> bool:
"""Check whether a URL addresses Anthropic's Messages API."""
parsed_url: Final = urlparse(url)

View file

@ -12,6 +12,7 @@ from pydantic_core import ErrorDetails
from typing_extensions import assert_never
import litellm
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.secret_managers.main import get_secret_str
@ -297,13 +298,7 @@ def _systemone_question(question: DecisionsIRQuestion) -> Mapping[str, object]:
assert_never(question)
def ir_to_systemone_request(
model: str, request: DecisionsIRRequest, *, supports_audio: bool = False, supports_video: bool = False
) -> Mapping[str, object] | UnsupportedDecisionsRequest:
if request.audio and not supports_audio:
return UnsupportedDecisionsRequest(reason="audio input is not supported by this Decisions model")
if request.videos and not supports_video:
return UnsupportedDecisionsRequest(reason="video input is not supported by this Decisions model")
def ir_to_systemone_request(model: str, request: DecisionsIRRequest) -> Mapping[str, object]:
state: Final = _systemone_state(request.input)
keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True)
images: Final = input_images(request.input)
@ -503,6 +498,16 @@ def ir_to_systemone_response(response: DecisionsIRResponse, request: DecisionsIR
)
def catalog_mode(model: str, custom_llm_provider: str) -> str | None:
try:
key: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)["key"]
except Exception: # noqa: BLE001 # get_model_info raises bare Exception for an unmapped model, which the catalog does not list
return None
entry: Final = GetModelCostMap.loaded_model_cost_map().get(key)
mode: Final = entry.get("mode") if entry is not None else None
return mode if isinstance(mode, str) else None
def _is_decisions_model(model: str, custom_llm_provider: str) -> bool:
try:
model_mode: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider).get("mode")
@ -640,12 +645,16 @@ class BaseDecisionsConfig(ABC):
model=model, custom_llm_provider=custom_llm_provider
)
def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool:
def supports_audio_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
if deployment_flag is not None:
return deployment_flag
return _is_decisions_model(model, custom_llm_provider) and litellm.supports_audio_input(
model=model, custom_llm_provider=custom_llm_provider
)
def supports_video_input(self, model: str, custom_llm_provider: str) -> bool:
def supports_video_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
if deployment_flag is not None:
return deployment_flag
return _is_decisions_model(model, custom_llm_provider) and litellm.supports_video_input(
model=model, custom_llm_provider=custom_llm_provider
)
@ -656,12 +665,7 @@ class BaseDecisionsConfig(ABC):
request: DecisionsIRRequest,
custom_llm_provider: str,
) -> Mapping[str, object] | UnsupportedDecisionsRequest:
return ir_to_systemone_request(
self.request_model(model),
request,
supports_audio=self.supports_audio_input(model, custom_llm_provider),
supports_video=self.supports_video_input(model, custom_llm_provider),
)
return ir_to_systemone_request(self.request_model(model), request)
def unwrap_response(self, payload: object) -> object:
return payload

View file

@ -25,6 +25,7 @@ from litellm.constants import (
)
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
drops_unsupported_params,
filter_exceptions_from_params,
filter_internal_params,
map_finish_reason,
@ -2129,6 +2130,24 @@ class AmazonConverseConfig(BaseConfig):
return data
@staticmethod
def _claude_messages_within_cache_control_cap(
messages: Sequence[AllMessageValues],
capability_model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> Sequence[AllMessageValues]:
if not drops_unsupported_params(litellm_params.get("drop_params")) or "claude" not in capability_model.lower():
return messages
from litellm.integrations.anthropic_cache_control_hook import (
messages_within_cache_control_cap,
tool_message_with_single_cache_mark,
)
return messages_within_cache_control_cap(
tuple(map(tool_message_with_single_cache_mark, messages)), optional_params.get("tools")
)
async def _async_transform_request(
self,
model: str,
@ -2140,7 +2159,13 @@ class AmazonConverseConfig(BaseConfig):
base_model: Final = litellm_params.get("base_model")
capability_model: Final = bedrock_capability_model(model, base_model)
messages, system_content_blocks = self._transform_system_message(
messages, model=capability_model, deployment_model=model
list(
self._claude_messages_within_cache_control_cap(
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
)
),
model=capability_model,
deployment_model=model,
)
# Convert last user message to guarded_text if guardrailConfig is present
@ -2221,7 +2246,13 @@ class AmazonConverseConfig(BaseConfig):
base_model: Final = litellm_params.get("base_model")
capability_model: Final = bedrock_capability_model(model, base_model)
messages, system_content_blocks = self._transform_system_message(
messages, model=capability_model, deployment_model=model
list(
self._claude_messages_within_cache_control_cap(
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
)
),
model=capability_model,
deployment_model=model,
)
# Convert last user message to guarded_text if guardrailConfig is present

View file

@ -110,6 +110,15 @@ def map_aiohttp_exceptions() -> Generator[None, None, None]:
raise mapped_exc(message) from exc
def _unread_payload(content: object) -> bytes:
"""Bytes the upstream sent before the connection dropped that aiohttp buffered but never handed out, since its
reads raise the stored payload error ahead of the buffer."""
if not isinstance(content, aiohttp.StreamReader):
return b""
buffered: Final = b"".join(content._buffer) # pyright: ignore[reportPrivateUsage] # no public read drains it
return buffered[content._buffer_offset :] # pyright: ignore[reportPrivateUsage] # see above
def _release_transport_anchor(_transport: object) -> None:
"""Release a transport anchored to a response stream. See ``_anchor_transport_to``."""
@ -144,6 +153,12 @@ class AiohttpResponseStream(httpx.AsyncByteStream):
raise httpx.ReadError(str(e)) from e
except aiohttp.http_exceptions.TransferEncodingError as e:
raise httpx.ReadError(str(e)) from e
except aiohttp.ClientPayloadError:
unread: Final = _unread_payload(self._aiohttp_response.content)
if unread:
yield unread
with map_aiohttp_exceptions():
raise
except Exception:
# For other exceptions, use the normal mapping
with map_aiohttp_exceptions():

View file

@ -26,6 +26,7 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase):
aembedding=None,
custom_endpoint: bool | None = None,
headers: dict | None = None,
drop_params: bool = False,
) -> EmbeddingResponse:
# Check for custom user agent in optional_params or environment
# This allows partners building on LiteLLM to set their own telemetry
@ -58,4 +59,5 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase):
aembedding=aembedding,
custom_endpoint=True,
headers=headers,
drop_params=drop_params,
)

View file

@ -130,7 +130,8 @@ class DeepInfraConfig(OpenAIGPTConfig):
- Input: {"role": "tool", "content": [{"type": "text", "text": "result"}]}
- Output: {"role": "tool", "content": "[{\"type\": \"text\", \"text\": \"result\"}]"}
"""
for message in messages:
copied_messages: Final = [message.copy() for message in messages]
for message in copied_messages:
if message.get("role") == "tool":
content = message.get("content")
@ -149,7 +150,7 @@ class DeepInfraConfig(OpenAIGPTConfig):
# For complex content, serialize the entire array as JSON string
message["content"] = json.dumps(content)
return messages
return copied_messages
@overload
def _transform_messages(

View file

@ -23,6 +23,7 @@ from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionToolParam,
ChatCompletionToolParamFunctionChunk,
OpenAIChatCompletionToolParam,
)
from litellm.types.utils import (
@ -45,6 +46,7 @@ from ...openai.chat.gpt_transformation import (
OpenAIChatCompletionStreamingHandler,
OpenAIGPTConfig,
)
from ...openai.common_utils import without_cache_control
from ..common_utils import (
FIREROUTER,
FireworksAIException,
@ -480,15 +482,23 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
return (("response_format", _json_schema_response_format(choice_schema, "choice")),)
def _transform_tools(self, tools: list[OpenAIChatCompletionToolParam]) -> list[OpenAIChatCompletionToolParam]:
for tool in tools:
if tool.get("type") != "function":
continue
function = tool["function"]
function.pop("strict", None)
params = function.get("parameters")
if isinstance(params, dict):
unpack_legacy_defs(params)
return tools
return [self._fireworks_tool(tool) for tool in tools]
@staticmethod
def _fireworks_tool(tool: OpenAIChatCompletionToolParam) -> OpenAIChatCompletionToolParam:
if tool.get("type") != "function":
return tool
function: Final = tool["function"]
params: Final = function.get("parameters")
unstrict_function: Final = cast( # cast-ok: the same function definition minus `strict`
ChatCompletionToolParamFunctionChunk, {key: value for key, value in function.items() if key != "strict"}
)
new_function: Final[ChatCompletionToolParamFunctionChunk] = (
{**unstrict_function, "parameters": unpack_legacy_defs(params, copy=True)}
if isinstance(params, dict)
else unstrict_function
)
return {**tool, "function": new_function}
def _transform_messages_helper(
self, messages: list[AllMessageValues], model: str, litellm_params: dict
@ -496,10 +506,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
"""
Strip fields not permitted by FireworksAI from messages.
"""
from litellm.litellm_core_utils.prompt_templates.common_utils import (
filter_value_from_dict,
)
supports_vision_value: Final = self._get_model_cost_capability_exact(model=model, capability="supports_vision")
for message in messages:
if message["role"] == "user":
@ -530,9 +536,11 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
model=model,
llm_provider="fireworks_ai",
)
filter_value_from_dict(cast(dict, message), "cache_control")
return [strip_litellm_response_only_fields(message) for message in messages]
return [
strip_litellm_response_only_fields(cast(AllMessageValues, without_cache_control(message)))
for message in messages
]
# Cached index of fireworks_ai/* entries from litellm.model_cost. Building
# this index requires a full scan of model_cost (tens of thousands of

View file

@ -140,7 +140,8 @@ class GroqChatConfig(OpenAILikeChatConfig):
def _transform_messages(
self, messages: list[AllMessageValues], model: str, is_async: bool = False
) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]:
for idx, message in enumerate(messages):
transformed_messages: Final = list(messages)
for idx, message in enumerate(transformed_messages):
"""
1. Don't pass 'null' function_call assistant message to groq - https://github.com/BerriAI/litellm/issues/5839
"""
@ -154,12 +155,12 @@ class GroqChatConfig(OpenAILikeChatConfig):
for k, v in _message.items():
if v is not None:
new_message[k] = v
messages[idx] = strip_litellm_internal_message_fields(new_message)
transformed_messages[idx] = strip_litellm_internal_message_fields(new_message)
if is_async:
return super()._transform_messages(messages=messages, model=model, is_async=True)
return super()._transform_messages(messages=transformed_messages, model=model, is_async=True)
else:
return super()._transform_messages(messages=messages, model=model, is_async=False)
return super()._transform_messages(messages=transformed_messages, model=model, is_async=False)
def transform_messages(
self,

View file

@ -2,6 +2,7 @@
Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
"""
import copy
import json
from collections.abc import Coroutine
from typing import Final, Literal, cast, overload
@ -88,7 +89,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
) -> dict:
_tools = non_default_params.pop("tools", None)
if _tools is not None:
_tools = remove_additional_properties(_tools)
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
_tools = remove_strict_from_schema(_tools)
if isinstance(_tools, list):
_tools = self._convert_custom_tools_to_function_tools(_tools)

View file

@ -755,7 +755,8 @@ class MistralConfig(OpenAIGPTConfig):
if _name is not None:
# Remove name if not a tool message
if message["role"] != "tool" or isinstance(_name, str) and len(_name.strip()) == 0:
message.pop("name", None)
message_without_name: Final = {k: v for k, v in message.items() if k != "name"}
return cast(AllMessageValues, message_without_name) # cast-ok: the same message minus name
return message

View file

@ -14,7 +14,7 @@ from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.constants import OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
extract_reasoning_content,
handle_invalid_parallel_tool_calls,
@ -64,7 +64,7 @@ from litellm.types.utils import (
)
from litellm.utils import convert_to_model_response_object, declared_mid_conversation_system_support
from ..common_utils import OpenAIError
from ..common_utils import OpenAIError, without_cache_control
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
if TYPE_CHECKING:
@ -128,6 +128,29 @@ def _litellm_params_str(litellm_params: Mapping[str, object] | None, key: str) -
return value if isinstance(value, str) else None
def _drops_params(litellm_params: Mapping[str, object]) -> bool:
return drops_unsupported_params(litellm_params.get("drop_params"))
def _names_claude(model: str, litellm_params: Mapping[str, object]) -> bool:
base_model: Final = litellm_params.get("base_model")
return "claude" in model.lower() or (isinstance(base_model, str) and "claude" in base_model.lower())
def _claude_messages_within_cache_control_cap(
model: str,
messages: Sequence[AllMessageValues],
tools: Sequence["ChatCompletionToolParam"] | None,
litellm_params: Mapping[str, object],
request_params: Mapping[str, object],
) -> Sequence[AllMessageValues]:
if not _names_claude(model, litellm_params) or not _drops_params(litellm_params):
return messages
from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap
return list(messages_within_cache_control_cap(messages, tools, request_params.get("cache_control"), request_params))
def _prompt_cache_options(optional_params: Mapping[str, object]) -> object | None:
try:
extra_body: Final = _OBJECT_MAPPING.validate_python(optional_params.get("extra_body"))
@ -311,52 +334,58 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
return content_copy
async def _async_handle_pdf_url(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile:
file_id: Final = content_item.get("file_id")
content_copy: Final = content_item.copy()
file_id: Final = content_copy.get("file_id")
if file_id is not None: # check for file id being url done in _handle_pdf_url
base64_data: Final = await async_convert_url_to_base64(file_id)
content_item["file_data"] = base64_data
content_item["filename"] = "my_file.pdf"
content_item.pop("file_id")
return content_item
content_copy["file_data"] = base64_data
content_copy["filename"] = "my_file.pdf"
content_copy.pop("file_id")
return content_copy
def _common_file_data_check(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile:
file_data: Final = content_item.get("file_data")
filename: Final = content_item.get("filename")
if file_data is not None and filename is None:
content_item["filename"] = "my_file.pdf"
return {**content_item, "filename": "my_file.pdf"}
return content_item
@staticmethod
def _image_item_for_openai(content_item: ChatCompletionImageObject) -> ChatCompletionImageObject:
image_url: Final = content_item["image_url"]
if isinstance(image_url, str):
return {**content_item, "image_url": {"url": image_url}}
if not isinstance(image_url, dict):
return content_item
new_image_url_obj: Final = cast(
ChatCompletionImageUrlObject,
{k: v for k, v in image_url.items() if k not in {"format", "video_metadata"}},
)
return {**content_item, "image_url": new_image_url_obj}
@staticmethod
def _file_item_for_openai(content_item: ChatCompletionFileObject) -> ChatCompletionFileObject:
litellm_specific_params: Final = {"format", "detail", "video_metadata"}
file_obj: Final = content_item.get("file")
if file_obj is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=None,
llm_provider="openai",
)
new_file_obj: Final = ChatCompletionFileObjectFile(
**{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
)
return {**content_item, "file": new_file_obj}
def _apply_common_transform_content_item(
self,
content_item: OpenAIMessageContentListBlock,
) -> OpenAIMessageContentListBlock:
litellm_specific_params: Final = {"format", "detail", "video_metadata"}
if content_item.get("type") == "image_url":
content_item = cast(ChatCompletionImageObject, content_item)
if isinstance(content_item["image_url"], str):
content_item["image_url"] = {
"url": content_item["image_url"],
}
elif isinstance(content_item["image_url"], dict):
new_image_url_obj: Final = cast(
ChatCompletionImageUrlObject,
{k: v for k, v in content_item["image_url"].items() if k not in {"format", "video_metadata"}},
)
content_item["image_url"] = new_image_url_obj
elif content_item.get("type") == "file":
content_item = cast(ChatCompletionFileObject, content_item)
file_obj: Final = content_item.get("file")
if file_obj is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=None,
llm_provider="openai",
)
new_file_obj: Final = ChatCompletionFileObjectFile(
**{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
)
content_item["file"] = new_file_obj
return self._image_item_for_openai(cast(ChatCompletionImageObject, content_item))
if content_item.get("type") == "file":
return self._file_item_for_openai(cast(ChatCompletionFileObject, content_item))
return content_item
def _transform_content_item(
@ -419,31 +448,34 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
hoisted_messages: Final = hoist_images_from_tool_messages(stripped_messages)
async def _async_transform():
for message in hoisted_messages:
message_content = message.get("content")
message_role = message.get("role")
if message_role == "user" and message_content and isinstance(message_content, list):
message_content_types = cast(list[OpenAIMessageContentListBlock], message_content)
for i, content_item in enumerate(message_content_types):
message_content_types[i] = await self._async_transform_content_item(
cast(OpenAIMessageContentListBlock, content_item),
)
return hoisted_messages
return [await self._async_user_message_for_openai(message) for message in hoisted_messages]
if is_async:
return _async_transform()
else:
for message in hoisted_messages:
message_content = message.get("content")
message_role = message.get("role")
if message_role == "user" and message_content and isinstance(message_content, list):
message_content_types = cast(list[OpenAIMessageContentListBlock], message_content)
for i, content_item in enumerate(message_content):
message_content_types[i] = self._transform_content_item(
cast(OpenAIMessageContentListBlock, content_item)
)
return hoisted_messages
return [self._user_message_for_openai(message) for message in hoisted_messages]
@staticmethod
def _user_content_parts(message: AllMessageValues) -> Sequence[OpenAIMessageContentListBlock] | None:
message_content: Final = message.get("content")
if message.get("role") != "user" or not message_content or not isinstance(message_content, list):
return None
return cast(Sequence[OpenAIMessageContentListBlock], message_content)
def _user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues:
parts: Final = self._user_content_parts(message)
if parts is None:
return message
return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI
AllMessageValues, {**message, "content": [self._transform_content_item(part) for part in parts]}
)
async def _async_user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues:
parts: Final = self._user_content_parts(message)
if parts is None:
return message
return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI
AllMessageValues, {**message, "content": [await self._async_transform_content_item(part) for part in parts]}
)
def transform_messages(
self,
@ -462,23 +494,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
messages: list[AllMessageValues],
tools: list["ChatCompletionToolParam"] | None = None,
) -> tuple[list[AllMessageValues], list["ChatCompletionToolParam"] | None]:
from litellm.litellm_core_utils.prompt_templates.common_utils import (
filter_value_from_dict,
)
from litellm.types.llms.openai import ChatCompletionToolParam
for i, message in enumerate(messages):
messages[i] = cast(
AllMessageValues,
filter_value_from_dict(message, "cache_control"),
)
if tools is not None:
for i, tool in enumerate(tools):
tools[i] = cast(
ChatCompletionToolParam,
filter_value_from_dict(tool, "cache_control"),
)
return messages, tools
new_messages: Final = [cast(AllMessageValues, without_cache_control(message)) for message in messages]
new_tools: Final = (
[cast(ChatCompletionToolParam, without_cache_control(tool)) for tool in tools]
if tools is not None
else None
)
return new_messages, new_tools
def translate_developer_role_to_system_role(
self,
@ -527,6 +551,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
tools: Sequence["ChatCompletionToolParam"] | None,
prompt_cache_options: object,
litellm_params: Mapping[str, object],
request_params: Mapping[str, object],
) -> tuple[Sequence[AllMessageValues], Sequence["ChatCompletionToolParam"] | None]:
"""
A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on
@ -543,10 +568,11 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
model, custom_llm_provider, api_base, prompt_cache_options
)
if not reads_breakpoints and self._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base):
return messages, tools
drops_breakpoints: Final = not reads_breakpoints and (
litellm.drop_params is True or litellm_params.get("drop_params") is True
)
return (
_claude_messages_within_cache_control_cap(model, messages, tools, litellm_params, request_params),
tools,
)
drops_breakpoints: Final = not reads_breakpoints and _drops_params(litellm_params)
renamed: Final = (
[AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages]
if reads_breakpoints
@ -561,7 +587,13 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
if drops_breakpoints
else list(tools)
)
return self.remove_cache_control_flag_from_messages_and_tools(model=model, messages=renamed, tools=kept_tools)
sent_messages, sent_tools = self.remove_cache_control_flag_from_messages_and_tools(
model=model, messages=renamed, tools=kept_tools
)
return (
_claude_messages_within_cache_control_cap(model, sent_messages, sent_tools, litellm_params, request_params),
sent_tools,
)
def _sanitized_tools_update_for_openai(
self,
@ -623,6 +655,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
tools=optional_params.get("tools", []),
prompt_cache_options=_prompt_cache_options(optional_params),
litellm_params=litellm_params,
request_params=optional_params,
)
if tools is not None and len(tools) > 0:
optional_params["tools"] = tools
@ -654,6 +687,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
tools=optional_params.get("tools", []),
prompt_cache_options=_prompt_cache_options(optional_params),
litellm_params=litellm_params,
request_params=optional_params,
)
if tools is not None and len(tools) > 0:
optional_params["tools"] = tools

View file

@ -153,12 +153,13 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig):
- role: system ==> translate to role 'user'
"""
_supports_system_messages: Final = supports_system_messages(model, "openai")
for i, message in enumerate(messages):
transformed_messages: Final = list(messages)
for i, message in enumerate(transformed_messages):
if message["role"] == "system" and not _supports_system_messages:
new_message = ChatCompletionUserMessage(content=message["content"], role="user")
messages[i] = new_message # Replace the old message with the new one
transformed_messages[i] = new_message # Replace the old message with the new one
if is_async:
return super()._transform_messages(messages, model, is_async=cast(Literal[True], True))
return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[True], True))
else:
return super()._transform_messages(messages, model, is_async=cast(Literal[False], False))
return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[False], False))

View file

@ -3,6 +3,7 @@ Common helpers / utils across al OpenAI endpoints
"""
import asyncio
import copy
import hashlib
import inspect
import json
@ -31,7 +32,7 @@ if TYPE_CHECKING:
import litellm
from litellm.exceptions import error_response_without_credentials
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, iter_items
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params
from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -43,6 +44,47 @@ from litellm.llms.custom_httpx.http_handler import (
)
def _is_plain_dict(value: object) -> bool:
return isinstance(value, dict)
def _is_list(value: object) -> bool:
return isinstance(value, list)
def _contains_cache_control(value: object) -> bool:
pending: Final[list[object]] = [value] # mutable-ok: a local worklist, never returned or stored
while pending:
node = pending.pop()
mapping = as_str_mapping(node)
if mapping is not None:
if "cache_control" in mapping:
return True
pending.extend(mapping.values())
elif _is_list(node):
pending.extend(iter_items(node))
return False
def without_cache_control(item: object) -> object:
"""Return `item` with every `cache_control` key removed, without touching the original.
`filter_value_from_dict` deletes the key in place and recurses into nested dicts and
lists, so the caller's own message, tool or input object has to be copied first: it
may reuse the same list on a provider that does support prompt caching. The copy is
skipped when there is nothing to strip, which keeps the object identity the
pass-through paths rely on and avoids a deep copy on every request.
"""
from litellm.litellm_core_utils.prompt_templates.common_utils import filter_value_from_dict
if not _is_plain_dict(item) or not _contains_cache_control(item):
return item
copied: Final = copy.deepcopy(item)
if isinstance(copied, dict):
filter_value_from_dict(copied, "cache_control")
return copied
def _get_client_init_params(cls: type) -> tuple[str, ...]:
"""Extract __init__ parameter names (excluding 'self') from a class."""
return tuple(p for p in inspect.signature(cls.__init__).parameters if p != "self")

View file

@ -354,10 +354,10 @@ class OpenAIDecisionsConfig(BaseDecisionsConfig):
) -> str | None:
return resolve_openai_bearer_token(api_key=api_key, api_base=api_base, litellm_params=litellm_params)
def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool:
def supports_audio_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
return False
def supports_video_input(self, model: str, custom_llm_provider: str) -> bool:
def supports_video_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
return False
def transform_decisions_request(

View file

@ -16,6 +16,7 @@ from openai._base_client import make_request_options
from openai._constants import RAW_RESPONSE_HEADER
from openai._legacy_response import LegacyAPIResponse
from openai._types import RequestOptions
from openai.pagination import SyncCursorPage
from openai.types import CreateEmbeddingResponse
from openai.types.beta.assistant_deleted import AssistantDeleted
from openai.types.chat import ChatCompletion, ChatCompletionChunk
@ -40,6 +41,7 @@ from litellm.types.utils import (
LiteLLMBatch,
ModelResponse,
ModelResponseStream,
openai_batch_status,
)
from litellm.utils import (
CustomStreamWrapper,
@ -2182,6 +2184,18 @@ class OpenAIFilesAPI(BaseLLM):
return response
def _with_openai_batch_fields(batches: Sequence[Batch]) -> Sequence[Batch]:
return [
batch.model_copy(
update={
"object": batch.object or "batch",
"status": openai_batch_status(batch.status),
}
)
for batch in batches
]
class OpenAIBatchesAPI(BaseLLM):
"""
OpenAI methods to support for batches
@ -2372,7 +2386,7 @@ class OpenAIBatchesAPI(BaseLLM):
):
verbose_logger.debug("listing batches, after= %s, limit= %s", after, limit)
response: Final = await openai_client.batches.list(after=after, limit=limit)
return response
return response.model_copy(update={"data": _with_openai_batch_fields(response.data)})
def list_batches(
self,
@ -2384,7 +2398,7 @@ class OpenAIBatchesAPI(BaseLLM):
organization: str | None,
after: str | None = None,
limit: int | None = None,
client: OpenAI | None = None,
client: OpenAI | AsyncOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
):
openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
@ -2409,7 +2423,9 @@ class OpenAIBatchesAPI(BaseLLM):
)
return self.alist_batches(openai_client=openai_client, after=after, limit=limit)
response: Final = openai_client.batches.list(after=after, limit=limit)
return response
if not isinstance(response, SyncCursorPage):
return response
return response.model_copy(update={"data": _with_openai_batch_fields(response.data)})
class OpenAIAssistantsAPI(BaseLLM):

View file

@ -52,7 +52,7 @@ from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
from litellm.types.workload_identity import OPENAI_WIF_KWARGS_KEYS
from ..common_utils import OpenAIError
from ..common_utils import OpenAIError, without_cache_control
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16
@ -658,13 +658,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
input=self._replayable_input(marker_safe_input, reads_breakpoints, maps_anthropic_tool_search),
tools=mapped_tools,
)
if marker_safe_input is not validated_input:
self._strip_caller_input_as_if_unmarked(
model=model,
caller_input=validated_input,
reads_breakpoints=reads_breakpoints,
maps_anthropic_tool_search=maps_anthropic_tool_search,
)
object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools)
sanitized_tools: Final = self._sanitized_tool_schemas_for_openai(
model=model, tools=object_schema_tools, litellm_params=litellm_params
@ -679,17 +672,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
return marked_input
return input_without_unanswered_anthropic_tool_search_calls(marked_input)
def _strip_caller_input_as_if_unmarked(
self,
model: str,
caller_input: str | ResponseInputParam,
reads_breakpoints: bool,
maps_anthropic_tool_search: bool,
) -> None:
self.remove_cache_control_flag_from_input_and_tools(
model=model, input=self._replayable_input(caller_input, reads_breakpoints, maps_anthropic_tool_search)
)
def _input_with_wire_safe_prompt_cache_breakpoints(
self,
model: str,
@ -800,24 +782,29 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
the chat path. Strips Anthropic-only `cache_control` markers from
Responses API input content blocks and tools.
`filter_value_from_dict` mutates each dict in place, so the same
objects are returned.
`filter_value_from_dict` deletes the key in place and recurses, so each
item is copied first: the caller keeps its own input list and may reuse
it on a provider that does support prompt caching.
"""
from litellm.litellm_core_utils.prompt_templates.common_utils import (
filter_value_from_dict,
new_input: Final = (
cast( # cast-ok: the comprehension rebuilds the input item for item
ResponseInputParam,
[without_cache_control(item) for item in input],
)
if isinstance(input, list)
else input
)
if isinstance(input, list):
for item in input:
if isinstance(item, dict):
filter_value_from_dict(cast(dict, item), "cache_control")
new_tools: Final = (
cast( # cast-ok: the comprehension rebuilds the tool list item for item
Sequence[ALL_RESPONSES_API_TOOL_PARAMS],
[without_cache_control(tool) for tool in tools],
)
if tools is not None
else None
)
if tools is not None:
for tool in tools:
if isinstance(tool, dict):
filter_value_from_dict(cast(dict, tool), "cache_control")
return input, tools
return new_input, new_tools
def _drop_bridge_minted_reasoning_items(self, input: str | ResponseInputParam) -> str | ResponseInputParam:
if self.custom_llm_provider not in _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING or not isinstance(input, list):

View file

@ -125,6 +125,20 @@ The provider inherits all request/response handling from OpenAI's Responses API
If `supported_endpoints` is omitted, it defaults to `[]` (only chat completions, which is always enabled for JSON providers).
When only some models serve an endpoint natively, list them under `model_supported_endpoints`. Every other model reaches `/v1/responses` and `/v1/messages` through chat completions:
```json
{
"your_provider": {
"supported_endpoints": ["/v1/chat/completions"],
"model_supported_endpoints": {
"model-a": ["/v1/messages"],
"model-b": ["/v1/responses"]
}
}
}
```
### How It Works
1. `json_loader.py` checks `supported_endpoints` for `/v1/responses`

View file

@ -3,11 +3,16 @@
## Allows jina ai embedding calls - which don't allow 'encoding_format' in payload.
import json
import re
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
import httpx
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@ -17,6 +22,57 @@ from litellm.types.utils import EmbeddingResponse
from ..common_utils import OpenAILikeBase, OpenAILikeError
_KEY_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, str])
_EXTRA_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_TRANSPORT_HEADER_NAMES: Final = frozenset(
{
"host",
"content-length",
"transfer-encoding",
"connection",
"keep-alive",
"proxy-connection",
"te",
"trailer",
"upgrade",
"expect",
}
)
_HEADER_NAME: Final = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
_HEADER_VALUE: Final = re.compile(r"[\t\x20-\x7e]*")
def _header_text(name: str, value: object) -> str | None:
if isinstance(value, bool) or not isinstance(value, str | int | float):
return None
if _HEADER_NAME.fullmatch(name) is None:
return None
return str(value).strip(" \t\r\n")
def _extra_headers_as_text(extra_headers: Mapping[str, object], model: str, drop_params: bool) -> Mapping[str, str]:
caller_set: Final = {
name: value for name, value in extra_headers.items() if name.lower() not in _TRANSPORT_HEADER_NAMES
}
texts: Final = MappingProxyType({name: _header_text(name, value) for name, value in caller_set.items()})
refused: Final = sorted(name for name, text in texts.items() if text is None)
if refused and not drop_params:
raise litellm.UnsupportedParamsError(
message=f"extra_headers {', '.join(map(repr, refused))} cannot be sent. A header needs a name made of "
"letters, digits or !#$%&'*+-.^_`|~ and a text or number value. "
"To drop them from the call, set `litellm.drop_params = True`.",
model=model,
)
named: Final = MappingProxyType({name: text for name, text in texts.items() if text is not None})
unsendable: Final = sorted(name for name, text in named.items() if _HEADER_VALUE.fullmatch(text) is None)
if unsendable:
verbose_logger.warning(
"extra_headers %s dropped for model %r, since a header value must be printable ASCII on one line",
unsendable,
model,
)
return MappingProxyType({name: text for name, text in named.items() if name not in unsendable})
class OpenAILikeEmbeddingHandler(OpenAILikeBase):
def __init__(self, **kwargs):
@ -94,6 +150,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
aembedding=None,
custom_endpoint: bool | None = None,
headers: dict | None = None,
drop_params: bool = False,
) -> EmbeddingResponse:
api_base, headers = self._validate_environment(
api_base=api_base,
@ -103,8 +160,20 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
custom_endpoint=custom_endpoint,
)
model = model
filtered_optional_params: Final = {k: v for k, v in optional_params.items() if v not in (None, "")}
data: Final = {"model": model, "input": input, **filtered_optional_params}
data: Final = {
"model": model,
"input": input,
**{k: v for k, v in optional_params.items() if k != "extra_headers" and v not in (None, "")},
}
extra_headers: Final = _extra_headers_as_text(
_EXTRA_HEADERS_ADAPTER.validate_python(optional_params.get("extra_headers") or {}), model, drop_params
)
key_headers: Final = _KEY_HEADERS_ADAPTER.validate_python(headers)
extra_header_names: Final = frozenset(name.lower() for name in extra_headers)
request_headers: Final = {
**{name: value for name, value in key_headers.items() if name.lower() not in extra_header_names},
**extra_headers,
}
## LOGGING
logging_obj.pre_call(
@ -123,7 +192,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
api_key=api_key,
timeout=timeout,
client=client,
headers=headers,
headers=request_headers,
)
if client is None or isinstance(client, AsyncHTTPHandler):
self.client = HTTPHandler(timeout=timeout)
@ -134,7 +203,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
try:
response: Final = self.client.post(
api_base,
headers=headers,
headers=request_headers,
data=json.dumps(data),
)

View file

@ -3,7 +3,7 @@ JSON-based provider configuration loader for OpenAI-compatible providers.
"""
import json
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Final
@ -14,6 +14,7 @@ from litellm.types.utils import ProviderSpecificModelInfo
_PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo)
_SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(Mapping[str, bool])
_MODEL_ENDPOINTS_ADAPTER: Final = TypeAdapter(Mapping[str, tuple[str, ...]])
class SimpleProviderConfig:
@ -30,7 +31,10 @@ class SimpleProviderConfig:
self.special_handling: Mapping[str, bool] = _SPECIAL_HANDLING_ADAPTER.validate_python(
data.get("special_handling", {})
)
self.supported_endpoints = data.get("supported_endpoints", [])
self.supported_endpoints: Sequence[str] = data.get("supported_endpoints", [])
self.model_supported_endpoints: Mapping[str, tuple[str, ...]] = _MODEL_ENDPOINTS_ADAPTER.validate_python(
data.get("model_supported_endpoints", {})
)
self.provider_info: ProviderSpecificModelInfo = (
_PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {}
)
@ -90,6 +94,23 @@ class JSONProviderRegistry:
return False
return "/v1/responses" in provider.supported_endpoints
@classmethod
def supports_embeddings(cls, slug: str) -> bool:
provider: Final = cls._providers.get(slug)
if provider is None:
return False
return "/v1/embeddings" in provider.supported_endpoints
@classmethod
def serves_endpoint(cls, slug: str, model: str | None, endpoint: str) -> bool:
provider: Final = cls._providers.get(slug)
if provider is None:
return False
model_endpoints: Final = (
provider.model_supported_endpoints.get(model.removeprefix(f"{slug}/"), ()) if model else ()
)
return endpoint in provider.supported_endpoints or endpoint in model_endpoints
@classmethod
def list_providers(cls) -> list:
"""List all registered provider slugs"""

View file

@ -211,7 +211,7 @@
"param_mappings": {
"max_completion_tokens": "max_tokens"
},
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"]
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"]
},
"scx-ai": {
"base_url": "https://api.scx.ai/v1",
@ -231,6 +231,16 @@
"api_base_env": "PRISM_API_BASE",
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
},
"hostinger_router": {
"base_url": "https://router.hostinger.com/v1",
"api_key_env": "HOSTINGER_ROUTER_API_KEY",
"api_base_env": "HOSTINGER_ROUTER_API_BASE",
"supported_endpoints": ["/v1/chat/completions", "/v1/embeddings"],
"model_supported_endpoints": {
"claude-sonnet-5": ["/v1/messages"],
"gpt-5.2": ["/v1/responses"]
}
},
"sail": {
"base_url": "https://api.sailresearch.com/v1",
"api_key_env": "SAIL_API_KEY",

View file

@ -110,38 +110,23 @@ def separate_cached_messages(
messages: list[AllMessageValues],
) -> tuple[list[AllMessageValues], list[AllMessageValues]]:
"""
Returns separated cached and non-cached messages.
Args:
messages: List of messages to be separated.
Returns:
Tuple containing:
- cached_messages: List of cached messages.
- non_cached_messages: List of non-cached messages.
Returns the first continuous block of cached messages, plus every system message once
any message is cached, since generateContent rejects a system_instruction beside a
cachedContent, and the remaining messages.
"""
cached_messages: list[AllMessageValues] = []
non_cached_messages: list[AllMessageValues] = []
filtered_messages: Final = [(idx, message) for idx, message in enumerate(messages) if is_cached_message(message)]
if not filtered_messages:
return [], messages
# Extract cached messages and their indices
filtered_messages: Final[list[tuple[int, AllMessageValues]]] = []
for idx, message in enumerate(messages):
if is_cached_message(message=message):
filtered_messages.append((idx, message))
# Validate only one block of continuous cached messages
last_continuous_block_idx: Final = get_first_continuous_block_idx(filtered_messages)
# Separate messages based on the block of cached messages
if filtered_messages and last_continuous_block_idx is not None:
first_cached_idx: Final = filtered_messages[0][0]
last_cached_idx: Final = filtered_messages[last_continuous_block_idx][0]
cached_messages = messages[first_cached_idx : last_cached_idx + 1]
non_cached_messages = messages[:first_cached_idx] + messages[last_cached_idx + 1 :]
else:
non_cached_messages = messages
return cached_messages, non_cached_messages
cached_block: Final = range(filtered_messages[0][0], filtered_messages[last_continuous_block_idx][0] + 1)
goes_in_cache: Final = tuple(
idx in cached_block or message.get("role") == "system" for idx, message in enumerate(messages)
)
return (
[message for message, cached in zip(messages, goes_in_cache) if cached],
[message for message, cached in zip(messages, goes_in_cache) if not cached],
)
def cached_messages_end_on_supported_turn(cached_messages: Sequence[AllMessageValues]) -> bool:

View file

@ -4,6 +4,7 @@ Translation from OpenAI's `/chat/completions` endpoint to IBM WatsonX's `/text/c
Docs: https://cloud.ibm.com/apidocs/watsonx-ai#text-chat
"""
import copy
from typing import Final
from litellm import verbose_logger
@ -57,7 +58,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig):
_tools = non_default_params.pop("tools", None)
if _tools is not None:
# remove 'additionalProperties' from tools
_tools = remove_additional_properties(_tools)
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
# remove 'strict' from tools
_tools = remove_strict_from_schema(_tools)
if _tools is not None:

View file

@ -1,3 +1,4 @@
import copy
from collections.abc import AsyncIterator, Iterator, Mapping
from types import MappingProxyType
from typing import Any, Final
@ -208,7 +209,8 @@ class XAIChatConfig(OpenAIGPTConfig):
tools = [
filtered_tool
for filtered_tool in (
_FILTERED_TOOL.validate_python(filter_value_from_dict(tool, "strict")) for tool in value
_FILTERED_TOOL.validate_python(filter_value_from_dict(copy.deepcopy(tool), "strict")) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
for tool in value
)
if filtered_tool is not None
]

View file

@ -6848,6 +6848,23 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse:
)
def _embedding_headers(param: str, configured: object, model: str, drop_params: bool) -> Mapping[str, object]:
if not configured:
return {}
if isinstance(configured, Mapping):
return dict(cast(Mapping[str, object], configured)) # cast-ok: header names from untyped request JSON
try:
return dict(cast(Iterable[tuple[str, object]], configured)) # cast-ok: name and value pairs from request JSON
except (TypeError, ValueError):
if drop_params:
return {}
raise litellm.UnsupportedParamsError(
message=f"{param} must be a JSON object of header names to values. "
"To drop it from the call, set `litellm.drop_params = True`.",
model=model,
)
# fmt: off
# Overload for when aembedding=True (returns coroutine)
@ -6962,11 +6979,10 @@ def embedding(
azure_ad_token_provider: Final = kwargs.get("azure_ad_token_provider", None)
aembedding: Final[bool | None] = kwargs.get("aembedding", None)
extra_headers: Final = kwargs.get("extra_headers", None)
headers = kwargs.get("headers", None) or extra_headers
if headers is None:
headers = {}
if extra_headers is not None:
headers.update(extra_headers)
headers = {
**_embedding_headers("headers", kwargs.get("headers"), model, drop_unsupported_params),
**_embedding_headers("extra_headers", extra_headers, model, drop_unsupported_params),
}
# Inject proxy auth headers if configured
if litellm.proxy_auth is not None:
try:
@ -7130,7 +7146,7 @@ def embedding(
# set API KEY
api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
if headers is not None and headers != {}:
if headers:
optional_params["extra_headers"] = headers
requested_encoding_format: Final = (
@ -7222,14 +7238,15 @@ def embedding(
custom_llm_provider == "openai_like"
or custom_llm_provider == "llamafile"
or custom_llm_provider == "lm_studio"
or JSONProviderRegistry.supports_embeddings(custom_llm_provider)
):
api_base = api_base or litellm.api_base or get_secret_str("OPENAI_LIKE_API_BASE")
# set API KEY
if api_key is None:
api_key = api_key or litellm.api_key or litellm.openai_like_key or get_secret_str("OPENAI_LIKE_API_KEY")
if api_key is None and not JSONProviderRegistry.supports_embeddings(custom_llm_provider):
api_key = litellm.api_key or litellm.openai_like_key or get_secret_str("OPENAI_LIKE_API_KEY")
if headers is not None and headers != {}:
if headers:
optional_params["extra_headers"] = headers
## EMBEDDING CALL
@ -7244,10 +7261,9 @@ def embedding(
optional_params=optional_params,
client=client,
aembedding=aembedding,
drop_params=drop_unsupported_params,
)
elif custom_llm_provider == "oci":
if headers is None:
headers = {}
response = base_llm_http_handler.embedding(
model=model,
input=input,
@ -7272,11 +7288,6 @@ def embedding(
or litellm.api_key
)
# Use the merged headers variable (already merged at the top of the function)
# Don't overwrite it with just extra_headers
if headers is None:
headers = {}
response = base_llm_http_handler.embedding(
model=model,
input=input,

View file

@ -39687,6 +39687,57 @@
"/v1/images/generations"
]
},
"hostinger_router/gpt-5.2": {
"cache_read_input_token_cost": 0.00000021,
"input_cost_per_token": 0.0000021,
"litellm_provider": "hostinger_router",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 0.0000168,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_vision": true,
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
"source": "https://router.hostinger.com/v1/models"
},
"hostinger_router/claude-sonnet-5": {
"cache_creation_input_token_cost": 3.3e-06,
"cache_creation_input_token_cost_above_1hr": 5.28e-06,
"cache_read_input_token_cost": 2.64e-07,
"input_cost_per_token": 2.64e-06,
"litellm_provider": "hostinger_router",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.32e-05,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_response_schema": true,
"supported_endpoints": ["/v1/chat/completions", "/v1/messages"],
"source": "https://router.hostinger.com/v1/models"
},
"hostinger_router/text-embedding-3-large": {
"input_cost_per_token": 1.716e-07,
"litellm_provider": "hostinger_router",
"max_input_tokens": 8191,
"mode": "embedding",
"output_cost_per_token": 0.0,
"source": "https://router.hostinger.com/v1/models"
},
"hostinger_router/text-embedding-3-small": {
"input_cost_per_token": 3e-08,
"litellm_provider": "hostinger_router",
"max_input_tokens": 8191,
"mode": "embedding",
"output_cost_per_token": 0.0,
"source": "https://router.hostinger.com/v1/models"
},
"hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": {
"input_cost_per_token": 1.2e-07,
"litellm_provider": "hyperbolic",
@ -51387,6 +51438,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 5e-06,
"output_cost_per_token_batches": 2.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -51415,6 +51471,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 5e-06,
"output_cost_per_token_batches": 2.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -52161,6 +52222,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_batches": 7.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -52266,6 +52332,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_batches": 7.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,

View file

@ -1087,6 +1087,32 @@
"generateContent": false
}
},
"hostinger_router": {
"display_name": "Hostinger Router (`hostinger_router`)",
"url": "https://docs.litellm.ai/docs/providers/hostinger_router",
"endpoints": {
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": true,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": false,
"a2a": true,
"generateContent": true
}
},
"crusoe": {
"display_name": "Crusoe (`crusoe`)",
"url": "https://docs.litellm.ai/docs/providers/crusoe",

View file

@ -58015,7 +58015,7 @@
"paths": {
"/toolset/{toolset_name}/mcp": {
"delete": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_delete",
"parameters": [
{
@ -58054,7 +58054,7 @@
]
},
"get": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_get",
"parameters": [
{
@ -58093,7 +58093,7 @@
]
},
"head": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_head",
"parameters": [
{
@ -58132,7 +58132,7 @@
]
},
"options": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_options",
"parameters": [
{
@ -58171,7 +58171,7 @@
]
},
"patch": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_patch",
"parameters": [
{
@ -58210,7 +58210,7 @@
]
},
"post": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_post",
"parameters": [
{
@ -58249,7 +58249,7 @@
]
},
"put": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put",
"parameters": [
{

View file

@ -5031,8 +5031,9 @@ class ProxyBaseLLMRequestProcessing:
if stream_usage is None:
return None
service_tier: Final = obj.get("service_tier")
served_model: Final = litellm_logging_obj.model if litellm_logging_obj is not None else None
cost_val: Final = ProxyBaseLLMRequestProcessing._streamed_usage_cost(
ModelResponse(usage=stream_usage),
ModelResponse(model=served_model, usage=stream_usage),
model_name,
service_tier if isinstance(service_tier, str) else None,
litellm_logging_obj,

View file

@ -31,6 +31,13 @@ class AccountingConflictError(ValueError):
pass
@dataclass(frozen=True, slots=True)
class UsageWrite:
event: InboundUsageEvent | OutboundUsageEvent
envelope: str
targets: tuple[DeliveryTarget, ...] = ()
_Transaction: TypeAlias = AbstractAsyncContextManager["Prisma"]
@ -107,6 +114,35 @@ SELECT event_id FROM "LiteLLM_AccountingEvent" WHERE kind = 'cost' AND source_ev
_LOCK_EVENT: Final = """
SELECT payload_hash FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 FOR UPDATE
"""
_INSERT_USAGE_BATCH: Final = """
INSERT INTO "LiteLLM_AccountingEvent"
(event_id, kind, schema_version, payload_hash, occurred_at, payload)
SELECT event_id, kind::"AccountingEventKind", 1, payload_hash, occurred_at::timestamp, payload
FROM jsonb_to_recordset($1::jsonb) AS records(
event_id text, kind text, payload_hash text, occurred_at text, payload jsonb)
ORDER BY event_id
ON CONFLICT (event_id) DO NOTHING RETURNING event_id
"""
_LOCK_USAGE_BATCH: Final = """
SELECT event_id, payload_hash FROM "LiteLLM_AccountingEvent"
WHERE event_id IN (SELECT value FROM jsonb_array_elements_text($1::jsonb))
ORDER BY event_id FOR UPDATE
"""
_LOAD_USAGE_BATCH: Final = """
SELECT event.payload, ARRAY(
SELECT target::text FROM "LiteLLM_AccountingDelivery" delivery
WHERE delivery.event_id = event.event_id ORDER BY target::text
) AS targets
FROM "LiteLLM_AccountingEvent" event
WHERE event.event_id IN (SELECT value FROM jsonb_array_elements_text($1::jsonb))
AND event.kind = 'outbound_usage' AND event.schema_version = 1
"""
_INSERT_USAGE_DELIVERIES: Final = """
INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version)
SELECT records.event_id, destination.value::"AccountingDeliveryTarget", 1
FROM jsonb_to_recordset($1::jsonb) AS records(event_id text, targets jsonb)
CROSS JOIN LATERAL jsonb_array_elements_text(records.targets) AS destination
"""
_COMPLETED_FAMILY: Final = """
root.source_event_id IS NULL AND root.kind IN ('inbound_usage', 'outbound_usage')
AND root.recorded_at < $1::timestamp
@ -199,6 +235,53 @@ class AccountingLedger:
return cls(transaction)
async def insert_usage_batch(self, writes: tuple[UsageWrite, ...]) -> None:
records: Final = tuple(_prepare(write.event, write.targets, write.envelope, 1, 1) for write in writes)
if not records:
return
unique: Final = {record.event_id: record for record in records}
if any(record.payload_hash != unique[record.event_id].payload_hash for record in records):
raise AccountingConflictError("Batch contains conflicting copies of a usage identity")
batch: Final = _json(
tuple(
{
"event_id": record.event_id,
"kind": record.kind.value,
"payload_hash": record.payload_hash,
"occurred_at": record.occurred_at.astimezone(timezone.utc).isoformat(),
"payload": _JSON.validate_json(record.payload),
}
for record in unique.values()
)
)
async with self._transaction() as db:
inserted: Final = _STRINGS.validate_python(await db.query_raw(_INSERT_USAGE_BATCH, batch))
accepted: Final = _STRINGS.validate_python(await db.query_raw(_LOCK_USAGE_BATCH, _json(tuple(unique))))
if len(accepted) != len(unique) or any(
row["payload_hash"] != unique[row["event_id"]].payload_hash for row in accepted
):
raise AccountingConflictError("Usage identity conflicts with accepted content or delivery set")
deliveries: Final = _json(
tuple({"event_id": row["event_id"], "targets": unique[row["event_id"]].targets} for row in inserted)
)
await db.execute_raw(_INSERT_USAGE_DELIVERIES, deliveries)
async def load_usage_batch(self, event_ids: tuple[str, ...]) -> tuple[UsageWrite, ...]:
if not event_ids:
return ()
async with self._transaction() as db:
rows: Final = TypeAdapter(tuple[dict[str, JsonValue], ...]).validate_python(
await db.query_raw(_LOAD_USAGE_BATCH, _json(event_ids))
)
return tuple(
UsageWrite(
OutboundUsageEvent.model_validate(_JSON.validate_python(row["payload"])["event"]),
_json(row["payload"]),
TypeAdapter(tuple[DeliveryTarget, ...]).validate_python(row["targets"]),
)
for row in rows
)
async def insert_event(
self,
event: AccountingEvent,

View file

@ -72,6 +72,22 @@ def _exception_types(*candidates: object) -> tuple[type[BaseException], ...]:
return tuple(c for c in candidates if isinstance(c, type) and issubclass(c, BaseException))
def _prisma_error_code(e: Exception) -> str | None:
import prisma.engine.errors
if isinstance(e, _exception_types(prisma.errors.DataError)):
code: Final[object] = getattr(e, "code", None)
return code if isinstance(code, str) else None
if not isinstance(e, _exception_types(prisma.engine.errors.EngineRequestError)):
return None
try:
body: Final = _DATABASE_ERROR_META.validate_json(str(e).partition(": ")[2])
except ValidationError:
return None
engine_code: Final = body.get("error_code")
return engine_code if isinstance(engine_code, str) else None
class PrismaDBExceptionHandler:
"""
Class to handle DB Exceptions or Connection Errors
@ -206,6 +222,8 @@ class PrismaDBExceptionHandler:
return True
if PrismaDBExceptionHandler.is_database_capacity_error(e):
return False
if _prisma_error_code(e) == "P1017":
return True
if isinstance(e, _exception_types(prisma.errors.PrismaError)):
error_message: Final = str(e).lower()
connection_keywords: Final = (

View file

@ -1,14 +1,15 @@
# What is this?
## Helper utils for the management endpoints (keys/users/teams)
from collections.abc import Callable, Mapping, MutableMapping, Sequence
from collections.abc import Awaitable, Callable, Mapping, MutableMapping, Sequence
from collections.abc import Set as AbstractSet
from datetime import datetime
from functools import wraps
from types import MappingProxyType
from typing import Any, Final, Protocol
from typing import Any, Final, ParamSpec, Protocol, TypeVar
from fastapi import HTTPException, Request
from pydantic import BaseModel, TypeAdapter
from starlette.exceptions import HTTPException as StarletteHTTPException
import litellm
from litellm._internal_context import with_service_target
@ -687,6 +688,7 @@ async def _emit_management_endpoint_otel_span(
end_time: datetime,
result: Any = None,
exception: Exception | None = None,
log_payload: bool = True,
) -> None:
"""Stamp + end the parent OTEL SERVER span for a management endpoint.
@ -716,8 +718,8 @@ async def _emit_management_endpoint_otel_span(
)
route = get_request_route(http_request)
request_body: dict = await read_request_body( # rebind-ok: pre-existing rebinding on a rename-only line
request=http_request
request_body: dict = ( # rebind-ok: pre-existing rebinding on a rename-only line
await read_request_body(request=http_request) if log_payload else {}
)
else:
route = func.__name__
@ -738,7 +740,7 @@ async def _emit_management_endpoint_otel_span(
)
_response: dict[str, object] | None = None
if exception is None and result is not None:
if log_payload and exception is None and result is not None:
try:
raw: Final[Mapping[str, object]] = dict(result)
_response = {k: v for k, v in raw.items() if k not in _CREDENTIAL_FIELDS}
@ -767,37 +769,60 @@ async def _emit_management_endpoint_otel_span(
)
def management_endpoint_wrapper(func):
_P: Final = ParamSpec("_P")
_R: Final = TypeVar("_R")
def management_endpoint_wrapper(func: Callable[_P, Awaitable[_R]]) -> Callable[_P, Awaitable[_R]]:
"""
This wrapper does the following:
1. Log I/O, Exceptions to OTEL
2. Create an Audit log for success calls
"""
return _wrap_management_endpoint(func, log_payload=True)
@wraps(func)
async def wrapper(*args, **kwargs):
def management_endpoint_span_wrapper(func: Callable[_P, Awaitable[_R]]) -> Callable[_P, Awaitable[_R]]:
return _wrap_management_endpoint(func, log_payload=False)
def _caller_auth(endpoint_kwargs: Mapping[str, object]) -> UserAPIKeyAuth:
caller: Final = endpoint_kwargs.get("user_api_key_dict")
return caller if isinstance(caller, UserAPIKeyAuth) else UserAPIKeyAuth()
def _proxy_exception_handler_ends_server_span(exception: Exception) -> bool:
return not isinstance(exception, StarletteHTTPException)
def _wrap_management_endpoint(
endpoint: Callable[_P, Awaitable[_R]], *, log_payload: bool
) -> Callable[_P, Awaitable[_R]]:
@wraps(endpoint)
async def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R:
start_time: Final = datetime.now()
try:
result: Final = await func(*args, **kwargs)
result: Final = await endpoint(*args, **kwargs)
end_time = datetime.now()
try:
user_api_key_dict: UserAPIKeyAuth = kwargs.get("user_api_key_dict") or UserAPIKeyAuth()
user_api_key_dict: UserAPIKeyAuth = _caller_auth(kwargs)
await send_management_endpoint_alert(
request_kwargs=kwargs,
user_api_key_dict=user_api_key_dict,
function_name=func.__name__,
function_name=endpoint.__name__,
)
parent_otel_span: Span | None = getattr(user_api_key_dict, "parent_otel_span", None)
if parent_otel_span is not None:
await _emit_management_endpoint_otel_span(
func=func,
func=endpoint,
kwargs=kwargs,
parent_otel_span=parent_otel_span,
start_time=start_time,
end_time=end_time,
result=result,
log_payload=log_payload,
)
# Delete updated/deleted info from cache
@ -813,17 +838,18 @@ def management_endpoint_wrapper(func):
except Exception as e:
end_time = datetime.now()
user_api_key_dict: UserAPIKeyAuth = kwargs.get("user_api_key_dict") or UserAPIKeyAuth()
user_api_key_dict: UserAPIKeyAuth = _caller_auth(kwargs)
parent_otel_span = getattr(user_api_key_dict, "parent_otel_span", None)
if parent_otel_span is not None:
if parent_otel_span is not None and (log_payload or not _proxy_exception_handler_ends_server_span(e)):
try:
await _emit_management_endpoint_otel_span(
func=func,
func=endpoint,
kwargs=kwargs,
parent_otel_span=parent_otel_span,
start_time=start_time,
end_time=end_time,
exception=e,
log_payload=log_payload,
)
except Exception as otel_exc:
# Non-Blocking Exception - never let OTEL failures swallow

View file

@ -438,6 +438,14 @@ def is_mcp_admitted_user_subject(user_api_key_auth: UserAPIKeyAuth | None) -> bo
_is_mcp_admitted_user_subject: Final = is_mcp_admitted_user_subject
def has_team_unbound_mcp_admin_view(user_api_key_auth: UserAPIKeyAuth | None) -> bool:
return (
user_api_key_auth is not None
and user_api_key_has_admin_view(user_api_key_auth)
and (not user_api_key_auth.team_id or user_api_key_auth.team_id == UI_TEAM_ID)
)
def _gateway_dcr_challenge_target(
route: str,
mcp_servers: list[str] | None,
@ -4290,19 +4298,19 @@ class MCPRequestHandler:
"""Whether this principal's admin-view role grants the unscoped MCP resolution, whatever
credential carries it (admin key, dashboard session, or OAuth-admitted session subject).
Two bounds disqualify, one per ownership of the row. A CREDENTIAL's explicit
``object_permission.mcp_servers`` scope wins even for admins, including the empty list. An
admitted subject's object_permission is the user's own row, whose ``mcp_servers`` column is
[] by DB default, so for that shape the row binds through the entitlement ceiling instead
(any non-empty entitlement, or an unresolved one, disqualifies), exactly as
``operator_open_server_ids`` reads the same row. The one owner of this predicate: the
server-axis registry resolution in ``get_allowed_mcp_servers`` and the tools-axis open
channel in ``resolve_admitted_subject_tools`` both consult it, so the two axes cannot
disagree."""
A credential on a real team never qualifies, a virtual key or a ``lite login`` session alike: its team
bounds its MCP access whoever owns it, as it bounds the credential's model access. Two more bounds disqualify, one per ownership of the row. A CREDENTIAL's
explicit ``object_permission.mcp_servers`` scope wins even for admins, including the empty list. An
admitted subject's object_permission is the user's own row, whose ``mcp_servers`` column is [] by DB
default, so for that shape the row binds through the entitlement ceiling instead (any non-empty
entitlement, or an unresolved one, disqualifies), exactly as ``operator_open_server_ids`` reads the same
row. The one owner of this predicate: the server-axis registry resolution in ``get_allowed_mcp_servers``
and the tools-axis open channel in ``resolve_admitted_subject_tools`` both consult it, so the two axes
cannot disagree."""
if (
user_api_key_auth is None
or user_api_key_auth.mcp_explicit_grants_only
or not user_api_key_has_admin_view(user_api_key_auth)
or not has_team_unbound_mcp_admin_view(user_api_key_auth)
):
return False
object_permission: Final = user_api_key_auth.object_permission

View file

@ -3705,7 +3705,7 @@ class MCPServerManager:
Priority:
1. If object_permission.mcp_servers is explicitly set, use it (even for admins)
2. If admin and no object_permission, return all servers
2. If admin, not bound to a real team, and no object_permission, return all servers
3. Otherwise, use standard permission checks
"""
if managed_agent_policy(user_api_key_auth) is not None:

View file

@ -223,6 +223,7 @@ __all__ = (
"mcp_get_prompt",
"mcp_read_resource",
"raise_denied_scoped_mcp_access",
"raise_generic_mcp_access_denial",
)
@ -763,8 +764,14 @@ async def raise_denied_scoped_mcp_access(
)
}
raise HTTPException(status_code=403, detail=group_denial)
raise_generic_mcp_access_denial(requested_names)
def raise_generic_mcp_access_denial(requested_names: Sequence[str]) -> NoReturn:
generic_denial: Final[_McpDeniedDetail] = {
"error": f"The key is not allowed to access the requested MCP servers: {', '.join(requested_names)}"
"error": (
f"The key is not allowed to access the requested MCP servers: {', '.join(dict.fromkeys(requested_names))}"
)
}
raise HTTPException(status_code=403, detail=generic_denial)

View file

@ -520,6 +520,7 @@ if MCP_AVAILABLE:
"mcp_get_prompt",
"mcp_read_resource",
"raise_denied_scoped_mcp_access",
"raise_generic_mcp_access_denial",
"redact_mcp_resource_url",
)
from mcp.server import Server
@ -1197,6 +1198,7 @@ if MCP_AVAILABLE:
apply_display_name_overrides,
filter_tools_by_allowed_tools,
raise_denied_scoped_mcp_access,
raise_generic_mcp_access_denial,
)
@contextlib.asynccontextmanager
@ -1824,10 +1826,10 @@ if MCP_AVAILABLE:
row capping it.
Raises HTTPException(403) unless the principal holds toolset_id through one
of its grant sources. Admins always pass.
of its grant sources. Admins pass unless the key is bound to a real team.
"""
from litellm.proxy._types import LiteLLM_ObjectPermissionTable
from litellm.proxy.management_endpoints.common_utils import user_api_key_has_admin_view
from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import has_team_unbound_mcp_admin_view
# A key scoped to no MCP servers opts out of every MCP path. Enforce it
# here too, since toolset scoping replaces mcp_servers and would otherwise
@ -1841,7 +1843,7 @@ if MCP_AVAILABLE:
)
acting: Final = await acting_user(user_api_key_auth)
is_admin: Final = user_api_key_has_admin_view(acting)
is_admin: Final = has_team_unbound_mcp_admin_view(acting)
if not is_admin and toolset_id not in await granted(acting):
raise HTTPException(
status_code=403,

View file

@ -52,7 +52,11 @@ from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming, is_guardrail_intervention
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.fallback_generalizations import match_routing_generalization
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers
from litellm.llms.anthropic.common_utils import (
AnthropicModelInfo,
anthropic_pass_through_api_base,
merge_anthropic_beta_headers,
)
from litellm.llms.azure.passthrough.transformation import (
foreign_azure_deployment,
is_azure_body_model_inference_endpoint,
@ -1047,7 +1051,7 @@ async def anthropic_proxy_route(
"""
[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)
"""
base_target_url = os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
base_target_url = anthropic_pass_through_api_base()
encoded_endpoint = httpx.URL(endpoint).path
# Ensure endpoint starts with '/' for proper URL construction

View file

@ -71,7 +71,11 @@ from litellm.litellm_core_utils.litellm_logging import get_masked_values
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.anthropic.common_utils import is_anthropic_messages_url
from litellm.llms.anthropic.common_utils import (
anthropic_accepts_user_id,
anthropic_upstream_user_id_limit,
is_anthropic_messages_url,
)
from litellm.llms.base_llm.managed_resources.utils import (
resolve_passthrough_managed_id_provider,
)
@ -148,6 +152,8 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
EndpointType,
PassthroughStandardLoggingPayload,
WebSocketSessionMeter,
WebSocketSessionUsage,
)
from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, CallTypes, Usage
@ -501,7 +507,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
url: httpx.URL,
headers: dict,
requested_query_params: dict | None = None,
_parsed_body: dict | None = None,
_parsed_body: Mapping[str, object] | None = None,
forward_multipart: bool = False,
) -> httpx.Response:
"""
@ -715,8 +721,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
# unenforced. Set after the client merge so a request body cannot supply its own budget.
#
# Only for the built-in provider routes. `get_model_from_request` returns
# None for a user-defined pass-through, deliberately: its body is forwarded
# verbatim, so `model` there names an UPSTREAM model rather than a
# None for a user-defined pass-through, deliberately: the client's `model`
# goes upstream unchanged, so it names an UPSTREAM model rather than a
# LiteLLM-managed one. Enforcement is therefore skipped on those routes, and
# charging a counter anyway would track spend that nothing can refuse, and
# would attribute it to a budget the operator scoped to a LiteLLM model that
@ -1185,6 +1191,36 @@ def _unfiltered_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
return headers
def _json_object(value: object) -> Mapping[str, object]:
return cast("Mapping[str, object]", value) if isinstance(value, Mapping) else MappingProxyType({}) # cast-ok: json
def _anthropic_user_id_to_forward(
body: Mapping[str, object],
url: str,
has_pre_call_guardrails: Callable[[Mapping[str, object]], bool],
) -> str | None:
user_id: Final = _json_object(body.get("metadata")).get("user_id")
if not isinstance(user_id, str):
return None
max_length: Final = anthropic_upstream_user_id_limit(url)
if max_length is None:
return None
if not anthropic_accepts_user_id(user_id, max_length):
verbose_proxy_logger.warning("Not forwarding metadata.user_id because the upstream would refuse it")
return None
if has_pre_call_guardrails(body):
verbose_proxy_logger.debug("Not forwarding metadata.user_id because a pre-call guardrail cannot scan it")
return None
return user_id
def _with_anthropic_user_id(body: Mapping[str, object] | None, user_id: str | None) -> Mapping[str, object] | None:
if body is None or user_id is None:
return body
return {**body, "metadata": {"user_id": user_id}}
def _eu_jev_client_body(custom_llm_provider: str | None, body: Mapping[str, object]) -> Mapping[str, object] | None:
return MappingProxyType(dict(body)) if custom_llm_provider == "eu_jev" else None
@ -1421,6 +1457,10 @@ async def pass_through_request(
request_method=getattr(request, "method", None),
cost_per_request=cost_per_request,
)
hooked_body: Final = _json_object(_parsed_body)
client_user_id: Final = _anthropic_user_id_to_forward(
hooked_body, str(url), proxy_logging_obj.request_has_pre_call_guardrails
)
kwargs = HttpPassThroughEndpointHelpers.init_kwargs_for_pass_through_endpoint( # rebind-ok: pre-existing rebinding on a rename-only line
user_api_key_dict=user_api_key_dict,
_parsed_body=_parsed_body,
@ -1588,11 +1628,13 @@ async def pass_through_request(
else:
logging_url = str(url) + "?" + requested_query_params_str
outgoing_body: Final = cast("Mapping[str, object] | None", _parsed_body) # cast-ok: json
upstream_body: Final = _with_anthropic_user_id(outgoing_body, client_user_id)
logging_obj.pre_call(
input=[{"role": "user", "content": safe_dumps(_parsed_body)}],
api_key="",
additional_args={
"complete_input_dict": _parsed_body,
"complete_input_dict": upstream_body,
"api_base": str(logging_url),
"headers": redact_credential_headers(upstream_headers),
},
@ -1632,7 +1674,7 @@ async def pass_through_request(
url,
params=requested_query_params,
headers=upstream_headers,
json=_parsed_body,
json=upstream_body,
)
)
@ -1731,7 +1773,7 @@ async def pass_through_request(
url=url,
headers=upstream_headers,
requested_query_params=requested_query_params,
_parsed_body=_parsed_body,
_parsed_body=upstream_body,
forward_multipart=is_multipart,
)
verbose_proxy_logger.debug("response.headers= %s", response.headers)
@ -2595,6 +2637,13 @@ def _with_trace_context(headers: Mapping[str, str], parent_span: object) -> dict
return inject_trace_context(headers, parent_span=parent_span)
def _log_session_usage(logging_obj: LiteLLMLoggingObj, usage: WebSocketSessionUsage) -> None:
logging_obj.model = usage.model # rebind-ok: the spend logger reads model and cost off the shared logging object
logging_obj.model_call_details["model"] = usage.model # rebind-ok: same shared logging object
logging_obj.model_call_details["custom_llm_provider"] = usage.custom_llm_provider # rebind-ok: same
logging_obj.model_call_details["response_cost"] = usage.response_cost # rebind-ok: same
async def websocket_passthrough_request(
websocket: WebSocket,
target: str,
@ -2607,6 +2656,7 @@ async def websocket_passthrough_request(
setup_model_rewriter: Callable[[str], str] | None = None,
builtin_provider_route: bool = False,
model: str | None = None,
meter: WebSocketSessionMeter | None = None,
):
"""
WebSocket passthrough request handler.
@ -2622,6 +2672,7 @@ async def websocket_passthrough_request(
setup_model_rewriter: Optional rewrite of the setup frame's model before it reaches the upstream
builtin_provider_route: Whether a built-in provider route serves the connection, so its URL names the provider
model: The model the caller named before connecting, logged on a failure before the setup frame arrives
meter: Prices the session from the client text frames delivered upstream, for an upstream that reports no usage
"""
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy.proxy_server import proxy_config, proxy_logging_obj
@ -2632,6 +2683,7 @@ async def websocket_passthrough_request(
# Initialize tracking variables
start_time: Final = datetime.now()
websocket_messages: Final[list[dict[str, object]]] = []
metered_total = 0 # rebind-ok: forward_client_to_upstream adds the measure of each client text frame it delivers
litellm_call_id: Final = str(uuid.uuid4())
verbose_proxy_logger.info("WebSocket passthrough (%s): Starting WebSocket connection to %s", endpoint, target)
@ -2750,6 +2802,7 @@ async def websocket_passthrough_request(
async def forward_client_to_upstream() -> None:
"""Forward messages from client to upstream WebSocket"""
nonlocal metered_total
try:
while True:
message = await websocket.receive()
@ -2819,6 +2872,8 @@ async def websocket_passthrough_request(
# Not a JSON message or doesn't contain setup data
await upstream_ws.send(_rewrite_vertex_live_setup_model(text_data, setup_model_rewriter))
if meter is not None:
metered_total += meter.measure(text_data)
elif bytes_data is not None:
await upstream_ws.send(bytes_data)
except asyncio.CancelledError:
@ -2945,6 +3000,8 @@ async def websocket_passthrough_request(
self.url = target_url
mock_response: Final = MockWebSocketResponse(target)
if meter is not None:
_log_session_usage(logging_obj, meter.price(metered_total))
# Use the same success handler as HTTP passthrough endpoints
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(

View file

@ -181,6 +181,7 @@ from litellm.proxy.common_utils.token_count_settings import (
)
from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog
from litellm.proxy.management_helpers.model_group_custom_fields import model_group_custom_fields
from litellm.proxy.management_helpers.utils import management_endpoint_span_wrapper
from litellm.repositories import database as database_runtime
from litellm.router_strategy.auto_router.auto_router_model_naming import (
GATED_AUTO_ROUTER_CAPABILITIES,
@ -12339,6 +12340,7 @@ async def _deployment_hidden_by_listing_callbacks(deployment: Deployment, user_a
@router.get(
"/models", dependencies=[Depends(user_api_key_auth)], tags=["model management"]
) # if project requires model list
@management_endpoint_span_wrapper
async def model_list(
request: Request = None, # pyright: ignore[reportArgumentType] # FastAPI always injects the Request; the None default only serves direct in-process callers
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
@ -12649,6 +12651,7 @@ async def model_list(
dependencies=[Depends(user_api_key_auth)],
tags=["model management"],
)
@management_endpoint_span_wrapper
async def model_info(
model_id: str,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
@ -21985,9 +21988,9 @@ async def toolset_mcp_route(toolset_name: str, request: Request):
Namespace a toolset as its own MCP endpoint.
Connecting to /toolset/<name>/mcp exposes exactly the tools defined in
the toolset. Access is enforced: non-admin API keys must have the toolset
listed in their object_permission.mcp_toolsets grant list, or the request
will be rejected with a 403.
the toolset. Access is enforced: API keys other than an admin's key with no
team must hold the toolset through one of their grant sources, or the
request will be rejected with a 403.
"""
try:
from litellm.proxy.mcp_server.mcp_server_manager import (

View file

@ -3343,6 +3343,34 @@
],
"default_model_placeholder": "prism/deepseek-v4.1-flash"
},
{
"provider": "HOSTINGER_ROUTER",
"provider_display_name": "Hostinger Router",
"litellm_provider": "hostinger_router",
"credential_fields": [
{
"key": "api_base",
"label": "API Base",
"placeholder": "https://router.hostinger.com/v1",
"tooltip": null,
"required": false,
"field_type": "text",
"options": null,
"default_value": null
},
{
"key": "api_key",
"label": "API Key",
"placeholder": null,
"tooltip": null,
"required": true,
"field_type": "password",
"options": null,
"default_value": null
}
],
"default_model_placeholder": "hostinger_router/claude-sonnet-5"
},
{
"provider": "RECRAFT",
"provider_display_name": "Recraft",

View file

@ -0,0 +1,268 @@
from __future__ import annotations
import asyncio
from collections import deque
from collections.abc import Awaitable, Callable
from concurrent.futures import Future
from dataclasses import dataclass
from itertools import accumulate, chain, count, takewhile
from threading import Lock
from typing import Final
from pydantic import BaseModel, ConfigDict, Field
from litellm._logging import verbose_proxy_logger
from litellm.accounting import (
DeliveryTarget,
Dimension,
DimensionName,
InboundUsageEvent,
OutboundUsageEvent,
)
from litellm.proxy.db.accounting_ledger import AccountingConflictError, UsageWrite
_DAILY_TARGETS: Final = (
(DimensionName.KEY, DeliveryTarget.DAILY_USER_SPEND),
(DimensionName.USER, DeliveryTarget.DAILY_USER_SPEND),
(DimensionName.TEAM, DeliveryTarget.DAILY_TEAM_SPEND),
(DimensionName.ORGANIZATION, DeliveryTarget.DAILY_ORGANIZATION_SPEND),
(DimensionName.END_USER, DeliveryTarget.DAILY_END_USER_SPEND),
(DimensionName.AGENT, DeliveryTarget.DAILY_AGENT_SPEND),
(DimensionName.TAG, DeliveryTarget.DAILY_TAG_SPEND),
(DimensionName.MODEL, DeliveryTarget.DAILY_MODEL_USAGE),
(DimensionName.TOOL, DeliveryTarget.DAILY_TOOL_SPEND),
)
def usage_delivery_targets(dimensions: tuple[Dimension, ...]) -> tuple[DeliveryTarget, ...]:
names: Final = frozenset(dimension.name for dimension in dimensions)
return tuple(dict.fromkeys(target for dimension, target in _DAILY_TARGETS if dimension in names))
class UsageCaptureSettings(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
batch_size: int = Field(default=1000, ge=2, le=1000)
flush_interval_seconds: float = Field(default=5.0, gt=0, le=10)
max_pending_bytes: int = Field(default=16 * 1024 * 1024, ge=1024)
max_pending_records: int = Field(default=10000, ge=2)
class UsageCaptureError(RuntimeError):
pass
@dataclass(frozen=True, slots=True)
class _Pending:
writes: tuple[UsageWrite, ...]
size: int
committed: Future[None]
finished: asyncio.Event
prerequisites: tuple[asyncio.Future[None], ...]
async def _wait_for_batch(wakeup: asyncio.Event, timeout: float) -> None:
try:
await asyncio.wait_for(wakeup.wait(), timeout)
except asyncio.TimeoutError:
return
def _report_failure(receipt: Future[None]) -> None:
if not receipt.cancelled() and receipt.exception() is not None:
verbose_proxy_logger.error("Usage capture did not acknowledge commit", exc_info=receipt.exception())
def _retryable_persistence_failure(error: Exception) -> bool:
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain
outage: Final = PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(error)
return (outage is not None and not PrismaDBExceptionHandler.is_permanent_database_fault(outage)) or any(
isinstance(cause, Exception)
and (
PrismaDBExceptionHandler.is_deadlock_error(cause)
or PrismaDBExceptionHandler.is_lock_timeout_error(cause)
or PrismaDBExceptionHandler.postgres_sqlstate(cause) == "40001"
)
for cause in exception_chain(error)
)
def _isolatable_persistence_failure(error: Exception) -> bool:
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain
return isinstance(error, AccountingConflictError) or any(
isinstance(cause, Exception) and (PrismaDBExceptionHandler.postgres_sqlstate(cause) or "").startswith("22")
for cause in exception_chain(error)
)
class DurableUsageMeter:
def __init__(
self,
write_batch: Callable[[tuple[UsageWrite, ...]], Awaitable[None]],
settings: UsageCaptureSettings,
*,
wait_for_batch: Callable[[asyncio.Event, float], Awaitable[None]] = _wait_for_batch,
retry_delay: Callable[[float], Awaitable[None]] = asyncio.sleep,
) -> None:
self._write_batch: Final = write_batch
self.settings: Final = settings
self._wait_for_batch: Final = wait_for_batch
self._retry_delay: Final = retry_delay
self._loop: Final = asyncio.get_running_loop()
self._admission: Final = Lock()
self._queue: Final[deque[_Pending]] = deque() # mutable-ok: one bounded producer/consumer queue
self._wakeup: Final = asyncio.Event()
self._task: asyncio.Task[None] | None = None
self._closed = False
self.pending_bytes = 0
self.pending_records = 0
def submit(
self, writes: tuple[UsageWrite, ...], *, prerequisites: tuple[asyncio.Future[None], ...] = ()
) -> asyncio.Future[None]:
size: Final = sum(len(write.envelope.encode("utf-8")) for write in writes)
with self._admission:
if (
self._closed
or not writes
or len(writes) > self.settings.batch_size
or self.pending_bytes + size > self.settings.max_pending_bytes
or self.pending_records + len(writes) > self.settings.max_pending_records
):
raise UsageCaptureError("Usage capture admission rejected: closed, empty or capacity exhausted")
receipt: Final = Future[None]()
receipt.set_running_or_notify_cancel()
receipt.add_done_callback(_report_failure)
self._queue.append(_Pending(writes, size, receipt, asyncio.Event(), prerequisites))
self.pending_bytes += size
self.pending_records += len(writes)
self._loop.call_soon_threadsafe(self._ensure_worker)
wrapped: Final = asyncio.wrap_future(receipt, loop=self._loop)
wrapped.add_done_callback(lambda done: None if done.cancelled() else done.exception())
return wrapped
def _ensure_worker(self) -> None:
if self.pending_records >= self.settings.batch_size or self._queue and self._queue[0].prerequisites:
self._wakeup.set()
if self._task is None or self._task.done():
self._task = asyncio.create_task(self._run(), name="accounting usage capture")
async def record(self, event: InboundUsageEvent | OutboundUsageEvent) -> None:
envelope: Final = '{"event":' + event.model_dump_json() + "}"
await asyncio.shield(self.submit((UsageWrite(event, envelope, usage_delivery_targets(event.dimensions)),)))
async def flush(self) -> None:
self._ensure_worker()
self._wakeup.set()
if self._task is not None:
await asyncio.shield(self._task)
async def close(self) -> None:
with self._admission:
self._closed = True
await self.flush()
async def stop(self, timeout: float) -> None:
with self._admission:
self._closed = True
try:
await asyncio.wait_for(self.close(), timeout)
except asyncio.TimeoutError:
if self._task is not None:
self._task.cancel()
await asyncio.gather(self._task, return_exceptions=True)
while self._queue:
self._finish(
self._take_batch(), UsageCaptureError("Usage capture stopped before commit acknowledgement")
)
verbose_proxy_logger.error("Usage capture shutdown deadline exhausted; commit acknowledgement incomplete")
def _take_batch(self) -> tuple[_Pending, ...]:
with self._admission:
if self._queue[0].prerequisites:
return (self._queue.popleft(),)
independent: Final = takewhile(lambda item: not item.prerequisites, self._queue)
sizes: Final = accumulate(len(item.writes) for item in independent)
count: Final = sum(1 for _ in takewhile(lambda size: size <= self.settings.batch_size, sizes))
return tuple(self._queue.popleft() for _ in range(count))
async def _persist(self, writes: tuple[UsageWrite, ...]) -> None:
for attempt in count():
try:
await self._write_batch(writes)
except Exception as error: # noqa: BLE001 # retry only classified database failures, preserving frozen writes
if not _retryable_persistence_failure(error):
raise
verbose_proxy_logger.warning("Usage persistence unavailable; retaining admitted batch for retry")
await self._retry_delay(min(10.0, 0.2 * 2 ** min(attempt, 6)))
else:
return
async def _commit(self, batch: tuple[_Pending, ...]) -> None:
writes: Final = tuple(chain.from_iterable(item.writes for item in batch))
try:
if batch[0].prerequisites:
outcomes: Final = await asyncio.gather(
*(asyncio.shield(receipt) for receipt in batch[0].prerequisites), return_exceptions=True
)
for outcome in outcomes:
if isinstance(outcome, BaseException):
raise UsageCaptureError("Usage dependency did not acknowledge commit") from outcome
await self._persist(writes)
except asyncio.CancelledError:
self._finish(batch, UsageCaptureError("Usage commit acknowledgement interrupted"))
raise
except Exception as error: # noqa: BLE001 # isolate data faults, surface systemic failures to all producers
if len(batch) > 1 and _isolatable_persistence_failure(error):
try:
for item in batch:
await self._commit((item,))
except asyncio.CancelledError:
self._finish(batch, UsageCaptureError("Usage commit acknowledgement interrupted"))
raise
return
self._finish(
batch,
UsageCaptureError("Usage identity conflicts with accepted content")
if isinstance(error, AccountingConflictError)
else error,
)
else:
self._finish(batch, None)
def _finish(self, batch: tuple[_Pending, ...], error: Exception | None) -> None:
for item in batch:
if item.finished.is_set():
continue
item.finished.set()
with self._admission:
self.pending_bytes -= item.size
self.pending_records -= len(item.writes)
if item.committed.done():
continue
if error is None:
item.committed.set_result(None)
else:
item.committed.set_exception(error)
async def _run(self) -> None:
try:
while self._queue:
await self._wait_for_batch(self._wakeup, self.settings.flush_interval_seconds)
self._wakeup.clear()
await self._commit(self._take_batch())
if (
self._closed
or self.pending_records >= self.settings.batch_size
or self._queue
and self._queue[0].prerequisites
):
self._wakeup.set()
except (Exception, asyncio.CancelledError): # noqa: BLE001 # reject queued receipts if the writer exits abnormally
while self._queue:
self._finish(
self._take_batch(), UsageCaptureError("Usage capture stopped before commit acknowledgement")
)
raise

View file

@ -3031,14 +3031,21 @@ class ProxyLogging:
hook to consult, but it judges the payload the same way, so a proxy configured only with
one of those still has something to say about every record.
"""
if request_metadata.get("_guardrail_pipelines"):
return self.request_has_pre_call_guardrails({"metadata": dict(request_metadata)}, event_type)
def request_has_pre_call_guardrails(
self,
request_data: Mapping[str, object],
event_type: GuardrailEventHooks = GuardrailEventHooks.pre_call,
) -> bool:
if _policy_state_metadata(request_data).get("_guardrail_pipelines"):
return True
caps: Final = ProxyLogging.callback_capabilities()
if caps.has_content_enforcer:
return True
probe: Final = {"metadata": dict(request_metadata)}
return any(
isinstance(callback, CustomGuardrail) and callback.should_run_guardrail(data=probe, event_type=event_type)
isinstance(callback, CustomGuardrail)
and callback.should_run_guardrail(data=request_data, event_type=event_type)
for callback in caps.resolved_callbacks
)

View file

@ -34,6 +34,14 @@ from litellm.types.utils import ModelResponse
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object])
def _chat_verbosity(responses_api_request: Mapping[str, object]) -> Mapping[str, object]:
match responses_api_request.get("text"):
case {"verbosity": str() as verbosity}:
return {"verbosity": verbosity}
case _:
return {}
@dataclass(frozen=True, slots=True)
class _BridgedRequest:
input: str | ResponseInputParam
@ -96,8 +104,8 @@ class LiteLLMCompletionTransformationHandler:
bridged: Final = _bridged_request(hoisted, responses_api_request, ())
bridged_input: Final = bridged.input
bridged_request: Final = bridged.responses_api_request
litellm_completion_request: Final[dict] = (
LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
litellm_completion_request: Final[dict] = {
**LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
model=model,
input=bridged_input,
responses_api_request=bridged_request,
@ -105,8 +113,9 @@ class LiteLLMCompletionTransformationHandler:
stream=stream,
extra_headers=extra_headers,
**kwargs,
)
)
),
**_chat_verbosity(bridged_request),
}
if _is_async:
return self.async_response_api_handler(
@ -173,8 +182,8 @@ class LiteLLMCompletionTransformationHandler:
return await self.async_response_api_handler(
previous_response_client_ids,
stored_turns,
litellm_completion_request=(
LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
litellm_completion_request={
**LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
model=model,
input=bridged.input,
responses_api_request=bridged.responses_api_request,
@ -182,8 +191,9 @@ class LiteLLMCompletionTransformationHandler:
stream=stream,
extra_headers=extra_headers,
**request_kwargs,
)
),
),
**_chat_verbosity(bridged.responses_api_request),
},
request_input=bridged.input,
responses_api_request=bridged.responses_api_request,
**request_kwargs,

View file

@ -8,7 +8,7 @@ import uuid
from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from functools import partial
from itertools import accumulate, chain, groupby, islice
from itertools import chain, groupby
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
@ -45,7 +45,15 @@ from litellm.constants import (
REDACTED_BY_LITELLM,
REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER,
)
from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook
from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
block_with_cache_control,
cache_control_count,
dropped_cache_control,
newest_marks_within_cache_control_limit,
whole_messages_within_cache_control_limit,
with_cache_control,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, normalize_drop_params
from litellm.litellm_core_utils.get_supported_openai_params import (
get_supported_openai_params,
@ -605,10 +613,6 @@ def _validated_cache_control(value: object) -> ChatCompletionCachedContent | Non
return None
def _dropped_cache_control(_: object) -> None:
return None
def _without_keys(item: object, keys: frozenset[str]) -> object:
if not isinstance(item, dict):
return item
@ -707,83 +711,6 @@ def _content_blocks_carrying(content: object, cache_control: ChatCompletionCache
return _with_cache_control_on_last_block(_OBJECT_LIST_ADAPTER.validate_python(content), cache_control)
_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls")
def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]:
lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS)
return tuple(
chain.from_iterable(_OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list))
)
def _block_carries_cache_control(block: object) -> bool:
return isinstance(block, dict) and _STR_KEY_DICT_ADAPTER.validate_python(block).get("cache_control") is not None
def _cache_control_count(message: object) -> int:
if not isinstance(message, dict):
return 0
fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message)
block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block))
return int(fields.get("cache_control") is not None) + block_marks
_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None]
def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]:
others: Final = {key: value for key, value in fields.items() if key != "cache_control"}
if "cache_control" not in fields:
return others
cache_control: Final = rewrite(fields["cache_control"])
return others if cache_control is None else {**others, "cache_control": cache_control}
def _block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object:
if not isinstance(block, dict):
return block
return _fields_with_cache_control(_STR_KEY_DICT_ADAPTER.validate_python(block), rewrite)
def _with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT:
if not isinstance(message, dict):
return message
fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message)
blocks_rewritten: Final = {
key: (
[_block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)]
if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list)
else value
)
for key, value in fields.items()
}
return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped
_MessageT, _fields_with_cache_control(blocks_rewritten, rewrite)
)
def _kept_within_budget(state: tuple[int, bool], marks: int) -> tuple[int, bool]:
remaining, _ = state
if marks > remaining:
return remaining, False
return remaining - marks, True
def _messages_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]:
budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0)
newest_first: Final = tuple(reversed(messages))
budget_after_each: Final = accumulate(
(_cache_control_count(message) for message in newest_first), _kept_within_budget, initial=(budget, True)
)
kept_flags: Final = tuple(kept for _, kept in islice(budget_after_each, 1, None))
kept_newest_first: Final = tuple(
message if kept else _with_cache_control(message, _dropped_cache_control)
for message, kept in zip(newest_first, kept_flags)
)
return tuple(reversed(kept_newest_first))
_SYSTEM_MESSAGE_FIELDS: Final = frozenset(("content", "cache_control"))
@ -812,7 +739,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT:
if fields is None:
return message
if fields.get("role") == "tool":
return _with_cache_control(message, _dropped_cache_control)
return with_cache_control(message, dropped_cache_control)
tool_calls: Final = fields.get("tool_calls")
if not isinstance(tool_calls, list):
return message
@ -821,7 +748,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT:
{
**fields,
"tool_calls": [
_block_with_cache_control(tool_call, _dropped_cache_control)
block_with_cache_control(tool_call, dropped_cache_control)
for tool_call in _OBJECT_LIST_ADAPTER.validate_python(tool_calls)
],
},
@ -835,7 +762,7 @@ class _ItemMarkRules:
external_marks: int
def sanitized(self, message: _MessageT) -> _MessageT:
valid: Final = _with_cache_control(message, _validated_cache_control)
valid: Final = with_cache_control(message, _validated_cache_control)
return _without_tool_block_marks(valid) if self.refuses_tool_block_marks else valid
def checked(self, message: _MessageT) -> _MessageT:
@ -858,15 +785,15 @@ def _request_messages_for_cache_points(messages: Sequence[_MessageT], rules: _It
as_system: Final = tuple(_developer_message_as_system_message(message) for message in messages)
if not rules.drop_params:
return as_system
return _messages_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks)
return newest_marks_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks)
def _replayed_messages_for_cache_points(
replayed_messages: Sequence[_MessageT], current_messages: Sequence[object], rules: _ItemMarkRules
) -> Sequence[_MessageT]:
as_system: Final = tuple(_developer_message_as_system_message(message) for message in replayed_messages)
current_marks: Final = sum(_cache_control_count(message) for message in current_messages)
return _messages_within_cache_control_limit(
current_marks: Final = sum(cache_control_count(message) for message in current_messages)
return whole_messages_within_cache_control_limit(
tuple(map(rules.sanitized, as_system)), current_marks + rules.external_marks
)

View file

@ -23,6 +23,7 @@ from litellm.responses.mcp.tool_precheck import (
LITELLM_PROXY_MCP_SERVER_URL_PREFIX,
may_name_litellm_mcp_gateway,
names_gateway_explicitly,
names_whole_gateway,
proxy_path_mcp_name,
)
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
@ -182,8 +183,8 @@ class LiteLLM_Proxy_MCP_Handler:
@staticmethod
async def _apply_toolset_permissions(
resolved_toolset_ids: list[str],
resolved_mcp_servers: list[str],
resolved_toolset_ids: Sequence[str],
resolved_mcp_servers: Sequence[str],
user_api_key_auth: "UserAPIKeyAuth",
) -> "UserAPIKeyAuth":
"""Apply resolved toolset permissions to user_api_key_auth and return updated auth."""
@ -195,7 +196,7 @@ class LiteLLM_Proxy_MCP_Handler:
)
tool_permissions: Final = await global_mcp_server_manager.resolve_toolset_tool_permissions(
toolset_ids=resolved_toolset_ids
toolset_ids=list(resolved_toolset_ids)
)
all_server_ids: Final = list(set(tool_permissions.keys()) | set(resolved_mcp_servers))
existing_op: Final = user_api_key_auth.object_permission
@ -224,6 +225,62 @@ class LiteLLM_Proxy_MCP_Handler:
verbose_logger.debug("Could not apply toolset permissions: %s", _e)
return user_api_key_auth
@staticmethod
async def _resolve_proxy_mcp_name(
name: str,
user_api_key_auth: "UserAPIKeyAuth | None",
granted_toolsets: "GrantedToolsetIds | None",
) -> tuple[Literal["server", "toolset", "denied_toolset"], str]:
from litellm.proxy.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
if global_mcp_server_manager.get_mcp_server_by_name(name):
return ("server", name)
try:
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
return ("server", name)
toolset: Final = await global_mcp_server_manager.get_toolset_by_name_cached(prisma_client, name)
if toolset is None:
return ("server", name)
from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import (
has_team_unbound_mcp_admin_view,
)
from litellm.proxy.mcp_server.ui_session_utils import (
granted_toolset_ids,
)
denied: Final = (
user_api_key_auth is not None
and not has_team_unbound_mcp_admin_view(user_api_key_auth)
and toolset.toolset_id not in await (granted_toolsets or granted_toolset_ids)(user_api_key_auth)
)
except Exception as _e:
verbose_logger.debug("Could not resolve '%s' as toolset: %s", name, _e)
return ("server", name)
if not denied:
return ("toolset", toolset.toolset_id)
if await LiteLLM_Proxy_MCP_Handler._holds_access_group_named(name, user_api_key_auth):
return ("server", name)
verbose_logger.debug("Key does not have access to toolset '%s', denying it.", name)
return ("denied_toolset", name)
@staticmethod
async def _holds_access_group_named(name: str, user_api_key_auth: "UserAPIKeyAuth | None") -> bool:
from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import (
MCPRequestHandler,
)
from litellm.proxy.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
group_server_ids: Final = await MCPRequestHandler.get_mcp_servers_from_access_groups([name])
return bool(group_server_ids) and not frozenset(group_server_ids).isdisjoint(
await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth)
)
@staticmethod
@catalog_operation(global_manager)
async def _get_mcp_tools_from_manager(
@ -256,6 +313,7 @@ class LiteLLM_Proxy_MCP_Handler:
from litellm.proxy.mcp_server.server import (
_get_allowed_mcp_servers_from_mcp_server_names,
_get_tools_from_mcp_servers,
raise_generic_mcp_access_denial,
)
mcp_servers: Final = [
@ -268,36 +326,15 @@ class LiteLLM_Proxy_MCP_Handler:
# Resolve toolset names: collect all toolset IDs first, then apply their
# combined permissions in a single pass so multiple toolsets are unioned
# rather than the last one overwriting the others.
resolved_mcp_servers: Final[list[str]] = []
resolved_toolset_ids: Final[list[str]] = []
for name in mcp_servers:
if not global_mcp_server_manager.get_mcp_server_by_name(name):
try:
from litellm.proxy.proxy_server import prisma_client
if prisma_client is not None:
toolset = await global_mcp_server_manager.get_toolset_by_name_cached(prisma_client, name)
if toolset is not None:
if user_api_key_auth is not None:
from litellm.proxy.management_endpoints.common_utils import (
user_api_key_has_admin_view,
)
from litellm.proxy.mcp_server.ui_session_utils import (
granted_toolset_ids,
)
if not user_api_key_has_admin_view(user_api_key_auth) and toolset.toolset_id not in (
await (granted_toolsets or granted_toolset_ids)(user_api_key_auth)
):
verbose_logger.debug("Key does not have access to toolset '%s', skipping.", name)
continue
resolved_toolset_ids.append(toolset.toolset_id)
# Don't add to resolved_mcp_servers — toolset scope
# restricts via object_permission, not server name filter.
continue
except Exception as _e:
verbose_logger.debug("Could not resolve '%s' as toolset: %s", name, _e)
resolved_mcp_servers.append(name)
resolutions: Final = tuple(
[
await LiteLLM_Proxy_MCP_Handler._resolve_proxy_mcp_name(name, user_api_key_auth, granted_toolsets)
for name in mcp_servers
]
)
resolved_mcp_servers: Final = tuple(value for kind, value in resolutions if kind == "server")
resolved_toolset_ids: Final = tuple(value for kind, value in resolutions if kind == "toolset")
denied_toolset_names: Final = tuple(value for kind, value in resolutions if kind == "denied_toolset")
# Apply all resolved toolsets at once (union), avoiding permission overwrite.
if resolved_toolset_ids and user_api_key_auth is not None:
@ -311,7 +348,10 @@ class LiteLLM_Proxy_MCP_Handler:
# full union (toolset server IDs + direct server names). Passing a name-based
# filter here would exclude those toolset server IDs (which are UUIDs, not
# names), so use None and let the auth object's mcp_servers do the filtering.
effective_server_filter: Final = None if resolved_toolset_ids else (resolved_mcp_servers or None)
asks_whole_gateway: Final = any(map(names_whole_gateway, mcp_tools_with_litellm_proxy or ()))
if denied_toolset_names and not (asks_whole_gateway or resolved_mcp_servers or resolved_toolset_ids):
raise_generic_mcp_access_denial(denied_toolset_names)
effective_server_filter: Final = None if resolved_toolset_ids else (list(resolved_mcp_servers) or None)
listing: Final = await _get_tools_from_mcp_servers(
user_api_key_auth=user_api_key_auth,

View file

@ -22,6 +22,13 @@ def names_gateway_explicitly(tool: object) -> bool:
return (_mcp_server_url(tool) or "").startswith(LITELLM_PROXY_MCP_SERVER_URL)
def names_whole_gateway(tool: object) -> bool:
server_url: Final = _mcp_server_url(tool) or ""
return server_url.startswith(LITELLM_PROXY_MCP_SERVER_URL) and not server_url.startswith(
LITELLM_PROXY_MCP_SERVER_URL_PREFIX
)
def proxy_path_mcp_name(tool: object) -> str | None:
server_url: Final = _mcp_server_url(tool)
match: Final = None if server_url is None else _PROXY_MCP_PATH_RE.match(server_url)

View file

@ -637,6 +637,7 @@ _PRE_CALL_CHECK_FILTERED_PARAMS: Final = frozenset({"response_format"})
_STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS: Final = frozenset({"anthropic_messages", "aresponses", "responses"})
_video_call_deployment_id: Final[ContextVar[str | None]] = ContextVar("video_call_deployment_id", default=None)
_routed_call_type: Final[ContextVar[str | None]] = ContextVar("routed_call_type", default=None)
_DECISIONS_MEDIA_CALL_TYPES: Final = frozenset({"adecisions", "decisions", "asystemone", "systemone"})
_ROUTE_CAPABILITY_RESOLVERS: Final[Mapping[str, Callable[[str, LlmProviders], object | None]]] = MappingProxyType(
{
"adecisions": ProviderConfigManager.get_provider_decisions_config,
@ -4184,13 +4185,16 @@ class Router(DeprecatedRouterMembers):
raise e.original_exception from e
raise e
def _active_default_litellm_params(self) -> Mapping[str, object]:
default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self._core_default_litellm_params)
return {key: value for key, value in default_params.items() if value is not None}
def _params_sent_to_deployment(
self, request_kwargs: Mapping[str, object], deployment: Deployment | None
) -> Mapping[str, object]:
"""The request as the deployment received it, merged in the order _update_kwargs_with_deployment applies."""
deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {}
default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self._core_default_litellm_params)
defaults: Final = {key: value for key, value in default_params.items() if value is not None}
defaults: Final = self._active_default_litellm_params()
deployment_tool_choice: Final = (
{"tool_choice": deployment_params["tool_choice"]}
if deployment_params.get("tool_choice") is not None
@ -14949,6 +14953,62 @@ class Router(DeprecatedRouterMembers):
return healthy_deployments
return cast(_RouteDeploymentsT, filtered_deployments) # cast-ok: same list shape as the input candidates
@staticmethod
def _decisions_deployment_refuses_media(
deployment: DeploymentTypedDict, group: str, request_kwargs: Mapping[str, object], *, audio: bool, videos: bool
) -> bool:
from litellm.decisions.call import deployment_refuses_media
plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts
model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict
deployment_params: Final = cast(Mapping[str, object], deployment["litellm_params"]) # cast-ok: plain dict
try:
model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group)
return deployment_refuses_media(
model,
custom_llm_provider,
model_info,
{**deployment_params, **request_kwargs},
audio=audio,
videos=videos,
)
except Exception as e: # noqa: BLE001 # best-effort filter: a deployment it cannot resolve answers for itself
verbose_router_logger.debug(
"litellm.router.py::_decisions_deployment_refuses_media: keeping a deployment of model=%s. Got - %s",
group,
e,
)
return False
def _without_decisions_deployments_refusing_media(
self, healthy_deployments: _RouteDeploymentsT, group: str, request_kwargs: Mapping[str, object] | None
) -> _RouteDeploymentsT:
call_kwargs: Final = request_kwargs or {}
audio: Final = bool(call_kwargs.get("audio"))
videos: Final = bool(call_kwargs.get("videos"))
if (
_routed_call_type.get() not in _DECISIONS_MEDIA_CALL_TYPES
or not (audio or videos)
or isinstance(healthy_deployments, Mapping)
):
return healthy_deployments
sent_kwargs: Final = {**self._active_default_litellm_params(), **call_kwargs}
deployments_taking_media: Final = [
deployment
for deployment in healthy_deployments
if not Router._decisions_deployment_refuses_media(
cast(DeploymentTypedDict, deployment), # cast-ok: router candidates use the deployment shape
group,
sent_kwargs,
audio=audio,
videos=videos,
)
]
if not deployments_taking_media:
return healthy_deployments
return cast(_RouteDeploymentsT, deployments_taking_media) # cast-ok: same list shape as the input candidates
@staticmethod
def _without_declared_text_only_deployments(
healthy_deployments: _RouteDeploymentsT,
@ -15673,10 +15733,14 @@ class Router(DeprecatedRouterMembers):
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
_target_order: Final = (request_kwargs or {}).pop("_target_order", None)
batch_candidates: Final = self._without_declared_text_only_deployments(
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
batch_candidates: Final = self._without_decisions_deployments_refusing_media(
self._without_declared_text_only_deployments(
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
model,
messages,
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
),
model,
messages,
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
)
embedding_batch_admitting: Final = deployments_admitting_embedding_batch(
@ -16916,10 +16980,14 @@ class Router(DeprecatedRouterMembers):
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
_target_order: Final = (request_kwargs or {}).pop("_target_order", None)
batch_candidates: Final = self._without_declared_text_only_deployments(
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
batch_candidates: Final = self._without_decisions_deployments_refusing_media(
self._without_declared_text_only_deployments(
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
model,
messages,
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
),
model,
messages,
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
)
embedding_batch_admitting: Final = deployments_admitting_embedding_batch(

View file

@ -402,7 +402,7 @@ class OpenAIFileObject(LiteLLMBaseModel):
filename: str
"""The name of the file."""
object: Literal["file"]
object: Literal["file"] = "file"
"""The object type, which is always `file`."""
purpose: OpenAIFilesPurpose
@ -440,6 +440,11 @@ class OpenAIFileObject(LiteLLMBaseModel):
_hidden_params: dict = PrivateAttr(default={"response_cost": 0.0}) # no cost for writing a file
@field_validator("object", mode="before")
@classmethod
def _default_missing_object_type(cls, value: _JsonValue) -> _JsonValue:
return "file" if value is None else value
@property
def hidden_params(self) -> dict[str, builtins.object]: # mutable-ok: API requires mutation
return self._hidden_params

View file

@ -1,5 +1,6 @@
from dataclasses import dataclass
from enum import Enum
from typing import Final
from typing import Final, Protocol
from pydantic import Field
from typing_extensions import TypedDict
@ -75,6 +76,19 @@ class PassthroughStandardLoggingPayload(TypedDict, total=False):
"""
@dataclass(frozen=True, slots=True)
class WebSocketSessionUsage:
model: str
custom_llm_provider: str
response_cost: float | None
class WebSocketSessionMeter(Protocol):
def measure(self, client_text: str) -> int: ...
def price(self, total: int) -> WebSocketSessionUsage: ...
class PassThroughGuardrailSettings(LiteLLMPydanticObjectBase):
"""
Settings for a specific guardrail on a passthrough endpoint.

View file

@ -106,6 +106,7 @@ from .llms.openai import (
OpenAIRealtimeStreamList,
ResponsesAPIResponse,
WebSearchOptions,
_JsonValue,
)
from .rerank import RerankResponse as RerankResponse
@ -4682,6 +4683,7 @@ class LlmProviders(str, Enum):
PRISM = "prism"
OPENINFER = "openinfer"
INFERENCEAPIS = "inferenceapis"
HOSTINGER_ROUTER = "hostinger_router"
ZEROGPU = "zerogpu"
DARKBLOOM = "darkbloom"
META = "meta"
@ -4963,29 +4965,50 @@ def _batch_usage_from_chat_usage(usage: Usage) -> BatchUsage:
)
OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS: Final = MappingProxyType(
{"pending": "validating", "running": "in_progress"}
)
def openai_batch_status(status: _JsonValue) -> _JsonValue:
return OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(status, status) if isinstance(status, str) else status
class LiteLLMBatch(Batch):
_hidden_params: dict = {}
object: Literal["batch"] = "batch"
@field_validator("object", mode="before")
@classmethod
def _default_missing_object_type(cls, value: _JsonValue) -> _JsonValue:
return "batch" if value is None else value
@field_validator("status", mode="before")
@classmethod
def _map_vllm_router_status(cls, value: _JsonValue) -> _JsonValue:
return openai_batch_status(value)
@property
def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation
def hidden_params(self) -> dict[str, _JsonValue]: # mutable-ok: API requires mutation
return self._hidden_params
@hidden_params.setter
def hidden_params(self, hidden_params: dict[str, object]) -> None: # mutable-ok: API requires mutation
def hidden_params(self, hidden_params: dict[str, _JsonValue]) -> None: # mutable-ok: API requires mutation
self._hidden_params = hidden_params
usage: Usage | None = None
@field_validator("usage", mode="before")
@classmethod
def _usage_from_batch_token_counts(cls, value: object) -> object:
def _usage_from_batch_token_counts(cls, value: _JsonValue) -> _JsonValue:
counts: Final = _parse_batch_token_counts(value)
return value if counts is None else _usage_with_batch_token_counts(Usage.model_validate(value), counts)
@field_serializer("usage")
def _serialize_usage_with_batch_fields(
self, usage: Usage | None, info: FieldSerializationInfo
) -> Mapping[str, object] | None:
) -> Mapping[str, _JsonValue] | None:
if not isinstance(usage, Usage):
return usage
chat_fields: Final = usage.model_dump(

View file

@ -10159,7 +10159,7 @@ class ProviderConfigManager:
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
json_provider: Final = JSONProviderRegistry.get(provider.value)
if json_provider is not None and "/v1/messages" in json_provider.supported_endpoints:
if json_provider is not None and JSONProviderRegistry.serves_endpoint(provider.value, model, "/v1/messages"):
from litellm.llms.openai_like.messages.transformation import (
JSONProviderAnthropicMessagesConfig,
)
@ -10298,7 +10298,7 @@ class ProviderConfigManager:
return result
# Fall back to JSON providers (generic OpenAI-compatible)
if JSONProviderRegistry.exists(provider_str) and JSONProviderRegistry.supports_responses_api(provider_str):
if JSONProviderRegistry.serves_endpoint(provider_str, model, "/v1/responses"):
provider_config: Final = JSONProviderRegistry.get(provider_str)
if provider_config is not None:
return create_responses_config_class(provider_config)()

View file

@ -39687,6 +39687,57 @@
"/v1/images/generations"
]
},
"hostinger_router/gpt-5.2": {
"cache_read_input_token_cost": 0.00000021,
"input_cost_per_token": 0.0000021,
"litellm_provider": "hostinger_router",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 0.0000168,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_vision": true,
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
"source": "https://router.hostinger.com/v1/models"
},
"hostinger_router/claude-sonnet-5": {
"cache_creation_input_token_cost": 3.3e-06,
"cache_creation_input_token_cost_above_1hr": 5.28e-06,
"cache_read_input_token_cost": 2.64e-07,
"input_cost_per_token": 2.64e-06,
"litellm_provider": "hostinger_router",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.32e-05,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_response_schema": true,
"supported_endpoints": ["/v1/chat/completions", "/v1/messages"],
"source": "https://router.hostinger.com/v1/models"
},
"hostinger_router/text-embedding-3-large": {
"input_cost_per_token": 1.716e-07,
"litellm_provider": "hostinger_router",
"max_input_tokens": 8191,
"mode": "embedding",
"output_cost_per_token": 0.0,
"source": "https://router.hostinger.com/v1/models"
},
"hostinger_router/text-embedding-3-small": {
"input_cost_per_token": 3e-08,
"litellm_provider": "hostinger_router",
"max_input_tokens": 8191,
"mode": "embedding",
"output_cost_per_token": 0.0,
"source": "https://router.hostinger.com/v1/models"
},
"hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": {
"input_cost_per_token": 1.2e-07,
"litellm_provider": "hyperbolic",
@ -51387,6 +51438,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 5e-06,
"output_cost_per_token_batches": 2.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -51415,6 +51471,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 5e-06,
"output_cost_per_token_batches": 2.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -52161,6 +52222,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_batches": 7.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -52266,6 +52332,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_batches": 7.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,

View file

@ -1087,6 +1087,32 @@
"generateContent": false
}
},
"hostinger_router": {
"display_name": "Hostinger Router (`hostinger_router`)",
"url": "https://docs.litellm.ai/docs/providers/hostinger_router",
"endpoints": {
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": true,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": false,
"a2a": true,
"generateContent": true
}
},
"crusoe": {
"display_name": "Crusoe (`crusoe`)",
"url": "https://docs.litellm.ai/docs/providers/crusoe",

View file

@ -383,6 +383,8 @@ exclude-newer = "3 days"
[tool.uv.exclude-newer-package]
# TODO: remove after 2026-10-25, once brotli 1.2.1 (CVE-2026-111497) clears the 3-day window
brotli = "2026-10-22T14:01:00Z"
# TODO: remove after 2026-10-31, once pypdf 6.21.4 clears the 3-day window
pypdf = "2026-10-28T17:01:00Z"
[tool.uv.sources]
litellm-proxy-extras = { workspace = true }

View file

@ -185,6 +185,9 @@ ignored_function_names = [
"_aresponses_background_stream_can_release", # Tested through the background Responses stream tests in test_router.py
"_raise_released_stream_error", # Tested through the background Responses stream content-policy tests in test_router.py
"_mid_stream_error_before_output", # Tested through the Responses mid-stream fallback and background stream tests
"_without_decisions_deployments_refusing_media", # Tested through the mixed media-flag group tests in tests/unit/decisions/test_main.py
"_decisions_deployment_refuses_media", # Read by _without_decisions_deployments_refusing_media, tested through the same group tests
"_active_default_litellm_params", # Tested through the router default drop_params test in tests/unit/decisions/test_main.py
]

View file

@ -3,7 +3,7 @@
PARITY pairs the streamed case with its plain twin where both bill the same row."""
import json
from typing import Final, cast
from typing import Final, Literal, cast
from pydantic import JsonValue
@ -41,6 +41,7 @@ from integration.cost_calculation.cases.anthropic import (
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs
from integration.cost_calculation.cost_tracking_case import (
CostTrackingTestCase,
Deployment,
ExactExpected,
JsonResponse,
RecountExpected,
@ -3387,6 +3388,232 @@ VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS: Final = bridge_pairs(
)
VERTEX_AI_CLAUDE_DEPLOYMENT_RATES: Final = Deployment(
input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.01
)
def _vertex_claude_web_search_message(web_search_requests: int) -> JsonResponse:
return JsonResponse(
content_type="application/json",
body={
"id": "msg_$REQUEST_ID",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5-5",
"content": [{"type": "text", "text": "scripted answer for cost fixtures"}],
"stop_reason": "end_turn",
"usage": {
"input_tokens": 1840,
"output_tokens": 412,
"server_tool_use": {"web_search_requests": web_search_requests},
},
},
)
def _vertex_claude_web_search_stream(web_search_requests: int) -> SseResponse:
return SseResponse(
content_type="text/event-stream",
frames=sse_frames(
{
"type": "message_start",
"message": {
"id": "msg_$REQUEST_ID",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5-5",
"content": [],
"stop_reason": None,
"usage": {"input_tokens": 1840},
},
},
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
{
"type": "content_block_delta",
"index": 0,
"delta": {"type": "text_delta", "text": "scripted answer for cost fixtures"},
},
{"type": "content_block_stop", "index": 0},
{
"type": "message_delta",
"delta": {"stop_reason": "end_turn"},
"usage": {"output_tokens": 412, "server_tool_use": {"web_search_requests": web_search_requests}},
},
{"type": "message_stop"},
),
)
def _vertex_claude_deployment_web_search_case(
name: str,
endpoint: Literal["/v1/chat/completions", "/v1/responses", "/v1/messages"],
request: dict[str, JsonValue],
response: JsonResponse | SseResponse,
tool_usage_cost: float,
deployment: Deployment = VERTEX_AI_CLAUDE_DEPLOYMENT_RATES,
) -> CostTrackingTestCase:
return CostTrackingTestCase(
name=name,
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="vertex_ai/claude-sonnet-5-5",
endpoint=endpoint,
deployment=deployment,
request={"model": "$MODEL", **request},
response=response,
expected=ExactExpected(
spend=round(0.0039 + tool_usage_cost, 10),
input_cost=0.00184,
output_cost=0.00206,
prompt_tokens=1840,
completion_tokens=412,
tool_usage_cost=tool_usage_cost,
),
)
VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL: Final[dict[str, JsonValue]] = {
"type": "web_search_20250305",
"name": "web_search",
"max_uses": 3,
}
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_rates_messages_web_search_two_requests",
"/v1/messages",
{
"max_tokens": 412,
"messages": [{"role": "user", "content": "vertex claude deployment rates messages web search"}],
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
},
_vertex_claude_web_search_message(2),
0.02,
)
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM: Final = _vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_rates_messages_web_search_two_requests_stream",
"/v1/messages",
{
"max_tokens": 412,
"messages": [{"role": "user", "content": "vertex claude deployment rates messages web search stream"}],
"stream": True,
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
},
_vertex_claude_web_search_stream(2),
0.02,
)
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE: Final = (
_vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_zero_search_rate_messages_two_requests",
"/v1/messages",
{
"max_tokens": 412,
"messages": [{"role": "user", "content": "vertex claude deployment zero search rate messages"}],
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
},
_vertex_claude_web_search_message(2),
0.0,
Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.0),
)
)
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY: Final = (
_vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_token_rates_only_messages_web_search_two_requests",
"/v1/messages",
{
"max_tokens": 412,
"messages": [{"role": "user", "content": "vertex claude deployment token rates only messages web search"}],
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
},
_vertex_claude_web_search_message(2),
0.034,
Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06),
)
)
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_rates_chat_web_search_three_requests",
"/v1/chat/completions",
{
"messages": [{"role": "user", "content": "vertex claude deployment rates chat web search"}],
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
"allowed_openai_params": ["web_search_options"],
},
_vertex_claude_web_search_message(3),
0.03,
)
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH_STREAM: Final = _vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_rates_chat_web_search_two_requests_stream",
"/v1/chat/completions",
{
"messages": [{"role": "user", "content": "vertex claude deployment rates chat web search stream"}],
"stream": True,
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
"allowed_openai_params": ["web_search_options"],
},
_vertex_claude_web_search_stream(2),
0.02,
)
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_RESPONSES_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_rates_responses_web_search_two_requests",
"/v1/responses",
{"input": "vertex claude deployment rates responses web search", "tools": [{"type": "web_search"}]},
_vertex_claude_web_search_message(2),
0.02,
)
GEMINI_3_1_PRO_PREVIEW_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM: Final = CostTrackingTestCase(
name="vertex_ai/gemini-3.1-pro-preview-deployment_rates_messages_web_search_two_queries_stream",
covers="quota_management.spend_tracking.scripted_wire.logs_cost",
model="vertex_ai/gemini-3.1-pro-preview",
endpoint="/v1/messages",
deployment=Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.03),
request={
"model": "$MODEL",
"max_tokens": 412,
"messages": [{"role": "user", "content": "vertex gemini deployment rates messages web search stream"}],
"stream": True,
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
},
response=SseResponse(
content_type="text/event-stream",
frames=sse_frames(
{
"candidates": [
{
"content": {"parts": [{"text": "scripted answer for cost fixtures"}], "role": "model"},
"index": 0,
}
],
"modelVersion": "gemini-3.1-pro-preview",
},
{
"candidates": [
{
"content": {"parts": [{"text": ""}], "role": "model"},
"finishReason": "STOP",
"index": 0,
"groundingMetadata": {"webSearchQueries": ["query 0", "query 1"]},
}
],
"usageMetadata": {
"promptTokenCount": 1840,
"candidatesTokenCount": 412,
"totalTokenCount": 2252,
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}],
},
"modelVersion": "gemini-3.1-pro-preview",
},
),
),
expected=ExactExpected(
spend=0.0339,
input_cost=0.00184,
output_cost=0.00206,
prompt_tokens=1840,
completion_tokens=412,
tool_usage_cost=0.03,
),
)
GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT: Final = _gemini_case(
GEMINI_3_8_FLASH_INPUT_TEXT, "vertex_ai/gemini-3.1-pro-preview-input_text", "vertex_ai/gemini-3.1-pro-preview",
ExactExpected(spend=0.0094864, input_cost=0.004048, output_cost=0.0054384, prompt_tokens=1840, completion_tokens=412),
@ -3725,6 +3952,14 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT,
VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_STREAM,
*(pair.bridged for pair in VERTEX_AI_CLAUDE_SONNET_5_5_BRIDGE_PAIRS),
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH_STREAM,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_RESPONSES_WEB_SEARCH,
GEMINI_3_1_PRO_PREVIEW_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM,
GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT,
GEMINI_3_1_PRO_PREVIEW_STREAM,
GEMINI_3_1_PRO_PREVIEW_CACHE_READ,

View file

@ -32,7 +32,7 @@ from litellm.accounting import (
PricingSnapshot,
)
from litellm.proxy.db import accounting_ledger
from litellm.proxy.db.accounting_ledger import AccountingConflictError, AccountingEvent, AccountingLedger
from litellm.proxy.db.accounting_ledger import AccountingConflictError, AccountingEvent, AccountingLedger, UsageWrite
from litellm.types.utils import CallTypes
from tests.integration._support.database import scratch_database
from tests.integration._support.prisma_clients import connected_proxy_prisma
@ -532,3 +532,74 @@ def test_family_cleanup_lookups_use_the_kind_source_index(ledger_database: str,
name == "LiteLLM_AccountingEvent_kind_source_event_id_idx" and "source_event_id" in str(condition)
for name, condition in conditions
), conditions
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ["before_commit", "after_commit", "conflict"])
async def test_usage_batch_is_atomic_and_replay_preserves_receipts(
writer: _Writer, ledger_database: str, failure: str
) -> None:
client, store = writer
events: Final = (_usage("caller", inbound=True), _usage("attempt"))
writes: Final = tuple(UsageWrite(event, _envelope(event), _TARGETS) for event in events)
@asynccontextmanager
async def interrupted() -> AsyncGenerator[Prisma]:
async with client.writer_db.tx() as tx:
yield tx
if failure == "before_commit":
raise RuntimeError("injected pre-commit failure")
if failure == "after_commit":
raise RuntimeError("injected lost acknowledgement")
if failure == "conflict":
await store.insert_usage_batch((writes[1],))
_complete_and_age(ledger_database)
rejected: Final = (writes[0], UsageWrite(events[1], _envelope(events[1], context={"changed": True}), _TARGETS))
with pytest.raises(AccountingConflictError):
await store.insert_usage_batch(rejected)
assert _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent"') == [("attempt",)]
else:
with pytest.raises(RuntimeError, match="injected"):
await AccountingLedger(interrupted).insert_usage_batch(writes)
count: Final = 2 if failure == "after_commit" else 0
assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(count,)]
assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery"') == [(count * len(_TARGETS),)]
await store.insert_usage_batch(writes)
recovered: Final = await store.load_usage_batch(tuple(event.event_id for event in events))
assert len(recovered) == 1
assert recovered[0].event == events[1]
assert json.loads(recovered[0].envelope) == json.loads(writes[1].envelope)
assert set(recovered[0].targets) == set(_TARGETS)
_complete_and_age(ledger_database)
before: Final = _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target')
await store.insert_usage_batch((*writes, *writes))
assert _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target') == before
assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(2,)]
@pytest.mark.asyncio
async def test_invalid_jsonb_submission_cannot_reject_another_producers_commit(
writer: _Writer, ledger_database: str
) -> None:
from prisma.errors import DataError
from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings
_, store = writer
invalid: Final = _usage("invalid")
valid: Final = _usage("valid")
meter: Final = DurableUsageMeter(store.insert_usage_batch, UsageCaptureSettings(batch_size=2))
rejected: Final = meter.submit(
(UsageWrite(invalid, _envelope(invalid, context={"parent": "bad\x00text"}), _TARGETS),)
)
accepted: Final = meter.submit((UsageWrite(valid, _envelope(valid), _TARGETS),))
try:
with pytest.raises(DataError):
await rejected
await accepted
finally:
await meter.close()
assert _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent"') == [("valid",)]
assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery"') == [(len(_TARGETS),)]
assert (meter.pending_bytes, meter.pending_records) == (0, 0)

View file

@ -317,7 +317,6 @@ def test_credentialless_proxy_forwards_the_caller_anthropic_credential_and_never
def test_anthropic_sdk_metadata_user_id_reaches_upstream(api_key_proxy: Gateway, wire: Wire) -> None:
pytest.skip("BUG: /anthropic/v1/messages drops the native metadata.user_id field before forwarding upstream")
with api_key_proxy.scenario() as scenario:
key: Final = scenario.key()
sent: Final[list[httpx.Request]] = []

View file

@ -0,0 +1,123 @@
import json
import uuid
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Final, Literal
import httpx
import pytest
from integration._support.client import Gateway, eventually, object_value
from integration._support.database import read_rows
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
_API_KEY: Final = "synthetic-decisions-key"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_INPUT: Final = "Listen to the attached clip."
_AUDIO: Final = ("data:audio/wav;base64,UklGRiQAAABXQVZF",)
_VIDEOS: Final = ("data:video/mp4;base64,AAAAIGZ0eXBpc29t",)
_QUESTION: Final[dict[str, JsonValue]] = {
"type": "predicate",
"name": "banana",
"instructions": "Does the clip name a banana?",
}
_SPEND_QUERY: Final = 'SELECT status, model_group FROM "LiteLLM_SpendLogs" WHERE request_id = %s'
Route = Literal["/v1/decisions", "/v1/systemone"]
Media = Literal["audio", "videos"]
@dataclass(frozen=True, slots=True)
class _Upstream:
model: str
def reply(self, request: Request) -> Reply:
questions: Final = object_value(_JSON_OBJECT.validate_json(request.body)["questions"])
plain: Final[dict[str, JsonValue]] = {
"model": self.model,
"answers": {name: {"type": "noul", "noul": 0.9} for name in questions},
"usage": {"input_tokens": 41, "output_tokens": 0},
}
return Reply(body=json.dumps(plain).encode())
def _unlisted_openrouter_model() -> _Upstream:
return _Upstream(f"openrouter/example/omni-{uuid.uuid4().hex[:12]}")
def _decide(gateway: Gateway, model: str, route: Route, media: Mapping[str, JsonValue]) -> httpx.Response:
body: Final[dict[str, JsonValue]] = (
{"model": model, "input": _INPUT, "questions": [_QUESTION], "num_retries": 0, **media}
if route == "/v1/decisions"
else {
"model": model,
"state": _INPUT,
"questions": {"banana": {"type": "noul", "instructions": _QUESTION["instructions"]}},
"num_retries": 0,
**media,
}
)
return gateway.request("POST", route, body)
def _sent_media(request: Request) -> dict[str, JsonValue]:
body: Final = _JSON_OBJECT.validate_json(request.body)
return {kind: body[kind] for kind in ("audio", "videos") if kind in body}
def _assert_refused(response: httpx.Response, kinds: tuple[Media, ...]) -> None:
assert response.status_code == 400, response.text
assert f"does not support parameters: {list(kinds)}" in response.text, response.text
_BOTH: Final[dict[str, JsonValue]] = {"audio": list(_AUDIO), "videos": list(_VIDEOS)}
@pytest.mark.parametrize("route", ("/v1/decisions", "/v1/systemone"))
def test_a_deployment_flagged_for_audio_and_video_sends_both_clips_for_a_model_the_cost_map_does_not_list(
gateway: Gateway, route: Route
) -> None:
upstream: Final = _unlisted_openrouter_model()
with wire_server(upstream.reply) as wire, gateway.scenario() as scenario:
flags: Final[dict[str, JsonValue]] = {"supports_audio_input": True, "supports_video_input": True}
flagged: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info=flags)
flagged_dropping: Final = scenario.model(
model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info=flags, drop_params=True
)
audio_only: Final = scenario.model(
model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info={"supports_audio_input": True}
)
unflagged: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY)
_assert_refused(_decide(gateway, unflagged, route, _BOTH), ("audio", "videos"))
_assert_refused(_decide(gateway, audio_only, route, _BOTH), ("videos",))
assert wire.drain() == ()
answered: Final = _decide(gateway, flagged, route, _BOTH)
assert answered.status_code == 200, answered.text
(call,) = wire.drain()
assert _sent_media(call) == _BOTH
rows: Final = eventually(
lambda: read_rows(_SPEND_QUERY, (answered.headers["x-litellm-call-id"],)),
lambda found: len(found) == 1,
seconds=70,
)
assert (rows[0]["status"], rows[0]["model_group"]) == ("success", flagged)
kept: Final = _decide(gateway, flagged_dropping, route, _BOTH)
assert kept.status_code == 200, kept.text
(kept_call,) = wire.drain()
assert _sent_media(kept_call) == _BOTH
@pytest.mark.parametrize(
"value",
({"supports_audio_input": True, "supports_video_input": True}, 5, [{"supports_audio_input": True}], "x" * 5000),
ids=("flags", "int", "list", "5kb"),
)
def test_a_caller_sent_model_info_never_turns_media_on(gateway: Gateway, value: JsonValue) -> None:
upstream: Final = _unlisted_openrouter_model()
with wire_server(upstream.reply) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY)
_assert_refused(_decide(gateway, model, "/v1/decisions", {**_BOTH, "model_info": value}), ("audio", "videos"))
assert wire.drain() == ()

View file

@ -0,0 +1,413 @@
import uuid
from collections.abc import Iterator, Mapping
from contextlib import contextmanager
from dataclasses import dataclass
from pathlib import Path
from typing import Final, Literal
import httpx
import pytest
import yaml
from integration._support import responses_vendor as rv
from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment
from integration._support.openai_wire import answering_model_discovery, chat_reply, responses_reply
from integration._support.process import OwnedProxy, owned_proxy_process
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue, TypeAdapter
pytestmark: Final = pytest.mark.timeout(120)
_GPT: Final = "gpt-5.6"
_DATABRICKS: Final = "databricks-meta-llama-4-maverick"
_PROVIDER_KEY: Final = "synthetic-provider-key"
_NO_CACHE: Final[dict[str, JsonValue]] = {"no-cache": True}
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_WEATHER_SCHEMA: Final[dict[str, JsonValue]] = {
"type": "object",
"properties": {"city": {"type": "string"}, "celsius": {"type": "number"}},
"required": ["city", "celsius"],
"additionalProperties": False,
}
_TEXT_FORMAT: Final[dict[str, JsonValue]] = {
"type": "json_schema",
"name": "weather",
"schema": _WEATHER_SCHEMA,
"strict": True,
}
_RESPONSE_FORMAT: Final[dict[str, JsonValue]] = {
"type": "json_schema",
"json_schema": {"name": "weather", "schema": _WEATHER_SCHEMA, "strict": True},
}
_STREAM_OPTIONS: Final[dict[str, JsonValue]] = {"stream_options": {"include_usage": True}}
Verbosity = Literal["low", "high"]
def _prompt(marker: str) -> str:
return f"verbosity probe marker-{marker}"
def _answer(marker: str) -> str:
return f"scripted answer marker-{marker}"
def _marker_of(request: Request) -> str:
body: Final = request.body.decode()
start: Final = body.rfind("marker-")
return body[start + len("marker-") : start + len("marker-") + 32] if start >= 0 else "health"
def _vendor(request: Request) -> Reply:
body: Final = _JSON_OBJECT.validate_json(request.body or b"{}")
marker: Final = _marker_of(request)
stream: Final = body.get("stream") is True
if request.target.endswith("/responses"):
return responses_reply(f"resp_{marker}", _GPT, _answer(marker), stream=stream)
return chat_reply(f"chatcmpl-{marker}", str(body.get("model")), _answer(marker), stream=stream)
@contextmanager
def _upstream() -> Iterator[Wire]:
with wire_server(answering_model_discovery(_vendor)) as wire:
yield wire
def _bridged_gpt(scenario: Scenario, wire: Wire, **parameters: JsonValue) -> str:
return scenario.model(
model=f"openai/{_GPT}",
api_base=f"{wire.url}/v1",
api_key=_PROVIDER_KEY,
use_chat_completions_api=True,
**parameters,
)
def _native_gpt(scenario: Scenario, wire: Wire) -> str:
return scenario.model(model=f"openai/{_GPT}", api_base=f"{wire.url}/v1", api_key=_PROVIDER_KEY)
def _databricks(scenario: Scenario, wire: Wire, **parameters: JsonValue) -> str:
return scenario.model(
model=f"databricks/{_DATABRICKS}",
api_base=f"{wire.url}/serving-endpoints",
api_key=_PROVIDER_KEY,
**parameters,
)
def _responses(gateway: Gateway, model: str, marker: str, **fields: JsonValue) -> httpx.Response:
return gateway.request(
"POST", "/v1/responses", {"model": model, "input": _prompt(marker), "cache": _NO_CACHE, **fields}
)
def _streamed_events(gateway: Gateway, model: str, marker: str, **fields: JsonValue) -> list[dict[str, JsonValue]]:
with gateway.client.stream(
"POST",
"/v1/responses",
json={"model": model, "input": _prompt(marker), "cache": _NO_CACHE, "stream": True, **fields},
headers={"Authorization": f"Bearer {gateway.key}"},
) as response:
assert response.status_code == 200, response.read()
return [
_JSON_OBJECT.validate_json(line.removeprefix("data: "))
for line in response.iter_lines()
if line.startswith("data: {")
]
def _output_text(response: Mapping[str, JsonValue]) -> str:
output: Final = response["output"]
assert isinstance(output, list), response
return "".join(
str(part["text"])
for item in output
if isinstance(item, dict) and item.get("type") == "message"
for part in item["content"]
if isinstance(part, dict) and part.get("type") == "output_text"
)
def _answered(response: httpx.Response, marker: str) -> None:
assert response.status_code == 200, response.text
body: Final = _JSON_OBJECT.validate_json(response.text)
assert body["status"] == "completed" and _output_text(body) == _answer(marker), body
def _streamed_answer(events: list[dict[str, JsonValue]], marker: str) -> None:
completed: Final = [event for event in events if event["type"] == "response.completed"]
assert len(completed) == 1, [event["type"] for event in events]
final: Final = completed[0]["response"]
assert isinstance(final, dict) and _output_text(final) == _answer(marker), final
deltas: Final = "".join(str(event["delta"]) for event in events if event["type"] == "response.output_text.delta")
assert deltas == _answer(marker), events
def _posted(wire: Wire) -> tuple[Request, ...]:
return tuple(request for request in wire.drain() if request.method == "POST")
def _only_post(wire: Wire, target: str) -> dict[str, JsonValue]:
posted: Final = _posted(wire)
assert [request.target for request in posted] == [target], posted
return _JSON_OBJECT.validate_json(posted[0].body)
def _chat_messages(marker: str) -> list[JsonValue]:
return [{"role": "user", "content": _prompt(marker)}]
def _unsupported_param_error(response: httpx.Response) -> None:
assert response.status_code == 400, response.text
error: Final = _JSON_OBJECT.validate_json(response.text)["error"]
assert isinstance(error, dict) and "verbosity" in str(error["message"]), response.text
@pytest.mark.parametrize("verbosity", ["low", "high"])
def test_regression_explicit_bridge_sends_text_verbosity_as_top_level_verbosity(
gateway: Gateway, verbosity: Verbosity
) -> None:
marker: Final = uuid.uuid4().hex
with _upstream() as wire, gateway.scenario() as scenario:
model: Final = _bridged_gpt(scenario, wire)
_answered(_responses(gateway, model, marker, text={"verbosity": verbosity}), marker)
sent: Final = _only_post(wire, "/v1/chat/completions")
assert sent == {"model": _GPT, "messages": _chat_messages(marker), "verbosity": verbosity}, sent
@pytest.mark.parametrize("verbosity", ["low", "high"])
def test_regression_explicit_bridge_stream_sends_text_verbosity_as_top_level_verbosity(
gateway: Gateway, verbosity: Verbosity
) -> None:
marker: Final = uuid.uuid4().hex
with _upstream() as wire, gateway.scenario() as scenario:
model: Final = _bridged_gpt(scenario, wire)
_streamed_answer(_streamed_events(gateway, model, marker, text={"verbosity": verbosity}), marker)
sent: Final = _only_post(wire, "/v1/chat/completions")
assert sent == {
"model": _GPT,
"messages": _chat_messages(marker),
"stream": True,
**_STREAM_OPTIONS,
"verbosity": verbosity,
}, sent
def test_regression_explicit_bridge_sends_verbosity_beside_the_json_schema_response_format(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with _upstream() as wire, gateway.scenario() as scenario:
model: Final = _bridged_gpt(scenario, wire)
_answered(_responses(gateway, model, marker, text={"format": _TEXT_FORMAT, "verbosity": "low"}), marker)
sent: Final = _only_post(wire, "/v1/chat/completions")
assert sent == {
"model": _GPT,
"messages": _chat_messages(marker),
"response_format": _RESPONSE_FORMAT,
"verbosity": "low",
}, sent
def test_regression_automatic_bridge_refuses_unsupported_verbosity_before_the_wire(gateway: Gateway) -> None:
refused: Final = uuid.uuid4().hex
control: Final = uuid.uuid4().hex
with _upstream() as wire, gateway.scenario() as scenario:
model: Final = _databricks(scenario, wire, drop_params=False)
_unsupported_param_error(_responses(gateway, model, refused, text={"verbosity": "low"}))
_answered(_responses(gateway, model, control), control)
posted: Final = _posted(wire)
assert [(request.target, _marker_of(request)) for request in posted] == [
("/serving-endpoints/chat/completions", control)
], posted
def test_regression_automatic_bridge_stream_refuses_unsupported_verbosity_before_the_wire(gateway: Gateway) -> None:
refused: Final = uuid.uuid4().hex
control: Final = uuid.uuid4().hex
with _upstream() as wire, gateway.scenario() as scenario:
model: Final = _databricks(scenario, wire, drop_params=False)
_unsupported_param_error(_responses(gateway, model, refused, text={"verbosity": "high"}, stream=True))
_answered(_responses(gateway, model, control), control)
posted: Final = _posted(wire)
assert [(request.target, _marker_of(request)) for request in posted] == [
("/serving-endpoints/chat/completions", control)
], posted
@pytest.mark.parametrize("source", ["deployment", "request"])
def test_regression_allowed_openai_params_forwards_verbosity_through_the_automatic_bridge(
gateway: Gateway, source: Literal["deployment", "request"]
) -> None:
marker: Final = uuid.uuid4().hex
allowed: Final[dict[str, JsonValue]] = {"allowed_openai_params": ["verbosity"]}
with _upstream() as wire, gateway.scenario() as scenario:
model: Final = _databricks(scenario, wire, drop_params=False, **(allowed if source == "deployment" else {}))
extra: Final = allowed if source == "request" else {}
_answered(_responses(gateway, model, marker, text={"verbosity": "low"}, **extra), marker)
sent: Final = _only_post(wire, "/serving-endpoints/chat/completions")
assert sent == {"model": _DATABRICKS, "messages": _chat_messages(marker), "verbosity": "low"}, sent
@pytest.mark.parametrize("source", ["deployment", "request"])
def test_guard_drop_params_drops_unsupported_verbosity_and_keeps_the_json_schema_format(
gateway: Gateway, source: Literal["deployment", "request"]
) -> None:
marker: Final = uuid.uuid4().hex
with _upstream() as wire, gateway.scenario() as scenario:
model: Final = _databricks(scenario, wire, drop_params=source == "deployment")
extra: Final[dict[str, JsonValue]] = {"drop_params": True} if source == "request" else {}
_answered(
_responses(gateway, model, marker, text={"format": _TEXT_FORMAT, "verbosity": "low"}, **extra), marker
)
sent: Final = _only_post(wire, "/serving-endpoints/chat/completions")
assert sent == {"model": _DATABRICKS, "messages": _chat_messages(marker), "response_format": _RESPONSE_FORMAT}, sent
def test_guard_drop_params_stream_drops_unsupported_verbosity(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with _upstream() as wire, gateway.scenario() as scenario:
model: Final = _databricks(scenario, wire, drop_params=True)
_streamed_answer(_streamed_events(gateway, model, marker, text={"verbosity": "high"}), marker)
sent: Final = _only_post(wire, "/serving-endpoints/chat/completions")
assert sent == {"model": _DATABRICKS, "messages": _chat_messages(marker), "stream": True}, sent
def test_guard_explicit_bridge_without_verbosity_sends_only_the_json_schema_format(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with _upstream() as wire, gateway.scenario() as scenario:
model: Final = _bridged_gpt(scenario, wire)
_answered(_responses(gateway, model, marker, text={"format": _TEXT_FORMAT}), marker)
sent: Final = _only_post(wire, "/v1/chat/completions")
assert sent == {"model": _GPT, "messages": _chat_messages(marker), "response_format": _RESPONSE_FORMAT}, sent
def test_guard_explicit_bridge_without_text_sends_no_verbosity(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with _upstream() as wire, gateway.scenario() as scenario:
model: Final = _bridged_gpt(scenario, wire)
_answered(_responses(gateway, model, marker), marker)
sent: Final = _only_post(wire, "/v1/chat/completions")
assert sent == {"model": _GPT, "messages": _chat_messages(marker)}, sent
@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"])
def test_guard_native_responses_route_forwards_text_verbosity_unchanged(gateway: Gateway, stream: bool) -> None:
marker: Final = uuid.uuid4().hex
text: Final[dict[str, JsonValue]] = {"format": _TEXT_FORMAT, "verbosity": "high"}
with _upstream() as wire, gateway.scenario() as scenario:
model: Final = _native_gpt(scenario, wire)
if stream:
_streamed_answer(_streamed_events(gateway, model, marker, text=text), marker)
else:
_answered(_responses(gateway, model, marker, text=text), marker)
sent: Final = _only_post(wire, "/v1/responses")
assert sent == {"model": _GPT, "input": _prompt(marker), "text": text, **({"stream": True} if stream else {})}, sent
_CLIENT_TOOL_SEARCH: Final[dict[str, JsonValue]] = {"type": "tool_search", "execution": "client"}
_CLIENT_TOOL_SEARCH_ON_THE_WIRE: Final[dict[str, JsonValue]] = {
"type": "function",
"function": {
"name": "tool_search",
"description": (
"Search the deferred tools by keyword. The matching tools are loaded and become callable on the next turn."
),
"parameters": {
"type": "object",
"properties": {
"query": {"type": "string", "description": "Search query for deferred tools."},
"limit": {"type": "integer", "description": "Maximum number of tools to return. Defaults to 8."},
},
"required": ["query"],
"additionalProperties": False,
},
},
}
@dataclass(frozen=True, slots=True)
class _FollowUpWire:
sent: JsonValue
expected: dict[str, JsonValue]
def _stored_prompts_config(directory: Path) -> Path:
stock: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
general: Final = _JSON_OBJECT.validate_python(stock["general_settings"])
path: Final = directory / "verbosity-stored-turns.yaml"
path.write_text(yaml.safe_dump({**stock, "general_settings": {**general, "store_prompts_in_spend_logs": True}}))
return path
@pytest.fixture(scope="module")
def stored_turns_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[OwnedProxy, Wire]]:
directory: Final = tmp_path_factory.mktemp("verbosity-stored-turns")
with _upstream() as wire, gateway_from_environment() as gateway:
with owned_proxy_process(gateway, directory, {}, config=_stored_prompts_config(directory)) as owned:
yield owned, wire
def _stored_first_turn(proxy: Gateway, model: str, marker: str) -> str:
first: Final = _responses(proxy, model, marker, tools=[_CLIENT_TOOL_SEARCH])
_answered(first, marker)
response_id: Final = str(_JSON_OBJECT.validate_json(first.text)["id"])
eventually(
lambda: proxy.rows('SELECT request_id FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)),
lambda rows: any(rv.same_response(str(row["request_id"]), response_id) for row in rows),
seconds=70,
)
return response_id
def _client_tool_search_follow_up(
stored_turns_proxy: tuple[OwnedProxy, Wire], stream: bool, **fields: JsonValue
) -> _FollowUpWire:
owned, wire = stored_turns_proxy
proxy: Final = owned.gateway
first: Final = uuid.uuid4().hex
second: Final = uuid.uuid4().hex
with proxy.scenario() as scenario:
model: Final = _bridged_gpt(scenario, wire)
previous: Final = _stored_first_turn(proxy, model, first)
wire.drain()
follow_up: Final[dict[str, JsonValue]] = {
"previous_response_id": previous,
"tools": [_CLIENT_TOOL_SEARCH],
**fields,
}
if stream:
_streamed_answer(_streamed_events(proxy, model, second, **follow_up), second)
else:
_answered(_responses(proxy, model, second, **follow_up), second)
sent: Final = _only_post(wire, "/v1/chat/completions")
return _FollowUpWire(
sent,
{
"model": _GPT,
"messages": [
*_chat_messages(first),
{"role": "assistant", "content": _answer(first)},
*_chat_messages(second),
],
"tools": [_CLIENT_TOOL_SEARCH_ON_THE_WIRE],
**({"stream": True, **_STREAM_OPTIONS} if stream else {}),
},
)
@pytest.mark.timeout(240)
@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"])
@pytest.mark.parametrize("verbosity", ["low", "high"])
def test_regression_client_tool_search_follow_up_sends_text_verbosity_as_top_level_verbosity(
stored_turns_proxy: tuple[OwnedProxy, Wire], stream: bool, verbosity: Verbosity
) -> None:
observed: Final = _client_tool_search_follow_up(stored_turns_proxy, stream, text={"verbosity": verbosity})
assert observed.sent == {**observed.expected, "verbosity": verbosity}, observed.sent
@pytest.mark.timeout(240)
@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"])
def test_guard_client_tool_search_follow_up_without_verbosity_sends_no_verbosity(
stored_turns_proxy: tuple[OwnedProxy, Wire], stream: bool
) -> None:
observed: Final = _client_tool_search_follow_up(stored_turns_proxy, stream)
assert observed.sent == observed.expected, observed.sent

View file

@ -715,8 +715,11 @@ def test_cyberark_incremental_update_encrypts_reloads_on_restart_and_delete_clea
}, saved.text
stored_full: Final = _stored_override(database_url, "cyberark")
assert set(stored_full) == set(full_config), stored_full
assert all(stored_full[key] != value for key, value in full_config.items()), stored_full
serialized_full: Final = json.dumps(stored_full)
assert all(value not in serialized_full for value in full_config.values()), serialized_full
assert all(value not in serialized_full for value in (cyberark_wire.url, account, username, api_key)), (
serialized_full
)
model_one: Final = "integration-cyberark-" + uuid.uuid4().hex
_new_model(first, model_one, provider_wire.url, secret_names[0])

View file

@ -37,13 +37,15 @@ import litellm.batches.main as bm
import asyncio
import datetime
import json
from collections.abc import Mapping
from collections.abc import Awaitable, Callable, Mapping, Sequence
from typing import Final
import httpx
import respx
from pydantic import TypeAdapter
from pydantic import TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
from litellm.integrations.custom_logger import CustomLogger
from openai.types.batch import Batch
from litellm.types.utils import LiteLLMBatch
# --------------------------------------------------------------------------- #
@ -1204,3 +1206,183 @@ async def test_delete_batch_output_file(respx_mock: respx.MockRouter, monkeypatc
assert delete_route.call_count == 1
assert delete_response.id == "file-output123"
assert delete_response.deleted is True
VLLM_ROUTER_API_BASE: Final = "http://vllm-router.test/v1"
VLLM_ROUTER_BATCH: Final = MappingProxyType(
{
"id": "batch_5f8dac",
"status": "pending",
"input_file_id": "file-b73f4c",
"created_at": 1793239207,
"endpoint": "/v1/chat/completions",
"completion_window": "24h",
"output_file_id": None,
"total_requests": None,
"completed_requests": 0,
"failed_requests": 0,
"metadata": {},
}
)
async def _create_on_vllm_router() -> LiteLLMBatch:
return await litellm.acreate_batch(
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id="file-b73f4c",
custom_llm_provider="hosted_vllm",
api_key="sk-test",
api_base=VLLM_ROUTER_API_BASE,
)
async def _retrieve_from_vllm_router() -> LiteLLMBatch:
return await litellm.aretrieve_batch(
batch_id="batch_5f8dac", custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE
)
VLLM_ROUTER_BATCH_CALLS: Final = (
pytest.param("POST", "/batches", _create_on_vllm_router, id="acreate_batch"),
pytest.param("GET", "/batches/batch_5f8dac", _retrieve_from_vllm_router, id="aretrieve_batch"),
)
@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_BATCH_CALLS)
@pytest.mark.parametrize(
"router_status, openai_status",
(
("pending", "validating"),
("running", "in_progress"),
("completed", "completed"),
("failed", "failed"),
("cancelled", "cancelled"),
),
)
async def test_hosted_vllm_router_batch_comes_back_as_an_openai_batch(
respx_mock: respx.MockRouter,
method: str,
path: str,
call_router: Callable[[], Awaitable[LiteLLMBatch]],
router_status: str,
openai_status: str,
):
respx_mock.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond(
200, json={**VLLM_ROUTER_BATCH, "status": router_status}
)
batch: Final = await call_router()
assert (batch.id, batch.object, batch.status) == ("batch_5f8dac", "batch", openai_status)
@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_BATCH_CALLS)
@pytest.mark.parametrize("field, value", (("status", "queued"), ("object", "file")))
async def test_hosted_vllm_router_batch_with_an_unknown_value_is_rejected(
respx_mock: respx.MockRouter,
method: str,
path: str,
call_router: Callable[[], Awaitable[LiteLLMBatch]],
field: str,
value: str,
):
respx_mock.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond(
200, json={**VLLM_ROUTER_BATCH, field: value}
)
with pytest.raises(ValidationError) as exc_info:
await call_router()
assert [error["loc"] for error in exc_info.value.errors()] == [(field,)]
async def _list_on_vllm_router() -> Sequence[Batch]:
page: Final = await litellm.alist_batches(
custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE
)
return page.data
async def _list_on_vllm_router_sync() -> Sequence[Batch]:
return litellm.list_batches(custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE).data
@pytest.mark.parametrize("list_router", (_list_on_vllm_router, _list_on_vllm_router_sync))
async def test_hosted_vllm_router_batch_list_comes_back_as_openai_batches(
respx_mock: respx.MockRouter, list_router: Callable[[], Awaitable[Sequence[Batch]]]
):
respx_mock.get(f"{VLLM_ROUTER_API_BASE}/batches").respond(
200,
json={
"object": "list",
"data": [
{**VLLM_ROUTER_BATCH, "id": "batch_5f8dac", "status": "pending"},
{**VLLM_ROUTER_BATCH, "id": "batch_c74802", "status": "running"},
],
"first_id": "batch_5f8dac",
"last_id": "batch_c74802",
"has_more": False,
},
)
batches: Final = await list_router()
assert [(batch.id, batch.object, batch.status) for batch in batches] == [
("batch_5f8dac", "batch", "validating"),
("batch_c74802", "batch", "in_progress"),
]
async def test_hosted_vllm_router_batch_list_passes_through_a_status_that_is_not_a_string(
respx_mock: respx.MockRouter,
):
respx_mock.get(f"{VLLM_ROUTER_API_BASE}/batches").respond(
200,
json={
"object": "list",
"data": [{**VLLM_ROUTER_BATCH, "status": ["pending"]}],
"first_id": "batch_5f8dac",
"last_id": "batch_5f8dac",
"has_more": False,
},
)
batches: Final = await _list_on_vllm_router()
assert [(batch.id, batch.object, batch.status) for batch in batches] == [("batch_5f8dac", "batch", ["pending"])]
async def test_openai_batch_list_items_come_back_as_openai_sent_them(respx_mock: respx.MockRouter):
respx_mock.get("http://openai.test/v1/batches").respond(
200,
json={
"object": "list",
"data": [
{
**_OPENAI_BATCH_JSON,
"status": "completed",
"usage": {
"input_tokens": 10,
"input_tokens_details": {"cached_tokens": 4},
"output_tokens": 5,
"output_tokens_details": {"reasoning_tokens": 2},
"total_tokens": 15,
},
},
{**_OPENAI_BATCH_JSON, "id": "batch_def456", "status": "queued"},
],
"first_id": "batch_abc123",
"last_id": "batch_def456",
"has_more": False,
},
)
page: Final = await litellm.alist_batches(
custom_llm_provider="openai", api_key="sk-test", api_base="http://openai.test/v1"
)
first, second = page.data
assert first.usage is not None
assert (first.usage.input_tokens_details.cached_tokens, first.usage.output_tokens_details.reasoning_tokens) == (4, 2)
assert (second.id, second.object, second.status) == ("batch_def456", "batch", "queued")

View file

@ -6,10 +6,11 @@ import copy
import json
import re
import threading
from collections.abc import Mapping, Sequence
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
from concurrent.futures import wait
from functools import partial
from types import MappingProxyType
from typing import Final
from typing import Final, TypeAlias
import httpx
import pytest
@ -23,6 +24,7 @@ import litellm
from litellm.constants import AWS_SIGNING_MAX_THREADS
from litellm.cost_calculator import get_response_cost_from_hidden_params
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded, refetch_model_cost_map
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers
from litellm.llms.bedrock.base_aws_llm import AWS_SIGNING_EXECUTOR
@ -3702,6 +3704,387 @@ async def test_a_model_info_object_instead_of_a_dict_still_reaches_the_provider(
assert response.answers[0].model_dump(mode="json")["name"] == "is_defect"
_AUDIO_CLIPS: Final = ("data:audio/wav;base64,YQ==",)
_VIDEO_CLIPS: Final = ("data:video/mp4;base64,YQ==",)
@pytest.mark.asyncio
async def test_a_deployment_that_advertises_media_sends_it_for_a_model_the_cost_map_does_not_list(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
backend: Final = "openrouter/example/omni-preview"
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
router: Final = litellm.Router(
model_list=[
{
"model_name": "hears",
"litellm_params": {"model": backend, "api_key": "k"},
"model_info": {"id": "media-hears", "supports_audio_input": True},
},
{
"model_name": "hears-and-sees",
"litellm_params": {"model": backend, "api_key": "k"},
"model_info": {"id": "media-both", "supports_audio_input": True, "supports_video_input": True},
},
]
)
send: Final = partial(
router.adecisions, input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS
)
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['videos']")):
await send(model="hears")
assert not route.called
await send(model="hears-and-sees")
sent: Final = json.loads(route.calls.last.request.content)
assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS))
@pytest.mark.asyncio
async def test_media_flags_on_a_deployment_whose_own_mode_is_not_decisions_never_send_clips(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
advertised: Final = {"supports_audio_input": True, "supports_video_input": True}
router: Final = litellm.Router(
model_list=[
{
"model_name": mode,
"litellm_params": {"model": f"openrouter/example/omni-{mode}-preview", "api_key": "k"},
"model_info": {"id": f"media-{mode}", "mode": mode, **advertised},
}
for mode in ("chat", "decisions")
]
)
send: Final = partial(
router.adecisions, input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS
)
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio', 'videos']")):
await send(model="chat")
assert not route.called
await send(model="decisions")
sent: Final = json.loads(route.calls.last.request.content)
assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS))
CatalogLoader: TypeAlias = Callable[[Mapping[str, Mapping[str, object]]], Awaitable[None]]
@pytest.fixture
def load_catalog_with(monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None) -> Iterator[CatalogLoader]:
async def load(rows: Mapping[str, Mapping[str, object]]) -> None:
catalog: Final = {**litellm.get_model_cost_map(url=""), **rows}
transport: Final = httpx.MockTransport(lambda _: httpx.Response(200, json=catalog))
monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP")
async with httpx.AsyncClient(transport=transport) as client:
loaded: Final = await refetch_model_cost_map(url="https://catalog.example/prices.json", client=client)
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
assert isinstance(loaded, ModelCostMapReloaded)
monkeypatch.setattr(litellm, "model_cost", loaded.model_cost_map)
litellm.get_model_info.cache_clear()
yield load
litellm.get_model_cost_map(url="")
@pytest.mark.asyncio
async def test_media_flags_on_a_deployment_without_its_own_mode_follow_the_mode_the_catalog_lists(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, load_catalog_with: CatalogLoader
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
await load_catalog_with(
{
f"openrouter/example/omni-{mode}": {"litellm_provider": "openrouter", "mode": mode}
for mode in ("chat", "decisions")
}
)
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
router: Final = litellm.Router(
model_list=[
{
"model_name": mode,
"litellm_params": {"model": f"openrouter/example/omni-{mode}", "api_key": "k"},
"model_info": {"id": f"listed-{mode}", "supports_audio_input": True, "supports_video_input": True},
}
for mode in ("chat", "decisions")
]
)
send: Final = partial(
router.adecisions, input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS
)
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio', 'videos']")):
await send(model="chat")
assert not route.called
monkeypatch.setattr(litellm, "drop_params", True)
await send(model="chat")
dropped: Final = json.loads(route.calls.last.request.content)
assert ("audio" in dropped, "videos" in dropped) == (False, False)
await send(model="decisions")
sent: Final = json.loads(route.calls.last.request.content)
assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS))
@pytest.mark.asyncio
@pytest.mark.parametrize("sibling_mode", ("responses", "embedding"))
async def test_another_deployments_mode_never_switches_off_a_deployments_own_media_flags(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, sibling_mode: str
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
shared_params: Final = {"model": "openrouter/example/omni-shared", "api_key": "k"}
router: Final = litellm.Router(
model_list=[
{"model_name": "sibling", "litellm_params": shared_params, "model_info": {"id": "s", "mode": sibling_mode}},
{
"model_name": "flagged",
"litellm_params": shared_params,
"model_info": {"id": "f", "supports_audio_input": True, "supports_video_input": True},
},
]
)
await router.adecisions(
model="flagged", input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS
)
sent: Final = json.loads(route.calls.last.request.content)
assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS))
_MEDIA_SURFACES: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType(
{
"adecisions": {"input": "review", "questions": _PREDICATE_QUESTIONS},
"asystemone": {"state": "review", "questions": _QUESTIONS},
"systemone": {"state": "review", "questions": _QUESTIONS},
}
)
def _group_member(
group: str, model: str, key: str, weight: int, model_info: Mapping[str, object], **params: object
) -> dict[str, object]:
return {
"model_name": group,
"litellm_params": {"model": model, "api_key": key, "weight": weight, **params},
"model_info": {"id": f"{group}-{key}", **model_info},
}
@pytest.mark.asyncio
@pytest.mark.parametrize("surface", tuple(_MEDIA_SURFACES))
async def test_a_group_sends_clips_to_a_deployment_that_takes_them_and_refuses_only_when_none_does(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, surface: str
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct")
monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False)
monkeypatch.setitem(
litellm.model_cost,
"cloudflare/@cf/cloudflare/clef-omni",
{"litellm_provider": "cloudflare", "mode": "evaluation", "supports_audio_input": True},
)
respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
respx_mock.post("https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef-omni").respond(
json=_RESPONSE
)
unlisted: Final = "openrouter/example/omni-preview"
listed: Final = "cloudflare/clef-omni"
router: Final = litellm.Router(
model_list=[
_group_member("unlisted", unlisted, "refuses", 1, {}),
_group_member("unlisted", unlisted, "takes", 0, {"supports_audio_input": True}),
_group_member("listed", listed, "refuses", 1, {"supports_audio_input": False}),
_group_member("listed", listed, "takes", 0, {}),
_group_member("none", unlisted, "refuses", 1, {}),
_group_member("none", listed, "refuses", 1, {"supports_audio_input": False}),
_group_member("drops", unlisted, "drops", 1, {}, drop_params=True),
_group_member("drops", listed, "takes", 0, {}),
_group_member("unresolvable", "databricks/not a serving endpoint", "unresolvable", 0, {}),
_group_member("unresolvable", listed, "takes", 1, {}),
]
)
call: Final = getattr(router, surface)
async def send(group: str, **media: object) -> object:
response: Final = call(model=group, **_MEDIA_SURFACES[surface], **media)
return await response if asyncio.iscoroutine(response) else response
for group in ("unlisted", "listed", "unresolvable"):
await send(group, audio=_AUDIO_CLIPS)
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes"
assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS)
await send("drops", audio=_AUDIO_CLIPS)
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer drops"
assert "audio" not in json.loads(respx_mock.calls.last.request.content)
await send("unlisted")
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer refuses"
sent_before_refusal: Final = len(respx_mock.calls)
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio']")):
await send("none", audio=_AUDIO_CLIPS)
assert len(respx_mock.calls) == sent_before_refusal
def _answer_unless_key_is(rejected_key: str) -> Callable[[httpx.Request], httpx.Response]:
def answer(request: httpx.Request) -> httpx.Response:
if request.headers["authorization"] == f"Bearer {rejected_key}":
return httpx.Response(401, json={"error": {"message": "invalid key"}})
return httpx.Response(200, json=_RESPONSE)
return answer
@pytest.mark.asyncio
async def test_clips_to_a_group_whose_only_taker_is_cooling_down_are_refused_rather_than_rate_limited(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").mock(
side_effect=_answer_unless_key_is("takes")
)
unlisted: Final = "openrouter/example/omni-preview"
router: Final = litellm.Router(
model_list=[
_group_member("g", unlisted, "refuses", 1, {}),
_group_member("g", unlisted, "takes", 1, {"supports_audio_input": True}),
],
num_retries=0,
cooldown_time=120,
)
with pytest.raises(litellm.AuthenticationError):
await router.adecisions(model="g-takes", input="review", questions=_PREDICATE_QUESTIONS)
sent_before_clips: Final = len(route.calls)
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio']")):
await router.adecisions(model="g", input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS)
assert len(route.calls) == sent_before_clips
@pytest.mark.asyncio
@pytest.mark.parametrize("surface", tuple(_MEDIA_SURFACES))
async def test_a_router_default_drop_params_overrides_a_deployment_when_choosing_who_takes_clips(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, surface: str
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
unlisted: Final = "openrouter/example/omni-preview"
router: Final = litellm.Router(
model_list=[
_group_member("g", unlisted, "drops", 1, {}, drop_params=True),
_group_member("g", unlisted, "takes", 0, {"supports_audio_input": True}),
],
default_litellm_params={"drop_params": False},
)
call: Final = getattr(router, surface)
async def send(**params: object) -> object:
response: Final = call(model="g", **_MEDIA_SURFACES[surface], audio=_AUDIO_CLIPS, **params)
return await response if asyncio.iscoroutine(response) else response
await send()
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes"
assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS)
await send(drop_params=True)
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer drops"
assert "audio" not in json.loads(respx_mock.calls.last.request.content)
@pytest.mark.asyncio
async def test_a_deployment_that_turns_audio_off_is_refused_or_dropped_even_when_the_cost_map_allows_it(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
monkeypatch.setitem(
litellm.model_cost,
"typesafe/jev-omni",
{
"litellm_provider": "typesafe",
"mode": "decisions",
"supports_audio_input": True,
"supports_video_input": True,
},
)
route: Final = respx_mock.post("https://example.com/v1/systemone").respond(json=_RESPONSE)
send: Final = partial(
litellm.asystemone,
model="typesafe/jev-omni",
state="test",
questions=_QUESTIONS,
api_base="https://example.com",
api_key="test",
audio=_AUDIO_CLIPS,
videos=_VIDEO_CLIPS,
model_info={"supports_audio_input": False},
)
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("does not support parameters: ['audio']")):
await send()
assert not route.called
await send(drop_params=True)
sent: Final = json.loads(route.calls.last.request.content)
assert "audio" not in sent
assert sent["videos"] == list(_VIDEO_CLIPS)
@pytest.mark.asyncio
async def test_a_media_flag_that_is_not_a_boolean_leaves_the_cost_map_in_charge_of_that_kind_only(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
route: Final = respx_mock.post("https://example.com/v1/systemone").respond(json=_RESPONSE)
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("does not support parameters: ['audio']")):
await litellm.asystemone(
model="typesafe/jev-unlisted",
state="test",
questions=_QUESTIONS,
api_base="https://example.com",
api_key="test",
audio=_AUDIO_CLIPS,
videos=_VIDEO_CLIPS,
model_info={"supports_audio_input": "true", "supports_video_input": True},
)
assert not route.called
@pytest.mark.asyncio
async def test_an_openai_format_deployment_that_advertises_media_is_still_refused_it(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE)
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio', 'videos']")):
await litellm.adecisions(
model="openai/gpt-6-luna",
input="review",
questions=_PREDICATE_QUESTIONS,
api_key="deployment-key",
audio=_AUDIO_CLIPS,
videos=_VIDEO_CLIPS,
model_info={"supports_audio_input": True, "supports_video_input": True},
)
assert not route.called
def test_dropping_every_question_answers_400_before_http(respx_mock: respx.MockRouter) -> None:
route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE)

View file

@ -1,3 +1,4 @@
from collections.abc import Awaitable, Callable
from types import MappingProxyType
from typing import Final
from urllib.parse import parse_qs, urlparse
@ -124,6 +125,61 @@ async def test_afile_retrieve_rejects_a_provider_file_without_its_size():
assert [error["loc"] for error in exc_info.value.errors()] == [("bytes",)]
VLLM_ROUTER_API_BASE: Final = "http://vllm-router.test/v1"
VLLM_ROUTER_FILE: Final = MappingProxyType(
{"id": "file-b73f4c", "bytes": 156, "created_at": 1793238707, "filename": "batch_requests.jsonl", "purpose": "batch"}
)
async def _upload_to_vllm_router() -> OpenAIFileObject:
return await litellm.acreate_file(
file=("batch_requests.jsonl", b'{"custom_id": "request-1"}'),
purpose="batch",
custom_llm_provider="hosted_vllm",
api_key="sk-test",
api_base=VLLM_ROUTER_API_BASE,
)
async def _retrieve_from_vllm_router() -> OpenAIFileObject:
return await litellm.afile_retrieve(
file_id="file-b73f4c", custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE
)
VLLM_ROUTER_FILE_CALLS: Final = (
pytest.param("POST", "/files", _upload_to_vllm_router, id="acreate_file"),
pytest.param("GET", "/files/file-b73f4c", _retrieve_from_vllm_router, id="afile_retrieve"),
)
@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_FILE_CALLS)
@respx.mock
async def test_hosted_vllm_file_without_an_object_type_comes_back_as_a_file(
method: str, path: str, call_router: Callable[[], Awaitable[OpenAIFileObject]]
):
respx.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond(200, json=dict(VLLM_ROUTER_FILE))
file_object: Final = await call_router()
assert file_object.model_dump(exclude_none=True) == {**VLLM_ROUTER_FILE, "object": "file"}
@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_FILE_CALLS)
@respx.mock
async def test_hosted_vllm_file_with_a_different_object_type_is_rejected(
method: str, path: str, call_router: Callable[[], Awaitable[OpenAIFileObject]]
):
respx.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond(
200, json={**VLLM_ROUTER_FILE, "object": "batch"}
)
with pytest.raises(ValidationError) as exc_info:
await call_router()
assert [error["loc"] for error in exc_info.value.errors()] == [("object",)]
_FILE_BODY: Final = b'{"prompt": "Hello", "completion": "Hi"}'
_FINE_TUNE_FILE_JSON: Final = MappingProxyType(
{

View file

@ -3,10 +3,14 @@ async_management_endpoint_{success,failure}_hook integration points."""
import asyncio
from datetime import datetime
from typing import Final
from unittest.mock import MagicMock
import httpx
import pytest
from fastapi import Request
from litellm import Router
from litellm.proxy._types import (
ManagementEndpointLoggingPayload,
UserAPIKeyAuth,
@ -340,3 +344,203 @@ def test_management_wrapper_swallows_post_success_errors(server_span_factory, ot
result = asyncio.run(fake_fn(data={}, user_api_key_dict=_real_user_api_key_dict(server_span)))
assert result == {"ok": True}
TEAM_NEW_PATH: Final = "/team/new"
MODEL_LIST_PATH: Final = "/v1/models"
MODEL_RETRIEVE_PATH: Final = "/v1/models/gpt-4o"
INBOUND_TRACEPARENT: Final = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"
def _request(method: str, path: str, body: bytes = b"") -> Request:
async def receive() -> dict[str, object]:
return {"type": "http.request", "body": body, "more_body": False}
return Request(
{
"type": "http",
"method": method,
"path": path,
"root_path": "",
"query_string": b"",
"headers": [(b"content-type", b"application/json")],
},
receive,
)
async def _answer_like_the_proxy(request: Request, exception: Exception) -> None:
from litellm.proxy.proxy_server import app
handler: Final = next(
app.exception_handlers[cls] for cls in type(exception).__mro__ if cls in app.exception_handlers
)
await handler(request, exception)
@pytest.mark.parametrize(
("wrapper_name", "exports_payload"),
[("management_endpoint_wrapper", True), ("management_endpoint_span_wrapper", False)],
)
def test_span_wrapper_omits_payload(server_span_factory, otel_with_exporter, monkeypatch, wrapper_name, exports_payload):
import litellm.proxy.proxy_server as proxy_server
from litellm.proxy.management_helpers import utils as mgmt_utils
otel, exporter = otel_with_exporter
monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False)
server_span: Final = server_span_factory(TEAM_NEW_PATH)
wrap: Final = getattr(mgmt_utils, wrapper_name)
@wrap
async def fake_fn(http_request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]:
return {"data": [{"id": "gpt-4o"}]}
asyncio.run(
fake_fn(
http_request=_request("POST", TEAM_NEW_PATH, b'{"team_alias": "t"}'),
user_api_key_dict=_real_user_api_key_dict(server_span),
)
)
spans: Final = [s for s in exporter.get_finished_spans() if s.name == TEAM_NEW_PATH]
assert len(spans) == 1
attribute_keys: Final = tuple(spans[0].attributes)
assert any(key.startswith("request.") for key in attribute_keys) is exports_payload
assert any(key.startswith("response.") for key in attribute_keys) is exports_payload
@pytest.mark.parametrize(
("wrapper_name", "exports_request_body"),
[("management_endpoint_wrapper", True), ("management_endpoint_span_wrapper", False)],
)
def test_span_wrapper_omits_request_body_on_failure(
server_span_factory, otel_with_exporter, monkeypatch, wrapper_name, exports_request_body
):
import litellm.proxy.proxy_server as proxy_server
from litellm.proxy.management_helpers import utils as mgmt_utils
otel, exporter = otel_with_exporter
monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False)
server_span: Final = server_span_factory(TEAM_NEW_PATH)
wrap: Final = getattr(mgmt_utils, wrapper_name)
@wrap
async def fake_fn(http_request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]:
raise make_fastapi_http_exception(400, "team_alias taken")
with pytest.raises(Exception, match="team_alias taken"):
asyncio.run(
fake_fn(
http_request=_request("POST", TEAM_NEW_PATH, b'{"team_alias": "t"}'),
user_api_key_dict=_real_user_api_key_dict(server_span),
)
)
spans: Final = [s for s in exporter.get_finished_spans() if s.name == TEAM_NEW_PATH]
assert len(spans) == 1
assert any(key.startswith("request.") for key in spans[0].attributes) is exports_request_body
@pytest.mark.parametrize(
("raised", "status_code"),
[
(RuntimeError("router settings lookup failed"), 500),
(httpx.ConnectError("All connection attempts failed"), 503),
(make_fastapi_http_exception(400, "Invalid scope parameter"), 400),
],
)
def test_span_wrapper_failure_keeps_the_answered_status(otel_with_exporter, monkeypatch, raised, status_code):
import litellm.proxy.proxy_server as proxy_server
from litellm.proxy.management_helpers import utils as mgmt_utils
otel, exporter = otel_with_exporter
monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False)
server_span: Final = _server_span(otel, MODEL_LIST_PATH, headers={})
request: Final = _request("GET", MODEL_LIST_PATH)
request.state.parent_otel_span = server_span
@mgmt_utils.management_endpoint_span_wrapper
async def failing_model_list(request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]:
raise raised
async def call_and_answer() -> None:
try:
await failing_model_list(request=request, user_api_key_dict=_real_user_api_key_dict(server_span))
except Exception as exc:
await _answer_like_the_proxy(request, exc)
asyncio.run(call_and_answer())
assert_server_span_attrs(
exporter,
expected_status=status_code,
expected_url_path=MODEL_LIST_PATH,
where=f"model list {type(raised).__name__}",
)
exported: Final = get_server_span(exporter)
assert exported is not None
assert exported.attributes.get("error.code") == str(status_code)
def _server_span(otel, path: str, headers: dict[str, str]):
server_span: Final = otel.create_litellm_proxy_request_started_span(start_time=datetime.now(), headers=headers)
otel.set_proxy_request_route_attributes(server_span, url_path=path, http_route=path)
return server_span
def _assert_ended_in_inbound_trace(exporter, path: str, handler_name: str) -> None:
assert_server_span_attrs(exporter, expected_status=200, expected_url_path=path, where=f"{path} success")
exported: Final = get_server_span(exporter)
assert exported is not None
assert exported.context.trace_id == int(INBOUND_TRACEPARENT.split("-")[1], 16)
endpoint_spans: Final = [s for s in exporter.get_finished_spans() if s.name == handler_name]
assert len(endpoint_spans) == 1
assert endpoint_spans[0].parent is not None
assert endpoint_spans[0].parent.span_id == exported.context.span_id
assert not any(key.startswith("response.") for key in endpoint_spans[0].attributes)
def _install_proxy_state(monkeypatch, router: Router | None) -> None:
import litellm.proxy.proxy_server as proxy_server
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "prisma_client", None)
monkeypatch.setattr(proxy_server, "general_settings", {})
monkeypatch.setattr(proxy_server, "user_model", None)
def test_model_list_ends_server_span_in_inbound_trace(otel_with_exporter, monkeypatch):
import litellm.proxy.proxy_server as proxy_server
otel, exporter = otel_with_exporter
monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False)
_install_proxy_state(monkeypatch, None)
server_span: Final = _server_span(otel, MODEL_LIST_PATH, headers={"traceparent": INBOUND_TRACEPARENT})
result: Final = asyncio.run(proxy_server.model_list(user_api_key_dict=_real_user_api_key_dict(server_span)))
assert result == {"data": [], "object": "list"}
_assert_ended_in_inbound_trace(exporter, MODEL_LIST_PATH, "model_list")
def test_model_retrieve_ends_server_span_in_inbound_trace(otel_with_exporter, monkeypatch):
import litellm.proxy.proxy_server as proxy_server
otel, exporter = otel_with_exporter
monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False)
_install_proxy_state(
monkeypatch,
Router(model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-x"}}]),
)
server_span: Final = _server_span(otel, MODEL_RETRIEVE_PATH, headers={"traceparent": INBOUND_TRACEPARENT})
result: Final = asyncio.run(
proxy_server.model_info(model_id="gpt-4o", user_api_key_dict=_real_user_api_key_dict(server_span))
)
assert result["id"] == "gpt-4o"
_assert_ended_in_inbound_trace(exporter, MODEL_RETRIEVE_PATH, "model_info")

View file

@ -1,3 +1,4 @@
from collections.abc import Callable
from typing import Final
import pytest
@ -7,7 +8,7 @@ from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
StandardBuiltInToolCostTracking,
)
from litellm.types.llms.openai import FileSearchTool, ResponsesAPIResponse, WebSearchOptions
from litellm.types.utils import ModelResponse, StandardBuiltInToolsParams
from litellm.types.utils import ModelResponse, ServerToolUse, StandardBuiltInToolsParams, Usage
def test_web_search_cost_low():
@ -1052,6 +1053,175 @@ def test_azure_ai_non_claude_web_search_keeps_generic_fallback() -> None:
assert get_cost_for_web_search_request(custom_llm_provider="azure_ai", usage=usage, model_info=model_info) is None
def _register_priced_deployment(
model: str, search_cost_per_query: float | None, deployment_id: str | None = None
) -> str:
from litellm import Router
token_rates: Final = {"input_cost_per_token": 3e-6, "output_cost_per_token": 1.5e-5}
search_rates: Final = (
{}
if search_cost_per_query is None
else {
"search_context_cost_per_query": {
"search_context_size_low": search_cost_per_query,
"search_context_size_medium": search_cost_per_query,
"search_context_size_high": search_cost_per_query,
}
}
)
router: Final = Router(
model_list=[
{
"model_name": "priced-deployment",
"litellm_params": {"model": model, "api_key": "k", "vertex_project": "p", "vertex_location": "global"},
"model_info": {**token_rates, **search_rates, **({} if deployment_id is None else {"id": deployment_id})},
}
]
)
return router.model_list[0]["model_info"]["id"]
def _claude_chat_response(
web_search_requests: int, model: str = "claude-sonnet-5-5"
) -> tuple[object, Usage | None]:
usage: Final = Usage(
prompt_tokens=1840,
completion_tokens=412,
total_tokens=2252,
server_tool_use=ServerToolUse(web_search_requests=web_search_requests),
)
return ModelResponse(model=model, usage=usage), usage
def _claude_messages_response(
web_search_requests: int, model: str = "claude-sonnet-5-5"
) -> tuple[object, Usage | None]:
return {
"id": "msg_vertex_claude_web_search",
"type": "message",
"role": "assistant",
"model": model,
"content": [{"type": "text", "text": "scripted answer"}],
"usage": {
"input_tokens": 1840,
"output_tokens": 412,
"server_tool_use": {"web_search_requests": web_search_requests},
},
}, None
@pytest.mark.parametrize("build_response", [_claude_chat_response, _claude_messages_response])
@pytest.mark.parametrize(("web_search_requests", "search_cost_per_query"), [(2, 0.01), (3, 0.01), (2, 0.0)])
def test_vertex_claude_deployment_priced_web_search_billed_per_request(
local_model_cost_map: None,
build_response: Callable[[int], tuple[object, Usage | None]],
web_search_requests: int,
search_cost_per_query: float,
) -> None:
deployment_id: Final = _register_priced_deployment("vertex_ai/claude-sonnet-5-5", search_cost_per_query)
response_object, usage = build_response(web_search_requests)
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model=deployment_id,
response_object=response_object,
usage=usage,
custom_llm_provider="vertex_ai",
standard_built_in_tools_params=None,
)
assert cost == pytest.approx(web_search_requests * search_cost_per_query)
@pytest.mark.parametrize("build_response", [_claude_chat_response, _claude_messages_response])
@pytest.mark.parametrize(
("deployment_model", "served_model", "custom_llm_provider"),
# Google bills Claude web search on Sonnet 4.5 and Haiku 4.5 too:
# https://cloud.google.com/model-garden/partner-models/pricing (read 2026-10-30)
[
("vertex_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "vertex_ai"),
("vertex_ai/claude-sonnet-4-5@20250929", "claude-sonnet-4-5-20250929", "vertex_ai"),
("vertex_ai/claude-haiku-4-5@20251001", "claude-haiku-4-5-20251001", "vertex_ai"),
("anthropic/claude-sonnet-5-5", "claude-sonnet-5-5", "anthropic"),
("azure_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "azure_ai"),
],
)
@pytest.mark.parametrize("deployment_id", [None, "claude-token-priced"], ids=["hashed_id", "claude_in_id"])
def test_claude_deployment_with_token_rates_only_bills_served_model_search_price(
local_model_cost_map: None,
build_response: Callable[[int, str], tuple[object, Usage | None]],
deployment_model: str,
served_model: str,
custom_llm_provider: str,
deployment_id: str | None,
) -> None:
served_search_pricing: Final = (
litellm.get_model_info(model=deployment_model).get("search_context_cost_per_query") or {}
)
served_search_price: Final = served_search_pricing.get("search_context_size_medium") or 0.0
registered_id: Final = _register_priced_deployment(deployment_model, None, deployment_id)
response_object, usage = build_response(2, served_model)
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model=registered_id,
response_object=response_object,
usage=usage,
custom_llm_provider=custom_llm_provider,
standard_built_in_tools_params=None,
)
assert served_search_price > 0
assert cost == pytest.approx(2 * served_search_price)
def test_vertex_claude_deployment_with_token_rates_only_and_unmapped_served_model_bills_no_search(
local_model_cost_map: None,
) -> None:
deployment_id: Final = _register_priced_deployment("vertex_ai/claude-sonnet-5-5", None)
usage: Final = Usage(
prompt_tokens=1840,
completion_tokens=412,
total_tokens=2252,
server_tool_use=ServerToolUse(web_search_requests=2),
)
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model=deployment_id,
response_object=ModelResponse(model="claude-unmapped-preview", usage=usage),
usage=usage,
custom_llm_provider="vertex_ai",
standard_built_in_tools_params=None,
)
assert cost == 0.0
@pytest.mark.parametrize("deployment_id", [None, "claude-code-gemini"], ids=["hashed_id", "claude_in_id"])
def test_vertex_gemini_deployment_priced_web_search_stays_per_prompt(
local_model_cost_map: None, deployment_id: str | None
) -> None:
search_cost_per_prompt: Final = 0.03
registered_id: Final = _register_priced_deployment(
"vertex_ai/gemini-2.5-flash", search_cost_per_prompt, deployment_id
)
usage: Final = Usage(
prompt_tokens=1840,
completion_tokens=412,
total_tokens=2252,
server_tool_use=ServerToolUse(web_search_requests=2),
)
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model=registered_id,
response_object=ModelResponse(model="gemini-2.5-flash", usage=usage),
usage=usage,
custom_llm_provider="vertex_ai",
standard_built_in_tools_params=None,
)
assert cost == pytest.approx(search_cost_per_prompt)
@pytest.mark.parametrize(
"kwargs",
[{"tools": 5}, {"web_search_options": None}, {"web_search_options": 5}],

View file

@ -25,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_semantic_cache_prompt_from_messages,
get_str_from_messages,
handle_any_messages_to_chat_completion_str_messages_conversion,
handle_messages_with_content_list_to_str_conversion,
hoist_images_from_tool_messages,
is_encrypted_reasoning_block,
merge_consecutive_system_messages,
@ -2675,3 +2676,16 @@ def test_tpm_projection_preserves_non_media_identity_and_opaque_values() -> None
projected, media = split_media_for_tpm(request)
assert projected is request
assert media == ()
def test_content_list_flattening_returns_new_messages_and_leaves_the_callers_untouched() -> None:
messages: Final = [
{"role": "user", "content": [{"type": "text", "text": "Hello"}, {"type": "text", "text": " world"}]},
{"role": "assistant", "content": "Hi"},
]
messages_before: Final = copy.deepcopy(messages)
flattened: Final = handle_messages_with_content_list_to_str_conversion(messages)
assert flattened == [{"role": "user", "content": "Hello world"}, {"role": "assistant", "content": "Hi"}]
assert messages == messages_before

View file

@ -3,7 +3,10 @@ Tests for litellm.litellm_core_utils.logging_utils — base64 truncation helpers
"""
import asyncio, datetime, importlib, litellm, os, pytest_asyncio
import sys
import threading
from itertools import chain
from types import FrameType
from unittest.mock import MagicMock
import pytest
@ -17,7 +20,7 @@ from litellm.litellm_core_utils.logging_utils import (
truncate_base64_in_messages,
truncate_base64_in_messages_async,
)
from collections.abc import AsyncIterator
from collections.abc import AsyncIterator, Callable, Coroutine, Sequence
from datetime import datetime as datetime_assemble_streaming
from litellm import(
Choices,
@ -28,6 +31,7 @@ from litellm import(
)
from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.types.llms.openai import AllMessageValues
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
from typing import Final
@ -99,8 +103,8 @@ class TestTruncateBase64InString:
@pytest.mark.parametrize(
"text",
[
'data: {"choices": [{"delta": {"content": "hi"}}]}\n\n' * 50_000,
"data:" * 200_000,
'data: {"choices": [{"delta": {"content": "hi"}}]}\n\n' * 50_000 + ";base64,",
"data:" * 200_000 + ";base64,",
],
ids=["sse_lines", "whitespace_free_prefixes"],
)
@ -241,6 +245,49 @@ def scan_threads(monkeypatch):
return threads
def _run_without_yielding(coroutine: Coroutine[object, object, object]) -> object:
try:
coroutine.send(None)
except StopIteration as finished:
return finished.value
coroutine.close()
raise AssertionError("the coroutine yielded to the event loop")
def _regex_calls_while(run: Callable[[], object]) -> tuple[object, tuple[str, ...]]:
calls: Final[list[str]] = [] # mutable-ok: sys.setprofile reports each call to a callback that has to record it
previous: Final = sys.getprofile()
def record(frame: FrameType, event: str, arg: object) -> None:
name: Final = getattr(arg, "__qualname__", "")
if event == "c_call" and name.startswith("Pattern."):
calls.append(name)
sys.setprofile(record)
try:
result: Final = run()
finally:
sys.setprofile(previous)
return result, tuple(calls)
def _tool_result_messages(tool_output: str) -> list[AllMessageValues]: # mutable-ok: the truncator only walks lists
return [
{"role": "user", "content": "Summarize the rows the tool returned."},
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "dump_rows", "arguments": "{}"}}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": tool_output},
]
_PLAIN_TOOL_OUTPUT_CHARS: Final = 400 * 1024
class TestTruncateBase64InMessagesAsync:
@pytest.mark.asyncio
async def test_large_payload_is_scanned_off_the_event_loop(self, monkeypatch, scan_threads):
@ -268,6 +315,83 @@ class TestTruncateBase64InMessagesAsync:
assert scan_threads
assert set(scan_threads) == {threading.get_ident()}
@pytest.mark.parametrize(
"row",
[
'{"id": 7, "name": "widget", "status": "ok"}\n',
"data: {\"choices\": [{\"delta\": {\"content\": \"hi\"}}]}\n\n",
"const total = rows.map((r) => r.price).reduce((a, b) => a + b, 0);\n",
'data: {"choices": [{"delta": {"content": "a; b"}}]}\n\n',
"Content-Type: text/plain;base64,aGVsbG8=\n",
],
ids=["json_rows", "sse_lines", "code_with_semicolons", "sse_lines_with_semicolons", "base64_marker_without_data"],
)
def test_tool_message_without_data_uri_is_copied_without_a_regex_scan(self, row: str):
tool_output: Final = (row * (_PLAIN_TOOL_OUTPUT_CHARS // len(row) + 1))[:_PLAIN_TOOL_OUTPUT_CHARS]
messages: Final = _tool_result_messages(tool_output)
result, regex_calls = _regex_calls_while(lambda: truncate_base64_in_messages(messages))
assert result == messages
assert result is not messages
assert regex_calls == ()
def test_small_data_uri_is_truncated_inline_with_a_regex_scan(self):
payload: Final = "L" * 2_000
messages: Final = _image_messages(payload)
result, regex_calls = _regex_calls_while(
lambda: _run_without_yielding(truncate_base64_in_messages_async(messages))
)
assert regex_calls
assert payload not in result[0]["content"][1]["image_url"]["url"]
assert "base64_data truncated" in result[0]["content"][1]["image_url"]["url"]
def test_large_plain_text_is_copied_inline_without_rescanning_or_a_thread_hop(
self, monkeypatch: pytest.MonkeyPatch, scan_threads: Sequence[int]
):
monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS", 1_000)
messages: Final = _tool_result_messages("plain tool output\n" * 100)
result: Final = _run_without_yielding(truncate_base64_in_messages_async(messages))
assert result == messages
assert result is not messages
assert scan_threads == []
@pytest.mark.asyncio
async def test_long_plain_text_history_is_copied_off_the_event_loop(
self, monkeypatch: pytest.MonkeyPatch, scan_threads: Sequence[int]
):
monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS", 1_000)
monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS", 10)
messages: Final = list(chain.from_iterable(_tool_result_messages(f"rows {turn}\n" * 100) for turn in range(5)))
result: Final = await truncate_base64_in_messages_async(messages)
offload_threads: Final = tuple(scan_threads)
assert result == messages
assert result is not messages
assert offload_threads
assert threading.get_ident() not in offload_threads
@pytest.mark.asyncio
async def test_plain_text_over_the_inline_limit_is_copied_off_the_event_loop(
self, monkeypatch: pytest.MonkeyPatch, scan_threads: Sequence[int]
):
monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS", 1_000)
monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS", 10_000)
messages: Final = _tool_result_messages('data: {"choices": [{"delta": {"content": "a; b"}}]}\n\n' * 400)
result: Final = await truncate_base64_in_messages_async(messages)
offload_threads: Final = tuple(scan_threads)
assert result == messages
assert result is not messages
assert offload_threads
assert threading.get_ident() not in offload_threads
@pytest.mark.asyncio
async def test_none_and_disabled_truncation_short_circuit(self, monkeypatch, scan_threads):
assert await truncate_base64_in_messages_async(None) is None

View file

@ -1,6 +1,7 @@
import copy
import json
import logging
from collections.abc import Iterator
from itertools import chain
from typing import Final
from unittest.mock import MagicMock, patch
@ -9849,3 +9850,186 @@ def test_transform_request_tool_config_point_without_tools_drops_the_key():
assert "cache_control_injection_points" not in data
assert "tools" not in data
assert litellm_params["metadata"] == {}
def _five_cache_hint_conversation() -> list[dict[str, object]]:
return [
{"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]},
{"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]},
{"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]},
{"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]},
{"role": "assistant", "content": "answer 2"},
{"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]},
]
def _cached_texts(node: object) -> Iterator[str]:
if isinstance(node, dict):
yield from ((node["text"],) if "cache_control" in node and "text" in node else ())
yield from chain.from_iterable(map(_cached_texts, node.values()))
elif isinstance(node, list):
yield from chain.from_iterable(map(_cached_texts, node))
@pytest.mark.parametrize(
("config", "model"),
[
(AnthropicConfig(), "claude-haiku-5-5"),
(VertexAIAnthropicConfig(), "claude-haiku-5-5"),
(AmazonAnthropicClaudeConfig(), "anthropic.claude-haiku-5-5"),
(AzureAnthropicConfig(), "claude-haiku-5-5"),
],
ids=["anthropic", "vertex_ai", "bedrock_invoke", "azure_ai"],
)
@pytest.mark.parametrize(
("drop_params", "expected_cached_texts"),
[
(True, ["answer 1", "turn 1", "turn 2", "turn 3"]),
("true", ["answer 1", "turn 1", "turn 2", "turn 3"]),
(False, ["answer 1", "system prompt", "turn 1", "turn 2", "turn 3"]),
],
ids=[
"drop_params_keeps_the_newest_four",
"string_drop_params_keeps_the_newest_four",
"without_drop_params_all_five_pass_through",
],
)
def test_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params(
config: AnthropicConfig,
model: str,
drop_params: bool | str,
expected_cached_texts: list[str],
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
messages: Final = _five_cache_hint_conversation()
messages_before: Final = copy.deepcopy(messages)
body: Final = config.transform_request(
model=model,
messages=messages,
optional_params={},
litellm_params={"drop_params": drop_params},
headers={},
)
assert sorted(_cached_texts(body)) == expected_cached_texts
assert json.dumps(body).count('"cache_control"') == len(expected_cached_texts)
assert messages == messages_before
def test_claude_cache_control_cap_keeps_a_cached_tool_and_drops_the_oldest_message_hint(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "drop_params", True)
messages: Final = _five_cache_hint_conversation()[1:]
tool: Final = {
"name": "get_weather",
"description": "Get the weather",
"input_schema": {"type": "object", "properties": {}},
"cache_control": {"type": "ephemeral"},
}
body: Final = AnthropicConfig().transform_request(
model="claude-haiku-5-5",
messages=messages,
optional_params={"tools": [tool]},
litellm_params={},
headers={},
)
assert body["tools"][0]["cache_control"] == {"type": "ephemeral"}
assert sorted(_cached_texts(body["messages"])) == ["answer 1", "turn 2", "turn 3"]
def test_claude_cache_control_cap_drops_a_string_assistant_turns_text_hint_before_its_tool_call_hints(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "drop_params", True)
hint: Final = {"type": "ephemeral"}
tool_calls: Final = [
{"id": f"call_{name}", "type": "function", "function": {"name": name, "arguments": "{}"}, "cache_control": hint}
for name in ("lookup", "search")
]
follow_up: Final = [
{"type": "text", "text": "follow-up", "cache_control": hint},
{"type": "text", "text": "and more", "cache_control": hint},
]
messages: Final = [
{"role": "user", "content": "question"},
{"role": "assistant", "content": "checking both", "cache_control": hint, "tool_calls": tool_calls},
{"role": "tool", "tool_call_id": "call_lookup", "content": "found"},
{"role": "tool", "tool_call_id": "call_search", "content": "none"},
{"role": "user", "content": follow_up},
]
body: Final = AnthropicConfig().transform_request(
model="claude-haiku-5-5",
messages=messages,
optional_params={},
litellm_params={},
headers={},
)
assistant_blocks: Final = body["messages"][1]["content"]
assert [block["type"] for block in assistant_blocks] == ["text", "tool_use", "tool_use"]
assert ["cache_control" in block for block in assistant_blocks] == [False, True, True]
assert json.dumps(body).count('"cache_control"') == 4
assert json.dumps(body).count('"cache_control"') == 4
def test_claude_cache_control_cap_keeps_the_newest_hints_inside_one_message(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "drop_params", True)
messages: Final = [
{
"role": "user",
"content": [
*({"type": "text", "text": f"part {n}", "cache_control": {"type": "ephemeral"}} for n in range(1, 6)),
{"type": "text", "text": "Reply with the single word OK."},
],
}
]
messages_before: Final = copy.deepcopy(messages)
body: Final = AnthropicConfig().transform_request(
model="claude-haiku-5-5",
messages=messages,
optional_params={},
litellm_params={},
headers={},
)
assert sorted(_cached_texts(body["messages"])) == ["part 2", "part 3", "part 4", "part 5"]
assert json.dumps(body).count('"cache_control"') == 4
assert messages == messages_before
def test_claude_cache_control_cap_does_not_count_a_message_level_hint_that_list_content_never_sends(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
messages: Final = [
{"role": "system", "content": "system prompt", "cache_control": {"type": "ephemeral"}},
{"role": "user", "content": "turn 1", "cache_control": {"type": "ephemeral"}},
{"role": "assistant", "content": "answer 1"},
{"role": "user", "content": "turn 2", "cache_control": {"type": "ephemeral"}},
{"role": "assistant", "content": "answer 2"},
{
"role": "user",
"content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}],
"cache_control": {"type": "ephemeral"},
},
]
body: Final = AnthropicConfig().transform_request(
model="claude-haiku-5-5",
messages=messages,
optional_params={},
litellm_params={"drop_params": True},
headers={},
)
assert sorted(_cached_texts(body)) == ["system prompt", "turn 1", "turn 2", "turn 3"]
assert json.dumps(body).count('"cache_control"') == 4

View file

@ -56,6 +56,98 @@ def test_is_anthropic_messages_url(url: str, expected: bool) -> None:
assert is_anthropic_messages_url(url) is expected
@pytest.mark.parametrize(
("configured_base", "url", "expected"),
[
pytest.param(None, "https://api.anthropic.com/v1/messages", 512, id="default-upstream"),
pytest.param(None, "https://api.anthropic.com:443/v1/messages/", 512, id="default-upstream-explicit-port"),
pytest.param("http://relay.test:8080", "https://api.anthropic.com/v1/messages", 512, id="direct-target"),
pytest.param("https://api.anthropic.com", "https://api.anthropic.com/v1/messages", 512, id="configured-default"),
pytest.param("http://relay.test:8080", "http://relay.test:8080/v1/messages", 255, id="configured-upstream"),
pytest.param("http://relay.test:8080/a", "http://relay.test:8080/a/v1/messages", 255, id="configured-prefix"),
pytest.param("https://Relay.Test", "https://relay.test/v1/messages", 255, id="configured-upper-case-host"),
pytest.param("https://relay.test:443", "https://relay.test/v1/messages", 255, id="configured-https-port"),
pytest.param("http://relay.test:80/a", "http://relay.test/a/v1/messages", 255, id="configured-http-port"),
pytest.param("http://relay.test:8080", "http://relay.test:8081/v1/messages", None, id="other-port"),
pytest.param("http://relay.test:notaport", "http://relay.test/v1/messages", None, id="invalid-configured-base"),
pytest.param(None, "https://other-vendor.test/v1/messages", None, id="other-vendor"),
pytest.param(None, "https://api.anthropic.com/v1/messages/count_tokens", None, id="count-tokens"),
],
)
def test_anthropic_upstream_user_id_limit(
monkeypatch: pytest.MonkeyPatch, configured_base: str | None, url: str, expected: int | None
) -> None:
from litellm.llms.anthropic.common_utils import anthropic_upstream_user_id_limit
monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False)
monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False)
if configured_base is not None:
monkeypatch.setenv("ANTHROPIC_API_BASE", configured_base)
assert anthropic_upstream_user_id_limit(url) == expected
def test_anthropic_pass_through_api_base_prefers_the_api_base_variable(monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.llms.anthropic.common_utils import anthropic_pass_through_api_base
monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://fallback.test")
monkeypatch.setenv("ANTHROPIC_API_BASE", "http://preferred.test")
assert anthropic_pass_through_api_base() == "http://preferred.test"
monkeypatch.delenv("ANTHROPIC_API_BASE")
assert anthropic_pass_through_api_base() == "http://fallback.test"
# Each row was sent to https://api.anthropic.com/v1/messages on 2026-10-30, False rows answered 400.
@pytest.mark.parametrize(
("user_id", "expected"),
[
pytest.param("user-1", True, id="opaque"),
pytest.param("", True, id="empty"),
pytest.param('{"device_id":"abc","session_id":"def"}', True, id="claude-code-shape"),
pytest.param("1234567890", True, id="digits"),
pytest.param("+14155550100", True, id="phone-shape"),
pytest.param("a@b", True, id="no-dot"),
pytest.param("@b.co", True, id="nothing-before-at-sign"),
pytest.param("a@.co", True, id="nothing-between-at-sign-and-dot"),
pytest.param("a@b.", True, id="nothing-after-dot"),
pytest.param("a @ b.co", True, id="spaces-around-at-sign"),
pytest.param("a\u00a0@b.co", False, id="no-break-space-before-at-sign"),
pytest.param("a@b\n.co", False, id="newline-before-dot"),
pytest.param("line one\nline two", False, id="newline"),
pytest.param("a\tb", False, id="tab"),
pytest.param("a\x1f@b.co", False, id="unit-separator-before-at-sign"),
pytest.param("a@\x1cb.co", False, id="file-separator-after-at-sign"),
pytest.param("a\u200bb", False, id="zero-width-space"),
pytest.param("a@b.co", False, id="email"),
pytest.param("user a@b.co here", False, id="email-inside-text"),
pytest.param('{"device_id":"a@b.co"}', False, id="email-inside-json"),
pytest.param("a@@example.com", False, id="doubled-at-sign"),
pytest.param("@@example.com", False, id="leading-doubled-at-sign"),
pytest.param("a@b@c.co", False, id="two-at-signs"),
pytest.param("a@b..co", False, id="doubled-dot"),
pytest.param("a@.b.co", False, id="dot-after-at-sign"),
pytest.param("a@b.c d", False, id="email-before-space"),
],
)
def test_anthropic_accepts_user_id(user_id: str, expected: bool) -> None:
from litellm.constants import ANTHROPIC_MAX_USER_ID_LENGTH
from litellm.llms.anthropic.common_utils import anthropic_accepts_user_id
assert anthropic_accepts_user_id(user_id, ANTHROPIC_MAX_USER_ID_LENGTH) is expected
@pytest.mark.parametrize("character", ["a", "\u00e9", "\U0001f600"], ids=["ascii", "two-bytes", "four-bytes"])
def test_anthropic_accepts_user_id_counts_characters_up_to_the_limit(character: str) -> None:
from litellm.constants import ANTHROPIC_MAX_USER_ID_LENGTH
from litellm.llms.anthropic.common_utils import anthropic_accepts_user_id
assert anthropic_accepts_user_id(character * ANTHROPIC_MAX_USER_ID_LENGTH, ANTHROPIC_MAX_USER_ID_LENGTH) is True
assert (
anthropic_accepts_user_id(character * (ANTHROPIC_MAX_USER_ID_LENGTH + 1), ANTHROPIC_MAX_USER_ID_LENGTH) is False
)
def test_anthropic_model_not_found_maps_to_not_found_but_authentication_error_does_not() -> None:
with respx.mock(assert_all_called=True) as router:
router.post("https://api.anthropic.com/v1/messages").mock(

View file

@ -26,7 +26,6 @@ from litellm.types.decisions import (
DecisionsIRScoreAnswer,
DecisionsRequestBody,
OpenAIDecisionRequestBody,
UnsupportedDecisionsRequest,
)
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
@ -279,7 +278,7 @@ def test_a_malformed_systemone_response_names_the_field_at_fault(
assert [item["loc"] for item in error.value.errors()] == [field]
def test_each_media_kind_is_gated_by_its_own_support_flag() -> None:
def test_each_media_kind_is_sent_in_its_own_field() -> None:
audio: Final = ["data:audio/wav;base64,YQ=="]
videos: Final = ["data:video/mp4;base64,YQ=="]
audio_only: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "audio": audio}))
@ -287,22 +286,16 @@ def test_each_media_kind_is_gated_by_its_own_support_flag() -> None:
_SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "videos": videos})
)
assert ir_to_systemone_request("clef-omni", audio_only, supports_audio=True) == {
assert ir_to_systemone_request("clef-omni", audio_only) == {
"model": "clef-omni",
**_SYSTEMONE_REQUEST,
"audio": audio,
}
assert ir_to_systemone_request("clef-omni", video_only, supports_video=True) == {
assert ir_to_systemone_request("clef-omni", video_only) == {
"model": "clef-omni",
**_SYSTEMONE_REQUEST,
"videos": videos,
}
assert isinstance(
ir_to_systemone_request("clef-omni", audio_only, supports_video=True), UnsupportedDecisionsRequest
)
assert isinstance(
ir_to_systemone_request("clef-omni", video_only, supports_audio=True), UnsupportedDecisionsRequest
)
_ASSET_REQUEST: Final[Mapping[str, object]] = {

View file

@ -11259,3 +11259,86 @@ def test_converse_sends_only_the_header_betas_when_header_and_body_both_carry_be
"thinking-binding-controls-2026-08-01",
"thinking-display-updates-2026-08-18",
]
def _five_cache_hint_conversation() -> list[dict[str, object]]:
return [
{"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]},
{"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]},
{"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]},
{"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]},
{"role": "assistant", "content": "answer 2"},
{"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]},
]
@pytest.mark.parametrize(
("drop_params", "expected_system_cache_points"),
[(True, 0), ("true", 0), (False, 1)],
ids=[
"drop_params_drops_the_oldest_hint",
"string_drop_params_drops_the_oldest_hint",
"without_drop_params_all_five_pass_through",
],
)
def test_converse_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params(
drop_params: bool | str, expected_system_cache_points: int, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
messages: Final = _five_cache_hint_conversation()
messages_before: Final = copy.deepcopy(messages)
request: Final = AmazonConverseConfig().transform_request(
model="anthropic.claude-haiku-5-5",
messages=messages,
optional_params={},
litellm_params={"drop_params": drop_params},
headers={},
)
assert len(list(_cache_points(request["system"]))) == expected_system_cache_points
assert [len(list(_cache_points(message))) for message in request["messages"]] == [1, 1, 1, 0, 1]
assert messages == messages_before
def test_converse_claude_tool_message_with_several_marked_blocks_counts_as_one_cache_point_against_the_cap(
monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
messages: Final = [
{
"role": "user",
"content": [
{"type": "text", "text": "policy", "cache_control": {"type": "ephemeral"}},
{"type": "text", "text": "question", "cache_control": {"type": "ephemeral"}},
],
},
{
"role": "assistant",
"content": None,
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}],
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": [
{"type": "text", "text": "part 1", "cache_control": {"type": "ephemeral"}},
{"type": "text", "text": "part 2", "cache_control": {"type": "ephemeral"}},
{"type": "text", "text": "part 3", "cache_control": {"type": "ephemeral"}},
],
},
]
messages_before: Final = copy.deepcopy(messages)
request: Final = AmazonConverseConfig().transform_request(
model="anthropic.claude-haiku-5-5",
messages=messages,
optional_params={
"tools": [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}]
},
litellm_params={"drop_params": True},
headers={},
)
assert len(list(_cache_points(request["messages"]))) == 3
assert messages == messages_before

View file

@ -1,5 +1,5 @@
import asyncio, importlib, litellm, litellm.litellm_core_utils.get_model_cost_map as bedrock_govcloud_model_cost_map, pytest
import asyncio, importlib, litellm, pytest
from typing import Final
@ -1563,7 +1563,6 @@ def _pr4_bedrock_env(monkeypatch: pytest.MonkeyPatch) -> None:
def _use_local_model_cost_map() -> Iterator[None]:
with pytest.MonkeyPatch.context() as monkeypatch:
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
importlib.reload(bedrock_govcloud_model_cost_map)
importlib.reload(litellm)
yield

View file

@ -144,6 +144,49 @@ async def test_client_payload_error_mid_stream_raises_read_error():
assert mock_response.closed is True
@pytest.mark.asyncio
async def test_bytes_buffered_before_a_dropped_connection_are_yielded_before_the_read_error():
loop: Final = asyncio.get_running_loop()
content: Final = aiohttp.StreamReader(aiohttp.base_protocol.BaseProtocol(loop), 2**16, loop=loop)
content.feed_data(b"event: message_start\n\n")
content.set_exception(aiohttp.ClientPayloadError("Response payload is not completed"))
mock_response: Final = MockAiohttpResponse()
mock_response.content = content
received: Final[list[bytes]] = []
async def _drain() -> None:
async for chunk in AiohttpResponseStream(mock_response): # pyright: ignore[reportArgumentType] # mock response
received.append(chunk)
with pytest.raises(httpx.ReadError):
await _drain()
assert received == [b"event: message_start\n\n"]
assert mock_response.closed is True
@pytest.mark.asyncio
async def test_bytes_already_read_before_a_dropped_connection_are_not_yielded_again():
loop: Final = asyncio.get_running_loop()
content: Final = aiohttp.StreamReader(aiohttp.base_protocol.BaseProtocol(loop), 2**16, loop=loop)
content.feed_data(b"event: message_start\n\n")
content.feed_data(b"event: ping\n\n")
assert await content.read(7) == b"event: "
content.set_exception(aiohttp.ClientPayloadError("Response payload is not completed"))
mock_response: Final = MockAiohttpResponse()
mock_response.content = content
received: Final[list[bytes]] = []
async def _drain() -> None:
async for chunk in AiohttpResponseStream(mock_response): # pyright: ignore[reportArgumentType] # mock response
received.append(chunk)
with pytest.raises(httpx.ReadError):
await _drain()
assert received == [b"message_start\n\nevent: ping\n\n"]
@pytest.mark.asyncio
async def test_client_payload_error_before_first_chunk_raises_read_error():
"""A connection reset before any body byte must surface, not yield an empty 200 body"""

View file

@ -2773,3 +2773,66 @@ def test_claude_json_schema_stream_split_across_json_tools_keeps_the_non_stream_
choices: Final = _json_schema_stream_choices(respx_mock, call, split_stream)
assert [choice.finish_reason for choice in choices if choice.finish_reason] == [answer.finish_reason]
@pytest.mark.parametrize(
("model", "base_model"),
[("databricks-claude-haiku-5-5", None), ("support-bot-endpoint", "databricks/databricks-claude-haiku-5-5")],
ids=["claude_in_the_endpoint_name", "claude_named_only_by_base_model"],
)
def test_databricks_claude_request_over_the_cache_control_cap_keeps_the_newest_four_hints_under_drop_params(
model: str, base_model: str | None, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
hint: Final = {"type": "ephemeral"}
messages: Final = [
{"role": "system", "content": "system prompt", "cache_control": hint},
{"role": "user", "content": "turn 1", "cache_control": hint},
{"role": "assistant", "content": "answer 1", "cache_control": hint},
{"role": "user", "content": "turn 2", "cache_control": hint},
{"role": "assistant", "content": "answer 2"},
{"role": "user", "content": "turn 3", "cache_control": hint},
]
messages_before: Final = copy.deepcopy(messages)
body: Final = DatabricksConfig().transform_request(
model=model,
messages=messages,
optional_params={},
litellm_params={"drop_params": True, "custom_llm_provider": "databricks", "base_model": base_model},
headers={},
)
assert "cache_control" not in json.dumps(body["messages"][0])
assert json.dumps(body["messages"]).count('"cache_control"') == 4
assert messages == messages_before
@pytest.mark.parametrize(
"optional_params",
[{"cache_control": {"type": "ephemeral"}}, {"extra_body": {"cache_control": {"type": "ephemeral"}}}],
ids=["top_level_cache_control", "extra_body_cache_control"],
)
def test_databricks_claude_cache_control_cap_keeps_room_for_a_top_level_cache_control_under_drop_params(
optional_params: dict[str, object], monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
hint: Final = {"type": "ephemeral"}
messages: Final = [
{"role": "user", "content": "turn 1", "cache_control": hint},
{"role": "assistant", "content": "answer 1", "cache_control": hint},
{"role": "user", "content": "turn 2", "cache_control": hint},
{"role": "assistant", "content": "answer 2", "cache_control": hint},
{"role": "user", "content": "turn 3"},
]
body: Final = DatabricksConfig().transform_request(
model="databricks-claude-haiku-5-5",
messages=messages,
optional_params=copy.deepcopy(optional_params),
litellm_params={"drop_params": True, "custom_llm_provider": "databricks"},
headers={},
)
assert "cache_control" not in json.dumps(body["messages"][0])
assert json.dumps(body["messages"]).count('"cache_control"') == 3

View file

@ -1,6 +1,8 @@
import asyncio
import copy
import json
import os
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@ -200,3 +202,38 @@ async def test_deepinfra_tool_message_content_transformation_async():
print(f"✓ Async test passed: {tool_message['content']}")
print("\n✅ DeepInfra async tool message transformation test passed!")
@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"])
async def test_deepinfra_request_stringifies_tool_content_without_touching_the_callers_message(is_async: bool) -> None:
from litellm.llms.deepinfra.chat.transformation import DeepInfraConfig
config: Final = DeepInfraConfig()
messages: Final = [
{"role": "user", "content": "Calculate 10 + 10"},
{
"role": "assistant",
"content": "",
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "calculator", "arguments": "{}"}}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": [{"type": "text", "text": "20"}]},
]
messages_before: Final = copy.deepcopy(messages)
request_kwargs: Final = {
"model": "deepseek-ai/DeepSeek-V3.2",
"messages": messages,
"optional_params": {},
"litellm_params": {},
"headers": {},
}
body: Final = (
await config.async_transform_request(**request_kwargs)
if is_async
else config.transform_request(**request_kwargs)
)
assert body["messages"][2]["content"] == "20"
assert messages == messages_before

View file

@ -1,3 +1,4 @@
import copy
import json
import logging
from collections.abc import Callable
@ -537,6 +538,26 @@ def test_tool_search_reply_sent_to_fireworks_drops_litellm_response_fields():
}
def test_cache_control_is_stripped_from_the_request_but_kept_on_the_callers_messages():
hint: Final = {"type": "ephemeral"}
messages: Final = [
{"role": "system", "content": [{"type": "text", "text": "a long cached prompt", "cache_control": hint}]},
{"role": "user", "content": "hi", "cache_control": hint},
]
caller_messages_before: Final = json.dumps(messages)
request: Final = FireworksAIConfig().transform_request(
model="accounts/fireworks/models/gpt-oss-120b",
messages=messages,
optional_params={},
litellm_params={},
headers={},
)
assert "cache_control" not in json.dumps(request["messages"])
assert json.dumps(messages) == caller_messages_before
# -----------------------------------------------------------------------------
# Regression tests for legacy / OpenAPI $ref defs in tool parameters.
#
@ -2477,3 +2498,43 @@ def test_translate_developer_role_follows_the_deployment_model_info_over_the_cos
)
assert [message["role"] for message in messages] == expected_roles
def test_fireworks_request_rewrites_tools_without_touching_the_callers_tool() -> None:
tools: Final = [
{
"type": "function",
"function": {
"name": "create_thing",
"strict": True,
"parameters": {
"type": "object",
"properties": {"thing": {"$ref": "#/definitions/Thing"}},
"definitions": {"Thing": {"type": "object", "properties": {"id": {"type": "string"}}}},
},
},
}
]
tools_before: Final = copy.deepcopy(tools)
request: Final = FireworksAIConfig().transform_request(
model="accounts/fireworks/models/gpt-oss-120b",
messages=[{"role": "user", "content": "Make a thing"}],
optional_params={"tools": tools},
litellm_params={},
headers={},
)
assert request["tools"] == [
{
"type": "function",
"function": {
"name": "create_thing",
"parameters": {
"type": "object",
"properties": {"thing": {"type": "object", "properties": {"id": {"type": "string"}}}},
},
},
}
]
assert tools == tools_before

View file

@ -1,3 +1,4 @@
import copy
import logging
from typing import Final
from unittest.mock import patch
@ -589,3 +590,19 @@ async def test_streamed_reasoning_reply_can_be_replayed(is_async: bool, with_too
}
]
assert reply.reasoning_content == "I should check the weather."
def test_groq_request_drops_null_assistant_fields_without_touching_the_callers_messages() -> None:
messages: Final = [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hello", "function_call": None, "tool_calls": None},
{"role": "user", "content": "Again"},
]
messages_before: Final = copy.deepcopy(messages)
body: Final = GroqChatConfig().transform_request(
model="openai/gpt-oss-120b", messages=messages, optional_params={}, litellm_params={}, headers={}
)
assert body["messages"][1] == {"role": "assistant", "content": "Hello"}
assert messages == messages_before

View file

@ -1,4 +1,6 @@
import copy
import json
from typing import Final
from unittest.mock import MagicMock, patch
import pytest
@ -397,3 +399,43 @@ def test_hosted_vllm_custom_tools_use_top_level_input_schema():
assert tools[0]["function"]["name"] == "search"
assert tools[0]["function"]["description"] == "Search docs"
assert tools[0]["function"]["parameters"] == input_schema
def _strict_weather_tool() -> dict[str, object]:
return {
"type": "function",
"function": {
"name": "get_weather",
"strict": True,
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
"additionalProperties": False,
},
},
}
def test_hosted_vllm_tools_are_rewritten_without_touching_the_callers_tool() -> None:
tools: Final = [_strict_weather_tool()]
tools_before: Final = copy.deepcopy(tools)
optional_params: Final = HostedVLLMChatConfig().map_openai_params(
non_default_params={"tools": tools}, optional_params={}, model="gpt-oss-120b", drop_params=False
)
assert optional_params["tools"] == [
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
}
]
assert tools == tools_before

View file

@ -1,3 +1,4 @@
import copy
import json
import logging
from collections.abc import Mapping
@ -1212,7 +1213,10 @@ def test_mistral_sends_only_the_text_items_of_a_native_thinking_chunk_beside_an_
"content": [
{
"type": "thinking",
"thinking": [{"type": "text", "text": "It has whiskers."}, {"type": "reference", "reference_ids": [1]}],
"thinking": [
{"type": "text", "text": "It has whiskers."},
{"type": "reference", "reference_ids": [1]},
],
},
{"type": "text", "text": "A cat."},
],
@ -1672,6 +1676,18 @@ def test_mistral_transform_request_drops_an_empty_tool_message_whose_call_id_is_
assert [m.get("role") for m in request["messages"]] == ["user", "assistant"]
def test_mistral_request_drops_a_user_name_without_touching_the_callers_message() -> None:
messages: Final = [{"role": "user", "content": "Hi"}, {"role": "user", "content": "", "name": "alice"}]
messages_before: Final = copy.deepcopy(messages)
body: Final = MistralConfig().transform_request(
model="mistral-medium-latest", messages=messages, optional_params={}, litellm_params={}, headers={}
)
assert "name" not in json.dumps(body["messages"])
assert messages == messages_before
def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]:
return tuple(
r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage()

View file

@ -1824,6 +1824,94 @@ class TestToolSchemaCombinatorFlatteningForOpenAI:
assert request["tools"][0] is tool
class TestCacheControlStrippingDoesNotMutateCallerInput:
"""
Stripping cache_control for a provider that cannot use it must not reach back
into the caller's own message and tool objects.
filter_value_from_dict deletes the key in place and recurses into nested dicts
and lists, and remove_cache_control_flag_from_messages_and_tools assigned the
result back into the caller's list, so one call to any OpenAI-compatible
provider permanently stripped cache_control from a list the caller still holds.
Reusing that list on Anthropic or Bedrock afterwards then silently lost prompt
caching, with no error and full-price billing.
"""
def setup_method(self):
self.config = OpenAIGPTConfig()
@pytest.fixture(autouse=True)
def _clean_openai_base_env(self, monkeypatch):
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
monkeypatch.setattr(litellm, "api_base", None, raising=False)
@staticmethod
def _messages():
return [
{
"role": "system",
"content": [
{
"type": "text",
"text": "a long cached system prompt",
"cache_control": {"type": "ephemeral"},
}
],
},
{
"role": "user",
"content": "Hello",
"cache_control": {"type": "ephemeral"},
},
]
@staticmethod
def _tools():
return [
{
"type": "function",
"function": {"name": "get_weather", "parameters": {"type": "object"}},
"cache_control": {"type": "ephemeral"},
}
]
def _transform(self, messages, tools):
return self.config.transform_request(
model="gpt-4o",
messages=messages,
optional_params={"tools": tools},
litellm_params={"custom_llm_provider": "openai", "api_base": None},
headers={},
)
def test_caller_messages_and_tools_keep_cache_control(self):
messages, tools = self._messages(), self._tools()
messages_before, tools_before = copy.deepcopy(messages), copy.deepcopy(tools)
request = self._transform(messages, tools)
assert "cache_control" not in json.dumps(request)
assert messages == messages_before
assert tools == tools_before
def test_a_later_anthropic_call_still_sees_cache_control(self):
"""The user-visible consequence: the same message list reused on a provider
that does support caching must still carry the cache breakpoints."""
messages = self._messages()
self._transform(messages, self._tools())
anthropic_body = litellm.AnthropicConfig().transform_request(
model="claude-3-5-sonnet-20240620",
messages=messages,
optional_params={},
litellm_params={},
headers={},
)
assert "cache_control" in json.dumps(anthropic_body)
def _developer_message_after_a_user_turn() -> list[dict[str, str]]:
return [
{"role": "system", "content": "You are terse."},
@ -1916,3 +2004,53 @@ def test_completion_leaves_the_callers_closing_developer_message_unchanged_on_a_
)
assert json.loads(chat.calls.last.request.content)["messages"][-1]["role"] == "user"
assert messages == sent
def _user_turn_with_litellm_only_media_fields() -> list[dict[str, object]]:
return [
{
"role": "user",
"content": [
{"type": "text", "text": "Describe these"},
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png", "format": "image/png"}},
{"type": "image_url", "image_url": "https://example.com/dog.png"},
{
"type": "file",
"file": {"file_data": "data:application/pdf;base64,JVBERi0=", "format": "application/pdf"},
},
],
}
]
@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"])
async def test_openai_request_rewrites_media_parts_without_touching_the_callers_message(
is_async: bool, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
monkeypatch.setattr(litellm, "api_base", None, raising=False)
config: Final = OpenAIGPTConfig()
messages: Final = _user_turn_with_litellm_only_media_fields()
messages_before: Final = copy.deepcopy(messages)
request_kwargs: Final = {
"model": "gpt-5.5",
"messages": messages,
"optional_params": {},
"litellm_params": {"custom_llm_provider": "openai", "api_base": None},
"headers": {},
}
body: Final = (
await config.async_transform_request(**request_kwargs)
if is_async
else config.transform_request(**request_kwargs)
)
assert body["messages"][0]["content"] == [
{"type": "text", "text": "Describe these"},
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}},
{"type": "image_url", "image_url": {"url": "https://example.com/dog.png"}},
{"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0=", "filename": "my_file.pdf"}},
]
assert messages == messages_before

View file

@ -1,3 +1,4 @@
import copy
import json
import logging
from collections.abc import Callable, Mapping
@ -541,6 +542,50 @@ class TestOpenAIResponsesAPIConfig:
assert "cache_control" not in result["tools"][0]
assert result["tools"][0]["name"] == "get_weather"
def test_transform_does_not_strip_cache_control_from_the_callers_input(self):
"""Stripping for OpenAI must not reach back into the caller's own objects.
`filter_value_from_dict` deletes the key in place and recurses, and
`_validate_input_param` passes plain dict items through by reference, so
the caller's input list used to lose its cache breakpoints. Reusing that
list on Anthropic or Bedrock afterwards then silently lost prompt caching.
Same defect as the Chat Completions path.
"""
input_with_cache_control = [
{
"role": "system",
"content": [
{
"type": "input_text",
"text": "a long cached system prompt",
"cache_control": {"type": "ephemeral"},
}
],
}
]
tools_with_cache_control = [
{
"type": "function",
"name": "get_weather",
"parameters": {"type": "object"},
"cache_control": {"type": "ephemeral"},
}
]
input_before = copy.deepcopy(input_with_cache_control)
tools_before = copy.deepcopy(tools_with_cache_control)
result = self.config.transform_responses_api_request(
model=self.model,
input=input_with_cache_control,
response_api_optional_request_params={"tools": tools_with_cache_control},
litellm_params={},
headers={},
)
assert "cache_control" not in json.dumps(result)
assert input_with_cache_control == input_before
assert tools_with_cache_control == tools_before
def test_transform_preserves_input_without_cache_control(self):
"""Inputs without cache_control must pass through unmodified."""
input_clean = [
@ -4607,10 +4652,10 @@ def test_cache_control_still_marks_a_part_whose_own_marker_is_malformed_under_dr
}
def _caller_input_after_transform(model: str, marker: object) -> str:
def _hinted_caller_input(marker: object) -> list[dict[str, object]]:
hint: Final = {"type": "ephemeral"}
marked: Final = {} if marker is None else {"prompt_cache_breakpoint": marker}
input: Final = [
return [
{
"type": "message",
"role": "user",
@ -4626,6 +4671,10 @@ def _caller_input_after_transform(model: str, marker: object) -> str:
"content": [{"type": "output_text", "text": "ok", "cache_control": hint, **marked}],
},
]
def _caller_input_after_transform(model: str, marker: object) -> str:
input: Final = _hinted_caller_input(marker)
_CONFIGS[0].transform_responses_api_request(
model=model,
input=cast("ResponseInputParam", input), # cast-ok: plain dicts shaped like the SDK's input items
@ -4647,6 +4696,15 @@ def test_a_marker_leaves_the_caller_request_that_fallbacks_and_logs_read_as_an_u
assert _caller_input_after_transform(model, marker).replace(marker_json, "") == unmarked
@pytest.mark.parametrize("model", ("gpt-4.1", "gpt-6.1-sol"))
@pytest.mark.parametrize("marker", (None, _VALID_MARKERS[0], "yes"), ids=("unmarked", "explicit", "malformed"))
def test_the_caller_input_keeps_its_cache_control_hints_for_a_fallback_deployment(
monkeypatch: pytest.MonkeyPatch, model: str, marker: object
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
assert _caller_input_after_transform(model, marker) == json.dumps(_hinted_caller_input(marker))
@pytest.mark.asyncio
@pytest.mark.usefixtures(httpx_transport.__name__)
async def test_aresponses_forwards_previous_response_id_to_openai() -> None:

View file

@ -1,3 +1,4 @@
import copy
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
@ -198,3 +199,23 @@ def test_translate_developer_role_keeps_developer_messages_untouched() -> None:
)
== messages
)
def test_o_series_request_turns_system_into_user_without_touching_the_callers_messages(
monkeypatch: pytest.MonkeyPatch,
) -> None:
model: Final = "o-series-test-model-without-system-role"
monkeypatch.setitem(
litellm.model_cost,
model,
{"litellm_provider": "openai", "mode": "chat", "supports_system_messages": False},
)
messages: Final = [{"role": "system", "content": "Be terse"}, {"role": "user", "content": "Hi"}]
messages_before: Final = copy.deepcopy(messages)
body: Final = OpenAIOSeriesConfig().transform_request(
model=model, messages=messages, optional_params={}, litellm_params={}, headers={}
)
assert [message["role"] for message in body["messages"]] == ["user", "user"]
assert messages == messages_before

View file

@ -24,6 +24,7 @@ from litellm.llms.openai.openai import (
AsyncAssistantEventHandler,
AsyncCursorPage,
MessageData,
OpenAIBatchesAPI,
OpenAIChatCompletion,
OpenAIConfig,
OpenAIMessage as Message,
@ -2320,3 +2321,43 @@ def test_openai_config_hoists_a_later_developer_message_only_for_a_model_off_ope
model_info=None,
)
assert [message["role"] for message in messages] == expected_roles
@pytest.mark.asyncio
async def test_sync_batch_list_given_an_async_client_returns_a_page_the_caller_can_await(
http_backend: ModuleType,
):
def respond(request: _Request) -> httpx.Response | SDKResponse:
assert str(request.url).startswith("http://vllm-router.test/v1/batches")
return http_backend.Response(
200,
json={
"object": "list",
"data": [
{
"id": "batch_5f8dac",
"status": "pending",
"input_file_id": "file-b73f4c",
"created_at": 1793239207,
"endpoint": "/v1/chat/completions",
"completion_window": "24h",
}
],
"first_id": "batch_5f8dac",
"last_id": "batch_5f8dac",
"has_more": False,
},
)
async with http_backend.AsyncClient(transport=http_backend.MockTransport(respond)) as http_client:
page: Final = OpenAIBatchesAPI().list_batches(
_is_async=False,
api_key="sk-test",
api_base="http://vllm-router.test/v1",
timeout=5.0,
max_retries=0,
organization=None,
client=AsyncOpenAI(api_key="sk-test", base_url="http://vllm-router.test/v1", http_client=http_client),
)
assert [(batch.id, batch.status) for batch in (await page).data] == [("batch_5f8dac", "pending")]

View file

@ -0,0 +1,706 @@
import copy
import json
from typing import Final
import pytest
import respx
import litellm
from litellm.caching.llm_caching_handler import LLMClientCache
def test_hostinger_router_provider_resolution(monkeypatch: pytest.MonkeyPatch):
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
monkeypatch.setenv("HOSTINGER_ROUTER_API_KEY", "hostinger-test-key")
model, provider, api_key, api_base = get_llm_provider(
model="hostinger_router/claude-sonnet-5",
custom_llm_provider=None,
api_base=None,
api_key=None,
)
assert model == "claude-sonnet-5"
assert provider == "hostinger_router"
assert api_key == "hostinger-test-key"
assert api_base == "https://router.hostinger.com/v1"
def test_hostinger_router_provider_keeps_explicit_credentials(monkeypatch: pytest.MonkeyPatch):
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
monkeypatch.setenv("HOSTINGER_ROUTER_API_KEY", "hostinger-env-key")
_, provider, api_key, api_base = get_llm_provider(
model="hostinger_router/claude-sonnet-5",
custom_llm_provider=None,
api_base="https://router.internal.example/v1",
api_key="hostinger-explicit-key",
)
assert provider == "hostinger_router"
assert api_key == "hostinger-explicit-key"
assert api_base == "https://router.internal.example/v1"
@pytest.mark.parametrize("model", ["gpt-5-nano", "claude-sonnet-5"])
def test_hostinger_router_api_base_keeps_unprefixed_model_provider(model: str):
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
_, provider_without_api_base, _, _ = get_llm_provider(model=model)
_, provider_with_hostinger_api_base, _, _ = get_llm_provider(
model=model,
api_base="https://router.hostinger.com/v1",
api_key="hostinger-test-key",
)
assert provider_with_hostinger_api_base == provider_without_api_base
HOSTINGER_ROUTER_MODELS = tuple(sorted(name for name in litellm.model_cost if name.startswith("hostinger_router/")))
HOSTINGER_ROUTER_CHAT_MODELS = tuple(
name for name in HOSTINGER_ROUTER_MODELS if litellm.get_model_info(name)["mode"] == "chat"
)
@pytest.mark.parametrize("model", HOSTINGER_ROUTER_CHAT_MODELS)
def test_hostinger_router_model_cost_and_capabilities(model: str):
from litellm.cost_calculator import cost_per_token
prompt_cost, completion_cost = cost_per_token(
model=model,
prompt_tokens=1_000_000,
completion_tokens=1_000_000,
custom_llm_provider="hostinger_router",
)
model_info = litellm.get_model_info(model)
assert prompt_cost == pytest.approx(model_info["input_cost_per_token"] * 1_000_000)
assert completion_cost == pytest.approx(model_info["output_cost_per_token"] * 1_000_000)
assert model_info["output_cost_per_token"] > 0
assert model_info["max_tokens"] == model_info["max_output_tokens"] <= model_info["max_input_tokens"]
assert model_info["litellm_provider"] == "hostinger_router"
assert model_info["mode"] == "chat"
if model_info.get("supports_prompt_caching"):
assert 0 < model_info["cache_read_input_token_cost"] < model_info["input_cost_per_token"]
capability_checks: Final = (
("supports_function_calling", litellm.supports_function_calling),
("supports_reasoning", litellm.supports_reasoning),
("supports_response_schema", litellm.supports_response_schema),
("supports_vision", litellm.supports_vision),
)
assert {name: check(model) for name, check in capability_checks} == {
name: model_info.get(name) is True for name, _ in capability_checks
}
def test_hostinger_router_chat_completions_request():
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond(
200,
json={
"id": "chatcmpl_hostinger",
"object": "chat.completion",
"created": 1_789_550_000,
"model": "claude-sonnet-5",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "Hello from Hostinger Router"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7},
},
)
response: Final = litellm.completion(
model="hostinger_router/claude-sonnet-5",
messages=[{"role": "user", "content": "Say hello"}],
api_key="hostinger-test-key",
)
request: Final = route.calls.last.request
body: Final = json.loads(request.content)
assert route.call_count == 1
assert request.headers["authorization"] == "Bearer hostinger-test-key"
assert body["model"] == "claude-sonnet-5"
assert response.choices[0].message.content == "Hello from Hostinger Router"
def test_hostinger_router_chat_completions_streaming_request():
chunks = [
'data: {"id":"chatcmpl_hostinger","object":"chat.completion.chunk","created":1789550000,"model":"claude-sonnet-5","choices":[{"index":0,"delta":{"role":"assistant","content":"Hi"},"finish_reason":null}]}\n\n',
'data: {"id":"chatcmpl_hostinger","object":"chat.completion.chunk","created":1789550000,"model":"claude-sonnet-5","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":4,"completion_tokens":1,"total_tokens":5}}\n\n',
"data: [DONE]\n\n",
]
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond(
200,
headers={"content-type": "text/event-stream"},
content="".join(chunks).encode(),
)
response: Final = litellm.completion(
model="hostinger_router/claude-sonnet-5",
messages=[{"role": "user", "content": "Say hello"}],
stream=True,
api_key="hostinger-test-key",
)
collected = [chunk for chunk in response]
request: Final = route.calls.last.request
body: Final = json.loads(request.content)
assert route.call_count == 1
assert body["stream"] is True
assert any(chunk.choices and chunk.choices[0].delta.content == "Hi" for chunk in collected)
def test_hostinger_router_responses_request():
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/responses").respond(
200,
json={
"id": "resp_hostinger",
"object": "response",
"created_at": 1_789_550_000,
"model": "gpt-5.2",
"status": "completed",
"output": [
{
"id": "msg_hostinger",
"type": "message",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "Hello from Hostinger Router", "annotations": []}],
}
],
"usage": {"input_tokens": 4, "output_tokens": 3, "total_tokens": 7},
},
)
response: Final = litellm.responses(
model="hostinger_router/gpt-5.2",
input="Say hello",
api_key="hostinger-test-key",
)
request: Final = route.calls.last.request
body: Final = json.loads(request.content)
assert route.call_count == 1
assert str(request.url) == "https://router.hostinger.com/v1/responses"
assert request.headers["authorization"] == "Bearer hostinger-test-key"
assert body["model"] == "gpt-5.2"
assert body["input"] == "Say hello"
assert response.output[0].content[0].text == "Hello from Hostinger Router"
EMBEDDING_RESPONSE: Final = {
"object": "list",
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}],
"model": "text-embedding-3-small",
"usage": {"prompt_tokens": 2, "total_tokens": 2},
}
def test_hostinger_router_embeddings_request():
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
response: Final = litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
)
request: Final = route.calls.last.request
body: Final = json.loads(request.content)
assert route.call_count == 1
assert request.headers["authorization"] == "Bearer hostinger-test-key"
assert body["model"] == "text-embedding-3-small"
assert response.data[0]["embedding"] == [0.1, 0.2, 0.3]
def test_hostinger_router_embeddings_send_extra_headers_as_http_headers():
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
extra_headers={"X-Trace-Id": "trace-123"},
)
request: Final = route.calls.last.request
body: Final = json.loads(request.content)
assert request.headers["x-trace-id"] == "trace-123"
assert request.headers["authorization"] == "Bearer hostinger-test-key"
assert request.headers["content-type"] == "application/json"
assert set(body) == {"model", "input"}
def test_hostinger_router_embeddings_extra_headers_replace_key_headers_of_any_case():
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
extra_headers={"authorization": "Bearer caller-key", "x-trace-id": "trace-1"},
)
request: Final = route.calls.last.request
assert request.headers.get_list("authorization") == ["Bearer caller-key"]
assert request.headers.get_list("content-type") == ["application/json"]
assert request.headers.get_list("x-trace-id") == ["trace-1"]
@pytest.mark.parametrize(("configured", "sent"), [(12345, "12345"), (1.5, "1.5")])
def test_hostinger_router_embeddings_send_numeric_extra_headers_as_text(configured: int | float, sent: str):
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
extra_headers={"X-Org-Id": configured},
)
assert route.calls.last.request.headers.get_list("x-org-id") == [sent]
@pytest.mark.parametrize("configured", [True, None, ["a", "b"]])
def test_hostinger_router_embeddings_refuse_non_text_extra_headers(configured: object):
with respx.mock(assert_all_called=False) as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
with pytest.raises(litellm.UnsupportedParamsError, match="X-Debug") as refused:
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
extra_headers={"X-Debug": configured},
)
assert refused.value.status_code == 400
assert route.call_count == 0
@pytest.mark.parametrize("configured", [True, None, ["a", "b"]])
def test_hostinger_router_embeddings_drop_non_text_extra_headers_under_drop_params(configured: object):
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
extra_headers={"X-Debug": configured, "X-Trace-Id": "trace-1"},
drop_params=True,
)
request: Final = route.calls.last.request
assert "x-debug" not in request.headers
assert request.headers.get_list("x-trace-id") == ["trace-1"]
assert set(json.loads(request.content)) == {"model", "input"}
TRANSPORT_EXTRA_HEADERS: Final = [
("Host", "other.example"),
("content-length", "1"),
("Transfer-Encoding", "chunked"),
("Connection", "close"),
("Keep-Alive", "timeout=5"),
("Proxy-Connection", "close"),
("TE", "gzip"),
("Trailer", "X-Checksum"),
("Upgrade", "websocket"),
("Expect", "100-continue"),
]
@pytest.mark.parametrize(("name", "configured"), TRANSPORT_EXTRA_HEADERS)
def test_hostinger_router_embeddings_ignore_transport_extra_headers(name: str, configured: str):
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
extra_headers={name: configured, "X-Trace-Id": "trace-1"},
)
request: Final = route.calls.last.request
assert request.headers.get_list("host") == ["router.hostinger.com"]
assert request.headers.get_list("content-length") == [str(len(request.content))]
assert "transfer-encoding" not in request.headers
assert configured not in request.headers.get_list(name)
assert request.headers.get_list("x-trace-id") == ["trace-1"]
MALFORMED_HEADER_NAMES: Final = ["Host:evil.example", "X Note", "X-Caf\u00e9"]
@pytest.mark.parametrize("name", MALFORMED_HEADER_NAMES)
def test_hostinger_router_embeddings_refuse_malformed_extra_header_names(name: str):
with respx.mock(assert_all_called=False) as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
with pytest.raises(litellm.UnsupportedParamsError) as refused:
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
extra_headers={name: "x"},
)
assert refused.value.status_code == 400
assert repr(name) in refused.value.message
assert route.call_count == 0
@pytest.mark.parametrize("name", MALFORMED_HEADER_NAMES)
def test_hostinger_router_embeddings_drop_malformed_extra_header_names_under_drop_params(name: str):
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
extra_headers={name: "x", "X-Trace-Id": "trace-1"},
drop_params=True,
)
request: Final = route.calls.last.request
assert request.headers.get_list("host") == ["router.hostinger.com"]
assert request.headers.get_list("x-trace-id") == ["trace-1"]
assert name.lower() not in request.headers.keys()
@pytest.mark.parametrize("configured", ["a\r\nHost: evil.example", "line\nbreak", "caf\u00e9", "a\x7fb"])
def test_hostinger_router_embeddings_drop_extra_header_values_http_cannot_carry(configured: str):
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
extra_headers={"x-litellm-user_api_key_alias": configured, "X-Trace-Id": "trace-1"},
)
request: Final = route.calls.last.request
assert request.headers.get_list("host") == ["router.hostinger.com"]
assert request.headers.get_list("x-trace-id") == ["trace-1"]
assert "x-litellm-user_api_key_alias" not in request.headers
@pytest.mark.parametrize(
("configured", "sent"),
[(" x ", "x"), ("\t", ""), (" a\tb ", "a\tb"), ("env-secret\n", "env-secret"), ("\r\n x \r\n", "x")],
)
def test_hostinger_router_embeddings_trim_whitespace_around_extra_header_values(configured: str, sent: str):
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
extra_headers={"X-Rig": configured},
)
assert route.calls.last.request.headers.get_list("x-rig") == [sent]
def test_hostinger_router_embeddings_extra_headers_stay_with_their_request():
router: Final = litellm.Router(
model_list=[
{
"model_name": "shared",
"litellm_params": {
"model": "hostinger_router/text-embedding-3-small",
"api_key": "hostinger-test-key",
"headers": {"X-Org-Id": "org-1"},
},
}
]
)
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
router.embedding(
model="shared", input=["first"], extra_headers={"Authorization": "Bearer caller-a", "X-Leak": "a"}
)
router.embedding(model="shared", input=["second"])
first: Final = route.calls[0].request
second: Final = route.calls[1].request
assert first.headers.get_list("x-leak") == ["a"]
assert second.headers.get_list("authorization") == ["Bearer hostinger-test-key"]
assert "x-leak" not in second.headers
assert second.headers.get_list("x-org-id") == ["org-1"]
@pytest.mark.parametrize("param", ["headers", "extra_headers"])
def test_hostinger_router_embeddings_send_headers_given_as_name_value_pairs(param: str):
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
**{param: [["X-Trace-Id", "trace-1"]]},
)
request: Final = route.calls.last.request
assert request.headers.get_list("x-trace-id") == ["trace-1"]
assert request.headers.get_list("authorization") == ["Bearer hostinger-test-key"]
@pytest.mark.parametrize("param", ["headers", "extra_headers"])
def test_hostinger_router_embeddings_refuse_headers_that_are_not_an_object(param: str):
with respx.mock(assert_all_called=False) as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
with pytest.raises(litellm.UnsupportedParamsError, match=rf"\b{param} must be a JSON object") as refused:
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
**{param: "X-Trace-Id: trace-1"},
)
assert refused.value.status_code == 400
assert route.call_count == 0
@pytest.mark.parametrize("param", ["headers", "extra_headers"])
def test_hostinger_router_embeddings_drop_headers_that_are_not_an_object_under_drop_params(param: str):
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
drop_params=True,
**{param: "X-Trace-Id: trace-1"},
)
request: Final = route.calls.last.request
assert "x-trace-id" not in request.headers
assert request.headers.get_list("authorization") == ["Bearer hostinger-test-key"]
assert set(json.loads(request.content)) == {"model", "input"}
def test_hostinger_router_embeddings_never_borrow_another_providers_key(monkeypatch: pytest.MonkeyPatch):
monkeypatch.delenv("HOSTINGER_ROUTER_API_KEY", raising=False)
monkeypatch.setenv("OPENAI_LIKE_API_KEY", "other-provider-key")
monkeypatch.setenv("OPENAI_API_KEY", "other-provider-key")
with respx.mock(assert_all_called=False) as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
with pytest.raises(litellm.BadRequestError, match="Missing API Key"):
litellm.embedding(model="hostinger_router/text-embedding-3-small", input=["Say hello"])
assert route.call_count == 0
def test_json_provider_without_embeddings_endpoint_refuses_embeddings():
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
slug: Final = next(
slug
for slug in JSONProviderRegistry.list_providers()
if not JSONProviderRegistry.supports_embeddings(slug) and slug not in litellm.provider_list
)
with respx.mock(assert_all_called=False) as upstream:
route: Final = upstream.route().respond(200, json=EMBEDDING_RESPONSE)
with pytest.raises(litellm.BadRequestError):
litellm.embedding(model=f"{slug}/text-embedding-3-small", input=["Say hello"], api_key="test-key")
assert route.call_count == 0
@pytest.mark.asyncio
async def test_hostinger_router_anthropic_messages_request(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/messages").respond(
200,
json={
"id": "msg_hostinger",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5",
"content": [{"type": "text", "text": "Hello from Hostinger Router"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 4, "output_tokens": 3},
},
)
response: Final = await litellm.anthropic.messages.acreate(
model="hostinger_router/claude-sonnet-5",
messages=[{"role": "user", "content": "Say hello"}],
max_tokens=32,
api_key="hostinger-test-key",
)
request: Final = route.calls.last.request
body: Final = json.loads(request.content)
assert route.call_count == 1
assert str(request.url) == "https://router.hostinger.com/v1/messages"
assert request.headers["authorization"] == "Bearer hostinger-test-key"
assert request.headers["anthropic-version"] == "2023-06-01"
assert body["model"] == "claude-sonnet-5"
assert body["messages"] == [{"role": "user", "content": "Say hello"}]
assert response["content"][0]["text"] == "Hello from Hostinger Router"
CHAT_COMPLETION_RESPONSE: Final = {
"id": "chatcmpl_hostinger",
"object": "chat.completion",
"created": 1_789_550_000,
"model": "gpt-5.2",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "Hello from Hostinger Router"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7},
}
UNSUPPORTED_ENDPOINT: Final = {"error": {"message": "Model does not support this endpoint."}}
@pytest.mark.asyncio
@pytest.mark.parametrize("model", ["gpt-5.2", "gpt-5-nano"])
async def test_hostinger_router_messages_bridge_models_without_native_messages(
model: str, monkeypatch: pytest.MonkeyPatch
):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
with respx.mock(assert_all_called=False) as upstream:
native: Final = upstream.post("https://router.hostinger.com/v1/messages").respond(
403, json=UNSUPPORTED_ENDPOINT
)
chat: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond(
200, json=CHAT_COMPLETION_RESPONSE
)
response: Final = await litellm.anthropic.messages.acreate(
model=f"hostinger_router/{model}",
messages=[{"role": "user", "content": "Say hello"}],
max_tokens=32,
api_key="hostinger-test-key",
)
assert native.call_count == 0
assert chat.call_count == 1
assert json.loads(chat.calls.last.request.content)["model"] == model
assert response["content"][0]["text"] == "Hello from Hostinger Router"
def test_hostinger_router_responses_bridge_models_without_native_responses():
with respx.mock(assert_all_called=False) as upstream:
native: Final = upstream.post("https://router.hostinger.com/v1/responses").respond(
403, json=UNSUPPORTED_ENDPOINT
)
chat: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond(
200, json=CHAT_COMPLETION_RESPONSE
)
response: Final = litellm.responses(
model="hostinger_router/claude-sonnet-5",
input="Say hello",
api_key="hostinger-test-key",
)
assert native.call_count == 0
assert chat.call_count == 1
assert json.loads(chat.calls.last.request.content)["model"] == "claude-sonnet-5"
assert response.output[0].content[0].text == "Hello from Hostinger Router"
ANTHROPIC_MESSAGE_RESPONSE: Final = {
"id": "msg_hostinger",
"type": "message",
"role": "assistant",
"model": "claude-haiku-4-5",
"content": [{"type": "text", "text": "Hello from Hostinger Router"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 4, "output_tokens": 3},
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"sibling_model_info",
[
{"supported_endpoints": ["/v1/chat/completions", "/v1/messages", "/v1/responses"]},
{"supported_endpoints": "/v1/messages"},
],
)
async def test_hostinger_router_endpoint_opt_in_stays_with_its_own_deployment(
sibling_model_info: dict[str, object], monkeypatch: pytest.MonkeyPatch
):
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
deployment_params: Final = {"model": "hostinger_router/gpt-5-nano", "api_key": "hostinger-test-key"}
router: Final = litellm.Router(
model_list=[
{"model_name": "opted-in", "litellm_params": deployment_params, "model_info": sibling_model_info},
{"model_name": "plain", "litellm_params": deployment_params},
]
)
with respx.mock(assert_all_called=False) as upstream:
native_messages: Final = upstream.post("https://router.hostinger.com/v1/messages").respond(
403, json=UNSUPPORTED_ENDPOINT
)
native_responses: Final = upstream.post("https://router.hostinger.com/v1/responses").respond(
403, json=UNSUPPORTED_ENDPOINT
)
chat: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond(
200, json=CHAT_COMPLETION_RESPONSE
)
messages_response: Final = await router.aanthropic_messages(
model="plain", messages=[{"role": "user", "content": "Say hello"}], max_tokens=32
)
responses_response: Final = await router.aresponses(model="plain", input="Say hello")
assert native_messages.call_count == 0
assert native_responses.call_count == 0
assert chat.call_count == 2
assert messages_response["content"][0]["text"] == "Hello from Hostinger Router"
assert responses_response.output[0].content[0].text == "Hello from Hostinger Router"
@pytest.mark.asyncio
async def test_hostinger_router_deployment_cache_ttl_opt_in_keeps_one_hour_ttl(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
router: Final = litellm.Router(
model_list=[
{
"model_name": "opted-in",
"litellm_params": {
"model": "hostinger_router/claude-haiku-4-5",
"api_base": "https://router.hostinger.com/v1",
"api_key": "hostinger-test-key",
},
"model_info": {"supported_endpoints": ["/v1/messages"], "cache_control_ttl": True},
}
]
)
one_hour_cache: Final = {"type": "ephemeral", "ttl": "1h"}
with respx.mock() as upstream:
native: Final = upstream.post("https://router.hostinger.com/v1/messages").respond(
200, json=ANTHROPIC_MESSAGE_RESPONSE
)
await router.aanthropic_messages(
model="opted-in",
messages=[
{"role": "user", "content": [{"type": "text", "text": "Say hello", "cache_control": one_hour_cache}]}
],
max_tokens=32,
)
body: Final = json.loads(native.calls.last.request.content)
assert body["messages"][0]["content"][0]["cache_control"] == one_hour_cache

View file

@ -1,5 +1,8 @@
import copy
import json
from collections.abc import Iterator
from itertools import chain
from typing import Final
from unittest.mock import MagicMock
@ -874,6 +877,58 @@ def test_openrouter_streaming_reasoning_is_preserved() -> None:
assert assembled.choices[0].message.reasoning_content == "Consider the evidence. Apply the rule."
def _five_message_level_cache_hints() -> list[dict[str, object]]:
return [
{"role": "system", "content": "system prompt", "cache_control": {"type": "ephemeral"}},
{"role": "user", "content": "turn 1", "cache_control": {"type": "ephemeral"}},
{"role": "assistant", "content": "answer 1", "cache_control": {"type": "ephemeral"}},
{"role": "user", "content": "turn 2", "cache_control": {"type": "ephemeral"}},
{"role": "assistant", "content": "answer 2"},
{"role": "user", "content": "turn 3", "cache_control": {"type": "ephemeral"}},
]
def _cached_texts(node: object) -> Iterator[str]:
if isinstance(node, dict):
yield from ((node["text"],) if "cache_control" in node and "text" in node else ())
yield from chain.from_iterable(map(_cached_texts, node.values()))
elif isinstance(node, list):
yield from chain.from_iterable(map(_cached_texts, node))
@pytest.mark.parametrize(
("drop_params", "expected_cached_texts"),
[
(True, ["answer 1", "turn 1", "turn 2", "turn 3"]),
("true", ["answer 1", "turn 1", "turn 2", "turn 3"]),
(False, ["answer 1", "system prompt", "turn 1", "turn 2", "turn 3"]),
],
ids=[
"drop_params_keeps_the_newest_four",
"string_drop_params_keeps_the_newest_four",
"without_drop_params_all_five_pass_through",
],
)
def test_openrouter_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params(
drop_params: bool | str, expected_cached_texts: list[str], monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
messages: Final = _five_message_level_cache_hints()
messages_before: Final = copy.deepcopy(messages)
body: Final = OpenrouterConfig().transform_request(
model="anthropic/claude-haiku-5.5",
messages=messages,
optional_params={},
litellm_params={"drop_params": drop_params, "custom_llm_provider": "openrouter"},
headers={},
)
assert sorted(_cached_texts(body["messages"])) == expected_cached_texts
assert json.dumps(body).count('"cache_control"') == len(expected_cached_texts)
assert messages == messages_before
def _transform_openrouter_response(body: dict[str, object]) -> litellm.ModelResponse:
return OpenrouterConfig().transform_response(
model="openrouter/openai/gpt-4o-mini",

View file

@ -1,3 +1,4 @@
import json
from typing import Final, Literal
from unittest.mock import AsyncMock, MagicMock, patch
@ -2507,3 +2508,46 @@ def test_check_and_create_cache_clamps_negative_creation_tokens() -> None:
assert cache_name == "cachedContents/created"
assert logging_obj.vertex_ai_cache_creation_input_tokens == 0
def _recording_gemini_client(calls: list[httpx.Request]) -> HTTPHandler:
def route(request: httpx.Request) -> httpx.Response:
calls.append(request)
if request.url.path.endswith("/cachedContents"):
return httpx.Response(200, json={} if request.method == "GET" else _created_cache_body(3000))
return httpx.Response(
200,
json={
"candidates": [{"content": {"role": "model", "parts": [{"text": "OK"}]}, "finishReason": "STOP"}],
"usageMetadata": {"promptTokenCount": 3000, "candidatesTokenCount": 1, "totalTokenCount": 3001},
},
)
return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(route)))
def test_a_plain_system_prompt_goes_into_the_gemini_cache_instead_of_beside_it(local_model_cost_map) -> None:
calls: Final[list[httpx.Request]] = []
litellm.completion(
model="gemini/gemini-flash-latest",
messages=[
{"role": "system", "content": "You are terse."},
{
"role": "user",
"content": [
{"type": "text", "text": "plain system prefix " * 3000, "cache_control": {"type": "ephemeral"}},
{"type": "text", "text": "Reply with OK only."},
],
},
],
api_key="test-key",
client=_recording_gemini_client(calls),
)
created: Final = next(
json.loads(call.content) for call in calls if call.method == "POST" and call.url.path.endswith("/cachedContents")
)
generated: Final = next(json.loads(call.content) for call in calls if call.url.path.endswith(":generateContent"))
assert created["system_instruction"] == {"parts": [{"text": "You are terse."}]}
assert generated["cachedContent"] == "cachedContents/created"
assert "system_instruction" not in generated

View file

@ -1,4 +1,5 @@
import asyncio
import copy
import importlib
import json
from typing import Final, Optional, cast
@ -10,6 +11,7 @@ import pytest
import litellm
from litellm import completion, embedding
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.watsonx.chat.transformation import IBMWatsonXChatConfig
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
@ -542,3 +544,43 @@ def test_watsonx_deployment_space_id_embedding(monkeypatch, watsonx_embedding_ca
# nor space_id or project_id is required by wx.ai API when inferencing deployment
assert "project_id" not in json_data and "space_id" not in json_data
def _strict_weather_tool() -> dict[str, object]:
return {
"type": "function",
"function": {
"name": "get_weather",
"strict": True,
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
"additionalProperties": False,
},
},
}
def test_watsonx_chat_tools_are_rewritten_without_touching_the_callers_tool() -> None:
tools: Final = [_strict_weather_tool()]
tools_before: Final = copy.deepcopy(tools)
optional_params: Final = IBMWatsonXChatConfig().map_openai_params(
non_default_params={"tools": tools}, optional_params={}, model="ibm/granite-4-h-small", drop_params=False
)
assert optional_params["tools"] == [
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
}
]
assert tools == tools_before

View file

@ -1,3 +1,4 @@
import copy
import json
import os
from typing import Final
@ -581,3 +582,44 @@ def test_xai_grok_4_frequency_penalty_not_supported(model):
"""
supported_params = XAIChatConfig().get_supported_openai_params(model=model)
assert "frequency_penalty" not in supported_params
def _strict_weather_tool() -> dict[str, object]:
return {
"type": "function",
"function": {
"name": "get_weather",
"strict": True,
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
"additionalProperties": False,
},
},
}
def test_xai_tools_are_rewritten_without_touching_the_callers_tool() -> None:
tools: Final = [_strict_weather_tool()]
tools_before: Final = copy.deepcopy(tools)
optional_params: Final = XAIChatConfig().map_openai_params(
non_default_params={"tools": tools}, optional_params={}, model="grok-4.3", drop_params=False
)
assert optional_params["tools"] == [
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
"additionalProperties": False,
},
},
}
]
assert tools == tools_before

View file

@ -6,7 +6,7 @@ from typing import Final, Literal, NoReturn
import pytest
from litellm.accounting import InboundUsageEvent, Metric, MetricName, Outcome
from litellm.proxy.db.accounting_ledger import AccountingLedger
from litellm.proxy.db.accounting_ledger import AccountingConflictError, AccountingLedger, UsageWrite
from litellm.types.utils import CallTypes
@ -101,3 +101,18 @@ async def test_unknown_delivery_target_is_rejected_before_database_access(event:
("unknown",), # pyright: ignore[reportArgumentType] # Exercise an unknown runtime target
envelope=envelope,
)
@pytest.mark.asyncio
async def test_empty_usage_batches_do_not_open_a_transaction() -> None:
ledger: Final = AccountingLedger(_no_transaction)
await ledger.insert_usage_batch(())
assert await ledger.load_usage_batch(()) == ()
@pytest.mark.asyncio
async def test_conflicting_usage_within_one_batch_is_rejected_before_database_access(event: InboundUsageEvent) -> None:
original: Final = UsageWrite(event, json.dumps({"event": event.model_dump(mode="json")}))
changed: Final = UsageWrite(event, json.dumps({"event": event.model_dump(mode="json"), "pricing": {"rate": 2}}))
with pytest.raises(AccountingConflictError, match="conflicting copies"):
await AccountingLedger(_no_transaction).insert_usage_batch((original, changed))

View file

@ -951,3 +951,33 @@ def test_is_lock_timeout_error_recognises_a_statement_cancelled_under_lock_timeo
)
def test_is_lock_timeout_error_excludes_other_failures(error: Exception) -> None:
assert PrismaDBExceptionHandler.is_lock_timeout_error(error) is False
@pytest.mark.parametrize(
("body", "retryable"),
[
('{"error_code":"P1017","message":"Server has closed the connection."}', True),
('{"error_code":"P1000","message":"Authentication failed."}', False),
('{"error_code":"P2021","message":"Table does not exist."}', False),
('{"error_code":"P9999","message":"Unknown engine failure."}', False),
("invalid engine response", False),
],
)
def test_engine_connection_closure_is_recoverable_without_allowing_unverified_auth(
body: str,
retryable: bool,
) -> None:
from prisma.http import Response
error: Final = EngineRequestError(Response(httpx.Response(500)), body)
assert PrismaDBExceptionHandler.is_database_transport_error(error) is retryable
assert PrismaDBExceptionHandler.is_permanent_database_fault(error) is not retryable
assert PrismaDBExceptionHandler.is_database_connection_error(error) is False
def test_connection_closure_code_survives_prisma_data_error_wrapping() -> None:
error: Final = DataError(
data={"user_facing_error": {"error_code": "P1017", "message": "Server has closed the connection."}}
)
assert PrismaDBExceptionHandler.is_database_transport_error(error) is True
assert PrismaDBExceptionHandler.is_permanent_database_fault(error) is False

View file

@ -21,6 +21,7 @@ from litellm.proxy._types import (
LiteLLM_ObjectPermissionTable,
LiteLLM_TeamTable,
LiteLLM_UserTable,
LitellmUserRoles,
ProxyException,
SpecialHeaders,
SpecialMCPServerNames,
@ -13196,3 +13197,58 @@ async def test_personal_key_mcp_access_is_denied_when_owner_teams_cannot_be_read
assert await MCPRequestHandler.get_allowed_mcp_servers(personal_key_owner_teams) == []
assert await MCPRequestHandler.get_allowed_tools_for_server("srv-a", personal_key_owner_teams) == []
assert await MCPRequestHandler.get_allowed_tools_for_server("srv-open", personal_key_owner_teams) == []
@pytest.mark.asyncio
@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True)
@pytest.mark.parametrize(
"owner_role",
[LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY],
ids=["proxy_admin", "proxy_admin_viewer"],
)
@pytest.mark.parametrize(
("key_team_id", "expected_servers"),
[("team-a", {"srv-a", "srv-open"}), ("team-without-mcp", {"srv-open"})],
ids=["team_grants_one_server", "team_grants_no_server"],
)
@pytest.mark.parametrize(
"credential_shape",
[{"is_session_token": False}, {"is_session_token": True}, {"api_key": None, "jwt_claims": {"sub": "admin"}}],
ids=["virtual_key", "cli_session", "jwt"],
)
async def test_admin_owned_team_key_reaches_only_its_team_mcp_servers(
monkeypatch: pytest.MonkeyPatch,
personal_key_owner_teams: UserAPIKeyAuth,
owner_role: LitellmUserRoles,
key_team_id: str,
expected_servers: set[str],
credential_shape: dict[str, object],
) -> None:
from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager
from litellm.proxy.auth import auth_checks
async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable:
return _personal_key_team(team_id) if team_id in _PERSONAL_KEY_TEAMS else LiteLLM_TeamTable(team_id=team_id)
monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object)
team_key: Final = personal_key_owner_teams.model_copy(
update={"team_id": key_team_id, "user_role": owner_role, **credential_shape}
)
assert set(await global_mcp_server_manager.get_allowed_mcp_servers(team_key)) == expected_servers
@pytest.mark.asyncio
@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True)
async def test_admin_dashboard_session_keeps_the_whole_mcp_registry(
personal_key_owner_teams: UserAPIKeyAuth,
) -> None:
from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager
dashboard_session: Final = personal_key_owner_teams.model_copy(
update={"team_id": UI_TEAM_ID, "user_role": LitellmUserRoles.PROXY_ADMIN}
)
assert set(await global_mcp_server_manager.get_allowed_mcp_servers(dashboard_session)) == set(
global_mcp_server_manager.get_registry()
)

View file

@ -2,7 +2,7 @@
import asyncio
from collections.abc import Awaitable, Callable
from typing import Dict, List, Optional
from typing import Dict, Final, List, Optional
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@ -112,6 +112,23 @@ class TestApplyToolsetScope:
assert op.mcp_servers == ["server-a"]
assert op.mcp_tool_permissions == toolset_perms
@pytest.mark.asyncio
@pytest.mark.parametrize(
"owner_role",
[LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY],
ids=["proxy_admin", "proxy_admin_viewer"],
)
async def test_admin_owned_team_key_without_the_toolset_grant_is_denied(self, owner_role: LitellmUserRoles):
from litellm.proxy.mcp_server.server import apply_toolset_scope
team_key: Final = UserAPIKeyAuth(
api_key="sk-test", team_id="team-a", user_role=owner_role, object_permission=None
)
with pytest.raises(HTTPException) as exc_info:
await apply_toolset_scope(team_key, "toolset-123", granted=_granted_through_team("toolset-other"))
assert exc_info.value.status_code == 403
@pytest.mark.asyncio
async def test_team_granted_toolset_is_served_to_a_key_without_its_own_grant(self):
"""A team key whose own row carries no toolset grant is admitted to the toolset its team

Some files were not shown because too many files have changed in this diff Show more