diff --git a/litellm/constants.py b/litellm/constants.py index c17e776f54b..1a1e66ecf86 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -177,6 +177,8 @@ MID_STREAM_FAILURE_LOG_WAIT_SECONDS: Final = 1.0 # Set to 0 to disable truncation. MAX_BASE64_LENGTH_FOR_LOGGING: Final = get_env_int("MAX_BASE64_LENGTH_FOR_LOGGING", 64) BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS: Final = 256 * 1024 +BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS: Final = 256 +BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS: Final = 8 * 1024 * 1024 REDACTED_BY_LITELLM: Final = "redacted-by-litellm" # in-memory stand-in handed to provider converters for redacted arguments; never stored REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}" @@ -634,6 +636,8 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60 BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024) # Anthropic's Messages API rejects thinking.budget_tokens < 1024. ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024 +ANTHROPIC_MAX_USER_ID_LENGTH: Final = 512 +ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH: Final = 255 ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:" REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5) DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000) @@ -1163,6 +1167,7 @@ openai_compatible_providers: Final[list[str]] = [ "cortecs", "scx-ai", "prism", + "hostinger_router", "sail", "reka", "streamlake", diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index ae01aa8d0f7..871fa8227ea 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -3,10 +3,10 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass, field, replace from functools import partial from types import MappingProxyType -from typing import Final, Literal, TypeAlias +from typing import Annotated, Final, Literal, TypeAlias import httpx -from pydantic import BaseModel, ConfigDict, StrictBool, TypeAdapter, ValidationError +from pydantic import BaseModel, BeforeValidator, ConfigDict, TypeAdapter, ValidationError from pydantic_core import to_jsonable_python from typing_extensions import assert_never @@ -23,6 +23,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.decisions.transformation import ( BaseDecisionsConfig, DecisionsResponseValidationError, + catalog_mode, image_media_type, image_param, input_images, @@ -61,7 +62,7 @@ from litellm.types.decisions import ( UnsupportedDecisionsRequest, ) from litellm.types.llms.openai import ChatCompletionUserMessage -from litellm.types.utils import LlmProviders, ProviderSpecificHeader +from litellm.types.utils import LlmProviders, ProviderSpecificHeader, is_decisions_model_mode from litellm.utils import ProviderConfigManager, get_model_cost_key SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]] @@ -257,19 +258,105 @@ def _request_safety_identifier( return None -def _fields_without_unsupported_media( - fields: RequestFields, +def _bool_or_none(value: object) -> bool | None: + return value if isinstance(value, bool) else None + + +def _str_or_none(value: object) -> str | None: + return value if isinstance(value, str) else None + + +_DeploymentFlag: TypeAlias = Annotated[bool | None, BeforeValidator(_bool_or_none)] + + +class _DeploymentCapabilities(BaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + mode: Annotated[str | None, BeforeValidator(_str_or_none)] = None + supports_box_questions: _DeploymentFlag = None + supports_audio_input: _DeploymentFlag = None + supports_video_input: _DeploymentFlag = None + + def decisions_media_flag(self, flag: bool | None, model: str, custom_llm_provider: str) -> bool | None: + if flag is None: + return None + mode: Final = self.mode if self.mode is not None else catalog_mode(model, custom_llm_provider) + return flag if mode is None or is_decisions_model_mode(mode) else None + + +@dataclass(frozen=True, slots=True) +class _ResolvedSupport: + box_questions: bool + audio: bool + videos: bool + + +def _takes_audio( + provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities +) -> bool: + return provider_config.supports_audio_input( + model, + custom_llm_provider, + deployment.decisions_media_flag(deployment.supports_audio_input, model, custom_llm_provider), + ) + + +def _takes_videos( + provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities +) -> bool: + return provider_config.supports_video_input( + model, + custom_llm_provider, + deployment.decisions_media_flag(deployment.supports_video_input, model, custom_llm_provider), + ) + + +def deployment_refuses_media( + model: str, + custom_llm_provider: str, + model_info: Mapping[str, object], + call_kwargs: Mapping[str, object], + *, + audio: bool, + videos: bool, +) -> bool: + if _drops_params(call_kwargs): + return False + provider_config: Final = get_decisions_provider_config(model, custom_llm_provider) + canonical_model: Final = provider_config.canonical_model(model) + deployment: Final = _DeploymentCapabilities.model_validate(model_info) + return (audio and not _takes_audio(provider_config, canonical_model, custom_llm_provider, deployment)) or ( + videos and not _takes_videos(provider_config, canonical_model, custom_llm_provider, deployment) + ) + + +def _resolved_support( provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, + headers: Mapping[str, str], kwargs: Mapping[str, object], + fields: RequestFields, +) -> _ResolvedSupport: + deployment: Final = _DeploymentCapabilities.model_validate(kwargs.get("model_info") or {}) + return _ResolvedSupport( + box_questions=provider_config.supports_box_questions( + model, custom_llm_provider, headers, deployment.supports_box_questions + ), + audio=bool(fields.audio) and _takes_audio(provider_config, model, custom_llm_provider, deployment), + videos=bool(fields.videos) and _takes_videos(provider_config, model, custom_llm_provider, deployment), + ) + + +def _fields_without_unsupported_media( + fields: RequestFields, support: _ResolvedSupport, kwargs: Mapping[str, object] ) -> RequestFields: if not _drops_params(kwargs): return fields return replace( fields, - audio=fields.audio if provider_config.supports_audio_input(model, custom_llm_provider) else None, - videos=fields.videos if provider_config.supports_video_input(model, custom_llm_provider) else None, + audio=fields.audio if support.audio else None, + videos=fields.videos if support.videos else None, ) @@ -292,10 +379,8 @@ class _MalformedImages: def _unsupported_params( ir_request: DecisionsIRRequest, provider_config: BaseDecisionsConfig, - model: str, - custom_llm_provider: str, headers: Mapping[str, str], - supports_box_questions: bool, + support: _ResolvedSupport, ) -> tuple[str, ...]: refuses_safety_identifier: Final = ( ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier @@ -304,13 +389,9 @@ def _unsupported_params( None if provider_config.supports_images_in(ir_request.input, headers) else image_param(ir_request.input) ) refuses_assets: Final = bool(ir_request.assets) and not provider_config.supports_assets(headers) - refuses_box_questions: Final = _has_box_question(ir_request.questions) and not supports_box_questions - refuses_audio: Final = bool(ir_request.audio) and not provider_config.supports_audio_input( - model, custom_llm_provider - ) - refuses_videos: Final = bool(ir_request.videos) and not provider_config.supports_video_input( - model, custom_llm_provider - ) + refuses_box_questions: Final = _has_box_question(ir_request.questions) and not support.box_questions + refuses_audio: Final = bool(ir_request.audio) and not support.audio + refuses_videos: Final = bool(ir_request.videos) and not support.videos return ( *(("safety_identifier",) if refuses_safety_identifier else ()), *(() if refused_image_param is None else (refused_image_param,)), @@ -333,7 +414,7 @@ def _without_unsupported_params( ir_request: DecisionsIRRequest, provider_config: BaseDecisionsConfig, headers: Mapping[str, str], - supports_box_questions: bool, + support: _ResolvedSupport, ) -> DecisionsIRRequest: return replace( ir_request, @@ -345,9 +426,7 @@ def _without_unsupported_params( safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None, assets=ir_request.assets if provider_config.supports_assets(headers) else MappingProxyType({}), questions=( - ir_request.questions - if supports_box_questions - else tuple(filter(_is_not_box_question, ir_request.questions)) + ir_request.questions if support.box_questions else tuple(filter(_is_not_box_question, ir_request.questions)) ), ) @@ -368,39 +447,20 @@ def _without_malformed_images( return well_formed -class _DeploymentBoxQuestions(BaseModel): - model_config = ConfigDict(frozen=True, from_attributes=True) - - supports_box_questions: StrictBool | None = None - - -def _deployment_supports_box_questions(kwargs: Mapping[str, object]) -> bool | None: - try: - return _DeploymentBoxQuestions.model_validate(kwargs.get("model_info") or {}).supports_box_questions - except ValidationError: - return None - - def _provider_ir_request( ir_request: DecisionsIRRequest, *, provider_config: BaseDecisionsConfig, - model: str, - custom_llm_provider: str, headers: Mapping[str, str], + support: _ResolvedSupport, kwargs: Mapping[str, object], ) -> DecisionsIRRequest | _UnsupportedParams | _NothingLeftAfterDrop: - supports_box_questions: Final = provider_config.supports_box_questions( - model, custom_llm_provider, headers, _deployment_supports_box_questions(kwargs) - ) - unsupported: Final = _unsupported_params( - ir_request, provider_config, model, custom_llm_provider, headers, supports_box_questions - ) + unsupported: Final = _unsupported_params(ir_request, provider_config, headers, support) if not unsupported: return ir_request if not _drops_params(kwargs): return _UnsupportedParams(names=unsupported) - supported: Final = _without_unsupported_params(ir_request, provider_config, headers, supports_box_questions) + supported: Final = _without_unsupported_params(ir_request, provider_config, headers, support) if _leaves_no_input(supported.input): return _NothingLeftAfterDrop(names=unsupported, missing="input to decide on") if not supported.questions: @@ -479,10 +539,11 @@ def prepare_call( raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error try: headers: Final = decisions_outbound_headers(extra_headers, provider, kwargs) + support: Final = _resolved_support(provider_config, canonical_model, provider, headers, kwargs, fields) request_safety_identifier: Final = _request_safety_identifier(safety_identifier, provider_config, kwargs) - request: Final = _fields_without_unsupported_media( - fields, provider_config, canonical_model, provider, kwargs - ).validate(safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs)) + request: Final = _fields_without_unsupported_media(fields, support, kwargs).validate( + safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs) + ) except ValidationError as error: raise litellm.BadRequestError( message=f"Invalid Decisions request: {error}", @@ -523,12 +584,7 @@ def prepare_call( llm_provider=provider, ) ir_request: Final = _provider_ir_request( - caller_ir_request, - provider_config=provider_config, - model=canonical_model, - custom_llm_provider=provider, - headers=headers, - kwargs=kwargs, + caller_ir_request, provider_config=provider_config, headers=headers, support=support, kwargs=kwargs ) if isinstance(ir_request, _UnsupportedParams): raise litellm.UnsupportedParamsError( diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index d816bc661f3..39368dee47c 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -14,7 +14,7 @@ import itertools import os import re from collections.abc import Callable, Container, Iterable, Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast from urllib.parse import urlparse from pydantic import TypeAdapter, ValidationError @@ -102,6 +102,192 @@ def _validated_object_list(value: object) -> list[object] | None: return None +_MessageT: Final = TypeVar("_MessageT") +_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None] +_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls") + + +def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]: + lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS) + return tuple( + itertools.chain.from_iterable( + _OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list) + ) + ) + + +def _block_carries_cache_control(block: object) -> bool: + return isinstance(block, dict) and _CONTENT_PART_ADAPTER.validate_python(block).get("cache_control") is not None + + +_ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT: Final = frozenset({"tool", "function"}) + + +def _message_mark_reaches_provider(fields: Mapping[str, object]) -> bool: + if fields.get("cache_control") is None: + return False + return ( + not isinstance(fields.get("content"), list) + or fields.get("role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT + ) + + +def cache_control_count(message: object) -> int: + if not isinstance(message, dict): + return 0 + fields: Final = _CONTENT_PART_ADAPTER.validate_python(message) + block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block)) + return int(_message_mark_reaches_provider(fields)) + block_marks + + +def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]: + others: Final = {key: value for key, value in fields.items() if key != "cache_control"} + if "cache_control" not in fields: + return others + cache_control: Final = rewrite(fields["cache_control"]) + return others if cache_control is None else {**others, "cache_control": cache_control} + + +def block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object: + if not isinstance(block, dict): + return block + return _fields_with_cache_control(_CONTENT_PART_ADAPTER.validate_python(block), rewrite) + + +def with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT: + if not isinstance(message, dict): + return message + fields: Final = _CONTENT_PART_ADAPTER.validate_python(message) + blocks_rewritten: Final = { + key: ( + [block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)] + if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list) + else value + ) + for key, value in fields.items() + } + return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped + _MessageT, _fields_with_cache_control(blocks_rewritten, rewrite) + ) + + +def dropped_cache_control(_: object) -> None: + return None + + +def _first_marked_index(value: object) -> int | None: + if not isinstance(value, list): + return None + blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value) + return next((index for index, block in enumerate(blocks) if _block_carries_cache_control(block)), None) + + +_MarkSlot: TypeAlias = tuple[str, int] +_MESSAGE_MARK_SLOT: Final[_MarkSlot] = ("cache_control", -1) + + +def _block_mark_slots(fields: Mapping[str, object], key: str) -> tuple[_MarkSlot, ...]: + value: Final = fields.get(key) + if not isinstance(value, list): + return () + blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value) + return tuple((key, index) for index, block in enumerate(blocks) if _block_carries_cache_control(block)) + + +def _mark_slots_in_wire_order(fields: Mapping[str, object]) -> tuple[_MarkSlot, ...]: + content_slots: Final = _block_mark_slots(fields, "content") + tool_call_slots: Final = _block_mark_slots(fields, "tool_calls") + if not _message_mark_reaches_provider(fields): + return content_slots + tool_call_slots + if isinstance(fields.get("content"), list): + return content_slots + (_MESSAGE_MARK_SLOT,) + tool_call_slots + return (_MESSAGE_MARK_SLOT,) + content_slots + tool_call_slots + + +def _blocks_without_marks(key: str, value: object, dropped_slots: frozenset[_MarkSlot]) -> object: + if key not in _MARK_CARRYING_MESSAGE_FIELDS or not isinstance(value, list): + return value + return [ + block_with_cache_control(block, dropped_cache_control) if (key, index) in dropped_slots else block + for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(value)) + ] + + +def _without_oldest_marks(message: _MessageT, dropped: int) -> _MessageT: + if dropped == 0 or not isinstance(message, dict): + return message + fields: Final = _CONTENT_PART_ADAPTER.validate_python(message) + dropped_slots: Final = frozenset(_mark_slots_in_wire_order(fields)[:dropped]) + blocks_unmarked: Final = {key: _blocks_without_marks(key, value, dropped_slots) for key, value in fields.items()} + return cast( # cast-ok: the same message with its oldest wire-order marks dropped + _MessageT, + _fields_with_cache_control(blocks_unmarked, dropped_cache_control) + if _MESSAGE_MARK_SLOT in dropped_slots + else blocks_unmarked, + ) + + +def tool_message_with_single_cache_mark(message: _MessageT) -> _MessageT: + if not isinstance(message, dict): + return message + fields: Final = _CONTENT_PART_ADAPTER.validate_python(message) + content: Final = fields.get("content") + if fields.get("role") != "tool" or not isinstance(content, list): + return message + kept_index: Final = None if fields.get("cache_control") is not None else _first_marked_index(content) + return cast( # cast-ok: the same tool message keeping only the mark its single cache point reads + _MessageT, + { + **fields, + "content": [ + block if index == kept_index else block_with_cache_control(block, dropped_cache_control) + for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(content)) + ], + }, + ) + + +_BudgetSpend: TypeAlias = Callable[[tuple[int, int], int], tuple[int, int]] + + +def _newest_marks_kept(state: tuple[int, int], marks: int) -> tuple[int, int]: + remaining, _ = state + kept: Final = min(marks, remaining) + return remaining - kept, marks - kept + + +def _whole_message_kept(state: tuple[int, int], marks: int) -> tuple[int, int]: + remaining, _ = state + if marks > remaining: + return remaining, marks + return remaining - marks, 0 + + +def _messages_within_budget( + messages: Sequence[_MessageT], reserved_marks: int, spend: _BudgetSpend +) -> Sequence[_MessageT]: + budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0) + newest_first: Final = tuple(reversed(messages)) + budget_after_each: Final = itertools.accumulate( + (cache_control_count(message) for message in newest_first), spend, initial=(budget, 0) + ) + dropped_counts: Final = tuple(dropped for _, dropped in itertools.islice(budget_after_each, 1, None)) + kept_newest_first: Final = tuple( + _without_oldest_marks(message, dropped) for message, dropped in zip(newest_first, dropped_counts) + ) + return tuple(reversed(kept_newest_first)) + + +def newest_marks_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]: + return _messages_within_budget(messages, reserved_marks, _newest_marks_kept) + + +def whole_messages_within_cache_control_limit( + messages: Sequence[_MessageT], reserved_marks: int +) -> Sequence[_MessageT]: + return _messages_within_budget(messages, reserved_marks, _whole_message_kept) + + def _part_with_cache_control_renamed(part: object) -> object: if not _accepts_prompt_cache_breakpoint(part): return part @@ -1439,3 +1625,25 @@ class AnthropicCacheControlHook(CustomPromptManagement): llm_router=None, ) return None + + +def messages_within_cache_control_cap( + messages: Sequence[_MessageT], tools: object, cache_control: object = None, request_kwargs: object = None +) -> Sequence[_MessageT]: + """Under drop_params, drop the oldest message hints the provider's cache_control cap has no room for. + + Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first. + """ + reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints( + _validated_object_list(tools), cache_control, request_kwargs + ) + kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks) + dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept)) + if dropped > 0: + verbose_logger.warning( + "Dropped %d cache_control hint(s) from the oldest messages because drop_params is on and the " + "provider accepts at most %d cache_control blocks per request", + dropped, + MAX_CACHE_CONTROL_BLOCKS, + ) + return kept diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index 045869e4547..54213f40ac0 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -208,11 +208,14 @@ class StandardBuiltInToolCostTracking: usage=usage, response_object=response_object ) + response_model: Final = _output_item_field(response_object, "model") if model_info is not None and resolved_usage is not None and resolved_provider is not None: result: Final = get_cost_for_web_search_request( custom_llm_provider=resolved_provider, usage=resolved_usage, model_info=model_info, + response_model=response_model if isinstance(response_model, str) else None, + lookup_model_info=StandardBuiltInToolCostTracking._safe_get_model_info, ) if result is not None: return result diff --git a/litellm/litellm_core_utils/logging_utils.py b/litellm/litellm_core_utils/logging_utils.py index 8f58e8b7051..88d84201beb 100644 --- a/litellm/litellm_core_utils/logging_utils.py +++ b/litellm/litellm_core_utils/logging_utils.py @@ -3,13 +3,15 @@ import functools import inspect import re import time -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence from datetime import datetime from typing import TYPE_CHECKING, Any, Final, TypeAlias, cast from litellm._logging import format_base64_size, verbose_logger from litellm.constants import ( + BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS, BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS, + BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS, MAX_BASE64_LENGTH_FOR_LOGGING, ) from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs @@ -44,6 +46,8 @@ Helper utils used for logging callbacks # Regex matching data-URI base64 content: "data:;base64," # Captures: group(1)=mime_type, group(2)=base64_payload _DATA_URI_RE: Final = re.compile(r"data:([^;,\s]{1,255});base64,([A-Za-z0-9+/=]+)") +_DATA_URI_PREFIX: Final = "data:" +_DATA_URI_BASE64_MARKER: Final = ";base64," # Maximum nesting depth for _truncate_base64_in_value to guard against # pathological payloads. OpenAI message format is typically 3-4 levels deep. @@ -60,15 +64,20 @@ def _base64_data_uri_replacer(match: re.Match[str]) -> str: return f"data:{mime_type};base64,[base64_data truncated: {size_str}]" +def _may_contain_base64_data_uri(value: str) -> bool: + return ";" in value and _DATA_URI_PREFIX in value and _DATA_URI_BASE64_MARKER in value + + def _truncate_base64_in_string(value: str) -> str: """Replace long base64 data-URI payloads in a string with a size placeholder.""" - if MAX_BASE64_LENGTH_FOR_LOGGING <= 0: + if MAX_BASE64_LENGTH_FOR_LOGGING <= 0 or not _may_contain_base64_data_uri(value): return value return _DATA_URI_RE.sub(_base64_data_uri_replacer, value) def _truncate_base64_in_value( value: str | dict[str, object] | list[object] | None, + truncate_string: Callable[[str], str], ) -> str | dict[str, object] | list[object] | None: """Iteratively truncate base64 data URIs in a JSON-like value (str/list/dict). @@ -78,13 +87,13 @@ def _truncate_base64_in_value( # Stack entries: (source_value, depth, parent_container, key_or_index) # We mutate *copies* of dicts/lists in-place via parent references. if isinstance(value, str): - return _truncate_base64_in_string(value) + return truncate_string(value) if not isinstance(value, (dict, list)): return value # Shallow-copy the root so we don't mutate the caller's data. root: Final = {k: v for k, v in value.items()} if isinstance(value, dict) else list(value) - stack: Final[list] = [(root, 0)] + stack: Final[list[tuple[dict[str, object] | list[object], int]]] = [(root, 0)] # mutable-ok: explicit stack while stack: container, depth = stack.pop() @@ -93,19 +102,19 @@ def _truncate_base64_in_value( if isinstance(container, dict): for k, v in container.items(): if isinstance(v, str): - container[k] = _truncate_base64_in_string(v) + container[k] = truncate_string(v) elif isinstance(v, dict): - copy: dict | list = {ck: cv for ck, cv in v.items()} + copy: dict[str, object] | list[object] = {ck: cv for ck, cv in v.items()} container[k] = copy stack.append((copy, depth + 1)) elif isinstance(v, list): copy = list(v) container[k] = copy stack.append((copy, depth + 1)) - elif isinstance(container, list): + else: for i, v in enumerate(container): if isinstance(v, str): - container[i] = _truncate_base64_in_string(v) + container[i] = truncate_string(v) elif isinstance(v, dict): copy = {ck: cv for ck, cv in v.items()} container[i] = copy @@ -127,8 +136,15 @@ def truncate_base64_in_messages( """ if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0: return messages + return _copy_messages(messages, _truncate_base64_in_string) + + +def _copy_messages( + messages: str | dict[str, object] | list[object], # mutable-ok: same contract as truncate_base64_in_messages + truncate_string: Callable[[str], str], +) -> str | dict[str, object] | list[object] | None: # mutable-ok: same contract as truncate_base64_in_messages try: - return _truncate_base64_in_value(messages) + return _truncate_base64_in_value(messages, truncate_string) except Exception as e: verbose_logger.debug("Failed to truncate base64 in messages: %s", e) return messages @@ -151,19 +167,31 @@ def _iter_string_leaves(value: _StringTree) -> Iterator[str]: pass +def _is_few_plain_strings(leaves: tuple[str, ...]) -> bool: + return len(leaves) < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS and not any( + _may_contain_base64_data_uri(leaf) for leaf in leaves + ) + + async def truncate_base64_in_messages_async( messages: str | list | dict | None, # mutable-ok: same contract as truncate_base64_in_messages ) -> str | list | dict | None: # mutable-ok: same contract as truncate_base64_in_messages """ Same result as truncate_base64_in_messages, but payloads whose string content - reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are scanned in a worker - thread so the regex pass over multi-MB base64 images does not block the event loop. + reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are copied in a worker thread, + so neither the regex pass over multi-MB base64 images nor the copy of a long + history blocks the event loop. A few strings with no data URI, under + BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS in total, are copied inline, since + checking them for data URIs and copying them costs less than the thread hop. """ if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0: return messages - total_chars: Final = sum(len(leaf) for leaf in _iter_string_leaves(messages)) + leaves: Final = tuple(_iter_string_leaves(messages)) + total_chars: Final = sum(len(leaf) for leaf in leaves) if total_chars < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS: return truncate_base64_in_messages(messages) + if total_chars < BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS and _is_few_plain_strings(leaves): + return _copy_messages(messages, lambda text: text) return await asyncio.to_thread(truncate_base64_in_messages, messages) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 862e61cb631..e733f48ffd7 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -88,17 +88,22 @@ def handle_any_messages_to_chat_completion_str_messages_conversion( return [{"input": str(messages)}] +def _message_with_str_content(message: AllMessageValues) -> AllMessageValues: + texts: Final = convert_content_list_to_str(message=message) + if not texts: + return message + return cast( # cast-ok: the same message with its content flattened to one string + AllMessageValues, {**message, "content": texts} + ) + + def handle_messages_with_content_list_to_str_conversion( messages: list[AllMessageValues], ) -> list[AllMessageValues]: """ - Handles messages with content list conversion + Returns new messages with each content list flattened to a string, leaving the caller's messages untouched """ - for message in messages: - texts = convert_content_list_to_str(message=message) - if texts: - message["content"] = texts - return messages + return [_message_with_str_content(message) for message in messages] def strip_name_from_message(message: AllMessageValues, allowed_name_roles: list[str] = ["user"]) -> AllMessageValues: diff --git a/litellm/llms/__init__.py b/litellm/llms/__init__.py index c18ce8cea68..84c91be9386 100644 --- a/litellm/llms/__init__.py +++ b/litellm/llms/__init__.py @@ -9,6 +9,8 @@ from litellm.types.utils import CallTypes from . import * if TYPE_CHECKING: + from collections.abc import Callable + from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, ) @@ -30,7 +32,29 @@ def get_cost_for_google_maps_grounding_request( return cost_per_google_maps_grounding_request(usage=usage, model_info=model_info) -def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", model_info: "ModelInfo") -> float | None: +def _with_served_model_search_pricing( + model_info: "ModelInfo", + response_model: str | None, + served_model_provider: str, + lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None", +) -> "ModelInfo": + if model_info.get("search_context_cost_per_query") is not None or response_model is None: + return model_info + served_model_info: Final = ( + lookup_model_info(response_model, served_model_provider) if lookup_model_info is not None else None + ) + if served_model_info is None: + return model_info + return {**model_info, "search_context_cost_per_query": served_model_info.get("search_context_cost_per_query")} + + +def get_cost_for_web_search_request( + custom_llm_provider: str, + usage: "Usage", + model_info: "ModelInfo", + response_model: str | None = None, + lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None" = None, +) -> float | None: """ Get the cost for a web search request for a given model. @@ -38,6 +62,8 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo custom_llm_provider: The custom LLM provider. usage: The usage object. model_info: The model info. + response_model: The model the provider response names. + lookup_model_info: Looks up a model's cost map entry, returning None when it is not mapped. """ if custom_llm_provider == "gemini": from .gemini.cost_calculator import cost_per_web_search_request @@ -46,17 +72,25 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo elif custom_llm_provider == "anthropic": from .anthropic.cost_calculation import get_cost_for_anthropic_web_search - return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage) + return get_cost_for_anthropic_web_search( + model_info=_with_served_model_search_pricing(model_info, response_model, "anthropic", lookup_model_info), + usage=usage, + ) elif custom_llm_provider.startswith("vertex_ai"): # Anthropic Claude models on Vertex AI populate server_tool_use.web_search_requests # (same as the direct Anthropic API), not prompt_tokens_details.web_search_requests # (which is the Gemini field). Route claude-* models to the Anthropic calculator. model_key: Final[str] = model_info.get("key", "") if model_info else "" - if "claude" in model_key.lower(): + if "claude" in (response_model or model_key).lower(): from .anthropic.cost_calculation import get_cost_for_anthropic_web_search verbose_logger.debug("vertex_ai/claude model detected — routing web search cost to Anthropic calculator") - return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage) + return get_cost_for_anthropic_web_search( + model_info=_with_served_model_search_pricing( + model_info, response_model, "vertex_ai", lookup_model_info + ), + usage=usage, + ) from .vertex_ai.gemini.cost_calculator import ( cost_per_web_search_request as cost_per_web_search_request_vertex_ai, @@ -71,7 +105,10 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo return None from .anthropic.cost_calculation import get_cost_for_anthropic_web_search - return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage) + return get_cost_for_anthropic_web_search( + model_info=_with_served_model_search_pricing(model_info, response_model, "azure_ai", lookup_model_info), + usage=usage, + ) elif custom_llm_provider == "perplexity": # Perplexity handles search costs internally in its own cost calculator # Return 0.0 to indicate costs are already accounted for diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index d773e31c733..25245c347f4 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -167,6 +167,7 @@ _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool) _TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _TOOLS_ADAPTER: Final = TypeAdapter(list[object]) _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_REQUEST_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset( ("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid") ) @@ -2254,6 +2255,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): headers=headers, ) + @staticmethod + def _messages_within_cache_control_cap( + messages: Sequence[AllMessageValues], optional_params: Mapping[str, object] + ) -> Sequence[AllMessageValues]: + from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap + + return messages_within_cache_control_cap( + messages, optional_params.get("tools"), optional_params.get("cache_control"), optional_params + ) + def transform_request( self, model: str, @@ -2271,9 +2282,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): anthropic_messages_pt, ) - drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True + drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params")) + capped_messages: Final = ( + list( + self._messages_within_cache_control_cap( + messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params) + ) + ) + if drop_params + else messages + ) - if "tools" not in optional_params and messages is not None and has_tool_call_blocks(messages): + if "tools" not in optional_params and has_tool_call_blocks(capped_messages): optional_params["tools"], _ = self._map_tools(add_dummy_tool(custom_llm_provider="anthropic")) # Drop thinking param if thinking is enabled but thinking_blocks are missing @@ -2285,9 +2305,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # Related issue: https://github.com/BerriAI/litellm/issues/18926 if ( optional_params.get("thinking") is not None - and messages is not None - and last_assistant_with_tool_calls_has_no_thinking_blocks(messages) - and not any_assistant_message_has_thinking_blocks(messages) + and last_assistant_with_tool_calls_has_no_thinking_blocks(capped_messages) + and not any_assistant_message_has_thinking_blocks(capped_messages) ): if litellm.modify_params: optional_params.pop("thinking", None) @@ -2332,15 +2351,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): _name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request( optional_params=optional_params, ) - if _name_forward_map: - messages = self._rewrite_tool_names_in_messages(messages, _name_forward_map) + request_messages: Final = ( + self._rewrite_tool_names_in_messages(capped_messages, _name_forward_map) + if _name_forward_map + else capped_messages + ) if _name_reverse_map and isinstance(litellm_params, dict): litellm_params[ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY] = _name_reverse_map # Only the leading system run becomes the top-level system prompt. A later # system message stays in the conversation: hoisting it rewrites the cached # prefix and re-bills the whole history at cache-write pricing (#36559). - leading_system_run, later_messages = split_leading_system_run(messages) + leading_system_run, later_messages = split_leading_system_run(request_messages) anthropic_system_message_list: Final = self.translate_system_message(messages=list(leading_system_run)) # Handling anthropic API Prompt Caching if len(anthropic_system_message_list) > 0: @@ -2362,7 +2384,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): except Exception as e: raise AnthropicError( status_code=400, - message=f"{e}\nReceived Messages={messages}", + message=f"{e}\nReceived Messages={request_messages}", ) # don't use verbose_logger.exception, if exception is raised self.update_headers_with_optional_anthropic_beta( diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index e1f9a7d540f..e0da8943a4a 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -4,6 +4,7 @@ This file contains common utils for anthropic calls. import copy import json +import os import re from collections.abc import Container, Mapping, MutableMapping, Sequence from dataclasses import dataclass @@ -19,6 +20,8 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm.constants import ( + ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH, + ANTHROPIC_MAX_USER_ID_LENGTH, DEFAULT_MODEL_CREATED_AT_TIME, DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, @@ -178,6 +181,40 @@ def tools_without_custom_field(tools: Sequence[object], hoist_defer_loading: boo return [_tool_without_custom_field(tool, hoist_defer_loading, drop_custom) for tool in tools] +def anthropic_pass_through_api_base() -> str: + return os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com" + + +def _same_host_and_port(url: httpx.URL, other: str) -> bool: + try: + other_url: Final = httpx.URL(other) + except httpx.InvalidURL: + return False + return (url.host, url.port) == (other_url.host, other_url.port) + + +def anthropic_upstream_user_id_limit(url: str) -> int | None: + upstream: Final = httpx.URL(url) + if not upstream.path.removesuffix("/").endswith("/v1/messages"): + return None + if upstream.host == "api.anthropic.com": + return ANTHROPIC_MAX_USER_ID_LENGTH + return ( + ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH + if _same_host_and_port(upstream, anthropic_pass_through_api_base()) + else None + ) + + +def _has_email_shape(token: str) -> bool: + at_sign: Final = token.find("@", 1) + return at_sign != -1 and token.find(".", at_sign + 2, -1) != -1 + + +def anthropic_accepts_user_id(user_id: str, max_length: int) -> bool: + return len(user_id) <= max_length and user_id.isprintable() and not any(map(_has_email_shape, user_id.split())) + + def is_anthropic_messages_url(url: str) -> bool: """Check whether a URL addresses Anthropic's Messages API.""" parsed_url: Final = urlparse(url) diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index efd3e3fe9b5..c562c8d0f04 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -12,6 +12,7 @@ from pydantic_core import ErrorDetails from typing_extensions import assert_never import litellm +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str @@ -297,13 +298,7 @@ def _systemone_question(question: DecisionsIRQuestion) -> Mapping[str, object]: assert_never(question) -def ir_to_systemone_request( - model: str, request: DecisionsIRRequest, *, supports_audio: bool = False, supports_video: bool = False -) -> Mapping[str, object] | UnsupportedDecisionsRequest: - if request.audio and not supports_audio: - return UnsupportedDecisionsRequest(reason="audio input is not supported by this Decisions model") - if request.videos and not supports_video: - return UnsupportedDecisionsRequest(reason="video input is not supported by this Decisions model") +def ir_to_systemone_request(model: str, request: DecisionsIRRequest) -> Mapping[str, object]: state: Final = _systemone_state(request.input) keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True) images: Final = input_images(request.input) @@ -503,6 +498,16 @@ def ir_to_systemone_response(response: DecisionsIRResponse, request: DecisionsIR ) +def catalog_mode(model: str, custom_llm_provider: str) -> str | None: + try: + key: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)["key"] + except Exception: # noqa: BLE001 # get_model_info raises bare Exception for an unmapped model, which the catalog does not list + return None + entry: Final = GetModelCostMap.loaded_model_cost_map().get(key) + mode: Final = entry.get("mode") if entry is not None else None + return mode if isinstance(mode, str) else None + + def _is_decisions_model(model: str, custom_llm_provider: str) -> bool: try: model_mode: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider).get("mode") @@ -640,12 +645,16 @@ class BaseDecisionsConfig(ABC): model=model, custom_llm_provider=custom_llm_provider ) - def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool: + def supports_audio_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool: + if deployment_flag is not None: + return deployment_flag return _is_decisions_model(model, custom_llm_provider) and litellm.supports_audio_input( model=model, custom_llm_provider=custom_llm_provider ) - def supports_video_input(self, model: str, custom_llm_provider: str) -> bool: + def supports_video_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool: + if deployment_flag is not None: + return deployment_flag return _is_decisions_model(model, custom_llm_provider) and litellm.supports_video_input( model=model, custom_llm_provider=custom_llm_provider ) @@ -656,12 +665,7 @@ class BaseDecisionsConfig(ABC): request: DecisionsIRRequest, custom_llm_provider: str, ) -> Mapping[str, object] | UnsupportedDecisionsRequest: - return ir_to_systemone_request( - self.request_model(model), - request, - supports_audio=self.supports_audio_input(model, custom_llm_provider), - supports_video=self.supports_video_input(model, custom_llm_provider), - ) + return ir_to_systemone_request(self.request_model(model), request) def unwrap_response(self, payload: object) -> object: return payload diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 3800489ec05..21af97e523e 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -25,6 +25,7 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.core_helpers import ( as_str_mapping, + drops_unsupported_params, filter_exceptions_from_params, filter_internal_params, map_finish_reason, @@ -2129,6 +2130,24 @@ class AmazonConverseConfig(BaseConfig): return data + @staticmethod + def _claude_messages_within_cache_control_cap( + messages: Sequence[AllMessageValues], + capability_model: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + ) -> Sequence[AllMessageValues]: + if not drops_unsupported_params(litellm_params.get("drop_params")) or "claude" not in capability_model.lower(): + return messages + from litellm.integrations.anthropic_cache_control_hook import ( + messages_within_cache_control_cap, + tool_message_with_single_cache_mark, + ) + + return messages_within_cache_control_cap( + tuple(map(tool_message_with_single_cache_mark, messages)), optional_params.get("tools") + ) + async def _async_transform_request( self, model: str, @@ -2140,7 +2159,13 @@ class AmazonConverseConfig(BaseConfig): base_model: Final = litellm_params.get("base_model") capability_model: Final = bedrock_capability_model(model, base_model) messages, system_content_blocks = self._transform_system_message( - messages, model=capability_model, deployment_model=model + list( + self._claude_messages_within_cache_control_cap( + messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params + ) + ), + model=capability_model, + deployment_model=model, ) # Convert last user message to guarded_text if guardrailConfig is present @@ -2221,7 +2246,13 @@ class AmazonConverseConfig(BaseConfig): base_model: Final = litellm_params.get("base_model") capability_model: Final = bedrock_capability_model(model, base_model) messages, system_content_blocks = self._transform_system_message( - messages, model=capability_model, deployment_model=model + list( + self._claude_messages_within_cache_control_cap( + messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params + ) + ), + model=capability_model, + deployment_model=model, ) # Convert last user message to guarded_text if guardrailConfig is present diff --git a/litellm/llms/custom_httpx/aiohttp_transport.py b/litellm/llms/custom_httpx/aiohttp_transport.py index 9297e76624d..c32e75bafa8 100644 --- a/litellm/llms/custom_httpx/aiohttp_transport.py +++ b/litellm/llms/custom_httpx/aiohttp_transport.py @@ -110,6 +110,15 @@ def map_aiohttp_exceptions() -> Generator[None, None, None]: raise mapped_exc(message) from exc +def _unread_payload(content: object) -> bytes: + """Bytes the upstream sent before the connection dropped that aiohttp buffered but never handed out, since its + reads raise the stored payload error ahead of the buffer.""" + if not isinstance(content, aiohttp.StreamReader): + return b"" + buffered: Final = b"".join(content._buffer) # pyright: ignore[reportPrivateUsage] # no public read drains it + return buffered[content._buffer_offset :] # pyright: ignore[reportPrivateUsage] # see above + + def _release_transport_anchor(_transport: object) -> None: """Release a transport anchored to a response stream. See ``_anchor_transport_to``.""" @@ -144,6 +153,12 @@ class AiohttpResponseStream(httpx.AsyncByteStream): raise httpx.ReadError(str(e)) from e except aiohttp.http_exceptions.TransferEncodingError as e: raise httpx.ReadError(str(e)) from e + except aiohttp.ClientPayloadError: + unread: Final = _unread_payload(self._aiohttp_response.content) + if unread: + yield unread + with map_aiohttp_exceptions(): + raise except Exception: # For other exceptions, use the normal mapping with map_aiohttp_exceptions(): diff --git a/litellm/llms/databricks/embed/handler.py b/litellm/llms/databricks/embed/handler.py index 8b2fa768968..05807a118c3 100644 --- a/litellm/llms/databricks/embed/handler.py +++ b/litellm/llms/databricks/embed/handler.py @@ -26,6 +26,7 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase): aembedding=None, custom_endpoint: bool | None = None, headers: dict | None = None, + drop_params: bool = False, ) -> EmbeddingResponse: # Check for custom user agent in optional_params or environment # This allows partners building on LiteLLM to set their own telemetry @@ -58,4 +59,5 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase): aembedding=aembedding, custom_endpoint=True, headers=headers, + drop_params=drop_params, ) diff --git a/litellm/llms/deepinfra/chat/transformation.py b/litellm/llms/deepinfra/chat/transformation.py index 004128c0184..7caef58dc79 100644 --- a/litellm/llms/deepinfra/chat/transformation.py +++ b/litellm/llms/deepinfra/chat/transformation.py @@ -130,7 +130,8 @@ class DeepInfraConfig(OpenAIGPTConfig): - Input: {"role": "tool", "content": [{"type": "text", "text": "result"}]} - Output: {"role": "tool", "content": "[{\"type\": \"text\", \"text\": \"result\"}]"} """ - for message in messages: + copied_messages: Final = [message.copy() for message in messages] + for message in copied_messages: if message.get("role") == "tool": content = message.get("content") @@ -149,7 +150,7 @@ class DeepInfraConfig(OpenAIGPTConfig): # For complex content, serialize the entire array as JSON string message["content"] = json.dumps(content) - return messages + return copied_messages @overload def _transform_messages( diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 92c6c8439b3..5a9e5745c40 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -23,6 +23,7 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionToolParam, + ChatCompletionToolParamFunctionChunk, OpenAIChatCompletionToolParam, ) from litellm.types.utils import ( @@ -45,6 +46,7 @@ from ...openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, OpenAIGPTConfig, ) +from ...openai.common_utils import without_cache_control from ..common_utils import ( FIREROUTER, FireworksAIException, @@ -480,15 +482,23 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): return (("response_format", _json_schema_response_format(choice_schema, "choice")),) def _transform_tools(self, tools: list[OpenAIChatCompletionToolParam]) -> list[OpenAIChatCompletionToolParam]: - for tool in tools: - if tool.get("type") != "function": - continue - function = tool["function"] - function.pop("strict", None) - params = function.get("parameters") - if isinstance(params, dict): - unpack_legacy_defs(params) - return tools + return [self._fireworks_tool(tool) for tool in tools] + + @staticmethod + def _fireworks_tool(tool: OpenAIChatCompletionToolParam) -> OpenAIChatCompletionToolParam: + if tool.get("type") != "function": + return tool + function: Final = tool["function"] + params: Final = function.get("parameters") + unstrict_function: Final = cast( # cast-ok: the same function definition minus `strict` + ChatCompletionToolParamFunctionChunk, {key: value for key, value in function.items() if key != "strict"} + ) + new_function: Final[ChatCompletionToolParamFunctionChunk] = ( + {**unstrict_function, "parameters": unpack_legacy_defs(params, copy=True)} + if isinstance(params, dict) + else unstrict_function + ) + return {**tool, "function": new_function} def _transform_messages_helper( self, messages: list[AllMessageValues], model: str, litellm_params: dict @@ -496,10 +506,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): """ Strip fields not permitted by FireworksAI from messages. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - filter_value_from_dict, - ) - supports_vision_value: Final = self._get_model_cost_capability_exact(model=model, capability="supports_vision") for message in messages: if message["role"] == "user": @@ -530,9 +536,11 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): model=model, llm_provider="fireworks_ai", ) - filter_value_from_dict(cast(dict, message), "cache_control") - return [strip_litellm_response_only_fields(message) for message in messages] + return [ + strip_litellm_response_only_fields(cast(AllMessageValues, without_cache_control(message))) + for message in messages + ] # Cached index of fireworks_ai/* entries from litellm.model_cost. Building # this index requires a full scan of model_cost (tens of thousands of diff --git a/litellm/llms/groq/chat/transformation.py b/litellm/llms/groq/chat/transformation.py index 23f68335ea9..fd6b81fce8a 100644 --- a/litellm/llms/groq/chat/transformation.py +++ b/litellm/llms/groq/chat/transformation.py @@ -140,7 +140,8 @@ class GroqChatConfig(OpenAILikeChatConfig): def _transform_messages( self, messages: list[AllMessageValues], model: str, is_async: bool = False ) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]: - for idx, message in enumerate(messages): + transformed_messages: Final = list(messages) + for idx, message in enumerate(transformed_messages): """ 1. Don't pass 'null' function_call assistant message to groq - https://github.com/BerriAI/litellm/issues/5839 """ @@ -154,12 +155,12 @@ class GroqChatConfig(OpenAILikeChatConfig): for k, v in _message.items(): if v is not None: new_message[k] = v - messages[idx] = strip_litellm_internal_message_fields(new_message) + transformed_messages[idx] = strip_litellm_internal_message_fields(new_message) if is_async: - return super()._transform_messages(messages=messages, model=model, is_async=True) + return super()._transform_messages(messages=transformed_messages, model=model, is_async=True) else: - return super()._transform_messages(messages=messages, model=model, is_async=False) + return super()._transform_messages(messages=transformed_messages, model=model, is_async=False) def transform_messages( self, diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index 9f9503f93b6..3386cd452da 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -2,6 +2,7 @@ Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions` """ +import copy import json from collections.abc import Coroutine from typing import Final, Literal, cast, overload @@ -88,7 +89,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): ) -> dict: _tools = non_default_params.pop("tools", None) if _tools is not None: - _tools = remove_additional_properties(_tools) + _tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params _tools = remove_strict_from_schema(_tools) if isinstance(_tools, list): _tools = self._convert_custom_tools_to_function_tools(_tools) diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index db8dd3fa7f0..1ecef46ffb1 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -755,7 +755,8 @@ class MistralConfig(OpenAIGPTConfig): if _name is not None: # Remove name if not a tool message if message["role"] != "tool" or isinstance(_name, str) and len(_name.strip()) == 0: - message.pop("name", None) + message_without_name: Final = {k: v for k, v in message.items() if k != "name"} + return cast(AllMessageValues, message_without_name) # cast-ok: the same message minus name return message diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 22af51aaabe..9d551f6f859 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -14,7 +14,7 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm.constants import OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS -from litellm.litellm_core_utils.core_helpers import map_finish_reason +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( extract_reasoning_content, handle_invalid_parallel_tool_calls, @@ -64,7 +64,7 @@ from litellm.types.utils import ( ) from litellm.utils import convert_to_model_response_object, declared_mid_conversation_system_support -from ..common_utils import OpenAIError +from ..common_utils import OpenAIError, without_cache_control from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config if TYPE_CHECKING: @@ -128,6 +128,29 @@ def _litellm_params_str(litellm_params: Mapping[str, object] | None, key: str) - return value if isinstance(value, str) else None +def _drops_params(litellm_params: Mapping[str, object]) -> bool: + return drops_unsupported_params(litellm_params.get("drop_params")) + + +def _names_claude(model: str, litellm_params: Mapping[str, object]) -> bool: + base_model: Final = litellm_params.get("base_model") + return "claude" in model.lower() or (isinstance(base_model, str) and "claude" in base_model.lower()) + + +def _claude_messages_within_cache_control_cap( + model: str, + messages: Sequence[AllMessageValues], + tools: Sequence["ChatCompletionToolParam"] | None, + litellm_params: Mapping[str, object], + request_params: Mapping[str, object], +) -> Sequence[AllMessageValues]: + if not _names_claude(model, litellm_params) or not _drops_params(litellm_params): + return messages + from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap + + return list(messages_within_cache_control_cap(messages, tools, request_params.get("cache_control"), request_params)) + + def _prompt_cache_options(optional_params: Mapping[str, object]) -> object | None: try: extra_body: Final = _OBJECT_MAPPING.validate_python(optional_params.get("extra_body")) @@ -311,52 +334,58 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): return content_copy async def _async_handle_pdf_url(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile: - file_id: Final = content_item.get("file_id") + content_copy: Final = content_item.copy() + file_id: Final = content_copy.get("file_id") if file_id is not None: # check for file id being url done in _handle_pdf_url base64_data: Final = await async_convert_url_to_base64(file_id) - content_item["file_data"] = base64_data - content_item["filename"] = "my_file.pdf" - content_item.pop("file_id") - return content_item + content_copy["file_data"] = base64_data + content_copy["filename"] = "my_file.pdf" + content_copy.pop("file_id") + return content_copy def _common_file_data_check(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile: file_data: Final = content_item.get("file_data") filename: Final = content_item.get("filename") if file_data is not None and filename is None: - content_item["filename"] = "my_file.pdf" + return {**content_item, "filename": "my_file.pdf"} return content_item + @staticmethod + def _image_item_for_openai(content_item: ChatCompletionImageObject) -> ChatCompletionImageObject: + image_url: Final = content_item["image_url"] + if isinstance(image_url, str): + return {**content_item, "image_url": {"url": image_url}} + if not isinstance(image_url, dict): + return content_item + new_image_url_obj: Final = cast( + ChatCompletionImageUrlObject, + {k: v for k, v in image_url.items() if k not in {"format", "video_metadata"}}, + ) + return {**content_item, "image_url": new_image_url_obj} + + @staticmethod + def _file_item_for_openai(content_item: ChatCompletionFileObject) -> ChatCompletionFileObject: + litellm_specific_params: Final = {"format", "detail", "video_metadata"} + file_obj: Final = content_item.get("file") + if file_obj is None: + raise litellm.BadRequestError( + message="Content block has type='file' but is missing the required 'file' field", + model=None, + llm_provider="openai", + ) + new_file_obj: Final = ChatCompletionFileObjectFile( + **{k: v for k, v in file_obj.items() if k not in litellm_specific_params} + ) + return {**content_item, "file": new_file_obj} + def _apply_common_transform_content_item( self, content_item: OpenAIMessageContentListBlock, ) -> OpenAIMessageContentListBlock: - litellm_specific_params: Final = {"format", "detail", "video_metadata"} if content_item.get("type") == "image_url": - content_item = cast(ChatCompletionImageObject, content_item) - if isinstance(content_item["image_url"], str): - content_item["image_url"] = { - "url": content_item["image_url"], - } - elif isinstance(content_item["image_url"], dict): - new_image_url_obj: Final = cast( - ChatCompletionImageUrlObject, - {k: v for k, v in content_item["image_url"].items() if k not in {"format", "video_metadata"}}, - ) - content_item["image_url"] = new_image_url_obj - elif content_item.get("type") == "file": - content_item = cast(ChatCompletionFileObject, content_item) - file_obj: Final = content_item.get("file") - if file_obj is None: - raise litellm.BadRequestError( - message="Content block has type='file' but is missing the required 'file' field", - model=None, - llm_provider="openai", - ) - new_file_obj: Final = ChatCompletionFileObjectFile( - **{k: v for k, v in file_obj.items() if k not in litellm_specific_params} - ) - content_item["file"] = new_file_obj - + return self._image_item_for_openai(cast(ChatCompletionImageObject, content_item)) + if content_item.get("type") == "file": + return self._file_item_for_openai(cast(ChatCompletionFileObject, content_item)) return content_item def _transform_content_item( @@ -419,31 +448,34 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): hoisted_messages: Final = hoist_images_from_tool_messages(stripped_messages) async def _async_transform(): - for message in hoisted_messages: - message_content = message.get("content") - message_role = message.get("role") - - if message_role == "user" and message_content and isinstance(message_content, list): - message_content_types = cast(list[OpenAIMessageContentListBlock], message_content) - for i, content_item in enumerate(message_content_types): - message_content_types[i] = await self._async_transform_content_item( - cast(OpenAIMessageContentListBlock, content_item), - ) - return hoisted_messages + return [await self._async_user_message_for_openai(message) for message in hoisted_messages] if is_async: return _async_transform() - else: - for message in hoisted_messages: - message_content = message.get("content") - message_role = message.get("role") - if message_role == "user" and message_content and isinstance(message_content, list): - message_content_types = cast(list[OpenAIMessageContentListBlock], message_content) - for i, content_item in enumerate(message_content): - message_content_types[i] = self._transform_content_item( - cast(OpenAIMessageContentListBlock, content_item) - ) - return hoisted_messages + return [self._user_message_for_openai(message) for message in hoisted_messages] + + @staticmethod + def _user_content_parts(message: AllMessageValues) -> Sequence[OpenAIMessageContentListBlock] | None: + message_content: Final = message.get("content") + if message.get("role") != "user" or not message_content or not isinstance(message_content, list): + return None + return cast(Sequence[OpenAIMessageContentListBlock], message_content) + + def _user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues: + parts: Final = self._user_content_parts(message) + if parts is None: + return message + return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI + AllMessageValues, {**message, "content": [self._transform_content_item(part) for part in parts]} + ) + + async def _async_user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues: + parts: Final = self._user_content_parts(message) + if parts is None: + return message + return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI + AllMessageValues, {**message, "content": [await self._async_transform_content_item(part) for part in parts]} + ) def transform_messages( self, @@ -462,23 +494,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): messages: list[AllMessageValues], tools: list["ChatCompletionToolParam"] | None = None, ) -> tuple[list[AllMessageValues], list["ChatCompletionToolParam"] | None]: - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - filter_value_from_dict, - ) from litellm.types.llms.openai import ChatCompletionToolParam - for i, message in enumerate(messages): - messages[i] = cast( - AllMessageValues, - filter_value_from_dict(message, "cache_control"), - ) - if tools is not None: - for i, tool in enumerate(tools): - tools[i] = cast( - ChatCompletionToolParam, - filter_value_from_dict(tool, "cache_control"), - ) - return messages, tools + new_messages: Final = [cast(AllMessageValues, without_cache_control(message)) for message in messages] + new_tools: Final = ( + [cast(ChatCompletionToolParam, without_cache_control(tool)) for tool in tools] + if tools is not None + else None + ) + return new_messages, new_tools def translate_developer_role_to_system_role( self, @@ -527,6 +551,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): tools: Sequence["ChatCompletionToolParam"] | None, prompt_cache_options: object, litellm_params: Mapping[str, object], + request_params: Mapping[str, object], ) -> tuple[Sequence[AllMessageValues], Sequence["ChatCompletionToolParam"] | None]: """ A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on @@ -543,10 +568,11 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): model, custom_llm_provider, api_base, prompt_cache_options ) if not reads_breakpoints and self._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base): - return messages, tools - drops_breakpoints: Final = not reads_breakpoints and ( - litellm.drop_params is True or litellm_params.get("drop_params") is True - ) + return ( + _claude_messages_within_cache_control_cap(model, messages, tools, litellm_params, request_params), + tools, + ) + drops_breakpoints: Final = not reads_breakpoints and _drops_params(litellm_params) renamed: Final = ( [AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages] if reads_breakpoints @@ -561,7 +587,13 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): if drops_breakpoints else list(tools) ) - return self.remove_cache_control_flag_from_messages_and_tools(model=model, messages=renamed, tools=kept_tools) + sent_messages, sent_tools = self.remove_cache_control_flag_from_messages_and_tools( + model=model, messages=renamed, tools=kept_tools + ) + return ( + _claude_messages_within_cache_control_cap(model, sent_messages, sent_tools, litellm_params, request_params), + sent_tools, + ) def _sanitized_tools_update_for_openai( self, @@ -623,6 +655,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): tools=optional_params.get("tools", []), prompt_cache_options=_prompt_cache_options(optional_params), litellm_params=litellm_params, + request_params=optional_params, ) if tools is not None and len(tools) > 0: optional_params["tools"] = tools @@ -654,6 +687,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): tools=optional_params.get("tools", []), prompt_cache_options=_prompt_cache_options(optional_params), litellm_params=litellm_params, + request_params=optional_params, ) if tools is not None and len(tools) > 0: optional_params["tools"] = tools diff --git a/litellm/llms/openai/chat/o_series_transformation.py b/litellm/llms/openai/chat/o_series_transformation.py index 64285aafdb0..4d56a3b38c5 100644 --- a/litellm/llms/openai/chat/o_series_transformation.py +++ b/litellm/llms/openai/chat/o_series_transformation.py @@ -153,12 +153,13 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig): - role: system ==> translate to role 'user' """ _supports_system_messages: Final = supports_system_messages(model, "openai") - for i, message in enumerate(messages): + transformed_messages: Final = list(messages) + for i, message in enumerate(transformed_messages): if message["role"] == "system" and not _supports_system_messages: new_message = ChatCompletionUserMessage(content=message["content"], role="user") - messages[i] = new_message # Replace the old message with the new one + transformed_messages[i] = new_message # Replace the old message with the new one if is_async: - return super()._transform_messages(messages, model, is_async=cast(Literal[True], True)) + return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[True], True)) else: - return super()._transform_messages(messages, model, is_async=cast(Literal[False], False)) + return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[False], False)) diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index c113cb351d8..d476cf62626 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -3,6 +3,7 @@ Common helpers / utils across al OpenAI endpoints """ import asyncio +import copy import hashlib import inspect import json @@ -31,7 +32,7 @@ if TYPE_CHECKING: import litellm from litellm.exceptions import error_response_without_credentials from litellm.litellm_core_utils.completion_timeout import CompletionTimeout -from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object +from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, iter_items from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -43,6 +44,47 @@ from litellm.llms.custom_httpx.http_handler import ( ) +def _is_plain_dict(value: object) -> bool: + return isinstance(value, dict) + + +def _is_list(value: object) -> bool: + return isinstance(value, list) + + +def _contains_cache_control(value: object) -> bool: + pending: Final[list[object]] = [value] # mutable-ok: a local worklist, never returned or stored + while pending: + node = pending.pop() + mapping = as_str_mapping(node) + if mapping is not None: + if "cache_control" in mapping: + return True + pending.extend(mapping.values()) + elif _is_list(node): + pending.extend(iter_items(node)) + return False + + +def without_cache_control(item: object) -> object: + """Return `item` with every `cache_control` key removed, without touching the original. + + `filter_value_from_dict` deletes the key in place and recurses into nested dicts and + lists, so the caller's own message, tool or input object has to be copied first: it + may reuse the same list on a provider that does support prompt caching. The copy is + skipped when there is nothing to strip, which keeps the object identity the + pass-through paths rely on and avoids a deep copy on every request. + """ + from litellm.litellm_core_utils.prompt_templates.common_utils import filter_value_from_dict + + if not _is_plain_dict(item) or not _contains_cache_control(item): + return item + copied: Final = copy.deepcopy(item) + if isinstance(copied, dict): + filter_value_from_dict(copied, "cache_control") + return copied + + def _get_client_init_params(cls: type) -> tuple[str, ...]: """Extract __init__ parameter names (excluding 'self') from a class.""" return tuple(p for p in inspect.signature(cls.__init__).parameters if p != "self") diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index 4518fe6a77e..369b6959411 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -354,10 +354,10 @@ class OpenAIDecisionsConfig(BaseDecisionsConfig): ) -> str | None: return resolve_openai_bearer_token(api_key=api_key, api_base=api_base, litellm_params=litellm_params) - def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool: + def supports_audio_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool: return False - def supports_video_input(self, model: str, custom_llm_provider: str) -> bool: + def supports_video_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool: return False def transform_decisions_request( diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index bdd9f285823..6e22ef9112e 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -16,6 +16,7 @@ from openai._base_client import make_request_options from openai._constants import RAW_RESPONSE_HEADER from openai._legacy_response import LegacyAPIResponse from openai._types import RequestOptions +from openai.pagination import SyncCursorPage from openai.types import CreateEmbeddingResponse from openai.types.beta.assistant_deleted import AssistantDeleted from openai.types.chat import ChatCompletion, ChatCompletionChunk @@ -40,6 +41,7 @@ from litellm.types.utils import ( LiteLLMBatch, ModelResponse, ModelResponseStream, + openai_batch_status, ) from litellm.utils import ( CustomStreamWrapper, @@ -2182,6 +2184,18 @@ class OpenAIFilesAPI(BaseLLM): return response +def _with_openai_batch_fields(batches: Sequence[Batch]) -> Sequence[Batch]: + return [ + batch.model_copy( + update={ + "object": batch.object or "batch", + "status": openai_batch_status(batch.status), + } + ) + for batch in batches + ] + + class OpenAIBatchesAPI(BaseLLM): """ OpenAI methods to support for batches @@ -2372,7 +2386,7 @@ class OpenAIBatchesAPI(BaseLLM): ): verbose_logger.debug("listing batches, after= %s, limit= %s", after, limit) response: Final = await openai_client.batches.list(after=after, limit=limit) - return response + return response.model_copy(update={"data": _with_openai_batch_fields(response.data)}) def list_batches( self, @@ -2384,7 +2398,7 @@ class OpenAIBatchesAPI(BaseLLM): organization: str | None, after: str | None = None, limit: int | None = None, - client: OpenAI | None = None, + client: OpenAI | AsyncOpenAI | None = None, litellm_params: Mapping[str, object] | None = None, ): openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client( @@ -2409,7 +2423,9 @@ class OpenAIBatchesAPI(BaseLLM): ) return self.alist_batches(openai_client=openai_client, after=after, limit=limit) response: Final = openai_client.batches.list(after=after, limit=limit) - return response + if not isinstance(response, SyncCursorPage): + return response + return response.model_copy(update={"data": _with_openai_batch_fields(response.data)}) class OpenAIAssistantsAPI(BaseLLM): diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 9b8515a9ecd..7cca31c5d3f 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -52,7 +52,7 @@ from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders from litellm.types.workload_identity import OPENAI_WIF_KWARGS_KEYS -from ..common_utils import OpenAIError +from ..common_utils import OpenAIError, without_cache_control from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16 @@ -658,13 +658,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): input=self._replayable_input(marker_safe_input, reads_breakpoints, maps_anthropic_tool_search), tools=mapped_tools, ) - if marker_safe_input is not validated_input: - self._strip_caller_input_as_if_unmarked( - model=model, - caller_input=validated_input, - reads_breakpoints=reads_breakpoints, - maps_anthropic_tool_search=maps_anthropic_tool_search, - ) object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools) sanitized_tools: Final = self._sanitized_tool_schemas_for_openai( model=model, tools=object_schema_tools, litellm_params=litellm_params @@ -679,17 +672,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return marked_input return input_without_unanswered_anthropic_tool_search_calls(marked_input) - def _strip_caller_input_as_if_unmarked( - self, - model: str, - caller_input: str | ResponseInputParam, - reads_breakpoints: bool, - maps_anthropic_tool_search: bool, - ) -> None: - self.remove_cache_control_flag_from_input_and_tools( - model=model, input=self._replayable_input(caller_input, reads_breakpoints, maps_anthropic_tool_search) - ) - def _input_with_wire_safe_prompt_cache_breakpoints( self, model: str, @@ -800,24 +782,29 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): the chat path. Strips Anthropic-only `cache_control` markers from Responses API input content blocks and tools. - `filter_value_from_dict` mutates each dict in place, so the same - objects are returned. + `filter_value_from_dict` deletes the key in place and recurses, so each + item is copied first: the caller keeps its own input list and may reuse + it on a provider that does support prompt caching. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - filter_value_from_dict, + new_input: Final = ( + cast( # cast-ok: the comprehension rebuilds the input item for item + ResponseInputParam, + [without_cache_control(item) for item in input], + ) + if isinstance(input, list) + else input ) - if isinstance(input, list): - for item in input: - if isinstance(item, dict): - filter_value_from_dict(cast(dict, item), "cache_control") + new_tools: Final = ( + cast( # cast-ok: the comprehension rebuilds the tool list item for item + Sequence[ALL_RESPONSES_API_TOOL_PARAMS], + [without_cache_control(tool) for tool in tools], + ) + if tools is not None + else None + ) - if tools is not None: - for tool in tools: - if isinstance(tool, dict): - filter_value_from_dict(cast(dict, tool), "cache_control") - - return input, tools + return new_input, new_tools def _drop_bridge_minted_reasoning_items(self, input: str | ResponseInputParam) -> str | ResponseInputParam: if self.custom_llm_provider not in _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING or not isinstance(input, list): diff --git a/litellm/llms/openai_like/README.md b/litellm/llms/openai_like/README.md index c56f58b24c9..fb00b07c986 100644 --- a/litellm/llms/openai_like/README.md +++ b/litellm/llms/openai_like/README.md @@ -125,6 +125,20 @@ The provider inherits all request/response handling from OpenAI's Responses API If `supported_endpoints` is omitted, it defaults to `[]` (only chat completions, which is always enabled for JSON providers). +When only some models serve an endpoint natively, list them under `model_supported_endpoints`. Every other model reaches `/v1/responses` and `/v1/messages` through chat completions: + +```json +{ + "your_provider": { + "supported_endpoints": ["/v1/chat/completions"], + "model_supported_endpoints": { + "model-a": ["/v1/messages"], + "model-b": ["/v1/responses"] + } + } +} +``` + ### How It Works 1. `json_loader.py` checks `supported_endpoints` for `/v1/responses` diff --git a/litellm/llms/openai_like/embedding/handler.py b/litellm/llms/openai_like/embedding/handler.py index 77b6b673707..540feeca6e7 100644 --- a/litellm/llms/openai_like/embedding/handler.py +++ b/litellm/llms/openai_like/embedding/handler.py @@ -3,11 +3,16 @@ ## Allows jina ai embedding calls - which don't allow 'encoding_format' in payload. import json +import re +from collections.abc import Mapping +from types import MappingProxyType from typing import Final import httpx +from pydantic import TypeAdapter import litellm +from litellm._logging import verbose_logger from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, @@ -17,6 +22,57 @@ from litellm.types.utils import EmbeddingResponse from ..common_utils import OpenAILikeBase, OpenAILikeError +_KEY_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, str]) +_EXTRA_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_TRANSPORT_HEADER_NAMES: Final = frozenset( + { + "host", + "content-length", + "transfer-encoding", + "connection", + "keep-alive", + "proxy-connection", + "te", + "trailer", + "upgrade", + "expect", + } +) +_HEADER_NAME: Final = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+") +_HEADER_VALUE: Final = re.compile(r"[\t\x20-\x7e]*") + + +def _header_text(name: str, value: object) -> str | None: + if isinstance(value, bool) or not isinstance(value, str | int | float): + return None + if _HEADER_NAME.fullmatch(name) is None: + return None + return str(value).strip(" \t\r\n") + + +def _extra_headers_as_text(extra_headers: Mapping[str, object], model: str, drop_params: bool) -> Mapping[str, str]: + caller_set: Final = { + name: value for name, value in extra_headers.items() if name.lower() not in _TRANSPORT_HEADER_NAMES + } + texts: Final = MappingProxyType({name: _header_text(name, value) for name, value in caller_set.items()}) + refused: Final = sorted(name for name, text in texts.items() if text is None) + if refused and not drop_params: + raise litellm.UnsupportedParamsError( + message=f"extra_headers {', '.join(map(repr, refused))} cannot be sent. A header needs a name made of " + "letters, digits or !#$%&'*+-.^_`|~ and a text or number value. " + "To drop them from the call, set `litellm.drop_params = True`.", + model=model, + ) + named: Final = MappingProxyType({name: text for name, text in texts.items() if text is not None}) + unsendable: Final = sorted(name for name, text in named.items() if _HEADER_VALUE.fullmatch(text) is None) + if unsendable: + verbose_logger.warning( + "extra_headers %s dropped for model %r, since a header value must be printable ASCII on one line", + unsendable, + model, + ) + return MappingProxyType({name: text for name, text in named.items() if name not in unsendable}) + class OpenAILikeEmbeddingHandler(OpenAILikeBase): def __init__(self, **kwargs): @@ -94,6 +150,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase): aembedding=None, custom_endpoint: bool | None = None, headers: dict | None = None, + drop_params: bool = False, ) -> EmbeddingResponse: api_base, headers = self._validate_environment( api_base=api_base, @@ -103,8 +160,20 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase): custom_endpoint=custom_endpoint, ) model = model - filtered_optional_params: Final = {k: v for k, v in optional_params.items() if v not in (None, "")} - data: Final = {"model": model, "input": input, **filtered_optional_params} + data: Final = { + "model": model, + "input": input, + **{k: v for k, v in optional_params.items() if k != "extra_headers" and v not in (None, "")}, + } + extra_headers: Final = _extra_headers_as_text( + _EXTRA_HEADERS_ADAPTER.validate_python(optional_params.get("extra_headers") or {}), model, drop_params + ) + key_headers: Final = _KEY_HEADERS_ADAPTER.validate_python(headers) + extra_header_names: Final = frozenset(name.lower() for name in extra_headers) + request_headers: Final = { + **{name: value for name, value in key_headers.items() if name.lower() not in extra_header_names}, + **extra_headers, + } ## LOGGING logging_obj.pre_call( @@ -123,7 +192,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase): api_key=api_key, timeout=timeout, client=client, - headers=headers, + headers=request_headers, ) if client is None or isinstance(client, AsyncHTTPHandler): self.client = HTTPHandler(timeout=timeout) @@ -134,7 +203,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase): try: response: Final = self.client.post( api_base, - headers=headers, + headers=request_headers, data=json.dumps(data), ) diff --git a/litellm/llms/openai_like/json_loader.py b/litellm/llms/openai_like/json_loader.py index 6e99b9e7fec..658e4176f22 100644 --- a/litellm/llms/openai_like/json_loader.py +++ b/litellm/llms/openai_like/json_loader.py @@ -3,7 +3,7 @@ JSON-based provider configuration loader for OpenAI-compatible providers. """ import json -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from pathlib import Path from typing import Final @@ -14,6 +14,7 @@ from litellm.types.utils import ProviderSpecificModelInfo _PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo) _SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(Mapping[str, bool]) +_MODEL_ENDPOINTS_ADAPTER: Final = TypeAdapter(Mapping[str, tuple[str, ...]]) class SimpleProviderConfig: @@ -30,7 +31,10 @@ class SimpleProviderConfig: self.special_handling: Mapping[str, bool] = _SPECIAL_HANDLING_ADAPTER.validate_python( data.get("special_handling", {}) ) - self.supported_endpoints = data.get("supported_endpoints", []) + self.supported_endpoints: Sequence[str] = data.get("supported_endpoints", []) + self.model_supported_endpoints: Mapping[str, tuple[str, ...]] = _MODEL_ENDPOINTS_ADAPTER.validate_python( + data.get("model_supported_endpoints", {}) + ) self.provider_info: ProviderSpecificModelInfo = ( _PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {} ) @@ -90,6 +94,23 @@ class JSONProviderRegistry: return False return "/v1/responses" in provider.supported_endpoints + @classmethod + def supports_embeddings(cls, slug: str) -> bool: + provider: Final = cls._providers.get(slug) + if provider is None: + return False + return "/v1/embeddings" in provider.supported_endpoints + + @classmethod + def serves_endpoint(cls, slug: str, model: str | None, endpoint: str) -> bool: + provider: Final = cls._providers.get(slug) + if provider is None: + return False + model_endpoints: Final = ( + provider.model_supported_endpoints.get(model.removeprefix(f"{slug}/"), ()) if model else () + ) + return endpoint in provider.supported_endpoints or endpoint in model_endpoints + @classmethod def list_providers(cls) -> list: """List all registered provider slugs""" diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index ad47b334ed4..a71a9c3bb9b 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -211,7 +211,7 @@ "param_mappings": { "max_completion_tokens": "max_tokens" }, - "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"] + "supported_endpoints": ["/v1/chat/completions", "/v1/responses"] }, "scx-ai": { "base_url": "https://api.scx.ai/v1", @@ -231,6 +231,16 @@ "api_base_env": "PRISM_API_BASE", "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"] }, + "hostinger_router": { + "base_url": "https://router.hostinger.com/v1", + "api_key_env": "HOSTINGER_ROUTER_API_KEY", + "api_base_env": "HOSTINGER_ROUTER_API_BASE", + "supported_endpoints": ["/v1/chat/completions", "/v1/embeddings"], + "model_supported_endpoints": { + "claude-sonnet-5": ["/v1/messages"], + "gpt-5.2": ["/v1/responses"] + } + }, "sail": { "base_url": "https://api.sailresearch.com/v1", "api_key_env": "SAIL_API_KEY", diff --git a/litellm/llms/vertex_ai/context_caching/transformation.py b/litellm/llms/vertex_ai/context_caching/transformation.py index f683ad0e377..a4601af944b 100644 --- a/litellm/llms/vertex_ai/context_caching/transformation.py +++ b/litellm/llms/vertex_ai/context_caching/transformation.py @@ -110,38 +110,23 @@ def separate_cached_messages( messages: list[AllMessageValues], ) -> tuple[list[AllMessageValues], list[AllMessageValues]]: """ - Returns separated cached and non-cached messages. - - Args: - messages: List of messages to be separated. - - Returns: - Tuple containing: - - cached_messages: List of cached messages. - - non_cached_messages: List of non-cached messages. + Returns the first continuous block of cached messages, plus every system message once + any message is cached, since generateContent rejects a system_instruction beside a + cachedContent, and the remaining messages. """ - cached_messages: list[AllMessageValues] = [] - non_cached_messages: list[AllMessageValues] = [] + filtered_messages: Final = [(idx, message) for idx, message in enumerate(messages) if is_cached_message(message)] + if not filtered_messages: + return [], messages - # Extract cached messages and their indices - filtered_messages: Final[list[tuple[int, AllMessageValues]]] = [] - for idx, message in enumerate(messages): - if is_cached_message(message=message): - filtered_messages.append((idx, message)) - - # Validate only one block of continuous cached messages last_continuous_block_idx: Final = get_first_continuous_block_idx(filtered_messages) - # Separate messages based on the block of cached messages - if filtered_messages and last_continuous_block_idx is not None: - first_cached_idx: Final = filtered_messages[0][0] - last_cached_idx: Final = filtered_messages[last_continuous_block_idx][0] - - cached_messages = messages[first_cached_idx : last_cached_idx + 1] - non_cached_messages = messages[:first_cached_idx] + messages[last_cached_idx + 1 :] - else: - non_cached_messages = messages - - return cached_messages, non_cached_messages + cached_block: Final = range(filtered_messages[0][0], filtered_messages[last_continuous_block_idx][0] + 1) + goes_in_cache: Final = tuple( + idx in cached_block or message.get("role") == "system" for idx, message in enumerate(messages) + ) + return ( + [message for message, cached in zip(messages, goes_in_cache) if cached], + [message for message, cached in zip(messages, goes_in_cache) if not cached], + ) def cached_messages_end_on_supported_turn(cached_messages: Sequence[AllMessageValues]) -> bool: diff --git a/litellm/llms/watsonx/chat/transformation.py b/litellm/llms/watsonx/chat/transformation.py index 071ff7108b9..25981b0f04c 100644 --- a/litellm/llms/watsonx/chat/transformation.py +++ b/litellm/llms/watsonx/chat/transformation.py @@ -4,6 +4,7 @@ Translation from OpenAI's `/chat/completions` endpoint to IBM WatsonX's `/text/c Docs: https://cloud.ibm.com/apidocs/watsonx-ai#text-chat """ +import copy from typing import Final from litellm import verbose_logger @@ -57,7 +58,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig): _tools = non_default_params.pop("tools", None) if _tools is not None: # remove 'additionalProperties' from tools - _tools = remove_additional_properties(_tools) + _tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params # remove 'strict' from tools _tools = remove_strict_from_schema(_tools) if _tools is not None: diff --git a/litellm/llms/xai/chat/transformation.py b/litellm/llms/xai/chat/transformation.py index 2b943fa4107..f53b71a5a9a 100644 --- a/litellm/llms/xai/chat/transformation.py +++ b/litellm/llms/xai/chat/transformation.py @@ -1,3 +1,4 @@ +import copy from collections.abc import AsyncIterator, Iterator, Mapping from types import MappingProxyType from typing import Any, Final @@ -208,7 +209,8 @@ class XAIChatConfig(OpenAIGPTConfig): tools = [ filtered_tool for filtered_tool in ( - _FILTERED_TOOL.validate_python(filter_value_from_dict(tool, "strict")) for tool in value + _FILTERED_TOOL.validate_python(filter_value_from_dict(copy.deepcopy(tool), "strict")) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params + for tool in value ) if filtered_tool is not None ] diff --git a/litellm/main.py b/litellm/main.py index 01dbafaff9c..3a97fc7d5e6 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -6848,6 +6848,23 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse: ) +def _embedding_headers(param: str, configured: object, model: str, drop_params: bool) -> Mapping[str, object]: + if not configured: + return {} + if isinstance(configured, Mapping): + return dict(cast(Mapping[str, object], configured)) # cast-ok: header names from untyped request JSON + try: + return dict(cast(Iterable[tuple[str, object]], configured)) # cast-ok: name and value pairs from request JSON + except (TypeError, ValueError): + if drop_params: + return {} + raise litellm.UnsupportedParamsError( + message=f"{param} must be a JSON object of header names to values. " + "To drop it from the call, set `litellm.drop_params = True`.", + model=model, + ) + + # fmt: off # Overload for when aembedding=True (returns coroutine) @@ -6962,11 +6979,10 @@ def embedding( azure_ad_token_provider: Final = kwargs.get("azure_ad_token_provider", None) aembedding: Final[bool | None] = kwargs.get("aembedding", None) extra_headers: Final = kwargs.get("extra_headers", None) - headers = kwargs.get("headers", None) or extra_headers - if headers is None: - headers = {} - if extra_headers is not None: - headers.update(extra_headers) + headers = { + **_embedding_headers("headers", kwargs.get("headers"), model, drop_unsupported_params), + **_embedding_headers("extra_headers", extra_headers, model, drop_unsupported_params), + } # Inject proxy auth headers if configured if litellm.proxy_auth is not None: try: @@ -7130,7 +7146,7 @@ def embedding( # set API KEY api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") - if headers is not None and headers != {}: + if headers: optional_params["extra_headers"] = headers requested_encoding_format: Final = ( @@ -7222,14 +7238,15 @@ def embedding( custom_llm_provider == "openai_like" or custom_llm_provider == "llamafile" or custom_llm_provider == "lm_studio" + or JSONProviderRegistry.supports_embeddings(custom_llm_provider) ): api_base = api_base or litellm.api_base or get_secret_str("OPENAI_LIKE_API_BASE") # set API KEY - if api_key is None: - api_key = api_key or litellm.api_key or litellm.openai_like_key or get_secret_str("OPENAI_LIKE_API_KEY") + if api_key is None and not JSONProviderRegistry.supports_embeddings(custom_llm_provider): + api_key = litellm.api_key or litellm.openai_like_key or get_secret_str("OPENAI_LIKE_API_KEY") - if headers is not None and headers != {}: + if headers: optional_params["extra_headers"] = headers ## EMBEDDING CALL @@ -7244,10 +7261,9 @@ def embedding( optional_params=optional_params, client=client, aembedding=aembedding, + drop_params=drop_unsupported_params, ) elif custom_llm_provider == "oci": - if headers is None: - headers = {} response = base_llm_http_handler.embedding( model=model, input=input, @@ -7272,11 +7288,6 @@ def embedding( or litellm.api_key ) - # Use the merged headers variable (already merged at the top of the function) - # Don't overwrite it with just extra_headers - if headers is None: - headers = {} - response = base_llm_http_handler.embedding( model=model, input=input, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4d503952600..7eede4a395f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -39687,6 +39687,57 @@ "/v1/images/generations" ] }, + "hostinger_router/gpt-5.2": { + "cache_read_input_token_cost": 0.00000021, + "input_cost_per_token": 0.0000021, + "litellm_provider": "hostinger_router", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.0000168, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true, + "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "source": "https://router.hostinger.com/v1/models" + }, + "hostinger_router/claude-sonnet-5": { + "cache_creation_input_token_cost": 3.3e-06, + "cache_creation_input_token_cost_above_1hr": 5.28e-06, + "cache_read_input_token_cost": 2.64e-07, + "input_cost_per_token": 2.64e-06, + "litellm_provider": "hostinger_router", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.32e-05, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_response_schema": true, + "supported_endpoints": ["/v1/chat/completions", "/v1/messages"], + "source": "https://router.hostinger.com/v1/models" + }, + "hostinger_router/text-embedding-3-large": { + "input_cost_per_token": 1.716e-07, + "litellm_provider": "hostinger_router", + "max_input_tokens": 8191, + "mode": "embedding", + "output_cost_per_token": 0.0, + "source": "https://router.hostinger.com/v1/models" + }, + "hostinger_router/text-embedding-3-small": { + "input_cost_per_token": 3e-08, + "litellm_provider": "hostinger_router", + "max_input_tokens": 8191, + "mode": "embedding", + "output_cost_per_token": 0.0, + "source": "https://router.hostinger.com/v1/models" + }, "hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": { "input_cost_per_token": 1.2e-07, "litellm_provider": "hyperbolic", @@ -51387,6 +51438,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -51415,6 +51471,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -52161,6 +52222,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -52266,6 +52332,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, "regional_endpoint_uplift_multiplier": 1.1, diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index d33e61026df..700a35edb06 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -1087,6 +1087,32 @@ "generateContent": false } }, + "hostinger_router": { + "display_name": "Hostinger Router (`hostinger_router`)", + "url": "https://docs.litellm.ai/docs/providers/hostinger_router", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": true, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "crusoe": { "display_name": "Crusoe (`crusoe`)", "url": "https://docs.litellm.ai/docs/providers/crusoe", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index dc8b9415020..41ab7fcaf88 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -58015,7 +58015,7 @@ "paths": { "/toolset/{toolset_name}/mcp": { "delete": { - "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.", "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_delete", "parameters": [ { @@ -58054,7 +58054,7 @@ ] }, "get": { - "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.", "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_get", "parameters": [ { @@ -58093,7 +58093,7 @@ ] }, "head": { - "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.", "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_head", "parameters": [ { @@ -58132,7 +58132,7 @@ ] }, "options": { - "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.", "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_options", "parameters": [ { @@ -58171,7 +58171,7 @@ ] }, "patch": { - "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.", "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_patch", "parameters": [ { @@ -58210,7 +58210,7 @@ ] }, "post": { - "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.", "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_post", "parameters": [ { @@ -58249,7 +58249,7 @@ ] }, "put": { - "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.", + "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset//mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.", "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put", "parameters": [ { diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index c9b6fff3a00..5599086d5c1 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -5031,8 +5031,9 @@ class ProxyBaseLLMRequestProcessing: if stream_usage is None: return None service_tier: Final = obj.get("service_tier") + served_model: Final = litellm_logging_obj.model if litellm_logging_obj is not None else None cost_val: Final = ProxyBaseLLMRequestProcessing._streamed_usage_cost( - ModelResponse(usage=stream_usage), + ModelResponse(model=served_model, usage=stream_usage), model_name, service_tier if isinstance(service_tier, str) else None, litellm_logging_obj, diff --git a/litellm/proxy/db/accounting_ledger.py b/litellm/proxy/db/accounting_ledger.py index 91f8307283e..e37ce8d14d4 100644 --- a/litellm/proxy/db/accounting_ledger.py +++ b/litellm/proxy/db/accounting_ledger.py @@ -31,6 +31,13 @@ class AccountingConflictError(ValueError): pass +@dataclass(frozen=True, slots=True) +class UsageWrite: + event: InboundUsageEvent | OutboundUsageEvent + envelope: str + targets: tuple[DeliveryTarget, ...] = () + + _Transaction: TypeAlias = AbstractAsyncContextManager["Prisma"] @@ -107,6 +114,35 @@ SELECT event_id FROM "LiteLLM_AccountingEvent" WHERE kind = 'cost' AND source_ev _LOCK_EVENT: Final = """ SELECT payload_hash FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 FOR UPDATE """ +_INSERT_USAGE_BATCH: Final = """ +INSERT INTO "LiteLLM_AccountingEvent" + (event_id, kind, schema_version, payload_hash, occurred_at, payload) +SELECT event_id, kind::"AccountingEventKind", 1, payload_hash, occurred_at::timestamp, payload +FROM jsonb_to_recordset($1::jsonb) AS records( + event_id text, kind text, payload_hash text, occurred_at text, payload jsonb) +ORDER BY event_id +ON CONFLICT (event_id) DO NOTHING RETURNING event_id +""" +_LOCK_USAGE_BATCH: Final = """ +SELECT event_id, payload_hash FROM "LiteLLM_AccountingEvent" +WHERE event_id IN (SELECT value FROM jsonb_array_elements_text($1::jsonb)) +ORDER BY event_id FOR UPDATE +""" +_LOAD_USAGE_BATCH: Final = """ +SELECT event.payload, ARRAY( + SELECT target::text FROM "LiteLLM_AccountingDelivery" delivery + WHERE delivery.event_id = event.event_id ORDER BY target::text +) AS targets +FROM "LiteLLM_AccountingEvent" event +WHERE event.event_id IN (SELECT value FROM jsonb_array_elements_text($1::jsonb)) + AND event.kind = 'outbound_usage' AND event.schema_version = 1 +""" +_INSERT_USAGE_DELIVERIES: Final = """ +INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version) +SELECT records.event_id, destination.value::"AccountingDeliveryTarget", 1 +FROM jsonb_to_recordset($1::jsonb) AS records(event_id text, targets jsonb) +CROSS JOIN LATERAL jsonb_array_elements_text(records.targets) AS destination +""" _COMPLETED_FAMILY: Final = """ root.source_event_id IS NULL AND root.kind IN ('inbound_usage', 'outbound_usage') AND root.recorded_at < $1::timestamp @@ -199,6 +235,53 @@ class AccountingLedger: return cls(transaction) + async def insert_usage_batch(self, writes: tuple[UsageWrite, ...]) -> None: + records: Final = tuple(_prepare(write.event, write.targets, write.envelope, 1, 1) for write in writes) + if not records: + return + unique: Final = {record.event_id: record for record in records} + if any(record.payload_hash != unique[record.event_id].payload_hash for record in records): + raise AccountingConflictError("Batch contains conflicting copies of a usage identity") + batch: Final = _json( + tuple( + { + "event_id": record.event_id, + "kind": record.kind.value, + "payload_hash": record.payload_hash, + "occurred_at": record.occurred_at.astimezone(timezone.utc).isoformat(), + "payload": _JSON.validate_json(record.payload), + } + for record in unique.values() + ) + ) + async with self._transaction() as db: + inserted: Final = _STRINGS.validate_python(await db.query_raw(_INSERT_USAGE_BATCH, batch)) + accepted: Final = _STRINGS.validate_python(await db.query_raw(_LOCK_USAGE_BATCH, _json(tuple(unique)))) + if len(accepted) != len(unique) or any( + row["payload_hash"] != unique[row["event_id"]].payload_hash for row in accepted + ): + raise AccountingConflictError("Usage identity conflicts with accepted content or delivery set") + deliveries: Final = _json( + tuple({"event_id": row["event_id"], "targets": unique[row["event_id"]].targets} for row in inserted) + ) + await db.execute_raw(_INSERT_USAGE_DELIVERIES, deliveries) + + async def load_usage_batch(self, event_ids: tuple[str, ...]) -> tuple[UsageWrite, ...]: + if not event_ids: + return () + async with self._transaction() as db: + rows: Final = TypeAdapter(tuple[dict[str, JsonValue], ...]).validate_python( + await db.query_raw(_LOAD_USAGE_BATCH, _json(event_ids)) + ) + return tuple( + UsageWrite( + OutboundUsageEvent.model_validate(_JSON.validate_python(row["payload"])["event"]), + _json(row["payload"]), + TypeAdapter(tuple[DeliveryTarget, ...]).validate_python(row["targets"]), + ) + for row in rows + ) + async def insert_event( self, event: AccountingEvent, diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index baf53604ea5..13a14e95e56 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -72,6 +72,22 @@ def _exception_types(*candidates: object) -> tuple[type[BaseException], ...]: return tuple(c for c in candidates if isinstance(c, type) and issubclass(c, BaseException)) +def _prisma_error_code(e: Exception) -> str | None: + import prisma.engine.errors + + if isinstance(e, _exception_types(prisma.errors.DataError)): + code: Final[object] = getattr(e, "code", None) + return code if isinstance(code, str) else None + if not isinstance(e, _exception_types(prisma.engine.errors.EngineRequestError)): + return None + try: + body: Final = _DATABASE_ERROR_META.validate_json(str(e).partition(": ")[2]) + except ValidationError: + return None + engine_code: Final = body.get("error_code") + return engine_code if isinstance(engine_code, str) else None + + class PrismaDBExceptionHandler: """ Class to handle DB Exceptions or Connection Errors @@ -206,6 +222,8 @@ class PrismaDBExceptionHandler: return True if PrismaDBExceptionHandler.is_database_capacity_error(e): return False + if _prisma_error_code(e) == "P1017": + return True if isinstance(e, _exception_types(prisma.errors.PrismaError)): error_message: Final = str(e).lower() connection_keywords: Final = ( diff --git a/litellm/proxy/management_helpers/utils.py b/litellm/proxy/management_helpers/utils.py index 30aaaacad04..ea19befccc6 100644 --- a/litellm/proxy/management_helpers/utils.py +++ b/litellm/proxy/management_helpers/utils.py @@ -1,14 +1,15 @@ # What is this? ## Helper utils for the management endpoints (keys/users/teams) -from collections.abc import Callable, Mapping, MutableMapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, MutableMapping, Sequence from collections.abc import Set as AbstractSet from datetime import datetime from functools import wraps from types import MappingProxyType -from typing import Any, Final, Protocol +from typing import Any, Final, ParamSpec, Protocol, TypeVar from fastapi import HTTPException, Request from pydantic import BaseModel, TypeAdapter +from starlette.exceptions import HTTPException as StarletteHTTPException import litellm from litellm._internal_context import with_service_target @@ -687,6 +688,7 @@ async def _emit_management_endpoint_otel_span( end_time: datetime, result: Any = None, exception: Exception | None = None, + log_payload: bool = True, ) -> None: """Stamp + end the parent OTEL SERVER span for a management endpoint. @@ -716,8 +718,8 @@ async def _emit_management_endpoint_otel_span( ) route = get_request_route(http_request) - request_body: dict = await read_request_body( # rebind-ok: pre-existing rebinding on a rename-only line - request=http_request + request_body: dict = ( # rebind-ok: pre-existing rebinding on a rename-only line + await read_request_body(request=http_request) if log_payload else {} ) else: route = func.__name__ @@ -738,7 +740,7 @@ async def _emit_management_endpoint_otel_span( ) _response: dict[str, object] | None = None - if exception is None and result is not None: + if log_payload and exception is None and result is not None: try: raw: Final[Mapping[str, object]] = dict(result) _response = {k: v for k, v in raw.items() if k not in _CREDENTIAL_FIELDS} @@ -767,37 +769,60 @@ async def _emit_management_endpoint_otel_span( ) -def management_endpoint_wrapper(func): +_P: Final = ParamSpec("_P") +_R: Final = TypeVar("_R") + + +def management_endpoint_wrapper(func: Callable[_P, Awaitable[_R]]) -> Callable[_P, Awaitable[_R]]: """ This wrapper does the following: 1. Log I/O, Exceptions to OTEL 2. Create an Audit log for success calls """ + return _wrap_management_endpoint(func, log_payload=True) - @wraps(func) - async def wrapper(*args, **kwargs): + +def management_endpoint_span_wrapper(func: Callable[_P, Awaitable[_R]]) -> Callable[_P, Awaitable[_R]]: + return _wrap_management_endpoint(func, log_payload=False) + + +def _caller_auth(endpoint_kwargs: Mapping[str, object]) -> UserAPIKeyAuth: + caller: Final = endpoint_kwargs.get("user_api_key_dict") + return caller if isinstance(caller, UserAPIKeyAuth) else UserAPIKeyAuth() + + +def _proxy_exception_handler_ends_server_span(exception: Exception) -> bool: + return not isinstance(exception, StarletteHTTPException) + + +def _wrap_management_endpoint( + endpoint: Callable[_P, Awaitable[_R]], *, log_payload: bool +) -> Callable[_P, Awaitable[_R]]: + @wraps(endpoint) + async def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: start_time: Final = datetime.now() try: - result: Final = await func(*args, **kwargs) + result: Final = await endpoint(*args, **kwargs) end_time = datetime.now() try: - user_api_key_dict: UserAPIKeyAuth = kwargs.get("user_api_key_dict") or UserAPIKeyAuth() + user_api_key_dict: UserAPIKeyAuth = _caller_auth(kwargs) await send_management_endpoint_alert( request_kwargs=kwargs, user_api_key_dict=user_api_key_dict, - function_name=func.__name__, + function_name=endpoint.__name__, ) parent_otel_span: Span | None = getattr(user_api_key_dict, "parent_otel_span", None) if parent_otel_span is not None: await _emit_management_endpoint_otel_span( - func=func, + func=endpoint, kwargs=kwargs, parent_otel_span=parent_otel_span, start_time=start_time, end_time=end_time, result=result, + log_payload=log_payload, ) # Delete updated/deleted info from cache @@ -813,17 +838,18 @@ def management_endpoint_wrapper(func): except Exception as e: end_time = datetime.now() - user_api_key_dict: UserAPIKeyAuth = kwargs.get("user_api_key_dict") or UserAPIKeyAuth() + user_api_key_dict: UserAPIKeyAuth = _caller_auth(kwargs) parent_otel_span = getattr(user_api_key_dict, "parent_otel_span", None) - if parent_otel_span is not None: + if parent_otel_span is not None and (log_payload or not _proxy_exception_handler_ends_server_span(e)): try: await _emit_management_endpoint_otel_span( - func=func, + func=endpoint, kwargs=kwargs, parent_otel_span=parent_otel_span, start_time=start_time, end_time=end_time, exception=e, + log_payload=log_payload, ) except Exception as otel_exc: # Non-Blocking Exception - never let OTEL failures swallow diff --git a/litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py index db640b6d689..be24b1630b0 100644 --- a/litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py @@ -438,6 +438,14 @@ def is_mcp_admitted_user_subject(user_api_key_auth: UserAPIKeyAuth | None) -> bo _is_mcp_admitted_user_subject: Final = is_mcp_admitted_user_subject +def has_team_unbound_mcp_admin_view(user_api_key_auth: UserAPIKeyAuth | None) -> bool: + return ( + user_api_key_auth is not None + and user_api_key_has_admin_view(user_api_key_auth) + and (not user_api_key_auth.team_id or user_api_key_auth.team_id == UI_TEAM_ID) + ) + + def _gateway_dcr_challenge_target( route: str, mcp_servers: list[str] | None, @@ -4290,19 +4298,19 @@ class MCPRequestHandler: """Whether this principal's admin-view role grants the unscoped MCP resolution, whatever credential carries it (admin key, dashboard session, or OAuth-admitted session subject). - Two bounds disqualify, one per ownership of the row. A CREDENTIAL's explicit - ``object_permission.mcp_servers`` scope wins even for admins, including the empty list. An - admitted subject's object_permission is the user's own row, whose ``mcp_servers`` column is - [] by DB default, so for that shape the row binds through the entitlement ceiling instead - (any non-empty entitlement, or an unresolved one, disqualifies), exactly as - ``operator_open_server_ids`` reads the same row. The one owner of this predicate: the - server-axis registry resolution in ``get_allowed_mcp_servers`` and the tools-axis open - channel in ``resolve_admitted_subject_tools`` both consult it, so the two axes cannot - disagree.""" + A credential on a real team never qualifies, a virtual key or a ``lite login`` session alike: its team + bounds its MCP access whoever owns it, as it bounds the credential's model access. Two more bounds disqualify, one per ownership of the row. A CREDENTIAL's + explicit ``object_permission.mcp_servers`` scope wins even for admins, including the empty list. An + admitted subject's object_permission is the user's own row, whose ``mcp_servers`` column is [] by DB + default, so for that shape the row binds through the entitlement ceiling instead (any non-empty + entitlement, or an unresolved one, disqualifies), exactly as ``operator_open_server_ids`` reads the same + row. The one owner of this predicate: the server-axis registry resolution in ``get_allowed_mcp_servers`` + and the tools-axis open channel in ``resolve_admitted_subject_tools`` both consult it, so the two axes + cannot disagree.""" if ( user_api_key_auth is None or user_api_key_auth.mcp_explicit_grants_only - or not user_api_key_has_admin_view(user_api_key_auth) + or not has_team_unbound_mcp_admin_view(user_api_key_auth) ): return False object_permission: Final = user_api_key_auth.object_permission diff --git a/litellm/proxy/mcp_server/mcp_server_manager.py b/litellm/proxy/mcp_server/mcp_server_manager.py index c772e2e9d09..eacba25257e 100644 --- a/litellm/proxy/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/mcp_server/mcp_server_manager.py @@ -3705,7 +3705,7 @@ class MCPServerManager: Priority: 1. If object_permission.mcp_servers is explicitly set, use it (even for admins) - 2. If admin and no object_permission, return all servers + 2. If admin, not bound to a real team, and no object_permission, return all servers 3. Otherwise, use standard permission checks """ if managed_agent_policy(user_api_key_auth) is not None: diff --git a/litellm/proxy/mcp_server/operations.py b/litellm/proxy/mcp_server/operations.py index 2526522f97f..66e46c73f98 100644 --- a/litellm/proxy/mcp_server/operations.py +++ b/litellm/proxy/mcp_server/operations.py @@ -223,6 +223,7 @@ __all__ = ( "mcp_get_prompt", "mcp_read_resource", "raise_denied_scoped_mcp_access", + "raise_generic_mcp_access_denial", ) @@ -763,8 +764,14 @@ async def raise_denied_scoped_mcp_access( ) } raise HTTPException(status_code=403, detail=group_denial) + raise_generic_mcp_access_denial(requested_names) + + +def raise_generic_mcp_access_denial(requested_names: Sequence[str]) -> NoReturn: generic_denial: Final[_McpDeniedDetail] = { - "error": f"The key is not allowed to access the requested MCP servers: {', '.join(requested_names)}" + "error": ( + f"The key is not allowed to access the requested MCP servers: {', '.join(dict.fromkeys(requested_names))}" + ) } raise HTTPException(status_code=403, detail=generic_denial) diff --git a/litellm/proxy/mcp_server/server.py b/litellm/proxy/mcp_server/server.py index 8e0826b105e..aeb640a7c8b 100644 --- a/litellm/proxy/mcp_server/server.py +++ b/litellm/proxy/mcp_server/server.py @@ -520,6 +520,7 @@ if MCP_AVAILABLE: "mcp_get_prompt", "mcp_read_resource", "raise_denied_scoped_mcp_access", + "raise_generic_mcp_access_denial", "redact_mcp_resource_url", ) from mcp.server import Server @@ -1197,6 +1198,7 @@ if MCP_AVAILABLE: apply_display_name_overrides, filter_tools_by_allowed_tools, raise_denied_scoped_mcp_access, + raise_generic_mcp_access_denial, ) @contextlib.asynccontextmanager @@ -1824,10 +1826,10 @@ if MCP_AVAILABLE: row capping it. Raises HTTPException(403) unless the principal holds toolset_id through one - of its grant sources. Admins always pass. + of its grant sources. Admins pass unless the key is bound to a real team. """ from litellm.proxy._types import LiteLLM_ObjectPermissionTable - from litellm.proxy.management_endpoints.common_utils import user_api_key_has_admin_view + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import has_team_unbound_mcp_admin_view # A key scoped to no MCP servers opts out of every MCP path. Enforce it # here too, since toolset scoping replaces mcp_servers and would otherwise @@ -1841,7 +1843,7 @@ if MCP_AVAILABLE: ) acting: Final = await acting_user(user_api_key_auth) - is_admin: Final = user_api_key_has_admin_view(acting) + is_admin: Final = has_team_unbound_mcp_admin_view(acting) if not is_admin and toolset_id not in await granted(acting): raise HTTPException( status_code=403, diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 33b18c3d939..151938d73d7 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -52,7 +52,11 @@ from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming, is_guardrail_intervention from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.litellm_core_utils.fallback_generalizations import match_routing_generalization -from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers +from litellm.llms.anthropic.common_utils import ( + AnthropicModelInfo, + anthropic_pass_through_api_base, + merge_anthropic_beta_headers, +) from litellm.llms.azure.passthrough.transformation import ( foreign_azure_deployment, is_azure_body_model_inference_endpoint, @@ -1047,7 +1051,7 @@ async def anthropic_proxy_route( """ [Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion) """ - base_target_url = os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com" + base_target_url = anthropic_pass_through_api_base() encoded_endpoint = httpx.URL(endpoint).path # Ensure endpoint starts with '/' for proper URL construction diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index cfac28db1a9..bda4f4c2ad2 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -71,7 +71,11 @@ from litellm.litellm_core_utils.litellm_logging import get_masked_values from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.litellm_core_utils.redact_messages import should_redact_message_logging from litellm.litellm_core_utils.safe_json_dumps import safe_dumps -from litellm.llms.anthropic.common_utils import is_anthropic_messages_url +from litellm.llms.anthropic.common_utils import ( + anthropic_accepts_user_id, + anthropic_upstream_user_id_limit, + is_anthropic_messages_url, +) from litellm.llms.base_llm.managed_resources.utils import ( resolve_passthrough_managed_id_provider, ) @@ -148,6 +152,8 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, EndpointType, PassthroughStandardLoggingPayload, + WebSocketSessionMeter, + WebSocketSessionUsage, ) from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, CallTypes, Usage @@ -501,7 +507,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): url: httpx.URL, headers: dict, requested_query_params: dict | None = None, - _parsed_body: dict | None = None, + _parsed_body: Mapping[str, object] | None = None, forward_multipart: bool = False, ) -> httpx.Response: """ @@ -715,8 +721,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): # unenforced. Set after the client merge so a request body cannot supply its own budget. # # Only for the built-in provider routes. `get_model_from_request` returns - # None for a user-defined pass-through, deliberately: its body is forwarded - # verbatim, so `model` there names an UPSTREAM model rather than a + # None for a user-defined pass-through, deliberately: the client's `model` + # goes upstream unchanged, so it names an UPSTREAM model rather than a # LiteLLM-managed one. Enforcement is therefore skipped on those routes, and # charging a counter anyway would track spend that nothing can refuse, and # would attribute it to a budget the operator scoped to a LiteLLM model that @@ -1185,6 +1191,36 @@ def _unfiltered_headers(headers: Mapping[str, str]) -> Mapping[str, str]: return headers +def _json_object(value: object) -> Mapping[str, object]: + return cast("Mapping[str, object]", value) if isinstance(value, Mapping) else MappingProxyType({}) # cast-ok: json + + +def _anthropic_user_id_to_forward( + body: Mapping[str, object], + url: str, + has_pre_call_guardrails: Callable[[Mapping[str, object]], bool], +) -> str | None: + user_id: Final = _json_object(body.get("metadata")).get("user_id") + if not isinstance(user_id, str): + return None + max_length: Final = anthropic_upstream_user_id_limit(url) + if max_length is None: + return None + if not anthropic_accepts_user_id(user_id, max_length): + verbose_proxy_logger.warning("Not forwarding metadata.user_id because the upstream would refuse it") + return None + if has_pre_call_guardrails(body): + verbose_proxy_logger.debug("Not forwarding metadata.user_id because a pre-call guardrail cannot scan it") + return None + return user_id + + +def _with_anthropic_user_id(body: Mapping[str, object] | None, user_id: str | None) -> Mapping[str, object] | None: + if body is None or user_id is None: + return body + return {**body, "metadata": {"user_id": user_id}} + + def _eu_jev_client_body(custom_llm_provider: str | None, body: Mapping[str, object]) -> Mapping[str, object] | None: return MappingProxyType(dict(body)) if custom_llm_provider == "eu_jev" else None @@ -1421,6 +1457,10 @@ async def pass_through_request( request_method=getattr(request, "method", None), cost_per_request=cost_per_request, ) + hooked_body: Final = _json_object(_parsed_body) + client_user_id: Final = _anthropic_user_id_to_forward( + hooked_body, str(url), proxy_logging_obj.request_has_pre_call_guardrails + ) kwargs = HttpPassThroughEndpointHelpers.init_kwargs_for_pass_through_endpoint( # rebind-ok: pre-existing rebinding on a rename-only line user_api_key_dict=user_api_key_dict, _parsed_body=_parsed_body, @@ -1588,11 +1628,13 @@ async def pass_through_request( else: logging_url = str(url) + "?" + requested_query_params_str + outgoing_body: Final = cast("Mapping[str, object] | None", _parsed_body) # cast-ok: json + upstream_body: Final = _with_anthropic_user_id(outgoing_body, client_user_id) logging_obj.pre_call( input=[{"role": "user", "content": safe_dumps(_parsed_body)}], api_key="", additional_args={ - "complete_input_dict": _parsed_body, + "complete_input_dict": upstream_body, "api_base": str(logging_url), "headers": redact_credential_headers(upstream_headers), }, @@ -1632,7 +1674,7 @@ async def pass_through_request( url, params=requested_query_params, headers=upstream_headers, - json=_parsed_body, + json=upstream_body, ) ) @@ -1731,7 +1773,7 @@ async def pass_through_request( url=url, headers=upstream_headers, requested_query_params=requested_query_params, - _parsed_body=_parsed_body, + _parsed_body=upstream_body, forward_multipart=is_multipart, ) verbose_proxy_logger.debug("response.headers= %s", response.headers) @@ -2595,6 +2637,13 @@ def _with_trace_context(headers: Mapping[str, str], parent_span: object) -> dict return inject_trace_context(headers, parent_span=parent_span) +def _log_session_usage(logging_obj: LiteLLMLoggingObj, usage: WebSocketSessionUsage) -> None: + logging_obj.model = usage.model # rebind-ok: the spend logger reads model and cost off the shared logging object + logging_obj.model_call_details["model"] = usage.model # rebind-ok: same shared logging object + logging_obj.model_call_details["custom_llm_provider"] = usage.custom_llm_provider # rebind-ok: same + logging_obj.model_call_details["response_cost"] = usage.response_cost # rebind-ok: same + + async def websocket_passthrough_request( websocket: WebSocket, target: str, @@ -2607,6 +2656,7 @@ async def websocket_passthrough_request( setup_model_rewriter: Callable[[str], str] | None = None, builtin_provider_route: bool = False, model: str | None = None, + meter: WebSocketSessionMeter | None = None, ): """ WebSocket passthrough request handler. @@ -2622,6 +2672,7 @@ async def websocket_passthrough_request( setup_model_rewriter: Optional rewrite of the setup frame's model before it reaches the upstream builtin_provider_route: Whether a built-in provider route serves the connection, so its URL names the provider model: The model the caller named before connecting, logged on a failure before the setup frame arrives + meter: Prices the session from the client text frames delivered upstream, for an upstream that reports no usage """ from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy.proxy_server import proxy_config, proxy_logging_obj @@ -2632,6 +2683,7 @@ async def websocket_passthrough_request( # Initialize tracking variables start_time: Final = datetime.now() websocket_messages: Final[list[dict[str, object]]] = [] + metered_total = 0 # rebind-ok: forward_client_to_upstream adds the measure of each client text frame it delivers litellm_call_id: Final = str(uuid.uuid4()) verbose_proxy_logger.info("WebSocket passthrough (%s): Starting WebSocket connection to %s", endpoint, target) @@ -2750,6 +2802,7 @@ async def websocket_passthrough_request( async def forward_client_to_upstream() -> None: """Forward messages from client to upstream WebSocket""" + nonlocal metered_total try: while True: message = await websocket.receive() @@ -2819,6 +2872,8 @@ async def websocket_passthrough_request( # Not a JSON message or doesn't contain setup data await upstream_ws.send(_rewrite_vertex_live_setup_model(text_data, setup_model_rewriter)) + if meter is not None: + metered_total += meter.measure(text_data) elif bytes_data is not None: await upstream_ws.send(bytes_data) except asyncio.CancelledError: @@ -2945,6 +3000,8 @@ async def websocket_passthrough_request( self.url = target_url mock_response: Final = MockWebSocketResponse(target) + if meter is not None: + _log_session_usage(logging_obj, meter.price(metered_total)) # Use the same success handler as HTTP passthrough endpoints GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b9b15d332e9..7d4a42cf025 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -181,6 +181,7 @@ from litellm.proxy.common_utils.token_count_settings import ( ) from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog from litellm.proxy.management_helpers.model_group_custom_fields import model_group_custom_fields +from litellm.proxy.management_helpers.utils import management_endpoint_span_wrapper from litellm.repositories import database as database_runtime from litellm.router_strategy.auto_router.auto_router_model_naming import ( GATED_AUTO_ROUTER_CAPABILITIES, @@ -12339,6 +12340,7 @@ async def _deployment_hidden_by_listing_callbacks(deployment: Deployment, user_a @router.get( "/models", dependencies=[Depends(user_api_key_auth)], tags=["model management"] ) # if project requires model list +@management_endpoint_span_wrapper async def model_list( request: Request = None, # pyright: ignore[reportArgumentType] # FastAPI always injects the Request; the None default only serves direct in-process callers user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), @@ -12649,6 +12651,7 @@ async def model_list( dependencies=[Depends(user_api_key_auth)], tags=["model management"], ) +@management_endpoint_span_wrapper async def model_info( model_id: str, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), @@ -21985,9 +21988,9 @@ async def toolset_mcp_route(toolset_name: str, request: Request): Namespace a toolset as its own MCP endpoint. Connecting to /toolset//mcp exposes exactly the tools defined in - the toolset. Access is enforced: non-admin API keys must have the toolset - listed in their object_permission.mcp_toolsets grant list, or the request - will be rejected with a 403. + the toolset. Access is enforced: API keys other than an admin's key with no + team must hold the toolset through one of their grant sources, or the + request will be rejected with a 403. """ try: from litellm.proxy.mcp_server.mcp_server_manager import ( diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 01a822c5d93..044b8a1de6e 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -3343,6 +3343,34 @@ ], "default_model_placeholder": "prism/deepseek-v4.1-flash" }, + { + "provider": "HOSTINGER_ROUTER", + "provider_display_name": "Hostinger Router", + "litellm_provider": "hostinger_router", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://router.hostinger.com/v1", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "hostinger_router/claude-sonnet-5" + }, { "provider": "RECRAFT", "provider_display_name": "Recraft", diff --git a/litellm/proxy/spend_tracking/usage_meter.py b/litellm/proxy/spend_tracking/usage_meter.py new file mode 100644 index 00000000000..3bc7572b476 --- /dev/null +++ b/litellm/proxy/spend_tracking/usage_meter.py @@ -0,0 +1,268 @@ +from __future__ import annotations + +import asyncio +from collections import deque +from collections.abc import Awaitable, Callable +from concurrent.futures import Future +from dataclasses import dataclass +from itertools import accumulate, chain, count, takewhile +from threading import Lock +from typing import Final + +from pydantic import BaseModel, ConfigDict, Field + +from litellm._logging import verbose_proxy_logger +from litellm.accounting import ( + DeliveryTarget, + Dimension, + DimensionName, + InboundUsageEvent, + OutboundUsageEvent, +) +from litellm.proxy.db.accounting_ledger import AccountingConflictError, UsageWrite + +_DAILY_TARGETS: Final = ( + (DimensionName.KEY, DeliveryTarget.DAILY_USER_SPEND), + (DimensionName.USER, DeliveryTarget.DAILY_USER_SPEND), + (DimensionName.TEAM, DeliveryTarget.DAILY_TEAM_SPEND), + (DimensionName.ORGANIZATION, DeliveryTarget.DAILY_ORGANIZATION_SPEND), + (DimensionName.END_USER, DeliveryTarget.DAILY_END_USER_SPEND), + (DimensionName.AGENT, DeliveryTarget.DAILY_AGENT_SPEND), + (DimensionName.TAG, DeliveryTarget.DAILY_TAG_SPEND), + (DimensionName.MODEL, DeliveryTarget.DAILY_MODEL_USAGE), + (DimensionName.TOOL, DeliveryTarget.DAILY_TOOL_SPEND), +) + + +def usage_delivery_targets(dimensions: tuple[Dimension, ...]) -> tuple[DeliveryTarget, ...]: + names: Final = frozenset(dimension.name for dimension in dimensions) + return tuple(dict.fromkeys(target for dimension, target in _DAILY_TARGETS if dimension in names)) + + +class UsageCaptureSettings(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + batch_size: int = Field(default=1000, ge=2, le=1000) + flush_interval_seconds: float = Field(default=5.0, gt=0, le=10) + max_pending_bytes: int = Field(default=16 * 1024 * 1024, ge=1024) + max_pending_records: int = Field(default=10000, ge=2) + + +class UsageCaptureError(RuntimeError): + pass + + +@dataclass(frozen=True, slots=True) +class _Pending: + writes: tuple[UsageWrite, ...] + size: int + committed: Future[None] + finished: asyncio.Event + prerequisites: tuple[asyncio.Future[None], ...] + + +async def _wait_for_batch(wakeup: asyncio.Event, timeout: float) -> None: + try: + await asyncio.wait_for(wakeup.wait(), timeout) + except asyncio.TimeoutError: + return + + +def _report_failure(receipt: Future[None]) -> None: + if not receipt.cancelled() and receipt.exception() is not None: + verbose_proxy_logger.error("Usage capture did not acknowledge commit", exc_info=receipt.exception()) + + +def _retryable_persistence_failure(error: Exception) -> bool: + from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain + + outage: Final = PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(error) + return (outage is not None and not PrismaDBExceptionHandler.is_permanent_database_fault(outage)) or any( + isinstance(cause, Exception) + and ( + PrismaDBExceptionHandler.is_deadlock_error(cause) + or PrismaDBExceptionHandler.is_lock_timeout_error(cause) + or PrismaDBExceptionHandler.postgres_sqlstate(cause) == "40001" + ) + for cause in exception_chain(error) + ) + + +def _isolatable_persistence_failure(error: Exception) -> bool: + from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain + + return isinstance(error, AccountingConflictError) or any( + isinstance(cause, Exception) and (PrismaDBExceptionHandler.postgres_sqlstate(cause) or "").startswith("22") + for cause in exception_chain(error) + ) + + +class DurableUsageMeter: + def __init__( + self, + write_batch: Callable[[tuple[UsageWrite, ...]], Awaitable[None]], + settings: UsageCaptureSettings, + *, + wait_for_batch: Callable[[asyncio.Event, float], Awaitable[None]] = _wait_for_batch, + retry_delay: Callable[[float], Awaitable[None]] = asyncio.sleep, + ) -> None: + self._write_batch: Final = write_batch + self.settings: Final = settings + self._wait_for_batch: Final = wait_for_batch + self._retry_delay: Final = retry_delay + self._loop: Final = asyncio.get_running_loop() + self._admission: Final = Lock() + self._queue: Final[deque[_Pending]] = deque() # mutable-ok: one bounded producer/consumer queue + self._wakeup: Final = asyncio.Event() + self._task: asyncio.Task[None] | None = None + self._closed = False + self.pending_bytes = 0 + self.pending_records = 0 + + def submit( + self, writes: tuple[UsageWrite, ...], *, prerequisites: tuple[asyncio.Future[None], ...] = () + ) -> asyncio.Future[None]: + size: Final = sum(len(write.envelope.encode("utf-8")) for write in writes) + with self._admission: + if ( + self._closed + or not writes + or len(writes) > self.settings.batch_size + or self.pending_bytes + size > self.settings.max_pending_bytes + or self.pending_records + len(writes) > self.settings.max_pending_records + ): + raise UsageCaptureError("Usage capture admission rejected: closed, empty or capacity exhausted") + receipt: Final = Future[None]() + receipt.set_running_or_notify_cancel() + receipt.add_done_callback(_report_failure) + self._queue.append(_Pending(writes, size, receipt, asyncio.Event(), prerequisites)) + self.pending_bytes += size + self.pending_records += len(writes) + self._loop.call_soon_threadsafe(self._ensure_worker) + wrapped: Final = asyncio.wrap_future(receipt, loop=self._loop) + wrapped.add_done_callback(lambda done: None if done.cancelled() else done.exception()) + return wrapped + + def _ensure_worker(self) -> None: + if self.pending_records >= self.settings.batch_size or self._queue and self._queue[0].prerequisites: + self._wakeup.set() + if self._task is None or self._task.done(): + self._task = asyncio.create_task(self._run(), name="accounting usage capture") + + async def record(self, event: InboundUsageEvent | OutboundUsageEvent) -> None: + envelope: Final = '{"event":' + event.model_dump_json() + "}" + await asyncio.shield(self.submit((UsageWrite(event, envelope, usage_delivery_targets(event.dimensions)),))) + + async def flush(self) -> None: + self._ensure_worker() + self._wakeup.set() + if self._task is not None: + await asyncio.shield(self._task) + + async def close(self) -> None: + with self._admission: + self._closed = True + await self.flush() + + async def stop(self, timeout: float) -> None: + with self._admission: + self._closed = True + try: + await asyncio.wait_for(self.close(), timeout) + except asyncio.TimeoutError: + if self._task is not None: + self._task.cancel() + await asyncio.gather(self._task, return_exceptions=True) + while self._queue: + self._finish( + self._take_batch(), UsageCaptureError("Usage capture stopped before commit acknowledgement") + ) + verbose_proxy_logger.error("Usage capture shutdown deadline exhausted; commit acknowledgement incomplete") + + def _take_batch(self) -> tuple[_Pending, ...]: + with self._admission: + if self._queue[0].prerequisites: + return (self._queue.popleft(),) + independent: Final = takewhile(lambda item: not item.prerequisites, self._queue) + sizes: Final = accumulate(len(item.writes) for item in independent) + count: Final = sum(1 for _ in takewhile(lambda size: size <= self.settings.batch_size, sizes)) + return tuple(self._queue.popleft() for _ in range(count)) + + async def _persist(self, writes: tuple[UsageWrite, ...]) -> None: + for attempt in count(): + try: + await self._write_batch(writes) + except Exception as error: # noqa: BLE001 # retry only classified database failures, preserving frozen writes + if not _retryable_persistence_failure(error): + raise + verbose_proxy_logger.warning("Usage persistence unavailable; retaining admitted batch for retry") + await self._retry_delay(min(10.0, 0.2 * 2 ** min(attempt, 6))) + else: + return + + async def _commit(self, batch: tuple[_Pending, ...]) -> None: + writes: Final = tuple(chain.from_iterable(item.writes for item in batch)) + try: + if batch[0].prerequisites: + outcomes: Final = await asyncio.gather( + *(asyncio.shield(receipt) for receipt in batch[0].prerequisites), return_exceptions=True + ) + for outcome in outcomes: + if isinstance(outcome, BaseException): + raise UsageCaptureError("Usage dependency did not acknowledge commit") from outcome + await self._persist(writes) + except asyncio.CancelledError: + self._finish(batch, UsageCaptureError("Usage commit acknowledgement interrupted")) + raise + except Exception as error: # noqa: BLE001 # isolate data faults, surface systemic failures to all producers + if len(batch) > 1 and _isolatable_persistence_failure(error): + try: + for item in batch: + await self._commit((item,)) + except asyncio.CancelledError: + self._finish(batch, UsageCaptureError("Usage commit acknowledgement interrupted")) + raise + return + self._finish( + batch, + UsageCaptureError("Usage identity conflicts with accepted content") + if isinstance(error, AccountingConflictError) + else error, + ) + else: + self._finish(batch, None) + + def _finish(self, batch: tuple[_Pending, ...], error: Exception | None) -> None: + for item in batch: + if item.finished.is_set(): + continue + item.finished.set() + with self._admission: + self.pending_bytes -= item.size + self.pending_records -= len(item.writes) + if item.committed.done(): + continue + if error is None: + item.committed.set_result(None) + else: + item.committed.set_exception(error) + + async def _run(self) -> None: + try: + while self._queue: + await self._wait_for_batch(self._wakeup, self.settings.flush_interval_seconds) + self._wakeup.clear() + await self._commit(self._take_batch()) + if ( + self._closed + or self.pending_records >= self.settings.batch_size + or self._queue + and self._queue[0].prerequisites + ): + self._wakeup.set() + except (Exception, asyncio.CancelledError): # noqa: BLE001 # reject queued receipts if the writer exits abnormally + while self._queue: + self._finish( + self._take_batch(), UsageCaptureError("Usage capture stopped before commit acknowledgement") + ) + raise diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 9e237ffcb57..c7efbf9dacf 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3031,14 +3031,21 @@ class ProxyLogging: hook to consult, but it judges the payload the same way, so a proxy configured only with one of those still has something to say about every record. """ - if request_metadata.get("_guardrail_pipelines"): + return self.request_has_pre_call_guardrails({"metadata": dict(request_metadata)}, event_type) + + def request_has_pre_call_guardrails( + self, + request_data: Mapping[str, object], + event_type: GuardrailEventHooks = GuardrailEventHooks.pre_call, + ) -> bool: + if _policy_state_metadata(request_data).get("_guardrail_pipelines"): return True caps: Final = ProxyLogging.callback_capabilities() if caps.has_content_enforcer: return True - probe: Final = {"metadata": dict(request_metadata)} return any( - isinstance(callback, CustomGuardrail) and callback.should_run_guardrail(data=probe, event_type=event_type) + isinstance(callback, CustomGuardrail) + and callback.should_run_guardrail(data=request_data, event_type=event_type) for callback in caps.resolved_callbacks ) diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index 26d63db00cd..04fe027cbcc 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -34,6 +34,14 @@ from litellm.types.utils import ModelResponse _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object]) +def _chat_verbosity(responses_api_request: Mapping[str, object]) -> Mapping[str, object]: + match responses_api_request.get("text"): + case {"verbosity": str() as verbosity}: + return {"verbosity": verbosity} + case _: + return {} + + @dataclass(frozen=True, slots=True) class _BridgedRequest: input: str | ResponseInputParam @@ -96,8 +104,8 @@ class LiteLLMCompletionTransformationHandler: bridged: Final = _bridged_request(hoisted, responses_api_request, ()) bridged_input: Final = bridged.input bridged_request: Final = bridged.responses_api_request - litellm_completion_request: Final[dict] = ( - LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + litellm_completion_request: Final[dict] = { + **LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( model=model, input=bridged_input, responses_api_request=bridged_request, @@ -105,8 +113,9 @@ class LiteLLMCompletionTransformationHandler: stream=stream, extra_headers=extra_headers, **kwargs, - ) - ) + ), + **_chat_verbosity(bridged_request), + } if _is_async: return self.async_response_api_handler( @@ -173,8 +182,8 @@ class LiteLLMCompletionTransformationHandler: return await self.async_response_api_handler( previous_response_client_ids, stored_turns, - litellm_completion_request=( - LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + litellm_completion_request={ + **LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( model=model, input=bridged.input, responses_api_request=bridged.responses_api_request, @@ -182,8 +191,9 @@ class LiteLLMCompletionTransformationHandler: stream=stream, extra_headers=extra_headers, **request_kwargs, - ) - ), + ), + **_chat_verbosity(bridged.responses_api_request), + }, request_input=bridged.input, responses_api_request=bridged.responses_api_request, **request_kwargs, diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 4ac0d9f1257..165981f9294 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -8,7 +8,7 @@ import uuid from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from functools import partial -from itertools import accumulate, chain, groupby, islice +from itertools import chain, groupby from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -45,7 +45,15 @@ from litellm.constants import ( REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER, ) -from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook +from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + block_with_cache_control, + cache_control_count, + dropped_cache_control, + newest_marks_within_cache_control_limit, + whole_messages_within_cache_control_limit, + with_cache_control, +) from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, normalize_drop_params from litellm.litellm_core_utils.get_supported_openai_params import ( get_supported_openai_params, @@ -605,10 +613,6 @@ def _validated_cache_control(value: object) -> ChatCompletionCachedContent | Non return None -def _dropped_cache_control(_: object) -> None: - return None - - def _without_keys(item: object, keys: frozenset[str]) -> object: if not isinstance(item, dict): return item @@ -707,83 +711,6 @@ def _content_blocks_carrying(content: object, cache_control: ChatCompletionCache return _with_cache_control_on_last_block(_OBJECT_LIST_ADAPTER.validate_python(content), cache_control) -_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls") - - -def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]: - lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS) - return tuple( - chain.from_iterable(_OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list)) - ) - - -def _block_carries_cache_control(block: object) -> bool: - return isinstance(block, dict) and _STR_KEY_DICT_ADAPTER.validate_python(block).get("cache_control") is not None - - -def _cache_control_count(message: object) -> int: - if not isinstance(message, dict): - return 0 - fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message) - block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block)) - return int(fields.get("cache_control") is not None) + block_marks - - -_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None] - - -def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]: - others: Final = {key: value for key, value in fields.items() if key != "cache_control"} - if "cache_control" not in fields: - return others - cache_control: Final = rewrite(fields["cache_control"]) - return others if cache_control is None else {**others, "cache_control": cache_control} - - -def _block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object: - if not isinstance(block, dict): - return block - return _fields_with_cache_control(_STR_KEY_DICT_ADAPTER.validate_python(block), rewrite) - - -def _with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT: - if not isinstance(message, dict): - return message - fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message) - blocks_rewritten: Final = { - key: ( - [_block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)] - if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list) - else value - ) - for key, value in fields.items() - } - return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped - _MessageT, _fields_with_cache_control(blocks_rewritten, rewrite) - ) - - -def _kept_within_budget(state: tuple[int, bool], marks: int) -> tuple[int, bool]: - remaining, _ = state - if marks > remaining: - return remaining, False - return remaining - marks, True - - -def _messages_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]: - budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0) - newest_first: Final = tuple(reversed(messages)) - budget_after_each: Final = accumulate( - (_cache_control_count(message) for message in newest_first), _kept_within_budget, initial=(budget, True) - ) - kept_flags: Final = tuple(kept for _, kept in islice(budget_after_each, 1, None)) - kept_newest_first: Final = tuple( - message if kept else _with_cache_control(message, _dropped_cache_control) - for message, kept in zip(newest_first, kept_flags) - ) - return tuple(reversed(kept_newest_first)) - - _SYSTEM_MESSAGE_FIELDS: Final = frozenset(("content", "cache_control")) @@ -812,7 +739,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT: if fields is None: return message if fields.get("role") == "tool": - return _with_cache_control(message, _dropped_cache_control) + return with_cache_control(message, dropped_cache_control) tool_calls: Final = fields.get("tool_calls") if not isinstance(tool_calls, list): return message @@ -821,7 +748,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT: { **fields, "tool_calls": [ - _block_with_cache_control(tool_call, _dropped_cache_control) + block_with_cache_control(tool_call, dropped_cache_control) for tool_call in _OBJECT_LIST_ADAPTER.validate_python(tool_calls) ], }, @@ -835,7 +762,7 @@ class _ItemMarkRules: external_marks: int def sanitized(self, message: _MessageT) -> _MessageT: - valid: Final = _with_cache_control(message, _validated_cache_control) + valid: Final = with_cache_control(message, _validated_cache_control) return _without_tool_block_marks(valid) if self.refuses_tool_block_marks else valid def checked(self, message: _MessageT) -> _MessageT: @@ -858,15 +785,15 @@ def _request_messages_for_cache_points(messages: Sequence[_MessageT], rules: _It as_system: Final = tuple(_developer_message_as_system_message(message) for message in messages) if not rules.drop_params: return as_system - return _messages_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks) + return newest_marks_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks) def _replayed_messages_for_cache_points( replayed_messages: Sequence[_MessageT], current_messages: Sequence[object], rules: _ItemMarkRules ) -> Sequence[_MessageT]: as_system: Final = tuple(_developer_message_as_system_message(message) for message in replayed_messages) - current_marks: Final = sum(_cache_control_count(message) for message in current_messages) - return _messages_within_cache_control_limit( + current_marks: Final = sum(cache_control_count(message) for message in current_messages) + return whole_messages_within_cache_control_limit( tuple(map(rules.sanitized, as_system)), current_marks + rules.external_marks ) diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index c88a8c1cc28..1104aee353e 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -23,6 +23,7 @@ from litellm.responses.mcp.tool_precheck import ( LITELLM_PROXY_MCP_SERVER_URL_PREFIX, may_name_litellm_mcp_gateway, names_gateway_explicitly, + names_whole_gateway, proxy_path_mcp_name, ) from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator @@ -182,8 +183,8 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod async def _apply_toolset_permissions( - resolved_toolset_ids: list[str], - resolved_mcp_servers: list[str], + resolved_toolset_ids: Sequence[str], + resolved_mcp_servers: Sequence[str], user_api_key_auth: "UserAPIKeyAuth", ) -> "UserAPIKeyAuth": """Apply resolved toolset permissions to user_api_key_auth and return updated auth.""" @@ -195,7 +196,7 @@ class LiteLLM_Proxy_MCP_Handler: ) tool_permissions: Final = await global_mcp_server_manager.resolve_toolset_tool_permissions( - toolset_ids=resolved_toolset_ids + toolset_ids=list(resolved_toolset_ids) ) all_server_ids: Final = list(set(tool_permissions.keys()) | set(resolved_mcp_servers)) existing_op: Final = user_api_key_auth.object_permission @@ -224,6 +225,62 @@ class LiteLLM_Proxy_MCP_Handler: verbose_logger.debug("Could not apply toolset permissions: %s", _e) return user_api_key_auth + @staticmethod + async def _resolve_proxy_mcp_name( + name: str, + user_api_key_auth: "UserAPIKeyAuth | None", + granted_toolsets: "GrantedToolsetIds | None", + ) -> tuple[Literal["server", "toolset", "denied_toolset"], str]: + from litellm.proxy.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + + if global_mcp_server_manager.get_mcp_server_by_name(name): + return ("server", name) + try: + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + return ("server", name) + toolset: Final = await global_mcp_server_manager.get_toolset_by_name_cached(prisma_client, name) + if toolset is None: + return ("server", name) + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( + has_team_unbound_mcp_admin_view, + ) + from litellm.proxy.mcp_server.ui_session_utils import ( + granted_toolset_ids, + ) + + denied: Final = ( + user_api_key_auth is not None + and not has_team_unbound_mcp_admin_view(user_api_key_auth) + and toolset.toolset_id not in await (granted_toolsets or granted_toolset_ids)(user_api_key_auth) + ) + except Exception as _e: + verbose_logger.debug("Could not resolve '%s' as toolset: %s", name, _e) + return ("server", name) + if not denied: + return ("toolset", toolset.toolset_id) + if await LiteLLM_Proxy_MCP_Handler._holds_access_group_named(name, user_api_key_auth): + return ("server", name) + verbose_logger.debug("Key does not have access to toolset '%s', denying it.", name) + return ("denied_toolset", name) + + @staticmethod + async def _holds_access_group_named(name: str, user_api_key_auth: "UserAPIKeyAuth | None") -> bool: + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( + MCPRequestHandler, + ) + from litellm.proxy.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + + group_server_ids: Final = await MCPRequestHandler.get_mcp_servers_from_access_groups([name]) + return bool(group_server_ids) and not frozenset(group_server_ids).isdisjoint( + await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth) + ) + @staticmethod @catalog_operation(global_manager) async def _get_mcp_tools_from_manager( @@ -256,6 +313,7 @@ class LiteLLM_Proxy_MCP_Handler: from litellm.proxy.mcp_server.server import ( _get_allowed_mcp_servers_from_mcp_server_names, _get_tools_from_mcp_servers, + raise_generic_mcp_access_denial, ) mcp_servers: Final = [ @@ -268,36 +326,15 @@ class LiteLLM_Proxy_MCP_Handler: # Resolve toolset names: collect all toolset IDs first, then apply their # combined permissions in a single pass so multiple toolsets are unioned # rather than the last one overwriting the others. - resolved_mcp_servers: Final[list[str]] = [] - resolved_toolset_ids: Final[list[str]] = [] - for name in mcp_servers: - if not global_mcp_server_manager.get_mcp_server_by_name(name): - try: - from litellm.proxy.proxy_server import prisma_client - - if prisma_client is not None: - toolset = await global_mcp_server_manager.get_toolset_by_name_cached(prisma_client, name) - if toolset is not None: - if user_api_key_auth is not None: - from litellm.proxy.management_endpoints.common_utils import ( - user_api_key_has_admin_view, - ) - from litellm.proxy.mcp_server.ui_session_utils import ( - granted_toolset_ids, - ) - - if not user_api_key_has_admin_view(user_api_key_auth) and toolset.toolset_id not in ( - await (granted_toolsets or granted_toolset_ids)(user_api_key_auth) - ): - verbose_logger.debug("Key does not have access to toolset '%s', skipping.", name) - continue - resolved_toolset_ids.append(toolset.toolset_id) - # Don't add to resolved_mcp_servers — toolset scope - # restricts via object_permission, not server name filter. - continue - except Exception as _e: - verbose_logger.debug("Could not resolve '%s' as toolset: %s", name, _e) - resolved_mcp_servers.append(name) + resolutions: Final = tuple( + [ + await LiteLLM_Proxy_MCP_Handler._resolve_proxy_mcp_name(name, user_api_key_auth, granted_toolsets) + for name in mcp_servers + ] + ) + resolved_mcp_servers: Final = tuple(value for kind, value in resolutions if kind == "server") + resolved_toolset_ids: Final = tuple(value for kind, value in resolutions if kind == "toolset") + denied_toolset_names: Final = tuple(value for kind, value in resolutions if kind == "denied_toolset") # Apply all resolved toolsets at once (union), avoiding permission overwrite. if resolved_toolset_ids and user_api_key_auth is not None: @@ -311,7 +348,10 @@ class LiteLLM_Proxy_MCP_Handler: # full union (toolset server IDs + direct server names). Passing a name-based # filter here would exclude those toolset server IDs (which are UUIDs, not # names), so use None and let the auth object's mcp_servers do the filtering. - effective_server_filter: Final = None if resolved_toolset_ids else (resolved_mcp_servers or None) + asks_whole_gateway: Final = any(map(names_whole_gateway, mcp_tools_with_litellm_proxy or ())) + if denied_toolset_names and not (asks_whole_gateway or resolved_mcp_servers or resolved_toolset_ids): + raise_generic_mcp_access_denial(denied_toolset_names) + effective_server_filter: Final = None if resolved_toolset_ids else (list(resolved_mcp_servers) or None) listing: Final = await _get_tools_from_mcp_servers( user_api_key_auth=user_api_key_auth, diff --git a/litellm/responses/mcp/tool_precheck.py b/litellm/responses/mcp/tool_precheck.py index a29e6895f9a..a67d81e70ae 100644 --- a/litellm/responses/mcp/tool_precheck.py +++ b/litellm/responses/mcp/tool_precheck.py @@ -22,6 +22,13 @@ def names_gateway_explicitly(tool: object) -> bool: return (_mcp_server_url(tool) or "").startswith(LITELLM_PROXY_MCP_SERVER_URL) +def names_whole_gateway(tool: object) -> bool: + server_url: Final = _mcp_server_url(tool) or "" + return server_url.startswith(LITELLM_PROXY_MCP_SERVER_URL) and not server_url.startswith( + LITELLM_PROXY_MCP_SERVER_URL_PREFIX + ) + + def proxy_path_mcp_name(tool: object) -> str | None: server_url: Final = _mcp_server_url(tool) match: Final = None if server_url is None else _PROXY_MCP_PATH_RE.match(server_url) diff --git a/litellm/router.py b/litellm/router.py index 381faf350ff..af670d8a681 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -637,6 +637,7 @@ _PRE_CALL_CHECK_FILTERED_PARAMS: Final = frozenset({"response_format"}) _STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS: Final = frozenset({"anthropic_messages", "aresponses", "responses"}) _video_call_deployment_id: Final[ContextVar[str | None]] = ContextVar("video_call_deployment_id", default=None) _routed_call_type: Final[ContextVar[str | None]] = ContextVar("routed_call_type", default=None) +_DECISIONS_MEDIA_CALL_TYPES: Final = frozenset({"adecisions", "decisions", "asystemone", "systemone"}) _ROUTE_CAPABILITY_RESOLVERS: Final[Mapping[str, Callable[[str, LlmProviders], object | None]]] = MappingProxyType( { "adecisions": ProviderConfigManager.get_provider_decisions_config, @@ -4184,13 +4185,16 @@ class Router(DeprecatedRouterMembers): raise e.original_exception from e raise e + def _active_default_litellm_params(self) -> Mapping[str, object]: + default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self._core_default_litellm_params) + return {key: value for key, value in default_params.items() if value is not None} + def _params_sent_to_deployment( self, request_kwargs: Mapping[str, object], deployment: Deployment | None ) -> Mapping[str, object]: """The request as the deployment received it, merged in the order _update_kwargs_with_deployment applies.""" deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {} - default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self._core_default_litellm_params) - defaults: Final = {key: value for key, value in default_params.items() if value is not None} + defaults: Final = self._active_default_litellm_params() deployment_tool_choice: Final = ( {"tool_choice": deployment_params["tool_choice"]} if deployment_params.get("tool_choice") is not None @@ -14949,6 +14953,62 @@ class Router(DeprecatedRouterMembers): return healthy_deployments return cast(_RouteDeploymentsT, filtered_deployments) # cast-ok: same list shape as the input candidates + @staticmethod + def _decisions_deployment_refuses_media( + deployment: DeploymentTypedDict, group: str, request_kwargs: Mapping[str, object], *, audio: bool, videos: bool + ) -> bool: + from litellm.decisions.call import deployment_refuses_media + + plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts + model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict + deployment_params: Final = cast(Mapping[str, object], deployment["litellm_params"]) # cast-ok: plain dict + try: + model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group) + return deployment_refuses_media( + model, + custom_llm_provider, + model_info, + {**deployment_params, **request_kwargs}, + audio=audio, + videos=videos, + ) + except Exception as e: # noqa: BLE001 # best-effort filter: a deployment it cannot resolve answers for itself + verbose_router_logger.debug( + "litellm.router.py::_decisions_deployment_refuses_media: keeping a deployment of model=%s. Got - %s", + group, + e, + ) + return False + + def _without_decisions_deployments_refusing_media( + self, healthy_deployments: _RouteDeploymentsT, group: str, request_kwargs: Mapping[str, object] | None + ) -> _RouteDeploymentsT: + call_kwargs: Final = request_kwargs or {} + audio: Final = bool(call_kwargs.get("audio")) + videos: Final = bool(call_kwargs.get("videos")) + if ( + _routed_call_type.get() not in _DECISIONS_MEDIA_CALL_TYPES + or not (audio or videos) + or isinstance(healthy_deployments, Mapping) + ): + return healthy_deployments + + sent_kwargs: Final = {**self._active_default_litellm_params(), **call_kwargs} + deployments_taking_media: Final = [ + deployment + for deployment in healthy_deployments + if not Router._decisions_deployment_refuses_media( + cast(DeploymentTypedDict, deployment), # cast-ok: router candidates use the deployment shape + group, + sent_kwargs, + audio=audio, + videos=videos, + ) + ] + if not deployments_taking_media: + return healthy_deployments + return cast(_RouteDeploymentsT, deployments_taking_media) # cast-ok: same list shape as the input candidates + @staticmethod def _without_declared_text_only_deployments( healthy_deployments: _RouteDeploymentsT, @@ -15673,10 +15733,14 @@ class Router(DeprecatedRouterMembers): ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) _target_order: Final = (request_kwargs or {}).pop("_target_order", None) - batch_candidates: Final = self._without_declared_text_only_deployments( - cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + batch_candidates: Final = self._without_decisions_deployments_refusing_media( + self._without_declared_text_only_deployments( + cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + model, + messages, + cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict + ), model, - messages, cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict ) embedding_batch_admitting: Final = deployments_admitting_embedding_batch( @@ -16916,10 +16980,14 @@ class Router(DeprecatedRouterMembers): ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) _target_order: Final = (request_kwargs or {}).pop("_target_order", None) - batch_candidates: Final = self._without_declared_text_only_deployments( - cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + batch_candidates: Final = self._without_decisions_deployments_refusing_media( + self._without_declared_text_only_deployments( + cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + model, + messages, + cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict + ), model, - messages, cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict ) embedding_batch_admitting: Final = deployments_admitting_embedding_batch( diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index a54d66c4beb..ee86ec52327 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -402,7 +402,7 @@ class OpenAIFileObject(LiteLLMBaseModel): filename: str """The name of the file.""" - object: Literal["file"] + object: Literal["file"] = "file" """The object type, which is always `file`.""" purpose: OpenAIFilesPurpose @@ -440,6 +440,11 @@ class OpenAIFileObject(LiteLLMBaseModel): _hidden_params: dict = PrivateAttr(default={"response_cost": 0.0}) # no cost for writing a file + @field_validator("object", mode="before") + @classmethod + def _default_missing_object_type(cls, value: _JsonValue) -> _JsonValue: + return "file" if value is None else value + @property def hidden_params(self) -> dict[str, builtins.object]: # mutable-ok: API requires mutation return self._hidden_params diff --git a/litellm/types/passthrough_endpoints/pass_through_endpoints.py b/litellm/types/passthrough_endpoints/pass_through_endpoints.py index 0d99cd54a6b..2f269dc87d9 100644 --- a/litellm/types/passthrough_endpoints/pass_through_endpoints.py +++ b/litellm/types/passthrough_endpoints/pass_through_endpoints.py @@ -1,5 +1,6 @@ +from dataclasses import dataclass from enum import Enum -from typing import Final +from typing import Final, Protocol from pydantic import Field from typing_extensions import TypedDict @@ -75,6 +76,19 @@ class PassthroughStandardLoggingPayload(TypedDict, total=False): """ +@dataclass(frozen=True, slots=True) +class WebSocketSessionUsage: + model: str + custom_llm_provider: str + response_cost: float | None + + +class WebSocketSessionMeter(Protocol): + def measure(self, client_text: str) -> int: ... + + def price(self, total: int) -> WebSocketSessionUsage: ... + + class PassThroughGuardrailSettings(LiteLLMPydanticObjectBase): """ Settings for a specific guardrail on a passthrough endpoint. diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 0c3440e88f7..24f4acaf5df 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -106,6 +106,7 @@ from .llms.openai import ( OpenAIRealtimeStreamList, ResponsesAPIResponse, WebSearchOptions, + _JsonValue, ) from .rerank import RerankResponse as RerankResponse @@ -4682,6 +4683,7 @@ class LlmProviders(str, Enum): PRISM = "prism" OPENINFER = "openinfer" INFERENCEAPIS = "inferenceapis" + HOSTINGER_ROUTER = "hostinger_router" ZEROGPU = "zerogpu" DARKBLOOM = "darkbloom" META = "meta" @@ -4963,29 +4965,50 @@ def _batch_usage_from_chat_usage(usage: Usage) -> BatchUsage: ) +OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS: Final = MappingProxyType( + {"pending": "validating", "running": "in_progress"} +) + + +def openai_batch_status(status: _JsonValue) -> _JsonValue: + return OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(status, status) if isinstance(status, str) else status + + class LiteLLMBatch(Batch): _hidden_params: dict = {} + object: Literal["batch"] = "batch" + + @field_validator("object", mode="before") + @classmethod + def _default_missing_object_type(cls, value: _JsonValue) -> _JsonValue: + return "batch" if value is None else value + + @field_validator("status", mode="before") + @classmethod + def _map_vllm_router_status(cls, value: _JsonValue) -> _JsonValue: + return openai_batch_status(value) + @property - def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation + def hidden_params(self) -> dict[str, _JsonValue]: # mutable-ok: API requires mutation return self._hidden_params @hidden_params.setter - def hidden_params(self, hidden_params: dict[str, object]) -> None: # mutable-ok: API requires mutation + def hidden_params(self, hidden_params: dict[str, _JsonValue]) -> None: # mutable-ok: API requires mutation self._hidden_params = hidden_params usage: Usage | None = None @field_validator("usage", mode="before") @classmethod - def _usage_from_batch_token_counts(cls, value: object) -> object: + def _usage_from_batch_token_counts(cls, value: _JsonValue) -> _JsonValue: counts: Final = _parse_batch_token_counts(value) return value if counts is None else _usage_with_batch_token_counts(Usage.model_validate(value), counts) @field_serializer("usage") def _serialize_usage_with_batch_fields( self, usage: Usage | None, info: FieldSerializationInfo - ) -> Mapping[str, object] | None: + ) -> Mapping[str, _JsonValue] | None: if not isinstance(usage, Usage): return usage chat_fields: Final = usage.model_dump( diff --git a/litellm/utils.py b/litellm/utils.py index 8fe3fd84fbd..f19fa0dbb34 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10159,7 +10159,7 @@ class ProviderConfigManager: from litellm.llms.openai_like.json_loader import JSONProviderRegistry json_provider: Final = JSONProviderRegistry.get(provider.value) - if json_provider is not None and "/v1/messages" in json_provider.supported_endpoints: + if json_provider is not None and JSONProviderRegistry.serves_endpoint(provider.value, model, "/v1/messages"): from litellm.llms.openai_like.messages.transformation import ( JSONProviderAnthropicMessagesConfig, ) @@ -10298,7 +10298,7 @@ class ProviderConfigManager: return result # Fall back to JSON providers (generic OpenAI-compatible) - if JSONProviderRegistry.exists(provider_str) and JSONProviderRegistry.supports_responses_api(provider_str): + if JSONProviderRegistry.serves_endpoint(provider_str, model, "/v1/responses"): provider_config: Final = JSONProviderRegistry.get(provider_str) if provider_config is not None: return create_responses_config_class(provider_config)() diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4d503952600..7eede4a395f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -39687,6 +39687,57 @@ "/v1/images/generations" ] }, + "hostinger_router/gpt-5.2": { + "cache_read_input_token_cost": 0.00000021, + "input_cost_per_token": 0.0000021, + "litellm_provider": "hostinger_router", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.0000168, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true, + "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "source": "https://router.hostinger.com/v1/models" + }, + "hostinger_router/claude-sonnet-5": { + "cache_creation_input_token_cost": 3.3e-06, + "cache_creation_input_token_cost_above_1hr": 5.28e-06, + "cache_read_input_token_cost": 2.64e-07, + "input_cost_per_token": 2.64e-06, + "litellm_provider": "hostinger_router", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.32e-05, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_response_schema": true, + "supported_endpoints": ["/v1/chat/completions", "/v1/messages"], + "source": "https://router.hostinger.com/v1/models" + }, + "hostinger_router/text-embedding-3-large": { + "input_cost_per_token": 1.716e-07, + "litellm_provider": "hostinger_router", + "max_input_tokens": 8191, + "mode": "embedding", + "output_cost_per_token": 0.0, + "source": "https://router.hostinger.com/v1/models" + }, + "hostinger_router/text-embedding-3-small": { + "input_cost_per_token": 3e-08, + "litellm_provider": "hostinger_router", + "max_input_tokens": 8191, + "mode": "embedding", + "output_cost_per_token": 0.0, + "source": "https://router.hostinger.com/v1/models" + }, "hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": { "input_cost_per_token": 1.2e-07, "litellm_provider": "hyperbolic", @@ -51387,6 +51438,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -51415,6 +51471,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -52161,6 +52222,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -52266,6 +52332,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, "regional_endpoint_uplift_multiplier": 1.1, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index d33e61026df..700a35edb06 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1087,6 +1087,32 @@ "generateContent": false } }, + "hostinger_router": { + "display_name": "Hostinger Router (`hostinger_router`)", + "url": "https://docs.litellm.ai/docs/providers/hostinger_router", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": true, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "crusoe": { "display_name": "Crusoe (`crusoe`)", "url": "https://docs.litellm.ai/docs/providers/crusoe", diff --git a/pyproject.toml b/pyproject.toml index 6e9e75a5b7f..5c23a64ebb3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -383,6 +383,8 @@ exclude-newer = "3 days" [tool.uv.exclude-newer-package] # TODO: remove after 2026-10-25, once brotli 1.2.1 (CVE-2026-111497) clears the 3-day window brotli = "2026-10-22T14:01:00Z" +# TODO: remove after 2026-10-31, once pypdf 6.21.4 clears the 3-day window +pypdf = "2026-10-28T17:01:00Z" [tool.uv.sources] litellm-proxy-extras = { workspace = true } diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 7df1cc8c1b0..1b1ee484afc 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -185,6 +185,9 @@ ignored_function_names = [ "_aresponses_background_stream_can_release", # Tested through the background Responses stream tests in test_router.py "_raise_released_stream_error", # Tested through the background Responses stream content-policy tests in test_router.py "_mid_stream_error_before_output", # Tested through the Responses mid-stream fallback and background stream tests + "_without_decisions_deployments_refusing_media", # Tested through the mixed media-flag group tests in tests/unit/decisions/test_main.py + "_decisions_deployment_refuses_media", # Read by _without_decisions_deployments_refusing_media, tested through the same group tests + "_active_default_litellm_params", # Tested through the router default drop_params test in tests/unit/decisions/test_main.py ] diff --git a/tests/integration/cost_calculation/cases/vertex_ai.py b/tests/integration/cost_calculation/cases/vertex_ai.py index 2fb005dbeca..8590a19646b 100644 --- a/tests/integration/cost_calculation/cases/vertex_ai.py +++ b/tests/integration/cost_calculation/cases/vertex_ai.py @@ -3,7 +3,7 @@ PARITY pairs the streamed case with its plain twin where both bill the same row.""" import json -from typing import Final, cast +from typing import Final, Literal, cast from pydantic import JsonValue @@ -41,6 +41,7 @@ from integration.cost_calculation.cases.anthropic import ( from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs from integration.cost_calculation.cost_tracking_case import ( CostTrackingTestCase, + Deployment, ExactExpected, JsonResponse, RecountExpected, @@ -3387,6 +3388,232 @@ VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS: Final = bridge_pairs( ) +VERTEX_AI_CLAUDE_DEPLOYMENT_RATES: Final = Deployment( + input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.01 +) + + +def _vertex_claude_web_search_message(web_search_requests: int) -> JsonResponse: + return JsonResponse( + content_type="application/json", + body={ + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": "scripted answer for cost fixtures"}], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "server_tool_use": {"web_search_requests": web_search_requests}, + }, + }, + ) + + +def _vertex_claude_web_search_stream(web_search_requests: int) -> SseResponse: + return SseResponse( + content_type="text/event-stream", + frames=sse_frames( + { + "type": "message_start", + "message": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 1840}, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "scripted answer for cost fixtures"}, + }, + {"type": "content_block_stop", "index": 0}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": 412, "server_tool_use": {"web_search_requests": web_search_requests}}, + }, + {"type": "message_stop"}, + ), + ) + + +def _vertex_claude_deployment_web_search_case( + name: str, + endpoint: Literal["/v1/chat/completions", "/v1/responses", "/v1/messages"], + request: dict[str, JsonValue], + response: JsonResponse | SseResponse, + tool_usage_cost: float, + deployment: Deployment = VERTEX_AI_CLAUDE_DEPLOYMENT_RATES, +) -> CostTrackingTestCase: + return CostTrackingTestCase( + name=name, + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="vertex_ai/claude-sonnet-5-5", + endpoint=endpoint, + deployment=deployment, + request={"model": "$MODEL", **request}, + response=response, + expected=ExactExpected( + spend=round(0.0039 + tool_usage_cost, 10), + input_cost=0.00184, + output_cost=0.00206, + prompt_tokens=1840, + completion_tokens=412, + tool_usage_cost=tool_usage_cost, + ), + ) + + +VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL: Final[dict[str, JsonValue]] = { + "type": "web_search_20250305", + "name": "web_search", + "max_uses": 3, +} + +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_rates_messages_web_search_two_requests", + "/v1/messages", + { + "max_tokens": 412, + "messages": [{"role": "user", "content": "vertex claude deployment rates messages web search"}], + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + }, + _vertex_claude_web_search_message(2), + 0.02, +) +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM: Final = _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_rates_messages_web_search_two_requests_stream", + "/v1/messages", + { + "max_tokens": 412, + "messages": [{"role": "user", "content": "vertex claude deployment rates messages web search stream"}], + "stream": True, + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + }, + _vertex_claude_web_search_stream(2), + 0.02, +) +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE: Final = ( + _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_zero_search_rate_messages_two_requests", + "/v1/messages", + { + "max_tokens": 412, + "messages": [{"role": "user", "content": "vertex claude deployment zero search rate messages"}], + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + }, + _vertex_claude_web_search_message(2), + 0.0, + Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.0), + ) +) +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY: Final = ( + _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_token_rates_only_messages_web_search_two_requests", + "/v1/messages", + { + "max_tokens": 412, + "messages": [{"role": "user", "content": "vertex claude deployment token rates only messages web search"}], + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + }, + _vertex_claude_web_search_message(2), + 0.034, + Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06), + ) +) +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_rates_chat_web_search_three_requests", + "/v1/chat/completions", + { + "messages": [{"role": "user", "content": "vertex claude deployment rates chat web search"}], + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + "allowed_openai_params": ["web_search_options"], + }, + _vertex_claude_web_search_message(3), + 0.03, +) +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH_STREAM: Final = _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_rates_chat_web_search_two_requests_stream", + "/v1/chat/completions", + { + "messages": [{"role": "user", "content": "vertex claude deployment rates chat web search stream"}], + "stream": True, + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + "allowed_openai_params": ["web_search_options"], + }, + _vertex_claude_web_search_stream(2), + 0.02, +) +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_RESPONSES_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_rates_responses_web_search_two_requests", + "/v1/responses", + {"input": "vertex claude deployment rates responses web search", "tools": [{"type": "web_search"}]}, + _vertex_claude_web_search_message(2), + 0.02, +) + +GEMINI_3_1_PRO_PREVIEW_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM: Final = CostTrackingTestCase( + name="vertex_ai/gemini-3.1-pro-preview-deployment_rates_messages_web_search_two_queries_stream", + covers="quota_management.spend_tracking.scripted_wire.logs_cost", + model="vertex_ai/gemini-3.1-pro-preview", + endpoint="/v1/messages", + deployment=Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.03), + request={ + "model": "$MODEL", + "max_tokens": 412, + "messages": [{"role": "user", "content": "vertex gemini deployment rates messages web search stream"}], + "stream": True, + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + }, + response=SseResponse( + content_type="text/event-stream", + frames=sse_frames( + { + "candidates": [ + { + "content": {"parts": [{"text": "scripted answer for cost fixtures"}], "role": "model"}, + "index": 0, + } + ], + "modelVersion": "gemini-3.1-pro-preview", + }, + { + "candidates": [ + { + "content": {"parts": [{"text": ""}], "role": "model"}, + "finishReason": "STOP", + "index": 0, + "groundingMetadata": {"webSearchQueries": ["query 0", "query 1"]}, + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}], + }, + "modelVersion": "gemini-3.1-pro-preview", + }, + ), + ), + expected=ExactExpected( + spend=0.0339, + input_cost=0.00184, + output_cost=0.00206, + prompt_tokens=1840, + completion_tokens=412, + tool_usage_cost=0.03, + ), +) + GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT: Final = _gemini_case( GEMINI_3_8_FLASH_INPUT_TEXT, "vertex_ai/gemini-3.1-pro-preview-input_text", "vertex_ai/gemini-3.1-pro-preview", ExactExpected(spend=0.0094864, input_cost=0.004048, output_cost=0.0054384, prompt_tokens=1840, completion_tokens=412), @@ -3725,6 +3952,14 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT, VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_STREAM, *(pair.bridged for pair in VERTEX_AI_CLAUDE_SONNET_5_5_BRIDGE_PAIRS), + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH, + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM, + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE, + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY, + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH, + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH_STREAM, + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_RESPONSES_WEB_SEARCH, + GEMINI_3_1_PRO_PREVIEW_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM, GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT, GEMINI_3_1_PRO_PREVIEW_STREAM, GEMINI_3_1_PRO_PREVIEW_CACHE_READ, diff --git a/tests/integration/database/test_accounting_ledger.py b/tests/integration/database/test_accounting_ledger.py index 0c2ceb62eaf..86c2b325226 100644 --- a/tests/integration/database/test_accounting_ledger.py +++ b/tests/integration/database/test_accounting_ledger.py @@ -32,7 +32,7 @@ from litellm.accounting import ( PricingSnapshot, ) from litellm.proxy.db import accounting_ledger -from litellm.proxy.db.accounting_ledger import AccountingConflictError, AccountingEvent, AccountingLedger +from litellm.proxy.db.accounting_ledger import AccountingConflictError, AccountingEvent, AccountingLedger, UsageWrite from litellm.types.utils import CallTypes from tests.integration._support.database import scratch_database from tests.integration._support.prisma_clients import connected_proxy_prisma @@ -532,3 +532,74 @@ def test_family_cleanup_lookups_use_the_kind_source_index(ledger_database: str, name == "LiteLLM_AccountingEvent_kind_source_event_id_idx" and "source_event_id" in str(condition) for name, condition in conditions ), conditions + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", ["before_commit", "after_commit", "conflict"]) +async def test_usage_batch_is_atomic_and_replay_preserves_receipts( + writer: _Writer, ledger_database: str, failure: str +) -> None: + client, store = writer + events: Final = (_usage("caller", inbound=True), _usage("attempt")) + writes: Final = tuple(UsageWrite(event, _envelope(event), _TARGETS) for event in events) + + @asynccontextmanager + async def interrupted() -> AsyncGenerator[Prisma]: + async with client.writer_db.tx() as tx: + yield tx + if failure == "before_commit": + raise RuntimeError("injected pre-commit failure") + if failure == "after_commit": + raise RuntimeError("injected lost acknowledgement") + + if failure == "conflict": + await store.insert_usage_batch((writes[1],)) + _complete_and_age(ledger_database) + rejected: Final = (writes[0], UsageWrite(events[1], _envelope(events[1], context={"changed": True}), _TARGETS)) + with pytest.raises(AccountingConflictError): + await store.insert_usage_batch(rejected) + assert _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent"') == [("attempt",)] + else: + with pytest.raises(RuntimeError, match="injected"): + await AccountingLedger(interrupted).insert_usage_batch(writes) + count: Final = 2 if failure == "after_commit" else 0 + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(count,)] + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery"') == [(count * len(_TARGETS),)] + await store.insert_usage_batch(writes) + recovered: Final = await store.load_usage_batch(tuple(event.event_id for event in events)) + assert len(recovered) == 1 + assert recovered[0].event == events[1] + assert json.loads(recovered[0].envelope) == json.loads(writes[1].envelope) + assert set(recovered[0].targets) == set(_TARGETS) + _complete_and_age(ledger_database) + before: Final = _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target') + await store.insert_usage_batch((*writes, *writes)) + assert _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target') == before + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(2,)] + + +@pytest.mark.asyncio +async def test_invalid_jsonb_submission_cannot_reject_another_producers_commit( + writer: _Writer, ledger_database: str +) -> None: + from prisma.errors import DataError + + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + + _, store = writer + invalid: Final = _usage("invalid") + valid: Final = _usage("valid") + meter: Final = DurableUsageMeter(store.insert_usage_batch, UsageCaptureSettings(batch_size=2)) + rejected: Final = meter.submit( + (UsageWrite(invalid, _envelope(invalid, context={"parent": "bad\x00text"}), _TARGETS),) + ) + accepted: Final = meter.submit((UsageWrite(valid, _envelope(valid), _TARGETS),)) + try: + with pytest.raises(DataError): + await rejected + await accepted + finally: + await meter.close() + assert _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent"') == [("valid",)] + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery"') == [(len(_TARGETS),)] + assert (meter.pending_bytes, meter.pending_records) == (0, 0) diff --git a/tests/integration/providers/test_anthropic_passthrough_wire.py b/tests/integration/providers/test_anthropic_passthrough_wire.py index a791eb9c2a2..0a16520f626 100644 --- a/tests/integration/providers/test_anthropic_passthrough_wire.py +++ b/tests/integration/providers/test_anthropic_passthrough_wire.py @@ -317,7 +317,6 @@ def test_credentialless_proxy_forwards_the_caller_anthropic_credential_and_never def test_anthropic_sdk_metadata_user_id_reaches_upstream(api_key_proxy: Gateway, wire: Wire) -> None: - pytest.skip("BUG: /anthropic/v1/messages drops the native metadata.user_id field before forwarding upstream") with api_key_proxy.scenario() as scenario: key: Final = scenario.key() sent: Final[list[httpx.Request]] = [] diff --git a/tests/integration/providers/test_decisions_media_deployment_flag_wire.py b/tests/integration/providers/test_decisions_media_deployment_flag_wire.py new file mode 100644 index 00000000000..a2e53ef47ca --- /dev/null +++ b/tests/integration/providers/test_decisions_media_deployment_flag_wire.py @@ -0,0 +1,123 @@ +import json +import uuid +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final, Literal + +import httpx +import pytest +from integration._support.client import Gateway, eventually, object_value +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_API_KEY: Final = "synthetic-decisions-key" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_INPUT: Final = "Listen to the attached clip." +_AUDIO: Final = ("data:audio/wav;base64,UklGRiQAAABXQVZF",) +_VIDEOS: Final = ("data:video/mp4;base64,AAAAIGZ0eXBpc29t",) +_QUESTION: Final[dict[str, JsonValue]] = { + "type": "predicate", + "name": "banana", + "instructions": "Does the clip name a banana?", +} +_SPEND_QUERY: Final = 'SELECT status, model_group FROM "LiteLLM_SpendLogs" WHERE request_id = %s' + +Route = Literal["/v1/decisions", "/v1/systemone"] +Media = Literal["audio", "videos"] + + +@dataclass(frozen=True, slots=True) +class _Upstream: + model: str + + def reply(self, request: Request) -> Reply: + questions: Final = object_value(_JSON_OBJECT.validate_json(request.body)["questions"]) + plain: Final[dict[str, JsonValue]] = { + "model": self.model, + "answers": {name: {"type": "noul", "noul": 0.9} for name in questions}, + "usage": {"input_tokens": 41, "output_tokens": 0}, + } + return Reply(body=json.dumps(plain).encode()) + + +def _unlisted_openrouter_model() -> _Upstream: + return _Upstream(f"openrouter/example/omni-{uuid.uuid4().hex[:12]}") + + +def _decide(gateway: Gateway, model: str, route: Route, media: Mapping[str, JsonValue]) -> httpx.Response: + body: Final[dict[str, JsonValue]] = ( + {"model": model, "input": _INPUT, "questions": [_QUESTION], "num_retries": 0, **media} + if route == "/v1/decisions" + else { + "model": model, + "state": _INPUT, + "questions": {"banana": {"type": "noul", "instructions": _QUESTION["instructions"]}}, + "num_retries": 0, + **media, + } + ) + return gateway.request("POST", route, body) + + +def _sent_media(request: Request) -> dict[str, JsonValue]: + body: Final = _JSON_OBJECT.validate_json(request.body) + return {kind: body[kind] for kind in ("audio", "videos") if kind in body} + + +def _assert_refused(response: httpx.Response, kinds: tuple[Media, ...]) -> None: + assert response.status_code == 400, response.text + assert f"does not support parameters: {list(kinds)}" in response.text, response.text + + +_BOTH: Final[dict[str, JsonValue]] = {"audio": list(_AUDIO), "videos": list(_VIDEOS)} + + +@pytest.mark.parametrize("route", ("/v1/decisions", "/v1/systemone")) +def test_a_deployment_flagged_for_audio_and_video_sends_both_clips_for_a_model_the_cost_map_does_not_list( + gateway: Gateway, route: Route +) -> None: + upstream: Final = _unlisted_openrouter_model() + with wire_server(upstream.reply) as wire, gateway.scenario() as scenario: + flags: Final[dict[str, JsonValue]] = {"supports_audio_input": True, "supports_video_input": True} + flagged: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info=flags) + flagged_dropping: Final = scenario.model( + model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info=flags, drop_params=True + ) + audio_only: Final = scenario.model( + model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info={"supports_audio_input": True} + ) + unflagged: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY) + + _assert_refused(_decide(gateway, unflagged, route, _BOTH), ("audio", "videos")) + _assert_refused(_decide(gateway, audio_only, route, _BOTH), ("videos",)) + assert wire.drain() == () + + answered: Final = _decide(gateway, flagged, route, _BOTH) + assert answered.status_code == 200, answered.text + (call,) = wire.drain() + assert _sent_media(call) == _BOTH + rows: Final = eventually( + lambda: read_rows(_SPEND_QUERY, (answered.headers["x-litellm-call-id"],)), + lambda found: len(found) == 1, + seconds=70, + ) + assert (rows[0]["status"], rows[0]["model_group"]) == ("success", flagged) + + kept: Final = _decide(gateway, flagged_dropping, route, _BOTH) + assert kept.status_code == 200, kept.text + (kept_call,) = wire.drain() + assert _sent_media(kept_call) == _BOTH + + +@pytest.mark.parametrize( + "value", + ({"supports_audio_input": True, "supports_video_input": True}, 5, [{"supports_audio_input": True}], "x" * 5000), + ids=("flags", "int", "list", "5kb"), +) +def test_a_caller_sent_model_info_never_turns_media_on(gateway: Gateway, value: JsonValue) -> None: + upstream: Final = _unlisted_openrouter_model() + with wire_server(upstream.reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY) + _assert_refused(_decide(gateway, model, "/v1/decisions", {**_BOTH, "model_info": value}), ("audio", "videos")) + assert wire.drain() == () diff --git a/tests/integration/providers/test_responses_bridge_verbosity_wire.py b/tests/integration/providers/test_responses_bridge_verbosity_wire.py new file mode 100644 index 00000000000..037450ba820 --- /dev/null +++ b/tests/integration/providers/test_responses_bridge_verbosity_wire.py @@ -0,0 +1,413 @@ +import uuid +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal + +import httpx +import pytest +import yaml +from integration._support import responses_vendor as rv +from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment +from integration._support.openai_wire import answering_model_discovery, chat_reply, responses_reply +from integration._support.process import OwnedProxy, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +pytestmark: Final = pytest.mark.timeout(120) + +_GPT: Final = "gpt-5.6" +_DATABRICKS: Final = "databricks-meta-llama-4-maverick" +_PROVIDER_KEY: Final = "synthetic-provider-key" +_NO_CACHE: Final[dict[str, JsonValue]] = {"no-cache": True} +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_WEATHER_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"city": {"type": "string"}, "celsius": {"type": "number"}}, + "required": ["city", "celsius"], + "additionalProperties": False, +} +_TEXT_FORMAT: Final[dict[str, JsonValue]] = { + "type": "json_schema", + "name": "weather", + "schema": _WEATHER_SCHEMA, + "strict": True, +} +_RESPONSE_FORMAT: Final[dict[str, JsonValue]] = { + "type": "json_schema", + "json_schema": {"name": "weather", "schema": _WEATHER_SCHEMA, "strict": True}, +} +_STREAM_OPTIONS: Final[dict[str, JsonValue]] = {"stream_options": {"include_usage": True}} + +Verbosity = Literal["low", "high"] + + +def _prompt(marker: str) -> str: + return f"verbosity probe marker-{marker}" + + +def _answer(marker: str) -> str: + return f"scripted answer marker-{marker}" + + +def _marker_of(request: Request) -> str: + body: Final = request.body.decode() + start: Final = body.rfind("marker-") + return body[start + len("marker-") : start + len("marker-") + 32] if start >= 0 else "health" + + +def _vendor(request: Request) -> Reply: + body: Final = _JSON_OBJECT.validate_json(request.body or b"{}") + marker: Final = _marker_of(request) + stream: Final = body.get("stream") is True + if request.target.endswith("/responses"): + return responses_reply(f"resp_{marker}", _GPT, _answer(marker), stream=stream) + return chat_reply(f"chatcmpl-{marker}", str(body.get("model")), _answer(marker), stream=stream) + + +@contextmanager +def _upstream() -> Iterator[Wire]: + with wire_server(answering_model_discovery(_vendor)) as wire: + yield wire + + +def _bridged_gpt(scenario: Scenario, wire: Wire, **parameters: JsonValue) -> str: + return scenario.model( + model=f"openai/{_GPT}", + api_base=f"{wire.url}/v1", + api_key=_PROVIDER_KEY, + use_chat_completions_api=True, + **parameters, + ) + + +def _native_gpt(scenario: Scenario, wire: Wire) -> str: + return scenario.model(model=f"openai/{_GPT}", api_base=f"{wire.url}/v1", api_key=_PROVIDER_KEY) + + +def _databricks(scenario: Scenario, wire: Wire, **parameters: JsonValue) -> str: + return scenario.model( + model=f"databricks/{_DATABRICKS}", + api_base=f"{wire.url}/serving-endpoints", + api_key=_PROVIDER_KEY, + **parameters, + ) + + +def _responses(gateway: Gateway, model: str, marker: str, **fields: JsonValue) -> httpx.Response: + return gateway.request( + "POST", "/v1/responses", {"model": model, "input": _prompt(marker), "cache": _NO_CACHE, **fields} + ) + + +def _streamed_events(gateway: Gateway, model: str, marker: str, **fields: JsonValue) -> list[dict[str, JsonValue]]: + with gateway.client.stream( + "POST", + "/v1/responses", + json={"model": model, "input": _prompt(marker), "cache": _NO_CACHE, "stream": True, **fields}, + headers={"Authorization": f"Bearer {gateway.key}"}, + ) as response: + assert response.status_code == 200, response.read() + return [ + _JSON_OBJECT.validate_json(line.removeprefix("data: ")) + for line in response.iter_lines() + if line.startswith("data: {") + ] + + +def _output_text(response: Mapping[str, JsonValue]) -> str: + output: Final = response["output"] + assert isinstance(output, list), response + return "".join( + str(part["text"]) + for item in output + if isinstance(item, dict) and item.get("type") == "message" + for part in item["content"] + if isinstance(part, dict) and part.get("type") == "output_text" + ) + + +def _answered(response: httpx.Response, marker: str) -> None: + assert response.status_code == 200, response.text + body: Final = _JSON_OBJECT.validate_json(response.text) + assert body["status"] == "completed" and _output_text(body) == _answer(marker), body + + +def _streamed_answer(events: list[dict[str, JsonValue]], marker: str) -> None: + completed: Final = [event for event in events if event["type"] == "response.completed"] + assert len(completed) == 1, [event["type"] for event in events] + final: Final = completed[0]["response"] + assert isinstance(final, dict) and _output_text(final) == _answer(marker), final + deltas: Final = "".join(str(event["delta"]) for event in events if event["type"] == "response.output_text.delta") + assert deltas == _answer(marker), events + + +def _posted(wire: Wire) -> tuple[Request, ...]: + return tuple(request for request in wire.drain() if request.method == "POST") + + +def _only_post(wire: Wire, target: str) -> dict[str, JsonValue]: + posted: Final = _posted(wire) + assert [request.target for request in posted] == [target], posted + return _JSON_OBJECT.validate_json(posted[0].body) + + +def _chat_messages(marker: str) -> list[JsonValue]: + return [{"role": "user", "content": _prompt(marker)}] + + +def _unsupported_param_error(response: httpx.Response) -> None: + assert response.status_code == 400, response.text + error: Final = _JSON_OBJECT.validate_json(response.text)["error"] + assert isinstance(error, dict) and "verbosity" in str(error["message"]), response.text + + +@pytest.mark.parametrize("verbosity", ["low", "high"]) +def test_regression_explicit_bridge_sends_text_verbosity_as_top_level_verbosity( + gateway: Gateway, verbosity: Verbosity +) -> None: + marker: Final = uuid.uuid4().hex + with _upstream() as wire, gateway.scenario() as scenario: + model: Final = _bridged_gpt(scenario, wire) + _answered(_responses(gateway, model, marker, text={"verbosity": verbosity}), marker) + sent: Final = _only_post(wire, "/v1/chat/completions") + assert sent == {"model": _GPT, "messages": _chat_messages(marker), "verbosity": verbosity}, sent + + +@pytest.mark.parametrize("verbosity", ["low", "high"]) +def test_regression_explicit_bridge_stream_sends_text_verbosity_as_top_level_verbosity( + gateway: Gateway, verbosity: Verbosity +) -> None: + marker: Final = uuid.uuid4().hex + with _upstream() as wire, gateway.scenario() as scenario: + model: Final = _bridged_gpt(scenario, wire) + _streamed_answer(_streamed_events(gateway, model, marker, text={"verbosity": verbosity}), marker) + sent: Final = _only_post(wire, "/v1/chat/completions") + assert sent == { + "model": _GPT, + "messages": _chat_messages(marker), + "stream": True, + **_STREAM_OPTIONS, + "verbosity": verbosity, + }, sent + + +def test_regression_explicit_bridge_sends_verbosity_beside_the_json_schema_response_format(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with _upstream() as wire, gateway.scenario() as scenario: + model: Final = _bridged_gpt(scenario, wire) + _answered(_responses(gateway, model, marker, text={"format": _TEXT_FORMAT, "verbosity": "low"}), marker) + sent: Final = _only_post(wire, "/v1/chat/completions") + assert sent == { + "model": _GPT, + "messages": _chat_messages(marker), + "response_format": _RESPONSE_FORMAT, + "verbosity": "low", + }, sent + + +def test_regression_automatic_bridge_refuses_unsupported_verbosity_before_the_wire(gateway: Gateway) -> None: + refused: Final = uuid.uuid4().hex + control: Final = uuid.uuid4().hex + with _upstream() as wire, gateway.scenario() as scenario: + model: Final = _databricks(scenario, wire, drop_params=False) + _unsupported_param_error(_responses(gateway, model, refused, text={"verbosity": "low"})) + _answered(_responses(gateway, model, control), control) + posted: Final = _posted(wire) + assert [(request.target, _marker_of(request)) for request in posted] == [ + ("/serving-endpoints/chat/completions", control) + ], posted + + +def test_regression_automatic_bridge_stream_refuses_unsupported_verbosity_before_the_wire(gateway: Gateway) -> None: + refused: Final = uuid.uuid4().hex + control: Final = uuid.uuid4().hex + with _upstream() as wire, gateway.scenario() as scenario: + model: Final = _databricks(scenario, wire, drop_params=False) + _unsupported_param_error(_responses(gateway, model, refused, text={"verbosity": "high"}, stream=True)) + _answered(_responses(gateway, model, control), control) + posted: Final = _posted(wire) + assert [(request.target, _marker_of(request)) for request in posted] == [ + ("/serving-endpoints/chat/completions", control) + ], posted + + +@pytest.mark.parametrize("source", ["deployment", "request"]) +def test_regression_allowed_openai_params_forwards_verbosity_through_the_automatic_bridge( + gateway: Gateway, source: Literal["deployment", "request"] +) -> None: + marker: Final = uuid.uuid4().hex + allowed: Final[dict[str, JsonValue]] = {"allowed_openai_params": ["verbosity"]} + with _upstream() as wire, gateway.scenario() as scenario: + model: Final = _databricks(scenario, wire, drop_params=False, **(allowed if source == "deployment" else {})) + extra: Final = allowed if source == "request" else {} + _answered(_responses(gateway, model, marker, text={"verbosity": "low"}, **extra), marker) + sent: Final = _only_post(wire, "/serving-endpoints/chat/completions") + assert sent == {"model": _DATABRICKS, "messages": _chat_messages(marker), "verbosity": "low"}, sent + + +@pytest.mark.parametrize("source", ["deployment", "request"]) +def test_guard_drop_params_drops_unsupported_verbosity_and_keeps_the_json_schema_format( + gateway: Gateway, source: Literal["deployment", "request"] +) -> None: + marker: Final = uuid.uuid4().hex + with _upstream() as wire, gateway.scenario() as scenario: + model: Final = _databricks(scenario, wire, drop_params=source == "deployment") + extra: Final[dict[str, JsonValue]] = {"drop_params": True} if source == "request" else {} + _answered( + _responses(gateway, model, marker, text={"format": _TEXT_FORMAT, "verbosity": "low"}, **extra), marker + ) + sent: Final = _only_post(wire, "/serving-endpoints/chat/completions") + assert sent == {"model": _DATABRICKS, "messages": _chat_messages(marker), "response_format": _RESPONSE_FORMAT}, sent + + +def test_guard_drop_params_stream_drops_unsupported_verbosity(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with _upstream() as wire, gateway.scenario() as scenario: + model: Final = _databricks(scenario, wire, drop_params=True) + _streamed_answer(_streamed_events(gateway, model, marker, text={"verbosity": "high"}), marker) + sent: Final = _only_post(wire, "/serving-endpoints/chat/completions") + assert sent == {"model": _DATABRICKS, "messages": _chat_messages(marker), "stream": True}, sent + + +def test_guard_explicit_bridge_without_verbosity_sends_only_the_json_schema_format(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with _upstream() as wire, gateway.scenario() as scenario: + model: Final = _bridged_gpt(scenario, wire) + _answered(_responses(gateway, model, marker, text={"format": _TEXT_FORMAT}), marker) + sent: Final = _only_post(wire, "/v1/chat/completions") + assert sent == {"model": _GPT, "messages": _chat_messages(marker), "response_format": _RESPONSE_FORMAT}, sent + + +def test_guard_explicit_bridge_without_text_sends_no_verbosity(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with _upstream() as wire, gateway.scenario() as scenario: + model: Final = _bridged_gpt(scenario, wire) + _answered(_responses(gateway, model, marker), marker) + sent: Final = _only_post(wire, "/v1/chat/completions") + assert sent == {"model": _GPT, "messages": _chat_messages(marker)}, sent + + +@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"]) +def test_guard_native_responses_route_forwards_text_verbosity_unchanged(gateway: Gateway, stream: bool) -> None: + marker: Final = uuid.uuid4().hex + text: Final[dict[str, JsonValue]] = {"format": _TEXT_FORMAT, "verbosity": "high"} + with _upstream() as wire, gateway.scenario() as scenario: + model: Final = _native_gpt(scenario, wire) + if stream: + _streamed_answer(_streamed_events(gateway, model, marker, text=text), marker) + else: + _answered(_responses(gateway, model, marker, text=text), marker) + sent: Final = _only_post(wire, "/v1/responses") + assert sent == {"model": _GPT, "input": _prompt(marker), "text": text, **({"stream": True} if stream else {})}, sent + + +_CLIENT_TOOL_SEARCH: Final[dict[str, JsonValue]] = {"type": "tool_search", "execution": "client"} +_CLIENT_TOOL_SEARCH_ON_THE_WIRE: Final[dict[str, JsonValue]] = { + "type": "function", + "function": { + "name": "tool_search", + "description": ( + "Search the deferred tools by keyword. The matching tools are loaded and become callable on the next turn." + ), + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string", "description": "Search query for deferred tools."}, + "limit": {"type": "integer", "description": "Maximum number of tools to return. Defaults to 8."}, + }, + "required": ["query"], + "additionalProperties": False, + }, + }, +} + + +@dataclass(frozen=True, slots=True) +class _FollowUpWire: + sent: JsonValue + expected: dict[str, JsonValue] + + +def _stored_prompts_config(directory: Path) -> Path: + stock: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + general: Final = _JSON_OBJECT.validate_python(stock["general_settings"]) + path: Final = directory / "verbosity-stored-turns.yaml" + path.write_text(yaml.safe_dump({**stock, "general_settings": {**general, "store_prompts_in_spend_logs": True}})) + return path + + +@pytest.fixture(scope="module") +def stored_turns_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[OwnedProxy, Wire]]: + directory: Final = tmp_path_factory.mktemp("verbosity-stored-turns") + with _upstream() as wire, gateway_from_environment() as gateway: + with owned_proxy_process(gateway, directory, {}, config=_stored_prompts_config(directory)) as owned: + yield owned, wire + + +def _stored_first_turn(proxy: Gateway, model: str, marker: str) -> str: + first: Final = _responses(proxy, model, marker, tools=[_CLIENT_TOOL_SEARCH]) + _answered(first, marker) + response_id: Final = str(_JSON_OBJECT.validate_json(first.text)["id"]) + eventually( + lambda: proxy.rows('SELECT request_id FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)), + lambda rows: any(rv.same_response(str(row["request_id"]), response_id) for row in rows), + seconds=70, + ) + return response_id + + +def _client_tool_search_follow_up( + stored_turns_proxy: tuple[OwnedProxy, Wire], stream: bool, **fields: JsonValue +) -> _FollowUpWire: + owned, wire = stored_turns_proxy + proxy: Final = owned.gateway + first: Final = uuid.uuid4().hex + second: Final = uuid.uuid4().hex + with proxy.scenario() as scenario: + model: Final = _bridged_gpt(scenario, wire) + previous: Final = _stored_first_turn(proxy, model, first) + wire.drain() + follow_up: Final[dict[str, JsonValue]] = { + "previous_response_id": previous, + "tools": [_CLIENT_TOOL_SEARCH], + **fields, + } + if stream: + _streamed_answer(_streamed_events(proxy, model, second, **follow_up), second) + else: + _answered(_responses(proxy, model, second, **follow_up), second) + sent: Final = _only_post(wire, "/v1/chat/completions") + return _FollowUpWire( + sent, + { + "model": _GPT, + "messages": [ + *_chat_messages(first), + {"role": "assistant", "content": _answer(first)}, + *_chat_messages(second), + ], + "tools": [_CLIENT_TOOL_SEARCH_ON_THE_WIRE], + **({"stream": True, **_STREAM_OPTIONS} if stream else {}), + }, + ) + + +@pytest.mark.timeout(240) +@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"]) +@pytest.mark.parametrize("verbosity", ["low", "high"]) +def test_regression_client_tool_search_follow_up_sends_text_verbosity_as_top_level_verbosity( + stored_turns_proxy: tuple[OwnedProxy, Wire], stream: bool, verbosity: Verbosity +) -> None: + observed: Final = _client_tool_search_follow_up(stored_turns_proxy, stream, text={"verbosity": verbosity}) + assert observed.sent == {**observed.expected, "verbosity": verbosity}, observed.sent + + +@pytest.mark.timeout(240) +@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"]) +def test_guard_client_tool_search_follow_up_without_verbosity_sends_no_verbosity( + stored_turns_proxy: tuple[OwnedProxy, Wire], stream: bool +) -> None: + observed: Final = _client_tool_search_follow_up(stored_turns_proxy, stream) + assert observed.sent == observed.expected, observed.sent diff --git a/tests/integration/security/test_config_overrides.py b/tests/integration/security/test_config_overrides.py index 70f9226ea79..6b8e8791b93 100644 --- a/tests/integration/security/test_config_overrides.py +++ b/tests/integration/security/test_config_overrides.py @@ -715,8 +715,11 @@ def test_cyberark_incremental_update_encrypts_reloads_on_restart_and_delete_clea }, saved.text stored_full: Final = _stored_override(database_url, "cyberark") assert set(stored_full) == set(full_config), stored_full + assert all(stored_full[key] != value for key, value in full_config.items()), stored_full serialized_full: Final = json.dumps(stored_full) - assert all(value not in serialized_full for value in full_config.values()), serialized_full + assert all(value not in serialized_full for value in (cyberark_wire.url, account, username, api_key)), ( + serialized_full + ) model_one: Final = "integration-cyberark-" + uuid.uuid4().hex _new_model(first, model_one, provider_wire.url, secret_names[0]) diff --git a/tests/unit/batches/test_main.py b/tests/unit/batches/test_main.py index fb97e0e4919..8b22395f462 100644 --- a/tests/unit/batches/test_main.py +++ b/tests/unit/batches/test_main.py @@ -37,13 +37,15 @@ import litellm.batches.main as bm import asyncio import datetime import json -from collections.abc import Mapping +from collections.abc import Awaitable, Callable, Mapping, Sequence from typing import Final import httpx import respx -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict from litellm.integrations.custom_logger import CustomLogger +from openai.types.batch import Batch +from litellm.types.utils import LiteLLMBatch # --------------------------------------------------------------------------- # @@ -1204,3 +1206,183 @@ async def test_delete_batch_output_file(respx_mock: respx.MockRouter, monkeypatc assert delete_route.call_count == 1 assert delete_response.id == "file-output123" assert delete_response.deleted is True + + +VLLM_ROUTER_API_BASE: Final = "http://vllm-router.test/v1" +VLLM_ROUTER_BATCH: Final = MappingProxyType( + { + "id": "batch_5f8dac", + "status": "pending", + "input_file_id": "file-b73f4c", + "created_at": 1793239207, + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + "output_file_id": None, + "total_requests": None, + "completed_requests": 0, + "failed_requests": 0, + "metadata": {}, + } +) + + +async def _create_on_vllm_router() -> LiteLLMBatch: + return await litellm.acreate_batch( + completion_window="24h", + endpoint="/v1/chat/completions", + input_file_id="file-b73f4c", + custom_llm_provider="hosted_vllm", + api_key="sk-test", + api_base=VLLM_ROUTER_API_BASE, + ) + + +async def _retrieve_from_vllm_router() -> LiteLLMBatch: + return await litellm.aretrieve_batch( + batch_id="batch_5f8dac", custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE + ) + + +VLLM_ROUTER_BATCH_CALLS: Final = ( + pytest.param("POST", "/batches", _create_on_vllm_router, id="acreate_batch"), + pytest.param("GET", "/batches/batch_5f8dac", _retrieve_from_vllm_router, id="aretrieve_batch"), +) + + +@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_BATCH_CALLS) +@pytest.mark.parametrize( + "router_status, openai_status", + ( + ("pending", "validating"), + ("running", "in_progress"), + ("completed", "completed"), + ("failed", "failed"), + ("cancelled", "cancelled"), + ), +) +async def test_hosted_vllm_router_batch_comes_back_as_an_openai_batch( + respx_mock: respx.MockRouter, + method: str, + path: str, + call_router: Callable[[], Awaitable[LiteLLMBatch]], + router_status: str, + openai_status: str, +): + respx_mock.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond( + 200, json={**VLLM_ROUTER_BATCH, "status": router_status} + ) + + batch: Final = await call_router() + + assert (batch.id, batch.object, batch.status) == ("batch_5f8dac", "batch", openai_status) + + +@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_BATCH_CALLS) +@pytest.mark.parametrize("field, value", (("status", "queued"), ("object", "file"))) +async def test_hosted_vllm_router_batch_with_an_unknown_value_is_rejected( + respx_mock: respx.MockRouter, + method: str, + path: str, + call_router: Callable[[], Awaitable[LiteLLMBatch]], + field: str, + value: str, +): + respx_mock.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond( + 200, json={**VLLM_ROUTER_BATCH, field: value} + ) + + with pytest.raises(ValidationError) as exc_info: + await call_router() + + assert [error["loc"] for error in exc_info.value.errors()] == [(field,)] + + +async def _list_on_vllm_router() -> Sequence[Batch]: + page: Final = await litellm.alist_batches( + custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE + ) + return page.data + + +async def _list_on_vllm_router_sync() -> Sequence[Batch]: + return litellm.list_batches(custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE).data + + +@pytest.mark.parametrize("list_router", (_list_on_vllm_router, _list_on_vllm_router_sync)) +async def test_hosted_vllm_router_batch_list_comes_back_as_openai_batches( + respx_mock: respx.MockRouter, list_router: Callable[[], Awaitable[Sequence[Batch]]] +): + respx_mock.get(f"{VLLM_ROUTER_API_BASE}/batches").respond( + 200, + json={ + "object": "list", + "data": [ + {**VLLM_ROUTER_BATCH, "id": "batch_5f8dac", "status": "pending"}, + {**VLLM_ROUTER_BATCH, "id": "batch_c74802", "status": "running"}, + ], + "first_id": "batch_5f8dac", + "last_id": "batch_c74802", + "has_more": False, + }, + ) + + batches: Final = await list_router() + + assert [(batch.id, batch.object, batch.status) for batch in batches] == [ + ("batch_5f8dac", "batch", "validating"), + ("batch_c74802", "batch", "in_progress"), + ] + + +async def test_hosted_vllm_router_batch_list_passes_through_a_status_that_is_not_a_string( + respx_mock: respx.MockRouter, +): + respx_mock.get(f"{VLLM_ROUTER_API_BASE}/batches").respond( + 200, + json={ + "object": "list", + "data": [{**VLLM_ROUTER_BATCH, "status": ["pending"]}], + "first_id": "batch_5f8dac", + "last_id": "batch_5f8dac", + "has_more": False, + }, + ) + + batches: Final = await _list_on_vllm_router() + + assert [(batch.id, batch.object, batch.status) for batch in batches] == [("batch_5f8dac", "batch", ["pending"])] + + +async def test_openai_batch_list_items_come_back_as_openai_sent_them(respx_mock: respx.MockRouter): + respx_mock.get("http://openai.test/v1/batches").respond( + 200, + json={ + "object": "list", + "data": [ + { + **_OPENAI_BATCH_JSON, + "status": "completed", + "usage": { + "input_tokens": 10, + "input_tokens_details": {"cached_tokens": 4}, + "output_tokens": 5, + "output_tokens_details": {"reasoning_tokens": 2}, + "total_tokens": 15, + }, + }, + {**_OPENAI_BATCH_JSON, "id": "batch_def456", "status": "queued"}, + ], + "first_id": "batch_abc123", + "last_id": "batch_def456", + "has_more": False, + }, + ) + + page: Final = await litellm.alist_batches( + custom_llm_provider="openai", api_key="sk-test", api_base="http://openai.test/v1" + ) + + first, second = page.data + assert first.usage is not None + assert (first.usage.input_tokens_details.cached_tokens, first.usage.output_tokens_details.reasoning_tokens) == (4, 2) + assert (second.id, second.object, second.status) == ("batch_def456", "batch", "queued") diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 0c357837d31..789d59bd613 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -6,10 +6,11 @@ import copy import json import re import threading -from collections.abc import Mapping, Sequence +from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from concurrent.futures import wait +from functools import partial from types import MappingProxyType -from typing import Final +from typing import Final, TypeAlias import httpx import pytest @@ -23,6 +24,7 @@ import litellm from litellm.constants import AWS_SIGNING_MAX_THREADS from litellm.cost_calculator import get_response_cost_from_hidden_params from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded, refetch_model_cost_map from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers from litellm.llms.bedrock.base_aws_llm import AWS_SIGNING_EXECUTOR @@ -3702,6 +3704,387 @@ async def test_a_model_info_object_instead_of_a_dict_still_reaches_the_provider( assert response.answers[0].model_dump(mode="json")["name"] == "is_defect" +_AUDIO_CLIPS: Final = ("data:audio/wav;base64,YQ==",) +_VIDEO_CLIPS: Final = ("data:video/mp4;base64,YQ==",) + + +@pytest.mark.asyncio +async def test_a_deployment_that_advertises_media_sends_it_for_a_model_the_cost_map_does_not_list( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + backend: Final = "openrouter/example/omni-preview" + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "hears", + "litellm_params": {"model": backend, "api_key": "k"}, + "model_info": {"id": "media-hears", "supports_audio_input": True}, + }, + { + "model_name": "hears-and-sees", + "litellm_params": {"model": backend, "api_key": "k"}, + "model_info": {"id": "media-both", "supports_audio_input": True, "supports_video_input": True}, + }, + ] + ) + send: Final = partial( + router.adecisions, input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS + ) + + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['videos']")): + await send(model="hears") + assert not route.called + + await send(model="hears-and-sees") + + sent: Final = json.loads(route.calls.last.request.content) + assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS)) + + +@pytest.mark.asyncio +async def test_media_flags_on_a_deployment_whose_own_mode_is_not_decisions_never_send_clips( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + advertised: Final = {"supports_audio_input": True, "supports_video_input": True} + router: Final = litellm.Router( + model_list=[ + { + "model_name": mode, + "litellm_params": {"model": f"openrouter/example/omni-{mode}-preview", "api_key": "k"}, + "model_info": {"id": f"media-{mode}", "mode": mode, **advertised}, + } + for mode in ("chat", "decisions") + ] + ) + send: Final = partial( + router.adecisions, input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS + ) + + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio', 'videos']")): + await send(model="chat") + assert not route.called + + await send(model="decisions") + + sent: Final = json.loads(route.calls.last.request.content) + assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS)) + + +CatalogLoader: TypeAlias = Callable[[Mapping[str, Mapping[str, object]]], Awaitable[None]] + + +@pytest.fixture +def load_catalog_with(monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None) -> Iterator[CatalogLoader]: + async def load(rows: Mapping[str, Mapping[str, object]]) -> None: + catalog: Final = {**litellm.get_model_cost_map(url=""), **rows} + transport: Final = httpx.MockTransport(lambda _: httpx.Response(200, json=catalog)) + monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP") + async with httpx.AsyncClient(transport=transport) as client: + loaded: Final = await refetch_model_cost_map(url="https://catalog.example/prices.json", client=client) + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + assert isinstance(loaded, ModelCostMapReloaded) + monkeypatch.setattr(litellm, "model_cost", loaded.model_cost_map) + litellm.get_model_info.cache_clear() + + yield load + litellm.get_model_cost_map(url="") + + +@pytest.mark.asyncio +async def test_media_flags_on_a_deployment_without_its_own_mode_follow_the_mode_the_catalog_lists( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, load_catalog_with: CatalogLoader +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + await load_catalog_with( + { + f"openrouter/example/omni-{mode}": {"litellm_provider": "openrouter", "mode": mode} + for mode in ("chat", "decisions") + } + ) + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + router: Final = litellm.Router( + model_list=[ + { + "model_name": mode, + "litellm_params": {"model": f"openrouter/example/omni-{mode}", "api_key": "k"}, + "model_info": {"id": f"listed-{mode}", "supports_audio_input": True, "supports_video_input": True}, + } + for mode in ("chat", "decisions") + ] + ) + send: Final = partial( + router.adecisions, input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS + ) + + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio', 'videos']")): + await send(model="chat") + assert not route.called + + monkeypatch.setattr(litellm, "drop_params", True) + await send(model="chat") + dropped: Final = json.loads(route.calls.last.request.content) + assert ("audio" in dropped, "videos" in dropped) == (False, False) + + await send(model="decisions") + sent: Final = json.loads(route.calls.last.request.content) + assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sibling_mode", ("responses", "embedding")) +async def test_another_deployments_mode_never_switches_off_a_deployments_own_media_flags( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, sibling_mode: str +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + shared_params: Final = {"model": "openrouter/example/omni-shared", "api_key": "k"} + router: Final = litellm.Router( + model_list=[ + {"model_name": "sibling", "litellm_params": shared_params, "model_info": {"id": "s", "mode": sibling_mode}}, + { + "model_name": "flagged", + "litellm_params": shared_params, + "model_info": {"id": "f", "supports_audio_input": True, "supports_video_input": True}, + }, + ] + ) + + await router.adecisions( + model="flagged", input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS)) + + +_MEDIA_SURFACES: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType( + { + "adecisions": {"input": "review", "questions": _PREDICATE_QUESTIONS}, + "asystemone": {"state": "review", "questions": _QUESTIONS}, + "systemone": {"state": "review", "questions": _QUESTIONS}, + } +) + + +def _group_member( + group: str, model: str, key: str, weight: int, model_info: Mapping[str, object], **params: object +) -> dict[str, object]: + return { + "model_name": group, + "litellm_params": {"model": model, "api_key": key, "weight": weight, **params}, + "model_info": {"id": f"{group}-{key}", **model_info}, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("surface", tuple(_MEDIA_SURFACES)) +async def test_a_group_sends_clips_to_a_deployment_that_takes_them_and_refuses_only_when_none_does( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, surface: str +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False) + monkeypatch.setitem( + litellm.model_cost, + "cloudflare/@cf/cloudflare/clef-omni", + {"litellm_provider": "cloudflare", "mode": "evaluation", "supports_audio_input": True}, + ) + respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + respx_mock.post("https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef-omni").respond( + json=_RESPONSE + ) + unlisted: Final = "openrouter/example/omni-preview" + listed: Final = "cloudflare/clef-omni" + router: Final = litellm.Router( + model_list=[ + _group_member("unlisted", unlisted, "refuses", 1, {}), + _group_member("unlisted", unlisted, "takes", 0, {"supports_audio_input": True}), + _group_member("listed", listed, "refuses", 1, {"supports_audio_input": False}), + _group_member("listed", listed, "takes", 0, {}), + _group_member("none", unlisted, "refuses", 1, {}), + _group_member("none", listed, "refuses", 1, {"supports_audio_input": False}), + _group_member("drops", unlisted, "drops", 1, {}, drop_params=True), + _group_member("drops", listed, "takes", 0, {}), + _group_member("unresolvable", "databricks/not a serving endpoint", "unresolvable", 0, {}), + _group_member("unresolvable", listed, "takes", 1, {}), + ] + ) + call: Final = getattr(router, surface) + + async def send(group: str, **media: object) -> object: + response: Final = call(model=group, **_MEDIA_SURFACES[surface], **media) + return await response if asyncio.iscoroutine(response) else response + + for group in ("unlisted", "listed", "unresolvable"): + await send(group, audio=_AUDIO_CLIPS) + assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes" + assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS) + + await send("drops", audio=_AUDIO_CLIPS) + assert respx_mock.calls.last.request.headers["authorization"] == "Bearer drops" + assert "audio" not in json.loads(respx_mock.calls.last.request.content) + + await send("unlisted") + assert respx_mock.calls.last.request.headers["authorization"] == "Bearer refuses" + + sent_before_refusal: Final = len(respx_mock.calls) + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio']")): + await send("none", audio=_AUDIO_CLIPS) + assert len(respx_mock.calls) == sent_before_refusal + + +def _answer_unless_key_is(rejected_key: str) -> Callable[[httpx.Request], httpx.Response]: + def answer(request: httpx.Request) -> httpx.Response: + if request.headers["authorization"] == f"Bearer {rejected_key}": + return httpx.Response(401, json={"error": {"message": "invalid key"}}) + return httpx.Response(200, json=_RESPONSE) + + return answer + + +@pytest.mark.asyncio +async def test_clips_to_a_group_whose_only_taker_is_cooling_down_are_refused_rather_than_rate_limited( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").mock( + side_effect=_answer_unless_key_is("takes") + ) + unlisted: Final = "openrouter/example/omni-preview" + router: Final = litellm.Router( + model_list=[ + _group_member("g", unlisted, "refuses", 1, {}), + _group_member("g", unlisted, "takes", 1, {"supports_audio_input": True}), + ], + num_retries=0, + cooldown_time=120, + ) + with pytest.raises(litellm.AuthenticationError): + await router.adecisions(model="g-takes", input="review", questions=_PREDICATE_QUESTIONS) + sent_before_clips: Final = len(route.calls) + + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio']")): + await router.adecisions(model="g", input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS) + + assert len(route.calls) == sent_before_clips + + +@pytest.mark.asyncio +@pytest.mark.parametrize("surface", tuple(_MEDIA_SURFACES)) +async def test_a_router_default_drop_params_overrides_a_deployment_when_choosing_who_takes_clips( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, surface: str +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + unlisted: Final = "openrouter/example/omni-preview" + router: Final = litellm.Router( + model_list=[ + _group_member("g", unlisted, "drops", 1, {}, drop_params=True), + _group_member("g", unlisted, "takes", 0, {"supports_audio_input": True}), + ], + default_litellm_params={"drop_params": False}, + ) + call: Final = getattr(router, surface) + + async def send(**params: object) -> object: + response: Final = call(model="g", **_MEDIA_SURFACES[surface], audio=_AUDIO_CLIPS, **params) + return await response if asyncio.iscoroutine(response) else response + + await send() + assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes" + assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS) + + await send(drop_params=True) + assert respx_mock.calls.last.request.headers["authorization"] == "Bearer drops" + assert "audio" not in json.loads(respx_mock.calls.last.request.content) + + +@pytest.mark.asyncio +async def test_a_deployment_that_turns_audio_off_is_refused_or_dropped_even_when_the_cost_map_allows_it( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setitem( + litellm.model_cost, + "typesafe/jev-omni", + { + "litellm_provider": "typesafe", + "mode": "decisions", + "supports_audio_input": True, + "supports_video_input": True, + }, + ) + route: Final = respx_mock.post("https://example.com/v1/systemone").respond(json=_RESPONSE) + send: Final = partial( + litellm.asystemone, + model="typesafe/jev-omni", + state="test", + questions=_QUESTIONS, + api_base="https://example.com", + api_key="test", + audio=_AUDIO_CLIPS, + videos=_VIDEO_CLIPS, + model_info={"supports_audio_input": False}, + ) + + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("does not support parameters: ['audio']")): + await send() + assert not route.called + + await send(drop_params=True) + + sent: Final = json.loads(route.calls.last.request.content) + assert "audio" not in sent + assert sent["videos"] == list(_VIDEO_CLIPS) + + +@pytest.mark.asyncio +async def test_a_media_flag_that_is_not_a_boolean_leaves_the_cost_map_in_charge_of_that_kind_only( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://example.com/v1/systemone").respond(json=_RESPONSE) + + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("does not support parameters: ['audio']")): + await litellm.asystemone( + model="typesafe/jev-unlisted", + state="test", + questions=_QUESTIONS, + api_base="https://example.com", + api_key="test", + audio=_AUDIO_CLIPS, + videos=_VIDEO_CLIPS, + model_info={"supports_audio_input": "true", "supports_video_input": True}, + ) + + assert not route.called + + +@pytest.mark.asyncio +async def test_an_openai_format_deployment_that_advertises_media_is_still_refused_it( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE) + + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio', 'videos']")): + await litellm.adecisions( + model="openai/gpt-6-luna", + input="review", + questions=_PREDICATE_QUESTIONS, + api_key="deployment-key", + audio=_AUDIO_CLIPS, + videos=_VIDEO_CLIPS, + model_info={"supports_audio_input": True, "supports_video_input": True}, + ) + + assert not route.called + + def test_dropping_every_question_answers_400_before_http(respx_mock: respx.MockRouter) -> None: route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE) diff --git a/tests/unit/files/test_main.py b/tests/unit/files/test_main.py index e91951fb5ba..44aadc9014e 100644 --- a/tests/unit/files/test_main.py +++ b/tests/unit/files/test_main.py @@ -1,3 +1,4 @@ +from collections.abc import Awaitable, Callable from types import MappingProxyType from typing import Final from urllib.parse import parse_qs, urlparse @@ -124,6 +125,61 @@ async def test_afile_retrieve_rejects_a_provider_file_without_its_size(): assert [error["loc"] for error in exc_info.value.errors()] == [("bytes",)] +VLLM_ROUTER_API_BASE: Final = "http://vllm-router.test/v1" +VLLM_ROUTER_FILE: Final = MappingProxyType( + {"id": "file-b73f4c", "bytes": 156, "created_at": 1793238707, "filename": "batch_requests.jsonl", "purpose": "batch"} +) + + +async def _upload_to_vllm_router() -> OpenAIFileObject: + return await litellm.acreate_file( + file=("batch_requests.jsonl", b'{"custom_id": "request-1"}'), + purpose="batch", + custom_llm_provider="hosted_vllm", + api_key="sk-test", + api_base=VLLM_ROUTER_API_BASE, + ) + + +async def _retrieve_from_vllm_router() -> OpenAIFileObject: + return await litellm.afile_retrieve( + file_id="file-b73f4c", custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE + ) + + +VLLM_ROUTER_FILE_CALLS: Final = ( + pytest.param("POST", "/files", _upload_to_vllm_router, id="acreate_file"), + pytest.param("GET", "/files/file-b73f4c", _retrieve_from_vllm_router, id="afile_retrieve"), +) + + +@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_FILE_CALLS) +@respx.mock +async def test_hosted_vllm_file_without_an_object_type_comes_back_as_a_file( + method: str, path: str, call_router: Callable[[], Awaitable[OpenAIFileObject]] +): + respx.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond(200, json=dict(VLLM_ROUTER_FILE)) + + file_object: Final = await call_router() + + assert file_object.model_dump(exclude_none=True) == {**VLLM_ROUTER_FILE, "object": "file"} + + +@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_FILE_CALLS) +@respx.mock +async def test_hosted_vllm_file_with_a_different_object_type_is_rejected( + method: str, path: str, call_router: Callable[[], Awaitable[OpenAIFileObject]] +): + respx.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond( + 200, json={**VLLM_ROUTER_FILE, "object": "batch"} + ) + + with pytest.raises(ValidationError) as exc_info: + await call_router() + + assert [error["loc"] for error in exc_info.value.errors()] == [("object",)] + + _FILE_BODY: Final = b'{"prompt": "Hello", "completion": "Hi"}' _FINE_TUNE_FILE_JSON: Final = MappingProxyType( { diff --git a/tests/unit/integrations/open_telemetry/test_otel_admin_endpoints.py b/tests/unit/integrations/open_telemetry/test_otel_admin_endpoints.py index ebeb514ffff..1965fba94c6 100644 --- a/tests/unit/integrations/open_telemetry/test_otel_admin_endpoints.py +++ b/tests/unit/integrations/open_telemetry/test_otel_admin_endpoints.py @@ -3,10 +3,14 @@ async_management_endpoint_{success,failure}_hook integration points.""" import asyncio from datetime import datetime +from typing import Final from unittest.mock import MagicMock +import httpx import pytest +from fastapi import Request +from litellm import Router from litellm.proxy._types import ( ManagementEndpointLoggingPayload, UserAPIKeyAuth, @@ -340,3 +344,203 @@ def test_management_wrapper_swallows_post_success_errors(server_span_factory, ot result = asyncio.run(fake_fn(data={}, user_api_key_dict=_real_user_api_key_dict(server_span))) assert result == {"ok": True} + + +TEAM_NEW_PATH: Final = "/team/new" +MODEL_LIST_PATH: Final = "/v1/models" +MODEL_RETRIEVE_PATH: Final = "/v1/models/gpt-4o" +INBOUND_TRACEPARENT: Final = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" + + +def _request(method: str, path: str, body: bytes = b"") -> Request: + async def receive() -> dict[str, object]: + return {"type": "http.request", "body": body, "more_body": False} + + return Request( + { + "type": "http", + "method": method, + "path": path, + "root_path": "", + "query_string": b"", + "headers": [(b"content-type", b"application/json")], + }, + receive, + ) + + +async def _answer_like_the_proxy(request: Request, exception: Exception) -> None: + from litellm.proxy.proxy_server import app + + handler: Final = next( + app.exception_handlers[cls] for cls in type(exception).__mro__ if cls in app.exception_handlers + ) + await handler(request, exception) + + +@pytest.mark.parametrize( + ("wrapper_name", "exports_payload"), + [("management_endpoint_wrapper", True), ("management_endpoint_span_wrapper", False)], +) +def test_span_wrapper_omits_payload(server_span_factory, otel_with_exporter, monkeypatch, wrapper_name, exports_payload): + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.management_helpers import utils as mgmt_utils + + otel, exporter = otel_with_exporter + monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False) + + server_span: Final = server_span_factory(TEAM_NEW_PATH) + wrap: Final = getattr(mgmt_utils, wrapper_name) + + @wrap + async def fake_fn(http_request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: + return {"data": [{"id": "gpt-4o"}]} + + asyncio.run( + fake_fn( + http_request=_request("POST", TEAM_NEW_PATH, b'{"team_alias": "t"}'), + user_api_key_dict=_real_user_api_key_dict(server_span), + ) + ) + + spans: Final = [s for s in exporter.get_finished_spans() if s.name == TEAM_NEW_PATH] + assert len(spans) == 1 + attribute_keys: Final = tuple(spans[0].attributes) + assert any(key.startswith("request.") for key in attribute_keys) is exports_payload + assert any(key.startswith("response.") for key in attribute_keys) is exports_payload + + +@pytest.mark.parametrize( + ("wrapper_name", "exports_request_body"), + [("management_endpoint_wrapper", True), ("management_endpoint_span_wrapper", False)], +) +def test_span_wrapper_omits_request_body_on_failure( + server_span_factory, otel_with_exporter, monkeypatch, wrapper_name, exports_request_body +): + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.management_helpers import utils as mgmt_utils + + otel, exporter = otel_with_exporter + monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False) + + server_span: Final = server_span_factory(TEAM_NEW_PATH) + wrap: Final = getattr(mgmt_utils, wrapper_name) + + @wrap + async def fake_fn(http_request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: + raise make_fastapi_http_exception(400, "team_alias taken") + + with pytest.raises(Exception, match="team_alias taken"): + asyncio.run( + fake_fn( + http_request=_request("POST", TEAM_NEW_PATH, b'{"team_alias": "t"}'), + user_api_key_dict=_real_user_api_key_dict(server_span), + ) + ) + + spans: Final = [s for s in exporter.get_finished_spans() if s.name == TEAM_NEW_PATH] + assert len(spans) == 1 + assert any(key.startswith("request.") for key in spans[0].attributes) is exports_request_body + + +@pytest.mark.parametrize( + ("raised", "status_code"), + [ + (RuntimeError("router settings lookup failed"), 500), + (httpx.ConnectError("All connection attempts failed"), 503), + (make_fastapi_http_exception(400, "Invalid scope parameter"), 400), + ], +) +def test_span_wrapper_failure_keeps_the_answered_status(otel_with_exporter, monkeypatch, raised, status_code): + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.management_helpers import utils as mgmt_utils + + otel, exporter = otel_with_exporter + monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False) + + server_span: Final = _server_span(otel, MODEL_LIST_PATH, headers={}) + request: Final = _request("GET", MODEL_LIST_PATH) + request.state.parent_otel_span = server_span + + @mgmt_utils.management_endpoint_span_wrapper + async def failing_model_list(request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: + raise raised + + async def call_and_answer() -> None: + try: + await failing_model_list(request=request, user_api_key_dict=_real_user_api_key_dict(server_span)) + except Exception as exc: + await _answer_like_the_proxy(request, exc) + + asyncio.run(call_and_answer()) + + assert_server_span_attrs( + exporter, + expected_status=status_code, + expected_url_path=MODEL_LIST_PATH, + where=f"model list {type(raised).__name__}", + ) + exported: Final = get_server_span(exporter) + assert exported is not None + assert exported.attributes.get("error.code") == str(status_code) + + +def _server_span(otel, path: str, headers: dict[str, str]): + server_span: Final = otel.create_litellm_proxy_request_started_span(start_time=datetime.now(), headers=headers) + otel.set_proxy_request_route_attributes(server_span, url_path=path, http_route=path) + return server_span + + +def _assert_ended_in_inbound_trace(exporter, path: str, handler_name: str) -> None: + assert_server_span_attrs(exporter, expected_status=200, expected_url_path=path, where=f"{path} success") + exported: Final = get_server_span(exporter) + assert exported is not None + assert exported.context.trace_id == int(INBOUND_TRACEPARENT.split("-")[1], 16) + + endpoint_spans: Final = [s for s in exporter.get_finished_spans() if s.name == handler_name] + assert len(endpoint_spans) == 1 + assert endpoint_spans[0].parent is not None + assert endpoint_spans[0].parent.span_id == exported.context.span_id + assert not any(key.startswith("response.") for key in endpoint_spans[0].attributes) + + +def _install_proxy_state(monkeypatch, router: Router | None) -> None: + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "user_model", None) + + +def test_model_list_ends_server_span_in_inbound_trace(otel_with_exporter, monkeypatch): + import litellm.proxy.proxy_server as proxy_server + + otel, exporter = otel_with_exporter + monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False) + _install_proxy_state(monkeypatch, None) + server_span: Final = _server_span(otel, MODEL_LIST_PATH, headers={"traceparent": INBOUND_TRACEPARENT}) + + result: Final = asyncio.run(proxy_server.model_list(user_api_key_dict=_real_user_api_key_dict(server_span))) + + assert result == {"data": [], "object": "list"} + _assert_ended_in_inbound_trace(exporter, MODEL_LIST_PATH, "model_list") + + +def test_model_retrieve_ends_server_span_in_inbound_trace(otel_with_exporter, monkeypatch): + import litellm.proxy.proxy_server as proxy_server + + otel, exporter = otel_with_exporter + monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False) + _install_proxy_state( + monkeypatch, + Router(model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-x"}}]), + ) + server_span: Final = _server_span(otel, MODEL_RETRIEVE_PATH, headers={"traceparent": INBOUND_TRACEPARENT}) + + result: Final = asyncio.run( + proxy_server.model_info(model_id="gpt-4o", user_api_key_dict=_real_user_api_key_dict(server_span)) + ) + + assert result["id"] == "gpt-4o" + _assert_ended_in_inbound_trace(exporter, MODEL_RETRIEVE_PATH, "model_info") diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index e2452a928a4..9da2632943c 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -1,3 +1,4 @@ +from collections.abc import Callable from typing import Final import pytest @@ -7,7 +8,7 @@ from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) from litellm.types.llms.openai import FileSearchTool, ResponsesAPIResponse, WebSearchOptions -from litellm.types.utils import ModelResponse, StandardBuiltInToolsParams +from litellm.types.utils import ModelResponse, ServerToolUse, StandardBuiltInToolsParams, Usage def test_web_search_cost_low(): @@ -1052,6 +1053,175 @@ def test_azure_ai_non_claude_web_search_keeps_generic_fallback() -> None: assert get_cost_for_web_search_request(custom_llm_provider="azure_ai", usage=usage, model_info=model_info) is None +def _register_priced_deployment( + model: str, search_cost_per_query: float | None, deployment_id: str | None = None +) -> str: + from litellm import Router + + token_rates: Final = {"input_cost_per_token": 3e-6, "output_cost_per_token": 1.5e-5} + search_rates: Final = ( + {} + if search_cost_per_query is None + else { + "search_context_cost_per_query": { + "search_context_size_low": search_cost_per_query, + "search_context_size_medium": search_cost_per_query, + "search_context_size_high": search_cost_per_query, + } + } + ) + router: Final = Router( + model_list=[ + { + "model_name": "priced-deployment", + "litellm_params": {"model": model, "api_key": "k", "vertex_project": "p", "vertex_location": "global"}, + "model_info": {**token_rates, **search_rates, **({} if deployment_id is None else {"id": deployment_id})}, + } + ] + ) + return router.model_list[0]["model_info"]["id"] + + +def _claude_chat_response( + web_search_requests: int, model: str = "claude-sonnet-5-5" +) -> tuple[object, Usage | None]: + usage: Final = Usage( + prompt_tokens=1840, + completion_tokens=412, + total_tokens=2252, + server_tool_use=ServerToolUse(web_search_requests=web_search_requests), + ) + return ModelResponse(model=model, usage=usage), usage + + +def _claude_messages_response( + web_search_requests: int, model: str = "claude-sonnet-5-5" +) -> tuple[object, Usage | None]: + return { + "id": "msg_vertex_claude_web_search", + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": "scripted answer"}], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "server_tool_use": {"web_search_requests": web_search_requests}, + }, + }, None + + +@pytest.mark.parametrize("build_response", [_claude_chat_response, _claude_messages_response]) +@pytest.mark.parametrize(("web_search_requests", "search_cost_per_query"), [(2, 0.01), (3, 0.01), (2, 0.0)]) +def test_vertex_claude_deployment_priced_web_search_billed_per_request( + local_model_cost_map: None, + build_response: Callable[[int], tuple[object, Usage | None]], + web_search_requests: int, + search_cost_per_query: float, +) -> None: + deployment_id: Final = _register_priced_deployment("vertex_ai/claude-sonnet-5-5", search_cost_per_query) + response_object, usage = build_response(web_search_requests) + + cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=deployment_id, + response_object=response_object, + usage=usage, + custom_llm_provider="vertex_ai", + standard_built_in_tools_params=None, + ) + + assert cost == pytest.approx(web_search_requests * search_cost_per_query) + + +@pytest.mark.parametrize("build_response", [_claude_chat_response, _claude_messages_response]) +@pytest.mark.parametrize( + ("deployment_model", "served_model", "custom_llm_provider"), + # Google bills Claude web search on Sonnet 4.5 and Haiku 4.5 too: + # https://cloud.google.com/model-garden/partner-models/pricing (read 2026-10-30) + [ + ("vertex_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "vertex_ai"), + ("vertex_ai/claude-sonnet-4-5@20250929", "claude-sonnet-4-5-20250929", "vertex_ai"), + ("vertex_ai/claude-haiku-4-5@20251001", "claude-haiku-4-5-20251001", "vertex_ai"), + ("anthropic/claude-sonnet-5-5", "claude-sonnet-5-5", "anthropic"), + ("azure_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "azure_ai"), + ], +) +@pytest.mark.parametrize("deployment_id", [None, "claude-token-priced"], ids=["hashed_id", "claude_in_id"]) +def test_claude_deployment_with_token_rates_only_bills_served_model_search_price( + local_model_cost_map: None, + build_response: Callable[[int, str], tuple[object, Usage | None]], + deployment_model: str, + served_model: str, + custom_llm_provider: str, + deployment_id: str | None, +) -> None: + served_search_pricing: Final = ( + litellm.get_model_info(model=deployment_model).get("search_context_cost_per_query") or {} + ) + served_search_price: Final = served_search_pricing.get("search_context_size_medium") or 0.0 + registered_id: Final = _register_priced_deployment(deployment_model, None, deployment_id) + response_object, usage = build_response(2, served_model) + + cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=registered_id, + response_object=response_object, + usage=usage, + custom_llm_provider=custom_llm_provider, + standard_built_in_tools_params=None, + ) + + assert served_search_price > 0 + assert cost == pytest.approx(2 * served_search_price) + + +def test_vertex_claude_deployment_with_token_rates_only_and_unmapped_served_model_bills_no_search( + local_model_cost_map: None, +) -> None: + deployment_id: Final = _register_priced_deployment("vertex_ai/claude-sonnet-5-5", None) + usage: Final = Usage( + prompt_tokens=1840, + completion_tokens=412, + total_tokens=2252, + server_tool_use=ServerToolUse(web_search_requests=2), + ) + + cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=deployment_id, + response_object=ModelResponse(model="claude-unmapped-preview", usage=usage), + usage=usage, + custom_llm_provider="vertex_ai", + standard_built_in_tools_params=None, + ) + + assert cost == 0.0 + + +@pytest.mark.parametrize("deployment_id", [None, "claude-code-gemini"], ids=["hashed_id", "claude_in_id"]) +def test_vertex_gemini_deployment_priced_web_search_stays_per_prompt( + local_model_cost_map: None, deployment_id: str | None +) -> None: + search_cost_per_prompt: Final = 0.03 + registered_id: Final = _register_priced_deployment( + "vertex_ai/gemini-2.5-flash", search_cost_per_prompt, deployment_id + ) + usage: Final = Usage( + prompt_tokens=1840, + completion_tokens=412, + total_tokens=2252, + server_tool_use=ServerToolUse(web_search_requests=2), + ) + + cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=registered_id, + response_object=ModelResponse(model="gemini-2.5-flash", usage=usage), + usage=usage, + custom_llm_provider="vertex_ai", + standard_built_in_tools_params=None, + ) + + assert cost == pytest.approx(search_cost_per_prompt) + + @pytest.mark.parametrize( "kwargs", [{"tools": 5}, {"web_search_options": None}, {"web_search_options": 5}], diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 72e72714cd4..4ce6aee1619 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -25,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_semantic_cache_prompt_from_messages, get_str_from_messages, handle_any_messages_to_chat_completion_str_messages_conversion, + handle_messages_with_content_list_to_str_conversion, hoist_images_from_tool_messages, is_encrypted_reasoning_block, merge_consecutive_system_messages, @@ -2675,3 +2676,16 @@ def test_tpm_projection_preserves_non_media_identity_and_opaque_values() -> None projected, media = split_media_for_tpm(request) assert projected is request assert media == () + + +def test_content_list_flattening_returns_new_messages_and_leaves_the_callers_untouched() -> None: + messages: Final = [ + {"role": "user", "content": [{"type": "text", "text": "Hello"}, {"type": "text", "text": " world"}]}, + {"role": "assistant", "content": "Hi"}, + ] + messages_before: Final = copy.deepcopy(messages) + + flattened: Final = handle_messages_with_content_list_to_str_conversion(messages) + + assert flattened == [{"role": "user", "content": "Hello world"}, {"role": "assistant", "content": "Hi"}] + assert messages == messages_before diff --git a/tests/unit/litellm_core_utils/test_logging_utils.py b/tests/unit/litellm_core_utils/test_logging_utils.py index edf0dc7960b..a367c02fd6f 100644 --- a/tests/unit/litellm_core_utils/test_logging_utils.py +++ b/tests/unit/litellm_core_utils/test_logging_utils.py @@ -3,7 +3,10 @@ Tests for litellm.litellm_core_utils.logging_utils — base64 truncation helpers """ import asyncio, datetime, importlib, litellm, os, pytest_asyncio +import sys import threading +from itertools import chain +from types import FrameType from unittest.mock import MagicMock import pytest @@ -17,7 +20,7 @@ from litellm.litellm_core_utils.logging_utils import ( truncate_base64_in_messages, truncate_base64_in_messages_async, ) -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Callable, Coroutine, Sequence from datetime import datetime as datetime_assemble_streaming from litellm import( Choices, @@ -28,6 +31,7 @@ from litellm import( ) from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.types.llms.openai import AllMessageValues from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome from typing import Final @@ -99,8 +103,8 @@ class TestTruncateBase64InString: @pytest.mark.parametrize( "text", [ - 'data: {"choices": [{"delta": {"content": "hi"}}]}\n\n' * 50_000, - "data:" * 200_000, + 'data: {"choices": [{"delta": {"content": "hi"}}]}\n\n' * 50_000 + ";base64,", + "data:" * 200_000 + ";base64,", ], ids=["sse_lines", "whitespace_free_prefixes"], ) @@ -241,6 +245,49 @@ def scan_threads(monkeypatch): return threads +def _run_without_yielding(coroutine: Coroutine[object, object, object]) -> object: + try: + coroutine.send(None) + except StopIteration as finished: + return finished.value + coroutine.close() + raise AssertionError("the coroutine yielded to the event loop") + + +def _regex_calls_while(run: Callable[[], object]) -> tuple[object, tuple[str, ...]]: + calls: Final[list[str]] = [] # mutable-ok: sys.setprofile reports each call to a callback that has to record it + previous: Final = sys.getprofile() + + def record(frame: FrameType, event: str, arg: object) -> None: + name: Final = getattr(arg, "__qualname__", "") + if event == "c_call" and name.startswith("Pattern."): + calls.append(name) + + sys.setprofile(record) + try: + result: Final = run() + finally: + sys.setprofile(previous) + return result, tuple(calls) + + +def _tool_result_messages(tool_output: str) -> list[AllMessageValues]: # mutable-ok: the truncator only walks lists + return [ + {"role": "user", "content": "Summarize the rows the tool returned."}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "dump_rows", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": tool_output}, + ] + + +_PLAIN_TOOL_OUTPUT_CHARS: Final = 400 * 1024 + + class TestTruncateBase64InMessagesAsync: @pytest.mark.asyncio async def test_large_payload_is_scanned_off_the_event_loop(self, monkeypatch, scan_threads): @@ -268,6 +315,83 @@ class TestTruncateBase64InMessagesAsync: assert scan_threads assert set(scan_threads) == {threading.get_ident()} + @pytest.mark.parametrize( + "row", + [ + '{"id": 7, "name": "widget", "status": "ok"}\n', + "data: {\"choices\": [{\"delta\": {\"content\": \"hi\"}}]}\n\n", + "const total = rows.map((r) => r.price).reduce((a, b) => a + b, 0);\n", + 'data: {"choices": [{"delta": {"content": "a; b"}}]}\n\n', + "Content-Type: text/plain;base64,aGVsbG8=\n", + ], + ids=["json_rows", "sse_lines", "code_with_semicolons", "sse_lines_with_semicolons", "base64_marker_without_data"], + ) + def test_tool_message_without_data_uri_is_copied_without_a_regex_scan(self, row: str): + tool_output: Final = (row * (_PLAIN_TOOL_OUTPUT_CHARS // len(row) + 1))[:_PLAIN_TOOL_OUTPUT_CHARS] + messages: Final = _tool_result_messages(tool_output) + + result, regex_calls = _regex_calls_while(lambda: truncate_base64_in_messages(messages)) + + assert result == messages + assert result is not messages + assert regex_calls == () + + def test_small_data_uri_is_truncated_inline_with_a_regex_scan(self): + payload: Final = "L" * 2_000 + messages: Final = _image_messages(payload) + + result, regex_calls = _regex_calls_while( + lambda: _run_without_yielding(truncate_base64_in_messages_async(messages)) + ) + + assert regex_calls + assert payload not in result[0]["content"][1]["image_url"]["url"] + assert "base64_data truncated" in result[0]["content"][1]["image_url"]["url"] + + def test_large_plain_text_is_copied_inline_without_rescanning_or_a_thread_hop( + self, monkeypatch: pytest.MonkeyPatch, scan_threads: Sequence[int] + ): + monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS", 1_000) + messages: Final = _tool_result_messages("plain tool output\n" * 100) + + result: Final = _run_without_yielding(truncate_base64_in_messages_async(messages)) + + assert result == messages + assert result is not messages + assert scan_threads == [] + + @pytest.mark.asyncio + async def test_long_plain_text_history_is_copied_off_the_event_loop( + self, monkeypatch: pytest.MonkeyPatch, scan_threads: Sequence[int] + ): + monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS", 1_000) + monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS", 10) + messages: Final = list(chain.from_iterable(_tool_result_messages(f"rows {turn}\n" * 100) for turn in range(5))) + + result: Final = await truncate_base64_in_messages_async(messages) + offload_threads: Final = tuple(scan_threads) + + assert result == messages + assert result is not messages + assert offload_threads + assert threading.get_ident() not in offload_threads + + @pytest.mark.asyncio + async def test_plain_text_over_the_inline_limit_is_copied_off_the_event_loop( + self, monkeypatch: pytest.MonkeyPatch, scan_threads: Sequence[int] + ): + monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS", 1_000) + monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS", 10_000) + messages: Final = _tool_result_messages('data: {"choices": [{"delta": {"content": "a; b"}}]}\n\n' * 400) + + result: Final = await truncate_base64_in_messages_async(messages) + offload_threads: Final = tuple(scan_threads) + + assert result == messages + assert result is not messages + assert offload_threads + assert threading.get_ident() not in offload_threads + @pytest.mark.asyncio async def test_none_and_disabled_truncation_short_circuit(self, monkeypatch, scan_threads): assert await truncate_base64_in_messages_async(None) is None diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index cbb60caa57b..501dd5bd0c6 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,6 +1,7 @@ import copy import json import logging +from collections.abc import Iterator from itertools import chain from typing import Final from unittest.mock import MagicMock, patch @@ -9849,3 +9850,186 @@ def test_transform_request_tool_config_point_without_tools_drops_the_key(): assert "cache_control_injection_points" not in data assert "tools" not in data assert litellm_params["metadata"] == {} + + +def _five_cache_hint_conversation() -> list[dict[str, object]]: + return [ + {"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]}, + {"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]}, + {"role": "assistant", "content": "answer 2"}, + {"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]}, + ] + + +def _cached_texts(node: object) -> Iterator[str]: + if isinstance(node, dict): + yield from ((node["text"],) if "cache_control" in node and "text" in node else ()) + yield from chain.from_iterable(map(_cached_texts, node.values())) + elif isinstance(node, list): + yield from chain.from_iterable(map(_cached_texts, node)) + + +@pytest.mark.parametrize( + ("config", "model"), + [ + (AnthropicConfig(), "claude-haiku-5-5"), + (VertexAIAnthropicConfig(), "claude-haiku-5-5"), + (AmazonAnthropicClaudeConfig(), "anthropic.claude-haiku-5-5"), + (AzureAnthropicConfig(), "claude-haiku-5-5"), + ], + ids=["anthropic", "vertex_ai", "bedrock_invoke", "azure_ai"], +) +@pytest.mark.parametrize( + ("drop_params", "expected_cached_texts"), + [ + (True, ["answer 1", "turn 1", "turn 2", "turn 3"]), + ("true", ["answer 1", "turn 1", "turn 2", "turn 3"]), + (False, ["answer 1", "system prompt", "turn 1", "turn 2", "turn 3"]), + ], + ids=[ + "drop_params_keeps_the_newest_four", + "string_drop_params_keeps_the_newest_four", + "without_drop_params_all_five_pass_through", + ], +) +def test_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params( + config: AnthropicConfig, + model: str, + drop_params: bool | str, + expected_cached_texts: list[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + messages: Final = _five_cache_hint_conversation() + messages_before: Final = copy.deepcopy(messages) + + body: Final = config.transform_request( + model=model, + messages=messages, + optional_params={}, + litellm_params={"drop_params": drop_params}, + headers={}, + ) + + assert sorted(_cached_texts(body)) == expected_cached_texts + assert json.dumps(body).count('"cache_control"') == len(expected_cached_texts) + assert messages == messages_before + + +def test_claude_cache_control_cap_keeps_a_cached_tool_and_drops_the_oldest_message_hint( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + messages: Final = _five_cache_hint_conversation()[1:] + tool: Final = { + "name": "get_weather", + "description": "Get the weather", + "input_schema": {"type": "object", "properties": {}}, + "cache_control": {"type": "ephemeral"}, + } + + body: Final = AnthropicConfig().transform_request( + model="claude-haiku-5-5", + messages=messages, + optional_params={"tools": [tool]}, + litellm_params={}, + headers={}, + ) + + assert body["tools"][0]["cache_control"] == {"type": "ephemeral"} + assert sorted(_cached_texts(body["messages"])) == ["answer 1", "turn 2", "turn 3"] + + +def test_claude_cache_control_cap_drops_a_string_assistant_turns_text_hint_before_its_tool_call_hints( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + hint: Final = {"type": "ephemeral"} + tool_calls: Final = [ + {"id": f"call_{name}", "type": "function", "function": {"name": name, "arguments": "{}"}, "cache_control": hint} + for name in ("lookup", "search") + ] + follow_up: Final = [ + {"type": "text", "text": "follow-up", "cache_control": hint}, + {"type": "text", "text": "and more", "cache_control": hint}, + ] + messages: Final = [ + {"role": "user", "content": "question"}, + {"role": "assistant", "content": "checking both", "cache_control": hint, "tool_calls": tool_calls}, + {"role": "tool", "tool_call_id": "call_lookup", "content": "found"}, + {"role": "tool", "tool_call_id": "call_search", "content": "none"}, + {"role": "user", "content": follow_up}, + ] + + body: Final = AnthropicConfig().transform_request( + model="claude-haiku-5-5", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + assistant_blocks: Final = body["messages"][1]["content"] + assert [block["type"] for block in assistant_blocks] == ["text", "tool_use", "tool_use"] + assert ["cache_control" in block for block in assistant_blocks] == [False, True, True] + assert json.dumps(body).count('"cache_control"') == 4 + assert json.dumps(body).count('"cache_control"') == 4 + + +def test_claude_cache_control_cap_keeps_the_newest_hints_inside_one_message( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + messages: Final = [ + { + "role": "user", + "content": [ + *({"type": "text", "text": f"part {n}", "cache_control": {"type": "ephemeral"}} for n in range(1, 6)), + {"type": "text", "text": "Reply with the single word OK."}, + ], + } + ] + messages_before: Final = copy.deepcopy(messages) + + body: Final = AnthropicConfig().transform_request( + model="claude-haiku-5-5", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert sorted(_cached_texts(body["messages"])) == ["part 2", "part 3", "part 4", "part 5"] + assert json.dumps(body).count('"cache_control"') == 4 + assert messages == messages_before + + +def test_claude_cache_control_cap_does_not_count_a_message_level_hint_that_list_content_never_sends( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + messages: Final = [ + {"role": "system", "content": "system prompt", "cache_control": {"type": "ephemeral"}}, + {"role": "user", "content": "turn 1", "cache_control": {"type": "ephemeral"}}, + {"role": "assistant", "content": "answer 1"}, + {"role": "user", "content": "turn 2", "cache_control": {"type": "ephemeral"}}, + {"role": "assistant", "content": "answer 2"}, + { + "role": "user", + "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}], + "cache_control": {"type": "ephemeral"}, + }, + ] + + body: Final = AnthropicConfig().transform_request( + model="claude-haiku-5-5", + messages=messages, + optional_params={}, + litellm_params={"drop_params": True}, + headers={}, + ) + + assert sorted(_cached_texts(body)) == ["system prompt", "turn 1", "turn 2", "turn 3"] + assert json.dumps(body).count('"cache_control"') == 4 diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index d18ef28c68e..cf0b8a39865 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -56,6 +56,98 @@ def test_is_anthropic_messages_url(url: str, expected: bool) -> None: assert is_anthropic_messages_url(url) is expected +@pytest.mark.parametrize( + ("configured_base", "url", "expected"), + [ + pytest.param(None, "https://api.anthropic.com/v1/messages", 512, id="default-upstream"), + pytest.param(None, "https://api.anthropic.com:443/v1/messages/", 512, id="default-upstream-explicit-port"), + pytest.param("http://relay.test:8080", "https://api.anthropic.com/v1/messages", 512, id="direct-target"), + pytest.param("https://api.anthropic.com", "https://api.anthropic.com/v1/messages", 512, id="configured-default"), + pytest.param("http://relay.test:8080", "http://relay.test:8080/v1/messages", 255, id="configured-upstream"), + pytest.param("http://relay.test:8080/a", "http://relay.test:8080/a/v1/messages", 255, id="configured-prefix"), + pytest.param("https://Relay.Test", "https://relay.test/v1/messages", 255, id="configured-upper-case-host"), + pytest.param("https://relay.test:443", "https://relay.test/v1/messages", 255, id="configured-https-port"), + pytest.param("http://relay.test:80/a", "http://relay.test/a/v1/messages", 255, id="configured-http-port"), + pytest.param("http://relay.test:8080", "http://relay.test:8081/v1/messages", None, id="other-port"), + pytest.param("http://relay.test:notaport", "http://relay.test/v1/messages", None, id="invalid-configured-base"), + pytest.param(None, "https://other-vendor.test/v1/messages", None, id="other-vendor"), + pytest.param(None, "https://api.anthropic.com/v1/messages/count_tokens", None, id="count-tokens"), + ], +) +def test_anthropic_upstream_user_id_limit( + monkeypatch: pytest.MonkeyPatch, configured_base: str | None, url: str, expected: int | None +) -> None: + from litellm.llms.anthropic.common_utils import anthropic_upstream_user_id_limit + + monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) + monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False) + if configured_base is not None: + monkeypatch.setenv("ANTHROPIC_API_BASE", configured_base) + + assert anthropic_upstream_user_id_limit(url) == expected + + +def test_anthropic_pass_through_api_base_prefers_the_api_base_variable(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.llms.anthropic.common_utils import anthropic_pass_through_api_base + + monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://fallback.test") + monkeypatch.setenv("ANTHROPIC_API_BASE", "http://preferred.test") + assert anthropic_pass_through_api_base() == "http://preferred.test" + + monkeypatch.delenv("ANTHROPIC_API_BASE") + assert anthropic_pass_through_api_base() == "http://fallback.test" + + +# Each row was sent to https://api.anthropic.com/v1/messages on 2026-10-30, False rows answered 400. +@pytest.mark.parametrize( + ("user_id", "expected"), + [ + pytest.param("user-1", True, id="opaque"), + pytest.param("", True, id="empty"), + pytest.param('{"device_id":"abc","session_id":"def"}', True, id="claude-code-shape"), + pytest.param("1234567890", True, id="digits"), + pytest.param("+14155550100", True, id="phone-shape"), + pytest.param("a@b", True, id="no-dot"), + pytest.param("@b.co", True, id="nothing-before-at-sign"), + pytest.param("a@.co", True, id="nothing-between-at-sign-and-dot"), + pytest.param("a@b.", True, id="nothing-after-dot"), + pytest.param("a @ b.co", True, id="spaces-around-at-sign"), + pytest.param("a\u00a0@b.co", False, id="no-break-space-before-at-sign"), + pytest.param("a@b\n.co", False, id="newline-before-dot"), + pytest.param("line one\nline two", False, id="newline"), + pytest.param("a\tb", False, id="tab"), + pytest.param("a\x1f@b.co", False, id="unit-separator-before-at-sign"), + pytest.param("a@\x1cb.co", False, id="file-separator-after-at-sign"), + pytest.param("a\u200bb", False, id="zero-width-space"), + pytest.param("a@b.co", False, id="email"), + pytest.param("user a@b.co here", False, id="email-inside-text"), + pytest.param('{"device_id":"a@b.co"}', False, id="email-inside-json"), + pytest.param("a@@example.com", False, id="doubled-at-sign"), + pytest.param("@@example.com", False, id="leading-doubled-at-sign"), + pytest.param("a@b@c.co", False, id="two-at-signs"), + pytest.param("a@b..co", False, id="doubled-dot"), + pytest.param("a@.b.co", False, id="dot-after-at-sign"), + pytest.param("a@b.c d", False, id="email-before-space"), + ], +) +def test_anthropic_accepts_user_id(user_id: str, expected: bool) -> None: + from litellm.constants import ANTHROPIC_MAX_USER_ID_LENGTH + from litellm.llms.anthropic.common_utils import anthropic_accepts_user_id + + assert anthropic_accepts_user_id(user_id, ANTHROPIC_MAX_USER_ID_LENGTH) is expected + + +@pytest.mark.parametrize("character", ["a", "\u00e9", "\U0001f600"], ids=["ascii", "two-bytes", "four-bytes"]) +def test_anthropic_accepts_user_id_counts_characters_up_to_the_limit(character: str) -> None: + from litellm.constants import ANTHROPIC_MAX_USER_ID_LENGTH + from litellm.llms.anthropic.common_utils import anthropic_accepts_user_id + + assert anthropic_accepts_user_id(character * ANTHROPIC_MAX_USER_ID_LENGTH, ANTHROPIC_MAX_USER_ID_LENGTH) is True + assert ( + anthropic_accepts_user_id(character * (ANTHROPIC_MAX_USER_ID_LENGTH + 1), ANTHROPIC_MAX_USER_ID_LENGTH) is False + ) + + def test_anthropic_model_not_found_maps_to_not_found_but_authentication_error_does_not() -> None: with respx.mock(assert_all_called=True) as router: router.post("https://api.anthropic.com/v1/messages").mock( diff --git a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py index ec6f118837f..041c625eaef 100644 --- a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py +++ b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py @@ -26,7 +26,6 @@ from litellm.types.decisions import ( DecisionsIRScoreAnswer, DecisionsRequestBody, OpenAIDecisionRequestBody, - UnsupportedDecisionsRequest, ) from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager @@ -279,7 +278,7 @@ def test_a_malformed_systemone_response_names_the_field_at_fault( assert [item["loc"] for item in error.value.errors()] == [field] -def test_each_media_kind_is_gated_by_its_own_support_flag() -> None: +def test_each_media_kind_is_sent_in_its_own_field() -> None: audio: Final = ["data:audio/wav;base64,YQ=="] videos: Final = ["data:video/mp4;base64,YQ=="] audio_only: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "audio": audio})) @@ -287,22 +286,16 @@ def test_each_media_kind_is_gated_by_its_own_support_flag() -> None: _SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "videos": videos}) ) - assert ir_to_systemone_request("clef-omni", audio_only, supports_audio=True) == { + assert ir_to_systemone_request("clef-omni", audio_only) == { "model": "clef-omni", **_SYSTEMONE_REQUEST, "audio": audio, } - assert ir_to_systemone_request("clef-omni", video_only, supports_video=True) == { + assert ir_to_systemone_request("clef-omni", video_only) == { "model": "clef-omni", **_SYSTEMONE_REQUEST, "videos": videos, } - assert isinstance( - ir_to_systemone_request("clef-omni", audio_only, supports_video=True), UnsupportedDecisionsRequest - ) - assert isinstance( - ir_to_systemone_request("clef-omni", video_only, supports_audio=True), UnsupportedDecisionsRequest - ) _ASSET_REQUEST: Final[Mapping[str, object]] = { diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 524caaba7a1..9ab7060d766 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -11259,3 +11259,86 @@ def test_converse_sends_only_the_header_betas_when_header_and_body_both_carry_be "thinking-binding-controls-2026-08-01", "thinking-display-updates-2026-08-18", ] + + +def _five_cache_hint_conversation() -> list[dict[str, object]]: + return [ + {"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]}, + {"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]}, + {"role": "assistant", "content": "answer 2"}, + {"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]}, + ] + + +@pytest.mark.parametrize( + ("drop_params", "expected_system_cache_points"), + [(True, 0), ("true", 0), (False, 1)], + ids=[ + "drop_params_drops_the_oldest_hint", + "string_drop_params_drops_the_oldest_hint", + "without_drop_params_all_five_pass_through", + ], +) +def test_converse_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params( + drop_params: bool | str, expected_system_cache_points: int, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + messages: Final = _five_cache_hint_conversation() + messages_before: Final = copy.deepcopy(messages) + + request: Final = AmazonConverseConfig().transform_request( + model="anthropic.claude-haiku-5-5", + messages=messages, + optional_params={}, + litellm_params={"drop_params": drop_params}, + headers={}, + ) + + assert len(list(_cache_points(request["system"]))) == expected_system_cache_points + assert [len(list(_cache_points(message))) for message in request["messages"]] == [1, 1, 1, 0, 1] + assert messages == messages_before + + +def test_converse_claude_tool_message_with_several_marked_blocks_counts_as_one_cache_point_against_the_cap( + monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + messages: Final = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "policy", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "question", "cache_control": {"type": "ephemeral"}}, + ], + }, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}], + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [ + {"type": "text", "text": "part 1", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "part 2", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "part 3", "cache_control": {"type": "ephemeral"}}, + ], + }, + ] + messages_before: Final = copy.deepcopy(messages) + + request: Final = AmazonConverseConfig().transform_request( + model="anthropic.claude-haiku-5-5", + messages=messages, + optional_params={ + "tools": [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}] + }, + litellm_params={"drop_params": True}, + headers={}, + ) + + assert len(list(_cache_points(request["messages"]))) == 3 + assert messages == messages_before diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index 20c0e10cb81..38c69e0ce27 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -1,5 +1,5 @@ -import asyncio, importlib, litellm, litellm.litellm_core_utils.get_model_cost_map as bedrock_govcloud_model_cost_map, pytest +import asyncio, importlib, litellm, pytest from typing import Final @@ -1563,7 +1563,6 @@ def _pr4_bedrock_env(monkeypatch: pytest.MonkeyPatch) -> None: def _use_local_model_cost_map() -> Iterator[None]: with pytest.MonkeyPatch.context() as monkeypatch: monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - importlib.reload(bedrock_govcloud_model_cost_map) importlib.reload(litellm) yield diff --git a/tests/unit/llms/custom_httpx/test_aiohttp_transport.py b/tests/unit/llms/custom_httpx/test_aiohttp_transport.py index aed808223bd..4e4d5df9fb0 100644 --- a/tests/unit/llms/custom_httpx/test_aiohttp_transport.py +++ b/tests/unit/llms/custom_httpx/test_aiohttp_transport.py @@ -144,6 +144,49 @@ async def test_client_payload_error_mid_stream_raises_read_error(): assert mock_response.closed is True +@pytest.mark.asyncio +async def test_bytes_buffered_before_a_dropped_connection_are_yielded_before_the_read_error(): + loop: Final = asyncio.get_running_loop() + content: Final = aiohttp.StreamReader(aiohttp.base_protocol.BaseProtocol(loop), 2**16, loop=loop) + content.feed_data(b"event: message_start\n\n") + content.set_exception(aiohttp.ClientPayloadError("Response payload is not completed")) + mock_response: Final = MockAiohttpResponse() + mock_response.content = content + received: Final[list[bytes]] = [] + + async def _drain() -> None: + async for chunk in AiohttpResponseStream(mock_response): # pyright: ignore[reportArgumentType] # mock response + received.append(chunk) + + with pytest.raises(httpx.ReadError): + await _drain() + + assert received == [b"event: message_start\n\n"] + assert mock_response.closed is True + + +@pytest.mark.asyncio +async def test_bytes_already_read_before_a_dropped_connection_are_not_yielded_again(): + loop: Final = asyncio.get_running_loop() + content: Final = aiohttp.StreamReader(aiohttp.base_protocol.BaseProtocol(loop), 2**16, loop=loop) + content.feed_data(b"event: message_start\n\n") + content.feed_data(b"event: ping\n\n") + assert await content.read(7) == b"event: " + content.set_exception(aiohttp.ClientPayloadError("Response payload is not completed")) + mock_response: Final = MockAiohttpResponse() + mock_response.content = content + received: Final[list[bytes]] = [] + + async def _drain() -> None: + async for chunk in AiohttpResponseStream(mock_response): # pyright: ignore[reportArgumentType] # mock response + received.append(chunk) + + with pytest.raises(httpx.ReadError): + await _drain() + + assert received == [b"message_start\n\nevent: ping\n\n"] + + @pytest.mark.asyncio async def test_client_payload_error_before_first_chunk_raises_read_error(): """A connection reset before any body byte must surface, not yield an empty 200 body""" diff --git a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py index 19234767b19..7a6f9c7dc18 100644 --- a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py @@ -2773,3 +2773,66 @@ def test_claude_json_schema_stream_split_across_json_tools_keeps_the_non_stream_ choices: Final = _json_schema_stream_choices(respx_mock, call, split_stream) assert [choice.finish_reason for choice in choices if choice.finish_reason] == [answer.finish_reason] + + +@pytest.mark.parametrize( + ("model", "base_model"), + [("databricks-claude-haiku-5-5", None), ("support-bot-endpoint", "databricks/databricks-claude-haiku-5-5")], + ids=["claude_in_the_endpoint_name", "claude_named_only_by_base_model"], +) +def test_databricks_claude_request_over_the_cache_control_cap_keeps_the_newest_four_hints_under_drop_params( + model: str, base_model: str | None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + hint: Final = {"type": "ephemeral"} + messages: Final = [ + {"role": "system", "content": "system prompt", "cache_control": hint}, + {"role": "user", "content": "turn 1", "cache_control": hint}, + {"role": "assistant", "content": "answer 1", "cache_control": hint}, + {"role": "user", "content": "turn 2", "cache_control": hint}, + {"role": "assistant", "content": "answer 2"}, + {"role": "user", "content": "turn 3", "cache_control": hint}, + ] + messages_before: Final = copy.deepcopy(messages) + + body: Final = DatabricksConfig().transform_request( + model=model, + messages=messages, + optional_params={}, + litellm_params={"drop_params": True, "custom_llm_provider": "databricks", "base_model": base_model}, + headers={}, + ) + + assert "cache_control" not in json.dumps(body["messages"][0]) + assert json.dumps(body["messages"]).count('"cache_control"') == 4 + assert messages == messages_before + + +@pytest.mark.parametrize( + "optional_params", + [{"cache_control": {"type": "ephemeral"}}, {"extra_body": {"cache_control": {"type": "ephemeral"}}}], + ids=["top_level_cache_control", "extra_body_cache_control"], +) +def test_databricks_claude_cache_control_cap_keeps_room_for_a_top_level_cache_control_under_drop_params( + optional_params: dict[str, object], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + hint: Final = {"type": "ephemeral"} + messages: Final = [ + {"role": "user", "content": "turn 1", "cache_control": hint}, + {"role": "assistant", "content": "answer 1", "cache_control": hint}, + {"role": "user", "content": "turn 2", "cache_control": hint}, + {"role": "assistant", "content": "answer 2", "cache_control": hint}, + {"role": "user", "content": "turn 3"}, + ] + + body: Final = DatabricksConfig().transform_request( + model="databricks-claude-haiku-5-5", + messages=messages, + optional_params=copy.deepcopy(optional_params), + litellm_params={"drop_params": True, "custom_llm_provider": "databricks"}, + headers={}, + ) + + assert "cache_control" not in json.dumps(body["messages"][0]) + assert json.dumps(body["messages"]).count('"cache_control"') == 3 diff --git a/tests/unit/llms/deepinfra/test_deepinfra_chat_transformation.py b/tests/unit/llms/deepinfra/test_deepinfra_chat_transformation.py index 7ec9216df7b..773c1d056f0 100644 --- a/tests/unit/llms/deepinfra/test_deepinfra_chat_transformation.py +++ b/tests/unit/llms/deepinfra/test_deepinfra_chat_transformation.py @@ -1,6 +1,8 @@ import asyncio +import copy import json import os +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -200,3 +202,38 @@ async def test_deepinfra_tool_message_content_transformation_async(): print(f"✓ Async test passed: {tool_message['content']}") print("\n✅ DeepInfra async tool message transformation test passed!") + + +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +async def test_deepinfra_request_stringifies_tool_content_without_touching_the_callers_message(is_async: bool) -> None: + from litellm.llms.deepinfra.chat.transformation import DeepInfraConfig + + config: Final = DeepInfraConfig() + messages: Final = [ + {"role": "user", "content": "Calculate 10 + 10"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "calculator", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": [{"type": "text", "text": "20"}]}, + ] + messages_before: Final = copy.deepcopy(messages) + request_kwargs: Final = { + "model": "deepseek-ai/DeepSeek-V3.2", + "messages": messages, + "optional_params": {}, + "litellm_params": {}, + "headers": {}, + } + + body: Final = ( + await config.async_transform_request(**request_kwargs) + if is_async + else config.transform_request(**request_kwargs) + ) + + assert body["messages"][2]["content"] == "20" + assert messages == messages_before diff --git a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index ed166d206c8..dd352aac66d 100644 --- a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -1,3 +1,4 @@ +import copy import json import logging from collections.abc import Callable @@ -537,6 +538,26 @@ def test_tool_search_reply_sent_to_fireworks_drops_litellm_response_fields(): } +def test_cache_control_is_stripped_from_the_request_but_kept_on_the_callers_messages(): + hint: Final = {"type": "ephemeral"} + messages: Final = [ + {"role": "system", "content": [{"type": "text", "text": "a long cached prompt", "cache_control": hint}]}, + {"role": "user", "content": "hi", "cache_control": hint}, + ] + caller_messages_before: Final = json.dumps(messages) + + request: Final = FireworksAIConfig().transform_request( + model="accounts/fireworks/models/gpt-oss-120b", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert "cache_control" not in json.dumps(request["messages"]) + assert json.dumps(messages) == caller_messages_before + + # ----------------------------------------------------------------------------- # Regression tests for legacy / OpenAPI $ref defs in tool parameters. # @@ -2477,3 +2498,43 @@ def test_translate_developer_role_follows_the_deployment_model_info_over_the_cos ) assert [message["role"] for message in messages] == expected_roles + + +def test_fireworks_request_rewrites_tools_without_touching_the_callers_tool() -> None: + tools: Final = [ + { + "type": "function", + "function": { + "name": "create_thing", + "strict": True, + "parameters": { + "type": "object", + "properties": {"thing": {"$ref": "#/definitions/Thing"}}, + "definitions": {"Thing": {"type": "object", "properties": {"id": {"type": "string"}}}}, + }, + }, + } + ] + tools_before: Final = copy.deepcopy(tools) + + request: Final = FireworksAIConfig().transform_request( + model="accounts/fireworks/models/gpt-oss-120b", + messages=[{"role": "user", "content": "Make a thing"}], + optional_params={"tools": tools}, + litellm_params={}, + headers={}, + ) + + assert request["tools"] == [ + { + "type": "function", + "function": { + "name": "create_thing", + "parameters": { + "type": "object", + "properties": {"thing": {"type": "object", "properties": {"id": {"type": "string"}}}}, + }, + }, + } + ] + assert tools == tools_before diff --git a/tests/unit/llms/groq/chat/test_groq_chat_transformation.py b/tests/unit/llms/groq/chat/test_groq_chat_transformation.py index 7aa8c65215e..a493a259f39 100644 --- a/tests/unit/llms/groq/chat/test_groq_chat_transformation.py +++ b/tests/unit/llms/groq/chat/test_groq_chat_transformation.py @@ -1,3 +1,4 @@ +import copy import logging from typing import Final from unittest.mock import patch @@ -589,3 +590,19 @@ async def test_streamed_reasoning_reply_can_be_replayed(is_async: bool, with_too } ] assert reply.reasoning_content == "I should check the weather." + + +def test_groq_request_drops_null_assistant_fields_without_touching_the_callers_messages() -> None: + messages: Final = [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello", "function_call": None, "tool_calls": None}, + {"role": "user", "content": "Again"}, + ] + messages_before: Final = copy.deepcopy(messages) + + body: Final = GroqChatConfig().transform_request( + model="openai/gpt-oss-120b", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert body["messages"][1] == {"role": "assistant", "content": "Hello"} + assert messages == messages_before diff --git a/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py b/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py index c792d5dffcd..fc5ae3631ff 100644 --- a/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py +++ b/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py @@ -1,4 +1,6 @@ +import copy import json +from typing import Final from unittest.mock import MagicMock, patch import pytest @@ -397,3 +399,43 @@ def test_hosted_vllm_custom_tools_use_top_level_input_schema(): assert tools[0]["function"]["name"] == "search" assert tools[0]["function"]["description"] == "Search docs" assert tools[0]["function"]["parameters"] == input_schema + + +def _strict_weather_tool() -> dict[str, object]: + return { + "type": "function", + "function": { + "name": "get_weather", + "strict": True, + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + "additionalProperties": False, + }, + }, + } + + +def test_hosted_vllm_tools_are_rewritten_without_touching_the_callers_tool() -> None: + tools: Final = [_strict_weather_tool()] + tools_before: Final = copy.deepcopy(tools) + + optional_params: Final = HostedVLLMChatConfig().map_openai_params( + non_default_params={"tools": tools}, optional_params={}, model="gpt-oss-120b", drop_params=False + ) + + assert optional_params["tools"] == [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ] + assert tools == tools_before diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index 720a1b4f458..af18a9caf3d 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -1,3 +1,4 @@ +import copy import json import logging from collections.abc import Mapping @@ -1212,7 +1213,10 @@ def test_mistral_sends_only_the_text_items_of_a_native_thinking_chunk_beside_an_ "content": [ { "type": "thinking", - "thinking": [{"type": "text", "text": "It has whiskers."}, {"type": "reference", "reference_ids": [1]}], + "thinking": [ + {"type": "text", "text": "It has whiskers."}, + {"type": "reference", "reference_ids": [1]}, + ], }, {"type": "text", "text": "A cat."}, ], @@ -1672,6 +1676,18 @@ def test_mistral_transform_request_drops_an_empty_tool_message_whose_call_id_is_ assert [m.get("role") for m in request["messages"]] == ["user", "assistant"] +def test_mistral_request_drops_a_user_name_without_touching_the_callers_message() -> None: + messages: Final = [{"role": "user", "content": "Hi"}, {"role": "user", "content": "", "name": "alice"}] + messages_before: Final = copy.deepcopy(messages) + + body: Final = MistralConfig().transform_request( + model="mistral-medium-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert "name" not in json.dumps(body["messages"]) + assert messages == messages_before + + def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: return tuple( r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 5b85a94b24d..79fe7061297 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -1824,6 +1824,94 @@ class TestToolSchemaCombinatorFlatteningForOpenAI: assert request["tools"][0] is tool +class TestCacheControlStrippingDoesNotMutateCallerInput: + """ + Stripping cache_control for a provider that cannot use it must not reach back + into the caller's own message and tool objects. + + filter_value_from_dict deletes the key in place and recurses into nested dicts + and lists, and remove_cache_control_flag_from_messages_and_tools assigned the + result back into the caller's list, so one call to any OpenAI-compatible + provider permanently stripped cache_control from a list the caller still holds. + Reusing that list on Anthropic or Bedrock afterwards then silently lost prompt + caching, with no error and full-price billing. + """ + + def setup_method(self): + self.config = OpenAIGPTConfig() + + @pytest.fixture(autouse=True) + def _clean_openai_base_env(self, monkeypatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None, raising=False) + + @staticmethod + def _messages(): + return [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "a long cached system prompt", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + { + "role": "user", + "content": "Hello", + "cache_control": {"type": "ephemeral"}, + }, + ] + + @staticmethod + def _tools(): + return [ + { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object"}}, + "cache_control": {"type": "ephemeral"}, + } + ] + + def _transform(self, messages, tools): + return self.config.transform_request( + model="gpt-4o", + messages=messages, + optional_params={"tools": tools}, + litellm_params={"custom_llm_provider": "openai", "api_base": None}, + headers={}, + ) + + def test_caller_messages_and_tools_keep_cache_control(self): + messages, tools = self._messages(), self._tools() + messages_before, tools_before = copy.deepcopy(messages), copy.deepcopy(tools) + + request = self._transform(messages, tools) + + assert "cache_control" not in json.dumps(request) + assert messages == messages_before + assert tools == tools_before + + def test_a_later_anthropic_call_still_sees_cache_control(self): + """The user-visible consequence: the same message list reused on a provider + that does support caching must still carry the cache breakpoints.""" + messages = self._messages() + + self._transform(messages, self._tools()) + + anthropic_body = litellm.AnthropicConfig().transform_request( + model="claude-3-5-sonnet-20240620", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + assert "cache_control" in json.dumps(anthropic_body) + + def _developer_message_after_a_user_turn() -> list[dict[str, str]]: return [ {"role": "system", "content": "You are terse."}, @@ -1916,3 +2004,53 @@ def test_completion_leaves_the_callers_closing_developer_message_unchanged_on_a_ ) assert json.loads(chat.calls.last.request.content)["messages"][-1]["role"] == "user" assert messages == sent + + +def _user_turn_with_litellm_only_media_fields() -> list[dict[str, object]]: + return [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe these"}, + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png", "format": "image/png"}}, + {"type": "image_url", "image_url": "https://example.com/dog.png"}, + { + "type": "file", + "file": {"file_data": "data:application/pdf;base64,JVBERi0=", "format": "application/pdf"}, + }, + ], + } + ] + + +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +async def test_openai_request_rewrites_media_parts_without_touching_the_callers_message( + is_async: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None, raising=False) + config: Final = OpenAIGPTConfig() + messages: Final = _user_turn_with_litellm_only_media_fields() + messages_before: Final = copy.deepcopy(messages) + request_kwargs: Final = { + "model": "gpt-5.5", + "messages": messages, + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai", "api_base": None}, + "headers": {}, + } + + body: Final = ( + await config.async_transform_request(**request_kwargs) + if is_async + else config.transform_request(**request_kwargs) + ) + + assert body["messages"][0]["content"] == [ + {"type": "text", "text": "Describe these"}, + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}, + {"type": "image_url", "image_url": {"url": "https://example.com/dog.png"}}, + {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0=", "filename": "my_file.pdf"}}, + ] + assert messages == messages_before diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index f5a33f87fd0..41c54bf6cc9 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -1,3 +1,4 @@ +import copy import json import logging from collections.abc import Callable, Mapping @@ -541,6 +542,50 @@ class TestOpenAIResponsesAPIConfig: assert "cache_control" not in result["tools"][0] assert result["tools"][0]["name"] == "get_weather" + def test_transform_does_not_strip_cache_control_from_the_callers_input(self): + """Stripping for OpenAI must not reach back into the caller's own objects. + + `filter_value_from_dict` deletes the key in place and recurses, and + `_validate_input_param` passes plain dict items through by reference, so + the caller's input list used to lose its cache breakpoints. Reusing that + list on Anthropic or Bedrock afterwards then silently lost prompt caching. + Same defect as the Chat Completions path. + """ + input_with_cache_control = [ + { + "role": "system", + "content": [ + { + "type": "input_text", + "text": "a long cached system prompt", + "cache_control": {"type": "ephemeral"}, + } + ], + } + ] + tools_with_cache_control = [ + { + "type": "function", + "name": "get_weather", + "parameters": {"type": "object"}, + "cache_control": {"type": "ephemeral"}, + } + ] + input_before = copy.deepcopy(input_with_cache_control) + tools_before = copy.deepcopy(tools_with_cache_control) + + result = self.config.transform_responses_api_request( + model=self.model, + input=input_with_cache_control, + response_api_optional_request_params={"tools": tools_with_cache_control}, + litellm_params={}, + headers={}, + ) + + assert "cache_control" not in json.dumps(result) + assert input_with_cache_control == input_before + assert tools_with_cache_control == tools_before + def test_transform_preserves_input_without_cache_control(self): """Inputs without cache_control must pass through unmodified.""" input_clean = [ @@ -4607,10 +4652,10 @@ def test_cache_control_still_marks_a_part_whose_own_marker_is_malformed_under_dr } -def _caller_input_after_transform(model: str, marker: object) -> str: +def _hinted_caller_input(marker: object) -> list[dict[str, object]]: hint: Final = {"type": "ephemeral"} marked: Final = {} if marker is None else {"prompt_cache_breakpoint": marker} - input: Final = [ + return [ { "type": "message", "role": "user", @@ -4626,6 +4671,10 @@ def _caller_input_after_transform(model: str, marker: object) -> str: "content": [{"type": "output_text", "text": "ok", "cache_control": hint, **marked}], }, ] + + +def _caller_input_after_transform(model: str, marker: object) -> str: + input: Final = _hinted_caller_input(marker) _CONFIGS[0].transform_responses_api_request( model=model, input=cast("ResponseInputParam", input), # cast-ok: plain dicts shaped like the SDK's input items @@ -4647,6 +4696,15 @@ def test_a_marker_leaves_the_caller_request_that_fallbacks_and_logs_read_as_an_u assert _caller_input_after_transform(model, marker).replace(marker_json, "") == unmarked +@pytest.mark.parametrize("model", ("gpt-4.1", "gpt-6.1-sol")) +@pytest.mark.parametrize("marker", (None, _VALID_MARKERS[0], "yes"), ids=("unmarked", "explicit", "malformed")) +def test_the_caller_input_keeps_its_cache_control_hints_for_a_fallback_deployment( + monkeypatch: pytest.MonkeyPatch, model: str, marker: object +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + assert _caller_input_after_transform(model, marker) == json.dumps(_hinted_caller_input(marker)) + + @pytest.mark.asyncio @pytest.mark.usefixtures(httpx_transport.__name__) async def test_aresponses_forwards_previous_response_id_to_openai() -> None: diff --git a/tests/unit/llms/openai/test_o_series_transformation.py b/tests/unit/llms/openai/test_o_series_transformation.py index e78e1a6f12b..9d39dfda60a 100644 --- a/tests/unit/llms/openai/test_o_series_transformation.py +++ b/tests/unit/llms/openai/test_o_series_transformation.py @@ -1,3 +1,4 @@ +import copy from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -198,3 +199,23 @@ def test_translate_developer_role_keeps_developer_messages_untouched() -> None: ) == messages ) + + +def test_o_series_request_turns_system_into_user_without_touching_the_callers_messages( + monkeypatch: pytest.MonkeyPatch, +) -> None: + model: Final = "o-series-test-model-without-system-role" + monkeypatch.setitem( + litellm.model_cost, + model, + {"litellm_provider": "openai", "mode": "chat", "supports_system_messages": False}, + ) + messages: Final = [{"role": "system", "content": "Be terse"}, {"role": "user", "content": "Hi"}] + messages_before: Final = copy.deepcopy(messages) + + body: Final = OpenAIOSeriesConfig().transform_request( + model=model, messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert [message["role"] for message in body["messages"]] == ["user", "user"] + assert messages == messages_before diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index 3e1cbad9308..77a7c3b83fd 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -24,6 +24,7 @@ from litellm.llms.openai.openai import ( AsyncAssistantEventHandler, AsyncCursorPage, MessageData, + OpenAIBatchesAPI, OpenAIChatCompletion, OpenAIConfig, OpenAIMessage as Message, @@ -2320,3 +2321,43 @@ def test_openai_config_hoists_a_later_developer_message_only_for_a_model_off_ope model_info=None, ) assert [message["role"] for message in messages] == expected_roles + + +@pytest.mark.asyncio +async def test_sync_batch_list_given_an_async_client_returns_a_page_the_caller_can_await( + http_backend: ModuleType, +): + def respond(request: _Request) -> httpx.Response | SDKResponse: + assert str(request.url).startswith("http://vllm-router.test/v1/batches") + return http_backend.Response( + 200, + json={ + "object": "list", + "data": [ + { + "id": "batch_5f8dac", + "status": "pending", + "input_file_id": "file-b73f4c", + "created_at": 1793239207, + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + } + ], + "first_id": "batch_5f8dac", + "last_id": "batch_5f8dac", + "has_more": False, + }, + ) + + async with http_backend.AsyncClient(transport=http_backend.MockTransport(respond)) as http_client: + page: Final = OpenAIBatchesAPI().list_batches( + _is_async=False, + api_key="sk-test", + api_base="http://vllm-router.test/v1", + timeout=5.0, + max_retries=0, + organization=None, + client=AsyncOpenAI(api_key="sk-test", base_url="http://vllm-router.test/v1", http_client=http_client), + ) + + assert [(batch.id, batch.status) for batch in (await page).data] == [("batch_5f8dac", "pending")] diff --git a/tests/unit/llms/openai_like/test_hostinger_router_provider.py b/tests/unit/llms/openai_like/test_hostinger_router_provider.py new file mode 100644 index 00000000000..cbc16c8567a --- /dev/null +++ b/tests/unit/llms/openai_like/test_hostinger_router_provider.py @@ -0,0 +1,706 @@ +import copy +import json +from typing import Final + +import pytest +import respx + +import litellm +from litellm.caching.llm_caching_handler import LLMClientCache + + +def test_hostinger_router_provider_resolution(monkeypatch: pytest.MonkeyPatch): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + monkeypatch.setenv("HOSTINGER_ROUTER_API_KEY", "hostinger-test-key") + + model, provider, api_key, api_base = get_llm_provider( + model="hostinger_router/claude-sonnet-5", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert model == "claude-sonnet-5" + assert provider == "hostinger_router" + assert api_key == "hostinger-test-key" + assert api_base == "https://router.hostinger.com/v1" + + +def test_hostinger_router_provider_keeps_explicit_credentials(monkeypatch: pytest.MonkeyPatch): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + monkeypatch.setenv("HOSTINGER_ROUTER_API_KEY", "hostinger-env-key") + + _, provider, api_key, api_base = get_llm_provider( + model="hostinger_router/claude-sonnet-5", + custom_llm_provider=None, + api_base="https://router.internal.example/v1", + api_key="hostinger-explicit-key", + ) + + assert provider == "hostinger_router" + assert api_key == "hostinger-explicit-key" + assert api_base == "https://router.internal.example/v1" + + +@pytest.mark.parametrize("model", ["gpt-5-nano", "claude-sonnet-5"]) +def test_hostinger_router_api_base_keeps_unprefixed_model_provider(model: str): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + _, provider_without_api_base, _, _ = get_llm_provider(model=model) + _, provider_with_hostinger_api_base, _, _ = get_llm_provider( + model=model, + api_base="https://router.hostinger.com/v1", + api_key="hostinger-test-key", + ) + + assert provider_with_hostinger_api_base == provider_without_api_base + + +HOSTINGER_ROUTER_MODELS = tuple(sorted(name for name in litellm.model_cost if name.startswith("hostinger_router/"))) +HOSTINGER_ROUTER_CHAT_MODELS = tuple( + name for name in HOSTINGER_ROUTER_MODELS if litellm.get_model_info(name)["mode"] == "chat" +) + + +@pytest.mark.parametrize("model", HOSTINGER_ROUTER_CHAT_MODELS) +def test_hostinger_router_model_cost_and_capabilities(model: str): + from litellm.cost_calculator import cost_per_token + + prompt_cost, completion_cost = cost_per_token( + model=model, + prompt_tokens=1_000_000, + completion_tokens=1_000_000, + custom_llm_provider="hostinger_router", + ) + model_info = litellm.get_model_info(model) + + assert prompt_cost == pytest.approx(model_info["input_cost_per_token"] * 1_000_000) + assert completion_cost == pytest.approx(model_info["output_cost_per_token"] * 1_000_000) + assert model_info["output_cost_per_token"] > 0 + assert model_info["max_tokens"] == model_info["max_output_tokens"] <= model_info["max_input_tokens"] + assert model_info["litellm_provider"] == "hostinger_router" + assert model_info["mode"] == "chat" + if model_info.get("supports_prompt_caching"): + assert 0 < model_info["cache_read_input_token_cost"] < model_info["input_cost_per_token"] + capability_checks: Final = ( + ("supports_function_calling", litellm.supports_function_calling), + ("supports_reasoning", litellm.supports_reasoning), + ("supports_response_schema", litellm.supports_response_schema), + ("supports_vision", litellm.supports_vision), + ) + assert {name: check(model) for name, check in capability_checks} == { + name: model_info.get(name) is True for name, _ in capability_checks + } + + +def test_hostinger_router_chat_completions_request(): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond( + 200, + json={ + "id": "chatcmpl_hostinger", + "object": "chat.completion", + "created": 1_789_550_000, + "model": "claude-sonnet-5", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello from Hostinger Router"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}, + }, + ) + response: Final = litellm.completion( + model="hostinger_router/claude-sonnet-5", + messages=[{"role": "user", "content": "Say hello"}], + api_key="hostinger-test-key", + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert request.headers["authorization"] == "Bearer hostinger-test-key" + assert body["model"] == "claude-sonnet-5" + assert response.choices[0].message.content == "Hello from Hostinger Router" + + +def test_hostinger_router_chat_completions_streaming_request(): + chunks = [ + 'data: {"id":"chatcmpl_hostinger","object":"chat.completion.chunk","created":1789550000,"model":"claude-sonnet-5","choices":[{"index":0,"delta":{"role":"assistant","content":"Hi"},"finish_reason":null}]}\n\n', + 'data: {"id":"chatcmpl_hostinger","object":"chat.completion.chunk","created":1789550000,"model":"claude-sonnet-5","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":4,"completion_tokens":1,"total_tokens":5}}\n\n', + "data: [DONE]\n\n", + ] + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond( + 200, + headers={"content-type": "text/event-stream"}, + content="".join(chunks).encode(), + ) + response: Final = litellm.completion( + model="hostinger_router/claude-sonnet-5", + messages=[{"role": "user", "content": "Say hello"}], + stream=True, + api_key="hostinger-test-key", + ) + collected = [chunk for chunk in response] + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert body["stream"] is True + assert any(chunk.choices and chunk.choices[0].delta.content == "Hi" for chunk in collected) + + +def test_hostinger_router_responses_request(): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/responses").respond( + 200, + json={ + "id": "resp_hostinger", + "object": "response", + "created_at": 1_789_550_000, + "model": "gpt-5.2", + "status": "completed", + "output": [ + { + "id": "msg_hostinger", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "Hello from Hostinger Router", "annotations": []}], + } + ], + "usage": {"input_tokens": 4, "output_tokens": 3, "total_tokens": 7}, + }, + ) + response: Final = litellm.responses( + model="hostinger_router/gpt-5.2", + input="Say hello", + api_key="hostinger-test-key", + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert str(request.url) == "https://router.hostinger.com/v1/responses" + assert request.headers["authorization"] == "Bearer hostinger-test-key" + assert body["model"] == "gpt-5.2" + assert body["input"] == "Say hello" + assert response.output[0].content[0].text == "Hello from Hostinger Router" + + +EMBEDDING_RESPONSE: Final = { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}], + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 2, "total_tokens": 2}, +} + + +def test_hostinger_router_embeddings_request(): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + response: Final = litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert request.headers["authorization"] == "Bearer hostinger-test-key" + assert body["model"] == "text-embedding-3-small" + assert response.data[0]["embedding"] == [0.1, 0.2, 0.3] + + +def test_hostinger_router_embeddings_send_extra_headers_as_http_headers(): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + extra_headers={"X-Trace-Id": "trace-123"}, + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert request.headers["x-trace-id"] == "trace-123" + assert request.headers["authorization"] == "Bearer hostinger-test-key" + assert request.headers["content-type"] == "application/json" + assert set(body) == {"model", "input"} + + +def test_hostinger_router_embeddings_extra_headers_replace_key_headers_of_any_case(): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + extra_headers={"authorization": "Bearer caller-key", "x-trace-id": "trace-1"}, + ) + + request: Final = route.calls.last.request + assert request.headers.get_list("authorization") == ["Bearer caller-key"] + assert request.headers.get_list("content-type") == ["application/json"] + assert request.headers.get_list("x-trace-id") == ["trace-1"] + + +@pytest.mark.parametrize(("configured", "sent"), [(12345, "12345"), (1.5, "1.5")]) +def test_hostinger_router_embeddings_send_numeric_extra_headers_as_text(configured: int | float, sent: str): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + extra_headers={"X-Org-Id": configured}, + ) + + assert route.calls.last.request.headers.get_list("x-org-id") == [sent] + + +@pytest.mark.parametrize("configured", [True, None, ["a", "b"]]) +def test_hostinger_router_embeddings_refuse_non_text_extra_headers(configured: object): + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + with pytest.raises(litellm.UnsupportedParamsError, match="X-Debug") as refused: + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + extra_headers={"X-Debug": configured}, + ) + + assert refused.value.status_code == 400 + assert route.call_count == 0 + + +@pytest.mark.parametrize("configured", [True, None, ["a", "b"]]) +def test_hostinger_router_embeddings_drop_non_text_extra_headers_under_drop_params(configured: object): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + extra_headers={"X-Debug": configured, "X-Trace-Id": "trace-1"}, + drop_params=True, + ) + + request: Final = route.calls.last.request + assert "x-debug" not in request.headers + assert request.headers.get_list("x-trace-id") == ["trace-1"] + assert set(json.loads(request.content)) == {"model", "input"} + + +TRANSPORT_EXTRA_HEADERS: Final = [ + ("Host", "other.example"), + ("content-length", "1"), + ("Transfer-Encoding", "chunked"), + ("Connection", "close"), + ("Keep-Alive", "timeout=5"), + ("Proxy-Connection", "close"), + ("TE", "gzip"), + ("Trailer", "X-Checksum"), + ("Upgrade", "websocket"), + ("Expect", "100-continue"), +] + + +@pytest.mark.parametrize(("name", "configured"), TRANSPORT_EXTRA_HEADERS) +def test_hostinger_router_embeddings_ignore_transport_extra_headers(name: str, configured: str): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + extra_headers={name: configured, "X-Trace-Id": "trace-1"}, + ) + + request: Final = route.calls.last.request + assert request.headers.get_list("host") == ["router.hostinger.com"] + assert request.headers.get_list("content-length") == [str(len(request.content))] + assert "transfer-encoding" not in request.headers + assert configured not in request.headers.get_list(name) + assert request.headers.get_list("x-trace-id") == ["trace-1"] + + +MALFORMED_HEADER_NAMES: Final = ["Host:evil.example", "X Note", "X-Caf\u00e9"] + + +@pytest.mark.parametrize("name", MALFORMED_HEADER_NAMES) +def test_hostinger_router_embeddings_refuse_malformed_extra_header_names(name: str): + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + with pytest.raises(litellm.UnsupportedParamsError) as refused: + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + extra_headers={name: "x"}, + ) + + assert refused.value.status_code == 400 + assert repr(name) in refused.value.message + assert route.call_count == 0 + + +@pytest.mark.parametrize("name", MALFORMED_HEADER_NAMES) +def test_hostinger_router_embeddings_drop_malformed_extra_header_names_under_drop_params(name: str): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + extra_headers={name: "x", "X-Trace-Id": "trace-1"}, + drop_params=True, + ) + + request: Final = route.calls.last.request + assert request.headers.get_list("host") == ["router.hostinger.com"] + assert request.headers.get_list("x-trace-id") == ["trace-1"] + assert name.lower() not in request.headers.keys() + + +@pytest.mark.parametrize("configured", ["a\r\nHost: evil.example", "line\nbreak", "caf\u00e9", "a\x7fb"]) +def test_hostinger_router_embeddings_drop_extra_header_values_http_cannot_carry(configured: str): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + extra_headers={"x-litellm-user_api_key_alias": configured, "X-Trace-Id": "trace-1"}, + ) + + request: Final = route.calls.last.request + assert request.headers.get_list("host") == ["router.hostinger.com"] + assert request.headers.get_list("x-trace-id") == ["trace-1"] + assert "x-litellm-user_api_key_alias" not in request.headers + + +@pytest.mark.parametrize( + ("configured", "sent"), + [(" x ", "x"), ("\t", ""), (" a\tb ", "a\tb"), ("env-secret\n", "env-secret"), ("\r\n x \r\n", "x")], +) +def test_hostinger_router_embeddings_trim_whitespace_around_extra_header_values(configured: str, sent: str): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + extra_headers={"X-Rig": configured}, + ) + + assert route.calls.last.request.headers.get_list("x-rig") == [sent] + + +def test_hostinger_router_embeddings_extra_headers_stay_with_their_request(): + router: Final = litellm.Router( + model_list=[ + { + "model_name": "shared", + "litellm_params": { + "model": "hostinger_router/text-embedding-3-small", + "api_key": "hostinger-test-key", + "headers": {"X-Org-Id": "org-1"}, + }, + } + ] + ) + + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + router.embedding( + model="shared", input=["first"], extra_headers={"Authorization": "Bearer caller-a", "X-Leak": "a"} + ) + router.embedding(model="shared", input=["second"]) + + first: Final = route.calls[0].request + second: Final = route.calls[1].request + assert first.headers.get_list("x-leak") == ["a"] + assert second.headers.get_list("authorization") == ["Bearer hostinger-test-key"] + assert "x-leak" not in second.headers + assert second.headers.get_list("x-org-id") == ["org-1"] + + +@pytest.mark.parametrize("param", ["headers", "extra_headers"]) +def test_hostinger_router_embeddings_send_headers_given_as_name_value_pairs(param: str): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + **{param: [["X-Trace-Id", "trace-1"]]}, + ) + + request: Final = route.calls.last.request + assert request.headers.get_list("x-trace-id") == ["trace-1"] + assert request.headers.get_list("authorization") == ["Bearer hostinger-test-key"] + + +@pytest.mark.parametrize("param", ["headers", "extra_headers"]) +def test_hostinger_router_embeddings_refuse_headers_that_are_not_an_object(param: str): + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + with pytest.raises(litellm.UnsupportedParamsError, match=rf"\b{param} must be a JSON object") as refused: + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + **{param: "X-Trace-Id: trace-1"}, + ) + + assert refused.value.status_code == 400 + assert route.call_count == 0 + + +@pytest.mark.parametrize("param", ["headers", "extra_headers"]) +def test_hostinger_router_embeddings_drop_headers_that_are_not_an_object_under_drop_params(param: str): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + drop_params=True, + **{param: "X-Trace-Id: trace-1"}, + ) + + request: Final = route.calls.last.request + assert "x-trace-id" not in request.headers + assert request.headers.get_list("authorization") == ["Bearer hostinger-test-key"] + assert set(json.loads(request.content)) == {"model", "input"} + + +def test_hostinger_router_embeddings_never_borrow_another_providers_key(monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("HOSTINGER_ROUTER_API_KEY", raising=False) + monkeypatch.setenv("OPENAI_LIKE_API_KEY", "other-provider-key") + monkeypatch.setenv("OPENAI_API_KEY", "other-provider-key") + + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + with pytest.raises(litellm.BadRequestError, match="Missing API Key"): + litellm.embedding(model="hostinger_router/text-embedding-3-small", input=["Say hello"]) + + assert route.call_count == 0 + + +def test_json_provider_without_embeddings_endpoint_refuses_embeddings(): + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + slug: Final = next( + slug + for slug in JSONProviderRegistry.list_providers() + if not JSONProviderRegistry.supports_embeddings(slug) and slug not in litellm.provider_list + ) + + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.route().respond(200, json=EMBEDDING_RESPONSE) + with pytest.raises(litellm.BadRequestError): + litellm.embedding(model=f"{slug}/text-embedding-3-small", input=["Say hello"], api_key="test-key") + + assert route.call_count == 0 + + +@pytest.mark.asyncio +async def test_hostinger_router_anthropic_messages_request(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/messages").respond( + 200, + json={ + "id": "msg_hostinger", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [{"type": "text", "text": "Hello from Hostinger Router"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 4, "output_tokens": 3}, + }, + ) + response: Final = await litellm.anthropic.messages.acreate( + model="hostinger_router/claude-sonnet-5", + messages=[{"role": "user", "content": "Say hello"}], + max_tokens=32, + api_key="hostinger-test-key", + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert str(request.url) == "https://router.hostinger.com/v1/messages" + assert request.headers["authorization"] == "Bearer hostinger-test-key" + assert request.headers["anthropic-version"] == "2023-06-01" + assert body["model"] == "claude-sonnet-5" + assert body["messages"] == [{"role": "user", "content": "Say hello"}] + assert response["content"][0]["text"] == "Hello from Hostinger Router" + + +CHAT_COMPLETION_RESPONSE: Final = { + "id": "chatcmpl_hostinger", + "object": "chat.completion", + "created": 1_789_550_000, + "model": "gpt-5.2", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello from Hostinger Router"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}, +} +UNSUPPORTED_ENDPOINT: Final = {"error": {"message": "Model does not support this endpoint."}} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ["gpt-5.2", "gpt-5-nano"]) +async def test_hostinger_router_messages_bridge_models_without_native_messages( + model: str, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + with respx.mock(assert_all_called=False) as upstream: + native: Final = upstream.post("https://router.hostinger.com/v1/messages").respond( + 403, json=UNSUPPORTED_ENDPOINT + ) + chat: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond( + 200, json=CHAT_COMPLETION_RESPONSE + ) + response: Final = await litellm.anthropic.messages.acreate( + model=f"hostinger_router/{model}", + messages=[{"role": "user", "content": "Say hello"}], + max_tokens=32, + api_key="hostinger-test-key", + ) + + assert native.call_count == 0 + assert chat.call_count == 1 + assert json.loads(chat.calls.last.request.content)["model"] == model + assert response["content"][0]["text"] == "Hello from Hostinger Router" + + +def test_hostinger_router_responses_bridge_models_without_native_responses(): + with respx.mock(assert_all_called=False) as upstream: + native: Final = upstream.post("https://router.hostinger.com/v1/responses").respond( + 403, json=UNSUPPORTED_ENDPOINT + ) + chat: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond( + 200, json=CHAT_COMPLETION_RESPONSE + ) + response: Final = litellm.responses( + model="hostinger_router/claude-sonnet-5", + input="Say hello", + api_key="hostinger-test-key", + ) + + assert native.call_count == 0 + assert chat.call_count == 1 + assert json.loads(chat.calls.last.request.content)["model"] == "claude-sonnet-5" + assert response.output[0].content[0].text == "Hello from Hostinger Router" + + +ANTHROPIC_MESSAGE_RESPONSE: Final = { + "id": "msg_hostinger", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [{"type": "text", "text": "Hello from Hostinger Router"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 4, "output_tokens": 3}, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "sibling_model_info", + [ + {"supported_endpoints": ["/v1/chat/completions", "/v1/messages", "/v1/responses"]}, + {"supported_endpoints": "/v1/messages"}, + ], +) +async def test_hostinger_router_endpoint_opt_in_stays_with_its_own_deployment( + sibling_model_info: dict[str, object], monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + deployment_params: Final = {"model": "hostinger_router/gpt-5-nano", "api_key": "hostinger-test-key"} + router: Final = litellm.Router( + model_list=[ + {"model_name": "opted-in", "litellm_params": deployment_params, "model_info": sibling_model_info}, + {"model_name": "plain", "litellm_params": deployment_params}, + ] + ) + + with respx.mock(assert_all_called=False) as upstream: + native_messages: Final = upstream.post("https://router.hostinger.com/v1/messages").respond( + 403, json=UNSUPPORTED_ENDPOINT + ) + native_responses: Final = upstream.post("https://router.hostinger.com/v1/responses").respond( + 403, json=UNSUPPORTED_ENDPOINT + ) + chat: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond( + 200, json=CHAT_COMPLETION_RESPONSE + ) + messages_response: Final = await router.aanthropic_messages( + model="plain", messages=[{"role": "user", "content": "Say hello"}], max_tokens=32 + ) + responses_response: Final = await router.aresponses(model="plain", input="Say hello") + + assert native_messages.call_count == 0 + assert native_responses.call_count == 0 + assert chat.call_count == 2 + assert messages_response["content"][0]["text"] == "Hello from Hostinger Router" + assert responses_response.output[0].content[0].text == "Hello from Hostinger Router" + + +@pytest.mark.asyncio +async def test_hostinger_router_deployment_cache_ttl_opt_in_keeps_one_hour_ttl(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "opted-in", + "litellm_params": { + "model": "hostinger_router/claude-haiku-4-5", + "api_base": "https://router.hostinger.com/v1", + "api_key": "hostinger-test-key", + }, + "model_info": {"supported_endpoints": ["/v1/messages"], "cache_control_ttl": True}, + } + ] + ) + one_hour_cache: Final = {"type": "ephemeral", "ttl": "1h"} + + with respx.mock() as upstream: + native: Final = upstream.post("https://router.hostinger.com/v1/messages").respond( + 200, json=ANTHROPIC_MESSAGE_RESPONSE + ) + await router.aanthropic_messages( + model="opted-in", + messages=[ + {"role": "user", "content": [{"type": "text", "text": "Say hello", "cache_control": one_hour_cache}]} + ], + max_tokens=32, + ) + + body: Final = json.loads(native.calls.last.request.content) + assert body["messages"][0]["content"][0]["cache_control"] == one_hour_cache diff --git a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py index 3d659641f54..6cef65f8a88 100644 --- a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py +++ b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py @@ -1,5 +1,8 @@ +import copy import json +from collections.abc import Iterator +from itertools import chain from typing import Final from unittest.mock import MagicMock @@ -874,6 +877,58 @@ def test_openrouter_streaming_reasoning_is_preserved() -> None: assert assembled.choices[0].message.reasoning_content == "Consider the evidence. Apply the rule." +def _five_message_level_cache_hints() -> list[dict[str, object]]: + return [ + {"role": "system", "content": "system prompt", "cache_control": {"type": "ephemeral"}}, + {"role": "user", "content": "turn 1", "cache_control": {"type": "ephemeral"}}, + {"role": "assistant", "content": "answer 1", "cache_control": {"type": "ephemeral"}}, + {"role": "user", "content": "turn 2", "cache_control": {"type": "ephemeral"}}, + {"role": "assistant", "content": "answer 2"}, + {"role": "user", "content": "turn 3", "cache_control": {"type": "ephemeral"}}, + ] + + +def _cached_texts(node: object) -> Iterator[str]: + if isinstance(node, dict): + yield from ((node["text"],) if "cache_control" in node and "text" in node else ()) + yield from chain.from_iterable(map(_cached_texts, node.values())) + elif isinstance(node, list): + yield from chain.from_iterable(map(_cached_texts, node)) + + +@pytest.mark.parametrize( + ("drop_params", "expected_cached_texts"), + [ + (True, ["answer 1", "turn 1", "turn 2", "turn 3"]), + ("true", ["answer 1", "turn 1", "turn 2", "turn 3"]), + (False, ["answer 1", "system prompt", "turn 1", "turn 2", "turn 3"]), + ], + ids=[ + "drop_params_keeps_the_newest_four", + "string_drop_params_keeps_the_newest_four", + "without_drop_params_all_five_pass_through", + ], +) +def test_openrouter_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params( + drop_params: bool | str, expected_cached_texts: list[str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + messages: Final = _five_message_level_cache_hints() + messages_before: Final = copy.deepcopy(messages) + + body: Final = OpenrouterConfig().transform_request( + model="anthropic/claude-haiku-5.5", + messages=messages, + optional_params={}, + litellm_params={"drop_params": drop_params, "custom_llm_provider": "openrouter"}, + headers={}, + ) + + assert sorted(_cached_texts(body["messages"])) == expected_cached_texts + assert json.dumps(body).count('"cache_control"') == len(expected_cached_texts) + assert messages == messages_before + + def _transform_openrouter_response(body: dict[str, object]) -> litellm.ModelResponse: return OpenrouterConfig().transform_response( model="openrouter/openai/gpt-4o-mini", diff --git a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py index 78b9e930001..bc8421c80f8 100644 --- a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py +++ b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py @@ -1,3 +1,4 @@ +import json from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -2507,3 +2508,46 @@ def test_check_and_create_cache_clamps_negative_creation_tokens() -> None: assert cache_name == "cachedContents/created" assert logging_obj.vertex_ai_cache_creation_input_tokens == 0 + + +def _recording_gemini_client(calls: list[httpx.Request]) -> HTTPHandler: + def route(request: httpx.Request) -> httpx.Response: + calls.append(request) + if request.url.path.endswith("/cachedContents"): + return httpx.Response(200, json={} if request.method == "GET" else _created_cache_body(3000)) + return httpx.Response( + 200, + json={ + "candidates": [{"content": {"role": "model", "parts": [{"text": "OK"}]}, "finishReason": "STOP"}], + "usageMetadata": {"promptTokenCount": 3000, "candidatesTokenCount": 1, "totalTokenCount": 3001}, + }, + ) + + return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(route))) + + +def test_a_plain_system_prompt_goes_into_the_gemini_cache_instead_of_beside_it(local_model_cost_map) -> None: + calls: Final[list[httpx.Request]] = [] + litellm.completion( + model="gemini/gemini-flash-latest", + messages=[ + {"role": "system", "content": "You are terse."}, + { + "role": "user", + "content": [ + {"type": "text", "text": "plain system prefix " * 3000, "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "Reply with OK only."}, + ], + }, + ], + api_key="test-key", + client=_recording_gemini_client(calls), + ) + + created: Final = next( + json.loads(call.content) for call in calls if call.method == "POST" and call.url.path.endswith("/cachedContents") + ) + generated: Final = next(json.loads(call.content) for call in calls if call.url.path.endswith(":generateContent")) + assert created["system_instruction"] == {"parts": [{"text": "You are terse."}]} + assert generated["cachedContent"] == "cachedContents/created" + assert "system_instruction" not in generated diff --git a/tests/unit/llms/watsonx/test_watsonx.py b/tests/unit/llms/watsonx/test_watsonx.py index 4afa717debe..31d2875d2b9 100644 --- a/tests/unit/llms/watsonx/test_watsonx.py +++ b/tests/unit/llms/watsonx/test_watsonx.py @@ -1,4 +1,5 @@ import asyncio +import copy import importlib import json from typing import Final, Optional, cast @@ -10,6 +11,7 @@ import pytest import litellm from litellm import completion, embedding from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.watsonx.chat.transformation import IBMWatsonXChatConfig from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -542,3 +544,43 @@ def test_watsonx_deployment_space_id_embedding(monkeypatch, watsonx_embedding_ca # nor space_id or project_id is required by wx.ai API when inferencing deployment assert "project_id" not in json_data and "space_id" not in json_data + + +def _strict_weather_tool() -> dict[str, object]: + return { + "type": "function", + "function": { + "name": "get_weather", + "strict": True, + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + "additionalProperties": False, + }, + }, + } + + +def test_watsonx_chat_tools_are_rewritten_without_touching_the_callers_tool() -> None: + tools: Final = [_strict_weather_tool()] + tools_before: Final = copy.deepcopy(tools) + + optional_params: Final = IBMWatsonXChatConfig().map_openai_params( + non_default_params={"tools": tools}, optional_params={}, model="ibm/granite-4-h-small", drop_params=False + ) + + assert optional_params["tools"] == [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ] + assert tools == tools_before diff --git a/tests/unit/llms/xai/test_xai_chat_transformation.py b/tests/unit/llms/xai/test_xai_chat_transformation.py index 69e620dbc8a..37086f797fa 100644 --- a/tests/unit/llms/xai/test_xai_chat_transformation.py +++ b/tests/unit/llms/xai/test_xai_chat_transformation.py @@ -1,3 +1,4 @@ +import copy import json import os from typing import Final @@ -581,3 +582,44 @@ def test_xai_grok_4_frequency_penalty_not_supported(model): """ supported_params = XAIChatConfig().get_supported_openai_params(model=model) assert "frequency_penalty" not in supported_params + + +def _strict_weather_tool() -> dict[str, object]: + return { + "type": "function", + "function": { + "name": "get_weather", + "strict": True, + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + "additionalProperties": False, + }, + }, + } + + +def test_xai_tools_are_rewritten_without_touching_the_callers_tool() -> None: + tools: Final = [_strict_weather_tool()] + tools_before: Final = copy.deepcopy(tools) + + optional_params: Final = XAIChatConfig().map_openai_params( + non_default_params={"tools": tools}, optional_params={}, model="grok-4.3", drop_params=False + ) + + assert optional_params["tools"] == [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + "additionalProperties": False, + }, + }, + } + ] + assert tools == tools_before diff --git a/tests/unit/proxy/db/test_accounting_ledger.py b/tests/unit/proxy/db/test_accounting_ledger.py index 3916fe869e6..6ca719ed849 100644 --- a/tests/unit/proxy/db/test_accounting_ledger.py +++ b/tests/unit/proxy/db/test_accounting_ledger.py @@ -6,7 +6,7 @@ from typing import Final, Literal, NoReturn import pytest from litellm.accounting import InboundUsageEvent, Metric, MetricName, Outcome -from litellm.proxy.db.accounting_ledger import AccountingLedger +from litellm.proxy.db.accounting_ledger import AccountingConflictError, AccountingLedger, UsageWrite from litellm.types.utils import CallTypes @@ -101,3 +101,18 @@ async def test_unknown_delivery_target_is_rejected_before_database_access(event: ("unknown",), # pyright: ignore[reportArgumentType] # Exercise an unknown runtime target envelope=envelope, ) + + +@pytest.mark.asyncio +async def test_empty_usage_batches_do_not_open_a_transaction() -> None: + ledger: Final = AccountingLedger(_no_transaction) + await ledger.insert_usage_batch(()) + assert await ledger.load_usage_batch(()) == () + + +@pytest.mark.asyncio +async def test_conflicting_usage_within_one_batch_is_rejected_before_database_access(event: InboundUsageEvent) -> None: + original: Final = UsageWrite(event, json.dumps({"event": event.model_dump(mode="json")})) + changed: Final = UsageWrite(event, json.dumps({"event": event.model_dump(mode="json"), "pricing": {"rate": 2}})) + with pytest.raises(AccountingConflictError, match="conflicting copies"): + await AccountingLedger(_no_transaction).insert_usage_batch((original, changed)) diff --git a/tests/unit/proxy/db/test_exception_handler.py b/tests/unit/proxy/db/test_exception_handler.py index 75346ca08a6..d70d4403962 100644 --- a/tests/unit/proxy/db/test_exception_handler.py +++ b/tests/unit/proxy/db/test_exception_handler.py @@ -951,3 +951,33 @@ def test_is_lock_timeout_error_recognises_a_statement_cancelled_under_lock_timeo ) def test_is_lock_timeout_error_excludes_other_failures(error: Exception) -> None: assert PrismaDBExceptionHandler.is_lock_timeout_error(error) is False + + +@pytest.mark.parametrize( + ("body", "retryable"), + [ + ('{"error_code":"P1017","message":"Server has closed the connection."}', True), + ('{"error_code":"P1000","message":"Authentication failed."}', False), + ('{"error_code":"P2021","message":"Table does not exist."}', False), + ('{"error_code":"P9999","message":"Unknown engine failure."}', False), + ("invalid engine response", False), + ], +) +def test_engine_connection_closure_is_recoverable_without_allowing_unverified_auth( + body: str, + retryable: bool, +) -> None: + from prisma.http import Response + + error: Final = EngineRequestError(Response(httpx.Response(500)), body) + assert PrismaDBExceptionHandler.is_database_transport_error(error) is retryable + assert PrismaDBExceptionHandler.is_permanent_database_fault(error) is not retryable + assert PrismaDBExceptionHandler.is_database_connection_error(error) is False + + +def test_connection_closure_code_survives_prisma_data_error_wrapping() -> None: + error: Final = DataError( + data={"user_facing_error": {"error_code": "P1017", "message": "Server has closed the connection."}} + ) + assert PrismaDBExceptionHandler.is_database_transport_error(error) is True + assert PrismaDBExceptionHandler.is_permanent_database_fault(error) is False diff --git a/tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py index ee3ef3f5323..1b95cbf3c94 100644 --- a/tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -21,6 +21,7 @@ from litellm.proxy._types import ( LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, LiteLLM_UserTable, + LitellmUserRoles, ProxyException, SpecialHeaders, SpecialMCPServerNames, @@ -13196,3 +13197,58 @@ async def test_personal_key_mcp_access_is_denied_when_owner_teams_cannot_be_read assert await MCPRequestHandler.get_allowed_mcp_servers(personal_key_owner_teams) == [] assert await MCPRequestHandler.get_allowed_tools_for_server("srv-a", personal_key_owner_teams) == [] assert await MCPRequestHandler.get_allowed_tools_for_server("srv-open", personal_key_owner_teams) == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True) +@pytest.mark.parametrize( + "owner_role", + [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY], + ids=["proxy_admin", "proxy_admin_viewer"], +) +@pytest.mark.parametrize( + ("key_team_id", "expected_servers"), + [("team-a", {"srv-a", "srv-open"}), ("team-without-mcp", {"srv-open"})], + ids=["team_grants_one_server", "team_grants_no_server"], +) +@pytest.mark.parametrize( + "credential_shape", + [{"is_session_token": False}, {"is_session_token": True}, {"api_key": None, "jwt_claims": {"sub": "admin"}}], + ids=["virtual_key", "cli_session", "jwt"], +) +async def test_admin_owned_team_key_reaches_only_its_team_mcp_servers( + monkeypatch: pytest.MonkeyPatch, + personal_key_owner_teams: UserAPIKeyAuth, + owner_role: LitellmUserRoles, + key_team_id: str, + expected_servers: set[str], + credential_shape: dict[str, object], +) -> None: + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return _personal_key_team(team_id) if team_id in _PERSONAL_KEY_TEAMS else LiteLLM_TeamTable(team_id=team_id) + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + team_key: Final = personal_key_owner_teams.model_copy( + update={"team_id": key_team_id, "user_role": owner_role, **credential_shape} + ) + + assert set(await global_mcp_server_manager.get_allowed_mcp_servers(team_key)) == expected_servers + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True) +async def test_admin_dashboard_session_keeps_the_whole_mcp_registry( + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager + + dashboard_session: Final = personal_key_owner_teams.model_copy( + update={"team_id": UI_TEAM_ID, "user_role": LitellmUserRoles.PROXY_ADMIN} + ) + + assert set(await global_mcp_server_manager.get_allowed_mcp_servers(dashboard_session)) == set( + global_mcp_server_manager.get_registry() + ) diff --git a/tests/unit/proxy/mcp_server/test_mcp_toolset_scope.py b/tests/unit/proxy/mcp_server/test_mcp_toolset_scope.py index f3cc38eacf9..d1337138cbc 100644 --- a/tests/unit/proxy/mcp_server/test_mcp_toolset_scope.py +++ b/tests/unit/proxy/mcp_server/test_mcp_toolset_scope.py @@ -2,7 +2,7 @@ import asyncio from collections.abc import Awaitable, Callable -from typing import Dict, List, Optional +from typing import Dict, Final, List, Optional from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -112,6 +112,23 @@ class TestApplyToolsetScope: assert op.mcp_servers == ["server-a"] assert op.mcp_tool_permissions == toolset_perms + @pytest.mark.asyncio + @pytest.mark.parametrize( + "owner_role", + [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY], + ids=["proxy_admin", "proxy_admin_viewer"], + ) + async def test_admin_owned_team_key_without_the_toolset_grant_is_denied(self, owner_role: LitellmUserRoles): + from litellm.proxy.mcp_server.server import apply_toolset_scope + + team_key: Final = UserAPIKeyAuth( + api_key="sk-test", team_id="team-a", user_role=owner_role, object_permission=None + ) + with pytest.raises(HTTPException) as exc_info: + await apply_toolset_scope(team_key, "toolset-123", granted=_granted_through_team("toolset-other")) + + assert exc_info.value.status_code == 403 + @pytest.mark.asyncio async def test_team_granted_toolset_is_served_to_a_key_without_its_own_grant(self): """A team key whose own row carries no toolset grant is admitted to the toolset its team diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index a9098ac3d59..4fe358c38a3 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -31,7 +31,7 @@ from starlette.datastructures import UploadFile as StarletteUploadFile import litellm from litellm._logging import verbose_proxy_logger from litellm.caching.dual_cache import DualCache -from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS +from litellm.constants import ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH, ANTHROPIC_MAX_USER_ID_LENGTH, DEFAULT_REQUEST_TIMEOUT_SECONDS from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.prometheus import PrometheusLogger @@ -77,6 +77,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, EndpointType, PassthroughStandardLoggingPayload, + WebSocketSessionUsage, ) from tests._master_key import MASTER_KEY from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -6620,6 +6621,272 @@ async def test_pass_through_body_stays_byte_identical_outside_the_opt_in( assert "x-litellm-call-id" in response.headers +def _echo_upstream_body(upstream_request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, content=upstream_request.content, headers={"content-type": "application/json"}, request=upstream_request + ) + + +async def _relayed_body(response: Response) -> bytes: + if isinstance(response, StreamingResponse): + return b"".join([chunk async for chunk in response.body_iterator]) + return bytes(response.body) + + +async def _relay_through_pass_through( + client_body: Mapping[str, object], + target: str, + guardrails_config: dict[str, object] | None, + custom_body: dict[str, object] | None = None, +) -> Mapping[str, object]: + async def receive() -> Mapping[str, object]: + return {"type": "http.request", "body": json.dumps(dict(client_body)).encode()} + + request: Final = Request( + { + "type": "http", + "method": "POST", + "path": "/my-claude-route", + "query_string": b"", + "headers": [(b"content-type", b"application/json")], + }, + receive, + ) + fake_client, cleanup = _inject_fake_passthrough_client(httpx.MockTransport(_echo_upstream_body), timeout=None) + try: + with ExitStack() as stack: + _enter_relay_logging_mocks(stack, {}) + stack.enter_context( + patch("litellm.proxy.proxy_server.proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache())) + ) + response: Final = await pass_through_request( + request=request, + target=target, + custom_headers={}, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-relay-test"), + custom_body=custom_body, + guardrails_config=guardrails_config, + ) + relayed: Final = await _relayed_body(response) + finally: + cleanup() + await fake_client.aclose() + + assert response.status_code == 200 + return MappingProxyType(json.loads(relayed)) + + +_MESSAGES_BODY: Final = MappingProxyType( + {"model": "claude-sonnet-4-5", "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]} +) +_ANTHROPIC_MESSAGES: Final = "https://api.anthropic.com/v1/messages" + + +@pytest.fixture +def no_configured_anthropic_base(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +@pytest.mark.parametrize( + "client_metadata, target, forwarded_metadata", + [ + ({"user_id": "user-1"}, _ANTHROPIC_MESSAGES, {"user_id": "user-1"}), + ({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/", {"user_id": "user-1"}), + ({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}?beta=true", {"user_id": "user-1"}), + ( + {"user_id": "user-1", "tags": ["a"], "trace_id": "t", "generation_name": "g", "foo": "bar"}, + _ANTHROPIC_MESSAGES, + {"user_id": "user-1"}, + ), + ({"user_id": "1234567890"}, _ANTHROPIC_MESSAGES, {"user_id": "1234567890"}), + ({"tags": ["a"], "foo": "bar"}, _ANTHROPIC_MESSAGES, None), + ({"user_id": "a@example.com"}, _ANTHROPIC_MESSAGES, None), + ({"user_id": "x" * (ANTHROPIC_MAX_USER_ID_LENGTH + 1)}, _ANTHROPIC_MESSAGES, None), + ({"user_id": 42}, _ANTHROPIC_MESSAGES, None), + ({"user_id": "line one\nline two"}, _ANTHROPIC_MESSAGES, None), + ({"user_id": "a\x1f@example.com"}, _ANTHROPIC_MESSAGES, None), + ({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/count_tokens", None), + ({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/batches", None), + ({"user_id": "user-1"}, "https://api.anthropic.com/v1/models", None), + ({"user_id": "user-1"}, "https://other-vendor.test/anthropic/v1/messages", None), + ({"user_id": "user-1"}, "https://other-vendor.test/v1/responses", None), + ({"user_id": "user-1"}, "https://other-vendor.test/v1/chat/completions", None), + ], + ids=[ + "messages", + "messages-trailing-slash", + "messages-with-query", + "user-id-beside-litellm-and-unknown-keys", + "digits-user-id", + "no-user-id", + "email-user-id", + "over-long-user-id", + "integer-user-id", + "newline-user-id", + "control-character-before-at-sign", + "count-tokens", + "batches", + "other-anthropic-path", + "other-vendor-messages", + "other-vendor-responses", + "other-vendor-chat-completions", + ], +) +async def test_pass_through_forwards_only_an_acceptable_user_id_to_anthropic_messages( + stream: bool, client_metadata: object, target: str, forwarded_metadata: Mapping[str, object] | None +) -> None: + client_body: Final = MappingProxyType( + {**_MESSAGES_BODY, "metadata": client_metadata, **({"stream": True} if stream else {})} + ) + body_without_metadata: Final = {k: v for k, v in client_body.items() if k != "metadata"} + + relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None) + + assert relayed == ( + body_without_metadata if forwarded_metadata is None else {**body_without_metadata, "metadata": forwarded_metadata} + ) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +@pytest.mark.parametrize("base_env", ["ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"]) +@pytest.mark.parametrize( + "target, forwarded", + [ + ("http://gateway.test:8080/anthropic/v1/messages", True), + ("http://gateway.test:9090/v1/messages", False), + ("http://other-gateway.test:8080/v1/messages", False), + ], + ids=["configured-upstream", "other-port", "other-host"], +) +async def test_pass_through_treats_the_configured_anthropic_base_as_anthropic( + monkeypatch: pytest.MonkeyPatch, base_env: str, target: str, forwarded: bool +) -> None: + monkeypatch.setenv(base_env, "http://gateway.test:8080/anthropic") + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}}) + + relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None) + + assert relayed == (client_body if forwarded else _MESSAGES_BODY) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +@pytest.mark.parametrize( + "user_id_length, target, forwarded", + [ + (ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH, "http://gateway.test/v1/messages", True), + (ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH + 1, "http://gateway.test/v1/messages", False), + (ANTHROPIC_MAX_USER_ID_LENGTH, _ANTHROPIC_MESSAGES, True), + ], + ids=["at-the-configured-base-limit", "over-the-configured-base-limit", "direct-anthropic-keeps-its-limit"], +) +async def test_pass_through_caps_the_user_id_for_a_configured_anthropic_compatible_base( + monkeypatch: pytest.MonkeyPatch, user_id_length: int, target: str, forwarded: bool +) -> None: + monkeypatch.setenv("ANTHROPIC_API_BASE", "http://gateway.test") + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "u" * user_id_length}}) + + relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None) + + assert relayed == (client_body if forwarded else _MESSAGES_BODY) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +@pytest.mark.parametrize( + "default_on, event_hook, guardrails_config, client_fields, stream_scope, forwarded", + [ + (False, GuardrailEventHooks.pre_call, None, {}, None, True), + (False, GuardrailEventHooks.pre_call, {"unregistered-guard": {}}, {}, None, True), + (True, GuardrailEventHooks.post_call, None, {}, None, True), + (True, GuardrailEventHooks.pre_call, None, {}, None, False), + (False, GuardrailEventHooks.pre_call, {"pii-guard": {}}, {}, None, False), + (False, GuardrailEventHooks.pre_call, None, {"litellm_metadata": {"guardrails": ["pii-guard"]}}, None, False), + (False, GuardrailEventHooks.pre_call, {"pii-guard": {}}, {"litellm_metadata": {"guardrails": []}}, None, False), + (True, GuardrailEventHooks.pre_call, None, {"stream": True}, "streaming", False), + (True, GuardrailEventHooks.pre_call, None, {}, "streaming", True), + (True, GuardrailEventHooks.pre_call, None, {"stream": True}, "non_streaming", True), + (True, GuardrailEventHooks.pre_call, None, {}, "non_streaming", False), + ], + ids=[ + "guardrail-not-requested", + "guardrail-not-registered", + "default-on-post-call-guardrail", + "default-on-pre-call-guardrail", + "guardrail-on-the-route", + "guardrail-in-litellm-metadata", + "route-guardrail-beside-an-empty-client-list", + "streaming-guardrail-on-a-stream", + "streaming-guardrail-off-a-stream", + "non-streaming-guardrail-on-a-stream", + "non-streaming-guardrail-off-a-stream", + ], +) +async def test_pass_through_keeps_the_user_id_from_routes_a_guardrail_scans( + default_on: bool, + event_hook: GuardrailEventHooks, + guardrails_config: dict[str, object] | None, + client_fields: Mapping[str, object], + stream_scope: str | None, + forwarded: bool, +) -> None: + guardrail: Final = CustomGuardrail( + guardrail_name="pii-guard", default_on=default_on, event_hook=event_hook, stream_scope=stream_scope + ) + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}, **client_fields}) + body_without_client_fields: Final = {k: v for k, v in client_body.items() if k not in ("metadata", "litellm_metadata")} + + with patch.object(litellm, "callbacks", [guardrail]): + relayed: Final = await _relay_through_pass_through( + client_body, _ANTHROPIC_MESSAGES, guardrails_config=guardrails_config + ) + + assert relayed == ( + {**body_without_client_fields, "metadata": {"user_id": "user-1"}} if forwarded else body_without_client_fields + ) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +async def test_pass_through_warns_without_the_value_when_it_drops_a_refused_user_id( + caplog: pytest.LogCaptureFixture, +) -> None: + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "person@example.com"}}) + + with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name): + await _relay_through_pass_through(client_body, _ANTHROPIC_MESSAGES, guardrails_config=None) + + assert "Not forwarding metadata.user_id" in caplog.text + assert "person@example.com" not in caplog.text + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +async def test_pass_through_drops_metadata_that_is_not_an_object() -> None: + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": "user-1"}) + + relayed: Final = await _relay_through_pass_through(client_body, _ANTHROPIC_MESSAGES, guardrails_config=None) + + assert relayed == _MESSAGES_BODY + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +async def test_pass_through_custom_body_replaces_the_client_user_id() -> None: + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}}) + + relayed: Final = await _relay_through_pass_through( + client_body, _ANTHROPIC_MESSAGES, guardrails_config=None, custom_body=dict(_MESSAGES_BODY) + ) + + assert relayed == _MESSAGES_BODY + + _PARTIAL_RELAY_WARNING_MARKER = "ended before upstream body was fully relayed" @@ -7736,6 +8003,102 @@ async def test_websocket_passthrough_sends_deepgram_audio_bytes_and_control_text upstream_ws.close.assert_awaited_once() +@dataclass(frozen=True, slots=True) +class _LengthMeter: + rate: float + + def measure(self, client_text: str) -> int: + return len(client_text) + + def price(self, total: int) -> WebSocketSessionUsage: + return WebSocketSessionUsage(model="voice-1", custom_llm_provider="tts-provider", response_cost=total * self.rate) + + +@pytest.mark.asyncio +async def test_websocket_passthrough_logs_the_meters_price_for_the_client_text_it_delivered(): + """Text frames reach the upstream unchanged and are measured, binary frames are not, and the session is logged at + the model, provider and cost the meter prices for the total.""" + frames = ("hello", b"\x00\x01binary", "world!") + upstream_ws = RecordingUpstreamWebSocket() + websocket = _client_websocket( + AsyncMock( + side_effect=[ + *( + {"type": "websocket.receive", "bytes": frame} + if isinstance(frame, bytes) + else {"type": "websocket.receive", "text": frame} + for frame in frames + ), + {"type": "websocket.disconnect"}, + ] + ) + ) + + with ( + _patched_websocket_passthrough_environment(upstream_ws), + patch( # test-quality-ok: pass_through_endpoint_logging is a module global read inside websocket_passthrough_request; there is no injection seam + "litellm.proxy.pass_through_endpoints.pass_through_endpoints." + "pass_through_endpoint_logging.pass_through_async_success_handler", + new=AsyncMock(), + ) as success_handler, + ): + await websocket_passthrough_request( + websocket=websocket, + target="wss://upstream.example/v1/stream", + custom_headers={}, + user_api_key_dict=UserAPIKeyAuth(), + forward_headers=False, + endpoint="/provider/v1/stream", + accept_websocket=False, + meter=_LengthMeter(rate=0.5), + ) + + assert [call.args[0] for call in upstream_ws.send.await_args_list] == list(frames) + logging_obj = success_handler.call_args.kwargs["logging_obj"] + assert logging_obj.model == "voice-1" + assert logging_obj.model_call_details["model"] == "voice-1" + assert logging_obj.model_call_details["custom_llm_provider"] == "tts-provider" + assert logging_obj.model_call_details["response_cost"] == (len("hello") + len("world!")) * 0.5 + + +@pytest.mark.asyncio +async def test_websocket_passthrough_meters_only_client_text_the_upstream_accepted(): + upstream_ws = RecordingUpstreamWebSocket() + upstream_ws.send = AsyncMock(side_effect=[None, ConnectionError("upstream went away")]) + websocket = _client_websocket( + AsyncMock( + side_effect=[ + {"type": "websocket.receive", "text": "hello"}, + {"type": "websocket.receive", "text": "never delivered"}, + {"type": "websocket.disconnect"}, + ] + ) + ) + + with ( + _patched_websocket_passthrough_environment(upstream_ws), + patch( # test-quality-ok: pass_through_endpoint_logging is a module global read inside websocket_passthrough_request; there is no injection seam + "litellm.proxy.pass_through_endpoints.pass_through_endpoints." + "pass_through_endpoint_logging.pass_through_async_success_handler", + new=AsyncMock(), + ) as success_handler, + ): + await websocket_passthrough_request( + websocket=websocket, + target="wss://upstream.example/v1/stream", + custom_headers={}, + user_api_key_dict=UserAPIKeyAuth(), + forward_headers=False, + endpoint="/provider/v1/stream", + accept_websocket=False, + meter=_LengthMeter(rate=0.5), + ) + + logging_obj = success_handler.call_args.kwargs["logging_obj"] + assert [call.args[0] for call in upstream_ws.send.await_args_list] == ["hello", "never delivered"] + assert logging_obj.model_call_details["response_cost"] == len("hello") * 0.5 + + @pytest.mark.asyncio async def test_websocket_passthrough_vertex_live_setup_ack_names_the_model_but_is_not_billed_as_usage(): """Vertex Live keeps its special first frame: the setup acknowledgement is forwarded verbatim, read for the diff --git a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py index ae95c27950c..8127ffe558b 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py +++ b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py @@ -41,12 +41,13 @@ def _make_streaming_response(chunks): return mock -def _unarmed_logging_obj(): +def _unarmed_logging_obj(model: str = "unknown"): """Real Logging objects only carry _on_deferred_stream_complete when the proxy arms deferred dispatch; a bare MagicMock's auto-attribute is truthy and would spuriously trigger the deferral branch.""" obj = MagicMock() obj._on_deferred_stream_complete = None + obj.model = model return obj @@ -409,7 +410,7 @@ async def _collect_openai_passthrough_chunks(chunks, endpoint_type): async for chunk in PassThroughStreamingHandler.chunk_processor( response=response, request_body={"model": "gpt-4o-mini", "stream": True}, - litellm_logging_obj=_unarmed_logging_obj(), + litellm_logging_obj=_unarmed_logging_obj(model="gpt-4o-mini"), endpoint_type=endpoint_type, start_time=datetime(2026, 1, 1), passthrough_success_handler_obj=MagicMock(), diff --git a/tests/unit/proxy/pass_through_endpoints/test_success_handler.py b/tests/unit/proxy/pass_through_endpoints/test_success_handler.py index c721e808ed5..cf00545f331 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_success_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/test_success_handler.py @@ -346,6 +346,7 @@ async def test_vertex_ai_anthropic_streaming_cost_injection_enabled(): # Setup logging object with model info litellm_logging_obj = MagicMock(spec=LiteLLMLoggingObj) + litellm_logging_obj.model = "claude-sonnet-4@20250514" litellm_logging_obj.litellm_params = {} litellm_logging_obj.model_call_details = {"model": "claude-sonnet-4@20250514"} litellm_logging_obj.completion_start_time = None @@ -544,6 +545,7 @@ async def test_vertex_ai_anthropic_streaming_model_extraction(): response.aiter_bytes = mock_aiter_bytes litellm_logging_obj = MagicMock(spec=LiteLLMLoggingObj) + litellm_logging_obj.model = "claude-sonnet-4@20250514" litellm_logging_obj.litellm_params = {} litellm_logging_obj.model_call_details = {} litellm_logging_obj.completion_start_time = None diff --git a/tests/unit/proxy/spend_tracking/test_usage_meter.py b/tests/unit/proxy/spend_tracking/test_usage_meter.py new file mode 100644 index 00000000000..de7d7c6591d --- /dev/null +++ b/tests/unit/proxy/spend_tracking/test_usage_meter.py @@ -0,0 +1,389 @@ +import asyncio +import json +from datetime import datetime, timezone +from decimal import Decimal +from typing import Final + +import pytest + +from litellm.accounting import InboundUsageEvent, Metric, MetricName, Outcome +from litellm.proxy.db.accounting_ledger import AccountingConflictError, UsageWrite +from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureError, UsageCaptureSettings +from litellm.types.utils import CallTypes + + +def _write(identity: str) -> UsageWrite: + event: Final = InboundUsageEvent( + event_id=identity, + request_id=identity, + call_type=CallTypes.acompletion, + started_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + ended_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + outcome=Outcome.SUCCEEDED, + dimensions=(), + metrics=(Metric(name=MetricName.REQUESTS, value=Decimal(1)),), + ) + return UsageWrite(event, json.dumps({"event": event.model_dump(mode="json")})) + + +class CommitBarrier: + def __init__(self, *, conflict: str | None = None, failure: Exception | None = None) -> None: + self.entered: Final = asyncio.Event() + self.release: Final = asyncio.Event() + self.committed: tuple[str, ...] = () + self.transactions = 0 + self.conflict: Final = conflict + self.failure: Final = failure + + async def write(self, writes: tuple[UsageWrite, ...]) -> None: + self.transactions += 1 + self.entered.set() + await self.release.wait() + if self.failure is not None: + raise self.failure + identities: Final = tuple(write.event.event_id for write in writes) + if self.conflict in identities: + raise AccountingConflictError("Conflicting test event") + self.committed += identities + + +async def _manual_flush(wakeup: asyncio.Event, timeout: float) -> None: + await wakeup.wait() + + +@pytest.mark.asyncio +async def test_receipts_wait_for_one_batched_commit_and_keep_inflight_capacity_reserved() -> None: + database: Final = CommitBarrier() + meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=2, max_pending_records=2)) + first: Final = meter.submit((_write("one"),)) + second: Final = meter.submit((_write("two"),)) + await database.entered.wait() + assert not first.done() and not second.done() + assert meter.pending_records == 2 + with pytest.raises(UsageCaptureError, match="capacity"): + meter.submit((_write("three"),)) + database.release.set() + await asyncio.gather(first, second) + await meter.close() + assert database.committed == ("one", "two") + assert database.transactions == 1 + assert (meter.pending_bytes, meter.pending_records) == (0, 0) + + +@pytest.mark.asyncio +async def test_byte_limit_rejects_new_work_without_evicting_admitted_work() -> None: + database: Final = CommitBarrier() + write: Final = _write("large" * 300) + size: Final = len(write.envelope.encode()) + meter: Final = DurableUsageMeter( + database.write, UsageCaptureSettings(max_pending_bytes=size), wait_for_batch=_manual_flush + ) + receipt: Final = meter.submit((write,)) + with pytest.raises(UsageCaptureError, match="capacity"): + meter.submit((_write("extra"),)) + assert meter.pending_bytes == size + database.release.set() + await meter.close() + await receipt + assert database.committed == (write.event.event_id,) + with pytest.raises(UsageCaptureError, match="closed"): + meter.submit((_write("after-close"),)) + + +@pytest.mark.asyncio +async def test_conflicting_group_rolls_back_and_valid_neighbor_still_commits() -> None: + database: Final = CommitBarrier(conflict="conflict") + meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=3)) + rejected: Final = meter.submit((_write("caller"), _write("conflict"))) + accepted: Final = meter.submit((_write("valid"),)) + await database.entered.wait() + database.release.set() + with pytest.raises(UsageCaptureError, match="identity"): + await rejected + await accepted + await meter.close() + assert database.committed == ("valid",) + assert meter.pending_records == 0 + + +@pytest.mark.asyncio +async def test_permanent_persistence_failure_reaches_each_receipt_and_releases_capacity() -> None: + failure: Final = ValueError("invalid envelope") + database: Final = CommitBarrier(failure=failure) + meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=2)) + first: Final = meter.submit((_write("one"),)) + second: Final = meter.submit((_write("two"),)) + await database.entered.wait() + database.release.set() + results: Final = await asyncio.gather(first, second, return_exceptions=True) + await meter.close() + assert results == [failure, failure] + assert meter.pending_bytes == 0 + + +@pytest.mark.asyncio +async def test_cancelling_record_waiter_does_not_cancel_persistence() -> None: + database: Final = CommitBarrier() + meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=2)) + waiting: Final = asyncio.create_task(meter.record(_write("caller").event)) + await database.entered.wait() + waiting.cancel() + with pytest.raises(asyncio.CancelledError): + await waiting + database.release.set() + await meter.close() + assert database.committed == ("caller",) + assert meter.pending_records == 0 + + +@pytest.mark.asyncio +async def test_shutdown_deadline_rejects_unacknowledged_work_and_releases_capacity() -> None: + database: Final = CommitBarrier() + meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=2)) + first: Final = meter.submit((_write("in-flight"), _write("in-flight-two"))) + await database.entered.wait() + queued: Final = meter.submit((_write("queued"),)) + await meter.stop(0) + results: Final = await asyncio.gather(first, queued, return_exceptions=True) + assert all(isinstance(result, UsageCaptureError) for result in results) + assert meter.pending_records == 0 + assert meter.pending_bytes == 0 + assert database.committed == () + + +@pytest.mark.asyncio +async def test_shutdown_during_database_backoff_releases_reserved_capacity() -> None: + retrying: Final = asyncio.Event() + blocked: Final = asyncio.Event() + + async def unavailable(writes: tuple[UsageWrite, ...]) -> None: + raise ConnectionError("database unavailable") + + async def backoff(delay: float) -> None: + retrying.set() + await blocked.wait() + + meter: Final = DurableUsageMeter(unavailable, UsageCaptureSettings(batch_size=2), retry_delay=backoff) + receipt: Final = meter.submit((_write("one"), _write("two"))) + await retrying.wait() + await meter.stop(0) + with pytest.raises(UsageCaptureError, match="interrupted"): + await receipt + assert (meter.pending_records, meter.pending_bytes) == (0, 0) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("wrapped", [False, True]) +async def test_closed_pooled_connection_retains_batch_until_automatic_recovery(wrapped: bool) -> None: + import httpx + from prisma.engine.errors import EngineRequestError + from prisma.http import Response + + database: Final = CommitBarrier() + attempts: Final = asyncio.Queue[tuple[UsageWrite, ...]]() + retrying: Final = asyncio.Event() + recovered: Final = asyncio.Event() + closure: Final = EngineRequestError( + Response(httpx.Response(500)), '{"error_code":"P1017","message":"Server has closed the connection."}' + ) + failure: Final = RuntimeError("ledger transaction failed") if wrapped else closure + if wrapped: + failure.__cause__ = closure + + async def persist(writes: tuple[UsageWrite, ...]) -> None: + attempts.put_nowait(writes) + if attempts.qsize() == 1: + raise failure + await database.write(writes) + + async def backoff(delay: float) -> None: + retrying.set() + await recovered.wait() + + meter: Final = DurableUsageMeter( + persist, UsageCaptureSettings(batch_size=2, max_pending_records=2), retry_delay=backoff + ) + writes: Final = (_write("caller"), _write("attempt")) + receipt: Final = meter.submit(writes) + retry_signal: Final = asyncio.create_task(retrying.wait()) + reserved: Final = meter.pending_bytes + try: + await asyncio.wait((receipt, retry_signal), return_when=asyncio.FIRST_COMPLETED) + assert not receipt.done(), "A dropped pooled connection must retain admitted usage for retry" + assert (meter.pending_records, meter.pending_bytes) == (2, reserved) + with pytest.raises(UsageCaptureError, match="capacity"): + meter.submit((_write("overflow"),)) + database.release.set() + recovered.set() + await receipt + assert attempts.get_nowait() == attempts.get_nowait() == writes + assert database.committed == ("caller", "attempt") + assert (meter.pending_records, meter.pending_bytes) == (0, 0) + finally: + retry_signal.cancel() + await asyncio.gather(retry_signal, return_exceptions=True) + await meter.stop(0) + await asyncio.gather(receipt, return_exceptions=True) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("first_commits", [False, True]) +async def test_shutdown_during_conflict_isolation_completes_every_receipt(first_commits: bool) -> None: + isolating: Final = asyncio.Event() + blocked: Final = asyncio.Event() + + async def persist(writes: tuple[UsageWrite, ...]) -> None: + if len(writes) > 1: + raise AccountingConflictError("one producer conflicts") + if first_commits and writes[0].event.event_id == "one": + return + isolating.set() + await blocked.wait() + + meter: Final = DurableUsageMeter(persist, UsageCaptureSettings(batch_size=2)) + first: Final = meter.submit((_write("one"),)) + second: Final = meter.submit((_write("two"),)) + await isolating.wait() + await meter.stop(0) + results: Final = await asyncio.gather(first, second, return_exceptions=True) + assert results[0] is None if first_commits else isinstance(results[0], UsageCaptureError) + assert isinstance(results[1], UsageCaptureError) + assert (meter.pending_records, meter.pending_bytes) == (0, 0) + + +@pytest.mark.asyncio +async def test_elapsed_flush_deadline_commits_a_partial_batch() -> None: + from litellm.proxy.spend_tracking.usage_meter import _wait_for_batch + + database: Final = CommitBarrier() + database.release.set() + + async def expired_flush(wakeup: asyncio.Event, timeout: float) -> None: + assert timeout == 5.0 + await _wait_for_batch(wakeup, 0) + + meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=10), wait_for_batch=expired_flush) + receipt: Final = meter.submit((_write("partial-batch"),)) + try: + await receipt + assert database.committed == ("partial-batch",) + assert database.transactions == 1 + assert (meter.pending_records, meter.pending_bytes) == (0, 0) + finally: + await meter.stop(0) + + +@pytest.mark.asyncio +async def test_burst_drains_full_batches_without_waiting_for_partial_batch_deadline() -> None: + database: Final = CommitBarrier() + database.release.set() + waits: Final = asyncio.Queue[bool]() + + async def observe_wait(wakeup: asyncio.Event, timeout: float) -> None: + waits.put_nowait(wakeup.is_set()) + await wakeup.wait() + + meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=2), wait_for_batch=observe_wait) + receipts: Final = tuple(meter.submit((_write(str(index)),)) for index in range(5)) + try: + assert await waits.get() is True + await asyncio.gather(*receipts[:2]) + assert await waits.get() is True, "A full queued batch must not wait for the flush deadline" + await asyncio.gather(*receipts[2:4]) + assert await waits.get() is False + assert database.committed == ("0", "1", "2", "3") + assert meter.pending_records == 1 + assert not receipts[-1].done() + await meter.close() + await receipts[-1] + assert database.committed == ("0", "1", "2", "3", "4") + assert database.transactions == 3 + assert (meter.pending_records, meter.pending_bytes) == (0, 0) + finally: + await meter.stop(0) + await asyncio.gather(*receipts, return_exceptions=True) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sqlstate", ["22P05", "22003", "42P01"]) +async def test_invalid_submission_isolated_without_retrying_systemic_faults(sqlstate: str) -> None: + from prisma.errors import DataError + + database: Final = CommitBarrier() + database.release.set() + attempts: Final = asyncio.Queue[tuple[str, ...]]() + + async def persist(writes: tuple[UsageWrite, ...]) -> None: + identities: Final = tuple(write.event.event_id for write in writes) + attempts.put_nowait(identities) + if "invalid" in identities or sqlstate == "42P01": + raise DataError( + data={"user_facing_error": {"message": "database rejected input", "meta": {"code": sqlstate}}} + ) + await database.write(writes) + + meter: Final = DurableUsageMeter(persist, UsageCaptureSettings(batch_size=3)) + rejected: Final = meter.submit((_write("caller"), _write("invalid"))) + neighbor: Final = meter.submit((_write("neighbor"),)) + try: + with pytest.raises(DataError): + await rejected + if sqlstate == "42P01": + with pytest.raises(DataError): + await neighbor + assert attempts.qsize() == 1 + else: + await neighbor + assert database.committed == ("neighbor",) + assert attempts.qsize() == 3 + finally: + await meter.close() + assert (meter.pending_records, meter.pending_bytes) == (0, 0) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("dependency_outcome", ["commit", "failure", "cancel"]) +async def test_commit_dependencies_preserve_capacity_and_isolate_failed_dependents(dependency_outcome: str) -> None: + database: Final = CommitBarrier() + meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=3, max_pending_records=4)) + prerequisite: Final[asyncio.Future[None]] = asyncio.get_running_loop().create_future() + dependent: Final = meter.submit((_write("parent"), _write("final")), prerequisites=(prerequisite,)) + neighbor: Final = meter.submit((_write("neighbor"),)) + assert meter.pending_records == 3 + assert not dependent.done() + with pytest.raises(UsageCaptureError, match="capacity"): + meter.submit((_write("excess-one"), _write("excess-two"))) + if dependency_outcome == "commit": + prerequisite.set_result(None) + elif dependency_outcome == "failure": + prerequisite.set_exception(ValueError("dependency failed")) + else: + prerequisite.cancel() + database.release.set() + await meter.close() + await neighbor + if dependency_outcome == "commit": + await dependent + assert database.committed == ("parent", "final", "neighbor") + else: + with pytest.raises(UsageCaptureError, match="dependency"): + await dependent + assert database.committed == ("neighbor",) + assert meter.pending_records == 0 + assert meter.pending_bytes == 0 + + +@pytest.mark.asyncio +async def test_shutdown_does_not_cancel_dependency_receipt_or_leak_reserved_capacity() -> None: + database: Final = CommitBarrier() + meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=2)) + prerequisite: Final[asyncio.Future[None]] = asyncio.get_running_loop().create_future() + dependent: Final = meter.submit((_write("parent"), _write("final")), prerequisites=(prerequisite,)) + await meter.stop(0) + with pytest.raises(UsageCaptureError, match="acknowledgement"): + await dependent + assert not prerequisite.done() + assert database.committed == () + assert meter.pending_records == 0 + assert meter.pending_bytes == 0 + prerequisite.set_result(None) diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index fe6b894b368..4d1b67f9212 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -74,6 +74,7 @@ from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitErro from litellm.proxy.common_utils.sse_keepalive import ANTHROPIC_PING_SSE_CHUNK from litellm.proxy.dd_span_tagger import DDSpanTagger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.proxy._types import Litellm_EntityType, ProxyErrorTypes, ProxyException from litellm.proxy._types import UserAPIKeyAuth as ProxyUserAPIKeyAuth from litellm.proxy.auth.fallback_budget import RouterFallbackBudgetCheck @@ -10221,6 +10222,8 @@ class TestInjectCostIntoUsageDict: discount streamed a ``usage.cost`` that disagreed with the callback's ``response_cost``.""" class _StubLoggingObj: + model = "claude-haiku-4-5" + def __init__(self, cost): self._cost = cost self.captured_result = None @@ -10257,6 +10260,8 @@ class TestInjectCostIntoUsageDict: def test_message_delta_falls_back_to_model_pricing_when_the_logging_obj_returns_no_cost(self): class _StubLoggingObj: + model = "claude-haiku-4-5" + def response_cost_calculator(self, result): return None @@ -10282,6 +10287,8 @@ class TestInjectCostIntoUsageDict: model-name pricing rather than propagating into the response body.""" class _StubLoggingObj: + model = "claude-haiku-4-5" + def response_cost_calculator(self, result): raise ValueError("no pricing for this deployment") @@ -10363,6 +10370,8 @@ class TestInjectCostIntoUsageDict: streaming /v1/chat/completions gets its negotiated price instead of sticker.""" class _StubLoggingObj: + model = "gpt-4o-mini" + def __init__(self, cost): self._cost = cost self.captured_result = None @@ -10391,6 +10400,8 @@ class TestInjectCostIntoUsageDict: def test_openai_chunk_falls_back_to_model_pricing_when_the_logging_obj_returns_no_cost(self): class _StubLoggingObj: + model = "gpt-4o-mini" + def response_cost_calculator(self, result): return None @@ -10406,6 +10417,60 @@ class TestInjectCostIntoUsageDict: assert result is not None assert result["usage"]["cost"] == pytest.approx(self._expected_cost("gpt-4o-mini", 11, 4)) + @pytest.mark.parametrize( + ("deployment_model", "served_model", "custom_llm_provider"), + [ + ("vertex_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "vertex_ai"), + ("anthropic/claude-sonnet-5-5", "claude-sonnet-5-5", "anthropic"), + ("azure_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "azure_ai"), + ], + ) + def test_message_delta_bills_search_like_the_logged_response_on_a_token_priced_claude_deployment( + self, local_model_cost_map: None, deployment_model: str, served_model: str, custom_llm_provider: str + ) -> None: + token_rates: Final = {"input_cost_per_token": 3e-6, "output_cost_per_token": 1.5e-5} + router: Final = Router( + model_list=[ + { + "model_name": "priced-claude", + "litellm_params": { + "model": deployment_model, + "api_key": "k", + "vertex_project": "p", + "vertex_location": "global", + }, + "model_info": token_rates, + } + ] + ) + deployment_id: Final = router.model_list[0]["model_info"]["id"] + logging_obj: Final = LiteLLMLoggingObj( + model=deployment_model, + messages=[{"role": "user", "content": "test"}], + stream=True, + call_type="anthropic_messages", + start_time=None, + litellm_call_id="token-priced-claude-search", + function_id="token-priced-claude-search", + ) + logging_obj.update_environment_variables( + litellm_params={"metadata": {"model_info": {"id": deployment_id, **token_rates}}}, optional_params={} + ) + logging_obj.model_call_details["custom_llm_provider"] = custom_llm_provider + wire_usage: Final = {"input_tokens": 1840, "output_tokens": 412, "server_tool_use": {"web_search_requests": 2}} + event: Final = {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": wire_usage} + + result: Final = ProxyBaseLLMRequestProcessing._inject_cost_into_usage_dict(event, "priced-claude", logging_obj) + + stream_usage: Final = AnthropicConfig().calculate_usage(usage_object=wire_usage, reasoning_content=None) + logged_cost: Final = logging_obj.response_cost_calculator( + result=litellm.ModelResponse(model=served_model, usage=stream_usage) + ) + token_cost: Final = 1840 * token_rates["input_cost_per_token"] + 412 * token_rates["output_cost_per_token"] + assert logged_cost is not None and logged_cost > token_cost + assert result is not None + assert result["usage"]["cost"] == pytest.approx(logged_cost) + class TestProcessChunkWithCostInjection: @pytest.mark.parametrize("model", ["claude-opus-5-5", "vertex_ai/claude-opus-5-5"]) @@ -10472,6 +10537,8 @@ class TestProcessChunkWithCostInjection: monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True) class _StubLoggingObj: + model = "claude-haiku-4-5" + def response_cost_calculator(self, result): return 0.00042 diff --git a/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py b/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py index 5013ecd98a2..26ee9f04d01 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py +++ b/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py @@ -996,3 +996,21 @@ async def test_guardrails_only_and_skip_guardrails_are_mutually_exclusive( guardrails_only=True, skip_guardrails=True, ) + + +@pytest.mark.parametrize( + "request_data, expected", + [ + ({"metadata": {"_guardrail_pipelines": ["p1"]}}, True), + ({"metadata": {"user_id": "u"}, "litellm_metadata": {"_guardrail_pipelines": ["p1"]}}, True), + ({"metadata": {"_guardrail_pipelines": 5}}, True), + ({"metadata": {"user_id": "u"}}, False), + ], + ids=["pipeline-in-metadata", "pipeline-in-litellm-metadata", "non-list-pipeline-value", "no-pipeline"], +) +def test_request_has_pre_call_guardrails_reads_pipelines_from_both_metadata_buckets( + proxy_logging, monkeypatch, request_data, expected +): + monkeypatch.setattr(litellm, "callbacks", []) + + assert proxy_logging.request_has_pre_call_guardrails(request_data) is expected diff --git a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py index 0341511496c..5627167432d 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py +++ b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py @@ -1595,3 +1595,57 @@ async def test_previous_response_id_keeps_the_tools_an_earlier_codex_tool_search [("tool_result", "toolu_standup"), ("text", None)], ] assert prisma_client.db.queries == 1 + + +async def _codex_follow_up_body(**kwargs: object) -> Mapping[str, object]: + anthropic: Final = _RecordingRawAnthropicMessages() + with patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([_SEARCH_TURN, _CALENDAR_TURN])): + await litellm.aresponses( + model=_MODEL, + previous_response_id="chatcmpl-calendar-turn", + input=[{"type": "function_call_output", "call_id": "toolu_standup", "output": "Created Standup."}], + tools=[_CODEX_TOOL_SEARCH, _SHELL_TOOL], + api_key="sk-ant-fake", + client=AsyncHTTPHandler(transport=httpx.MockTransport(anthropic)), + **kwargs, + ) + return json.loads(anthropic.body) + + +@pytest.mark.asyncio +async def test_codex_tool_search_follow_up_rejects_text_verbosity_anthropic_does_not_support( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + anthropic: Final = _RecordingRawAnthropicMessages() + + with ( + patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([_SEARCH_TURN, _CALENDAR_TURN])), + pytest.raises(litellm.UnsupportedParamsError, match="verbosity"), + ): + await litellm.aresponses( + model=_MODEL, + previous_response_id="chatcmpl-calendar-turn", + input=[{"type": "function_call_output", "call_id": "toolu_standup", "output": "Created Standup."}], + tools=[_CODEX_TOOL_SEARCH, _SHELL_TOOL], + text={"verbosity": "low"}, + api_key="sk-ant-fake", + num_retries=0, + client=AsyncHTTPHandler(transport=httpx.MockTransport(anthropic)), + ) + + assert anthropic.body == "" + + +@pytest.mark.asyncio +async def test_codex_tool_search_follow_up_text_verbosity_follows_provider_param_validation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + without_text: Final = await _codex_follow_up_body() + + dropped: Final = await _codex_follow_up_body(text={"verbosity": "low"}, drop_params=True) + allowed: Final = await _codex_follow_up_body(text={"verbosity": "low"}, allowed_openai_params=["verbosity"]) + + assert dropped == without_text + assert allowed == {**without_text, "verbosity": "low"} diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 53c5ba61b7f..6a57e625c34 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -6345,6 +6345,17 @@ def test_marks_past_the_provider_cap_keep_the_newest_under_drop_params(drop_para assert messages[1] == {"role": "user", "content": "turn 0"} +def test_marks_past_the_provider_cap_inside_one_message_keep_the_newest_under_drop_params(): + parts: Final = tuple( + {"type": "input_text", "text": f"part {index}", "cache_control": _EPHEMERAL} for index in range(1, 6) + ) + + messages: Final = _bridged_request({"role": "user", "content": list(parts)}, drop_params=True) + + content: Final = TypeAdapter(list[dict[str, object]]).validate_python(messages[-1]["content"]) + assert ["cache_control" in part for part in content] == [False, True, True, True, True] + + def test_global_drop_params_keeps_the_newest_marks(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(litellm, "drop_params", True) diff --git a/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py index 2f27622d77c..a16661c0174 100644 --- a/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -4,6 +4,8 @@ import subprocess import sys import textwrap import types +from types import MappingProxyType +from collections.abc import Mapping, Sequence from contextlib import nullcontext from typing import Any, Final, Literal, cast from unittest.mock import AsyncMock, MagicMock, patch @@ -18,10 +20,14 @@ import litellm from litellm.caching.caching import DualCache from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.mcp_server import operations as mcp_operations +from litellm.proxy.common_utils.user_api_key_cache import object_permission_cache_key from litellm.proxy.mcp_server.faults.list_outcomes import AggregateToolListing from litellm.proxy.mcp_server.mcp_server_manager import ListedToolsCaller, MCPServerManager +from litellm.proxy._types import UI_TEAM_ID, LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTableCachedObj +from litellm.proxy.mcp_server.ui_session_utils import GrantedToolsetIds, granted_toolset_ids +from litellm.types.mcp_server.mcp_toolset import MCPToolset from litellm.proxy.utils import ProxyLogging from litellm.responses import main as responses_main from litellm.responses.mcp import litellm_proxy_mcp_handler as mcp_handler_module @@ -1575,61 +1581,309 @@ async def test_concurrent_bridge_calls_use_their_own_served_metadata(monkeypatch ProxyLogging._callback_capabilities_cache.clear() -def _toolset_gateway_manager(toolset_id: str, server_id: str) -> types.SimpleNamespace: - return types.SimpleNamespace( - catalog=types.SimpleNamespace(operation=nullcontext), - get_registry=MagicMock(return_value={}), - get_allowed_mcp_servers=AsyncMock(return_value=[]), - get_mcp_servers_from_ids=MagicMock(return_value=[]), - get_mcp_server_by_name=MagicMock(return_value=None), - get_toolset_by_name_cached=AsyncMock(return_value=types.SimpleNamespace(toolset_id=toolset_id)), - resolve_toolset_tool_permissions=AsyncMock(return_value={server_id: ["add"]}), - ) +TOOLSET_ROWS: Final = ( + MCPToolset( + toolset_id="ts-granted", toolset_name="team-toolset", tools=[{"server_id": "srv-1", "tool_name": "add"}] + ), + MCPToolset( + toolset_id="ts-other", toolset_name="other-toolset", tools=[{"server_id": "srv-2", "tool_name": "lookup"}] + ), + MCPToolset(toolset_id="ts-shouty", toolset_name="CALC", tools=[{"server_id": "srv-2", "tool_name": "lookup"}]), + MCPToolset(toolset_id="ts-group", toolset_name="reference", tools=[{"server_id": "srv-1", "tool_name": "add"}]), +) +GATEWAY_SERVERS: Final = ( + MCPServer(server_id="srv-1", name="calc", alias="calc", transport=MCPTransport.http, access_groups=["wiki"]), + MCPServer(server_id="srv-2", name="docs", alias="docs", transport=MCPTransport.http, access_groups=["reference"]), +) +UPSTREAM_TOOLS: Final = MappingProxyType( + { + "calc": ( + MCPTool(name="add", inputSchema={"type": "object"}), + MCPTool(name="subtract", inputSchema={"type": "object"}), + ), + "docs": (MCPTool(name="lookup", inputSchema={"type": "object"}),), + } +) -async def _tools_listing_kwargs_for_toolset_url(monkeypatch, team_toolset_id: str) -> dict[str, object]: - from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LitellmUserRoles, UserAPIKeyAuth - from litellm.proxy.mcp_server.ui_session_utils import granted_toolset_ids +def _toolset_table(lookup_error: Exception | None = None) -> types.SimpleNamespace: + async def find_first(*, where: Mapping[str, str]) -> MCPToolset | None: + if lookup_error is not None: + raise lookup_error + return next((row for row in TOOLSET_ROWS if row.toolset_name == where["toolset_name"]), None) - mock_get_tools = AsyncMock(return_value=AggregateToolListing(tools=[], outcomes={})) - monkeypatch.setattr("litellm.proxy.mcp_server.server._get_tools_from_mcp_servers", mock_get_tools) - monkeypatch.setattr( - "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", - _toolset_gateway_manager("ts-granted", "srv-1"), - ) - monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", MagicMock()) + async def find_many(*, where: Mapping[str, Mapping[str, Sequence[str]]]) -> list[MCPToolset]: + return [row for row in TOOLSET_ROWS if row.toolset_id in where["toolset_id"]["in"]] + return types.SimpleNamespace(find_first=find_first, find_many=find_many) + + +def _team_grants(toolset_id: str) -> GrantedToolsetIds: async def team_permission(context: UserAPIKeyAuth) -> LiteLLM_ObjectPermissionTable: - return LiteLLM_ObjectPermissionTable(object_permission_id="op-team", mcp_toolsets=[team_toolset_id]) + return LiteLLM_ObjectPermissionTable(object_permission_id="op-team", mcp_toolsets=[toolset_id]) async def granted_through_team(context: UserAPIKeyAuth) -> frozenset[str]: return await granted_toolset_ids(context, team_object_permission=team_permission, require_key_access=False) - team_key: Final = UserAPIKeyAuth(api_key="sk-team", team_id="team-1", user_role=LitellmUserRoles.INTERNAL_USER) - await LiteLLM_Proxy_MCP_Handler._get_mcp_tools_from_manager( - user_api_key_auth=team_key, - mcp_tools_with_litellm_proxy=[{"type": "mcp", "server_url": "litellm_proxy/mcp/team-toolset"}], - granted_toolsets=granted_through_team, + return granted_through_team + + +async def _no_toolset_grants(context: UserAPIKeyAuth) -> frozenset[str]: + return frozenset() + + +async def _tools_listed_through_the_gateway( + monkeypatch: pytest.MonkeyPatch, + user_api_key_auth: UserAPIKeyAuth, + server_urls: tuple[str, ...], + granted_toolsets: GrantedToolsetIds, + toolset_table: types.SimpleNamespace | None = None, + agent_permission: LiteLLM_ObjectPermissionTable | None = None, +) -> list[str]: + async def upstream_listing(_client: object, server_name: str) -> list[MCPTool]: + return list(UPSTREAM_TOOLS[server_name]) + + manager: Final = MCPServerManager() + manager.registry = {server.server_id: server for server in GATEWAY_SERVERS} + manager.config_mcp_servers = dict(manager.registry) + manager.create_mcp_client = AsyncMock(return_value=object()) + manager._fetch_tools_with_timeout = AsyncMock(side_effect=upstream_listing) + tables: Final = types.SimpleNamespace(litellm_mcptoolsettable=toolset_table or _toolset_table()) + monkeypatch.setattr("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", manager) + monkeypatch.setattr(mcp_operations, "global_mcp_server_manager", manager) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", types.SimpleNamespace(db=tables, writer_db=tables)) + cache: Final = DualCache() + await cache.async_set_cache(key="team_id:team-1", value=LiteLLM_TeamTableCachedObj(team_id="team-1")) + if agent_permission is not None: + await cache.async_set_cache( + key=f"agent_object_permission_id:{user_api_key_auth.agent_id}", + value=agent_permission.object_permission_id, + ) + await cache.async_set_cache( + key=object_permission_cache_key(agent_permission.object_permission_id), value=agent_permission + ) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", cache) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"supported_db_objects": []}) + try: + tools, _server_names = await LiteLLM_Proxy_MCP_Handler._get_mcp_tools_from_manager( + user_api_key_auth=user_api_key_auth, + mcp_tools_with_litellm_proxy=[{"type": "mcp", "server_url": url} for url in server_urls], + granted_toolsets=granted_toolsets, + ) + finally: + for server in GATEWAY_SERVERS: + manager._drop_listed_tools(server.server_id) + return sorted(tool.name for tool in tools) + + +def _key_holding(*server_ids: str, role: LitellmUserRoles = LitellmUserRoles.INTERNAL_USER) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + api_key="sk-team", + team_id="team-1", + user_role=role, + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="op-key", mcp_servers=list(server_ids)), ) - assert mock_get_tools.await_args is not None - return mock_get_tools.await_args.kwargs @pytest.mark.asyncio -async def test_toolset_gateway_url_scopes_a_team_granted_toolset_for_a_key_without_its_own_grant(monkeypatch): - kwargs: Final = await _tools_listing_kwargs_for_toolset_url(monkeypatch, team_toolset_id="ts-granted") - scoped = kwargs["user_api_key_auth"].object_permission - assert scoped is not None - assert scoped.mcp_servers == ["srv-1"] - assert scoped.mcp_tool_permissions == {"srv-1": ["add"]} - assert kwargs["mcp_servers"] is None +async def test_toolset_gateway_url_scopes_a_team_granted_toolset_for_a_key_without_its_own_grant( + monkeypatch: pytest.MonkeyPatch, +): + listed: Final = await _tools_listed_through_the_gateway( + monkeypatch, + UserAPIKeyAuth(api_key="sk-team", team_id="team-1", user_role=LitellmUserRoles.INTERNAL_USER), + ("litellm_proxy/mcp/team-toolset",), + _team_grants("ts-granted"), + ) + assert listed == ["calc-add"] @pytest.mark.asyncio -async def test_toolset_gateway_url_skips_a_toolset_the_team_does_not_grant(monkeypatch): - kwargs: Final = await _tools_listing_kwargs_for_toolset_url(monkeypatch, team_toolset_id="ts-other") - assert kwargs["user_api_key_auth"].object_permission is None - assert kwargs["mcp_servers"] is None +@pytest.mark.parametrize( + "owner_role", + [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY], + ids=["internal_user", "proxy_admin", "proxy_admin_viewer"], +) +async def test_toolset_gateway_url_answers_403_for_a_toolset_the_team_does_not_grant( + monkeypatch: pytest.MonkeyPatch, owner_role: LitellmUserRoles +): + with pytest.raises(HTTPException) as denial: + await _tools_listed_through_the_gateway( + monkeypatch, + UserAPIKeyAuth(api_key="sk-team", team_id="team-1", user_role=owner_role), + ("litellm_proxy/mcp/team-toolset",), + _team_grants("ts-other"), + ) + assert denial.value.status_code == 403 + assert "team-toolset" in str(denial.value.detail) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "requested_toolsets", + [("team-toolset", "other-toolset"), ("other-toolset", "team-toolset")], + ids=["granted_first", "denied_first"], +) +async def test_a_denied_toolset_beside_a_granted_one_adds_nothing_to_the_scoped_key( + monkeypatch: pytest.MonkeyPatch, requested_toolsets: tuple[str, ...] +): + listed: Final = await _tools_listed_through_the_gateway( + monkeypatch, + UserAPIKeyAuth(api_key="sk-team", team_id="team-1", user_role=LitellmUserRoles.PROXY_ADMIN), + tuple(f"litellm_proxy/mcp/{name}" for name in requested_toolsets), + _team_grants("ts-granted"), + ) + assert listed == ["calc-add"] + + +@pytest.mark.asyncio +async def test_a_whole_gateway_entry_beside_an_ungranted_toolset_still_lists_the_keys_own_servers( + monkeypatch: pytest.MonkeyPatch, +): + listed: Final = await _tools_listed_through_the_gateway( + monkeypatch, + _key_holding("srv-1", role=LitellmUserRoles.PROXY_ADMIN), + ("litellm_proxy", "litellm_proxy/mcp/other-toolset"), + _team_grants("ts-granted"), + ) + assert listed == ["calc-add", "calc-subtract"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("team_id", [None, UI_TEAM_ID], ids=["no_team", "dashboard_session"]) +async def test_an_admin_credential_without_a_team_keeps_a_toolset_no_grant_names( + monkeypatch: pytest.MonkeyPatch, team_id: str | None +): + listed: Final = await _tools_listed_through_the_gateway( + monkeypatch, + UserAPIKeyAuth(api_key="sk-admin", team_id=team_id, user_role=LitellmUserRoles.PROXY_ADMIN), + ("litellm_proxy/mcp/team-toolset",), + _team_grants("ts-other"), + ) + assert listed == ["calc-add"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "lookup_error", [None, ConnectionError("database unreachable")], ids=["no_such_toolset", "toolset_lookup_fails"] +) +async def test_a_name_that_does_not_resolve_to_a_toolset_is_listed_as_a_server_name( + monkeypatch: pytest.MonkeyPatch, lookup_error: Exception | None +): + listed: Final = await _tools_listed_through_the_gateway( + monkeypatch, + _key_holding("srv-1", "srv-2"), + ("litellm_proxy", "litellm_proxy/mcp/wiki"), + _no_toolset_grants, + _toolset_table(lookup_error), + ) + assert listed == ["calc-add", "calc-subtract"] + + +@pytest.mark.asyncio +async def test_an_ungranted_toolset_named_like_a_server_the_key_holds_answers_403_once( + monkeypatch: pytest.MonkeyPatch, +): + with pytest.raises(HTTPException) as denial: + await _tools_listed_through_the_gateway( + monkeypatch, + _key_holding("srv-1", "srv-2"), + ("litellm_proxy/mcp/CALC", "litellm_proxy/mcp/CALC"), + _no_toolset_grants, + ) + assert denial.value.status_code == 403 + assert str(denial.value.detail).count("CALC") == 1 + + +@pytest.mark.asyncio +async def test_an_ungranted_toolset_named_like_a_server_adds_no_server_beside_a_named_one( + monkeypatch: pytest.MonkeyPatch, +): + listed: Final = await _tools_listed_through_the_gateway( + monkeypatch, + _key_holding("srv-1", "srv-2"), + ("litellm_proxy/mcp/docs", "litellm_proxy/mcp/CALC"), + _no_toolset_grants, + ) + assert listed == ["docs-lookup"] + + +@pytest.mark.asyncio +async def test_ungranted_toolsets_are_named_in_request_order_in_the_403(monkeypatch: pytest.MonkeyPatch): + with pytest.raises(HTTPException) as denial: + await _tools_listed_through_the_gateway( + monkeypatch, + _key_holding("srv-1", "srv-2"), + ("litellm_proxy/mcp/other-toolset", "litellm_proxy/mcp/CALC"), + _no_toolset_grants, + ) + assert denial.value.status_code == 403 + assert str(denial.value.detail).endswith("other-toolset, CALC'}") + + +@pytest.mark.asyncio +async def test_an_agent_bound_key_naming_an_ungranted_toolset_gets_the_generic_403(monkeypatch: pytest.MonkeyPatch): + with pytest.raises(HTTPException) as denial: + await _tools_listed_through_the_gateway( + monkeypatch, + _key_holding("srv-1", "srv-2").model_copy(update={"agent_id": "agent-1"}), + ("litellm_proxy/mcp/CALC",), + _no_toolset_grants, + ) + assert denial.value.status_code == 403 + assert denial.value.detail == {"error": "The key is not allowed to access the requested MCP servers: CALC"} + + +@pytest.mark.asyncio +async def test_an_access_group_the_key_holds_outranks_an_ungranted_toolset_of_the_same_name( + monkeypatch: pytest.MonkeyPatch, +): + listed: Final = await _tools_listed_through_the_gateway( + monkeypatch, + _key_holding("srv-1", "srv-2"), + ("litellm_proxy/mcp/reference",), + _no_toolset_grants, + ) + assert listed == ["docs-lookup"] + + +@pytest.mark.asyncio +async def test_an_access_group_the_keys_agent_does_not_grant_leaves_the_same_named_toolset_denied( + monkeypatch: pytest.MonkeyPatch, +): + key_holding_the_group: Final = UserAPIKeyAuth( + api_key="sk-team", + team_id="team-1", + user_role=LitellmUserRoles.INTERNAL_USER, + agent_id="agent-1", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="op-key", mcp_servers=["srv-1"], mcp_access_groups=["reference"] + ), + ) + with pytest.raises(HTTPException) as denial: + await _tools_listed_through_the_gateway( + monkeypatch, + key_holding_the_group, + ("litellm_proxy/mcp/reference",), + _no_toolset_grants, + agent_permission=LiteLLM_ObjectPermissionTable(object_permission_id="op-agent", mcp_servers=["srv-1"]), + ) + assert denial.value.status_code == 403 + assert denial.value.detail == {"error": "The key is not allowed to access the requested MCP servers: reference"} + + +@pytest.mark.asyncio +async def test_a_granted_toolset_outranks_an_access_group_of_the_same_name(monkeypatch: pytest.MonkeyPatch): + async def granted_reference_toolset(context: UserAPIKeyAuth) -> frozenset[str]: + return frozenset({"ts-group"}) + + listed: Final = await _tools_listed_through_the_gateway( + monkeypatch, + _key_holding("srv-1", "srv-2"), + ("litellm_proxy/mcp/reference",), + granted_reference_toolset, + ) + assert listed == ["calc-add"] @pytest.mark.asyncio diff --git a/tests/unit/responses/test_responses_api_bridge_flag.py b/tests/unit/responses/test_responses_api_bridge_flag.py index de96e995dd4..210488f0888 100644 --- a/tests/unit/responses/test_responses_api_bridge_flag.py +++ b/tests/unit/responses/test_responses_api_bridge_flag.py @@ -554,7 +554,6 @@ class TestUseResponsesApiBridgeFlag: truncation="auto", metadata={"k": "v"}, background=False, - text={"verbosity": "low"}, api_base="https://example.databricks.test/serving-endpoints", api_key="fake-databricks-api-key", num_retries=0, @@ -1103,3 +1102,281 @@ async def test_chatgpt_completion_routes_to_responses_without_a_provider_catalog def test_missing_catalog_entry_does_not_force_other_providers_to_responses(provider: str) -> None: info, model = responses_api_bridge_check(model="test-chatgpt-routing-control", custom_llm_provider=provider) assert (info.get("mode"), model) == (None, "test-chatgpt-routing-control") + + +_GPT_5_6_CHUNKS: Final = ( + { + "id": "chatcmpl-123", + "object": "chat.completion.chunk", + "created": 1677652288, + "model": "gpt-5.6", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "Answer"}, "finish_reason": None}], + }, + { + "id": "chatcmpl-123", + "object": "chat.completion.chunk", + "created": 1677652288, + "model": "gpt-5.6", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, +) + + +def _openai_chat_upstream(respx_mock: respx.MockRouter, stream: bool) -> respx.Route: + route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions") + if stream: + return route.respond( + 200, + headers={"content-type": "text/event-stream"}, + text="".join(f"data: {json.dumps(chunk)}\n\n" for chunk in _GPT_5_6_CHUNKS) + "data: [DONE]\n\n", + ) + return route.respond( + 200, + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-5.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + + +async def _bridged_responses_call(asynchronous: bool, stream: bool, **kwargs: object) -> None: + if not asynchronous: + response = litellm.responses( + input="Hello", stream=stream, api_key="fake-provider-api-key", num_retries=0, **kwargs + ) + _ = list(response) if stream else response + return + response = await litellm.aresponses( + input="Hello", stream=stream, api_key="fake-provider-api-key", num_retries=0, **kwargs + ) + _ = [event async for event in response] if stream else response + + +@pytest.mark.parametrize("verbosity", ["low", "high"]) +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.asyncio +async def test_bridge_sends_text_verbosity_as_chat_verbosity( + verbosity: str, asynchronous: bool, stream: bool, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + upstream: Final = _openai_chat_upstream(respx_mock, stream) + + await _bridged_responses_call( + asynchronous, stream, model="openai/gpt-5.6", use_chat_completions_api=True, text={"verbosity": verbosity} + ) + + assert upstream.call_count == 1 + assert json.loads(upstream.calls[0].request.read()) == { + "model": "gpt-5.6", + "messages": [{"role": "user", "content": "Hello"}], + "verbosity": verbosity, + **({"stream": True, "stream_options": {"include_usage": True}} if stream else {}), + } + + +_JSON_SCHEMA_TEXT_FORMAT: Final = { + "type": "json_schema", + "name": "answer", + "schema": {"type": "object", "properties": {"answer": {"type": "string"}}}, + "strict": True, +} +_JSON_SCHEMA_RESPONSE_FORMAT: Final = { + "type": "json_schema", + "json_schema": { + "name": "answer", + "schema": {"type": "object", "properties": {"answer": {"type": "string"}}}, + "strict": True, + }, +} + + +@pytest.mark.parametrize( + ("text", "expected_fields"), + [ + pytest.param( + {"verbosity": "low", "format": _JSON_SCHEMA_TEXT_FORMAT}, + {"response_format": _JSON_SCHEMA_RESPONSE_FORMAT, "verbosity": "low"}, + id="verbosity-and-format", + ), + pytest.param( + {"format": _JSON_SCHEMA_TEXT_FORMAT}, {"response_format": _JSON_SCHEMA_RESPONSE_FORMAT}, id="format-only" + ), + pytest.param(None, {}, id="no-text"), + ], +) +def test_bridge_maps_text_format_alongside_verbosity( + text: dict[str, object] | None, + expected_fields: dict[str, object], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + upstream: Final = _openai_chat_upstream(respx_mock, stream=False) + + litellm.responses( + model="openai/gpt-5.6", + input="Hello", + use_chat_completions_api=True, + api_key="fake-provider-api-key", + num_retries=0, + **({"text": text} if text is not None else {}), + ) + + assert upstream.call_count == 1 + assert json.loads(upstream.calls[0].request.read()) == { + "model": "gpt-5.6", + "messages": [{"role": "user", "content": "Hello"}], + **expected_fields, + } + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.asyncio +async def test_bridge_rejects_text_verbosity_the_provider_does_not_support_when_drop_params_is_off( + asynchronous: bool, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + upstream: Final = _databricks_chat_upstream(respx_mock) + + with pytest.raises(litellm.UnsupportedParamsError) as raised: + await _bridged_responses_call( + asynchronous, + False, + model="databricks/my-custom-model", + text={"verbosity": "low"}, + api_base="https://example.databricks.test/serving-endpoints", + ) + + assert upstream.call_count == 0, "the provider must not be called with a verbosity it does not support" + assert raised.value.status_code == 400 + assert "verbosity" in str(raised.value) + + +@pytest.mark.parametrize( + ("call_kwargs", "global_drop_params", "expected_fields"), + [ + pytest.param({"drop_params": True}, False, {}, id="request-drop-params"), + pytest.param({}, True, {}, id="global-drop-params"), + pytest.param({"allowed_openai_params": ["verbosity"]}, False, {"verbosity": "low"}, id="allowed-openai-params"), + ], +) +def test_bridge_text_verbosity_follows_provider_param_validation( + call_kwargs: dict[str, object], + global_drop_params: bool, + expected_fields: dict[str, object], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm, "drop_params", global_drop_params) + upstream: Final = _databricks_chat_upstream(respx_mock) + + response: Final = litellm.responses( + model="databricks/my-custom-model", + input="Hello", + text={"verbosity": "low"}, + api_base="https://example.databricks.test/serving-endpoints", + api_key="fake-databricks-api-key", + num_retries=0, + **call_kwargs, + ) + + assert upstream.call_count == 1 + assert json.loads(upstream.calls[0].request.read()) == { + "model": "my-custom-model", + "messages": [{"role": "user", "content": "Hello"}], + **expected_fields, + } + assert response.output[0].content[0].text == "Answer" + + +def test_native_responses_route_keeps_text_verbosity_nested( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( + 200, + json={ + "id": "resp_1", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-5.6", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + }, + ) + + litellm.responses( + model="openai/gpt-5.6", + input="Hello", + text={"verbosity": "low"}, + api_key="fake-provider-api-key", + num_retries=0, + ) + + assert upstream.call_count == 1 + assert json.loads(upstream.calls[0].request.read()) == { + "model": "gpt-5.6", + "input": "Hello", + "text": {"verbosity": "low"}, + } + + +_CLIENT_TOOL_SEARCH: Final = { + "type": "tool_search", + "execution": "client", + "description": "Search the deferred app tools.", + "parameters": {"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]}, +} + + +@pytest.mark.parametrize("verbosity", ["low", "high"]) +@pytest.mark.asyncio +async def test_bridge_sends_text_verbosity_on_a_client_tool_search_follow_up_turn( + verbosity: str, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + upstream: Final = _openai_chat_upstream(respx_mock, stream=False) + + await _bridged_responses_call( + True, + False, + model="openai/gpt-5.6", + use_chat_completions_api=True, + previous_response_id="chatcmpl-earlier-turn", + tools=[_CLIENT_TOOL_SEARCH], + text={"verbosity": verbosity}, + ) + + assert upstream.call_count == 1 + assert json.loads(upstream.calls[0].request.read()) == { + "model": "gpt-5.6", + "messages": [{"role": "user", "content": "Hello"}], + "tools": [ + { + "type": "function", + "function": { + "name": "tool_search", + "description": _CLIENT_TOOL_SEARCH["description"], + "parameters": _CLIENT_TOOL_SEARCH["parameters"], + }, + } + ], + "verbosity": verbosity, + } diff --git a/tests/unit/types/llms/test_types_llms_openai.py b/tests/unit/types/llms/test_types_llms_openai.py index 1b7a9693f7b..05801c9feec 100644 --- a/tests/unit/types/llms/test_types_llms_openai.py +++ b/tests/unit/types/llms/test_types_llms_openai.py @@ -607,6 +607,23 @@ def test_openai_file_object_accepts_pending_status(): assert file_obj.status == "pending" +@pytest.mark.parametrize("object_type", ({}, {"object": None}), ids=["missing", "null"]) +def test_openai_file_object_without_an_object_type_validates_as_a_file(object_type: dict[str, None]) -> None: + from litellm.types.llms.openai import OpenAIFileObject + + server_file: Final = { + "id": "file-b73f4c", + "bytes": 156, + "created_at": 1793238707, + "filename": "batch_requests.jsonl", + "purpose": "batch", + } + + file_obj: Final = OpenAIFileObject.model_validate({**server_file, **object_type}) + + assert file_obj.model_dump(exclude_none=True) == {**server_file, "object": "file"} + + class TestOpenAIFileObjectBatchGuardrailSerialization: """The proxy-only `litellm_batch_guardrail` key must reach the wire only when something set it.""" diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 48def46e3fe..54cd9087c4e 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -2037,6 +2037,13 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@isaacs/nonrecursive-glob-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.0.2.tgz", + "integrity": "sha512-GJPHIgPPtFl+nLhFHyF0zZoFcOFLsZWbHFFzo+Zr+PWDXJbl9hHxb2w+9SiF2fHmLgKaXIpUzQo3SbBBZ9WObw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", @@ -9657,13 +9664,14 @@ } }, "node_modules/minimatch": { - "version": "10.2.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", - "integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==", + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.2.tgz", + "integrity": "sha512-pWTgz5CN3tjqyZNSK2tePlSmu9IwM41q8yn8Usdqqwe9TIHh+9dEpLLjF4uApsfPEAZY71qtSiA2B2APEr+1Yw==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.2" + "@isaacs/nonrecursive-glob-match": "^1.0.2", + "brace-expansion": "^5.0.12" }, "engines": { "node": "18 || 20 || >=22" diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index f65316439bf..0707496a6bc 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -117,7 +117,7 @@ "js-yaml": "4.3.2", "brace-expansion": "5.0.12", "glob": "13.0.0", - "minimatch": "10.2.4", + "minimatch": "11.0.2", "ws": "8.21.0", "braces": "3.0.3", "axios": "1.13.6", diff --git a/ui/litellm-dashboard/public/assets/logos/hostinger.png b/ui/litellm-dashboard/public/assets/logos/hostinger.png new file mode 100644 index 00000000000..9ae3f4bcca1 Binary files /dev/null and b/ui/litellm-dashboard/public/assets/logos/hostinger.png differ diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx index 78782e755ac..d4c74785107 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx @@ -113,8 +113,9 @@ const { mockFetchAvailableModels, mockFetchAllModelDeployments } = vi.hoisted(() mockFetchAllModelDeployments: vi.fn(), })); -const { validateAutoRouterConfig } = vi.hoisted(() => ({ +const { validateAutoRouterConfig, getRouterSettingsCall } = vi.hoisted(() => ({ validateAutoRouterConfig: vi.fn().mockResolvedValue({ valid: true }), + getRouterSettingsCall: vi.fn(), })); vi.mock("../networking", () => ({ @@ -129,6 +130,7 @@ vi.mock("../networking", () => ({ modelAvailableCall: vi.fn().mockResolvedValue({ data: [] }), testAutoRouterRouting: vi.fn(), validateAutoRouterConfig, + getRouterSettingsCall, })); vi.mock("@/components/llm_calls/fetch_models", () => ({ @@ -196,6 +198,7 @@ describe("AddAutoRouterTab", () => { testQueryClient.clear(); mockFetchAvailableModels.mockResolvedValue([]); mockFetchAllModelDeployments.mockResolvedValue([]); + getRouterSettingsCall.mockResolvedValue({ current_values: {} }); vi.mocked(modelCostMap).mockResolvedValue({}); }); @@ -2194,6 +2197,102 @@ describe("AddAutoRouterTab", () => { expect(optionByLabel("Anthropic Family")!).toHaveTextContent(/Matches your deployments/); }); + it("enables a preset whose models a bare-star catch-all lists over a partial wildcard", async () => { + mockFetchAvailableModels.mockResolvedValue( + [...getRequiredModelsInPreset(ANTHROPIC_PRESET)].map((model) => ({ + model_group: model.replace(/^claude-/, ""), + mode: "chat", + providers: ["anthropic"], + })), + ); + mockFetchAllModelDeployments.mockResolvedValue([ + { model_name: "*", litellm_params: { model: "anthropic/claude-*" } }, + ]); + + renderAdvanced(); + openTemplateDropdown(); + + await waitFor(() => { + expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false); + }); + }); + + it("names a preset model missing when an alias over its listed id routes elsewhere", async () => { + mockFetchAvailableModels.mockResolvedValue( + [...getRequiredModelsInPreset(ANTHROPIC_PRESET)].map((model) => ({ + model_group: model.replace(/^claude-/, ""), + mode: "chat", + providers: ["anthropic"], + })), + ); + mockFetchAllModelDeployments.mockResolvedValue([ + { model_name: "*", litellm_params: { model: "anthropic/claude-*" } }, + { model_name: "corp-opus", litellm_params: { model: "anthropic/claude-opus-5-5" } }, + ]); + getRouterSettingsCall.mockResolvedValue({ current_values: { model_group_alias: { "haiku-4-5": "corp-opus" } } }); + + renderAdvanced(); + openTemplateDropdown(); + + await waitFor(() => { + expect(optionByLabel("Anthropic Family")!).toHaveTextContent(/Missing: claude-haiku-4-5$/); + }); + expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(true); + }); + + it("withholds templates until the alias over a listed id has loaded", async () => { + let completeSettings: ((settings: unknown) => void) | undefined; + getRouterSettingsCall.mockReturnValue( + new Promise((resolve) => { + completeSettings = resolve; + }), + ); + mockFetchAvailableModels.mockResolvedValue( + [...getRequiredModelsInPreset(ANTHROPIC_PRESET)].map((model) => ({ + model_group: model.replace(/^claude-/, ""), + mode: "chat", + providers: ["anthropic"], + })), + ); + mockFetchAllModelDeployments.mockResolvedValue([ + { model_name: "*", litellm_params: { model: "anthropic/claude-*" } }, + { model_name: "corp-opus", litellm_params: { model: "anthropic/claude-opus-5-5" } }, + ]); + + renderAdvanced(); + openTemplateDropdown(); + await waitFor(() => expect(screen.getByLabelText("Auto Router Name")).toHaveValue("auto-router")); + + expect(optionByLabel("Anthropic Family")!).toHaveTextContent("Checking model availability..."); + expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(true); + + completeSettings?.({ current_values: { model_group_alias: { "haiku-4-5": "corp-opus" } } }); + await waitFor(() => { + expect(optionByLabel("Anthropic Family")!).toHaveTextContent(/Missing: claude-haiku-4-5$/); + }); + }); + + it("credits a catch-all's listed ids for a non-admin without reading admin-only router settings", async () => { + mockFetchAvailableModels.mockResolvedValue( + [...getRequiredModelsInPreset(ANTHROPIC_PRESET)].map((model) => ({ + model_group: model.replace(/^claude-/, ""), + mode: "chat", + providers: ["anthropic"], + })), + ); + mockFetchAllModelDeployments.mockResolvedValue([ + { model_name: "*", litellm_params: { model: "anthropic/claude-*" } }, + ]); + + renderAdvanced(); + openTemplateDropdown(); + + await waitFor(() => { + expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false); + }); + expect(getRouterSettingsCall).not.toHaveBeenCalled(); + }); + it("prefills the expanded group names and submits them", async () => { const user = userEvent.setup(); mockFetchAvailableModels.mockResolvedValue(EXPANDED_HUB_GROUPS); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 8c3f402df1f..ceca03a924e 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -18,9 +18,9 @@ import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/comp import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; import { useZodForm } from "@/lib/forms/useZodForm"; import AccessGroupTagsCombobox from "./AccessGroupTagsCombobox"; -import { modelAvailableCall, validateAutoRouterConfig, type Team } from "../networking"; +import { getRouterSettingsCall, modelAvailableCall, validateAutoRouterConfig, type Team } from "../networking"; import { labelWithHint } from "@/components/shared/form/LabelWithHint"; -import { all_admin_roles } from "@/utils/roles"; +import { all_admin_roles, isProxyAdminRole } from "@/utils/roles"; import { canCreateAutoRouterForTeam, canModifyModel, type ModelWriteScope } from "@/utils/modelPermissions"; import TeamDropdown from "../common_components/team_dropdown"; import { type AddAutoRouterValues, handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit"; @@ -73,6 +73,7 @@ import { buildPresetPrefill, buildModelAvailability, deploymentRefsFromModelInfo, + modelGroupAliasTargets, ModelAvailability, PresetPrefill, AutoRouterPreset, @@ -105,6 +106,7 @@ interface AddAutoRouterTabProps { } const NO_PRESETS: AutoRouterPreset[] = []; +const NO_ALIASES: ReadonlyMap = new Map(); // Why the submit is unavailable, or null when it is available. The button reads this to disable // itself and to say what is missing, so the two can never give different answers. Checks the @@ -256,7 +258,13 @@ const AddAutoRouterTab: React.FC = ({ queryFn: () => fetchAllModelDeployments(accessToken, userId ?? "", userRole), enabled: Boolean(accessToken), }); - const modelsLoading = groupsLoading || deploymentsLoading; + const { data: aliasTargets = NO_ALIASES, isLoading: aliasesLoading } = useQuery({ + queryKey: ["routerSettings", "modelGroupAlias", accessToken], + queryFn: async () => + modelGroupAliasTargets((await getRouterSettingsCall(accessToken))?.current_values?.model_group_alias), + enabled: Boolean(accessToken) && isProxyAdminRole(userRole), + }); + const modelsLoading = groupsLoading || deploymentsLoading || aliasesLoading; useEffect(() => { if (!deployments || form.getFieldState("auto_router_name").isDirty) return; form.setValue( @@ -287,8 +295,10 @@ const AddAutoRouterTab: React.FC = ({ buildModelAvailability( modelInfo.map((m) => m.model_group), deploymentRefsFromModelInfo(deployments ?? []), + new Map(modelInfo.map((m) => [m.model_group, m.providers ?? []])), + aliasTargets, ), - [modelInfo, deployments], + [modelInfo, deployments, aliasTargets], ); const groupsOnlyAvailability = React.useMemo( () => @@ -299,8 +309,8 @@ const AddAutoRouterTab: React.FC = ({ [modelInfo], ); const solverModels = React.useMemo( - () => deployedRouterModels(modelInfo, deployments ?? []), - [modelInfo, deployments], + () => deployedRouterModels(modelInfo, deployments ?? [], aliasTargets), + [modelInfo, deployments, aliasTargets], ); const recommended = React.useMemo( () => recommendSolverPair(solverModels, goal, family), diff --git a/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts b/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts index c0d4931873f..6b20cab8b1f 100644 --- a/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts @@ -160,6 +160,66 @@ describe("deployedRouterModels", () => { expect(models[0]).toMatchObject({ name: "group", version: [5, 5], provider: "Anthropic, Bedrock" }); }); + it.each([ + ["a bare-star catch-all", "*", ""], + ["a named alias", "claude/*", "claude/"], + ])("suggests the Claude ids %s lists over a partial wildcard", (_label, name, prefix) => { + const groups = ["haiku-4-5", "opus-5-5"].map((id) => ({ + model_group: `${prefix}${id}`, + mode: "chat", + providers: ["anthropic"], + })); + const suggest = (row: AutoRouterDeployment) => + names(recommendSolverPair(deployedRouterModels(groups, [row]), "balanced", "auto")); + expect(suggest(deployment(name, "anthropic/claude-*"))).toEqual([`${prefix}haiku-4-5`, `${prefix}opus-5-5`]); + expect(suggest(deployment(name, "anthropic/claude-*", { model_info: { blocked: true } }))).toBeNull(); + }); + + it.each(["team/", ""])( + "suggests no id whose most specific wildcard %j is paused, even when a broader one is live", + (prefix) => { + const groups = ["haiku-4-5", "opus-5-5"].map((id) => ({ + model_group: `${prefix}${id}`, + mode: "chat", + providers: ["anthropic"], + })); + const rows = [ + deployment(`${prefix}*`, "anthropic/claude-*"), + deployment(`${prefix}opus-*`, "anthropic/claude-opus-*", { model_info: { blocked: true } }), + ]; + expect(names(recommendSolverPair(deployedRouterModels(groups, rows), "balanced", "auto"))).toBeNull(); + expect(names(recommendSolverPair(deployedRouterModels(groups, rows.slice(0, 1)), "balanced", "auto"))).toEqual([ + `${prefix}haiku-4-5`, + `${prefix}opus-5-5`, + ]); + }, + ); + + it("takes no Claude identity for an alias the router sends to another provider", () => { + const [model] = deployedRouterModels( + [{ model_group: "sonnet-5", mode: "chat", providers: ["openai"] }], + [deployment("*", "anthropic/claude-*")], + ); + expect(model).toEqual({ name: "sonnet-5", model: "sonnet-5", provider: "OpenAI" }); + }); + + it("takes the identity of the model an alias over a listed id routes to", () => { + const [model] = deployedRouterModels( + [{ model_group: "haiku-4-5", mode: "chat", providers: ["anthropic"] }], + [deployment("*", "anthropic/claude-*"), deployment("corp-opus", "anthropic/claude-opus-5-5")], + new Map([["haiku-4-5", "corp-opus"]]), + ); + expect(model).toMatchObject({ name: "haiku-4-5", line: "opus", version: [5, 5] }); + }); + + it("keeps a canonical alias's own identity when the catch-all's listed id does not parse", () => { + const [model] = deployedRouterModels( + [{ model_group: "claude-opus-5-5", mode: "chat", providers: ["anthropic"] }], + [deployment("*", "anthropic/claude-*")], + ); + expect(model).toMatchObject({ name: "claude-opus-5-5", line: "opus", version: [5, 5] }); + }); + it("keeps arbitrary canonical names available without pretending an unversioned alias is latest", () => { expect( deployedRouterModels([{ model_group: "claude-opus-8-12" }, { model_group: "claude-opus-latest" }], []).map( diff --git a/ui/litellm-dashboard/src/components/add_model/auto_setup.ts b/ui/litellm-dashboard/src/components/add_model/auto_setup.ts index eb21e17c265..dc5774d813a 100644 --- a/ui/litellm-dashboard/src/components/add_model/auto_setup.ts +++ b/ui/litellm-dashboard/src/components/add_model/auto_setup.ts @@ -1,6 +1,7 @@ import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels"; import { isAutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels"; import type { ModelGroup } from "@/components/llm_calls/fetch_models"; +import { type WildcardDeployment, type WildcardRoute, wildcardRoutes } from "@/lib/wildcard_routes"; import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; export const ROUTER_FAMILIES = { @@ -105,6 +106,19 @@ const identityOf = (deployment: AutoRouterDeployment): ModelIdentity | null => parseModel(deployment.litellm_params?.model ?? "") ?? parseModel(deployment.model_info?.base_model ?? deployment.litellm_params?.base_model ?? ""); +const wildcardDeployments = (deployments: readonly AutoRouterDeployment[]): WildcardDeployment[] => + deployments.flatMap((row) => { + const model = row.litellm_params?.model; + return row.model_name?.includes("*") && model + ? [{ modelGroup: row.model_name, underlyingModels: [model], blocked: Boolean(row.model_info?.blocked) }] + : []; + }); + +const routedIdentities = (routes: readonly WildcardRoute[], group: ModelGroup): (ModelIdentity | null)[] => { + const routed = routes.filter(({ paused }) => !paused).map(({ model }) => parseModel(model)); + return routed.some((identity) => identity !== null) ? routed : [parseModel(group.model_group)]; +}; + const providerOf = (deployment: AutoRouterDeployment, group: ModelGroup, family?: RouterFamily): string => { const raw = deployment.litellm_params?.model ?? ""; const provider = @@ -152,16 +166,21 @@ export const routingModelGroups = (groups: readonly ModelGroup[]): ModelGroup[] export const deployedRouterModels = ( groups: readonly ModelGroup[], deployments: readonly AutoRouterDeployment[], -): SolverModel[] => - routingModelGroups(groups) + aliasTargets: ReadonlyMap = new Map(), +): SolverModel[] => { + const wildcards = wildcardDeployments(deployments); + return routingModelGroups(groups) .map((group) => { - const rows = deployments.filter((row) => row.model_name === group.model_group); + const routed = { ...group, model_group: aliasTargets.get(group.model_group) ?? group.model_group }; + const rows = deployments.filter((row) => row.model_name === routed.model_group); const active = rows.filter((row) => !row.model_info?.blocked); - const isRouter = rows.some(isAutoRouterDeployment) || group.model_group.startsWith("auto_router/"); + const isRouter = rows.some(isAutoRouterDeployment) || routed.model_group.startsWith("auto_router/"); const hasWildcard = - group.model_group.includes("*") || active.some((row) => row.litellm_params?.model?.includes("*")); - const manualOnly = isRouter || hasWildcard || (rows.length > 0 && active.length === 0); - const identities = active.length > 0 ? active.map(identityOf) : [parseModel(group.model_group)]; + routed.model_group.includes("*") || active.some((row) => row.litellm_params?.model?.includes("*")); + const routes = rows.length > 0 ? [] : wildcardRoutes(routed.model_group, wildcards, routed.providers ?? []); + const paused = rows.length > 0 ? active.length === 0 : routes.length > 0 && routes.every((route) => route.paused); + const manualOnly = isRouter || hasWildcard || paused; + const identities = active.length > 0 ? active.map(identityOf) : routedIdentities(routes, routed); const first = identities[0]; const homogeneous = first && @@ -183,6 +202,7 @@ export const deployedRouterModels = ( }; }) .toSorted((a, b) => a.name.localeCompare(b.name)); +}; const latestLines = (models: readonly RouterModel[]): RouterModel[] => models.filter( diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index a8491f656a0..a8879eb570b 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -30,6 +30,7 @@ import googleLogo from "../../public/assets/logos/google.svg"; import groqLogo from "../../public/assets/logos/groq.svg"; import huggingfaceLogo from "../../public/assets/logos/huggingface.svg"; import hyperbolicLogo from "../../public/assets/logos/hyperbolic.svg"; +import hostingerLogo from "../../public/assets/logos/hostinger.png"; import infinityLogo from "../../public/assets/logos/infinity.png"; import jinaLogo from "../../public/assets/logos/jina.png"; import lambdaLogo from "../../public/assets/logos/lambda.svg"; @@ -131,6 +132,7 @@ export enum Providers { Hosted_Vllm = "Hosted vLLM", HUGGINGFACE = "Huggingface", HYPERBOLIC = "Hyperbolic", + HOSTINGER_ROUTER = "Hostinger Router", Infinity = "Infinity", JinaAI = "Jina AI", OPENCODE_GO = "OpenCode Go", @@ -263,6 +265,7 @@ export const provider_map: Record = { Hosted_Vllm: "hosted_vllm", HUGGINGFACE: "huggingface", HYPERBOLIC: "hyperbolic", + HOSTINGER_ROUTER: "hostinger_router", Infinity: "infinity", JinaAI: "jina_ai", OPENCODE_GO: "opencode_go", @@ -389,6 +392,7 @@ export const providerLogoMap: Partial> = { [Providers.Hosted_Vllm]: vllmLogo.src, [Providers.HUGGINGFACE]: huggingfaceLogo.src, [Providers.HYPERBOLIC]: hyperbolicLogo.src, + [Providers.HOSTINGER_ROUTER]: hostingerLogo.src, [Providers.Infinity]: infinityLogo.src, [Providers.JinaAI]: jinaLogo.src, [Providers.OPENCODE_GO]: opencodeLogo.src, @@ -493,6 +497,7 @@ const providerPlaceholderMap: Partial> = { [Providers.EDENAI]: "edenai/openai/gpt-mini-latest", [Providers.FalAI]: "fal_ai/fal-ai/flux-pro/v1.1-ultra", [Providers.Google_AI_Studio]: "gemini-pro", + [Providers.HOSTINGER_ROUTER]: "hostinger_router/claude-sonnet-5", [Providers.JinaAI]: "jina_ai/", [Providers.META]: "meta/muse-spark-1.1", [Providers.MICROSOFT_365_COPILOT]: "microsoft_365_copilot/chat", diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index f4e45e473df..f5f19c745c4 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -12,6 +12,7 @@ import { buildPresetPrefill, buildModelAvailability, deploymentRefsFromModelInfo, + modelGroupAliasTargets, normalizeModelName, resolveAvailableModel, resolveAvailableModels, @@ -646,11 +647,284 @@ describe("autorouter_presets", () => { expect(availability.underlyingIndex.size).toBe(0); }); - it("derives no pattern from a slashless wildcard model_name", () => { - const availability = buildModelAvailability(["gpt-5.4"], [wildcardDeployment("gpt*")]); + const servedBy = (provider: string, ...groups: string[]) => new Map(groups.map((group) => [group, [provider]])); + + it("resolves a model through the id a bare-star catch-all lists over a partial wildcard", () => { + const availability = buildModelAvailability( + ["claude-haiku-4-5-20251001-v1:0"], + [{ modelGroup: "*", underlyingModels: ["bedrock/us.anthropic.*"] }], + servedBy("bedrock", "claude-haiku-4-5-20251001-v1:0"), + ); + const config = simpleTierConfig("claude-haiku-4-5"); + expect(getMissingModels(config, availability)).toEqual([]); + expect(buildPresetPrefill(config, availability).complexityRouterConfig.tiers.SIMPLE).toEqual([ + "claude-haiku-4-5-20251001-v1:0", + ]); + }); + + it("keys a catch-all's listed id by the model it routes to, not by its own name", () => { + const availability = buildModelAvailability( + ["opus-5-5"], + [{ modelGroup: "*", underlyingModels: ["anthropic/claude-*"] }], + servedBy("anthropic", "opus-5-5"), + ); + expect(resolveAvailableModel("claude-opus-5-5", availability)).toBe("opus-5-5"); + }); + + it("credits no listed id to an alias the router sends to another provider", () => { + const availability = buildModelAvailability( + ["sonnet-5"], + [{ modelGroup: "*", underlyingModels: ["anthropic/claude-*"] }], + servedBy("openai", "sonnet-5"), + ); + expect(resolveAvailableModel("claude-sonnet-5", availability)).toBeUndefined(); + }); + + it("credits no listed id to a paused catch-all", () => { + const availability = buildModelAvailability( + ["opus-5-5"], + deploymentRefsFromModelInfo([ + { model_name: "*", litellm_params: { model: "anthropic/claude-*" }, model_info: { blocked: true } }, + ]), + servedBy("anthropic", "opus-5-5"), + ); + expect(resolveAvailableModel("claude-opus-5-5", availability)).toBeUndefined(); + }); + + it("keys an alias that shadows a listed id by the model its target serves", () => { + const availability = buildModelAvailability( + ["corp-opus", "haiku-4-5"], + [ + { modelGroup: "*", underlyingModels: ["anthropic/claude-*"] }, + { modelGroup: "corp-opus", underlyingModels: ["anthropic/claude-opus-5-5"] }, + ], + servedBy("anthropic", "corp-opus", "haiku-4-5"), + new Map([["haiku-4-5", "corp-opus"]]), + ); + expect(resolveAvailableModel("claude-haiku-4-5", availability)).toBeUndefined(); + expect(resolveAvailableModels("claude-opus-5-5", availability)).toEqual(["corp-opus", "haiku-4-5"]); + }); + + it("keys an alias to a listed id by the model the catch-all routes that id to", () => { + const availability = buildModelAvailability( + ["fast"], + [{ modelGroup: "*", underlyingModels: ["anthropic/claude-*"] }], + servedBy("anthropic", "fast"), + new Map([["fast", "haiku-4-5"]]), + ); + expect(resolveAvailableModel("claude-haiku-4-5", availability)).toBe("fast"); + }); + + it("keys an alias to a deployed group by that group's model, not by the id the catch-all lists", () => { + const availability = buildModelAvailability( + ["latest"], + [ + { modelGroup: "*", underlyingModels: ["anthropic/claude-*"] }, + { modelGroup: "sonnet-5", underlyingModels: ["anthropic/claude-opus-5-5"] }, + ], + servedBy("anthropic", "latest"), + new Map([["latest", "sonnet-5"]]), + ); + expect(resolveAvailableModel("claude-sonnet-5", availability)).toBeUndefined(); + expect(resolveAvailableModel("claude-opus-5-5", availability)).toBe("latest"); + }); + + it("credits no model to a group whose only deployment is paused", () => { + const availability = buildModelAvailability( + ["corp-opus"], + deploymentRefsFromModelInfo([ + { + model_name: "corp-opus", + litellm_params: { model: "anthropic/claude-opus-5-5" }, + model_info: { blocked: true }, + }, + ]), + servedBy("anthropic", "corp-opus"), + ); + expect(resolveAvailableModel("claude-opus-5-5", availability)).toBeUndefined(); + }); + + it("credits no model to a paused group named like the model", () => { + const availability = buildModelAvailability( + ["claude-opus-5-5"], + deploymentRefsFromModelInfo([ + { + model_name: "claude-opus-5-5", + litellm_params: { model: "anthropic/claude-opus-5-5" }, + model_info: { blocked: true }, + }, + ]), + servedBy("anthropic", "claude-opus-5-5"), + ); + expect(resolveAvailableModel("claude-opus-5-5", availability)).toBeUndefined(); + }); + + it("reads both alias forms the router accepts and skips malformed entries", () => { + const aliases = { + fast: "haiku-4-5", + big: { model: "corp-opus", hidden: true }, + unnamed: { hidden: true }, + blank: "", + numeric: 3, + }; + expect(modelGroupAliasTargets(aliases)).toEqual( + new Map([ + ["fast", "haiku-4-5"], + ["big", "corp-opus"], + ]), + ); + expect(modelGroupAliasTargets(undefined).size).toBe(0); + }); + + it("leaves a group another wildcard lists to that wildcard instead of the catch-all", () => { + const availability = buildModelAvailability( + ["a-cloud", "openai/gpt-6-sol", "sonnet-5"], + [ + { modelGroup: "a-cloud", underlyingModels: ["azure/gpt-6-sol"] }, + { modelGroup: "openai/*", underlyingModels: ["openai/*"] }, + { modelGroup: "*", underlyingModels: ["anthropic/claude-*"] }, + ], + servedBy("anthropic", "sonnet-5"), + ); + expect(resolveAvailableModel("gpt-6-sol", availability)).toBe("openai/gpt-6-sol"); + expect(resolveAvailableModel("claude-sonnet-5", availability)).toBe("sonnet-5"); + }); + + it.each([ + ["the text before its star ends mid-word", "anthropic/claude*"], + ["it names no provider", "claude-*"], + ])("lists nothing for a bare-star catch-all whose partial wildcard %s", (_label, underlying) => { + const availability = buildModelAvailability(["opus-5-5"], [{ modelGroup: "*", underlyingModels: [underlying] }]); expect(availability.underlyingIndex.size).toBe(0); }); + it("keeps a catch-all's declared base model from claiming every id the catch-all lists", () => { + const availability = buildModelAvailability( + ["fable-5", "sonnet-5"], + [{ modelGroup: "*", underlyingModels: ["anthropic/claude-*", "anthropic/claude-sonnet-5"] }], + servedBy("anthropic", "fable-5", "sonnet-5"), + ); + expect(resolveAvailableModel("claude-sonnet-5", availability)).toBe("sonnet-5"); + expect(resolveAvailableModel("claude-fable-5", availability)).toBe("fable-5"); + }); + + it("credits no listed id to a catch-all over a single model", () => { + const availability = buildModelAvailability( + ["team-alias"], + [{ modelGroup: "*", underlyingModels: ["anthropic/claude-opus-5-5"] }], + ); + expect(availability.underlyingIndex.size).toBe(0); + }); + + it("keys a named alias over a partial wildcard by the model it routes to", () => { + const availability = buildModelAvailability( + ["claude/haiku-4-5"], + [{ modelGroup: "claude/*", underlyingModels: ["anthropic/claude-*"] }], + servedBy("anthropic", "claude/haiku-4-5"), + ); + expect(resolveAvailableModel("claude-haiku-4-5", availability)).toBe("claude/haiku-4-5"); + }); + + it("keys a named wildcard whose prefix runs into the routed id's own words", () => { + const availability = buildModelAvailability( + ["team/gpt-4o-prod-mini"], + [{ modelGroup: "team/gpt-4o-prod*", underlyingModels: ["openai/gpt-4o*"] }], + servedBy("openai", "team/gpt-4o-prod-mini"), + ); + expect(resolveAvailableModel("gpt-4o-mini", availability)).toBe("team/gpt-4o-prod-mini"); + }); + + const anthropicTeam = { modelGroup: "team/*", underlyingModels: ["anthropic/claude-*"] }; + const bedrockOpusTeam = { modelGroup: "team/opus-*", underlyingModels: ["bedrock/anthropic.claude-sonnet-*-v1:0"] }; + const opuPrefixTeam = { modelGroup: "team/opu*", underlyingModels: ["anthropic/claude-opus-4-1"] }; + const dashFiveTeam = { modelGroup: "team/*-5", underlyingModels: ["anthropic/claude-*-5"] }; + + it.each([ + [ + "a longer pattern listed after a shorter one", + "team/opus-5-5", + [anthropicTeam, bedrockOpusTeam], + "claude-sonnet-5-5", + ], + [ + "a longer pattern listed before a shorter one", + "team/opus-5-5", + [bedrockOpusTeam, anthropicTeam], + "claude-sonnet-5-5", + ], + [ + "more regex characters breaking a length tie", + "team/opus-4-5", + [opuPrefixTeam, dashFiveTeam], + "claude-opus-4-5", + ], + ])("keys a listed id only by the wildcard the router picks, %s", (_label, group, deployments, routedModel) => { + const availability = buildModelAvailability([group], deployments, new Map([[group, ["anthropic", "bedrock"]]])); + expect(resolveAvailableModel(routedModel, availability)).toBe(group); + }); + + it("does not credit a listed id to a wildcard the router never reaches", () => { + const availability = buildModelAvailability( + ["team/opus-5-5"], + [bedrockOpusTeam, anthropicTeam], + servedBy("bedrock", "team/opus-5-5"), + ); + expect(resolveAvailableModel("claude-opus-5-5", availability)).toBeUndefined(); + }); + + it.each(["team/", ""])("credits nothing to a listed id whose most specific wildcard %j is paused", (prefix) => { + const pausedOpus = { + modelGroup: `${prefix}opus-*`, + underlyingModels: ["anthropic/claude-opus-*"], + blocked: true, + }; + const availability = buildModelAvailability( + [`${prefix}opus-5-5`, `${prefix}haiku-4-5`], + [{ ...anthropicTeam, modelGroup: `${prefix}*` }, pausedOpus], + servedBy("anthropic", `${prefix}opus-5-5`, `${prefix}haiku-4-5`), + ); + expect(resolveAvailableModel("claude-opus-5-5", availability)).toBeUndefined(); + expect(resolveAvailableModel("claude-haiku-4-5", availability)).toBe(`${prefix}haiku-4-5`); + }); + + it("keeps replacement patterns in a listed id literal", () => { + const availability = buildModelAvailability( + ["haiku-4-5$&"], + [{ modelGroup: "*", underlyingModels: ["anthropic/claude-*"] }], + servedBy("anthropic", "haiku-4-5$&"), + ); + expect(resolveAvailableModel("claude-haiku-4-5$&", availability)).toBe("haiku-4-5$&"); + }); + + it("prefers an id a native catch-all lists over a cloud group serving the same model", () => { + const availability = buildModelAvailability( + ["a-cloud", "opus-5-5"], + [ + { modelGroup: "a-cloud", underlyingModels: ["bedrock/anthropic.claude-opus-5-5"] }, + { modelGroup: "*", underlyingModels: ["anthropic/claude-*"] }, + ], + servedBy("anthropic", "opus-5-5"), + ); + expect(resolveAvailableModel("claude-opus-5-5", availability)).toBe("opus-5-5"); + }); + + it.each([ + ["its head and tail would overlap", "team/a*a", "team/a"], + ["an earlier middle part is missing", "team/*x*y*", "team/y"], + ])("does not match a named wildcard when %s", (_label, pattern, group) => { + const availability = buildModelAvailability([group], [wildcardDeployment(pattern)]); + expect(availability.underlyingIndex.size).toBe(0); + }); + + it("credits a listed id through the slashless wildcard the router sends it to", () => { + const availability = buildModelAvailability( + ["opus-5-5"], + [{ modelGroup: "opus-*", underlyingModels: ["anthropic/claude-opus-*"] }], + servedBy("anthropic", "opus-5-5"), + ); + expect(resolveAvailableModel("claude-opus-5-5", availability)).toBe("opus-5-5"); + }); + it("does not trust a group's name when no wildcard deployment covers it", () => { const availability = buildModelAvailability( ["team-a/claude-opus-5", "openai/*"], diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts index fd400a8163a..827361839a0 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts @@ -29,6 +29,7 @@ import { } from "@/components/add_model/complexity_router_tiers"; import { DEFAULT_ESCALATION_KEYWORDS } from "@/components/add_model/EscalationKeywords"; import { DEFAULT_MATCH_THRESHOLD } from "@/components/add_model/SemanticKeywordMatching"; +import { wildcardRoutes } from "@/lib/wildcard_routes"; // `key` is the stable JSON object key (e.g. "anthropic_family"); `label` is display text and // never an identity. @@ -68,6 +69,7 @@ export interface DeploymentModelRef { modelGroup: string; underlyingModels: readonly string[]; provider?: string; + blocked?: boolean; } export interface ModelAvailability { @@ -110,29 +112,14 @@ const normalizeUnderlyingModel = (model: string): string | null => { return stripped.toLowerCase() || null; }; -// A linear glob scan rather than a RegExp: patterns are admin-controlled model_name values, and a -// backtracking regex built from one ("a*a*a*...") can freeze another admin's dashboard. -const matchesWildcard = (pattern: string, name: string): boolean => { - const parts = pattern.split("*"); - if (parts.length === 1) return pattern === name; - const head = parts[0]; - const tail = parts[parts.length - 1]; - if (!name.startsWith(head) || !name.endsWith(tail)) return false; - if (name.length < head.length + tail.length) return false; - const scanEnd = name.length - tail.length; - const scanResult = parts.slice(1, -1).reduce((searchFrom: number, part: string) => { - if (searchFrom < 0) return -1; - const found = name.indexOf(part, searchFrom); - return found === -1 || found + part.length > scanEnd ? -1 : found + part.length; - }, head.length); - return scanResult >= 0; -}; - export const buildModelAvailability = ( modelGroups: Iterable, deployments: readonly DeploymentModelRef[], + groupProviders: ReadonlyMap = new Map(), + aliasTargets: ReadonlyMap = new Map(), ): ModelAvailability => { const groups = new Set(modelGroups); + const routedGroup = (group: string): string => aliasTargets.get(group) ?? group; const deploymentGroups = new Set(deployments.map((deployment) => deployment.modelGroup)); const deploymentProviders = new Map>(); for (const deployment of deployments) { @@ -140,27 +127,33 @@ export const buildModelAvailability = ( providers.add(deploymentProvider(deployment)); deploymentProviders.set(deployment.modelGroup, providers); } - const literalEntries = deployments - .filter((deployment) => groups.has(deployment.modelGroup)) - .flatMap((deployment) => + const routableDeployments = deployments.filter((deployment) => !deployment.blocked); + const routableByGroup = new Map(); + for (const deployment of routableDeployments) { + routableByGroup.set(deployment.modelGroup, [...(routableByGroup.get(deployment.modelGroup) ?? []), deployment]); + } + const literalEntries = Array.from(groups).flatMap((group) => + (routableByGroup.get(routedGroup(group)) ?? []).flatMap((deployment) => deployment.underlyingModels .map(normalizeUnderlyingModel) - .map((key) => ({ key, modelGroup: deployment.modelGroup, sourceGroup: deployment.modelGroup })), - ); - // Mirrors get_known_models_from_wildcard: a bare "*" model_name expands via its underlying - // wildcard (or not at all), and a wildcard without a "/" expands to nothing. - const wildcardPatterns = deployments.flatMap((deployment) => - (deployment.modelGroup === "*" ? deployment.underlyingModels : [deployment.modelGroup]) - .filter((pattern) => pattern !== "*" && pattern.includes("*") && pattern.includes("/")) - .map((pattern) => ({ pattern, sourceGroup: deployment.modelGroup })), + .map((key) => ({ key, modelGroup: group, sourceGroup: deployment.modelGroup })), + ), + ); + const pausedGroups = new Set(Array.from(deploymentGroups).filter((group) => !routableByGroup.has(group))); + const servedGroups = new Set(Array.from(groups).filter((group) => !pausedGroups.has(routedGroup(group)))); + const unlistedGroups = Array.from(groups).filter( + (group) => !group.includes("*") && !deploymentGroups.has(routedGroup(group)), + ); + const patternDeployments = deployments.filter(({ modelGroup }) => modelGroup.includes("*")); + const wildcardEntries = unlistedGroups.flatMap((group) => + wildcardRoutes(routedGroup(group), patternDeployments, groupProviders.get(group) ?? []) + .filter(({ paused }) => !paused) + .map(({ model, sourceGroup }) => ({ + key: normalizeUnderlyingModel(model), + modelGroup: group, + sourceGroup, + })), ); - const wildcardEntries = Array.from(groups) - .filter((group) => !group.includes("*") && !deploymentGroups.has(group)) - .flatMap((group) => - wildcardPatterns - .filter(({ pattern }) => matchesWildcard(pattern, group)) - .map(({ sourceGroup }) => ({ key: normalizeUnderlyingModel(group), modelGroup: group, sourceGroup })), - ); const entries = [...literalEntries, ...wildcardEntries]; const grouped = new Map>(); const providersByGroup = new Map>(); @@ -186,14 +179,14 @@ export const buildModelAvailability = ( }), ]), ); - return { modelGroups: groups, underlyingIndex, nativeUnderlyingIndex }; + return { modelGroups: servedGroups, underlyingIndex, nativeUnderlyingIndex }; }; export const deploymentRefsFromModelInfo = ( rows: readonly { model_name?: string | null; litellm_params?: { model?: string | null; base_model?: string | null; custom_llm_provider?: string | null } | null; - model_info?: { base_model?: string | null } | null; + model_info?: { base_model?: string | null; blocked?: boolean | null } | null; }[], ): DeploymentModelRef[] => rows.flatMap((row) => { @@ -204,10 +197,18 @@ export const deploymentRefsFromModelInfo = ( ].filter((model): model is string => Boolean(model)); const provider = row.litellm_params?.custom_llm_provider || routingProvider(row.litellm_params?.model ?? ""); return row.model_name && underlyingModels.length > 0 - ? [{ modelGroup: row.model_name, underlyingModels, provider }] + ? [{ modelGroup: row.model_name, underlyingModels, provider, blocked: row.model_info?.blocked || undefined }] : []; }); +export const modelGroupAliasTargets = (aliases: unknown): ReadonlyMap => + new Map( + Object.entries(typeof aliases === "object" && aliases !== null ? aliases : {}).flatMap(([alias, entry]) => { + const target: unknown = typeof entry === "object" && entry !== null && "model" in entry ? entry.model : entry; + return typeof target === "string" && target ? [[alias, target] as const] : []; + }), + ); + export const resolveAvailableModels = (requiredModel: string, availability: ModelAvailability): readonly string[] => { const { modelGroups, underlyingIndex } = availability; if (modelGroups.has(requiredModel)) return [requiredModel]; diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 9ce96f44e75..0ea6ac172ef 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -18082,9 +18082,9 @@ export interface paths { * @description Namespace a toolset as its own MCP endpoint. * * Connecting to /toolset//mcp exposes exactly the tools defined in - * the toolset. Access is enforced: non-admin API keys must have the toolset - * listed in their object_permission.mcp_toolsets grant list, or the request - * will be rejected with a 403. + * the toolset. Access is enforced: API keys other than an admin's key with no + * team must hold the toolset through one of their grant sources, or the + * request will be rejected with a 403. */ get: operations["toolset_mcp_route_toolset__toolset_name__mcp_get"]; /** @@ -18092,9 +18092,9 @@ export interface paths { * @description Namespace a toolset as its own MCP endpoint. * * Connecting to /toolset//mcp exposes exactly the tools defined in - * the toolset. Access is enforced: non-admin API keys must have the toolset - * listed in their object_permission.mcp_toolsets grant list, or the request - * will be rejected with a 403. + * the toolset. Access is enforced: API keys other than an admin's key with no + * team must hold the toolset through one of their grant sources, or the + * request will be rejected with a 403. */ put: operations["toolset_mcp_route_toolset__toolset_name__mcp_put"]; /** @@ -18102,9 +18102,9 @@ export interface paths { * @description Namespace a toolset as its own MCP endpoint. * * Connecting to /toolset//mcp exposes exactly the tools defined in - * the toolset. Access is enforced: non-admin API keys must have the toolset - * listed in their object_permission.mcp_toolsets grant list, or the request - * will be rejected with a 403. + * the toolset. Access is enforced: API keys other than an admin's key with no + * team must hold the toolset through one of their grant sources, or the + * request will be rejected with a 403. */ post: operations["toolset_mcp_route_toolset__toolset_name__mcp_post"]; /** @@ -18112,9 +18112,9 @@ export interface paths { * @description Namespace a toolset as its own MCP endpoint. * * Connecting to /toolset//mcp exposes exactly the tools defined in - * the toolset. Access is enforced: non-admin API keys must have the toolset - * listed in their object_permission.mcp_toolsets grant list, or the request - * will be rejected with a 403. + * the toolset. Access is enforced: API keys other than an admin's key with no + * team must hold the toolset through one of their grant sources, or the + * request will be rejected with a 403. */ delete: operations["toolset_mcp_route_toolset__toolset_name__mcp_delete"]; /** @@ -18122,9 +18122,9 @@ export interface paths { * @description Namespace a toolset as its own MCP endpoint. * * Connecting to /toolset//mcp exposes exactly the tools defined in - * the toolset. Access is enforced: non-admin API keys must have the toolset - * listed in their object_permission.mcp_toolsets grant list, or the request - * will be rejected with a 403. + * the toolset. Access is enforced: API keys other than an admin's key with no + * team must hold the toolset through one of their grant sources, or the + * request will be rejected with a 403. */ options: operations["toolset_mcp_route_toolset__toolset_name__mcp_options"]; /** @@ -18132,9 +18132,9 @@ export interface paths { * @description Namespace a toolset as its own MCP endpoint. * * Connecting to /toolset//mcp exposes exactly the tools defined in - * the toolset. Access is enforced: non-admin API keys must have the toolset - * listed in their object_permission.mcp_toolsets grant list, or the request - * will be rejected with a 403. + * the toolset. Access is enforced: API keys other than an admin's key with no + * team must hold the toolset through one of their grant sources, or the + * request will be rejected with a 403. */ head: operations["toolset_mcp_route_toolset__toolset_name__mcp_head"]; /** @@ -18142,9 +18142,9 @@ export interface paths { * @description Namespace a toolset as its own MCP endpoint. * * Connecting to /toolset//mcp exposes exactly the tools defined in - * the toolset. Access is enforced: non-admin API keys must have the toolset - * listed in their object_permission.mcp_toolsets grant list, or the request - * will be rejected with a 403. + * the toolset. Access is enforced: API keys other than an admin's key with no + * team must hold the toolset through one of their grant sources, or the + * request will be rejected with a 403. */ patch: operations["toolset_mcp_route_toolset__toolset_name__mcp_patch"]; trace?: never; diff --git a/ui/litellm-dashboard/src/lib/wildcard_routes.ts b/ui/litellm-dashboard/src/lib/wildcard_routes.ts new file mode 100644 index 00000000000..607e5d106a3 --- /dev/null +++ b/ui/litellm-dashboard/src/lib/wildcard_routes.ts @@ -0,0 +1,116 @@ +export type WildcardDeployment = { + readonly modelGroup: string; + readonly underlyingModels: readonly string[]; + readonly blocked?: boolean; +}; +export type WildcardRoute = { readonly model: string; readonly sourceGroup: string; readonly paused: boolean }; + +// A linear glob scan rather than a RegExp: patterns are admin-controlled model_name values, and a +// backtracking regex built from one ("a*a*a*...") can freeze another admin's dashboard. +const matchesWildcard = (pattern: string, name: string): boolean => { + const parts = pattern.split("*"); + const head = parts[0]; + const tail = parts[parts.length - 1]; + if (!name.startsWith(head) || !name.endsWith(tail)) return false; + if (name.length < head.length + tail.length) return false; + const scanEnd = name.length - tail.length; + const scanResult = parts.slice(1, -1).reduce((searchFrom: number, part: string) => { + if (searchFrom < 0) return -1; + const found = name.indexOf(part, searchFrom); + return found === -1 || found + part.length > scanEnd ? -1 : found + part.length; + }, head.length); + return scanResult >= 0; +}; + +const isProviderWide = (pattern: string): boolean => pattern.slice(pattern.indexOf("/") + 1) === "*"; + +const partialWordBeforeStar = (pattern: string): string => + pattern.slice(0, pattern.indexOf("*")).split("/").at(-1) ?? ""; + +const routesCapturedIds = (namePattern: string, deploymentPattern: string): boolean => { + const singleStar = namePattern.split("*").length === 2 && deploymentPattern.split("*").length === 2; + const listsCatalogIds = isProviderWide(namePattern) && isProviderWide(deploymentPattern); + if (!deploymentPattern.includes("/") || listsCatalogIds) return false; + const listsNameFragments = + /[a-z0-9]$/i.test(partialWordBeforeStar(deploymentPattern)) && partialWordBeforeStar(namePattern) === ""; + return singleStar && !listsNameFragments; +}; + +const substituteCapture = (namePattern: string, deploymentPattern: string, group: string): string => { + const [head, tail] = namePattern.split("*"); + return deploymentPattern.replace("*", () => group.slice(head.length, group.length - tail.length)); +}; + +const routingPatterns = (deployment: WildcardDeployment): readonly string[] => + deployment.modelGroup === "*" + ? deployment.underlyingModels.filter((model) => model !== "*" && isProviderWide(model)) + : [deployment.modelGroup].filter((pattern) => pattern.includes("*")); + +const routesThroughPattern = ( + deployment: WildcardDeployment, + pattern: string, + group: string, +): readonly WildcardRoute[] => { + const capturing = + deployment.modelGroup === "*" + ? [] + : deployment.underlyingModels.filter((model) => routesCapturedIds(pattern, model)); + const models = capturing.length > 0 ? capturing.map((model) => substituteCapture(pattern, model, group)) : [group]; + return models.map((model) => ({ model, sourceGroup: deployment.modelGroup, paused: Boolean(deployment.blocked) })); +}; + +const PYTHON_ESCAPED_CHARS = new Set(Array.from("()[]{}?*+-|^$\\.&~# \t\n\r\v\f")); +const ROUTER_COMPLEXITY_CHARS = new Set(Array.from("*+?\\^$|()")); + +const routerRegexPart = (char: string): string => { + if (char === "*") return "(.*)"; + return PYTHON_ESCAPED_CHARS.has(char) ? `\\${char}` : char; +}; + +const routerSpecificity = (namePattern: string): readonly [number, number] => { + const regex = Array.from(namePattern, routerRegexPart).join(""); + return [regex.length, Array.from(regex).filter((char) => ROUTER_COMPLEXITY_CHARS.has(char)).length]; +}; + +const isMoreSpecific = (namePattern: string, than: string): boolean => { + const [length, complexity] = routerSpecificity(namePattern); + const [otherLength, otherComplexity] = routerSpecificity(than); + return length > otherLength || (length === otherLength && complexity > otherComplexity); +}; + +const catchAllRoutes = (group: string, deployments: readonly WildcardDeployment[]): readonly WildcardRoute[] => + deployments + .filter((deployment) => deployment.modelGroup === "*") + .flatMap((deployment) => + deployment.underlyingModels + .filter((model) => routesCapturedIds("*", model)) + .map((model) => ({ + model: substituteCapture("*", model, group), + sourceGroup: "*", + paused: Boolean(deployment.blocked), + })), + ); + +const routesByPrecedence = (group: string, deployments: readonly WildcardDeployment[]): readonly WildcardRoute[] => { + const matches = deployments.flatMap((deployment) => + routingPatterns(deployment) + .filter((pattern) => matchesWildcard(pattern, group)) + .map((pattern) => ({ deployment, pattern })), + ); + if (matches.length === 0) return catchAllRoutes(group, deployments); + const routedName = matches + .map(({ deployment }) => deployment.modelGroup) + .reduce((best, name) => (isMoreSpecific(name, best) ? name : best)); + return matches + .filter(({ deployment }) => deployment.modelGroup === routedName) + .flatMap(({ deployment, pattern }) => routesThroughPattern(deployment, pattern, group)); +}; + +export const wildcardRoutes = ( + group: string, + deployments: readonly WildcardDeployment[], + groupProviders: readonly string[], +): readonly WildcardRoute[] => + routesByPrecedence(group, deployments).filter( + ({ model }) => model === group || groupProviders.includes(model.split("/")[0]), + ); diff --git a/uv.lock b/uv.lock index 8cd31eb0828..19d3a1ffd4f 100644 --- a/uv.lock +++ b/uv.lock @@ -16,6 +16,7 @@ exclude-newer-span = "P3D" [options.exclude-newer-package] brotli = "2026-10-22T14:01:00Z" +pypdf = "2026-10-28T17:01:00Z" [manifest] members = [ @@ -8372,11 +8373,11 @@ wheels = [ [[package]] name = "pyasn1" -version = "0.6.4" +version = "0.6.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" } +sdist = { url = "https://files.pythonhosted.org/packages/88/7e/3c0da41061a2021cac7cd524fd57092adbe5b515e8bf9005702e366802e2/pyasn1-0.6.5.tar.gz", hash = "sha256:6b11616a6bcf3a40e635f082bb232fa57b5e0dd7f517bd7ddd69ec3794508a47", size = 155152, upload-time = "2026-10-27T06:32:49.125Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" }, + { url = "https://files.pythonhosted.org/packages/19/1f/3eefd5c84c35cae44549ce1c7f7e58f7945b34f5a6bce07a135438522073/pyasn1-0.6.5-py3-none-any.whl", hash = "sha256:4017b0fce125096b8573b39c989fdb1a4b19cf761e683398184550fea0e66587", size = 84807, upload-time = "2026-10-27T06:32:48.045Z" }, ] [[package]] @@ -8869,14 +8870,14 @@ wheels = [ [[package]] name = "pypdf" -version = "6.21.2" +version = "6.21.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/06/28/081f1cbcca7042441dafc8de00f48ba60853f89e5bff92c5f26477346d81/pypdf-6.21.2.tar.gz", hash = "sha256:0a3e1c5c886a1be03646c16a04827cc166c2a0575c971b26a4bf89686551c2d4", size = 7144571, upload-time = "2026-10-21T16:44:55.272Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/e1/ec8f4d155c997c45917153a5521e60d6ee6389d68abdcbc8e44e564caffc/pypdf-6.21.4.tar.gz", hash = "sha256:c8817e3f888d47a087872ded69812c1ef30cacb871d6e053e0c00bbaf0bc2163", size = 7109255, upload-time = "2026-10-28T17:00:54.81Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/4d/c9d2d67f204eff463237a1a8f5e25a6923b7b1043982b91fcaf104c8ef60/pypdf-6.21.2-py3-none-any.whl", hash = "sha256:1d44d1a3a200f25062adff0085b02040e4646431ac2ccac22e42679e91466084", size = 407766, upload-time = "2026-10-21T16:44:53.626Z" }, + { url = "https://files.pythonhosted.org/packages/3f/4b/77e8ad4ac42c9d93600f40541a50e9feb3a61d2bbee2d76f659346072051/pypdf-6.21.4-py3-none-any.whl", hash = "sha256:33e4fb41ca352810eb4e9838aa0d82d6888a5f8bf682f43e99cce5faa666255c", size = 410430, upload-time = "2026-10-28T17:00:53.146Z" }, ] [[package]] diff --git a/vscode-extension/package-lock.json b/vscode-extension/package-lock.json index 5f6cf7edf4e..80b868e528b 100644 --- a/vscode-extension/package-lock.json +++ b/vscode-extension/package-lock.json @@ -644,6 +644,13 @@ "node": ">=18" } }, + "node_modules/@isaacs/nonrecursive-glob-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.0.2.tgz", + "integrity": "sha512-GJPHIgPPtFl+nLhFHyF0zZoFcOFLsZWbHFFzo+Zr+PWDXJbl9hHxb2w+9SiF2fHmLgKaXIpUzQo3SbBBZ9WObw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", @@ -2700,13 +2707,14 @@ } }, "node_modules/minimatch": { - "version": "10.2.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", - "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.2.tgz", + "integrity": "sha512-pWTgz5CN3tjqyZNSK2tePlSmu9IwM41q8yn8Usdqqwe9TIHh+9dEpLLjF4uApsfPEAZY71qtSiA2B2APEr+1Yw==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.8" + "@isaacs/nonrecursive-glob-match": "^1.0.2", + "brace-expansion": "^5.0.12" }, "engines": { "node": "18 || 20 || >=22" diff --git a/vscode-extension/package.json b/vscode-extension/package.json index a3cf2dc7a5f..42626439aca 100644 --- a/vscode-extension/package.json +++ b/vscode-extension/package.json @@ -83,5 +83,8 @@ "esbuild": "^0.28.2", "typescript": "^5.9.3", "vitest": "^4.1.12" + }, + "overrides": { + "minimatch": "11.0.2" } }