mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge remote-tracking branch 'origin/main' into litellm_gemini_tool_call_id_length
# Conflicts: # litellm/responses/litellm_completion_transformation/transformation.py
This commit is contained in:
commit
0cf5d818db
126 changed files with 8055 additions and 662 deletions
|
|
@ -177,6 +177,8 @@ MID_STREAM_FAILURE_LOG_WAIT_SECONDS: Final = 1.0
|
|||
# Set to 0 to disable truncation.
|
||||
MAX_BASE64_LENGTH_FOR_LOGGING: Final = get_env_int("MAX_BASE64_LENGTH_FOR_LOGGING", 64)
|
||||
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS: Final = 256 * 1024
|
||||
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS: Final = 256
|
||||
BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS: Final = 8 * 1024 * 1024
|
||||
REDACTED_BY_LITELLM: Final = "redacted-by-litellm"
|
||||
# in-memory stand-in handed to provider converters for redacted arguments; never stored
|
||||
REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}"
|
||||
|
|
@ -634,6 +636,8 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60
|
|||
BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024)
|
||||
# Anthropic's Messages API rejects thinking.budget_tokens < 1024.
|
||||
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024
|
||||
ANTHROPIC_MAX_USER_ID_LENGTH: Final = 512
|
||||
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH: Final = 255
|
||||
ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:"
|
||||
REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5)
|
||||
DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000)
|
||||
|
|
@ -1163,6 +1167,7 @@ openai_compatible_providers: Final[list[str]] = [
|
|||
"cortecs",
|
||||
"scx-ai",
|
||||
"prism",
|
||||
"hostinger_router",
|
||||
"sail",
|
||||
"reka",
|
||||
"streamlake",
|
||||
|
|
|
|||
|
|
@ -3,10 +3,10 @@ from collections.abc import Mapping, Sequence
|
|||
from dataclasses import dataclass, field, replace
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypeAlias
|
||||
from typing import Annotated, Final, Literal, TypeAlias
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, StrictBool, TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, BeforeValidator, ConfigDict, TypeAdapter, ValidationError
|
||||
from pydantic_core import to_jsonable_python
|
||||
from typing_extensions import assert_never
|
||||
|
||||
|
|
@ -23,6 +23,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
|||
from litellm.llms.base_llm.decisions.transformation import (
|
||||
BaseDecisionsConfig,
|
||||
DecisionsResponseValidationError,
|
||||
catalog_mode,
|
||||
image_media_type,
|
||||
image_param,
|
||||
input_images,
|
||||
|
|
@ -61,7 +62,7 @@ from litellm.types.decisions import (
|
|||
UnsupportedDecisionsRequest,
|
||||
)
|
||||
from litellm.types.llms.openai import ChatCompletionUserMessage
|
||||
from litellm.types.utils import LlmProviders, ProviderSpecificHeader
|
||||
from litellm.types.utils import LlmProviders, ProviderSpecificHeader, is_decisions_model_mode
|
||||
from litellm.utils import ProviderConfigManager, get_model_cost_key
|
||||
|
||||
SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]]
|
||||
|
|
@ -257,19 +258,105 @@ def _request_safety_identifier(
|
|||
return None
|
||||
|
||||
|
||||
def _fields_without_unsupported_media(
|
||||
fields: RequestFields,
|
||||
def _bool_or_none(value: object) -> bool | None:
|
||||
return value if isinstance(value, bool) else None
|
||||
|
||||
|
||||
def _str_or_none(value: object) -> str | None:
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
_DeploymentFlag: TypeAlias = Annotated[bool | None, BeforeValidator(_bool_or_none)]
|
||||
|
||||
|
||||
class _DeploymentCapabilities(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, from_attributes=True)
|
||||
|
||||
mode: Annotated[str | None, BeforeValidator(_str_or_none)] = None
|
||||
supports_box_questions: _DeploymentFlag = None
|
||||
supports_audio_input: _DeploymentFlag = None
|
||||
supports_video_input: _DeploymentFlag = None
|
||||
|
||||
def decisions_media_flag(self, flag: bool | None, model: str, custom_llm_provider: str) -> bool | None:
|
||||
if flag is None:
|
||||
return None
|
||||
mode: Final = self.mode if self.mode is not None else catalog_mode(model, custom_llm_provider)
|
||||
return flag if mode is None or is_decisions_model_mode(mode) else None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ResolvedSupport:
|
||||
box_questions: bool
|
||||
audio: bool
|
||||
videos: bool
|
||||
|
||||
|
||||
def _takes_audio(
|
||||
provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities
|
||||
) -> bool:
|
||||
return provider_config.supports_audio_input(
|
||||
model,
|
||||
custom_llm_provider,
|
||||
deployment.decisions_media_flag(deployment.supports_audio_input, model, custom_llm_provider),
|
||||
)
|
||||
|
||||
|
||||
def _takes_videos(
|
||||
provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities
|
||||
) -> bool:
|
||||
return provider_config.supports_video_input(
|
||||
model,
|
||||
custom_llm_provider,
|
||||
deployment.decisions_media_flag(deployment.supports_video_input, model, custom_llm_provider),
|
||||
)
|
||||
|
||||
|
||||
def deployment_refuses_media(
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
model_info: Mapping[str, object],
|
||||
call_kwargs: Mapping[str, object],
|
||||
*,
|
||||
audio: bool,
|
||||
videos: bool,
|
||||
) -> bool:
|
||||
if _drops_params(call_kwargs):
|
||||
return False
|
||||
provider_config: Final = get_decisions_provider_config(model, custom_llm_provider)
|
||||
canonical_model: Final = provider_config.canonical_model(model)
|
||||
deployment: Final = _DeploymentCapabilities.model_validate(model_info)
|
||||
return (audio and not _takes_audio(provider_config, canonical_model, custom_llm_provider, deployment)) or (
|
||||
videos and not _takes_videos(provider_config, canonical_model, custom_llm_provider, deployment)
|
||||
)
|
||||
|
||||
|
||||
def _resolved_support(
|
||||
provider_config: BaseDecisionsConfig,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
headers: Mapping[str, str],
|
||||
kwargs: Mapping[str, object],
|
||||
fields: RequestFields,
|
||||
) -> _ResolvedSupport:
|
||||
deployment: Final = _DeploymentCapabilities.model_validate(kwargs.get("model_info") or {})
|
||||
return _ResolvedSupport(
|
||||
box_questions=provider_config.supports_box_questions(
|
||||
model, custom_llm_provider, headers, deployment.supports_box_questions
|
||||
),
|
||||
audio=bool(fields.audio) and _takes_audio(provider_config, model, custom_llm_provider, deployment),
|
||||
videos=bool(fields.videos) and _takes_videos(provider_config, model, custom_llm_provider, deployment),
|
||||
)
|
||||
|
||||
|
||||
def _fields_without_unsupported_media(
|
||||
fields: RequestFields, support: _ResolvedSupport, kwargs: Mapping[str, object]
|
||||
) -> RequestFields:
|
||||
if not _drops_params(kwargs):
|
||||
return fields
|
||||
return replace(
|
||||
fields,
|
||||
audio=fields.audio if provider_config.supports_audio_input(model, custom_llm_provider) else None,
|
||||
videos=fields.videos if provider_config.supports_video_input(model, custom_llm_provider) else None,
|
||||
audio=fields.audio if support.audio else None,
|
||||
videos=fields.videos if support.videos else None,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -292,10 +379,8 @@ class _MalformedImages:
|
|||
def _unsupported_params(
|
||||
ir_request: DecisionsIRRequest,
|
||||
provider_config: BaseDecisionsConfig,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
headers: Mapping[str, str],
|
||||
supports_box_questions: bool,
|
||||
support: _ResolvedSupport,
|
||||
) -> tuple[str, ...]:
|
||||
refuses_safety_identifier: Final = (
|
||||
ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier
|
||||
|
|
@ -304,13 +389,9 @@ def _unsupported_params(
|
|||
None if provider_config.supports_images_in(ir_request.input, headers) else image_param(ir_request.input)
|
||||
)
|
||||
refuses_assets: Final = bool(ir_request.assets) and not provider_config.supports_assets(headers)
|
||||
refuses_box_questions: Final = _has_box_question(ir_request.questions) and not supports_box_questions
|
||||
refuses_audio: Final = bool(ir_request.audio) and not provider_config.supports_audio_input(
|
||||
model, custom_llm_provider
|
||||
)
|
||||
refuses_videos: Final = bool(ir_request.videos) and not provider_config.supports_video_input(
|
||||
model, custom_llm_provider
|
||||
)
|
||||
refuses_box_questions: Final = _has_box_question(ir_request.questions) and not support.box_questions
|
||||
refuses_audio: Final = bool(ir_request.audio) and not support.audio
|
||||
refuses_videos: Final = bool(ir_request.videos) and not support.videos
|
||||
return (
|
||||
*(("safety_identifier",) if refuses_safety_identifier else ()),
|
||||
*(() if refused_image_param is None else (refused_image_param,)),
|
||||
|
|
@ -333,7 +414,7 @@ def _without_unsupported_params(
|
|||
ir_request: DecisionsIRRequest,
|
||||
provider_config: BaseDecisionsConfig,
|
||||
headers: Mapping[str, str],
|
||||
supports_box_questions: bool,
|
||||
support: _ResolvedSupport,
|
||||
) -> DecisionsIRRequest:
|
||||
return replace(
|
||||
ir_request,
|
||||
|
|
@ -345,9 +426,7 @@ def _without_unsupported_params(
|
|||
safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None,
|
||||
assets=ir_request.assets if provider_config.supports_assets(headers) else MappingProxyType({}),
|
||||
questions=(
|
||||
ir_request.questions
|
||||
if supports_box_questions
|
||||
else tuple(filter(_is_not_box_question, ir_request.questions))
|
||||
ir_request.questions if support.box_questions else tuple(filter(_is_not_box_question, ir_request.questions))
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -368,39 +447,20 @@ def _without_malformed_images(
|
|||
return well_formed
|
||||
|
||||
|
||||
class _DeploymentBoxQuestions(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, from_attributes=True)
|
||||
|
||||
supports_box_questions: StrictBool | None = None
|
||||
|
||||
|
||||
def _deployment_supports_box_questions(kwargs: Mapping[str, object]) -> bool | None:
|
||||
try:
|
||||
return _DeploymentBoxQuestions.model_validate(kwargs.get("model_info") or {}).supports_box_questions
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _provider_ir_request(
|
||||
ir_request: DecisionsIRRequest,
|
||||
*,
|
||||
provider_config: BaseDecisionsConfig,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
headers: Mapping[str, str],
|
||||
support: _ResolvedSupport,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> DecisionsIRRequest | _UnsupportedParams | _NothingLeftAfterDrop:
|
||||
supports_box_questions: Final = provider_config.supports_box_questions(
|
||||
model, custom_llm_provider, headers, _deployment_supports_box_questions(kwargs)
|
||||
)
|
||||
unsupported: Final = _unsupported_params(
|
||||
ir_request, provider_config, model, custom_llm_provider, headers, supports_box_questions
|
||||
)
|
||||
unsupported: Final = _unsupported_params(ir_request, provider_config, headers, support)
|
||||
if not unsupported:
|
||||
return ir_request
|
||||
if not _drops_params(kwargs):
|
||||
return _UnsupportedParams(names=unsupported)
|
||||
supported: Final = _without_unsupported_params(ir_request, provider_config, headers, supports_box_questions)
|
||||
supported: Final = _without_unsupported_params(ir_request, provider_config, headers, support)
|
||||
if _leaves_no_input(supported.input):
|
||||
return _NothingLeftAfterDrop(names=unsupported, missing="input to decide on")
|
||||
if not supported.questions:
|
||||
|
|
@ -479,10 +539,11 @@ def prepare_call(
|
|||
raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error
|
||||
try:
|
||||
headers: Final = decisions_outbound_headers(extra_headers, provider, kwargs)
|
||||
support: Final = _resolved_support(provider_config, canonical_model, provider, headers, kwargs, fields)
|
||||
request_safety_identifier: Final = _request_safety_identifier(safety_identifier, provider_config, kwargs)
|
||||
request: Final = _fields_without_unsupported_media(
|
||||
fields, provider_config, canonical_model, provider, kwargs
|
||||
).validate(safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs))
|
||||
request: Final = _fields_without_unsupported_media(fields, support, kwargs).validate(
|
||||
safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs)
|
||||
)
|
||||
except ValidationError as error:
|
||||
raise litellm.BadRequestError(
|
||||
message=f"Invalid Decisions request: {error}",
|
||||
|
|
@ -523,12 +584,7 @@ def prepare_call(
|
|||
llm_provider=provider,
|
||||
)
|
||||
ir_request: Final = _provider_ir_request(
|
||||
caller_ir_request,
|
||||
provider_config=provider_config,
|
||||
model=canonical_model,
|
||||
custom_llm_provider=provider,
|
||||
headers=headers,
|
||||
kwargs=kwargs,
|
||||
caller_ir_request, provider_config=provider_config, headers=headers, support=support, kwargs=kwargs
|
||||
)
|
||||
if isinstance(ir_request, _UnsupportedParams):
|
||||
raise litellm.UnsupportedParamsError(
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import itertools
|
|||
import os
|
||||
import re
|
||||
from collections.abc import Callable, Container, Iterable, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
|
@ -102,6 +102,192 @@ def _validated_object_list(value: object) -> list[object] | None:
|
|||
return None
|
||||
|
||||
|
||||
_MessageT: Final = TypeVar("_MessageT")
|
||||
_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None]
|
||||
_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls")
|
||||
|
||||
|
||||
def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]:
|
||||
lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS)
|
||||
return tuple(
|
||||
itertools.chain.from_iterable(
|
||||
_OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _block_carries_cache_control(block: object) -> bool:
|
||||
return isinstance(block, dict) and _CONTENT_PART_ADAPTER.validate_python(block).get("cache_control") is not None
|
||||
|
||||
|
||||
_ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT: Final = frozenset({"tool", "function"})
|
||||
|
||||
|
||||
def _message_mark_reaches_provider(fields: Mapping[str, object]) -> bool:
|
||||
if fields.get("cache_control") is None:
|
||||
return False
|
||||
return (
|
||||
not isinstance(fields.get("content"), list)
|
||||
or fields.get("role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT
|
||||
)
|
||||
|
||||
|
||||
def cache_control_count(message: object) -> int:
|
||||
if not isinstance(message, dict):
|
||||
return 0
|
||||
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
|
||||
block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block))
|
||||
return int(_message_mark_reaches_provider(fields)) + block_marks
|
||||
|
||||
|
||||
def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]:
|
||||
others: Final = {key: value for key, value in fields.items() if key != "cache_control"}
|
||||
if "cache_control" not in fields:
|
||||
return others
|
||||
cache_control: Final = rewrite(fields["cache_control"])
|
||||
return others if cache_control is None else {**others, "cache_control": cache_control}
|
||||
|
||||
|
||||
def block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object:
|
||||
if not isinstance(block, dict):
|
||||
return block
|
||||
return _fields_with_cache_control(_CONTENT_PART_ADAPTER.validate_python(block), rewrite)
|
||||
|
||||
|
||||
def with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT:
|
||||
if not isinstance(message, dict):
|
||||
return message
|
||||
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
|
||||
blocks_rewritten: Final = {
|
||||
key: (
|
||||
[block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)]
|
||||
if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list)
|
||||
else value
|
||||
)
|
||||
for key, value in fields.items()
|
||||
}
|
||||
return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped
|
||||
_MessageT, _fields_with_cache_control(blocks_rewritten, rewrite)
|
||||
)
|
||||
|
||||
|
||||
def dropped_cache_control(_: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _first_marked_index(value: object) -> int | None:
|
||||
if not isinstance(value, list):
|
||||
return None
|
||||
blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value)
|
||||
return next((index for index, block in enumerate(blocks) if _block_carries_cache_control(block)), None)
|
||||
|
||||
|
||||
_MarkSlot: TypeAlias = tuple[str, int]
|
||||
_MESSAGE_MARK_SLOT: Final[_MarkSlot] = ("cache_control", -1)
|
||||
|
||||
|
||||
def _block_mark_slots(fields: Mapping[str, object], key: str) -> tuple[_MarkSlot, ...]:
|
||||
value: Final = fields.get(key)
|
||||
if not isinstance(value, list):
|
||||
return ()
|
||||
blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value)
|
||||
return tuple((key, index) for index, block in enumerate(blocks) if _block_carries_cache_control(block))
|
||||
|
||||
|
||||
def _mark_slots_in_wire_order(fields: Mapping[str, object]) -> tuple[_MarkSlot, ...]:
|
||||
content_slots: Final = _block_mark_slots(fields, "content")
|
||||
tool_call_slots: Final = _block_mark_slots(fields, "tool_calls")
|
||||
if not _message_mark_reaches_provider(fields):
|
||||
return content_slots + tool_call_slots
|
||||
if isinstance(fields.get("content"), list):
|
||||
return content_slots + (_MESSAGE_MARK_SLOT,) + tool_call_slots
|
||||
return (_MESSAGE_MARK_SLOT,) + content_slots + tool_call_slots
|
||||
|
||||
|
||||
def _blocks_without_marks(key: str, value: object, dropped_slots: frozenset[_MarkSlot]) -> object:
|
||||
if key not in _MARK_CARRYING_MESSAGE_FIELDS or not isinstance(value, list):
|
||||
return value
|
||||
return [
|
||||
block_with_cache_control(block, dropped_cache_control) if (key, index) in dropped_slots else block
|
||||
for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(value))
|
||||
]
|
||||
|
||||
|
||||
def _without_oldest_marks(message: _MessageT, dropped: int) -> _MessageT:
|
||||
if dropped == 0 or not isinstance(message, dict):
|
||||
return message
|
||||
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
|
||||
dropped_slots: Final = frozenset(_mark_slots_in_wire_order(fields)[:dropped])
|
||||
blocks_unmarked: Final = {key: _blocks_without_marks(key, value, dropped_slots) for key, value in fields.items()}
|
||||
return cast( # cast-ok: the same message with its oldest wire-order marks dropped
|
||||
_MessageT,
|
||||
_fields_with_cache_control(blocks_unmarked, dropped_cache_control)
|
||||
if _MESSAGE_MARK_SLOT in dropped_slots
|
||||
else blocks_unmarked,
|
||||
)
|
||||
|
||||
|
||||
def tool_message_with_single_cache_mark(message: _MessageT) -> _MessageT:
|
||||
if not isinstance(message, dict):
|
||||
return message
|
||||
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
|
||||
content: Final = fields.get("content")
|
||||
if fields.get("role") != "tool" or not isinstance(content, list):
|
||||
return message
|
||||
kept_index: Final = None if fields.get("cache_control") is not None else _first_marked_index(content)
|
||||
return cast( # cast-ok: the same tool message keeping only the mark its single cache point reads
|
||||
_MessageT,
|
||||
{
|
||||
**fields,
|
||||
"content": [
|
||||
block if index == kept_index else block_with_cache_control(block, dropped_cache_control)
|
||||
for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(content))
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
_BudgetSpend: TypeAlias = Callable[[tuple[int, int], int], tuple[int, int]]
|
||||
|
||||
|
||||
def _newest_marks_kept(state: tuple[int, int], marks: int) -> tuple[int, int]:
|
||||
remaining, _ = state
|
||||
kept: Final = min(marks, remaining)
|
||||
return remaining - kept, marks - kept
|
||||
|
||||
|
||||
def _whole_message_kept(state: tuple[int, int], marks: int) -> tuple[int, int]:
|
||||
remaining, _ = state
|
||||
if marks > remaining:
|
||||
return remaining, marks
|
||||
return remaining - marks, 0
|
||||
|
||||
|
||||
def _messages_within_budget(
|
||||
messages: Sequence[_MessageT], reserved_marks: int, spend: _BudgetSpend
|
||||
) -> Sequence[_MessageT]:
|
||||
budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0)
|
||||
newest_first: Final = tuple(reversed(messages))
|
||||
budget_after_each: Final = itertools.accumulate(
|
||||
(cache_control_count(message) for message in newest_first), spend, initial=(budget, 0)
|
||||
)
|
||||
dropped_counts: Final = tuple(dropped for _, dropped in itertools.islice(budget_after_each, 1, None))
|
||||
kept_newest_first: Final = tuple(
|
||||
_without_oldest_marks(message, dropped) for message, dropped in zip(newest_first, dropped_counts)
|
||||
)
|
||||
return tuple(reversed(kept_newest_first))
|
||||
|
||||
|
||||
def newest_marks_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]:
|
||||
return _messages_within_budget(messages, reserved_marks, _newest_marks_kept)
|
||||
|
||||
|
||||
def whole_messages_within_cache_control_limit(
|
||||
messages: Sequence[_MessageT], reserved_marks: int
|
||||
) -> Sequence[_MessageT]:
|
||||
return _messages_within_budget(messages, reserved_marks, _whole_message_kept)
|
||||
|
||||
|
||||
def _part_with_cache_control_renamed(part: object) -> object:
|
||||
if not _accepts_prompt_cache_breakpoint(part):
|
||||
return part
|
||||
|
|
@ -1439,3 +1625,25 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
llm_router=None,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def messages_within_cache_control_cap(
|
||||
messages: Sequence[_MessageT], tools: object, cache_control: object = None, request_kwargs: object = None
|
||||
) -> Sequence[_MessageT]:
|
||||
"""Under drop_params, drop the oldest message hints the provider's cache_control cap has no room for.
|
||||
|
||||
Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first.
|
||||
"""
|
||||
reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints(
|
||||
_validated_object_list(tools), cache_control, request_kwargs
|
||||
)
|
||||
kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks)
|
||||
dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept))
|
||||
if dropped > 0:
|
||||
verbose_logger.warning(
|
||||
"Dropped %d cache_control hint(s) from the oldest messages because drop_params is on and the "
|
||||
"provider accepts at most %d cache_control blocks per request",
|
||||
dropped,
|
||||
MAX_CACHE_CONTROL_BLOCKS,
|
||||
)
|
||||
return kept
|
||||
|
|
|
|||
|
|
@ -208,11 +208,14 @@ class StandardBuiltInToolCostTracking:
|
|||
usage=usage, response_object=response_object
|
||||
)
|
||||
|
||||
response_model: Final = _output_item_field(response_object, "model")
|
||||
if model_info is not None and resolved_usage is not None and resolved_provider is not None:
|
||||
result: Final = get_cost_for_web_search_request(
|
||||
custom_llm_provider=resolved_provider,
|
||||
usage=resolved_usage,
|
||||
model_info=model_info,
|
||||
response_model=response_model if isinstance(response_model, str) else None,
|
||||
lookup_model_info=StandardBuiltInToolCostTracking._safe_get_model_info,
|
||||
)
|
||||
if result is not None:
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -3,13 +3,15 @@ import functools
|
|||
import inspect
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any, Final, TypeAlias, cast
|
||||
|
||||
from litellm._logging import format_base64_size, verbose_logger
|
||||
from litellm.constants import (
|
||||
BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS,
|
||||
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS,
|
||||
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS,
|
||||
MAX_BASE64_LENGTH_FOR_LOGGING,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
|
||||
|
|
@ -44,6 +46,8 @@ Helper utils used for logging callbacks
|
|||
# Regex matching data-URI base64 content: "data:<mime>;base64,<payload>"
|
||||
# Captures: group(1)=mime_type, group(2)=base64_payload
|
||||
_DATA_URI_RE: Final = re.compile(r"data:([^;,\s]{1,255});base64,([A-Za-z0-9+/=]+)")
|
||||
_DATA_URI_PREFIX: Final = "data:"
|
||||
_DATA_URI_BASE64_MARKER: Final = ";base64,"
|
||||
|
||||
# Maximum nesting depth for _truncate_base64_in_value to guard against
|
||||
# pathological payloads. OpenAI message format is typically 3-4 levels deep.
|
||||
|
|
@ -60,15 +64,20 @@ def _base64_data_uri_replacer(match: re.Match[str]) -> str:
|
|||
return f"data:{mime_type};base64,[base64_data truncated: {size_str}]"
|
||||
|
||||
|
||||
def _may_contain_base64_data_uri(value: str) -> bool:
|
||||
return ";" in value and _DATA_URI_PREFIX in value and _DATA_URI_BASE64_MARKER in value
|
||||
|
||||
|
||||
def _truncate_base64_in_string(value: str) -> str:
|
||||
"""Replace long base64 data-URI payloads in a string with a size placeholder."""
|
||||
if MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
|
||||
if MAX_BASE64_LENGTH_FOR_LOGGING <= 0 or not _may_contain_base64_data_uri(value):
|
||||
return value
|
||||
return _DATA_URI_RE.sub(_base64_data_uri_replacer, value)
|
||||
|
||||
|
||||
def _truncate_base64_in_value(
|
||||
value: str | dict[str, object] | list[object] | None,
|
||||
truncate_string: Callable[[str], str],
|
||||
) -> str | dict[str, object] | list[object] | None:
|
||||
"""Iteratively truncate base64 data URIs in a JSON-like value (str/list/dict).
|
||||
|
||||
|
|
@ -78,13 +87,13 @@ def _truncate_base64_in_value(
|
|||
# Stack entries: (source_value, depth, parent_container, key_or_index)
|
||||
# We mutate *copies* of dicts/lists in-place via parent references.
|
||||
if isinstance(value, str):
|
||||
return _truncate_base64_in_string(value)
|
||||
return truncate_string(value)
|
||||
if not isinstance(value, (dict, list)):
|
||||
return value
|
||||
|
||||
# Shallow-copy the root so we don't mutate the caller's data.
|
||||
root: Final = {k: v for k, v in value.items()} if isinstance(value, dict) else list(value)
|
||||
stack: Final[list] = [(root, 0)]
|
||||
stack: Final[list[tuple[dict[str, object] | list[object], int]]] = [(root, 0)] # mutable-ok: explicit stack
|
||||
|
||||
while stack:
|
||||
container, depth = stack.pop()
|
||||
|
|
@ -93,19 +102,19 @@ def _truncate_base64_in_value(
|
|||
if isinstance(container, dict):
|
||||
for k, v in container.items():
|
||||
if isinstance(v, str):
|
||||
container[k] = _truncate_base64_in_string(v)
|
||||
container[k] = truncate_string(v)
|
||||
elif isinstance(v, dict):
|
||||
copy: dict | list = {ck: cv for ck, cv in v.items()}
|
||||
copy: dict[str, object] | list[object] = {ck: cv for ck, cv in v.items()}
|
||||
container[k] = copy
|
||||
stack.append((copy, depth + 1))
|
||||
elif isinstance(v, list):
|
||||
copy = list(v)
|
||||
container[k] = copy
|
||||
stack.append((copy, depth + 1))
|
||||
elif isinstance(container, list):
|
||||
else:
|
||||
for i, v in enumerate(container):
|
||||
if isinstance(v, str):
|
||||
container[i] = _truncate_base64_in_string(v)
|
||||
container[i] = truncate_string(v)
|
||||
elif isinstance(v, dict):
|
||||
copy = {ck: cv for ck, cv in v.items()}
|
||||
container[i] = copy
|
||||
|
|
@ -127,8 +136,15 @@ def truncate_base64_in_messages(
|
|||
"""
|
||||
if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
|
||||
return messages
|
||||
return _copy_messages(messages, _truncate_base64_in_string)
|
||||
|
||||
|
||||
def _copy_messages(
|
||||
messages: str | dict[str, object] | list[object], # mutable-ok: same contract as truncate_base64_in_messages
|
||||
truncate_string: Callable[[str], str],
|
||||
) -> str | dict[str, object] | list[object] | None: # mutable-ok: same contract as truncate_base64_in_messages
|
||||
try:
|
||||
return _truncate_base64_in_value(messages)
|
||||
return _truncate_base64_in_value(messages, truncate_string)
|
||||
except Exception as e:
|
||||
verbose_logger.debug("Failed to truncate base64 in messages: %s", e)
|
||||
return messages
|
||||
|
|
@ -151,19 +167,31 @@ def _iter_string_leaves(value: _StringTree) -> Iterator[str]:
|
|||
pass
|
||||
|
||||
|
||||
def _is_few_plain_strings(leaves: tuple[str, ...]) -> bool:
|
||||
return len(leaves) < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS and not any(
|
||||
_may_contain_base64_data_uri(leaf) for leaf in leaves
|
||||
)
|
||||
|
||||
|
||||
async def truncate_base64_in_messages_async(
|
||||
messages: str | list | dict | None, # mutable-ok: same contract as truncate_base64_in_messages
|
||||
) -> str | list | dict | None: # mutable-ok: same contract as truncate_base64_in_messages
|
||||
"""
|
||||
Same result as truncate_base64_in_messages, but payloads whose string content
|
||||
reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are scanned in a worker
|
||||
thread so the regex pass over multi-MB base64 images does not block the event loop.
|
||||
reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are copied in a worker thread,
|
||||
so neither the regex pass over multi-MB base64 images nor the copy of a long
|
||||
history blocks the event loop. A few strings with no data URI, under
|
||||
BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS in total, are copied inline, since
|
||||
checking them for data URIs and copying them costs less than the thread hop.
|
||||
"""
|
||||
if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
|
||||
return messages
|
||||
total_chars: Final = sum(len(leaf) for leaf in _iter_string_leaves(messages))
|
||||
leaves: Final = tuple(_iter_string_leaves(messages))
|
||||
total_chars: Final = sum(len(leaf) for leaf in leaves)
|
||||
if total_chars < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS:
|
||||
return truncate_base64_in_messages(messages)
|
||||
if total_chars < BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS and _is_few_plain_strings(leaves):
|
||||
return _copy_messages(messages, lambda text: text)
|
||||
return await asyncio.to_thread(truncate_base64_in_messages, messages)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -88,17 +88,22 @@ def handle_any_messages_to_chat_completion_str_messages_conversion(
|
|||
return [{"input": str(messages)}]
|
||||
|
||||
|
||||
def _message_with_str_content(message: AllMessageValues) -> AllMessageValues:
|
||||
texts: Final = convert_content_list_to_str(message=message)
|
||||
if not texts:
|
||||
return message
|
||||
return cast( # cast-ok: the same message with its content flattened to one string
|
||||
AllMessageValues, {**message, "content": texts}
|
||||
)
|
||||
|
||||
|
||||
def handle_messages_with_content_list_to_str_conversion(
|
||||
messages: list[AllMessageValues],
|
||||
) -> list[AllMessageValues]:
|
||||
"""
|
||||
Handles messages with content list conversion
|
||||
Returns new messages with each content list flattened to a string, leaving the caller's messages untouched
|
||||
"""
|
||||
for message in messages:
|
||||
texts = convert_content_list_to_str(message=message)
|
||||
if texts:
|
||||
message["content"] = texts
|
||||
return messages
|
||||
return [_message_with_str_content(message) for message in messages]
|
||||
|
||||
|
||||
def strip_name_from_message(message: AllMessageValues, allowed_name_roles: list[str] = ["user"]) -> AllMessageValues:
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ from litellm.types.utils import CallTypes
|
|||
from . import *
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import (
|
||||
BaseTranslation,
|
||||
)
|
||||
|
|
@ -30,7 +32,29 @@ def get_cost_for_google_maps_grounding_request(
|
|||
return cost_per_google_maps_grounding_request(usage=usage, model_info=model_info)
|
||||
|
||||
|
||||
def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", model_info: "ModelInfo") -> float | None:
|
||||
def _with_served_model_search_pricing(
|
||||
model_info: "ModelInfo",
|
||||
response_model: str | None,
|
||||
served_model_provider: str,
|
||||
lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None",
|
||||
) -> "ModelInfo":
|
||||
if model_info.get("search_context_cost_per_query") is not None or response_model is None:
|
||||
return model_info
|
||||
served_model_info: Final = (
|
||||
lookup_model_info(response_model, served_model_provider) if lookup_model_info is not None else None
|
||||
)
|
||||
if served_model_info is None:
|
||||
return model_info
|
||||
return {**model_info, "search_context_cost_per_query": served_model_info.get("search_context_cost_per_query")}
|
||||
|
||||
|
||||
def get_cost_for_web_search_request(
|
||||
custom_llm_provider: str,
|
||||
usage: "Usage",
|
||||
model_info: "ModelInfo",
|
||||
response_model: str | None = None,
|
||||
lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None" = None,
|
||||
) -> float | None:
|
||||
"""
|
||||
Get the cost for a web search request for a given model.
|
||||
|
||||
|
|
@ -38,6 +62,8 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
|
|||
custom_llm_provider: The custom LLM provider.
|
||||
usage: The usage object.
|
||||
model_info: The model info.
|
||||
response_model: The model the provider response names.
|
||||
lookup_model_info: Looks up a model's cost map entry, returning None when it is not mapped.
|
||||
"""
|
||||
if custom_llm_provider == "gemini":
|
||||
from .gemini.cost_calculator import cost_per_web_search_request
|
||||
|
|
@ -46,17 +72,25 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
|
|||
elif custom_llm_provider == "anthropic":
|
||||
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
|
||||
|
||||
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
|
||||
return get_cost_for_anthropic_web_search(
|
||||
model_info=_with_served_model_search_pricing(model_info, response_model, "anthropic", lookup_model_info),
|
||||
usage=usage,
|
||||
)
|
||||
elif custom_llm_provider.startswith("vertex_ai"):
|
||||
# Anthropic Claude models on Vertex AI populate server_tool_use.web_search_requests
|
||||
# (same as the direct Anthropic API), not prompt_tokens_details.web_search_requests
|
||||
# (which is the Gemini field). Route claude-* models to the Anthropic calculator.
|
||||
model_key: Final[str] = model_info.get("key", "") if model_info else ""
|
||||
if "claude" in model_key.lower():
|
||||
if "claude" in (response_model or model_key).lower():
|
||||
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
|
||||
|
||||
verbose_logger.debug("vertex_ai/claude model detected — routing web search cost to Anthropic calculator")
|
||||
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
|
||||
return get_cost_for_anthropic_web_search(
|
||||
model_info=_with_served_model_search_pricing(
|
||||
model_info, response_model, "vertex_ai", lookup_model_info
|
||||
),
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
from .vertex_ai.gemini.cost_calculator import (
|
||||
cost_per_web_search_request as cost_per_web_search_request_vertex_ai,
|
||||
|
|
@ -71,7 +105,10 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
|
|||
return None
|
||||
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
|
||||
|
||||
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
|
||||
return get_cost_for_anthropic_web_search(
|
||||
model_info=_with_served_model_search_pricing(model_info, response_model, "azure_ai", lookup_model_info),
|
||||
usage=usage,
|
||||
)
|
||||
elif custom_llm_provider == "perplexity":
|
||||
# Perplexity handles search costs internally in its own cost calculator
|
||||
# Return 0.0 to indicate costs are already accounted for
|
||||
|
|
|
|||
|
|
@ -167,6 +167,7 @@ _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool)
|
|||
_TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_TOOLS_ADAPTER: Final = TypeAdapter(list[object])
|
||||
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_REQUEST_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
|
||||
("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
|
||||
)
|
||||
|
|
@ -2254,6 +2255,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
headers=headers,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _messages_within_cache_control_cap(
|
||||
messages: Sequence[AllMessageValues], optional_params: Mapping[str, object]
|
||||
) -> Sequence[AllMessageValues]:
|
||||
from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap
|
||||
|
||||
return messages_within_cache_control_cap(
|
||||
messages, optional_params.get("tools"), optional_params.get("cache_control"), optional_params
|
||||
)
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -2271,9 +2282,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
anthropic_messages_pt,
|
||||
)
|
||||
|
||||
drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True
|
||||
drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params"))
|
||||
capped_messages: Final = (
|
||||
list(
|
||||
self._messages_within_cache_control_cap(
|
||||
messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params)
|
||||
)
|
||||
)
|
||||
if drop_params
|
||||
else messages
|
||||
)
|
||||
|
||||
if "tools" not in optional_params and messages is not None and has_tool_call_blocks(messages):
|
||||
if "tools" not in optional_params and has_tool_call_blocks(capped_messages):
|
||||
optional_params["tools"], _ = self._map_tools(add_dummy_tool(custom_llm_provider="anthropic"))
|
||||
|
||||
# Drop thinking param if thinking is enabled but thinking_blocks are missing
|
||||
|
|
@ -2285,9 +2305,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
# Related issue: https://github.com/BerriAI/litellm/issues/18926
|
||||
if (
|
||||
optional_params.get("thinking") is not None
|
||||
and messages is not None
|
||||
and last_assistant_with_tool_calls_has_no_thinking_blocks(messages)
|
||||
and not any_assistant_message_has_thinking_blocks(messages)
|
||||
and last_assistant_with_tool_calls_has_no_thinking_blocks(capped_messages)
|
||||
and not any_assistant_message_has_thinking_blocks(capped_messages)
|
||||
):
|
||||
if litellm.modify_params:
|
||||
optional_params.pop("thinking", None)
|
||||
|
|
@ -2332,15 +2351,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
_name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request(
|
||||
optional_params=optional_params,
|
||||
)
|
||||
if _name_forward_map:
|
||||
messages = self._rewrite_tool_names_in_messages(messages, _name_forward_map)
|
||||
request_messages: Final = (
|
||||
self._rewrite_tool_names_in_messages(capped_messages, _name_forward_map)
|
||||
if _name_forward_map
|
||||
else capped_messages
|
||||
)
|
||||
if _name_reverse_map and isinstance(litellm_params, dict):
|
||||
litellm_params[ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY] = _name_reverse_map
|
||||
|
||||
# Only the leading system run becomes the top-level system prompt. A later
|
||||
# system message stays in the conversation: hoisting it rewrites the cached
|
||||
# prefix and re-bills the whole history at cache-write pricing (#36559).
|
||||
leading_system_run, later_messages = split_leading_system_run(messages)
|
||||
leading_system_run, later_messages = split_leading_system_run(request_messages)
|
||||
anthropic_system_message_list: Final = self.translate_system_message(messages=list(leading_system_run))
|
||||
# Handling anthropic API Prompt Caching
|
||||
if len(anthropic_system_message_list) > 0:
|
||||
|
|
@ -2362,7 +2384,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
except Exception as e:
|
||||
raise AnthropicError(
|
||||
status_code=400,
|
||||
message=f"{e}\nReceived Messages={messages}",
|
||||
message=f"{e}\nReceived Messages={request_messages}",
|
||||
) # don't use verbose_logger.exception, if exception is raised
|
||||
|
||||
self.update_headers_with_optional_anthropic_beta(
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ This file contains common utils for anthropic calls.
|
|||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Container, Mapping, MutableMapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
|
|
@ -19,6 +20,8 @@ from typing_extensions import ReadOnly, TypedDict
|
|||
|
||||
import litellm
|
||||
from litellm.constants import (
|
||||
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH,
|
||||
ANTHROPIC_MAX_USER_ID_LENGTH,
|
||||
DEFAULT_MODEL_CREATED_AT_TIME,
|
||||
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
|
||||
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
|
||||
|
|
@ -178,6 +181,40 @@ def tools_without_custom_field(tools: Sequence[object], hoist_defer_loading: boo
|
|||
return [_tool_without_custom_field(tool, hoist_defer_loading, drop_custom) for tool in tools]
|
||||
|
||||
|
||||
def anthropic_pass_through_api_base() -> str:
|
||||
return os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
|
||||
|
||||
|
||||
def _same_host_and_port(url: httpx.URL, other: str) -> bool:
|
||||
try:
|
||||
other_url: Final = httpx.URL(other)
|
||||
except httpx.InvalidURL:
|
||||
return False
|
||||
return (url.host, url.port) == (other_url.host, other_url.port)
|
||||
|
||||
|
||||
def anthropic_upstream_user_id_limit(url: str) -> int | None:
|
||||
upstream: Final = httpx.URL(url)
|
||||
if not upstream.path.removesuffix("/").endswith("/v1/messages"):
|
||||
return None
|
||||
if upstream.host == "api.anthropic.com":
|
||||
return ANTHROPIC_MAX_USER_ID_LENGTH
|
||||
return (
|
||||
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH
|
||||
if _same_host_and_port(upstream, anthropic_pass_through_api_base())
|
||||
else None
|
||||
)
|
||||
|
||||
|
||||
def _has_email_shape(token: str) -> bool:
|
||||
at_sign: Final = token.find("@", 1)
|
||||
return at_sign != -1 and token.find(".", at_sign + 2, -1) != -1
|
||||
|
||||
|
||||
def anthropic_accepts_user_id(user_id: str, max_length: int) -> bool:
|
||||
return len(user_id) <= max_length and user_id.isprintable() and not any(map(_has_email_shape, user_id.split()))
|
||||
|
||||
|
||||
def is_anthropic_messages_url(url: str) -> bool:
|
||||
"""Check whether a URL addresses Anthropic's Messages API."""
|
||||
parsed_url: Final = urlparse(url)
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from pydantic_core import ErrorDetails
|
|||
from typing_extensions import assert_never
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
|
@ -297,13 +298,7 @@ def _systemone_question(question: DecisionsIRQuestion) -> Mapping[str, object]:
|
|||
assert_never(question)
|
||||
|
||||
|
||||
def ir_to_systemone_request(
|
||||
model: str, request: DecisionsIRRequest, *, supports_audio: bool = False, supports_video: bool = False
|
||||
) -> Mapping[str, object] | UnsupportedDecisionsRequest:
|
||||
if request.audio and not supports_audio:
|
||||
return UnsupportedDecisionsRequest(reason="audio input is not supported by this Decisions model")
|
||||
if request.videos and not supports_video:
|
||||
return UnsupportedDecisionsRequest(reason="video input is not supported by this Decisions model")
|
||||
def ir_to_systemone_request(model: str, request: DecisionsIRRequest) -> Mapping[str, object]:
|
||||
state: Final = _systemone_state(request.input)
|
||||
keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True)
|
||||
images: Final = input_images(request.input)
|
||||
|
|
@ -503,6 +498,16 @@ def ir_to_systemone_response(response: DecisionsIRResponse, request: DecisionsIR
|
|||
)
|
||||
|
||||
|
||||
def catalog_mode(model: str, custom_llm_provider: str) -> str | None:
|
||||
try:
|
||||
key: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)["key"]
|
||||
except Exception: # noqa: BLE001 # get_model_info raises bare Exception for an unmapped model, which the catalog does not list
|
||||
return None
|
||||
entry: Final = GetModelCostMap.loaded_model_cost_map().get(key)
|
||||
mode: Final = entry.get("mode") if entry is not None else None
|
||||
return mode if isinstance(mode, str) else None
|
||||
|
||||
|
||||
def _is_decisions_model(model: str, custom_llm_provider: str) -> bool:
|
||||
try:
|
||||
model_mode: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider).get("mode")
|
||||
|
|
@ -640,12 +645,16 @@ class BaseDecisionsConfig(ABC):
|
|||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
||||
def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool:
|
||||
def supports_audio_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
|
||||
if deployment_flag is not None:
|
||||
return deployment_flag
|
||||
return _is_decisions_model(model, custom_llm_provider) and litellm.supports_audio_input(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
||||
def supports_video_input(self, model: str, custom_llm_provider: str) -> bool:
|
||||
def supports_video_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
|
||||
if deployment_flag is not None:
|
||||
return deployment_flag
|
||||
return _is_decisions_model(model, custom_llm_provider) and litellm.supports_video_input(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
|
@ -656,12 +665,7 @@ class BaseDecisionsConfig(ABC):
|
|||
request: DecisionsIRRequest,
|
||||
custom_llm_provider: str,
|
||||
) -> Mapping[str, object] | UnsupportedDecisionsRequest:
|
||||
return ir_to_systemone_request(
|
||||
self.request_model(model),
|
||||
request,
|
||||
supports_audio=self.supports_audio_input(model, custom_llm_provider),
|
||||
supports_video=self.supports_video_input(model, custom_llm_provider),
|
||||
)
|
||||
return ir_to_systemone_request(self.request_model(model), request)
|
||||
|
||||
def unwrap_response(self, payload: object) -> object:
|
||||
return payload
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from litellm.constants import (
|
|||
)
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
as_str_mapping,
|
||||
drops_unsupported_params,
|
||||
filter_exceptions_from_params,
|
||||
filter_internal_params,
|
||||
map_finish_reason,
|
||||
|
|
@ -2129,6 +2130,24 @@ class AmazonConverseConfig(BaseConfig):
|
|||
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def _claude_messages_within_cache_control_cap(
|
||||
messages: Sequence[AllMessageValues],
|
||||
capability_model: str,
|
||||
optional_params: Mapping[str, object],
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> Sequence[AllMessageValues]:
|
||||
if not drops_unsupported_params(litellm_params.get("drop_params")) or "claude" not in capability_model.lower():
|
||||
return messages
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
messages_within_cache_control_cap,
|
||||
tool_message_with_single_cache_mark,
|
||||
)
|
||||
|
||||
return messages_within_cache_control_cap(
|
||||
tuple(map(tool_message_with_single_cache_mark, messages)), optional_params.get("tools")
|
||||
)
|
||||
|
||||
async def _async_transform_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -2140,7 +2159,13 @@ class AmazonConverseConfig(BaseConfig):
|
|||
base_model: Final = litellm_params.get("base_model")
|
||||
capability_model: Final = bedrock_capability_model(model, base_model)
|
||||
messages, system_content_blocks = self._transform_system_message(
|
||||
messages, model=capability_model, deployment_model=model
|
||||
list(
|
||||
self._claude_messages_within_cache_control_cap(
|
||||
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
|
||||
)
|
||||
),
|
||||
model=capability_model,
|
||||
deployment_model=model,
|
||||
)
|
||||
|
||||
# Convert last user message to guarded_text if guardrailConfig is present
|
||||
|
|
@ -2221,7 +2246,13 @@ class AmazonConverseConfig(BaseConfig):
|
|||
base_model: Final = litellm_params.get("base_model")
|
||||
capability_model: Final = bedrock_capability_model(model, base_model)
|
||||
messages, system_content_blocks = self._transform_system_message(
|
||||
messages, model=capability_model, deployment_model=model
|
||||
list(
|
||||
self._claude_messages_within_cache_control_cap(
|
||||
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
|
||||
)
|
||||
),
|
||||
model=capability_model,
|
||||
deployment_model=model,
|
||||
)
|
||||
|
||||
# Convert last user message to guarded_text if guardrailConfig is present
|
||||
|
|
|
|||
|
|
@ -110,6 +110,15 @@ def map_aiohttp_exceptions() -> Generator[None, None, None]:
|
|||
raise mapped_exc(message) from exc
|
||||
|
||||
|
||||
def _unread_payload(content: object) -> bytes:
|
||||
"""Bytes the upstream sent before the connection dropped that aiohttp buffered but never handed out, since its
|
||||
reads raise the stored payload error ahead of the buffer."""
|
||||
if not isinstance(content, aiohttp.StreamReader):
|
||||
return b""
|
||||
buffered: Final = b"".join(content._buffer) # pyright: ignore[reportPrivateUsage] # no public read drains it
|
||||
return buffered[content._buffer_offset :] # pyright: ignore[reportPrivateUsage] # see above
|
||||
|
||||
|
||||
def _release_transport_anchor(_transport: object) -> None:
|
||||
"""Release a transport anchored to a response stream. See ``_anchor_transport_to``."""
|
||||
|
||||
|
|
@ -144,6 +153,12 @@ class AiohttpResponseStream(httpx.AsyncByteStream):
|
|||
raise httpx.ReadError(str(e)) from e
|
||||
except aiohttp.http_exceptions.TransferEncodingError as e:
|
||||
raise httpx.ReadError(str(e)) from e
|
||||
except aiohttp.ClientPayloadError:
|
||||
unread: Final = _unread_payload(self._aiohttp_response.content)
|
||||
if unread:
|
||||
yield unread
|
||||
with map_aiohttp_exceptions():
|
||||
raise
|
||||
except Exception:
|
||||
# For other exceptions, use the normal mapping
|
||||
with map_aiohttp_exceptions():
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase):
|
|||
aembedding=None,
|
||||
custom_endpoint: bool | None = None,
|
||||
headers: dict | None = None,
|
||||
drop_params: bool = False,
|
||||
) -> EmbeddingResponse:
|
||||
# Check for custom user agent in optional_params or environment
|
||||
# This allows partners building on LiteLLM to set their own telemetry
|
||||
|
|
@ -58,4 +59,5 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase):
|
|||
aembedding=aembedding,
|
||||
custom_endpoint=True,
|
||||
headers=headers,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -130,7 +130,8 @@ class DeepInfraConfig(OpenAIGPTConfig):
|
|||
- Input: {"role": "tool", "content": [{"type": "text", "text": "result"}]}
|
||||
- Output: {"role": "tool", "content": "[{\"type\": \"text\", \"text\": \"result\"}]"}
|
||||
"""
|
||||
for message in messages:
|
||||
copied_messages: Final = [message.copy() for message in messages]
|
||||
for message in copied_messages:
|
||||
if message.get("role") == "tool":
|
||||
content = message.get("content")
|
||||
|
||||
|
|
@ -149,7 +150,7 @@ class DeepInfraConfig(OpenAIGPTConfig):
|
|||
# For complex content, serialize the entire array as JSON string
|
||||
message["content"] = json.dumps(content)
|
||||
|
||||
return messages
|
||||
return copied_messages
|
||||
|
||||
@overload
|
||||
def _transform_messages(
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from litellm.secret_managers.main import get_secret_str
|
|||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionToolParam,
|
||||
ChatCompletionToolParamFunctionChunk,
|
||||
OpenAIChatCompletionToolParam,
|
||||
)
|
||||
from litellm.types.utils import (
|
||||
|
|
@ -45,6 +46,7 @@ from ...openai.chat.gpt_transformation import (
|
|||
OpenAIChatCompletionStreamingHandler,
|
||||
OpenAIGPTConfig,
|
||||
)
|
||||
from ...openai.common_utils import without_cache_control
|
||||
from ..common_utils import (
|
||||
FIREROUTER,
|
||||
FireworksAIException,
|
||||
|
|
@ -480,15 +482,23 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
|
|||
return (("response_format", _json_schema_response_format(choice_schema, "choice")),)
|
||||
|
||||
def _transform_tools(self, tools: list[OpenAIChatCompletionToolParam]) -> list[OpenAIChatCompletionToolParam]:
|
||||
for tool in tools:
|
||||
if tool.get("type") != "function":
|
||||
continue
|
||||
function = tool["function"]
|
||||
function.pop("strict", None)
|
||||
params = function.get("parameters")
|
||||
if isinstance(params, dict):
|
||||
unpack_legacy_defs(params)
|
||||
return tools
|
||||
return [self._fireworks_tool(tool) for tool in tools]
|
||||
|
||||
@staticmethod
|
||||
def _fireworks_tool(tool: OpenAIChatCompletionToolParam) -> OpenAIChatCompletionToolParam:
|
||||
if tool.get("type") != "function":
|
||||
return tool
|
||||
function: Final = tool["function"]
|
||||
params: Final = function.get("parameters")
|
||||
unstrict_function: Final = cast( # cast-ok: the same function definition minus `strict`
|
||||
ChatCompletionToolParamFunctionChunk, {key: value for key, value in function.items() if key != "strict"}
|
||||
)
|
||||
new_function: Final[ChatCompletionToolParamFunctionChunk] = (
|
||||
{**unstrict_function, "parameters": unpack_legacy_defs(params, copy=True)}
|
||||
if isinstance(params, dict)
|
||||
else unstrict_function
|
||||
)
|
||||
return {**tool, "function": new_function}
|
||||
|
||||
def _transform_messages_helper(
|
||||
self, messages: list[AllMessageValues], model: str, litellm_params: dict
|
||||
|
|
@ -496,10 +506,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
|
|||
"""
|
||||
Strip fields not permitted by FireworksAI from messages.
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
filter_value_from_dict,
|
||||
)
|
||||
|
||||
supports_vision_value: Final = self._get_model_cost_capability_exact(model=model, capability="supports_vision")
|
||||
for message in messages:
|
||||
if message["role"] == "user":
|
||||
|
|
@ -530,9 +536,11 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
|
|||
model=model,
|
||||
llm_provider="fireworks_ai",
|
||||
)
|
||||
filter_value_from_dict(cast(dict, message), "cache_control")
|
||||
|
||||
return [strip_litellm_response_only_fields(message) for message in messages]
|
||||
return [
|
||||
strip_litellm_response_only_fields(cast(AllMessageValues, without_cache_control(message)))
|
||||
for message in messages
|
||||
]
|
||||
|
||||
# Cached index of fireworks_ai/* entries from litellm.model_cost. Building
|
||||
# this index requires a full scan of model_cost (tens of thousands of
|
||||
|
|
|
|||
|
|
@ -140,7 +140,8 @@ class GroqChatConfig(OpenAILikeChatConfig):
|
|||
def _transform_messages(
|
||||
self, messages: list[AllMessageValues], model: str, is_async: bool = False
|
||||
) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]:
|
||||
for idx, message in enumerate(messages):
|
||||
transformed_messages: Final = list(messages)
|
||||
for idx, message in enumerate(transformed_messages):
|
||||
"""
|
||||
1. Don't pass 'null' function_call assistant message to groq - https://github.com/BerriAI/litellm/issues/5839
|
||||
"""
|
||||
|
|
@ -154,12 +155,12 @@ class GroqChatConfig(OpenAILikeChatConfig):
|
|||
for k, v in _message.items():
|
||||
if v is not None:
|
||||
new_message[k] = v
|
||||
messages[idx] = strip_litellm_internal_message_fields(new_message)
|
||||
transformed_messages[idx] = strip_litellm_internal_message_fields(new_message)
|
||||
|
||||
if is_async:
|
||||
return super()._transform_messages(messages=messages, model=model, is_async=True)
|
||||
return super()._transform_messages(messages=transformed_messages, model=model, is_async=True)
|
||||
else:
|
||||
return super()._transform_messages(messages=messages, model=model, is_async=False)
|
||||
return super()._transform_messages(messages=transformed_messages, model=model, is_async=False)
|
||||
|
||||
def transform_messages(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
|
||||
"""
|
||||
|
||||
import copy
|
||||
import json
|
||||
from collections.abc import Coroutine
|
||||
from typing import Final, Literal, cast, overload
|
||||
|
|
@ -88,7 +89,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
|
|||
) -> dict:
|
||||
_tools = non_default_params.pop("tools", None)
|
||||
if _tools is not None:
|
||||
_tools = remove_additional_properties(_tools)
|
||||
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
|
||||
_tools = remove_strict_from_schema(_tools)
|
||||
if isinstance(_tools, list):
|
||||
_tools = self._convert_custom_tools_to_function_tools(_tools)
|
||||
|
|
|
|||
|
|
@ -755,7 +755,8 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
if _name is not None:
|
||||
# Remove name if not a tool message
|
||||
if message["role"] != "tool" or isinstance(_name, str) and len(_name.strip()) == 0:
|
||||
message.pop("name", None)
|
||||
message_without_name: Final = {k: v for k, v in message.items() if k != "name"}
|
||||
return cast(AllMessageValues, message_without_name) # cast-ok: the same message minus name
|
||||
|
||||
return message
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from pydantic import TypeAdapter, ValidationError
|
|||
|
||||
import litellm
|
||||
from litellm.constants import OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS
|
||||
from litellm.litellm_core_utils.core_helpers import map_finish_reason
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
extract_reasoning_content,
|
||||
handle_invalid_parallel_tool_calls,
|
||||
|
|
@ -64,7 +64,7 @@ from litellm.types.utils import (
|
|||
)
|
||||
from litellm.utils import convert_to_model_response_object, declared_mid_conversation_system_support
|
||||
|
||||
from ..common_utils import OpenAIError
|
||||
from ..common_utils import OpenAIError, without_cache_control
|
||||
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -128,6 +128,29 @@ def _litellm_params_str(litellm_params: Mapping[str, object] | None, key: str) -
|
|||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
def _drops_params(litellm_params: Mapping[str, object]) -> bool:
|
||||
return drops_unsupported_params(litellm_params.get("drop_params"))
|
||||
|
||||
|
||||
def _names_claude(model: str, litellm_params: Mapping[str, object]) -> bool:
|
||||
base_model: Final = litellm_params.get("base_model")
|
||||
return "claude" in model.lower() or (isinstance(base_model, str) and "claude" in base_model.lower())
|
||||
|
||||
|
||||
def _claude_messages_within_cache_control_cap(
|
||||
model: str,
|
||||
messages: Sequence[AllMessageValues],
|
||||
tools: Sequence["ChatCompletionToolParam"] | None,
|
||||
litellm_params: Mapping[str, object],
|
||||
request_params: Mapping[str, object],
|
||||
) -> Sequence[AllMessageValues]:
|
||||
if not _names_claude(model, litellm_params) or not _drops_params(litellm_params):
|
||||
return messages
|
||||
from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap
|
||||
|
||||
return list(messages_within_cache_control_cap(messages, tools, request_params.get("cache_control"), request_params))
|
||||
|
||||
|
||||
def _prompt_cache_options(optional_params: Mapping[str, object]) -> object | None:
|
||||
try:
|
||||
extra_body: Final = _OBJECT_MAPPING.validate_python(optional_params.get("extra_body"))
|
||||
|
|
@ -311,52 +334,58 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
return content_copy
|
||||
|
||||
async def _async_handle_pdf_url(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile:
|
||||
file_id: Final = content_item.get("file_id")
|
||||
content_copy: Final = content_item.copy()
|
||||
file_id: Final = content_copy.get("file_id")
|
||||
if file_id is not None: # check for file id being url done in _handle_pdf_url
|
||||
base64_data: Final = await async_convert_url_to_base64(file_id)
|
||||
content_item["file_data"] = base64_data
|
||||
content_item["filename"] = "my_file.pdf"
|
||||
content_item.pop("file_id")
|
||||
return content_item
|
||||
content_copy["file_data"] = base64_data
|
||||
content_copy["filename"] = "my_file.pdf"
|
||||
content_copy.pop("file_id")
|
||||
return content_copy
|
||||
|
||||
def _common_file_data_check(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile:
|
||||
file_data: Final = content_item.get("file_data")
|
||||
filename: Final = content_item.get("filename")
|
||||
if file_data is not None and filename is None:
|
||||
content_item["filename"] = "my_file.pdf"
|
||||
return {**content_item, "filename": "my_file.pdf"}
|
||||
return content_item
|
||||
|
||||
@staticmethod
|
||||
def _image_item_for_openai(content_item: ChatCompletionImageObject) -> ChatCompletionImageObject:
|
||||
image_url: Final = content_item["image_url"]
|
||||
if isinstance(image_url, str):
|
||||
return {**content_item, "image_url": {"url": image_url}}
|
||||
if not isinstance(image_url, dict):
|
||||
return content_item
|
||||
new_image_url_obj: Final = cast(
|
||||
ChatCompletionImageUrlObject,
|
||||
{k: v for k, v in image_url.items() if k not in {"format", "video_metadata"}},
|
||||
)
|
||||
return {**content_item, "image_url": new_image_url_obj}
|
||||
|
||||
@staticmethod
|
||||
def _file_item_for_openai(content_item: ChatCompletionFileObject) -> ChatCompletionFileObject:
|
||||
litellm_specific_params: Final = {"format", "detail", "video_metadata"}
|
||||
file_obj: Final = content_item.get("file")
|
||||
if file_obj is None:
|
||||
raise litellm.BadRequestError(
|
||||
message="Content block has type='file' but is missing the required 'file' field",
|
||||
model=None,
|
||||
llm_provider="openai",
|
||||
)
|
||||
new_file_obj: Final = ChatCompletionFileObjectFile(
|
||||
**{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
|
||||
)
|
||||
return {**content_item, "file": new_file_obj}
|
||||
|
||||
def _apply_common_transform_content_item(
|
||||
self,
|
||||
content_item: OpenAIMessageContentListBlock,
|
||||
) -> OpenAIMessageContentListBlock:
|
||||
litellm_specific_params: Final = {"format", "detail", "video_metadata"}
|
||||
if content_item.get("type") == "image_url":
|
||||
content_item = cast(ChatCompletionImageObject, content_item)
|
||||
if isinstance(content_item["image_url"], str):
|
||||
content_item["image_url"] = {
|
||||
"url": content_item["image_url"],
|
||||
}
|
||||
elif isinstance(content_item["image_url"], dict):
|
||||
new_image_url_obj: Final = cast(
|
||||
ChatCompletionImageUrlObject,
|
||||
{k: v for k, v in content_item["image_url"].items() if k not in {"format", "video_metadata"}},
|
||||
)
|
||||
content_item["image_url"] = new_image_url_obj
|
||||
elif content_item.get("type") == "file":
|
||||
content_item = cast(ChatCompletionFileObject, content_item)
|
||||
file_obj: Final = content_item.get("file")
|
||||
if file_obj is None:
|
||||
raise litellm.BadRequestError(
|
||||
message="Content block has type='file' but is missing the required 'file' field",
|
||||
model=None,
|
||||
llm_provider="openai",
|
||||
)
|
||||
new_file_obj: Final = ChatCompletionFileObjectFile(
|
||||
**{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
|
||||
)
|
||||
content_item["file"] = new_file_obj
|
||||
|
||||
return self._image_item_for_openai(cast(ChatCompletionImageObject, content_item))
|
||||
if content_item.get("type") == "file":
|
||||
return self._file_item_for_openai(cast(ChatCompletionFileObject, content_item))
|
||||
return content_item
|
||||
|
||||
def _transform_content_item(
|
||||
|
|
@ -419,31 +448,34 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
hoisted_messages: Final = hoist_images_from_tool_messages(stripped_messages)
|
||||
|
||||
async def _async_transform():
|
||||
for message in hoisted_messages:
|
||||
message_content = message.get("content")
|
||||
message_role = message.get("role")
|
||||
|
||||
if message_role == "user" and message_content and isinstance(message_content, list):
|
||||
message_content_types = cast(list[OpenAIMessageContentListBlock], message_content)
|
||||
for i, content_item in enumerate(message_content_types):
|
||||
message_content_types[i] = await self._async_transform_content_item(
|
||||
cast(OpenAIMessageContentListBlock, content_item),
|
||||
)
|
||||
return hoisted_messages
|
||||
return [await self._async_user_message_for_openai(message) for message in hoisted_messages]
|
||||
|
||||
if is_async:
|
||||
return _async_transform()
|
||||
else:
|
||||
for message in hoisted_messages:
|
||||
message_content = message.get("content")
|
||||
message_role = message.get("role")
|
||||
if message_role == "user" and message_content and isinstance(message_content, list):
|
||||
message_content_types = cast(list[OpenAIMessageContentListBlock], message_content)
|
||||
for i, content_item in enumerate(message_content):
|
||||
message_content_types[i] = self._transform_content_item(
|
||||
cast(OpenAIMessageContentListBlock, content_item)
|
||||
)
|
||||
return hoisted_messages
|
||||
return [self._user_message_for_openai(message) for message in hoisted_messages]
|
||||
|
||||
@staticmethod
|
||||
def _user_content_parts(message: AllMessageValues) -> Sequence[OpenAIMessageContentListBlock] | None:
|
||||
message_content: Final = message.get("content")
|
||||
if message.get("role") != "user" or not message_content or not isinstance(message_content, list):
|
||||
return None
|
||||
return cast(Sequence[OpenAIMessageContentListBlock], message_content)
|
||||
|
||||
def _user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues:
|
||||
parts: Final = self._user_content_parts(message)
|
||||
if parts is None:
|
||||
return message
|
||||
return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI
|
||||
AllMessageValues, {**message, "content": [self._transform_content_item(part) for part in parts]}
|
||||
)
|
||||
|
||||
async def _async_user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues:
|
||||
parts: Final = self._user_content_parts(message)
|
||||
if parts is None:
|
||||
return message
|
||||
return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI
|
||||
AllMessageValues, {**message, "content": [await self._async_transform_content_item(part) for part in parts]}
|
||||
)
|
||||
|
||||
def transform_messages(
|
||||
self,
|
||||
|
|
@ -462,23 +494,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
tools: list["ChatCompletionToolParam"] | None = None,
|
||||
) -> tuple[list[AllMessageValues], list["ChatCompletionToolParam"] | None]:
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
filter_value_from_dict,
|
||||
)
|
||||
from litellm.types.llms.openai import ChatCompletionToolParam
|
||||
|
||||
for i, message in enumerate(messages):
|
||||
messages[i] = cast(
|
||||
AllMessageValues,
|
||||
filter_value_from_dict(message, "cache_control"),
|
||||
)
|
||||
if tools is not None:
|
||||
for i, tool in enumerate(tools):
|
||||
tools[i] = cast(
|
||||
ChatCompletionToolParam,
|
||||
filter_value_from_dict(tool, "cache_control"),
|
||||
)
|
||||
return messages, tools
|
||||
new_messages: Final = [cast(AllMessageValues, without_cache_control(message)) for message in messages]
|
||||
new_tools: Final = (
|
||||
[cast(ChatCompletionToolParam, without_cache_control(tool)) for tool in tools]
|
||||
if tools is not None
|
||||
else None
|
||||
)
|
||||
return new_messages, new_tools
|
||||
|
||||
def translate_developer_role_to_system_role(
|
||||
self,
|
||||
|
|
@ -527,6 +551,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
tools: Sequence["ChatCompletionToolParam"] | None,
|
||||
prompt_cache_options: object,
|
||||
litellm_params: Mapping[str, object],
|
||||
request_params: Mapping[str, object],
|
||||
) -> tuple[Sequence[AllMessageValues], Sequence["ChatCompletionToolParam"] | None]:
|
||||
"""
|
||||
A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on
|
||||
|
|
@ -543,10 +568,11 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
model, custom_llm_provider, api_base, prompt_cache_options
|
||||
)
|
||||
if not reads_breakpoints and self._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base):
|
||||
return messages, tools
|
||||
drops_breakpoints: Final = not reads_breakpoints and (
|
||||
litellm.drop_params is True or litellm_params.get("drop_params") is True
|
||||
)
|
||||
return (
|
||||
_claude_messages_within_cache_control_cap(model, messages, tools, litellm_params, request_params),
|
||||
tools,
|
||||
)
|
||||
drops_breakpoints: Final = not reads_breakpoints and _drops_params(litellm_params)
|
||||
renamed: Final = (
|
||||
[AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages]
|
||||
if reads_breakpoints
|
||||
|
|
@ -561,7 +587,13 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
if drops_breakpoints
|
||||
else list(tools)
|
||||
)
|
||||
return self.remove_cache_control_flag_from_messages_and_tools(model=model, messages=renamed, tools=kept_tools)
|
||||
sent_messages, sent_tools = self.remove_cache_control_flag_from_messages_and_tools(
|
||||
model=model, messages=renamed, tools=kept_tools
|
||||
)
|
||||
return (
|
||||
_claude_messages_within_cache_control_cap(model, sent_messages, sent_tools, litellm_params, request_params),
|
||||
sent_tools,
|
||||
)
|
||||
|
||||
def _sanitized_tools_update_for_openai(
|
||||
self,
|
||||
|
|
@ -623,6 +655,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
tools=optional_params.get("tools", []),
|
||||
prompt_cache_options=_prompt_cache_options(optional_params),
|
||||
litellm_params=litellm_params,
|
||||
request_params=optional_params,
|
||||
)
|
||||
if tools is not None and len(tools) > 0:
|
||||
optional_params["tools"] = tools
|
||||
|
|
@ -654,6 +687,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
tools=optional_params.get("tools", []),
|
||||
prompt_cache_options=_prompt_cache_options(optional_params),
|
||||
litellm_params=litellm_params,
|
||||
request_params=optional_params,
|
||||
)
|
||||
if tools is not None and len(tools) > 0:
|
||||
optional_params["tools"] = tools
|
||||
|
|
|
|||
|
|
@ -153,12 +153,13 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig):
|
|||
- role: system ==> translate to role 'user'
|
||||
"""
|
||||
_supports_system_messages: Final = supports_system_messages(model, "openai")
|
||||
for i, message in enumerate(messages):
|
||||
transformed_messages: Final = list(messages)
|
||||
for i, message in enumerate(transformed_messages):
|
||||
if message["role"] == "system" and not _supports_system_messages:
|
||||
new_message = ChatCompletionUserMessage(content=message["content"], role="user")
|
||||
messages[i] = new_message # Replace the old message with the new one
|
||||
transformed_messages[i] = new_message # Replace the old message with the new one
|
||||
|
||||
if is_async:
|
||||
return super()._transform_messages(messages, model, is_async=cast(Literal[True], True))
|
||||
return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[True], True))
|
||||
else:
|
||||
return super()._transform_messages(messages, model, is_async=cast(Literal[False], False))
|
||||
return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[False], False))
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Common helpers / utils across al OpenAI endpoints
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import hashlib
|
||||
import inspect
|
||||
import json
|
||||
|
|
@ -31,7 +32,7 @@ if TYPE_CHECKING:
|
|||
import litellm
|
||||
from litellm.exceptions import error_response_without_credentials
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, iter_items
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params
|
||||
from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
|
@ -43,6 +44,47 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
)
|
||||
|
||||
|
||||
def _is_plain_dict(value: object) -> bool:
|
||||
return isinstance(value, dict)
|
||||
|
||||
|
||||
def _is_list(value: object) -> bool:
|
||||
return isinstance(value, list)
|
||||
|
||||
|
||||
def _contains_cache_control(value: object) -> bool:
|
||||
pending: Final[list[object]] = [value] # mutable-ok: a local worklist, never returned or stored
|
||||
while pending:
|
||||
node = pending.pop()
|
||||
mapping = as_str_mapping(node)
|
||||
if mapping is not None:
|
||||
if "cache_control" in mapping:
|
||||
return True
|
||||
pending.extend(mapping.values())
|
||||
elif _is_list(node):
|
||||
pending.extend(iter_items(node))
|
||||
return False
|
||||
|
||||
|
||||
def without_cache_control(item: object) -> object:
|
||||
"""Return `item` with every `cache_control` key removed, without touching the original.
|
||||
|
||||
`filter_value_from_dict` deletes the key in place and recurses into nested dicts and
|
||||
lists, so the caller's own message, tool or input object has to be copied first: it
|
||||
may reuse the same list on a provider that does support prompt caching. The copy is
|
||||
skipped when there is nothing to strip, which keeps the object identity the
|
||||
pass-through paths rely on and avoids a deep copy on every request.
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import filter_value_from_dict
|
||||
|
||||
if not _is_plain_dict(item) or not _contains_cache_control(item):
|
||||
return item
|
||||
copied: Final = copy.deepcopy(item)
|
||||
if isinstance(copied, dict):
|
||||
filter_value_from_dict(copied, "cache_control")
|
||||
return copied
|
||||
|
||||
|
||||
def _get_client_init_params(cls: type) -> tuple[str, ...]:
|
||||
"""Extract __init__ parameter names (excluding 'self') from a class."""
|
||||
return tuple(p for p in inspect.signature(cls.__init__).parameters if p != "self")
|
||||
|
|
|
|||
|
|
@ -354,10 +354,10 @@ class OpenAIDecisionsConfig(BaseDecisionsConfig):
|
|||
) -> str | None:
|
||||
return resolve_openai_bearer_token(api_key=api_key, api_base=api_base, litellm_params=litellm_params)
|
||||
|
||||
def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool:
|
||||
def supports_audio_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
|
||||
return False
|
||||
|
||||
def supports_video_input(self, model: str, custom_llm_provider: str) -> bool:
|
||||
def supports_video_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
|
||||
return False
|
||||
|
||||
def transform_decisions_request(
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from openai._base_client import make_request_options
|
|||
from openai._constants import RAW_RESPONSE_HEADER
|
||||
from openai._legacy_response import LegacyAPIResponse
|
||||
from openai._types import RequestOptions
|
||||
from openai.pagination import SyncCursorPage
|
||||
from openai.types import CreateEmbeddingResponse
|
||||
from openai.types.beta.assistant_deleted import AssistantDeleted
|
||||
from openai.types.chat import ChatCompletion, ChatCompletionChunk
|
||||
|
|
@ -40,6 +41,7 @@ from litellm.types.utils import (
|
|||
LiteLLMBatch,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
openai_batch_status,
|
||||
)
|
||||
from litellm.utils import (
|
||||
CustomStreamWrapper,
|
||||
|
|
@ -2182,6 +2184,18 @@ class OpenAIFilesAPI(BaseLLM):
|
|||
return response
|
||||
|
||||
|
||||
def _with_openai_batch_fields(batches: Sequence[Batch]) -> Sequence[Batch]:
|
||||
return [
|
||||
batch.model_copy(
|
||||
update={
|
||||
"object": batch.object or "batch",
|
||||
"status": openai_batch_status(batch.status),
|
||||
}
|
||||
)
|
||||
for batch in batches
|
||||
]
|
||||
|
||||
|
||||
class OpenAIBatchesAPI(BaseLLM):
|
||||
"""
|
||||
OpenAI methods to support for batches
|
||||
|
|
@ -2372,7 +2386,7 @@ class OpenAIBatchesAPI(BaseLLM):
|
|||
):
|
||||
verbose_logger.debug("listing batches, after= %s, limit= %s", after, limit)
|
||||
response: Final = await openai_client.batches.list(after=after, limit=limit)
|
||||
return response
|
||||
return response.model_copy(update={"data": _with_openai_batch_fields(response.data)})
|
||||
|
||||
def list_batches(
|
||||
self,
|
||||
|
|
@ -2384,7 +2398,7 @@ class OpenAIBatchesAPI(BaseLLM):
|
|||
organization: str | None,
|
||||
after: str | None = None,
|
||||
limit: int | None = None,
|
||||
client: OpenAI | None = None,
|
||||
client: OpenAI | AsyncOpenAI | None = None,
|
||||
litellm_params: Mapping[str, object] | None = None,
|
||||
):
|
||||
openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
|
||||
|
|
@ -2409,7 +2423,9 @@ class OpenAIBatchesAPI(BaseLLM):
|
|||
)
|
||||
return self.alist_batches(openai_client=openai_client, after=after, limit=limit)
|
||||
response: Final = openai_client.batches.list(after=after, limit=limit)
|
||||
return response
|
||||
if not isinstance(response, SyncCursorPage):
|
||||
return response
|
||||
return response.model_copy(update={"data": _with_openai_batch_fields(response.data)})
|
||||
|
||||
|
||||
class OpenAIAssistantsAPI(BaseLLM):
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ from litellm.types.router import GenericLiteLLMParams
|
|||
from litellm.types.utils import LlmProviders
|
||||
from litellm.types.workload_identity import OPENAI_WIF_KWARGS_KEYS
|
||||
|
||||
from ..common_utils import OpenAIError
|
||||
from ..common_utils import OpenAIError, without_cache_control
|
||||
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
|
||||
|
||||
OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16
|
||||
|
|
@ -658,13 +658,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
input=self._replayable_input(marker_safe_input, reads_breakpoints, maps_anthropic_tool_search),
|
||||
tools=mapped_tools,
|
||||
)
|
||||
if marker_safe_input is not validated_input:
|
||||
self._strip_caller_input_as_if_unmarked(
|
||||
model=model,
|
||||
caller_input=validated_input,
|
||||
reads_breakpoints=reads_breakpoints,
|
||||
maps_anthropic_tool_search=maps_anthropic_tool_search,
|
||||
)
|
||||
object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools)
|
||||
sanitized_tools: Final = self._sanitized_tool_schemas_for_openai(
|
||||
model=model, tools=object_schema_tools, litellm_params=litellm_params
|
||||
|
|
@ -679,17 +672,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
return marked_input
|
||||
return input_without_unanswered_anthropic_tool_search_calls(marked_input)
|
||||
|
||||
def _strip_caller_input_as_if_unmarked(
|
||||
self,
|
||||
model: str,
|
||||
caller_input: str | ResponseInputParam,
|
||||
reads_breakpoints: bool,
|
||||
maps_anthropic_tool_search: bool,
|
||||
) -> None:
|
||||
self.remove_cache_control_flag_from_input_and_tools(
|
||||
model=model, input=self._replayable_input(caller_input, reads_breakpoints, maps_anthropic_tool_search)
|
||||
)
|
||||
|
||||
def _input_with_wire_safe_prompt_cache_breakpoints(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -800,24 +782,29 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
the chat path. Strips Anthropic-only `cache_control` markers from
|
||||
Responses API input content blocks and tools.
|
||||
|
||||
`filter_value_from_dict` mutates each dict in place, so the same
|
||||
objects are returned.
|
||||
`filter_value_from_dict` deletes the key in place and recurses, so each
|
||||
item is copied first: the caller keeps its own input list and may reuse
|
||||
it on a provider that does support prompt caching.
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
filter_value_from_dict,
|
||||
new_input: Final = (
|
||||
cast( # cast-ok: the comprehension rebuilds the input item for item
|
||||
ResponseInputParam,
|
||||
[without_cache_control(item) for item in input],
|
||||
)
|
||||
if isinstance(input, list)
|
||||
else input
|
||||
)
|
||||
|
||||
if isinstance(input, list):
|
||||
for item in input:
|
||||
if isinstance(item, dict):
|
||||
filter_value_from_dict(cast(dict, item), "cache_control")
|
||||
new_tools: Final = (
|
||||
cast( # cast-ok: the comprehension rebuilds the tool list item for item
|
||||
Sequence[ALL_RESPONSES_API_TOOL_PARAMS],
|
||||
[without_cache_control(tool) for tool in tools],
|
||||
)
|
||||
if tools is not None
|
||||
else None
|
||||
)
|
||||
|
||||
if tools is not None:
|
||||
for tool in tools:
|
||||
if isinstance(tool, dict):
|
||||
filter_value_from_dict(cast(dict, tool), "cache_control")
|
||||
|
||||
return input, tools
|
||||
return new_input, new_tools
|
||||
|
||||
def _drop_bridge_minted_reasoning_items(self, input: str | ResponseInputParam) -> str | ResponseInputParam:
|
||||
if self.custom_llm_provider not in _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING or not isinstance(input, list):
|
||||
|
|
|
|||
|
|
@ -125,6 +125,20 @@ The provider inherits all request/response handling from OpenAI's Responses API
|
|||
|
||||
If `supported_endpoints` is omitted, it defaults to `[]` (only chat completions, which is always enabled for JSON providers).
|
||||
|
||||
When only some models serve an endpoint natively, list them under `model_supported_endpoints`. Every other model reaches `/v1/responses` and `/v1/messages` through chat completions:
|
||||
|
||||
```json
|
||||
{
|
||||
"your_provider": {
|
||||
"supported_endpoints": ["/v1/chat/completions"],
|
||||
"model_supported_endpoints": {
|
||||
"model-a": ["/v1/messages"],
|
||||
"model-b": ["/v1/responses"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### How It Works
|
||||
|
||||
1. `json_loader.py` checks `supported_endpoints` for `/v1/responses`
|
||||
|
|
|
|||
|
|
@ -3,11 +3,16 @@
|
|||
## Allows jina ai embedding calls - which don't allow 'encoding_format' in payload.
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
HTTPHandler,
|
||||
|
|
@ -17,6 +22,57 @@ from litellm.types.utils import EmbeddingResponse
|
|||
|
||||
from ..common_utils import OpenAILikeBase, OpenAILikeError
|
||||
|
||||
_KEY_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, str])
|
||||
_EXTRA_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_TRANSPORT_HEADER_NAMES: Final = frozenset(
|
||||
{
|
||||
"host",
|
||||
"content-length",
|
||||
"transfer-encoding",
|
||||
"connection",
|
||||
"keep-alive",
|
||||
"proxy-connection",
|
||||
"te",
|
||||
"trailer",
|
||||
"upgrade",
|
||||
"expect",
|
||||
}
|
||||
)
|
||||
_HEADER_NAME: Final = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
|
||||
_HEADER_VALUE: Final = re.compile(r"[\t\x20-\x7e]*")
|
||||
|
||||
|
||||
def _header_text(name: str, value: object) -> str | None:
|
||||
if isinstance(value, bool) or not isinstance(value, str | int | float):
|
||||
return None
|
||||
if _HEADER_NAME.fullmatch(name) is None:
|
||||
return None
|
||||
return str(value).strip(" \t\r\n")
|
||||
|
||||
|
||||
def _extra_headers_as_text(extra_headers: Mapping[str, object], model: str, drop_params: bool) -> Mapping[str, str]:
|
||||
caller_set: Final = {
|
||||
name: value for name, value in extra_headers.items() if name.lower() not in _TRANSPORT_HEADER_NAMES
|
||||
}
|
||||
texts: Final = MappingProxyType({name: _header_text(name, value) for name, value in caller_set.items()})
|
||||
refused: Final = sorted(name for name, text in texts.items() if text is None)
|
||||
if refused and not drop_params:
|
||||
raise litellm.UnsupportedParamsError(
|
||||
message=f"extra_headers {', '.join(map(repr, refused))} cannot be sent. A header needs a name made of "
|
||||
"letters, digits or !#$%&'*+-.^_`|~ and a text or number value. "
|
||||
"To drop them from the call, set `litellm.drop_params = True`.",
|
||||
model=model,
|
||||
)
|
||||
named: Final = MappingProxyType({name: text for name, text in texts.items() if text is not None})
|
||||
unsendable: Final = sorted(name for name, text in named.items() if _HEADER_VALUE.fullmatch(text) is None)
|
||||
if unsendable:
|
||||
verbose_logger.warning(
|
||||
"extra_headers %s dropped for model %r, since a header value must be printable ASCII on one line",
|
||||
unsendable,
|
||||
model,
|
||||
)
|
||||
return MappingProxyType({name: text for name, text in named.items() if name not in unsendable})
|
||||
|
||||
|
||||
class OpenAILikeEmbeddingHandler(OpenAILikeBase):
|
||||
def __init__(self, **kwargs):
|
||||
|
|
@ -94,6 +150,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
|
|||
aembedding=None,
|
||||
custom_endpoint: bool | None = None,
|
||||
headers: dict | None = None,
|
||||
drop_params: bool = False,
|
||||
) -> EmbeddingResponse:
|
||||
api_base, headers = self._validate_environment(
|
||||
api_base=api_base,
|
||||
|
|
@ -103,8 +160,20 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
|
|||
custom_endpoint=custom_endpoint,
|
||||
)
|
||||
model = model
|
||||
filtered_optional_params: Final = {k: v for k, v in optional_params.items() if v not in (None, "")}
|
||||
data: Final = {"model": model, "input": input, **filtered_optional_params}
|
||||
data: Final = {
|
||||
"model": model,
|
||||
"input": input,
|
||||
**{k: v for k, v in optional_params.items() if k != "extra_headers" and v not in (None, "")},
|
||||
}
|
||||
extra_headers: Final = _extra_headers_as_text(
|
||||
_EXTRA_HEADERS_ADAPTER.validate_python(optional_params.get("extra_headers") or {}), model, drop_params
|
||||
)
|
||||
key_headers: Final = _KEY_HEADERS_ADAPTER.validate_python(headers)
|
||||
extra_header_names: Final = frozenset(name.lower() for name in extra_headers)
|
||||
request_headers: Final = {
|
||||
**{name: value for name, value in key_headers.items() if name.lower() not in extra_header_names},
|
||||
**extra_headers,
|
||||
}
|
||||
|
||||
## LOGGING
|
||||
logging_obj.pre_call(
|
||||
|
|
@ -123,7 +192,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
|
|||
api_key=api_key,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
headers=headers,
|
||||
headers=request_headers,
|
||||
)
|
||||
if client is None or isinstance(client, AsyncHTTPHandler):
|
||||
self.client = HTTPHandler(timeout=timeout)
|
||||
|
|
@ -134,7 +203,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
|
|||
try:
|
||||
response: Final = self.client.post(
|
||||
api_base,
|
||||
headers=headers,
|
||||
headers=request_headers,
|
||||
data=json.dumps(data),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ JSON-based provider configuration loader for OpenAI-compatible providers.
|
|||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -14,6 +14,7 @@ from litellm.types.utils import ProviderSpecificModelInfo
|
|||
|
||||
_PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo)
|
||||
_SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(Mapping[str, bool])
|
||||
_MODEL_ENDPOINTS_ADAPTER: Final = TypeAdapter(Mapping[str, tuple[str, ...]])
|
||||
|
||||
|
||||
class SimpleProviderConfig:
|
||||
|
|
@ -30,7 +31,10 @@ class SimpleProviderConfig:
|
|||
self.special_handling: Mapping[str, bool] = _SPECIAL_HANDLING_ADAPTER.validate_python(
|
||||
data.get("special_handling", {})
|
||||
)
|
||||
self.supported_endpoints = data.get("supported_endpoints", [])
|
||||
self.supported_endpoints: Sequence[str] = data.get("supported_endpoints", [])
|
||||
self.model_supported_endpoints: Mapping[str, tuple[str, ...]] = _MODEL_ENDPOINTS_ADAPTER.validate_python(
|
||||
data.get("model_supported_endpoints", {})
|
||||
)
|
||||
self.provider_info: ProviderSpecificModelInfo = (
|
||||
_PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {}
|
||||
)
|
||||
|
|
@ -90,6 +94,23 @@ class JSONProviderRegistry:
|
|||
return False
|
||||
return "/v1/responses" in provider.supported_endpoints
|
||||
|
||||
@classmethod
|
||||
def supports_embeddings(cls, slug: str) -> bool:
|
||||
provider: Final = cls._providers.get(slug)
|
||||
if provider is None:
|
||||
return False
|
||||
return "/v1/embeddings" in provider.supported_endpoints
|
||||
|
||||
@classmethod
|
||||
def serves_endpoint(cls, slug: str, model: str | None, endpoint: str) -> bool:
|
||||
provider: Final = cls._providers.get(slug)
|
||||
if provider is None:
|
||||
return False
|
||||
model_endpoints: Final = (
|
||||
provider.model_supported_endpoints.get(model.removeprefix(f"{slug}/"), ()) if model else ()
|
||||
)
|
||||
return endpoint in provider.supported_endpoints or endpoint in model_endpoints
|
||||
|
||||
@classmethod
|
||||
def list_providers(cls) -> list:
|
||||
"""List all registered provider slugs"""
|
||||
|
|
|
|||
|
|
@ -211,7 +211,7 @@
|
|||
"param_mappings": {
|
||||
"max_completion_tokens": "max_tokens"
|
||||
},
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"]
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"]
|
||||
},
|
||||
"scx-ai": {
|
||||
"base_url": "https://api.scx.ai/v1",
|
||||
|
|
@ -231,6 +231,16 @@
|
|||
"api_base_env": "PRISM_API_BASE",
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
|
||||
},
|
||||
"hostinger_router": {
|
||||
"base_url": "https://router.hostinger.com/v1",
|
||||
"api_key_env": "HOSTINGER_ROUTER_API_KEY",
|
||||
"api_base_env": "HOSTINGER_ROUTER_API_BASE",
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/embeddings"],
|
||||
"model_supported_endpoints": {
|
||||
"claude-sonnet-5": ["/v1/messages"],
|
||||
"gpt-5.2": ["/v1/responses"]
|
||||
}
|
||||
},
|
||||
"sail": {
|
||||
"base_url": "https://api.sailresearch.com/v1",
|
||||
"api_key_env": "SAIL_API_KEY",
|
||||
|
|
|
|||
|
|
@ -110,38 +110,23 @@ def separate_cached_messages(
|
|||
messages: list[AllMessageValues],
|
||||
) -> tuple[list[AllMessageValues], list[AllMessageValues]]:
|
||||
"""
|
||||
Returns separated cached and non-cached messages.
|
||||
|
||||
Args:
|
||||
messages: List of messages to be separated.
|
||||
|
||||
Returns:
|
||||
Tuple containing:
|
||||
- cached_messages: List of cached messages.
|
||||
- non_cached_messages: List of non-cached messages.
|
||||
Returns the first continuous block of cached messages, plus every system message once
|
||||
any message is cached, since generateContent rejects a system_instruction beside a
|
||||
cachedContent, and the remaining messages.
|
||||
"""
|
||||
cached_messages: list[AllMessageValues] = []
|
||||
non_cached_messages: list[AllMessageValues] = []
|
||||
filtered_messages: Final = [(idx, message) for idx, message in enumerate(messages) if is_cached_message(message)]
|
||||
if not filtered_messages:
|
||||
return [], messages
|
||||
|
||||
# Extract cached messages and their indices
|
||||
filtered_messages: Final[list[tuple[int, AllMessageValues]]] = []
|
||||
for idx, message in enumerate(messages):
|
||||
if is_cached_message(message=message):
|
||||
filtered_messages.append((idx, message))
|
||||
|
||||
# Validate only one block of continuous cached messages
|
||||
last_continuous_block_idx: Final = get_first_continuous_block_idx(filtered_messages)
|
||||
# Separate messages based on the block of cached messages
|
||||
if filtered_messages and last_continuous_block_idx is not None:
|
||||
first_cached_idx: Final = filtered_messages[0][0]
|
||||
last_cached_idx: Final = filtered_messages[last_continuous_block_idx][0]
|
||||
|
||||
cached_messages = messages[first_cached_idx : last_cached_idx + 1]
|
||||
non_cached_messages = messages[:first_cached_idx] + messages[last_cached_idx + 1 :]
|
||||
else:
|
||||
non_cached_messages = messages
|
||||
|
||||
return cached_messages, non_cached_messages
|
||||
cached_block: Final = range(filtered_messages[0][0], filtered_messages[last_continuous_block_idx][0] + 1)
|
||||
goes_in_cache: Final = tuple(
|
||||
idx in cached_block or message.get("role") == "system" for idx, message in enumerate(messages)
|
||||
)
|
||||
return (
|
||||
[message for message, cached in zip(messages, goes_in_cache) if cached],
|
||||
[message for message, cached in zip(messages, goes_in_cache) if not cached],
|
||||
)
|
||||
|
||||
|
||||
def cached_messages_end_on_supported_turn(cached_messages: Sequence[AllMessageValues]) -> bool:
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ Translation from OpenAI's `/chat/completions` endpoint to IBM WatsonX's `/text/c
|
|||
Docs: https://cloud.ibm.com/apidocs/watsonx-ai#text-chat
|
||||
"""
|
||||
|
||||
import copy
|
||||
from typing import Final
|
||||
|
||||
from litellm import verbose_logger
|
||||
|
|
@ -57,7 +58,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig):
|
|||
_tools = non_default_params.pop("tools", None)
|
||||
if _tools is not None:
|
||||
# remove 'additionalProperties' from tools
|
||||
_tools = remove_additional_properties(_tools)
|
||||
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
|
||||
# remove 'strict' from tools
|
||||
_tools = remove_strict_from_schema(_tools)
|
||||
if _tools is not None:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
from collections.abc import AsyncIterator, Iterator, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final
|
||||
|
|
@ -208,7 +209,8 @@ class XAIChatConfig(OpenAIGPTConfig):
|
|||
tools = [
|
||||
filtered_tool
|
||||
for filtered_tool in (
|
||||
_FILTERED_TOOL.validate_python(filter_value_from_dict(tool, "strict")) for tool in value
|
||||
_FILTERED_TOOL.validate_python(filter_value_from_dict(copy.deepcopy(tool), "strict")) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
|
||||
for tool in value
|
||||
)
|
||||
if filtered_tool is not None
|
||||
]
|
||||
|
|
|
|||
|
|
@ -6848,6 +6848,23 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse:
|
|||
)
|
||||
|
||||
|
||||
def _embedding_headers(param: str, configured: object, model: str, drop_params: bool) -> Mapping[str, object]:
|
||||
if not configured:
|
||||
return {}
|
||||
if isinstance(configured, Mapping):
|
||||
return dict(cast(Mapping[str, object], configured)) # cast-ok: header names from untyped request JSON
|
||||
try:
|
||||
return dict(cast(Iterable[tuple[str, object]], configured)) # cast-ok: name and value pairs from request JSON
|
||||
except (TypeError, ValueError):
|
||||
if drop_params:
|
||||
return {}
|
||||
raise litellm.UnsupportedParamsError(
|
||||
message=f"{param} must be a JSON object of header names to values. "
|
||||
"To drop it from the call, set `litellm.drop_params = True`.",
|
||||
model=model,
|
||||
)
|
||||
|
||||
|
||||
# fmt: off
|
||||
|
||||
# Overload for when aembedding=True (returns coroutine)
|
||||
|
|
@ -6962,11 +6979,10 @@ def embedding(
|
|||
azure_ad_token_provider: Final = kwargs.get("azure_ad_token_provider", None)
|
||||
aembedding: Final[bool | None] = kwargs.get("aembedding", None)
|
||||
extra_headers: Final = kwargs.get("extra_headers", None)
|
||||
headers = kwargs.get("headers", None) or extra_headers
|
||||
if headers is None:
|
||||
headers = {}
|
||||
if extra_headers is not None:
|
||||
headers.update(extra_headers)
|
||||
headers = {
|
||||
**_embedding_headers("headers", kwargs.get("headers"), model, drop_unsupported_params),
|
||||
**_embedding_headers("extra_headers", extra_headers, model, drop_unsupported_params),
|
||||
}
|
||||
# Inject proxy auth headers if configured
|
||||
if litellm.proxy_auth is not None:
|
||||
try:
|
||||
|
|
@ -7130,7 +7146,7 @@ def embedding(
|
|||
# set API KEY
|
||||
api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
|
||||
|
||||
if headers is not None and headers != {}:
|
||||
if headers:
|
||||
optional_params["extra_headers"] = headers
|
||||
|
||||
requested_encoding_format: Final = (
|
||||
|
|
@ -7222,14 +7238,15 @@ def embedding(
|
|||
custom_llm_provider == "openai_like"
|
||||
or custom_llm_provider == "llamafile"
|
||||
or custom_llm_provider == "lm_studio"
|
||||
or JSONProviderRegistry.supports_embeddings(custom_llm_provider)
|
||||
):
|
||||
api_base = api_base or litellm.api_base or get_secret_str("OPENAI_LIKE_API_BASE")
|
||||
|
||||
# set API KEY
|
||||
if api_key is None:
|
||||
api_key = api_key or litellm.api_key or litellm.openai_like_key or get_secret_str("OPENAI_LIKE_API_KEY")
|
||||
if api_key is None and not JSONProviderRegistry.supports_embeddings(custom_llm_provider):
|
||||
api_key = litellm.api_key or litellm.openai_like_key or get_secret_str("OPENAI_LIKE_API_KEY")
|
||||
|
||||
if headers is not None and headers != {}:
|
||||
if headers:
|
||||
optional_params["extra_headers"] = headers
|
||||
|
||||
## EMBEDDING CALL
|
||||
|
|
@ -7244,10 +7261,9 @@ def embedding(
|
|||
optional_params=optional_params,
|
||||
client=client,
|
||||
aembedding=aembedding,
|
||||
drop_params=drop_unsupported_params,
|
||||
)
|
||||
elif custom_llm_provider == "oci":
|
||||
if headers is None:
|
||||
headers = {}
|
||||
response = base_llm_http_handler.embedding(
|
||||
model=model,
|
||||
input=input,
|
||||
|
|
@ -7272,11 +7288,6 @@ def embedding(
|
|||
or litellm.api_key
|
||||
)
|
||||
|
||||
# Use the merged headers variable (already merged at the top of the function)
|
||||
# Don't overwrite it with just extra_headers
|
||||
if headers is None:
|
||||
headers = {}
|
||||
|
||||
response = base_llm_http_handler.embedding(
|
||||
model=model,
|
||||
input=input,
|
||||
|
|
|
|||
|
|
@ -39687,6 +39687,57 @@
|
|||
"/v1/images/generations"
|
||||
]
|
||||
},
|
||||
"hostinger_router/gpt-5.2": {
|
||||
"cache_read_input_token_cost": 0.00000021,
|
||||
"input_cost_per_token": 0.0000021,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 272000,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 0.0000168,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hostinger_router/claude-sonnet-5": {
|
||||
"cache_creation_input_token_cost": 3.3e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 5.28e-06,
|
||||
"cache_read_input_token_cost": 2.64e-07,
|
||||
"input_cost_per_token": 2.64e-06,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.32e-05,
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_response_schema": true,
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/messages"],
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hostinger_router/text-embedding-3-large": {
|
||||
"input_cost_per_token": 1.716e-07,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 8191,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hostinger_router/text-embedding-3-small": {
|
||||
"input_cost_per_token": 3e-08,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 8191,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": {
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
|
|
@ -51387,6 +51438,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 5e-06,
|
||||
"output_cost_per_token_batches": 2.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
@ -51415,6 +51471,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 5e-06,
|
||||
"output_cost_per_token_batches": 2.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
@ -52161,6 +52222,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 1.5e-05,
|
||||
"output_cost_per_token_batches": 7.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
@ -52266,6 +52332,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 1.5e-05,
|
||||
"output_cost_per_token_batches": 7.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
|
|||
|
|
@ -1087,6 +1087,32 @@
|
|||
"generateContent": false
|
||||
}
|
||||
},
|
||||
"hostinger_router": {
|
||||
"display_name": "Hostinger Router (`hostinger_router`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/hostinger_router",
|
||||
"endpoints": {
|
||||
"chat_completions": true,
|
||||
"responses": true,
|
||||
"text_completion": true,
|
||||
"embeddings": true,
|
||||
"image_generations": false,
|
||||
"image_edits": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": false,
|
||||
"moderations": false,
|
||||
"realtime": false,
|
||||
"video_generations": false,
|
||||
"batches": false,
|
||||
"messages": true,
|
||||
"interactions": true,
|
||||
"systemone": false,
|
||||
"a2a": true,
|
||||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"crusoe": {
|
||||
"display_name": "Crusoe (`crusoe`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/crusoe",
|
||||
|
|
|
|||
|
|
@ -58015,7 +58015,7 @@
|
|||
"paths": {
|
||||
"/toolset/{toolset_name}/mcp": {
|
||||
"delete": {
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
|
||||
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_delete",
|
||||
"parameters": [
|
||||
{
|
||||
|
|
@ -58054,7 +58054,7 @@
|
|||
]
|
||||
},
|
||||
"get": {
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
|
||||
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_get",
|
||||
"parameters": [
|
||||
{
|
||||
|
|
@ -58093,7 +58093,7 @@
|
|||
]
|
||||
},
|
||||
"head": {
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
|
||||
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_head",
|
||||
"parameters": [
|
||||
{
|
||||
|
|
@ -58132,7 +58132,7 @@
|
|||
]
|
||||
},
|
||||
"options": {
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
|
||||
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_options",
|
||||
"parameters": [
|
||||
{
|
||||
|
|
@ -58171,7 +58171,7 @@
|
|||
]
|
||||
},
|
||||
"patch": {
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
|
||||
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_patch",
|
||||
"parameters": [
|
||||
{
|
||||
|
|
@ -58210,7 +58210,7 @@
|
|||
]
|
||||
},
|
||||
"post": {
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
|
||||
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_post",
|
||||
"parameters": [
|
||||
{
|
||||
|
|
@ -58249,7 +58249,7 @@
|
|||
]
|
||||
},
|
||||
"put": {
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
|
||||
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
|
||||
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put",
|
||||
"parameters": [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -5031,8 +5031,9 @@ class ProxyBaseLLMRequestProcessing:
|
|||
if stream_usage is None:
|
||||
return None
|
||||
service_tier: Final = obj.get("service_tier")
|
||||
served_model: Final = litellm_logging_obj.model if litellm_logging_obj is not None else None
|
||||
cost_val: Final = ProxyBaseLLMRequestProcessing._streamed_usage_cost(
|
||||
ModelResponse(usage=stream_usage),
|
||||
ModelResponse(model=served_model, usage=stream_usage),
|
||||
model_name,
|
||||
service_tier if isinstance(service_tier, str) else None,
|
||||
litellm_logging_obj,
|
||||
|
|
|
|||
|
|
@ -31,6 +31,13 @@ class AccountingConflictError(ValueError):
|
|||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class UsageWrite:
|
||||
event: InboundUsageEvent | OutboundUsageEvent
|
||||
envelope: str
|
||||
targets: tuple[DeliveryTarget, ...] = ()
|
||||
|
||||
|
||||
_Transaction: TypeAlias = AbstractAsyncContextManager["Prisma"]
|
||||
|
||||
|
||||
|
|
@ -107,6 +114,35 @@ SELECT event_id FROM "LiteLLM_AccountingEvent" WHERE kind = 'cost' AND source_ev
|
|||
_LOCK_EVENT: Final = """
|
||||
SELECT payload_hash FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 FOR UPDATE
|
||||
"""
|
||||
_INSERT_USAGE_BATCH: Final = """
|
||||
INSERT INTO "LiteLLM_AccountingEvent"
|
||||
(event_id, kind, schema_version, payload_hash, occurred_at, payload)
|
||||
SELECT event_id, kind::"AccountingEventKind", 1, payload_hash, occurred_at::timestamp, payload
|
||||
FROM jsonb_to_recordset($1::jsonb) AS records(
|
||||
event_id text, kind text, payload_hash text, occurred_at text, payload jsonb)
|
||||
ORDER BY event_id
|
||||
ON CONFLICT (event_id) DO NOTHING RETURNING event_id
|
||||
"""
|
||||
_LOCK_USAGE_BATCH: Final = """
|
||||
SELECT event_id, payload_hash FROM "LiteLLM_AccountingEvent"
|
||||
WHERE event_id IN (SELECT value FROM jsonb_array_elements_text($1::jsonb))
|
||||
ORDER BY event_id FOR UPDATE
|
||||
"""
|
||||
_LOAD_USAGE_BATCH: Final = """
|
||||
SELECT event.payload, ARRAY(
|
||||
SELECT target::text FROM "LiteLLM_AccountingDelivery" delivery
|
||||
WHERE delivery.event_id = event.event_id ORDER BY target::text
|
||||
) AS targets
|
||||
FROM "LiteLLM_AccountingEvent" event
|
||||
WHERE event.event_id IN (SELECT value FROM jsonb_array_elements_text($1::jsonb))
|
||||
AND event.kind = 'outbound_usage' AND event.schema_version = 1
|
||||
"""
|
||||
_INSERT_USAGE_DELIVERIES: Final = """
|
||||
INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version)
|
||||
SELECT records.event_id, destination.value::"AccountingDeliveryTarget", 1
|
||||
FROM jsonb_to_recordset($1::jsonb) AS records(event_id text, targets jsonb)
|
||||
CROSS JOIN LATERAL jsonb_array_elements_text(records.targets) AS destination
|
||||
"""
|
||||
_COMPLETED_FAMILY: Final = """
|
||||
root.source_event_id IS NULL AND root.kind IN ('inbound_usage', 'outbound_usage')
|
||||
AND root.recorded_at < $1::timestamp
|
||||
|
|
@ -199,6 +235,53 @@ class AccountingLedger:
|
|||
|
||||
return cls(transaction)
|
||||
|
||||
async def insert_usage_batch(self, writes: tuple[UsageWrite, ...]) -> None:
|
||||
records: Final = tuple(_prepare(write.event, write.targets, write.envelope, 1, 1) for write in writes)
|
||||
if not records:
|
||||
return
|
||||
unique: Final = {record.event_id: record for record in records}
|
||||
if any(record.payload_hash != unique[record.event_id].payload_hash for record in records):
|
||||
raise AccountingConflictError("Batch contains conflicting copies of a usage identity")
|
||||
batch: Final = _json(
|
||||
tuple(
|
||||
{
|
||||
"event_id": record.event_id,
|
||||
"kind": record.kind.value,
|
||||
"payload_hash": record.payload_hash,
|
||||
"occurred_at": record.occurred_at.astimezone(timezone.utc).isoformat(),
|
||||
"payload": _JSON.validate_json(record.payload),
|
||||
}
|
||||
for record in unique.values()
|
||||
)
|
||||
)
|
||||
async with self._transaction() as db:
|
||||
inserted: Final = _STRINGS.validate_python(await db.query_raw(_INSERT_USAGE_BATCH, batch))
|
||||
accepted: Final = _STRINGS.validate_python(await db.query_raw(_LOCK_USAGE_BATCH, _json(tuple(unique))))
|
||||
if len(accepted) != len(unique) or any(
|
||||
row["payload_hash"] != unique[row["event_id"]].payload_hash for row in accepted
|
||||
):
|
||||
raise AccountingConflictError("Usage identity conflicts with accepted content or delivery set")
|
||||
deliveries: Final = _json(
|
||||
tuple({"event_id": row["event_id"], "targets": unique[row["event_id"]].targets} for row in inserted)
|
||||
)
|
||||
await db.execute_raw(_INSERT_USAGE_DELIVERIES, deliveries)
|
||||
|
||||
async def load_usage_batch(self, event_ids: tuple[str, ...]) -> tuple[UsageWrite, ...]:
|
||||
if not event_ids:
|
||||
return ()
|
||||
async with self._transaction() as db:
|
||||
rows: Final = TypeAdapter(tuple[dict[str, JsonValue], ...]).validate_python(
|
||||
await db.query_raw(_LOAD_USAGE_BATCH, _json(event_ids))
|
||||
)
|
||||
return tuple(
|
||||
UsageWrite(
|
||||
OutboundUsageEvent.model_validate(_JSON.validate_python(row["payload"])["event"]),
|
||||
_json(row["payload"]),
|
||||
TypeAdapter(tuple[DeliveryTarget, ...]).validate_python(row["targets"]),
|
||||
)
|
||||
for row in rows
|
||||
)
|
||||
|
||||
async def insert_event(
|
||||
self,
|
||||
event: AccountingEvent,
|
||||
|
|
|
|||
|
|
@ -72,6 +72,22 @@ def _exception_types(*candidates: object) -> tuple[type[BaseException], ...]:
|
|||
return tuple(c for c in candidates if isinstance(c, type) and issubclass(c, BaseException))
|
||||
|
||||
|
||||
def _prisma_error_code(e: Exception) -> str | None:
|
||||
import prisma.engine.errors
|
||||
|
||||
if isinstance(e, _exception_types(prisma.errors.DataError)):
|
||||
code: Final[object] = getattr(e, "code", None)
|
||||
return code if isinstance(code, str) else None
|
||||
if not isinstance(e, _exception_types(prisma.engine.errors.EngineRequestError)):
|
||||
return None
|
||||
try:
|
||||
body: Final = _DATABASE_ERROR_META.validate_json(str(e).partition(": ")[2])
|
||||
except ValidationError:
|
||||
return None
|
||||
engine_code: Final = body.get("error_code")
|
||||
return engine_code if isinstance(engine_code, str) else None
|
||||
|
||||
|
||||
class PrismaDBExceptionHandler:
|
||||
"""
|
||||
Class to handle DB Exceptions or Connection Errors
|
||||
|
|
@ -206,6 +222,8 @@ class PrismaDBExceptionHandler:
|
|||
return True
|
||||
if PrismaDBExceptionHandler.is_database_capacity_error(e):
|
||||
return False
|
||||
if _prisma_error_code(e) == "P1017":
|
||||
return True
|
||||
if isinstance(e, _exception_types(prisma.errors.PrismaError)):
|
||||
error_message: Final = str(e).lower()
|
||||
connection_keywords: Final = (
|
||||
|
|
|
|||
|
|
@ -1,14 +1,15 @@
|
|||
# What is this?
|
||||
## Helper utils for the management endpoints (keys/users/teams)
|
||||
from collections.abc import Callable, Mapping, MutableMapping, Sequence
|
||||
from collections.abc import Awaitable, Callable, Mapping, MutableMapping, Sequence
|
||||
from collections.abc import Set as AbstractSet
|
||||
from datetime import datetime
|
||||
from functools import wraps
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final, Protocol
|
||||
from typing import Any, Final, ParamSpec, Protocol, TypeVar
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
import litellm
|
||||
from litellm._internal_context import with_service_target
|
||||
|
|
@ -687,6 +688,7 @@ async def _emit_management_endpoint_otel_span(
|
|||
end_time: datetime,
|
||||
result: Any = None,
|
||||
exception: Exception | None = None,
|
||||
log_payload: bool = True,
|
||||
) -> None:
|
||||
"""Stamp + end the parent OTEL SERVER span for a management endpoint.
|
||||
|
||||
|
|
@ -716,8 +718,8 @@ async def _emit_management_endpoint_otel_span(
|
|||
)
|
||||
|
||||
route = get_request_route(http_request)
|
||||
request_body: dict = await read_request_body( # rebind-ok: pre-existing rebinding on a rename-only line
|
||||
request=http_request
|
||||
request_body: dict = ( # rebind-ok: pre-existing rebinding on a rename-only line
|
||||
await read_request_body(request=http_request) if log_payload else {}
|
||||
)
|
||||
else:
|
||||
route = func.__name__
|
||||
|
|
@ -738,7 +740,7 @@ async def _emit_management_endpoint_otel_span(
|
|||
)
|
||||
|
||||
_response: dict[str, object] | None = None
|
||||
if exception is None and result is not None:
|
||||
if log_payload and exception is None and result is not None:
|
||||
try:
|
||||
raw: Final[Mapping[str, object]] = dict(result)
|
||||
_response = {k: v for k, v in raw.items() if k not in _CREDENTIAL_FIELDS}
|
||||
|
|
@ -767,37 +769,60 @@ async def _emit_management_endpoint_otel_span(
|
|||
)
|
||||
|
||||
|
||||
def management_endpoint_wrapper(func):
|
||||
_P: Final = ParamSpec("_P")
|
||||
_R: Final = TypeVar("_R")
|
||||
|
||||
|
||||
def management_endpoint_wrapper(func: Callable[_P, Awaitable[_R]]) -> Callable[_P, Awaitable[_R]]:
|
||||
"""
|
||||
This wrapper does the following:
|
||||
|
||||
1. Log I/O, Exceptions to OTEL
|
||||
2. Create an Audit log for success calls
|
||||
"""
|
||||
return _wrap_management_endpoint(func, log_payload=True)
|
||||
|
||||
@wraps(func)
|
||||
async def wrapper(*args, **kwargs):
|
||||
|
||||
def management_endpoint_span_wrapper(func: Callable[_P, Awaitable[_R]]) -> Callable[_P, Awaitable[_R]]:
|
||||
return _wrap_management_endpoint(func, log_payload=False)
|
||||
|
||||
|
||||
def _caller_auth(endpoint_kwargs: Mapping[str, object]) -> UserAPIKeyAuth:
|
||||
caller: Final = endpoint_kwargs.get("user_api_key_dict")
|
||||
return caller if isinstance(caller, UserAPIKeyAuth) else UserAPIKeyAuth()
|
||||
|
||||
|
||||
def _proxy_exception_handler_ends_server_span(exception: Exception) -> bool:
|
||||
return not isinstance(exception, StarletteHTTPException)
|
||||
|
||||
|
||||
def _wrap_management_endpoint(
|
||||
endpoint: Callable[_P, Awaitable[_R]], *, log_payload: bool
|
||||
) -> Callable[_P, Awaitable[_R]]:
|
||||
@wraps(endpoint)
|
||||
async def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R:
|
||||
start_time: Final = datetime.now()
|
||||
try:
|
||||
result: Final = await func(*args, **kwargs)
|
||||
result: Final = await endpoint(*args, **kwargs)
|
||||
end_time = datetime.now()
|
||||
try:
|
||||
user_api_key_dict: UserAPIKeyAuth = kwargs.get("user_api_key_dict") or UserAPIKeyAuth()
|
||||
user_api_key_dict: UserAPIKeyAuth = _caller_auth(kwargs)
|
||||
|
||||
await send_management_endpoint_alert(
|
||||
request_kwargs=kwargs,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
function_name=func.__name__,
|
||||
function_name=endpoint.__name__,
|
||||
)
|
||||
parent_otel_span: Span | None = getattr(user_api_key_dict, "parent_otel_span", None)
|
||||
if parent_otel_span is not None:
|
||||
await _emit_management_endpoint_otel_span(
|
||||
func=func,
|
||||
func=endpoint,
|
||||
kwargs=kwargs,
|
||||
parent_otel_span=parent_otel_span,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
result=result,
|
||||
log_payload=log_payload,
|
||||
)
|
||||
|
||||
# Delete updated/deleted info from cache
|
||||
|
|
@ -813,17 +838,18 @@ def management_endpoint_wrapper(func):
|
|||
except Exception as e:
|
||||
end_time = datetime.now()
|
||||
|
||||
user_api_key_dict: UserAPIKeyAuth = kwargs.get("user_api_key_dict") or UserAPIKeyAuth()
|
||||
user_api_key_dict: UserAPIKeyAuth = _caller_auth(kwargs)
|
||||
parent_otel_span = getattr(user_api_key_dict, "parent_otel_span", None)
|
||||
if parent_otel_span is not None:
|
||||
if parent_otel_span is not None and (log_payload or not _proxy_exception_handler_ends_server_span(e)):
|
||||
try:
|
||||
await _emit_management_endpoint_otel_span(
|
||||
func=func,
|
||||
func=endpoint,
|
||||
kwargs=kwargs,
|
||||
parent_otel_span=parent_otel_span,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
exception=e,
|
||||
log_payload=log_payload,
|
||||
)
|
||||
except Exception as otel_exc:
|
||||
# Non-Blocking Exception - never let OTEL failures swallow
|
||||
|
|
|
|||
|
|
@ -438,6 +438,14 @@ def is_mcp_admitted_user_subject(user_api_key_auth: UserAPIKeyAuth | None) -> bo
|
|||
_is_mcp_admitted_user_subject: Final = is_mcp_admitted_user_subject
|
||||
|
||||
|
||||
def has_team_unbound_mcp_admin_view(user_api_key_auth: UserAPIKeyAuth | None) -> bool:
|
||||
return (
|
||||
user_api_key_auth is not None
|
||||
and user_api_key_has_admin_view(user_api_key_auth)
|
||||
and (not user_api_key_auth.team_id or user_api_key_auth.team_id == UI_TEAM_ID)
|
||||
)
|
||||
|
||||
|
||||
def _gateway_dcr_challenge_target(
|
||||
route: str,
|
||||
mcp_servers: list[str] | None,
|
||||
|
|
@ -4290,19 +4298,19 @@ class MCPRequestHandler:
|
|||
"""Whether this principal's admin-view role grants the unscoped MCP resolution, whatever
|
||||
credential carries it (admin key, dashboard session, or OAuth-admitted session subject).
|
||||
|
||||
Two bounds disqualify, one per ownership of the row. A CREDENTIAL's explicit
|
||||
``object_permission.mcp_servers`` scope wins even for admins, including the empty list. An
|
||||
admitted subject's object_permission is the user's own row, whose ``mcp_servers`` column is
|
||||
[] by DB default, so for that shape the row binds through the entitlement ceiling instead
|
||||
(any non-empty entitlement, or an unresolved one, disqualifies), exactly as
|
||||
``operator_open_server_ids`` reads the same row. The one owner of this predicate: the
|
||||
server-axis registry resolution in ``get_allowed_mcp_servers`` and the tools-axis open
|
||||
channel in ``resolve_admitted_subject_tools`` both consult it, so the two axes cannot
|
||||
disagree."""
|
||||
A credential on a real team never qualifies, a virtual key or a ``lite login`` session alike: its team
|
||||
bounds its MCP access whoever owns it, as it bounds the credential's model access. Two more bounds disqualify, one per ownership of the row. A CREDENTIAL's
|
||||
explicit ``object_permission.mcp_servers`` scope wins even for admins, including the empty list. An
|
||||
admitted subject's object_permission is the user's own row, whose ``mcp_servers`` column is [] by DB
|
||||
default, so for that shape the row binds through the entitlement ceiling instead (any non-empty
|
||||
entitlement, or an unresolved one, disqualifies), exactly as ``operator_open_server_ids`` reads the same
|
||||
row. The one owner of this predicate: the server-axis registry resolution in ``get_allowed_mcp_servers``
|
||||
and the tools-axis open channel in ``resolve_admitted_subject_tools`` both consult it, so the two axes
|
||||
cannot disagree."""
|
||||
if (
|
||||
user_api_key_auth is None
|
||||
or user_api_key_auth.mcp_explicit_grants_only
|
||||
or not user_api_key_has_admin_view(user_api_key_auth)
|
||||
or not has_team_unbound_mcp_admin_view(user_api_key_auth)
|
||||
):
|
||||
return False
|
||||
object_permission: Final = user_api_key_auth.object_permission
|
||||
|
|
|
|||
|
|
@ -3705,7 +3705,7 @@ class MCPServerManager:
|
|||
|
||||
Priority:
|
||||
1. If object_permission.mcp_servers is explicitly set, use it (even for admins)
|
||||
2. If admin and no object_permission, return all servers
|
||||
2. If admin, not bound to a real team, and no object_permission, return all servers
|
||||
3. Otherwise, use standard permission checks
|
||||
"""
|
||||
if managed_agent_policy(user_api_key_auth) is not None:
|
||||
|
|
|
|||
|
|
@ -223,6 +223,7 @@ __all__ = (
|
|||
"mcp_get_prompt",
|
||||
"mcp_read_resource",
|
||||
"raise_denied_scoped_mcp_access",
|
||||
"raise_generic_mcp_access_denial",
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -763,8 +764,14 @@ async def raise_denied_scoped_mcp_access(
|
|||
)
|
||||
}
|
||||
raise HTTPException(status_code=403, detail=group_denial)
|
||||
raise_generic_mcp_access_denial(requested_names)
|
||||
|
||||
|
||||
def raise_generic_mcp_access_denial(requested_names: Sequence[str]) -> NoReturn:
|
||||
generic_denial: Final[_McpDeniedDetail] = {
|
||||
"error": f"The key is not allowed to access the requested MCP servers: {', '.join(requested_names)}"
|
||||
"error": (
|
||||
f"The key is not allowed to access the requested MCP servers: {', '.join(dict.fromkeys(requested_names))}"
|
||||
)
|
||||
}
|
||||
raise HTTPException(status_code=403, detail=generic_denial)
|
||||
|
||||
|
|
|
|||
|
|
@ -520,6 +520,7 @@ if MCP_AVAILABLE:
|
|||
"mcp_get_prompt",
|
||||
"mcp_read_resource",
|
||||
"raise_denied_scoped_mcp_access",
|
||||
"raise_generic_mcp_access_denial",
|
||||
"redact_mcp_resource_url",
|
||||
)
|
||||
from mcp.server import Server
|
||||
|
|
@ -1197,6 +1198,7 @@ if MCP_AVAILABLE:
|
|||
apply_display_name_overrides,
|
||||
filter_tools_by_allowed_tools,
|
||||
raise_denied_scoped_mcp_access,
|
||||
raise_generic_mcp_access_denial,
|
||||
)
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
|
|
@ -1824,10 +1826,10 @@ if MCP_AVAILABLE:
|
|||
row capping it.
|
||||
|
||||
Raises HTTPException(403) unless the principal holds toolset_id through one
|
||||
of its grant sources. Admins always pass.
|
||||
of its grant sources. Admins pass unless the key is bound to a real team.
|
||||
"""
|
||||
from litellm.proxy._types import LiteLLM_ObjectPermissionTable
|
||||
from litellm.proxy.management_endpoints.common_utils import user_api_key_has_admin_view
|
||||
from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import has_team_unbound_mcp_admin_view
|
||||
|
||||
# A key scoped to no MCP servers opts out of every MCP path. Enforce it
|
||||
# here too, since toolset scoping replaces mcp_servers and would otherwise
|
||||
|
|
@ -1841,7 +1843,7 @@ if MCP_AVAILABLE:
|
|||
)
|
||||
|
||||
acting: Final = await acting_user(user_api_key_auth)
|
||||
is_admin: Final = user_api_key_has_admin_view(acting)
|
||||
is_admin: Final = has_team_unbound_mcp_admin_view(acting)
|
||||
if not is_admin and toolset_id not in await granted(acting):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
|
|
|
|||
|
|
@ -52,7 +52,11 @@ from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
|
|||
from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming, is_guardrail_intervention
|
||||
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
|
||||
from litellm.litellm_core_utils.fallback_generalizations import match_routing_generalization
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
AnthropicModelInfo,
|
||||
anthropic_pass_through_api_base,
|
||||
merge_anthropic_beta_headers,
|
||||
)
|
||||
from litellm.llms.azure.passthrough.transformation import (
|
||||
foreign_azure_deployment,
|
||||
is_azure_body_model_inference_endpoint,
|
||||
|
|
@ -1047,7 +1051,7 @@ async def anthropic_proxy_route(
|
|||
"""
|
||||
[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)
|
||||
"""
|
||||
base_target_url = os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
|
||||
base_target_url = anthropic_pass_through_api_base()
|
||||
encoded_endpoint = httpx.URL(endpoint).path
|
||||
|
||||
# Ensure endpoint starts with '/' for proper URL construction
|
||||
|
|
|
|||
|
|
@ -71,7 +71,11 @@ from litellm.litellm_core_utils.litellm_logging import get_masked_values
|
|||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.llms.anthropic.common_utils import is_anthropic_messages_url
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
anthropic_accepts_user_id,
|
||||
anthropic_upstream_user_id_limit,
|
||||
is_anthropic_messages_url,
|
||||
)
|
||||
from litellm.llms.base_llm.managed_resources.utils import (
|
||||
resolve_passthrough_managed_id_provider,
|
||||
)
|
||||
|
|
@ -148,6 +152,8 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
|
|||
LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
|
||||
EndpointType,
|
||||
PassthroughStandardLoggingPayload,
|
||||
WebSocketSessionMeter,
|
||||
WebSocketSessionUsage,
|
||||
)
|
||||
from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, CallTypes, Usage
|
||||
|
||||
|
|
@ -501,7 +507,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
|
|||
url: httpx.URL,
|
||||
headers: dict,
|
||||
requested_query_params: dict | None = None,
|
||||
_parsed_body: dict | None = None,
|
||||
_parsed_body: Mapping[str, object] | None = None,
|
||||
forward_multipart: bool = False,
|
||||
) -> httpx.Response:
|
||||
"""
|
||||
|
|
@ -715,8 +721,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
|
|||
# unenforced. Set after the client merge so a request body cannot supply its own budget.
|
||||
#
|
||||
# Only for the built-in provider routes. `get_model_from_request` returns
|
||||
# None for a user-defined pass-through, deliberately: its body is forwarded
|
||||
# verbatim, so `model` there names an UPSTREAM model rather than a
|
||||
# None for a user-defined pass-through, deliberately: the client's `model`
|
||||
# goes upstream unchanged, so it names an UPSTREAM model rather than a
|
||||
# LiteLLM-managed one. Enforcement is therefore skipped on those routes, and
|
||||
# charging a counter anyway would track spend that nothing can refuse, and
|
||||
# would attribute it to a budget the operator scoped to a LiteLLM model that
|
||||
|
|
@ -1185,6 +1191,36 @@ def _unfiltered_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
|
|||
return headers
|
||||
|
||||
|
||||
def _json_object(value: object) -> Mapping[str, object]:
|
||||
return cast("Mapping[str, object]", value) if isinstance(value, Mapping) else MappingProxyType({}) # cast-ok: json
|
||||
|
||||
|
||||
def _anthropic_user_id_to_forward(
|
||||
body: Mapping[str, object],
|
||||
url: str,
|
||||
has_pre_call_guardrails: Callable[[Mapping[str, object]], bool],
|
||||
) -> str | None:
|
||||
user_id: Final = _json_object(body.get("metadata")).get("user_id")
|
||||
if not isinstance(user_id, str):
|
||||
return None
|
||||
max_length: Final = anthropic_upstream_user_id_limit(url)
|
||||
if max_length is None:
|
||||
return None
|
||||
if not anthropic_accepts_user_id(user_id, max_length):
|
||||
verbose_proxy_logger.warning("Not forwarding metadata.user_id because the upstream would refuse it")
|
||||
return None
|
||||
if has_pre_call_guardrails(body):
|
||||
verbose_proxy_logger.debug("Not forwarding metadata.user_id because a pre-call guardrail cannot scan it")
|
||||
return None
|
||||
return user_id
|
||||
|
||||
|
||||
def _with_anthropic_user_id(body: Mapping[str, object] | None, user_id: str | None) -> Mapping[str, object] | None:
|
||||
if body is None or user_id is None:
|
||||
return body
|
||||
return {**body, "metadata": {"user_id": user_id}}
|
||||
|
||||
|
||||
def _eu_jev_client_body(custom_llm_provider: str | None, body: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
return MappingProxyType(dict(body)) if custom_llm_provider == "eu_jev" else None
|
||||
|
||||
|
|
@ -1421,6 +1457,10 @@ async def pass_through_request(
|
|||
request_method=getattr(request, "method", None),
|
||||
cost_per_request=cost_per_request,
|
||||
)
|
||||
hooked_body: Final = _json_object(_parsed_body)
|
||||
client_user_id: Final = _anthropic_user_id_to_forward(
|
||||
hooked_body, str(url), proxy_logging_obj.request_has_pre_call_guardrails
|
||||
)
|
||||
kwargs = HttpPassThroughEndpointHelpers.init_kwargs_for_pass_through_endpoint( # rebind-ok: pre-existing rebinding on a rename-only line
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
_parsed_body=_parsed_body,
|
||||
|
|
@ -1588,11 +1628,13 @@ async def pass_through_request(
|
|||
else:
|
||||
logging_url = str(url) + "?" + requested_query_params_str
|
||||
|
||||
outgoing_body: Final = cast("Mapping[str, object] | None", _parsed_body) # cast-ok: json
|
||||
upstream_body: Final = _with_anthropic_user_id(outgoing_body, client_user_id)
|
||||
logging_obj.pre_call(
|
||||
input=[{"role": "user", "content": safe_dumps(_parsed_body)}],
|
||||
api_key="",
|
||||
additional_args={
|
||||
"complete_input_dict": _parsed_body,
|
||||
"complete_input_dict": upstream_body,
|
||||
"api_base": str(logging_url),
|
||||
"headers": redact_credential_headers(upstream_headers),
|
||||
},
|
||||
|
|
@ -1632,7 +1674,7 @@ async def pass_through_request(
|
|||
url,
|
||||
params=requested_query_params,
|
||||
headers=upstream_headers,
|
||||
json=_parsed_body,
|
||||
json=upstream_body,
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -1731,7 +1773,7 @@ async def pass_through_request(
|
|||
url=url,
|
||||
headers=upstream_headers,
|
||||
requested_query_params=requested_query_params,
|
||||
_parsed_body=_parsed_body,
|
||||
_parsed_body=upstream_body,
|
||||
forward_multipart=is_multipart,
|
||||
)
|
||||
verbose_proxy_logger.debug("response.headers= %s", response.headers)
|
||||
|
|
@ -2595,6 +2637,13 @@ def _with_trace_context(headers: Mapping[str, str], parent_span: object) -> dict
|
|||
return inject_trace_context(headers, parent_span=parent_span)
|
||||
|
||||
|
||||
def _log_session_usage(logging_obj: LiteLLMLoggingObj, usage: WebSocketSessionUsage) -> None:
|
||||
logging_obj.model = usage.model # rebind-ok: the spend logger reads model and cost off the shared logging object
|
||||
logging_obj.model_call_details["model"] = usage.model # rebind-ok: same shared logging object
|
||||
logging_obj.model_call_details["custom_llm_provider"] = usage.custom_llm_provider # rebind-ok: same
|
||||
logging_obj.model_call_details["response_cost"] = usage.response_cost # rebind-ok: same
|
||||
|
||||
|
||||
async def websocket_passthrough_request(
|
||||
websocket: WebSocket,
|
||||
target: str,
|
||||
|
|
@ -2607,6 +2656,7 @@ async def websocket_passthrough_request(
|
|||
setup_model_rewriter: Callable[[str], str] | None = None,
|
||||
builtin_provider_route: bool = False,
|
||||
model: str | None = None,
|
||||
meter: WebSocketSessionMeter | None = None,
|
||||
):
|
||||
"""
|
||||
WebSocket passthrough request handler.
|
||||
|
|
@ -2622,6 +2672,7 @@ async def websocket_passthrough_request(
|
|||
setup_model_rewriter: Optional rewrite of the setup frame's model before it reaches the upstream
|
||||
builtin_provider_route: Whether a built-in provider route serves the connection, so its URL names the provider
|
||||
model: The model the caller named before connecting, logged on a failure before the setup frame arrives
|
||||
meter: Prices the session from the client text frames delivered upstream, for an upstream that reports no usage
|
||||
"""
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.proxy.proxy_server import proxy_config, proxy_logging_obj
|
||||
|
|
@ -2632,6 +2683,7 @@ async def websocket_passthrough_request(
|
|||
# Initialize tracking variables
|
||||
start_time: Final = datetime.now()
|
||||
websocket_messages: Final[list[dict[str, object]]] = []
|
||||
metered_total = 0 # rebind-ok: forward_client_to_upstream adds the measure of each client text frame it delivers
|
||||
litellm_call_id: Final = str(uuid.uuid4())
|
||||
|
||||
verbose_proxy_logger.info("WebSocket passthrough (%s): Starting WebSocket connection to %s", endpoint, target)
|
||||
|
|
@ -2750,6 +2802,7 @@ async def websocket_passthrough_request(
|
|||
|
||||
async def forward_client_to_upstream() -> None:
|
||||
"""Forward messages from client to upstream WebSocket"""
|
||||
nonlocal metered_total
|
||||
try:
|
||||
while True:
|
||||
message = await websocket.receive()
|
||||
|
|
@ -2819,6 +2872,8 @@ async def websocket_passthrough_request(
|
|||
# Not a JSON message or doesn't contain setup data
|
||||
|
||||
await upstream_ws.send(_rewrite_vertex_live_setup_model(text_data, setup_model_rewriter))
|
||||
if meter is not None:
|
||||
metered_total += meter.measure(text_data)
|
||||
elif bytes_data is not None:
|
||||
await upstream_ws.send(bytes_data)
|
||||
except asyncio.CancelledError:
|
||||
|
|
@ -2945,6 +3000,8 @@ async def websocket_passthrough_request(
|
|||
self.url = target_url
|
||||
|
||||
mock_response: Final = MockWebSocketResponse(target)
|
||||
if meter is not None:
|
||||
_log_session_usage(logging_obj, meter.price(metered_total))
|
||||
|
||||
# Use the same success handler as HTTP passthrough endpoints
|
||||
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
|
||||
|
|
|
|||
|
|
@ -181,6 +181,7 @@ from litellm.proxy.common_utils.token_count_settings import (
|
|||
)
|
||||
from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog
|
||||
from litellm.proxy.management_helpers.model_group_custom_fields import model_group_custom_fields
|
||||
from litellm.proxy.management_helpers.utils import management_endpoint_span_wrapper
|
||||
from litellm.repositories import database as database_runtime
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
GATED_AUTO_ROUTER_CAPABILITIES,
|
||||
|
|
@ -12339,6 +12340,7 @@ async def _deployment_hidden_by_listing_callbacks(deployment: Deployment, user_a
|
|||
@router.get(
|
||||
"/models", dependencies=[Depends(user_api_key_auth)], tags=["model management"]
|
||||
) # if project requires model list
|
||||
@management_endpoint_span_wrapper
|
||||
async def model_list(
|
||||
request: Request = None, # pyright: ignore[reportArgumentType] # FastAPI always injects the Request; the None default only serves direct in-process callers
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
|
|
@ -12649,6 +12651,7 @@ async def model_list(
|
|||
dependencies=[Depends(user_api_key_auth)],
|
||||
tags=["model management"],
|
||||
)
|
||||
@management_endpoint_span_wrapper
|
||||
async def model_info(
|
||||
model_id: str,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
|
|
@ -21985,9 +21988,9 @@ async def toolset_mcp_route(toolset_name: str, request: Request):
|
|||
Namespace a toolset as its own MCP endpoint.
|
||||
|
||||
Connecting to /toolset/<name>/mcp exposes exactly the tools defined in
|
||||
the toolset. Access is enforced: non-admin API keys must have the toolset
|
||||
listed in their object_permission.mcp_toolsets grant list, or the request
|
||||
will be rejected with a 403.
|
||||
the toolset. Access is enforced: API keys other than an admin's key with no
|
||||
team must hold the toolset through one of their grant sources, or the
|
||||
request will be rejected with a 403.
|
||||
"""
|
||||
try:
|
||||
from litellm.proxy.mcp_server.mcp_server_manager import (
|
||||
|
|
|
|||
|
|
@ -3343,6 +3343,34 @@
|
|||
],
|
||||
"default_model_placeholder": "prism/deepseek-v4.1-flash"
|
||||
},
|
||||
{
|
||||
"provider": "HOSTINGER_ROUTER",
|
||||
"provider_display_name": "Hostinger Router",
|
||||
"litellm_provider": "hostinger_router",
|
||||
"credential_fields": [
|
||||
{
|
||||
"key": "api_base",
|
||||
"label": "API Base",
|
||||
"placeholder": "https://router.hostinger.com/v1",
|
||||
"tooltip": null,
|
||||
"required": false,
|
||||
"field_type": "text",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
},
|
||||
{
|
||||
"key": "api_key",
|
||||
"label": "API Key",
|
||||
"placeholder": null,
|
||||
"tooltip": null,
|
||||
"required": true,
|
||||
"field_type": "password",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
}
|
||||
],
|
||||
"default_model_placeholder": "hostinger_router/claude-sonnet-5"
|
||||
},
|
||||
{
|
||||
"provider": "RECRAFT",
|
||||
"provider_display_name": "Recraft",
|
||||
|
|
|
|||
268
litellm/proxy/spend_tracking/usage_meter.py
Normal file
268
litellm/proxy/spend_tracking/usage_meter.py
Normal file
|
|
@ -0,0 +1,268 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections import deque
|
||||
from collections.abc import Awaitable, Callable
|
||||
from concurrent.futures import Future
|
||||
from dataclasses import dataclass
|
||||
from itertools import accumulate, chain, count, takewhile
|
||||
from threading import Lock
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.accounting import (
|
||||
DeliveryTarget,
|
||||
Dimension,
|
||||
DimensionName,
|
||||
InboundUsageEvent,
|
||||
OutboundUsageEvent,
|
||||
)
|
||||
from litellm.proxy.db.accounting_ledger import AccountingConflictError, UsageWrite
|
||||
|
||||
_DAILY_TARGETS: Final = (
|
||||
(DimensionName.KEY, DeliveryTarget.DAILY_USER_SPEND),
|
||||
(DimensionName.USER, DeliveryTarget.DAILY_USER_SPEND),
|
||||
(DimensionName.TEAM, DeliveryTarget.DAILY_TEAM_SPEND),
|
||||
(DimensionName.ORGANIZATION, DeliveryTarget.DAILY_ORGANIZATION_SPEND),
|
||||
(DimensionName.END_USER, DeliveryTarget.DAILY_END_USER_SPEND),
|
||||
(DimensionName.AGENT, DeliveryTarget.DAILY_AGENT_SPEND),
|
||||
(DimensionName.TAG, DeliveryTarget.DAILY_TAG_SPEND),
|
||||
(DimensionName.MODEL, DeliveryTarget.DAILY_MODEL_USAGE),
|
||||
(DimensionName.TOOL, DeliveryTarget.DAILY_TOOL_SPEND),
|
||||
)
|
||||
|
||||
|
||||
def usage_delivery_targets(dimensions: tuple[Dimension, ...]) -> tuple[DeliveryTarget, ...]:
|
||||
names: Final = frozenset(dimension.name for dimension in dimensions)
|
||||
return tuple(dict.fromkeys(target for dimension, target in _DAILY_TARGETS if dimension in names))
|
||||
|
||||
|
||||
class UsageCaptureSettings(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
batch_size: int = Field(default=1000, ge=2, le=1000)
|
||||
flush_interval_seconds: float = Field(default=5.0, gt=0, le=10)
|
||||
max_pending_bytes: int = Field(default=16 * 1024 * 1024, ge=1024)
|
||||
max_pending_records: int = Field(default=10000, ge=2)
|
||||
|
||||
|
||||
class UsageCaptureError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Pending:
|
||||
writes: tuple[UsageWrite, ...]
|
||||
size: int
|
||||
committed: Future[None]
|
||||
finished: asyncio.Event
|
||||
prerequisites: tuple[asyncio.Future[None], ...]
|
||||
|
||||
|
||||
async def _wait_for_batch(wakeup: asyncio.Event, timeout: float) -> None:
|
||||
try:
|
||||
await asyncio.wait_for(wakeup.wait(), timeout)
|
||||
except asyncio.TimeoutError:
|
||||
return
|
||||
|
||||
|
||||
def _report_failure(receipt: Future[None]) -> None:
|
||||
if not receipt.cancelled() and receipt.exception() is not None:
|
||||
verbose_proxy_logger.error("Usage capture did not acknowledge commit", exc_info=receipt.exception())
|
||||
|
||||
|
||||
def _retryable_persistence_failure(error: Exception) -> bool:
|
||||
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain
|
||||
|
||||
outage: Final = PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(error)
|
||||
return (outage is not None and not PrismaDBExceptionHandler.is_permanent_database_fault(outage)) or any(
|
||||
isinstance(cause, Exception)
|
||||
and (
|
||||
PrismaDBExceptionHandler.is_deadlock_error(cause)
|
||||
or PrismaDBExceptionHandler.is_lock_timeout_error(cause)
|
||||
or PrismaDBExceptionHandler.postgres_sqlstate(cause) == "40001"
|
||||
)
|
||||
for cause in exception_chain(error)
|
||||
)
|
||||
|
||||
|
||||
def _isolatable_persistence_failure(error: Exception) -> bool:
|
||||
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain
|
||||
|
||||
return isinstance(error, AccountingConflictError) or any(
|
||||
isinstance(cause, Exception) and (PrismaDBExceptionHandler.postgres_sqlstate(cause) or "").startswith("22")
|
||||
for cause in exception_chain(error)
|
||||
)
|
||||
|
||||
|
||||
class DurableUsageMeter:
|
||||
def __init__(
|
||||
self,
|
||||
write_batch: Callable[[tuple[UsageWrite, ...]], Awaitable[None]],
|
||||
settings: UsageCaptureSettings,
|
||||
*,
|
||||
wait_for_batch: Callable[[asyncio.Event, float], Awaitable[None]] = _wait_for_batch,
|
||||
retry_delay: Callable[[float], Awaitable[None]] = asyncio.sleep,
|
||||
) -> None:
|
||||
self._write_batch: Final = write_batch
|
||||
self.settings: Final = settings
|
||||
self._wait_for_batch: Final = wait_for_batch
|
||||
self._retry_delay: Final = retry_delay
|
||||
self._loop: Final = asyncio.get_running_loop()
|
||||
self._admission: Final = Lock()
|
||||
self._queue: Final[deque[_Pending]] = deque() # mutable-ok: one bounded producer/consumer queue
|
||||
self._wakeup: Final = asyncio.Event()
|
||||
self._task: asyncio.Task[None] | None = None
|
||||
self._closed = False
|
||||
self.pending_bytes = 0
|
||||
self.pending_records = 0
|
||||
|
||||
def submit(
|
||||
self, writes: tuple[UsageWrite, ...], *, prerequisites: tuple[asyncio.Future[None], ...] = ()
|
||||
) -> asyncio.Future[None]:
|
||||
size: Final = sum(len(write.envelope.encode("utf-8")) for write in writes)
|
||||
with self._admission:
|
||||
if (
|
||||
self._closed
|
||||
or not writes
|
||||
or len(writes) > self.settings.batch_size
|
||||
or self.pending_bytes + size > self.settings.max_pending_bytes
|
||||
or self.pending_records + len(writes) > self.settings.max_pending_records
|
||||
):
|
||||
raise UsageCaptureError("Usage capture admission rejected: closed, empty or capacity exhausted")
|
||||
receipt: Final = Future[None]()
|
||||
receipt.set_running_or_notify_cancel()
|
||||
receipt.add_done_callback(_report_failure)
|
||||
self._queue.append(_Pending(writes, size, receipt, asyncio.Event(), prerequisites))
|
||||
self.pending_bytes += size
|
||||
self.pending_records += len(writes)
|
||||
self._loop.call_soon_threadsafe(self._ensure_worker)
|
||||
wrapped: Final = asyncio.wrap_future(receipt, loop=self._loop)
|
||||
wrapped.add_done_callback(lambda done: None if done.cancelled() else done.exception())
|
||||
return wrapped
|
||||
|
||||
def _ensure_worker(self) -> None:
|
||||
if self.pending_records >= self.settings.batch_size or self._queue and self._queue[0].prerequisites:
|
||||
self._wakeup.set()
|
||||
if self._task is None or self._task.done():
|
||||
self._task = asyncio.create_task(self._run(), name="accounting usage capture")
|
||||
|
||||
async def record(self, event: InboundUsageEvent | OutboundUsageEvent) -> None:
|
||||
envelope: Final = '{"event":' + event.model_dump_json() + "}"
|
||||
await asyncio.shield(self.submit((UsageWrite(event, envelope, usage_delivery_targets(event.dimensions)),)))
|
||||
|
||||
async def flush(self) -> None:
|
||||
self._ensure_worker()
|
||||
self._wakeup.set()
|
||||
if self._task is not None:
|
||||
await asyncio.shield(self._task)
|
||||
|
||||
async def close(self) -> None:
|
||||
with self._admission:
|
||||
self._closed = True
|
||||
await self.flush()
|
||||
|
||||
async def stop(self, timeout: float) -> None:
|
||||
with self._admission:
|
||||
self._closed = True
|
||||
try:
|
||||
await asyncio.wait_for(self.close(), timeout)
|
||||
except asyncio.TimeoutError:
|
||||
if self._task is not None:
|
||||
self._task.cancel()
|
||||
await asyncio.gather(self._task, return_exceptions=True)
|
||||
while self._queue:
|
||||
self._finish(
|
||||
self._take_batch(), UsageCaptureError("Usage capture stopped before commit acknowledgement")
|
||||
)
|
||||
verbose_proxy_logger.error("Usage capture shutdown deadline exhausted; commit acknowledgement incomplete")
|
||||
|
||||
def _take_batch(self) -> tuple[_Pending, ...]:
|
||||
with self._admission:
|
||||
if self._queue[0].prerequisites:
|
||||
return (self._queue.popleft(),)
|
||||
independent: Final = takewhile(lambda item: not item.prerequisites, self._queue)
|
||||
sizes: Final = accumulate(len(item.writes) for item in independent)
|
||||
count: Final = sum(1 for _ in takewhile(lambda size: size <= self.settings.batch_size, sizes))
|
||||
return tuple(self._queue.popleft() for _ in range(count))
|
||||
|
||||
async def _persist(self, writes: tuple[UsageWrite, ...]) -> None:
|
||||
for attempt in count():
|
||||
try:
|
||||
await self._write_batch(writes)
|
||||
except Exception as error: # noqa: BLE001 # retry only classified database failures, preserving frozen writes
|
||||
if not _retryable_persistence_failure(error):
|
||||
raise
|
||||
verbose_proxy_logger.warning("Usage persistence unavailable; retaining admitted batch for retry")
|
||||
await self._retry_delay(min(10.0, 0.2 * 2 ** min(attempt, 6)))
|
||||
else:
|
||||
return
|
||||
|
||||
async def _commit(self, batch: tuple[_Pending, ...]) -> None:
|
||||
writes: Final = tuple(chain.from_iterable(item.writes for item in batch))
|
||||
try:
|
||||
if batch[0].prerequisites:
|
||||
outcomes: Final = await asyncio.gather(
|
||||
*(asyncio.shield(receipt) for receipt in batch[0].prerequisites), return_exceptions=True
|
||||
)
|
||||
for outcome in outcomes:
|
||||
if isinstance(outcome, BaseException):
|
||||
raise UsageCaptureError("Usage dependency did not acknowledge commit") from outcome
|
||||
await self._persist(writes)
|
||||
except asyncio.CancelledError:
|
||||
self._finish(batch, UsageCaptureError("Usage commit acknowledgement interrupted"))
|
||||
raise
|
||||
except Exception as error: # noqa: BLE001 # isolate data faults, surface systemic failures to all producers
|
||||
if len(batch) > 1 and _isolatable_persistence_failure(error):
|
||||
try:
|
||||
for item in batch:
|
||||
await self._commit((item,))
|
||||
except asyncio.CancelledError:
|
||||
self._finish(batch, UsageCaptureError("Usage commit acknowledgement interrupted"))
|
||||
raise
|
||||
return
|
||||
self._finish(
|
||||
batch,
|
||||
UsageCaptureError("Usage identity conflicts with accepted content")
|
||||
if isinstance(error, AccountingConflictError)
|
||||
else error,
|
||||
)
|
||||
else:
|
||||
self._finish(batch, None)
|
||||
|
||||
def _finish(self, batch: tuple[_Pending, ...], error: Exception | None) -> None:
|
||||
for item in batch:
|
||||
if item.finished.is_set():
|
||||
continue
|
||||
item.finished.set()
|
||||
with self._admission:
|
||||
self.pending_bytes -= item.size
|
||||
self.pending_records -= len(item.writes)
|
||||
if item.committed.done():
|
||||
continue
|
||||
if error is None:
|
||||
item.committed.set_result(None)
|
||||
else:
|
||||
item.committed.set_exception(error)
|
||||
|
||||
async def _run(self) -> None:
|
||||
try:
|
||||
while self._queue:
|
||||
await self._wait_for_batch(self._wakeup, self.settings.flush_interval_seconds)
|
||||
self._wakeup.clear()
|
||||
await self._commit(self._take_batch())
|
||||
if (
|
||||
self._closed
|
||||
or self.pending_records >= self.settings.batch_size
|
||||
or self._queue
|
||||
and self._queue[0].prerequisites
|
||||
):
|
||||
self._wakeup.set()
|
||||
except (Exception, asyncio.CancelledError): # noqa: BLE001 # reject queued receipts if the writer exits abnormally
|
||||
while self._queue:
|
||||
self._finish(
|
||||
self._take_batch(), UsageCaptureError("Usage capture stopped before commit acknowledgement")
|
||||
)
|
||||
raise
|
||||
|
|
@ -3031,14 +3031,21 @@ class ProxyLogging:
|
|||
hook to consult, but it judges the payload the same way, so a proxy configured only with
|
||||
one of those still has something to say about every record.
|
||||
"""
|
||||
if request_metadata.get("_guardrail_pipelines"):
|
||||
return self.request_has_pre_call_guardrails({"metadata": dict(request_metadata)}, event_type)
|
||||
|
||||
def request_has_pre_call_guardrails(
|
||||
self,
|
||||
request_data: Mapping[str, object],
|
||||
event_type: GuardrailEventHooks = GuardrailEventHooks.pre_call,
|
||||
) -> bool:
|
||||
if _policy_state_metadata(request_data).get("_guardrail_pipelines"):
|
||||
return True
|
||||
caps: Final = ProxyLogging.callback_capabilities()
|
||||
if caps.has_content_enforcer:
|
||||
return True
|
||||
probe: Final = {"metadata": dict(request_metadata)}
|
||||
return any(
|
||||
isinstance(callback, CustomGuardrail) and callback.should_run_guardrail(data=probe, event_type=event_type)
|
||||
isinstance(callback, CustomGuardrail)
|
||||
and callback.should_run_guardrail(data=request_data, event_type=event_type)
|
||||
for callback in caps.resolved_callbacks
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -34,6 +34,14 @@ from litellm.types.utils import ModelResponse
|
|||
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
def _chat_verbosity(responses_api_request: Mapping[str, object]) -> Mapping[str, object]:
|
||||
match responses_api_request.get("text"):
|
||||
case {"verbosity": str() as verbosity}:
|
||||
return {"verbosity": verbosity}
|
||||
case _:
|
||||
return {}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _BridgedRequest:
|
||||
input: str | ResponseInputParam
|
||||
|
|
@ -96,8 +104,8 @@ class LiteLLMCompletionTransformationHandler:
|
|||
bridged: Final = _bridged_request(hoisted, responses_api_request, ())
|
||||
bridged_input: Final = bridged.input
|
||||
bridged_request: Final = bridged.responses_api_request
|
||||
litellm_completion_request: Final[dict] = (
|
||||
LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
|
||||
litellm_completion_request: Final[dict] = {
|
||||
**LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
|
||||
model=model,
|
||||
input=bridged_input,
|
||||
responses_api_request=bridged_request,
|
||||
|
|
@ -105,8 +113,9 @@ class LiteLLMCompletionTransformationHandler:
|
|||
stream=stream,
|
||||
extra_headers=extra_headers,
|
||||
**kwargs,
|
||||
)
|
||||
)
|
||||
),
|
||||
**_chat_verbosity(bridged_request),
|
||||
}
|
||||
|
||||
if _is_async:
|
||||
return self.async_response_api_handler(
|
||||
|
|
@ -173,8 +182,8 @@ class LiteLLMCompletionTransformationHandler:
|
|||
return await self.async_response_api_handler(
|
||||
previous_response_client_ids,
|
||||
stored_turns,
|
||||
litellm_completion_request=(
|
||||
LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
|
||||
litellm_completion_request={
|
||||
**LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
|
||||
model=model,
|
||||
input=bridged.input,
|
||||
responses_api_request=bridged.responses_api_request,
|
||||
|
|
@ -182,8 +191,9 @@ class LiteLLMCompletionTransformationHandler:
|
|||
stream=stream,
|
||||
extra_headers=extra_headers,
|
||||
**request_kwargs,
|
||||
)
|
||||
),
|
||||
),
|
||||
**_chat_verbosity(bridged.responses_api_request),
|
||||
},
|
||||
request_input=bridged.input,
|
||||
responses_api_request=bridged.responses_api_request,
|
||||
**request_kwargs,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import uuid
|
|||
from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from itertools import accumulate, chain, groupby, islice
|
||||
from itertools import chain, groupby
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
|
|
@ -45,7 +45,15 @@ from litellm.constants import (
|
|||
REDACTED_BY_LITELLM,
|
||||
REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER,
|
||||
)
|
||||
from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
AnthropicCacheControlHook,
|
||||
block_with_cache_control,
|
||||
cache_control_count,
|
||||
dropped_cache_control,
|
||||
newest_marks_within_cache_control_limit,
|
||||
whole_messages_within_cache_control_limit,
|
||||
with_cache_control,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, normalize_drop_params
|
||||
from litellm.litellm_core_utils.get_supported_openai_params import (
|
||||
get_supported_openai_params,
|
||||
|
|
@ -605,10 +613,6 @@ def _validated_cache_control(value: object) -> ChatCompletionCachedContent | Non
|
|||
return None
|
||||
|
||||
|
||||
def _dropped_cache_control(_: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _without_keys(item: object, keys: frozenset[str]) -> object:
|
||||
if not isinstance(item, dict):
|
||||
return item
|
||||
|
|
@ -707,83 +711,6 @@ def _content_blocks_carrying(content: object, cache_control: ChatCompletionCache
|
|||
return _with_cache_control_on_last_block(_OBJECT_LIST_ADAPTER.validate_python(content), cache_control)
|
||||
|
||||
|
||||
_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls")
|
||||
|
||||
|
||||
def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]:
|
||||
lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS)
|
||||
return tuple(
|
||||
chain.from_iterable(_OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list))
|
||||
)
|
||||
|
||||
|
||||
def _block_carries_cache_control(block: object) -> bool:
|
||||
return isinstance(block, dict) and _STR_KEY_DICT_ADAPTER.validate_python(block).get("cache_control") is not None
|
||||
|
||||
|
||||
def _cache_control_count(message: object) -> int:
|
||||
if not isinstance(message, dict):
|
||||
return 0
|
||||
fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message)
|
||||
block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block))
|
||||
return int(fields.get("cache_control") is not None) + block_marks
|
||||
|
||||
|
||||
_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None]
|
||||
|
||||
|
||||
def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]:
|
||||
others: Final = {key: value for key, value in fields.items() if key != "cache_control"}
|
||||
if "cache_control" not in fields:
|
||||
return others
|
||||
cache_control: Final = rewrite(fields["cache_control"])
|
||||
return others if cache_control is None else {**others, "cache_control": cache_control}
|
||||
|
||||
|
||||
def _block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object:
|
||||
if not isinstance(block, dict):
|
||||
return block
|
||||
return _fields_with_cache_control(_STR_KEY_DICT_ADAPTER.validate_python(block), rewrite)
|
||||
|
||||
|
||||
def _with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT:
|
||||
if not isinstance(message, dict):
|
||||
return message
|
||||
fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message)
|
||||
blocks_rewritten: Final = {
|
||||
key: (
|
||||
[_block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)]
|
||||
if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list)
|
||||
else value
|
||||
)
|
||||
for key, value in fields.items()
|
||||
}
|
||||
return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped
|
||||
_MessageT, _fields_with_cache_control(blocks_rewritten, rewrite)
|
||||
)
|
||||
|
||||
|
||||
def _kept_within_budget(state: tuple[int, bool], marks: int) -> tuple[int, bool]:
|
||||
remaining, _ = state
|
||||
if marks > remaining:
|
||||
return remaining, False
|
||||
return remaining - marks, True
|
||||
|
||||
|
||||
def _messages_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]:
|
||||
budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0)
|
||||
newest_first: Final = tuple(reversed(messages))
|
||||
budget_after_each: Final = accumulate(
|
||||
(_cache_control_count(message) for message in newest_first), _kept_within_budget, initial=(budget, True)
|
||||
)
|
||||
kept_flags: Final = tuple(kept for _, kept in islice(budget_after_each, 1, None))
|
||||
kept_newest_first: Final = tuple(
|
||||
message if kept else _with_cache_control(message, _dropped_cache_control)
|
||||
for message, kept in zip(newest_first, kept_flags)
|
||||
)
|
||||
return tuple(reversed(kept_newest_first))
|
||||
|
||||
|
||||
_SYSTEM_MESSAGE_FIELDS: Final = frozenset(("content", "cache_control"))
|
||||
|
||||
|
||||
|
|
@ -812,7 +739,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT:
|
|||
if fields is None:
|
||||
return message
|
||||
if fields.get("role") == "tool":
|
||||
return _with_cache_control(message, _dropped_cache_control)
|
||||
return with_cache_control(message, dropped_cache_control)
|
||||
tool_calls: Final = fields.get("tool_calls")
|
||||
if not isinstance(tool_calls, list):
|
||||
return message
|
||||
|
|
@ -821,7 +748,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT:
|
|||
{
|
||||
**fields,
|
||||
"tool_calls": [
|
||||
_block_with_cache_control(tool_call, _dropped_cache_control)
|
||||
block_with_cache_control(tool_call, dropped_cache_control)
|
||||
for tool_call in _OBJECT_LIST_ADAPTER.validate_python(tool_calls)
|
||||
],
|
||||
},
|
||||
|
|
@ -835,7 +762,7 @@ class _ItemMarkRules:
|
|||
external_marks: int
|
||||
|
||||
def sanitized(self, message: _MessageT) -> _MessageT:
|
||||
valid: Final = _with_cache_control(message, _validated_cache_control)
|
||||
valid: Final = with_cache_control(message, _validated_cache_control)
|
||||
return _without_tool_block_marks(valid) if self.refuses_tool_block_marks else valid
|
||||
|
||||
def checked(self, message: _MessageT) -> _MessageT:
|
||||
|
|
@ -858,15 +785,15 @@ def _request_messages_for_cache_points(messages: Sequence[_MessageT], rules: _It
|
|||
as_system: Final = tuple(_developer_message_as_system_message(message) for message in messages)
|
||||
if not rules.drop_params:
|
||||
return as_system
|
||||
return _messages_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks)
|
||||
return newest_marks_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks)
|
||||
|
||||
|
||||
def _replayed_messages_for_cache_points(
|
||||
replayed_messages: Sequence[_MessageT], current_messages: Sequence[object], rules: _ItemMarkRules
|
||||
) -> Sequence[_MessageT]:
|
||||
as_system: Final = tuple(_developer_message_as_system_message(message) for message in replayed_messages)
|
||||
current_marks: Final = sum(_cache_control_count(message) for message in current_messages)
|
||||
return _messages_within_cache_control_limit(
|
||||
current_marks: Final = sum(cache_control_count(message) for message in current_messages)
|
||||
return whole_messages_within_cache_control_limit(
|
||||
tuple(map(rules.sanitized, as_system)), current_marks + rules.external_marks
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from litellm.responses.mcp.tool_precheck import (
|
|||
LITELLM_PROXY_MCP_SERVER_URL_PREFIX,
|
||||
may_name_litellm_mcp_gateway,
|
||||
names_gateway_explicitly,
|
||||
names_whole_gateway,
|
||||
proxy_path_mcp_name,
|
||||
)
|
||||
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
|
||||
|
|
@ -182,8 +183,8 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
|
||||
@staticmethod
|
||||
async def _apply_toolset_permissions(
|
||||
resolved_toolset_ids: list[str],
|
||||
resolved_mcp_servers: list[str],
|
||||
resolved_toolset_ids: Sequence[str],
|
||||
resolved_mcp_servers: Sequence[str],
|
||||
user_api_key_auth: "UserAPIKeyAuth",
|
||||
) -> "UserAPIKeyAuth":
|
||||
"""Apply resolved toolset permissions to user_api_key_auth and return updated auth."""
|
||||
|
|
@ -195,7 +196,7 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
)
|
||||
|
||||
tool_permissions: Final = await global_mcp_server_manager.resolve_toolset_tool_permissions(
|
||||
toolset_ids=resolved_toolset_ids
|
||||
toolset_ids=list(resolved_toolset_ids)
|
||||
)
|
||||
all_server_ids: Final = list(set(tool_permissions.keys()) | set(resolved_mcp_servers))
|
||||
existing_op: Final = user_api_key_auth.object_permission
|
||||
|
|
@ -224,6 +225,62 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
verbose_logger.debug("Could not apply toolset permissions: %s", _e)
|
||||
return user_api_key_auth
|
||||
|
||||
@staticmethod
|
||||
async def _resolve_proxy_mcp_name(
|
||||
name: str,
|
||||
user_api_key_auth: "UserAPIKeyAuth | None",
|
||||
granted_toolsets: "GrantedToolsetIds | None",
|
||||
) -> tuple[Literal["server", "toolset", "denied_toolset"], str]:
|
||||
from litellm.proxy.mcp_server.mcp_server_manager import (
|
||||
global_mcp_server_manager,
|
||||
)
|
||||
|
||||
if global_mcp_server_manager.get_mcp_server_by_name(name):
|
||||
return ("server", name)
|
||||
try:
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
if prisma_client is None:
|
||||
return ("server", name)
|
||||
toolset: Final = await global_mcp_server_manager.get_toolset_by_name_cached(prisma_client, name)
|
||||
if toolset is None:
|
||||
return ("server", name)
|
||||
from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import (
|
||||
has_team_unbound_mcp_admin_view,
|
||||
)
|
||||
from litellm.proxy.mcp_server.ui_session_utils import (
|
||||
granted_toolset_ids,
|
||||
)
|
||||
|
||||
denied: Final = (
|
||||
user_api_key_auth is not None
|
||||
and not has_team_unbound_mcp_admin_view(user_api_key_auth)
|
||||
and toolset.toolset_id not in await (granted_toolsets or granted_toolset_ids)(user_api_key_auth)
|
||||
)
|
||||
except Exception as _e:
|
||||
verbose_logger.debug("Could not resolve '%s' as toolset: %s", name, _e)
|
||||
return ("server", name)
|
||||
if not denied:
|
||||
return ("toolset", toolset.toolset_id)
|
||||
if await LiteLLM_Proxy_MCP_Handler._holds_access_group_named(name, user_api_key_auth):
|
||||
return ("server", name)
|
||||
verbose_logger.debug("Key does not have access to toolset '%s', denying it.", name)
|
||||
return ("denied_toolset", name)
|
||||
|
||||
@staticmethod
|
||||
async def _holds_access_group_named(name: str, user_api_key_auth: "UserAPIKeyAuth | None") -> bool:
|
||||
from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import (
|
||||
MCPRequestHandler,
|
||||
)
|
||||
from litellm.proxy.mcp_server.mcp_server_manager import (
|
||||
global_mcp_server_manager,
|
||||
)
|
||||
|
||||
group_server_ids: Final = await MCPRequestHandler.get_mcp_servers_from_access_groups([name])
|
||||
return bool(group_server_ids) and not frozenset(group_server_ids).isdisjoint(
|
||||
await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@catalog_operation(global_manager)
|
||||
async def _get_mcp_tools_from_manager(
|
||||
|
|
@ -256,6 +313,7 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
from litellm.proxy.mcp_server.server import (
|
||||
_get_allowed_mcp_servers_from_mcp_server_names,
|
||||
_get_tools_from_mcp_servers,
|
||||
raise_generic_mcp_access_denial,
|
||||
)
|
||||
|
||||
mcp_servers: Final = [
|
||||
|
|
@ -268,36 +326,15 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
# Resolve toolset names: collect all toolset IDs first, then apply their
|
||||
# combined permissions in a single pass so multiple toolsets are unioned
|
||||
# rather than the last one overwriting the others.
|
||||
resolved_mcp_servers: Final[list[str]] = []
|
||||
resolved_toolset_ids: Final[list[str]] = []
|
||||
for name in mcp_servers:
|
||||
if not global_mcp_server_manager.get_mcp_server_by_name(name):
|
||||
try:
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
if prisma_client is not None:
|
||||
toolset = await global_mcp_server_manager.get_toolset_by_name_cached(prisma_client, name)
|
||||
if toolset is not None:
|
||||
if user_api_key_auth is not None:
|
||||
from litellm.proxy.management_endpoints.common_utils import (
|
||||
user_api_key_has_admin_view,
|
||||
)
|
||||
from litellm.proxy.mcp_server.ui_session_utils import (
|
||||
granted_toolset_ids,
|
||||
)
|
||||
|
||||
if not user_api_key_has_admin_view(user_api_key_auth) and toolset.toolset_id not in (
|
||||
await (granted_toolsets or granted_toolset_ids)(user_api_key_auth)
|
||||
):
|
||||
verbose_logger.debug("Key does not have access to toolset '%s', skipping.", name)
|
||||
continue
|
||||
resolved_toolset_ids.append(toolset.toolset_id)
|
||||
# Don't add to resolved_mcp_servers — toolset scope
|
||||
# restricts via object_permission, not server name filter.
|
||||
continue
|
||||
except Exception as _e:
|
||||
verbose_logger.debug("Could not resolve '%s' as toolset: %s", name, _e)
|
||||
resolved_mcp_servers.append(name)
|
||||
resolutions: Final = tuple(
|
||||
[
|
||||
await LiteLLM_Proxy_MCP_Handler._resolve_proxy_mcp_name(name, user_api_key_auth, granted_toolsets)
|
||||
for name in mcp_servers
|
||||
]
|
||||
)
|
||||
resolved_mcp_servers: Final = tuple(value for kind, value in resolutions if kind == "server")
|
||||
resolved_toolset_ids: Final = tuple(value for kind, value in resolutions if kind == "toolset")
|
||||
denied_toolset_names: Final = tuple(value for kind, value in resolutions if kind == "denied_toolset")
|
||||
|
||||
# Apply all resolved toolsets at once (union), avoiding permission overwrite.
|
||||
if resolved_toolset_ids and user_api_key_auth is not None:
|
||||
|
|
@ -311,7 +348,10 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
# full union (toolset server IDs + direct server names). Passing a name-based
|
||||
# filter here would exclude those toolset server IDs (which are UUIDs, not
|
||||
# names), so use None and let the auth object's mcp_servers do the filtering.
|
||||
effective_server_filter: Final = None if resolved_toolset_ids else (resolved_mcp_servers or None)
|
||||
asks_whole_gateway: Final = any(map(names_whole_gateway, mcp_tools_with_litellm_proxy or ()))
|
||||
if denied_toolset_names and not (asks_whole_gateway or resolved_mcp_servers or resolved_toolset_ids):
|
||||
raise_generic_mcp_access_denial(denied_toolset_names)
|
||||
effective_server_filter: Final = None if resolved_toolset_ids else (list(resolved_mcp_servers) or None)
|
||||
|
||||
listing: Final = await _get_tools_from_mcp_servers(
|
||||
user_api_key_auth=user_api_key_auth,
|
||||
|
|
|
|||
|
|
@ -22,6 +22,13 @@ def names_gateway_explicitly(tool: object) -> bool:
|
|||
return (_mcp_server_url(tool) or "").startswith(LITELLM_PROXY_MCP_SERVER_URL)
|
||||
|
||||
|
||||
def names_whole_gateway(tool: object) -> bool:
|
||||
server_url: Final = _mcp_server_url(tool) or ""
|
||||
return server_url.startswith(LITELLM_PROXY_MCP_SERVER_URL) and not server_url.startswith(
|
||||
LITELLM_PROXY_MCP_SERVER_URL_PREFIX
|
||||
)
|
||||
|
||||
|
||||
def proxy_path_mcp_name(tool: object) -> str | None:
|
||||
server_url: Final = _mcp_server_url(tool)
|
||||
match: Final = None if server_url is None else _PROXY_MCP_PATH_RE.match(server_url)
|
||||
|
|
|
|||
|
|
@ -637,6 +637,7 @@ _PRE_CALL_CHECK_FILTERED_PARAMS: Final = frozenset({"response_format"})
|
|||
_STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS: Final = frozenset({"anthropic_messages", "aresponses", "responses"})
|
||||
_video_call_deployment_id: Final[ContextVar[str | None]] = ContextVar("video_call_deployment_id", default=None)
|
||||
_routed_call_type: Final[ContextVar[str | None]] = ContextVar("routed_call_type", default=None)
|
||||
_DECISIONS_MEDIA_CALL_TYPES: Final = frozenset({"adecisions", "decisions", "asystemone", "systemone"})
|
||||
_ROUTE_CAPABILITY_RESOLVERS: Final[Mapping[str, Callable[[str, LlmProviders], object | None]]] = MappingProxyType(
|
||||
{
|
||||
"adecisions": ProviderConfigManager.get_provider_decisions_config,
|
||||
|
|
@ -4184,13 +4185,16 @@ class Router(DeprecatedRouterMembers):
|
|||
raise e.original_exception from e
|
||||
raise e
|
||||
|
||||
def _active_default_litellm_params(self) -> Mapping[str, object]:
|
||||
default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self._core_default_litellm_params)
|
||||
return {key: value for key, value in default_params.items() if value is not None}
|
||||
|
||||
def _params_sent_to_deployment(
|
||||
self, request_kwargs: Mapping[str, object], deployment: Deployment | None
|
||||
) -> Mapping[str, object]:
|
||||
"""The request as the deployment received it, merged in the order _update_kwargs_with_deployment applies."""
|
||||
deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {}
|
||||
default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self._core_default_litellm_params)
|
||||
defaults: Final = {key: value for key, value in default_params.items() if value is not None}
|
||||
defaults: Final = self._active_default_litellm_params()
|
||||
deployment_tool_choice: Final = (
|
||||
{"tool_choice": deployment_params["tool_choice"]}
|
||||
if deployment_params.get("tool_choice") is not None
|
||||
|
|
@ -14949,6 +14953,62 @@ class Router(DeprecatedRouterMembers):
|
|||
return healthy_deployments
|
||||
return cast(_RouteDeploymentsT, filtered_deployments) # cast-ok: same list shape as the input candidates
|
||||
|
||||
@staticmethod
|
||||
def _decisions_deployment_refuses_media(
|
||||
deployment: DeploymentTypedDict, group: str, request_kwargs: Mapping[str, object], *, audio: bool, videos: bool
|
||||
) -> bool:
|
||||
from litellm.decisions.call import deployment_refuses_media
|
||||
|
||||
plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts
|
||||
model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict
|
||||
deployment_params: Final = cast(Mapping[str, object], deployment["litellm_params"]) # cast-ok: plain dict
|
||||
try:
|
||||
model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group)
|
||||
return deployment_refuses_media(
|
||||
model,
|
||||
custom_llm_provider,
|
||||
model_info,
|
||||
{**deployment_params, **request_kwargs},
|
||||
audio=audio,
|
||||
videos=videos,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # best-effort filter: a deployment it cannot resolve answers for itself
|
||||
verbose_router_logger.debug(
|
||||
"litellm.router.py::_decisions_deployment_refuses_media: keeping a deployment of model=%s. Got - %s",
|
||||
group,
|
||||
e,
|
||||
)
|
||||
return False
|
||||
|
||||
def _without_decisions_deployments_refusing_media(
|
||||
self, healthy_deployments: _RouteDeploymentsT, group: str, request_kwargs: Mapping[str, object] | None
|
||||
) -> _RouteDeploymentsT:
|
||||
call_kwargs: Final = request_kwargs or {}
|
||||
audio: Final = bool(call_kwargs.get("audio"))
|
||||
videos: Final = bool(call_kwargs.get("videos"))
|
||||
if (
|
||||
_routed_call_type.get() not in _DECISIONS_MEDIA_CALL_TYPES
|
||||
or not (audio or videos)
|
||||
or isinstance(healthy_deployments, Mapping)
|
||||
):
|
||||
return healthy_deployments
|
||||
|
||||
sent_kwargs: Final = {**self._active_default_litellm_params(), **call_kwargs}
|
||||
deployments_taking_media: Final = [
|
||||
deployment
|
||||
for deployment in healthy_deployments
|
||||
if not Router._decisions_deployment_refuses_media(
|
||||
cast(DeploymentTypedDict, deployment), # cast-ok: router candidates use the deployment shape
|
||||
group,
|
||||
sent_kwargs,
|
||||
audio=audio,
|
||||
videos=videos,
|
||||
)
|
||||
]
|
||||
if not deployments_taking_media:
|
||||
return healthy_deployments
|
||||
return cast(_RouteDeploymentsT, deployments_taking_media) # cast-ok: same list shape as the input candidates
|
||||
|
||||
@staticmethod
|
||||
def _without_declared_text_only_deployments(
|
||||
healthy_deployments: _RouteDeploymentsT,
|
||||
|
|
@ -15673,10 +15733,14 @@ class Router(DeprecatedRouterMembers):
|
|||
|
||||
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
|
||||
_target_order: Final = (request_kwargs or {}).pop("_target_order", None)
|
||||
batch_candidates: Final = self._without_declared_text_only_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
batch_candidates: Final = self._without_decisions_deployments_refusing_media(
|
||||
self._without_declared_text_only_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
model,
|
||||
messages,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
|
||||
),
|
||||
model,
|
||||
messages,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
|
||||
)
|
||||
embedding_batch_admitting: Final = deployments_admitting_embedding_batch(
|
||||
|
|
@ -16916,10 +16980,14 @@ class Router(DeprecatedRouterMembers):
|
|||
|
||||
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
|
||||
_target_order: Final = (request_kwargs or {}).pop("_target_order", None)
|
||||
batch_candidates: Final = self._without_declared_text_only_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
batch_candidates: Final = self._without_decisions_deployments_refusing_media(
|
||||
self._without_declared_text_only_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
model,
|
||||
messages,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
|
||||
),
|
||||
model,
|
||||
messages,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
|
||||
)
|
||||
embedding_batch_admitting: Final = deployments_admitting_embedding_batch(
|
||||
|
|
|
|||
|
|
@ -402,7 +402,7 @@ class OpenAIFileObject(LiteLLMBaseModel):
|
|||
filename: str
|
||||
"""The name of the file."""
|
||||
|
||||
object: Literal["file"]
|
||||
object: Literal["file"] = "file"
|
||||
"""The object type, which is always `file`."""
|
||||
|
||||
purpose: OpenAIFilesPurpose
|
||||
|
|
@ -440,6 +440,11 @@ class OpenAIFileObject(LiteLLMBaseModel):
|
|||
|
||||
_hidden_params: dict = PrivateAttr(default={"response_cost": 0.0}) # no cost for writing a file
|
||||
|
||||
@field_validator("object", mode="before")
|
||||
@classmethod
|
||||
def _default_missing_object_type(cls, value: _JsonValue) -> _JsonValue:
|
||||
return "file" if value is None else value
|
||||
|
||||
@property
|
||||
def hidden_params(self) -> dict[str, builtins.object]: # mutable-ok: API requires mutation
|
||||
return self._hidden_params
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Final
|
||||
from typing import Final, Protocol
|
||||
|
||||
from pydantic import Field
|
||||
from typing_extensions import TypedDict
|
||||
|
|
@ -75,6 +76,19 @@ class PassthroughStandardLoggingPayload(TypedDict, total=False):
|
|||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class WebSocketSessionUsage:
|
||||
model: str
|
||||
custom_llm_provider: str
|
||||
response_cost: float | None
|
||||
|
||||
|
||||
class WebSocketSessionMeter(Protocol):
|
||||
def measure(self, client_text: str) -> int: ...
|
||||
|
||||
def price(self, total: int) -> WebSocketSessionUsage: ...
|
||||
|
||||
|
||||
class PassThroughGuardrailSettings(LiteLLMPydanticObjectBase):
|
||||
"""
|
||||
Settings for a specific guardrail on a passthrough endpoint.
|
||||
|
|
|
|||
|
|
@ -106,6 +106,7 @@ from .llms.openai import (
|
|||
OpenAIRealtimeStreamList,
|
||||
ResponsesAPIResponse,
|
||||
WebSearchOptions,
|
||||
_JsonValue,
|
||||
)
|
||||
from .rerank import RerankResponse as RerankResponse
|
||||
|
||||
|
|
@ -4682,6 +4683,7 @@ class LlmProviders(str, Enum):
|
|||
PRISM = "prism"
|
||||
OPENINFER = "openinfer"
|
||||
INFERENCEAPIS = "inferenceapis"
|
||||
HOSTINGER_ROUTER = "hostinger_router"
|
||||
ZEROGPU = "zerogpu"
|
||||
DARKBLOOM = "darkbloom"
|
||||
META = "meta"
|
||||
|
|
@ -4963,29 +4965,50 @@ def _batch_usage_from_chat_usage(usage: Usage) -> BatchUsage:
|
|||
)
|
||||
|
||||
|
||||
OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS: Final = MappingProxyType(
|
||||
{"pending": "validating", "running": "in_progress"}
|
||||
)
|
||||
|
||||
|
||||
def openai_batch_status(status: _JsonValue) -> _JsonValue:
|
||||
return OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(status, status) if isinstance(status, str) else status
|
||||
|
||||
|
||||
class LiteLLMBatch(Batch):
|
||||
_hidden_params: dict = {}
|
||||
|
||||
object: Literal["batch"] = "batch"
|
||||
|
||||
@field_validator("object", mode="before")
|
||||
@classmethod
|
||||
def _default_missing_object_type(cls, value: _JsonValue) -> _JsonValue:
|
||||
return "batch" if value is None else value
|
||||
|
||||
@field_validator("status", mode="before")
|
||||
@classmethod
|
||||
def _map_vllm_router_status(cls, value: _JsonValue) -> _JsonValue:
|
||||
return openai_batch_status(value)
|
||||
|
||||
@property
|
||||
def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation
|
||||
def hidden_params(self) -> dict[str, _JsonValue]: # mutable-ok: API requires mutation
|
||||
return self._hidden_params
|
||||
|
||||
@hidden_params.setter
|
||||
def hidden_params(self, hidden_params: dict[str, object]) -> None: # mutable-ok: API requires mutation
|
||||
def hidden_params(self, hidden_params: dict[str, _JsonValue]) -> None: # mutable-ok: API requires mutation
|
||||
self._hidden_params = hidden_params
|
||||
|
||||
usage: Usage | None = None
|
||||
|
||||
@field_validator("usage", mode="before")
|
||||
@classmethod
|
||||
def _usage_from_batch_token_counts(cls, value: object) -> object:
|
||||
def _usage_from_batch_token_counts(cls, value: _JsonValue) -> _JsonValue:
|
||||
counts: Final = _parse_batch_token_counts(value)
|
||||
return value if counts is None else _usage_with_batch_token_counts(Usage.model_validate(value), counts)
|
||||
|
||||
@field_serializer("usage")
|
||||
def _serialize_usage_with_batch_fields(
|
||||
self, usage: Usage | None, info: FieldSerializationInfo
|
||||
) -> Mapping[str, object] | None:
|
||||
) -> Mapping[str, _JsonValue] | None:
|
||||
if not isinstance(usage, Usage):
|
||||
return usage
|
||||
chat_fields: Final = usage.model_dump(
|
||||
|
|
|
|||
|
|
@ -10159,7 +10159,7 @@ class ProviderConfigManager:
|
|||
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
||||
|
||||
json_provider: Final = JSONProviderRegistry.get(provider.value)
|
||||
if json_provider is not None and "/v1/messages" in json_provider.supported_endpoints:
|
||||
if json_provider is not None and JSONProviderRegistry.serves_endpoint(provider.value, model, "/v1/messages"):
|
||||
from litellm.llms.openai_like.messages.transformation import (
|
||||
JSONProviderAnthropicMessagesConfig,
|
||||
)
|
||||
|
|
@ -10298,7 +10298,7 @@ class ProviderConfigManager:
|
|||
return result
|
||||
|
||||
# Fall back to JSON providers (generic OpenAI-compatible)
|
||||
if JSONProviderRegistry.exists(provider_str) and JSONProviderRegistry.supports_responses_api(provider_str):
|
||||
if JSONProviderRegistry.serves_endpoint(provider_str, model, "/v1/responses"):
|
||||
provider_config: Final = JSONProviderRegistry.get(provider_str)
|
||||
if provider_config is not None:
|
||||
return create_responses_config_class(provider_config)()
|
||||
|
|
|
|||
|
|
@ -39687,6 +39687,57 @@
|
|||
"/v1/images/generations"
|
||||
]
|
||||
},
|
||||
"hostinger_router/gpt-5.2": {
|
||||
"cache_read_input_token_cost": 0.00000021,
|
||||
"input_cost_per_token": 0.0000021,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 272000,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 0.0000168,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hostinger_router/claude-sonnet-5": {
|
||||
"cache_creation_input_token_cost": 3.3e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 5.28e-06,
|
||||
"cache_read_input_token_cost": 2.64e-07,
|
||||
"input_cost_per_token": 2.64e-06,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.32e-05,
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_response_schema": true,
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/messages"],
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hostinger_router/text-embedding-3-large": {
|
||||
"input_cost_per_token": 1.716e-07,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 8191,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hostinger_router/text-embedding-3-small": {
|
||||
"input_cost_per_token": 3e-08,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 8191,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": {
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
|
|
@ -51387,6 +51438,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 5e-06,
|
||||
"output_cost_per_token_batches": 2.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
@ -51415,6 +51471,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 5e-06,
|
||||
"output_cost_per_token_batches": 2.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
@ -52161,6 +52222,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 1.5e-05,
|
||||
"output_cost_per_token_batches": 7.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
@ -52266,6 +52332,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 1.5e-05,
|
||||
"output_cost_per_token_batches": 7.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
|
|||
|
|
@ -1087,6 +1087,32 @@
|
|||
"generateContent": false
|
||||
}
|
||||
},
|
||||
"hostinger_router": {
|
||||
"display_name": "Hostinger Router (`hostinger_router`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/hostinger_router",
|
||||
"endpoints": {
|
||||
"chat_completions": true,
|
||||
"responses": true,
|
||||
"text_completion": true,
|
||||
"embeddings": true,
|
||||
"image_generations": false,
|
||||
"image_edits": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": false,
|
||||
"moderations": false,
|
||||
"realtime": false,
|
||||
"video_generations": false,
|
||||
"batches": false,
|
||||
"messages": true,
|
||||
"interactions": true,
|
||||
"systemone": false,
|
||||
"a2a": true,
|
||||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"crusoe": {
|
||||
"display_name": "Crusoe (`crusoe`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/crusoe",
|
||||
|
|
|
|||
|
|
@ -383,6 +383,8 @@ exclude-newer = "3 days"
|
|||
[tool.uv.exclude-newer-package]
|
||||
# TODO: remove after 2026-10-25, once brotli 1.2.1 (CVE-2026-111497) clears the 3-day window
|
||||
brotli = "2026-10-22T14:01:00Z"
|
||||
# TODO: remove after 2026-10-31, once pypdf 6.21.4 clears the 3-day window
|
||||
pypdf = "2026-10-28T17:01:00Z"
|
||||
|
||||
[tool.uv.sources]
|
||||
litellm-proxy-extras = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -185,6 +185,9 @@ ignored_function_names = [
|
|||
"_aresponses_background_stream_can_release", # Tested through the background Responses stream tests in test_router.py
|
||||
"_raise_released_stream_error", # Tested through the background Responses stream content-policy tests in test_router.py
|
||||
"_mid_stream_error_before_output", # Tested through the Responses mid-stream fallback and background stream tests
|
||||
"_without_decisions_deployments_refusing_media", # Tested through the mixed media-flag group tests in tests/unit/decisions/test_main.py
|
||||
"_decisions_deployment_refuses_media", # Read by _without_decisions_deployments_refusing_media, tested through the same group tests
|
||||
"_active_default_litellm_params", # Tested through the router default drop_params test in tests/unit/decisions/test_main.py
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
PARITY pairs the streamed case with its plain twin where both bill the same row."""
|
||||
|
||||
import json
|
||||
from typing import Final, cast
|
||||
from typing import Final, Literal, cast
|
||||
|
||||
from pydantic import JsonValue
|
||||
|
||||
|
|
@ -41,6 +41,7 @@ from integration.cost_calculation.cases.anthropic import (
|
|||
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs
|
||||
from integration.cost_calculation.cost_tracking_case import (
|
||||
CostTrackingTestCase,
|
||||
Deployment,
|
||||
ExactExpected,
|
||||
JsonResponse,
|
||||
RecountExpected,
|
||||
|
|
@ -3387,6 +3388,232 @@ VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS: Final = bridge_pairs(
|
|||
)
|
||||
|
||||
|
||||
VERTEX_AI_CLAUDE_DEPLOYMENT_RATES: Final = Deployment(
|
||||
input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.01
|
||||
)
|
||||
|
||||
|
||||
def _vertex_claude_web_search_message(web_search_requests: int) -> JsonResponse:
|
||||
return JsonResponse(
|
||||
content_type="application/json",
|
||||
body={
|
||||
"id": "msg_$REQUEST_ID",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5-5",
|
||||
"content": [{"type": "text", "text": "scripted answer for cost fixtures"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {
|
||||
"input_tokens": 1840,
|
||||
"output_tokens": 412,
|
||||
"server_tool_use": {"web_search_requests": web_search_requests},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _vertex_claude_web_search_stream(web_search_requests: int) -> SseResponse:
|
||||
return SseResponse(
|
||||
content_type="text/event-stream",
|
||||
frames=sse_frames(
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": "msg_$REQUEST_ID",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5-5",
|
||||
"content": [],
|
||||
"stop_reason": None,
|
||||
"usage": {"input_tokens": 1840},
|
||||
},
|
||||
},
|
||||
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": 0,
|
||||
"delta": {"type": "text_delta", "text": "scripted answer for cost fixtures"},
|
||||
},
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
{
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": "end_turn"},
|
||||
"usage": {"output_tokens": 412, "server_tool_use": {"web_search_requests": web_search_requests}},
|
||||
},
|
||||
{"type": "message_stop"},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _vertex_claude_deployment_web_search_case(
|
||||
name: str,
|
||||
endpoint: Literal["/v1/chat/completions", "/v1/responses", "/v1/messages"],
|
||||
request: dict[str, JsonValue],
|
||||
response: JsonResponse | SseResponse,
|
||||
tool_usage_cost: float,
|
||||
deployment: Deployment = VERTEX_AI_CLAUDE_DEPLOYMENT_RATES,
|
||||
) -> CostTrackingTestCase:
|
||||
return CostTrackingTestCase(
|
||||
name=name,
|
||||
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
|
||||
model="vertex_ai/claude-sonnet-5-5",
|
||||
endpoint=endpoint,
|
||||
deployment=deployment,
|
||||
request={"model": "$MODEL", **request},
|
||||
response=response,
|
||||
expected=ExactExpected(
|
||||
spend=round(0.0039 + tool_usage_cost, 10),
|
||||
input_cost=0.00184,
|
||||
output_cost=0.00206,
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
tool_usage_cost=tool_usage_cost,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL: Final[dict[str, JsonValue]] = {
|
||||
"type": "web_search_20250305",
|
||||
"name": "web_search",
|
||||
"max_uses": 3,
|
||||
}
|
||||
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_rates_messages_web_search_two_requests",
|
||||
"/v1/messages",
|
||||
{
|
||||
"max_tokens": 412,
|
||||
"messages": [{"role": "user", "content": "vertex claude deployment rates messages web search"}],
|
||||
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
|
||||
},
|
||||
_vertex_claude_web_search_message(2),
|
||||
0.02,
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM: Final = _vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_rates_messages_web_search_two_requests_stream",
|
||||
"/v1/messages",
|
||||
{
|
||||
"max_tokens": 412,
|
||||
"messages": [{"role": "user", "content": "vertex claude deployment rates messages web search stream"}],
|
||||
"stream": True,
|
||||
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
|
||||
},
|
||||
_vertex_claude_web_search_stream(2),
|
||||
0.02,
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE: Final = (
|
||||
_vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_zero_search_rate_messages_two_requests",
|
||||
"/v1/messages",
|
||||
{
|
||||
"max_tokens": 412,
|
||||
"messages": [{"role": "user", "content": "vertex claude deployment zero search rate messages"}],
|
||||
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
|
||||
},
|
||||
_vertex_claude_web_search_message(2),
|
||||
0.0,
|
||||
Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.0),
|
||||
)
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY: Final = (
|
||||
_vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_token_rates_only_messages_web_search_two_requests",
|
||||
"/v1/messages",
|
||||
{
|
||||
"max_tokens": 412,
|
||||
"messages": [{"role": "user", "content": "vertex claude deployment token rates only messages web search"}],
|
||||
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
|
||||
},
|
||||
_vertex_claude_web_search_message(2),
|
||||
0.034,
|
||||
Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06),
|
||||
)
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_rates_chat_web_search_three_requests",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"messages": [{"role": "user", "content": "vertex claude deployment rates chat web search"}],
|
||||
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
|
||||
"allowed_openai_params": ["web_search_options"],
|
||||
},
|
||||
_vertex_claude_web_search_message(3),
|
||||
0.03,
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH_STREAM: Final = _vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_rates_chat_web_search_two_requests_stream",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"messages": [{"role": "user", "content": "vertex claude deployment rates chat web search stream"}],
|
||||
"stream": True,
|
||||
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
|
||||
"allowed_openai_params": ["web_search_options"],
|
||||
},
|
||||
_vertex_claude_web_search_stream(2),
|
||||
0.02,
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_RESPONSES_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_rates_responses_web_search_two_requests",
|
||||
"/v1/responses",
|
||||
{"input": "vertex claude deployment rates responses web search", "tools": [{"type": "web_search"}]},
|
||||
_vertex_claude_web_search_message(2),
|
||||
0.02,
|
||||
)
|
||||
|
||||
GEMINI_3_1_PRO_PREVIEW_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM: Final = CostTrackingTestCase(
|
||||
name="vertex_ai/gemini-3.1-pro-preview-deployment_rates_messages_web_search_two_queries_stream",
|
||||
covers="quota_management.spend_tracking.scripted_wire.logs_cost",
|
||||
model="vertex_ai/gemini-3.1-pro-preview",
|
||||
endpoint="/v1/messages",
|
||||
deployment=Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.03),
|
||||
request={
|
||||
"model": "$MODEL",
|
||||
"max_tokens": 412,
|
||||
"messages": [{"role": "user", "content": "vertex gemini deployment rates messages web search stream"}],
|
||||
"stream": True,
|
||||
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
|
||||
},
|
||||
response=SseResponse(
|
||||
content_type="text/event-stream",
|
||||
frames=sse_frames(
|
||||
{
|
||||
"candidates": [
|
||||
{
|
||||
"content": {"parts": [{"text": "scripted answer for cost fixtures"}], "role": "model"},
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
"modelVersion": "gemini-3.1-pro-preview",
|
||||
},
|
||||
{
|
||||
"candidates": [
|
||||
{
|
||||
"content": {"parts": [{"text": ""}], "role": "model"},
|
||||
"finishReason": "STOP",
|
||||
"index": 0,
|
||||
"groundingMetadata": {"webSearchQueries": ["query 0", "query 1"]},
|
||||
}
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 1840,
|
||||
"candidatesTokenCount": 412,
|
||||
"totalTokenCount": 2252,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}],
|
||||
},
|
||||
"modelVersion": "gemini-3.1-pro-preview",
|
||||
},
|
||||
),
|
||||
),
|
||||
expected=ExactExpected(
|
||||
spend=0.0339,
|
||||
input_cost=0.00184,
|
||||
output_cost=0.00206,
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
tool_usage_cost=0.03,
|
||||
),
|
||||
)
|
||||
|
||||
GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT: Final = _gemini_case(
|
||||
GEMINI_3_8_FLASH_INPUT_TEXT, "vertex_ai/gemini-3.1-pro-preview-input_text", "vertex_ai/gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.0094864, input_cost=0.004048, output_cost=0.0054384, prompt_tokens=1840, completion_tokens=412),
|
||||
|
|
@ -3725,6 +3952,14 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_STREAM,
|
||||
*(pair.bridged for pair in VERTEX_AI_CLAUDE_SONNET_5_5_BRIDGE_PAIRS),
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH_STREAM,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_RESPONSES_WEB_SEARCH,
|
||||
GEMINI_3_1_PRO_PREVIEW_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM,
|
||||
GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT,
|
||||
GEMINI_3_1_PRO_PREVIEW_STREAM,
|
||||
GEMINI_3_1_PRO_PREVIEW_CACHE_READ,
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ from litellm.accounting import (
|
|||
PricingSnapshot,
|
||||
)
|
||||
from litellm.proxy.db import accounting_ledger
|
||||
from litellm.proxy.db.accounting_ledger import AccountingConflictError, AccountingEvent, AccountingLedger
|
||||
from litellm.proxy.db.accounting_ledger import AccountingConflictError, AccountingEvent, AccountingLedger, UsageWrite
|
||||
from litellm.types.utils import CallTypes
|
||||
from tests.integration._support.database import scratch_database
|
||||
from tests.integration._support.prisma_clients import connected_proxy_prisma
|
||||
|
|
@ -532,3 +532,74 @@ def test_family_cleanup_lookups_use_the_kind_source_index(ledger_database: str,
|
|||
name == "LiteLLM_AccountingEvent_kind_source_event_id_idx" and "source_event_id" in str(condition)
|
||||
for name, condition in conditions
|
||||
), conditions
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("failure", ["before_commit", "after_commit", "conflict"])
|
||||
async def test_usage_batch_is_atomic_and_replay_preserves_receipts(
|
||||
writer: _Writer, ledger_database: str, failure: str
|
||||
) -> None:
|
||||
client, store = writer
|
||||
events: Final = (_usage("caller", inbound=True), _usage("attempt"))
|
||||
writes: Final = tuple(UsageWrite(event, _envelope(event), _TARGETS) for event in events)
|
||||
|
||||
@asynccontextmanager
|
||||
async def interrupted() -> AsyncGenerator[Prisma]:
|
||||
async with client.writer_db.tx() as tx:
|
||||
yield tx
|
||||
if failure == "before_commit":
|
||||
raise RuntimeError("injected pre-commit failure")
|
||||
if failure == "after_commit":
|
||||
raise RuntimeError("injected lost acknowledgement")
|
||||
|
||||
if failure == "conflict":
|
||||
await store.insert_usage_batch((writes[1],))
|
||||
_complete_and_age(ledger_database)
|
||||
rejected: Final = (writes[0], UsageWrite(events[1], _envelope(events[1], context={"changed": True}), _TARGETS))
|
||||
with pytest.raises(AccountingConflictError):
|
||||
await store.insert_usage_batch(rejected)
|
||||
assert _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent"') == [("attempt",)]
|
||||
else:
|
||||
with pytest.raises(RuntimeError, match="injected"):
|
||||
await AccountingLedger(interrupted).insert_usage_batch(writes)
|
||||
count: Final = 2 if failure == "after_commit" else 0
|
||||
assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(count,)]
|
||||
assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery"') == [(count * len(_TARGETS),)]
|
||||
await store.insert_usage_batch(writes)
|
||||
recovered: Final = await store.load_usage_batch(tuple(event.event_id for event in events))
|
||||
assert len(recovered) == 1
|
||||
assert recovered[0].event == events[1]
|
||||
assert json.loads(recovered[0].envelope) == json.loads(writes[1].envelope)
|
||||
assert set(recovered[0].targets) == set(_TARGETS)
|
||||
_complete_and_age(ledger_database)
|
||||
before: Final = _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target')
|
||||
await store.insert_usage_batch((*writes, *writes))
|
||||
assert _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target') == before
|
||||
assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(2,)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_jsonb_submission_cannot_reject_another_producers_commit(
|
||||
writer: _Writer, ledger_database: str
|
||||
) -> None:
|
||||
from prisma.errors import DataError
|
||||
|
||||
from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings
|
||||
|
||||
_, store = writer
|
||||
invalid: Final = _usage("invalid")
|
||||
valid: Final = _usage("valid")
|
||||
meter: Final = DurableUsageMeter(store.insert_usage_batch, UsageCaptureSettings(batch_size=2))
|
||||
rejected: Final = meter.submit(
|
||||
(UsageWrite(invalid, _envelope(invalid, context={"parent": "bad\x00text"}), _TARGETS),)
|
||||
)
|
||||
accepted: Final = meter.submit((UsageWrite(valid, _envelope(valid), _TARGETS),))
|
||||
try:
|
||||
with pytest.raises(DataError):
|
||||
await rejected
|
||||
await accepted
|
||||
finally:
|
||||
await meter.close()
|
||||
assert _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent"') == [("valid",)]
|
||||
assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery"') == [(len(_TARGETS),)]
|
||||
assert (meter.pending_bytes, meter.pending_records) == (0, 0)
|
||||
|
|
|
|||
|
|
@ -317,7 +317,6 @@ def test_credentialless_proxy_forwards_the_caller_anthropic_credential_and_never
|
|||
|
||||
|
||||
def test_anthropic_sdk_metadata_user_id_reaches_upstream(api_key_proxy: Gateway, wire: Wire) -> None:
|
||||
pytest.skip("BUG: /anthropic/v1/messages drops the native metadata.user_id field before forwarding upstream")
|
||||
with api_key_proxy.scenario() as scenario:
|
||||
key: Final = scenario.key()
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
|
|
|||
|
|
@ -0,0 +1,123 @@
|
|||
import json
|
||||
import uuid
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Literal
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from integration._support.client import Gateway, eventually, object_value
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_API_KEY: Final = "synthetic-decisions-key"
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_INPUT: Final = "Listen to the attached clip."
|
||||
_AUDIO: Final = ("data:audio/wav;base64,UklGRiQAAABXQVZF",)
|
||||
_VIDEOS: Final = ("data:video/mp4;base64,AAAAIGZ0eXBpc29t",)
|
||||
_QUESTION: Final[dict[str, JsonValue]] = {
|
||||
"type": "predicate",
|
||||
"name": "banana",
|
||||
"instructions": "Does the clip name a banana?",
|
||||
}
|
||||
_SPEND_QUERY: Final = 'SELECT status, model_group FROM "LiteLLM_SpendLogs" WHERE request_id = %s'
|
||||
|
||||
Route = Literal["/v1/decisions", "/v1/systemone"]
|
||||
Media = Literal["audio", "videos"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Upstream:
|
||||
model: str
|
||||
|
||||
def reply(self, request: Request) -> Reply:
|
||||
questions: Final = object_value(_JSON_OBJECT.validate_json(request.body)["questions"])
|
||||
plain: Final[dict[str, JsonValue]] = {
|
||||
"model": self.model,
|
||||
"answers": {name: {"type": "noul", "noul": 0.9} for name in questions},
|
||||
"usage": {"input_tokens": 41, "output_tokens": 0},
|
||||
}
|
||||
return Reply(body=json.dumps(plain).encode())
|
||||
|
||||
|
||||
def _unlisted_openrouter_model() -> _Upstream:
|
||||
return _Upstream(f"openrouter/example/omni-{uuid.uuid4().hex[:12]}")
|
||||
|
||||
|
||||
def _decide(gateway: Gateway, model: str, route: Route, media: Mapping[str, JsonValue]) -> httpx.Response:
|
||||
body: Final[dict[str, JsonValue]] = (
|
||||
{"model": model, "input": _INPUT, "questions": [_QUESTION], "num_retries": 0, **media}
|
||||
if route == "/v1/decisions"
|
||||
else {
|
||||
"model": model,
|
||||
"state": _INPUT,
|
||||
"questions": {"banana": {"type": "noul", "instructions": _QUESTION["instructions"]}},
|
||||
"num_retries": 0,
|
||||
**media,
|
||||
}
|
||||
)
|
||||
return gateway.request("POST", route, body)
|
||||
|
||||
|
||||
def _sent_media(request: Request) -> dict[str, JsonValue]:
|
||||
body: Final = _JSON_OBJECT.validate_json(request.body)
|
||||
return {kind: body[kind] for kind in ("audio", "videos") if kind in body}
|
||||
|
||||
|
||||
def _assert_refused(response: httpx.Response, kinds: tuple[Media, ...]) -> None:
|
||||
assert response.status_code == 400, response.text
|
||||
assert f"does not support parameters: {list(kinds)}" in response.text, response.text
|
||||
|
||||
|
||||
_BOTH: Final[dict[str, JsonValue]] = {"audio": list(_AUDIO), "videos": list(_VIDEOS)}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", ("/v1/decisions", "/v1/systemone"))
|
||||
def test_a_deployment_flagged_for_audio_and_video_sends_both_clips_for_a_model_the_cost_map_does_not_list(
|
||||
gateway: Gateway, route: Route
|
||||
) -> None:
|
||||
upstream: Final = _unlisted_openrouter_model()
|
||||
with wire_server(upstream.reply) as wire, gateway.scenario() as scenario:
|
||||
flags: Final[dict[str, JsonValue]] = {"supports_audio_input": True, "supports_video_input": True}
|
||||
flagged: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info=flags)
|
||||
flagged_dropping: Final = scenario.model(
|
||||
model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info=flags, drop_params=True
|
||||
)
|
||||
audio_only: Final = scenario.model(
|
||||
model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info={"supports_audio_input": True}
|
||||
)
|
||||
unflagged: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY)
|
||||
|
||||
_assert_refused(_decide(gateway, unflagged, route, _BOTH), ("audio", "videos"))
|
||||
_assert_refused(_decide(gateway, audio_only, route, _BOTH), ("videos",))
|
||||
assert wire.drain() == ()
|
||||
|
||||
answered: Final = _decide(gateway, flagged, route, _BOTH)
|
||||
assert answered.status_code == 200, answered.text
|
||||
(call,) = wire.drain()
|
||||
assert _sent_media(call) == _BOTH
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(_SPEND_QUERY, (answered.headers["x-litellm-call-id"],)),
|
||||
lambda found: len(found) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
assert (rows[0]["status"], rows[0]["model_group"]) == ("success", flagged)
|
||||
|
||||
kept: Final = _decide(gateway, flagged_dropping, route, _BOTH)
|
||||
assert kept.status_code == 200, kept.text
|
||||
(kept_call,) = wire.drain()
|
||||
assert _sent_media(kept_call) == _BOTH
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value",
|
||||
({"supports_audio_input": True, "supports_video_input": True}, 5, [{"supports_audio_input": True}], "x" * 5000),
|
||||
ids=("flags", "int", "list", "5kb"),
|
||||
)
|
||||
def test_a_caller_sent_model_info_never_turns_media_on(gateway: Gateway, value: JsonValue) -> None:
|
||||
upstream: Final = _unlisted_openrouter_model()
|
||||
with wire_server(upstream.reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY)
|
||||
_assert_refused(_decide(gateway, model, "/v1/decisions", {**_BOTH, "model_info": value}), ("audio", "videos"))
|
||||
assert wire.drain() == ()
|
||||
|
|
@ -0,0 +1,413 @@
|
|||
import uuid
|
||||
from collections.abc import Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final, Literal
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support import responses_vendor as rv
|
||||
from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment
|
||||
from integration._support.openai_wire import answering_model_discovery, chat_reply, responses_reply
|
||||
from integration._support.process import OwnedProxy, owned_proxy_process
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
pytestmark: Final = pytest.mark.timeout(120)
|
||||
|
||||
_GPT: Final = "gpt-5.6"
|
||||
_DATABRICKS: Final = "databricks-meta-llama-4-maverick"
|
||||
_PROVIDER_KEY: Final = "synthetic-provider-key"
|
||||
_NO_CACHE: Final[dict[str, JsonValue]] = {"no-cache": True}
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_WEATHER_SCHEMA: Final[dict[str, JsonValue]] = {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}, "celsius": {"type": "number"}},
|
||||
"required": ["city", "celsius"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
_TEXT_FORMAT: Final[dict[str, JsonValue]] = {
|
||||
"type": "json_schema",
|
||||
"name": "weather",
|
||||
"schema": _WEATHER_SCHEMA,
|
||||
"strict": True,
|
||||
}
|
||||
_RESPONSE_FORMAT: Final[dict[str, JsonValue]] = {
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "weather", "schema": _WEATHER_SCHEMA, "strict": True},
|
||||
}
|
||||
_STREAM_OPTIONS: Final[dict[str, JsonValue]] = {"stream_options": {"include_usage": True}}
|
||||
|
||||
Verbosity = Literal["low", "high"]
|
||||
|
||||
|
||||
def _prompt(marker: str) -> str:
|
||||
return f"verbosity probe marker-{marker}"
|
||||
|
||||
|
||||
def _answer(marker: str) -> str:
|
||||
return f"scripted answer marker-{marker}"
|
||||
|
||||
|
||||
def _marker_of(request: Request) -> str:
|
||||
body: Final = request.body.decode()
|
||||
start: Final = body.rfind("marker-")
|
||||
return body[start + len("marker-") : start + len("marker-") + 32] if start >= 0 else "health"
|
||||
|
||||
|
||||
def _vendor(request: Request) -> Reply:
|
||||
body: Final = _JSON_OBJECT.validate_json(request.body or b"{}")
|
||||
marker: Final = _marker_of(request)
|
||||
stream: Final = body.get("stream") is True
|
||||
if request.target.endswith("/responses"):
|
||||
return responses_reply(f"resp_{marker}", _GPT, _answer(marker), stream=stream)
|
||||
return chat_reply(f"chatcmpl-{marker}", str(body.get("model")), _answer(marker), stream=stream)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _upstream() -> Iterator[Wire]:
|
||||
with wire_server(answering_model_discovery(_vendor)) as wire:
|
||||
yield wire
|
||||
|
||||
|
||||
def _bridged_gpt(scenario: Scenario, wire: Wire, **parameters: JsonValue) -> str:
|
||||
return scenario.model(
|
||||
model=f"openai/{_GPT}",
|
||||
api_base=f"{wire.url}/v1",
|
||||
api_key=_PROVIDER_KEY,
|
||||
use_chat_completions_api=True,
|
||||
**parameters,
|
||||
)
|
||||
|
||||
|
||||
def _native_gpt(scenario: Scenario, wire: Wire) -> str:
|
||||
return scenario.model(model=f"openai/{_GPT}", api_base=f"{wire.url}/v1", api_key=_PROVIDER_KEY)
|
||||
|
||||
|
||||
def _databricks(scenario: Scenario, wire: Wire, **parameters: JsonValue) -> str:
|
||||
return scenario.model(
|
||||
model=f"databricks/{_DATABRICKS}",
|
||||
api_base=f"{wire.url}/serving-endpoints",
|
||||
api_key=_PROVIDER_KEY,
|
||||
**parameters,
|
||||
)
|
||||
|
||||
|
||||
def _responses(gateway: Gateway, model: str, marker: str, **fields: JsonValue) -> httpx.Response:
|
||||
return gateway.request(
|
||||
"POST", "/v1/responses", {"model": model, "input": _prompt(marker), "cache": _NO_CACHE, **fields}
|
||||
)
|
||||
|
||||
|
||||
def _streamed_events(gateway: Gateway, model: str, marker: str, **fields: JsonValue) -> list[dict[str, JsonValue]]:
|
||||
with gateway.client.stream(
|
||||
"POST",
|
||||
"/v1/responses",
|
||||
json={"model": model, "input": _prompt(marker), "cache": _NO_CACHE, "stream": True, **fields},
|
||||
headers={"Authorization": f"Bearer {gateway.key}"},
|
||||
) as response:
|
||||
assert response.status_code == 200, response.read()
|
||||
return [
|
||||
_JSON_OBJECT.validate_json(line.removeprefix("data: "))
|
||||
for line in response.iter_lines()
|
||||
if line.startswith("data: {")
|
||||
]
|
||||
|
||||
|
||||
def _output_text(response: Mapping[str, JsonValue]) -> str:
|
||||
output: Final = response["output"]
|
||||
assert isinstance(output, list), response
|
||||
return "".join(
|
||||
str(part["text"])
|
||||
for item in output
|
||||
if isinstance(item, dict) and item.get("type") == "message"
|
||||
for part in item["content"]
|
||||
if isinstance(part, dict) and part.get("type") == "output_text"
|
||||
)
|
||||
|
||||
|
||||
def _answered(response: httpx.Response, marker: str) -> None:
|
||||
assert response.status_code == 200, response.text
|
||||
body: Final = _JSON_OBJECT.validate_json(response.text)
|
||||
assert body["status"] == "completed" and _output_text(body) == _answer(marker), body
|
||||
|
||||
|
||||
def _streamed_answer(events: list[dict[str, JsonValue]], marker: str) -> None:
|
||||
completed: Final = [event for event in events if event["type"] == "response.completed"]
|
||||
assert len(completed) == 1, [event["type"] for event in events]
|
||||
final: Final = completed[0]["response"]
|
||||
assert isinstance(final, dict) and _output_text(final) == _answer(marker), final
|
||||
deltas: Final = "".join(str(event["delta"]) for event in events if event["type"] == "response.output_text.delta")
|
||||
assert deltas == _answer(marker), events
|
||||
|
||||
|
||||
def _posted(wire: Wire) -> tuple[Request, ...]:
|
||||
return tuple(request for request in wire.drain() if request.method == "POST")
|
||||
|
||||
|
||||
def _only_post(wire: Wire, target: str) -> dict[str, JsonValue]:
|
||||
posted: Final = _posted(wire)
|
||||
assert [request.target for request in posted] == [target], posted
|
||||
return _JSON_OBJECT.validate_json(posted[0].body)
|
||||
|
||||
|
||||
def _chat_messages(marker: str) -> list[JsonValue]:
|
||||
return [{"role": "user", "content": _prompt(marker)}]
|
||||
|
||||
|
||||
def _unsupported_param_error(response: httpx.Response) -> None:
|
||||
assert response.status_code == 400, response.text
|
||||
error: Final = _JSON_OBJECT.validate_json(response.text)["error"]
|
||||
assert isinstance(error, dict) and "verbosity" in str(error["message"]), response.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("verbosity", ["low", "high"])
|
||||
def test_regression_explicit_bridge_sends_text_verbosity_as_top_level_verbosity(
|
||||
gateway: Gateway, verbosity: Verbosity
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with _upstream() as wire, gateway.scenario() as scenario:
|
||||
model: Final = _bridged_gpt(scenario, wire)
|
||||
_answered(_responses(gateway, model, marker, text={"verbosity": verbosity}), marker)
|
||||
sent: Final = _only_post(wire, "/v1/chat/completions")
|
||||
assert sent == {"model": _GPT, "messages": _chat_messages(marker), "verbosity": verbosity}, sent
|
||||
|
||||
|
||||
@pytest.mark.parametrize("verbosity", ["low", "high"])
|
||||
def test_regression_explicit_bridge_stream_sends_text_verbosity_as_top_level_verbosity(
|
||||
gateway: Gateway, verbosity: Verbosity
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with _upstream() as wire, gateway.scenario() as scenario:
|
||||
model: Final = _bridged_gpt(scenario, wire)
|
||||
_streamed_answer(_streamed_events(gateway, model, marker, text={"verbosity": verbosity}), marker)
|
||||
sent: Final = _only_post(wire, "/v1/chat/completions")
|
||||
assert sent == {
|
||||
"model": _GPT,
|
||||
"messages": _chat_messages(marker),
|
||||
"stream": True,
|
||||
**_STREAM_OPTIONS,
|
||||
"verbosity": verbosity,
|
||||
}, sent
|
||||
|
||||
|
||||
def test_regression_explicit_bridge_sends_verbosity_beside_the_json_schema_response_format(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with _upstream() as wire, gateway.scenario() as scenario:
|
||||
model: Final = _bridged_gpt(scenario, wire)
|
||||
_answered(_responses(gateway, model, marker, text={"format": _TEXT_FORMAT, "verbosity": "low"}), marker)
|
||||
sent: Final = _only_post(wire, "/v1/chat/completions")
|
||||
assert sent == {
|
||||
"model": _GPT,
|
||||
"messages": _chat_messages(marker),
|
||||
"response_format": _RESPONSE_FORMAT,
|
||||
"verbosity": "low",
|
||||
}, sent
|
||||
|
||||
|
||||
def test_regression_automatic_bridge_refuses_unsupported_verbosity_before_the_wire(gateway: Gateway) -> None:
|
||||
refused: Final = uuid.uuid4().hex
|
||||
control: Final = uuid.uuid4().hex
|
||||
with _upstream() as wire, gateway.scenario() as scenario:
|
||||
model: Final = _databricks(scenario, wire, drop_params=False)
|
||||
_unsupported_param_error(_responses(gateway, model, refused, text={"verbosity": "low"}))
|
||||
_answered(_responses(gateway, model, control), control)
|
||||
posted: Final = _posted(wire)
|
||||
assert [(request.target, _marker_of(request)) for request in posted] == [
|
||||
("/serving-endpoints/chat/completions", control)
|
||||
], posted
|
||||
|
||||
|
||||
def test_regression_automatic_bridge_stream_refuses_unsupported_verbosity_before_the_wire(gateway: Gateway) -> None:
|
||||
refused: Final = uuid.uuid4().hex
|
||||
control: Final = uuid.uuid4().hex
|
||||
with _upstream() as wire, gateway.scenario() as scenario:
|
||||
model: Final = _databricks(scenario, wire, drop_params=False)
|
||||
_unsupported_param_error(_responses(gateway, model, refused, text={"verbosity": "high"}, stream=True))
|
||||
_answered(_responses(gateway, model, control), control)
|
||||
posted: Final = _posted(wire)
|
||||
assert [(request.target, _marker_of(request)) for request in posted] == [
|
||||
("/serving-endpoints/chat/completions", control)
|
||||
], posted
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source", ["deployment", "request"])
|
||||
def test_regression_allowed_openai_params_forwards_verbosity_through_the_automatic_bridge(
|
||||
gateway: Gateway, source: Literal["deployment", "request"]
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
allowed: Final[dict[str, JsonValue]] = {"allowed_openai_params": ["verbosity"]}
|
||||
with _upstream() as wire, gateway.scenario() as scenario:
|
||||
model: Final = _databricks(scenario, wire, drop_params=False, **(allowed if source == "deployment" else {}))
|
||||
extra: Final = allowed if source == "request" else {}
|
||||
_answered(_responses(gateway, model, marker, text={"verbosity": "low"}, **extra), marker)
|
||||
sent: Final = _only_post(wire, "/serving-endpoints/chat/completions")
|
||||
assert sent == {"model": _DATABRICKS, "messages": _chat_messages(marker), "verbosity": "low"}, sent
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source", ["deployment", "request"])
|
||||
def test_guard_drop_params_drops_unsupported_verbosity_and_keeps_the_json_schema_format(
|
||||
gateway: Gateway, source: Literal["deployment", "request"]
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with _upstream() as wire, gateway.scenario() as scenario:
|
||||
model: Final = _databricks(scenario, wire, drop_params=source == "deployment")
|
||||
extra: Final[dict[str, JsonValue]] = {"drop_params": True} if source == "request" else {}
|
||||
_answered(
|
||||
_responses(gateway, model, marker, text={"format": _TEXT_FORMAT, "verbosity": "low"}, **extra), marker
|
||||
)
|
||||
sent: Final = _only_post(wire, "/serving-endpoints/chat/completions")
|
||||
assert sent == {"model": _DATABRICKS, "messages": _chat_messages(marker), "response_format": _RESPONSE_FORMAT}, sent
|
||||
|
||||
|
||||
def test_guard_drop_params_stream_drops_unsupported_verbosity(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with _upstream() as wire, gateway.scenario() as scenario:
|
||||
model: Final = _databricks(scenario, wire, drop_params=True)
|
||||
_streamed_answer(_streamed_events(gateway, model, marker, text={"verbosity": "high"}), marker)
|
||||
sent: Final = _only_post(wire, "/serving-endpoints/chat/completions")
|
||||
assert sent == {"model": _DATABRICKS, "messages": _chat_messages(marker), "stream": True}, sent
|
||||
|
||||
|
||||
def test_guard_explicit_bridge_without_verbosity_sends_only_the_json_schema_format(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with _upstream() as wire, gateway.scenario() as scenario:
|
||||
model: Final = _bridged_gpt(scenario, wire)
|
||||
_answered(_responses(gateway, model, marker, text={"format": _TEXT_FORMAT}), marker)
|
||||
sent: Final = _only_post(wire, "/v1/chat/completions")
|
||||
assert sent == {"model": _GPT, "messages": _chat_messages(marker), "response_format": _RESPONSE_FORMAT}, sent
|
||||
|
||||
|
||||
def test_guard_explicit_bridge_without_text_sends_no_verbosity(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with _upstream() as wire, gateway.scenario() as scenario:
|
||||
model: Final = _bridged_gpt(scenario, wire)
|
||||
_answered(_responses(gateway, model, marker), marker)
|
||||
sent: Final = _only_post(wire, "/v1/chat/completions")
|
||||
assert sent == {"model": _GPT, "messages": _chat_messages(marker)}, sent
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"])
|
||||
def test_guard_native_responses_route_forwards_text_verbosity_unchanged(gateway: Gateway, stream: bool) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
text: Final[dict[str, JsonValue]] = {"format": _TEXT_FORMAT, "verbosity": "high"}
|
||||
with _upstream() as wire, gateway.scenario() as scenario:
|
||||
model: Final = _native_gpt(scenario, wire)
|
||||
if stream:
|
||||
_streamed_answer(_streamed_events(gateway, model, marker, text=text), marker)
|
||||
else:
|
||||
_answered(_responses(gateway, model, marker, text=text), marker)
|
||||
sent: Final = _only_post(wire, "/v1/responses")
|
||||
assert sent == {"model": _GPT, "input": _prompt(marker), "text": text, **({"stream": True} if stream else {})}, sent
|
||||
|
||||
|
||||
_CLIENT_TOOL_SEARCH: Final[dict[str, JsonValue]] = {"type": "tool_search", "execution": "client"}
|
||||
_CLIENT_TOOL_SEARCH_ON_THE_WIRE: Final[dict[str, JsonValue]] = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "tool_search",
|
||||
"description": (
|
||||
"Search the deferred tools by keyword. The matching tools are loaded and become callable on the next turn."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string", "description": "Search query for deferred tools."},
|
||||
"limit": {"type": "integer", "description": "Maximum number of tools to return. Defaults to 8."},
|
||||
},
|
||||
"required": ["query"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _FollowUpWire:
|
||||
sent: JsonValue
|
||||
expected: dict[str, JsonValue]
|
||||
|
||||
|
||||
def _stored_prompts_config(directory: Path) -> Path:
|
||||
stock: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
|
||||
general: Final = _JSON_OBJECT.validate_python(stock["general_settings"])
|
||||
path: Final = directory / "verbosity-stored-turns.yaml"
|
||||
path.write_text(yaml.safe_dump({**stock, "general_settings": {**general, "store_prompts_in_spend_logs": True}}))
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def stored_turns_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[OwnedProxy, Wire]]:
|
||||
directory: Final = tmp_path_factory.mktemp("verbosity-stored-turns")
|
||||
with _upstream() as wire, gateway_from_environment() as gateway:
|
||||
with owned_proxy_process(gateway, directory, {}, config=_stored_prompts_config(directory)) as owned:
|
||||
yield owned, wire
|
||||
|
||||
|
||||
def _stored_first_turn(proxy: Gateway, model: str, marker: str) -> str:
|
||||
first: Final = _responses(proxy, model, marker, tools=[_CLIENT_TOOL_SEARCH])
|
||||
_answered(first, marker)
|
||||
response_id: Final = str(_JSON_OBJECT.validate_json(first.text)["id"])
|
||||
eventually(
|
||||
lambda: proxy.rows('SELECT request_id FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)),
|
||||
lambda rows: any(rv.same_response(str(row["request_id"]), response_id) for row in rows),
|
||||
seconds=70,
|
||||
)
|
||||
return response_id
|
||||
|
||||
|
||||
def _client_tool_search_follow_up(
|
||||
stored_turns_proxy: tuple[OwnedProxy, Wire], stream: bool, **fields: JsonValue
|
||||
) -> _FollowUpWire:
|
||||
owned, wire = stored_turns_proxy
|
||||
proxy: Final = owned.gateway
|
||||
first: Final = uuid.uuid4().hex
|
||||
second: Final = uuid.uuid4().hex
|
||||
with proxy.scenario() as scenario:
|
||||
model: Final = _bridged_gpt(scenario, wire)
|
||||
previous: Final = _stored_first_turn(proxy, model, first)
|
||||
wire.drain()
|
||||
follow_up: Final[dict[str, JsonValue]] = {
|
||||
"previous_response_id": previous,
|
||||
"tools": [_CLIENT_TOOL_SEARCH],
|
||||
**fields,
|
||||
}
|
||||
if stream:
|
||||
_streamed_answer(_streamed_events(proxy, model, second, **follow_up), second)
|
||||
else:
|
||||
_answered(_responses(proxy, model, second, **follow_up), second)
|
||||
sent: Final = _only_post(wire, "/v1/chat/completions")
|
||||
return _FollowUpWire(
|
||||
sent,
|
||||
{
|
||||
"model": _GPT,
|
||||
"messages": [
|
||||
*_chat_messages(first),
|
||||
{"role": "assistant", "content": _answer(first)},
|
||||
*_chat_messages(second),
|
||||
],
|
||||
"tools": [_CLIENT_TOOL_SEARCH_ON_THE_WIRE],
|
||||
**({"stream": True, **_STREAM_OPTIONS} if stream else {}),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.timeout(240)
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"])
|
||||
@pytest.mark.parametrize("verbosity", ["low", "high"])
|
||||
def test_regression_client_tool_search_follow_up_sends_text_verbosity_as_top_level_verbosity(
|
||||
stored_turns_proxy: tuple[OwnedProxy, Wire], stream: bool, verbosity: Verbosity
|
||||
) -> None:
|
||||
observed: Final = _client_tool_search_follow_up(stored_turns_proxy, stream, text={"verbosity": verbosity})
|
||||
assert observed.sent == {**observed.expected, "verbosity": verbosity}, observed.sent
|
||||
|
||||
|
||||
@pytest.mark.timeout(240)
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"])
|
||||
def test_guard_client_tool_search_follow_up_without_verbosity_sends_no_verbosity(
|
||||
stored_turns_proxy: tuple[OwnedProxy, Wire], stream: bool
|
||||
) -> None:
|
||||
observed: Final = _client_tool_search_follow_up(stored_turns_proxy, stream)
|
||||
assert observed.sent == observed.expected, observed.sent
|
||||
|
|
@ -715,8 +715,11 @@ def test_cyberark_incremental_update_encrypts_reloads_on_restart_and_delete_clea
|
|||
}, saved.text
|
||||
stored_full: Final = _stored_override(database_url, "cyberark")
|
||||
assert set(stored_full) == set(full_config), stored_full
|
||||
assert all(stored_full[key] != value for key, value in full_config.items()), stored_full
|
||||
serialized_full: Final = json.dumps(stored_full)
|
||||
assert all(value not in serialized_full for value in full_config.values()), serialized_full
|
||||
assert all(value not in serialized_full for value in (cyberark_wire.url, account, username, api_key)), (
|
||||
serialized_full
|
||||
)
|
||||
|
||||
model_one: Final = "integration-cyberark-" + uuid.uuid4().hex
|
||||
_new_model(first, model_one, provider_wire.url, secret_names[0])
|
||||
|
|
|
|||
|
|
@ -37,13 +37,15 @@ import litellm.batches.main as bm
|
|||
import asyncio
|
||||
import datetime
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from typing import Final
|
||||
import httpx
|
||||
import respx
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from openai.types.batch import Batch
|
||||
from litellm.types.utils import LiteLLMBatch
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
|
@ -1204,3 +1206,183 @@ async def test_delete_batch_output_file(respx_mock: respx.MockRouter, monkeypatc
|
|||
assert delete_route.call_count == 1
|
||||
assert delete_response.id == "file-output123"
|
||||
assert delete_response.deleted is True
|
||||
|
||||
|
||||
VLLM_ROUTER_API_BASE: Final = "http://vllm-router.test/v1"
|
||||
VLLM_ROUTER_BATCH: Final = MappingProxyType(
|
||||
{
|
||||
"id": "batch_5f8dac",
|
||||
"status": "pending",
|
||||
"input_file_id": "file-b73f4c",
|
||||
"created_at": 1793239207,
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"completion_window": "24h",
|
||||
"output_file_id": None,
|
||||
"total_requests": None,
|
||||
"completed_requests": 0,
|
||||
"failed_requests": 0,
|
||||
"metadata": {},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
async def _create_on_vllm_router() -> LiteLLMBatch:
|
||||
return await litellm.acreate_batch(
|
||||
completion_window="24h",
|
||||
endpoint="/v1/chat/completions",
|
||||
input_file_id="file-b73f4c",
|
||||
custom_llm_provider="hosted_vllm",
|
||||
api_key="sk-test",
|
||||
api_base=VLLM_ROUTER_API_BASE,
|
||||
)
|
||||
|
||||
|
||||
async def _retrieve_from_vllm_router() -> LiteLLMBatch:
|
||||
return await litellm.aretrieve_batch(
|
||||
batch_id="batch_5f8dac", custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE
|
||||
)
|
||||
|
||||
|
||||
VLLM_ROUTER_BATCH_CALLS: Final = (
|
||||
pytest.param("POST", "/batches", _create_on_vllm_router, id="acreate_batch"),
|
||||
pytest.param("GET", "/batches/batch_5f8dac", _retrieve_from_vllm_router, id="aretrieve_batch"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_BATCH_CALLS)
|
||||
@pytest.mark.parametrize(
|
||||
"router_status, openai_status",
|
||||
(
|
||||
("pending", "validating"),
|
||||
("running", "in_progress"),
|
||||
("completed", "completed"),
|
||||
("failed", "failed"),
|
||||
("cancelled", "cancelled"),
|
||||
),
|
||||
)
|
||||
async def test_hosted_vllm_router_batch_comes_back_as_an_openai_batch(
|
||||
respx_mock: respx.MockRouter,
|
||||
method: str,
|
||||
path: str,
|
||||
call_router: Callable[[], Awaitable[LiteLLMBatch]],
|
||||
router_status: str,
|
||||
openai_status: str,
|
||||
):
|
||||
respx_mock.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond(
|
||||
200, json={**VLLM_ROUTER_BATCH, "status": router_status}
|
||||
)
|
||||
|
||||
batch: Final = await call_router()
|
||||
|
||||
assert (batch.id, batch.object, batch.status) == ("batch_5f8dac", "batch", openai_status)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_BATCH_CALLS)
|
||||
@pytest.mark.parametrize("field, value", (("status", "queued"), ("object", "file")))
|
||||
async def test_hosted_vllm_router_batch_with_an_unknown_value_is_rejected(
|
||||
respx_mock: respx.MockRouter,
|
||||
method: str,
|
||||
path: str,
|
||||
call_router: Callable[[], Awaitable[LiteLLMBatch]],
|
||||
field: str,
|
||||
value: str,
|
||||
):
|
||||
respx_mock.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond(
|
||||
200, json={**VLLM_ROUTER_BATCH, field: value}
|
||||
)
|
||||
|
||||
with pytest.raises(ValidationError) as exc_info:
|
||||
await call_router()
|
||||
|
||||
assert [error["loc"] for error in exc_info.value.errors()] == [(field,)]
|
||||
|
||||
|
||||
async def _list_on_vllm_router() -> Sequence[Batch]:
|
||||
page: Final = await litellm.alist_batches(
|
||||
custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE
|
||||
)
|
||||
return page.data
|
||||
|
||||
|
||||
async def _list_on_vllm_router_sync() -> Sequence[Batch]:
|
||||
return litellm.list_batches(custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE).data
|
||||
|
||||
|
||||
@pytest.mark.parametrize("list_router", (_list_on_vllm_router, _list_on_vllm_router_sync))
|
||||
async def test_hosted_vllm_router_batch_list_comes_back_as_openai_batches(
|
||||
respx_mock: respx.MockRouter, list_router: Callable[[], Awaitable[Sequence[Batch]]]
|
||||
):
|
||||
respx_mock.get(f"{VLLM_ROUTER_API_BASE}/batches").respond(
|
||||
200,
|
||||
json={
|
||||
"object": "list",
|
||||
"data": [
|
||||
{**VLLM_ROUTER_BATCH, "id": "batch_5f8dac", "status": "pending"},
|
||||
{**VLLM_ROUTER_BATCH, "id": "batch_c74802", "status": "running"},
|
||||
],
|
||||
"first_id": "batch_5f8dac",
|
||||
"last_id": "batch_c74802",
|
||||
"has_more": False,
|
||||
},
|
||||
)
|
||||
|
||||
batches: Final = await list_router()
|
||||
|
||||
assert [(batch.id, batch.object, batch.status) for batch in batches] == [
|
||||
("batch_5f8dac", "batch", "validating"),
|
||||
("batch_c74802", "batch", "in_progress"),
|
||||
]
|
||||
|
||||
|
||||
async def test_hosted_vllm_router_batch_list_passes_through_a_status_that_is_not_a_string(
|
||||
respx_mock: respx.MockRouter,
|
||||
):
|
||||
respx_mock.get(f"{VLLM_ROUTER_API_BASE}/batches").respond(
|
||||
200,
|
||||
json={
|
||||
"object": "list",
|
||||
"data": [{**VLLM_ROUTER_BATCH, "status": ["pending"]}],
|
||||
"first_id": "batch_5f8dac",
|
||||
"last_id": "batch_5f8dac",
|
||||
"has_more": False,
|
||||
},
|
||||
)
|
||||
|
||||
batches: Final = await _list_on_vllm_router()
|
||||
|
||||
assert [(batch.id, batch.object, batch.status) for batch in batches] == [("batch_5f8dac", "batch", ["pending"])]
|
||||
|
||||
|
||||
async def test_openai_batch_list_items_come_back_as_openai_sent_them(respx_mock: respx.MockRouter):
|
||||
respx_mock.get("http://openai.test/v1/batches").respond(
|
||||
200,
|
||||
json={
|
||||
"object": "list",
|
||||
"data": [
|
||||
{
|
||||
**_OPENAI_BATCH_JSON,
|
||||
"status": "completed",
|
||||
"usage": {
|
||||
"input_tokens": 10,
|
||||
"input_tokens_details": {"cached_tokens": 4},
|
||||
"output_tokens": 5,
|
||||
"output_tokens_details": {"reasoning_tokens": 2},
|
||||
"total_tokens": 15,
|
||||
},
|
||||
},
|
||||
{**_OPENAI_BATCH_JSON, "id": "batch_def456", "status": "queued"},
|
||||
],
|
||||
"first_id": "batch_abc123",
|
||||
"last_id": "batch_def456",
|
||||
"has_more": False,
|
||||
},
|
||||
)
|
||||
|
||||
page: Final = await litellm.alist_batches(
|
||||
custom_llm_provider="openai", api_key="sk-test", api_base="http://openai.test/v1"
|
||||
)
|
||||
|
||||
first, second = page.data
|
||||
assert first.usage is not None
|
||||
assert (first.usage.input_tokens_details.cached_tokens, first.usage.output_tokens_details.reasoning_tokens) == (4, 2)
|
||||
assert (second.id, second.object, second.status) == ("batch_def456", "batch", "queued")
|
||||
|
|
|
|||
|
|
@ -6,10 +6,11 @@ import copy
|
|||
import json
|
||||
import re
|
||||
import threading
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
|
||||
from concurrent.futures import wait
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
|
@ -23,6 +24,7 @@ import litellm
|
|||
from litellm.constants import AWS_SIGNING_MAX_THREADS
|
||||
from litellm.cost_calculator import get_response_cost_from_hidden_params
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded, refetch_model_cost_map
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers
|
||||
from litellm.llms.bedrock.base_aws_llm import AWS_SIGNING_EXECUTOR
|
||||
|
|
@ -3702,6 +3704,387 @@ async def test_a_model_info_object_instead_of_a_dict_still_reaches_the_provider(
|
|||
assert response.answers[0].model_dump(mode="json")["name"] == "is_defect"
|
||||
|
||||
|
||||
_AUDIO_CLIPS: Final = ("data:audio/wav;base64,YQ==",)
|
||||
_VIDEO_CLIPS: Final = ("data:video/mp4;base64,YQ==",)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_deployment_that_advertises_media_sends_it_for_a_model_the_cost_map_does_not_list(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
backend: Final = "openrouter/example/omni-preview"
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "hears",
|
||||
"litellm_params": {"model": backend, "api_key": "k"},
|
||||
"model_info": {"id": "media-hears", "supports_audio_input": True},
|
||||
},
|
||||
{
|
||||
"model_name": "hears-and-sees",
|
||||
"litellm_params": {"model": backend, "api_key": "k"},
|
||||
"model_info": {"id": "media-both", "supports_audio_input": True, "supports_video_input": True},
|
||||
},
|
||||
]
|
||||
)
|
||||
send: Final = partial(
|
||||
router.adecisions, input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['videos']")):
|
||||
await send(model="hears")
|
||||
assert not route.called
|
||||
|
||||
await send(model="hears-and-sees")
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_media_flags_on_a_deployment_whose_own_mode_is_not_decisions_never_send_clips(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
advertised: Final = {"supports_audio_input": True, "supports_video_input": True}
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": mode,
|
||||
"litellm_params": {"model": f"openrouter/example/omni-{mode}-preview", "api_key": "k"},
|
||||
"model_info": {"id": f"media-{mode}", "mode": mode, **advertised},
|
||||
}
|
||||
for mode in ("chat", "decisions")
|
||||
]
|
||||
)
|
||||
send: Final = partial(
|
||||
router.adecisions, input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio', 'videos']")):
|
||||
await send(model="chat")
|
||||
assert not route.called
|
||||
|
||||
await send(model="decisions")
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS))
|
||||
|
||||
|
||||
CatalogLoader: TypeAlias = Callable[[Mapping[str, Mapping[str, object]]], Awaitable[None]]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def load_catalog_with(monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None) -> Iterator[CatalogLoader]:
|
||||
async def load(rows: Mapping[str, Mapping[str, object]]) -> None:
|
||||
catalog: Final = {**litellm.get_model_cost_map(url=""), **rows}
|
||||
transport: Final = httpx.MockTransport(lambda _: httpx.Response(200, json=catalog))
|
||||
monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP")
|
||||
async with httpx.AsyncClient(transport=transport) as client:
|
||||
loaded: Final = await refetch_model_cost_map(url="https://catalog.example/prices.json", client=client)
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
assert isinstance(loaded, ModelCostMapReloaded)
|
||||
monkeypatch.setattr(litellm, "model_cost", loaded.model_cost_map)
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
yield load
|
||||
litellm.get_model_cost_map(url="")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_media_flags_on_a_deployment_without_its_own_mode_follow_the_mode_the_catalog_lists(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, load_catalog_with: CatalogLoader
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
await load_catalog_with(
|
||||
{
|
||||
f"openrouter/example/omni-{mode}": {"litellm_provider": "openrouter", "mode": mode}
|
||||
for mode in ("chat", "decisions")
|
||||
}
|
||||
)
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": mode,
|
||||
"litellm_params": {"model": f"openrouter/example/omni-{mode}", "api_key": "k"},
|
||||
"model_info": {"id": f"listed-{mode}", "supports_audio_input": True, "supports_video_input": True},
|
||||
}
|
||||
for mode in ("chat", "decisions")
|
||||
]
|
||||
)
|
||||
send: Final = partial(
|
||||
router.adecisions, input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio', 'videos']")):
|
||||
await send(model="chat")
|
||||
assert not route.called
|
||||
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
await send(model="chat")
|
||||
dropped: Final = json.loads(route.calls.last.request.content)
|
||||
assert ("audio" in dropped, "videos" in dropped) == (False, False)
|
||||
|
||||
await send(model="decisions")
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sibling_mode", ("responses", "embedding"))
|
||||
async def test_another_deployments_mode_never_switches_off_a_deployments_own_media_flags(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, sibling_mode: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
shared_params: Final = {"model": "openrouter/example/omni-shared", "api_key": "k"}
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "sibling", "litellm_params": shared_params, "model_info": {"id": "s", "mode": sibling_mode}},
|
||||
{
|
||||
"model_name": "flagged",
|
||||
"litellm_params": shared_params,
|
||||
"model_info": {"id": "f", "supports_audio_input": True, "supports_video_input": True},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
await router.adecisions(
|
||||
model="flagged", input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS
|
||||
)
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS))
|
||||
|
||||
|
||||
_MEDIA_SURFACES: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType(
|
||||
{
|
||||
"adecisions": {"input": "review", "questions": _PREDICATE_QUESTIONS},
|
||||
"asystemone": {"state": "review", "questions": _QUESTIONS},
|
||||
"systemone": {"state": "review", "questions": _QUESTIONS},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _group_member(
|
||||
group: str, model: str, key: str, weight: int, model_info: Mapping[str, object], **params: object
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"model_name": group,
|
||||
"litellm_params": {"model": model, "api_key": key, "weight": weight, **params},
|
||||
"model_info": {"id": f"{group}-{key}", **model_info},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("surface", tuple(_MEDIA_SURFACES))
|
||||
async def test_a_group_sends_clips_to_a_deployment_that_takes_them_and_refuses_only_when_none_does(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, surface: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct")
|
||||
monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False)
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"cloudflare/@cf/cloudflare/clef-omni",
|
||||
{"litellm_provider": "cloudflare", "mode": "evaluation", "supports_audio_input": True},
|
||||
)
|
||||
respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
respx_mock.post("https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef-omni").respond(
|
||||
json=_RESPONSE
|
||||
)
|
||||
unlisted: Final = "openrouter/example/omni-preview"
|
||||
listed: Final = "cloudflare/clef-omni"
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
_group_member("unlisted", unlisted, "refuses", 1, {}),
|
||||
_group_member("unlisted", unlisted, "takes", 0, {"supports_audio_input": True}),
|
||||
_group_member("listed", listed, "refuses", 1, {"supports_audio_input": False}),
|
||||
_group_member("listed", listed, "takes", 0, {}),
|
||||
_group_member("none", unlisted, "refuses", 1, {}),
|
||||
_group_member("none", listed, "refuses", 1, {"supports_audio_input": False}),
|
||||
_group_member("drops", unlisted, "drops", 1, {}, drop_params=True),
|
||||
_group_member("drops", listed, "takes", 0, {}),
|
||||
_group_member("unresolvable", "databricks/not a serving endpoint", "unresolvable", 0, {}),
|
||||
_group_member("unresolvable", listed, "takes", 1, {}),
|
||||
]
|
||||
)
|
||||
call: Final = getattr(router, surface)
|
||||
|
||||
async def send(group: str, **media: object) -> object:
|
||||
response: Final = call(model=group, **_MEDIA_SURFACES[surface], **media)
|
||||
return await response if asyncio.iscoroutine(response) else response
|
||||
|
||||
for group in ("unlisted", "listed", "unresolvable"):
|
||||
await send(group, audio=_AUDIO_CLIPS)
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes"
|
||||
assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS)
|
||||
|
||||
await send("drops", audio=_AUDIO_CLIPS)
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer drops"
|
||||
assert "audio" not in json.loads(respx_mock.calls.last.request.content)
|
||||
|
||||
await send("unlisted")
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer refuses"
|
||||
|
||||
sent_before_refusal: Final = len(respx_mock.calls)
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio']")):
|
||||
await send("none", audio=_AUDIO_CLIPS)
|
||||
assert len(respx_mock.calls) == sent_before_refusal
|
||||
|
||||
|
||||
def _answer_unless_key_is(rejected_key: str) -> Callable[[httpx.Request], httpx.Response]:
|
||||
def answer(request: httpx.Request) -> httpx.Response:
|
||||
if request.headers["authorization"] == f"Bearer {rejected_key}":
|
||||
return httpx.Response(401, json={"error": {"message": "invalid key"}})
|
||||
return httpx.Response(200, json=_RESPONSE)
|
||||
|
||||
return answer
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clips_to_a_group_whose_only_taker_is_cooling_down_are_refused_rather_than_rate_limited(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").mock(
|
||||
side_effect=_answer_unless_key_is("takes")
|
||||
)
|
||||
unlisted: Final = "openrouter/example/omni-preview"
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
_group_member("g", unlisted, "refuses", 1, {}),
|
||||
_group_member("g", unlisted, "takes", 1, {"supports_audio_input": True}),
|
||||
],
|
||||
num_retries=0,
|
||||
cooldown_time=120,
|
||||
)
|
||||
with pytest.raises(litellm.AuthenticationError):
|
||||
await router.adecisions(model="g-takes", input="review", questions=_PREDICATE_QUESTIONS)
|
||||
sent_before_clips: Final = len(route.calls)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio']")):
|
||||
await router.adecisions(model="g", input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS)
|
||||
|
||||
assert len(route.calls) == sent_before_clips
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("surface", tuple(_MEDIA_SURFACES))
|
||||
async def test_a_router_default_drop_params_overrides_a_deployment_when_choosing_who_takes_clips(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, surface: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
unlisted: Final = "openrouter/example/omni-preview"
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
_group_member("g", unlisted, "drops", 1, {}, drop_params=True),
|
||||
_group_member("g", unlisted, "takes", 0, {"supports_audio_input": True}),
|
||||
],
|
||||
default_litellm_params={"drop_params": False},
|
||||
)
|
||||
call: Final = getattr(router, surface)
|
||||
|
||||
async def send(**params: object) -> object:
|
||||
response: Final = call(model="g", **_MEDIA_SURFACES[surface], audio=_AUDIO_CLIPS, **params)
|
||||
return await response if asyncio.iscoroutine(response) else response
|
||||
|
||||
await send()
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes"
|
||||
assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS)
|
||||
|
||||
await send(drop_params=True)
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer drops"
|
||||
assert "audio" not in json.loads(respx_mock.calls.last.request.content)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_deployment_that_turns_audio_off_is_refused_or_dropped_even_when_the_cost_map_allows_it(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"typesafe/jev-omni",
|
||||
{
|
||||
"litellm_provider": "typesafe",
|
||||
"mode": "decisions",
|
||||
"supports_audio_input": True,
|
||||
"supports_video_input": True,
|
||||
},
|
||||
)
|
||||
route: Final = respx_mock.post("https://example.com/v1/systemone").respond(json=_RESPONSE)
|
||||
send: Final = partial(
|
||||
litellm.asystemone,
|
||||
model="typesafe/jev-omni",
|
||||
state="test",
|
||||
questions=_QUESTIONS,
|
||||
api_base="https://example.com",
|
||||
api_key="test",
|
||||
audio=_AUDIO_CLIPS,
|
||||
videos=_VIDEO_CLIPS,
|
||||
model_info={"supports_audio_input": False},
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("does not support parameters: ['audio']")):
|
||||
await send()
|
||||
assert not route.called
|
||||
|
||||
await send(drop_params=True)
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert "audio" not in sent
|
||||
assert sent["videos"] == list(_VIDEO_CLIPS)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_media_flag_that_is_not_a_boolean_leaves_the_cost_map_in_charge_of_that_kind_only(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post("https://example.com/v1/systemone").respond(json=_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("does not support parameters: ['audio']")):
|
||||
await litellm.asystemone(
|
||||
model="typesafe/jev-unlisted",
|
||||
state="test",
|
||||
questions=_QUESTIONS,
|
||||
api_base="https://example.com",
|
||||
api_key="test",
|
||||
audio=_AUDIO_CLIPS,
|
||||
videos=_VIDEO_CLIPS,
|
||||
model_info={"supports_audio_input": "true", "supports_video_input": True},
|
||||
)
|
||||
|
||||
assert not route.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_openai_format_deployment_that_advertises_media_is_still_refused_it(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio', 'videos']")):
|
||||
await litellm.adecisions(
|
||||
model="openai/gpt-6-luna",
|
||||
input="review",
|
||||
questions=_PREDICATE_QUESTIONS,
|
||||
api_key="deployment-key",
|
||||
audio=_AUDIO_CLIPS,
|
||||
videos=_VIDEO_CLIPS,
|
||||
model_info={"supports_audio_input": True, "supports_video_input": True},
|
||||
)
|
||||
|
||||
assert not route.called
|
||||
|
||||
|
||||
def test_dropping_every_question_answers_400_before_http(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from collections.abc import Awaitable, Callable
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
|
@ -124,6 +125,61 @@ async def test_afile_retrieve_rejects_a_provider_file_without_its_size():
|
|||
assert [error["loc"] for error in exc_info.value.errors()] == [("bytes",)]
|
||||
|
||||
|
||||
VLLM_ROUTER_API_BASE: Final = "http://vllm-router.test/v1"
|
||||
VLLM_ROUTER_FILE: Final = MappingProxyType(
|
||||
{"id": "file-b73f4c", "bytes": 156, "created_at": 1793238707, "filename": "batch_requests.jsonl", "purpose": "batch"}
|
||||
)
|
||||
|
||||
|
||||
async def _upload_to_vllm_router() -> OpenAIFileObject:
|
||||
return await litellm.acreate_file(
|
||||
file=("batch_requests.jsonl", b'{"custom_id": "request-1"}'),
|
||||
purpose="batch",
|
||||
custom_llm_provider="hosted_vllm",
|
||||
api_key="sk-test",
|
||||
api_base=VLLM_ROUTER_API_BASE,
|
||||
)
|
||||
|
||||
|
||||
async def _retrieve_from_vllm_router() -> OpenAIFileObject:
|
||||
return await litellm.afile_retrieve(
|
||||
file_id="file-b73f4c", custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE
|
||||
)
|
||||
|
||||
|
||||
VLLM_ROUTER_FILE_CALLS: Final = (
|
||||
pytest.param("POST", "/files", _upload_to_vllm_router, id="acreate_file"),
|
||||
pytest.param("GET", "/files/file-b73f4c", _retrieve_from_vllm_router, id="afile_retrieve"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_FILE_CALLS)
|
||||
@respx.mock
|
||||
async def test_hosted_vllm_file_without_an_object_type_comes_back_as_a_file(
|
||||
method: str, path: str, call_router: Callable[[], Awaitable[OpenAIFileObject]]
|
||||
):
|
||||
respx.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond(200, json=dict(VLLM_ROUTER_FILE))
|
||||
|
||||
file_object: Final = await call_router()
|
||||
|
||||
assert file_object.model_dump(exclude_none=True) == {**VLLM_ROUTER_FILE, "object": "file"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_FILE_CALLS)
|
||||
@respx.mock
|
||||
async def test_hosted_vllm_file_with_a_different_object_type_is_rejected(
|
||||
method: str, path: str, call_router: Callable[[], Awaitable[OpenAIFileObject]]
|
||||
):
|
||||
respx.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond(
|
||||
200, json={**VLLM_ROUTER_FILE, "object": "batch"}
|
||||
)
|
||||
|
||||
with pytest.raises(ValidationError) as exc_info:
|
||||
await call_router()
|
||||
|
||||
assert [error["loc"] for error in exc_info.value.errors()] == [("object",)]
|
||||
|
||||
|
||||
_FILE_BODY: Final = b'{"prompt": "Hello", "completion": "Hi"}'
|
||||
_FINE_TUNE_FILE_JSON: Final = MappingProxyType(
|
||||
{
|
||||
|
|
|
|||
|
|
@ -3,10 +3,14 @@ async_management_endpoint_{success,failure}_hook integration points."""
|
|||
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi import Request
|
||||
|
||||
from litellm import Router
|
||||
from litellm.proxy._types import (
|
||||
ManagementEndpointLoggingPayload,
|
||||
UserAPIKeyAuth,
|
||||
|
|
@ -340,3 +344,203 @@ def test_management_wrapper_swallows_post_success_errors(server_span_factory, ot
|
|||
result = asyncio.run(fake_fn(data={}, user_api_key_dict=_real_user_api_key_dict(server_span)))
|
||||
|
||||
assert result == {"ok": True}
|
||||
|
||||
|
||||
TEAM_NEW_PATH: Final = "/team/new"
|
||||
MODEL_LIST_PATH: Final = "/v1/models"
|
||||
MODEL_RETRIEVE_PATH: Final = "/v1/models/gpt-4o"
|
||||
INBOUND_TRACEPARENT: Final = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"
|
||||
|
||||
|
||||
def _request(method: str, path: str, body: bytes = b"") -> Request:
|
||||
async def receive() -> dict[str, object]:
|
||||
return {"type": "http.request", "body": body, "more_body": False}
|
||||
|
||||
return Request(
|
||||
{
|
||||
"type": "http",
|
||||
"method": method,
|
||||
"path": path,
|
||||
"root_path": "",
|
||||
"query_string": b"",
|
||||
"headers": [(b"content-type", b"application/json")],
|
||||
},
|
||||
receive,
|
||||
)
|
||||
|
||||
|
||||
async def _answer_like_the_proxy(request: Request, exception: Exception) -> None:
|
||||
from litellm.proxy.proxy_server import app
|
||||
|
||||
handler: Final = next(
|
||||
app.exception_handlers[cls] for cls in type(exception).__mro__ if cls in app.exception_handlers
|
||||
)
|
||||
await handler(request, exception)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("wrapper_name", "exports_payload"),
|
||||
[("management_endpoint_wrapper", True), ("management_endpoint_span_wrapper", False)],
|
||||
)
|
||||
def test_span_wrapper_omits_payload(server_span_factory, otel_with_exporter, monkeypatch, wrapper_name, exports_payload):
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy.management_helpers import utils as mgmt_utils
|
||||
|
||||
otel, exporter = otel_with_exporter
|
||||
monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False)
|
||||
|
||||
server_span: Final = server_span_factory(TEAM_NEW_PATH)
|
||||
wrap: Final = getattr(mgmt_utils, wrapper_name)
|
||||
|
||||
@wrap
|
||||
async def fake_fn(http_request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]:
|
||||
return {"data": [{"id": "gpt-4o"}]}
|
||||
|
||||
asyncio.run(
|
||||
fake_fn(
|
||||
http_request=_request("POST", TEAM_NEW_PATH, b'{"team_alias": "t"}'),
|
||||
user_api_key_dict=_real_user_api_key_dict(server_span),
|
||||
)
|
||||
)
|
||||
|
||||
spans: Final = [s for s in exporter.get_finished_spans() if s.name == TEAM_NEW_PATH]
|
||||
assert len(spans) == 1
|
||||
attribute_keys: Final = tuple(spans[0].attributes)
|
||||
assert any(key.startswith("request.") for key in attribute_keys) is exports_payload
|
||||
assert any(key.startswith("response.") for key in attribute_keys) is exports_payload
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("wrapper_name", "exports_request_body"),
|
||||
[("management_endpoint_wrapper", True), ("management_endpoint_span_wrapper", False)],
|
||||
)
|
||||
def test_span_wrapper_omits_request_body_on_failure(
|
||||
server_span_factory, otel_with_exporter, monkeypatch, wrapper_name, exports_request_body
|
||||
):
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy.management_helpers import utils as mgmt_utils
|
||||
|
||||
otel, exporter = otel_with_exporter
|
||||
monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False)
|
||||
|
||||
server_span: Final = server_span_factory(TEAM_NEW_PATH)
|
||||
wrap: Final = getattr(mgmt_utils, wrapper_name)
|
||||
|
||||
@wrap
|
||||
async def fake_fn(http_request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]:
|
||||
raise make_fastapi_http_exception(400, "team_alias taken")
|
||||
|
||||
with pytest.raises(Exception, match="team_alias taken"):
|
||||
asyncio.run(
|
||||
fake_fn(
|
||||
http_request=_request("POST", TEAM_NEW_PATH, b'{"team_alias": "t"}'),
|
||||
user_api_key_dict=_real_user_api_key_dict(server_span),
|
||||
)
|
||||
)
|
||||
|
||||
spans: Final = [s for s in exporter.get_finished_spans() if s.name == TEAM_NEW_PATH]
|
||||
assert len(spans) == 1
|
||||
assert any(key.startswith("request.") for key in spans[0].attributes) is exports_request_body
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raised", "status_code"),
|
||||
[
|
||||
(RuntimeError("router settings lookup failed"), 500),
|
||||
(httpx.ConnectError("All connection attempts failed"), 503),
|
||||
(make_fastapi_http_exception(400, "Invalid scope parameter"), 400),
|
||||
],
|
||||
)
|
||||
def test_span_wrapper_failure_keeps_the_answered_status(otel_with_exporter, monkeypatch, raised, status_code):
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy.management_helpers import utils as mgmt_utils
|
||||
|
||||
otel, exporter = otel_with_exporter
|
||||
monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False)
|
||||
|
||||
server_span: Final = _server_span(otel, MODEL_LIST_PATH, headers={})
|
||||
request: Final = _request("GET", MODEL_LIST_PATH)
|
||||
request.state.parent_otel_span = server_span
|
||||
|
||||
@mgmt_utils.management_endpoint_span_wrapper
|
||||
async def failing_model_list(request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]:
|
||||
raise raised
|
||||
|
||||
async def call_and_answer() -> None:
|
||||
try:
|
||||
await failing_model_list(request=request, user_api_key_dict=_real_user_api_key_dict(server_span))
|
||||
except Exception as exc:
|
||||
await _answer_like_the_proxy(request, exc)
|
||||
|
||||
asyncio.run(call_and_answer())
|
||||
|
||||
assert_server_span_attrs(
|
||||
exporter,
|
||||
expected_status=status_code,
|
||||
expected_url_path=MODEL_LIST_PATH,
|
||||
where=f"model list {type(raised).__name__}",
|
||||
)
|
||||
exported: Final = get_server_span(exporter)
|
||||
assert exported is not None
|
||||
assert exported.attributes.get("error.code") == str(status_code)
|
||||
|
||||
|
||||
def _server_span(otel, path: str, headers: dict[str, str]):
|
||||
server_span: Final = otel.create_litellm_proxy_request_started_span(start_time=datetime.now(), headers=headers)
|
||||
otel.set_proxy_request_route_attributes(server_span, url_path=path, http_route=path)
|
||||
return server_span
|
||||
|
||||
|
||||
def _assert_ended_in_inbound_trace(exporter, path: str, handler_name: str) -> None:
|
||||
assert_server_span_attrs(exporter, expected_status=200, expected_url_path=path, where=f"{path} success")
|
||||
exported: Final = get_server_span(exporter)
|
||||
assert exported is not None
|
||||
assert exported.context.trace_id == int(INBOUND_TRACEPARENT.split("-")[1], 16)
|
||||
|
||||
endpoint_spans: Final = [s for s in exporter.get_finished_spans() if s.name == handler_name]
|
||||
assert len(endpoint_spans) == 1
|
||||
assert endpoint_spans[0].parent is not None
|
||||
assert endpoint_spans[0].parent.span_id == exported.context.span_id
|
||||
assert not any(key.startswith("response.") for key in endpoint_spans[0].attributes)
|
||||
|
||||
|
||||
def _install_proxy_state(monkeypatch, router: Router | None) -> None:
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", None)
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {})
|
||||
monkeypatch.setattr(proxy_server, "user_model", None)
|
||||
|
||||
|
||||
def test_model_list_ends_server_span_in_inbound_trace(otel_with_exporter, monkeypatch):
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
otel, exporter = otel_with_exporter
|
||||
monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False)
|
||||
_install_proxy_state(monkeypatch, None)
|
||||
server_span: Final = _server_span(otel, MODEL_LIST_PATH, headers={"traceparent": INBOUND_TRACEPARENT})
|
||||
|
||||
result: Final = asyncio.run(proxy_server.model_list(user_api_key_dict=_real_user_api_key_dict(server_span)))
|
||||
|
||||
assert result == {"data": [], "object": "list"}
|
||||
_assert_ended_in_inbound_trace(exporter, MODEL_LIST_PATH, "model_list")
|
||||
|
||||
|
||||
def test_model_retrieve_ends_server_span_in_inbound_trace(otel_with_exporter, monkeypatch):
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
otel, exporter = otel_with_exporter
|
||||
monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False)
|
||||
_install_proxy_state(
|
||||
monkeypatch,
|
||||
Router(model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-x"}}]),
|
||||
)
|
||||
server_span: Final = _server_span(otel, MODEL_RETRIEVE_PATH, headers={"traceparent": INBOUND_TRACEPARENT})
|
||||
|
||||
result: Final = asyncio.run(
|
||||
proxy_server.model_info(model_id="gpt-4o", user_api_key_dict=_real_user_api_key_dict(server_span))
|
||||
)
|
||||
|
||||
assert result["id"] == "gpt-4o"
|
||||
_assert_ended_in_inbound_trace(exporter, MODEL_RETRIEVE_PATH, "model_info")
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from collections.abc import Callable
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
|
@ -7,7 +8,7 @@ from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
|
|||
StandardBuiltInToolCostTracking,
|
||||
)
|
||||
from litellm.types.llms.openai import FileSearchTool, ResponsesAPIResponse, WebSearchOptions
|
||||
from litellm.types.utils import ModelResponse, StandardBuiltInToolsParams
|
||||
from litellm.types.utils import ModelResponse, ServerToolUse, StandardBuiltInToolsParams, Usage
|
||||
|
||||
|
||||
def test_web_search_cost_low():
|
||||
|
|
@ -1052,6 +1053,175 @@ def test_azure_ai_non_claude_web_search_keeps_generic_fallback() -> None:
|
|||
assert get_cost_for_web_search_request(custom_llm_provider="azure_ai", usage=usage, model_info=model_info) is None
|
||||
|
||||
|
||||
def _register_priced_deployment(
|
||||
model: str, search_cost_per_query: float | None, deployment_id: str | None = None
|
||||
) -> str:
|
||||
from litellm import Router
|
||||
|
||||
token_rates: Final = {"input_cost_per_token": 3e-6, "output_cost_per_token": 1.5e-5}
|
||||
search_rates: Final = (
|
||||
{}
|
||||
if search_cost_per_query is None
|
||||
else {
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_low": search_cost_per_query,
|
||||
"search_context_size_medium": search_cost_per_query,
|
||||
"search_context_size_high": search_cost_per_query,
|
||||
}
|
||||
}
|
||||
)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "priced-deployment",
|
||||
"litellm_params": {"model": model, "api_key": "k", "vertex_project": "p", "vertex_location": "global"},
|
||||
"model_info": {**token_rates, **search_rates, **({} if deployment_id is None else {"id": deployment_id})},
|
||||
}
|
||||
]
|
||||
)
|
||||
return router.model_list[0]["model_info"]["id"]
|
||||
|
||||
|
||||
def _claude_chat_response(
|
||||
web_search_requests: int, model: str = "claude-sonnet-5-5"
|
||||
) -> tuple[object, Usage | None]:
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
total_tokens=2252,
|
||||
server_tool_use=ServerToolUse(web_search_requests=web_search_requests),
|
||||
)
|
||||
return ModelResponse(model=model, usage=usage), usage
|
||||
|
||||
|
||||
def _claude_messages_response(
|
||||
web_search_requests: int, model: str = "claude-sonnet-5-5"
|
||||
) -> tuple[object, Usage | None]:
|
||||
return {
|
||||
"id": "msg_vertex_claude_web_search",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": model,
|
||||
"content": [{"type": "text", "text": "scripted answer"}],
|
||||
"usage": {
|
||||
"input_tokens": 1840,
|
||||
"output_tokens": 412,
|
||||
"server_tool_use": {"web_search_requests": web_search_requests},
|
||||
},
|
||||
}, None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("build_response", [_claude_chat_response, _claude_messages_response])
|
||||
@pytest.mark.parametrize(("web_search_requests", "search_cost_per_query"), [(2, 0.01), (3, 0.01), (2, 0.0)])
|
||||
def test_vertex_claude_deployment_priced_web_search_billed_per_request(
|
||||
local_model_cost_map: None,
|
||||
build_response: Callable[[int], tuple[object, Usage | None]],
|
||||
web_search_requests: int,
|
||||
search_cost_per_query: float,
|
||||
) -> None:
|
||||
deployment_id: Final = _register_priced_deployment("vertex_ai/claude-sonnet-5-5", search_cost_per_query)
|
||||
response_object, usage = build_response(web_search_requests)
|
||||
|
||||
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
|
||||
model=deployment_id,
|
||||
response_object=response_object,
|
||||
usage=usage,
|
||||
custom_llm_provider="vertex_ai",
|
||||
standard_built_in_tools_params=None,
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(web_search_requests * search_cost_per_query)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("build_response", [_claude_chat_response, _claude_messages_response])
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_model", "served_model", "custom_llm_provider"),
|
||||
# Google bills Claude web search on Sonnet 4.5 and Haiku 4.5 too:
|
||||
# https://cloud.google.com/model-garden/partner-models/pricing (read 2026-10-30)
|
||||
[
|
||||
("vertex_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "vertex_ai"),
|
||||
("vertex_ai/claude-sonnet-4-5@20250929", "claude-sonnet-4-5-20250929", "vertex_ai"),
|
||||
("vertex_ai/claude-haiku-4-5@20251001", "claude-haiku-4-5-20251001", "vertex_ai"),
|
||||
("anthropic/claude-sonnet-5-5", "claude-sonnet-5-5", "anthropic"),
|
||||
("azure_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "azure_ai"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("deployment_id", [None, "claude-token-priced"], ids=["hashed_id", "claude_in_id"])
|
||||
def test_claude_deployment_with_token_rates_only_bills_served_model_search_price(
|
||||
local_model_cost_map: None,
|
||||
build_response: Callable[[int, str], tuple[object, Usage | None]],
|
||||
deployment_model: str,
|
||||
served_model: str,
|
||||
custom_llm_provider: str,
|
||||
deployment_id: str | None,
|
||||
) -> None:
|
||||
served_search_pricing: Final = (
|
||||
litellm.get_model_info(model=deployment_model).get("search_context_cost_per_query") or {}
|
||||
)
|
||||
served_search_price: Final = served_search_pricing.get("search_context_size_medium") or 0.0
|
||||
registered_id: Final = _register_priced_deployment(deployment_model, None, deployment_id)
|
||||
response_object, usage = build_response(2, served_model)
|
||||
|
||||
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
|
||||
model=registered_id,
|
||||
response_object=response_object,
|
||||
usage=usage,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
standard_built_in_tools_params=None,
|
||||
)
|
||||
|
||||
assert served_search_price > 0
|
||||
assert cost == pytest.approx(2 * served_search_price)
|
||||
|
||||
|
||||
def test_vertex_claude_deployment_with_token_rates_only_and_unmapped_served_model_bills_no_search(
|
||||
local_model_cost_map: None,
|
||||
) -> None:
|
||||
deployment_id: Final = _register_priced_deployment("vertex_ai/claude-sonnet-5-5", None)
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
total_tokens=2252,
|
||||
server_tool_use=ServerToolUse(web_search_requests=2),
|
||||
)
|
||||
|
||||
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
|
||||
model=deployment_id,
|
||||
response_object=ModelResponse(model="claude-unmapped-preview", usage=usage),
|
||||
usage=usage,
|
||||
custom_llm_provider="vertex_ai",
|
||||
standard_built_in_tools_params=None,
|
||||
)
|
||||
|
||||
assert cost == 0.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("deployment_id", [None, "claude-code-gemini"], ids=["hashed_id", "claude_in_id"])
|
||||
def test_vertex_gemini_deployment_priced_web_search_stays_per_prompt(
|
||||
local_model_cost_map: None, deployment_id: str | None
|
||||
) -> None:
|
||||
search_cost_per_prompt: Final = 0.03
|
||||
registered_id: Final = _register_priced_deployment(
|
||||
"vertex_ai/gemini-2.5-flash", search_cost_per_prompt, deployment_id
|
||||
)
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
total_tokens=2252,
|
||||
server_tool_use=ServerToolUse(web_search_requests=2),
|
||||
)
|
||||
|
||||
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
|
||||
model=registered_id,
|
||||
response_object=ModelResponse(model="gemini-2.5-flash", usage=usage),
|
||||
usage=usage,
|
||||
custom_llm_provider="vertex_ai",
|
||||
standard_built_in_tools_params=None,
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(search_cost_per_prompt)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kwargs",
|
||||
[{"tools": 5}, {"web_search_options": None}, {"web_search_options": 5}],
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
get_semantic_cache_prompt_from_messages,
|
||||
get_str_from_messages,
|
||||
handle_any_messages_to_chat_completion_str_messages_conversion,
|
||||
handle_messages_with_content_list_to_str_conversion,
|
||||
hoist_images_from_tool_messages,
|
||||
is_encrypted_reasoning_block,
|
||||
merge_consecutive_system_messages,
|
||||
|
|
@ -2675,3 +2676,16 @@ def test_tpm_projection_preserves_non_media_identity_and_opaque_values() -> None
|
|||
projected, media = split_media_for_tpm(request)
|
||||
assert projected is request
|
||||
assert media == ()
|
||||
|
||||
|
||||
def test_content_list_flattening_returns_new_messages_and_leaves_the_callers_untouched() -> None:
|
||||
messages: Final = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "Hello"}, {"type": "text", "text": " world"}]},
|
||||
{"role": "assistant", "content": "Hi"},
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
flattened: Final = handle_messages_with_content_list_to_str_conversion(messages)
|
||||
|
||||
assert flattened == [{"role": "user", "content": "Hello world"}, {"role": "assistant", "content": "Hi"}]
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -3,7 +3,10 @@ Tests for litellm.litellm_core_utils.logging_utils — base64 truncation helpers
|
|||
"""
|
||||
|
||||
import asyncio, datetime, importlib, litellm, os, pytest_asyncio
|
||||
import sys
|
||||
import threading
|
||||
from itertools import chain
|
||||
from types import FrameType
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
|
@ -17,7 +20,7 @@ from litellm.litellm_core_utils.logging_utils import (
|
|||
truncate_base64_in_messages,
|
||||
truncate_base64_in_messages_async,
|
||||
)
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import AsyncIterator, Callable, Coroutine, Sequence
|
||||
from datetime import datetime as datetime_assemble_streaming
|
||||
from litellm import(
|
||||
Choices,
|
||||
|
|
@ -28,6 +31,7 @@ from litellm import(
|
|||
)
|
||||
from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -99,8 +103,8 @@ class TestTruncateBase64InString:
|
|||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
'data: {"choices": [{"delta": {"content": "hi"}}]}\n\n' * 50_000,
|
||||
"data:" * 200_000,
|
||||
'data: {"choices": [{"delta": {"content": "hi"}}]}\n\n' * 50_000 + ";base64,",
|
||||
"data:" * 200_000 + ";base64,",
|
||||
],
|
||||
ids=["sse_lines", "whitespace_free_prefixes"],
|
||||
)
|
||||
|
|
@ -241,6 +245,49 @@ def scan_threads(monkeypatch):
|
|||
return threads
|
||||
|
||||
|
||||
def _run_without_yielding(coroutine: Coroutine[object, object, object]) -> object:
|
||||
try:
|
||||
coroutine.send(None)
|
||||
except StopIteration as finished:
|
||||
return finished.value
|
||||
coroutine.close()
|
||||
raise AssertionError("the coroutine yielded to the event loop")
|
||||
|
||||
|
||||
def _regex_calls_while(run: Callable[[], object]) -> tuple[object, tuple[str, ...]]:
|
||||
calls: Final[list[str]] = [] # mutable-ok: sys.setprofile reports each call to a callback that has to record it
|
||||
previous: Final = sys.getprofile()
|
||||
|
||||
def record(frame: FrameType, event: str, arg: object) -> None:
|
||||
name: Final = getattr(arg, "__qualname__", "")
|
||||
if event == "c_call" and name.startswith("Pattern."):
|
||||
calls.append(name)
|
||||
|
||||
sys.setprofile(record)
|
||||
try:
|
||||
result: Final = run()
|
||||
finally:
|
||||
sys.setprofile(previous)
|
||||
return result, tuple(calls)
|
||||
|
||||
|
||||
def _tool_result_messages(tool_output: str) -> list[AllMessageValues]: # mutable-ok: the truncator only walks lists
|
||||
return [
|
||||
{"role": "user", "content": "Summarize the rows the tool returned."},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{"id": "call_1", "type": "function", "function": {"name": "dump_rows", "arguments": "{}"}}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": tool_output},
|
||||
]
|
||||
|
||||
|
||||
_PLAIN_TOOL_OUTPUT_CHARS: Final = 400 * 1024
|
||||
|
||||
|
||||
class TestTruncateBase64InMessagesAsync:
|
||||
@pytest.mark.asyncio
|
||||
async def test_large_payload_is_scanned_off_the_event_loop(self, monkeypatch, scan_threads):
|
||||
|
|
@ -268,6 +315,83 @@ class TestTruncateBase64InMessagesAsync:
|
|||
assert scan_threads
|
||||
assert set(scan_threads) == {threading.get_ident()}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"row",
|
||||
[
|
||||
'{"id": 7, "name": "widget", "status": "ok"}\n',
|
||||
"data: {\"choices\": [{\"delta\": {\"content\": \"hi\"}}]}\n\n",
|
||||
"const total = rows.map((r) => r.price).reduce((a, b) => a + b, 0);\n",
|
||||
'data: {"choices": [{"delta": {"content": "a; b"}}]}\n\n',
|
||||
"Content-Type: text/plain;base64,aGVsbG8=\n",
|
||||
],
|
||||
ids=["json_rows", "sse_lines", "code_with_semicolons", "sse_lines_with_semicolons", "base64_marker_without_data"],
|
||||
)
|
||||
def test_tool_message_without_data_uri_is_copied_without_a_regex_scan(self, row: str):
|
||||
tool_output: Final = (row * (_PLAIN_TOOL_OUTPUT_CHARS // len(row) + 1))[:_PLAIN_TOOL_OUTPUT_CHARS]
|
||||
messages: Final = _tool_result_messages(tool_output)
|
||||
|
||||
result, regex_calls = _regex_calls_while(lambda: truncate_base64_in_messages(messages))
|
||||
|
||||
assert result == messages
|
||||
assert result is not messages
|
||||
assert regex_calls == ()
|
||||
|
||||
def test_small_data_uri_is_truncated_inline_with_a_regex_scan(self):
|
||||
payload: Final = "L" * 2_000
|
||||
messages: Final = _image_messages(payload)
|
||||
|
||||
result, regex_calls = _regex_calls_while(
|
||||
lambda: _run_without_yielding(truncate_base64_in_messages_async(messages))
|
||||
)
|
||||
|
||||
assert regex_calls
|
||||
assert payload not in result[0]["content"][1]["image_url"]["url"]
|
||||
assert "base64_data truncated" in result[0]["content"][1]["image_url"]["url"]
|
||||
|
||||
def test_large_plain_text_is_copied_inline_without_rescanning_or_a_thread_hop(
|
||||
self, monkeypatch: pytest.MonkeyPatch, scan_threads: Sequence[int]
|
||||
):
|
||||
monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS", 1_000)
|
||||
messages: Final = _tool_result_messages("plain tool output\n" * 100)
|
||||
|
||||
result: Final = _run_without_yielding(truncate_base64_in_messages_async(messages))
|
||||
|
||||
assert result == messages
|
||||
assert result is not messages
|
||||
assert scan_threads == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_long_plain_text_history_is_copied_off_the_event_loop(
|
||||
self, monkeypatch: pytest.MonkeyPatch, scan_threads: Sequence[int]
|
||||
):
|
||||
monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS", 1_000)
|
||||
monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS", 10)
|
||||
messages: Final = list(chain.from_iterable(_tool_result_messages(f"rows {turn}\n" * 100) for turn in range(5)))
|
||||
|
||||
result: Final = await truncate_base64_in_messages_async(messages)
|
||||
offload_threads: Final = tuple(scan_threads)
|
||||
|
||||
assert result == messages
|
||||
assert result is not messages
|
||||
assert offload_threads
|
||||
assert threading.get_ident() not in offload_threads
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plain_text_over_the_inline_limit_is_copied_off_the_event_loop(
|
||||
self, monkeypatch: pytest.MonkeyPatch, scan_threads: Sequence[int]
|
||||
):
|
||||
monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS", 1_000)
|
||||
monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS", 10_000)
|
||||
messages: Final = _tool_result_messages('data: {"choices": [{"delta": {"content": "a; b"}}]}\n\n' * 400)
|
||||
|
||||
result: Final = await truncate_base64_in_messages_async(messages)
|
||||
offload_threads: Final = tuple(scan_threads)
|
||||
|
||||
assert result == messages
|
||||
assert result is not messages
|
||||
assert offload_threads
|
||||
assert threading.get_ident() not in offload_threads
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_none_and_disabled_truncation_short_circuit(self, monkeypatch, scan_threads):
|
||||
assert await truncate_base64_in_messages_async(None) is None
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import copy
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Iterator
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
|
@ -9849,3 +9850,186 @@ def test_transform_request_tool_config_point_without_tools_drops_the_key():
|
|||
assert "cache_control_injection_points" not in data
|
||||
assert "tools" not in data
|
||||
assert litellm_params["metadata"] == {}
|
||||
|
||||
|
||||
def _five_cache_hint_conversation() -> list[dict[str, object]]:
|
||||
return [
|
||||
{"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "assistant", "content": "answer 2"},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]},
|
||||
]
|
||||
|
||||
|
||||
def _cached_texts(node: object) -> Iterator[str]:
|
||||
if isinstance(node, dict):
|
||||
yield from ((node["text"],) if "cache_control" in node and "text" in node else ())
|
||||
yield from chain.from_iterable(map(_cached_texts, node.values()))
|
||||
elif isinstance(node, list):
|
||||
yield from chain.from_iterable(map(_cached_texts, node))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("config", "model"),
|
||||
[
|
||||
(AnthropicConfig(), "claude-haiku-5-5"),
|
||||
(VertexAIAnthropicConfig(), "claude-haiku-5-5"),
|
||||
(AmazonAnthropicClaudeConfig(), "anthropic.claude-haiku-5-5"),
|
||||
(AzureAnthropicConfig(), "claude-haiku-5-5"),
|
||||
],
|
||||
ids=["anthropic", "vertex_ai", "bedrock_invoke", "azure_ai"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("drop_params", "expected_cached_texts"),
|
||||
[
|
||||
(True, ["answer 1", "turn 1", "turn 2", "turn 3"]),
|
||||
("true", ["answer 1", "turn 1", "turn 2", "turn 3"]),
|
||||
(False, ["answer 1", "system prompt", "turn 1", "turn 2", "turn 3"]),
|
||||
],
|
||||
ids=[
|
||||
"drop_params_keeps_the_newest_four",
|
||||
"string_drop_params_keeps_the_newest_four",
|
||||
"without_drop_params_all_five_pass_through",
|
||||
],
|
||||
)
|
||||
def test_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params(
|
||||
config: AnthropicConfig,
|
||||
model: str,
|
||||
drop_params: bool | str,
|
||||
expected_cached_texts: list[str],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
messages: Final = _five_cache_hint_conversation()
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = config.transform_request(
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={"drop_params": drop_params},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert sorted(_cached_texts(body)) == expected_cached_texts
|
||||
assert json.dumps(body).count('"cache_control"') == len(expected_cached_texts)
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
def test_claude_cache_control_cap_keeps_a_cached_tool_and_drops_the_oldest_message_hint(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
messages: Final = _five_cache_hint_conversation()[1:]
|
||||
tool: Final = {
|
||||
"name": "get_weather",
|
||||
"description": "Get the weather",
|
||||
"input_schema": {"type": "object", "properties": {}},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
|
||||
body: Final = AnthropicConfig().transform_request(
|
||||
model="claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={"tools": [tool]},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert body["tools"][0]["cache_control"] == {"type": "ephemeral"}
|
||||
assert sorted(_cached_texts(body["messages"])) == ["answer 1", "turn 2", "turn 3"]
|
||||
|
||||
|
||||
def test_claude_cache_control_cap_drops_a_string_assistant_turns_text_hint_before_its_tool_call_hints(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
hint: Final = {"type": "ephemeral"}
|
||||
tool_calls: Final = [
|
||||
{"id": f"call_{name}", "type": "function", "function": {"name": name, "arguments": "{}"}, "cache_control": hint}
|
||||
for name in ("lookup", "search")
|
||||
]
|
||||
follow_up: Final = [
|
||||
{"type": "text", "text": "follow-up", "cache_control": hint},
|
||||
{"type": "text", "text": "and more", "cache_control": hint},
|
||||
]
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "question"},
|
||||
{"role": "assistant", "content": "checking both", "cache_control": hint, "tool_calls": tool_calls},
|
||||
{"role": "tool", "tool_call_id": "call_lookup", "content": "found"},
|
||||
{"role": "tool", "tool_call_id": "call_search", "content": "none"},
|
||||
{"role": "user", "content": follow_up},
|
||||
]
|
||||
|
||||
body: Final = AnthropicConfig().transform_request(
|
||||
model="claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assistant_blocks: Final = body["messages"][1]["content"]
|
||||
assert [block["type"] for block in assistant_blocks] == ["text", "tool_use", "tool_use"]
|
||||
assert ["cache_control" in block for block in assistant_blocks] == [False, True, True]
|
||||
assert json.dumps(body).count('"cache_control"') == 4
|
||||
assert json.dumps(body).count('"cache_control"') == 4
|
||||
|
||||
|
||||
def test_claude_cache_control_cap_keeps_the_newest_hints_inside_one_message(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
messages: Final = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
*({"type": "text", "text": f"part {n}", "cache_control": {"type": "ephemeral"}} for n in range(1, 6)),
|
||||
{"type": "text", "text": "Reply with the single word OK."},
|
||||
],
|
||||
}
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = AnthropicConfig().transform_request(
|
||||
model="claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert sorted(_cached_texts(body["messages"])) == ["part 2", "part 3", "part 4", "part 5"]
|
||||
assert json.dumps(body).count('"cache_control"') == 4
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
def test_claude_cache_control_cap_does_not_count_a_message_level_hint_that_list_content_never_sends(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
messages: Final = [
|
||||
{"role": "system", "content": "system prompt", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "user", "content": "turn 1", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "assistant", "content": "answer 1"},
|
||||
{"role": "user", "content": "turn 2", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "assistant", "content": "answer 2"},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}],
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
]
|
||||
|
||||
body: Final = AnthropicConfig().transform_request(
|
||||
model="claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={"drop_params": True},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert sorted(_cached_texts(body)) == ["system prompt", "turn 1", "turn 2", "turn 3"]
|
||||
assert json.dumps(body).count('"cache_control"') == 4
|
||||
|
|
|
|||
|
|
@ -56,6 +56,98 @@ def test_is_anthropic_messages_url(url: str, expected: bool) -> None:
|
|||
assert is_anthropic_messages_url(url) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("configured_base", "url", "expected"),
|
||||
[
|
||||
pytest.param(None, "https://api.anthropic.com/v1/messages", 512, id="default-upstream"),
|
||||
pytest.param(None, "https://api.anthropic.com:443/v1/messages/", 512, id="default-upstream-explicit-port"),
|
||||
pytest.param("http://relay.test:8080", "https://api.anthropic.com/v1/messages", 512, id="direct-target"),
|
||||
pytest.param("https://api.anthropic.com", "https://api.anthropic.com/v1/messages", 512, id="configured-default"),
|
||||
pytest.param("http://relay.test:8080", "http://relay.test:8080/v1/messages", 255, id="configured-upstream"),
|
||||
pytest.param("http://relay.test:8080/a", "http://relay.test:8080/a/v1/messages", 255, id="configured-prefix"),
|
||||
pytest.param("https://Relay.Test", "https://relay.test/v1/messages", 255, id="configured-upper-case-host"),
|
||||
pytest.param("https://relay.test:443", "https://relay.test/v1/messages", 255, id="configured-https-port"),
|
||||
pytest.param("http://relay.test:80/a", "http://relay.test/a/v1/messages", 255, id="configured-http-port"),
|
||||
pytest.param("http://relay.test:8080", "http://relay.test:8081/v1/messages", None, id="other-port"),
|
||||
pytest.param("http://relay.test:notaport", "http://relay.test/v1/messages", None, id="invalid-configured-base"),
|
||||
pytest.param(None, "https://other-vendor.test/v1/messages", None, id="other-vendor"),
|
||||
pytest.param(None, "https://api.anthropic.com/v1/messages/count_tokens", None, id="count-tokens"),
|
||||
],
|
||||
)
|
||||
def test_anthropic_upstream_user_id_limit(
|
||||
monkeypatch: pytest.MonkeyPatch, configured_base: str | None, url: str, expected: int | None
|
||||
) -> None:
|
||||
from litellm.llms.anthropic.common_utils import anthropic_upstream_user_id_limit
|
||||
|
||||
monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False)
|
||||
if configured_base is not None:
|
||||
monkeypatch.setenv("ANTHROPIC_API_BASE", configured_base)
|
||||
|
||||
assert anthropic_upstream_user_id_limit(url) == expected
|
||||
|
||||
|
||||
def test_anthropic_pass_through_api_base_prefers_the_api_base_variable(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.llms.anthropic.common_utils import anthropic_pass_through_api_base
|
||||
|
||||
monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://fallback.test")
|
||||
monkeypatch.setenv("ANTHROPIC_API_BASE", "http://preferred.test")
|
||||
assert anthropic_pass_through_api_base() == "http://preferred.test"
|
||||
|
||||
monkeypatch.delenv("ANTHROPIC_API_BASE")
|
||||
assert anthropic_pass_through_api_base() == "http://fallback.test"
|
||||
|
||||
|
||||
# Each row was sent to https://api.anthropic.com/v1/messages on 2026-10-30, False rows answered 400.
|
||||
@pytest.mark.parametrize(
|
||||
("user_id", "expected"),
|
||||
[
|
||||
pytest.param("user-1", True, id="opaque"),
|
||||
pytest.param("", True, id="empty"),
|
||||
pytest.param('{"device_id":"abc","session_id":"def"}', True, id="claude-code-shape"),
|
||||
pytest.param("1234567890", True, id="digits"),
|
||||
pytest.param("+14155550100", True, id="phone-shape"),
|
||||
pytest.param("a@b", True, id="no-dot"),
|
||||
pytest.param("@b.co", True, id="nothing-before-at-sign"),
|
||||
pytest.param("a@.co", True, id="nothing-between-at-sign-and-dot"),
|
||||
pytest.param("a@b.", True, id="nothing-after-dot"),
|
||||
pytest.param("a @ b.co", True, id="spaces-around-at-sign"),
|
||||
pytest.param("a\u00a0@b.co", False, id="no-break-space-before-at-sign"),
|
||||
pytest.param("a@b\n.co", False, id="newline-before-dot"),
|
||||
pytest.param("line one\nline two", False, id="newline"),
|
||||
pytest.param("a\tb", False, id="tab"),
|
||||
pytest.param("a\x1f@b.co", False, id="unit-separator-before-at-sign"),
|
||||
pytest.param("a@\x1cb.co", False, id="file-separator-after-at-sign"),
|
||||
pytest.param("a\u200bb", False, id="zero-width-space"),
|
||||
pytest.param("a@b.co", False, id="email"),
|
||||
pytest.param("user a@b.co here", False, id="email-inside-text"),
|
||||
pytest.param('{"device_id":"a@b.co"}', False, id="email-inside-json"),
|
||||
pytest.param("a@@example.com", False, id="doubled-at-sign"),
|
||||
pytest.param("@@example.com", False, id="leading-doubled-at-sign"),
|
||||
pytest.param("a@b@c.co", False, id="two-at-signs"),
|
||||
pytest.param("a@b..co", False, id="doubled-dot"),
|
||||
pytest.param("a@.b.co", False, id="dot-after-at-sign"),
|
||||
pytest.param("a@b.c d", False, id="email-before-space"),
|
||||
],
|
||||
)
|
||||
def test_anthropic_accepts_user_id(user_id: str, expected: bool) -> None:
|
||||
from litellm.constants import ANTHROPIC_MAX_USER_ID_LENGTH
|
||||
from litellm.llms.anthropic.common_utils import anthropic_accepts_user_id
|
||||
|
||||
assert anthropic_accepts_user_id(user_id, ANTHROPIC_MAX_USER_ID_LENGTH) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("character", ["a", "\u00e9", "\U0001f600"], ids=["ascii", "two-bytes", "four-bytes"])
|
||||
def test_anthropic_accepts_user_id_counts_characters_up_to_the_limit(character: str) -> None:
|
||||
from litellm.constants import ANTHROPIC_MAX_USER_ID_LENGTH
|
||||
from litellm.llms.anthropic.common_utils import anthropic_accepts_user_id
|
||||
|
||||
assert anthropic_accepts_user_id(character * ANTHROPIC_MAX_USER_ID_LENGTH, ANTHROPIC_MAX_USER_ID_LENGTH) is True
|
||||
assert (
|
||||
anthropic_accepts_user_id(character * (ANTHROPIC_MAX_USER_ID_LENGTH + 1), ANTHROPIC_MAX_USER_ID_LENGTH) is False
|
||||
)
|
||||
|
||||
|
||||
def test_anthropic_model_not_found_maps_to_not_found_but_authentication_error_does_not() -> None:
|
||||
with respx.mock(assert_all_called=True) as router:
|
||||
router.post("https://api.anthropic.com/v1/messages").mock(
|
||||
|
|
|
|||
|
|
@ -26,7 +26,6 @@ from litellm.types.decisions import (
|
|||
DecisionsIRScoreAnswer,
|
||||
DecisionsRequestBody,
|
||||
OpenAIDecisionRequestBody,
|
||||
UnsupportedDecisionsRequest,
|
||||
)
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
|
@ -279,7 +278,7 @@ def test_a_malformed_systemone_response_names_the_field_at_fault(
|
|||
assert [item["loc"] for item in error.value.errors()] == [field]
|
||||
|
||||
|
||||
def test_each_media_kind_is_gated_by_its_own_support_flag() -> None:
|
||||
def test_each_media_kind_is_sent_in_its_own_field() -> None:
|
||||
audio: Final = ["data:audio/wav;base64,YQ=="]
|
||||
videos: Final = ["data:video/mp4;base64,YQ=="]
|
||||
audio_only: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "audio": audio}))
|
||||
|
|
@ -287,22 +286,16 @@ def test_each_media_kind_is_gated_by_its_own_support_flag() -> None:
|
|||
_SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "videos": videos})
|
||||
)
|
||||
|
||||
assert ir_to_systemone_request("clef-omni", audio_only, supports_audio=True) == {
|
||||
assert ir_to_systemone_request("clef-omni", audio_only) == {
|
||||
"model": "clef-omni",
|
||||
**_SYSTEMONE_REQUEST,
|
||||
"audio": audio,
|
||||
}
|
||||
assert ir_to_systemone_request("clef-omni", video_only, supports_video=True) == {
|
||||
assert ir_to_systemone_request("clef-omni", video_only) == {
|
||||
"model": "clef-omni",
|
||||
**_SYSTEMONE_REQUEST,
|
||||
"videos": videos,
|
||||
}
|
||||
assert isinstance(
|
||||
ir_to_systemone_request("clef-omni", audio_only, supports_video=True), UnsupportedDecisionsRequest
|
||||
)
|
||||
assert isinstance(
|
||||
ir_to_systemone_request("clef-omni", video_only, supports_audio=True), UnsupportedDecisionsRequest
|
||||
)
|
||||
|
||||
|
||||
_ASSET_REQUEST: Final[Mapping[str, object]] = {
|
||||
|
|
|
|||
|
|
@ -11259,3 +11259,86 @@ def test_converse_sends_only_the_header_betas_when_header_and_body_both_carry_be
|
|||
"thinking-binding-controls-2026-08-01",
|
||||
"thinking-display-updates-2026-08-18",
|
||||
]
|
||||
|
||||
|
||||
def _five_cache_hint_conversation() -> list[dict[str, object]]:
|
||||
return [
|
||||
{"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "assistant", "content": "answer 2"},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("drop_params", "expected_system_cache_points"),
|
||||
[(True, 0), ("true", 0), (False, 1)],
|
||||
ids=[
|
||||
"drop_params_drops_the_oldest_hint",
|
||||
"string_drop_params_drops_the_oldest_hint",
|
||||
"without_drop_params_all_five_pass_through",
|
||||
],
|
||||
)
|
||||
def test_converse_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params(
|
||||
drop_params: bool | str, expected_system_cache_points: int, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
messages: Final = _five_cache_hint_conversation()
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
request: Final = AmazonConverseConfig().transform_request(
|
||||
model="anthropic.claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={"drop_params": drop_params},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert len(list(_cache_points(request["system"]))) == expected_system_cache_points
|
||||
assert [len(list(_cache_points(message))) for message in request["messages"]] == [1, 1, 1, 0, 1]
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
def test_converse_claude_tool_message_with_several_marked_blocks_counts_as_one_cache_point_against_the_cap(
|
||||
monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
messages: Final = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "policy", "cache_control": {"type": "ephemeral"}},
|
||||
{"type": "text", "text": "question", "cache_control": {"type": "ephemeral"}},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_1",
|
||||
"content": [
|
||||
{"type": "text", "text": "part 1", "cache_control": {"type": "ephemeral"}},
|
||||
{"type": "text", "text": "part 2", "cache_control": {"type": "ephemeral"}},
|
||||
{"type": "text", "text": "part 3", "cache_control": {"type": "ephemeral"}},
|
||||
],
|
||||
},
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
request: Final = AmazonConverseConfig().transform_request(
|
||||
model="anthropic.claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={
|
||||
"tools": [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}]
|
||||
},
|
||||
litellm_params={"drop_params": True},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert len(list(_cache_points(request["messages"]))) == 3
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
|
||||
import asyncio, importlib, litellm, litellm.litellm_core_utils.get_model_cost_map as bedrock_govcloud_model_cost_map, pytest
|
||||
import asyncio, importlib, litellm, pytest
|
||||
from typing import Final
|
||||
|
||||
|
||||
|
|
@ -1563,7 +1563,6 @@ def _pr4_bedrock_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||
def _use_local_model_cost_map() -> Iterator[None]:
|
||||
with pytest.MonkeyPatch.context() as monkeypatch:
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
importlib.reload(bedrock_govcloud_model_cost_map)
|
||||
importlib.reload(litellm)
|
||||
yield
|
||||
|
||||
|
|
|
|||
|
|
@ -144,6 +144,49 @@ async def test_client_payload_error_mid_stream_raises_read_error():
|
|||
assert mock_response.closed is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bytes_buffered_before_a_dropped_connection_are_yielded_before_the_read_error():
|
||||
loop: Final = asyncio.get_running_loop()
|
||||
content: Final = aiohttp.StreamReader(aiohttp.base_protocol.BaseProtocol(loop), 2**16, loop=loop)
|
||||
content.feed_data(b"event: message_start\n\n")
|
||||
content.set_exception(aiohttp.ClientPayloadError("Response payload is not completed"))
|
||||
mock_response: Final = MockAiohttpResponse()
|
||||
mock_response.content = content
|
||||
received: Final[list[bytes]] = []
|
||||
|
||||
async def _drain() -> None:
|
||||
async for chunk in AiohttpResponseStream(mock_response): # pyright: ignore[reportArgumentType] # mock response
|
||||
received.append(chunk)
|
||||
|
||||
with pytest.raises(httpx.ReadError):
|
||||
await _drain()
|
||||
|
||||
assert received == [b"event: message_start\n\n"]
|
||||
assert mock_response.closed is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bytes_already_read_before_a_dropped_connection_are_not_yielded_again():
|
||||
loop: Final = asyncio.get_running_loop()
|
||||
content: Final = aiohttp.StreamReader(aiohttp.base_protocol.BaseProtocol(loop), 2**16, loop=loop)
|
||||
content.feed_data(b"event: message_start\n\n")
|
||||
content.feed_data(b"event: ping\n\n")
|
||||
assert await content.read(7) == b"event: "
|
||||
content.set_exception(aiohttp.ClientPayloadError("Response payload is not completed"))
|
||||
mock_response: Final = MockAiohttpResponse()
|
||||
mock_response.content = content
|
||||
received: Final[list[bytes]] = []
|
||||
|
||||
async def _drain() -> None:
|
||||
async for chunk in AiohttpResponseStream(mock_response): # pyright: ignore[reportArgumentType] # mock response
|
||||
received.append(chunk)
|
||||
|
||||
with pytest.raises(httpx.ReadError):
|
||||
await _drain()
|
||||
|
||||
assert received == [b"message_start\n\nevent: ping\n\n"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_payload_error_before_first_chunk_raises_read_error():
|
||||
"""A connection reset before any body byte must surface, not yield an empty 200 body"""
|
||||
|
|
|
|||
|
|
@ -2773,3 +2773,66 @@ def test_claude_json_schema_stream_split_across_json_tools_keeps_the_non_stream_
|
|||
choices: Final = _json_schema_stream_choices(respx_mock, call, split_stream)
|
||||
|
||||
assert [choice.finish_reason for choice in choices if choice.finish_reason] == [answer.finish_reason]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "base_model"),
|
||||
[("databricks-claude-haiku-5-5", None), ("support-bot-endpoint", "databricks/databricks-claude-haiku-5-5")],
|
||||
ids=["claude_in_the_endpoint_name", "claude_named_only_by_base_model"],
|
||||
)
|
||||
def test_databricks_claude_request_over_the_cache_control_cap_keeps_the_newest_four_hints_under_drop_params(
|
||||
model: str, base_model: str | None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
hint: Final = {"type": "ephemeral"}
|
||||
messages: Final = [
|
||||
{"role": "system", "content": "system prompt", "cache_control": hint},
|
||||
{"role": "user", "content": "turn 1", "cache_control": hint},
|
||||
{"role": "assistant", "content": "answer 1", "cache_control": hint},
|
||||
{"role": "user", "content": "turn 2", "cache_control": hint},
|
||||
{"role": "assistant", "content": "answer 2"},
|
||||
{"role": "user", "content": "turn 3", "cache_control": hint},
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = DatabricksConfig().transform_request(
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={"drop_params": True, "custom_llm_provider": "databricks", "base_model": base_model},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert "cache_control" not in json.dumps(body["messages"][0])
|
||||
assert json.dumps(body["messages"]).count('"cache_control"') == 4
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"optional_params",
|
||||
[{"cache_control": {"type": "ephemeral"}}, {"extra_body": {"cache_control": {"type": "ephemeral"}}}],
|
||||
ids=["top_level_cache_control", "extra_body_cache_control"],
|
||||
)
|
||||
def test_databricks_claude_cache_control_cap_keeps_room_for_a_top_level_cache_control_under_drop_params(
|
||||
optional_params: dict[str, object], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
hint: Final = {"type": "ephemeral"}
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "turn 1", "cache_control": hint},
|
||||
{"role": "assistant", "content": "answer 1", "cache_control": hint},
|
||||
{"role": "user", "content": "turn 2", "cache_control": hint},
|
||||
{"role": "assistant", "content": "answer 2", "cache_control": hint},
|
||||
{"role": "user", "content": "turn 3"},
|
||||
]
|
||||
|
||||
body: Final = DatabricksConfig().transform_request(
|
||||
model="databricks-claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params=copy.deepcopy(optional_params),
|
||||
litellm_params={"drop_params": True, "custom_llm_provider": "databricks"},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert "cache_control" not in json.dumps(body["messages"][0])
|
||||
assert json.dumps(body["messages"]).count('"cache_control"') == 3
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -200,3 +202,38 @@ async def test_deepinfra_tool_message_content_transformation_async():
|
|||
print(f"✓ Async test passed: {tool_message['content']}")
|
||||
|
||||
print("\n✅ DeepInfra async tool message transformation test passed!")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"])
|
||||
async def test_deepinfra_request_stringifies_tool_content_without_touching_the_callers_message(is_async: bool) -> None:
|
||||
from litellm.llms.deepinfra.chat.transformation import DeepInfraConfig
|
||||
|
||||
config: Final = DeepInfraConfig()
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "Calculate 10 + 10"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{"id": "call_1", "type": "function", "function": {"name": "calculator", "arguments": "{}"}}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": [{"type": "text", "text": "20"}]},
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
request_kwargs: Final = {
|
||||
"model": "deepseek-ai/DeepSeek-V3.2",
|
||||
"messages": messages,
|
||||
"optional_params": {},
|
||||
"litellm_params": {},
|
||||
"headers": {},
|
||||
}
|
||||
|
||||
body: Final = (
|
||||
await config.async_transform_request(**request_kwargs)
|
||||
if is_async
|
||||
else config.transform_request(**request_kwargs)
|
||||
)
|
||||
|
||||
assert body["messages"][2]["content"] == "20"
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
|
|
@ -537,6 +538,26 @@ def test_tool_search_reply_sent_to_fireworks_drops_litellm_response_fields():
|
|||
}
|
||||
|
||||
|
||||
def test_cache_control_is_stripped_from_the_request_but_kept_on_the_callers_messages():
|
||||
hint: Final = {"type": "ephemeral"}
|
||||
messages: Final = [
|
||||
{"role": "system", "content": [{"type": "text", "text": "a long cached prompt", "cache_control": hint}]},
|
||||
{"role": "user", "content": "hi", "cache_control": hint},
|
||||
]
|
||||
caller_messages_before: Final = json.dumps(messages)
|
||||
|
||||
request: Final = FireworksAIConfig().transform_request(
|
||||
model="accounts/fireworks/models/gpt-oss-120b",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert "cache_control" not in json.dumps(request["messages"])
|
||||
assert json.dumps(messages) == caller_messages_before
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Regression tests for legacy / OpenAPI $ref defs in tool parameters.
|
||||
#
|
||||
|
|
@ -2477,3 +2498,43 @@ def test_translate_developer_role_follows_the_deployment_model_info_over_the_cos
|
|||
)
|
||||
|
||||
assert [message["role"] for message in messages] == expected_roles
|
||||
|
||||
|
||||
def test_fireworks_request_rewrites_tools_without_touching_the_callers_tool() -> None:
|
||||
tools: Final = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "create_thing",
|
||||
"strict": True,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"thing": {"$ref": "#/definitions/Thing"}},
|
||||
"definitions": {"Thing": {"type": "object", "properties": {"id": {"type": "string"}}}},
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
tools_before: Final = copy.deepcopy(tools)
|
||||
|
||||
request: Final = FireworksAIConfig().transform_request(
|
||||
model="accounts/fireworks/models/gpt-oss-120b",
|
||||
messages=[{"role": "user", "content": "Make a thing"}],
|
||||
optional_params={"tools": tools},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request["tools"] == [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "create_thing",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"thing": {"type": "object", "properties": {"id": {"type": "string"}}}},
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
assert tools == tools_before
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import logging
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
|
@ -589,3 +590,19 @@ async def test_streamed_reasoning_reply_can_be_replayed(is_async: bool, with_too
|
|||
}
|
||||
]
|
||||
assert reply.reasoning_content == "I should check the weather."
|
||||
|
||||
|
||||
def test_groq_request_drops_null_assistant_fields_without_touching_the_callers_messages() -> None:
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "Hi"},
|
||||
{"role": "assistant", "content": "Hello", "function_call": None, "tool_calls": None},
|
||||
{"role": "user", "content": "Again"},
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = GroqChatConfig().transform_request(
|
||||
model="openai/gpt-oss-120b", messages=messages, optional_params={}, litellm_params={}, headers={}
|
||||
)
|
||||
|
||||
assert body["messages"][1] == {"role": "assistant", "content": "Hello"}
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
import copy
|
||||
import json
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -397,3 +399,43 @@ def test_hosted_vllm_custom_tools_use_top_level_input_schema():
|
|||
assert tools[0]["function"]["name"] == "search"
|
||||
assert tools[0]["function"]["description"] == "Search docs"
|
||||
assert tools[0]["function"]["parameters"] == input_schema
|
||||
|
||||
|
||||
def _strict_weather_tool() -> dict[str, object]:
|
||||
return {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"strict": True,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_hosted_vllm_tools_are_rewritten_without_touching_the_callers_tool() -> None:
|
||||
tools: Final = [_strict_weather_tool()]
|
||||
tools_before: Final = copy.deepcopy(tools)
|
||||
|
||||
optional_params: Final = HostedVLLMChatConfig().map_openai_params(
|
||||
non_default_params={"tools": tools}, optional_params={}, model="gpt-oss-120b", drop_params=False
|
||||
)
|
||||
|
||||
assert optional_params["tools"] == [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
assert tools == tools_before
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Mapping
|
||||
|
|
@ -1212,7 +1213,10 @@ def test_mistral_sends_only_the_text_items_of_a_native_thinking_chunk_beside_an_
|
|||
"content": [
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": [{"type": "text", "text": "It has whiskers."}, {"type": "reference", "reference_ids": [1]}],
|
||||
"thinking": [
|
||||
{"type": "text", "text": "It has whiskers."},
|
||||
{"type": "reference", "reference_ids": [1]},
|
||||
],
|
||||
},
|
||||
{"type": "text", "text": "A cat."},
|
||||
],
|
||||
|
|
@ -1672,6 +1676,18 @@ def test_mistral_transform_request_drops_an_empty_tool_message_whose_call_id_is_
|
|||
assert [m.get("role") for m in request["messages"]] == ["user", "assistant"]
|
||||
|
||||
|
||||
def test_mistral_request_drops_a_user_name_without_touching_the_callers_message() -> None:
|
||||
messages: Final = [{"role": "user", "content": "Hi"}, {"role": "user", "content": "", "name": "alice"}]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = MistralConfig().transform_request(
|
||||
model="mistral-medium-latest", messages=messages, optional_params={}, litellm_params={}, headers={}
|
||||
)
|
||||
|
||||
assert "name" not in json.dumps(body["messages"])
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage()
|
||||
|
|
|
|||
|
|
@ -1824,6 +1824,94 @@ class TestToolSchemaCombinatorFlatteningForOpenAI:
|
|||
assert request["tools"][0] is tool
|
||||
|
||||
|
||||
class TestCacheControlStrippingDoesNotMutateCallerInput:
|
||||
"""
|
||||
Stripping cache_control for a provider that cannot use it must not reach back
|
||||
into the caller's own message and tool objects.
|
||||
|
||||
filter_value_from_dict deletes the key in place and recurses into nested dicts
|
||||
and lists, and remove_cache_control_flag_from_messages_and_tools assigned the
|
||||
result back into the caller's list, so one call to any OpenAI-compatible
|
||||
provider permanently stripped cache_control from a list the caller still holds.
|
||||
Reusing that list on Anthropic or Bedrock afterwards then silently lost prompt
|
||||
caching, with no error and full-price billing.
|
||||
"""
|
||||
|
||||
def setup_method(self):
|
||||
self.config = OpenAIGPTConfig()
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_openai_base_env(self, monkeypatch):
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", None, raising=False)
|
||||
|
||||
@staticmethod
|
||||
def _messages():
|
||||
return [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "a long cached system prompt",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hello",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _tools():
|
||||
return [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "parameters": {"type": "object"}},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
]
|
||||
|
||||
def _transform(self, messages, tools):
|
||||
return self.config.transform_request(
|
||||
model="gpt-4o",
|
||||
messages=messages,
|
||||
optional_params={"tools": tools},
|
||||
litellm_params={"custom_llm_provider": "openai", "api_base": None},
|
||||
headers={},
|
||||
)
|
||||
|
||||
def test_caller_messages_and_tools_keep_cache_control(self):
|
||||
messages, tools = self._messages(), self._tools()
|
||||
messages_before, tools_before = copy.deepcopy(messages), copy.deepcopy(tools)
|
||||
|
||||
request = self._transform(messages, tools)
|
||||
|
||||
assert "cache_control" not in json.dumps(request)
|
||||
assert messages == messages_before
|
||||
assert tools == tools_before
|
||||
|
||||
def test_a_later_anthropic_call_still_sees_cache_control(self):
|
||||
"""The user-visible consequence: the same message list reused on a provider
|
||||
that does support caching must still carry the cache breakpoints."""
|
||||
messages = self._messages()
|
||||
|
||||
self._transform(messages, self._tools())
|
||||
|
||||
anthropic_body = litellm.AnthropicConfig().transform_request(
|
||||
model="claude-3-5-sonnet-20240620",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
assert "cache_control" in json.dumps(anthropic_body)
|
||||
|
||||
|
||||
def _developer_message_after_a_user_turn() -> list[dict[str, str]]:
|
||||
return [
|
||||
{"role": "system", "content": "You are terse."},
|
||||
|
|
@ -1916,3 +2004,53 @@ def test_completion_leaves_the_callers_closing_developer_message_unchanged_on_a_
|
|||
)
|
||||
assert json.loads(chat.calls.last.request.content)["messages"][-1]["role"] == "user"
|
||||
assert messages == sent
|
||||
|
||||
|
||||
def _user_turn_with_litellm_only_media_fields() -> list[dict[str, object]]:
|
||||
return [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe these"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png", "format": "image/png"}},
|
||||
{"type": "image_url", "image_url": "https://example.com/dog.png"},
|
||||
{
|
||||
"type": "file",
|
||||
"file": {"file_data": "data:application/pdf;base64,JVBERi0=", "format": "application/pdf"},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"])
|
||||
async def test_openai_request_rewrites_media_parts_without_touching_the_callers_message(
|
||||
is_async: bool, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", None, raising=False)
|
||||
config: Final = OpenAIGPTConfig()
|
||||
messages: Final = _user_turn_with_litellm_only_media_fields()
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
request_kwargs: Final = {
|
||||
"model": "gpt-5.5",
|
||||
"messages": messages,
|
||||
"optional_params": {},
|
||||
"litellm_params": {"custom_llm_provider": "openai", "api_base": None},
|
||||
"headers": {},
|
||||
}
|
||||
|
||||
body: Final = (
|
||||
await config.async_transform_request(**request_kwargs)
|
||||
if is_async
|
||||
else config.transform_request(**request_kwargs)
|
||||
)
|
||||
|
||||
assert body["messages"][0]["content"] == [
|
||||
{"type": "text", "text": "Describe these"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/dog.png"}},
|
||||
{"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0=", "filename": "my_file.pdf"}},
|
||||
]
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Callable, Mapping
|
||||
|
|
@ -541,6 +542,50 @@ class TestOpenAIResponsesAPIConfig:
|
|||
assert "cache_control" not in result["tools"][0]
|
||||
assert result["tools"][0]["name"] == "get_weather"
|
||||
|
||||
def test_transform_does_not_strip_cache_control_from_the_callers_input(self):
|
||||
"""Stripping for OpenAI must not reach back into the caller's own objects.
|
||||
|
||||
`filter_value_from_dict` deletes the key in place and recurses, and
|
||||
`_validate_input_param` passes plain dict items through by reference, so
|
||||
the caller's input list used to lose its cache breakpoints. Reusing that
|
||||
list on Anthropic or Bedrock afterwards then silently lost prompt caching.
|
||||
Same defect as the Chat Completions path.
|
||||
"""
|
||||
input_with_cache_control = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": "a long cached system prompt",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
tools_with_cache_control = [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object"},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
]
|
||||
input_before = copy.deepcopy(input_with_cache_control)
|
||||
tools_before = copy.deepcopy(tools_with_cache_control)
|
||||
|
||||
result = self.config.transform_responses_api_request(
|
||||
model=self.model,
|
||||
input=input_with_cache_control,
|
||||
response_api_optional_request_params={"tools": tools_with_cache_control},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert "cache_control" not in json.dumps(result)
|
||||
assert input_with_cache_control == input_before
|
||||
assert tools_with_cache_control == tools_before
|
||||
|
||||
def test_transform_preserves_input_without_cache_control(self):
|
||||
"""Inputs without cache_control must pass through unmodified."""
|
||||
input_clean = [
|
||||
|
|
@ -4607,10 +4652,10 @@ def test_cache_control_still_marks_a_part_whose_own_marker_is_malformed_under_dr
|
|||
}
|
||||
|
||||
|
||||
def _caller_input_after_transform(model: str, marker: object) -> str:
|
||||
def _hinted_caller_input(marker: object) -> list[dict[str, object]]:
|
||||
hint: Final = {"type": "ephemeral"}
|
||||
marked: Final = {} if marker is None else {"prompt_cache_breakpoint": marker}
|
||||
input: Final = [
|
||||
return [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
|
|
@ -4626,6 +4671,10 @@ def _caller_input_after_transform(model: str, marker: object) -> str:
|
|||
"content": [{"type": "output_text", "text": "ok", "cache_control": hint, **marked}],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _caller_input_after_transform(model: str, marker: object) -> str:
|
||||
input: Final = _hinted_caller_input(marker)
|
||||
_CONFIGS[0].transform_responses_api_request(
|
||||
model=model,
|
||||
input=cast("ResponseInputParam", input), # cast-ok: plain dicts shaped like the SDK's input items
|
||||
|
|
@ -4647,6 +4696,15 @@ def test_a_marker_leaves_the_caller_request_that_fallbacks_and_logs_read_as_an_u
|
|||
assert _caller_input_after_transform(model, marker).replace(marker_json, "") == unmarked
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ("gpt-4.1", "gpt-6.1-sol"))
|
||||
@pytest.mark.parametrize("marker", (None, _VALID_MARKERS[0], "yes"), ids=("unmarked", "explicit", "malformed"))
|
||||
def test_the_caller_input_keeps_its_cache_control_hints_for_a_fallback_deployment(
|
||||
monkeypatch: pytest.MonkeyPatch, model: str, marker: object
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
assert _caller_input_after_transform(model, marker) == json.dumps(_hinted_caller_input(marker))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures(httpx_transport.__name__)
|
||||
async def test_aresponses_forwards_previous_response_id_to_openai() -> None:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -198,3 +199,23 @@ def test_translate_developer_role_keeps_developer_messages_untouched() -> None:
|
|||
)
|
||||
== messages
|
||||
)
|
||||
|
||||
|
||||
def test_o_series_request_turns_system_into_user_without_touching_the_callers_messages(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
model: Final = "o-series-test-model-without-system-role"
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
model,
|
||||
{"litellm_provider": "openai", "mode": "chat", "supports_system_messages": False},
|
||||
)
|
||||
messages: Final = [{"role": "system", "content": "Be terse"}, {"role": "user", "content": "Hi"}]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = OpenAIOSeriesConfig().transform_request(
|
||||
model=model, messages=messages, optional_params={}, litellm_params={}, headers={}
|
||||
)
|
||||
|
||||
assert [message["role"] for message in body["messages"]] == ["user", "user"]
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from litellm.llms.openai.openai import (
|
|||
AsyncAssistantEventHandler,
|
||||
AsyncCursorPage,
|
||||
MessageData,
|
||||
OpenAIBatchesAPI,
|
||||
OpenAIChatCompletion,
|
||||
OpenAIConfig,
|
||||
OpenAIMessage as Message,
|
||||
|
|
@ -2320,3 +2321,43 @@ def test_openai_config_hoists_a_later_developer_message_only_for_a_model_off_ope
|
|||
model_info=None,
|
||||
)
|
||||
assert [message["role"] for message in messages] == expected_roles
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_batch_list_given_an_async_client_returns_a_page_the_caller_can_await(
|
||||
http_backend: ModuleType,
|
||||
):
|
||||
def respond(request: _Request) -> httpx.Response | SDKResponse:
|
||||
assert str(request.url).startswith("http://vllm-router.test/v1/batches")
|
||||
return http_backend.Response(
|
||||
200,
|
||||
json={
|
||||
"object": "list",
|
||||
"data": [
|
||||
{
|
||||
"id": "batch_5f8dac",
|
||||
"status": "pending",
|
||||
"input_file_id": "file-b73f4c",
|
||||
"created_at": 1793239207,
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"completion_window": "24h",
|
||||
}
|
||||
],
|
||||
"first_id": "batch_5f8dac",
|
||||
"last_id": "batch_5f8dac",
|
||||
"has_more": False,
|
||||
},
|
||||
)
|
||||
|
||||
async with http_backend.AsyncClient(transport=http_backend.MockTransport(respond)) as http_client:
|
||||
page: Final = OpenAIBatchesAPI().list_batches(
|
||||
_is_async=False,
|
||||
api_key="sk-test",
|
||||
api_base="http://vllm-router.test/v1",
|
||||
timeout=5.0,
|
||||
max_retries=0,
|
||||
organization=None,
|
||||
client=AsyncOpenAI(api_key="sk-test", base_url="http://vllm-router.test/v1", http_client=http_client),
|
||||
)
|
||||
|
||||
assert [(batch.id, batch.status) for batch in (await page).data] == [("batch_5f8dac", "pending")]
|
||||
|
|
|
|||
706
tests/unit/llms/openai_like/test_hostinger_router_provider.py
Normal file
706
tests/unit/llms/openai_like/test_hostinger_router_provider.py
Normal file
|
|
@ -0,0 +1,706 @@
|
|||
import copy
|
||||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.caching.llm_caching_handler import LLMClientCache
|
||||
|
||||
|
||||
def test_hostinger_router_provider_resolution(monkeypatch: pytest.MonkeyPatch):
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
monkeypatch.setenv("HOSTINGER_ROUTER_API_KEY", "hostinger-test-key")
|
||||
|
||||
model, provider, api_key, api_base = get_llm_provider(
|
||||
model="hostinger_router/claude-sonnet-5",
|
||||
custom_llm_provider=None,
|
||||
api_base=None,
|
||||
api_key=None,
|
||||
)
|
||||
|
||||
assert model == "claude-sonnet-5"
|
||||
assert provider == "hostinger_router"
|
||||
assert api_key == "hostinger-test-key"
|
||||
assert api_base == "https://router.hostinger.com/v1"
|
||||
|
||||
|
||||
def test_hostinger_router_provider_keeps_explicit_credentials(monkeypatch: pytest.MonkeyPatch):
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
monkeypatch.setenv("HOSTINGER_ROUTER_API_KEY", "hostinger-env-key")
|
||||
|
||||
_, provider, api_key, api_base = get_llm_provider(
|
||||
model="hostinger_router/claude-sonnet-5",
|
||||
custom_llm_provider=None,
|
||||
api_base="https://router.internal.example/v1",
|
||||
api_key="hostinger-explicit-key",
|
||||
)
|
||||
|
||||
assert provider == "hostinger_router"
|
||||
assert api_key == "hostinger-explicit-key"
|
||||
assert api_base == "https://router.internal.example/v1"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["gpt-5-nano", "claude-sonnet-5"])
|
||||
def test_hostinger_router_api_base_keeps_unprefixed_model_provider(model: str):
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
_, provider_without_api_base, _, _ = get_llm_provider(model=model)
|
||||
_, provider_with_hostinger_api_base, _, _ = get_llm_provider(
|
||||
model=model,
|
||||
api_base="https://router.hostinger.com/v1",
|
||||
api_key="hostinger-test-key",
|
||||
)
|
||||
|
||||
assert provider_with_hostinger_api_base == provider_without_api_base
|
||||
|
||||
|
||||
HOSTINGER_ROUTER_MODELS = tuple(sorted(name for name in litellm.model_cost if name.startswith("hostinger_router/")))
|
||||
HOSTINGER_ROUTER_CHAT_MODELS = tuple(
|
||||
name for name in HOSTINGER_ROUTER_MODELS if litellm.get_model_info(name)["mode"] == "chat"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", HOSTINGER_ROUTER_CHAT_MODELS)
|
||||
def test_hostinger_router_model_cost_and_capabilities(model: str):
|
||||
from litellm.cost_calculator import cost_per_token
|
||||
|
||||
prompt_cost, completion_cost = cost_per_token(
|
||||
model=model,
|
||||
prompt_tokens=1_000_000,
|
||||
completion_tokens=1_000_000,
|
||||
custom_llm_provider="hostinger_router",
|
||||
)
|
||||
model_info = litellm.get_model_info(model)
|
||||
|
||||
assert prompt_cost == pytest.approx(model_info["input_cost_per_token"] * 1_000_000)
|
||||
assert completion_cost == pytest.approx(model_info["output_cost_per_token"] * 1_000_000)
|
||||
assert model_info["output_cost_per_token"] > 0
|
||||
assert model_info["max_tokens"] == model_info["max_output_tokens"] <= model_info["max_input_tokens"]
|
||||
assert model_info["litellm_provider"] == "hostinger_router"
|
||||
assert model_info["mode"] == "chat"
|
||||
if model_info.get("supports_prompt_caching"):
|
||||
assert 0 < model_info["cache_read_input_token_cost"] < model_info["input_cost_per_token"]
|
||||
capability_checks: Final = (
|
||||
("supports_function_calling", litellm.supports_function_calling),
|
||||
("supports_reasoning", litellm.supports_reasoning),
|
||||
("supports_response_schema", litellm.supports_response_schema),
|
||||
("supports_vision", litellm.supports_vision),
|
||||
)
|
||||
assert {name: check(model) for name, check in capability_checks} == {
|
||||
name: model_info.get(name) is True for name, _ in capability_checks
|
||||
}
|
||||
|
||||
|
||||
def test_hostinger_router_chat_completions_request():
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl_hostinger",
|
||||
"object": "chat.completion",
|
||||
"created": 1_789_550_000,
|
||||
"model": "claude-sonnet-5",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "Hello from Hostinger Router"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7},
|
||||
},
|
||||
)
|
||||
response: Final = litellm.completion(
|
||||
model="hostinger_router/claude-sonnet-5",
|
||||
messages=[{"role": "user", "content": "Say hello"}],
|
||||
api_key="hostinger-test-key",
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
body: Final = json.loads(request.content)
|
||||
assert route.call_count == 1
|
||||
assert request.headers["authorization"] == "Bearer hostinger-test-key"
|
||||
assert body["model"] == "claude-sonnet-5"
|
||||
assert response.choices[0].message.content == "Hello from Hostinger Router"
|
||||
|
||||
|
||||
def test_hostinger_router_chat_completions_streaming_request():
|
||||
chunks = [
|
||||
'data: {"id":"chatcmpl_hostinger","object":"chat.completion.chunk","created":1789550000,"model":"claude-sonnet-5","choices":[{"index":0,"delta":{"role":"assistant","content":"Hi"},"finish_reason":null}]}\n\n',
|
||||
'data: {"id":"chatcmpl_hostinger","object":"chat.completion.chunk","created":1789550000,"model":"claude-sonnet-5","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":4,"completion_tokens":1,"total_tokens":5}}\n\n',
|
||||
"data: [DONE]\n\n",
|
||||
]
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond(
|
||||
200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
content="".join(chunks).encode(),
|
||||
)
|
||||
response: Final = litellm.completion(
|
||||
model="hostinger_router/claude-sonnet-5",
|
||||
messages=[{"role": "user", "content": "Say hello"}],
|
||||
stream=True,
|
||||
api_key="hostinger-test-key",
|
||||
)
|
||||
collected = [chunk for chunk in response]
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
body: Final = json.loads(request.content)
|
||||
assert route.call_count == 1
|
||||
assert body["stream"] is True
|
||||
assert any(chunk.choices and chunk.choices[0].delta.content == "Hi" for chunk in collected)
|
||||
|
||||
|
||||
def test_hostinger_router_responses_request():
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/responses").respond(
|
||||
200,
|
||||
json={
|
||||
"id": "resp_hostinger",
|
||||
"object": "response",
|
||||
"created_at": 1_789_550_000,
|
||||
"model": "gpt-5.2",
|
||||
"status": "completed",
|
||||
"output": [
|
||||
{
|
||||
"id": "msg_hostinger",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": "Hello from Hostinger Router", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {"input_tokens": 4, "output_tokens": 3, "total_tokens": 7},
|
||||
},
|
||||
)
|
||||
response: Final = litellm.responses(
|
||||
model="hostinger_router/gpt-5.2",
|
||||
input="Say hello",
|
||||
api_key="hostinger-test-key",
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
body: Final = json.loads(request.content)
|
||||
assert route.call_count == 1
|
||||
assert str(request.url) == "https://router.hostinger.com/v1/responses"
|
||||
assert request.headers["authorization"] == "Bearer hostinger-test-key"
|
||||
assert body["model"] == "gpt-5.2"
|
||||
assert body["input"] == "Say hello"
|
||||
assert response.output[0].content[0].text == "Hello from Hostinger Router"
|
||||
|
||||
|
||||
EMBEDDING_RESPONSE: Final = {
|
||||
"object": "list",
|
||||
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}],
|
||||
"model": "text-embedding-3-small",
|
||||
"usage": {"prompt_tokens": 2, "total_tokens": 2},
|
||||
}
|
||||
|
||||
|
||||
def test_hostinger_router_embeddings_request():
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
response: Final = litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
body: Final = json.loads(request.content)
|
||||
assert route.call_count == 1
|
||||
assert request.headers["authorization"] == "Bearer hostinger-test-key"
|
||||
assert body["model"] == "text-embedding-3-small"
|
||||
assert response.data[0]["embedding"] == [0.1, 0.2, 0.3]
|
||||
|
||||
|
||||
def test_hostinger_router_embeddings_send_extra_headers_as_http_headers():
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
extra_headers={"X-Trace-Id": "trace-123"},
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
body: Final = json.loads(request.content)
|
||||
assert request.headers["x-trace-id"] == "trace-123"
|
||||
assert request.headers["authorization"] == "Bearer hostinger-test-key"
|
||||
assert request.headers["content-type"] == "application/json"
|
||||
assert set(body) == {"model", "input"}
|
||||
|
||||
|
||||
def test_hostinger_router_embeddings_extra_headers_replace_key_headers_of_any_case():
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
extra_headers={"authorization": "Bearer caller-key", "x-trace-id": "trace-1"},
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
assert request.headers.get_list("authorization") == ["Bearer caller-key"]
|
||||
assert request.headers.get_list("content-type") == ["application/json"]
|
||||
assert request.headers.get_list("x-trace-id") == ["trace-1"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("configured", "sent"), [(12345, "12345"), (1.5, "1.5")])
|
||||
def test_hostinger_router_embeddings_send_numeric_extra_headers_as_text(configured: int | float, sent: str):
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
extra_headers={"X-Org-Id": configured},
|
||||
)
|
||||
|
||||
assert route.calls.last.request.headers.get_list("x-org-id") == [sent]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("configured", [True, None, ["a", "b"]])
|
||||
def test_hostinger_router_embeddings_refuse_non_text_extra_headers(configured: object):
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match="X-Debug") as refused:
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
extra_headers={"X-Debug": configured},
|
||||
)
|
||||
|
||||
assert refused.value.status_code == 400
|
||||
assert route.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("configured", [True, None, ["a", "b"]])
|
||||
def test_hostinger_router_embeddings_drop_non_text_extra_headers_under_drop_params(configured: object):
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
extra_headers={"X-Debug": configured, "X-Trace-Id": "trace-1"},
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
assert "x-debug" not in request.headers
|
||||
assert request.headers.get_list("x-trace-id") == ["trace-1"]
|
||||
assert set(json.loads(request.content)) == {"model", "input"}
|
||||
|
||||
|
||||
TRANSPORT_EXTRA_HEADERS: Final = [
|
||||
("Host", "other.example"),
|
||||
("content-length", "1"),
|
||||
("Transfer-Encoding", "chunked"),
|
||||
("Connection", "close"),
|
||||
("Keep-Alive", "timeout=5"),
|
||||
("Proxy-Connection", "close"),
|
||||
("TE", "gzip"),
|
||||
("Trailer", "X-Checksum"),
|
||||
("Upgrade", "websocket"),
|
||||
("Expect", "100-continue"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("name", "configured"), TRANSPORT_EXTRA_HEADERS)
|
||||
def test_hostinger_router_embeddings_ignore_transport_extra_headers(name: str, configured: str):
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
extra_headers={name: configured, "X-Trace-Id": "trace-1"},
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
assert request.headers.get_list("host") == ["router.hostinger.com"]
|
||||
assert request.headers.get_list("content-length") == [str(len(request.content))]
|
||||
assert "transfer-encoding" not in request.headers
|
||||
assert configured not in request.headers.get_list(name)
|
||||
assert request.headers.get_list("x-trace-id") == ["trace-1"]
|
||||
|
||||
|
||||
MALFORMED_HEADER_NAMES: Final = ["Host:evil.example", "X Note", "X-Caf\u00e9"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", MALFORMED_HEADER_NAMES)
|
||||
def test_hostinger_router_embeddings_refuse_malformed_extra_header_names(name: str):
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
with pytest.raises(litellm.UnsupportedParamsError) as refused:
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
extra_headers={name: "x"},
|
||||
)
|
||||
|
||||
assert refused.value.status_code == 400
|
||||
assert repr(name) in refused.value.message
|
||||
assert route.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", MALFORMED_HEADER_NAMES)
|
||||
def test_hostinger_router_embeddings_drop_malformed_extra_header_names_under_drop_params(name: str):
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
extra_headers={name: "x", "X-Trace-Id": "trace-1"},
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
assert request.headers.get_list("host") == ["router.hostinger.com"]
|
||||
assert request.headers.get_list("x-trace-id") == ["trace-1"]
|
||||
assert name.lower() not in request.headers.keys()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("configured", ["a\r\nHost: evil.example", "line\nbreak", "caf\u00e9", "a\x7fb"])
|
||||
def test_hostinger_router_embeddings_drop_extra_header_values_http_cannot_carry(configured: str):
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
extra_headers={"x-litellm-user_api_key_alias": configured, "X-Trace-Id": "trace-1"},
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
assert request.headers.get_list("host") == ["router.hostinger.com"]
|
||||
assert request.headers.get_list("x-trace-id") == ["trace-1"]
|
||||
assert "x-litellm-user_api_key_alias" not in request.headers
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("configured", "sent"),
|
||||
[(" x ", "x"), ("\t", ""), (" a\tb ", "a\tb"), ("env-secret\n", "env-secret"), ("\r\n x \r\n", "x")],
|
||||
)
|
||||
def test_hostinger_router_embeddings_trim_whitespace_around_extra_header_values(configured: str, sent: str):
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
extra_headers={"X-Rig": configured},
|
||||
)
|
||||
|
||||
assert route.calls.last.request.headers.get_list("x-rig") == [sent]
|
||||
|
||||
|
||||
def test_hostinger_router_embeddings_extra_headers_stay_with_their_request():
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "shared",
|
||||
"litellm_params": {
|
||||
"model": "hostinger_router/text-embedding-3-small",
|
||||
"api_key": "hostinger-test-key",
|
||||
"headers": {"X-Org-Id": "org-1"},
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
router.embedding(
|
||||
model="shared", input=["first"], extra_headers={"Authorization": "Bearer caller-a", "X-Leak": "a"}
|
||||
)
|
||||
router.embedding(model="shared", input=["second"])
|
||||
|
||||
first: Final = route.calls[0].request
|
||||
second: Final = route.calls[1].request
|
||||
assert first.headers.get_list("x-leak") == ["a"]
|
||||
assert second.headers.get_list("authorization") == ["Bearer hostinger-test-key"]
|
||||
assert "x-leak" not in second.headers
|
||||
assert second.headers.get_list("x-org-id") == ["org-1"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("param", ["headers", "extra_headers"])
|
||||
def test_hostinger_router_embeddings_send_headers_given_as_name_value_pairs(param: str):
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
**{param: [["X-Trace-Id", "trace-1"]]},
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
assert request.headers.get_list("x-trace-id") == ["trace-1"]
|
||||
assert request.headers.get_list("authorization") == ["Bearer hostinger-test-key"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("param", ["headers", "extra_headers"])
|
||||
def test_hostinger_router_embeddings_refuse_headers_that_are_not_an_object(param: str):
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=rf"\b{param} must be a JSON object") as refused:
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
**{param: "X-Trace-Id: trace-1"},
|
||||
)
|
||||
|
||||
assert refused.value.status_code == 400
|
||||
assert route.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("param", ["headers", "extra_headers"])
|
||||
def test_hostinger_router_embeddings_drop_headers_that_are_not_an_object_under_drop_params(param: str):
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
drop_params=True,
|
||||
**{param: "X-Trace-Id: trace-1"},
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
assert "x-trace-id" not in request.headers
|
||||
assert request.headers.get_list("authorization") == ["Bearer hostinger-test-key"]
|
||||
assert set(json.loads(request.content)) == {"model", "input"}
|
||||
|
||||
|
||||
def test_hostinger_router_embeddings_never_borrow_another_providers_key(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.delenv("HOSTINGER_ROUTER_API_KEY", raising=False)
|
||||
monkeypatch.setenv("OPENAI_LIKE_API_KEY", "other-provider-key")
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "other-provider-key")
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
with pytest.raises(litellm.BadRequestError, match="Missing API Key"):
|
||||
litellm.embedding(model="hostinger_router/text-embedding-3-small", input=["Say hello"])
|
||||
|
||||
assert route.call_count == 0
|
||||
|
||||
|
||||
def test_json_provider_without_embeddings_endpoint_refuses_embeddings():
|
||||
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
||||
|
||||
slug: Final = next(
|
||||
slug
|
||||
for slug in JSONProviderRegistry.list_providers()
|
||||
if not JSONProviderRegistry.supports_embeddings(slug) and slug not in litellm.provider_list
|
||||
)
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
route: Final = upstream.route().respond(200, json=EMBEDDING_RESPONSE)
|
||||
with pytest.raises(litellm.BadRequestError):
|
||||
litellm.embedding(model=f"{slug}/text-embedding-3-small", input=["Say hello"], api_key="test-key")
|
||||
|
||||
assert route.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hostinger_router_anthropic_messages_request(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/messages").respond(
|
||||
200,
|
||||
json={
|
||||
"id": "msg_hostinger",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5",
|
||||
"content": [{"type": "text", "text": "Hello from Hostinger Router"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 4, "output_tokens": 3},
|
||||
},
|
||||
)
|
||||
response: Final = await litellm.anthropic.messages.acreate(
|
||||
model="hostinger_router/claude-sonnet-5",
|
||||
messages=[{"role": "user", "content": "Say hello"}],
|
||||
max_tokens=32,
|
||||
api_key="hostinger-test-key",
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
body: Final = json.loads(request.content)
|
||||
assert route.call_count == 1
|
||||
assert str(request.url) == "https://router.hostinger.com/v1/messages"
|
||||
assert request.headers["authorization"] == "Bearer hostinger-test-key"
|
||||
assert request.headers["anthropic-version"] == "2023-06-01"
|
||||
assert body["model"] == "claude-sonnet-5"
|
||||
assert body["messages"] == [{"role": "user", "content": "Say hello"}]
|
||||
assert response["content"][0]["text"] == "Hello from Hostinger Router"
|
||||
|
||||
|
||||
CHAT_COMPLETION_RESPONSE: Final = {
|
||||
"id": "chatcmpl_hostinger",
|
||||
"object": "chat.completion",
|
||||
"created": 1_789_550_000,
|
||||
"model": "gpt-5.2",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "Hello from Hostinger Router"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7},
|
||||
}
|
||||
UNSUPPORTED_ENDPOINT: Final = {"error": {"message": "Model does not support this endpoint."}}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model", ["gpt-5.2", "gpt-5-nano"])
|
||||
async def test_hostinger_router_messages_bridge_models_without_native_messages(
|
||||
model: str, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
native: Final = upstream.post("https://router.hostinger.com/v1/messages").respond(
|
||||
403, json=UNSUPPORTED_ENDPOINT
|
||||
)
|
||||
chat: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond(
|
||||
200, json=CHAT_COMPLETION_RESPONSE
|
||||
)
|
||||
response: Final = await litellm.anthropic.messages.acreate(
|
||||
model=f"hostinger_router/{model}",
|
||||
messages=[{"role": "user", "content": "Say hello"}],
|
||||
max_tokens=32,
|
||||
api_key="hostinger-test-key",
|
||||
)
|
||||
|
||||
assert native.call_count == 0
|
||||
assert chat.call_count == 1
|
||||
assert json.loads(chat.calls.last.request.content)["model"] == model
|
||||
assert response["content"][0]["text"] == "Hello from Hostinger Router"
|
||||
|
||||
|
||||
def test_hostinger_router_responses_bridge_models_without_native_responses():
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
native: Final = upstream.post("https://router.hostinger.com/v1/responses").respond(
|
||||
403, json=UNSUPPORTED_ENDPOINT
|
||||
)
|
||||
chat: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond(
|
||||
200, json=CHAT_COMPLETION_RESPONSE
|
||||
)
|
||||
response: Final = litellm.responses(
|
||||
model="hostinger_router/claude-sonnet-5",
|
||||
input="Say hello",
|
||||
api_key="hostinger-test-key",
|
||||
)
|
||||
|
||||
assert native.call_count == 0
|
||||
assert chat.call_count == 1
|
||||
assert json.loads(chat.calls.last.request.content)["model"] == "claude-sonnet-5"
|
||||
assert response.output[0].content[0].text == "Hello from Hostinger Router"
|
||||
|
||||
|
||||
ANTHROPIC_MESSAGE_RESPONSE: Final = {
|
||||
"id": "msg_hostinger",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-haiku-4-5",
|
||||
"content": [{"type": "text", "text": "Hello from Hostinger Router"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 4, "output_tokens": 3},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"sibling_model_info",
|
||||
[
|
||||
{"supported_endpoints": ["/v1/chat/completions", "/v1/messages", "/v1/responses"]},
|
||||
{"supported_endpoints": "/v1/messages"},
|
||||
],
|
||||
)
|
||||
async def test_hostinger_router_endpoint_opt_in_stays_with_its_own_deployment(
|
||||
sibling_model_info: dict[str, object], monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
deployment_params: Final = {"model": "hostinger_router/gpt-5-nano", "api_key": "hostinger-test-key"}
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "opted-in", "litellm_params": deployment_params, "model_info": sibling_model_info},
|
||||
{"model_name": "plain", "litellm_params": deployment_params},
|
||||
]
|
||||
)
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
native_messages: Final = upstream.post("https://router.hostinger.com/v1/messages").respond(
|
||||
403, json=UNSUPPORTED_ENDPOINT
|
||||
)
|
||||
native_responses: Final = upstream.post("https://router.hostinger.com/v1/responses").respond(
|
||||
403, json=UNSUPPORTED_ENDPOINT
|
||||
)
|
||||
chat: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond(
|
||||
200, json=CHAT_COMPLETION_RESPONSE
|
||||
)
|
||||
messages_response: Final = await router.aanthropic_messages(
|
||||
model="plain", messages=[{"role": "user", "content": "Say hello"}], max_tokens=32
|
||||
)
|
||||
responses_response: Final = await router.aresponses(model="plain", input="Say hello")
|
||||
|
||||
assert native_messages.call_count == 0
|
||||
assert native_responses.call_count == 0
|
||||
assert chat.call_count == 2
|
||||
assert messages_response["content"][0]["text"] == "Hello from Hostinger Router"
|
||||
assert responses_response.output[0].content[0].text == "Hello from Hostinger Router"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hostinger_router_deployment_cache_ttl_opt_in_keeps_one_hour_ttl(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "opted-in",
|
||||
"litellm_params": {
|
||||
"model": "hostinger_router/claude-haiku-4-5",
|
||||
"api_base": "https://router.hostinger.com/v1",
|
||||
"api_key": "hostinger-test-key",
|
||||
},
|
||||
"model_info": {"supported_endpoints": ["/v1/messages"], "cache_control_ttl": True},
|
||||
}
|
||||
]
|
||||
)
|
||||
one_hour_cache: Final = {"type": "ephemeral", "ttl": "1h"}
|
||||
|
||||
with respx.mock() as upstream:
|
||||
native: Final = upstream.post("https://router.hostinger.com/v1/messages").respond(
|
||||
200, json=ANTHROPIC_MESSAGE_RESPONSE
|
||||
)
|
||||
await router.aanthropic_messages(
|
||||
model="opted-in",
|
||||
messages=[
|
||||
{"role": "user", "content": [{"type": "text", "text": "Say hello", "cache_control": one_hour_cache}]}
|
||||
],
|
||||
max_tokens=32,
|
||||
)
|
||||
|
||||
body: Final = json.loads(native.calls.last.request.content)
|
||||
assert body["messages"][0]["content"][0]["cache_control"] == one_hour_cache
|
||||
|
|
@ -1,5 +1,8 @@
|
|||
|
||||
import copy
|
||||
import json
|
||||
from collections.abc import Iterator
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
|
|
@ -874,6 +877,58 @@ def test_openrouter_streaming_reasoning_is_preserved() -> None:
|
|||
assert assembled.choices[0].message.reasoning_content == "Consider the evidence. Apply the rule."
|
||||
|
||||
|
||||
def _five_message_level_cache_hints() -> list[dict[str, object]]:
|
||||
return [
|
||||
{"role": "system", "content": "system prompt", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "user", "content": "turn 1", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "assistant", "content": "answer 1", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "user", "content": "turn 2", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "assistant", "content": "answer 2"},
|
||||
{"role": "user", "content": "turn 3", "cache_control": {"type": "ephemeral"}},
|
||||
]
|
||||
|
||||
|
||||
def _cached_texts(node: object) -> Iterator[str]:
|
||||
if isinstance(node, dict):
|
||||
yield from ((node["text"],) if "cache_control" in node and "text" in node else ())
|
||||
yield from chain.from_iterable(map(_cached_texts, node.values()))
|
||||
elif isinstance(node, list):
|
||||
yield from chain.from_iterable(map(_cached_texts, node))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("drop_params", "expected_cached_texts"),
|
||||
[
|
||||
(True, ["answer 1", "turn 1", "turn 2", "turn 3"]),
|
||||
("true", ["answer 1", "turn 1", "turn 2", "turn 3"]),
|
||||
(False, ["answer 1", "system prompt", "turn 1", "turn 2", "turn 3"]),
|
||||
],
|
||||
ids=[
|
||||
"drop_params_keeps_the_newest_four",
|
||||
"string_drop_params_keeps_the_newest_four",
|
||||
"without_drop_params_all_five_pass_through",
|
||||
],
|
||||
)
|
||||
def test_openrouter_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params(
|
||||
drop_params: bool | str, expected_cached_texts: list[str], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
messages: Final = _five_message_level_cache_hints()
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = OpenrouterConfig().transform_request(
|
||||
model="anthropic/claude-haiku-5.5",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={"drop_params": drop_params, "custom_llm_provider": "openrouter"},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert sorted(_cached_texts(body["messages"])) == expected_cached_texts
|
||||
assert json.dumps(body).count('"cache_control"') == len(expected_cached_texts)
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
def _transform_openrouter_response(body: dict[str, object]) -> litellm.ModelResponse:
|
||||
return OpenrouterConfig().transform_response(
|
||||
model="openrouter/openai/gpt-4o-mini",
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import json
|
||||
from typing import Final, Literal
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -2507,3 +2508,46 @@ def test_check_and_create_cache_clamps_negative_creation_tokens() -> None:
|
|||
|
||||
assert cache_name == "cachedContents/created"
|
||||
assert logging_obj.vertex_ai_cache_creation_input_tokens == 0
|
||||
|
||||
|
||||
def _recording_gemini_client(calls: list[httpx.Request]) -> HTTPHandler:
|
||||
def route(request: httpx.Request) -> httpx.Response:
|
||||
calls.append(request)
|
||||
if request.url.path.endswith("/cachedContents"):
|
||||
return httpx.Response(200, json={} if request.method == "GET" else _created_cache_body(3000))
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"candidates": [{"content": {"role": "model", "parts": [{"text": "OK"}]}, "finishReason": "STOP"}],
|
||||
"usageMetadata": {"promptTokenCount": 3000, "candidatesTokenCount": 1, "totalTokenCount": 3001},
|
||||
},
|
||||
)
|
||||
|
||||
return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(route)))
|
||||
|
||||
|
||||
def test_a_plain_system_prompt_goes_into_the_gemini_cache_instead_of_beside_it(local_model_cost_map) -> None:
|
||||
calls: Final[list[httpx.Request]] = []
|
||||
litellm.completion(
|
||||
model="gemini/gemini-flash-latest",
|
||||
messages=[
|
||||
{"role": "system", "content": "You are terse."},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "plain system prefix " * 3000, "cache_control": {"type": "ephemeral"}},
|
||||
{"type": "text", "text": "Reply with OK only."},
|
||||
],
|
||||
},
|
||||
],
|
||||
api_key="test-key",
|
||||
client=_recording_gemini_client(calls),
|
||||
)
|
||||
|
||||
created: Final = next(
|
||||
json.loads(call.content) for call in calls if call.method == "POST" and call.url.path.endswith("/cachedContents")
|
||||
)
|
||||
generated: Final = next(json.loads(call.content) for call in calls if call.url.path.endswith(":generateContent"))
|
||||
assert created["system_instruction"] == {"parts": [{"text": "You are terse."}]}
|
||||
assert generated["cachedContent"] == "cachedContents/created"
|
||||
assert "system_instruction" not in generated
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
import copy
|
||||
import importlib
|
||||
import json
|
||||
from typing import Final, Optional, cast
|
||||
|
|
@ -10,6 +11,7 @@ import pytest
|
|||
import litellm
|
||||
from litellm import completion, embedding
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.llms.watsonx.chat.transformation import IBMWatsonXChatConfig
|
||||
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
|
||||
|
||||
|
||||
|
|
@ -542,3 +544,43 @@ def test_watsonx_deployment_space_id_embedding(monkeypatch, watsonx_embedding_ca
|
|||
|
||||
# nor space_id or project_id is required by wx.ai API when inferencing deployment
|
||||
assert "project_id" not in json_data and "space_id" not in json_data
|
||||
|
||||
|
||||
def _strict_weather_tool() -> dict[str, object]:
|
||||
return {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"strict": True,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_watsonx_chat_tools_are_rewritten_without_touching_the_callers_tool() -> None:
|
||||
tools: Final = [_strict_weather_tool()]
|
||||
tools_before: Final = copy.deepcopy(tools)
|
||||
|
||||
optional_params: Final = IBMWatsonXChatConfig().map_openai_params(
|
||||
non_default_params={"tools": tools}, optional_params={}, model="ibm/granite-4-h-small", drop_params=False
|
||||
)
|
||||
|
||||
assert optional_params["tools"] == [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
assert tools == tools_before
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import json
|
||||
import os
|
||||
from typing import Final
|
||||
|
|
@ -581,3 +582,44 @@ def test_xai_grok_4_frequency_penalty_not_supported(model):
|
|||
"""
|
||||
supported_params = XAIChatConfig().get_supported_openai_params(model=model)
|
||||
assert "frequency_penalty" not in supported_params
|
||||
|
||||
|
||||
def _strict_weather_tool() -> dict[str, object]:
|
||||
return {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"strict": True,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_xai_tools_are_rewritten_without_touching_the_callers_tool() -> None:
|
||||
tools: Final = [_strict_weather_tool()]
|
||||
tools_before: Final = copy.deepcopy(tools)
|
||||
|
||||
optional_params: Final = XAIChatConfig().map_openai_params(
|
||||
non_default_params={"tools": tools}, optional_params={}, model="grok-4.3", drop_params=False
|
||||
)
|
||||
|
||||
assert optional_params["tools"] == [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
assert tools == tools_before
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from typing import Final, Literal, NoReturn
|
|||
import pytest
|
||||
|
||||
from litellm.accounting import InboundUsageEvent, Metric, MetricName, Outcome
|
||||
from litellm.proxy.db.accounting_ledger import AccountingLedger
|
||||
from litellm.proxy.db.accounting_ledger import AccountingConflictError, AccountingLedger, UsageWrite
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
|
||||
|
|
@ -101,3 +101,18 @@ async def test_unknown_delivery_target_is_rejected_before_database_access(event:
|
|||
("unknown",), # pyright: ignore[reportArgumentType] # Exercise an unknown runtime target
|
||||
envelope=envelope,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_usage_batches_do_not_open_a_transaction() -> None:
|
||||
ledger: Final = AccountingLedger(_no_transaction)
|
||||
await ledger.insert_usage_batch(())
|
||||
assert await ledger.load_usage_batch(()) == ()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_conflicting_usage_within_one_batch_is_rejected_before_database_access(event: InboundUsageEvent) -> None:
|
||||
original: Final = UsageWrite(event, json.dumps({"event": event.model_dump(mode="json")}))
|
||||
changed: Final = UsageWrite(event, json.dumps({"event": event.model_dump(mode="json"), "pricing": {"rate": 2}}))
|
||||
with pytest.raises(AccountingConflictError, match="conflicting copies"):
|
||||
await AccountingLedger(_no_transaction).insert_usage_batch((original, changed))
|
||||
|
|
|
|||
|
|
@ -951,3 +951,33 @@ def test_is_lock_timeout_error_recognises_a_statement_cancelled_under_lock_timeo
|
|||
)
|
||||
def test_is_lock_timeout_error_excludes_other_failures(error: Exception) -> None:
|
||||
assert PrismaDBExceptionHandler.is_lock_timeout_error(error) is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("body", "retryable"),
|
||||
[
|
||||
('{"error_code":"P1017","message":"Server has closed the connection."}', True),
|
||||
('{"error_code":"P1000","message":"Authentication failed."}', False),
|
||||
('{"error_code":"P2021","message":"Table does not exist."}', False),
|
||||
('{"error_code":"P9999","message":"Unknown engine failure."}', False),
|
||||
("invalid engine response", False),
|
||||
],
|
||||
)
|
||||
def test_engine_connection_closure_is_recoverable_without_allowing_unverified_auth(
|
||||
body: str,
|
||||
retryable: bool,
|
||||
) -> None:
|
||||
from prisma.http import Response
|
||||
|
||||
error: Final = EngineRequestError(Response(httpx.Response(500)), body)
|
||||
assert PrismaDBExceptionHandler.is_database_transport_error(error) is retryable
|
||||
assert PrismaDBExceptionHandler.is_permanent_database_fault(error) is not retryable
|
||||
assert PrismaDBExceptionHandler.is_database_connection_error(error) is False
|
||||
|
||||
|
||||
def test_connection_closure_code_survives_prisma_data_error_wrapping() -> None:
|
||||
error: Final = DataError(
|
||||
data={"user_facing_error": {"error_code": "P1017", "message": "Server has closed the connection."}}
|
||||
)
|
||||
assert PrismaDBExceptionHandler.is_database_transport_error(error) is True
|
||||
assert PrismaDBExceptionHandler.is_permanent_database_fault(error) is False
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from litellm.proxy._types import (
|
|||
LiteLLM_ObjectPermissionTable,
|
||||
LiteLLM_TeamTable,
|
||||
LiteLLM_UserTable,
|
||||
LitellmUserRoles,
|
||||
ProxyException,
|
||||
SpecialHeaders,
|
||||
SpecialMCPServerNames,
|
||||
|
|
@ -13196,3 +13197,58 @@ async def test_personal_key_mcp_access_is_denied_when_owner_teams_cannot_be_read
|
|||
assert await MCPRequestHandler.get_allowed_mcp_servers(personal_key_owner_teams) == []
|
||||
assert await MCPRequestHandler.get_allowed_tools_for_server("srv-a", personal_key_owner_teams) == []
|
||||
assert await MCPRequestHandler.get_allowed_tools_for_server("srv-open", personal_key_owner_teams) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True)
|
||||
@pytest.mark.parametrize(
|
||||
"owner_role",
|
||||
[LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY],
|
||||
ids=["proxy_admin", "proxy_admin_viewer"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("key_team_id", "expected_servers"),
|
||||
[("team-a", {"srv-a", "srv-open"}), ("team-without-mcp", {"srv-open"})],
|
||||
ids=["team_grants_one_server", "team_grants_no_server"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"credential_shape",
|
||||
[{"is_session_token": False}, {"is_session_token": True}, {"api_key": None, "jwt_claims": {"sub": "admin"}}],
|
||||
ids=["virtual_key", "cli_session", "jwt"],
|
||||
)
|
||||
async def test_admin_owned_team_key_reaches_only_its_team_mcp_servers(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
personal_key_owner_teams: UserAPIKeyAuth,
|
||||
owner_role: LitellmUserRoles,
|
||||
key_team_id: str,
|
||||
expected_servers: set[str],
|
||||
credential_shape: dict[str, object],
|
||||
) -> None:
|
||||
from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager
|
||||
from litellm.proxy.auth import auth_checks
|
||||
|
||||
async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable:
|
||||
return _personal_key_team(team_id) if team_id in _PERSONAL_KEY_TEAMS else LiteLLM_TeamTable(team_id=team_id)
|
||||
|
||||
monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object)
|
||||
team_key: Final = personal_key_owner_teams.model_copy(
|
||||
update={"team_id": key_team_id, "user_role": owner_role, **credential_shape}
|
||||
)
|
||||
|
||||
assert set(await global_mcp_server_manager.get_allowed_mcp_servers(team_key)) == expected_servers
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True)
|
||||
async def test_admin_dashboard_session_keeps_the_whole_mcp_registry(
|
||||
personal_key_owner_teams: UserAPIKeyAuth,
|
||||
) -> None:
|
||||
from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager
|
||||
|
||||
dashboard_session: Final = personal_key_owner_teams.model_copy(
|
||||
update={"team_id": UI_TEAM_ID, "user_role": LitellmUserRoles.PROXY_ADMIN}
|
||||
)
|
||||
|
||||
assert set(await global_mcp_server_manager.get_allowed_mcp_servers(dashboard_session)) == set(
|
||||
global_mcp_server_manager.get_registry()
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
import asyncio
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Dict, List, Optional
|
||||
from typing import Dict, Final, List, Optional
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -112,6 +112,23 @@ class TestApplyToolsetScope:
|
|||
assert op.mcp_servers == ["server-a"]
|
||||
assert op.mcp_tool_permissions == toolset_perms
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"owner_role",
|
||||
[LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY],
|
||||
ids=["proxy_admin", "proxy_admin_viewer"],
|
||||
)
|
||||
async def test_admin_owned_team_key_without_the_toolset_grant_is_denied(self, owner_role: LitellmUserRoles):
|
||||
from litellm.proxy.mcp_server.server import apply_toolset_scope
|
||||
|
||||
team_key: Final = UserAPIKeyAuth(
|
||||
api_key="sk-test", team_id="team-a", user_role=owner_role, object_permission=None
|
||||
)
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await apply_toolset_scope(team_key, "toolset-123", granted=_granted_through_team("toolset-other"))
|
||||
|
||||
assert exc_info.value.status_code == 403
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_granted_toolset_is_served_to_a_key_without_its_own_grant(self):
|
||||
"""A team key whose own row carries no toolset grant is admitted to the toolset its team
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue