Merge remote-tracking branch 'origin/main' into litellm_least_busy_burst_reservation

# Conflicts:
#	tests/unit/decisions/test_main.py
This commit is contained in:
mateo-berri 2026-10-31 01:47:40 -07:00
commit ce90f4c10f
76 changed files with 3916 additions and 439 deletions

View file

@ -177,6 +177,8 @@ MID_STREAM_FAILURE_LOG_WAIT_SECONDS: Final = 1.0
# Set to 0 to disable truncation.
MAX_BASE64_LENGTH_FOR_LOGGING: Final = get_env_int("MAX_BASE64_LENGTH_FOR_LOGGING", 64)
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS: Final = 256 * 1024
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS: Final = 256
BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS: Final = 8 * 1024 * 1024
REDACTED_BY_LITELLM: Final = "redacted-by-litellm"
# in-memory stand-in handed to provider converters for redacted arguments; never stored
REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}"
@ -634,6 +636,8 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60
BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024)
# Anthropic's Messages API rejects thinking.budget_tokens < 1024.
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024
ANTHROPIC_MAX_USER_ID_LENGTH: Final = 512
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH: Final = 255
ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:"
REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5)
DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000)
@ -1163,6 +1167,7 @@ openai_compatible_providers: Final[list[str]] = [
"cortecs",
"scx-ai",
"prism",
"hostinger_router",
"sail",
"reka",
"streamlake",

View file

@ -3,10 +3,10 @@ from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field, replace
from functools import partial
from types import MappingProxyType
from typing import Final, Literal, TypeAlias
from typing import Annotated, Final, Literal, TypeAlias
import httpx
from pydantic import BaseModel, ConfigDict, StrictBool, TypeAdapter, ValidationError
from pydantic import BaseModel, BeforeValidator, ConfigDict, TypeAdapter, ValidationError
from pydantic_core import to_jsonable_python
from typing_extensions import assert_never
@ -23,6 +23,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.decisions.transformation import (
BaseDecisionsConfig,
DecisionsResponseValidationError,
catalog_mode,
image_media_type,
image_param,
input_images,
@ -61,7 +62,7 @@ from litellm.types.decisions import (
UnsupportedDecisionsRequest,
)
from litellm.types.llms.openai import ChatCompletionUserMessage
from litellm.types.utils import LlmProviders, ProviderSpecificHeader
from litellm.types.utils import LlmProviders, ProviderSpecificHeader, is_decisions_model_mode
from litellm.utils import ProviderConfigManager, get_model_cost_key
SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]]
@ -257,19 +258,105 @@ def _request_safety_identifier(
return None
def _fields_without_unsupported_media(
fields: RequestFields,
def _bool_or_none(value: object) -> bool | None:
return value if isinstance(value, bool) else None
def _str_or_none(value: object) -> str | None:
return value if isinstance(value, str) else None
_DeploymentFlag: TypeAlias = Annotated[bool | None, BeforeValidator(_bool_or_none)]
class _DeploymentCapabilities(BaseModel):
model_config = ConfigDict(frozen=True, from_attributes=True)
mode: Annotated[str | None, BeforeValidator(_str_or_none)] = None
supports_box_questions: _DeploymentFlag = None
supports_audio_input: _DeploymentFlag = None
supports_video_input: _DeploymentFlag = None
def decisions_media_flag(self, flag: bool | None, model: str, custom_llm_provider: str) -> bool | None:
if flag is None:
return None
mode: Final = self.mode if self.mode is not None else catalog_mode(model, custom_llm_provider)
return flag if mode is None or is_decisions_model_mode(mode) else None
@dataclass(frozen=True, slots=True)
class _ResolvedSupport:
box_questions: bool
audio: bool
videos: bool
def _takes_audio(
provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities
) -> bool:
return provider_config.supports_audio_input(
model,
custom_llm_provider,
deployment.decisions_media_flag(deployment.supports_audio_input, model, custom_llm_provider),
)
def _takes_videos(
provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities
) -> bool:
return provider_config.supports_video_input(
model,
custom_llm_provider,
deployment.decisions_media_flag(deployment.supports_video_input, model, custom_llm_provider),
)
def deployment_refuses_media(
model: str,
custom_llm_provider: str,
model_info: Mapping[str, object],
call_kwargs: Mapping[str, object],
*,
audio: bool,
videos: bool,
) -> bool:
if _drops_params(call_kwargs):
return False
provider_config: Final = get_decisions_provider_config(model, custom_llm_provider)
canonical_model: Final = provider_config.canonical_model(model)
deployment: Final = _DeploymentCapabilities.model_validate(model_info)
return (audio and not _takes_audio(provider_config, canonical_model, custom_llm_provider, deployment)) or (
videos and not _takes_videos(provider_config, canonical_model, custom_llm_provider, deployment)
)
def _resolved_support(
provider_config: BaseDecisionsConfig,
model: str,
custom_llm_provider: str,
headers: Mapping[str, str],
kwargs: Mapping[str, object],
fields: RequestFields,
) -> _ResolvedSupport:
deployment: Final = _DeploymentCapabilities.model_validate(kwargs.get("model_info") or {})
return _ResolvedSupport(
box_questions=provider_config.supports_box_questions(
model, custom_llm_provider, headers, deployment.supports_box_questions
),
audio=bool(fields.audio) and _takes_audio(provider_config, model, custom_llm_provider, deployment),
videos=bool(fields.videos) and _takes_videos(provider_config, model, custom_llm_provider, deployment),
)
def _fields_without_unsupported_media(
fields: RequestFields, support: _ResolvedSupport, kwargs: Mapping[str, object]
) -> RequestFields:
if not _drops_params(kwargs):
return fields
return replace(
fields,
audio=fields.audio if provider_config.supports_audio_input(model, custom_llm_provider) else None,
videos=fields.videos if provider_config.supports_video_input(model, custom_llm_provider) else None,
audio=fields.audio if support.audio else None,
videos=fields.videos if support.videos else None,
)
@ -292,10 +379,8 @@ class _MalformedImages:
def _unsupported_params(
ir_request: DecisionsIRRequest,
provider_config: BaseDecisionsConfig,
model: str,
custom_llm_provider: str,
headers: Mapping[str, str],
supports_box_questions: bool,
support: _ResolvedSupport,
) -> tuple[str, ...]:
refuses_safety_identifier: Final = (
ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier
@ -304,13 +389,9 @@ def _unsupported_params(
None if provider_config.supports_images_in(ir_request.input, headers) else image_param(ir_request.input)
)
refuses_assets: Final = bool(ir_request.assets) and not provider_config.supports_assets(headers)
refuses_box_questions: Final = _has_box_question(ir_request.questions) and not supports_box_questions
refuses_audio: Final = bool(ir_request.audio) and not provider_config.supports_audio_input(
model, custom_llm_provider
)
refuses_videos: Final = bool(ir_request.videos) and not provider_config.supports_video_input(
model, custom_llm_provider
)
refuses_box_questions: Final = _has_box_question(ir_request.questions) and not support.box_questions
refuses_audio: Final = bool(ir_request.audio) and not support.audio
refuses_videos: Final = bool(ir_request.videos) and not support.videos
return (
*(("safety_identifier",) if refuses_safety_identifier else ()),
*(() if refused_image_param is None else (refused_image_param,)),
@ -333,7 +414,7 @@ def _without_unsupported_params(
ir_request: DecisionsIRRequest,
provider_config: BaseDecisionsConfig,
headers: Mapping[str, str],
supports_box_questions: bool,
support: _ResolvedSupport,
) -> DecisionsIRRequest:
return replace(
ir_request,
@ -345,9 +426,7 @@ def _without_unsupported_params(
safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None,
assets=ir_request.assets if provider_config.supports_assets(headers) else MappingProxyType({}),
questions=(
ir_request.questions
if supports_box_questions
else tuple(filter(_is_not_box_question, ir_request.questions))
ir_request.questions if support.box_questions else tuple(filter(_is_not_box_question, ir_request.questions))
),
)
@ -368,39 +447,20 @@ def _without_malformed_images(
return well_formed
class _DeploymentBoxQuestions(BaseModel):
model_config = ConfigDict(frozen=True, from_attributes=True)
supports_box_questions: StrictBool | None = None
def _deployment_supports_box_questions(kwargs: Mapping[str, object]) -> bool | None:
try:
return _DeploymentBoxQuestions.model_validate(kwargs.get("model_info") or {}).supports_box_questions
except ValidationError:
return None
def _provider_ir_request(
ir_request: DecisionsIRRequest,
*,
provider_config: BaseDecisionsConfig,
model: str,
custom_llm_provider: str,
headers: Mapping[str, str],
support: _ResolvedSupport,
kwargs: Mapping[str, object],
) -> DecisionsIRRequest | _UnsupportedParams | _NothingLeftAfterDrop:
supports_box_questions: Final = provider_config.supports_box_questions(
model, custom_llm_provider, headers, _deployment_supports_box_questions(kwargs)
)
unsupported: Final = _unsupported_params(
ir_request, provider_config, model, custom_llm_provider, headers, supports_box_questions
)
unsupported: Final = _unsupported_params(ir_request, provider_config, headers, support)
if not unsupported:
return ir_request
if not _drops_params(kwargs):
return _UnsupportedParams(names=unsupported)
supported: Final = _without_unsupported_params(ir_request, provider_config, headers, supports_box_questions)
supported: Final = _without_unsupported_params(ir_request, provider_config, headers, support)
if _leaves_no_input(supported.input):
return _NothingLeftAfterDrop(names=unsupported, missing="input to decide on")
if not supported.questions:
@ -479,10 +539,11 @@ def prepare_call(
raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error
try:
headers: Final = decisions_outbound_headers(extra_headers, provider, kwargs)
support: Final = _resolved_support(provider_config, canonical_model, provider, headers, kwargs, fields)
request_safety_identifier: Final = _request_safety_identifier(safety_identifier, provider_config, kwargs)
request: Final = _fields_without_unsupported_media(
fields, provider_config, canonical_model, provider, kwargs
).validate(safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs))
request: Final = _fields_without_unsupported_media(fields, support, kwargs).validate(
safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs)
)
except ValidationError as error:
raise litellm.BadRequestError(
message=f"Invalid Decisions request: {error}",
@ -523,12 +584,7 @@ def prepare_call(
llm_provider=provider,
)
ir_request: Final = _provider_ir_request(
caller_ir_request,
provider_config=provider_config,
model=canonical_model,
custom_llm_provider=provider,
headers=headers,
kwargs=kwargs,
caller_ir_request, provider_config=provider_config, headers=headers, support=support, kwargs=kwargs
)
if isinstance(ir_request, _UnsupportedParams):
raise litellm.UnsupportedParamsError(

View file

@ -14,7 +14,7 @@ import itertools
import os
import re
from collections.abc import Callable, Container, Iterable, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast
from urllib.parse import urlparse
from pydantic import TypeAdapter, ValidationError
@ -102,6 +102,192 @@ def _validated_object_list(value: object) -> list[object] | None:
return None
_MessageT: Final = TypeVar("_MessageT")
_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None]
_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls")
def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]:
lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS)
return tuple(
itertools.chain.from_iterable(
_OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list)
)
)
def _block_carries_cache_control(block: object) -> bool:
return isinstance(block, dict) and _CONTENT_PART_ADAPTER.validate_python(block).get("cache_control") is not None
_ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT: Final = frozenset({"tool", "function"})
def _message_mark_reaches_provider(fields: Mapping[str, object]) -> bool:
if fields.get("cache_control") is None:
return False
return (
not isinstance(fields.get("content"), list)
or fields.get("role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT
)
def cache_control_count(message: object) -> int:
if not isinstance(message, dict):
return 0
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block))
return int(_message_mark_reaches_provider(fields)) + block_marks
def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]:
others: Final = {key: value for key, value in fields.items() if key != "cache_control"}
if "cache_control" not in fields:
return others
cache_control: Final = rewrite(fields["cache_control"])
return others if cache_control is None else {**others, "cache_control": cache_control}
def block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object:
if not isinstance(block, dict):
return block
return _fields_with_cache_control(_CONTENT_PART_ADAPTER.validate_python(block), rewrite)
def with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT:
if not isinstance(message, dict):
return message
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
blocks_rewritten: Final = {
key: (
[block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)]
if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list)
else value
)
for key, value in fields.items()
}
return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped
_MessageT, _fields_with_cache_control(blocks_rewritten, rewrite)
)
def dropped_cache_control(_: object) -> None:
return None
def _first_marked_index(value: object) -> int | None:
if not isinstance(value, list):
return None
blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value)
return next((index for index, block in enumerate(blocks) if _block_carries_cache_control(block)), None)
_MarkSlot: TypeAlias = tuple[str, int]
_MESSAGE_MARK_SLOT: Final[_MarkSlot] = ("cache_control", -1)
def _block_mark_slots(fields: Mapping[str, object], key: str) -> tuple[_MarkSlot, ...]:
value: Final = fields.get(key)
if not isinstance(value, list):
return ()
blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value)
return tuple((key, index) for index, block in enumerate(blocks) if _block_carries_cache_control(block))
def _mark_slots_in_wire_order(fields: Mapping[str, object]) -> tuple[_MarkSlot, ...]:
content_slots: Final = _block_mark_slots(fields, "content")
tool_call_slots: Final = _block_mark_slots(fields, "tool_calls")
if not _message_mark_reaches_provider(fields):
return content_slots + tool_call_slots
if isinstance(fields.get("content"), list):
return content_slots + (_MESSAGE_MARK_SLOT,) + tool_call_slots
return (_MESSAGE_MARK_SLOT,) + content_slots + tool_call_slots
def _blocks_without_marks(key: str, value: object, dropped_slots: frozenset[_MarkSlot]) -> object:
if key not in _MARK_CARRYING_MESSAGE_FIELDS or not isinstance(value, list):
return value
return [
block_with_cache_control(block, dropped_cache_control) if (key, index) in dropped_slots else block
for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(value))
]
def _without_oldest_marks(message: _MessageT, dropped: int) -> _MessageT:
if dropped == 0 or not isinstance(message, dict):
return message
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
dropped_slots: Final = frozenset(_mark_slots_in_wire_order(fields)[:dropped])
blocks_unmarked: Final = {key: _blocks_without_marks(key, value, dropped_slots) for key, value in fields.items()}
return cast( # cast-ok: the same message with its oldest wire-order marks dropped
_MessageT,
_fields_with_cache_control(blocks_unmarked, dropped_cache_control)
if _MESSAGE_MARK_SLOT in dropped_slots
else blocks_unmarked,
)
def tool_message_with_single_cache_mark(message: _MessageT) -> _MessageT:
if not isinstance(message, dict):
return message
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
content: Final = fields.get("content")
if fields.get("role") != "tool" or not isinstance(content, list):
return message
kept_index: Final = None if fields.get("cache_control") is not None else _first_marked_index(content)
return cast( # cast-ok: the same tool message keeping only the mark its single cache point reads
_MessageT,
{
**fields,
"content": [
block if index == kept_index else block_with_cache_control(block, dropped_cache_control)
for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(content))
],
},
)
_BudgetSpend: TypeAlias = Callable[[tuple[int, int], int], tuple[int, int]]
def _newest_marks_kept(state: tuple[int, int], marks: int) -> tuple[int, int]:
remaining, _ = state
kept: Final = min(marks, remaining)
return remaining - kept, marks - kept
def _whole_message_kept(state: tuple[int, int], marks: int) -> tuple[int, int]:
remaining, _ = state
if marks > remaining:
return remaining, marks
return remaining - marks, 0
def _messages_within_budget(
messages: Sequence[_MessageT], reserved_marks: int, spend: _BudgetSpend
) -> Sequence[_MessageT]:
budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0)
newest_first: Final = tuple(reversed(messages))
budget_after_each: Final = itertools.accumulate(
(cache_control_count(message) for message in newest_first), spend, initial=(budget, 0)
)
dropped_counts: Final = tuple(dropped for _, dropped in itertools.islice(budget_after_each, 1, None))
kept_newest_first: Final = tuple(
_without_oldest_marks(message, dropped) for message, dropped in zip(newest_first, dropped_counts)
)
return tuple(reversed(kept_newest_first))
def newest_marks_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]:
return _messages_within_budget(messages, reserved_marks, _newest_marks_kept)
def whole_messages_within_cache_control_limit(
messages: Sequence[_MessageT], reserved_marks: int
) -> Sequence[_MessageT]:
return _messages_within_budget(messages, reserved_marks, _whole_message_kept)
def _part_with_cache_control_renamed(part: object) -> object:
if not _accepts_prompt_cache_breakpoint(part):
return part
@ -1439,3 +1625,25 @@ class AnthropicCacheControlHook(CustomPromptManagement):
llm_router=None,
)
return None
def messages_within_cache_control_cap(
messages: Sequence[_MessageT], tools: object, cache_control: object = None, request_kwargs: object = None
) -> Sequence[_MessageT]:
"""Under drop_params, drop the oldest message hints the provider's cache_control cap has no room for.
Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first.
"""
reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints(
_validated_object_list(tools), cache_control, request_kwargs
)
kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks)
dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept))
if dropped > 0:
verbose_logger.warning(
"Dropped %d cache_control hint(s) from the oldest messages because drop_params is on and the "
"provider accepts at most %d cache_control blocks per request",
dropped,
MAX_CACHE_CONTROL_BLOCKS,
)
return kept

View file

@ -3,13 +3,15 @@ import functools
import inspect
import re
import time
from collections.abc import Iterator, Mapping, Sequence
from collections.abc import Callable, Iterator, Mapping, Sequence
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final, TypeAlias, cast
from litellm._logging import format_base64_size, verbose_logger
from litellm.constants import (
BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS,
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS,
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS,
MAX_BASE64_LENGTH_FOR_LOGGING,
)
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
@ -44,6 +46,8 @@ Helper utils used for logging callbacks
# Regex matching data-URI base64 content: "data:<mime>;base64,<payload>"
# Captures: group(1)=mime_type, group(2)=base64_payload
_DATA_URI_RE: Final = re.compile(r"data:([^;,\s]{1,255});base64,([A-Za-z0-9+/=]+)")
_DATA_URI_PREFIX: Final = "data:"
_DATA_URI_BASE64_MARKER: Final = ";base64,"
# Maximum nesting depth for _truncate_base64_in_value to guard against
# pathological payloads. OpenAI message format is typically 3-4 levels deep.
@ -60,15 +64,20 @@ def _base64_data_uri_replacer(match: re.Match[str]) -> str:
return f"data:{mime_type};base64,[base64_data truncated: {size_str}]"
def _may_contain_base64_data_uri(value: str) -> bool:
return ";" in value and _DATA_URI_PREFIX in value and _DATA_URI_BASE64_MARKER in value
def _truncate_base64_in_string(value: str) -> str:
"""Replace long base64 data-URI payloads in a string with a size placeholder."""
if MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
if MAX_BASE64_LENGTH_FOR_LOGGING <= 0 or not _may_contain_base64_data_uri(value):
return value
return _DATA_URI_RE.sub(_base64_data_uri_replacer, value)
def _truncate_base64_in_value(
value: str | dict[str, object] | list[object] | None,
truncate_string: Callable[[str], str],
) -> str | dict[str, object] | list[object] | None:
"""Iteratively truncate base64 data URIs in a JSON-like value (str/list/dict).
@ -78,13 +87,13 @@ def _truncate_base64_in_value(
# Stack entries: (source_value, depth, parent_container, key_or_index)
# We mutate *copies* of dicts/lists in-place via parent references.
if isinstance(value, str):
return _truncate_base64_in_string(value)
return truncate_string(value)
if not isinstance(value, (dict, list)):
return value
# Shallow-copy the root so we don't mutate the caller's data.
root: Final = {k: v for k, v in value.items()} if isinstance(value, dict) else list(value)
stack: Final[list] = [(root, 0)]
stack: Final[list[tuple[dict[str, object] | list[object], int]]] = [(root, 0)] # mutable-ok: explicit stack
while stack:
container, depth = stack.pop()
@ -93,19 +102,19 @@ def _truncate_base64_in_value(
if isinstance(container, dict):
for k, v in container.items():
if isinstance(v, str):
container[k] = _truncate_base64_in_string(v)
container[k] = truncate_string(v)
elif isinstance(v, dict):
copy: dict | list = {ck: cv for ck, cv in v.items()}
copy: dict[str, object] | list[object] = {ck: cv for ck, cv in v.items()}
container[k] = copy
stack.append((copy, depth + 1))
elif isinstance(v, list):
copy = list(v)
container[k] = copy
stack.append((copy, depth + 1))
elif isinstance(container, list):
else:
for i, v in enumerate(container):
if isinstance(v, str):
container[i] = _truncate_base64_in_string(v)
container[i] = truncate_string(v)
elif isinstance(v, dict):
copy = {ck: cv for ck, cv in v.items()}
container[i] = copy
@ -127,8 +136,15 @@ def truncate_base64_in_messages(
"""
if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
return messages
return _copy_messages(messages, _truncate_base64_in_string)
def _copy_messages(
messages: str | dict[str, object] | list[object], # mutable-ok: same contract as truncate_base64_in_messages
truncate_string: Callable[[str], str],
) -> str | dict[str, object] | list[object] | None: # mutable-ok: same contract as truncate_base64_in_messages
try:
return _truncate_base64_in_value(messages)
return _truncate_base64_in_value(messages, truncate_string)
except Exception as e:
verbose_logger.debug("Failed to truncate base64 in messages: %s", e)
return messages
@ -151,19 +167,31 @@ def _iter_string_leaves(value: _StringTree) -> Iterator[str]:
pass
def _is_few_plain_strings(leaves: tuple[str, ...]) -> bool:
return len(leaves) < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS and not any(
_may_contain_base64_data_uri(leaf) for leaf in leaves
)
async def truncate_base64_in_messages_async(
messages: str | list | dict | None, # mutable-ok: same contract as truncate_base64_in_messages
) -> str | list | dict | None: # mutable-ok: same contract as truncate_base64_in_messages
"""
Same result as truncate_base64_in_messages, but payloads whose string content
reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are scanned in a worker
thread so the regex pass over multi-MB base64 images does not block the event loop.
reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are copied in a worker thread,
so neither the regex pass over multi-MB base64 images nor the copy of a long
history blocks the event loop. A few strings with no data URI, under
BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS in total, are copied inline, since
checking them for data URIs and copying them costs less than the thread hop.
"""
if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
return messages
total_chars: Final = sum(len(leaf) for leaf in _iter_string_leaves(messages))
leaves: Final = tuple(_iter_string_leaves(messages))
total_chars: Final = sum(len(leaf) for leaf in leaves)
if total_chars < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS:
return truncate_base64_in_messages(messages)
if total_chars < BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS and _is_few_plain_strings(leaves):
return _copy_messages(messages, lambda text: text)
return await asyncio.to_thread(truncate_base64_in_messages, messages)

View file

@ -88,17 +88,22 @@ def handle_any_messages_to_chat_completion_str_messages_conversion(
return [{"input": str(messages)}]
def _message_with_str_content(message: AllMessageValues) -> AllMessageValues:
texts: Final = convert_content_list_to_str(message=message)
if not texts:
return message
return cast( # cast-ok: the same message with its content flattened to one string
AllMessageValues, {**message, "content": texts}
)
def handle_messages_with_content_list_to_str_conversion(
messages: list[AllMessageValues],
) -> list[AllMessageValues]:
"""
Handles messages with content list conversion
Returns new messages with each content list flattened to a string, leaving the caller's messages untouched
"""
for message in messages:
texts = convert_content_list_to_str(message=message)
if texts:
message["content"] = texts
return messages
return [_message_with_str_content(message) for message in messages]
def strip_name_from_message(message: AllMessageValues, allowed_name_roles: list[str] = ["user"]) -> AllMessageValues:

View file

@ -167,6 +167,7 @@ _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool)
_TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_TOOLS_ADAPTER: Final = TypeAdapter(list[object])
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_REQUEST_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
)
@ -2254,6 +2255,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
headers=headers,
)
@staticmethod
def _messages_within_cache_control_cap(
messages: Sequence[AllMessageValues], optional_params: Mapping[str, object]
) -> Sequence[AllMessageValues]:
from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap
return messages_within_cache_control_cap(
messages, optional_params.get("tools"), optional_params.get("cache_control"), optional_params
)
def transform_request(
self,
model: str,
@ -2271,9 +2282,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
anthropic_messages_pt,
)
drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True
drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params"))
capped_messages: Final = (
list(
self._messages_within_cache_control_cap(
messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params)
)
)
if drop_params
else messages
)
if "tools" not in optional_params and messages is not None and has_tool_call_blocks(messages):
if "tools" not in optional_params and has_tool_call_blocks(capped_messages):
optional_params["tools"], _ = self._map_tools(add_dummy_tool(custom_llm_provider="anthropic"))
# Drop thinking param if thinking is enabled but thinking_blocks are missing
@ -2285,9 +2305,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# Related issue: https://github.com/BerriAI/litellm/issues/18926
if (
optional_params.get("thinking") is not None
and messages is not None
and last_assistant_with_tool_calls_has_no_thinking_blocks(messages)
and not any_assistant_message_has_thinking_blocks(messages)
and last_assistant_with_tool_calls_has_no_thinking_blocks(capped_messages)
and not any_assistant_message_has_thinking_blocks(capped_messages)
):
if litellm.modify_params:
optional_params.pop("thinking", None)
@ -2332,15 +2351,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
_name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request(
optional_params=optional_params,
)
if _name_forward_map:
messages = self._rewrite_tool_names_in_messages(messages, _name_forward_map)
request_messages: Final = (
self._rewrite_tool_names_in_messages(capped_messages, _name_forward_map)
if _name_forward_map
else capped_messages
)
if _name_reverse_map and isinstance(litellm_params, dict):
litellm_params[ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY] = _name_reverse_map
# Only the leading system run becomes the top-level system prompt. A later
# system message stays in the conversation: hoisting it rewrites the cached
# prefix and re-bills the whole history at cache-write pricing (#36559).
leading_system_run, later_messages = split_leading_system_run(messages)
leading_system_run, later_messages = split_leading_system_run(request_messages)
anthropic_system_message_list: Final = self.translate_system_message(messages=list(leading_system_run))
# Handling anthropic API Prompt Caching
if len(anthropic_system_message_list) > 0:
@ -2362,7 +2384,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
except Exception as e:
raise AnthropicError(
status_code=400,
message=f"{e}\nReceived Messages={messages}",
message=f"{e}\nReceived Messages={request_messages}",
) # don't use verbose_logger.exception, if exception is raised
self.update_headers_with_optional_anthropic_beta(

View file

@ -4,6 +4,7 @@ This file contains common utils for anthropic calls.
import copy
import json
import os
import re
from collections.abc import Container, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
@ -19,6 +20,8 @@ from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm.constants import (
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH,
ANTHROPIC_MAX_USER_ID_LENGTH,
DEFAULT_MODEL_CREATED_AT_TIME,
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
@ -178,6 +181,40 @@ def tools_without_custom_field(tools: Sequence[object], hoist_defer_loading: boo
return [_tool_without_custom_field(tool, hoist_defer_loading, drop_custom) for tool in tools]
def anthropic_pass_through_api_base() -> str:
return os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
def _same_host_and_port(url: httpx.URL, other: str) -> bool:
try:
other_url: Final = httpx.URL(other)
except httpx.InvalidURL:
return False
return (url.host, url.port) == (other_url.host, other_url.port)
def anthropic_upstream_user_id_limit(url: str) -> int | None:
upstream: Final = httpx.URL(url)
if not upstream.path.removesuffix("/").endswith("/v1/messages"):
return None
if upstream.host == "api.anthropic.com":
return ANTHROPIC_MAX_USER_ID_LENGTH
return (
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH
if _same_host_and_port(upstream, anthropic_pass_through_api_base())
else None
)
def _has_email_shape(token: str) -> bool:
at_sign: Final = token.find("@", 1)
return at_sign != -1 and token.find(".", at_sign + 2, -1) != -1
def anthropic_accepts_user_id(user_id: str, max_length: int) -> bool:
return len(user_id) <= max_length and user_id.isprintable() and not any(map(_has_email_shape, user_id.split()))
def is_anthropic_messages_url(url: str) -> bool:
"""Check whether a URL addresses Anthropic's Messages API."""
parsed_url: Final = urlparse(url)

View file

@ -12,6 +12,7 @@ from pydantic_core import ErrorDetails
from typing_extensions import assert_never
import litellm
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.secret_managers.main import get_secret_str
@ -297,13 +298,7 @@ def _systemone_question(question: DecisionsIRQuestion) -> Mapping[str, object]:
assert_never(question)
def ir_to_systemone_request(
model: str, request: DecisionsIRRequest, *, supports_audio: bool = False, supports_video: bool = False
) -> Mapping[str, object] | UnsupportedDecisionsRequest:
if request.audio and not supports_audio:
return UnsupportedDecisionsRequest(reason="audio input is not supported by this Decisions model")
if request.videos and not supports_video:
return UnsupportedDecisionsRequest(reason="video input is not supported by this Decisions model")
def ir_to_systemone_request(model: str, request: DecisionsIRRequest) -> Mapping[str, object]:
state: Final = _systemone_state(request.input)
keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True)
images: Final = input_images(request.input)
@ -503,6 +498,16 @@ def ir_to_systemone_response(response: DecisionsIRResponse, request: DecisionsIR
)
def catalog_mode(model: str, custom_llm_provider: str) -> str | None:
try:
key: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)["key"]
except Exception: # noqa: BLE001 # get_model_info raises bare Exception for an unmapped model, which the catalog does not list
return None
entry: Final = GetModelCostMap.loaded_model_cost_map().get(key)
mode: Final = entry.get("mode") if entry is not None else None
return mode if isinstance(mode, str) else None
def _is_decisions_model(model: str, custom_llm_provider: str) -> bool:
try:
model_mode: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider).get("mode")
@ -640,12 +645,16 @@ class BaseDecisionsConfig(ABC):
model=model, custom_llm_provider=custom_llm_provider
)
def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool:
def supports_audio_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
if deployment_flag is not None:
return deployment_flag
return _is_decisions_model(model, custom_llm_provider) and litellm.supports_audio_input(
model=model, custom_llm_provider=custom_llm_provider
)
def supports_video_input(self, model: str, custom_llm_provider: str) -> bool:
def supports_video_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
if deployment_flag is not None:
return deployment_flag
return _is_decisions_model(model, custom_llm_provider) and litellm.supports_video_input(
model=model, custom_llm_provider=custom_llm_provider
)
@ -656,12 +665,7 @@ class BaseDecisionsConfig(ABC):
request: DecisionsIRRequest,
custom_llm_provider: str,
) -> Mapping[str, object] | UnsupportedDecisionsRequest:
return ir_to_systemone_request(
self.request_model(model),
request,
supports_audio=self.supports_audio_input(model, custom_llm_provider),
supports_video=self.supports_video_input(model, custom_llm_provider),
)
return ir_to_systemone_request(self.request_model(model), request)
def unwrap_response(self, payload: object) -> object:
return payload

View file

@ -25,6 +25,7 @@ from litellm.constants import (
)
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
drops_unsupported_params,
filter_exceptions_from_params,
filter_internal_params,
map_finish_reason,
@ -2129,6 +2130,24 @@ class AmazonConverseConfig(BaseConfig):
return data
@staticmethod
def _claude_messages_within_cache_control_cap(
messages: Sequence[AllMessageValues],
capability_model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> Sequence[AllMessageValues]:
if not drops_unsupported_params(litellm_params.get("drop_params")) or "claude" not in capability_model.lower():
return messages
from litellm.integrations.anthropic_cache_control_hook import (
messages_within_cache_control_cap,
tool_message_with_single_cache_mark,
)
return messages_within_cache_control_cap(
tuple(map(tool_message_with_single_cache_mark, messages)), optional_params.get("tools")
)
async def _async_transform_request(
self,
model: str,
@ -2140,7 +2159,13 @@ class AmazonConverseConfig(BaseConfig):
base_model: Final = litellm_params.get("base_model")
capability_model: Final = bedrock_capability_model(model, base_model)
messages, system_content_blocks = self._transform_system_message(
messages, model=capability_model, deployment_model=model
list(
self._claude_messages_within_cache_control_cap(
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
)
),
model=capability_model,
deployment_model=model,
)
# Convert last user message to guarded_text if guardrailConfig is present
@ -2221,7 +2246,13 @@ class AmazonConverseConfig(BaseConfig):
base_model: Final = litellm_params.get("base_model")
capability_model: Final = bedrock_capability_model(model, base_model)
messages, system_content_blocks = self._transform_system_message(
messages, model=capability_model, deployment_model=model
list(
self._claude_messages_within_cache_control_cap(
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
)
),
model=capability_model,
deployment_model=model,
)
# Convert last user message to guarded_text if guardrailConfig is present

View file

@ -26,6 +26,7 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase):
aembedding=None,
custom_endpoint: bool | None = None,
headers: dict | None = None,
drop_params: bool = False,
) -> EmbeddingResponse:
# Check for custom user agent in optional_params or environment
# This allows partners building on LiteLLM to set their own telemetry
@ -58,4 +59,5 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase):
aembedding=aembedding,
custom_endpoint=True,
headers=headers,
drop_params=drop_params,
)

View file

@ -130,7 +130,8 @@ class DeepInfraConfig(OpenAIGPTConfig):
- Input: {"role": "tool", "content": [{"type": "text", "text": "result"}]}
- Output: {"role": "tool", "content": "[{\"type\": \"text\", \"text\": \"result\"}]"}
"""
for message in messages:
copied_messages: Final = [message.copy() for message in messages]
for message in copied_messages:
if message.get("role") == "tool":
content = message.get("content")
@ -149,7 +150,7 @@ class DeepInfraConfig(OpenAIGPTConfig):
# For complex content, serialize the entire array as JSON string
message["content"] = json.dumps(content)
return messages
return copied_messages
@overload
def _transform_messages(

View file

@ -23,6 +23,7 @@ from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionToolParam,
ChatCompletionToolParamFunctionChunk,
OpenAIChatCompletionToolParam,
)
from litellm.types.utils import (
@ -45,6 +46,7 @@ from ...openai.chat.gpt_transformation import (
OpenAIChatCompletionStreamingHandler,
OpenAIGPTConfig,
)
from ...openai.common_utils import without_cache_control
from ..common_utils import (
FIREROUTER,
FireworksAIException,
@ -480,15 +482,23 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
return (("response_format", _json_schema_response_format(choice_schema, "choice")),)
def _transform_tools(self, tools: list[OpenAIChatCompletionToolParam]) -> list[OpenAIChatCompletionToolParam]:
for tool in tools:
if tool.get("type") != "function":
continue
function = tool["function"]
function.pop("strict", None)
params = function.get("parameters")
if isinstance(params, dict):
unpack_legacy_defs(params)
return tools
return [self._fireworks_tool(tool) for tool in tools]
@staticmethod
def _fireworks_tool(tool: OpenAIChatCompletionToolParam) -> OpenAIChatCompletionToolParam:
if tool.get("type") != "function":
return tool
function: Final = tool["function"]
params: Final = function.get("parameters")
unstrict_function: Final = cast( # cast-ok: the same function definition minus `strict`
ChatCompletionToolParamFunctionChunk, {key: value for key, value in function.items() if key != "strict"}
)
new_function: Final[ChatCompletionToolParamFunctionChunk] = (
{**unstrict_function, "parameters": unpack_legacy_defs(params, copy=True)}
if isinstance(params, dict)
else unstrict_function
)
return {**tool, "function": new_function}
def _transform_messages_helper(
self, messages: list[AllMessageValues], model: str, litellm_params: dict
@ -496,10 +506,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
"""
Strip fields not permitted by FireworksAI from messages.
"""
from litellm.litellm_core_utils.prompt_templates.common_utils import (
filter_value_from_dict,
)
supports_vision_value: Final = self._get_model_cost_capability_exact(model=model, capability="supports_vision")
for message in messages:
if message["role"] == "user":
@ -530,9 +536,11 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
model=model,
llm_provider="fireworks_ai",
)
filter_value_from_dict(cast(dict, message), "cache_control")
return [strip_litellm_response_only_fields(message) for message in messages]
return [
strip_litellm_response_only_fields(cast(AllMessageValues, without_cache_control(message)))
for message in messages
]
# Cached index of fireworks_ai/* entries from litellm.model_cost. Building
# this index requires a full scan of model_cost (tens of thousands of

View file

@ -140,7 +140,8 @@ class GroqChatConfig(OpenAILikeChatConfig):
def _transform_messages(
self, messages: list[AllMessageValues], model: str, is_async: bool = False
) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]:
for idx, message in enumerate(messages):
transformed_messages: Final = list(messages)
for idx, message in enumerate(transformed_messages):
"""
1. Don't pass 'null' function_call assistant message to groq - https://github.com/BerriAI/litellm/issues/5839
"""
@ -154,12 +155,12 @@ class GroqChatConfig(OpenAILikeChatConfig):
for k, v in _message.items():
if v is not None:
new_message[k] = v
messages[idx] = strip_litellm_internal_message_fields(new_message)
transformed_messages[idx] = strip_litellm_internal_message_fields(new_message)
if is_async:
return super()._transform_messages(messages=messages, model=model, is_async=True)
return super()._transform_messages(messages=transformed_messages, model=model, is_async=True)
else:
return super()._transform_messages(messages=messages, model=model, is_async=False)
return super()._transform_messages(messages=transformed_messages, model=model, is_async=False)
def transform_messages(
self,

View file

@ -2,6 +2,7 @@
Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
"""
import copy
import json
from collections.abc import Coroutine
from typing import Final, Literal, cast, overload
@ -88,7 +89,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
) -> dict:
_tools = non_default_params.pop("tools", None)
if _tools is not None:
_tools = remove_additional_properties(_tools)
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
_tools = remove_strict_from_schema(_tools)
if isinstance(_tools, list):
_tools = self._convert_custom_tools_to_function_tools(_tools)

View file

@ -755,7 +755,8 @@ class MistralConfig(OpenAIGPTConfig):
if _name is not None:
# Remove name if not a tool message
if message["role"] != "tool" or isinstance(_name, str) and len(_name.strip()) == 0:
message.pop("name", None)
message_without_name: Final = {k: v for k, v in message.items() if k != "name"}
return cast(AllMessageValues, message_without_name) # cast-ok: the same message minus name
return message

View file

@ -14,7 +14,7 @@ from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.constants import OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
extract_reasoning_content,
handle_invalid_parallel_tool_calls,
@ -64,7 +64,7 @@ from litellm.types.utils import (
)
from litellm.utils import convert_to_model_response_object, declared_mid_conversation_system_support
from ..common_utils import OpenAIError
from ..common_utils import OpenAIError, without_cache_control
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
if TYPE_CHECKING:
@ -128,6 +128,29 @@ def _litellm_params_str(litellm_params: Mapping[str, object] | None, key: str) -
return value if isinstance(value, str) else None
def _drops_params(litellm_params: Mapping[str, object]) -> bool:
return drops_unsupported_params(litellm_params.get("drop_params"))
def _names_claude(model: str, litellm_params: Mapping[str, object]) -> bool:
base_model: Final = litellm_params.get("base_model")
return "claude" in model.lower() or (isinstance(base_model, str) and "claude" in base_model.lower())
def _claude_messages_within_cache_control_cap(
model: str,
messages: Sequence[AllMessageValues],
tools: Sequence["ChatCompletionToolParam"] | None,
litellm_params: Mapping[str, object],
request_params: Mapping[str, object],
) -> Sequence[AllMessageValues]:
if not _names_claude(model, litellm_params) or not _drops_params(litellm_params):
return messages
from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap
return list(messages_within_cache_control_cap(messages, tools, request_params.get("cache_control"), request_params))
def _prompt_cache_options(optional_params: Mapping[str, object]) -> object | None:
try:
extra_body: Final = _OBJECT_MAPPING.validate_python(optional_params.get("extra_body"))
@ -311,52 +334,58 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
return content_copy
async def _async_handle_pdf_url(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile:
file_id: Final = content_item.get("file_id")
content_copy: Final = content_item.copy()
file_id: Final = content_copy.get("file_id")
if file_id is not None: # check for file id being url done in _handle_pdf_url
base64_data: Final = await async_convert_url_to_base64(file_id)
content_item["file_data"] = base64_data
content_item["filename"] = "my_file.pdf"
content_item.pop("file_id")
return content_item
content_copy["file_data"] = base64_data
content_copy["filename"] = "my_file.pdf"
content_copy.pop("file_id")
return content_copy
def _common_file_data_check(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile:
file_data: Final = content_item.get("file_data")
filename: Final = content_item.get("filename")
if file_data is not None and filename is None:
content_item["filename"] = "my_file.pdf"
return {**content_item, "filename": "my_file.pdf"}
return content_item
@staticmethod
def _image_item_for_openai(content_item: ChatCompletionImageObject) -> ChatCompletionImageObject:
image_url: Final = content_item["image_url"]
if isinstance(image_url, str):
return {**content_item, "image_url": {"url": image_url}}
if not isinstance(image_url, dict):
return content_item
new_image_url_obj: Final = cast(
ChatCompletionImageUrlObject,
{k: v for k, v in image_url.items() if k not in {"format", "video_metadata"}},
)
return {**content_item, "image_url": new_image_url_obj}
@staticmethod
def _file_item_for_openai(content_item: ChatCompletionFileObject) -> ChatCompletionFileObject:
litellm_specific_params: Final = {"format", "detail", "video_metadata"}
file_obj: Final = content_item.get("file")
if file_obj is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=None,
llm_provider="openai",
)
new_file_obj: Final = ChatCompletionFileObjectFile(
**{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
)
return {**content_item, "file": new_file_obj}
def _apply_common_transform_content_item(
self,
content_item: OpenAIMessageContentListBlock,
) -> OpenAIMessageContentListBlock:
litellm_specific_params: Final = {"format", "detail", "video_metadata"}
if content_item.get("type") == "image_url":
content_item = cast(ChatCompletionImageObject, content_item)
if isinstance(content_item["image_url"], str):
content_item["image_url"] = {
"url": content_item["image_url"],
}
elif isinstance(content_item["image_url"], dict):
new_image_url_obj: Final = cast(
ChatCompletionImageUrlObject,
{k: v for k, v in content_item["image_url"].items() if k not in {"format", "video_metadata"}},
)
content_item["image_url"] = new_image_url_obj
elif content_item.get("type") == "file":
content_item = cast(ChatCompletionFileObject, content_item)
file_obj: Final = content_item.get("file")
if file_obj is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=None,
llm_provider="openai",
)
new_file_obj: Final = ChatCompletionFileObjectFile(
**{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
)
content_item["file"] = new_file_obj
return self._image_item_for_openai(cast(ChatCompletionImageObject, content_item))
if content_item.get("type") == "file":
return self._file_item_for_openai(cast(ChatCompletionFileObject, content_item))
return content_item
def _transform_content_item(
@ -419,31 +448,34 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
hoisted_messages: Final = hoist_images_from_tool_messages(stripped_messages)
async def _async_transform():
for message in hoisted_messages:
message_content = message.get("content")
message_role = message.get("role")
if message_role == "user" and message_content and isinstance(message_content, list):
message_content_types = cast(list[OpenAIMessageContentListBlock], message_content)
for i, content_item in enumerate(message_content_types):
message_content_types[i] = await self._async_transform_content_item(
cast(OpenAIMessageContentListBlock, content_item),
)
return hoisted_messages
return [await self._async_user_message_for_openai(message) for message in hoisted_messages]
if is_async:
return _async_transform()
else:
for message in hoisted_messages:
message_content = message.get("content")
message_role = message.get("role")
if message_role == "user" and message_content and isinstance(message_content, list):
message_content_types = cast(list[OpenAIMessageContentListBlock], message_content)
for i, content_item in enumerate(message_content):
message_content_types[i] = self._transform_content_item(
cast(OpenAIMessageContentListBlock, content_item)
)
return hoisted_messages
return [self._user_message_for_openai(message) for message in hoisted_messages]
@staticmethod
def _user_content_parts(message: AllMessageValues) -> Sequence[OpenAIMessageContentListBlock] | None:
message_content: Final = message.get("content")
if message.get("role") != "user" or not message_content or not isinstance(message_content, list):
return None
return cast(Sequence[OpenAIMessageContentListBlock], message_content)
def _user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues:
parts: Final = self._user_content_parts(message)
if parts is None:
return message
return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI
AllMessageValues, {**message, "content": [self._transform_content_item(part) for part in parts]}
)
async def _async_user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues:
parts: Final = self._user_content_parts(message)
if parts is None:
return message
return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI
AllMessageValues, {**message, "content": [await self._async_transform_content_item(part) for part in parts]}
)
def transform_messages(
self,
@ -462,23 +494,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
messages: list[AllMessageValues],
tools: list["ChatCompletionToolParam"] | None = None,
) -> tuple[list[AllMessageValues], list["ChatCompletionToolParam"] | None]:
from litellm.litellm_core_utils.prompt_templates.common_utils import (
filter_value_from_dict,
)
from litellm.types.llms.openai import ChatCompletionToolParam
for i, message in enumerate(messages):
messages[i] = cast(
AllMessageValues,
filter_value_from_dict(message, "cache_control"),
)
if tools is not None:
for i, tool in enumerate(tools):
tools[i] = cast(
ChatCompletionToolParam,
filter_value_from_dict(tool, "cache_control"),
)
return messages, tools
new_messages: Final = [cast(AllMessageValues, without_cache_control(message)) for message in messages]
new_tools: Final = (
[cast(ChatCompletionToolParam, without_cache_control(tool)) for tool in tools]
if tools is not None
else None
)
return new_messages, new_tools
def translate_developer_role_to_system_role(
self,
@ -527,6 +551,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
tools: Sequence["ChatCompletionToolParam"] | None,
prompt_cache_options: object,
litellm_params: Mapping[str, object],
request_params: Mapping[str, object],
) -> tuple[Sequence[AllMessageValues], Sequence["ChatCompletionToolParam"] | None]:
"""
A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on
@ -543,10 +568,11 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
model, custom_llm_provider, api_base, prompt_cache_options
)
if not reads_breakpoints and self._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base):
return messages, tools
drops_breakpoints: Final = not reads_breakpoints and (
litellm.drop_params is True or litellm_params.get("drop_params") is True
)
return (
_claude_messages_within_cache_control_cap(model, messages, tools, litellm_params, request_params),
tools,
)
drops_breakpoints: Final = not reads_breakpoints and _drops_params(litellm_params)
renamed: Final = (
[AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages]
if reads_breakpoints
@ -561,7 +587,13 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
if drops_breakpoints
else list(tools)
)
return self.remove_cache_control_flag_from_messages_and_tools(model=model, messages=renamed, tools=kept_tools)
sent_messages, sent_tools = self.remove_cache_control_flag_from_messages_and_tools(
model=model, messages=renamed, tools=kept_tools
)
return (
_claude_messages_within_cache_control_cap(model, sent_messages, sent_tools, litellm_params, request_params),
sent_tools,
)
def _sanitized_tools_update_for_openai(
self,
@ -623,6 +655,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
tools=optional_params.get("tools", []),
prompt_cache_options=_prompt_cache_options(optional_params),
litellm_params=litellm_params,
request_params=optional_params,
)
if tools is not None and len(tools) > 0:
optional_params["tools"] = tools
@ -654,6 +687,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
tools=optional_params.get("tools", []),
prompt_cache_options=_prompt_cache_options(optional_params),
litellm_params=litellm_params,
request_params=optional_params,
)
if tools is not None and len(tools) > 0:
optional_params["tools"] = tools

View file

@ -153,12 +153,13 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig):
- role: system ==> translate to role 'user'
"""
_supports_system_messages: Final = supports_system_messages(model, "openai")
for i, message in enumerate(messages):
transformed_messages: Final = list(messages)
for i, message in enumerate(transformed_messages):
if message["role"] == "system" and not _supports_system_messages:
new_message = ChatCompletionUserMessage(content=message["content"], role="user")
messages[i] = new_message # Replace the old message with the new one
transformed_messages[i] = new_message # Replace the old message with the new one
if is_async:
return super()._transform_messages(messages, model, is_async=cast(Literal[True], True))
return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[True], True))
else:
return super()._transform_messages(messages, model, is_async=cast(Literal[False], False))
return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[False], False))

View file

@ -3,6 +3,7 @@ Common helpers / utils across al OpenAI endpoints
"""
import asyncio
import copy
import hashlib
import inspect
import json
@ -31,7 +32,7 @@ if TYPE_CHECKING:
import litellm
from litellm.exceptions import error_response_without_credentials
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, iter_items
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params
from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -43,6 +44,47 @@ from litellm.llms.custom_httpx.http_handler import (
)
def _is_plain_dict(value: object) -> bool:
return isinstance(value, dict)
def _is_list(value: object) -> bool:
return isinstance(value, list)
def _contains_cache_control(value: object) -> bool:
pending: Final[list[object]] = [value] # mutable-ok: a local worklist, never returned or stored
while pending:
node = pending.pop()
mapping = as_str_mapping(node)
if mapping is not None:
if "cache_control" in mapping:
return True
pending.extend(mapping.values())
elif _is_list(node):
pending.extend(iter_items(node))
return False
def without_cache_control(item: object) -> object:
"""Return `item` with every `cache_control` key removed, without touching the original.
`filter_value_from_dict` deletes the key in place and recurses into nested dicts and
lists, so the caller's own message, tool or input object has to be copied first: it
may reuse the same list on a provider that does support prompt caching. The copy is
skipped when there is nothing to strip, which keeps the object identity the
pass-through paths rely on and avoids a deep copy on every request.
"""
from litellm.litellm_core_utils.prompt_templates.common_utils import filter_value_from_dict
if not _is_plain_dict(item) or not _contains_cache_control(item):
return item
copied: Final = copy.deepcopy(item)
if isinstance(copied, dict):
filter_value_from_dict(copied, "cache_control")
return copied
def _get_client_init_params(cls: type) -> tuple[str, ...]:
"""Extract __init__ parameter names (excluding 'self') from a class."""
return tuple(p for p in inspect.signature(cls.__init__).parameters if p != "self")

View file

@ -354,10 +354,10 @@ class OpenAIDecisionsConfig(BaseDecisionsConfig):
) -> str | None:
return resolve_openai_bearer_token(api_key=api_key, api_base=api_base, litellm_params=litellm_params)
def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool:
def supports_audio_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
return False
def supports_video_input(self, model: str, custom_llm_provider: str) -> bool:
def supports_video_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
return False
def transform_decisions_request(

View file

@ -52,7 +52,7 @@ from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
from litellm.types.workload_identity import OPENAI_WIF_KWARGS_KEYS
from ..common_utils import OpenAIError
from ..common_utils import OpenAIError, without_cache_control
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16
@ -658,13 +658,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
input=self._replayable_input(marker_safe_input, reads_breakpoints, maps_anthropic_tool_search),
tools=mapped_tools,
)
if marker_safe_input is not validated_input:
self._strip_caller_input_as_if_unmarked(
model=model,
caller_input=validated_input,
reads_breakpoints=reads_breakpoints,
maps_anthropic_tool_search=maps_anthropic_tool_search,
)
object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools)
sanitized_tools: Final = self._sanitized_tool_schemas_for_openai(
model=model, tools=object_schema_tools, litellm_params=litellm_params
@ -679,17 +672,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
return marked_input
return input_without_unanswered_anthropic_tool_search_calls(marked_input)
def _strip_caller_input_as_if_unmarked(
self,
model: str,
caller_input: str | ResponseInputParam,
reads_breakpoints: bool,
maps_anthropic_tool_search: bool,
) -> None:
self.remove_cache_control_flag_from_input_and_tools(
model=model, input=self._replayable_input(caller_input, reads_breakpoints, maps_anthropic_tool_search)
)
def _input_with_wire_safe_prompt_cache_breakpoints(
self,
model: str,
@ -800,24 +782,29 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
the chat path. Strips Anthropic-only `cache_control` markers from
Responses API input content blocks and tools.
`filter_value_from_dict` mutates each dict in place, so the same
objects are returned.
`filter_value_from_dict` deletes the key in place and recurses, so each
item is copied first: the caller keeps its own input list and may reuse
it on a provider that does support prompt caching.
"""
from litellm.litellm_core_utils.prompt_templates.common_utils import (
filter_value_from_dict,
new_input: Final = (
cast( # cast-ok: the comprehension rebuilds the input item for item
ResponseInputParam,
[without_cache_control(item) for item in input],
)
if isinstance(input, list)
else input
)
if isinstance(input, list):
for item in input:
if isinstance(item, dict):
filter_value_from_dict(cast(dict, item), "cache_control")
new_tools: Final = (
cast( # cast-ok: the comprehension rebuilds the tool list item for item
Sequence[ALL_RESPONSES_API_TOOL_PARAMS],
[without_cache_control(tool) for tool in tools],
)
if tools is not None
else None
)
if tools is not None:
for tool in tools:
if isinstance(tool, dict):
filter_value_from_dict(cast(dict, tool), "cache_control")
return input, tools
return new_input, new_tools
def _drop_bridge_minted_reasoning_items(self, input: str | ResponseInputParam) -> str | ResponseInputParam:
if self.custom_llm_provider not in _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING or not isinstance(input, list):

View file

@ -125,6 +125,20 @@ The provider inherits all request/response handling from OpenAI's Responses API
If `supported_endpoints` is omitted, it defaults to `[]` (only chat completions, which is always enabled for JSON providers).
When only some models serve an endpoint natively, list them under `model_supported_endpoints`. Every other model reaches `/v1/responses` and `/v1/messages` through chat completions:
```json
{
"your_provider": {
"supported_endpoints": ["/v1/chat/completions"],
"model_supported_endpoints": {
"model-a": ["/v1/messages"],
"model-b": ["/v1/responses"]
}
}
}
```
### How It Works
1. `json_loader.py` checks `supported_endpoints` for `/v1/responses`

View file

@ -3,11 +3,16 @@
## Allows jina ai embedding calls - which don't allow 'encoding_format' in payload.
import json
import re
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
import httpx
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@ -17,6 +22,57 @@ from litellm.types.utils import EmbeddingResponse
from ..common_utils import OpenAILikeBase, OpenAILikeError
_KEY_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, str])
_EXTRA_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_TRANSPORT_HEADER_NAMES: Final = frozenset(
{
"host",
"content-length",
"transfer-encoding",
"connection",
"keep-alive",
"proxy-connection",
"te",
"trailer",
"upgrade",
"expect",
}
)
_HEADER_NAME: Final = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
_HEADER_VALUE: Final = re.compile(r"[\t\x20-\x7e]*")
def _header_text(name: str, value: object) -> str | None:
if isinstance(value, bool) or not isinstance(value, str | int | float):
return None
if _HEADER_NAME.fullmatch(name) is None:
return None
return str(value).strip(" \t\r\n")
def _extra_headers_as_text(extra_headers: Mapping[str, object], model: str, drop_params: bool) -> Mapping[str, str]:
caller_set: Final = {
name: value for name, value in extra_headers.items() if name.lower() not in _TRANSPORT_HEADER_NAMES
}
texts: Final = MappingProxyType({name: _header_text(name, value) for name, value in caller_set.items()})
refused: Final = sorted(name for name, text in texts.items() if text is None)
if refused and not drop_params:
raise litellm.UnsupportedParamsError(
message=f"extra_headers {', '.join(map(repr, refused))} cannot be sent. A header needs a name made of "
"letters, digits or !#$%&'*+-.^_`|~ and a text or number value. "
"To drop them from the call, set `litellm.drop_params = True`.",
model=model,
)
named: Final = MappingProxyType({name: text for name, text in texts.items() if text is not None})
unsendable: Final = sorted(name for name, text in named.items() if _HEADER_VALUE.fullmatch(text) is None)
if unsendable:
verbose_logger.warning(
"extra_headers %s dropped for model %r, since a header value must be printable ASCII on one line",
unsendable,
model,
)
return MappingProxyType({name: text for name, text in named.items() if name not in unsendable})
class OpenAILikeEmbeddingHandler(OpenAILikeBase):
def __init__(self, **kwargs):
@ -94,6 +150,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
aembedding=None,
custom_endpoint: bool | None = None,
headers: dict | None = None,
drop_params: bool = False,
) -> EmbeddingResponse:
api_base, headers = self._validate_environment(
api_base=api_base,
@ -103,8 +160,20 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
custom_endpoint=custom_endpoint,
)
model = model
filtered_optional_params: Final = {k: v for k, v in optional_params.items() if v not in (None, "")}
data: Final = {"model": model, "input": input, **filtered_optional_params}
data: Final = {
"model": model,
"input": input,
**{k: v for k, v in optional_params.items() if k != "extra_headers" and v not in (None, "")},
}
extra_headers: Final = _extra_headers_as_text(
_EXTRA_HEADERS_ADAPTER.validate_python(optional_params.get("extra_headers") or {}), model, drop_params
)
key_headers: Final = _KEY_HEADERS_ADAPTER.validate_python(headers)
extra_header_names: Final = frozenset(name.lower() for name in extra_headers)
request_headers: Final = {
**{name: value for name, value in key_headers.items() if name.lower() not in extra_header_names},
**extra_headers,
}
## LOGGING
logging_obj.pre_call(
@ -123,7 +192,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
api_key=api_key,
timeout=timeout,
client=client,
headers=headers,
headers=request_headers,
)
if client is None or isinstance(client, AsyncHTTPHandler):
self.client = HTTPHandler(timeout=timeout)
@ -134,7 +203,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
try:
response: Final = self.client.post(
api_base,
headers=headers,
headers=request_headers,
data=json.dumps(data),
)

View file

@ -3,7 +3,7 @@ JSON-based provider configuration loader for OpenAI-compatible providers.
"""
import json
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Final
@ -14,6 +14,7 @@ from litellm.types.utils import ProviderSpecificModelInfo
_PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo)
_SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(Mapping[str, bool])
_MODEL_ENDPOINTS_ADAPTER: Final = TypeAdapter(Mapping[str, tuple[str, ...]])
class SimpleProviderConfig:
@ -30,7 +31,10 @@ class SimpleProviderConfig:
self.special_handling: Mapping[str, bool] = _SPECIAL_HANDLING_ADAPTER.validate_python(
data.get("special_handling", {})
)
self.supported_endpoints = data.get("supported_endpoints", [])
self.supported_endpoints: Sequence[str] = data.get("supported_endpoints", [])
self.model_supported_endpoints: Mapping[str, tuple[str, ...]] = _MODEL_ENDPOINTS_ADAPTER.validate_python(
data.get("model_supported_endpoints", {})
)
self.provider_info: ProviderSpecificModelInfo = (
_PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {}
)
@ -90,6 +94,23 @@ class JSONProviderRegistry:
return False
return "/v1/responses" in provider.supported_endpoints
@classmethod
def supports_embeddings(cls, slug: str) -> bool:
provider: Final = cls._providers.get(slug)
if provider is None:
return False
return "/v1/embeddings" in provider.supported_endpoints
@classmethod
def serves_endpoint(cls, slug: str, model: str | None, endpoint: str) -> bool:
provider: Final = cls._providers.get(slug)
if provider is None:
return False
model_endpoints: Final = (
provider.model_supported_endpoints.get(model.removeprefix(f"{slug}/"), ()) if model else ()
)
return endpoint in provider.supported_endpoints or endpoint in model_endpoints
@classmethod
def list_providers(cls) -> list:
"""List all registered provider slugs"""

View file

@ -211,7 +211,7 @@
"param_mappings": {
"max_completion_tokens": "max_tokens"
},
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"]
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"]
},
"scx-ai": {
"base_url": "https://api.scx.ai/v1",
@ -231,6 +231,16 @@
"api_base_env": "PRISM_API_BASE",
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
},
"hostinger_router": {
"base_url": "https://router.hostinger.com/v1",
"api_key_env": "HOSTINGER_ROUTER_API_KEY",
"api_base_env": "HOSTINGER_ROUTER_API_BASE",
"supported_endpoints": ["/v1/chat/completions", "/v1/embeddings"],
"model_supported_endpoints": {
"claude-sonnet-5": ["/v1/messages"],
"gpt-5.2": ["/v1/responses"]
}
},
"sail": {
"base_url": "https://api.sailresearch.com/v1",
"api_key_env": "SAIL_API_KEY",

View file

@ -110,38 +110,23 @@ def separate_cached_messages(
messages: list[AllMessageValues],
) -> tuple[list[AllMessageValues], list[AllMessageValues]]:
"""
Returns separated cached and non-cached messages.
Args:
messages: List of messages to be separated.
Returns:
Tuple containing:
- cached_messages: List of cached messages.
- non_cached_messages: List of non-cached messages.
Returns the first continuous block of cached messages, plus every system message once
any message is cached, since generateContent rejects a system_instruction beside a
cachedContent, and the remaining messages.
"""
cached_messages: list[AllMessageValues] = []
non_cached_messages: list[AllMessageValues] = []
filtered_messages: Final = [(idx, message) for idx, message in enumerate(messages) if is_cached_message(message)]
if not filtered_messages:
return [], messages
# Extract cached messages and their indices
filtered_messages: Final[list[tuple[int, AllMessageValues]]] = []
for idx, message in enumerate(messages):
if is_cached_message(message=message):
filtered_messages.append((idx, message))
# Validate only one block of continuous cached messages
last_continuous_block_idx: Final = get_first_continuous_block_idx(filtered_messages)
# Separate messages based on the block of cached messages
if filtered_messages and last_continuous_block_idx is not None:
first_cached_idx: Final = filtered_messages[0][0]
last_cached_idx: Final = filtered_messages[last_continuous_block_idx][0]
cached_messages = messages[first_cached_idx : last_cached_idx + 1]
non_cached_messages = messages[:first_cached_idx] + messages[last_cached_idx + 1 :]
else:
non_cached_messages = messages
return cached_messages, non_cached_messages
cached_block: Final = range(filtered_messages[0][0], filtered_messages[last_continuous_block_idx][0] + 1)
goes_in_cache: Final = tuple(
idx in cached_block or message.get("role") == "system" for idx, message in enumerate(messages)
)
return (
[message for message, cached in zip(messages, goes_in_cache) if cached],
[message for message, cached in zip(messages, goes_in_cache) if not cached],
)
def cached_messages_end_on_supported_turn(cached_messages: Sequence[AllMessageValues]) -> bool:

View file

@ -4,6 +4,7 @@ Translation from OpenAI's `/chat/completions` endpoint to IBM WatsonX's `/text/c
Docs: https://cloud.ibm.com/apidocs/watsonx-ai#text-chat
"""
import copy
from typing import Final
from litellm import verbose_logger
@ -57,7 +58,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig):
_tools = non_default_params.pop("tools", None)
if _tools is not None:
# remove 'additionalProperties' from tools
_tools = remove_additional_properties(_tools)
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
# remove 'strict' from tools
_tools = remove_strict_from_schema(_tools)
if _tools is not None:

View file

@ -1,3 +1,4 @@
import copy
from collections.abc import AsyncIterator, Iterator, Mapping
from types import MappingProxyType
from typing import Any, Final
@ -208,7 +209,8 @@ class XAIChatConfig(OpenAIGPTConfig):
tools = [
filtered_tool
for filtered_tool in (
_FILTERED_TOOL.validate_python(filter_value_from_dict(tool, "strict")) for tool in value
_FILTERED_TOOL.validate_python(filter_value_from_dict(copy.deepcopy(tool), "strict")) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
for tool in value
)
if filtered_tool is not None
]

View file

@ -6809,6 +6809,23 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse:
)
def _embedding_headers(param: str, configured: object, model: str, drop_params: bool) -> Mapping[str, object]:
if not configured:
return {}
if isinstance(configured, Mapping):
return dict(cast(Mapping[str, object], configured)) # cast-ok: header names from untyped request JSON
try:
return dict(cast(Iterable[tuple[str, object]], configured)) # cast-ok: name and value pairs from request JSON
except (TypeError, ValueError):
if drop_params:
return {}
raise litellm.UnsupportedParamsError(
message=f"{param} must be a JSON object of header names to values. "
"To drop it from the call, set `litellm.drop_params = True`.",
model=model,
)
# fmt: off
# Overload for when aembedding=True (returns coroutine)
@ -6923,11 +6940,10 @@ def embedding(
azure_ad_token_provider: Final = kwargs.get("azure_ad_token_provider", None)
aembedding: Final[bool | None] = kwargs.get("aembedding", None)
extra_headers: Final = kwargs.get("extra_headers", None)
headers = kwargs.get("headers", None) or extra_headers
if headers is None:
headers = {}
if extra_headers is not None:
headers.update(extra_headers)
headers = {
**_embedding_headers("headers", kwargs.get("headers"), model, drop_unsupported_params),
**_embedding_headers("extra_headers", extra_headers, model, drop_unsupported_params),
}
# Inject proxy auth headers if configured
if litellm.proxy_auth is not None:
try:
@ -7091,7 +7107,7 @@ def embedding(
# set API KEY
api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
if headers is not None and headers != {}:
if headers:
optional_params["extra_headers"] = headers
requested_encoding_format: Final = (
@ -7183,14 +7199,15 @@ def embedding(
custom_llm_provider == "openai_like"
or custom_llm_provider == "llamafile"
or custom_llm_provider == "lm_studio"
or JSONProviderRegistry.supports_embeddings(custom_llm_provider)
):
api_base = api_base or litellm.api_base or get_secret_str("OPENAI_LIKE_API_BASE")
# set API KEY
if api_key is None:
api_key = api_key or litellm.api_key or litellm.openai_like_key or get_secret_str("OPENAI_LIKE_API_KEY")
if api_key is None and not JSONProviderRegistry.supports_embeddings(custom_llm_provider):
api_key = litellm.api_key or litellm.openai_like_key or get_secret_str("OPENAI_LIKE_API_KEY")
if headers is not None and headers != {}:
if headers:
optional_params["extra_headers"] = headers
## EMBEDDING CALL
@ -7205,10 +7222,9 @@ def embedding(
optional_params=optional_params,
client=client,
aembedding=aembedding,
drop_params=drop_unsupported_params,
)
elif custom_llm_provider == "oci":
if headers is None:
headers = {}
response = base_llm_http_handler.embedding(
model=model,
input=input,
@ -7233,11 +7249,6 @@ def embedding(
or litellm.api_key
)
# Use the merged headers variable (already merged at the top of the function)
# Don't overwrite it with just extra_headers
if headers is None:
headers = {}
response = base_llm_http_handler.embedding(
model=model,
input=input,

View file

@ -39687,6 +39687,57 @@
"/v1/images/generations"
]
},
"hostinger_router/gpt-5.2": {
"cache_read_input_token_cost": 0.00000021,
"input_cost_per_token": 0.0000021,
"litellm_provider": "hostinger_router",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 0.0000168,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_vision": true,
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
"source": "https://router.hostinger.com/v1/models"
},
"hostinger_router/claude-sonnet-5": {
"cache_creation_input_token_cost": 3.3e-06,
"cache_creation_input_token_cost_above_1hr": 5.28e-06,
"cache_read_input_token_cost": 2.64e-07,
"input_cost_per_token": 2.64e-06,
"litellm_provider": "hostinger_router",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.32e-05,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_response_schema": true,
"supported_endpoints": ["/v1/chat/completions", "/v1/messages"],
"source": "https://router.hostinger.com/v1/models"
},
"hostinger_router/text-embedding-3-large": {
"input_cost_per_token": 1.716e-07,
"litellm_provider": "hostinger_router",
"max_input_tokens": 8191,
"mode": "embedding",
"output_cost_per_token": 0.0,
"source": "https://router.hostinger.com/v1/models"
},
"hostinger_router/text-embedding-3-small": {
"input_cost_per_token": 3e-08,
"litellm_provider": "hostinger_router",
"max_input_tokens": 8191,
"mode": "embedding",
"output_cost_per_token": 0.0,
"source": "https://router.hostinger.com/v1/models"
},
"hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": {
"input_cost_per_token": 1.2e-07,
"litellm_provider": "hyperbolic",

View file

@ -1087,6 +1087,32 @@
"generateContent": false
}
},
"hostinger_router": {
"display_name": "Hostinger Router (`hostinger_router`)",
"url": "https://docs.litellm.ai/docs/providers/hostinger_router",
"endpoints": {
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": true,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": false,
"a2a": true,
"generateContent": true
}
},
"crusoe": {
"display_name": "Crusoe (`crusoe`)",
"url": "https://docs.litellm.ai/docs/providers/crusoe",

View file

@ -52,7 +52,11 @@ from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming, is_guardrail_intervention
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.fallback_generalizations import match_routing_generalization
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers
from litellm.llms.anthropic.common_utils import (
AnthropicModelInfo,
anthropic_pass_through_api_base,
merge_anthropic_beta_headers,
)
from litellm.llms.azure.passthrough.transformation import (
foreign_azure_deployment,
is_azure_body_model_inference_endpoint,
@ -1047,7 +1051,7 @@ async def anthropic_proxy_route(
"""
[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)
"""
base_target_url = os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
base_target_url = anthropic_pass_through_api_base()
encoded_endpoint = httpx.URL(endpoint).path
# Ensure endpoint starts with '/' for proper URL construction

View file

@ -71,7 +71,11 @@ from litellm.litellm_core_utils.litellm_logging import get_masked_values
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.anthropic.common_utils import is_anthropic_messages_url
from litellm.llms.anthropic.common_utils import (
anthropic_accepts_user_id,
anthropic_upstream_user_id_limit,
is_anthropic_messages_url,
)
from litellm.llms.base_llm.managed_resources.utils import (
resolve_passthrough_managed_id_provider,
)
@ -503,7 +507,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
url: httpx.URL,
headers: dict,
requested_query_params: dict | None = None,
_parsed_body: dict | None = None,
_parsed_body: Mapping[str, object] | None = None,
forward_multipart: bool = False,
) -> httpx.Response:
"""
@ -717,8 +721,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
# unenforced. Set after the client merge so a request body cannot supply its own budget.
#
# Only for the built-in provider routes. `get_model_from_request` returns
# None for a user-defined pass-through, deliberately: its body is forwarded
# verbatim, so `model` there names an UPSTREAM model rather than a
# None for a user-defined pass-through, deliberately: the client's `model`
# goes upstream unchanged, so it names an UPSTREAM model rather than a
# LiteLLM-managed one. Enforcement is therefore skipped on those routes, and
# charging a counter anyway would track spend that nothing can refuse, and
# would attribute it to a budget the operator scoped to a LiteLLM model that
@ -1187,6 +1191,36 @@ def _unfiltered_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
return headers
def _json_object(value: object) -> Mapping[str, object]:
return cast("Mapping[str, object]", value) if isinstance(value, Mapping) else MappingProxyType({}) # cast-ok: json
def _anthropic_user_id_to_forward(
body: Mapping[str, object],
url: str,
has_pre_call_guardrails: Callable[[Mapping[str, object]], bool],
) -> str | None:
user_id: Final = _json_object(body.get("metadata")).get("user_id")
if not isinstance(user_id, str):
return None
max_length: Final = anthropic_upstream_user_id_limit(url)
if max_length is None:
return None
if not anthropic_accepts_user_id(user_id, max_length):
verbose_proxy_logger.warning("Not forwarding metadata.user_id because the upstream would refuse it")
return None
if has_pre_call_guardrails(body):
verbose_proxy_logger.debug("Not forwarding metadata.user_id because a pre-call guardrail cannot scan it")
return None
return user_id
def _with_anthropic_user_id(body: Mapping[str, object] | None, user_id: str | None) -> Mapping[str, object] | None:
if body is None or user_id is None:
return body
return {**body, "metadata": {"user_id": user_id}}
def _eu_jev_client_body(custom_llm_provider: str | None, body: Mapping[str, object]) -> Mapping[str, object] | None:
return MappingProxyType(dict(body)) if custom_llm_provider == "eu_jev" else None
@ -1423,6 +1457,10 @@ async def pass_through_request(
request_method=getattr(request, "method", None),
cost_per_request=cost_per_request,
)
hooked_body: Final = _json_object(_parsed_body)
client_user_id: Final = _anthropic_user_id_to_forward(
hooked_body, str(url), proxy_logging_obj.request_has_pre_call_guardrails
)
kwargs = HttpPassThroughEndpointHelpers.init_kwargs_for_pass_through_endpoint( # rebind-ok: pre-existing rebinding on a rename-only line
user_api_key_dict=user_api_key_dict,
_parsed_body=_parsed_body,
@ -1590,11 +1628,13 @@ async def pass_through_request(
else:
logging_url = str(url) + "?" + requested_query_params_str
outgoing_body: Final = cast("Mapping[str, object] | None", _parsed_body) # cast-ok: json
upstream_body: Final = _with_anthropic_user_id(outgoing_body, client_user_id)
logging_obj.pre_call(
input=[{"role": "user", "content": safe_dumps(_parsed_body)}],
api_key="",
additional_args={
"complete_input_dict": _parsed_body,
"complete_input_dict": upstream_body,
"api_base": str(logging_url),
"headers": redact_credential_headers(upstream_headers),
},
@ -1634,7 +1674,7 @@ async def pass_through_request(
url,
params=requested_query_params,
headers=upstream_headers,
json=_parsed_body,
json=upstream_body,
)
)
@ -1733,7 +1773,7 @@ async def pass_through_request(
url=url,
headers=upstream_headers,
requested_query_params=requested_query_params,
_parsed_body=_parsed_body,
_parsed_body=upstream_body,
forward_multipart=is_multipart,
)
verbose_proxy_logger.debug("response.headers= %s", response.headers)

View file

@ -3343,6 +3343,34 @@
],
"default_model_placeholder": "prism/deepseek-v4.1-flash"
},
{
"provider": "HOSTINGER_ROUTER",
"provider_display_name": "Hostinger Router",
"litellm_provider": "hostinger_router",
"credential_fields": [
{
"key": "api_base",
"label": "API Base",
"placeholder": "https://router.hostinger.com/v1",
"tooltip": null,
"required": false,
"field_type": "text",
"options": null,
"default_value": null
},
{
"key": "api_key",
"label": "API Key",
"placeholder": null,
"tooltip": null,
"required": true,
"field_type": "password",
"options": null,
"default_value": null
}
],
"default_model_placeholder": "hostinger_router/claude-sonnet-5"
},
{
"provider": "RECRAFT",
"provider_display_name": "Recraft",

View file

@ -3031,14 +3031,21 @@ class ProxyLogging:
hook to consult, but it judges the payload the same way, so a proxy configured only with
one of those still has something to say about every record.
"""
if request_metadata.get("_guardrail_pipelines"):
return self.request_has_pre_call_guardrails({"metadata": dict(request_metadata)}, event_type)
def request_has_pre_call_guardrails(
self,
request_data: Mapping[str, object],
event_type: GuardrailEventHooks = GuardrailEventHooks.pre_call,
) -> bool:
if _policy_state_metadata(request_data).get("_guardrail_pipelines"):
return True
caps: Final = ProxyLogging.callback_capabilities()
if caps.has_content_enforcer:
return True
probe: Final = {"metadata": dict(request_metadata)}
return any(
isinstance(callback, CustomGuardrail) and callback.should_run_guardrail(data=probe, event_type=event_type)
isinstance(callback, CustomGuardrail)
and callback.should_run_guardrail(data=request_data, event_type=event_type)
for callback in caps.resolved_callbacks
)

View file

@ -8,7 +8,7 @@ import uuid
from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from functools import partial
from itertools import accumulate, chain, groupby, islice
from itertools import chain, groupby
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
@ -45,7 +45,15 @@ from litellm.constants import (
REDACTED_BY_LITELLM,
REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER,
)
from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook
from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
block_with_cache_control,
cache_control_count,
dropped_cache_control,
newest_marks_within_cache_control_limit,
whole_messages_within_cache_control_limit,
with_cache_control,
)
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params
from litellm.litellm_core_utils.get_supported_openai_params import (
get_supported_openai_params,
@ -603,10 +611,6 @@ def _validated_cache_control(value: object) -> ChatCompletionCachedContent | Non
return None
def _dropped_cache_control(_: object) -> None:
return None
def _without_cache_control_key(item: object) -> object:
if not isinstance(item, dict):
return item
@ -694,83 +698,6 @@ def _content_blocks_carrying(content: object, cache_control: ChatCompletionCache
return _with_cache_control_on_last_block(_OBJECT_LIST_ADAPTER.validate_python(content), cache_control)
_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls")
def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]:
lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS)
return tuple(
chain.from_iterable(_OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list))
)
def _block_carries_cache_control(block: object) -> bool:
return isinstance(block, dict) and _STR_KEY_DICT_ADAPTER.validate_python(block).get("cache_control") is not None
def _cache_control_count(message: object) -> int:
if not isinstance(message, dict):
return 0
fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message)
block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block))
return int(fields.get("cache_control") is not None) + block_marks
_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None]
def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]:
others: Final = {key: value for key, value in fields.items() if key != "cache_control"}
if "cache_control" not in fields:
return others
cache_control: Final = rewrite(fields["cache_control"])
return others if cache_control is None else {**others, "cache_control": cache_control}
def _block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object:
if not isinstance(block, dict):
return block
return _fields_with_cache_control(_STR_KEY_DICT_ADAPTER.validate_python(block), rewrite)
def _with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT:
if not isinstance(message, dict):
return message
fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message)
blocks_rewritten: Final = {
key: (
[_block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)]
if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list)
else value
)
for key, value in fields.items()
}
return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped
_MessageT, _fields_with_cache_control(blocks_rewritten, rewrite)
)
def _kept_within_budget(state: tuple[int, bool], marks: int) -> tuple[int, bool]:
remaining, _ = state
if marks > remaining:
return remaining, False
return remaining - marks, True
def _messages_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]:
budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0)
newest_first: Final = tuple(reversed(messages))
budget_after_each: Final = accumulate(
(_cache_control_count(message) for message in newest_first), _kept_within_budget, initial=(budget, True)
)
kept_flags: Final = tuple(kept for _, kept in islice(budget_after_each, 1, None))
kept_newest_first: Final = tuple(
message if kept else _with_cache_control(message, _dropped_cache_control)
for message, kept in zip(newest_first, kept_flags)
)
return tuple(reversed(kept_newest_first))
_SYSTEM_MESSAGE_FIELDS: Final = frozenset(("content", "cache_control"))
@ -799,7 +726,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT:
if fields is None:
return message
if fields.get("role") == "tool":
return _with_cache_control(message, _dropped_cache_control)
return with_cache_control(message, dropped_cache_control)
tool_calls: Final = fields.get("tool_calls")
if not isinstance(tool_calls, list):
return message
@ -808,7 +735,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT:
{
**fields,
"tool_calls": [
_block_with_cache_control(tool_call, _dropped_cache_control)
block_with_cache_control(tool_call, dropped_cache_control)
for tool_call in _OBJECT_LIST_ADAPTER.validate_python(tool_calls)
],
},
@ -822,7 +749,7 @@ class _ItemMarkRules:
external_marks: int
def sanitized(self, message: _MessageT) -> _MessageT:
valid: Final = _with_cache_control(message, _validated_cache_control)
valid: Final = with_cache_control(message, _validated_cache_control)
return _without_tool_block_marks(valid) if self.refuses_tool_block_marks else valid
def checked(self, message: _MessageT) -> _MessageT:
@ -845,15 +772,15 @@ def _request_messages_for_cache_points(messages: Sequence[_MessageT], rules: _It
as_system: Final = tuple(_developer_message_as_system_message(message) for message in messages)
if not rules.drop_params:
return as_system
return _messages_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks)
return newest_marks_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks)
def _replayed_messages_for_cache_points(
replayed_messages: Sequence[_MessageT], current_messages: Sequence[object], rules: _ItemMarkRules
) -> Sequence[_MessageT]:
as_system: Final = tuple(_developer_message_as_system_message(message) for message in replayed_messages)
current_marks: Final = sum(_cache_control_count(message) for message in current_messages)
return _messages_within_cache_control_limit(
current_marks: Final = sum(cache_control_count(message) for message in current_messages)
return whole_messages_within_cache_control_limit(
tuple(map(rules.sanitized, as_system)), current_marks + rules.external_marks
)

View file

@ -643,6 +643,7 @@ _PRE_CALL_CHECK_FILTERED_PARAMS: Final = frozenset({"response_format"})
_STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS: Final = frozenset({"anthropic_messages", "aresponses", "responses"})
_video_call_deployment_id: Final[ContextVar[str | None]] = ContextVar("video_call_deployment_id", default=None)
_routed_call_type: Final[ContextVar[str | None]] = ContextVar("routed_call_type", default=None)
_DECISIONS_MEDIA_CALL_TYPES: Final = frozenset({"adecisions", "decisions", "asystemone", "systemone"})
_ROUTE_CAPABILITY_RESOLVERS: Final[Mapping[str, Callable[[str, LlmProviders], object | None]]] = MappingProxyType(
{
"adecisions": ProviderConfigManager.get_provider_decisions_config,
@ -4174,13 +4175,16 @@ class Router(DeprecatedRouterMembers):
raise e.original_exception from e
raise e
def _active_default_litellm_params(self) -> Mapping[str, object]:
default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self._core_default_litellm_params)
return {key: value for key, value in default_params.items() if value is not None}
def _params_sent_to_deployment(
self, request_kwargs: Mapping[str, object], deployment: Deployment | None
) -> Mapping[str, object]:
"""The request as the deployment received it, merged in the order _update_kwargs_with_deployment applies."""
deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {}
default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self._core_default_litellm_params)
defaults: Final = {key: value for key, value in default_params.items() if value is not None}
defaults: Final = self._active_default_litellm_params()
deployment_tool_choice: Final = (
{"tool_choice": deployment_params["tool_choice"]}
if deployment_params.get("tool_choice") is not None
@ -14945,6 +14949,62 @@ class Router(DeprecatedRouterMembers):
return healthy_deployments
return cast(_RouteDeploymentsT, filtered_deployments) # cast-ok: same list shape as the input candidates
@staticmethod
def _decisions_deployment_refuses_media(
deployment: DeploymentTypedDict, group: str, request_kwargs: Mapping[str, object], *, audio: bool, videos: bool
) -> bool:
from litellm.decisions.call import deployment_refuses_media
plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts
model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict
deployment_params: Final = cast(Mapping[str, object], deployment["litellm_params"]) # cast-ok: plain dict
try:
model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group)
return deployment_refuses_media(
model,
custom_llm_provider,
model_info,
{**deployment_params, **request_kwargs},
audio=audio,
videos=videos,
)
except Exception as e: # noqa: BLE001 # best-effort filter: a deployment it cannot resolve answers for itself
verbose_router_logger.debug(
"litellm.router.py::_decisions_deployment_refuses_media: keeping a deployment of model=%s. Got - %s",
group,
e,
)
return False
def _without_decisions_deployments_refusing_media(
self, healthy_deployments: _RouteDeploymentsT, group: str, request_kwargs: Mapping[str, object] | None
) -> _RouteDeploymentsT:
call_kwargs: Final = request_kwargs or {}
audio: Final = bool(call_kwargs.get("audio"))
videos: Final = bool(call_kwargs.get("videos"))
if (
_routed_call_type.get() not in _DECISIONS_MEDIA_CALL_TYPES
or not (audio or videos)
or isinstance(healthy_deployments, Mapping)
):
return healthy_deployments
sent_kwargs: Final = {**self._active_default_litellm_params(), **call_kwargs}
deployments_taking_media: Final = [
deployment
for deployment in healthy_deployments
if not Router._decisions_deployment_refuses_media(
cast(DeploymentTypedDict, deployment), # cast-ok: router candidates use the deployment shape
group,
sent_kwargs,
audio=audio,
videos=videos,
)
]
if not deployments_taking_media:
return healthy_deployments
return cast(_RouteDeploymentsT, deployments_taking_media) # cast-ok: same list shape as the input candidates
@staticmethod
def _without_declared_text_only_deployments(
healthy_deployments: _RouteDeploymentsT,
@ -15669,10 +15729,14 @@ class Router(DeprecatedRouterMembers):
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
_target_order: Final = (request_kwargs or {}).pop("_target_order", None)
batch_candidates: Final = self._without_declared_text_only_deployments(
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
batch_candidates: Final = self._without_decisions_deployments_refusing_media(
self._without_declared_text_only_deployments(
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
model,
messages,
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
),
model,
messages,
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
)
embedding_batch_admitting: Final = deployments_admitting_embedding_batch(
@ -16916,10 +16980,14 @@ class Router(DeprecatedRouterMembers):
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
_target_order: Final = (request_kwargs or {}).pop("_target_order", None)
batch_candidates: Final = self._without_declared_text_only_deployments(
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
batch_candidates: Final = self._without_decisions_deployments_refusing_media(
self._without_declared_text_only_deployments(
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
model,
messages,
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
),
model,
messages,
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
)
embedding_batch_admitting: Final = deployments_admitting_embedding_batch(

View file

@ -4683,6 +4683,7 @@ class LlmProviders(str, Enum):
PRISM = "prism"
OPENINFER = "openinfer"
INFERENCEAPIS = "inferenceapis"
HOSTINGER_ROUTER = "hostinger_router"
ZEROGPU = "zerogpu"
DARKBLOOM = "darkbloom"
META = "meta"

View file

@ -10159,7 +10159,7 @@ class ProviderConfigManager:
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
json_provider: Final = JSONProviderRegistry.get(provider.value)
if json_provider is not None and "/v1/messages" in json_provider.supported_endpoints:
if json_provider is not None and JSONProviderRegistry.serves_endpoint(provider.value, model, "/v1/messages"):
from litellm.llms.openai_like.messages.transformation import (
JSONProviderAnthropicMessagesConfig,
)
@ -10298,7 +10298,7 @@ class ProviderConfigManager:
return result
# Fall back to JSON providers (generic OpenAI-compatible)
if JSONProviderRegistry.exists(provider_str) and JSONProviderRegistry.supports_responses_api(provider_str):
if JSONProviderRegistry.serves_endpoint(provider_str, model, "/v1/responses"):
provider_config: Final = JSONProviderRegistry.get(provider_str)
if provider_config is not None:
return create_responses_config_class(provider_config)()

View file

@ -39687,6 +39687,57 @@
"/v1/images/generations"
]
},
"hostinger_router/gpt-5.2": {
"cache_read_input_token_cost": 0.00000021,
"input_cost_per_token": 0.0000021,
"litellm_provider": "hostinger_router",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 0.0000168,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_vision": true,
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
"source": "https://router.hostinger.com/v1/models"
},
"hostinger_router/claude-sonnet-5": {
"cache_creation_input_token_cost": 3.3e-06,
"cache_creation_input_token_cost_above_1hr": 5.28e-06,
"cache_read_input_token_cost": 2.64e-07,
"input_cost_per_token": 2.64e-06,
"litellm_provider": "hostinger_router",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.32e-05,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_response_schema": true,
"supported_endpoints": ["/v1/chat/completions", "/v1/messages"],
"source": "https://router.hostinger.com/v1/models"
},
"hostinger_router/text-embedding-3-large": {
"input_cost_per_token": 1.716e-07,
"litellm_provider": "hostinger_router",
"max_input_tokens": 8191,
"mode": "embedding",
"output_cost_per_token": 0.0,
"source": "https://router.hostinger.com/v1/models"
},
"hostinger_router/text-embedding-3-small": {
"input_cost_per_token": 3e-08,
"litellm_provider": "hostinger_router",
"max_input_tokens": 8191,
"mode": "embedding",
"output_cost_per_token": 0.0,
"source": "https://router.hostinger.com/v1/models"
},
"hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": {
"input_cost_per_token": 1.2e-07,
"litellm_provider": "hyperbolic",

View file

@ -1087,6 +1087,32 @@
"generateContent": false
}
},
"hostinger_router": {
"display_name": "Hostinger Router (`hostinger_router`)",
"url": "https://docs.litellm.ai/docs/providers/hostinger_router",
"endpoints": {
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": true,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": false,
"a2a": true,
"generateContent": true
}
},
"crusoe": {
"display_name": "Crusoe (`crusoe`)",
"url": "https://docs.litellm.ai/docs/providers/crusoe",

View file

@ -383,6 +383,8 @@ exclude-newer = "3 days"
[tool.uv.exclude-newer-package]
# TODO: remove after 2026-10-25, once brotli 1.2.1 (CVE-2026-111497) clears the 3-day window
brotli = "2026-10-22T14:01:00Z"
# TODO: remove after 2026-10-31, once pypdf 6.21.4 clears the 3-day window
pypdf = "2026-10-28T17:01:00Z"
[tool.uv.sources]
litellm-proxy-extras = { workspace = true }

View file

@ -183,6 +183,9 @@ ignored_function_names = [
"_aresponses_background_stream_can_release", # Tested through the background Responses stream tests in test_router.py
"_raise_released_stream_error", # Tested through the background Responses stream content-policy tests in test_router.py
"_mid_stream_error_before_output", # Tested through the Responses mid-stream fallback and background stream tests
"_without_decisions_deployments_refusing_media", # Tested through the mixed media-flag group tests in tests/unit/decisions/test_main.py
"_decisions_deployment_refuses_media", # Read by _without_decisions_deployments_refusing_media, tested through the same group tests
"_active_default_litellm_params", # Tested through the router default drop_params test in tests/unit/decisions/test_main.py
]

View file

@ -317,7 +317,6 @@ def test_credentialless_proxy_forwards_the_caller_anthropic_credential_and_never
def test_anthropic_sdk_metadata_user_id_reaches_upstream(api_key_proxy: Gateway, wire: Wire) -> None:
pytest.skip("BUG: /anthropic/v1/messages drops the native metadata.user_id field before forwarding upstream")
with api_key_proxy.scenario() as scenario:
key: Final = scenario.key()
sent: Final[list[httpx.Request]] = []

View file

@ -0,0 +1,123 @@
import json
import uuid
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Final, Literal
import httpx
import pytest
from integration._support.client import Gateway, eventually, object_value
from integration._support.database import read_rows
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
_API_KEY: Final = "synthetic-decisions-key"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_INPUT: Final = "Listen to the attached clip."
_AUDIO: Final = ("data:audio/wav;base64,UklGRiQAAABXQVZF",)
_VIDEOS: Final = ("data:video/mp4;base64,AAAAIGZ0eXBpc29t",)
_QUESTION: Final[dict[str, JsonValue]] = {
"type": "predicate",
"name": "banana",
"instructions": "Does the clip name a banana?",
}
_SPEND_QUERY: Final = 'SELECT status, model_group FROM "LiteLLM_SpendLogs" WHERE request_id = %s'
Route = Literal["/v1/decisions", "/v1/systemone"]
Media = Literal["audio", "videos"]
@dataclass(frozen=True, slots=True)
class _Upstream:
model: str
def reply(self, request: Request) -> Reply:
questions: Final = object_value(_JSON_OBJECT.validate_json(request.body)["questions"])
plain: Final[dict[str, JsonValue]] = {
"model": self.model,
"answers": {name: {"type": "noul", "noul": 0.9} for name in questions},
"usage": {"input_tokens": 41, "output_tokens": 0},
}
return Reply(body=json.dumps(plain).encode())
def _unlisted_openrouter_model() -> _Upstream:
return _Upstream(f"openrouter/example/omni-{uuid.uuid4().hex[:12]}")
def _decide(gateway: Gateway, model: str, route: Route, media: Mapping[str, JsonValue]) -> httpx.Response:
body: Final[dict[str, JsonValue]] = (
{"model": model, "input": _INPUT, "questions": [_QUESTION], "num_retries": 0, **media}
if route == "/v1/decisions"
else {
"model": model,
"state": _INPUT,
"questions": {"banana": {"type": "noul", "instructions": _QUESTION["instructions"]}},
"num_retries": 0,
**media,
}
)
return gateway.request("POST", route, body)
def _sent_media(request: Request) -> dict[str, JsonValue]:
body: Final = _JSON_OBJECT.validate_json(request.body)
return {kind: body[kind] for kind in ("audio", "videos") if kind in body}
def _assert_refused(response: httpx.Response, kinds: tuple[Media, ...]) -> None:
assert response.status_code == 400, response.text
assert f"does not support parameters: {list(kinds)}" in response.text, response.text
_BOTH: Final[dict[str, JsonValue]] = {"audio": list(_AUDIO), "videos": list(_VIDEOS)}
@pytest.mark.parametrize("route", ("/v1/decisions", "/v1/systemone"))
def test_a_deployment_flagged_for_audio_and_video_sends_both_clips_for_a_model_the_cost_map_does_not_list(
gateway: Gateway, route: Route
) -> None:
upstream: Final = _unlisted_openrouter_model()
with wire_server(upstream.reply) as wire, gateway.scenario() as scenario:
flags: Final[dict[str, JsonValue]] = {"supports_audio_input": True, "supports_video_input": True}
flagged: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info=flags)
flagged_dropping: Final = scenario.model(
model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info=flags, drop_params=True
)
audio_only: Final = scenario.model(
model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info={"supports_audio_input": True}
)
unflagged: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY)
_assert_refused(_decide(gateway, unflagged, route, _BOTH), ("audio", "videos"))
_assert_refused(_decide(gateway, audio_only, route, _BOTH), ("videos",))
assert wire.drain() == ()
answered: Final = _decide(gateway, flagged, route, _BOTH)
assert answered.status_code == 200, answered.text
(call,) = wire.drain()
assert _sent_media(call) == _BOTH
rows: Final = eventually(
lambda: read_rows(_SPEND_QUERY, (answered.headers["x-litellm-call-id"],)),
lambda found: len(found) == 1,
seconds=70,
)
assert (rows[0]["status"], rows[0]["model_group"]) == ("success", flagged)
kept: Final = _decide(gateway, flagged_dropping, route, _BOTH)
assert kept.status_code == 200, kept.text
(kept_call,) = wire.drain()
assert _sent_media(kept_call) == _BOTH
@pytest.mark.parametrize(
"value",
({"supports_audio_input": True, "supports_video_input": True}, 5, [{"supports_audio_input": True}], "x" * 5000),
ids=("flags", "int", "list", "5kb"),
)
def test_a_caller_sent_model_info_never_turns_media_on(gateway: Gateway, value: JsonValue) -> None:
upstream: Final = _unlisted_openrouter_model()
with wire_server(upstream.reply) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY)
_assert_refused(_decide(gateway, model, "/v1/decisions", {**_BOTH, "model_info": value}), ("audio", "videos"))
assert wire.drain() == ()

View file

@ -7,11 +7,12 @@ import json
import re
import threading
import uuid
from datetime import datetime
from collections.abc import Mapping, Sequence
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
from concurrent.futures import wait
from datetime import datetime
from functools import partial
from types import MappingProxyType
from typing import Final
from typing import Final, TypeAlias
import httpx
import pytest
@ -25,6 +26,7 @@ import litellm
from litellm.constants import AWS_SIGNING_MAX_THREADS
from litellm.cost_calculator import get_response_cost_from_hidden_params
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded, refetch_model_cost_map
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers
from litellm.llms.bedrock.base_aws_llm import AWS_SIGNING_EXECUTOR
@ -3741,6 +3743,387 @@ async def test_a_model_info_object_instead_of_a_dict_still_reaches_the_provider(
assert response.answers[0].model_dump(mode="json")["name"] == "is_defect"
_AUDIO_CLIPS: Final = ("data:audio/wav;base64,YQ==",)
_VIDEO_CLIPS: Final = ("data:video/mp4;base64,YQ==",)
@pytest.mark.asyncio
async def test_a_deployment_that_advertises_media_sends_it_for_a_model_the_cost_map_does_not_list(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
backend: Final = "openrouter/example/omni-preview"
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
router: Final = litellm.Router(
model_list=[
{
"model_name": "hears",
"litellm_params": {"model": backend, "api_key": "k"},
"model_info": {"id": "media-hears", "supports_audio_input": True},
},
{
"model_name": "hears-and-sees",
"litellm_params": {"model": backend, "api_key": "k"},
"model_info": {"id": "media-both", "supports_audio_input": True, "supports_video_input": True},
},
]
)
send: Final = partial(
router.adecisions, input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS
)
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['videos']")):
await send(model="hears")
assert not route.called
await send(model="hears-and-sees")
sent: Final = json.loads(route.calls.last.request.content)
assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS))
@pytest.mark.asyncio
async def test_media_flags_on_a_deployment_whose_own_mode_is_not_decisions_never_send_clips(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
advertised: Final = {"supports_audio_input": True, "supports_video_input": True}
router: Final = litellm.Router(
model_list=[
{
"model_name": mode,
"litellm_params": {"model": f"openrouter/example/omni-{mode}-preview", "api_key": "k"},
"model_info": {"id": f"media-{mode}", "mode": mode, **advertised},
}
for mode in ("chat", "decisions")
]
)
send: Final = partial(
router.adecisions, input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS
)
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio', 'videos']")):
await send(model="chat")
assert not route.called
await send(model="decisions")
sent: Final = json.loads(route.calls.last.request.content)
assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS))
CatalogLoader: TypeAlias = Callable[[Mapping[str, Mapping[str, object]]], Awaitable[None]]
@pytest.fixture
def load_catalog_with(monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None) -> Iterator[CatalogLoader]:
async def load(rows: Mapping[str, Mapping[str, object]]) -> None:
catalog: Final = {**litellm.get_model_cost_map(url=""), **rows}
transport: Final = httpx.MockTransport(lambda _: httpx.Response(200, json=catalog))
monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP")
async with httpx.AsyncClient(transport=transport) as client:
loaded: Final = await refetch_model_cost_map(url="https://catalog.example/prices.json", client=client)
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
assert isinstance(loaded, ModelCostMapReloaded)
monkeypatch.setattr(litellm, "model_cost", loaded.model_cost_map)
litellm.get_model_info.cache_clear()
yield load
litellm.get_model_cost_map(url="")
@pytest.mark.asyncio
async def test_media_flags_on_a_deployment_without_its_own_mode_follow_the_mode_the_catalog_lists(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, load_catalog_with: CatalogLoader
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
await load_catalog_with(
{
f"openrouter/example/omni-{mode}": {"litellm_provider": "openrouter", "mode": mode}
for mode in ("chat", "decisions")
}
)
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
router: Final = litellm.Router(
model_list=[
{
"model_name": mode,
"litellm_params": {"model": f"openrouter/example/omni-{mode}", "api_key": "k"},
"model_info": {"id": f"listed-{mode}", "supports_audio_input": True, "supports_video_input": True},
}
for mode in ("chat", "decisions")
]
)
send: Final = partial(
router.adecisions, input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS
)
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio', 'videos']")):
await send(model="chat")
assert not route.called
monkeypatch.setattr(litellm, "drop_params", True)
await send(model="chat")
dropped: Final = json.loads(route.calls.last.request.content)
assert ("audio" in dropped, "videos" in dropped) == (False, False)
await send(model="decisions")
sent: Final = json.loads(route.calls.last.request.content)
assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS))
@pytest.mark.asyncio
@pytest.mark.parametrize("sibling_mode", ("responses", "embedding"))
async def test_another_deployments_mode_never_switches_off_a_deployments_own_media_flags(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, sibling_mode: str
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
shared_params: Final = {"model": "openrouter/example/omni-shared", "api_key": "k"}
router: Final = litellm.Router(
model_list=[
{"model_name": "sibling", "litellm_params": shared_params, "model_info": {"id": "s", "mode": sibling_mode}},
{
"model_name": "flagged",
"litellm_params": shared_params,
"model_info": {"id": "f", "supports_audio_input": True, "supports_video_input": True},
},
]
)
await router.adecisions(
model="flagged", input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS
)
sent: Final = json.loads(route.calls.last.request.content)
assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS))
_MEDIA_SURFACES: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType(
{
"adecisions": {"input": "review", "questions": _PREDICATE_QUESTIONS},
"asystemone": {"state": "review", "questions": _QUESTIONS},
"systemone": {"state": "review", "questions": _QUESTIONS},
}
)
def _group_member(
group: str, model: str, key: str, weight: int, model_info: Mapping[str, object], **params: object
) -> dict[str, object]:
return {
"model_name": group,
"litellm_params": {"model": model, "api_key": key, "weight": weight, **params},
"model_info": {"id": f"{group}-{key}", **model_info},
}
@pytest.mark.asyncio
@pytest.mark.parametrize("surface", tuple(_MEDIA_SURFACES))
async def test_a_group_sends_clips_to_a_deployment_that_takes_them_and_refuses_only_when_none_does(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, surface: str
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct")
monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False)
monkeypatch.setitem(
litellm.model_cost,
"cloudflare/@cf/cloudflare/clef-omni",
{"litellm_provider": "cloudflare", "mode": "evaluation", "supports_audio_input": True},
)
respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
respx_mock.post("https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef-omni").respond(
json=_RESPONSE
)
unlisted: Final = "openrouter/example/omni-preview"
listed: Final = "cloudflare/clef-omni"
router: Final = litellm.Router(
model_list=[
_group_member("unlisted", unlisted, "refuses", 1, {}),
_group_member("unlisted", unlisted, "takes", 0, {"supports_audio_input": True}),
_group_member("listed", listed, "refuses", 1, {"supports_audio_input": False}),
_group_member("listed", listed, "takes", 0, {}),
_group_member("none", unlisted, "refuses", 1, {}),
_group_member("none", listed, "refuses", 1, {"supports_audio_input": False}),
_group_member("drops", unlisted, "drops", 1, {}, drop_params=True),
_group_member("drops", listed, "takes", 0, {}),
_group_member("unresolvable", "databricks/not a serving endpoint", "unresolvable", 0, {}),
_group_member("unresolvable", listed, "takes", 1, {}),
]
)
call: Final = getattr(router, surface)
async def send(group: str, **media: object) -> object:
response: Final = call(model=group, **_MEDIA_SURFACES[surface], **media)
return await response if asyncio.iscoroutine(response) else response
for group in ("unlisted", "listed", "unresolvable"):
await send(group, audio=_AUDIO_CLIPS)
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes"
assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS)
await send("drops", audio=_AUDIO_CLIPS)
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer drops"
assert "audio" not in json.loads(respx_mock.calls.last.request.content)
await send("unlisted")
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer refuses"
sent_before_refusal: Final = len(respx_mock.calls)
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio']")):
await send("none", audio=_AUDIO_CLIPS)
assert len(respx_mock.calls) == sent_before_refusal
def _answer_unless_key_is(rejected_key: str) -> Callable[[httpx.Request], httpx.Response]:
def answer(request: httpx.Request) -> httpx.Response:
if request.headers["authorization"] == f"Bearer {rejected_key}":
return httpx.Response(401, json={"error": {"message": "invalid key"}})
return httpx.Response(200, json=_RESPONSE)
return answer
@pytest.mark.asyncio
async def test_clips_to_a_group_whose_only_taker_is_cooling_down_are_refused_rather_than_rate_limited(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").mock(
side_effect=_answer_unless_key_is("takes")
)
unlisted: Final = "openrouter/example/omni-preview"
router: Final = litellm.Router(
model_list=[
_group_member("g", unlisted, "refuses", 1, {}),
_group_member("g", unlisted, "takes", 1, {"supports_audio_input": True}),
],
num_retries=0,
cooldown_time=120,
)
with pytest.raises(litellm.AuthenticationError):
await router.adecisions(model="g-takes", input="review", questions=_PREDICATE_QUESTIONS)
sent_before_clips: Final = len(route.calls)
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio']")):
await router.adecisions(model="g", input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS)
assert len(route.calls) == sent_before_clips
@pytest.mark.asyncio
@pytest.mark.parametrize("surface", tuple(_MEDIA_SURFACES))
async def test_a_router_default_drop_params_overrides_a_deployment_when_choosing_who_takes_clips(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, surface: str
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
unlisted: Final = "openrouter/example/omni-preview"
router: Final = litellm.Router(
model_list=[
_group_member("g", unlisted, "drops", 1, {}, drop_params=True),
_group_member("g", unlisted, "takes", 0, {"supports_audio_input": True}),
],
default_litellm_params={"drop_params": False},
)
call: Final = getattr(router, surface)
async def send(**params: object) -> object:
response: Final = call(model="g", **_MEDIA_SURFACES[surface], audio=_AUDIO_CLIPS, **params)
return await response if asyncio.iscoroutine(response) else response
await send()
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes"
assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS)
await send(drop_params=True)
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer drops"
assert "audio" not in json.loads(respx_mock.calls.last.request.content)
@pytest.mark.asyncio
async def test_a_deployment_that_turns_audio_off_is_refused_or_dropped_even_when_the_cost_map_allows_it(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
monkeypatch.setitem(
litellm.model_cost,
"typesafe/jev-omni",
{
"litellm_provider": "typesafe",
"mode": "decisions",
"supports_audio_input": True,
"supports_video_input": True,
},
)
route: Final = respx_mock.post("https://example.com/v1/systemone").respond(json=_RESPONSE)
send: Final = partial(
litellm.asystemone,
model="typesafe/jev-omni",
state="test",
questions=_QUESTIONS,
api_base="https://example.com",
api_key="test",
audio=_AUDIO_CLIPS,
videos=_VIDEO_CLIPS,
model_info={"supports_audio_input": False},
)
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("does not support parameters: ['audio']")):
await send()
assert not route.called
await send(drop_params=True)
sent: Final = json.loads(route.calls.last.request.content)
assert "audio" not in sent
assert sent["videos"] == list(_VIDEO_CLIPS)
@pytest.mark.asyncio
async def test_a_media_flag_that_is_not_a_boolean_leaves_the_cost_map_in_charge_of_that_kind_only(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
route: Final = respx_mock.post("https://example.com/v1/systemone").respond(json=_RESPONSE)
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("does not support parameters: ['audio']")):
await litellm.asystemone(
model="typesafe/jev-unlisted",
state="test",
questions=_QUESTIONS,
api_base="https://example.com",
api_key="test",
audio=_AUDIO_CLIPS,
videos=_VIDEO_CLIPS,
model_info={"supports_audio_input": "true", "supports_video_input": True},
)
assert not route.called
@pytest.mark.asyncio
async def test_an_openai_format_deployment_that_advertises_media_is_still_refused_it(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE)
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio', 'videos']")):
await litellm.adecisions(
model="openai/gpt-6-luna",
input="review",
questions=_PREDICATE_QUESTIONS,
api_key="deployment-key",
audio=_AUDIO_CLIPS,
videos=_VIDEO_CLIPS,
model_info={"supports_audio_input": True, "supports_video_input": True},
)
assert not route.called
def test_dropping_every_question_answers_400_before_http(respx_mock: respx.MockRouter) -> None:
route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE)

View file

@ -25,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_semantic_cache_prompt_from_messages,
get_str_from_messages,
handle_any_messages_to_chat_completion_str_messages_conversion,
handle_messages_with_content_list_to_str_conversion,
hoist_images_from_tool_messages,
is_encrypted_reasoning_block,
merge_consecutive_system_messages,
@ -2675,3 +2676,16 @@ def test_tpm_projection_preserves_non_media_identity_and_opaque_values() -> None
projected, media = split_media_for_tpm(request)
assert projected is request
assert media == ()
def test_content_list_flattening_returns_new_messages_and_leaves_the_callers_untouched() -> None:
messages: Final = [
{"role": "user", "content": [{"type": "text", "text": "Hello"}, {"type": "text", "text": " world"}]},
{"role": "assistant", "content": "Hi"},
]
messages_before: Final = copy.deepcopy(messages)
flattened: Final = handle_messages_with_content_list_to_str_conversion(messages)
assert flattened == [{"role": "user", "content": "Hello world"}, {"role": "assistant", "content": "Hi"}]
assert messages == messages_before

View file

@ -3,7 +3,10 @@ Tests for litellm.litellm_core_utils.logging_utils — base64 truncation helpers
"""
import asyncio, datetime, importlib, litellm, os, pytest_asyncio
import sys
import threading
from itertools import chain
from types import FrameType
from unittest.mock import MagicMock
import pytest
@ -17,7 +20,7 @@ from litellm.litellm_core_utils.logging_utils import (
truncate_base64_in_messages,
truncate_base64_in_messages_async,
)
from collections.abc import AsyncIterator
from collections.abc import AsyncIterator, Callable, Coroutine, Sequence
from datetime import datetime as datetime_assemble_streaming
from litellm import(
Choices,
@ -28,6 +31,7 @@ from litellm import(
)
from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.types.llms.openai import AllMessageValues
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
from typing import Final
@ -99,8 +103,8 @@ class TestTruncateBase64InString:
@pytest.mark.parametrize(
"text",
[
'data: {"choices": [{"delta": {"content": "hi"}}]}\n\n' * 50_000,
"data:" * 200_000,
'data: {"choices": [{"delta": {"content": "hi"}}]}\n\n' * 50_000 + ";base64,",
"data:" * 200_000 + ";base64,",
],
ids=["sse_lines", "whitespace_free_prefixes"],
)
@ -241,6 +245,49 @@ def scan_threads(monkeypatch):
return threads
def _run_without_yielding(coroutine: Coroutine[object, object, object]) -> object:
try:
coroutine.send(None)
except StopIteration as finished:
return finished.value
coroutine.close()
raise AssertionError("the coroutine yielded to the event loop")
def _regex_calls_while(run: Callable[[], object]) -> tuple[object, tuple[str, ...]]:
calls: Final[list[str]] = [] # mutable-ok: sys.setprofile reports each call to a callback that has to record it
previous: Final = sys.getprofile()
def record(frame: FrameType, event: str, arg: object) -> None:
name: Final = getattr(arg, "__qualname__", "")
if event == "c_call" and name.startswith("Pattern."):
calls.append(name)
sys.setprofile(record)
try:
result: Final = run()
finally:
sys.setprofile(previous)
return result, tuple(calls)
def _tool_result_messages(tool_output: str) -> list[AllMessageValues]: # mutable-ok: the truncator only walks lists
return [
{"role": "user", "content": "Summarize the rows the tool returned."},
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "dump_rows", "arguments": "{}"}}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": tool_output},
]
_PLAIN_TOOL_OUTPUT_CHARS: Final = 400 * 1024
class TestTruncateBase64InMessagesAsync:
@pytest.mark.asyncio
async def test_large_payload_is_scanned_off_the_event_loop(self, monkeypatch, scan_threads):
@ -268,6 +315,83 @@ class TestTruncateBase64InMessagesAsync:
assert scan_threads
assert set(scan_threads) == {threading.get_ident()}
@pytest.mark.parametrize(
"row",
[
'{"id": 7, "name": "widget", "status": "ok"}\n',
"data: {\"choices\": [{\"delta\": {\"content\": \"hi\"}}]}\n\n",
"const total = rows.map((r) => r.price).reduce((a, b) => a + b, 0);\n",
'data: {"choices": [{"delta": {"content": "a; b"}}]}\n\n',
"Content-Type: text/plain;base64,aGVsbG8=\n",
],
ids=["json_rows", "sse_lines", "code_with_semicolons", "sse_lines_with_semicolons", "base64_marker_without_data"],
)
def test_tool_message_without_data_uri_is_copied_without_a_regex_scan(self, row: str):
tool_output: Final = (row * (_PLAIN_TOOL_OUTPUT_CHARS // len(row) + 1))[:_PLAIN_TOOL_OUTPUT_CHARS]
messages: Final = _tool_result_messages(tool_output)
result, regex_calls = _regex_calls_while(lambda: truncate_base64_in_messages(messages))
assert result == messages
assert result is not messages
assert regex_calls == ()
def test_small_data_uri_is_truncated_inline_with_a_regex_scan(self):
payload: Final = "L" * 2_000
messages: Final = _image_messages(payload)
result, regex_calls = _regex_calls_while(
lambda: _run_without_yielding(truncate_base64_in_messages_async(messages))
)
assert regex_calls
assert payload not in result[0]["content"][1]["image_url"]["url"]
assert "base64_data truncated" in result[0]["content"][1]["image_url"]["url"]
def test_large_plain_text_is_copied_inline_without_rescanning_or_a_thread_hop(
self, monkeypatch: pytest.MonkeyPatch, scan_threads: Sequence[int]
):
monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS", 1_000)
messages: Final = _tool_result_messages("plain tool output\n" * 100)
result: Final = _run_without_yielding(truncate_base64_in_messages_async(messages))
assert result == messages
assert result is not messages
assert scan_threads == []
@pytest.mark.asyncio
async def test_long_plain_text_history_is_copied_off_the_event_loop(
self, monkeypatch: pytest.MonkeyPatch, scan_threads: Sequence[int]
):
monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS", 1_000)
monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS", 10)
messages: Final = list(chain.from_iterable(_tool_result_messages(f"rows {turn}\n" * 100) for turn in range(5)))
result: Final = await truncate_base64_in_messages_async(messages)
offload_threads: Final = tuple(scan_threads)
assert result == messages
assert result is not messages
assert offload_threads
assert threading.get_ident() not in offload_threads
@pytest.mark.asyncio
async def test_plain_text_over_the_inline_limit_is_copied_off_the_event_loop(
self, monkeypatch: pytest.MonkeyPatch, scan_threads: Sequence[int]
):
monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS", 1_000)
monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS", 10_000)
messages: Final = _tool_result_messages('data: {"choices": [{"delta": {"content": "a; b"}}]}\n\n' * 400)
result: Final = await truncate_base64_in_messages_async(messages)
offload_threads: Final = tuple(scan_threads)
assert result == messages
assert result is not messages
assert offload_threads
assert threading.get_ident() not in offload_threads
@pytest.mark.asyncio
async def test_none_and_disabled_truncation_short_circuit(self, monkeypatch, scan_threads):
assert await truncate_base64_in_messages_async(None) is None

View file

@ -1,6 +1,7 @@
import copy
import json
import logging
from collections.abc import Iterator
from itertools import chain
from typing import Final
from unittest.mock import MagicMock, patch
@ -9849,3 +9850,186 @@ def test_transform_request_tool_config_point_without_tools_drops_the_key():
assert "cache_control_injection_points" not in data
assert "tools" not in data
assert litellm_params["metadata"] == {}
def _five_cache_hint_conversation() -> list[dict[str, object]]:
return [
{"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]},
{"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]},
{"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]},
{"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]},
{"role": "assistant", "content": "answer 2"},
{"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]},
]
def _cached_texts(node: object) -> Iterator[str]:
if isinstance(node, dict):
yield from ((node["text"],) if "cache_control" in node and "text" in node else ())
yield from chain.from_iterable(map(_cached_texts, node.values()))
elif isinstance(node, list):
yield from chain.from_iterable(map(_cached_texts, node))
@pytest.mark.parametrize(
("config", "model"),
[
(AnthropicConfig(), "claude-haiku-5-5"),
(VertexAIAnthropicConfig(), "claude-haiku-5-5"),
(AmazonAnthropicClaudeConfig(), "anthropic.claude-haiku-5-5"),
(AzureAnthropicConfig(), "claude-haiku-5-5"),
],
ids=["anthropic", "vertex_ai", "bedrock_invoke", "azure_ai"],
)
@pytest.mark.parametrize(
("drop_params", "expected_cached_texts"),
[
(True, ["answer 1", "turn 1", "turn 2", "turn 3"]),
("true", ["answer 1", "turn 1", "turn 2", "turn 3"]),
(False, ["answer 1", "system prompt", "turn 1", "turn 2", "turn 3"]),
],
ids=[
"drop_params_keeps_the_newest_four",
"string_drop_params_keeps_the_newest_four",
"without_drop_params_all_five_pass_through",
],
)
def test_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params(
config: AnthropicConfig,
model: str,
drop_params: bool | str,
expected_cached_texts: list[str],
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
messages: Final = _five_cache_hint_conversation()
messages_before: Final = copy.deepcopy(messages)
body: Final = config.transform_request(
model=model,
messages=messages,
optional_params={},
litellm_params={"drop_params": drop_params},
headers={},
)
assert sorted(_cached_texts(body)) == expected_cached_texts
assert json.dumps(body).count('"cache_control"') == len(expected_cached_texts)
assert messages == messages_before
def test_claude_cache_control_cap_keeps_a_cached_tool_and_drops_the_oldest_message_hint(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "drop_params", True)
messages: Final = _five_cache_hint_conversation()[1:]
tool: Final = {
"name": "get_weather",
"description": "Get the weather",
"input_schema": {"type": "object", "properties": {}},
"cache_control": {"type": "ephemeral"},
}
body: Final = AnthropicConfig().transform_request(
model="claude-haiku-5-5",
messages=messages,
optional_params={"tools": [tool]},
litellm_params={},
headers={},
)
assert body["tools"][0]["cache_control"] == {"type": "ephemeral"}
assert sorted(_cached_texts(body["messages"])) == ["answer 1", "turn 2", "turn 3"]
def test_claude_cache_control_cap_drops_a_string_assistant_turns_text_hint_before_its_tool_call_hints(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "drop_params", True)
hint: Final = {"type": "ephemeral"}
tool_calls: Final = [
{"id": f"call_{name}", "type": "function", "function": {"name": name, "arguments": "{}"}, "cache_control": hint}
for name in ("lookup", "search")
]
follow_up: Final = [
{"type": "text", "text": "follow-up", "cache_control": hint},
{"type": "text", "text": "and more", "cache_control": hint},
]
messages: Final = [
{"role": "user", "content": "question"},
{"role": "assistant", "content": "checking both", "cache_control": hint, "tool_calls": tool_calls},
{"role": "tool", "tool_call_id": "call_lookup", "content": "found"},
{"role": "tool", "tool_call_id": "call_search", "content": "none"},
{"role": "user", "content": follow_up},
]
body: Final = AnthropicConfig().transform_request(
model="claude-haiku-5-5",
messages=messages,
optional_params={},
litellm_params={},
headers={},
)
assistant_blocks: Final = body["messages"][1]["content"]
assert [block["type"] for block in assistant_blocks] == ["text", "tool_use", "tool_use"]
assert ["cache_control" in block for block in assistant_blocks] == [False, True, True]
assert json.dumps(body).count('"cache_control"') == 4
assert json.dumps(body).count('"cache_control"') == 4
def test_claude_cache_control_cap_keeps_the_newest_hints_inside_one_message(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "drop_params", True)
messages: Final = [
{
"role": "user",
"content": [
*({"type": "text", "text": f"part {n}", "cache_control": {"type": "ephemeral"}} for n in range(1, 6)),
{"type": "text", "text": "Reply with the single word OK."},
],
}
]
messages_before: Final = copy.deepcopy(messages)
body: Final = AnthropicConfig().transform_request(
model="claude-haiku-5-5",
messages=messages,
optional_params={},
litellm_params={},
headers={},
)
assert sorted(_cached_texts(body["messages"])) == ["part 2", "part 3", "part 4", "part 5"]
assert json.dumps(body).count('"cache_control"') == 4
assert messages == messages_before
def test_claude_cache_control_cap_does_not_count_a_message_level_hint_that_list_content_never_sends(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
messages: Final = [
{"role": "system", "content": "system prompt", "cache_control": {"type": "ephemeral"}},
{"role": "user", "content": "turn 1", "cache_control": {"type": "ephemeral"}},
{"role": "assistant", "content": "answer 1"},
{"role": "user", "content": "turn 2", "cache_control": {"type": "ephemeral"}},
{"role": "assistant", "content": "answer 2"},
{
"role": "user",
"content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}],
"cache_control": {"type": "ephemeral"},
},
]
body: Final = AnthropicConfig().transform_request(
model="claude-haiku-5-5",
messages=messages,
optional_params={},
litellm_params={"drop_params": True},
headers={},
)
assert sorted(_cached_texts(body)) == ["system prompt", "turn 1", "turn 2", "turn 3"]
assert json.dumps(body).count('"cache_control"') == 4

View file

@ -56,6 +56,98 @@ def test_is_anthropic_messages_url(url: str, expected: bool) -> None:
assert is_anthropic_messages_url(url) is expected
@pytest.mark.parametrize(
("configured_base", "url", "expected"),
[
pytest.param(None, "https://api.anthropic.com/v1/messages", 512, id="default-upstream"),
pytest.param(None, "https://api.anthropic.com:443/v1/messages/", 512, id="default-upstream-explicit-port"),
pytest.param("http://relay.test:8080", "https://api.anthropic.com/v1/messages", 512, id="direct-target"),
pytest.param("https://api.anthropic.com", "https://api.anthropic.com/v1/messages", 512, id="configured-default"),
pytest.param("http://relay.test:8080", "http://relay.test:8080/v1/messages", 255, id="configured-upstream"),
pytest.param("http://relay.test:8080/a", "http://relay.test:8080/a/v1/messages", 255, id="configured-prefix"),
pytest.param("https://Relay.Test", "https://relay.test/v1/messages", 255, id="configured-upper-case-host"),
pytest.param("https://relay.test:443", "https://relay.test/v1/messages", 255, id="configured-https-port"),
pytest.param("http://relay.test:80/a", "http://relay.test/a/v1/messages", 255, id="configured-http-port"),
pytest.param("http://relay.test:8080", "http://relay.test:8081/v1/messages", None, id="other-port"),
pytest.param("http://relay.test:notaport", "http://relay.test/v1/messages", None, id="invalid-configured-base"),
pytest.param(None, "https://other-vendor.test/v1/messages", None, id="other-vendor"),
pytest.param(None, "https://api.anthropic.com/v1/messages/count_tokens", None, id="count-tokens"),
],
)
def test_anthropic_upstream_user_id_limit(
monkeypatch: pytest.MonkeyPatch, configured_base: str | None, url: str, expected: int | None
) -> None:
from litellm.llms.anthropic.common_utils import anthropic_upstream_user_id_limit
monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False)
monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False)
if configured_base is not None:
monkeypatch.setenv("ANTHROPIC_API_BASE", configured_base)
assert anthropic_upstream_user_id_limit(url) == expected
def test_anthropic_pass_through_api_base_prefers_the_api_base_variable(monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.llms.anthropic.common_utils import anthropic_pass_through_api_base
monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://fallback.test")
monkeypatch.setenv("ANTHROPIC_API_BASE", "http://preferred.test")
assert anthropic_pass_through_api_base() == "http://preferred.test"
monkeypatch.delenv("ANTHROPIC_API_BASE")
assert anthropic_pass_through_api_base() == "http://fallback.test"
# Each row was sent to https://api.anthropic.com/v1/messages on 2026-10-30, False rows answered 400.
@pytest.mark.parametrize(
("user_id", "expected"),
[
pytest.param("user-1", True, id="opaque"),
pytest.param("", True, id="empty"),
pytest.param('{"device_id":"abc","session_id":"def"}', True, id="claude-code-shape"),
pytest.param("1234567890", True, id="digits"),
pytest.param("+14155550100", True, id="phone-shape"),
pytest.param("a@b", True, id="no-dot"),
pytest.param("@b.co", True, id="nothing-before-at-sign"),
pytest.param("a@.co", True, id="nothing-between-at-sign-and-dot"),
pytest.param("a@b.", True, id="nothing-after-dot"),
pytest.param("a @ b.co", True, id="spaces-around-at-sign"),
pytest.param("a\u00a0@b.co", False, id="no-break-space-before-at-sign"),
pytest.param("a@b\n.co", False, id="newline-before-dot"),
pytest.param("line one\nline two", False, id="newline"),
pytest.param("a\tb", False, id="tab"),
pytest.param("a\x1f@b.co", False, id="unit-separator-before-at-sign"),
pytest.param("a@\x1cb.co", False, id="file-separator-after-at-sign"),
pytest.param("a\u200bb", False, id="zero-width-space"),
pytest.param("a@b.co", False, id="email"),
pytest.param("user a@b.co here", False, id="email-inside-text"),
pytest.param('{"device_id":"a@b.co"}', False, id="email-inside-json"),
pytest.param("a@@example.com", False, id="doubled-at-sign"),
pytest.param("@@example.com", False, id="leading-doubled-at-sign"),
pytest.param("a@b@c.co", False, id="two-at-signs"),
pytest.param("a@b..co", False, id="doubled-dot"),
pytest.param("a@.b.co", False, id="dot-after-at-sign"),
pytest.param("a@b.c d", False, id="email-before-space"),
],
)
def test_anthropic_accepts_user_id(user_id: str, expected: bool) -> None:
from litellm.constants import ANTHROPIC_MAX_USER_ID_LENGTH
from litellm.llms.anthropic.common_utils import anthropic_accepts_user_id
assert anthropic_accepts_user_id(user_id, ANTHROPIC_MAX_USER_ID_LENGTH) is expected
@pytest.mark.parametrize("character", ["a", "\u00e9", "\U0001f600"], ids=["ascii", "two-bytes", "four-bytes"])
def test_anthropic_accepts_user_id_counts_characters_up_to_the_limit(character: str) -> None:
from litellm.constants import ANTHROPIC_MAX_USER_ID_LENGTH
from litellm.llms.anthropic.common_utils import anthropic_accepts_user_id
assert anthropic_accepts_user_id(character * ANTHROPIC_MAX_USER_ID_LENGTH, ANTHROPIC_MAX_USER_ID_LENGTH) is True
assert (
anthropic_accepts_user_id(character * (ANTHROPIC_MAX_USER_ID_LENGTH + 1), ANTHROPIC_MAX_USER_ID_LENGTH) is False
)
def test_anthropic_model_not_found_maps_to_not_found_but_authentication_error_does_not() -> None:
with respx.mock(assert_all_called=True) as router:
router.post("https://api.anthropic.com/v1/messages").mock(

View file

@ -26,7 +26,6 @@ from litellm.types.decisions import (
DecisionsIRScoreAnswer,
DecisionsRequestBody,
OpenAIDecisionRequestBody,
UnsupportedDecisionsRequest,
)
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
@ -279,7 +278,7 @@ def test_a_malformed_systemone_response_names_the_field_at_fault(
assert [item["loc"] for item in error.value.errors()] == [field]
def test_each_media_kind_is_gated_by_its_own_support_flag() -> None:
def test_each_media_kind_is_sent_in_its_own_field() -> None:
audio: Final = ["data:audio/wav;base64,YQ=="]
videos: Final = ["data:video/mp4;base64,YQ=="]
audio_only: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "audio": audio}))
@ -287,22 +286,16 @@ def test_each_media_kind_is_gated_by_its_own_support_flag() -> None:
_SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "videos": videos})
)
assert ir_to_systemone_request("clef-omni", audio_only, supports_audio=True) == {
assert ir_to_systemone_request("clef-omni", audio_only) == {
"model": "clef-omni",
**_SYSTEMONE_REQUEST,
"audio": audio,
}
assert ir_to_systemone_request("clef-omni", video_only, supports_video=True) == {
assert ir_to_systemone_request("clef-omni", video_only) == {
"model": "clef-omni",
**_SYSTEMONE_REQUEST,
"videos": videos,
}
assert isinstance(
ir_to_systemone_request("clef-omni", audio_only, supports_video=True), UnsupportedDecisionsRequest
)
assert isinstance(
ir_to_systemone_request("clef-omni", video_only, supports_audio=True), UnsupportedDecisionsRequest
)
_ASSET_REQUEST: Final[Mapping[str, object]] = {

View file

@ -11259,3 +11259,86 @@ def test_converse_sends_only_the_header_betas_when_header_and_body_both_carry_be
"thinking-binding-controls-2026-08-01",
"thinking-display-updates-2026-08-18",
]
def _five_cache_hint_conversation() -> list[dict[str, object]]:
return [
{"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]},
{"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]},
{"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]},
{"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]},
{"role": "assistant", "content": "answer 2"},
{"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]},
]
@pytest.mark.parametrize(
("drop_params", "expected_system_cache_points"),
[(True, 0), ("true", 0), (False, 1)],
ids=[
"drop_params_drops_the_oldest_hint",
"string_drop_params_drops_the_oldest_hint",
"without_drop_params_all_five_pass_through",
],
)
def test_converse_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params(
drop_params: bool | str, expected_system_cache_points: int, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
messages: Final = _five_cache_hint_conversation()
messages_before: Final = copy.deepcopy(messages)
request: Final = AmazonConverseConfig().transform_request(
model="anthropic.claude-haiku-5-5",
messages=messages,
optional_params={},
litellm_params={"drop_params": drop_params},
headers={},
)
assert len(list(_cache_points(request["system"]))) == expected_system_cache_points
assert [len(list(_cache_points(message))) for message in request["messages"]] == [1, 1, 1, 0, 1]
assert messages == messages_before
def test_converse_claude_tool_message_with_several_marked_blocks_counts_as_one_cache_point_against_the_cap(
monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
messages: Final = [
{
"role": "user",
"content": [
{"type": "text", "text": "policy", "cache_control": {"type": "ephemeral"}},
{"type": "text", "text": "question", "cache_control": {"type": "ephemeral"}},
],
},
{
"role": "assistant",
"content": None,
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}],
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": [
{"type": "text", "text": "part 1", "cache_control": {"type": "ephemeral"}},
{"type": "text", "text": "part 2", "cache_control": {"type": "ephemeral"}},
{"type": "text", "text": "part 3", "cache_control": {"type": "ephemeral"}},
],
},
]
messages_before: Final = copy.deepcopy(messages)
request: Final = AmazonConverseConfig().transform_request(
model="anthropic.claude-haiku-5-5",
messages=messages,
optional_params={
"tools": [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}]
},
litellm_params={"drop_params": True},
headers={},
)
assert len(list(_cache_points(request["messages"]))) == 3
assert messages == messages_before

View file

@ -1,5 +1,5 @@
import asyncio, importlib, litellm, litellm.litellm_core_utils.get_model_cost_map as bedrock_govcloud_model_cost_map, pytest
import asyncio, importlib, litellm, pytest
from typing import Final
@ -1563,7 +1563,6 @@ def _pr4_bedrock_env(monkeypatch: pytest.MonkeyPatch) -> None:
def _use_local_model_cost_map() -> Iterator[None]:
with pytest.MonkeyPatch.context() as monkeypatch:
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
importlib.reload(bedrock_govcloud_model_cost_map)
importlib.reload(litellm)
yield

View file

@ -2773,3 +2773,66 @@ def test_claude_json_schema_stream_split_across_json_tools_keeps_the_non_stream_
choices: Final = _json_schema_stream_choices(respx_mock, call, split_stream)
assert [choice.finish_reason for choice in choices if choice.finish_reason] == [answer.finish_reason]
@pytest.mark.parametrize(
("model", "base_model"),
[("databricks-claude-haiku-5-5", None), ("support-bot-endpoint", "databricks/databricks-claude-haiku-5-5")],
ids=["claude_in_the_endpoint_name", "claude_named_only_by_base_model"],
)
def test_databricks_claude_request_over_the_cache_control_cap_keeps_the_newest_four_hints_under_drop_params(
model: str, base_model: str | None, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
hint: Final = {"type": "ephemeral"}
messages: Final = [
{"role": "system", "content": "system prompt", "cache_control": hint},
{"role": "user", "content": "turn 1", "cache_control": hint},
{"role": "assistant", "content": "answer 1", "cache_control": hint},
{"role": "user", "content": "turn 2", "cache_control": hint},
{"role": "assistant", "content": "answer 2"},
{"role": "user", "content": "turn 3", "cache_control": hint},
]
messages_before: Final = copy.deepcopy(messages)
body: Final = DatabricksConfig().transform_request(
model=model,
messages=messages,
optional_params={},
litellm_params={"drop_params": True, "custom_llm_provider": "databricks", "base_model": base_model},
headers={},
)
assert "cache_control" not in json.dumps(body["messages"][0])
assert json.dumps(body["messages"]).count('"cache_control"') == 4
assert messages == messages_before
@pytest.mark.parametrize(
"optional_params",
[{"cache_control": {"type": "ephemeral"}}, {"extra_body": {"cache_control": {"type": "ephemeral"}}}],
ids=["top_level_cache_control", "extra_body_cache_control"],
)
def test_databricks_claude_cache_control_cap_keeps_room_for_a_top_level_cache_control_under_drop_params(
optional_params: dict[str, object], monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
hint: Final = {"type": "ephemeral"}
messages: Final = [
{"role": "user", "content": "turn 1", "cache_control": hint},
{"role": "assistant", "content": "answer 1", "cache_control": hint},
{"role": "user", "content": "turn 2", "cache_control": hint},
{"role": "assistant", "content": "answer 2", "cache_control": hint},
{"role": "user", "content": "turn 3"},
]
body: Final = DatabricksConfig().transform_request(
model="databricks-claude-haiku-5-5",
messages=messages,
optional_params=copy.deepcopy(optional_params),
litellm_params={"drop_params": True, "custom_llm_provider": "databricks"},
headers={},
)
assert "cache_control" not in json.dumps(body["messages"][0])
assert json.dumps(body["messages"]).count('"cache_control"') == 3

View file

@ -1,6 +1,8 @@
import asyncio
import copy
import json
import os
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@ -200,3 +202,38 @@ async def test_deepinfra_tool_message_content_transformation_async():
print(f"✓ Async test passed: {tool_message['content']}")
print("\n✅ DeepInfra async tool message transformation test passed!")
@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"])
async def test_deepinfra_request_stringifies_tool_content_without_touching_the_callers_message(is_async: bool) -> None:
from litellm.llms.deepinfra.chat.transformation import DeepInfraConfig
config: Final = DeepInfraConfig()
messages: Final = [
{"role": "user", "content": "Calculate 10 + 10"},
{
"role": "assistant",
"content": "",
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "calculator", "arguments": "{}"}}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": [{"type": "text", "text": "20"}]},
]
messages_before: Final = copy.deepcopy(messages)
request_kwargs: Final = {
"model": "deepseek-ai/DeepSeek-V3.2",
"messages": messages,
"optional_params": {},
"litellm_params": {},
"headers": {},
}
body: Final = (
await config.async_transform_request(**request_kwargs)
if is_async
else config.transform_request(**request_kwargs)
)
assert body["messages"][2]["content"] == "20"
assert messages == messages_before

View file

@ -1,3 +1,4 @@
import copy
import json
import logging
from collections.abc import Callable
@ -537,6 +538,26 @@ def test_tool_search_reply_sent_to_fireworks_drops_litellm_response_fields():
}
def test_cache_control_is_stripped_from_the_request_but_kept_on_the_callers_messages():
hint: Final = {"type": "ephemeral"}
messages: Final = [
{"role": "system", "content": [{"type": "text", "text": "a long cached prompt", "cache_control": hint}]},
{"role": "user", "content": "hi", "cache_control": hint},
]
caller_messages_before: Final = json.dumps(messages)
request: Final = FireworksAIConfig().transform_request(
model="accounts/fireworks/models/gpt-oss-120b",
messages=messages,
optional_params={},
litellm_params={},
headers={},
)
assert "cache_control" not in json.dumps(request["messages"])
assert json.dumps(messages) == caller_messages_before
# -----------------------------------------------------------------------------
# Regression tests for legacy / OpenAPI $ref defs in tool parameters.
#
@ -2477,3 +2498,43 @@ def test_translate_developer_role_follows_the_deployment_model_info_over_the_cos
)
assert [message["role"] for message in messages] == expected_roles
def test_fireworks_request_rewrites_tools_without_touching_the_callers_tool() -> None:
tools: Final = [
{
"type": "function",
"function": {
"name": "create_thing",
"strict": True,
"parameters": {
"type": "object",
"properties": {"thing": {"$ref": "#/definitions/Thing"}},
"definitions": {"Thing": {"type": "object", "properties": {"id": {"type": "string"}}}},
},
},
}
]
tools_before: Final = copy.deepcopy(tools)
request: Final = FireworksAIConfig().transform_request(
model="accounts/fireworks/models/gpt-oss-120b",
messages=[{"role": "user", "content": "Make a thing"}],
optional_params={"tools": tools},
litellm_params={},
headers={},
)
assert request["tools"] == [
{
"type": "function",
"function": {
"name": "create_thing",
"parameters": {
"type": "object",
"properties": {"thing": {"type": "object", "properties": {"id": {"type": "string"}}}},
},
},
}
]
assert tools == tools_before

View file

@ -1,3 +1,4 @@
import copy
import logging
from typing import Final
from unittest.mock import patch
@ -589,3 +590,19 @@ async def test_streamed_reasoning_reply_can_be_replayed(is_async: bool, with_too
}
]
assert reply.reasoning_content == "I should check the weather."
def test_groq_request_drops_null_assistant_fields_without_touching_the_callers_messages() -> None:
messages: Final = [
{"role": "user", "content": "Hi"},
{"role": "assistant", "content": "Hello", "function_call": None, "tool_calls": None},
{"role": "user", "content": "Again"},
]
messages_before: Final = copy.deepcopy(messages)
body: Final = GroqChatConfig().transform_request(
model="openai/gpt-oss-120b", messages=messages, optional_params={}, litellm_params={}, headers={}
)
assert body["messages"][1] == {"role": "assistant", "content": "Hello"}
assert messages == messages_before

View file

@ -1,4 +1,6 @@
import copy
import json
from typing import Final
from unittest.mock import MagicMock, patch
import pytest
@ -397,3 +399,43 @@ def test_hosted_vllm_custom_tools_use_top_level_input_schema():
assert tools[0]["function"]["name"] == "search"
assert tools[0]["function"]["description"] == "Search docs"
assert tools[0]["function"]["parameters"] == input_schema
def _strict_weather_tool() -> dict[str, object]:
return {
"type": "function",
"function": {
"name": "get_weather",
"strict": True,
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
"additionalProperties": False,
},
},
}
def test_hosted_vllm_tools_are_rewritten_without_touching_the_callers_tool() -> None:
tools: Final = [_strict_weather_tool()]
tools_before: Final = copy.deepcopy(tools)
optional_params: Final = HostedVLLMChatConfig().map_openai_params(
non_default_params={"tools": tools}, optional_params={}, model="gpt-oss-120b", drop_params=False
)
assert optional_params["tools"] == [
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
}
]
assert tools == tools_before

View file

@ -1,3 +1,4 @@
import copy
import json
import logging
from collections.abc import Mapping
@ -1212,7 +1213,10 @@ def test_mistral_sends_only_the_text_items_of_a_native_thinking_chunk_beside_an_
"content": [
{
"type": "thinking",
"thinking": [{"type": "text", "text": "It has whiskers."}, {"type": "reference", "reference_ids": [1]}],
"thinking": [
{"type": "text", "text": "It has whiskers."},
{"type": "reference", "reference_ids": [1]},
],
},
{"type": "text", "text": "A cat."},
],
@ -1672,6 +1676,18 @@ def test_mistral_transform_request_drops_an_empty_tool_message_whose_call_id_is_
assert [m.get("role") for m in request["messages"]] == ["user", "assistant"]
def test_mistral_request_drops_a_user_name_without_touching_the_callers_message() -> None:
messages: Final = [{"role": "user", "content": "Hi"}, {"role": "user", "content": "", "name": "alice"}]
messages_before: Final = copy.deepcopy(messages)
body: Final = MistralConfig().transform_request(
model="mistral-medium-latest", messages=messages, optional_params={}, litellm_params={}, headers={}
)
assert "name" not in json.dumps(body["messages"])
assert messages == messages_before
def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]:
return tuple(
r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage()

View file

@ -1824,6 +1824,94 @@ class TestToolSchemaCombinatorFlatteningForOpenAI:
assert request["tools"][0] is tool
class TestCacheControlStrippingDoesNotMutateCallerInput:
"""
Stripping cache_control for a provider that cannot use it must not reach back
into the caller's own message and tool objects.
filter_value_from_dict deletes the key in place and recurses into nested dicts
and lists, and remove_cache_control_flag_from_messages_and_tools assigned the
result back into the caller's list, so one call to any OpenAI-compatible
provider permanently stripped cache_control from a list the caller still holds.
Reusing that list on Anthropic or Bedrock afterwards then silently lost prompt
caching, with no error and full-price billing.
"""
def setup_method(self):
self.config = OpenAIGPTConfig()
@pytest.fixture(autouse=True)
def _clean_openai_base_env(self, monkeypatch):
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
monkeypatch.setattr(litellm, "api_base", None, raising=False)
@staticmethod
def _messages():
return [
{
"role": "system",
"content": [
{
"type": "text",
"text": "a long cached system prompt",
"cache_control": {"type": "ephemeral"},
}
],
},
{
"role": "user",
"content": "Hello",
"cache_control": {"type": "ephemeral"},
},
]
@staticmethod
def _tools():
return [
{
"type": "function",
"function": {"name": "get_weather", "parameters": {"type": "object"}},
"cache_control": {"type": "ephemeral"},
}
]
def _transform(self, messages, tools):
return self.config.transform_request(
model="gpt-4o",
messages=messages,
optional_params={"tools": tools},
litellm_params={"custom_llm_provider": "openai", "api_base": None},
headers={},
)
def test_caller_messages_and_tools_keep_cache_control(self):
messages, tools = self._messages(), self._tools()
messages_before, tools_before = copy.deepcopy(messages), copy.deepcopy(tools)
request = self._transform(messages, tools)
assert "cache_control" not in json.dumps(request)
assert messages == messages_before
assert tools == tools_before
def test_a_later_anthropic_call_still_sees_cache_control(self):
"""The user-visible consequence: the same message list reused on a provider
that does support caching must still carry the cache breakpoints."""
messages = self._messages()
self._transform(messages, self._tools())
anthropic_body = litellm.AnthropicConfig().transform_request(
model="claude-3-5-sonnet-20240620",
messages=messages,
optional_params={},
litellm_params={},
headers={},
)
assert "cache_control" in json.dumps(anthropic_body)
def _developer_message_after_a_user_turn() -> list[dict[str, str]]:
return [
{"role": "system", "content": "You are terse."},
@ -1916,3 +2004,53 @@ def test_completion_leaves_the_callers_closing_developer_message_unchanged_on_a_
)
assert json.loads(chat.calls.last.request.content)["messages"][-1]["role"] == "user"
assert messages == sent
def _user_turn_with_litellm_only_media_fields() -> list[dict[str, object]]:
return [
{
"role": "user",
"content": [
{"type": "text", "text": "Describe these"},
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png", "format": "image/png"}},
{"type": "image_url", "image_url": "https://example.com/dog.png"},
{
"type": "file",
"file": {"file_data": "data:application/pdf;base64,JVBERi0=", "format": "application/pdf"},
},
],
}
]
@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"])
async def test_openai_request_rewrites_media_parts_without_touching_the_callers_message(
is_async: bool, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
monkeypatch.setattr(litellm, "api_base", None, raising=False)
config: Final = OpenAIGPTConfig()
messages: Final = _user_turn_with_litellm_only_media_fields()
messages_before: Final = copy.deepcopy(messages)
request_kwargs: Final = {
"model": "gpt-5.5",
"messages": messages,
"optional_params": {},
"litellm_params": {"custom_llm_provider": "openai", "api_base": None},
"headers": {},
}
body: Final = (
await config.async_transform_request(**request_kwargs)
if is_async
else config.transform_request(**request_kwargs)
)
assert body["messages"][0]["content"] == [
{"type": "text", "text": "Describe these"},
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}},
{"type": "image_url", "image_url": {"url": "https://example.com/dog.png"}},
{"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0=", "filename": "my_file.pdf"}},
]
assert messages == messages_before

View file

@ -1,3 +1,4 @@
import copy
import json
import logging
from collections.abc import Callable, Mapping
@ -541,6 +542,50 @@ class TestOpenAIResponsesAPIConfig:
assert "cache_control" not in result["tools"][0]
assert result["tools"][0]["name"] == "get_weather"
def test_transform_does_not_strip_cache_control_from_the_callers_input(self):
"""Stripping for OpenAI must not reach back into the caller's own objects.
`filter_value_from_dict` deletes the key in place and recurses, and
`_validate_input_param` passes plain dict items through by reference, so
the caller's input list used to lose its cache breakpoints. Reusing that
list on Anthropic or Bedrock afterwards then silently lost prompt caching.
Same defect as the Chat Completions path.
"""
input_with_cache_control = [
{
"role": "system",
"content": [
{
"type": "input_text",
"text": "a long cached system prompt",
"cache_control": {"type": "ephemeral"},
}
],
}
]
tools_with_cache_control = [
{
"type": "function",
"name": "get_weather",
"parameters": {"type": "object"},
"cache_control": {"type": "ephemeral"},
}
]
input_before = copy.deepcopy(input_with_cache_control)
tools_before = copy.deepcopy(tools_with_cache_control)
result = self.config.transform_responses_api_request(
model=self.model,
input=input_with_cache_control,
response_api_optional_request_params={"tools": tools_with_cache_control},
litellm_params={},
headers={},
)
assert "cache_control" not in json.dumps(result)
assert input_with_cache_control == input_before
assert tools_with_cache_control == tools_before
def test_transform_preserves_input_without_cache_control(self):
"""Inputs without cache_control must pass through unmodified."""
input_clean = [
@ -4607,10 +4652,10 @@ def test_cache_control_still_marks_a_part_whose_own_marker_is_malformed_under_dr
}
def _caller_input_after_transform(model: str, marker: object) -> str:
def _hinted_caller_input(marker: object) -> list[dict[str, object]]:
hint: Final = {"type": "ephemeral"}
marked: Final = {} if marker is None else {"prompt_cache_breakpoint": marker}
input: Final = [
return [
{
"type": "message",
"role": "user",
@ -4626,6 +4671,10 @@ def _caller_input_after_transform(model: str, marker: object) -> str:
"content": [{"type": "output_text", "text": "ok", "cache_control": hint, **marked}],
},
]
def _caller_input_after_transform(model: str, marker: object) -> str:
input: Final = _hinted_caller_input(marker)
_CONFIGS[0].transform_responses_api_request(
model=model,
input=cast("ResponseInputParam", input), # cast-ok: plain dicts shaped like the SDK's input items
@ -4647,6 +4696,15 @@ def test_a_marker_leaves_the_caller_request_that_fallbacks_and_logs_read_as_an_u
assert _caller_input_after_transform(model, marker).replace(marker_json, "") == unmarked
@pytest.mark.parametrize("model", ("gpt-4.1", "gpt-6.1-sol"))
@pytest.mark.parametrize("marker", (None, _VALID_MARKERS[0], "yes"), ids=("unmarked", "explicit", "malformed"))
def test_the_caller_input_keeps_its_cache_control_hints_for_a_fallback_deployment(
monkeypatch: pytest.MonkeyPatch, model: str, marker: object
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
assert _caller_input_after_transform(model, marker) == json.dumps(_hinted_caller_input(marker))
@pytest.mark.asyncio
@pytest.mark.usefixtures(httpx_transport.__name__)
async def test_aresponses_forwards_previous_response_id_to_openai() -> None:

View file

@ -1,3 +1,4 @@
import copy
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
@ -198,3 +199,23 @@ def test_translate_developer_role_keeps_developer_messages_untouched() -> None:
)
== messages
)
def test_o_series_request_turns_system_into_user_without_touching_the_callers_messages(
monkeypatch: pytest.MonkeyPatch,
) -> None:
model: Final = "o-series-test-model-without-system-role"
monkeypatch.setitem(
litellm.model_cost,
model,
{"litellm_provider": "openai", "mode": "chat", "supports_system_messages": False},
)
messages: Final = [{"role": "system", "content": "Be terse"}, {"role": "user", "content": "Hi"}]
messages_before: Final = copy.deepcopy(messages)
body: Final = OpenAIOSeriesConfig().transform_request(
model=model, messages=messages, optional_params={}, litellm_params={}, headers={}
)
assert [message["role"] for message in body["messages"]] == ["user", "user"]
assert messages == messages_before

View file

@ -0,0 +1,706 @@
import copy
import json
from typing import Final
import pytest
import respx
import litellm
from litellm.caching.llm_caching_handler import LLMClientCache
def test_hostinger_router_provider_resolution(monkeypatch: pytest.MonkeyPatch):
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
monkeypatch.setenv("HOSTINGER_ROUTER_API_KEY", "hostinger-test-key")
model, provider, api_key, api_base = get_llm_provider(
model="hostinger_router/claude-sonnet-5",
custom_llm_provider=None,
api_base=None,
api_key=None,
)
assert model == "claude-sonnet-5"
assert provider == "hostinger_router"
assert api_key == "hostinger-test-key"
assert api_base == "https://router.hostinger.com/v1"
def test_hostinger_router_provider_keeps_explicit_credentials(monkeypatch: pytest.MonkeyPatch):
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
monkeypatch.setenv("HOSTINGER_ROUTER_API_KEY", "hostinger-env-key")
_, provider, api_key, api_base = get_llm_provider(
model="hostinger_router/claude-sonnet-5",
custom_llm_provider=None,
api_base="https://router.internal.example/v1",
api_key="hostinger-explicit-key",
)
assert provider == "hostinger_router"
assert api_key == "hostinger-explicit-key"
assert api_base == "https://router.internal.example/v1"
@pytest.mark.parametrize("model", ["gpt-5-nano", "claude-sonnet-5"])
def test_hostinger_router_api_base_keeps_unprefixed_model_provider(model: str):
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
_, provider_without_api_base, _, _ = get_llm_provider(model=model)
_, provider_with_hostinger_api_base, _, _ = get_llm_provider(
model=model,
api_base="https://router.hostinger.com/v1",
api_key="hostinger-test-key",
)
assert provider_with_hostinger_api_base == provider_without_api_base
HOSTINGER_ROUTER_MODELS = tuple(sorted(name for name in litellm.model_cost if name.startswith("hostinger_router/")))
HOSTINGER_ROUTER_CHAT_MODELS = tuple(
name for name in HOSTINGER_ROUTER_MODELS if litellm.get_model_info(name)["mode"] == "chat"
)
@pytest.mark.parametrize("model", HOSTINGER_ROUTER_CHAT_MODELS)
def test_hostinger_router_model_cost_and_capabilities(model: str):
from litellm.cost_calculator import cost_per_token
prompt_cost, completion_cost = cost_per_token(
model=model,
prompt_tokens=1_000_000,
completion_tokens=1_000_000,
custom_llm_provider="hostinger_router",
)
model_info = litellm.get_model_info(model)
assert prompt_cost == pytest.approx(model_info["input_cost_per_token"] * 1_000_000)
assert completion_cost == pytest.approx(model_info["output_cost_per_token"] * 1_000_000)
assert model_info["output_cost_per_token"] > 0
assert model_info["max_tokens"] == model_info["max_output_tokens"] <= model_info["max_input_tokens"]
assert model_info["litellm_provider"] == "hostinger_router"
assert model_info["mode"] == "chat"
if model_info.get("supports_prompt_caching"):
assert 0 < model_info["cache_read_input_token_cost"] < model_info["input_cost_per_token"]
capability_checks: Final = (
("supports_function_calling", litellm.supports_function_calling),
("supports_reasoning", litellm.supports_reasoning),
("supports_response_schema", litellm.supports_response_schema),
("supports_vision", litellm.supports_vision),
)
assert {name: check(model) for name, check in capability_checks} == {
name: model_info.get(name) is True for name, _ in capability_checks
}
def test_hostinger_router_chat_completions_request():
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond(
200,
json={
"id": "chatcmpl_hostinger",
"object": "chat.completion",
"created": 1_789_550_000,
"model": "claude-sonnet-5",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "Hello from Hostinger Router"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7},
},
)
response: Final = litellm.completion(
model="hostinger_router/claude-sonnet-5",
messages=[{"role": "user", "content": "Say hello"}],
api_key="hostinger-test-key",
)
request: Final = route.calls.last.request
body: Final = json.loads(request.content)
assert route.call_count == 1
assert request.headers["authorization"] == "Bearer hostinger-test-key"
assert body["model"] == "claude-sonnet-5"
assert response.choices[0].message.content == "Hello from Hostinger Router"
def test_hostinger_router_chat_completions_streaming_request():
chunks = [
'data: {"id":"chatcmpl_hostinger","object":"chat.completion.chunk","created":1789550000,"model":"claude-sonnet-5","choices":[{"index":0,"delta":{"role":"assistant","content":"Hi"},"finish_reason":null}]}\n\n',
'data: {"id":"chatcmpl_hostinger","object":"chat.completion.chunk","created":1789550000,"model":"claude-sonnet-5","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":4,"completion_tokens":1,"total_tokens":5}}\n\n',
"data: [DONE]\n\n",
]
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond(
200,
headers={"content-type": "text/event-stream"},
content="".join(chunks).encode(),
)
response: Final = litellm.completion(
model="hostinger_router/claude-sonnet-5",
messages=[{"role": "user", "content": "Say hello"}],
stream=True,
api_key="hostinger-test-key",
)
collected = [chunk for chunk in response]
request: Final = route.calls.last.request
body: Final = json.loads(request.content)
assert route.call_count == 1
assert body["stream"] is True
assert any(chunk.choices and chunk.choices[0].delta.content == "Hi" for chunk in collected)
def test_hostinger_router_responses_request():
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/responses").respond(
200,
json={
"id": "resp_hostinger",
"object": "response",
"created_at": 1_789_550_000,
"model": "gpt-5.2",
"status": "completed",
"output": [
{
"id": "msg_hostinger",
"type": "message",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "Hello from Hostinger Router", "annotations": []}],
}
],
"usage": {"input_tokens": 4, "output_tokens": 3, "total_tokens": 7},
},
)
response: Final = litellm.responses(
model="hostinger_router/gpt-5.2",
input="Say hello",
api_key="hostinger-test-key",
)
request: Final = route.calls.last.request
body: Final = json.loads(request.content)
assert route.call_count == 1
assert str(request.url) == "https://router.hostinger.com/v1/responses"
assert request.headers["authorization"] == "Bearer hostinger-test-key"
assert body["model"] == "gpt-5.2"
assert body["input"] == "Say hello"
assert response.output[0].content[0].text == "Hello from Hostinger Router"
EMBEDDING_RESPONSE: Final = {
"object": "list",
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}],
"model": "text-embedding-3-small",
"usage": {"prompt_tokens": 2, "total_tokens": 2},
}
def test_hostinger_router_embeddings_request():
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
response: Final = litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
)
request: Final = route.calls.last.request
body: Final = json.loads(request.content)
assert route.call_count == 1
assert request.headers["authorization"] == "Bearer hostinger-test-key"
assert body["model"] == "text-embedding-3-small"
assert response.data[0]["embedding"] == [0.1, 0.2, 0.3]
def test_hostinger_router_embeddings_send_extra_headers_as_http_headers():
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
extra_headers={"X-Trace-Id": "trace-123"},
)
request: Final = route.calls.last.request
body: Final = json.loads(request.content)
assert request.headers["x-trace-id"] == "trace-123"
assert request.headers["authorization"] == "Bearer hostinger-test-key"
assert request.headers["content-type"] == "application/json"
assert set(body) == {"model", "input"}
def test_hostinger_router_embeddings_extra_headers_replace_key_headers_of_any_case():
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
extra_headers={"authorization": "Bearer caller-key", "x-trace-id": "trace-1"},
)
request: Final = route.calls.last.request
assert request.headers.get_list("authorization") == ["Bearer caller-key"]
assert request.headers.get_list("content-type") == ["application/json"]
assert request.headers.get_list("x-trace-id") == ["trace-1"]
@pytest.mark.parametrize(("configured", "sent"), [(12345, "12345"), (1.5, "1.5")])
def test_hostinger_router_embeddings_send_numeric_extra_headers_as_text(configured: int | float, sent: str):
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
extra_headers={"X-Org-Id": configured},
)
assert route.calls.last.request.headers.get_list("x-org-id") == [sent]
@pytest.mark.parametrize("configured", [True, None, ["a", "b"]])
def test_hostinger_router_embeddings_refuse_non_text_extra_headers(configured: object):
with respx.mock(assert_all_called=False) as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
with pytest.raises(litellm.UnsupportedParamsError, match="X-Debug") as refused:
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
extra_headers={"X-Debug": configured},
)
assert refused.value.status_code == 400
assert route.call_count == 0
@pytest.mark.parametrize("configured", [True, None, ["a", "b"]])
def test_hostinger_router_embeddings_drop_non_text_extra_headers_under_drop_params(configured: object):
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
extra_headers={"X-Debug": configured, "X-Trace-Id": "trace-1"},
drop_params=True,
)
request: Final = route.calls.last.request
assert "x-debug" not in request.headers
assert request.headers.get_list("x-trace-id") == ["trace-1"]
assert set(json.loads(request.content)) == {"model", "input"}
TRANSPORT_EXTRA_HEADERS: Final = [
("Host", "other.example"),
("content-length", "1"),
("Transfer-Encoding", "chunked"),
("Connection", "close"),
("Keep-Alive", "timeout=5"),
("Proxy-Connection", "close"),
("TE", "gzip"),
("Trailer", "X-Checksum"),
("Upgrade", "websocket"),
("Expect", "100-continue"),
]
@pytest.mark.parametrize(("name", "configured"), TRANSPORT_EXTRA_HEADERS)
def test_hostinger_router_embeddings_ignore_transport_extra_headers(name: str, configured: str):
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
extra_headers={name: configured, "X-Trace-Id": "trace-1"},
)
request: Final = route.calls.last.request
assert request.headers.get_list("host") == ["router.hostinger.com"]
assert request.headers.get_list("content-length") == [str(len(request.content))]
assert "transfer-encoding" not in request.headers
assert configured not in request.headers.get_list(name)
assert request.headers.get_list("x-trace-id") == ["trace-1"]
MALFORMED_HEADER_NAMES: Final = ["Host:evil.example", "X Note", "X-Caf\u00e9"]
@pytest.mark.parametrize("name", MALFORMED_HEADER_NAMES)
def test_hostinger_router_embeddings_refuse_malformed_extra_header_names(name: str):
with respx.mock(assert_all_called=False) as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
with pytest.raises(litellm.UnsupportedParamsError) as refused:
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
extra_headers={name: "x"},
)
assert refused.value.status_code == 400
assert repr(name) in refused.value.message
assert route.call_count == 0
@pytest.mark.parametrize("name", MALFORMED_HEADER_NAMES)
def test_hostinger_router_embeddings_drop_malformed_extra_header_names_under_drop_params(name: str):
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
extra_headers={name: "x", "X-Trace-Id": "trace-1"},
drop_params=True,
)
request: Final = route.calls.last.request
assert request.headers.get_list("host") == ["router.hostinger.com"]
assert request.headers.get_list("x-trace-id") == ["trace-1"]
assert name.lower() not in request.headers.keys()
@pytest.mark.parametrize("configured", ["a\r\nHost: evil.example", "line\nbreak", "caf\u00e9", "a\x7fb"])
def test_hostinger_router_embeddings_drop_extra_header_values_http_cannot_carry(configured: str):
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
extra_headers={"x-litellm-user_api_key_alias": configured, "X-Trace-Id": "trace-1"},
)
request: Final = route.calls.last.request
assert request.headers.get_list("host") == ["router.hostinger.com"]
assert request.headers.get_list("x-trace-id") == ["trace-1"]
assert "x-litellm-user_api_key_alias" not in request.headers
@pytest.mark.parametrize(
("configured", "sent"),
[(" x ", "x"), ("\t", ""), (" a\tb ", "a\tb"), ("env-secret\n", "env-secret"), ("\r\n x \r\n", "x")],
)
def test_hostinger_router_embeddings_trim_whitespace_around_extra_header_values(configured: str, sent: str):
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
extra_headers={"X-Rig": configured},
)
assert route.calls.last.request.headers.get_list("x-rig") == [sent]
def test_hostinger_router_embeddings_extra_headers_stay_with_their_request():
router: Final = litellm.Router(
model_list=[
{
"model_name": "shared",
"litellm_params": {
"model": "hostinger_router/text-embedding-3-small",
"api_key": "hostinger-test-key",
"headers": {"X-Org-Id": "org-1"},
},
}
]
)
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
router.embedding(
model="shared", input=["first"], extra_headers={"Authorization": "Bearer caller-a", "X-Leak": "a"}
)
router.embedding(model="shared", input=["second"])
first: Final = route.calls[0].request
second: Final = route.calls[1].request
assert first.headers.get_list("x-leak") == ["a"]
assert second.headers.get_list("authorization") == ["Bearer hostinger-test-key"]
assert "x-leak" not in second.headers
assert second.headers.get_list("x-org-id") == ["org-1"]
@pytest.mark.parametrize("param", ["headers", "extra_headers"])
def test_hostinger_router_embeddings_send_headers_given_as_name_value_pairs(param: str):
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
**{param: [["X-Trace-Id", "trace-1"]]},
)
request: Final = route.calls.last.request
assert request.headers.get_list("x-trace-id") == ["trace-1"]
assert request.headers.get_list("authorization") == ["Bearer hostinger-test-key"]
@pytest.mark.parametrize("param", ["headers", "extra_headers"])
def test_hostinger_router_embeddings_refuse_headers_that_are_not_an_object(param: str):
with respx.mock(assert_all_called=False) as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
with pytest.raises(litellm.UnsupportedParamsError, match=rf"\b{param} must be a JSON object") as refused:
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
**{param: "X-Trace-Id: trace-1"},
)
assert refused.value.status_code == 400
assert route.call_count == 0
@pytest.mark.parametrize("param", ["headers", "extra_headers"])
def test_hostinger_router_embeddings_drop_headers_that_are_not_an_object_under_drop_params(param: str):
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
litellm.embedding(
model="hostinger_router/text-embedding-3-small",
input=["Say hello"],
api_key="hostinger-test-key",
drop_params=True,
**{param: "X-Trace-Id: trace-1"},
)
request: Final = route.calls.last.request
assert "x-trace-id" not in request.headers
assert request.headers.get_list("authorization") == ["Bearer hostinger-test-key"]
assert set(json.loads(request.content)) == {"model", "input"}
def test_hostinger_router_embeddings_never_borrow_another_providers_key(monkeypatch: pytest.MonkeyPatch):
monkeypatch.delenv("HOSTINGER_ROUTER_API_KEY", raising=False)
monkeypatch.setenv("OPENAI_LIKE_API_KEY", "other-provider-key")
monkeypatch.setenv("OPENAI_API_KEY", "other-provider-key")
with respx.mock(assert_all_called=False) as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
with pytest.raises(litellm.BadRequestError, match="Missing API Key"):
litellm.embedding(model="hostinger_router/text-embedding-3-small", input=["Say hello"])
assert route.call_count == 0
def test_json_provider_without_embeddings_endpoint_refuses_embeddings():
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
slug: Final = next(
slug
for slug in JSONProviderRegistry.list_providers()
if not JSONProviderRegistry.supports_embeddings(slug) and slug not in litellm.provider_list
)
with respx.mock(assert_all_called=False) as upstream:
route: Final = upstream.route().respond(200, json=EMBEDDING_RESPONSE)
with pytest.raises(litellm.BadRequestError):
litellm.embedding(model=f"{slug}/text-embedding-3-small", input=["Say hello"], api_key="test-key")
assert route.call_count == 0
@pytest.mark.asyncio
async def test_hostinger_router_anthropic_messages_request(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
with respx.mock() as upstream:
route: Final = upstream.post("https://router.hostinger.com/v1/messages").respond(
200,
json={
"id": "msg_hostinger",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5",
"content": [{"type": "text", "text": "Hello from Hostinger Router"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 4, "output_tokens": 3},
},
)
response: Final = await litellm.anthropic.messages.acreate(
model="hostinger_router/claude-sonnet-5",
messages=[{"role": "user", "content": "Say hello"}],
max_tokens=32,
api_key="hostinger-test-key",
)
request: Final = route.calls.last.request
body: Final = json.loads(request.content)
assert route.call_count == 1
assert str(request.url) == "https://router.hostinger.com/v1/messages"
assert request.headers["authorization"] == "Bearer hostinger-test-key"
assert request.headers["anthropic-version"] == "2023-06-01"
assert body["model"] == "claude-sonnet-5"
assert body["messages"] == [{"role": "user", "content": "Say hello"}]
assert response["content"][0]["text"] == "Hello from Hostinger Router"
CHAT_COMPLETION_RESPONSE: Final = {
"id": "chatcmpl_hostinger",
"object": "chat.completion",
"created": 1_789_550_000,
"model": "gpt-5.2",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "Hello from Hostinger Router"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7},
}
UNSUPPORTED_ENDPOINT: Final = {"error": {"message": "Model does not support this endpoint."}}
@pytest.mark.asyncio
@pytest.mark.parametrize("model", ["gpt-5.2", "gpt-5-nano"])
async def test_hostinger_router_messages_bridge_models_without_native_messages(
model: str, monkeypatch: pytest.MonkeyPatch
):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
with respx.mock(assert_all_called=False) as upstream:
native: Final = upstream.post("https://router.hostinger.com/v1/messages").respond(
403, json=UNSUPPORTED_ENDPOINT
)
chat: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond(
200, json=CHAT_COMPLETION_RESPONSE
)
response: Final = await litellm.anthropic.messages.acreate(
model=f"hostinger_router/{model}",
messages=[{"role": "user", "content": "Say hello"}],
max_tokens=32,
api_key="hostinger-test-key",
)
assert native.call_count == 0
assert chat.call_count == 1
assert json.loads(chat.calls.last.request.content)["model"] == model
assert response["content"][0]["text"] == "Hello from Hostinger Router"
def test_hostinger_router_responses_bridge_models_without_native_responses():
with respx.mock(assert_all_called=False) as upstream:
native: Final = upstream.post("https://router.hostinger.com/v1/responses").respond(
403, json=UNSUPPORTED_ENDPOINT
)
chat: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond(
200, json=CHAT_COMPLETION_RESPONSE
)
response: Final = litellm.responses(
model="hostinger_router/claude-sonnet-5",
input="Say hello",
api_key="hostinger-test-key",
)
assert native.call_count == 0
assert chat.call_count == 1
assert json.loads(chat.calls.last.request.content)["model"] == "claude-sonnet-5"
assert response.output[0].content[0].text == "Hello from Hostinger Router"
ANTHROPIC_MESSAGE_RESPONSE: Final = {
"id": "msg_hostinger",
"type": "message",
"role": "assistant",
"model": "claude-haiku-4-5",
"content": [{"type": "text", "text": "Hello from Hostinger Router"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 4, "output_tokens": 3},
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"sibling_model_info",
[
{"supported_endpoints": ["/v1/chat/completions", "/v1/messages", "/v1/responses"]},
{"supported_endpoints": "/v1/messages"},
],
)
async def test_hostinger_router_endpoint_opt_in_stays_with_its_own_deployment(
sibling_model_info: dict[str, object], monkeypatch: pytest.MonkeyPatch
):
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
deployment_params: Final = {"model": "hostinger_router/gpt-5-nano", "api_key": "hostinger-test-key"}
router: Final = litellm.Router(
model_list=[
{"model_name": "opted-in", "litellm_params": deployment_params, "model_info": sibling_model_info},
{"model_name": "plain", "litellm_params": deployment_params},
]
)
with respx.mock(assert_all_called=False) as upstream:
native_messages: Final = upstream.post("https://router.hostinger.com/v1/messages").respond(
403, json=UNSUPPORTED_ENDPOINT
)
native_responses: Final = upstream.post("https://router.hostinger.com/v1/responses").respond(
403, json=UNSUPPORTED_ENDPOINT
)
chat: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond(
200, json=CHAT_COMPLETION_RESPONSE
)
messages_response: Final = await router.aanthropic_messages(
model="plain", messages=[{"role": "user", "content": "Say hello"}], max_tokens=32
)
responses_response: Final = await router.aresponses(model="plain", input="Say hello")
assert native_messages.call_count == 0
assert native_responses.call_count == 0
assert chat.call_count == 2
assert messages_response["content"][0]["text"] == "Hello from Hostinger Router"
assert responses_response.output[0].content[0].text == "Hello from Hostinger Router"
@pytest.mark.asyncio
async def test_hostinger_router_deployment_cache_ttl_opt_in_keeps_one_hour_ttl(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
router: Final = litellm.Router(
model_list=[
{
"model_name": "opted-in",
"litellm_params": {
"model": "hostinger_router/claude-haiku-4-5",
"api_base": "https://router.hostinger.com/v1",
"api_key": "hostinger-test-key",
},
"model_info": {"supported_endpoints": ["/v1/messages"], "cache_control_ttl": True},
}
]
)
one_hour_cache: Final = {"type": "ephemeral", "ttl": "1h"}
with respx.mock() as upstream:
native: Final = upstream.post("https://router.hostinger.com/v1/messages").respond(
200, json=ANTHROPIC_MESSAGE_RESPONSE
)
await router.aanthropic_messages(
model="opted-in",
messages=[
{"role": "user", "content": [{"type": "text", "text": "Say hello", "cache_control": one_hour_cache}]}
],
max_tokens=32,
)
body: Final = json.loads(native.calls.last.request.content)
assert body["messages"][0]["content"][0]["cache_control"] == one_hour_cache

View file

@ -1,5 +1,8 @@
import copy
import json
from collections.abc import Iterator
from itertools import chain
from typing import Final
from unittest.mock import MagicMock
@ -874,6 +877,58 @@ def test_openrouter_streaming_reasoning_is_preserved() -> None:
assert assembled.choices[0].message.reasoning_content == "Consider the evidence. Apply the rule."
def _five_message_level_cache_hints() -> list[dict[str, object]]:
return [
{"role": "system", "content": "system prompt", "cache_control": {"type": "ephemeral"}},
{"role": "user", "content": "turn 1", "cache_control": {"type": "ephemeral"}},
{"role": "assistant", "content": "answer 1", "cache_control": {"type": "ephemeral"}},
{"role": "user", "content": "turn 2", "cache_control": {"type": "ephemeral"}},
{"role": "assistant", "content": "answer 2"},
{"role": "user", "content": "turn 3", "cache_control": {"type": "ephemeral"}},
]
def _cached_texts(node: object) -> Iterator[str]:
if isinstance(node, dict):
yield from ((node["text"],) if "cache_control" in node and "text" in node else ())
yield from chain.from_iterable(map(_cached_texts, node.values()))
elif isinstance(node, list):
yield from chain.from_iterable(map(_cached_texts, node))
@pytest.mark.parametrize(
("drop_params", "expected_cached_texts"),
[
(True, ["answer 1", "turn 1", "turn 2", "turn 3"]),
("true", ["answer 1", "turn 1", "turn 2", "turn 3"]),
(False, ["answer 1", "system prompt", "turn 1", "turn 2", "turn 3"]),
],
ids=[
"drop_params_keeps_the_newest_four",
"string_drop_params_keeps_the_newest_four",
"without_drop_params_all_five_pass_through",
],
)
def test_openrouter_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params(
drop_params: bool | str, expected_cached_texts: list[str], monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
messages: Final = _five_message_level_cache_hints()
messages_before: Final = copy.deepcopy(messages)
body: Final = OpenrouterConfig().transform_request(
model="anthropic/claude-haiku-5.5",
messages=messages,
optional_params={},
litellm_params={"drop_params": drop_params, "custom_llm_provider": "openrouter"},
headers={},
)
assert sorted(_cached_texts(body["messages"])) == expected_cached_texts
assert json.dumps(body).count('"cache_control"') == len(expected_cached_texts)
assert messages == messages_before
def _transform_openrouter_response(body: dict[str, object]) -> litellm.ModelResponse:
return OpenrouterConfig().transform_response(
model="openrouter/openai/gpt-4o-mini",

View file

@ -1,3 +1,4 @@
import json
from typing import Final, Literal
from unittest.mock import AsyncMock, MagicMock, patch
@ -2507,3 +2508,46 @@ def test_check_and_create_cache_clamps_negative_creation_tokens() -> None:
assert cache_name == "cachedContents/created"
assert logging_obj.vertex_ai_cache_creation_input_tokens == 0
def _recording_gemini_client(calls: list[httpx.Request]) -> HTTPHandler:
def route(request: httpx.Request) -> httpx.Response:
calls.append(request)
if request.url.path.endswith("/cachedContents"):
return httpx.Response(200, json={} if request.method == "GET" else _created_cache_body(3000))
return httpx.Response(
200,
json={
"candidates": [{"content": {"role": "model", "parts": [{"text": "OK"}]}, "finishReason": "STOP"}],
"usageMetadata": {"promptTokenCount": 3000, "candidatesTokenCount": 1, "totalTokenCount": 3001},
},
)
return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(route)))
def test_a_plain_system_prompt_goes_into_the_gemini_cache_instead_of_beside_it(local_model_cost_map) -> None:
calls: Final[list[httpx.Request]] = []
litellm.completion(
model="gemini/gemini-flash-latest",
messages=[
{"role": "system", "content": "You are terse."},
{
"role": "user",
"content": [
{"type": "text", "text": "plain system prefix " * 3000, "cache_control": {"type": "ephemeral"}},
{"type": "text", "text": "Reply with OK only."},
],
},
],
api_key="test-key",
client=_recording_gemini_client(calls),
)
created: Final = next(
json.loads(call.content) for call in calls if call.method == "POST" and call.url.path.endswith("/cachedContents")
)
generated: Final = next(json.loads(call.content) for call in calls if call.url.path.endswith(":generateContent"))
assert created["system_instruction"] == {"parts": [{"text": "You are terse."}]}
assert generated["cachedContent"] == "cachedContents/created"
assert "system_instruction" not in generated

View file

@ -1,4 +1,5 @@
import asyncio
import copy
import importlib
import json
from typing import Final, Optional, cast
@ -10,6 +11,7 @@ import pytest
import litellm
from litellm import completion, embedding
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.watsonx.chat.transformation import IBMWatsonXChatConfig
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
@ -542,3 +544,43 @@ def test_watsonx_deployment_space_id_embedding(monkeypatch, watsonx_embedding_ca
# nor space_id or project_id is required by wx.ai API when inferencing deployment
assert "project_id" not in json_data and "space_id" not in json_data
def _strict_weather_tool() -> dict[str, object]:
return {
"type": "function",
"function": {
"name": "get_weather",
"strict": True,
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
"additionalProperties": False,
},
},
}
def test_watsonx_chat_tools_are_rewritten_without_touching_the_callers_tool() -> None:
tools: Final = [_strict_weather_tool()]
tools_before: Final = copy.deepcopy(tools)
optional_params: Final = IBMWatsonXChatConfig().map_openai_params(
non_default_params={"tools": tools}, optional_params={}, model="ibm/granite-4-h-small", drop_params=False
)
assert optional_params["tools"] == [
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
}
]
assert tools == tools_before

View file

@ -1,3 +1,4 @@
import copy
import json
import os
from typing import Final
@ -581,3 +582,44 @@ def test_xai_grok_4_frequency_penalty_not_supported(model):
"""
supported_params = XAIChatConfig().get_supported_openai_params(model=model)
assert "frequency_penalty" not in supported_params
def _strict_weather_tool() -> dict[str, object]:
return {
"type": "function",
"function": {
"name": "get_weather",
"strict": True,
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
"additionalProperties": False,
},
},
}
def test_xai_tools_are_rewritten_without_touching_the_callers_tool() -> None:
tools: Final = [_strict_weather_tool()]
tools_before: Final = copy.deepcopy(tools)
optional_params: Final = XAIChatConfig().map_openai_params(
non_default_params={"tools": tools}, optional_params={}, model="grok-4.3", drop_params=False
)
assert optional_params["tools"] == [
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
"additionalProperties": False,
},
},
}
]
assert tools == tools_before

View file

@ -31,7 +31,7 @@ from starlette.datastructures import UploadFile as StarletteUploadFile
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.caching.dual_cache import DualCache
from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS
from litellm.constants import ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH, ANTHROPIC_MAX_USER_ID_LENGTH, DEFAULT_REQUEST_TIMEOUT_SECONDS
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.prometheus import PrometheusLogger
@ -6621,6 +6621,272 @@ async def test_pass_through_body_stays_byte_identical_outside_the_opt_in(
assert "x-litellm-call-id" in response.headers
def _echo_upstream_body(upstream_request: httpx.Request) -> httpx.Response:
return httpx.Response(
200, content=upstream_request.content, headers={"content-type": "application/json"}, request=upstream_request
)
async def _relayed_body(response: Response) -> bytes:
if isinstance(response, StreamingResponse):
return b"".join([chunk async for chunk in response.body_iterator])
return bytes(response.body)
async def _relay_through_pass_through(
client_body: Mapping[str, object],
target: str,
guardrails_config: dict[str, object] | None,
custom_body: dict[str, object] | None = None,
) -> Mapping[str, object]:
async def receive() -> Mapping[str, object]:
return {"type": "http.request", "body": json.dumps(dict(client_body)).encode()}
request: Final = Request(
{
"type": "http",
"method": "POST",
"path": "/my-claude-route",
"query_string": b"",
"headers": [(b"content-type", b"application/json")],
},
receive,
)
fake_client, cleanup = _inject_fake_passthrough_client(httpx.MockTransport(_echo_upstream_body), timeout=None)
try:
with ExitStack() as stack:
_enter_relay_logging_mocks(stack, {})
stack.enter_context(
patch("litellm.proxy.proxy_server.proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache()))
)
response: Final = await pass_through_request(
request=request,
target=target,
custom_headers={},
user_api_key_dict=UserAPIKeyAuth(api_key="sk-relay-test"),
custom_body=custom_body,
guardrails_config=guardrails_config,
)
relayed: Final = await _relayed_body(response)
finally:
cleanup()
await fake_client.aclose()
assert response.status_code == 200
return MappingProxyType(json.loads(relayed))
_MESSAGES_BODY: Final = MappingProxyType(
{"model": "claude-sonnet-4-5", "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]}
)
_ANTHROPIC_MESSAGES: Final = "https://api.anthropic.com/v1/messages"
@pytest.fixture
def no_configured_anthropic_base(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False)
monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False)
@pytest.mark.asyncio
@pytest.mark.usefixtures("no_configured_anthropic_base")
@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"])
@pytest.mark.parametrize(
"client_metadata, target, forwarded_metadata",
[
({"user_id": "user-1"}, _ANTHROPIC_MESSAGES, {"user_id": "user-1"}),
({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/", {"user_id": "user-1"}),
({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}?beta=true", {"user_id": "user-1"}),
(
{"user_id": "user-1", "tags": ["a"], "trace_id": "t", "generation_name": "g", "foo": "bar"},
_ANTHROPIC_MESSAGES,
{"user_id": "user-1"},
),
({"user_id": "1234567890"}, _ANTHROPIC_MESSAGES, {"user_id": "1234567890"}),
({"tags": ["a"], "foo": "bar"}, _ANTHROPIC_MESSAGES, None),
({"user_id": "a@example.com"}, _ANTHROPIC_MESSAGES, None),
({"user_id": "x" * (ANTHROPIC_MAX_USER_ID_LENGTH + 1)}, _ANTHROPIC_MESSAGES, None),
({"user_id": 42}, _ANTHROPIC_MESSAGES, None),
({"user_id": "line one\nline two"}, _ANTHROPIC_MESSAGES, None),
({"user_id": "a\x1f@example.com"}, _ANTHROPIC_MESSAGES, None),
({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/count_tokens", None),
({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/batches", None),
({"user_id": "user-1"}, "https://api.anthropic.com/v1/models", None),
({"user_id": "user-1"}, "https://other-vendor.test/anthropic/v1/messages", None),
({"user_id": "user-1"}, "https://other-vendor.test/v1/responses", None),
({"user_id": "user-1"}, "https://other-vendor.test/v1/chat/completions", None),
],
ids=[
"messages",
"messages-trailing-slash",
"messages-with-query",
"user-id-beside-litellm-and-unknown-keys",
"digits-user-id",
"no-user-id",
"email-user-id",
"over-long-user-id",
"integer-user-id",
"newline-user-id",
"control-character-before-at-sign",
"count-tokens",
"batches",
"other-anthropic-path",
"other-vendor-messages",
"other-vendor-responses",
"other-vendor-chat-completions",
],
)
async def test_pass_through_forwards_only_an_acceptable_user_id_to_anthropic_messages(
stream: bool, client_metadata: object, target: str, forwarded_metadata: Mapping[str, object] | None
) -> None:
client_body: Final = MappingProxyType(
{**_MESSAGES_BODY, "metadata": client_metadata, **({"stream": True} if stream else {})}
)
body_without_metadata: Final = {k: v for k, v in client_body.items() if k != "metadata"}
relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None)
assert relayed == (
body_without_metadata if forwarded_metadata is None else {**body_without_metadata, "metadata": forwarded_metadata}
)
@pytest.mark.asyncio
@pytest.mark.usefixtures("no_configured_anthropic_base")
@pytest.mark.parametrize("base_env", ["ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"])
@pytest.mark.parametrize(
"target, forwarded",
[
("http://gateway.test:8080/anthropic/v1/messages", True),
("http://gateway.test:9090/v1/messages", False),
("http://other-gateway.test:8080/v1/messages", False),
],
ids=["configured-upstream", "other-port", "other-host"],
)
async def test_pass_through_treats_the_configured_anthropic_base_as_anthropic(
monkeypatch: pytest.MonkeyPatch, base_env: str, target: str, forwarded: bool
) -> None:
monkeypatch.setenv(base_env, "http://gateway.test:8080/anthropic")
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}})
relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None)
assert relayed == (client_body if forwarded else _MESSAGES_BODY)
@pytest.mark.asyncio
@pytest.mark.usefixtures("no_configured_anthropic_base")
@pytest.mark.parametrize(
"user_id_length, target, forwarded",
[
(ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH, "http://gateway.test/v1/messages", True),
(ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH + 1, "http://gateway.test/v1/messages", False),
(ANTHROPIC_MAX_USER_ID_LENGTH, _ANTHROPIC_MESSAGES, True),
],
ids=["at-the-configured-base-limit", "over-the-configured-base-limit", "direct-anthropic-keeps-its-limit"],
)
async def test_pass_through_caps_the_user_id_for_a_configured_anthropic_compatible_base(
monkeypatch: pytest.MonkeyPatch, user_id_length: int, target: str, forwarded: bool
) -> None:
monkeypatch.setenv("ANTHROPIC_API_BASE", "http://gateway.test")
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "u" * user_id_length}})
relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None)
assert relayed == (client_body if forwarded else _MESSAGES_BODY)
@pytest.mark.asyncio
@pytest.mark.usefixtures("no_configured_anthropic_base")
@pytest.mark.parametrize(
"default_on, event_hook, guardrails_config, client_fields, stream_scope, forwarded",
[
(False, GuardrailEventHooks.pre_call, None, {}, None, True),
(False, GuardrailEventHooks.pre_call, {"unregistered-guard": {}}, {}, None, True),
(True, GuardrailEventHooks.post_call, None, {}, None, True),
(True, GuardrailEventHooks.pre_call, None, {}, None, False),
(False, GuardrailEventHooks.pre_call, {"pii-guard": {}}, {}, None, False),
(False, GuardrailEventHooks.pre_call, None, {"litellm_metadata": {"guardrails": ["pii-guard"]}}, None, False),
(False, GuardrailEventHooks.pre_call, {"pii-guard": {}}, {"litellm_metadata": {"guardrails": []}}, None, False),
(True, GuardrailEventHooks.pre_call, None, {"stream": True}, "streaming", False),
(True, GuardrailEventHooks.pre_call, None, {}, "streaming", True),
(True, GuardrailEventHooks.pre_call, None, {"stream": True}, "non_streaming", True),
(True, GuardrailEventHooks.pre_call, None, {}, "non_streaming", False),
],
ids=[
"guardrail-not-requested",
"guardrail-not-registered",
"default-on-post-call-guardrail",
"default-on-pre-call-guardrail",
"guardrail-on-the-route",
"guardrail-in-litellm-metadata",
"route-guardrail-beside-an-empty-client-list",
"streaming-guardrail-on-a-stream",
"streaming-guardrail-off-a-stream",
"non-streaming-guardrail-on-a-stream",
"non-streaming-guardrail-off-a-stream",
],
)
async def test_pass_through_keeps_the_user_id_from_routes_a_guardrail_scans(
default_on: bool,
event_hook: GuardrailEventHooks,
guardrails_config: dict[str, object] | None,
client_fields: Mapping[str, object],
stream_scope: str | None,
forwarded: bool,
) -> None:
guardrail: Final = CustomGuardrail(
guardrail_name="pii-guard", default_on=default_on, event_hook=event_hook, stream_scope=stream_scope
)
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}, **client_fields})
body_without_client_fields: Final = {k: v for k, v in client_body.items() if k not in ("metadata", "litellm_metadata")}
with patch.object(litellm, "callbacks", [guardrail]):
relayed: Final = await _relay_through_pass_through(
client_body, _ANTHROPIC_MESSAGES, guardrails_config=guardrails_config
)
assert relayed == (
{**body_without_client_fields, "metadata": {"user_id": "user-1"}} if forwarded else body_without_client_fields
)
@pytest.mark.asyncio
@pytest.mark.usefixtures("no_configured_anthropic_base")
async def test_pass_through_warns_without_the_value_when_it_drops_a_refused_user_id(
caplog: pytest.LogCaptureFixture,
) -> None:
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "person@example.com"}})
with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name):
await _relay_through_pass_through(client_body, _ANTHROPIC_MESSAGES, guardrails_config=None)
assert "Not forwarding metadata.user_id" in caplog.text
assert "person@example.com" not in caplog.text
@pytest.mark.asyncio
@pytest.mark.usefixtures("no_configured_anthropic_base")
async def test_pass_through_drops_metadata_that_is_not_an_object() -> None:
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": "user-1"})
relayed: Final = await _relay_through_pass_through(client_body, _ANTHROPIC_MESSAGES, guardrails_config=None)
assert relayed == _MESSAGES_BODY
@pytest.mark.asyncio
@pytest.mark.usefixtures("no_configured_anthropic_base")
async def test_pass_through_custom_body_replaces_the_client_user_id() -> None:
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}})
relayed: Final = await _relay_through_pass_through(
client_body, _ANTHROPIC_MESSAGES, guardrails_config=None, custom_body=dict(_MESSAGES_BODY)
)
assert relayed == _MESSAGES_BODY
_PARTIAL_RELAY_WARNING_MARKER = "ended before upstream body was fully relayed"

View file

@ -996,3 +996,21 @@ async def test_guardrails_only_and_skip_guardrails_are_mutually_exclusive(
guardrails_only=True,
skip_guardrails=True,
)
@pytest.mark.parametrize(
"request_data, expected",
[
({"metadata": {"_guardrail_pipelines": ["p1"]}}, True),
({"metadata": {"user_id": "u"}, "litellm_metadata": {"_guardrail_pipelines": ["p1"]}}, True),
({"metadata": {"_guardrail_pipelines": 5}}, True),
({"metadata": {"user_id": "u"}}, False),
],
ids=["pipeline-in-metadata", "pipeline-in-litellm-metadata", "non-list-pipeline-value", "no-pipeline"],
)
def test_request_has_pre_call_guardrails_reads_pipelines_from_both_metadata_buckets(
proxy_logging, monkeypatch, request_data, expected
):
monkeypatch.setattr(litellm, "callbacks", [])
assert proxy_logging.request_has_pre_call_guardrails(request_data) is expected

View file

@ -6314,6 +6314,17 @@ def test_marks_past_the_provider_cap_keep_the_newest_under_drop_params(drop_para
assert messages[1] == {"role": "user", "content": "turn 0"}
def test_marks_past_the_provider_cap_inside_one_message_keep_the_newest_under_drop_params():
parts: Final = tuple(
{"type": "input_text", "text": f"part {index}", "cache_control": _EPHEMERAL} for index in range(1, 6)
)
messages: Final = _bridged_request({"role": "user", "content": list(parts)}, drop_params=True)
content: Final = TypeAdapter(list[dict[str, object]]).validate_python(messages[-1]["content"])
assert ["cache_control" in part for part in content] == [False, True, True, True, True]
def test_global_drop_params_keeps_the_newest_marks(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "drop_params", True)

View file

@ -2037,6 +2037,13 @@
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@isaacs/nonrecursive-glob-match": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.0.2.tgz",
"integrity": "sha512-GJPHIgPPtFl+nLhFHyF0zZoFcOFLsZWbHFFzo+Zr+PWDXJbl9hHxb2w+9SiF2fHmLgKaXIpUzQo3SbBBZ9WObw==",
"dev": true,
"license": "BlueOak-1.0.0"
},
"node_modules/@jridgewell/gen-mapping": {
"version": "0.3.13",
"resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz",
@ -9657,13 +9664,14 @@
}
},
"node_modules/minimatch": {
"version": "10.2.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz",
"integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==",
"version": "11.0.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.2.tgz",
"integrity": "sha512-pWTgz5CN3tjqyZNSK2tePlSmu9IwM41q8yn8Usdqqwe9TIHh+9dEpLLjF4uApsfPEAZY71qtSiA2B2APEr+1Yw==",
"dev": true,
"license": "BlueOak-1.0.0",
"dependencies": {
"brace-expansion": "^5.0.2"
"@isaacs/nonrecursive-glob-match": "^1.0.2",
"brace-expansion": "^5.0.12"
},
"engines": {
"node": "18 || 20 || >=22"

View file

@ -117,7 +117,7 @@
"js-yaml": "4.3.2",
"brace-expansion": "5.0.12",
"glob": "13.0.0",
"minimatch": "10.2.4",
"minimatch": "11.0.2",
"ws": "8.21.0",
"braces": "3.0.3",
"axios": "1.13.6",

Binary file not shown.

After

Width:  |  Height:  |  Size: 16 KiB

View file

@ -30,6 +30,7 @@ import googleLogo from "../../public/assets/logos/google.svg";
import groqLogo from "../../public/assets/logos/groq.svg";
import huggingfaceLogo from "../../public/assets/logos/huggingface.svg";
import hyperbolicLogo from "../../public/assets/logos/hyperbolic.svg";
import hostingerLogo from "../../public/assets/logos/hostinger.png";
import infinityLogo from "../../public/assets/logos/infinity.png";
import jinaLogo from "../../public/assets/logos/jina.png";
import lambdaLogo from "../../public/assets/logos/lambda.svg";
@ -131,6 +132,7 @@ export enum Providers {
Hosted_Vllm = "Hosted vLLM",
HUGGINGFACE = "Huggingface",
HYPERBOLIC = "Hyperbolic",
HOSTINGER_ROUTER = "Hostinger Router",
Infinity = "Infinity",
JinaAI = "Jina AI",
OPENCODE_GO = "OpenCode Go",
@ -263,6 +265,7 @@ export const provider_map: Record<string, string> = {
Hosted_Vllm: "hosted_vllm",
HUGGINGFACE: "huggingface",
HYPERBOLIC: "hyperbolic",
HOSTINGER_ROUTER: "hostinger_router",
Infinity: "infinity",
JinaAI: "jina_ai",
OPENCODE_GO: "opencode_go",
@ -389,6 +392,7 @@ export const providerLogoMap: Partial<Record<Providers, string>> = {
[Providers.Hosted_Vllm]: vllmLogo.src,
[Providers.HUGGINGFACE]: huggingfaceLogo.src,
[Providers.HYPERBOLIC]: hyperbolicLogo.src,
[Providers.HOSTINGER_ROUTER]: hostingerLogo.src,
[Providers.Infinity]: infinityLogo.src,
[Providers.JinaAI]: jinaLogo.src,
[Providers.OPENCODE_GO]: opencodeLogo.src,
@ -493,6 +497,7 @@ const providerPlaceholderMap: Partial<Record<Providers, string>> = {
[Providers.EDENAI]: "edenai/openai/gpt-mini-latest",
[Providers.FalAI]: "fal_ai/fal-ai/flux-pro/v1.1-ultra",
[Providers.Google_AI_Studio]: "gemini-pro",
[Providers.HOSTINGER_ROUTER]: "hostinger_router/claude-sonnet-5",
[Providers.JinaAI]: "jina_ai/",
[Providers.META]: "meta/muse-spark-1.1",
[Providers.MICROSOFT_365_COPILOT]: "microsoft_365_copilot/chat",

13
uv.lock generated
View file

@ -16,6 +16,7 @@ exclude-newer-span = "P3D"
[options.exclude-newer-package]
brotli = "2026-10-22T14:01:00Z"
pypdf = "2026-10-28T17:01:00Z"
[manifest]
members = [
@ -8372,11 +8373,11 @@ wheels = [
[[package]]
name = "pyasn1"
version = "0.6.4"
version = "0.6.5"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" }
sdist = { url = "https://files.pythonhosted.org/packages/88/7e/3c0da41061a2021cac7cd524fd57092adbe5b515e8bf9005702e366802e2/pyasn1-0.6.5.tar.gz", hash = "sha256:6b11616a6bcf3a40e635f082bb232fa57b5e0dd7f517bd7ddd69ec3794508a47", size = 155152, upload-time = "2026-10-27T06:32:49.125Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" },
{ url = "https://files.pythonhosted.org/packages/19/1f/3eefd5c84c35cae44549ce1c7f7e58f7945b34f5a6bce07a135438522073/pyasn1-0.6.5-py3-none-any.whl", hash = "sha256:4017b0fce125096b8573b39c989fdb1a4b19cf761e683398184550fea0e66587", size = 84807, upload-time = "2026-10-27T06:32:48.045Z" },
]
[[package]]
@ -8869,14 +8870,14 @@ wheels = [
[[package]]
name = "pypdf"
version = "6.21.2"
version = "6.21.4"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/06/28/081f1cbcca7042441dafc8de00f48ba60853f89e5bff92c5f26477346d81/pypdf-6.21.2.tar.gz", hash = "sha256:0a3e1c5c886a1be03646c16a04827cc166c2a0575c971b26a4bf89686551c2d4", size = 7144571, upload-time = "2026-10-21T16:44:55.272Z" }
sdist = { url = "https://files.pythonhosted.org/packages/1e/e1/ec8f4d155c997c45917153a5521e60d6ee6389d68abdcbc8e44e564caffc/pypdf-6.21.4.tar.gz", hash = "sha256:c8817e3f888d47a087872ded69812c1ef30cacb871d6e053e0c00bbaf0bc2163", size = 7109255, upload-time = "2026-10-28T17:00:54.81Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/1b/4d/c9d2d67f204eff463237a1a8f5e25a6923b7b1043982b91fcaf104c8ef60/pypdf-6.21.2-py3-none-any.whl", hash = "sha256:1d44d1a3a200f25062adff0085b02040e4646431ac2ccac22e42679e91466084", size = 407766, upload-time = "2026-10-21T16:44:53.626Z" },
{ url = "https://files.pythonhosted.org/packages/3f/4b/77e8ad4ac42c9d93600f40541a50e9feb3a61d2bbee2d76f659346072051/pypdf-6.21.4-py3-none-any.whl", hash = "sha256:33e4fb41ca352810eb4e9838aa0d82d6888a5f8bf682f43e99cce5faa666255c", size = 410430, upload-time = "2026-10-28T17:00:53.146Z" },
]
[[package]]

View file

@ -644,6 +644,13 @@
"node": ">=18"
}
},
"node_modules/@isaacs/nonrecursive-glob-match": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.0.2.tgz",
"integrity": "sha512-GJPHIgPPtFl+nLhFHyF0zZoFcOFLsZWbHFFzo+Zr+PWDXJbl9hHxb2w+9SiF2fHmLgKaXIpUzQo3SbBBZ9WObw==",
"dev": true,
"license": "BlueOak-1.0.0"
},
"node_modules/@jridgewell/sourcemap-codec": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz",
@ -2700,13 +2707,14 @@
}
},
"node_modules/minimatch": {
"version": "10.2.6",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz",
"integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==",
"version": "11.0.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.2.tgz",
"integrity": "sha512-pWTgz5CN3tjqyZNSK2tePlSmu9IwM41q8yn8Usdqqwe9TIHh+9dEpLLjF4uApsfPEAZY71qtSiA2B2APEr+1Yw==",
"dev": true,
"license": "BlueOak-1.0.0",
"dependencies": {
"brace-expansion": "^5.0.8"
"@isaacs/nonrecursive-glob-match": "^1.0.2",
"brace-expansion": "^5.0.12"
},
"engines": {
"node": "18 || 20 || >=22"

View file

@ -83,5 +83,8 @@
"esbuild": "^0.28.2",
"typescript": "^5.9.3",
"vitest": "^4.1.12"
},
"overrides": {
"minimatch": "11.0.2"
}
}